From b053fa14052aaa24883e73a3f899016f963b5d43 Mon Sep 17 00:00:00 2001 From: "(quasar) nebula" Date: Tue, 14 Nov 2023 21:45:08 -0400 Subject: data: expose CacheableObject directly via #cacheable-object import --- src/data/yaml.js | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) (limited to 'src/data/yaml.js') diff --git a/src/data/yaml.js b/src/data/yaml.js index 1d35bae8..986f25d1 100644 --- a/src/data/yaml.js +++ b/src/data/yaml.js @@ -7,15 +7,13 @@ import {inspect as nodeInspect} from 'node:util'; import yaml from 'js-yaml'; +import CacheableObject, {CacheableObjectPropertyValueError} + from '#cacheable-object'; import {colors, ENABLE_COLOR, logInfo, logWarn} from '#cli'; import find, {bindFind} from '#find'; import {traverse} from '#node-utils'; -import T, { - CacheableObject, - CacheableObjectPropertyValueError, - Thing, -} from '#things'; +import T, {Thing} from '#things'; import { annotateErrorWithFile, -- cgit 1.3.0-6-gf8a5 From 362dc0619b93d74ad34df1bfbfd9ebc632fa5156 Mon Sep 17 00:00:00 2001 From: "(quasar) nebula" Date: Tue, 14 Nov 2023 22:49:51 -0400 Subject: data, yaml: catch commentary artist ref errors --- src/data/yaml.js | 55 +++++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 41 insertions(+), 14 deletions(-) (limited to 'src/data/yaml.js') diff --git a/src/data/yaml.js b/src/data/yaml.js index 986f25d1..843e70b3 100644 --- a/src/data/yaml.js +++ b/src/data/yaml.js @@ -28,6 +28,7 @@ import { } from '#sugar'; import { + commentaryRegex, sortAlbumsTracksChronologically, sortAlphabetically, sortChronologically, @@ -1616,6 +1617,7 @@ export function filterReferenceErrors(wikiData) { bannerArtistContribs: '_contrib', groups: 'group', artTags: 'artTag', + commentary: '_commentary', }], ['trackData', processTrackDocument, { @@ -1626,6 +1628,7 @@ export function filterReferenceErrors(wikiData) { sampledTracks: '_trackNotRerelease', artTags: 'artTag', originalReleaseTrack: '_trackNotRerelease', + commentary: '_commentary', }], ['groupCategoryData', processGroupCategoryDocument, { @@ -1675,7 +1678,19 @@ export function filterReferenceErrors(wikiData) { nest({message: `Reference errors in ${inspect(thing)}`}, ({nest, push, filter}) => { for (const [property, findFnKey] of Object.entries(propSpec)) { - const value = CacheableObject.getUpdateValue(thing, property); + let value = CacheableObject.getUpdateValue(thing, property); + let writeProperty = true; + + switch (findFnKey) { + case '_commentary': + if (value) { + value = + Array.from(value.matchAll(commentaryRegex)) + .map(({groups}) => groups.artistReference); + } + writeProperty = false; + break; + } if (value === undefined) { push(new TypeError(`Property ${colors.red(property)} isn't valid for ${colors.green(thing.constructor.name)}`)); @@ -1688,19 +1703,25 @@ export function filterReferenceErrors(wikiData) { let findFn; + const findArtistOrAlias = artistRef => { + const alias = find.artist(artistRef, wikiData.artistAliasData, {mode: 'quiet'}); + if (alias) { + // No need to check if the original exists here. Aliases are automatically + // created from a field on the original, so the original certainly exists. + const original = alias.aliasedArtist; + throw new Error(`Reference ${colors.red(artistRef)} is to an alias, should be ${colors.green(original.name)}`); + } + + return boundFind.artist(artistRef); + }; + switch (findFnKey) { - case '_contrib': - findFn = contribRef => { - const alias = find.artist(contribRef.who, wikiData.artistAliasData, {mode: 'quiet'}); - if (alias) { - // No need to check if the original exists here. Aliases are automatically - // created from a field on the original, so the original certainly exists. - const original = alias.aliasedArtist; - throw new Error(`Reference ${colors.red(contribRef.who)} is to an alias, should be ${colors.green(original.name)}`); - } + case '_commentary': + findFn = findArtistOrAlias; + break; - return boundFind.artist(contribRef.who); - }; + case '_contrib': + findFn = contribRef => findArtistOrAlias(contribRef.who); break; case '_homepageSourceGroup': @@ -1781,8 +1802,10 @@ export function filterReferenceErrors(wikiData) { ? `Reference errors` + fieldPropertyMessage + findFnMessage : `Reference error` + fieldPropertyMessage + findFnMessage); + let newPropertyValue = value; + if (Array.isArray(value)) { - thing[property] = filter( + newPropertyValue = filter( value, decorateErrorWithIndex(suppress(findFn)), {message: errorMessage}); @@ -1792,11 +1815,15 @@ export function filterReferenceErrors(wikiData) { try { call(findFn, value); } catch (error) { - thing[property] = null; + newPropertyValue = null; throw error; } })); } + + if (writeProperty) { + thing[property] = newPropertyValue; + } } }); } -- cgit 1.3.0-6-gf8a5 From a34b8d027866fbe858a4d2ff3543bc84c9d5983a Mon Sep 17 00:00:00 2001 From: "(quasar) nebula" Date: Fri, 17 Nov 2023 06:53:34 -0400 Subject: data, yaml, content: support multiple artists per commentary entry --- src/data/yaml.js | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) (limited to 'src/data/yaml.js') diff --git a/src/data/yaml.js b/src/data/yaml.js index 843e70b3..0734d539 100644 --- a/src/data/yaml.js +++ b/src/data/yaml.js @@ -21,6 +21,7 @@ import { decorateErrorWithIndex, decorateErrorWithAnnotation, empty, + filterAggregate, filterProperties, openAggregate, showAggregate, @@ -1686,8 +1687,10 @@ export function filterReferenceErrors(wikiData) { if (value) { value = Array.from(value.matchAll(commentaryRegex)) - .map(({groups}) => groups.artistReference); + .map(({groups}) => groups.artistReferences) + .map(text => text.split(',').map(text => text.trim())); } + writeProperty = false; break; } @@ -1804,11 +1807,22 @@ export function filterReferenceErrors(wikiData) { let newPropertyValue = value; - if (Array.isArray(value)) { + if (findFnKey === '_commentary') { + // Commentary doesn't write a property value, so no need to set. + filter( + value, {message: errorMessage}, + decorateErrorWithIndex(refs => + (refs.length === 1 + ? suppress(findFn)(refs[0]) + : filterAggregate( + refs, {message: `Errors in entry's artist references`}, + decorateErrorWithIndex(suppress(findFn))) + .aggregate + .close()))); + } else if (Array.isArray(value)) { newPropertyValue = filter( - value, - decorateErrorWithIndex(suppress(findFn)), - {message: errorMessage}); + value, {message: errorMessage}, + decorateErrorWithIndex(suppress(findFn))); } else { nest({message: errorMessage}, suppress(({call}) => { -- cgit 1.3.0-6-gf8a5 From 9a35630d91ebd7227994a1009487794a65ec38e1 Mon Sep 17 00:00:00 2001 From: "(quasar) nebula" Date: Mon, 20 Nov 2023 13:53:36 -0400 Subject: data: tidy yaml error message construction, cut long strings ...Using maxStringLength, which is more than a bit annoying, because this isn't the same cut() algorithm we just added, looks bulkier, and can't be customized. But that's the cost of using util.inspect() here. It's better than displaying the entire long string or handling line breaks poorly. FOR NOW. --- src/data/yaml.js | 38 +++++++++++++++++++++++++++++++------- 1 file changed, 31 insertions(+), 7 deletions(-) (limited to 'src/data/yaml.js') diff --git a/src/data/yaml.js b/src/data/yaml.js index 0734d539..49e05266 100644 --- a/src/data/yaml.js +++ b/src/data/yaml.js @@ -38,8 +38,8 @@ import { // --> General supporting stuff -function inspect(value) { - return nodeInspect(value, {colors: ENABLE_COLOR}); +function inspect(value, opts = {}) { + return nodeInspect(value, {colors: ENABLE_COLOR, ...opts}); } // --> YAML data repository structure constants @@ -308,7 +308,12 @@ export class FieldCombinationError extends Error { constructor(fields, message) { const fieldNames = Object.keys(fields); - const mainMessage = `Don't combine ${fieldNames.map(field => colors.red(field)).join(', ')}`; + const fieldNamesText = + fieldNames + .map(field => colors.red(field)) + .join(', '); + + const mainMessage = `Don't combine ${fieldNamesText}`; const causeMessage = (typeof message === 'function' @@ -330,7 +335,12 @@ export class FieldCombinationError extends Error { export class FieldValueAggregateError extends AggregateError { constructor(thingConstructor, errors) { - super(errors, `Errors processing field values for ${colors.green(thingConstructor.name)}`); + const constructorText = + colors.green(thingConstructor.name); + + super( + errors, + `Errors processing field values for ${constructorText}`); } } @@ -341,8 +351,17 @@ export class FieldValueError extends Error { ? caughtError.cause : caughtError); + const fieldText = + colors.green(`"${field}"`); + + const propertyText = + colors.green(property); + + const valueText = + inspect(value, {maxStringLength: 40}); + super( - `Failed to set ${colors.green(`"${field}"`)} field (${colors.green(property)}) to ${inspect(value)}`, + `Failed to set ${fieldText} field (${propertyText}) to ${valueText}`, {cause}); } } @@ -354,13 +373,18 @@ export class SkippedFieldsSummaryError extends Error { const lines = entries.map(([field, value]) => ` - ${field}: ` + - inspect(value) + inspect(value, {maxStringLength: 70}) .split('\n') .map((line, index) => index === 0 ? line : ` ${line}`) .join('\n')); + const numFieldsText = + (entries.length === 1 + ? `1 field` + : `${entries.length} fields`); + super( - colors.bright(colors.yellow(`Altogether, skipped ${entries.length === 1 ? `1 field` : `${entries.length} fields`}:\n`)) + + colors.bright(colors.yellow(`Altogether, skipped ${numFieldsText}:\n`)) + lines.join('\n') + '\n' + colors.bright(colors.yellow(`See above errors for details.`))); } -- cgit 1.3.0-6-gf8a5 From f87fa920f91d36424e4613ac5da50f46418f4b19 Mon Sep 17 00:00:00 2001 From: "(quasar) nebula" Date: Mon, 20 Nov 2023 14:31:58 -0400 Subject: data, util: principle "translucent errors" & applications --- src/data/yaml.js | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) (limited to 'src/data/yaml.js') diff --git a/src/data/yaml.js b/src/data/yaml.js index 49e05266..dddf5fb2 100644 --- a/src/data/yaml.js +++ b/src/data/yaml.js @@ -334,6 +334,8 @@ export class FieldCombinationError extends Error { } export class FieldValueAggregateError extends AggregateError { + [Symbol.for('hsmusic.aggregate.translucent')] = true; + constructor(thingConstructor, errors) { const constructorText = colors.green(thingConstructor.name); @@ -1162,7 +1164,10 @@ export async function loadAndProcessDataDocuments({dataPath}) { for (const dataStep of dataSteps) { await processDataAggregate.nestAsync( - {message: `Errors during data step: ${colors.bright(dataStep.title)}`}, + { + message: `Errors during data step: ${colors.bright(dataStep.title)}`, + translucent: true, + }, async ({call, callAsync, map, mapAsync, push}) => { const {documentMode} = dataStep; @@ -1407,7 +1412,7 @@ export async function loadAndProcessDataDocuments({dataPath}) { switch (documentMode) { case documentModes.headerAndEntries: - map(yamlResults, {message: `Errors processing documents in data files`}, + map(yamlResults, {message: `Errors processing documents in data files`, translucent: true}, decorateErrorWithFile(({documents}) => { const headerDocument = documents[0]; const entryDocuments = documents.slice(1).filter(Boolean); -- cgit 1.3.0-6-gf8a5 From 765e039ef94f7cd1365ba9f211bd686beab5e8ec Mon Sep 17 00:00:00 2001 From: "(quasar) nebula" Date: Mon, 20 Nov 2023 19:31:05 -0400 Subject: data: move accent-parsing regex out of parseContributors Also use named capturing groups (and non-capturing groups) for generally better regex form. --- src/data/yaml.js | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) (limited to 'src/data/yaml.js') diff --git a/src/data/yaml.js b/src/data/yaml.js index 1d35bae8..67cd8db7 100644 --- a/src/data/yaml.js +++ b/src/data/yaml.js @@ -717,26 +717,28 @@ export function parseAdditionalFiles(array) { })); } -export function parseContributors(contributors) { +const extractAccentRegex = + /^(?
.*?)(?: \((?.*)\))?$/; + +export function parseContributors(contributionStrings) { // If this isn't something we can parse, just return it as-is. // The Thing object's validators will handle the data error better // than we're able to here. - if (!Array.isArray(contributors)) { - return contributors; + if (!Array.isArray(contributionStrings)) { + return contributionStrings; } - contributors = contributors.map((contrib) => { - if (typeof contrib !== 'string') return contrib; + return contributionStrings.map(contribString => { + if (typeof contribString !== 'string') return contribString; - const match = contrib.match(/^(.*?)( \((.*)\))?$/); - if (!match) return contrib; + const match = contribString.match(extractAccentRegex); + if (!match) return contribString; - const who = match[1]; - const what = match[3] || null; - return {who, what}; + return { + who: match.groups.main, + what: match.groups.accent ?? null, + }; }); - - return contributors; } function parseDimensions(string) { -- cgit 1.3.0-6-gf8a5 From bde469f4cede426bd9baa8981b876e82ae290972 Mon Sep 17 00:00:00 2001 From: "(quasar) nebula" Date: Mon, 20 Nov 2023 19:32:03 -0400 Subject: data: add additionalNames wiki property ("Additional Names") --- src/data/yaml.js | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) (limited to 'src/data/yaml.js') diff --git a/src/data/yaml.js b/src/data/yaml.js index 67cd8db7..cde4413b 100644 --- a/src/data/yaml.js +++ b/src/data/yaml.js @@ -436,6 +436,7 @@ export const processTrackSectionDocument = makeProcessDocument(T.TrackSectionHel export const processTrackDocument = makeProcessDocument(T.Track, { fieldTransformations: { + 'Additional Names': parseAdditionalNames, 'Duration': parseDuration, 'Date First Released': (value) => new Date(value), @@ -457,6 +458,7 @@ export const processTrackDocument = makeProcessDocument(T.Track, { propertyFieldMapping: { name: 'Track', directory: 'Directory', + additionalNames: 'Additional Names', duration: 'Duration', color: 'Color', urls: 'URLs', @@ -741,6 +743,24 @@ export function parseContributors(contributionStrings) { }); } +export function parseAdditionalNames(additionalNameStrings) { + if (!Array.isArray(additionalNameStrings)) { + return additionalNameStrings; + } + + return additionalNameStrings.map(additionalNameString => { + if (typeof additionalNameString !== 'string') return additionalNameString; + + const match = additionalNameString.match(extractAccentRegex); + if (!match) return additionalNameString; + + return { + name: match.groups.main, + annotation: match.groups.accent ?? null, + }; + }); +} + function parseDimensions(string) { // It's technically possible to pass an array like [30, 40] through here. // That's not really an issue because if it isn't of the appropriate shape, -- cgit 1.3.0-6-gf8a5 From c9847299d6a31b5fb39f82f80c92e80d53c44234 Mon Sep 17 00:00:00 2001 From: "(quasar) nebula" Date: Tue, 21 Nov 2023 07:20:16 -0400 Subject: data: parse contribs & additional names from object shape --- src/data/yaml.js | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) (limited to 'src/data/yaml.js') diff --git a/src/data/yaml.js b/src/data/yaml.js index cde4413b..5da66c93 100644 --- a/src/data/yaml.js +++ b/src/data/yaml.js @@ -730,11 +730,14 @@ export function parseContributors(contributionStrings) { return contributionStrings; } - return contributionStrings.map(contribString => { - if (typeof contribString !== 'string') return contribString; + return contributionStrings.map(item => { + if (typeof item === 'object' && item['Who']) + return {who: item['Who'], what: item['What'] ?? null}; - const match = contribString.match(extractAccentRegex); - if (!match) return contribString; + if (typeof item !== 'string') return item; + + const match = item.match(extractAccentRegex); + if (!match) return item; return { who: match.groups.main, @@ -748,11 +751,14 @@ export function parseAdditionalNames(additionalNameStrings) { return additionalNameStrings; } - return additionalNameStrings.map(additionalNameString => { - if (typeof additionalNameString !== 'string') return additionalNameString; + return additionalNameStrings.map(item => { + if (typeof item === 'object' && item['Name']) + return {name: item['Name'], annotation: item['Annotation'] ?? null}; + + if (typeof item !== 'string') return item; - const match = additionalNameString.match(extractAccentRegex); - if (!match) return additionalNameString; + const match = item.match(extractAccentRegex); + if (!match) return item; return { name: match.groups.main, -- cgit 1.3.0-6-gf8a5 From a65693efe23b97da173463f207979f81767d791c Mon Sep 17 00:00:00 2001 From: "(quasar) nebula" Date: Mon, 27 Nov 2023 21:13:39 -0400 Subject: data, content: embed scripts on static pages --- src/data/yaml.js | 1 + 1 file changed, 1 insertion(+) (limited to 'src/data/yaml.js') diff --git a/src/data/yaml.js b/src/data/yaml.js index 2c600341..dda06949 100644 --- a/src/data/yaml.js +++ b/src/data/yaml.js @@ -646,6 +646,7 @@ export const processStaticPageDocument = makeProcessDocument(T.StaticPage, { directory: 'Directory', stylesheet: 'Style', + script: 'Script', content: 'Content', }, }); -- cgit 1.3.0-6-gf8a5 From 084b5423d2a4fc60a91dd4aeb24ff0cd4d870fbc Mon Sep 17 00:00:00 2001 From: "(quasar) nebula" Date: Tue, 28 Nov 2023 13:04:32 -0400 Subject: data: tweak track album messaging in errors/inspect --- src/data/yaml.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'src/data/yaml.js') diff --git a/src/data/yaml.js b/src/data/yaml.js index dda06949..27d8721f 100644 --- a/src/data/yaml.js +++ b/src/data/yaml.js @@ -1625,8 +1625,7 @@ export function filterDuplicateDirectories(wikiData) { call(() => { throw new Error( `Duplicate directory ${colors.green(directory)}:\n` + - places.map((thing) => ` - ` + inspect(thing)).join('\n') - ); + places.map(thing => ` - ` + inspect(thing)).join('\n')); }); } -- cgit 1.3.0-6-gf8a5 From 7215aef076f9734f35dc4f25e54fbe2371630c5f Mon Sep 17 00:00:00 2001 From: "(quasar) nebula" Date: Tue, 28 Nov 2023 13:23:17 -0400 Subject: data, test: album.trackData -> album.ownTrackData --- src/data/yaml.js | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'src/data/yaml.js') diff --git a/src/data/yaml.js b/src/data/yaml.js index 27d8721f..82b7faf2 100644 --- a/src/data/yaml.js +++ b/src/data/yaml.js @@ -936,6 +936,7 @@ export const dataSteps = [ // an individual section before applying it, since those are just // generic objects; they aren't Things in and of themselves.) const trackSections = []; + const ownTrackData = []; let currentTrackSection = { name: `Default Track Section`, @@ -970,13 +971,16 @@ export const dataSteps = [ entry.dataSourceAlbum = albumRef; + ownTrackData.push(entry); currentTrackSection.tracks.push(Thing.getReference(entry)); } closeCurrentTrackSection(); - album.trackSections = trackSections; albumData.push(album); + + album.trackSections = trackSections; + album.ownTrackData = ownTrackData; } return {albumData, trackData}; @@ -1551,7 +1555,7 @@ export function linkWikiDataArrays(wikiData, { assignWikiData([WD.wikiInfo], 'groupData'); - assignWikiData(WD.albumData, 'artistData', 'artTagData', 'groupData', 'trackData'); + assignWikiData(WD.albumData, 'artistData', 'artTagData', 'groupData'); assignWikiData(WD.trackData, 'albumData', 'artistData', 'artTagData', 'flashData', 'trackData'); assignWikiData(WD.artistData, 'albumData', 'artistData', 'flashData', 'trackData'); assignWikiData(WD.groupData, 'albumData', 'groupCategoryData'); -- cgit 1.3.0-6-gf8a5