From b053fa14052aaa24883e73a3f899016f963b5d43 Mon Sep 17 00:00:00 2001 From: "(quasar) nebula" Date: Tue, 14 Nov 2023 21:45:08 -0400 Subject: data: expose CacheableObject directly via #cacheable-object import --- src/data/yaml.js | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) (limited to 'src/data/yaml.js') diff --git a/src/data/yaml.js b/src/data/yaml.js index 1d35bae8..986f25d1 100644 --- a/src/data/yaml.js +++ b/src/data/yaml.js @@ -7,15 +7,13 @@ import {inspect as nodeInspect} from 'node:util'; import yaml from 'js-yaml'; +import CacheableObject, {CacheableObjectPropertyValueError} + from '#cacheable-object'; import {colors, ENABLE_COLOR, logInfo, logWarn} from '#cli'; import find, {bindFind} from '#find'; import {traverse} from '#node-utils'; -import T, { - CacheableObject, - CacheableObjectPropertyValueError, - Thing, -} from '#things'; +import T, {Thing} from '#things'; import { annotateErrorWithFile, -- cgit 1.3.0-6-gf8a5 From 362dc0619b93d74ad34df1bfbfd9ebc632fa5156 Mon Sep 17 00:00:00 2001 From: "(quasar) nebula" Date: Tue, 14 Nov 2023 22:49:51 -0400 Subject: data, yaml: catch commentary artist ref errors --- src/data/yaml.js | 55 +++++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 41 insertions(+), 14 deletions(-) (limited to 'src/data/yaml.js') diff --git a/src/data/yaml.js b/src/data/yaml.js index 986f25d1..843e70b3 100644 --- a/src/data/yaml.js +++ b/src/data/yaml.js @@ -28,6 +28,7 @@ import { } from '#sugar'; import { + commentaryRegex, sortAlbumsTracksChronologically, sortAlphabetically, sortChronologically, @@ -1616,6 +1617,7 @@ export function filterReferenceErrors(wikiData) { bannerArtistContribs: '_contrib', groups: 'group', artTags: 'artTag', + commentary: '_commentary', }], ['trackData', processTrackDocument, { @@ -1626,6 +1628,7 @@ export function filterReferenceErrors(wikiData) { sampledTracks: '_trackNotRerelease', artTags: 'artTag', originalReleaseTrack: '_trackNotRerelease', + commentary: '_commentary', }], ['groupCategoryData', processGroupCategoryDocument, { @@ -1675,7 +1678,19 @@ export function filterReferenceErrors(wikiData) { nest({message: `Reference errors in ${inspect(thing)}`}, ({nest, push, filter}) => { for (const [property, findFnKey] of Object.entries(propSpec)) { - const value = CacheableObject.getUpdateValue(thing, property); + let value = CacheableObject.getUpdateValue(thing, property); + let writeProperty = true; + + switch (findFnKey) { + case '_commentary': + if (value) { + value = + Array.from(value.matchAll(commentaryRegex)) + .map(({groups}) => groups.artistReference); + } + writeProperty = false; + break; + } if (value === undefined) { push(new TypeError(`Property ${colors.red(property)} isn't valid for ${colors.green(thing.constructor.name)}`)); @@ -1688,19 +1703,25 @@ export function filterReferenceErrors(wikiData) { let findFn; + const findArtistOrAlias = artistRef => { + const alias = find.artist(artistRef, wikiData.artistAliasData, {mode: 'quiet'}); + if (alias) { + // No need to check if the original exists here. Aliases are automatically + // created from a field on the original, so the original certainly exists. + const original = alias.aliasedArtist; + throw new Error(`Reference ${colors.red(artistRef)} is to an alias, should be ${colors.green(original.name)}`); + } + + return boundFind.artist(artistRef); + }; + switch (findFnKey) { - case '_contrib': - findFn = contribRef => { - const alias = find.artist(contribRef.who, wikiData.artistAliasData, {mode: 'quiet'}); - if (alias) { - // No need to check if the original exists here. Aliases are automatically - // created from a field on the original, so the original certainly exists. - const original = alias.aliasedArtist; - throw new Error(`Reference ${colors.red(contribRef.who)} is to an alias, should be ${colors.green(original.name)}`); - } + case '_commentary': + findFn = findArtistOrAlias; + break; - return boundFind.artist(contribRef.who); - }; + case '_contrib': + findFn = contribRef => findArtistOrAlias(contribRef.who); break; case '_homepageSourceGroup': @@ -1781,8 +1802,10 @@ export function filterReferenceErrors(wikiData) { ? `Reference errors` + fieldPropertyMessage + findFnMessage : `Reference error` + fieldPropertyMessage + findFnMessage); + let newPropertyValue = value; + if (Array.isArray(value)) { - thing[property] = filter( + newPropertyValue = filter( value, decorateErrorWithIndex(suppress(findFn)), {message: errorMessage}); @@ -1792,11 +1815,15 @@ export function filterReferenceErrors(wikiData) { try { call(findFn, value); } catch (error) { - thing[property] = null; + newPropertyValue = null; throw error; } })); } + + if (writeProperty) { + thing[property] = newPropertyValue; + } } }); } -- cgit 1.3.0-6-gf8a5 From a34b8d027866fbe858a4d2ff3543bc84c9d5983a Mon Sep 17 00:00:00 2001 From: "(quasar) nebula" Date: Fri, 17 Nov 2023 06:53:34 -0400 Subject: data, yaml, content: support multiple artists per commentary entry --- src/data/yaml.js | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) (limited to 'src/data/yaml.js') diff --git a/src/data/yaml.js b/src/data/yaml.js index 843e70b3..0734d539 100644 --- a/src/data/yaml.js +++ b/src/data/yaml.js @@ -21,6 +21,7 @@ import { decorateErrorWithIndex, decorateErrorWithAnnotation, empty, + filterAggregate, filterProperties, openAggregate, showAggregate, @@ -1686,8 +1687,10 @@ export function filterReferenceErrors(wikiData) { if (value) { value = Array.from(value.matchAll(commentaryRegex)) - .map(({groups}) => groups.artistReference); + .map(({groups}) => groups.artistReferences) + .map(text => text.split(',').map(text => text.trim())); } + writeProperty = false; break; } @@ -1804,11 +1807,22 @@ export function filterReferenceErrors(wikiData) { let newPropertyValue = value; - if (Array.isArray(value)) { + if (findFnKey === '_commentary') { + // Commentary doesn't write a property value, so no need to set. + filter( + value, {message: errorMessage}, + decorateErrorWithIndex(refs => + (refs.length === 1 + ? suppress(findFn)(refs[0]) + : filterAggregate( + refs, {message: `Errors in entry's artist references`}, + decorateErrorWithIndex(suppress(findFn))) + .aggregate + .close()))); + } else if (Array.isArray(value)) { newPropertyValue = filter( - value, - decorateErrorWithIndex(suppress(findFn)), - {message: errorMessage}); + value, {message: errorMessage}, + decorateErrorWithIndex(suppress(findFn))); } else { nest({message: errorMessage}, suppress(({call}) => { -- cgit 1.3.0-6-gf8a5 From 9a35630d91ebd7227994a1009487794a65ec38e1 Mon Sep 17 00:00:00 2001 From: "(quasar) nebula" Date: Mon, 20 Nov 2023 13:53:36 -0400 Subject: data: tidy yaml error message construction, cut long strings ...Using maxStringLength, which is more than a bit annoying, because this isn't the same cut() algorithm we just added, looks bulkier, and can't be customized. But that's the cost of using util.inspect() here. It's better than displaying the entire long string or handling line breaks poorly. FOR NOW. --- src/data/yaml.js | 38 +++++++++++++++++++++++++++++++------- 1 file changed, 31 insertions(+), 7 deletions(-) (limited to 'src/data/yaml.js') diff --git a/src/data/yaml.js b/src/data/yaml.js index 0734d539..49e05266 100644 --- a/src/data/yaml.js +++ b/src/data/yaml.js @@ -38,8 +38,8 @@ import { // --> General supporting stuff -function inspect(value) { - return nodeInspect(value, {colors: ENABLE_COLOR}); +function inspect(value, opts = {}) { + return nodeInspect(value, {colors: ENABLE_COLOR, ...opts}); } // --> YAML data repository structure constants @@ -308,7 +308,12 @@ export class FieldCombinationError extends Error { constructor(fields, message) { const fieldNames = Object.keys(fields); - const mainMessage = `Don't combine ${fieldNames.map(field => colors.red(field)).join(', ')}`; + const fieldNamesText = + fieldNames + .map(field => colors.red(field)) + .join(', '); + + const mainMessage = `Don't combine ${fieldNamesText}`; const causeMessage = (typeof message === 'function' @@ -330,7 +335,12 @@ export class FieldCombinationError extends Error { export class FieldValueAggregateError extends AggregateError { constructor(thingConstructor, errors) { - super(errors, `Errors processing field values for ${colors.green(thingConstructor.name)}`); + const constructorText = + colors.green(thingConstructor.name); + + super( + errors, + `Errors processing field values for ${constructorText}`); } } @@ -341,8 +351,17 @@ export class FieldValueError extends Error { ? caughtError.cause : caughtError); + const fieldText = + colors.green(`"${field}"`); + + const propertyText = + colors.green(property); + + const valueText = + inspect(value, {maxStringLength: 40}); + super( - `Failed to set ${colors.green(`"${field}"`)} field (${colors.green(property)}) to ${inspect(value)}`, + `Failed to set ${fieldText} field (${propertyText}) to ${valueText}`, {cause}); } } @@ -354,13 +373,18 @@ export class SkippedFieldsSummaryError extends Error { const lines = entries.map(([field, value]) => ` - ${field}: ` + - inspect(value) + inspect(value, {maxStringLength: 70}) .split('\n') .map((line, index) => index === 0 ? line : ` ${line}`) .join('\n')); + const numFieldsText = + (entries.length === 1 + ? `1 field` + : `${entries.length} fields`); + super( - colors.bright(colors.yellow(`Altogether, skipped ${entries.length === 1 ? `1 field` : `${entries.length} fields`}:\n`)) + + colors.bright(colors.yellow(`Altogether, skipped ${numFieldsText}:\n`)) + lines.join('\n') + '\n' + colors.bright(colors.yellow(`See above errors for details.`))); } -- cgit 1.3.0-6-gf8a5 From f87fa920f91d36424e4613ac5da50f46418f4b19 Mon Sep 17 00:00:00 2001 From: "(quasar) nebula" Date: Mon, 20 Nov 2023 14:31:58 -0400 Subject: data, util: principle "translucent errors" & applications --- src/data/yaml.js | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) (limited to 'src/data/yaml.js') diff --git a/src/data/yaml.js b/src/data/yaml.js index 49e05266..dddf5fb2 100644 --- a/src/data/yaml.js +++ b/src/data/yaml.js @@ -334,6 +334,8 @@ export class FieldCombinationError extends Error { } export class FieldValueAggregateError extends AggregateError { + [Symbol.for('hsmusic.aggregate.translucent')] = true; + constructor(thingConstructor, errors) { const constructorText = colors.green(thingConstructor.name); @@ -1162,7 +1164,10 @@ export async function loadAndProcessDataDocuments({dataPath}) { for (const dataStep of dataSteps) { await processDataAggregate.nestAsync( - {message: `Errors during data step: ${colors.bright(dataStep.title)}`}, + { + message: `Errors during data step: ${colors.bright(dataStep.title)}`, + translucent: true, + }, async ({call, callAsync, map, mapAsync, push}) => { const {documentMode} = dataStep; @@ -1407,7 +1412,7 @@ export async function loadAndProcessDataDocuments({dataPath}) { switch (documentMode) { case documentModes.headerAndEntries: - map(yamlResults, {message: `Errors processing documents in data files`}, + map(yamlResults, {message: `Errors processing documents in data files`, translucent: true}, decorateErrorWithFile(({documents}) => { const headerDocument = documents[0]; const entryDocuments = documents.slice(1).filter(Boolean); -- cgit 1.3.0-6-gf8a5