From b053fa14052aaa24883e73a3f899016f963b5d43 Mon Sep 17 00:00:00 2001 From: "(quasar) nebula" Date: Tue, 14 Nov 2023 21:45:08 -0400 Subject: data: expose CacheableObject directly via #cacheable-object import --- src/data/things/index.js | 5 ----- src/data/yaml.js | 8 +++----- 2 files changed, 3 insertions(+), 10 deletions(-) (limited to 'src/data') diff --git a/src/data/things/index.js b/src/data/things/index.js index 4ea1f007..d1143b0a 100644 --- a/src/data/things/index.js +++ b/src/data/things/index.js @@ -22,11 +22,6 @@ import * as wikiInfoClasses from './wiki-info.js'; export {default as Thing} from './thing.js'; -export { - default as CacheableObject, - CacheableObjectPropertyValueError, -} from './cacheable-object.js'; - const allClassLists = { 'album.js': albumClasses, 'art-tag.js': artTagClasses, diff --git a/src/data/yaml.js b/src/data/yaml.js index 1d35bae8..986f25d1 100644 --- a/src/data/yaml.js +++ b/src/data/yaml.js @@ -7,15 +7,13 @@ import {inspect as nodeInspect} from 'node:util'; import yaml from 'js-yaml'; +import CacheableObject, {CacheableObjectPropertyValueError} + from '#cacheable-object'; import {colors, ENABLE_COLOR, logInfo, logWarn} from '#cli'; import find, {bindFind} from '#find'; import {traverse} from '#node-utils'; -import T, { - CacheableObject, - CacheableObjectPropertyValueError, - Thing, -} from '#things'; +import T, {Thing} from '#things'; import { annotateErrorWithFile, -- cgit 1.3.0-6-gf8a5 From 764b6a3a2c89d0e4d948862bfeb546d33bbf45e6 Mon Sep 17 00:00:00 2001 From: "(quasar) nebula" Date: Tue, 14 Nov 2023 21:45:35 -0400 Subject: data: generic composite dependency comments --- src/data/composite/control-flow/index.js | 5 +++++ src/data/composite/data/index.js | 5 +++++ src/data/composite/wiki-data/index.js | 7 +++++++ src/data/composite/wiki-properties/index.js | 5 +++++ 4 files changed, 22 insertions(+) (limited to 'src/data') diff --git a/src/data/composite/control-flow/index.js b/src/data/composite/control-flow/index.js index dfc53db7..7fad88b2 100644 --- a/src/data/composite/control-flow/index.js +++ b/src/data/composite/control-flow/index.js @@ -1,3 +1,8 @@ +// #composite/control-flow +// +// No entries depend on any other entries, except siblings in this directory. +// + export {default as exitWithoutDependency} from './exitWithoutDependency.js'; export {default as exitWithoutUpdateValue} from './exitWithoutUpdateValue.js'; export {default as exposeConstant} from './exposeConstant.js'; diff --git a/src/data/composite/data/index.js b/src/data/composite/data/index.js index ecd05129..db1c37cc 100644 --- a/src/data/composite/data/index.js +++ b/src/data/composite/data/index.js @@ -1,3 +1,8 @@ +// #composite/data +// +// Entries here may depend on entries in #composite/control-flow. +// + export {default as excludeFromList} from './excludeFromList.js'; export {default as fillMissingListItems} from './fillMissingListItems.js'; export {default as withFlattenedList} from './withFlattenedList.js'; diff --git a/src/data/composite/wiki-data/index.js b/src/data/composite/wiki-data/index.js index 1d0400fc..df50a2db 100644 --- a/src/data/composite/wiki-data/index.js +++ b/src/data/composite/wiki-data/index.js @@ -1,6 +1,13 @@ +// #composite/wiki-data +// +// Entries here may depend on entries in #composite/control-flow and in +// #composite/data. +// + export {default as exitWithoutContribs} from './exitWithoutContribs.js'; export {default as inputThingClass} from './inputThingClass.js'; export {default as inputWikiData} from './inputWikiData.js'; +export {default as withParsedCommentaryEntries} from './withParsedCommentaryEntries.js'; export {default as withResolvedContribs} from './withResolvedContribs.js'; export {default as withResolvedReference} from './withResolvedReference.js'; export {default as withResolvedReferenceList} from './withResolvedReferenceList.js'; diff --git a/src/data/composite/wiki-properties/index.js b/src/data/composite/wiki-properties/index.js index 2462b047..3a8b51d5 100644 --- a/src/data/composite/wiki-properties/index.js +++ b/src/data/composite/wiki-properties/index.js @@ -1,3 +1,8 @@ +// #composite/wiki-properties +// +// Entries here may depend on entries in #composite/control-flow, +// #composite/data, and #composite/wiki-data. + export {default as additionalFiles} from './additionalFiles.js'; export {default as color} from './color.js'; export {default as commentary} from './commentary.js'; -- cgit 1.3.0-6-gf8a5 From 6deea0629a3f3b9985d205d2f3a048893ea938c9 Mon Sep 17 00:00:00 2001 From: "(quasar) nebula" Date: Tue, 14 Nov 2023 22:20:45 -0400 Subject: data, test: withParsedCommentaryEntries --- .../wiki-data/withParsedCommentaryEntries.js | 181 +++++++++++++++++++++ src/data/composite/wiki-properties/commentary.js | 32 +++- 2 files changed, 206 insertions(+), 7 deletions(-) create mode 100644 src/data/composite/wiki-data/withParsedCommentaryEntries.js (limited to 'src/data') diff --git a/src/data/composite/wiki-data/withParsedCommentaryEntries.js b/src/data/composite/wiki-data/withParsedCommentaryEntries.js new file mode 100644 index 00000000..5bd72dc9 --- /dev/null +++ b/src/data/composite/wiki-data/withParsedCommentaryEntries.js @@ -0,0 +1,181 @@ +import {input, templateCompositeFrom} from '#composite'; +import find from '#find'; +import {stitchArrays} from '#sugar'; +import {isCommentary} from '#validators'; + +import {fillMissingListItems, withPropertiesFromList} from '#composite/data'; + +import withResolvedReferenceList from './withResolvedReferenceList.js'; + +// Matches in roughly the format: +// +// artistReference: (annotation, date) +// +// where capturing group "annotation" can be any text at all, except that the +// last entry (past a comma or the only content within parentheses), if parsed +// as a date, is the capturing group "date". "Parsing as a date" means one of +// these formats: +// +// * "25 December 2019" - one or two number digits, followed by any text, +// followed by four number digits +// * "12/25/2019" - one or two number digits, a slash, one or two number +// digits, a slash, and two to four number digits +// +// The artist reference can optionally be boldface (in ), which will be +// captured as non-null in "boldfaceArtist". Otherwise it is all the characters +// between and and is captured in "artistReference" and is either the +// name of an artist or an "artist:directory"-style reference. +// +export const commentaryRegex = + /^(?)?(?.+):(?:<\/b>)?<\/i>(?: \((?(?:.*?(?=[,)]))*?)(?:,? ?(?[0-9]{1,2} [^,]*[0-9]{4,4}|[0-9]{1,2}\/[0-9]{1,2}\/[0-9]{2,4}))?\))?/gm; + +export default templateCompositeFrom({ + annotation: `withParsedCommentaryEntries`, + + inputs: { + from: input({validate: isCommentary}), + }, + + outputs: ['#parsedCommentaryEntries'], + + steps: () => [ + { + dependencies: [input('from')], + + compute: (continuation, { + [input('from')]: commentaryText, + }) => continuation({ + ['#rawMatches']: + Array.from(commentaryText.matchAll(commentaryRegex)), + }), + }, + + withPropertiesFromList({ + list: '#rawMatches', + properties: input.value([ + '0', // The entire match as a string. + 'groups', + 'index', + ]), + }).outputs({ + '#rawMatches.0': '#rawMatches.text', + '#rawMatches.groups': '#rawMatches.groups', + '#rawMatches.index': '#rawMatches.startIndex', + }), + + { + dependencies: [ + '#rawMatches.text', + '#rawMatches.startIndex', + ], + + compute: (continuation, { + ['#rawMatches.text']: text, + ['#rawMatches.startIndex']: startIndex, + }) => continuation({ + ['#rawMatches.endIndex']: + stitchArrays({text, startIndex}) + .map(({text, startIndex}) => startIndex + text.length), + }), + }, + + { + dependencies: [ + input('from'), + '#rawMatches.startIndex', + '#rawMatches.endIndex', + ], + + compute: (continuation, { + [input('from')]: commentaryText, + ['#rawMatches.startIndex']: startIndex, + ['#rawMatches.endIndex']: endIndex, + }) => continuation({ + ['#entries.body']: + stitchArrays({startIndex, endIndex}) + .map(({endIndex}, index, stitched) => + (index === stitched.length - 1 + ? commentaryText.slice(endIndex) + : commentaryText.slice( + endIndex, + stitched[index + 1].startIndex))) + .map(body => body.trim()), + }), + }, + + withPropertiesFromList({ + list: '#rawMatches.groups', + prefix: input.value('#entries'), + properties: input.value([ + 'artistReference', + 'boldfaceArtist', + 'annotation', + 'date', + ]), + }), + + // The artistReference group will always have a value, since it's required + // for the line to match in the first place. + + withResolvedReferenceList({ + list: '#entries.artistReference', + data: 'artistData', + find: input.value(find.artist), + notFoundMode: input.value('null'), + }).outputs({ + '#resolvedReferenceList': '#entries.artist', + }), + + { + dependencies: ['#entries.boldfaceArtist'], + compute: (continuation, { + ['#entries.boldfaceArtist']: boldfaceArtist, + }) => continuation({ + ['#entries.boldfaceArtist']: + boldfaceArtist.map(boldface => boldface ? true : false), + }), + }, + + fillMissingListItems({ + list: '#entries.annotation', + fill: input.value(null), + }), + + { + dependencies: ['#entries.date'], + compute: (continuation, { + ['#entries.date']: date, + }) => continuation({ + ['#entries.date']: + date.map(date => date ? new Date(date) : null), + }), + }, + + { + dependencies: [ + '#entries.artist', + '#entries.boldfaceArtist', + '#entries.annotation', + '#entries.date', + '#entries.body', + ], + + compute: (continuation, { + ['#entries.artist']: artist, + ['#entries.boldfaceArtist']: boldfaceArtist, + ['#entries.annotation']: annotation, + ['#entries.date']: date, + ['#entries.body']: body, + }) => continuation({ + ['#parsedCommentaryEntries']: + stitchArrays({ + artist, + boldfaceArtist, + annotation, + date, + body, + }), + }), + }, + ], +}); diff --git a/src/data/composite/wiki-properties/commentary.js b/src/data/composite/wiki-properties/commentary.js index fbea9d5c..cd6b7ac4 100644 --- a/src/data/composite/wiki-properties/commentary.js +++ b/src/data/composite/wiki-properties/commentary.js @@ -1,12 +1,30 @@ // Artist commentary! Generally present on tracks and albums. +import {input, templateCompositeFrom} from '#composite'; import {isCommentary} from '#validators'; -// TODO: Not templateCompositeFrom. +import {exitWithoutDependency, exposeDependency} + from '#composite/control-flow'; +import {withParsedCommentaryEntries} from '#composite/wiki-data'; -export default function() { - return { - flags: {update: true, expose: true}, - update: {validate: isCommentary}, - }; -} +export default templateCompositeFrom({ + annotation: `commentary`, + + compose: false, + + steps: () => [ + exitWithoutDependency({ + dependency: input.updateValue({validate: isCommentary}), + mode: input.value('falsy'), + value: input.value(null), + }), + + withParsedCommentaryEntries({ + from: input.updateValue(), + }), + + exposeDependency({ + dependency: '#parsedCommentaryEntries', + }), + ], +}); -- cgit 1.3.0-6-gf8a5 From 09a4af31a3f8207dfe114926b0dbf27eeddf7de9 Mon Sep 17 00:00:00 2001 From: "(quasar) nebula" Date: Tue, 14 Nov 2023 22:26:25 -0400 Subject: data: commentatorArtists: use withParsedCommentaryEntries --- .../wiki-properties/commentatorArtists.js | 41 +++++++--------------- 1 file changed, 13 insertions(+), 28 deletions(-) (limited to 'src/data') diff --git a/src/data/composite/wiki-properties/commentatorArtists.js b/src/data/composite/wiki-properties/commentatorArtists.js index 52aeb868..8720e66d 100644 --- a/src/data/composite/wiki-properties/commentatorArtists.js +++ b/src/data/composite/wiki-properties/commentatorArtists.js @@ -1,13 +1,12 @@ -// This one's kinda tricky: it parses artist "references" from the -// commentary content, and finds the matching artist for each reference. +// List of artists referenced in commentary entries. // This is mostly useful for credits and listings on artist pages. import {input, templateCompositeFrom} from '#composite'; -import find from '#find'; import {unique} from '#sugar'; import {exitWithoutDependency} from '#composite/control-flow'; -import {withResolvedReferenceList} from '#composite/wiki-data'; +import {withPropertyFromList} from '#composite/data'; +import {withParsedCommentaryEntries} from '#composite/wiki-data'; export default templateCompositeFrom({ annotation: `commentatorArtists`, @@ -21,35 +20,21 @@ export default templateCompositeFrom({ value: input.value([]), }), - { - dependencies: ['commentary'], - compute: (continuation, {commentary}) => - continuation({ - '#artistRefs': - Array.from( - commentary - .replace(/<\/?b>/g, '') - .matchAll(/(?.*?):<\/i>/g)) - .map(({groups: {who}}) => who), - }), - }, + withParsedCommentaryEntries({ + from: 'commentary', + }), - withResolvedReferenceList({ - list: '#artistRefs', - data: 'artistData', - find: input.value(find.artist), + withPropertyFromList({ + list: '#parsedCommentaryEntries', + property: input.value('artist'), }).outputs({ - '#resolvedReferenceList': '#artists', + '#parsedCommentaryEntries.artist': '#artists', }), { - flags: {expose: true}, - - expose: { - dependencies: ['#artists'], - compute: ({'#artists': artists}) => - unique(artists), - }, + dependencies: ['#artists'], + compute: ({'#artists': artists}) => + unique(artists.filter(artist => artist !== null)), }, ], }); -- cgit 1.3.0-6-gf8a5 From 362dc0619b93d74ad34df1bfbfd9ebc632fa5156 Mon Sep 17 00:00:00 2001 From: "(quasar) nebula" Date: Tue, 14 Nov 2023 22:49:51 -0400 Subject: data, yaml: catch commentary artist ref errors --- .../wiki-data/withParsedCommentaryEntries.js | 23 +-------- src/data/yaml.js | 55 ++++++++++++++++------ 2 files changed, 42 insertions(+), 36 deletions(-) (limited to 'src/data') diff --git a/src/data/composite/wiki-data/withParsedCommentaryEntries.js b/src/data/composite/wiki-data/withParsedCommentaryEntries.js index 5bd72dc9..9e33cdac 100644 --- a/src/data/composite/wiki-data/withParsedCommentaryEntries.js +++ b/src/data/composite/wiki-data/withParsedCommentaryEntries.js @@ -2,33 +2,12 @@ import {input, templateCompositeFrom} from '#composite'; import find from '#find'; import {stitchArrays} from '#sugar'; import {isCommentary} from '#validators'; +import {commentaryRegex} from '#wiki-data'; import {fillMissingListItems, withPropertiesFromList} from '#composite/data'; import withResolvedReferenceList from './withResolvedReferenceList.js'; -// Matches in roughly the format: -// -// artistReference: (annotation, date) -// -// where capturing group "annotation" can be any text at all, except that the -// last entry (past a comma or the only content within parentheses), if parsed -// as a date, is the capturing group "date". "Parsing as a date" means one of -// these formats: -// -// * "25 December 2019" - one or two number digits, followed by any text, -// followed by four number digits -// * "12/25/2019" - one or two number digits, a slash, one or two number -// digits, a slash, and two to four number digits -// -// The artist reference can optionally be boldface (in ), which will be -// captured as non-null in "boldfaceArtist". Otherwise it is all the characters -// between and and is captured in "artistReference" and is either the -// name of an artist or an "artist:directory"-style reference. -// -export const commentaryRegex = - /^(?)?(?.+):(?:<\/b>)?<\/i>(?: \((?(?:.*?(?=[,)]))*?)(?:,? ?(?[0-9]{1,2} [^,]*[0-9]{4,4}|[0-9]{1,2}\/[0-9]{1,2}\/[0-9]{2,4}))?\))?/gm; - export default templateCompositeFrom({ annotation: `withParsedCommentaryEntries`, diff --git a/src/data/yaml.js b/src/data/yaml.js index 986f25d1..843e70b3 100644 --- a/src/data/yaml.js +++ b/src/data/yaml.js @@ -28,6 +28,7 @@ import { } from '#sugar'; import { + commentaryRegex, sortAlbumsTracksChronologically, sortAlphabetically, sortChronologically, @@ -1616,6 +1617,7 @@ export function filterReferenceErrors(wikiData) { bannerArtistContribs: '_contrib', groups: 'group', artTags: 'artTag', + commentary: '_commentary', }], ['trackData', processTrackDocument, { @@ -1626,6 +1628,7 @@ export function filterReferenceErrors(wikiData) { sampledTracks: '_trackNotRerelease', artTags: 'artTag', originalReleaseTrack: '_trackNotRerelease', + commentary: '_commentary', }], ['groupCategoryData', processGroupCategoryDocument, { @@ -1675,7 +1678,19 @@ export function filterReferenceErrors(wikiData) { nest({message: `Reference errors in ${inspect(thing)}`}, ({nest, push, filter}) => { for (const [property, findFnKey] of Object.entries(propSpec)) { - const value = CacheableObject.getUpdateValue(thing, property); + let value = CacheableObject.getUpdateValue(thing, property); + let writeProperty = true; + + switch (findFnKey) { + case '_commentary': + if (value) { + value = + Array.from(value.matchAll(commentaryRegex)) + .map(({groups}) => groups.artistReference); + } + writeProperty = false; + break; + } if (value === undefined) { push(new TypeError(`Property ${colors.red(property)} isn't valid for ${colors.green(thing.constructor.name)}`)); @@ -1688,19 +1703,25 @@ export function filterReferenceErrors(wikiData) { let findFn; + const findArtistOrAlias = artistRef => { + const alias = find.artist(artistRef, wikiData.artistAliasData, {mode: 'quiet'}); + if (alias) { + // No need to check if the original exists here. Aliases are automatically + // created from a field on the original, so the original certainly exists. + const original = alias.aliasedArtist; + throw new Error(`Reference ${colors.red(artistRef)} is to an alias, should be ${colors.green(original.name)}`); + } + + return boundFind.artist(artistRef); + }; + switch (findFnKey) { - case '_contrib': - findFn = contribRef => { - const alias = find.artist(contribRef.who, wikiData.artistAliasData, {mode: 'quiet'}); - if (alias) { - // No need to check if the original exists here. Aliases are automatically - // created from a field on the original, so the original certainly exists. - const original = alias.aliasedArtist; - throw new Error(`Reference ${colors.red(contribRef.who)} is to an alias, should be ${colors.green(original.name)}`); - } + case '_commentary': + findFn = findArtistOrAlias; + break; - return boundFind.artist(contribRef.who); - }; + case '_contrib': + findFn = contribRef => findArtistOrAlias(contribRef.who); break; case '_homepageSourceGroup': @@ -1781,8 +1802,10 @@ export function filterReferenceErrors(wikiData) { ? `Reference errors` + fieldPropertyMessage + findFnMessage : `Reference error` + fieldPropertyMessage + findFnMessage); + let newPropertyValue = value; + if (Array.isArray(value)) { - thing[property] = filter( + newPropertyValue = filter( value, decorateErrorWithIndex(suppress(findFn)), {message: errorMessage}); @@ -1792,11 +1815,15 @@ export function filterReferenceErrors(wikiData) { try { call(findFn, value); } catch (error) { - thing[property] = null; + newPropertyValue = null; throw error; } })); } + + if (writeProperty) { + thing[property] = newPropertyValue; + } } }); } -- cgit 1.3.0-6-gf8a5 From f754a8d9187e435a761db31b5053aa2e7ba22e13 Mon Sep 17 00:00:00 2001 From: "(quasar) nebula" Date: Tue, 14 Nov 2023 23:36:37 -0400 Subject: data, test: boldfaceArtist -> artistDisplayText --- .../wiki-data/withParsedCommentaryEntries.js | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) (limited to 'src/data') diff --git a/src/data/composite/wiki-data/withParsedCommentaryEntries.js b/src/data/composite/wiki-data/withParsedCommentaryEntries.js index 9e33cdac..7b1c9484 100644 --- a/src/data/composite/wiki-data/withParsedCommentaryEntries.js +++ b/src/data/composite/wiki-data/withParsedCommentaryEntries.js @@ -87,7 +87,7 @@ export default templateCompositeFrom({ prefix: input.value('#entries'), properties: input.value([ 'artistReference', - 'boldfaceArtist', + 'artistDisplayText', 'annotation', 'date', ]), @@ -105,15 +105,10 @@ export default templateCompositeFrom({ '#resolvedReferenceList': '#entries.artist', }), - { - dependencies: ['#entries.boldfaceArtist'], - compute: (continuation, { - ['#entries.boldfaceArtist']: boldfaceArtist, - }) => continuation({ - ['#entries.boldfaceArtist']: - boldfaceArtist.map(boldface => boldface ? true : false), - }), - }, + fillMissingListItems({ + list: '#entries.artistDisplayText', + fill: input.value(null), + }), fillMissingListItems({ list: '#entries.annotation', @@ -133,7 +128,7 @@ export default templateCompositeFrom({ { dependencies: [ '#entries.artist', - '#entries.boldfaceArtist', + '#entries.artistDisplayText', '#entries.annotation', '#entries.date', '#entries.body', @@ -141,7 +136,7 @@ export default templateCompositeFrom({ compute: (continuation, { ['#entries.artist']: artist, - ['#entries.boldfaceArtist']: boldfaceArtist, + ['#entries.artistDisplayText']: artistDisplayText, ['#entries.annotation']: annotation, ['#entries.date']: date, ['#entries.body']: body, @@ -149,7 +144,7 @@ export default templateCompositeFrom({ ['#parsedCommentaryEntries']: stitchArrays({ artist, - boldfaceArtist, + artistDisplayText, annotation, date, body, -- cgit 1.3.0-6-gf8a5 From f2a31006efa7c4d9c7c15823adc70cc40c46dedd Mon Sep 17 00:00:00 2001 From: "(quasar) nebula" Date: Wed, 15 Nov 2023 10:50:54 -0400 Subject: data: fix commentary entry serialization --- src/data/serialize.js | 4 ++++ src/data/things/album.js | 3 ++- 2 files changed, 6 insertions(+), 1 deletion(-) (limited to 'src/data') diff --git a/src/data/serialize.js b/src/data/serialize.js index 52aacb07..8cac3309 100644 --- a/src/data/serialize.js +++ b/src/data/serialize.js @@ -19,6 +19,10 @@ export function toContribRefs(contribs) { return contribs?.map(({who, what}) => ({who: toRef(who), what})); } +export function toCommentaryRefs(entries) { + return entries?.map(({artist, ...props}) => ({artist: toRef(artist), ...props})); +} + // Interface export const serializeDescriptors = Symbol(); diff --git a/src/data/things/album.js b/src/data/things/album.js index af3eb042..63ec1140 100644 --- a/src/data/things/album.js +++ b/src/data/things/album.js @@ -181,7 +181,8 @@ export class Album extends Thing { hasTrackArt: S.id, isListedOnHomepage: S.id, - commentary: S.id, + commentary: S.toCommentaryRefs, + additionalFiles: S.id, tracks: S.toRefs, -- cgit 1.3.0-6-gf8a5 From a34b8d027866fbe858a4d2ff3543bc84c9d5983a Mon Sep 17 00:00:00 2001 From: "(quasar) nebula" Date: Fri, 17 Nov 2023 06:53:34 -0400 Subject: data, yaml, content: support multiple artists per commentary entry --- .../wiki-data/withParsedCommentaryEntries.js | 41 +++++++++++++++++----- .../wiki-properties/commentatorArtists.js | 23 +++++++----- src/data/yaml.js | 24 ++++++++++--- 3 files changed, 66 insertions(+), 22 deletions(-) (limited to 'src/data') diff --git a/src/data/composite/wiki-data/withParsedCommentaryEntries.js b/src/data/composite/wiki-data/withParsedCommentaryEntries.js index 7b1c9484..25c07a37 100644 --- a/src/data/composite/wiki-data/withParsedCommentaryEntries.js +++ b/src/data/composite/wiki-data/withParsedCommentaryEntries.js @@ -4,7 +4,12 @@ import {stitchArrays} from '#sugar'; import {isCommentary} from '#validators'; import {commentaryRegex} from '#wiki-data'; -import {fillMissingListItems, withPropertiesFromList} from '#composite/data'; +import { + fillMissingListItems, + withFlattenedList, + withPropertiesFromList, + withUnflattenedList, +} from '#composite/data'; import withResolvedReferenceList from './withResolvedReferenceList.js'; @@ -86,23 +91,43 @@ export default templateCompositeFrom({ list: '#rawMatches.groups', prefix: input.value('#entries'), properties: input.value([ - 'artistReference', + 'artistReferences', 'artistDisplayText', 'annotation', 'date', ]), }), - // The artistReference group will always have a value, since it's required + // The artistReferences group will always have a value, since it's required // for the line to match in the first place. + { + dependencies: ['#entries.artistReferences'], + compute: (continuation, { + ['#entries.artistReferences']: artistReferenceTexts, + }) => continuation({ + ['#entries.artistReferences']: + artistReferenceTexts + .map(text => text.split(',').map(ref => ref.trim())), + }), + }, + + withFlattenedList({ + list: '#entries.artistReferences', + }), + withResolvedReferenceList({ - list: '#entries.artistReference', + list: '#flattenedList', data: 'artistData', find: input.value(find.artist), notFoundMode: input.value('null'), + }), + + withUnflattenedList({ + list: '#resolvedReferenceList', + filter: input.value(false), }).outputs({ - '#resolvedReferenceList': '#entries.artist', + '#unflattenedList': '#entries.artists', }), fillMissingListItems({ @@ -127,7 +152,7 @@ export default templateCompositeFrom({ { dependencies: [ - '#entries.artist', + '#entries.artists', '#entries.artistDisplayText', '#entries.annotation', '#entries.date', @@ -135,7 +160,7 @@ export default templateCompositeFrom({ ], compute: (continuation, { - ['#entries.artist']: artist, + ['#entries.artists']: artists, ['#entries.artistDisplayText']: artistDisplayText, ['#entries.annotation']: annotation, ['#entries.date']: date, @@ -143,7 +168,7 @@ export default templateCompositeFrom({ }) => continuation({ ['#parsedCommentaryEntries']: stitchArrays({ - artist, + artists, artistDisplayText, annotation, date, diff --git a/src/data/composite/wiki-properties/commentatorArtists.js b/src/data/composite/wiki-properties/commentatorArtists.js index 8720e66d..65ab1466 100644 --- a/src/data/composite/wiki-properties/commentatorArtists.js +++ b/src/data/composite/wiki-properties/commentatorArtists.js @@ -4,8 +4,9 @@ import {input, templateCompositeFrom} from '#composite'; import {unique} from '#sugar'; -import {exitWithoutDependency} from '#composite/control-flow'; -import {withPropertyFromList} from '#composite/data'; +import {exitWithoutDependency, exposeDependency} + from '#composite/control-flow'; +import {withFlattenedList, withPropertyFromList} from '#composite/data'; import {withParsedCommentaryEntries} from '#composite/wiki-data'; export default templateCompositeFrom({ @@ -26,15 +27,19 @@ export default templateCompositeFrom({ withPropertyFromList({ list: '#parsedCommentaryEntries', - property: input.value('artist'), + property: input.value('artists'), }).outputs({ - '#parsedCommentaryEntries.artist': '#artists', + '#parsedCommentaryEntries.artists': '#artistLists', }), - { - dependencies: ['#artists'], - compute: ({'#artists': artists}) => - unique(artists.filter(artist => artist !== null)), - }, + withFlattenedList({ + list: '#artistLists', + }).outputs({ + '#flattenedList': '#artists', + }), + + exposeDependency({ + dependency: '#artists', + }), ], }); diff --git a/src/data/yaml.js b/src/data/yaml.js index 843e70b3..0734d539 100644 --- a/src/data/yaml.js +++ b/src/data/yaml.js @@ -21,6 +21,7 @@ import { decorateErrorWithIndex, decorateErrorWithAnnotation, empty, + filterAggregate, filterProperties, openAggregate, showAggregate, @@ -1686,8 +1687,10 @@ export function filterReferenceErrors(wikiData) { if (value) { value = Array.from(value.matchAll(commentaryRegex)) - .map(({groups}) => groups.artistReference); + .map(({groups}) => groups.artistReferences) + .map(text => text.split(',').map(text => text.trim())); } + writeProperty = false; break; } @@ -1804,11 +1807,22 @@ export function filterReferenceErrors(wikiData) { let newPropertyValue = value; - if (Array.isArray(value)) { + if (findFnKey === '_commentary') { + // Commentary doesn't write a property value, so no need to set. + filter( + value, {message: errorMessage}, + decorateErrorWithIndex(refs => + (refs.length === 1 + ? suppress(findFn)(refs[0]) + : filterAggregate( + refs, {message: `Errors in entry's artist references`}, + decorateErrorWithIndex(suppress(findFn))) + .aggregate + .close()))); + } else if (Array.isArray(value)) { newPropertyValue = filter( - value, - decorateErrorWithIndex(suppress(findFn)), - {message: errorMessage}); + value, {message: errorMessage}, + decorateErrorWithIndex(suppress(findFn))); } else { nest({message: errorMessage}, suppress(({call}) => { -- cgit 1.3.0-6-gf8a5 From f82b74f62595c4def21b176c366c703da02c331e Mon Sep 17 00:00:00 2001 From: "(quasar) nebula" Date: Sat, 18 Nov 2023 20:15:07 -0400 Subject: data, test: withUniqueItemsOnly (#composite/data) --- src/data/composite/data/index.js | 1 + src/data/composite/data/withUniqueItemsOnly.js | 40 ++++++++++++++++++++++++++ 2 files changed, 41 insertions(+) create mode 100644 src/data/composite/data/withUniqueItemsOnly.js (limited to 'src/data') diff --git a/src/data/composite/data/index.js b/src/data/composite/data/index.js index db1c37cc..e2927afd 100644 --- a/src/data/composite/data/index.js +++ b/src/data/composite/data/index.js @@ -11,3 +11,4 @@ export {default as withPropertiesFromObject} from './withPropertiesFromObject.js export {default as withPropertyFromList} from './withPropertyFromList.js'; export {default as withPropertyFromObject} from './withPropertyFromObject.js'; export {default as withUnflattenedList} from './withUnflattenedList.js'; +export {default as withUniqueItemsOnly} from './withUniqueItemsOnly.js'; diff --git a/src/data/composite/data/withUniqueItemsOnly.js b/src/data/composite/data/withUniqueItemsOnly.js new file mode 100644 index 00000000..7ee08b08 --- /dev/null +++ b/src/data/composite/data/withUniqueItemsOnly.js @@ -0,0 +1,40 @@ +// Excludes duplicate items from a list and provides the results, overwriting +// the list in-place, if possible. + +import {input, templateCompositeFrom} from '#composite'; +import {unique} from '#sugar'; + +export default templateCompositeFrom({ + annotation: `withUniqueItemsOnly`, + + inputs: { + list: input({type: 'array'}), + }, + + outputs: ({ + [input.staticDependency('list')]: list, + }) => [list ?? '#uniqueItems'], + + steps: () => [ + { + dependencies: [input('list')], + compute: (continuation, { + [input('list')]: list, + }) => continuation({ + ['#values']: + unique(list), + }), + }, + + { + dependencies: ['#values', input.staticDependency('list')], + compute: (continuation, { + '#values': values, + [input.staticDependency('list')]: list, + }) => continuation({ + [list ?? '#uniqueItems']: + values, + }), + }, + ], +}); -- cgit 1.3.0-6-gf8a5 From 5b8060bb86d457a0d23b607aa866c4d7d6eb6f0f Mon Sep 17 00:00:00 2001 From: "(quasar) nebula" Date: Sat, 18 Nov 2023 20:16:40 -0400 Subject: data: withParsedCommentaryEntries: filter out null artists --- src/data/composite/wiki-data/withParsedCommentaryEntries.js | 1 - 1 file changed, 1 deletion(-) (limited to 'src/data') diff --git a/src/data/composite/wiki-data/withParsedCommentaryEntries.js b/src/data/composite/wiki-data/withParsedCommentaryEntries.js index 25c07a37..edfc9e3c 100644 --- a/src/data/composite/wiki-data/withParsedCommentaryEntries.js +++ b/src/data/composite/wiki-data/withParsedCommentaryEntries.js @@ -125,7 +125,6 @@ export default templateCompositeFrom({ withUnflattenedList({ list: '#resolvedReferenceList', - filter: input.value(false), }).outputs({ '#unflattenedList': '#entries.artists', }), -- cgit 1.3.0-6-gf8a5 From e35d23f4e9492b497138dce3f21382872e329e71 Mon Sep 17 00:00:00 2001 From: "(quasar) nebula" Date: Sat, 18 Nov 2023 20:16:57 -0400 Subject: data: commentatorArtists: filter out duplicate artists --- src/data/composite/wiki-properties/commentatorArtists.js | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'src/data') diff --git a/src/data/composite/wiki-properties/commentatorArtists.js b/src/data/composite/wiki-properties/commentatorArtists.js index 65ab1466..f400bbfc 100644 --- a/src/data/composite/wiki-properties/commentatorArtists.js +++ b/src/data/composite/wiki-properties/commentatorArtists.js @@ -6,7 +6,8 @@ import {unique} from '#sugar'; import {exitWithoutDependency, exposeDependency} from '#composite/control-flow'; -import {withFlattenedList, withPropertyFromList} from '#composite/data'; +import {withFlattenedList, withPropertyFromList, withUniqueItemsOnly} + from '#composite/data'; import {withParsedCommentaryEntries} from '#composite/wiki-data'; export default templateCompositeFrom({ @@ -38,6 +39,10 @@ export default templateCompositeFrom({ '#flattenedList': '#artists', }), + withUniqueItemsOnly({ + list: '#artists', + }), + exposeDependency({ dependency: '#artists', }), -- cgit 1.3.0-6-gf8a5 From ee02bc3efebf992c47694ec4065f658473b1f904 Mon Sep 17 00:00:00 2001 From: "(quasar) nebula" Date: Mon, 20 Nov 2023 13:32:10 -0400 Subject: data: validateArrayItems: annotate multiline errors nicely --- src/data/things/validators.js | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) (limited to 'src/data') diff --git a/src/data/things/validators.js b/src/data/things/validators.js index f60c363c..e213e933 100644 --- a/src/data/things/validators.js +++ b/src/data/things/validators.js @@ -1,5 +1,9 @@ import {inspect as nodeInspect} from 'node:util'; +// Heresy. +import printable_characters from 'printable-characters'; +const {strlen} = printable_characters; + import {colors, ENABLE_COLOR} from '#cli'; import {empty, typeAppearance, withAggregate} from '#sugar'; @@ -174,8 +178,19 @@ function validateArrayItemsHelper(itemValidator) { throw new Error(`Expected validator to return true`); } } catch (error) { - error.message = `(index: ${colors.yellow(`${index}`)}, item: ${inspect(item)}) ${error.message}`; + const annotation = `(index: ${colors.yellow(`${index}`)}, item: ${inspect(item)})`; + + error.message = + (error.message.includes('\n') || strlen(annotation) > 20 + ? annotation + '\n' + + error.message + .split('\n') + .map(line => ` ${line}`) + .join('\n') + : `${annotation} ${error}`); + error[Symbol.for('hsmusic.decorate.indexInSourceArray')] = index; + throw error; } }; -- cgit 1.3.0-6-gf8a5 From 2d58b70d0bd5bbc7cdd8789332a31a220c78da01 Mon Sep 17 00:00:00 2001 From: "(quasar) nebula" Date: Mon, 20 Nov 2023 13:48:52 -0400 Subject: data: validateArrayItems (etc): pass through index, array --- src/data/things/validators.js | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) (limited to 'src/data') diff --git a/src/data/things/validators.js b/src/data/things/validators.js index e213e933..2893e7fd 100644 --- a/src/data/things/validators.js +++ b/src/data/things/validators.js @@ -170,9 +170,9 @@ export function is(...values) { } function validateArrayItemsHelper(itemValidator) { - return (item, index) => { + return (item, index, array) => { try { - const value = itemValidator(item); + const value = itemValidator(item, index, array); if (value !== true) { throw new Error(`Expected validator to return true`); @@ -197,13 +197,15 @@ function validateArrayItemsHelper(itemValidator) { } export function validateArrayItems(itemValidator) { - const fn = validateArrayItemsHelper(itemValidator); + const helper = validateArrayItemsHelper(itemValidator); return (array) => { isArray(array); - withAggregate({message: 'Errors validating array items'}, ({wrap}) => { - array.forEach(wrap(fn)); + withAggregate({message: 'Errors validating array items'}, ({call}) => { + for (let index = 0; index < array.length; index++) { + call(helper, array[index], index, array); + } }); return true; @@ -215,12 +217,12 @@ export function strictArrayOf(itemValidator) { } export function sparseArrayOf(itemValidator) { - return validateArrayItems(item => { + return validateArrayItems((item, index, array) => { if (item === false || item === null) { return true; } - return itemValidator(item); + return itemValidator(item, index, array); }); } -- cgit 1.3.0-6-gf8a5 From 9a35630d91ebd7227994a1009487794a65ec38e1 Mon Sep 17 00:00:00 2001 From: "(quasar) nebula" Date: Mon, 20 Nov 2023 13:53:36 -0400 Subject: data: tidy yaml error message construction, cut long strings ...Using maxStringLength, which is more than a bit annoying, because this isn't the same cut() algorithm we just added, looks bulkier, and can't be customized. But that's the cost of using util.inspect() here. It's better than displaying the entire long string or handling line breaks poorly. FOR NOW. --- src/data/yaml.js | 38 +++++++++++++++++++++++++++++++------- 1 file changed, 31 insertions(+), 7 deletions(-) (limited to 'src/data') diff --git a/src/data/yaml.js b/src/data/yaml.js index 0734d539..49e05266 100644 --- a/src/data/yaml.js +++ b/src/data/yaml.js @@ -38,8 +38,8 @@ import { // --> General supporting stuff -function inspect(value) { - return nodeInspect(value, {colors: ENABLE_COLOR}); +function inspect(value, opts = {}) { + return nodeInspect(value, {colors: ENABLE_COLOR, ...opts}); } // --> YAML data repository structure constants @@ -308,7 +308,12 @@ export class FieldCombinationError extends Error { constructor(fields, message) { const fieldNames = Object.keys(fields); - const mainMessage = `Don't combine ${fieldNames.map(field => colors.red(field)).join(', ')}`; + const fieldNamesText = + fieldNames + .map(field => colors.red(field)) + .join(', '); + + const mainMessage = `Don't combine ${fieldNamesText}`; const causeMessage = (typeof message === 'function' @@ -330,7 +335,12 @@ export class FieldCombinationError extends Error { export class FieldValueAggregateError extends AggregateError { constructor(thingConstructor, errors) { - super(errors, `Errors processing field values for ${colors.green(thingConstructor.name)}`); + const constructorText = + colors.green(thingConstructor.name); + + super( + errors, + `Errors processing field values for ${constructorText}`); } } @@ -341,8 +351,17 @@ export class FieldValueError extends Error { ? caughtError.cause : caughtError); + const fieldText = + colors.green(`"${field}"`); + + const propertyText = + colors.green(property); + + const valueText = + inspect(value, {maxStringLength: 40}); + super( - `Failed to set ${colors.green(`"${field}"`)} field (${colors.green(property)}) to ${inspect(value)}`, + `Failed to set ${fieldText} field (${propertyText}) to ${valueText}`, {cause}); } } @@ -354,13 +373,18 @@ export class SkippedFieldsSummaryError extends Error { const lines = entries.map(([field, value]) => ` - ${field}: ` + - inspect(value) + inspect(value, {maxStringLength: 70}) .split('\n') .map((line, index) => index === 0 ? line : ` ${line}`) .join('\n')); + const numFieldsText = + (entries.length === 1 + ? `1 field` + : `${entries.length} fields`); + super( - colors.bright(colors.yellow(`Altogether, skipped ${entries.length === 1 ? `1 field` : `${entries.length} fields`}:\n`)) + + colors.bright(colors.yellow(`Altogether, skipped ${numFieldsText}:\n`)) + lines.join('\n') + '\n' + colors.bright(colors.yellow(`See above errors for details.`))); } -- cgit 1.3.0-6-gf8a5 From 87988954ad7314bee59932b0e5ef3474936ed33e Mon Sep 17 00:00:00 2001 From: "(quasar) nebula" Date: Mon, 20 Nov 2023 13:59:13 -0400 Subject: data: update and revamp isCommentary validator --- src/data/things/validators.js | 59 +++++++++++++++++++++++++++++++++++-------- 1 file changed, 49 insertions(+), 10 deletions(-) (limited to 'src/data') diff --git a/src/data/things/validators.js b/src/data/things/validators.js index 2893e7fd..569a7b34 100644 --- a/src/data/things/validators.js +++ b/src/data/things/validators.js @@ -5,7 +5,8 @@ import printable_characters from 'printable-characters'; const {strlen} = printable_characters; import {colors, ENABLE_COLOR} from '#cli'; -import {empty, typeAppearance, withAggregate} from '#sugar'; +import {cut, empty, typeAppearance, withAggregate} from '#sugar'; +import {commentaryRegex} from '#wiki-data'; function inspect(value) { return nodeInspect(value, {colors: ENABLE_COLOR}); @@ -248,18 +249,56 @@ export function isColor(color) { throw new TypeError(`Unknown color format`); } -export function isCommentary(commentary) { - isString(commentary); +export function isCommentary(commentaryText) { + isString(commentaryText); - const [firstLine] = commentary.match(/.*/); - if (!firstLine.replace(/<\/b>/g, '').includes(':')) { - throw new TypeError(`Missing commentary citation: "${ - firstLine.length > 40 - ? firstLine.slice(0, 40) + '...' - : firstLine - }"`); + const rawMatches = + Array.from(commentaryText.matchAll(commentaryRegex)); + + if (empty(rawMatches)) { + throw new TypeError(`Expected at least one commentary heading`); } + const niceMatches = + rawMatches.map(match => ({ + position: match.index, + length: match[0].length, + })); + + validateArrayItems(({position, length}, index) => { + if (index === 0 && position > 0) { + throw new TypeError(`Expected first commentary heading to be at top`); + } + + const ownInput = commentaryText.slice(position, position + length); + const restOfInput = commentaryText.slice(position + length); + const nextLineBreak = restOfInput.indexOf('\n'); + const upToNextLineBreak = restOfInput.slice(0, nextLineBreak); + + if (/\S/.test(upToNextLineBreak)) { + throw new TypeError( + `Expected commentary heading to occupy entire line, got extra text:\n` + + `${colors.green(`"${cut(ownInput, 40)}"`)} (<- heading)\n` + + `(extra on same line ->) ${colors.red(`"${cut(upToNextLineBreak, 30)}"`)}\n` + + `(Check for missing "|-" in YAML, or a misshapen annotation)`); + } + + const nextHeading = + (index === niceMatches.length - 1 + ? commentaryText.length + : niceMatches[index + 1].position); + + const upToNextHeading = + commentaryText.slice(position + length, nextHeading); + + if (!/\S/.test(upToNextHeading)) { + throw new TypeError( + `Expected commentary entry to have body text, only got a heading`); + } + + return true; + })(niceMatches); + return true; } -- cgit 1.3.0-6-gf8a5 From f87fa920f91d36424e4613ac5da50f46418f4b19 Mon Sep 17 00:00:00 2001 From: "(quasar) nebula" Date: Mon, 20 Nov 2023 14:31:58 -0400 Subject: data, util: principle "translucent errors" & applications --- src/data/yaml.js | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) (limited to 'src/data') diff --git a/src/data/yaml.js b/src/data/yaml.js index 49e05266..dddf5fb2 100644 --- a/src/data/yaml.js +++ b/src/data/yaml.js @@ -334,6 +334,8 @@ export class FieldCombinationError extends Error { } export class FieldValueAggregateError extends AggregateError { + [Symbol.for('hsmusic.aggregate.translucent')] = true; + constructor(thingConstructor, errors) { const constructorText = colors.green(thingConstructor.name); @@ -1162,7 +1164,10 @@ export async function loadAndProcessDataDocuments({dataPath}) { for (const dataStep of dataSteps) { await processDataAggregate.nestAsync( - {message: `Errors during data step: ${colors.bright(dataStep.title)}`}, + { + message: `Errors during data step: ${colors.bright(dataStep.title)}`, + translucent: true, + }, async ({call, callAsync, map, mapAsync, push}) => { const {documentMode} = dataStep; @@ -1407,7 +1412,7 @@ export async function loadAndProcessDataDocuments({dataPath}) { switch (documentMode) { case documentModes.headerAndEntries: - map(yamlResults, {message: `Errors processing documents in data files`}, + map(yamlResults, {message: `Errors processing documents in data files`, translucent: true}, decorateErrorWithFile(({documents}) => { const headerDocument = documents[0]; const entryDocuments = documents.slice(1).filter(Boolean); -- cgit 1.3.0-6-gf8a5