diff options
-rw-r--r-- | package.json | 1 | ||||
-rw-r--r-- | src/data/composite/control-flow/index.js | 5 | ||||
-rw-r--r-- | src/data/composite/data/index.js | 5 | ||||
-rw-r--r-- | src/data/composite/wiki-data/index.js | 7 | ||||
-rw-r--r-- | src/data/composite/wiki-data/withParsedCommentaryEntries.js | 155 | ||||
-rw-r--r-- | src/data/composite/wiki-properties/commentary.js | 32 | ||||
-rw-r--r-- | src/data/composite/wiki-properties/commentatorArtists.js | 41 | ||||
-rw-r--r-- | src/data/composite/wiki-properties/index.js | 5 | ||||
-rw-r--r-- | src/data/serialize.js | 4 | ||||
-rw-r--r-- | src/data/things/album.js | 3 | ||||
-rw-r--r-- | src/data/things/index.js | 5 | ||||
-rw-r--r-- | src/data/yaml.js | 63 | ||||
-rw-r--r-- | src/find.js | 2 | ||||
-rw-r--r-- | src/gen-thumbs.js | 2 | ||||
-rw-r--r-- | src/repl.js | 3 | ||||
-rwxr-xr-x | src/upd8.js | 2 | ||||
-rw-r--r-- | src/util/wiki-data.js | 25 | ||||
-rw-r--r-- | test/unit/data/cacheable-object.js | 2 | ||||
-rw-r--r-- | test/unit/data/composite/wiki-data/withParsedCommentaryEntries.js | 102 |
19 files changed, 399 insertions, 65 deletions
diff --git a/package.json b/package.json index 194c4060..34b4ebf0 100644 --- a/package.json +++ b/package.json @@ -12,6 +12,7 @@ "dev": "eslint src && node src/upd8.js" }, "imports": { + "#cacheable-object": "./src/data/things/cacheable-object.js", "#colors": "./src/util/colors.js", "#composite": "./src/data/things/composite.js", "#composite/control-flow": "./src/data/composite/control-flow/index.js", diff --git a/src/data/composite/control-flow/index.js b/src/data/composite/control-flow/index.js index dfc53db7..7fad88b2 100644 --- a/src/data/composite/control-flow/index.js +++ b/src/data/composite/control-flow/index.js @@ -1,3 +1,8 @@ +// #composite/control-flow +// +// No entries depend on any other entries, except siblings in this directory. +// + export {default as exitWithoutDependency} from './exitWithoutDependency.js'; export {default as exitWithoutUpdateValue} from './exitWithoutUpdateValue.js'; export {default as exposeConstant} from './exposeConstant.js'; diff --git a/src/data/composite/data/index.js b/src/data/composite/data/index.js index ecd05129..db1c37cc 100644 --- a/src/data/composite/data/index.js +++ b/src/data/composite/data/index.js @@ -1,3 +1,8 @@ +// #composite/data +// +// Entries here may depend on entries in #composite/control-flow. +// + export {default as excludeFromList} from './excludeFromList.js'; export {default as fillMissingListItems} from './fillMissingListItems.js'; export {default as withFlattenedList} from './withFlattenedList.js'; diff --git a/src/data/composite/wiki-data/index.js b/src/data/composite/wiki-data/index.js index 1d0400fc..df50a2db 100644 --- a/src/data/composite/wiki-data/index.js +++ b/src/data/composite/wiki-data/index.js @@ -1,6 +1,13 @@ +// #composite/wiki-data +// +// Entries here may depend on entries in #composite/control-flow and in +// #composite/data. +// + export {default as exitWithoutContribs} from './exitWithoutContribs.js'; export {default as inputThingClass} from './inputThingClass.js'; export {default as inputWikiData} from './inputWikiData.js'; +export {default as withParsedCommentaryEntries} from './withParsedCommentaryEntries.js'; export {default as withResolvedContribs} from './withResolvedContribs.js'; export {default as withResolvedReference} from './withResolvedReference.js'; export {default as withResolvedReferenceList} from './withResolvedReferenceList.js'; diff --git a/src/data/composite/wiki-data/withParsedCommentaryEntries.js b/src/data/composite/wiki-data/withParsedCommentaryEntries.js new file mode 100644 index 00000000..7b1c9484 --- /dev/null +++ b/src/data/composite/wiki-data/withParsedCommentaryEntries.js @@ -0,0 +1,155 @@ +import {input, templateCompositeFrom} from '#composite'; +import find from '#find'; +import {stitchArrays} from '#sugar'; +import {isCommentary} from '#validators'; +import {commentaryRegex} from '#wiki-data'; + +import {fillMissingListItems, withPropertiesFromList} from '#composite/data'; + +import withResolvedReferenceList from './withResolvedReferenceList.js'; + +export default templateCompositeFrom({ + annotation: `withParsedCommentaryEntries`, + + inputs: { + from: input({validate: isCommentary}), + }, + + outputs: ['#parsedCommentaryEntries'], + + steps: () => [ + { + dependencies: [input('from')], + + compute: (continuation, { + [input('from')]: commentaryText, + }) => continuation({ + ['#rawMatches']: + Array.from(commentaryText.matchAll(commentaryRegex)), + }), + }, + + withPropertiesFromList({ + list: '#rawMatches', + properties: input.value([ + '0', // The entire match as a string. + 'groups', + 'index', + ]), + }).outputs({ + '#rawMatches.0': '#rawMatches.text', + '#rawMatches.groups': '#rawMatches.groups', + '#rawMatches.index': '#rawMatches.startIndex', + }), + + { + dependencies: [ + '#rawMatches.text', + '#rawMatches.startIndex', + ], + + compute: (continuation, { + ['#rawMatches.text']: text, + ['#rawMatches.startIndex']: startIndex, + }) => continuation({ + ['#rawMatches.endIndex']: + stitchArrays({text, startIndex}) + .map(({text, startIndex}) => startIndex + text.length), + }), + }, + + { + dependencies: [ + input('from'), + '#rawMatches.startIndex', + '#rawMatches.endIndex', + ], + + compute: (continuation, { + [input('from')]: commentaryText, + ['#rawMatches.startIndex']: startIndex, + ['#rawMatches.endIndex']: endIndex, + }) => continuation({ + ['#entries.body']: + stitchArrays({startIndex, endIndex}) + .map(({endIndex}, index, stitched) => + (index === stitched.length - 1 + ? commentaryText.slice(endIndex) + : commentaryText.slice( + endIndex, + stitched[index + 1].startIndex))) + .map(body => body.trim()), + }), + }, + + withPropertiesFromList({ + list: '#rawMatches.groups', + prefix: input.value('#entries'), + properties: input.value([ + 'artistReference', + 'artistDisplayText', + 'annotation', + 'date', + ]), + }), + + // The artistReference group will always have a value, since it's required + // for the line to match in the first place. + + withResolvedReferenceList({ + list: '#entries.artistReference', + data: 'artistData', + find: input.value(find.artist), + notFoundMode: input.value('null'), + }).outputs({ + '#resolvedReferenceList': '#entries.artist', + }), + + fillMissingListItems({ + list: '#entries.artistDisplayText', + fill: input.value(null), + }), + + fillMissingListItems({ + list: '#entries.annotation', + fill: input.value(null), + }), + + { + dependencies: ['#entries.date'], + compute: (continuation, { + ['#entries.date']: date, + }) => continuation({ + ['#entries.date']: + date.map(date => date ? new Date(date) : null), + }), + }, + + { + dependencies: [ + '#entries.artist', + '#entries.artistDisplayText', + '#entries.annotation', + '#entries.date', + '#entries.body', + ], + + compute: (continuation, { + ['#entries.artist']: artist, + ['#entries.artistDisplayText']: artistDisplayText, + ['#entries.annotation']: annotation, + ['#entries.date']: date, + ['#entries.body']: body, + }) => continuation({ + ['#parsedCommentaryEntries']: + stitchArrays({ + artist, + artistDisplayText, + annotation, + date, + body, + }), + }), + }, + ], +}); diff --git a/src/data/composite/wiki-properties/commentary.js b/src/data/composite/wiki-properties/commentary.js index fbea9d5c..cd6b7ac4 100644 --- a/src/data/composite/wiki-properties/commentary.js +++ b/src/data/composite/wiki-properties/commentary.js @@ -1,12 +1,30 @@ // Artist commentary! Generally present on tracks and albums. +import {input, templateCompositeFrom} from '#composite'; import {isCommentary} from '#validators'; -// TODO: Not templateCompositeFrom. +import {exitWithoutDependency, exposeDependency} + from '#composite/control-flow'; +import {withParsedCommentaryEntries} from '#composite/wiki-data'; -export default function() { - return { - flags: {update: true, expose: true}, - update: {validate: isCommentary}, - }; -} +export default templateCompositeFrom({ + annotation: `commentary`, + + compose: false, + + steps: () => [ + exitWithoutDependency({ + dependency: input.updateValue({validate: isCommentary}), + mode: input.value('falsy'), + value: input.value(null), + }), + + withParsedCommentaryEntries({ + from: input.updateValue(), + }), + + exposeDependency({ + dependency: '#parsedCommentaryEntries', + }), + ], +}); diff --git a/src/data/composite/wiki-properties/commentatorArtists.js b/src/data/composite/wiki-properties/commentatorArtists.js index 52aeb868..8720e66d 100644 --- a/src/data/composite/wiki-properties/commentatorArtists.js +++ b/src/data/composite/wiki-properties/commentatorArtists.js @@ -1,13 +1,12 @@ -// This one's kinda tricky: it parses artist "references" from the -// commentary content, and finds the matching artist for each reference. +// List of artists referenced in commentary entries. // This is mostly useful for credits and listings on artist pages. import {input, templateCompositeFrom} from '#composite'; -import find from '#find'; import {unique} from '#sugar'; import {exitWithoutDependency} from '#composite/control-flow'; -import {withResolvedReferenceList} from '#composite/wiki-data'; +import {withPropertyFromList} from '#composite/data'; +import {withParsedCommentaryEntries} from '#composite/wiki-data'; export default templateCompositeFrom({ annotation: `commentatorArtists`, @@ -21,35 +20,21 @@ export default templateCompositeFrom({ value: input.value([]), }), - { - dependencies: ['commentary'], - compute: (continuation, {commentary}) => - continuation({ - '#artistRefs': - Array.from( - commentary - .replace(/<\/?b>/g, '') - .matchAll(/<i>(?<who>.*?):<\/i>/g)) - .map(({groups: {who}}) => who), - }), - }, + withParsedCommentaryEntries({ + from: 'commentary', + }), - withResolvedReferenceList({ - list: '#artistRefs', - data: 'artistData', - find: input.value(find.artist), + withPropertyFromList({ + list: '#parsedCommentaryEntries', + property: input.value('artist'), }).outputs({ - '#resolvedReferenceList': '#artists', + '#parsedCommentaryEntries.artist': '#artists', }), { - flags: {expose: true}, - - expose: { - dependencies: ['#artists'], - compute: ({'#artists': artists}) => - unique(artists), - }, + dependencies: ['#artists'], + compute: ({'#artists': artists}) => + unique(artists.filter(artist => artist !== null)), }, ], }); diff --git a/src/data/composite/wiki-properties/index.js b/src/data/composite/wiki-properties/index.js index 2462b047..3a8b51d5 100644 --- a/src/data/composite/wiki-properties/index.js +++ b/src/data/composite/wiki-properties/index.js @@ -1,3 +1,8 @@ +// #composite/wiki-properties +// +// Entries here may depend on entries in #composite/control-flow, +// #composite/data, and #composite/wiki-data. + export {default as additionalFiles} from './additionalFiles.js'; export {default as color} from './color.js'; export {default as commentary} from './commentary.js'; diff --git a/src/data/serialize.js b/src/data/serialize.js index 52aacb07..8cac3309 100644 --- a/src/data/serialize.js +++ b/src/data/serialize.js @@ -19,6 +19,10 @@ export function toContribRefs(contribs) { return contribs?.map(({who, what}) => ({who: toRef(who), what})); } +export function toCommentaryRefs(entries) { + return entries?.map(({artist, ...props}) => ({artist: toRef(artist), ...props})); +} + // Interface export const serializeDescriptors = Symbol(); diff --git a/src/data/things/album.js b/src/data/things/album.js index af3eb042..63ec1140 100644 --- a/src/data/things/album.js +++ b/src/data/things/album.js @@ -181,7 +181,8 @@ export class Album extends Thing { hasTrackArt: S.id, isListedOnHomepage: S.id, - commentary: S.id, + commentary: S.toCommentaryRefs, + additionalFiles: S.id, tracks: S.toRefs, diff --git a/src/data/things/index.js b/src/data/things/index.js index 4ea1f007..d1143b0a 100644 --- a/src/data/things/index.js +++ b/src/data/things/index.js @@ -22,11 +22,6 @@ import * as wikiInfoClasses from './wiki-info.js'; export {default as Thing} from './thing.js'; -export { - default as CacheableObject, - CacheableObjectPropertyValueError, -} from './cacheable-object.js'; - const allClassLists = { 'album.js': albumClasses, 'art-tag.js': artTagClasses, diff --git a/src/data/yaml.js b/src/data/yaml.js index 1d35bae8..843e70b3 100644 --- a/src/data/yaml.js +++ b/src/data/yaml.js @@ -7,15 +7,13 @@ import {inspect as nodeInspect} from 'node:util'; import yaml from 'js-yaml'; +import CacheableObject, {CacheableObjectPropertyValueError} + from '#cacheable-object'; import {colors, ENABLE_COLOR, logInfo, logWarn} from '#cli'; import find, {bindFind} from '#find'; import {traverse} from '#node-utils'; -import T, { - CacheableObject, - CacheableObjectPropertyValueError, - Thing, -} from '#things'; +import T, {Thing} from '#things'; import { annotateErrorWithFile, @@ -30,6 +28,7 @@ import { } from '#sugar'; import { + commentaryRegex, sortAlbumsTracksChronologically, sortAlphabetically, sortChronologically, @@ -1618,6 +1617,7 @@ export function filterReferenceErrors(wikiData) { bannerArtistContribs: '_contrib', groups: 'group', artTags: 'artTag', + commentary: '_commentary', }], ['trackData', processTrackDocument, { @@ -1628,6 +1628,7 @@ export function filterReferenceErrors(wikiData) { sampledTracks: '_trackNotRerelease', artTags: 'artTag', originalReleaseTrack: '_trackNotRerelease', + commentary: '_commentary', }], ['groupCategoryData', processGroupCategoryDocument, { @@ -1677,7 +1678,19 @@ export function filterReferenceErrors(wikiData) { nest({message: `Reference errors in ${inspect(thing)}`}, ({nest, push, filter}) => { for (const [property, findFnKey] of Object.entries(propSpec)) { - const value = CacheableObject.getUpdateValue(thing, property); + let value = CacheableObject.getUpdateValue(thing, property); + let writeProperty = true; + + switch (findFnKey) { + case '_commentary': + if (value) { + value = + Array.from(value.matchAll(commentaryRegex)) + .map(({groups}) => groups.artistReference); + } + writeProperty = false; + break; + } if (value === undefined) { push(new TypeError(`Property ${colors.red(property)} isn't valid for ${colors.green(thing.constructor.name)}`)); @@ -1690,19 +1703,25 @@ export function filterReferenceErrors(wikiData) { let findFn; + const findArtistOrAlias = artistRef => { + const alias = find.artist(artistRef, wikiData.artistAliasData, {mode: 'quiet'}); + if (alias) { + // No need to check if the original exists here. Aliases are automatically + // created from a field on the original, so the original certainly exists. + const original = alias.aliasedArtist; + throw new Error(`Reference ${colors.red(artistRef)} is to an alias, should be ${colors.green(original.name)}`); + } + + return boundFind.artist(artistRef); + }; + switch (findFnKey) { - case '_contrib': - findFn = contribRef => { - const alias = find.artist(contribRef.who, wikiData.artistAliasData, {mode: 'quiet'}); - if (alias) { - // No need to check if the original exists here. Aliases are automatically - // created from a field on the original, so the original certainly exists. - const original = alias.aliasedArtist; - throw new Error(`Reference ${colors.red(contribRef.who)} is to an alias, should be ${colors.green(original.name)}`); - } + case '_commentary': + findFn = findArtistOrAlias; + break; - return boundFind.artist(contribRef.who); - }; + case '_contrib': + findFn = contribRef => findArtistOrAlias(contribRef.who); break; case '_homepageSourceGroup': @@ -1783,8 +1802,10 @@ export function filterReferenceErrors(wikiData) { ? `Reference errors` + fieldPropertyMessage + findFnMessage : `Reference error` + fieldPropertyMessage + findFnMessage); + let newPropertyValue = value; + if (Array.isArray(value)) { - thing[property] = filter( + newPropertyValue = filter( value, decorateErrorWithIndex(suppress(findFn)), {message: errorMessage}); @@ -1794,11 +1815,15 @@ export function filterReferenceErrors(wikiData) { try { call(findFn, value); } catch (error) { - thing[property] = null; + newPropertyValue = null; throw error; } })); } + + if (writeProperty) { + thing[property] = newPropertyValue; + } } }); } diff --git a/src/find.js b/src/find.js index dfcaa9aa..4d3e996a 100644 --- a/src/find.js +++ b/src/find.js @@ -1,8 +1,8 @@ import {inspect} from 'node:util'; +import CacheableObject from '#cacheable-object'; import {colors, logWarn} from '#cli'; import {typeAppearance} from '#sugar'; -import {CacheableObject} from '#things'; function warnOrThrow(mode, message) { if (mode === 'error') { diff --git a/src/gen-thumbs.js b/src/gen-thumbs.js index 1bbcb9c1..e6c1f5c2 100644 --- a/src/gen-thumbs.js +++ b/src/gen-thumbs.js @@ -101,8 +101,8 @@ import { import dimensionsOf from 'image-size'; +import CacheableObject from '#cacheable-object'; import {delay, empty, queue, unique} from '#sugar'; -import {CacheableObject} from '#things'; import {sortByName} from '#wiki-data'; import { diff --git a/src/repl.js b/src/repl.js index 3f5d752a..dd61133c 100644 --- a/src/repl.js +++ b/src/repl.js @@ -11,7 +11,8 @@ import {generateURLs, urlSpec} from '#urls'; import {quickLoadAllFromYAML} from '#yaml'; import _find, {bindFind} from '#find'; -import thingConstructors, {CacheableObject} from '#things'; +import CacheableObject from '#cacheable-object'; +import thingConstructors from '#things'; import * as serialize from '#serialize'; import * as sugar from '#sugar'; import * as wikiDataUtils from '#wiki-data'; diff --git a/src/upd8.js b/src/upd8.js index ff7d7c5c..ebb278b2 100755 --- a/src/upd8.js +++ b/src/upd8.js @@ -38,13 +38,13 @@ import {fileURLToPath} from 'node:url'; import wrap from 'word-wrap'; +import CacheableObject from '#cacheable-object'; import {displayCompositeCacheAnalysis} from '#composite'; import {processLanguageFile, watchLanguageFile, internalDefaultStringsFile} from '#language'; import {isMain, traverse} from '#node-utils'; import bootRepl from '#repl'; import {empty, showAggregate, withEntries} from '#sugar'; -import {CacheableObject} from '#things'; import {generateURLs, urlSpec} from '#urls'; import {sortByName} from '#wiki-data'; diff --git a/src/util/wiki-data.js b/src/util/wiki-data.js index 0790ae91..75a141d3 100644 --- a/src/util/wiki-data.js +++ b/src/util/wiki-data.js @@ -629,6 +629,31 @@ export function sortFlashesChronologically(data, { // Specific data utilities +// Matches heading details from commentary data in roughly the formats: +// +// <i>artistReference:</i> (annotation, date) +// <i>artistReference|artistDisplayText:</i> (annotation, date) +// +// where capturing group "annotation" can be any text at all, except that the +// last entry (past a comma or the only content within parentheses), if parsed +// as a date, is the capturing group "date". "Parsing as a date" means one of +// these formats: +// +// * "25 December 2019" - one or two number digits, followed by any text, +// followed by four number digits +// * "12/25/2019" - one or two number digits, a slash, one or two number +// digits, a slash, and two to four number digits +// +// Capturing group "artistReference" is all the characters between <i> and </i> +// (apart from the pipe and "artistDisplayText" text, if present), and is either +// the name of an artist or an "artist:directory"-style reference. +// +// This regular expression *doesn't* match bodies, which will need to be parsed +// out of the original string based on the indices matched using this. +// +export const commentaryRegex = + /^<i>(?<artistReference>.+?)(?:\|(?<artistDisplayText>.+))?:<\/i>(?: \((?<annotation>(?:.*?(?=[,)]))*?)(?:,? ?(?<date>[a-zA-Z]+ [0-9]{1,2}, [0-9]{4,4}|[0-9]{1,2} [^,]*[0-9]{4,4}|[0-9]{1,2}\/[0-9]{1,2}\/[0-9]{2,4}))?\))?/gm; + export function filterAlbumsByCommentary(albums) { return albums .filter((album) => [album, ...album.tracks].some((x) => x.commentary)); diff --git a/test/unit/data/cacheable-object.js b/test/unit/data/cacheable-object.js index 57e562d8..5ed9a4a9 100644 --- a/test/unit/data/cacheable-object.js +++ b/test/unit/data/cacheable-object.js @@ -1,6 +1,6 @@ import t from 'tap'; -import {CacheableObject} from '#things'; +import CacheableObject from '#cacheable-object'; function newCacheableObject(PD) { return new (class extends CacheableObject { diff --git a/test/unit/data/composite/wiki-data/withParsedCommentaryEntries.js b/test/unit/data/composite/wiki-data/withParsedCommentaryEntries.js new file mode 100644 index 00000000..50570de6 --- /dev/null +++ b/test/unit/data/composite/wiki-data/withParsedCommentaryEntries.js @@ -0,0 +1,102 @@ +import t from 'tap'; + +import {compositeFrom, input} from '#composite'; +import thingConstructors from '#things'; + +import {exposeDependency} from '#composite/control-flow'; +import {withParsedCommentaryEntries} from '#composite/wiki-data'; + +const {Artist} = thingConstructors; + +const composite = compositeFrom({ + compose: false, + + steps: [ + withParsedCommentaryEntries({ + from: 'from', + }), + + exposeDependency({dependency: '#parsedCommentaryEntries'}), + ], +}); + +function stubArtist(artistName = `Test Artist`) { + const artist = new Artist(); + artist.name = artistName; + + return artist; +} + +t.test(`withParsedCommentaryEntries: basic behavior`, t => { + t.plan(3); + + const artist1 = stubArtist(`Mobius Trip`); + const artist2 = stubArtist(`Hadron Kaleido`); + + const artistData = [artist1, artist2]; + + t.match(composite, { + expose: { + dependencies: ['from', 'artistData'], + }, + }); + + t.match(composite.expose.compute({ + artistData, + from: + `<i>Mobius Trip:</i>\n` + + `Some commentary.\n` + + `Very cool.\n`, + }), [ + { + artist: artist1, + artistDisplayText: null, + annotation: null, + date: null, + body: `Some commentary.\nVery cool.`, + }, + ]); + + t.match(composite.expose.compute({ + artistData, + from: + `<i>Mobius Trip|Moo-bius Trip:</i> (music, art, 12 January 2015)\n` + + `First commentary entry.\n` + + `Very cool.\n` + + `<i>Hadron Kaleido|<b>[[artist:hadron-kaleido|The Ol' Hadron]]</b>:</i> (moral support, 4/4/2022)\n` + + `Second commentary entry. Yes. So cool.\n` + + `<i>Mystery Artist:</i> (pingas, August 25, 2023)\n` + + `Oh no.. Oh dear...\n` + + `<i>Mobius Trip:</i>\n` + + `And back around we go.`, + }), [ + { + artist: artist1, + artistDisplayText: `Moo-bius Trip`, + annotation: `music, art`, + date: new Date('12 January 2015'), + body: `First commentary entry.\nVery cool.`, + }, + { + artist: artist2, + artistDisplayText: `<b>[[artist:hadron-kaleido|The Ol' Hadron]]</b>`, + annotation: `moral support`, + date: new Date('4 April 2022'), + body: `Second commentary entry. Yes. So cool.`, + }, + { + artist: null, + artistDisplayText: null, + annotation: `pingas`, + date: new Date('25 August 2023'), + body: `Oh no.. Oh dear...`, + }, + { + artist: artist1, + artistDisplayText: null, + annotation: null, + date: null, + body: `And back around we go.`, + }, + ]); +}); |