diff options
Diffstat (limited to 'src/data')
30 files changed, 1197 insertions, 757 deletions
diff --git a/src/data/cacheable-object.js b/src/data/cacheable-object.js index 1e7c7aa..5e26c5f 100644 --- a/src/data/cacheable-object.js +++ b/src/data/cacheable-object.js @@ -83,6 +83,8 @@ function inspect(value) { } export default class CacheableObject { + static propertyDescriptors = Symbol.for('CacheableObject.propertyDescriptors'); + #propertyUpdateValues = Object.create(null); #propertyUpdateCacheInvalidators = Object.create(null); @@ -113,12 +115,21 @@ export default class CacheableObject { } } + #withEachPropertyDescriptor(callback) { + const {[CacheableObject.propertyDescriptors]: propertyDescriptors} = + this.constructor; + + for (const property of Reflect.ownKeys(propertyDescriptors)) { + callback(property, propertyDescriptors[property]); + } + } + #initializeUpdatingPropertyValues() { - for (const [property, descriptor] of Object.entries(this.constructor.propertyDescriptors)) { + this.#withEachPropertyDescriptor((property, descriptor) => { const {flags, update} = descriptor; if (!flags.update) { - continue; + return; } if (update?.default) { @@ -126,15 +137,15 @@ export default class CacheableObject { } else { this[property] = null; } - } + }); } #defineProperties() { - if (!this.constructor.propertyDescriptors) { - throw new Error(`Expected constructor ${this.constructor.name} to define propertyDescriptors`); + if (!this.constructor[CacheableObject.propertyDescriptors]) { + throw new Error(`Expected constructor ${this.constructor.name} to provide CacheableObject.propertyDescriptors`); } - for (const [property, descriptor] of Object.entries(this.constructor.propertyDescriptors)) { + this.#withEachPropertyDescriptor((property, descriptor) => { const {flags} = descriptor; const definition = { @@ -151,7 +162,7 @@ export default class CacheableObject { } Object.defineProperty(this, property, definition); - } + }); Object.seal(this); } @@ -191,7 +202,7 @@ export default class CacheableObject { } #getPropertyDescriptor(property) { - return this.constructor.propertyDescriptors[property]; + return this.constructor[CacheableObject.propertyDescriptors][property]; } #invalidateCachesDependentUpon(property) { @@ -244,7 +255,8 @@ export default class CacheableObject { if (expose.dependencies?.length > 0) { const dependencyKeys = expose.dependencies.slice(); - const shouldReflect = dependencyKeys.includes('this'); + const shouldReflectObject = dependencyKeys.includes('this'); + const shouldReflectProperty = dependencyKeys.includes('thisProperty'); getAllDependencies = () => { const dependencies = Object.create(null); @@ -253,10 +265,14 @@ export default class CacheableObject { dependencies[key] = this.#propertyUpdateValues[key]; } - if (shouldReflect) { + if (shouldReflectObject) { dependencies.this = this; } + if (shouldReflectProperty) { + dependencies.thisProperty = property; + } + return dependencies; }; } else { @@ -311,16 +327,16 @@ export default class CacheableObject { return; } - const {propertyDescriptors} = obj.constructor; + const {[CacheableObject.propertyDescriptors]: propertyDescriptors} = + obj.constructor; if (!propertyDescriptors) { console.warn('Missing property descriptors:', obj); return; } - for (const [property, descriptor] of Object.entries(propertyDescriptors)) { - const {flags} = descriptor; - + for (const property of Reflect.ownKeys(propertyDescriptors)) { + const {flags} = propertyDescriptors[property]; if (!flags.expose) { continue; } diff --git a/src/data/checks.js b/src/data/checks.js index 79a8d4c..fe4528e 100644 --- a/src/data/checks.js +++ b/src/data/checks.js @@ -37,7 +37,9 @@ export function reportDuplicateDirectories(wikiData, { const directoryPlaces = Object.create(null); const duplicateDirectories = new Set(); + const thingData = wikiData[findSpec.bindTo]; + if (!thingData) continue; for (const thing of thingData) { if (findSpec.include && !findSpec.include(thing)) { @@ -166,6 +168,10 @@ export function filterReferenceErrors(wikiData, { commentary: '_commentary', }], + ['flashData', { + commentary: '_commentary', + }], + ['groupCategoryData', { groups: 'group', }], @@ -257,7 +263,7 @@ export function filterReferenceErrors(wikiData, { break; case '_contrib': - findFn = contribRef => findArtistOrAlias(contribRef.who); + findFn = contribRef => findArtistOrAlias(contribRef.artist); break; case '_homepageSourceGroup': @@ -469,6 +475,10 @@ export class ContentNodeError extends Error { export function reportContentTextErrors(wikiData, { bindFind, }) { + const additionalFileShape = { + description: 'description', + }; + const commentaryShape = { body: 'commentary body', artistDisplayText: 'commentary artist display text', @@ -477,6 +487,7 @@ export function reportContentTextErrors(wikiData, { const contentTextSpec = [ ['albumData', { + additionalFiles: additionalFileShape, commentary: commentaryShape, }], @@ -484,8 +495,15 @@ export function reportContentTextErrors(wikiData, { contextNotes: '_content', }], + ['flashData', { + commentary: commentaryShape, + }], + ['flashActData', { - jump: '_content', + listTerminology: '_content', + }], + + ['flashSideData', { listTerminology: '_content', }], @@ -506,8 +524,11 @@ export function reportContentTextErrors(wikiData, { }], ['trackData', { + additionalFiles: additionalFileShape, commentary: commentaryShape, lyrics: '_content', + midiProjectFiles: additionalFileShape, + sheetMusicFiles: additionalFileShape, }], ['wikiInfo', { @@ -574,6 +595,16 @@ export function reportContentTextErrors(wikiData, { continue; } } + } else if (node.type === 'external-link') { + try { + new URL(node.data.href); + } catch (error) { + yield { + index, length, + message: + `Invalid URL ${colors.red(`"${node.data.href}"`)}`, + }; + } } } } diff --git a/src/data/composite.js b/src/data/composite.js index 7a98c42..33d69a6 100644 --- a/src/data/composite.js +++ b/src/data/composite.js @@ -29,6 +29,7 @@ input.value = _valueIntoToken('input.value'); input.dependency = _valueIntoToken('input.dependency'); input.myself = () => Symbol.for(`hsmusic.composite.input.myself`); +input.thisProperty = () => Symbol.for('hsmusic.composite.input.thisProperty'); input.updateValue = _valueIntoToken('input.updateValue'); @@ -284,6 +285,7 @@ export function templateCompositeFrom(description) { 'input.value', 'input.dependency', 'input.myself', + 'input.thisProperty', 'input.updateValue', ].includes(tokenShape)) { expectedValueProvidingTokenInputNames.push(name); @@ -567,6 +569,8 @@ export function compositeFrom(description) { return token; case 'input.myself': return 'this'; + case 'input.thisProperty': + return 'thisProperty'; default: return null; } @@ -721,6 +725,8 @@ export function compositeFrom(description) { return (tokenValue.startsWith('#') ? null : tokenValue); case 'input.myself': return 'this'; + case 'input.thisProperty': + return 'thisProperty'; default: return null; } @@ -774,16 +780,9 @@ export function compositeFrom(description) { (step.annotation ? ` (${step.annotation})` : ``); aggregate.nest({message}, ({push}) => { - if (isBase && stepComposes !== compositionNests) { + if (!isBase && !stepComposes) { return push(new TypeError( - (compositionNests - ? `Base must compose, this composition is nestable` - : `Base must not compose, this composition isn't nestable`))); - } else if (!isBase && !stepComposes) { - return push(new TypeError( - (compositionNests - ? `All steps must compose` - : `All steps (except base) must compose`))); + `All steps leading up to base must compose`)); } if ( @@ -877,6 +876,8 @@ export function compositeFrom(description) { return valueSoFar; case 'input.myself': return initialDependencies['this']; + case 'input.thisProperty': + return initialDependencies['thisProperty']; case 'input': return initialDependencies[token]; default: @@ -907,8 +908,16 @@ export function compositeFrom(description) { debug(() => colors.bright(`begin composition - not transforming`)); } - for (let i = 0; i < steps.length; i++) { - const step = steps[i]; + for ( + const [i, { + step, + stepComposes, + }] of + stitchArrays({ + step: steps, + stepComposes: stepsCompose, + }).entries() + ) { const isBase = i === steps.length - 1; debug(() => [ @@ -969,6 +978,7 @@ export function compositeFrom(description) { ? {[input.updateValue()]: valueSoFar} : {}), [input.myself()]: initialDependencies?.['this'] ?? null, + [input.thisProperty()]: initialDependencies?.['thisProperty'] ?? null, }; const selectDependencies = @@ -983,6 +993,8 @@ export function compositeFrom(description) { return dependency; case 'input.myself': return input.myself(); + case 'input.thisProperty': + return input.thisProperty(); case 'input.dependency': return tokenValue; case 'input.updateValue': @@ -1018,10 +1030,7 @@ export function compositeFrom(description) { let args; - if (isBase && !compositionNests) { - args = - argsLayout.filter(arg => arg !== continuationSymbol); - } else { + if (stepComposes) { let continuation; ({continuation, continuationStorage} = @@ -1032,6 +1041,9 @@ export function compositeFrom(description) { (arg === continuationSymbol ? continuation : arg)); + } else { + args = + argsLayout.filter(arg => arg !== continuationSymbol); } return expose[name](...args); @@ -1091,11 +1103,6 @@ export function compositeFrom(description) { if (result !== continuationSymbol) { debug(() => [`step #${i+1} - result: exit (inferred) ->`, result]); - - if (compositionNests) { - throw new TypeError(`Inferred early-exit is disallowed in nested compositions`); - } - debug(() => colors.bright(`end composition - exit (inferred)`)); return result; diff --git a/src/data/composite/control-flow/exposeConstant.js b/src/data/composite/control-flow/exposeConstant.js index e043547..e76699c 100644 --- a/src/data/composite/control-flow/exposeConstant.js +++ b/src/data/composite/control-flow/exposeConstant.js @@ -12,7 +12,7 @@ export default templateCompositeFrom({ compose: false, inputs: { - value: input.staticValue(), + value: input.staticValue({acceptsNull: true}), }, steps: () => [ diff --git a/src/data/composite/things/flash-act/index.js b/src/data/composite/things/flash-act/index.js new file mode 100644 index 0000000..40fecd2 --- /dev/null +++ b/src/data/composite/things/flash-act/index.js @@ -0,0 +1 @@ +export {default as withFlashSide} from './withFlashSide.js'; diff --git a/src/data/composite/things/flash-act/withFlashSide.js b/src/data/composite/things/flash-act/withFlashSide.js new file mode 100644 index 0000000..64daa1f --- /dev/null +++ b/src/data/composite/things/flash-act/withFlashSide.js @@ -0,0 +1,22 @@ +// Gets the flash act's side. This will early exit if flashSideData is missing. +// If there's no side whose list of flash acts includes this act, the output +// dependency will be null. + +import {input, templateCompositeFrom} from '#composite'; + +import {withUniqueReferencingThing} from '#composite/wiki-data'; + +export default templateCompositeFrom({ + annotation: `withFlashSide`, + + outputs: ['#flashSide'], + + steps: () => [ + withUniqueReferencingThing({ + data: 'flashSideData', + list: input.value('acts'), + }).outputs({ + ['#uniqueReferencingThing']: '#flashSide', + }), + ], +}); diff --git a/src/data/composite/things/flash/withFlashAct.js b/src/data/composite/things/flash/withFlashAct.js index ada2dcf..652b8bf 100644 --- a/src/data/composite/things/flash/withFlashAct.js +++ b/src/data/composite/things/flash/withFlashAct.js @@ -1,108 +1,22 @@ // Gets the flash's act. This will early exit if flashActData is missing. -// By default, if there's no flash whose list of flashes includes this flash, -// the output dependency will be null; set {notFoundMode: 'exit'} to early -// exit instead. -// -// This step models with Flash.withAlbum. +// If there's no flash whose list of flashes includes this flash, the output +// dependency will be null. import {input, templateCompositeFrom} from '#composite'; -import {is} from '#validators'; -import {exitWithoutDependency, withResultOfAvailabilityCheck} - from '#composite/control-flow'; -import {withPropertyFromList} from '#composite/data'; +import {withUniqueReferencingThing} from '#composite/wiki-data'; export default templateCompositeFrom({ annotation: `withFlashAct`, - inputs: { - notFoundMode: input({ - validate: is('exit', 'null'), - defaultValue: 'null', - }), - }, - outputs: ['#flashAct'], steps: () => [ - // null flashActData is always an early exit. - - exitWithoutDependency({ - dependency: 'flashActData', - mode: input.value('null'), - }), - - // empty flashActData conditionally exits early or outputs null. - - withResultOfAvailabilityCheck({ - from: 'flashActData', - mode: input.value('empty'), - }).outputs({ - '#availability': '#flashActDataAvailability', - }), - - { - dependencies: [input('notFoundMode'), '#flashActDataAvailability'], - compute(continuation, { - [input('notFoundMode')]: notFoundMode, - ['#flashActDataAvailability']: flashActDataIsAvailable, - }) { - if (flashActDataIsAvailable) return continuation(); - switch (notFoundMode) { - case 'exit': return continuation.exit(null); - case 'null': return continuation.raiseOutput({'#flashAct': null}); - } - }, - }, - - withPropertyFromList({ - list: 'flashActData', - property: input.value('flashes'), - }), - - { - dependencies: [input.myself(), '#flashActData.flashes'], - compute: (continuation, { - [input.myself()]: track, - ['#flashActData.flashes']: flashLists, - }) => continuation({ - ['#flashActIndex']: - flashLists.findIndex(flashes => flashes.includes(track)), - }), - }, - - // album not found conditionally exits or outputs null. - - withResultOfAvailabilityCheck({ - from: '#flashActIndex', - mode: input.value('index'), + withUniqueReferencingThing({ + data: 'flashActData', + list: input.value('flashes'), }).outputs({ - '#availability': '#flashActAvailability', + ['#uniqueReferencingThing']: '#flashAct', }), - - { - dependencies: [input('notFoundMode'), '#flashActAvailability'], - compute(continuation, { - [input('notFoundMode')]: notFoundMode, - ['#flashActAvailability']: flashActIsAvailable, - }) { - if (flashActIsAvailable) return continuation(); - switch (notFoundMode) { - case 'exit': return continuation.exit(null); - case 'null': return continuation.raiseOutput({'#flashAct': null}); - } - }, - }, - - { - dependencies: ['flashActData', '#flashActIndex'], - compute: (continuation, { - ['flashActData']: flashActData, - ['#flashActIndex']: flashActIndex, - }) => continuation.raiseOutput({ - ['#flashAct']: - flashActData[flashActIndex], - }), - }, ], }); diff --git a/src/data/composite/things/track/index.js b/src/data/composite/things/track/index.js index cc723a2..8959de9 100644 --- a/src/data/composite/things/track/index.js +++ b/src/data/composite/things/track/index.js @@ -9,3 +9,4 @@ export {default as withContainingTrackSection} from './withContainingTrackSectio export {default as withHasUniqueCoverArt} from './withHasUniqueCoverArt.js'; export {default as withOtherReleases} from './withOtherReleases.js'; export {default as withPropertyFromAlbum} from './withPropertyFromAlbum.js'; +export {default as withPropertyFromOriginalRelease} from './withPropertyFromOriginalRelease.js'; diff --git a/src/data/composite/things/track/inheritFromOriginalRelease.js b/src/data/composite/things/track/inheritFromOriginalRelease.js index 27ed138..38ab06b 100644 --- a/src/data/composite/things/track/inheritFromOriginalRelease.js +++ b/src/data/composite/things/track/inheritFromOriginalRelease.js @@ -1,8 +1,6 @@ -// Early exits with a value inherited from the original release, if -// this track is a rerelease, and otherwise continues with no further -// dependencies provided. If allowOverride is true, then the continuation -// will also be called if the original release exposed the requested -// property as null. +// Early exits with the value for the same property as specified on the +// original release, if this track is a rerelease, and otherwise continues +// without providing any further dependencies. // // Like withOriginalRelease, this will early exit (with notFoundValue) if the // original release is specified by reference and that reference doesn't @@ -10,41 +8,34 @@ import {input, templateCompositeFrom} from '#composite'; -import withOriginalRelease from './withOriginalRelease.js'; +import {exposeDependency, raiseOutputWithoutDependency} + from '#composite/control-flow'; + +import withPropertyFromOriginalRelease + from './withPropertyFromOriginalRelease.js'; export default templateCompositeFrom({ annotation: `inheritFromOriginalRelease`, inputs: { - property: input({type: 'string'}), - allowOverride: input({type: 'boolean', defaultValue: false}), - notFoundValue: input({defaultValue: null}), + notFoundValue: input({ + defaultValue: null, + }), }, steps: () => [ - withOriginalRelease({ + withPropertyFromOriginalRelease({ + property: input.thisProperty(), notFoundValue: input('notFoundValue'), }), - { - dependencies: [ - '#originalRelease', - input('property'), - input('allowOverride'), - ], - - compute: (continuation, { - ['#originalRelease']: originalRelease, - [input('property')]: originalProperty, - [input('allowOverride')]: allowOverride, - }) => { - if (!originalRelease) return continuation(); - - const value = originalRelease[originalProperty]; - if (allowOverride && value === null) return continuation(); - - return continuation.exit(value); - }, - }, + raiseOutputWithoutDependency({ + dependency: '#isRerelease', + mode: input.value('falsy'), + }), + + exposeDependency({ + dependency: '#originalValue', + }), ], }); diff --git a/src/data/composite/things/track/withAlbum.js b/src/data/composite/things/track/withAlbum.js index cbd16dc..03b840d 100644 --- a/src/data/composite/things/track/withAlbum.js +++ b/src/data/composite/things/track/withAlbum.js @@ -1,108 +1,22 @@ // Gets the track's album. This will early exit if albumData is missing. -// By default, if there's no album whose list of tracks includes this track, -// the output dependency will be null; set {notFoundMode: 'exit'} to early -// exit instead. -// -// This step models with Flash.withFlashAct. +// If there's no album whose list of tracks includes this track, the output +// dependency will be null. import {input, templateCompositeFrom} from '#composite'; -import {is} from '#validators'; -import {exitWithoutDependency, withResultOfAvailabilityCheck} - from '#composite/control-flow'; -import {withPropertyFromList} from '#composite/data'; +import {withUniqueReferencingThing} from '#composite/wiki-data'; export default templateCompositeFrom({ annotation: `withAlbum`, - inputs: { - notFoundMode: input({ - validate: is('exit', 'null'), - defaultValue: 'null', - }), - }, - outputs: ['#album'], steps: () => [ - // null albumData is always an early exit. - - exitWithoutDependency({ - dependency: 'albumData', - mode: input.value('null'), - }), - - // empty albumData conditionally exits early or outputs null. - - withResultOfAvailabilityCheck({ - from: 'albumData', - mode: input.value('empty'), - }).outputs({ - '#availability': '#albumDataAvailability', - }), - - { - dependencies: [input('notFoundMode'), '#albumDataAvailability'], - compute(continuation, { - [input('notFoundMode')]: notFoundMode, - ['#albumDataAvailability']: albumDataIsAvailable, - }) { - if (albumDataIsAvailable) return continuation(); - switch (notFoundMode) { - case 'exit': return continuation.exit(null); - case 'null': return continuation.raiseOutput({'#album': null}); - } - }, - }, - - withPropertyFromList({ - list: 'albumData', - property: input.value('tracks'), - }), - - { - dependencies: [input.myself(), '#albumData.tracks'], - compute: (continuation, { - [input.myself()]: track, - ['#albumData.tracks']: trackLists, - }) => continuation({ - ['#albumIndex']: - trackLists.findIndex(tracks => tracks.includes(track)), - }), - }, - - // album not found conditionally exits or outputs null. - - withResultOfAvailabilityCheck({ - from: '#albumIndex', - mode: input.value('index'), + withUniqueReferencingThing({ + data: 'albumData', + list: input.value('tracks'), }).outputs({ - '#availability': '#albumAvailability', + ['#uniqueReferencingThing']: '#album', }), - - { - dependencies: [input('notFoundMode'), '#albumAvailability'], - compute(continuation, { - [input('notFoundMode')]: notFoundMode, - ['#albumAvailability']: albumIsAvailable, - }) { - if (albumIsAvailable) return continuation(); - switch (notFoundMode) { - case 'exit': return continuation.exit(null); - case 'null': return continuation.raiseOutput({'#album': null}); - } - }, - }, - - { - dependencies: ['albumData', '#albumIndex'], - compute: (continuation, { - ['albumData']: albumData, - ['#albumIndex']: albumIndex, - }) => continuation.raiseOutput({ - ['#album']: - albumData[albumIndex], - }), - }, ], }); diff --git a/src/data/composite/things/track/withAlwaysReferenceByDirectory.js b/src/data/composite/things/track/withAlwaysReferenceByDirectory.js index fac8e21..e01720b 100644 --- a/src/data/composite/things/track/withAlwaysReferenceByDirectory.js +++ b/src/data/composite/things/track/withAlwaysReferenceByDirectory.js @@ -7,11 +7,15 @@ import {input, templateCompositeFrom} from '#composite'; import find from '#find'; import {isBoolean} from '#validators'; -import {exitWithoutDependency, exposeUpdateValueOrContinue} - from '#composite/control-flow'; import {withPropertyFromObject} from '#composite/data'; import {withResolvedReference} from '#composite/wiki-data'; +import { + exitWithoutDependency, + exposeDependencyOrContinue, + exposeUpdateValueOrContinue, +} from '#composite/control-flow'; + export default templateCompositeFrom({ annotation: `withAlwaysReferenceByDirectory`, @@ -22,6 +26,29 @@ export default templateCompositeFrom({ validate: input.value(isBoolean), }), + // withAlwaysReferenceByDirectory is sort of a fragile area - we can't + // find the track's album the normal way because albums' track lists + // recurse back into alwaysReferenceByDirectory! + withResolvedReference({ + ref: 'dataSourceAlbum', + data: 'albumData', + find: input.value(find.album), + }).outputs({ + '#resolvedReference': '#album', + }), + + withPropertyFromObject({ + object: '#album', + property: input.value('alwaysReferenceTracksByDirectory'), + }), + + // Falsy mode means this exposes true if the album's property is true, + // but continues if the property is false (which is also the default). + exposeDependencyOrContinue({ + dependency: '#album.alwaysReferenceTracksByDirectory', + mode: input.value('falsy'), + }), + // Remaining code is for defaulting to true if this track is a rerelease of // another with the same name, so everything further depends on access to // trackData as well as originalReleaseTrack. diff --git a/src/data/composite/things/track/withContainingTrackSection.js b/src/data/composite/things/track/withContainingTrackSection.js index b2e5f2b..eaac14d 100644 --- a/src/data/composite/things/track/withContainingTrackSection.js +++ b/src/data/composite/things/track/withContainingTrackSection.js @@ -1,63 +1,42 @@ // Gets the track section containing this track from its album's track list. -// If notFoundMode is set to 'exit', this will early exit if the album can't be -// found or if none of its trackSections includes the track for some reason. import {input, templateCompositeFrom} from '#composite'; import {is} from '#validators'; +import {raiseOutputWithoutDependency} from '#composite/control-flow'; + import withPropertyFromAlbum from './withPropertyFromAlbum.js'; export default templateCompositeFrom({ annotation: `withContainingTrackSection`, - inputs: { - notFoundMode: input({ - validate: is('exit', 'null'), - defaultValue: 'null', - }), - }, - outputs: ['#trackSection'], steps: () => [ withPropertyFromAlbum({ property: input.value('trackSections'), - notFoundMode: input('notFoundMode'), + }), + + raiseOutputWithoutDependency({ + dependency: '#album.trackSections', + output: input.value({'#trackSection': null}), }), { dependencies: [ input.myself(), - input('notFoundMode'), '#album.trackSections', ], - compute(continuation, { + compute: (continuation, { [input.myself()]: track, [input('notFoundMode')]: notFoundMode, ['#album.trackSections']: trackSections, - }) { - if (!trackSections) { - return continuation.raiseOutput({ - ['#trackSection']: null, - }); - } - - const trackSection = - trackSections.find(({tracks}) => tracks.includes(track)); - - if (trackSection) { - return continuation.raiseOutput({ - ['#trackSection']: trackSection, - }); - } else if (notFoundMode === 'exit') { - return continuation.exit(null); - } else { - return continuation.raiseOutput({ - ['#trackSection']: null, - }); - } - }, + }) => continuation({ + ['#trackSection']: + trackSections.find(({tracks}) => tracks.includes(track)) + ?? null, + }), }, ], }); diff --git a/src/data/composite/things/track/withPropertyFromAlbum.js b/src/data/composite/things/track/withPropertyFromAlbum.js index b236a6e..d41390f 100644 --- a/src/data/composite/things/track/withPropertyFromAlbum.js +++ b/src/data/composite/things/track/withPropertyFromAlbum.js @@ -1,7 +1,5 @@ // Gets a single property from this track's album, providing it as the same -// property name prefixed with '#album.' (by default). If the track's album -// isn't available, then by default, the property will be provided as null; -// set {notFoundMode: 'exit'} to early exit instead. +// property name prefixed with '#album.' (by default). import {input, templateCompositeFrom} from '#composite'; import {is} from '#validators'; @@ -15,11 +13,6 @@ export default templateCompositeFrom({ inputs: { property: input.staticValue({type: 'string'}), - - notFoundMode: input({ - validate: is('exit', 'null'), - defaultValue: 'null', - }), }, outputs: ({ @@ -27,9 +20,7 @@ export default templateCompositeFrom({ }) => ['#album.' + property], steps: () => [ - withAlbum({ - notFoundMode: input('notFoundMode'), - }), + withAlbum(), withPropertyFromObject({ object: '#album', diff --git a/src/data/composite/things/track/withPropertyFromOriginalRelease.js b/src/data/composite/things/track/withPropertyFromOriginalRelease.js new file mode 100644 index 0000000..fd37f6d --- /dev/null +++ b/src/data/composite/things/track/withPropertyFromOriginalRelease.js @@ -0,0 +1,86 @@ +// Provides a value inherited from the original release, if applicable, and a +// flag indicating if this track is a rerelase or not. +// +// Like withOriginalRelease, this will early exit (with notFoundValue) if the +// original release is specified by reference and that reference doesn't +// resolve to anything. + +import {input, templateCompositeFrom} from '#composite'; + +import {withResultOfAvailabilityCheck} from '#composite/control-flow'; +import {withPropertyFromObject} from '#composite/data'; + +import withOriginalRelease from './withOriginalRelease.js'; + +export default templateCompositeFrom({ + annotation: `inheritFromOriginalRelease`, + + inputs: { + property: input({type: 'string'}), + + notFoundValue: input({ + defaultValue: null, + }), + }, + + outputs: ({ + [input.staticValue('property')]: property, + }) => + ['#isRerelease'].concat( + (property + ? ['#original.' + property] + : ['#originalValue'])), + + steps: () => [ + withOriginalRelease({ + notFoundValue: input('notFoundValue'), + }), + + withResultOfAvailabilityCheck({ + from: '#originalRelease', + }), + + { + dependencies: [ + '#availability', + input.staticValue('property'), + ], + + compute: (continuation, { + ['#availability']: availability, + [input.staticValue('property')]: property, + }) => + (availability + ? continuation() + : continuation.raiseOutput( + Object.assign( + {'#isRerelease': false}, + (property + ? {['#original.' + property]: null} + : {'#originalValue': null})))), + }, + + withPropertyFromObject({ + object: '#originalRelease', + property: input('property'), + }), + + { + dependencies: [ + '#value', + input.staticValue('property'), + ], + + compute: (continuation, { + ['#value']: value, + [input.staticValue('property')]: property, + }) => + continuation.raiseOutput( + Object.assign( + {'#isRerelease': true}, + (property + ? {['#original.' + property]: value} + : {'#originalValue': value}))), + }, + ], +}); diff --git a/src/data/composite/wiki-data/index.js b/src/data/composite/wiki-data/index.js index 3ccfa75..b4cf6d1 100644 --- a/src/data/composite/wiki-data/index.js +++ b/src/data/composite/wiki-data/index.js @@ -13,3 +13,4 @@ export {default as withResolvedReferenceList} from './withResolvedReferenceList. export {default as withReverseContributionList} from './withReverseContributionList.js'; export {default as withReverseReferenceList} from './withReverseReferenceList.js'; export {default as withThingsSortedAlphabetically} from './withThingsSortedAlphabetically.js'; +export {default as withUniqueReferencingThing} from './withUniqueReferencingThing.js'; diff --git a/src/data/composite/wiki-data/withResolvedContribs.js b/src/data/composite/wiki-data/withResolvedContribs.js index 77b0f96..9526638 100644 --- a/src/data/composite/wiki-data/withResolvedContribs.js +++ b/src/data/composite/wiki-data/withResolvedContribs.js @@ -1,7 +1,8 @@ // Resolves the contribsByRef contained in the provided dependency, // providing (named by the second argument) the result. "Resolving" -// means mapping the "who" reference of each contribution to an artist -// object, and filtering out those whose "who" doesn't match any artist. +// means mapping the artist reference of each contribution to an artist +// object, and filtering out those whose artist reference doesn't match +// any artist. import {input, templateCompositeFrom} from '#composite'; import find from '#find'; @@ -46,29 +47,30 @@ export default templateCompositeFrom({ withPropertiesFromList({ list: input('from'), - properties: input.value(['who', 'what']), + properties: input.value(['artist', 'annotation']), prefix: input.value('#contribs'), }), withResolvedReferenceList({ - list: '#contribs.who', + list: '#contribs.artist', data: 'artistData', find: input.value(find.artist), notFoundMode: input('notFoundMode'), }).outputs({ - ['#resolvedReferenceList']: '#contribs.who', + ['#resolvedReferenceList']: '#contribs.artist', }), { - dependencies: ['#contribs.who', '#contribs.what'], + dependencies: ['#contribs.artist', '#contribs.annotation'], compute(continuation, { - ['#contribs.who']: who, - ['#contribs.what']: what, + ['#contribs.artist']: artist, + ['#contribs.annotation']: annotation, }) { - filterMultipleArrays(who, what, (who, _what) => who); + filterMultipleArrays(artist, annotation, (artist, _annotation) => artist); return continuation({ - ['#resolvedContribs']: stitchArrays({who, what}), + ['#resolvedContribs']: + stitchArrays({artist, annotation}), }); }, }, diff --git a/src/data/composite/wiki-data/withReverseContributionList.js b/src/data/composite/wiki-data/withReverseContributionList.js index eccb58b..91e125e 100644 --- a/src/data/composite/wiki-data/withReverseContributionList.js +++ b/src/data/composite/wiki-data/withReverseContributionList.js @@ -11,7 +11,8 @@ import {input, templateCompositeFrom} from '#composite'; -import {exitWithoutDependency} from '#composite/control-flow'; +import {exitWithoutDependency, raiseOutputWithoutDependency} + from '#composite/control-flow'; import inputWikiData from './inputWikiData.js'; @@ -32,10 +33,17 @@ export default templateCompositeFrom({ outputs: ['#reverseContributionList'], steps: () => [ + // Early exit with an empty array if the data list isn't available. exitWithoutDependency({ dependency: input('data'), value: input.value([]), + }), + + // Raise an empty array (don't early exit) if the data list is empty. + raiseOutputWithoutDependency({ + dependency: input('data'), mode: input.value('empty'), + output: input.value({'#reverseContributionList': []}), }), { @@ -58,10 +66,10 @@ export default templateCompositeFrom({ for (const referencingThing of data) { const referenceList = referencingThing[list]; - // Destructuring {who} is the only unique part of the + // Destructuring {artist} is the only unique part of the // withReverseContributionList implementation, compared to // withReverseReferneceList. - for (const {who: referencedThing} of referenceList) { + for (const {artist: referencedThing} of referenceList) { if (cacheRecord.has(referencedThing)) { cacheRecord.get(referencedThing).push(referencingThing); } else { diff --git a/src/data/composite/wiki-data/withReverseReferenceList.js b/src/data/composite/wiki-data/withReverseReferenceList.js index 2d7a421..8cd540a 100644 --- a/src/data/composite/wiki-data/withReverseReferenceList.js +++ b/src/data/composite/wiki-data/withReverseReferenceList.js @@ -13,7 +13,8 @@ import {input, templateCompositeFrom} from '#composite'; -import {exitWithoutDependency} from '#composite/control-flow'; +import {exitWithoutDependency, raiseOutputWithoutDependency} + from '#composite/control-flow'; import inputWikiData from './inputWikiData.js'; @@ -34,10 +35,17 @@ export default templateCompositeFrom({ outputs: ['#reverseReferenceList'], steps: () => [ + // Early exit with an empty array if the data list isn't available. exitWithoutDependency({ dependency: input('data'), value: input.value([]), + }), + + // Raise an empty array (don't early exit) if the data list is empty. + raiseOutputWithoutDependency({ + dependency: input('data'), mode: input.value('empty'), + output: input.value({'#reverseReferenceList': []}), }), { diff --git a/src/data/composite/wiki-data/withUniqueReferencingThing.js b/src/data/composite/wiki-data/withUniqueReferencingThing.js new file mode 100644 index 0000000..61c1061 --- /dev/null +++ b/src/data/composite/wiki-data/withUniqueReferencingThing.js @@ -0,0 +1,51 @@ +// Like withReverseReferenceList, but this is specifically for special "unique" +// references, meaning this thing is referenced by exactly one or zero things +// in the data list. + +import {input, templateCompositeFrom} from '#composite'; + +import {exitWithoutDependency, raiseOutputWithoutDependency} + from '#composite/control-flow'; + +import inputWikiData from './inputWikiData.js'; +import withReverseReferenceList from './withReverseReferenceList.js'; + +export default templateCompositeFrom({ + annotation: `withUniqueReferencingThing`, + + inputs: { + data: inputWikiData({allowMixedTypes: false}), + list: input({type: 'string'}), + }, + + outputs: ['#uniqueReferencingThing'], + + steps: () => [ + // Early exit with null (not an empty array) if the data list + // isn't available. + exitWithoutDependency({ + dependency: input('data'), + }), + + withReverseReferenceList({ + data: input('data'), + list: input('list'), + }), + + raiseOutputWithoutDependency({ + dependency: '#reverseReferenceList', + mode: input.value('empty'), + output: input.value({'#uniqueReferencingThing': null}), + }), + + { + dependencies: ['#reverseReferenceList'], + compute: (continuation, { + ['#reverseReferenceList']: reverseReferenceList, + }) => continuation({ + ['#uniqueReferencingThing']: + reverseReferenceList[0], + }), + }, + ], +}); diff --git a/src/data/composite/wiki-properties/contributionList.js b/src/data/composite/wiki-properties/contributionList.js index 8fde2ca..aad12a2 100644 --- a/src/data/composite/wiki-properties/contributionList.js +++ b/src/data/composite/wiki-properties/contributionList.js @@ -3,15 +3,15 @@ // into one property. Update value will look something like this: // // [ -// {who: 'Artist Name', what: 'Viola'}, -// {who: 'artist:john-cena', what: null}, +// {artist: 'Artist Name', annotation: 'Viola'}, +// {artist: 'artist:john-cena', annotation: null}, // ... // ] // // ...typically as processed from YAML, spreadsheet, or elsewhere. -// Exposes as the same, but with the "who" replaced with matches found in -// artistData - which means this always depends on an `artistData` property -// also existing on this object! +// Exposes as the same, but with the artist property replaced with matches +// found in artistData - which means this always depends on an `artistData` +// property also existing on this object! // import {input, templateCompositeFrom} from '#composite'; diff --git a/src/data/serialize.js b/src/data/serialize.js index 8cac330..2ecbf76 100644 --- a/src/data/serialize.js +++ b/src/data/serialize.js @@ -16,7 +16,10 @@ export function toRefs(things) { } export function toContribRefs(contribs) { - return contribs?.map(({who, what}) => ({who: toRef(who), what})); + return contribs?.map(({artist, annotation}) => ({ + artist: toRef(artist), + annotation, + })); } export function toCommentaryRefs(entries) { diff --git a/src/data/thing.js b/src/data/thing.js index 706e893..9a8cec9 100644 --- a/src/data/thing.js +++ b/src/data/thing.js @@ -17,6 +17,22 @@ export default class Thing extends CacheableObject { static yamlDocumentSpec = Symbol.for('Thing.yamlDocumentSpec'); static getYamlLoadingSpec = Symbol.for('Thing.getYamlLoadingSpec'); + static isThingConstructor = Symbol.for('Thing.isThingConstructor'); + static isThing = Symbol.for('Thing.isThing'); + + // To detect: + // Symbol.for('Thing.isThingConstructor') in constructor + static [Symbol.for('Thing.isThingConstructor')] = NaN; + + static [CacheableObject.propertyDescriptors] = { + // To detect: + // Object.hasOwn(object, Symbol.for('Thing.isThing')) + [Symbol.for('Thing.isThing')]: { + flags: {expose: true}, + expose: {compute: () => NaN}, + }, + }; + // Default custom inspect function, which may be overridden by Thing // subclasses. This will be used when displaying aggregate errors and other // command-line logging - it's the place to provide information useful in diff --git a/src/data/things/album.js b/src/data/things/album.js index 336d777..f835496 100644 --- a/src/data/things/album.js +++ b/src/data/things/album.js @@ -48,6 +48,8 @@ export class Album extends Thing { directory: directory(), urls: urls(), + alwaysReferenceTracksByDirectory: flag(false), + bandcampAlbumIdentifier: simpleString(), bandcampArtworkIdentifier: simpleString(), @@ -92,6 +94,11 @@ export class Album extends Thing { simpleString(), ], + coverArtDimensions: [ + exitWithoutContribs({contribs: 'coverArtistContribs'}), + dimensions(), + ], + bannerDimensions: [ exitWithoutContribs({contribs: 'bannerArtistContribs'}), dimensions(), @@ -221,6 +228,10 @@ export class Album extends Thing { 'Album': {property: 'name'}, 'Directory': {property: 'directory'}, + 'Always Reference Tracks By Directory': { + property: 'alwaysReferenceTracksByDirectory', + }, + 'Bandcamp Album ID': { property: 'bandcampAlbumIdentifier', transform: String, @@ -261,6 +272,11 @@ export class Album extends Thing { 'Cover Art File Extension': {property: 'coverArtFileExtension'}, 'Track Art File Extension': {property: 'trackCoverArtFileExtension'}, + 'Cover Art Dimensions': { + property: 'coverArtDimensions', + transform: parseDimensions, + }, + 'Wallpaper Artists': { property: 'wallpaperArtistContribs', transform: parseContributors, diff --git a/src/data/things/artist.js b/src/data/things/artist.js index 73acba6..841d652 100644 --- a/src/data/things/artist.js +++ b/src/data/things/artist.js @@ -220,6 +220,11 @@ export class Artist extends Thing { data: 'flashData', list: input.value('contributorContribs'), }), + + flashesAsCommentator: reverseReferenceList({ + data: 'flashData', + list: input.value('commentatorArtists'), + }), }); static [Thing.getSerializeDescriptors] = ({ diff --git a/src/data/things/flash.js b/src/data/things/flash.js index 81de327..ceed79f 100644 --- a/src/data/things/flash.js +++ b/src/data/things/flash.js @@ -2,17 +2,26 @@ export const FLASH_DATA_FILE = 'flashes.yaml'; import {input} from '#composite'; import find from '#find'; +import {empty} from '#sugar'; import {sortFlashesChronologically} from '#sort'; import Thing from '#thing'; -import {anyOf, isColor, isDirectory, isNumber, isString} from '#validators'; +import {anyOf, isColor, isContentString, isDirectory, isNumber, isString} + from '#validators'; import {parseDate, parseContributors} from '#yaml'; -import {exposeDependency, exposeUpdateValueOrContinue} - from '#composite/control-flow'; import {withPropertyFromObject} from '#composite/data'; import { + exposeConstant, + exposeDependency, + exposeDependencyOrContinue, + exposeUpdateValueOrContinue, +} from '#composite/control-flow'; + +import { color, + commentary, + commentatorArtists, contentString, contributionList, directory, @@ -25,6 +34,7 @@ import { } from '#composite/wiki-properties'; import {withFlashAct} from '#composite/things/flash'; +import {withFlashSide} from '#composite/things/flash-act'; export class Flash extends Thing { static [Thing.referenceType] = 'flash'; @@ -89,6 +99,8 @@ export class Flash extends Thing { urls: urls(), + commentary: commentary(), + // Update only artistData: wikiData({ @@ -105,16 +117,23 @@ export class Flash extends Thing { // Expose only - act: { - flags: {expose: true}, + commentatorArtists: commentatorArtists(), - expose: { - dependencies: ['this', 'flashActData'], + act: [ + withFlashAct(), + exposeDependency({dependency: '#flashAct'}), + ], - compute: ({this: flash, flashActData}) => - flashActData.find((act) => act.flashes.includes(flash)) ?? null, - }, - }, + side: [ + withFlashAct(), + + withPropertyFromObject({ + object: '#flashAct', + property: input.value('side'), + }), + + exposeDependency({dependency: '#flashAct.side'}), + ], }); static [Thing.getSerializeDescriptors] = ({ @@ -153,11 +172,14 @@ export class Flash extends Thing { 'Cover Art File Extension': {property: 'coverArtFileExtension'}, 'Featured Tracks': {property: 'featuredTracks'}, + 'Contributors': { property: 'contributorContribs', transform: parseContributors, }, + 'Commentary': {property: 'commentary'}, + 'Review Points': {ignore: true}, }, }; @@ -173,19 +195,27 @@ export class FlashAct extends Thing { name: name('Unnamed Flash Act'), directory: directory(), color: color(), - listTerminology: contentString(), - jump: contentString(), + listTerminology: [ + exposeUpdateValueOrContinue({ + validate: input.value(isContentString), + }), - jumpColor: { - flags: {update: true, expose: true}, - update: {validate: isColor}, - expose: { - dependencies: ['color'], - transform: (jumpColor, {color}) => - jumpColor ?? color, - } - }, + withFlashSide(), + + withPropertyFromObject({ + object: '#flashSide', + property: input.value('listTerminology'), + }), + + exposeDependencyOrContinue({ + dependency: '#flashSide.listTerminology', + }), + + exposeConstant({ + value: input.value(null), + }), + ], flashes: referenceList({ class: input.value(Flash), @@ -198,6 +228,17 @@ export class FlashAct extends Thing { flashData: wikiData({ class: input.value(Flash), }), + + flashSideData: wikiData({ + class: input.value(FlashSide), + }), + + // Expose only + + side: [ + withFlashSide(), + exposeDependency({dependency: '#flashSide'}), + ], }); static [Thing.findSpecs] = { @@ -215,12 +256,51 @@ export class FlashAct extends Thing { 'Color': {property: 'color'}, 'List Terminology': {property: 'listTerminology'}, - 'Jump': {property: 'jump'}, - 'Jump Color': {property: 'jumpColor'}, - 'Review Points': {ignore: true}, }, }; +} + +export class FlashSide extends Thing { + static [Thing.referenceType] = 'flash-side'; + static [Thing.friendlyName] = `Flash Side`; + + static [Thing.getPropertyDescriptors] = () => ({ + // Update & expose + + name: name('Unnamed Flash Side'), + directory: directory(), + color: color(), + listTerminology: contentString(), + + acts: referenceList({ + class: input.value(FlashAct), + find: input.value(find.flashAct), + data: 'flashActData', + }), + + // Update only + + flashActData: wikiData({ + class: input.value(FlashAct), + }), + }); + + static [Thing.yamlDocumentSpec] = { + fields: { + 'Side': {property: 'name'}, + 'Directory': {property: 'directory'}, + 'Color': {property: 'color'}, + 'List Terminology': {property: 'listTerminology'}, + }, + }; + + static [Thing.findSpecs] = { + flashSide: { + referenceTypes: ['flash-side'], + bindTo: 'flashSideData', + }, + }; static [Thing.getYamlLoadingSpec] = ({ documentModes: {allInOne}, @@ -231,39 +311,56 @@ export class FlashAct extends Thing { documentMode: allInOne, documentThing: document => - ('Act' in document + ('Side' in document + ? FlashSide + : 'Act' in document ? FlashAct : Flash), save(results) { - let flashAct; - let flashRefs = []; + // JavaScript likes you. - if (results[0] && !(results[0] instanceof FlashAct)) { - throw new Error(`Expected an act at top of flash data file`); + if (!empty(results) && !(results[0] instanceof FlashSide)) { + throw new Error(`Expected a side at top of flash data file`); } - for (const thing of results) { - if (thing instanceof FlashAct) { - if (flashAct) { - Object.assign(flashAct, {flashes: flashRefs}); - } + let index = 0; + let thing; + for (; thing = results[index]; index++) { + const flashSide = thing; + const flashActRefs = []; - flashAct = thing; - flashRefs = []; - } else { - flashRefs.push(Thing.getReference(thing)); + if (results[index + 1] instanceof Flash) { + throw new Error(`Expected an act to immediately follow a side`); } - } - if (flashAct) { - Object.assign(flashAct, {flashes: flashRefs}); + for ( + index++; + (thing = results[index]) && thing instanceof FlashAct; + index++ + ) { + const flashAct = thing; + const flashRefs = []; + for ( + index++; + (thing = results[index]) && thing instanceof Flash; + index++ + ) { + flashRefs.push(Thing.getReference(thing)); + } + index--; + flashAct.flashes = flashRefs; + flashActRefs.push(Thing.getReference(flashAct)); + } + index--; + flashSide.acts = flashActRefs; } const flashData = results.filter(x => x instanceof Flash); const flashActData = results.filter(x => x instanceof FlashAct); + const flashSideData = results.filter(x => x instanceof FlashSide); - return {flashData, flashActData}; + return {flashData, flashActData, flashSideData}; }, sort({flashData}) { diff --git a/src/data/things/index.js b/src/data/things/index.js index 3bf8409..4f87f49 100644 --- a/src/data/things/index.js +++ b/src/data/things/index.js @@ -2,10 +2,10 @@ import * as path from 'node:path'; import {fileURLToPath} from 'node:url'; import {openAggregate, showAggregate} from '#aggregate'; +import CacheableObject from '#cacheable-object'; import {logError} from '#cli'; import {compositeFrom} from '#composite'; import * as serialize from '#serialize'; - import Thing from '#thing'; import * as albumClasses from './album.js'; @@ -142,7 +142,10 @@ function evaluatePropertyDescriptors() { } } - constructor.propertyDescriptors = results; + constructor[CacheableObject.propertyDescriptors] = { + ...constructor[CacheableObject.propertyDescriptors] ?? {}, + ...results, + }; }, showFailedClasses(failedClasses) { diff --git a/src/data/things/language.js b/src/data/things/language.js index 93ed40b..dbe1ff3 100644 --- a/src/data/things/language.js +++ b/src/data/things/language.js @@ -2,6 +2,7 @@ import { Temporal, toTemporalInstant } from '@js-temporal/polyfill'; import {withAggregate} from '#aggregate'; import CacheableObject from '#cacheable-object'; +import {logWarn} from '#cli'; import * as html from '#html'; import {empty} from '#sugar'; import {isLanguageCode} from '#validators'; @@ -17,6 +18,8 @@ import { import {externalFunction, flag, name} from '#composite/wiki-properties'; +export const languageOptionRegex = /{(?<name>[A-Z0-9_]+)}/g; + export class Language extends Thing { static [Thing.getPropertyDescriptors] = () => ({ // Update & expose @@ -60,14 +63,46 @@ export class Language extends Thing { strings: { flags: {update: true, expose: true}, update: {validate: (t) => typeof t === 'object'}, + expose: { - dependencies: ['inheritedStrings'], - transform(strings, {inheritedStrings}) { - if (strings || inheritedStrings) { - return {...(inheritedStrings ?? {}), ...(strings ?? {})}; - } else { - return null; + dependencies: ['inheritedStrings', 'code'], + transform(strings, {inheritedStrings, code}) { + if (!strings && !inheritedStrings) return null; + if (!inheritedStrings) return strings; + + const validStrings = { + ...inheritedStrings, + ...strings, + }; + + const optionsFromTemplate = template => + Array.from(template.matchAll(languageOptionRegex)) + .map(({groups}) => groups.name); + + for (const [key, providedTemplate] of Object.entries(strings)) { + const inheritedTemplate = inheritedStrings[key]; + if (!inheritedTemplate) continue; + + const providedOptions = optionsFromTemplate(providedTemplate); + const inheritedOptions = optionsFromTemplate(inheritedTemplate); + + const missingOptionNames = + inheritedOptions.filter(name => !providedOptions.includes(name)); + + const misplacedOptionNames = + providedOptions.filter(name => !inheritedOptions.includes(name)); + + if (!empty(missingOptionNames) || !empty(misplacedOptionNames)) { + logWarn`Not using ${code ?? '(no code)'} string ${key}:`; + if (!empty(missingOptionNames)) + logWarn`- Missing options: ${missingOptionNames.join(', ')}`; + if (!empty(misplacedOptionNames)) + logWarn`- Unexpected options: ${misplacedOptionNames.join(', ')}`; + validStrings[key] = inheritedStrings[key]; + } } + + return validStrings; }, }, }, @@ -201,7 +236,7 @@ export class Language extends Thing { const output = this.#iterateOverTemplate({ template: this.strings[key], - match: /{(?<name>[A-Z0-9_]+)}/g, + match: languageOptionRegex, insert: ({name: optionName}, canceledForming) => { if (optionsMap.has(optionName)) { @@ -357,6 +392,7 @@ export class Language extends Thing { // contents, if needed. #wrapSanitized(content) { return html.tags(content, { + [html.blessAttributes]: true, [html.joinChildren]: '', [html.noEdgeWhitespace]: true, }); @@ -522,7 +558,7 @@ export class Language extends Thing { } formatExternalLink(url, { - style = 'normal', + style = 'platform', context = 'generic', } = {}) { if (!this.externalLinkSpec) { @@ -540,10 +576,16 @@ export class Language extends Thing { isExternalLinkStyle(style); - return getExternalLinkStringOfStyleFromDescriptors(url, style, this.externalLinkSpec, { - language: this, - context, - }); + const result = + getExternalLinkStringOfStyleFromDescriptors(url, style, this.externalLinkSpec, { + language: this, + context, + }); + + // It's possible for there to not actually be any string available for the + // given URL, style, and context, and we want this to be detectable via + // html.blank(). + return result ?? html.blank(); } formatIndex(value) { diff --git a/src/data/things/track.js b/src/data/things/track.js index 697dad4..725b1bb 100644 --- a/src/data/things/track.js +++ b/src/data/things/track.js @@ -13,6 +13,7 @@ import { parseAdditionalNames, parseContributors, parseDate, + parseDimensions, parseDuration, } from '#yaml'; @@ -34,6 +35,7 @@ import { commentatorArtists, contentString, contributionList, + dimensions, directory, duration, flag, @@ -158,13 +160,15 @@ export class Track extends Thing { exposeDependency({dependency: '#album.trackArtDate'}), ], + coverArtDimensions: [ + exitWithoutUniqueCoverArt(), + dimensions(), + ], + commentary: commentary(), lyrics: [ - inheritFromOriginalRelease({ - property: input.value('lyrics'), - }), - + inheritFromOriginalRelease(), contentString(), ], @@ -189,7 +193,6 @@ export class Track extends Thing { artistContribs: [ inheritFromOriginalRelease({ - property: input.value('artistContribs'), notFoundValue: input.value([]), }), @@ -213,7 +216,6 @@ export class Track extends Thing { contributorContribs: [ inheritFromOriginalRelease({ - property: input.value('contributorContribs'), notFoundValue: input.value([]), }), @@ -248,7 +250,6 @@ export class Track extends Thing { referencedTracks: [ inheritFromOriginalRelease({ - property: input.value('referencedTracks'), notFoundValue: input.value([]), }), @@ -261,7 +262,6 @@ export class Track extends Thing { sampledTracks: [ inheritFromOriginalRelease({ - property: input.value('sampledTracks'), notFoundValue: input.value([]), }), @@ -389,6 +389,11 @@ export class Track extends Thing { 'Cover Art File Extension': {property: 'coverArtFileExtension'}, + 'Cover Art Dimensions': { + property: 'coverArtDimensions', + transform: parseDimensions, + }, + 'Has Cover Art': { property: 'disableUniqueCoverArt', transform: value => diff --git a/src/data/validators.js b/src/data/validators.js index 4fc2ac6..5d68131 100644 --- a/src/data/validators.js +++ b/src/data/validators.js @@ -311,8 +311,11 @@ export function isCommentary(commentaryText) { const ownInput = commentaryText.slice(position, position + length); const restOfInput = commentaryText.slice(position + length); - const nextLineBreak = restOfInput.indexOf('\n'); - const upToNextLineBreak = restOfInput.slice(0, nextLineBreak); + + const upToNextLineBreak = + (restOfInput.includes('\n') + ? restOfInput.slice(0, restOfInput.indexOf('\n')) + : restOfInput); if (/\S/.test(upToNextLineBreak)) { throw new TypeError( @@ -420,6 +423,14 @@ const illegalContentSpec = [ {illegal: '\u2005', annotation: `four-per-em space`, ...illegalVisibleSpace}, {illegal: '\u205f', annotation: `medium mathematical space`, ...illegalVisibleSpace}, {illegal: '\xa0', annotation: `non-breaking space`, ...illegalVisibleSpace}, + + { + action: 'replace', + illegal: '<a href', + annotation: `HTML-style link`, + with: '[...](...)', + withAnnotation: `markdown`, + }, ]; for (const entry of illegalContentSpec) { @@ -432,24 +443,23 @@ for (const entry of illegalContentSpec) { } } -const illegalContentRegexp = - new RegExp( - illegalContentSpec - .map(entry => entry.illegal) - .map(illegal => `${illegal}+`) - .join('|'), - 'g'); - -const illegalCharactersInContent = +const illegalSequencesInContent = illegalContentSpec .map(entry => entry.illegal) - .join(''); + .map(illegal => + (illegal.length === 1 + ? `${illegal}+` + : `(?:${illegal})+`)) + .join('|'); + +const illegalContentRegexp = + new RegExp(illegalSequencesInContent, 'g'); const legalContentNearEndRegexp = - new RegExp(`[^\n${illegalCharactersInContent}]+$`); + new RegExp(`(?<=^|${illegalSequencesInContent})(?:(?!${illegalSequencesInContent}).)+$`); const legalContentNearStartRegexp = - new RegExp(`^[^\n${illegalCharactersInContent}]+`); + new RegExp(`^(?:(?!${illegalSequencesInContent}).)+`); const trimWhitespaceNearBothSidesRegexp = /^ +| +$/gm; @@ -595,16 +605,37 @@ export function isContentString(content) { export function isThingClass(thingClass) { isFunction(thingClass); - if (!Object.hasOwn(thingClass, Symbol.for('Thing.referenceType'))) { - throw new TypeError(`Expected a Thing constructor, missing Thing.referenceType`); + // This is *expressly* no faster than an instanceof check, because it's + // deliberately still walking the prototype chain for the provided object. + // (This is necessary because the symbol we're checking is defined only on + // the Thing constructor, and not directly on each subclass.) However, it's + // preferred over an instanceof check anyway, because instanceof would + // require that the #validators module has access to #thing, which it + // currently doesn't! + if (!(Symbol.for('Thing.isThingConstructor') in thingClass)) { + throw new TypeError(`Expected a Thing constructor, missing Thing.isThingConstructor`); + } + + return true; +} + +export function isThing(thing) { + isObject(thing); + + // This *is* faster than an instanceof check, because it doesn't walk the + // prototype chain. It works because this property is set as part of every + // Thing subclass's inherited "public class fields" - it's set directly on + // every constructed Thing. + if (!Object.hasOwn(thing, Symbol.for('Thing.isThing'))) { + throw new TypeError(`Expected a Thing, missing Thing.isThing`); } return true; } export const isContribution = validateProperties({ - who: isArtistRef, - what: optional(isStringNonEmpty), + artist: isArtistRef, + annotation: optional(isStringNonEmpty), }); export const isContributionList = validateArrayItems(isContribution); @@ -612,7 +643,7 @@ export const isContributionList = validateArrayItems(isContribution); export const isAdditionalFile = validateProperties({ title: isName, description: optional(isContentString), - files: validateArrayItems(isString), + files: optional(validateArrayItems(isString)), }); export const isAdditionalFileList = validateArrayItems(isAdditionalFile); @@ -632,10 +663,15 @@ export function isDimensions(dimensions) { if (dimensions.length !== 2) throw new TypeError(`Expected 2 item array`); - isPositive(dimensions[0]); - isInteger(dimensions[0]); - isPositive(dimensions[1]); - isInteger(dimensions[1]); + if (dimensions[0] !== null) { + isPositive(dimensions[0]); + isInteger(dimensions[0]); + } + + if (dimensions[1] !== null) { + isPositive(dimensions[1]); + isInteger(dimensions[1]); + } return true; } @@ -718,12 +754,31 @@ export function validateReferenceList(type = '') { return validateArrayItems(validateReference(type)); } +export function validateThing({ + referenceType: expectedReferenceType = '', +} = {}) { + return (thing) => { + isThing(thing); + + if (expectedReferenceType) { + const {[Symbol.for('Thing.referenceType')]: referenceType} = + thing.constructor; + + if (referenceType !== expectedReferenceType) { + throw new TypeError(`Expected only ${expectedReferenceType}, got other type: ${referenceType}`); + } + } + + return true; + }; +} + const validateWikiData_cache = {}; export function validateWikiData({ referenceType = '', allowMixedTypes = false, -}) { +} = {}) { if (referenceType && allowMixedTypes) { throw new TypeError(`Don't specify both referenceType and allowMixedTypes`); } @@ -752,25 +807,22 @@ export function validateWikiData({ let foundOtherObject = false; for (const object of array) { - const {[Symbol.for('Thing.referenceType')]: referenceType} = object.constructor; - - if (referenceType === undefined) { - foundOtherObject = true; - - // Early-exit if a Thing has been found - nothing more can be learned. - if (foundThing) { - throw new TypeError(`Expected array of wiki data objects, got mixed items`); - } - } else { - foundThing = true; - + if (Object.hasOwn(object, Symbol.for('Thing.isThing'))) { // Early-exit if a non-Thing object has been found - nothing more can // be learned. if (foundOtherObject) { throw new TypeError(`Expected array of wiki data objects, got mixed items`); } - allRefTypes.add(referenceType); + foundThing = true; + allRefTypes.add(object.constructor[Symbol.for('Thing.referenceType')]); + } else { + // Early-exit if a Thing has been found - nothing more can be learned. + if (foundThing) { + throw new TypeError(`Expected array of wiki data objects, got mixed items`); + } + + foundOtherObject = true; } } diff --git a/src/data/yaml.js b/src/data/yaml.js index f84f037..bd0b55d 100644 --- a/src/data/yaml.js +++ b/src/data/yaml.js @@ -9,26 +9,32 @@ import yaml from 'js-yaml'; import {colors, ENABLE_COLOR, logInfo, logWarn} from '#cli'; import {sortByName} from '#sort'; -import {atOffset, empty, filterProperties, typeAppearance, withEntries} - from '#sugar'; import Thing from '#thing'; import thingConstructors from '#things'; import { - filterReferenceErrors, - reportContentTextErrors, - reportDuplicateDirectories, -} from '#data-checks'; - -import { annotateErrorWithFile, decorateErrorWithIndex, decorateErrorWithAnnotation, openAggregate, showAggregate, - withAggregate, } from '#aggregate'; +import { + filterReferenceErrors, + reportContentTextErrors, + reportDuplicateDirectories, +} from '#data-checks'; + +import { + atOffset, + empty, + filterProperties, + stitchArrays, + typeAppearance, + withEntries, +} from '#sugar'; + function inspect(value, opts = {}) { return nodeInspect(value, {colors: ENABLE_COLOR, ...opts}); } @@ -393,7 +399,16 @@ export function parseContributors(contributionStrings) { return contributionStrings.map(item => { if (typeof item === 'object' && item['Who']) - return {who: item['Who'], what: item['What'] ?? null}; + return { + artist: item['Who'], + annotation: item['What'] ?? null, + }; + + if (typeof item === 'object' && item['Artist']) + return { + artist: item['Artist'], + annotation: item['Annotation'] ?? null, + }; if (typeof item !== 'string') return item; @@ -401,8 +416,8 @@ export function parseContributors(contributionStrings) { if (!match) return item; return { - who: match.groups.main, - what: match.groups.accent ?? null, + artist: match.groups.main, + annotation: match.groups.accent ?? null, }; }); } @@ -523,7 +538,13 @@ export const documentModes = { // them to each other, setting additional properties, etc). Input argument // format depends on documentMode. // -export const getDataSteps = () => { +export function getAllDataSteps() { + try { + thingConstructors; + } catch (error) { + throw new Error(`Thing constructors aren't ready yet, can't get all data steps`); + } + const steps = []; for (const thingConstructor of Object.values(thingConstructors)) { @@ -539,376 +560,501 @@ export const getDataSteps = () => { sortByName(steps, {getName: step => step.title}); return steps; -}; - -export async function loadAndProcessDataDocuments({dataPath}) { - const processDataAggregate = openAggregate({ - message: `Errors processing data files`, - }); - const wikiDataResult = {}; - - function decorateErrorWithFile(fn) { - return decorateErrorWithAnnotation(fn, - (caughtError, firstArg) => - annotateErrorWithFile( - caughtError, - path.relative( - dataPath, - (typeof firstArg === 'object' - ? firstArg.file - : firstArg)))); - } +} - function asyncDecorateErrorWithFile(fn) { - return decorateErrorWithFile(fn).async; - } +export async function getFilesFromDataStep(dataStep, {dataPath}) { + const {documentMode} = dataStep; - for (const dataStep of getDataSteps()) { - await processDataAggregate.nestAsync( - { - message: `Errors during data step: ${colors.bright(dataStep.title)}`, - translucent: true, - }, - async ({call, callAsync, map, mapAsync, push}) => { - const {documentMode} = dataStep; - - if (!Object.values(documentModes).includes(documentMode)) { - throw new Error(`Invalid documentMode: ${documentMode.toString()}`); - } + switch (documentMode) { + case documentModes.allInOne: + case documentModes.oneDocumentTotal: { + if (!dataStep.file) { + throw new Error(`Expected 'file' property for ${documentMode.toString()}`); + } - // Hear me out, it's been like 1200 years since I wrote the rest of - // this beautifully error-containing code and I don't know how to - // integrate this nicely. So I'm just returning the result and the - // error that should be thrown. Yes, we're back in callback hell, - // just without the callbacks. Thank you. - const filterBlankDocuments = documents => { - const aggregate = openAggregate({ - message: `Found blank documents - check for extra '${colors.cyan(`---`)}'`, + const localFile = + (typeof dataStep.file === 'function' + ? await dataStep.file(dataPath) + : dataStep.file); + + const fileUnderDataPath = + path.join(dataPath, localFile); + + const statResult = + await stat(fileUnderDataPath).then( + () => true, + error => { + if (error.code === 'ENOENT') { + return false; + } else { + throw error; + } }); - const filteredDocuments = - documents - .filter(doc => doc !== null); - - if (filteredDocuments.length !== documents.length) { - const blankIndexRangeInfo = - documents - .map((doc, index) => [doc, index]) - .filter(([doc]) => doc === null) - .map(([doc, index]) => index) - .reduce((accumulator, index) => { - if (accumulator.length === 0) { - return [[index, index]]; - } - const current = accumulator.at(-1); - const rest = accumulator.slice(0, -1); - if (current[1] === index - 1) { - return rest.concat([[current[0], index]]); - } else { - return accumulator.concat([[index, index]]); - } - }, []) - .map(([start, end]) => ({ - start, - end, - count: end - start + 1, - previous: atOffset(documents, start, -1), - next: atOffset(documents, end, +1), - })); - - for (const {start, end, count, previous, next} of blankIndexRangeInfo) { - const parts = []; - - if (count === 1) { - const range = `#${start + 1}`; - parts.push(`${count} document (${colors.yellow(range)}), `); - } else { - const range = `#${start + 1}-${end + 1}`; - parts.push(`${count} documents (${colors.yellow(range)}), `); - } - - if (previous === null) { - parts.push(`at start of file`); - } else if (next === null) { - parts.push(`at end of file`); - } else { - const previousDescription = Object.entries(previous).at(0).join(': '); - const nextDescription = Object.entries(next).at(0).join(': '); - parts.push(`between "${colors.cyan(previousDescription)}" and "${colors.cyan(nextDescription)}"`); - } - - aggregate.push(new Error(parts.join(''))); - } - } + if (statResult) { + return [fileUnderDataPath]; + } else { + return []; + } + } - return {documents: filteredDocuments, aggregate}; - }; + case documentModes.headerAndEntries: + case documentModes.onePerFile: { + if (!dataStep.files) { + throw new Error(`Expected 'files' property for ${documentMode.toString()}`); + } - const processDocument = (document, thingClassOrFn) => { - const thingClass = - (thingClassOrFn.prototype instanceof Thing - ? thingClassOrFn - : thingClassOrFn(document)); + const localFiles = + (typeof dataStep.files === 'function' + ? await dataStep.files(dataPath).then( + files => files, + error => { + if (error.code === 'ENOENT') { + return []; + } else { + throw error; + } + }) + : dataStep.files); - if (typeof thingClass !== 'function') { - throw new Error(`Expected a thing class, got ${typeAppearance(thingClass)}`); - } + const filesUnderDataPath = + localFiles + .map(file => path.join(dataPath, file)); - if (!(thingClass.prototype instanceof Thing)) { - throw new Error(`Expected a thing class, got ${thingClass.name}`); - } + return filesUnderDataPath; + } - const spec = thingClass[Thing.yamlDocumentSpec]; + default: + throw new Error(`Unknown document mode ${documentMode.toString()}`); + } +} - if (!spec) { - throw new Error(`Class "${thingClass.name}" doesn't specify Thing.yamlDocumentSpec`); - } +export async function loadYAMLDocumentsFromFile(file) { + let contents; + try { + contents = await readFile(file, 'utf-8'); + } catch (caughtError) { + throw new Error(`Failed to read data file`, {cause: caughtError}); + } - // TODO: Making a function to only call it just like that is - // obviously pretty jank! It should be created once per data step. - const fn = makeProcessDocument(thingClass, spec); - return fn(document); - }; - - if ( - documentMode === documentModes.allInOne || - documentMode === documentModes.oneDocumentTotal - ) { - if (!dataStep.file) { - throw new Error(`Expected 'file' property for ${documentMode.toString()}`); + let documents; + try { + documents = yaml.loadAll(contents); + } catch (caughtError) { + throw new Error(`Failed to parse valid YAML`, {cause: caughtError}); + } + + const aggregate = openAggregate({ + message: `Found blank documents - check for extra '${colors.cyan(`---`)}'`, + }); + + const filteredDocuments = + documents + .filter(doc => doc !== null); + + if (filteredDocuments.length !== documents.length) { + const blankIndexRangeInfo = + documents + .map((doc, index) => [doc, index]) + .filter(([doc]) => doc === null) + .map(([doc, index]) => index) + .reduce((accumulator, index) => { + if (accumulator.length === 0) { + return [[index, index]]; + } + const current = accumulator.at(-1); + const rest = accumulator.slice(0, -1); + if (current[1] === index - 1) { + return rest.concat([[current[0], index]]); + } else { + return accumulator.concat([[index, index]]); } + }, []) + .map(([start, end]) => ({ + start, + end, + count: end - start + 1, + previous: atOffset(documents, start, -1), + next: atOffset(documents, end, +1), + })); + + for (const {start, end, count, previous, next} of blankIndexRangeInfo) { + const parts = []; + + if (count === 1) { + const range = `#${start + 1}`; + parts.push(`${count} document (${colors.yellow(range)}), `); + } else { + const range = `#${start + 1}-${end + 1}`; + parts.push(`${count} documents (${colors.yellow(range)}), `); + } - const file = path.join( - dataPath, - typeof dataStep.file === 'function' - ? await callAsync(dataStep.file, dataPath) - : dataStep.file); + if (previous === null) { + parts.push(`at start of file`); + } else if (next === null) { + parts.push(`at end of file`); + } else { + const previousDescription = Object.entries(previous).at(0).join(': '); + const nextDescription = Object.entries(next).at(0).join(': '); + parts.push(`between "${colors.cyan(previousDescription)}" and "${colors.cyan(nextDescription)}"`); + } - const statResult = await callAsync(() => - stat(file).then( - () => true, - error => { - if (error.code === 'ENOENT') { - return false; - } else { - throw error; - } - })); + aggregate.push(new Error(parts.join(''))); + } + } - if (statResult === false) { - const saveResult = call(dataStep.save, { - [documentModes.allInOne]: [], - [documentModes.oneDocumentTotal]: {}, - }[documentMode]); + return {result: filteredDocuments, aggregate}; +} - if (!saveResult) return; +// Mapping from dataStep (spec) object each to a sub-map, from thing class to +// processDocument function. +const processDocumentFns = new WeakMap(); - Object.assign(wikiDataResult, saveResult); +export function processThingsFromDataStep(documents, dataStep) { + let submap; + if (processDocumentFns.has(dataStep)) { + submap = processDocumentFns.get(dataStep); + } else { + submap = new Map(); + processDocumentFns.set(dataStep, submap); + } - return; - } + function processDocument(document, thingClassOrFn) { + const thingClass = + (thingClassOrFn.prototype instanceof Thing + ? thingClassOrFn + : thingClassOrFn(document)); + + let fn; + if (submap.has(thingClass)) { + fn = submap.get(thingClass); + } else { + if (typeof thingClass !== 'function') { + throw new Error(`Expected a thing class, got ${typeAppearance(thingClass)}`); + } - const readResult = await callAsync(readFile, file, 'utf-8'); + if (!(thingClass.prototype instanceof Thing)) { + throw new Error(`Expected a thing class, got ${thingClass.name}`); + } - if (!readResult) { - return; - } + const spec = thingClass[Thing.yamlDocumentSpec]; - let processResults; + if (!spec) { + throw new Error(`Class "${thingClass.name}" doesn't specify Thing.yamlDocumentSpec`); + } - switch (documentMode) { - case documentModes.oneDocumentTotal: { - const yamlResult = call(yaml.load, readResult); + fn = makeProcessDocument(thingClass, spec); + submap.set(thingClass, fn); + } - if (!yamlResult) { - processResults = null; - break; - } + return fn(document); + } - const {thing, aggregate} = - processDocument(yamlResult, dataStep.documentThing); + const {documentMode} = dataStep; - processResults = thing; + switch (documentMode) { + case documentModes.allInOne: { + const result = []; + const aggregate = openAggregate({message: `Errors processing documents`}); - call(() => aggregate.close()); + documents.forEach( + decorateErrorWithIndex(document => { + const {thing, aggregate: subAggregate} = + processDocument(document, dataStep.documentThing); - break; - } + result.push(thing); + aggregate.call(subAggregate.close); + })); - case documentModes.allInOne: { - const yamlResults = call(yaml.loadAll, readResult); + return {aggregate, result}; + } - if (!yamlResults) { - processResults = []; - return; - } + case documentModes.oneDocumentTotal: { + if (documents.length > 1) + throw new Error(`Only expected one document to be present, got ${documents.length}`); - const {documents, aggregate: filterAggregate} = - filterBlankDocuments(yamlResults); + const {thing, aggregate} = + processDocument(documents[0], dataStep.documentThing); - call(filterAggregate.close); + return {aggregate, result: thing}; + } - processResults = []; + case documentModes.headerAndEntries: { + const headerDocument = documents[0]; + const entryDocuments = documents.slice(1).filter(Boolean); - map(documents, decorateErrorWithIndex(document => { - const {thing, aggregate} = - processDocument(document, dataStep.documentThing); + if (!headerDocument) + throw new Error(`Missing header document (empty file or erroneously starting with "---"?)`); - processResults.push(thing); - aggregate.close(); - }), {message: `Errors processing documents`}); + const aggregate = openAggregate({message: `Errors processing documents`}); - break; - } - } + const {thing: headerThing, aggregate: headerAggregate} = + processDocument(headerDocument, dataStep.headerDocumentThing); - if (!processResults) return; + try { + headerAggregate.close(); + } catch (caughtError) { + caughtError.message = `(${colors.yellow(`header`)}) ${caughtError.message}`; + aggregate.push(caughtError); + } - const saveResult = call(dataStep.save, processResults); + const entryThings = []; - if (!saveResult) return; + for (const [index, entryDocument] of entryDocuments.entries()) { + const {thing: entryThing, aggregate: entryAggregate} = + processDocument(entryDocument, dataStep.entryDocumentThing); - Object.assign(wikiDataResult, saveResult); + entryThings.push(entryThing); - return; + try { + entryAggregate.close(); + } catch (caughtError) { + caughtError.message = `(${colors.yellow(`entry #${index + 1}`)}) ${caughtError.message}`; + aggregate.push(caughtError); } + } - if (!dataStep.files) { - throw new Error(`Expected 'files' property for ${documentMode.toString()}`); - } + return { + aggregate, + result: { + header: headerThing, + entries: entryThings, + }, + }; + } - const filesFromDataStep = - (typeof dataStep.files === 'function' - ? await callAsync(() => - dataStep.files(dataPath).then( - files => files, - error => { - if (error.code === 'ENOENT') { - return []; - } else { - throw error; - } - })) - : dataStep.files); - - const filesUnderDataPath = - filesFromDataStep - .map(file => path.join(dataPath, file)); - - const yamlResults = []; - - await mapAsync(filesUnderDataPath, {message: `Errors loading data files`}, - asyncDecorateErrorWithFile(async file => { - let contents; - try { - contents = await readFile(file, 'utf-8'); - } catch (caughtError) { - throw new Error(`Failed to read data file`, {cause: caughtError}); - } + case documentModes.onePerFile: { + if (documents.length > 1) + throw new Error(`Only expected one document to be present per file, got ${documents.length} here`); - let documents; - try { - documents = yaml.loadAll(contents); - } catch (caughtError) { - throw new Error(`Failed to parse valid YAML`, {cause: caughtError}); - } + if (empty(documents) || !documents[0]) + throw new Error(`Expected a document, this file is empty`); - const {documents: filteredDocuments, aggregate: filterAggregate} = - filterBlankDocuments(documents); - - try { - filterAggregate.close(); - } catch (caughtError) { - // Blank documents aren't a critical error, they're just something - // that should be noted - the (filtered) documents still get pushed. - const pathToFile = path.relative(dataPath, file); - annotateErrorWithFile(caughtError, pathToFile); - push(caughtError); - } + const {thing, aggregate} = + processDocument(documents[0], dataStep.documentThing); + + return {aggregate, result: thing}; + } + + default: + throw new Error(`Unknown document mode ${documentMode.toString()}`); + } +} + +export function decorateErrorWithFileFromDataPath(fn, {dataPath}) { + return decorateErrorWithAnnotation(fn, + (caughtError, firstArg) => + annotateErrorWithFile( + caughtError, + path.relative( + dataPath, + (typeof firstArg === 'object' + ? firstArg.file + : firstArg)))); +} - yamlResults.push({file, documents: filteredDocuments}); +// Loads a list of files for each data step, and a list of documents +// for each file. +export async function loadYAMLDocumentsFromDataSteps(dataSteps, {dataPath}) { + const aggregate = + openAggregate({ + message: `Errors loading data files`, + translucent: true, + }); + + const fileLists = + await Promise.all( + dataSteps.map(dataStep => + getFilesFromDataStep(dataStep, {dataPath}))); + + const filePromises = + fileLists + .map(files => files + .map(file => + loadYAMLDocumentsFromFile(file).then( + ({result, aggregate}) => { + const close = + decorateErrorWithFileFromDataPath(aggregate.close, {dataPath}); + + aggregate.close = () => + close({file}); + + return {result, aggregate}; + }, + (error) => { + const aggregate = {}; + + annotateErrorWithFile(error, path.relative(dataPath, file)); + + aggregate.close = () => { + throw error; + }; + + return {result: [], aggregate}; + }))); + + const fileListPromises = + filePromises + .map(filePromises => Promise.all(filePromises)); + + const dataStepPromises = + stitchArrays({ + dataStep: dataSteps, + fileListPromise: fileListPromises, + }).map(async ({dataStep, fileListPromise}) => + openAggregate({ + message: `Errors loading data files for data step: ${colors.bright(dataStep.title)}`, + translucent: true, + }).contain(await fileListPromise)); + + const documentLists = + aggregate + .receive(await Promise.all(dataStepPromises)); + + return {aggregate, result: {documentLists, fileLists}}; +} + +// Loads a list of things from a list of documents for each file +// for each data step. Nesting! +export async function processThingsFromDataSteps(documentLists, fileLists, dataSteps, {dataPath}) { + const aggregate = + openAggregate({ + message: `Errors processing documents in data files`, + translucent: true, + }); + + const filePromises = + stitchArrays({ + dataStep: dataSteps, + files: fileLists, + documentLists: documentLists, + }).map(({dataStep, files, documentLists}) => + stitchArrays({ + file: files, + documents: documentLists, + }).map(({file, documents}) => { + const {result, aggregate} = + processThingsFromDataStep(documents, dataStep); + + const close = decorateErrorWithFileFromDataPath(aggregate.close, {dataPath}); + aggregate.close = () => close({file}); + + return {result, aggregate}; })); - const processResults = []; - - switch (documentMode) { - case documentModes.headerAndEntries: - map(yamlResults, {message: `Errors processing documents in data files`, translucent: true}, - decorateErrorWithFile(({documents}) => { - const headerDocument = documents[0]; - const entryDocuments = documents.slice(1).filter(Boolean); - - if (!headerDocument) - throw new Error(`Missing header document (empty file or erroneously starting with "---"?)`); - - withAggregate({message: `Errors processing documents`}, ({push}) => { - const {thing: headerObject, aggregate: headerAggregate} = - processDocument(headerDocument, dataStep.headerDocumentThing); - - try { - headerAggregate.close(); - } catch (caughtError) { - caughtError.message = `(${colors.yellow(`header`)}) ${caughtError.message}`; - push(caughtError); - } - - const entryObjects = []; - - for (let index = 0; index < entryDocuments.length; index++) { - const entryDocument = entryDocuments[index]; - - const {thing: entryObject, aggregate: entryAggregate} = - processDocument(entryDocument, dataStep.entryDocumentThing); - - entryObjects.push(entryObject); - - try { - entryAggregate.close(); - } catch (caughtError) { - caughtError.message = `(${colors.yellow(`entry #${index + 1}`)}) ${caughtError.message}`; - push(caughtError); - } - } - - processResults.push({ - header: headerObject, - entries: entryObjects, - }); - }); - })); - break; - - case documentModes.onePerFile: - map(yamlResults, {message: `Errors processing data files as valid documents`}, - decorateErrorWithFile(({documents}) => { - if (documents.length > 1) - throw new Error(`Only expected one document to be present per file, got ${documents.length} here`); - - if (empty(documents) || !documents[0]) - throw new Error(`Expected a document, this file is empty`); - - const {thing, aggregate} = - processDocument(documents[0], dataStep.documentThing); - - processResults.push(thing); - aggregate.close(); - })); - break; - } + const fileListPromises = + filePromises + .map(filePromises => Promise.all(filePromises)); + + const dataStepPromises = + stitchArrays({ + dataStep: dataSteps, + fileListPromise: fileListPromises, + }).map(async ({dataStep, fileListPromise}) => + openAggregate({ + message: `Errors loading data files for data step: ${colors.bright(dataStep.title)}`, + translucent: true, + }).contain(await fileListPromise)); + + const thingLists = + aggregate + .receive(await Promise.all(dataStepPromises)); + + return {aggregate, result: thingLists}; +} - const saveResult = call(dataStep.save, processResults); +// Flattens a list of *lists* of things for a given data step (each list +// corresponding to one YAML file) into results to be saved on the final +// wikiData object, routing thing lists into the step's save() function. +export function saveThingsFromDataStep(thingLists, dataStep) { + const {documentMode} = dataStep; - if (!saveResult) return; + switch (documentMode) { + case documentModes.allInOne: { + const things = + (empty(thingLists) + ? [] + : thingLists[0]); - Object.assign(wikiDataResult, saveResult); - } - ); + return dataStep.save(things); + } + + case documentModes.oneDocumentTotal: { + const thing = + (empty(thingLists) + ? {} + : thingLists[0]); + + return dataStep.save(thing); + } + + case documentModes.headerAndEntries: + case documentModes.onePerFile: { + return dataStep.save(thingLists); + } + + default: + throw new Error(`Invalid documentMode: ${documentMode.toString()}`); } +} - return { - aggregate: processDataAggregate, - result: wikiDataResult, - }; +// Flattens a list of *lists* of things for each data step (each list +// corresponding to one YAML file) into the final wikiData object, +// routing thing lists into each step's save() function. +export function saveThingsFromDataSteps(thingLists, dataSteps) { + const aggregate = + openAggregate({ + message: `Errors finalizing things from data files`, + translucent: true, + }); + + const wikiData = {}; + + stitchArrays({ + dataStep: dataSteps, + thingLists: thingLists, + }).map(({dataStep, thingLists}) => { + try { + return saveThingsFromDataStep(thingLists, dataStep); + } catch (caughtError) { + const error = new Error( + `Error finalizing things for data step: ${colors.bright(dataStep.title)}`, + {cause: caughtError}); + + error[Symbol.for('hsmusic.aggregate.translucent')] = true; + + aggregate.push(error); + + return null; + } + }) + .filter(Boolean) + .forEach(saveResult => { + Object.assign(wikiData, saveResult); + }); + + return {aggregate, result: wikiData}; +} + +export async function loadAndProcessDataDocuments(dataSteps, {dataPath}) { + const aggregate = + openAggregate({ + message: `Errors processing data files`, + }); + + const {documentLists, fileLists} = + aggregate.receive( + await loadYAMLDocumentsFromDataSteps(dataSteps, {dataPath})); + + const thingLists = + aggregate.receive( + await processThingsFromDataSteps(documentLists, fileLists, dataSteps, {dataPath})); + + const wikiData = + aggregate.receive( + saveThingsFromDataSteps(thingLists, dataSteps)); + + return {aggregate, result: wikiData}; } // Data linking! Basically, provide (portions of) wikiData to the Things which @@ -942,6 +1088,11 @@ export function linkWikiDataArrays(wikiData) { [wikiData.flashActData, [ 'flashData', + 'flashSideData', + ]], + + [wikiData.flashSideData, [ + 'flashActData', ]], [wikiData.groupData, [ @@ -983,15 +1134,13 @@ export function linkWikiDataArrays(wikiData) { } } -export function sortWikiDataArrays(wikiData) { +export function sortWikiDataArrays(dataSteps, wikiData) { for (const [key, value] of Object.entries(wikiData)) { if (!Array.isArray(value)) continue; wikiData[key] = value.slice(); } - const steps = getDataSteps(); - - for (const step of steps) { + for (const step of dataSteps) { if (!step.sort) continue; step.sort(wikiData); } @@ -1018,10 +1167,12 @@ export async function quickLoadAllFromYAML(dataPath, { }) { const showAggregate = customShowAggregate; + const dataSteps = getAllDataSteps(); + let wikiData; { - const {aggregate, result} = await loadAndProcessDataDocuments({dataPath}); + const {aggregate, result} = await loadAndProcessDataDocuments(dataSteps, {dataPath}); wikiData = result; @@ -1060,7 +1211,7 @@ export async function quickLoadAllFromYAML(dataPath, { logWarn`Content text errors found.`; } - sortWikiDataArrays(wikiData); + sortWikiDataArrays(dataSteps, wikiData); return wikiData; } |