diff options
Diffstat (limited to 'src/data')
33 files changed, 1702 insertions, 756 deletions
diff --git a/src/data/cacheable-object.js b/src/data/cacheable-object.js index 1e7c7aa8..71dc5bde 100644 --- a/src/data/cacheable-object.js +++ b/src/data/cacheable-object.js @@ -83,6 +83,8 @@ function inspect(value) { } export default class CacheableObject { + static propertyDescriptors = Symbol.for('CacheableObject.propertyDescriptors'); + #propertyUpdateValues = Object.create(null); #propertyUpdateCacheInvalidators = Object.create(null); @@ -113,28 +115,41 @@ export default class CacheableObject { } } + #withEachPropertyDescriptor(callback) { + const {[CacheableObject.propertyDescriptors]: propertyDescriptors} = + this.constructor; + + for (const property of Reflect.ownKeys(propertyDescriptors)) { + callback(property, propertyDescriptors[property]); + } + } + #initializeUpdatingPropertyValues() { - for (const [property, descriptor] of Object.entries(this.constructor.propertyDescriptors)) { + this.#withEachPropertyDescriptor((property, descriptor) => { const {flags, update} = descriptor; if (!flags.update) { - continue; + return; } - if (update?.default) { + if ( + typeof update === 'object' && + update !== null && + 'default' in update + ) { this[property] = update?.default; } else { this[property] = null; } - } + }); } #defineProperties() { - if (!this.constructor.propertyDescriptors) { - throw new Error(`Expected constructor ${this.constructor.name} to define propertyDescriptors`); + if (!this.constructor[CacheableObject.propertyDescriptors]) { + throw new Error(`Expected constructor ${this.constructor.name} to provide CacheableObject.propertyDescriptors`); } - for (const [property, descriptor] of Object.entries(this.constructor.propertyDescriptors)) { + this.#withEachPropertyDescriptor((property, descriptor) => { const {flags} = descriptor; const definition = { @@ -151,7 +166,7 @@ export default class CacheableObject { } Object.defineProperty(this, property, definition); - } + }); Object.seal(this); } @@ -191,7 +206,7 @@ export default class CacheableObject { } #getPropertyDescriptor(property) { - return this.constructor.propertyDescriptors[property]; + return this.constructor[CacheableObject.propertyDescriptors][property]; } #invalidateCachesDependentUpon(property) { @@ -244,7 +259,8 @@ export default class CacheableObject { if (expose.dependencies?.length > 0) { const dependencyKeys = expose.dependencies.slice(); - const shouldReflect = dependencyKeys.includes('this'); + const shouldReflectObject = dependencyKeys.includes('this'); + const shouldReflectProperty = dependencyKeys.includes('thisProperty'); getAllDependencies = () => { const dependencies = Object.create(null); @@ -253,10 +269,14 @@ export default class CacheableObject { dependencies[key] = this.#propertyUpdateValues[key]; } - if (shouldReflect) { + if (shouldReflectObject) { dependencies.this = this; } + if (shouldReflectProperty) { + dependencies.thisProperty = property; + } + return dependencies; }; } else { @@ -311,16 +331,16 @@ export default class CacheableObject { return; } - const {propertyDescriptors} = obj.constructor; + const {[CacheableObject.propertyDescriptors]: propertyDescriptors} = + obj.constructor; if (!propertyDescriptors) { console.warn('Missing property descriptors:', obj); return; } - for (const [property, descriptor] of Object.entries(propertyDescriptors)) { - const {flags} = descriptor; - + for (const property of Reflect.ownKeys(propertyDescriptors)) { + const {flags} = propertyDescriptors[property]; if (!flags.expose) { continue; } diff --git a/src/data/checks.js b/src/data/checks.js index 44f3efd7..ad621bab 100644 --- a/src/data/checks.js +++ b/src/data/checks.js @@ -24,20 +24,30 @@ function inspect(value, opts = {}) { return nodeInspect(value, {colors: ENABLE_COLOR, ...opts}); } -// Warn about directories which are reused across more than one of the same type -// of Thing. Directories are the unique identifier for most data objects across -// the wiki, so we have to make sure they aren't duplicated! -export function reportDuplicateDirectories(wikiData, { +// Warn about problems to do with directories. +// +// * Duplicate directories: these are the unique identifier for referencable +// data objects across the wiki, so duplicates introduce ambiguity where it +// can't fit. +// +// * Missing directories: in almost all cases directories can be computed, +// but in particularly brutal internal cases, it might not be possible, and +// a thing's directory is just null. This leaves it unable to be referenced. +// +export function reportDirectoryErrors(wikiData, { getAllFindSpecs, }) { const duplicateSets = []; + const missingDirectoryThings = new Set(); for (const findSpec of Object.values(getAllFindSpecs())) { if (!findSpec.bindTo) continue; const directoryPlaces = Object.create(null); const duplicateDirectories = new Set(); + const thingData = wikiData[findSpec.bindTo]; + if (!thingData) continue; for (const thing of thingData) { if (findSpec.include && !findSpec.include(thing)) { @@ -50,6 +60,11 @@ export function reportDuplicateDirectories(wikiData, { : [thing.directory]); for (const directory of directories) { + if (directory === null || directory === undefined) { + missingDirectoryThings.add(thing); + continue; + } + if (directory in directoryPlaces) { directoryPlaces[directory].push(thing); duplicateDirectories.add(directory); @@ -59,8 +74,6 @@ export function reportDuplicateDirectories(wikiData, { } } - if (empty(duplicateDirectories)) continue; - const sortedDuplicateDirectories = Array.from(duplicateDirectories) .sort((a, b) => { @@ -75,8 +88,6 @@ export function reportDuplicateDirectories(wikiData, { } } - if (empty(duplicateSets)) return; - // Multiple find functions may effectively have duplicates across the same // things. These only need to be reported once, because resolving one of them // will resolve the rest, so cut out duplicate sets before reporting. @@ -84,6 +95,7 @@ export function reportDuplicateDirectories(wikiData, { const seenDuplicateSets = new Map(); const deduplicateDuplicateSets = []; + iterateSets: for (const set of duplicateSets) { if (seenDuplicateSets.has(set.directory)) { const placeLists = seenDuplicateSets.get(set.directory); @@ -95,7 +107,7 @@ export function reportDuplicateDirectories(wikiData, { // Two artists named Foodog aren't going to match two tracks named // Foodog. if (compareArrays(places, set.places, {checkOrder: false})) { - continue; + continue iterateSets; } } @@ -107,12 +119,20 @@ export function reportDuplicateDirectories(wikiData, { deduplicateDuplicateSets.push(set); } - withAggregate({message: `Duplicate directories found`}, ({push}) => { + withAggregate({message: `Directory errors detected`}, ({push}) => { for (const {directory, places} of deduplicateDuplicateSets) { push(new Error( `Duplicate directory ${colors.green(`"${directory}"`)}:\n` + places.map(thing => ` - ` + inspect(thing)).join('\n'))); } + + if (!empty(missingDirectoryThings)) { + push(new Error( + `Couldn't figure out an implicit directory for:\n` + + Array.from(missingDirectoryThings) + .map(thing => `- ` + inspect(thing)) + .join('\n'))); + } }); } @@ -261,7 +281,7 @@ export function filterReferenceErrors(wikiData, { break; case '_contrib': - findFn = contribRef => findArtistOrAlias(contribRef.who); + findFn = contribRef => findArtistOrAlias(contribRef.artist); break; case '_homepageSourceGroup': diff --git a/src/data/composite.js b/src/data/composite.js index 7a98c424..ea7a3480 100644 --- a/src/data/composite.js +++ b/src/data/composite.js @@ -29,6 +29,7 @@ input.value = _valueIntoToken('input.value'); input.dependency = _valueIntoToken('input.dependency'); input.myself = () => Symbol.for(`hsmusic.composite.input.myself`); +input.thisProperty = () => Symbol.for('hsmusic.composite.input.thisProperty'); input.updateValue = _valueIntoToken('input.updateValue'); @@ -284,6 +285,7 @@ export function templateCompositeFrom(description) { 'input.value', 'input.dependency', 'input.myself', + 'input.thisProperty', 'input.updateValue', ].includes(tokenShape)) { expectedValueProvidingTokenInputNames.push(name); @@ -567,6 +569,8 @@ export function compositeFrom(description) { return token; case 'input.myself': return 'this'; + case 'input.thisProperty': + return 'thisProperty'; default: return null; } @@ -721,6 +725,8 @@ export function compositeFrom(description) { return (tokenValue.startsWith('#') ? null : tokenValue); case 'input.myself': return 'this'; + case 'input.thisProperty': + return 'thisProperty'; default: return null; } @@ -752,6 +758,9 @@ export function compositeFrom(description) { anyStepsUseUpdateValue || anyStepsUpdate; + const stepsFirstTimeCalling = + Array.from({length: steps.length}).fill(true); + const stepEntries = stitchArrays({ step: steps, stepComposes: stepsCompose, @@ -774,16 +783,9 @@ export function compositeFrom(description) { (step.annotation ? ` (${step.annotation})` : ``); aggregate.nest({message}, ({push}) => { - if (isBase && stepComposes !== compositionNests) { - return push(new TypeError( - (compositionNests - ? `Base must compose, this composition is nestable` - : `Base must not compose, this composition isn't nestable`))); - } else if (!isBase && !stepComposes) { + if (!isBase && !stepComposes) { return push(new TypeError( - (compositionNests - ? `All steps must compose` - : `All steps (except base) must compose`))); + `All steps leading up to base must compose`)); } if ( @@ -877,6 +879,8 @@ export function compositeFrom(description) { return valueSoFar; case 'input.myself': return initialDependencies['this']; + case 'input.thisProperty': + return initialDependencies['thisProperty']; case 'input': return initialDependencies[token]; default: @@ -907,8 +911,16 @@ export function compositeFrom(description) { debug(() => colors.bright(`begin composition - not transforming`)); } - for (let i = 0; i < steps.length; i++) { - const step = steps[i]; + for ( + const [i, { + step, + stepComposes, + }] of + stitchArrays({ + step: steps, + stepComposes: stepsCompose, + }).entries() + ) { const isBase = i === steps.length - 1; debug(() => [ @@ -968,7 +980,16 @@ export function compositeFrom(description) { (expectingTransform ? {[input.updateValue()]: valueSoFar} : {}), - [input.myself()]: initialDependencies?.['this'] ?? null, + + [input.myself()]: + (initialDependencies && Object.hasOwn(initialDependencies, 'this') + ? initialDependencies.this + : null), + + [input.thisProperty()]: + (initialDependencies && Object.hasOwn(initialDependencies, 'thisProperty') + ? initialDependencies.thisProperty + : null), }; const selectDependencies = @@ -983,6 +1004,8 @@ export function compositeFrom(description) { return dependency; case 'input.myself': return input.myself(); + case 'input.thisProperty': + return input.thisProperty(); case 'input.dependency': return tokenValue; case 'input.updateValue': @@ -1016,26 +1039,175 @@ export function compositeFrom(description) { const naturalEvaluate = () => { const [name, ...argsLayout] = getExpectedEvaluation(); - let args; + let args = argsLayout; - if (isBase && !compositionNests) { - args = - argsLayout.filter(arg => arg !== continuationSymbol); + let effectiveDependencies; + let reviewAccessedDependencies; + + if (stepsFirstTimeCalling[i]) { + const expressedDependencies = + selectDependencies; + + const remainingDependencies = + new Set(expressedDependencies); + + const unavailableDependencies = []; + const accessedDependencies = []; + + effectiveDependencies = + new Proxy(filteredDependencies, { + get(target, key) { + accessedDependencies.push(key); + remainingDependencies.delete(key); + + const value = target[key]; + + if (value === undefined) { + unavailableDependencies.push(key); + } + + return value; + }, + }); + + reviewAccessedDependencies = () => { + const topAggregate = + openAggregate({ + message: `Errors in accessed dependencies`, + }); + + const showDependency = dependency => + (isInputToken(dependency) + ? getInputTokenShape(dependency) + + `(` + + inspect(getInputTokenValue(dependency), {compact: true}) + + ')' + : dependency.toString()); + + let anyErrors = false; + + for (const dependency of remainingDependencies) { + topAggregate.push(new Error( + `Expected to access ${showDependency(dependency)}`)); + + anyErrors = true; + } + + for (const dependency of unavailableDependencies) { + const subAggregate = + openAggregate({ + message: + `Accessed ${showDependency(dependency)}, which is unavailable`, + }); + + let reason = false; + + if (!expressedDependencies.includes(dependency)) { + subAggregate.push(new Error( + `Missing from step's expressed dependencies`)); + reason = true; + } + + if (filterableDependencies[dependency] === undefined) { + subAggregate.push( + new Error( + `Not available` + + (isInputToken(dependency) + ? ` in input()-type dependencies` + : dependency.startsWith('#') + ? ` in local dependencies` + : ` on object dependencies`))); + reason = true; + } + + if (!reason) { + subAggregate.push(new Error( + `Not sure why this is unavailable, sorry!`)); + } + + topAggregate.call(subAggregate.close); + + anyErrors = true; + } + + if (anyErrors) { + topAggregate.push(new Error( + `These dependencies, in total, were accessed:` + + (empty(accessedDependencies) + ? ` (none)` + : accessedDependencies.length === 1 + ? showDependency(accessedDependencies[0]) + : `\n` + + accessedDependencies + .map(showDependency) + .map(line => ` - ${line}`) + .join('\n')))); + } + + topAggregate.close(); + }; } else { + effectiveDependencies = filteredDependencies; + reviewAccessedDependencies = null; + } + + args = + args.map(arg => + (arg === filteredDependencies + ? effectiveDependencies + : arg)); + + if (stepComposes) { let continuation; ({continuation, continuationStorage} = _prepareContinuation(callingTransformForThisStep)); args = - argsLayout.map(arg => + args.map(arg => (arg === continuationSymbol ? continuation : arg)); + } else { + args = + args.filter(arg => arg !== continuationSymbol); } - return expose[name](...args); - } + let stepError; + try { + return expose[name](...args); + } catch (error) { + stepError = error; + } finally { + stepsFirstTimeCalling[i] = false; + + let reviewError; + if (reviewAccessedDependencies) { + try { + reviewAccessedDependencies(); + } catch (error) { + reviewError = error; + } + } + + const stepPart = + `step ${i+1}` + + (isBase + ? ` (base)` + : ` of ${steps.length}`) + + (step.annotation ? `, ${step.annotation}` : ``); + + if (stepError && reviewError) { + throw new AggregateError( + [stepError, reviewError], + `Errors in ${stepPart}`); + } else if (stepError || reviewError) { + throw new Error( + `Error in ${stepPart}`, + {cause: stepError || reviewError}); + } + } + }; switch (step.cache) { // Warning! Highly WIP! @@ -1091,11 +1263,6 @@ export function compositeFrom(description) { if (result !== continuationSymbol) { debug(() => [`step #${i+1} - result: exit (inferred) ->`, result]); - - if (compositionNests) { - throw new TypeError(`Inferred early-exit is disallowed in nested compositions`); - } - debug(() => colors.bright(`end composition - exit (inferred)`)); return result; @@ -1216,6 +1383,7 @@ export function compositeFrom(description) { `Error computing composition` + (annotation ? ` ${annotation}` : '')); error.cause = thrownError; + error[Symbol.for('hsmusic.aggregate.translucent')] = true; throw error; } }; diff --git a/src/data/composite/things/album/index.js b/src/data/composite/things/album/index.js index 8139f10e..0ef91b87 100644 --- a/src/data/composite/things/album/index.js +++ b/src/data/composite/things/album/index.js @@ -1,2 +1,2 @@ -export {default as withTracks} from './withTracks.js'; export {default as withTrackSections} from './withTrackSections.js'; +export {default as withTracks} from './withTracks.js'; diff --git a/src/data/composite/things/album/withTrackSections.js b/src/data/composite/things/album/withTrackSections.js index 0a1ebebc..a56bda31 100644 --- a/src/data/composite/things/album/withTrackSections.js +++ b/src/data/composite/things/album/withTrackSections.js @@ -1,127 +1,21 @@ import {input, templateCompositeFrom} from '#composite'; + import find from '#find'; -import {empty, filterMultipleArrays, stitchArrays} from '#sugar'; -import {isTrackSectionList} from '#validators'; -import {exitWithoutDependency, exitWithoutUpdateValue} - from '#composite/control-flow'; import {withResolvedReferenceList} from '#composite/wiki-data'; -import { - fillMissingListItems, - withFlattenedList, - withPropertiesFromList, - withUnflattenedList, -} from '#composite/data'; - export default templateCompositeFrom({ annotation: `withTrackSections`, outputs: ['#trackSections'], steps: () => [ - exitWithoutDependency({ - dependency: 'ownTrackData', - value: input.value([]), - }), - - exitWithoutUpdateValue({ - mode: input.value('empty'), - value: input.value([]), - }), - - // TODO: input.updateValue description down here is a kludge. - withPropertiesFromList({ - list: input.updateValue({ - validate: isTrackSectionList, - }), - prefix: input.value('#sections'), - properties: input.value([ - 'tracks', - 'dateOriginallyReleased', - 'isDefaultTrackSection', - 'name', - 'color', - ]), - }), - - fillMissingListItems({ - list: '#sections.tracks', - fill: input.value([]), - }), - - fillMissingListItems({ - list: '#sections.isDefaultTrackSection', - fill: input.value(false), - }), - - fillMissingListItems({ - list: '#sections.name', - fill: input.value('Unnamed Track Section'), - }), - - fillMissingListItems({ - list: '#sections.color', - fill: input.dependency('color'), - }), - - withFlattenedList({ - list: '#sections.tracks', - }).outputs({ - ['#flattenedList']: '#trackRefs', - ['#flattenedIndices']: '#sections.startIndex', - }), - withResolvedReferenceList({ - list: '#trackRefs', - data: 'ownTrackData', - notFoundMode: input.value('null'), - find: input.value(find.track), + list: 'trackSections', + data: 'ownTrackSectionData', + find: input.value(find.unqualifiedTrackSection), }).outputs({ - ['#resolvedReferenceList']: '#tracks', + ['#resolvedReferenceList']: '#trackSections', }), - - withUnflattenedList({ - list: '#tracks', - indices: '#sections.startIndex', - }).outputs({ - ['#unflattenedList']: '#sections.tracks', - }), - - { - dependencies: [ - '#sections.tracks', - '#sections.name', - '#sections.color', - '#sections.dateOriginallyReleased', - '#sections.isDefaultTrackSection', - '#sections.startIndex', - ], - - compute: (continuation, { - '#sections.tracks': tracks, - '#sections.name': name, - '#sections.color': color, - '#sections.dateOriginallyReleased': dateOriginallyReleased, - '#sections.isDefaultTrackSection': isDefaultTrackSection, - '#sections.startIndex': startIndex, - }) => { - filterMultipleArrays( - tracks, name, color, dateOriginallyReleased, isDefaultTrackSection, startIndex, - tracks => !empty(tracks)); - - return continuation({ - ['#trackSections']: - stitchArrays({ - tracks, - name, - color, - dateOriginallyReleased, - isDefaultTrackSection, - startIndex, - }), - }); - }, - }, ], }); diff --git a/src/data/composite/things/album/withTracks.js b/src/data/composite/things/album/withTracks.js index fff3d5ae..c8d27c4c 100644 --- a/src/data/composite/things/album/withTracks.js +++ b/src/data/composite/things/album/withTracks.js @@ -1,51 +1,27 @@ import {input, templateCompositeFrom} from '#composite'; -import find from '#find'; -import {exitWithoutDependency, raiseOutputWithoutDependency} - from '#composite/control-flow'; +import {withFlattenedList, withPropertyFromList} from '#composite/data'; import {withResolvedReferenceList} from '#composite/wiki-data'; +import withTrackSections from './withTrackSections.js'; + export default templateCompositeFrom({ annotation: `withTracks`, outputs: ['#tracks'], steps: () => [ - exitWithoutDependency({ - dependency: 'ownTrackData', - value: input.value([]), - }), + withTrackSections(), - raiseOutputWithoutDependency({ - dependency: 'trackSections', - mode: input.value('empty'), - output: input.value({ - ['#tracks']: [], - }), + withPropertyFromList({ + list: '#trackSections', + property: input.value('tracks'), }), - { - dependencies: ['trackSections'], - compute: (continuation, {trackSections}) => - continuation({ - '#trackRefs': trackSections - .flatMap(section => section.tracks ?? []), - }), - }, - - withResolvedReferenceList({ - list: '#trackRefs', - data: 'ownTrackData', - find: input.value(find.track), + withFlattenedList({ + list: '#trackSections.tracks', + }).outputs({ + ['#flattenedList']: '#tracks', }), - - { - dependencies: ['#resolvedReferenceList'], - compute: (continuation, { - ['#resolvedReferenceList']: resolvedReferenceList, - }) => continuation({ - ['#tracks']: resolvedReferenceList, - }) - }, ], }); diff --git a/src/data/composite/things/track-section/index.js b/src/data/composite/things/track-section/index.js new file mode 100644 index 00000000..3202ed49 --- /dev/null +++ b/src/data/composite/things/track-section/index.js @@ -0,0 +1 @@ +export {default as withAlbum} from './withAlbum.js'; diff --git a/src/data/composite/things/track-section/withAlbum.js b/src/data/composite/things/track-section/withAlbum.js new file mode 100644 index 00000000..608cc0cd --- /dev/null +++ b/src/data/composite/things/track-section/withAlbum.js @@ -0,0 +1,22 @@ +// Gets the track section's album. This will early exit if ownAlbumData is +// missing. If there's no album whose list of track sections includes this one, +// the output dependency will be null. + +import {input, templateCompositeFrom} from '#composite'; + +import {withUniqueReferencingThing} from '#composite/wiki-data'; + +export default templateCompositeFrom({ + annotation: `withAlbum`, + + outputs: ['#album'], + + steps: () => [ + withUniqueReferencingThing({ + data: 'ownAlbumData', + list: input.value('trackSections'), + }).outputs({ + ['#uniqueReferencingThing']: '#album', + }), + ], +}); diff --git a/src/data/composite/things/track/index.js b/src/data/composite/things/track/index.js index cc723a24..8959de9f 100644 --- a/src/data/composite/things/track/index.js +++ b/src/data/composite/things/track/index.js @@ -9,3 +9,4 @@ export {default as withContainingTrackSection} from './withContainingTrackSectio export {default as withHasUniqueCoverArt} from './withHasUniqueCoverArt.js'; export {default as withOtherReleases} from './withOtherReleases.js'; export {default as withPropertyFromAlbum} from './withPropertyFromAlbum.js'; +export {default as withPropertyFromOriginalRelease} from './withPropertyFromOriginalRelease.js'; diff --git a/src/data/composite/things/track/inheritFromOriginalRelease.js b/src/data/composite/things/track/inheritFromOriginalRelease.js index 27ed1387..38ab06be 100644 --- a/src/data/composite/things/track/inheritFromOriginalRelease.js +++ b/src/data/composite/things/track/inheritFromOriginalRelease.js @@ -1,8 +1,6 @@ -// Early exits with a value inherited from the original release, if -// this track is a rerelease, and otherwise continues with no further -// dependencies provided. If allowOverride is true, then the continuation -// will also be called if the original release exposed the requested -// property as null. +// Early exits with the value for the same property as specified on the +// original release, if this track is a rerelease, and otherwise continues +// without providing any further dependencies. // // Like withOriginalRelease, this will early exit (with notFoundValue) if the // original release is specified by reference and that reference doesn't @@ -10,41 +8,34 @@ import {input, templateCompositeFrom} from '#composite'; -import withOriginalRelease from './withOriginalRelease.js'; +import {exposeDependency, raiseOutputWithoutDependency} + from '#composite/control-flow'; + +import withPropertyFromOriginalRelease + from './withPropertyFromOriginalRelease.js'; export default templateCompositeFrom({ annotation: `inheritFromOriginalRelease`, inputs: { - property: input({type: 'string'}), - allowOverride: input({type: 'boolean', defaultValue: false}), - notFoundValue: input({defaultValue: null}), + notFoundValue: input({ + defaultValue: null, + }), }, steps: () => [ - withOriginalRelease({ + withPropertyFromOriginalRelease({ + property: input.thisProperty(), notFoundValue: input('notFoundValue'), }), - { - dependencies: [ - '#originalRelease', - input('property'), - input('allowOverride'), - ], - - compute: (continuation, { - ['#originalRelease']: originalRelease, - [input('property')]: originalProperty, - [input('allowOverride')]: allowOverride, - }) => { - if (!originalRelease) return continuation(); - - const value = originalRelease[originalProperty]; - if (allowOverride && value === null) return continuation(); - - return continuation.exit(value); - }, - }, + raiseOutputWithoutDependency({ + dependency: '#isRerelease', + mode: input.value('falsy'), + }), + + exposeDependency({ + dependency: '#originalValue', + }), ], }); diff --git a/src/data/composite/things/track/withAlwaysReferenceByDirectory.js b/src/data/composite/things/track/withAlwaysReferenceByDirectory.js index fac8e213..e01720b4 100644 --- a/src/data/composite/things/track/withAlwaysReferenceByDirectory.js +++ b/src/data/composite/things/track/withAlwaysReferenceByDirectory.js @@ -7,11 +7,15 @@ import {input, templateCompositeFrom} from '#composite'; import find from '#find'; import {isBoolean} from '#validators'; -import {exitWithoutDependency, exposeUpdateValueOrContinue} - from '#composite/control-flow'; import {withPropertyFromObject} from '#composite/data'; import {withResolvedReference} from '#composite/wiki-data'; +import { + exitWithoutDependency, + exposeDependencyOrContinue, + exposeUpdateValueOrContinue, +} from '#composite/control-flow'; + export default templateCompositeFrom({ annotation: `withAlwaysReferenceByDirectory`, @@ -22,6 +26,29 @@ export default templateCompositeFrom({ validate: input.value(isBoolean), }), + // withAlwaysReferenceByDirectory is sort of a fragile area - we can't + // find the track's album the normal way because albums' track lists + // recurse back into alwaysReferenceByDirectory! + withResolvedReference({ + ref: 'dataSourceAlbum', + data: 'albumData', + find: input.value(find.album), + }).outputs({ + '#resolvedReference': '#album', + }), + + withPropertyFromObject({ + object: '#album', + property: input.value('alwaysReferenceTracksByDirectory'), + }), + + // Falsy mode means this exposes true if the album's property is true, + // but continues if the property is false (which is also the default). + exposeDependencyOrContinue({ + dependency: '#album.alwaysReferenceTracksByDirectory', + mode: input.value('falsy'), + }), + // Remaining code is for defaulting to true if this track is a rerelease of // another with the same name, so everything further depends on access to // trackData as well as originalReleaseTrack. diff --git a/src/data/composite/things/track/withContainingTrackSection.js b/src/data/composite/things/track/withContainingTrackSection.js index eaac14de..2c42709b 100644 --- a/src/data/composite/things/track/withContainingTrackSection.js +++ b/src/data/composite/things/track/withContainingTrackSection.js @@ -30,7 +30,6 @@ export default templateCompositeFrom({ compute: (continuation, { [input.myself()]: track, - [input('notFoundMode')]: notFoundMode, ['#album.trackSections']: trackSections, }) => continuation({ ['#trackSection']: diff --git a/src/data/composite/things/track/withPropertyFromOriginalRelease.js b/src/data/composite/things/track/withPropertyFromOriginalRelease.js new file mode 100644 index 00000000..fd37f6de --- /dev/null +++ b/src/data/composite/things/track/withPropertyFromOriginalRelease.js @@ -0,0 +1,86 @@ +// Provides a value inherited from the original release, if applicable, and a +// flag indicating if this track is a rerelase or not. +// +// Like withOriginalRelease, this will early exit (with notFoundValue) if the +// original release is specified by reference and that reference doesn't +// resolve to anything. + +import {input, templateCompositeFrom} from '#composite'; + +import {withResultOfAvailabilityCheck} from '#composite/control-flow'; +import {withPropertyFromObject} from '#composite/data'; + +import withOriginalRelease from './withOriginalRelease.js'; + +export default templateCompositeFrom({ + annotation: `inheritFromOriginalRelease`, + + inputs: { + property: input({type: 'string'}), + + notFoundValue: input({ + defaultValue: null, + }), + }, + + outputs: ({ + [input.staticValue('property')]: property, + }) => + ['#isRerelease'].concat( + (property + ? ['#original.' + property] + : ['#originalValue'])), + + steps: () => [ + withOriginalRelease({ + notFoundValue: input('notFoundValue'), + }), + + withResultOfAvailabilityCheck({ + from: '#originalRelease', + }), + + { + dependencies: [ + '#availability', + input.staticValue('property'), + ], + + compute: (continuation, { + ['#availability']: availability, + [input.staticValue('property')]: property, + }) => + (availability + ? continuation() + : continuation.raiseOutput( + Object.assign( + {'#isRerelease': false}, + (property + ? {['#original.' + property]: null} + : {'#originalValue': null})))), + }, + + withPropertyFromObject({ + object: '#originalRelease', + property: input('property'), + }), + + { + dependencies: [ + '#value', + input.staticValue('property'), + ], + + compute: (continuation, { + ['#value']: value, + [input.staticValue('property')]: property, + }) => + continuation.raiseOutput( + Object.assign( + {'#isRerelease': true}, + (property + ? {['#original.' + property]: value} + : {'#originalValue': value}))), + }, + ], +}); diff --git a/src/data/composite/wiki-data/index.js b/src/data/composite/wiki-data/index.js index b4cf6d13..15ebaffa 100644 --- a/src/data/composite/wiki-data/index.js +++ b/src/data/composite/wiki-data/index.js @@ -6,6 +6,8 @@ export {default as exitWithoutContribs} from './exitWithoutContribs.js'; export {default as inputWikiData} from './inputWikiData.js'; +export {default as withDirectory} from './withDirectory.js'; +export {default as withDirectoryFromName} from './withDirectoryFromName.js'; export {default as withParsedCommentaryEntries} from './withParsedCommentaryEntries.js'; export {default as withResolvedContribs} from './withResolvedContribs.js'; export {default as withResolvedReference} from './withResolvedReference.js'; diff --git a/src/data/composite/wiki-data/withDirectory.js b/src/data/composite/wiki-data/withDirectory.js new file mode 100644 index 00000000..b08b6153 --- /dev/null +++ b/src/data/composite/wiki-data/withDirectory.js @@ -0,0 +1,55 @@ +// Select a directory, either using a manually specified directory, or +// computing it from a name. By default these values are the current thing's +// 'directory' and 'name' properties, so it can be used without any options +// to get the current thing's effective directory (assuming no custom rules). + +import {input, templateCompositeFrom} from '#composite'; + +import {isDirectory, isName} from '#validators'; + +import {withResultOfAvailabilityCheck} from '#composite/control-flow'; + +import withDirectoryFromName from './withDirectoryFromName.js'; + +export default templateCompositeFrom({ + annotation: `withDirectory`, + + inputs: { + directory: input({ + validate: isDirectory, + defaultDependency: 'directory', + acceptsNull: true, + }), + + name: input({ + validate: isName, + defaultDependency: 'name', + acceptsNull: true, + }), + }, + + outputs: ['#directory'], + + steps: () => [ + withResultOfAvailabilityCheck({ + from: input('directory'), + }), + + { + dependencies: ['#availability', input('directory')], + compute: (continuation, { + ['#availability']: availability, + [input('directory')]: directory, + }) => + (availability + ? continuation.raiseOutput({ + ['#directory']: directory + }) + : continuation()), + }, + + withDirectoryFromName({ + name: input('name'), + }), + ], +}); diff --git a/src/data/composite/wiki-data/withDirectoryFromName.js b/src/data/composite/wiki-data/withDirectoryFromName.js new file mode 100644 index 00000000..034464e4 --- /dev/null +++ b/src/data/composite/wiki-data/withDirectoryFromName.js @@ -0,0 +1,42 @@ +// Compute a directory from a name - by default the current thing's own name. + +import {input, templateCompositeFrom} from '#composite'; + +import {isName} from '#validators'; +import {getKebabCase} from '#wiki-data'; + +import {raiseOutputWithoutDependency} from '#composite/control-flow'; + +export default templateCompositeFrom({ + annotation: `withDirectoryFromName`, + + inputs: { + name: input({ + validate: isName, + defaultDependency: 'name', + acceptsNull: true, + }), + }, + + outputs: ['#directory'], + + steps: () => [ + raiseOutputWithoutDependency({ + dependency: input('name'), + mode: input.value('falsy'), + output: input.value({ + ['#directory']: null, + }), + }), + + { + dependencies: [input('name')], + compute: (continuation, { + [input('name')]: name, + }) => continuation({ + ['#directory']: + getKebabCase(name), + }), + }, + ], +}); diff --git a/src/data/composite/wiki-data/withResolvedContribs.js b/src/data/composite/wiki-data/withResolvedContribs.js index 77b0f96d..95266382 100644 --- a/src/data/composite/wiki-data/withResolvedContribs.js +++ b/src/data/composite/wiki-data/withResolvedContribs.js @@ -1,7 +1,8 @@ // Resolves the contribsByRef contained in the provided dependency, // providing (named by the second argument) the result. "Resolving" -// means mapping the "who" reference of each contribution to an artist -// object, and filtering out those whose "who" doesn't match any artist. +// means mapping the artist reference of each contribution to an artist +// object, and filtering out those whose artist reference doesn't match +// any artist. import {input, templateCompositeFrom} from '#composite'; import find from '#find'; @@ -46,29 +47,30 @@ export default templateCompositeFrom({ withPropertiesFromList({ list: input('from'), - properties: input.value(['who', 'what']), + properties: input.value(['artist', 'annotation']), prefix: input.value('#contribs'), }), withResolvedReferenceList({ - list: '#contribs.who', + list: '#contribs.artist', data: 'artistData', find: input.value(find.artist), notFoundMode: input('notFoundMode'), }).outputs({ - ['#resolvedReferenceList']: '#contribs.who', + ['#resolvedReferenceList']: '#contribs.artist', }), { - dependencies: ['#contribs.who', '#contribs.what'], + dependencies: ['#contribs.artist', '#contribs.annotation'], compute(continuation, { - ['#contribs.who']: who, - ['#contribs.what']: what, + ['#contribs.artist']: artist, + ['#contribs.annotation']: annotation, }) { - filterMultipleArrays(who, what, (who, _what) => who); + filterMultipleArrays(artist, annotation, (artist, _annotation) => artist); return continuation({ - ['#resolvedContribs']: stitchArrays({who, what}), + ['#resolvedContribs']: + stitchArrays({artist, annotation}), }); }, }, diff --git a/src/data/composite/wiki-data/withReverseContributionList.js b/src/data/composite/wiki-data/withReverseContributionList.js index eccb58b7..91e125e4 100644 --- a/src/data/composite/wiki-data/withReverseContributionList.js +++ b/src/data/composite/wiki-data/withReverseContributionList.js @@ -11,7 +11,8 @@ import {input, templateCompositeFrom} from '#composite'; -import {exitWithoutDependency} from '#composite/control-flow'; +import {exitWithoutDependency, raiseOutputWithoutDependency} + from '#composite/control-flow'; import inputWikiData from './inputWikiData.js'; @@ -32,10 +33,17 @@ export default templateCompositeFrom({ outputs: ['#reverseContributionList'], steps: () => [ + // Early exit with an empty array if the data list isn't available. exitWithoutDependency({ dependency: input('data'), value: input.value([]), + }), + + // Raise an empty array (don't early exit) if the data list is empty. + raiseOutputWithoutDependency({ + dependency: input('data'), mode: input.value('empty'), + output: input.value({'#reverseContributionList': []}), }), { @@ -58,10 +66,10 @@ export default templateCompositeFrom({ for (const referencingThing of data) { const referenceList = referencingThing[list]; - // Destructuring {who} is the only unique part of the + // Destructuring {artist} is the only unique part of the // withReverseContributionList implementation, compared to // withReverseReferneceList. - for (const {who: referencedThing} of referenceList) { + for (const {artist: referencedThing} of referenceList) { if (cacheRecord.has(referencedThing)) { cacheRecord.get(referencedThing).push(referencingThing); } else { diff --git a/src/data/composite/wiki-data/withReverseReferenceList.js b/src/data/composite/wiki-data/withReverseReferenceList.js index 2d7a421b..8cd540a5 100644 --- a/src/data/composite/wiki-data/withReverseReferenceList.js +++ b/src/data/composite/wiki-data/withReverseReferenceList.js @@ -13,7 +13,8 @@ import {input, templateCompositeFrom} from '#composite'; -import {exitWithoutDependency} from '#composite/control-flow'; +import {exitWithoutDependency, raiseOutputWithoutDependency} + from '#composite/control-flow'; import inputWikiData from './inputWikiData.js'; @@ -34,10 +35,17 @@ export default templateCompositeFrom({ outputs: ['#reverseReferenceList'], steps: () => [ + // Early exit with an empty array if the data list isn't available. exitWithoutDependency({ dependency: input('data'), value: input.value([]), + }), + + // Raise an empty array (don't early exit) if the data list is empty. + raiseOutputWithoutDependency({ + dependency: input('data'), mode: input.value('empty'), + output: input.value({'#reverseReferenceList': []}), }), { diff --git a/src/data/composite/wiki-data/withUniqueReferencingThing.js b/src/data/composite/wiki-data/withUniqueReferencingThing.js index ce04f838..61c10618 100644 --- a/src/data/composite/wiki-data/withUniqueReferencingThing.js +++ b/src/data/composite/wiki-data/withUniqueReferencingThing.js @@ -21,11 +21,10 @@ export default templateCompositeFrom({ outputs: ['#uniqueReferencingThing'], steps: () => [ - // withReverseRefernceList does this check too, but it early exits with - // an empty array. That's no good here! + // Early exit with null (not an empty array) if the data list + // isn't available. exitWithoutDependency({ dependency: input('data'), - mode: input.value('empty'), }), withReverseReferenceList({ diff --git a/src/data/composite/wiki-properties/contributionList.js b/src/data/composite/wiki-properties/contributionList.js index 8fde2caa..aad12a2d 100644 --- a/src/data/composite/wiki-properties/contributionList.js +++ b/src/data/composite/wiki-properties/contributionList.js @@ -3,15 +3,15 @@ // into one property. Update value will look something like this: // // [ -// {who: 'Artist Name', what: 'Viola'}, -// {who: 'artist:john-cena', what: null}, +// {artist: 'Artist Name', annotation: 'Viola'}, +// {artist: 'artist:john-cena', annotation: null}, // ... // ] // // ...typically as processed from YAML, spreadsheet, or elsewhere. -// Exposes as the same, but with the "who" replaced with matches found in -// artistData - which means this always depends on an `artistData` property -// also existing on this object! +// Exposes as the same, but with the artist property replaced with matches +// found in artistData - which means this always depends on an `artistData` +// property also existing on this object! // import {input, templateCompositeFrom} from '#composite'; diff --git a/src/data/composite/wiki-properties/directory.js b/src/data/composite/wiki-properties/directory.js index 0b2181c9..41ce4b27 100644 --- a/src/data/composite/wiki-properties/directory.js +++ b/src/data/composite/wiki-properties/directory.js @@ -2,22 +2,32 @@ // almost any data object. Also corresponds to a part of the URL which pages of // such objects are visited at. -import {isDirectory} from '#validators'; -import {getKebabCase} from '#wiki-data'; - -// TODO: Not templateCompositeFrom. - -export default function() { - return { - flags: {update: true, expose: true}, - update: {validate: isDirectory}, - expose: { - dependencies: ['name'], - transform(directory, {name}) { - if (directory === null && name === null) return null; - else if (directory === null) return getKebabCase(name); - else return directory; - }, - }, - }; -} +import {input, templateCompositeFrom} from '#composite'; + +import {isDirectory, isName} from '#validators'; + +import {exposeDependency} from '#composite/control-flow'; +import {withDirectory} from '#composite/wiki-data'; + +export default templateCompositeFrom({ + annotation: `directory`, + + compose: false, + + inputs: { + name: input({ + validate: isName, + defaultDependency: 'name', + }), + }, + + steps: () => [ + withDirectory({ + directory: input.updateValue({validate: isDirectory}), + }), + + exposeDependency({ + dependency: '#directory', + }), + ], +}); diff --git a/src/data/composite/wiki-properties/referenceList.js b/src/data/composite/wiki-properties/referenceList.js index af634a68..ebd5947c 100644 --- a/src/data/composite/wiki-properties/referenceList.js +++ b/src/data/composite/wiki-properties/referenceList.js @@ -1,5 +1,6 @@ // Stores and exposes a list of references to other data objects; all items -// must be references to the same type, which is specified on the class input. +// must be references to the same type, which is either implied from the class +// input, or explicitly set on the referenceType input. // // See also: // - singleReference @@ -18,7 +19,17 @@ export default templateCompositeFrom({ compose: false, inputs: { - class: input.staticValue({validate: isThingClass}), + class: input.staticValue({ + validate: isThingClass, + acceptsNull: true, + defaultValue: null, + }), + + referenceType: input.staticValue({ + type: 'string', + acceptsNull: true, + defaultValue: null, + }), data: inputWikiData({allowMixedTypes: false}), @@ -27,10 +38,13 @@ export default templateCompositeFrom({ update: ({ [input.staticValue('class')]: thingClass, + [input.staticValue('referenceType')]: referenceType, }) => ({ validate: validateReferenceList( - thingClass[Symbol.for('Thing.referenceType')]), + (thingClass + ? thingClass[Symbol.for('Thing.referenceType')] + : referenceType)), }), steps: () => [ diff --git a/src/data/serialize.js b/src/data/serialize.js index 8cac3309..2ecbf76c 100644 --- a/src/data/serialize.js +++ b/src/data/serialize.js @@ -16,7 +16,10 @@ export function toRefs(things) { } export function toContribRefs(contribs) { - return contribs?.map(({who, what}) => ({who: toRef(who), what})); + return contribs?.map(({artist, annotation}) => ({ + artist: toRef(artist), + annotation, + })); } export function toCommentaryRefs(entries) { diff --git a/src/data/thing.js b/src/data/thing.js index 706e893d..29f50d23 100644 --- a/src/data/thing.js +++ b/src/data/thing.js @@ -17,17 +17,52 @@ export default class Thing extends CacheableObject { static yamlDocumentSpec = Symbol.for('Thing.yamlDocumentSpec'); static getYamlLoadingSpec = Symbol.for('Thing.getYamlLoadingSpec'); + static isThingConstructor = Symbol.for('Thing.isThingConstructor'); + static isThing = Symbol.for('Thing.isThing'); + + // To detect: + // Symbol.for('Thing.isThingConstructor') in constructor + static [Symbol.for('Thing.isThingConstructor')] = NaN; + + static [CacheableObject.propertyDescriptors] = { + // To detect: + // Object.hasOwn(object, Symbol.for('Thing.isThing')) + [Symbol.for('Thing.isThing')]: { + flags: {expose: true}, + expose: {compute: () => NaN}, + }, + }; + + static [Symbol.for('Thing.selectAll')] = _wikiData => []; + // Default custom inspect function, which may be overridden by Thing // subclasses. This will be used when displaying aggregate errors and other // command-line logging - it's the place to provide information useful in // identifying the Thing being presented. [inspect.custom]() { - const cname = this.constructor.name; + const constructorName = this.constructor.name; + + let name; + try { + if (this.name) { + name = colors.green(`"${this.name}"`); + } + } catch (error) { + name = colors.yellow(`couldn't get name`); + } + + let reference; + try { + if (this.directory) { + reference = colors.blue(Thing.getReference(this)); + } + } catch (error) { + reference = colors.yellow(`couldn't get reference`); + } return ( - (this.name ? `${cname} ${colors.green(`"${this.name}"`)}` : `${cname}`) + - (this.directory ? ` (${colors.blue(Thing.getReference(this))})` : '') - ); + (name ? `${constructorName} ${name}` : `${constructorName}`) + + (reference ? ` (${reference})` : '')); } static getReference(thing) { diff --git a/src/data/things/album.js b/src/data/things/album.js index 40cd4631..e9f55b2c 100644 --- a/src/data/things/album.js +++ b/src/data/things/album.js @@ -1,20 +1,25 @@ export const DATA_ALBUM_DIRECTORY = 'album'; import * as path from 'node:path'; +import {inspect} from 'node:util'; +import CacheableObject from '#cacheable-object'; +import {colors} from '#cli'; import {input} from '#composite'; import find from '#find'; import {traverse} from '#node-utils'; import {sortAlbumsTracksChronologically, sortChronologically} from '#sort'; -import {empty} from '#sugar'; +import {accumulateSum, empty} from '#sugar'; import Thing from '#thing'; -import {isDate} from '#validators'; +import {isColor, isDate, validateWikiData} from '#validators'; import {parseAdditionalFiles, parseContributors, parseDate, parseDimensions} from '#yaml'; -import {exposeDependency, exposeUpdateValueOrContinue} +import {exitWithoutDependency, exposeDependency, exposeUpdateValueOrContinue} from '#composite/control-flow'; -import {exitWithoutContribs} from '#composite/wiki-data'; +import {withPropertyFromObject} from '#composite/data'; +import {exitWithoutContribs, withDirectory, withResolvedReference} + from '#composite/wiki-data'; import { additionalFiles, @@ -31,16 +36,24 @@ import { referenceList, simpleDate, simpleString, + singleReference, urls, wikiData, } from '#composite/wiki-properties'; -import {withTracks, withTrackSections} from '#composite/things/album'; +import {withTracks} from '#composite/things/album'; +import {withAlbum} from '#composite/things/track-section'; export class Album extends Thing { static [Thing.referenceType] = 'album'; - static [Thing.getPropertyDescriptors] = ({ArtTag, Artist, Group, Track}) => ({ + static [Thing.getPropertyDescriptors] = ({ + ArtTag, + Artist, + Group, + Track, + TrackSection, + }) => ({ // Update & expose name: name('Unnamed Album'), @@ -48,6 +61,8 @@ export class Album extends Thing { directory: directory(), urls: urls(), + alwaysReferenceTracksByDirectory: flag(false), + bandcampAlbumIdentifier: simpleString(), bandcampArtworkIdentifier: simpleString(), @@ -109,10 +124,11 @@ export class Album extends Thing { commentary: commentary(), additionalFiles: additionalFiles(), - trackSections: [ - withTrackSections(), - exposeDependency({dependency: '#trackSections'}), - ], + trackSections: referenceList({ + referenceType: input.value('unqualified-track-section'), + data: 'ownTrackSectionData', + find: input.value(find.unqualifiedTrackSection), + }), artistContribs: contributionList(), coverArtistContribs: contributionList(), @@ -153,11 +169,8 @@ export class Album extends Thing { class: input.value(Group), }), - // Only the tracks which belong to this album. - // Necessary for computing the track list, so provide this statically - // or keep it updated. - ownTrackData: wikiData({ - class: input.value(Track), + ownTrackSectionData: wikiData({ + class: input.value(TrackSection), }), // Expose only @@ -226,6 +239,10 @@ export class Album extends Thing { 'Album': {property: 'name'}, 'Directory': {property: 'directory'}, + 'Always Reference Tracks By Directory': { + property: 'alwaysReferenceTracksByDirectory', + }, + 'Bandcamp Album ID': { property: 'bandcampAlbumIdentifier', transform: String, @@ -339,68 +356,77 @@ export class Album extends Thing { headerDocumentThing: Album, entryDocumentThing: document => ('Section' in document - ? TrackSectionHelper + ? TrackSection : Track), save(results) { const albumData = []; + const trackSectionData = []; const trackData = []; for (const {header: album, entries} of results) { - // We can't mutate an array once it's set as a property value, - // so prepare the track sections that will show up in a track list - // all the way before actually applying them. (It's okay to mutate - // an individual section before applying it, since those are just - // generic objects; they aren't Things in and of themselves.) const trackSections = []; - const ownTrackData = []; - let currentTrackSection = { + let currentTrackSection = new TrackSection(); + let currentTrackSectionTracks = []; + + Object.assign(currentTrackSection, { name: `Default Track Section`, isDefaultTrackSection: true, - tracks: [], - }; + }); const albumRef = Thing.getReference(album); const closeCurrentTrackSection = () => { - if (!empty(currentTrackSection.tracks)) { - trackSections.push(currentTrackSection); + if ( + currentTrackSection.isDefaultTrackSection && + empty(currentTrackSectionTracks) + ) { + return; } + + currentTrackSection.tracks = + currentTrackSectionTracks + .map(track => Thing.getReference(track)); + + currentTrackSection.ownTrackData = + currentTrackSectionTracks; + + currentTrackSection.ownAlbumData = + [album]; + + trackSections.push(currentTrackSection); + trackSectionData.push(currentTrackSection); }; for (const entry of entries) { - if (entry instanceof TrackSectionHelper) { + if (entry instanceof TrackSection) { closeCurrentTrackSection(); - - currentTrackSection = { - name: entry.name, - color: entry.color, - dateOriginallyReleased: entry.dateOriginallyReleased, - isDefaultTrackSection: false, - tracks: [], - }; - + currentTrackSection = entry; + currentTrackSectionTracks = []; continue; } + currentTrackSectionTracks.push(entry); trackData.push(entry); entry.dataSourceAlbum = albumRef; - - ownTrackData.push(entry); - currentTrackSection.tracks.push(Thing.getReference(entry)); } closeCurrentTrackSection(); albumData.push(album); - album.trackSections = trackSections; - album.ownTrackData = ownTrackData; + album.trackSections = + trackSections + .map(trackSection => + `unqualified-track-section:` + + trackSection.unqualifiedDirectory); + + album.ownTrackSectionData = trackSections; } - return {albumData, trackData}; + return {albumData, trackSectionData, trackData}; }, sort({albumData, trackData}) { @@ -410,15 +436,139 @@ export class Album extends Thing { }); } -export class TrackSectionHelper extends Thing { +export class TrackSection extends Thing { static [Thing.friendlyName] = `Track Section`; + static [Thing.referenceType] = `track-section`; + + static [Thing.getPropertyDescriptors] = ({Album, Track}) => ({ + // Update & expose - static [Thing.getPropertyDescriptors] = () => ({ name: name('Unnamed Track Section'), - color: color(), + + unqualifiedDirectory: directory(), + + color: [ + exposeUpdateValueOrContinue({ + validate: input.value(isColor), + }), + + withAlbum(), + + withPropertyFromObject({ + object: '#album', + property: input.value('color'), + }), + + exposeDependency({dependency: '#album.color'}), + ], + dateOriginallyReleased: simpleDate(), - isDefaultTrackGroup: flag(false), - }) + + isDefaultTrackSection: flag(false), + + album: [ + withAlbum(), + exposeDependency({dependency: '#album'}), + ], + + tracks: referenceList({ + class: input.value(Track), + data: 'ownTrackData', + find: input.value(find.track), + }), + + // Update only + + ownAlbumData: wikiData({ + class: input.value(Album), + }), + + ownTrackData: wikiData({ + class: input.value(Track), + }), + + // Expose only + + directory: [ + withAlbum(), + + exitWithoutDependency({ + dependency: '#album', + }), + + withPropertyFromObject({ + object: '#album', + property: input.value('directory'), + }), + + withDirectory({ + directory: 'unqualifiedDirectory', + }).outputs({ + '#directory': '#unqualifiedDirectory', + }), + + { + dependencies: ['#album.directory', '#unqualifiedDirectory'], + compute: ({ + ['#album.directory']: albumDirectory, + ['#unqualifiedDirectory']: unqualifiedDirectory, + }) => + albumDirectory + '/' + unqualifiedDirectory, + }, + ], + + startIndex: [ + withAlbum(), + + withPropertyFromObject({ + object: '#album', + property: input.value('trackSections'), + }), + + { + dependencies: ['#album.trackSections', input.myself()], + compute: (continuation, { + ['#album.trackSections']: trackSections, + [input.myself()]: myself, + }) => continuation({ + ['#index']: + trackSections.indexOf(myself), + }), + }, + + exitWithoutDependency({ + dependency: '#index', + mode: input.value('index'), + value: input.value(0), + }), + + { + dependencies: ['#album.trackSections', '#index'], + compute: ({ + ['#album.trackSections']: trackSections, + ['#index']: index, + }) => + accumulateSum( + trackSections + .slice(0, index) + .map(section => section.tracks.length)), + }, + ], + }); + + static [Thing.findSpecs] = { + trackSection: { + referenceTypes: ['track-section'], + bindTo: 'trackSectionData', + }, + + unqualifiedTrackSection: { + referenceTypes: ['unqualified-track-section'], + + getMatchableDirectories: trackSection => + [trackSection.unqualifiedDirectory], + }, + }; static [Thing.yamlDocumentSpec] = { fields: { @@ -431,4 +581,48 @@ export class TrackSectionHelper extends Thing { }, }, }; + + [inspect.custom](depth) { + const parts = []; + + parts.push(Thing.prototype[inspect.custom].apply(this)); + + if (depth >= 0) { + let album = null; + try { + album = this.album; + } catch {} + + let first = null; + try { + first = this.startIndex; + } catch {} + + let length = null; + try { + length = this.tracks.length; + } catch {} + + album ??= CacheableObject.getUpdateValue(this, 'ownAlbumData')?.[0]; + + if (album) { + const albumName = album.name; + const albumIndex = album.trackSections.indexOf(this); + + const num = + (albumIndex === -1 + ? 'indeterminate position' + : `#${albumIndex + 1}`); + + const range = + (albumIndex >= 0 && first !== null && length !== null + ? `: ${first + 1}-${first + length + 1}` + : ''); + + parts.push(` (${colors.yellow(num + range)} in ${colors.green(albumName)})`); + } + } + + return parts.join(''); + } } diff --git a/src/data/things/flash.js b/src/data/things/flash.js index ceed79f7..7038df86 100644 --- a/src/data/things/flash.js +++ b/src/data/things/flash.js @@ -7,7 +7,7 @@ import {sortFlashesChronologically} from '#sort'; import Thing from '#thing'; import {anyOf, isColor, isContentString, isDirectory, isNumber, isString} from '#validators'; -import {parseDate, parseContributors} from '#yaml'; +import {parseContributors, parseDate, parseDimensions} from '#yaml'; import {withPropertyFromObject} from '#composite/data'; @@ -24,6 +24,7 @@ import { commentatorArtists, contentString, contributionList, + dimensions, directory, fileExtension, name, @@ -89,6 +90,8 @@ export class Flash extends Thing { coverArtFileExtension: fileExtension('jpg'), + coverArtDimensions: dimensions(), + contributorContribs: contributionList(), featuredTracks: referenceList({ @@ -171,6 +174,11 @@ export class Flash extends Thing { 'Cover Art File Extension': {property: 'coverArtFileExtension'}, + 'Cover Art Dimensions': { + property: 'coverArtDimensions', + transform: parseDimensions, + }, + 'Featured Tracks': {property: 'featuredTracks'}, 'Contributors': { diff --git a/src/data/things/index.js b/src/data/things/index.js index 3bf84091..4f87f492 100644 --- a/src/data/things/index.js +++ b/src/data/things/index.js @@ -2,10 +2,10 @@ import * as path from 'node:path'; import {fileURLToPath} from 'node:url'; import {openAggregate, showAggregate} from '#aggregate'; +import CacheableObject from '#cacheable-object'; import {logError} from '#cli'; import {compositeFrom} from '#composite'; import * as serialize from '#serialize'; - import Thing from '#thing'; import * as albumClasses from './album.js'; @@ -142,7 +142,10 @@ function evaluatePropertyDescriptors() { } } - constructor.propertyDescriptors = results; + constructor[CacheableObject.propertyDescriptors] = { + ...constructor[CacheableObject.propertyDescriptors] ?? {}, + ...results, + }; }, showFailedClasses(failedClasses) { diff --git a/src/data/things/language.js b/src/data/things/language.js index dbe1ff3d..f20927a4 100644 --- a/src/data/things/language.js +++ b/src/data/things/language.js @@ -127,6 +127,13 @@ export class Language extends Thing { // Expose only + onlyIfOptions: { + flags: {expose: true}, + expose: { + compute: () => Symbol.for(`language.onlyIfOptions`), + }, + }, + intl_date: this.#intlHelper(Intl.DateTimeFormat, {full: true}), intl_number: this.#intlHelper(Intl.NumberFormat), intl_listConjunction: this.#intlHelper(Intl.ListFormat, {type: 'conjunction'}), @@ -218,18 +225,42 @@ export class Language extends Thing { throw new Error(`Invalid key ${key} accessed`); } + const constantCasify = name => + name + .replace(/[A-Z]/g, '_$&') + .toUpperCase(); + // These will be filled up as we iterate over the template, slotting in // each option (if it's present). const missingOptionNames = new Set(); + // These will also be filled. It's a bit different of an error, indicating + // a provided option was *expected,* but its value was null, undefined, or + // blank HTML content. + const valuelessOptionNames = new Set(); + + // These *might* be missing, and if they are, that's OK!! Instead of adding + // to the valueless set above, we'll just mark to return a blank for the + // whole string. + const expectedValuelessOptionNames = + new Set( + (options[this.onlyIfOptions] ?? []) + .map(constantCasify)); + + let seenExpectedValuelessOption = false; + + const isValueless = + value => + value === null || + value === undefined || + html.isBlank(value); + // And this will have entries deleted as they're encountered in the // template. Leftover entries are misplaced. const optionsMap = new Map( Object.entries(options).map(([name, value]) => [ - name - .replace(/[A-Z]/g, '_$&') - .toUpperCase(), + constantCasify(name), value, ])); @@ -239,32 +270,48 @@ export class Language extends Thing { match: languageOptionRegex, insert: ({name: optionName}, canceledForming) => { - if (optionsMap.has(optionName)) { - let optionValue; - - // We'll only need the option's value if we're going to use it as - // part of the formed output (see below). - if (!canceledForming) { - optionValue = optionsMap.get(optionName); - } - - // But we always have to delete expected options off the provided - // option map, since the leftovers are what will be used to tell - // which are misplaced. - optionsMap.delete(optionName); + if (!optionsMap.has(optionName)) { + missingOptionNames.add(optionName); - if (canceledForming) { - return undefined; - } else { - return optionValue; - } - } else { // We don't need to continue forming the output if we've hit a // missing option name, since the end result of this formatString // call will be a thrown error, and formed output won't be needed. - missingOptionNames.add(optionName); + // Return undefined to mark canceledForming for the following + // iterations (and exit early out of this iteration). + return undefined; + } + + // Even if we're not actually forming the output anymore, we'll still + // have to access this option's value to check if it is invalid. + const optionValue = optionsMap.get(optionName); + + // We always have to delete expected options off the provided option + // map, since the leftovers are what will be used to tell which are + // misplaced - information you want even (or doubly so) if we've + // already stopped forming the output thanks to missing options. + optionsMap.delete(optionName); + + // Just like if an option is missing, a valueless option cancels + // forming the rest of the output. + if (isValueless(optionValue)) { + // It's also an error, *except* if this option is one of the ones + // that we're indicated to *expect* might be valueless! In that case, + // we still need to stop forming the string (and mark a separate flag + // so that we return a blank), but it's not an error. + if (expectedValuelessOptionNames.has(optionName)) { + seenExpectedValuelessOption = true; + } else { + valuelessOptionNames.add(optionName); + } + return undefined; } + + if (canceledForming) { + return undefined; + } + + return optionValue; }, }); @@ -272,17 +319,30 @@ export class Language extends Thing { Array.from(optionsMap.keys()); withAggregate({message: `Errors in options for string "${key}"`}, ({push}) => { + const names = set => Array.from(set).join(', '); + if (!empty(missingOptionNames)) { - const names = Array.from(missingOptionNames).join(`, `); - push(new Error(`Missing options: ${names}`)); + push(new Error( + `Missing options: ${names(missingOptionNames)}`)); + } + + if (!empty(valuelessOptionNames)) { + push(new Error( + `Valueless options: ${names(valuelessOptionNames)}`)); } if (!empty(misplacedOptionNames)) { - const names = Array.from(misplacedOptionNames).join(`, `); - push(new Error(`Unexpected options: ${names}`)); + push(new Error( + `Unexpected options: ${names(misplacedOptionNames)}`)); } }); + // If an option was valueless as marked to expect, then that indicates + // the whole string should be treated as blank content. + if (seenExpectedValuelessOption) { + return html.blank(); + } + return output; } @@ -416,11 +476,32 @@ export class Language extends Thing { } formatDate(date) { + // Null or undefined date is blank content. + if (date === null || date === undefined) { + return html.blank(); + } + this.assertIntlAvailable('intl_date'); return this.intl_date.format(date); } formatDateRange(startDate, endDate) { + // formatDateRange expects both values to be present, but if both are null + // or both are undefined, that's just blank content. + const hasStart = startDate !== null && startDate !== undefined; + const hasEnd = endDate !== null && endDate !== undefined; + if (!hasStart || !hasEnd) { + if (startDate === endDate) { + return html.blank(); + } else if (hasStart) { + throw new Error(`Expected both start and end of date range, got only start`); + } else if (hasEnd) { + throw new Error(`Expected both start and end of date range, got only end`); + } else { + throw new Error(`Got mismatched ${startDate}/${endDate} for start and end`); + } + } + this.assertIntlAvailable('intl_date'); return this.intl_date.formatRange(startDate, endDate); } @@ -431,6 +512,17 @@ export class Language extends Thing { days: numDays = 0, approximate = false, }) { + // Give up if any of years, months, or days is null or undefined. + // These default to zero, so something's gone pretty badly wrong to + // pass in all or partial missing values. + if ( + numYears === undefined || numYears === null || + numMonths === undefined || numMonths === null || + numDays === undefined || numDays === null + ) { + throw new Error(`Expected values or default zero for years, months, and days`); + } + let basis; const years = this.countYears(numYears, {unit: true}); @@ -468,6 +560,14 @@ export class Language extends Thing { approximate = true, absolute = true, } = {}) { + // Give up if current and/or reference date is null or undefined. + if ( + currentDate === undefined || currentDate === null || + referenceDate === undefined || referenceDate === null + ) { + throw new Error(`Expected values for currentDate and referenceDate`); + } + const currentInstant = toTemporalInstant.apply(currentDate); const referenceInstant = toTemporalInstant.apply(referenceDate); @@ -528,6 +628,12 @@ export class Language extends Thing { } formatDuration(secTotal, {approximate = false, unit = false} = {}) { + // Null or undefined duration is blank content. + if (secTotal === null || secTotal === undefined) { + return html.blank(); + } + + // Zero duration is a "missing" string. if (secTotal === 0) { return this.formatString('count.duration.missing'); } @@ -565,6 +671,11 @@ export class Language extends Thing { throw new TypeError(`externalLinkSpec unavailable`); } + // Null or undefined url is blank content. + if (url === null || url === undefined) { + return html.blank(); + } + isExternalLinkContext(context); if (style === 'all') { @@ -589,16 +700,31 @@ export class Language extends Thing { } formatIndex(value) { + // Null or undefined value is blank content. + if (value === null || value === undefined) { + return html.blank(); + } + this.assertIntlAvailable('intl_pluralOrdinal'); return this.formatString('count.index.' + this.intl_pluralOrdinal.select(value), {index: value}); } formatNumber(value) { + // Null or undefined value is blank content. + if (value === null || value === undefined) { + return html.blank(); + } + this.assertIntlAvailable('intl_number'); return this.intl_number.format(value); } formatWordCount(value) { + // Null or undefined value is blank content. + if (value === null || value === undefined) { + return html.blank(); + } + const num = this.formatNumber( value > 1000 ? Math.floor(value / 100) / 10 : value ); @@ -612,6 +738,11 @@ export class Language extends Thing { } #formatListHelper(array, processFn) { + // Empty lists, null, and undefined are blank content. + if (empty(array) || array === null || array === undefined) { + return html.blank(); + } + // Operate on "insertion markers" instead of the actual contents of the // array, because the process function (likely an Intl operation) is taken // to only operate on strings. We'll insert the contents of the array back @@ -673,10 +804,22 @@ export class Language extends Thing { // File sizes: 42.5 kB, 127.2 MB, 4.13 GB, 998.82 TB formatFileSize(bytes) { - if (!bytes) return ''; + // Null or undefined bytes is blank content. + if (bytes === null || bytes === undefined) { + return html.blank(); + } + + // Zero bytes is blank content. + if (bytes === 0) { + return html.blank(); + } bytes = parseInt(bytes); - if (isNaN(bytes)) return ''; + + // Non-number bytes is blank content! Wow. + if (isNaN(bytes)) { + return html.blank(); + } const round = (exp) => Math.round(bytes / 10 ** (exp - 1)) / 10; @@ -704,6 +847,11 @@ export class Language extends Thing { const countHelper = (stringKey, optionName = stringKey) => function(value, {unit = false} = {}) { + // Null or undefined value is blank content. + if (value === null || value === undefined) { + return html.blank(); + } + return this.formatString( unit ? `count.${stringKey}.withUnit.` + this.getUnitForm(value) diff --git a/src/data/things/track.js b/src/data/things/track.js index cc49fc24..725b1bb7 100644 --- a/src/data/things/track.js +++ b/src/data/things/track.js @@ -168,10 +168,7 @@ export class Track extends Thing { commentary: commentary(), lyrics: [ - inheritFromOriginalRelease({ - property: input.value('lyrics'), - }), - + inheritFromOriginalRelease(), contentString(), ], @@ -196,7 +193,6 @@ export class Track extends Thing { artistContribs: [ inheritFromOriginalRelease({ - property: input.value('artistContribs'), notFoundValue: input.value([]), }), @@ -220,7 +216,6 @@ export class Track extends Thing { contributorContribs: [ inheritFromOriginalRelease({ - property: input.value('contributorContribs'), notFoundValue: input.value([]), }), @@ -255,7 +250,6 @@ export class Track extends Thing { referencedTracks: [ inheritFromOriginalRelease({ - property: input.value('referencedTracks'), notFoundValue: input.value([]), }), @@ -268,7 +262,6 @@ export class Track extends Thing { sampledTracks: [ inheritFromOriginalRelease({ - property: input.value('sampledTracks'), notFoundValue: input.value([]), }), diff --git a/src/data/things/wiki-info.js b/src/data/things/wiki-info.js index 316bd3bb..2a2c9986 100644 --- a/src/data/things/wiki-info.js +++ b/src/data/things/wiki-info.js @@ -3,8 +3,9 @@ export const WIKI_INFO_FILE = 'wiki-info.yaml'; import {input} from '#composite'; import find from '#find'; import Thing from '#thing'; -import {isColor, isLanguageCode, isName, isURL} from '#validators'; +import {isBoolean, isColor, isLanguageCode, isName, isURL} from '#validators'; +import {exitWithoutDependency} from '#composite/control-flow'; import {contentString, flag, name, referenceList, wikiData} from '#composite/wiki-properties'; @@ -64,8 +65,26 @@ export class WikiInfo extends Thing { enableArtTagUI: flag(false), enableGroupUI: flag(false), + enableSearch: [ + exitWithoutDependency({ + dependency: 'searchDataAvailable', + mode: input.value('falsy'), + value: input.value(false), + }), + + flag(true), + ], + // Update only + searchDataAvailable: { + flags: {update: true}, + update: { + validate: isBoolean, + default: false, + }, + }, + groupData: wikiData({ class: input.value(Group), }), diff --git a/src/data/validators.js b/src/data/validators.js index 987f806d..5d681311 100644 --- a/src/data/validators.js +++ b/src/data/validators.js @@ -443,24 +443,23 @@ for (const entry of illegalContentSpec) { } } -const illegalContentRegexp = - new RegExp( - illegalContentSpec - .map(entry => entry.illegal) - .map(illegal => `${illegal}+`) - .join('|'), - 'g'); - -const illegalCharactersInContent = +const illegalSequencesInContent = illegalContentSpec .map(entry => entry.illegal) - .join(''); + .map(illegal => + (illegal.length === 1 + ? `${illegal}+` + : `(?:${illegal})+`)) + .join('|'); + +const illegalContentRegexp = + new RegExp(illegalSequencesInContent, 'g'); const legalContentNearEndRegexp = - new RegExp(`[^\n${illegalCharactersInContent}]+$`); + new RegExp(`(?<=^|${illegalSequencesInContent})(?:(?!${illegalSequencesInContent}).)+$`); const legalContentNearStartRegexp = - new RegExp(`^[^\n${illegalCharactersInContent}]+`); + new RegExp(`^(?:(?!${illegalSequencesInContent}).)+`); const trimWhitespaceNearBothSidesRegexp = /^ +| +$/gm; @@ -606,16 +605,37 @@ export function isContentString(content) { export function isThingClass(thingClass) { isFunction(thingClass); - if (!Object.hasOwn(thingClass, Symbol.for('Thing.referenceType'))) { - throw new TypeError(`Expected a Thing constructor, missing Thing.referenceType`); + // This is *expressly* no faster than an instanceof check, because it's + // deliberately still walking the prototype chain for the provided object. + // (This is necessary because the symbol we're checking is defined only on + // the Thing constructor, and not directly on each subclass.) However, it's + // preferred over an instanceof check anyway, because instanceof would + // require that the #validators module has access to #thing, which it + // currently doesn't! + if (!(Symbol.for('Thing.isThingConstructor') in thingClass)) { + throw new TypeError(`Expected a Thing constructor, missing Thing.isThingConstructor`); + } + + return true; +} + +export function isThing(thing) { + isObject(thing); + + // This *is* faster than an instanceof check, because it doesn't walk the + // prototype chain. It works because this property is set as part of every + // Thing subclass's inherited "public class fields" - it's set directly on + // every constructed Thing. + if (!Object.hasOwn(thing, Symbol.for('Thing.isThing'))) { + throw new TypeError(`Expected a Thing, missing Thing.isThing`); } return true; } export const isContribution = validateProperties({ - who: isArtistRef, - what: optional(isStringNonEmpty), + artist: isArtistRef, + annotation: optional(isStringNonEmpty), }); export const isContributionList = validateArrayItems(isContribution); @@ -734,12 +754,31 @@ export function validateReferenceList(type = '') { return validateArrayItems(validateReference(type)); } +export function validateThing({ + referenceType: expectedReferenceType = '', +} = {}) { + return (thing) => { + isThing(thing); + + if (expectedReferenceType) { + const {[Symbol.for('Thing.referenceType')]: referenceType} = + thing.constructor; + + if (referenceType !== expectedReferenceType) { + throw new TypeError(`Expected only ${expectedReferenceType}, got other type: ${referenceType}`); + } + } + + return true; + }; +} + const validateWikiData_cache = {}; export function validateWikiData({ referenceType = '', allowMixedTypes = false, -}) { +} = {}) { if (referenceType && allowMixedTypes) { throw new TypeError(`Don't specify both referenceType and allowMixedTypes`); } @@ -768,25 +807,22 @@ export function validateWikiData({ let foundOtherObject = false; for (const object of array) { - const {[Symbol.for('Thing.referenceType')]: referenceType} = object.constructor; - - if (referenceType === undefined) { - foundOtherObject = true; - - // Early-exit if a Thing has been found - nothing more can be learned. - if (foundThing) { - throw new TypeError(`Expected array of wiki data objects, got mixed items`); - } - } else { - foundThing = true; - + if (Object.hasOwn(object, Symbol.for('Thing.isThing'))) { // Early-exit if a non-Thing object has been found - nothing more can // be learned. if (foundOtherObject) { throw new TypeError(`Expected array of wiki data objects, got mixed items`); } - allRefTypes.add(referenceType); + foundThing = true; + allRefTypes.add(object.constructor[Symbol.for('Thing.referenceType')]); + } else { + // Early-exit if a Thing has been found - nothing more can be learned. + if (foundThing) { + throw new TypeError(`Expected array of wiki data objects, got mixed items`); + } + + foundOtherObject = true; } } diff --git a/src/data/yaml.js b/src/data/yaml.js index 86f30143..7e470531 100644 --- a/src/data/yaml.js +++ b/src/data/yaml.js @@ -9,26 +9,32 @@ import yaml from 'js-yaml'; import {colors, ENABLE_COLOR, logInfo, logWarn} from '#cli'; import {sortByName} from '#sort'; -import {atOffset, empty, filterProperties, typeAppearance, withEntries} - from '#sugar'; import Thing from '#thing'; import thingConstructors from '#things'; import { - filterReferenceErrors, - reportContentTextErrors, - reportDuplicateDirectories, -} from '#data-checks'; - -import { annotateErrorWithFile, decorateErrorWithIndex, decorateErrorWithAnnotation, openAggregate, showAggregate, - withAggregate, } from '#aggregate'; +import { + filterReferenceErrors, + reportContentTextErrors, + reportDirectoryErrors, +} from '#data-checks'; + +import { + atOffset, + empty, + filterProperties, + stitchArrays, + typeAppearance, + withEntries, +} from '#sugar'; + function inspect(value, opts = {}) { return nodeInspect(value, {colors: ENABLE_COLOR, ...opts}); } @@ -364,36 +370,53 @@ export function parseDuration(string) { } } -export function parseAdditionalFiles(array) { - if (!Array.isArray(array)) { - // Error will be caught when validating against whatever this value is - return array; - } - - return array.map((item) => ({ - title: item['Title'], - description: item['Description'] ?? null, - files: item['Files'], - })); -} - export const extractAccentRegex = /^(?<main>.*?)(?: \((?<accent>.*)\))?$/; export const extractPrefixAccentRegex = /^(?:\((?<accent>.*)\) )?(?<main>.*?)$/; -export function parseContributors(contributionStrings) { +// TODO: Should this fit better within actual YAML loading infrastructure?? +export function parseArrayEntries(entries, mapFn) { // If this isn't something we can parse, just return it as-is. // The Thing object's validators will handle the data error better // than we're able to here. - if (!Array.isArray(contributionStrings)) { - return contributionStrings; + if (!Array.isArray(entries)) { + return entries; + } + + // If the array is REALLY ACTUALLY empty (it's represented in YAML + // as literally an empty []), that's something we want to reflect. + if (empty(entries)) { + return entries; } - return contributionStrings.map(item => { + const nonNullEntries = + entries.filter(value => value !== null); + + // On the other hand, if the array only contains null, it's just + // a placeholder, so skip over the field like it's not actually + // been put there yet. + if (empty(nonNullEntries)) { + return null; + } + + return entries.map(mapFn); +} + +export function parseContributors(entries) { + return parseArrayEntries(entries, item => { if (typeof item === 'object' && item['Who']) - return {who: item['Who'], what: item['What'] ?? null}; + return { + artist: item['Who'], + annotation: item['What'] ?? null, + }; + + if (typeof item === 'object' && item['Artist']) + return { + artist: item['Artist'], + annotation: item['Annotation'] ?? null, + }; if (typeof item !== 'string') return item; @@ -401,18 +424,26 @@ export function parseContributors(contributionStrings) { if (!match) return item; return { - who: match.groups.main, - what: match.groups.accent ?? null, + artist: match.groups.main, + annotation: match.groups.accent ?? null, }; }); } -export function parseAdditionalNames(additionalNameStrings) { - if (!Array.isArray(additionalNameStrings)) { - return additionalNameStrings; - } +export function parseAdditionalFiles(entries) { + return parseArrayEntries(entries, item => { + if (typeof item !== 'object') return item; + + return { + title: item['Title'], + description: item['Description'] ?? null, + files: item['Files'], + }; + }); +} - return additionalNameStrings.map(item => { +export function parseAdditionalNames(entries) { + return parseArrayEntries(entries, item => { if (typeof item === 'object' && item['Name']) return {name: item['Name'], annotation: item['Annotation'] ?? null}; @@ -523,7 +554,13 @@ export const documentModes = { // them to each other, setting additional properties, etc). Input argument // format depends on documentMode. // -export const getDataSteps = () => { +export function getAllDataSteps() { + try { + thingConstructors; + } catch (error) { + throw new Error(`Thing constructors aren't ready yet, can't get all data steps`); + } + const steps = []; for (const thingConstructor of Object.values(thingConstructors)) { @@ -539,376 +576,501 @@ export const getDataSteps = () => { sortByName(steps, {getName: step => step.title}); return steps; -}; - -export async function loadAndProcessDataDocuments({dataPath}) { - const processDataAggregate = openAggregate({ - message: `Errors processing data files`, - }); - const wikiDataResult = {}; - - function decorateErrorWithFile(fn) { - return decorateErrorWithAnnotation(fn, - (caughtError, firstArg) => - annotateErrorWithFile( - caughtError, - path.relative( - dataPath, - (typeof firstArg === 'object' - ? firstArg.file - : firstArg)))); - } +} - function asyncDecorateErrorWithFile(fn) { - return decorateErrorWithFile(fn).async; - } +export async function getFilesFromDataStep(dataStep, {dataPath}) { + const {documentMode} = dataStep; - for (const dataStep of getDataSteps()) { - await processDataAggregate.nestAsync( - { - message: `Errors during data step: ${colors.bright(dataStep.title)}`, - translucent: true, - }, - async ({call, callAsync, map, mapAsync, push}) => { - const {documentMode} = dataStep; - - if (!Object.values(documentModes).includes(documentMode)) { - throw new Error(`Invalid documentMode: ${documentMode.toString()}`); - } + switch (documentMode) { + case documentModes.allInOne: + case documentModes.oneDocumentTotal: { + if (!dataStep.file) { + throw new Error(`Expected 'file' property for ${documentMode.toString()}`); + } - // Hear me out, it's been like 1200 years since I wrote the rest of - // this beautifully error-containing code and I don't know how to - // integrate this nicely. So I'm just returning the result and the - // error that should be thrown. Yes, we're back in callback hell, - // just without the callbacks. Thank you. - const filterBlankDocuments = documents => { - const aggregate = openAggregate({ - message: `Found blank documents - check for extra '${colors.cyan(`---`)}'`, + const localFile = + (typeof dataStep.file === 'function' + ? await dataStep.file(dataPath) + : dataStep.file); + + const fileUnderDataPath = + path.join(dataPath, localFile); + + const statResult = + await stat(fileUnderDataPath).then( + () => true, + error => { + if (error.code === 'ENOENT') { + return false; + } else { + throw error; + } }); - const filteredDocuments = - documents - .filter(doc => doc !== null); - - if (filteredDocuments.length !== documents.length) { - const blankIndexRangeInfo = - documents - .map((doc, index) => [doc, index]) - .filter(([doc]) => doc === null) - .map(([doc, index]) => index) - .reduce((accumulator, index) => { - if (accumulator.length === 0) { - return [[index, index]]; - } - const current = accumulator.at(-1); - const rest = accumulator.slice(0, -1); - if (current[1] === index - 1) { - return rest.concat([[current[0], index]]); - } else { - return accumulator.concat([[index, index]]); - } - }, []) - .map(([start, end]) => ({ - start, - end, - count: end - start + 1, - previous: atOffset(documents, start, -1), - next: atOffset(documents, end, +1), - })); - - for (const {start, end, count, previous, next} of blankIndexRangeInfo) { - const parts = []; - - if (count === 1) { - const range = `#${start + 1}`; - parts.push(`${count} document (${colors.yellow(range)}), `); - } else { - const range = `#${start + 1}-${end + 1}`; - parts.push(`${count} documents (${colors.yellow(range)}), `); - } - - if (previous === null) { - parts.push(`at start of file`); - } else if (next === null) { - parts.push(`at end of file`); - } else { - const previousDescription = Object.entries(previous).at(0).join(': '); - const nextDescription = Object.entries(next).at(0).join(': '); - parts.push(`between "${colors.cyan(previousDescription)}" and "${colors.cyan(nextDescription)}"`); - } - - aggregate.push(new Error(parts.join(''))); - } - } + if (statResult) { + return [fileUnderDataPath]; + } else { + return []; + } + } - return {documents: filteredDocuments, aggregate}; - }; + case documentModes.headerAndEntries: + case documentModes.onePerFile: { + if (!dataStep.files) { + throw new Error(`Expected 'files' property for ${documentMode.toString()}`); + } - const processDocument = (document, thingClassOrFn) => { - const thingClass = - (thingClassOrFn.prototype instanceof Thing - ? thingClassOrFn - : thingClassOrFn(document)); + const localFiles = + (typeof dataStep.files === 'function' + ? await dataStep.files(dataPath).then( + files => files, + error => { + if (error.code === 'ENOENT') { + return []; + } else { + throw error; + } + }) + : dataStep.files); - if (typeof thingClass !== 'function') { - throw new Error(`Expected a thing class, got ${typeAppearance(thingClass)}`); - } + const filesUnderDataPath = + localFiles + .map(file => path.join(dataPath, file)); - if (!(thingClass.prototype instanceof Thing)) { - throw new Error(`Expected a thing class, got ${thingClass.name}`); - } + return filesUnderDataPath; + } - const spec = thingClass[Thing.yamlDocumentSpec]; + default: + throw new Error(`Unknown document mode ${documentMode.toString()}`); + } +} - if (!spec) { - throw new Error(`Class "${thingClass.name}" doesn't specify Thing.yamlDocumentSpec`); - } +export async function loadYAMLDocumentsFromFile(file) { + let contents; + try { + contents = await readFile(file, 'utf-8'); + } catch (caughtError) { + throw new Error(`Failed to read data file`, {cause: caughtError}); + } + + let documents; + try { + documents = yaml.loadAll(contents); + } catch (caughtError) { + throw new Error(`Failed to parse valid YAML`, {cause: caughtError}); + } + + const aggregate = openAggregate({ + message: `Found blank documents - check for extra '${colors.cyan(`---`)}'`, + }); - // TODO: Making a function to only call it just like that is - // obviously pretty jank! It should be created once per data step. - const fn = makeProcessDocument(thingClass, spec); - return fn(document); - }; - - if ( - documentMode === documentModes.allInOne || - documentMode === documentModes.oneDocumentTotal - ) { - if (!dataStep.file) { - throw new Error(`Expected 'file' property for ${documentMode.toString()}`); + const filteredDocuments = + documents + .filter(doc => doc !== null); + + if (filteredDocuments.length !== documents.length) { + const blankIndexRangeInfo = + documents + .map((doc, index) => [doc, index]) + .filter(([doc]) => doc === null) + .map(([doc, index]) => index) + .reduce((accumulator, index) => { + if (accumulator.length === 0) { + return [[index, index]]; } + const current = accumulator.at(-1); + const rest = accumulator.slice(0, -1); + if (current[1] === index - 1) { + return rest.concat([[current[0], index]]); + } else { + return accumulator.concat([[index, index]]); + } + }, []) + .map(([start, end]) => ({ + start, + end, + count: end - start + 1, + previous: atOffset(documents, start, -1), + next: atOffset(documents, end, +1), + })); + + for (const {start, end, count, previous, next} of blankIndexRangeInfo) { + const parts = []; + + if (count === 1) { + const range = `#${start + 1}`; + parts.push(`${count} document (${colors.yellow(range)}), `); + } else { + const range = `#${start + 1}-${end + 1}`; + parts.push(`${count} documents (${colors.yellow(range)}), `); + } - const file = path.join( - dataPath, - typeof dataStep.file === 'function' - ? await callAsync(dataStep.file, dataPath) - : dataStep.file); + if (previous === null) { + parts.push(`at start of file`); + } else if (next === null) { + parts.push(`at end of file`); + } else { + const previousDescription = Object.entries(previous).at(0).join(': '); + const nextDescription = Object.entries(next).at(0).join(': '); + parts.push(`between "${colors.cyan(previousDescription)}" and "${colors.cyan(nextDescription)}"`); + } - const statResult = await callAsync(() => - stat(file).then( - () => true, - error => { - if (error.code === 'ENOENT') { - return false; - } else { - throw error; - } - })); + aggregate.push(new Error(parts.join(''))); + } + } - if (statResult === false) { - const saveResult = call(dataStep.save, { - [documentModes.allInOne]: [], - [documentModes.oneDocumentTotal]: {}, - }[documentMode]); + return {result: filteredDocuments, aggregate}; +} - if (!saveResult) return; +// Mapping from dataStep (spec) object each to a sub-map, from thing class to +// processDocument function. +const processDocumentFns = new WeakMap(); - Object.assign(wikiDataResult, saveResult); +export function processThingsFromDataStep(documents, dataStep) { + let submap; + if (processDocumentFns.has(dataStep)) { + submap = processDocumentFns.get(dataStep); + } else { + submap = new Map(); + processDocumentFns.set(dataStep, submap); + } - return; - } + function processDocument(document, thingClassOrFn) { + const thingClass = + (thingClassOrFn.prototype instanceof Thing + ? thingClassOrFn + : thingClassOrFn(document)); + + let fn; + if (submap.has(thingClass)) { + fn = submap.get(thingClass); + } else { + if (typeof thingClass !== 'function') { + throw new Error(`Expected a thing class, got ${typeAppearance(thingClass)}`); + } - const readResult = await callAsync(readFile, file, 'utf-8'); + if (!(thingClass.prototype instanceof Thing)) { + throw new Error(`Expected a thing class, got ${thingClass.name}`); + } - if (!readResult) { - return; - } + const spec = thingClass[Thing.yamlDocumentSpec]; - let processResults; + if (!spec) { + throw new Error(`Class "${thingClass.name}" doesn't specify Thing.yamlDocumentSpec`); + } - switch (documentMode) { - case documentModes.oneDocumentTotal: { - const yamlResult = call(yaml.load, readResult); + fn = makeProcessDocument(thingClass, spec); + submap.set(thingClass, fn); + } - if (!yamlResult) { - processResults = null; - break; - } + return fn(document); + } - const {thing, aggregate} = - processDocument(yamlResult, dataStep.documentThing); + const {documentMode} = dataStep; - processResults = thing; + switch (documentMode) { + case documentModes.allInOne: { + const result = []; + const aggregate = openAggregate({message: `Errors processing documents`}); - call(() => aggregate.close()); + documents.forEach( + decorateErrorWithIndex(document => { + const {thing, aggregate: subAggregate} = + processDocument(document, dataStep.documentThing); - break; - } + result.push(thing); + aggregate.call(subAggregate.close); + })); - case documentModes.allInOne: { - const yamlResults = call(yaml.loadAll, readResult); + return {aggregate, result}; + } - if (!yamlResults) { - processResults = []; - return; - } + case documentModes.oneDocumentTotal: { + if (documents.length > 1) + throw new Error(`Only expected one document to be present, got ${documents.length}`); - const {documents, aggregate: filterAggregate} = - filterBlankDocuments(yamlResults); + const {thing, aggregate} = + processDocument(documents[0], dataStep.documentThing); - call(filterAggregate.close); + return {aggregate, result: thing}; + } - processResults = []; + case documentModes.headerAndEntries: { + const headerDocument = documents[0]; + const entryDocuments = documents.slice(1).filter(Boolean); - map(documents, decorateErrorWithIndex(document => { - const {thing, aggregate} = - processDocument(document, dataStep.documentThing); + if (!headerDocument) + throw new Error(`Missing header document (empty file or erroneously starting with "---"?)`); - processResults.push(thing); - aggregate.close(); - }), {message: `Errors processing documents`}); + const aggregate = openAggregate({message: `Errors processing documents`}); - break; - } - } + const {thing: headerThing, aggregate: headerAggregate} = + processDocument(headerDocument, dataStep.headerDocumentThing); - if (!processResults) return; + try { + headerAggregate.close(); + } catch (caughtError) { + caughtError.message = `(${colors.yellow(`header`)}) ${caughtError.message}`; + aggregate.push(caughtError); + } - const saveResult = call(dataStep.save, processResults); + const entryThings = []; - if (!saveResult) return; + for (const [index, entryDocument] of entryDocuments.entries()) { + const {thing: entryThing, aggregate: entryAggregate} = + processDocument(entryDocument, dataStep.entryDocumentThing); - Object.assign(wikiDataResult, saveResult); + entryThings.push(entryThing); - return; + try { + entryAggregate.close(); + } catch (caughtError) { + caughtError.message = `(${colors.yellow(`entry #${index + 1}`)}) ${caughtError.message}`; + aggregate.push(caughtError); } + } - if (!dataStep.files) { - throw new Error(`Expected 'files' property for ${documentMode.toString()}`); - } + return { + aggregate, + result: { + header: headerThing, + entries: entryThings, + }, + }; + } - const filesFromDataStep = - (typeof dataStep.files === 'function' - ? await callAsync(() => - dataStep.files(dataPath).then( - files => files, - error => { - if (error.code === 'ENOENT') { - return []; - } else { - throw error; - } - })) - : dataStep.files); - - const filesUnderDataPath = - filesFromDataStep - .map(file => path.join(dataPath, file)); - - const yamlResults = []; - - await mapAsync(filesUnderDataPath, {message: `Errors loading data files`}, - asyncDecorateErrorWithFile(async file => { - let contents; - try { - contents = await readFile(file, 'utf-8'); - } catch (caughtError) { - throw new Error(`Failed to read data file`, {cause: caughtError}); - } + case documentModes.onePerFile: { + if (documents.length > 1) + throw new Error(`Only expected one document to be present per file, got ${documents.length} here`); - let documents; - try { - documents = yaml.loadAll(contents); - } catch (caughtError) { - throw new Error(`Failed to parse valid YAML`, {cause: caughtError}); - } + if (empty(documents) || !documents[0]) + throw new Error(`Expected a document, this file is empty`); - const {documents: filteredDocuments, aggregate: filterAggregate} = - filterBlankDocuments(documents); - - try { - filterAggregate.close(); - } catch (caughtError) { - // Blank documents aren't a critical error, they're just something - // that should be noted - the (filtered) documents still get pushed. - const pathToFile = path.relative(dataPath, file); - annotateErrorWithFile(caughtError, pathToFile); - push(caughtError); - } + const {thing, aggregate} = + processDocument(documents[0], dataStep.documentThing); + + return {aggregate, result: thing}; + } + + default: + throw new Error(`Unknown document mode ${documentMode.toString()}`); + } +} + +export function decorateErrorWithFileFromDataPath(fn, {dataPath}) { + return decorateErrorWithAnnotation(fn, + (caughtError, firstArg) => + annotateErrorWithFile( + caughtError, + path.relative( + dataPath, + (typeof firstArg === 'object' + ? firstArg.file + : firstArg)))); +} + +// Loads a list of files for each data step, and a list of documents +// for each file. +export async function loadYAMLDocumentsFromDataSteps(dataSteps, {dataPath}) { + const aggregate = + openAggregate({ + message: `Errors loading data files`, + translucent: true, + }); - yamlResults.push({file, documents: filteredDocuments}); + const fileLists = + await Promise.all( + dataSteps.map(dataStep => + getFilesFromDataStep(dataStep, {dataPath}))); + + const filePromises = + fileLists + .map(files => files + .map(file => + loadYAMLDocumentsFromFile(file).then( + ({result, aggregate}) => { + const close = + decorateErrorWithFileFromDataPath(aggregate.close, {dataPath}); + + aggregate.close = () => + close({file}); + + return {result, aggregate}; + }, + (error) => { + const aggregate = {}; + + annotateErrorWithFile(error, path.relative(dataPath, file)); + + aggregate.close = () => { + throw error; + }; + + return {result: [], aggregate}; + }))); + + const fileListPromises = + filePromises + .map(filePromises => Promise.all(filePromises)); + + const dataStepPromises = + stitchArrays({ + dataStep: dataSteps, + fileListPromise: fileListPromises, + }).map(async ({dataStep, fileListPromise}) => + openAggregate({ + message: `Errors loading data files for data step: ${colors.bright(dataStep.title)}`, + translucent: true, + }).contain(await fileListPromise)); + + const documentLists = + aggregate + .receive(await Promise.all(dataStepPromises)); + + return {aggregate, result: {documentLists, fileLists}}; +} + +// Loads a list of things from a list of documents for each file +// for each data step. Nesting! +export async function processThingsFromDataSteps(documentLists, fileLists, dataSteps, {dataPath}) { + const aggregate = + openAggregate({ + message: `Errors processing documents in data files`, + translucent: true, + }); + + const filePromises = + stitchArrays({ + dataStep: dataSteps, + files: fileLists, + documentLists: documentLists, + }).map(({dataStep, files, documentLists}) => + stitchArrays({ + file: files, + documents: documentLists, + }).map(({file, documents}) => { + const {result, aggregate} = + processThingsFromDataStep(documents, dataStep); + + const close = decorateErrorWithFileFromDataPath(aggregate.close, {dataPath}); + aggregate.close = () => close({file}); + + return {result, aggregate}; })); - const processResults = []; - - switch (documentMode) { - case documentModes.headerAndEntries: - map(yamlResults, {message: `Errors processing documents in data files`, translucent: true}, - decorateErrorWithFile(({documents}) => { - const headerDocument = documents[0]; - const entryDocuments = documents.slice(1).filter(Boolean); - - if (!headerDocument) - throw new Error(`Missing header document (empty file or erroneously starting with "---"?)`); - - withAggregate({message: `Errors processing documents`}, ({push}) => { - const {thing: headerObject, aggregate: headerAggregate} = - processDocument(headerDocument, dataStep.headerDocumentThing); - - try { - headerAggregate.close(); - } catch (caughtError) { - caughtError.message = `(${colors.yellow(`header`)}) ${caughtError.message}`; - push(caughtError); - } - - const entryObjects = []; - - for (let index = 0; index < entryDocuments.length; index++) { - const entryDocument = entryDocuments[index]; - - const {thing: entryObject, aggregate: entryAggregate} = - processDocument(entryDocument, dataStep.entryDocumentThing); - - entryObjects.push(entryObject); - - try { - entryAggregate.close(); - } catch (caughtError) { - caughtError.message = `(${colors.yellow(`entry #${index + 1}`)}) ${caughtError.message}`; - push(caughtError); - } - } - - processResults.push({ - header: headerObject, - entries: entryObjects, - }); - }); - })); - break; - - case documentModes.onePerFile: - map(yamlResults, {message: `Errors processing data files as valid documents`}, - decorateErrorWithFile(({documents}) => { - if (documents.length > 1) - throw new Error(`Only expected one document to be present per file, got ${documents.length} here`); - - if (empty(documents) || !documents[0]) - throw new Error(`Expected a document, this file is empty`); - - const {thing, aggregate} = - processDocument(documents[0], dataStep.documentThing); - - processResults.push(thing); - aggregate.close(); - })); - break; - } + const fileListPromises = + filePromises + .map(filePromises => Promise.all(filePromises)); + + const dataStepPromises = + stitchArrays({ + dataStep: dataSteps, + fileListPromise: fileListPromises, + }).map(async ({dataStep, fileListPromise}) => + openAggregate({ + message: `Errors loading data files for data step: ${colors.bright(dataStep.title)}`, + translucent: true, + }).contain(await fileListPromise)); + + const thingLists = + aggregate + .receive(await Promise.all(dataStepPromises)); + + return {aggregate, result: thingLists}; +} - const saveResult = call(dataStep.save, processResults); +// Flattens a list of *lists* of things for a given data step (each list +// corresponding to one YAML file) into results to be saved on the final +// wikiData object, routing thing lists into the step's save() function. +export function saveThingsFromDataStep(thingLists, dataStep) { + const {documentMode} = dataStep; - if (!saveResult) return; + switch (documentMode) { + case documentModes.allInOne: { + const things = + (empty(thingLists) + ? [] + : thingLists[0]); - Object.assign(wikiDataResult, saveResult); - } - ); + return dataStep.save(things); + } + + case documentModes.oneDocumentTotal: { + const thing = + (empty(thingLists) + ? {} + : thingLists[0]); + + return dataStep.save(thing); + } + + case documentModes.headerAndEntries: + case documentModes.onePerFile: { + return dataStep.save(thingLists); + } + + default: + throw new Error(`Invalid documentMode: ${documentMode.toString()}`); } +} - return { - aggregate: processDataAggregate, - result: wikiDataResult, - }; +// Flattens a list of *lists* of things for each data step (each list +// corresponding to one YAML file) into the final wikiData object, +// routing thing lists into each step's save() function. +export function saveThingsFromDataSteps(thingLists, dataSteps) { + const aggregate = + openAggregate({ + message: `Errors finalizing things from data files`, + translucent: true, + }); + + const wikiData = {}; + + stitchArrays({ + dataStep: dataSteps, + thingLists: thingLists, + }).map(({dataStep, thingLists}) => { + try { + return saveThingsFromDataStep(thingLists, dataStep); + } catch (caughtError) { + const error = new Error( + `Error finalizing things for data step: ${colors.bright(dataStep.title)}`, + {cause: caughtError}); + + error[Symbol.for('hsmusic.aggregate.translucent')] = true; + + aggregate.push(error); + + return null; + } + }) + .filter(Boolean) + .forEach(saveResult => { + Object.assign(wikiData, saveResult); + }); + + return {aggregate, result: wikiData}; +} + +export async function loadAndProcessDataDocuments(dataSteps, {dataPath}) { + const aggregate = + openAggregate({ + message: `Errors processing data files`, + }); + + const {documentLists, fileLists} = + aggregate.receive( + await loadYAMLDocumentsFromDataSteps(dataSteps, {dataPath})); + + const thingLists = + aggregate.receive( + await processThingsFromDataSteps(documentLists, fileLists, dataSteps, {dataPath})); + + const wikiData = + aggregate.receive( + saveThingsFromDataSteps(thingLists, dataSteps)); + + return {aggregate, result: wikiData}; } // Data linking! Basically, provide (portions of) wikiData to the Things which @@ -988,15 +1150,13 @@ export function linkWikiDataArrays(wikiData) { } } -export function sortWikiDataArrays(wikiData) { +export function sortWikiDataArrays(dataSteps, wikiData) { for (const [key, value] of Object.entries(wikiData)) { if (!Array.isArray(value)) continue; wikiData[key] = value.slice(); } - const steps = getDataSteps(); - - for (const step of steps) { + for (const step of dataSteps) { if (!step.sort) continue; step.sort(wikiData); } @@ -1023,10 +1183,12 @@ export async function quickLoadAllFromYAML(dataPath, { }) { const showAggregate = customShowAggregate; + const dataSteps = getAllDataSteps(); + let wikiData; { - const {aggregate, result} = await loadAndProcessDataDocuments({dataPath}); + const {aggregate, result} = await loadAndProcessDataDocuments(dataSteps, {dataPath}); wikiData = result; @@ -1042,7 +1204,7 @@ export async function quickLoadAllFromYAML(dataPath, { linkWikiDataArrays(wikiData); try { - reportDuplicateDirectories(wikiData, {getAllFindSpecs}); + reportDirectoryErrors(wikiData, {getAllFindSpecs}); logInfo`No duplicate directories found. (complete data)`; } catch (error) { showAggregate(error); @@ -1065,7 +1227,7 @@ export async function quickLoadAllFromYAML(dataPath, { logWarn`Content text errors found.`; } - sortWikiDataArrays(wikiData); + sortWikiDataArrays(dataSteps, wikiData); return wikiData; } |