diff options
Diffstat (limited to 'src')
80 files changed, 2623 insertions, 1748 deletions
diff --git a/src/content/dependencies/generateAlbumAdditionalFilesList.js b/src/content/dependencies/generateAlbumAdditionalFilesList.js index 9818a43c..ad17206f 100644 --- a/src/content/dependencies/generateAlbumAdditionalFilesList.js +++ b/src/content/dependencies/generateAlbumAdditionalFilesList.js @@ -9,7 +9,7 @@ export default { 'transformContent', ], - extraDependencies: ['getSizeOfAdditionalFile', 'html', 'urls'], + extraDependencies: ['getSizeOfMediaFile', 'html', 'urls'], relations: (relation, album, additionalFiles) => ({ list: @@ -55,7 +55,7 @@ export default { showFileSizes: {type: 'boolean', default: true}, }, - generate: (data, relations, slots, {getSizeOfAdditionalFile, urls}) => + generate: (data, relations, slots, {getSizeOfMediaFile, urls}) => relations.list.slots({ chunks: stitchArrays({ @@ -86,7 +86,7 @@ export default { fileLink: fileLink, fileSize: (slots.showFileSizes - ? getSizeOfAdditionalFile( + ? getSizeOfMediaFile( urls .from('media.root') .to('media.albumAdditionalFile', data.albumDirectory, location)) diff --git a/src/content/dependencies/generatePageLayout.js b/src/content/dependencies/generatePageLayout.js index fa2cdc18..4c37c5af 100644 --- a/src/content/dependencies/generatePageLayout.js +++ b/src/content/dependencies/generatePageLayout.js @@ -578,6 +578,16 @@ export default { ])), ])); + const styleRulesCSS = + html.resolve(slots.styleRules, {normalize: 'string'}); + + const fallbackBackgroundStyleRule = + (styleRulesCSS.match(/body::before[^}]*background-image:/) + ? '' + : `body::before {\n` + + ` background-image: url("${to('media.path', 'bg.jpg')}");\n` + + `}`); + const numWallpaperParts = html.resolve(slots.styleRules, {normalize: 'string'}) .match(/\.wallpaper-part:nth-child/g) @@ -725,6 +735,8 @@ export default { html.tag('style', [ relations.colorStyleRules .slot('color', slots.color ?? data.wikiColor), + + fallbackBackgroundStyleRule, slots.styleRules, ]), diff --git a/src/content/dependencies/image.js b/src/content/dependencies/image.js index b1f02819..6cbcb7dd 100644 --- a/src/content/dependencies/image.js +++ b/src/content/dependencies/image.js @@ -5,7 +5,7 @@ export default { extraDependencies: [ 'checkIfImagePathHasCachedThumbnails', 'getDimensionsOfImagePath', - 'getSizeOfImagePath', + 'getSizeOfMediaFile', 'getThumbnailEqualOrSmaller', 'getThumbnailsAvailableForDimensions', 'html', @@ -83,7 +83,7 @@ export default { generate(data, relations, slots, { checkIfImagePathHasCachedThumbnails, getDimensionsOfImagePath, - getSizeOfImagePath, + getSizeOfMediaFile, getThumbnailEqualOrSmaller, getThumbnailsAvailableForDimensions, html, @@ -228,7 +228,7 @@ export default { const fileSize = (willLink && mediaSrc - ? getSizeOfImagePath(mediaSrc) + ? getSizeOfMediaFile(mediaSrc) : null); imgAttributes.add([ diff --git a/src/data/cacheable-object.js b/src/data/cacheable-object.js index 010d967a..4b354ef7 100644 --- a/src/data/cacheable-object.js +++ b/src/data/cacheable-object.js @@ -1,79 +1,3 @@ -// Generally extendable class for caching properties and handling dependencies, -// with a few key properties: -// -// 1) The behavior of every property is defined by its descriptor, which is a -// static value stored on the subclass (all instances share the same property -// descriptors). -// -// 1a) Additional properties may not be added past the time of object -// construction, and attempts to do so (including externally setting a -// property name which has no corresponding descriptor) will throw a -// TypeError. (This is done via an Object.seal(this) call after a newly -// created instance defines its own properties according to the descriptor -// on its constructor class.) -// -// 2) Properties may have two flags set: update and expose. Properties which -// update are provided values from the external. Properties which expose -// provide values to the external, generally dependent on other update -// properties (within the same object). -// -// 2a) Properties may be flagged as both updating and exposing. This is so -// that the same name may be used for both "output" and "input". -// -// 3) Exposed properties have values which are computations dependent on other -// properties, as described by a `compute` function on the descriptor. -// Depended-upon properties are explicitly listed on the descriptor next to -// this function, and are only provided as arguments to the function once -// listed. -// -// 3a) An exposed property may depend only upon updating properties, not other -// exposed properties (within the same object). This is to force the -// general complexity of a single object to be fairly simple: inputs -// directly determine outputs, with the only in-between step being the -// `compute` function, no multiple-layer dependencies. Note that this is -// only true within a given object - externally, values provided to one -// object's `update` may be (and regularly are) the exposed values of -// another object. -// -// 3b) If a property both updates and exposes, it is automatically regarded as -// a dependancy. (That is, its exposed value will depend on the value it is -// updated with.) Rather than a required `compute` function, these have an -// optional `transform` function, which takes the update value as its first -// argument and then the usual key-value dependencies as its second. If no -// `transform` function is provided, the expose value is the same as the -// update value. -// -// 4) Exposed properties are cached; that is, if no depended-upon properties are -// updated, the value of an exposed property is not recomputed. -// -// 4a) The cache for an exposed property is invalidated as soon as any of its -// dependencies are updated, but the cache itself is lazy: the exposed -// value will not be recomputed until it is again accessed. (Likewise, an -// exposed value won't be computed for the first time until it is first -// accessed.) -// -// 5) Updating a property may optionally apply validation checks before passing, -// declared by a `validate` function on the `update` block. This function -// should either throw an error (e.g. TypeError) or return false if the value -// is invalid. -// -// 6) Objects do not expect all updating properties to be provided at once. -// Incomplete objects are deliberately supported and enabled. -// -// 6a) The default value for every updating property is null; undefined is not -// accepted as a property value under any circumstances (it always errors). -// However, this default may be overridden by specifying a `default` value -// on a property's `update` block. (This value will be checked against -// the property's validate function.) Note that a property may always be -// updated to null, even if the default is non-null. (Null always bypasses -// the validate check.) -// -// 6b) It's required by the external consumer of an object to determine whether -// or not the object is ready for use (within the larger program). This is -// convenienced by the static CacheableObject.listAccessibleProperties() -// function, which provides a mapping of exposed property names to whether -// or not their dependencies are yet met. - import {inspect as nodeInspect} from 'node:util'; import {colors, ENABLE_COLOR} from '#cli'; @@ -84,53 +8,21 @@ function inspect(value) { export default class CacheableObject { static propertyDescriptors = Symbol.for('CacheableObject.propertyDescriptors'); + static constructorFinalized = Symbol.for('CacheableObject.constructorFinalized'); + static propertyDependants = Symbol.for('CacheableObject.propertyDependants'); - #propertyUpdateValues = Object.create(null); - #propertyUpdateCacheInvalidators = Object.create(null); - - // Note the constructor doesn't take an initial data source. Due to a quirk - // of JavaScript, private members can't be accessed before the superclass's - // constructor is finished processing - so if we call the overridden - // update() function from inside this constructor, it will error when - // writing to private members. Pretty bad! - // - // That means initial data must be provided by following up with update() - // after constructing the new instance of the Thing (sub)class. + static cacheValid = Symbol.for('CacheableObject.cacheValid'); + static updateValue = Symbol.for('CacheableObject.updateValues'); constructor() { - this.#defineProperties(); - this.#initializeUpdatingPropertyValues(); - - if (CacheableObject.DEBUG_SLOW_TRACK_INVALID_PROPERTIES) { - return new Proxy(this, { - get: (obj, key) => { - if (!Object.hasOwn(obj, key)) { - if (key !== 'constructor') { - CacheableObject._invalidAccesses.add(`(${obj.constructor.name}).${key}`); - } - } - return obj[key]; - }, - }); - } - } - - #withEachPropertyDescriptor(callback) { - const {[CacheableObject.propertyDescriptors]: propertyDescriptors} = - this.constructor; + this[CacheableObject.updateValue] = Object.create(null); + this[CacheableObject.cachedValue] = Object.create(null); + this[CacheableObject.cacheValid] = Object.create(null); + const propertyDescriptors = this.constructor[CacheableObject.propertyDescriptors]; for (const property of Reflect.ownKeys(propertyDescriptors)) { - callback(property, propertyDescriptors[property]); - } - } - - #initializeUpdatingPropertyValues() { - this.#withEachPropertyDescriptor((property, descriptor) => { - const {flags, update} = descriptor; - - if (!flags.update) { - return; - } + const {flags, update} = propertyDescriptors[property]; + if (!flags.update) continue; if ( typeof update === 'object' && @@ -141,188 +33,157 @@ export default class CacheableObject { } else { this[property] = null; } - }); + } } - #defineProperties() { - if (!this.constructor[CacheableObject.propertyDescriptors]) { - throw new Error(`Expected constructor ${this.constructor.name} to provide CacheableObject.propertyDescriptors`); + static finalizeCacheableObjectPrototype() { + if (this[CacheableObject.constructorFinalized]) { + throw new Error(`Constructor ${this.name} already finalized`); + } + + if (!this[CacheableObject.propertyDescriptors]) { + throw new Error(`Expected constructor ${this.name} to provide CacheableObject.propertyDescriptors`); } - this.#withEachPropertyDescriptor((property, descriptor) => { - const {flags} = descriptor; + this[CacheableObject.propertyDependants] = Object.create(null); + + const propertyDescriptors = this[CacheableObject.propertyDescriptors]; + for (const property of Reflect.ownKeys(propertyDescriptors)) { + const {flags, update, expose} = propertyDescriptors[property]; const definition = { configurable: false, enumerable: flags.expose, }; - if (flags.update) { - definition.set = this.#getUpdateObjectDefinitionSetterFunction(property); - } - - if (flags.expose) { - definition.get = this.#getExposeObjectDefinitionGetterFunction(property); - } - - Object.defineProperty(this, property, definition); - }); - - Object.seal(this); - } + if (flags.update) setSetter: { + definition.set = function(newValue) { + if (newValue === undefined) { + throw new TypeError(`Properties cannot be set to undefined`); + } - #getUpdateObjectDefinitionSetterFunction(property) { - const {update} = this.#getPropertyDescriptor(property); - const validate = update?.validate; + const oldValue = this[CacheableObject.updateValue][property]; - return (newValue) => { - const oldValue = this.#propertyUpdateValues[property]; + if (newValue === oldValue) { + return; + } - if (newValue === undefined) { - throw new TypeError(`Properties cannot be set to undefined`); - } + if (newValue !== null && update?.validate) { + try { + const result = update.validate(newValue); + if (result === undefined) { + throw new TypeError(`Validate function returned undefined`); + } else if (result !== true) { + throw new TypeError(`Validation failed for value ${newValue}`); + } + } catch (caughtError) { + throw new CacheableObjectPropertyValueError( + property, oldValue, newValue, {cause: caughtError}); + } + } - if (newValue === oldValue) { - return; - } + this[CacheableObject.updateValue][property] = newValue; - if (newValue !== null && validate) { - try { - const result = validate(newValue); - if (result === undefined) { - throw new TypeError(`Validate function returned undefined`); - } else if (result !== true) { - throw new TypeError(`Validation failed for value ${newValue}`); + const dependants = this.constructor[CacheableObject.propertyDependants][property]; + if (dependants) { + for (const dependant of dependants) { + this[CacheableObject.cacheValid][dependant] = false; + } } - } catch (caughtError) { - throw new CacheableObjectPropertyValueError( - property, oldValue, newValue, {cause: caughtError}); - } + }; } - this.#propertyUpdateValues[property] = newValue; - this.#invalidateCachesDependentUpon(property); - }; - } - - #getPropertyDescriptor(property) { - return this.constructor[CacheableObject.propertyDescriptors][property]; - } + if (flags.expose) setGetter: { + if (flags.update && !expose?.transform) { + definition.get = function() { + return this[CacheableObject.updateValue][property]; + }; - #invalidateCachesDependentUpon(property) { - const invalidators = this.#propertyUpdateCacheInvalidators[property]; - if (!invalidators) { - return; - } + break setGetter; + } - for (const invalidate of invalidators) { - invalidate(); - } - } + if (flags.update && expose?.compute) { + throw new Error(`Updating property ${property} has compute function, should be formatted as transform`); + } - #getExposeObjectDefinitionGetterFunction(property) { - const {flags} = this.#getPropertyDescriptor(property); - const compute = this.#getExposeComputeFunction(property); - - if (compute) { - let cachedValue; - const checkCacheValid = this.#getExposeCheckCacheValidFunction(property); - return () => { - if (checkCacheValid()) { - return cachedValue; - } else { - return (cachedValue = compute()); + if (!flags.update && !expose?.compute) { + throw new Error(`Exposed property ${property} does not update and is missing compute function`); } - }; - } else if (!flags.update && !compute) { - throw new Error(`Exposed property ${property} does not update and is missing compute function`); - } else { - return () => this.#propertyUpdateValues[property]; - } - } - #getExposeComputeFunction(property) { - const {flags, expose} = this.#getPropertyDescriptor(property); + definition.get = function() { + if (this[CacheableObject.cacheValid][property]) { + return this[CacheableObject.cachedValue][property]; + } - const compute = expose?.compute; - const transform = expose?.transform; + const dependencies = Object.create(null); + for (const key of expose.dependencies ?? []) { + switch (key) { + case 'this': + dependencies.this = this; + break; - if (flags.update && !transform) { - return null; - } else if (flags.update && compute) { - throw new Error(`Updating property ${property} has compute function, should be formatted as transform`); - } else if (!flags.update && !compute) { - throw new Error(`Exposed property ${property} does not update and is missing compute function`); - } + case 'thisProperty': + dependencies.thisProperty = property; + break; - let getAllDependencies; + default: + dependencies[key] = this[CacheableObject.updateValue][key]; + break; + } + } - if (expose.dependencies?.length > 0) { - const dependencyKeys = expose.dependencies.slice(); - const shouldReflectObject = dependencyKeys.includes('this'); - const shouldReflectProperty = dependencyKeys.includes('thisProperty'); + const value = + (flags.update + ? expose.transform(this[CacheableObject.updateValue][property], dependencies) + : expose.compute(dependencies)); - getAllDependencies = () => { - const dependencies = Object.create(null); + this[CacheableObject.cachedValue][property] = value; + this[CacheableObject.cacheValid][property] = true; - for (const key of dependencyKeys) { - dependencies[key] = this.#propertyUpdateValues[key]; - } + return value; + }; + } + + if (flags.expose) recordAsDependant: { + const dependantsMap = this[CacheableObject.propertyDependants]; - if (shouldReflectObject) { - dependencies.this = this; + if (flags.update && expose?.transform) { + if (dependantsMap[property]) { + dependantsMap[property].push(property); + } else { + dependantsMap[property] = [property]; + } } - if (shouldReflectProperty) { - dependencies.thisProperty = property; + for (const dependency of expose?.dependencies ?? []) { + switch (dependency) { + case 'this': + case 'thisProperty': + continue; + + default: { + if (dependantsMap[dependency]) { + dependantsMap[dependency].push(property); + } else { + dependantsMap[dependency] = [property]; + } + } + } } + } - return dependencies; - }; - } else { - const dependencies = Object.create(null); - Object.freeze(dependencies); - getAllDependencies = () => dependencies; + Object.defineProperty(this.prototype, property, definition); } - if (flags.update) { - return () => transform(this.#propertyUpdateValues[property], getAllDependencies()); - } else { - return () => compute(getAllDependencies()); - } + this[CacheableObject.constructorFinalized] = true; } - #getExposeCheckCacheValidFunction(property) { - const {flags, expose} = this.#getPropertyDescriptor(property); - - let valid = false; - - const invalidate = () => { - valid = false; - }; - - const dependencyKeys = new Set(expose?.dependencies); - - if (flags.update) { - dependencyKeys.add(property); - } - - for (const key of dependencyKeys) { - if (this.#propertyUpdateCacheInvalidators[key]) { - this.#propertyUpdateCacheInvalidators[key].push(invalidate); - } else { - this.#propertyUpdateCacheInvalidators[key] = [invalidate]; - } - } + static getPropertyDescriptor(property) { + return this[CacheableObject.propertyDescriptors][property]; + } - return () => { - if (!valid) { - valid = true; - return false; - } else { - return true; - } - }; + static hasPropertyDescriptor(property) { + return Object.hasOwn(this[CacheableObject.propertyDescriptors], property); } static cacheAllExposedProperties(obj) { @@ -349,30 +210,12 @@ export default class CacheableObject { } } - static DEBUG_SLOW_TRACK_INVALID_PROPERTIES = false; - static _invalidAccesses = new Set(); - - static showInvalidAccesses() { - if (!this.DEBUG_SLOW_TRACK_INVALID_PROPERTIES) { - return; - } - - if (!this._invalidAccesses.size) { - return; - } - - console.log(`${this._invalidAccesses.size} unique invalid accesses:`); - for (const line of this._invalidAccesses) { - console.log(` - ${line}`); - } - } - static getUpdateValue(object, key) { - if (!Object.hasOwn(object, key)) { + if (!object.constructor.hasPropertyDescriptor(key)) { return undefined; } - return object.#propertyUpdateValues[key] ?? null; + return object[CacheableObject.updateValue][key] ?? null; } static clone(object) { @@ -384,7 +227,7 @@ export default class CacheableObject { } static copyUpdateValuesOnto(source, target) { - Object.assign(target, source.#propertyUpdateValues); + Object.assign(target, source[CacheableObject.updateValue]); } } @@ -392,8 +235,22 @@ export class CacheableObjectPropertyValueError extends Error { [Symbol.for('hsmusic.aggregate.translucent')] = true; constructor(property, oldValue, newValue, options) { + let inspectOldValue, inspectNewValue; + + try { + inspectOldValue = inspect(oldValue); + } catch (error) { + inspectOldValue = colors.red(`(couldn't inspect)`); + } + + try { + inspectNewValue = inspect(newValue); + } catch (error) { + inspectNewValue = colors.red(`(couldn't inspect)`); + } + super( - `Error setting ${colors.green(property)} (${inspect(oldValue)} -> ${inspect(newValue)})`, + `Error setting ${colors.green(property)} (${inspectOldValue} -> ${inspectNewValue})`, options); this.property = property; diff --git a/src/data/composite/data/withMappedList.js b/src/data/composite/data/withMappedList.js index 0bc63a92..cd32058e 100644 --- a/src/data/composite/data/withMappedList.js +++ b/src/data/composite/data/withMappedList.js @@ -1,12 +1,16 @@ // Applies a map function to each item in a list, just like a normal JavaScript // map. // +// Pass a filter (e.g. from withAvailabilityFilter) to process only items +// kept by the filter. Other items will be left as-is. +// // See also: // - withFilteredList // - withSortedList // import {input, templateCompositeFrom} from '#composite'; +import {stitchArrays} from '#sugar'; export default templateCompositeFrom({ annotation: `withMappedList`, @@ -14,19 +18,31 @@ export default templateCompositeFrom({ inputs: { list: input({type: 'array'}), map: input({type: 'function'}), + + filter: input({ + type: 'array', + defaultValue: null, + }), }, outputs: ['#mappedList'], steps: () => [ { - dependencies: [input('list'), input('map')], + dependencies: [input('list'), input('map'), input('filter')], compute: (continuation, { [input('list')]: list, [input('map')]: mapFn, + [input('filter')]: filter, }) => continuation({ ['#mappedList']: - list.map(mapFn), + stitchArrays({ + item: list, + keep: filter ?? Array.from(list, () => true), + }).map(({item, keep}, index) => + (keep + ? mapFn(item, index, list) + : item)), }), }, ], diff --git a/src/data/composite/things/album/withTracks.js b/src/data/composite/things/album/withTracks.js index 348220e7..835ee570 100644 --- a/src/data/composite/things/album/withTracks.js +++ b/src/data/composite/things/album/withTracks.js @@ -1,7 +1,6 @@ import {input, templateCompositeFrom} from '#composite'; import {withFlattenedList, withPropertyFromList} from '#composite/data'; -import {withResolvedReferenceList} from '#composite/wiki-data'; import {raiseOutputWithoutDependency} from '#composite/control-flow'; diff --git a/src/data/composite/things/artist/artistTotalDuration.js b/src/data/composite/things/artist/artistTotalDuration.js index ff709f28..a4a33542 100644 --- a/src/data/composite/things/artist/artistTotalDuration.js +++ b/src/data/composite/things/artist/artistTotalDuration.js @@ -2,8 +2,9 @@ import {input, templateCompositeFrom} from '#composite'; import {exposeDependency} from '#composite/control-flow'; import {withFilteredList, withPropertyFromList} from '#composite/data'; -import {withContributionListSums, withReverseContributionList} +import {withContributionListSums, withReverseReferenceList} from '#composite/wiki-data'; +import {soupyReverse} from '#composite/wiki-properties'; export default templateCompositeFrom({ annotation: `artistTotalDuration`, @@ -11,18 +12,16 @@ export default templateCompositeFrom({ compose: false, steps: () => [ - withReverseContributionList({ - data: 'trackData', - list: input.value('artistContribs'), + withReverseReferenceList({ + reverse: soupyReverse.input('trackArtistContributionsBy'), }).outputs({ - '#reverseContributionList': '#contributionsAsArtist', + '#reverseReferenceList': '#contributionsAsArtist', }), - withReverseContributionList({ - data: 'trackData', - list: input.value('contributorContribs'), + withReverseReferenceList({ + reverse: soupyReverse.input('trackContributorContributionsBy'), }).outputs({ - '#reverseContributionList': '#contributionsAsContributor', + '#reverseReferenceList': '#contributionsAsContributor', }), { diff --git a/src/data/composite/things/contribution/inheritFromContributionPresets.js b/src/data/composite/things/contribution/inheritFromContributionPresets.js index 82425b9c..a74e6db3 100644 --- a/src/data/composite/things/contribution/inheritFromContributionPresets.js +++ b/src/data/composite/things/contribution/inheritFromContributionPresets.js @@ -1,7 +1,7 @@ import {input, templateCompositeFrom} from '#composite'; import {raiseOutputWithoutDependency} from '#composite/control-flow'; -import {withPropertyFromList, withPropertyFromObject} from '#composite/data'; +import {withPropertyFromList} from '#composite/data'; import withMatchingContributionPresets from './withMatchingContributionPresets.js'; diff --git a/src/data/composite/things/contribution/withContributionArtist.js b/src/data/composite/things/contribution/withContributionArtist.js index 5a611c1a..5f81c716 100644 --- a/src/data/composite/things/contribution/withContributionArtist.js +++ b/src/data/composite/things/contribution/withContributionArtist.js @@ -1,8 +1,7 @@ import {input, templateCompositeFrom} from '#composite'; -import find from '#find'; -import {withPropertyFromObject} from '#composite/data'; import {withResolvedReference} from '#composite/wiki-data'; +import {soupyFind} from '#composite/wiki-properties'; export default templateCompositeFrom({ annotation: `withContributionArtist`, @@ -17,16 +16,9 @@ export default templateCompositeFrom({ outputs: ['#artist'], steps: () => [ - withPropertyFromObject({ - object: 'thing', - property: input.value('artistData'), - internal: input.value(true), - }), - withResolvedReference({ ref: input('ref'), - data: '#thing.artistData', - find: input.value(find.artist), + find: soupyFind.input('artist'), }).outputs({ '#resolvedReference': '#artist', }), diff --git a/src/data/composite/things/flash-act/withFlashSide.js b/src/data/composite/things/flash-act/withFlashSide.js index 64daa1fb..e09f06e6 100644 --- a/src/data/composite/things/flash-act/withFlashSide.js +++ b/src/data/composite/things/flash-act/withFlashSide.js @@ -2,9 +2,10 @@ // If there's no side whose list of flash acts includes this act, the output // dependency will be null. -import {input, templateCompositeFrom} from '#composite'; +import {templateCompositeFrom} from '#composite'; import {withUniqueReferencingThing} from '#composite/wiki-data'; +import {soupyReverse} from '#composite/wiki-properties'; export default templateCompositeFrom({ annotation: `withFlashSide`, @@ -13,8 +14,7 @@ export default templateCompositeFrom({ steps: () => [ withUniqueReferencingThing({ - data: 'flashSideData', - list: input.value('acts'), + reverse: soupyReverse.input('flashSidesWhoseActsInclude'), }).outputs({ ['#uniqueReferencingThing']: '#flashSide', }), diff --git a/src/data/composite/things/flash/withFlashAct.js b/src/data/composite/things/flash/withFlashAct.js index 652b8bfb..87922aff 100644 --- a/src/data/composite/things/flash/withFlashAct.js +++ b/src/data/composite/things/flash/withFlashAct.js @@ -2,9 +2,10 @@ // If there's no flash whose list of flashes includes this flash, the output // dependency will be null. -import {input, templateCompositeFrom} from '#composite'; +import {templateCompositeFrom} from '#composite'; import {withUniqueReferencingThing} from '#composite/wiki-data'; +import {soupyReverse} from '#composite/wiki-properties'; export default templateCompositeFrom({ annotation: `withFlashAct`, @@ -13,8 +14,7 @@ export default templateCompositeFrom({ steps: () => [ withUniqueReferencingThing({ - data: 'flashActData', - list: input.value('flashes'), + reverse: soupyReverse.input('flashActsWhoseFlashesInclude'), }).outputs({ ['#uniqueReferencingThing']: '#flashAct', }), diff --git a/src/data/composite/things/track-section/withAlbum.js b/src/data/composite/things/track-section/withAlbum.js index a4dfff0d..e257062e 100644 --- a/src/data/composite/things/track-section/withAlbum.js +++ b/src/data/composite/things/track-section/withAlbum.js @@ -1,8 +1,9 @@ // Gets the track section's album. -import {input, templateCompositeFrom} from '#composite'; +import {templateCompositeFrom} from '#composite'; import {withUniqueReferencingThing} from '#composite/wiki-data'; +import {soupyReverse} from '#composite/wiki-properties'; export default templateCompositeFrom({ annotation: `withAlbum`, @@ -11,8 +12,7 @@ export default templateCompositeFrom({ steps: () => [ withUniqueReferencingThing({ - data: 'albumData', - list: input.value('trackSections'), + reverse: soupyReverse.input('albumsWhoseTrackSectionsInclude'), }).outputs({ ['#uniqueReferencingThing']: '#album', }), diff --git a/src/data/composite/things/track/index.js b/src/data/composite/things/track/index.js index 05ccaaba..32c72f78 100644 --- a/src/data/composite/things/track/index.js +++ b/src/data/composite/things/track/index.js @@ -1,7 +1,6 @@ export {default as exitWithoutUniqueCoverArt} from './exitWithoutUniqueCoverArt.js'; export {default as inheritContributionListFromOriginalRelease} from './inheritContributionListFromOriginalRelease.js'; export {default as inheritFromOriginalRelease} from './inheritFromOriginalRelease.js'; -export {default as trackReverseReferenceList} from './trackReverseReferenceList.js'; export {default as withAlbum} from './withAlbum.js'; export {default as withAlwaysReferenceByDirectory} from './withAlwaysReferenceByDirectory.js'; export {default as withContainingTrackSection} from './withContainingTrackSection.js'; diff --git a/src/data/composite/things/track/trackReverseReferenceList.js b/src/data/composite/things/track/trackReverseReferenceList.js deleted file mode 100644 index 44940ae7..00000000 --- a/src/data/composite/things/track/trackReverseReferenceList.js +++ /dev/null @@ -1,38 +0,0 @@ -// Like a normal reverse reference list ("objects which reference this object -// under a specified property"), only excluding rereleases from the possible -// outputs. While it's useful to travel from a rerelease to the tracks it -// references, rereleases aren't generally relevant from the perspective of -// the tracks *being* referenced. Apart from hiding rereleases from lists on -// the site, it also excludes keeps them from relational data processing, such -// as on the "Tracks - by Times Referenced" listing page. - -import {input, templateCompositeFrom} from '#composite'; -import {withReverseReferenceList} from '#composite/wiki-data'; - -export default templateCompositeFrom({ - annotation: `trackReverseReferenceList`, - - compose: false, - - inputs: { - list: input({type: 'string'}), - }, - - steps: () => [ - withReverseReferenceList({ - data: 'trackData', - list: input('list'), - }), - - { - flags: {expose: true}, - expose: { - dependencies: ['#reverseReferenceList'], - compute: ({ - ['#reverseReferenceList']: reverseReferenceList, - }) => - reverseReferenceList.filter(track => !track.originalReleaseTrack), - }, - }, - ], -}); diff --git a/src/data/composite/things/track/withAlbum.js b/src/data/composite/things/track/withAlbum.js index 03b840d4..4c55e1f4 100644 --- a/src/data/composite/things/track/withAlbum.js +++ b/src/data/composite/things/track/withAlbum.js @@ -2,9 +2,10 @@ // If there's no album whose list of tracks includes this track, the output // dependency will be null. -import {input, templateCompositeFrom} from '#composite'; +import {templateCompositeFrom} from '#composite'; import {withUniqueReferencingThing} from '#composite/wiki-data'; +import {soupyReverse} from '#composite/wiki-properties'; export default templateCompositeFrom({ annotation: `withAlbum`, @@ -13,8 +14,7 @@ export default templateCompositeFrom({ steps: () => [ withUniqueReferencingThing({ - data: 'albumData', - list: input.value('tracks'), + reverse: soupyReverse.input('albumsWhoseTracksInclude'), }).outputs({ ['#uniqueReferencingThing']: '#album', }), diff --git a/src/data/composite/things/track/withAlwaysReferenceByDirectory.js b/src/data/composite/things/track/withAlwaysReferenceByDirectory.js index e01720b4..26c5ba97 100644 --- a/src/data/composite/things/track/withAlwaysReferenceByDirectory.js +++ b/src/data/composite/things/track/withAlwaysReferenceByDirectory.js @@ -9,6 +9,7 @@ import {isBoolean} from '#validators'; import {withPropertyFromObject} from '#composite/data'; import {withResolvedReference} from '#composite/wiki-data'; +import {soupyFind} from '#composite/wiki-properties'; import { exitWithoutDependency, @@ -31,8 +32,7 @@ export default templateCompositeFrom({ // recurse back into alwaysReferenceByDirectory! withResolvedReference({ ref: 'dataSourceAlbum', - data: 'albumData', - find: input.value(find.album), + find: soupyFind.input('album'), }).outputs({ '#resolvedReference': '#album', }), diff --git a/src/data/composite/things/track/withContainingTrackSection.js b/src/data/composite/things/track/withContainingTrackSection.js index 9bbd9bd5..3d4d081e 100644 --- a/src/data/composite/things/track/withContainingTrackSection.js +++ b/src/data/composite/things/track/withContainingTrackSection.js @@ -1,8 +1,9 @@ // Gets the track section containing this track from its album's track list. -import {input, templateCompositeFrom} from '#composite'; +import {templateCompositeFrom} from '#composite'; import {withUniqueReferencingThing} from '#composite/wiki-data'; +import {soupyReverse} from '#composite/wiki-properties'; export default templateCompositeFrom({ annotation: `withContainingTrackSection`, @@ -11,8 +12,7 @@ export default templateCompositeFrom({ steps: () => [ withUniqueReferencingThing({ - data: 'trackSectionData', - list: input.value('tracks'), + reverse: soupyReverse.input('trackSectionsWhichInclude'), }).outputs({ ['#uniqueReferencingThing']: '#trackSection', }), diff --git a/src/data/composite/things/track/withOriginalRelease.js b/src/data/composite/things/track/withOriginalRelease.js index c7f49657..7aefc64a 100644 --- a/src/data/composite/things/track/withOriginalRelease.js +++ b/src/data/composite/things/track/withOriginalRelease.js @@ -5,24 +5,17 @@ // is specified by reference and that reference doesn't resolve to anything. import {input, templateCompositeFrom} from '#composite'; -import find from '#find'; -import {validateWikiData} from '#validators'; import {exitWithoutDependency, withResultOfAvailabilityCheck} from '#composite/control-flow'; import {withResolvedReference} from '#composite/wiki-data'; +import {soupyFind} from '#composite/wiki-properties'; export default templateCompositeFrom({ annotation: `withOriginalRelease`, inputs: { selfIfOriginal: input({type: 'boolean', defaultValue: false}), - - data: input({ - validate: validateWikiData({referenceType: 'track'}), - defaultDependency: 'trackData', - }), - notFoundValue: input({defaultValue: null}), }, @@ -55,8 +48,7 @@ export default templateCompositeFrom({ withResolvedReference({ ref: 'originalReleaseTrack', - data: input('data'), - find: input.value(find.track), + find: soupyFind.input('track'), }), exitWithoutDependency({ diff --git a/src/data/composite/things/track/withPropertyFromAlbum.js b/src/data/composite/things/track/withPropertyFromAlbum.js index d41390fa..e9c5b56e 100644 --- a/src/data/composite/things/track/withPropertyFromAlbum.js +++ b/src/data/composite/things/track/withPropertyFromAlbum.js @@ -2,7 +2,6 @@ // property name prefixed with '#album.' (by default). import {input, templateCompositeFrom} from '#composite'; -import {is} from '#validators'; import {withPropertyFromObject} from '#composite/data'; diff --git a/src/data/composite/wiki-data/gobbleSoupyFind.js b/src/data/composite/wiki-data/gobbleSoupyFind.js new file mode 100644 index 00000000..aec3f5b1 --- /dev/null +++ b/src/data/composite/wiki-data/gobbleSoupyFind.js @@ -0,0 +1,39 @@ +import {input, templateCompositeFrom} from '#composite'; + +import {withPropertyFromObject} from '#composite/data'; + +import inputSoupyFind, {getSoupyFindInputKey} from './inputSoupyFind.js'; + +export default templateCompositeFrom({ + annotation: `gobbleSoupyFind`, + + inputs: { + find: inputSoupyFind(), + }, + + outputs: ['#find'], + + steps: () => [ + { + dependencies: [input('find')], + compute: (continuation, { + [input('find')]: find, + }) => + (typeof find === 'function' + ? continuation.raiseOutput({ + ['#find']: find, + }) + : continuation({ + ['#key']: + getSoupyFindInputKey(find), + })), + }, + + withPropertyFromObject({ + object: 'find', + property: '#key', + }).outputs({ + '#value': '#find', + }), + ], +}); diff --git a/src/data/composite/wiki-data/gobbleSoupyReverse.js b/src/data/composite/wiki-data/gobbleSoupyReverse.js new file mode 100644 index 00000000..86a1061c --- /dev/null +++ b/src/data/composite/wiki-data/gobbleSoupyReverse.js @@ -0,0 +1,39 @@ +import {input, templateCompositeFrom} from '#composite'; + +import {withPropertyFromObject} from '#composite/data'; + +import inputSoupyReverse, {getSoupyReverseInputKey} from './inputSoupyReverse.js'; + +export default templateCompositeFrom({ + annotation: `gobbleSoupyReverse`, + + inputs: { + reverse: inputSoupyReverse(), + }, + + outputs: ['#reverse'], + + steps: () => [ + { + dependencies: [input('reverse')], + compute: (continuation, { + [input('reverse')]: reverse, + }) => + (typeof reverse === 'function' + ? continuation.raiseOutput({ + ['#reverse']: reverse, + }) + : continuation({ + ['#key']: + getSoupyReverseInputKey(reverse), + })), + }, + + withPropertyFromObject({ + object: 'reverse', + property: '#key', + }).outputs({ + '#value': '#reverse', + }), + ], +}); diff --git a/src/data/composite/wiki-data/helpers/withResolvedReverse.js b/src/data/composite/wiki-data/helpers/withResolvedReverse.js new file mode 100644 index 00000000..818f60b7 --- /dev/null +++ b/src/data/composite/wiki-data/helpers/withResolvedReverse.js @@ -0,0 +1,40 @@ +// Actually execute a reverse function. + +import {input, templateCompositeFrom} from '#composite'; + +import inputWikiData from '../inputWikiData.js'; + +export default templateCompositeFrom({ + annotation: `withReverseReferenceList`, + + inputs: { + data: inputWikiData({allowMixedTypes: true}), + reverse: input({type: 'function'}), + options: input({type: 'object', defaultValue: null}), + }, + + outputs: ['#resolvedReverse'], + + steps: () => [ + { + dependencies: [ + input.myself(), + input('data'), + input('reverse'), + input('options'), + ], + + compute: (continuation, { + [input.myself()]: myself, + [input('data')]: data, + [input('reverse')]: reverseFunction, + [input('options')]: opts, + }) => continuation({ + ['#resolvedReverse']: + (data + ? reverseFunction(myself, data, opts) + : reverseFunction(myself, opts)), + }), + }, + ], +}); diff --git a/src/data/composite/wiki-data/helpers/withReverseList-template.js b/src/data/composite/wiki-data/helpers/withReverseList-template.js deleted file mode 100644 index 6ffd5d70..00000000 --- a/src/data/composite/wiki-data/helpers/withReverseList-template.js +++ /dev/null @@ -1,193 +0,0 @@ -// Baseline implementation shared by or underlying reverse lists. -// -// This is a very rudimentary "these compositions have basically the same -// shape but slightly different guts midway through" kind of solution, -// and should use compositional subroutines instead, once those are ready. -// -// But, until then, this has the same effect of avoiding code duplication -// and clearly identifying differences. -// -// --- -// -// This implementation uses a global cache (via WeakMap) to attempt to speed -// up subsequent similar accesses. -// -// This has absolutely not been rigorously tested with altering properties of -// data objects in a wiki data array which is reused. If a new wiki data array -// is used, a fresh cache will always be created. -// - -import {input, templateCompositeFrom} from '#composite'; -import {sortByDate} from '#sort'; -import {stitchArrays} from '#sugar'; - -import {exitWithoutDependency, raiseOutputWithoutDependency} - from '#composite/control-flow'; -import {withFlattenedList, withMappedList} from '#composite/data'; - -import inputWikiData from '../inputWikiData.js'; - -export default function withReverseList_template({ - annotation, - - propertyInputName, - outputName, - - additionalInputs = {}, - - customCompositionSteps, -}) { - // Mapping of reference list property to WeakMap. - // Each WeakMap maps a wiki data array to another weak map, - // which in turn maps each referenced thing to an array of - // things referencing it. - const caches = new Map(); - - return templateCompositeFrom({ - annotation, - - inputs: { - data: inputWikiData({ - allowMixedTypes: true, - }), - - [propertyInputName]: input({ - type: 'string', - }), - - ...additionalInputs, - }, - - outputs: [outputName], - - steps: () => [ - // Early exit with an empty array if the data list isn't available. - exitWithoutDependency({ - dependency: input('data'), - value: input.value([]), - }), - - // Raise an empty array (don't early exit) if the data list is empty. - raiseOutputWithoutDependency({ - dependency: input('data'), - mode: input.value('empty'), - output: input.value({[outputName]: []}), - }), - - // Check for an existing cache record which corresponds to this - // property input and input('data'). If it exists, query it for the - // current thing, and raise that; if it doesn't, create it, put it - // where it needs to be, and provide it so the next steps can fill - // it in. - { - dependencies: [input(propertyInputName), input('data'), input.myself()], - - compute: (continuation, { - [input(propertyInputName)]: property, - [input('data')]: data, - [input.myself()]: myself, - }) => { - if (!caches.has(property)) { - const cache = new WeakMap(); - caches.set(property, cache); - - const cacheRecord = new WeakMap(); - cache.set(data, cacheRecord); - - return continuation({ - ['#cacheRecord']: cacheRecord, - }); - } - - const cache = caches.get(property); - - if (!cache.has(data)) { - const cacheRecord = new WeakMap(); - cache.set(data, cacheRecord); - - return continuation({ - ['#cacheRecord']: cacheRecord, - }); - } - - return continuation.raiseOutput({ - [outputName]: - cache.get(data).get(myself) ?? [], - }); - }, - }, - - ...customCompositionSteps(), - - // Actually fill in the cache record. Since we're building up a *reverse* - // reference list, track connections in terms of the referenced thing. - // Although we gather all referenced things into a set and provide that - // for sorting purposes in the next step, we *don't* reprovide the cache - // record, because we're mutating that in-place - we'll just reuse its - // existing '#cacheRecord' dependency. - { - dependencies: ['#cacheRecord', '#referencingThings', '#referencedThings'], - compute: (continuation, { - ['#cacheRecord']: cacheRecord, - ['#referencingThings']: referencingThings, - ['#referencedThings']: referencedThings, - }) => { - const allReferencedThings = new Set(); - - stitchArrays({ - referencingThing: referencingThings, - referencedThings: referencedThings, - }).forEach(({referencingThing, referencedThings}) => { - for (const referencedThing of referencedThings) { - if (cacheRecord.has(referencedThing)) { - cacheRecord.get(referencedThing).push(referencingThing); - } else { - cacheRecord.set(referencedThing, [referencingThing]); - allReferencedThings.add(referencedThing); - } - } - }); - - return continuation({ - ['#allReferencedThings']: - allReferencedThings, - }); - }, - }, - - // Sort the entries in the cache records, too, just by date - the rest of - // sorting should be handled outside of this composition, either preceding - // (changing the 'data' input) or following (sorting the output). - // Again we're mutating in place, so no need to reprovide '#cacheRecord' - // here. - { - dependencies: ['#cacheRecord', '#allReferencedThings'], - compute: (continuation, { - ['#cacheRecord']: cacheRecord, - ['#allReferencedThings']: allReferencedThings, - }) => { - for (const referencedThing of allReferencedThings) { - if (cacheRecord.has(referencedThing)) { - const referencingThings = cacheRecord.get(referencedThing); - sortByDate(referencingThings); - } - } - - return continuation(); - }, - }, - - // Then just pluck out the current object from the now-filled cache record! - { - dependencies: ['#cacheRecord', input.myself()], - compute: (continuation, { - ['#cacheRecord']: cacheRecord, - [input.myself()]: myself, - }) => continuation({ - [outputName]: - cacheRecord.get(myself) ?? [], - }), - }, - ], - }); -} diff --git a/src/data/composite/wiki-data/index.js b/src/data/composite/wiki-data/index.js index 51d07384..be83e4c9 100644 --- a/src/data/composite/wiki-data/index.js +++ b/src/data/composite/wiki-data/index.js @@ -5,7 +5,11 @@ // export {default as exitWithoutContribs} from './exitWithoutContribs.js'; +export {default as gobbleSoupyFind} from './gobbleSoupyFind.js'; +export {default as gobbleSoupyReverse} from './gobbleSoupyReverse.js'; export {default as inputNotFoundMode} from './inputNotFoundMode.js'; +export {default as inputSoupyFind} from './inputSoupyFind.js'; +export {default as inputSoupyReverse} from './inputSoupyReverse.js'; export {default as inputWikiData} from './inputWikiData.js'; export {default as withClonedThings} from './withClonedThings.js'; export {default as withContributionListSums} from './withContributionListSums.js'; @@ -19,9 +23,6 @@ export {default as withResolvedContribs} from './withResolvedContribs.js'; export {default as withResolvedReference} from './withResolvedReference.js'; export {default as withResolvedReferenceList} from './withResolvedReferenceList.js'; export {default as withResolvedSeriesList} from './withResolvedSeriesList.js'; -export {default as withReverseAnnotatedReferenceList} from './withReverseAnnotatedReferenceList.js'; -export {default as withReverseContributionList} from './withReverseContributionList.js'; export {default as withReverseReferenceList} from './withReverseReferenceList.js'; -export {default as withReverseSingleReferenceList} from './withReverseSingleReferenceList.js'; export {default as withThingsSortedAlphabetically} from './withThingsSortedAlphabetically.js'; export {default as withUniqueReferencingThing} from './withUniqueReferencingThing.js'; diff --git a/src/data/composite/wiki-data/inputSoupyFind.js b/src/data/composite/wiki-data/inputSoupyFind.js new file mode 100644 index 00000000..020f4990 --- /dev/null +++ b/src/data/composite/wiki-data/inputSoupyFind.js @@ -0,0 +1,28 @@ +import {input} from '#composite'; +import {anyOf, isFunction, isString} from '#validators'; + +function inputSoupyFind() { + return input({ + validate: + anyOf( + isFunction, + val => { + isString(val); + + if (!val.startsWith('_soupyFind:')) { + throw new Error(`Expected soupyFind.input() token`); + } + + return true; + }), + }); +} + +inputSoupyFind.input = key => + input.value('_soupyFind:' + key); + +export default inputSoupyFind; + +export function getSoupyFindInputKey(value) { + return value.slice('_soupyFind:'.length); +} diff --git a/src/data/composite/wiki-data/inputSoupyReverse.js b/src/data/composite/wiki-data/inputSoupyReverse.js new file mode 100644 index 00000000..0b0a23fe --- /dev/null +++ b/src/data/composite/wiki-data/inputSoupyReverse.js @@ -0,0 +1,32 @@ +import {input} from '#composite'; +import {anyOf, isFunction, isString} from '#validators'; + +function inputSoupyReverse() { + return input({ + validate: + anyOf( + isFunction, + val => { + isString(val); + + if (!val.startsWith('_soupyReverse:')) { + throw new Error(`Expected soupyReverse.input() token`); + } + + return true; + }), + }); +} + +inputSoupyReverse.input = key => + input.value('_soupyReverse:' + key); + +export default inputSoupyReverse; + +export function getSoupyReverseInputKey(value) { + return value.slice('_soupyReverse:'.length).replace(/\.unique$/, ''); +} + +export function doesSoupyReverseInputWantUnique(value) { + return value.endsWith('.unique'); +} diff --git a/src/data/composite/wiki-data/inputWikiData.js b/src/data/composite/wiki-data/inputWikiData.js index cf7a7c2c..b9021986 100644 --- a/src/data/composite/wiki-data/inputWikiData.js +++ b/src/data/composite/wiki-data/inputWikiData.js @@ -12,6 +12,6 @@ export default function inputWikiData({ } = {}) { return input({ validate: validateWikiData({referenceType, allowMixedTypes}), - acceptsNull: true, + defaultValue: null, }); } diff --git a/src/data/composite/wiki-data/withParsedCommentaryEntries.js b/src/data/composite/wiki-data/withParsedCommentaryEntries.js index 144781a8..9bf4278c 100644 --- a/src/data/composite/wiki-data/withParsedCommentaryEntries.js +++ b/src/data/composite/wiki-data/withParsedCommentaryEntries.js @@ -1,5 +1,4 @@ import {input, templateCompositeFrom} from '#composite'; -import find from '#find'; import {stitchArrays} from '#sugar'; import {isCommentary} from '#validators'; import {commentaryRegexCaseSensitive} from '#wiki-data'; @@ -11,6 +10,7 @@ import { withUnflattenedList, } from '#composite/data'; +import inputSoupyFind from './inputSoupyFind.js'; import withResolvedReferenceList from './withResolvedReferenceList.js'; export default templateCompositeFrom({ @@ -122,8 +122,7 @@ export default templateCompositeFrom({ withResolvedReferenceList({ list: '#flattenedList', - data: 'artistData', - find: input.value(find.artist), + find: inputSoupyFind.input('artist'), notFoundMode: input.value('null'), }), diff --git a/src/data/composite/wiki-data/withRecontextualizedContributionList.js b/src/data/composite/wiki-data/withRecontextualizedContributionList.js index d2401eac..bcc6e486 100644 --- a/src/data/composite/wiki-data/withRecontextualizedContributionList.js +++ b/src/data/composite/wiki-data/withRecontextualizedContributionList.js @@ -10,7 +10,6 @@ import {input, templateCompositeFrom} from '#composite'; import {isStringNonEmpty} from '#validators'; -import {raiseOutputWithoutDependency} from '#composite/control-flow'; import {withClonedThings} from '#composite/wiki-data'; export default templateCompositeFrom({ diff --git a/src/data/composite/wiki-data/withResolvedAnnotatedReferenceList.js b/src/data/composite/wiki-data/withResolvedAnnotatedReferenceList.js index 789a8844..c9a7c058 100644 --- a/src/data/composite/wiki-data/withResolvedAnnotatedReferenceList.js +++ b/src/data/composite/wiki-data/withResolvedAnnotatedReferenceList.js @@ -4,12 +4,10 @@ import {isDate, isObject, validateArrayItems} from '#validators'; import {withPropertyFromList} from '#composite/data'; -import { - exitWithoutDependency, - raiseOutputWithoutDependency, - withAvailabilityFilter, -} from '#composite/control-flow'; +import {raiseOutputWithoutDependency, withAvailabilityFilter} + from '#composite/control-flow'; +import inputSoupyFind from './inputSoupyFind.js'; import inputNotFoundMode from './inputNotFoundMode.js'; import inputWikiData from './inputWikiData.js'; import raiseResolvedReferenceList from './raiseResolvedReferenceList.js'; @@ -34,7 +32,7 @@ export default templateCompositeFrom({ thing: input({type: 'string', defaultValue: 'thing'}), data: inputWikiData({allowMixedTypes: true}), - find: input({type: 'function'}), + find: inputSoupyFind(), notFoundMode: inputNotFoundMode(), }, @@ -42,11 +40,6 @@ export default templateCompositeFrom({ outputs: ['#resolvedAnnotatedReferenceList'], steps: () => [ - exitWithoutDependency({ - dependency: input('data'), - value: input.value([]), - }), - raiseOutputWithoutDependency({ dependency: input('list'), mode: input.value('empty'), diff --git a/src/data/composite/wiki-data/withResolvedContribs.js b/src/data/composite/wiki-data/withResolvedContribs.js index fd3d8a0d..838c991f 100644 --- a/src/data/composite/wiki-data/withResolvedContribs.js +++ b/src/data/composite/wiki-data/withResolvedContribs.js @@ -110,6 +110,7 @@ export default templateCompositeFrom({ '#thingProperty', input('artistProperty'), input.myself(), + 'find', ], compute: (continuation, { @@ -117,6 +118,7 @@ export default templateCompositeFrom({ ['#thingProperty']: thingProperty, [input('artistProperty')]: artistProperty, [input.myself()]: myself, + ['find']: find, }) => continuation({ ['#contributions']: details.map(details => { @@ -127,6 +129,7 @@ export default templateCompositeFrom({ thing: myself, thingProperty: thingProperty, artistProperty: artistProperty, + find: find, }); return contrib; diff --git a/src/data/composite/wiki-data/withResolvedReference.js b/src/data/composite/wiki-data/withResolvedReference.js index ea71707e..6f422194 100644 --- a/src/data/composite/wiki-data/withResolvedReference.js +++ b/src/data/composite/wiki-data/withResolvedReference.js @@ -1,16 +1,14 @@ // Resolves a reference by using the provided find function to match it -// within the provided thingData dependency. This will early exit if the -// data dependency is null. Otherwise, the data object is provided on the -// output dependency, or null, if the reference doesn't match anything or +// within the provided thingData dependency. The data object is provided on +// the output dependency, or null, if the reference doesn't match anything or // itself was null to begin with. import {input, templateCompositeFrom} from '#composite'; -import { - exitWithoutDependency, - raiseOutputWithoutDependency, -} from '#composite/control-flow'; +import {raiseOutputWithoutDependency} from '#composite/control-flow'; +import gobbleSoupyFind from './gobbleSoupyFind.js'; +import inputSoupyFind from './inputSoupyFind.js'; import inputWikiData from './inputWikiData.js'; export default templateCompositeFrom({ @@ -20,7 +18,7 @@ export default templateCompositeFrom({ ref: input({type: 'string', acceptsNull: true}), data: inputWikiData({allowMixedTypes: false}), - find: input({type: 'function'}), + find: inputSoupyFind(), }, outputs: ['#resolvedReference'], @@ -33,24 +31,26 @@ export default templateCompositeFrom({ }), }), - exitWithoutDependency({ - dependency: input('data'), + gobbleSoupyFind({ + find: input('find'), }), { dependencies: [ input('ref'), input('data'), - input('find'), + '#find', ], compute: (continuation, { [input('ref')]: ref, [input('data')]: data, - [input('find')]: findFunction, + ['#find']: findFunction, }) => continuation({ ['#resolvedReference']: - findFunction(ref, data, {mode: 'quiet'}) ?? null, + (data + ? findFunction(ref, data, {mode: 'quiet'}) ?? null + : findFunction(ref, {mode: 'quiet'}) ?? null), }), }, ], diff --git a/src/data/composite/wiki-data/withResolvedReferenceList.js b/src/data/composite/wiki-data/withResolvedReferenceList.js index 790a962f..9dc960dd 100644 --- a/src/data/composite/wiki-data/withResolvedReferenceList.js +++ b/src/data/composite/wiki-data/withResolvedReferenceList.js @@ -1,19 +1,18 @@ // Resolves a list of references, with each reference matched with provided -// data in the same way as withResolvedReference. This will early exit if the -// data dependency is null (even if the reference list is empty). By default -// it will filter out references which don't match, but this can be changed -// to early exit ({notFoundMode: 'exit'}) or leave null in place ('null'). +// data in the same way as withResolvedReference. By default it will filter +// out references which don't match, but this can be changed to early exit +// ({notFoundMode: 'exit'}) or leave null in place ('null'). import {input, templateCompositeFrom} from '#composite'; import {isString, validateArrayItems} from '#validators'; -import { - exitWithoutDependency, - raiseOutputWithoutDependency, - withAvailabilityFilter, -} from '#composite/control-flow'; +import {raiseOutputWithoutDependency, withAvailabilityFilter} + from '#composite/control-flow'; +import {withMappedList} from '#composite/data'; +import gobbleSoupyFind from './gobbleSoupyFind.js'; import inputNotFoundMode from './inputNotFoundMode.js'; +import inputSoupyFind from './inputSoupyFind.js'; import inputWikiData from './inputWikiData.js'; import raiseResolvedReferenceList from './raiseResolvedReferenceList.js'; @@ -27,7 +26,7 @@ export default templateCompositeFrom({ }), data: inputWikiData({allowMixedTypes: true}), - find: input({type: 'function'}), + find: inputSoupyFind(), notFoundMode: inputNotFoundMode(), }, @@ -35,11 +34,6 @@ export default templateCompositeFrom({ outputs: ['#resolvedReferenceList'], steps: () => [ - exitWithoutDependency({ - dependency: input('data'), - value: input.value([]), - }), - raiseOutputWithoutDependency({ dependency: input('list'), mode: input.value('empty'), @@ -48,18 +42,30 @@ export default templateCompositeFrom({ }), }), + gobbleSoupyFind({ + find: input('find'), + }), + { - dependencies: [input('list'), input('data'), input('find')], + dependencies: [input('data'), '#find'], compute: (continuation, { - [input('list')]: list, [input('data')]: data, - [input('find')]: findFunction, - }) => - continuation({ - '#matches': list.map(ref => findFunction(ref, data, {mode: 'quiet'})), - }), + ['#find']: findFunction, + }) => continuation({ + ['#map']: + (data + ? ref => findFunction(ref, data, {mode: 'quiet'}) + : ref => findFunction(ref, {mode: 'quiet'})), + }), }, + withMappedList({ + list: input('list'), + map: '#map', + }).outputs({ + '#mappedList': '#matches', + }), + withAvailabilityFilter({ from: '#matches', }), diff --git a/src/data/composite/wiki-data/withResolvedSeriesList.js b/src/data/composite/wiki-data/withResolvedSeriesList.js index 4ac74cc3..deaab466 100644 --- a/src/data/composite/wiki-data/withResolvedSeriesList.js +++ b/src/data/composite/wiki-data/withResolvedSeriesList.js @@ -1,5 +1,4 @@ import {input, templateCompositeFrom} from '#composite'; -import find from '#find'; import {stitchArrays} from '#sugar'; import {isSeriesList, validateThing} from '#validators'; @@ -12,6 +11,7 @@ import { withPropertiesFromList, } from '#composite/data'; +import inputSoupyFind from './inputSoupyFind.js'; import withResolvedReferenceList from './withResolvedReferenceList.js'; export default templateCompositeFrom({ @@ -62,8 +62,7 @@ export default templateCompositeFrom({ withResolvedReferenceList({ list: '#flattenedList', - data: 'albumData', - find: input.value(find.album), + find: inputSoupyFind.input('album'), notFoundMode: input.value('null'), }), diff --git a/src/data/composite/wiki-data/withReverseAnnotatedReferenceList.js b/src/data/composite/wiki-data/withReverseAnnotatedReferenceList.js deleted file mode 100644 index feae9ccb..00000000 --- a/src/data/composite/wiki-data/withReverseAnnotatedReferenceList.js +++ /dev/null @@ -1,116 +0,0 @@ -// Analogous implementation for withReverseReferenceList, for annotated -// references. -// -// Unlike withReverseContributionList, this composition is responsible for -// "flipping" the directionality of references: in a forward reference list, -// `thing` points to the thing being referenced, while here, it points to the -// referencing thing. -// -// This behavior can be customized to respect reference lists which are shaped -// differently than the default and/or to customize the reversed property and -// provide a less generic label than just "thing". - -import withReverseList_template from './helpers/withReverseList-template.js'; - -import {input} from '#composite'; -import {stitchArrays} from '#sugar'; - -import { - withFlattenedList, - withMappedList, - withPropertyFromList, - withStretchedList, -} from '#composite/data'; - -export default withReverseList_template({ - annotation: `withReverseAnnotatedReferenceList`, - - propertyInputName: 'list', - outputName: '#reverseAnnotatedReferenceList', - - additionalInputs: { - forward: input({type: 'string', defaultValue: 'thing'}), - backward: input({type: 'string', defaultValue: 'thing'}), - annotation: input({type: 'string', defaultValue: 'annotation'}), - }, - - customCompositionSteps: () => [ - withPropertyFromList({ - list: input('data'), - property: input('list'), - }).outputs({ - '#values': '#referenceLists', - }), - - withPropertyFromList({ - list: '#referenceLists', - property: input.value('length'), - }), - - withFlattenedList({ - list: '#referenceLists', - }).outputs({ - '#flattenedList': '#references', - }), - - withStretchedList({ - list: input('data'), - lengths: '#referenceLists.length', - }).outputs({ - '#stretchedList': '#things', - }), - - withPropertyFromList({ - list: '#references', - property: input('annotation'), - }).outputs({ - '#values': '#annotations', - }), - - withPropertyFromList({ - list: '#references', - property: input.value('date'), - }).outputs({ - '#references.date': '#dates', - }), - - { - dependencies: [ - input('backward'), - input('annotation'), - '#things', - '#annotations', - '#dates', - ], - - compute: (continuation, { - [input('backward')]: thingProperty, - [input('annotation')]: annotationProperty, - ['#things']: things, - ['#annotations']: annotations, - ['#dates']: dates, - }) => continuation({ - '#referencingThings': - stitchArrays({ - [thingProperty]: things, - [annotationProperty]: annotations, - date: dates, - }), - }), - }, - - withPropertyFromList({ - list: '#references', - property: input('forward'), - }).outputs({ - '#values': '#individualReferencedThings', - }), - - withMappedList({ - list: '#individualReferencedThings', - map: input.value(thing => [thing]), - }).outputs({ - '#mappedList': '#referencedThings', - }), - ], -}); diff --git a/src/data/composite/wiki-data/withReverseContributionList.js b/src/data/composite/wiki-data/withReverseContributionList.js deleted file mode 100644 index 2396c3b4..00000000 --- a/src/data/composite/wiki-data/withReverseContributionList.js +++ /dev/null @@ -1,37 +0,0 @@ -// Analogous implementation for withReverseReferenceList, for contributions. - -import withReverseList_template from './helpers/withReverseList-template.js'; - -import {input} from '#composite'; - -import {withFlattenedList, withMappedList, withPropertyFromList} - from '#composite/data'; - -export default withReverseList_template({ - annotation: `withReverseContributionList`, - - propertyInputName: 'list', - outputName: '#reverseContributionList', - - customCompositionSteps: () => [ - withPropertyFromList({ - list: input('data'), - property: input('list'), - }).outputs({ - '#values': '#contributionLists', - }), - - withFlattenedList({ - list: '#contributionLists', - }).outputs({ - '#flattenedList': '#referencingThings', - }), - - withMappedList({ - list: '#referencingThings', - map: input.value(contrib => [contrib.artist]), - }).outputs({ - '#mappedList': '#referencedThings', - }), - ], -}); diff --git a/src/data/composite/wiki-data/withReverseReferenceList.js b/src/data/composite/wiki-data/withReverseReferenceList.js index 70d9a58d..906f5bc5 100644 --- a/src/data/composite/wiki-data/withReverseReferenceList.js +++ b/src/data/composite/wiki-data/withReverseReferenceList.js @@ -1,44 +1,36 @@ // Check out the info on reverseReferenceList! // This is its composable form. -import withReverseList_template from './helpers/withReverseList-template.js'; +import {input, templateCompositeFrom} from '#composite'; -import {input} from '#composite'; +import gobbleSoupyReverse from './gobbleSoupyReverse.js'; +import inputSoupyReverse from './inputSoupyReverse.js'; +import inputWikiData from './inputWikiData.js'; -import {withMappedList} from '#composite/data'; +import withResolvedReverse from './helpers/withResolvedReverse.js'; -export default withReverseList_template({ +export default templateCompositeFrom({ annotation: `withReverseReferenceList`, - propertyInputName: 'list', - outputName: '#reverseReferenceList', - - customCompositionSteps: () => [ - { - dependencies: [input('list')], - compute: (continuation, { - [input('list')]: list, - }) => continuation({ - ['#referenceMap']: - thing => thing[list], - }), - }, - - withMappedList({ - list: input('data'), - map: '#referenceMap', - }).outputs({ - '#mappedList': '#referencedThings', + inputs: { + data: inputWikiData({allowMixedTypes: true}), + reverse: inputSoupyReverse(), + }, + + outputs: ['#reverseReferenceList'], + + steps: () => [ + gobbleSoupyReverse({ + reverse: input('reverse'), }), - { - dependencies: [input('data')], - compute: (continuation, { - [input('data')]: data, - }) => continuation({ - ['#referencingThings']: - data, - }), - }, + // TODO: Check that the reverse spec returns a list. + + withResolvedReverse({ + data: input('data'), + reverse: '#reverse', + }).outputs({ + '#resolvedReverse': '#reverseReferenceList', + }), ], }); diff --git a/src/data/composite/wiki-data/withReverseSingleReferenceList.js b/src/data/composite/wiki-data/withReverseSingleReferenceList.js deleted file mode 100644 index dd97dc66..00000000 --- a/src/data/composite/wiki-data/withReverseSingleReferenceList.js +++ /dev/null @@ -1,50 +0,0 @@ -// Like withReverseReferenceList, but for finding all things which reference -// the current thing by a property that contains a single reference, rather -// than within a reference list. - -import withReverseList_template from './helpers/withReverseList-template.js'; - -import {input} from '#composite'; - -import {withMappedList} from '#composite/data'; - -export default withReverseList_template({ - annotation: `withReverseSingleReferenceList`, - - propertyInputName: 'ref', - outputName: '#reverseSingleReferenceList', - - customCompositionSteps: () => [ - { - dependencies: [input('data')], - compute: (continuation, { - [input('data')]: data, - }) => continuation({ - ['#referencingThings']: - data, - }), - }, - - // This map wraps each referenced thing in a single-item array. - // Each referencing thing references exactly one thing, if any. - { - dependencies: [input('ref')], - compute: (continuation, { - [input('ref')]: ref, - }) => continuation({ - ['#singleReferenceMap']: - thing => - (thing[ref] - ? [thing[ref]] - : []), - }), - }, - - withMappedList({ - list: '#referencingThings', - map: '#singleReferenceMap', - }).outputs({ - '#mappedList': '#referencedThings', - }), - ], -}); diff --git a/src/data/composite/wiki-data/withUniqueReferencingThing.js b/src/data/composite/wiki-data/withUniqueReferencingThing.js index 61c10618..7c267038 100644 --- a/src/data/composite/wiki-data/withUniqueReferencingThing.js +++ b/src/data/composite/wiki-data/withUniqueReferencingThing.js @@ -4,48 +4,33 @@ import {input, templateCompositeFrom} from '#composite'; -import {exitWithoutDependency, raiseOutputWithoutDependency} - from '#composite/control-flow'; - +import gobbleSoupyReverse from './gobbleSoupyReverse.js'; +import inputSoupyReverse from './inputSoupyReverse.js'; import inputWikiData from './inputWikiData.js'; -import withReverseReferenceList from './withReverseReferenceList.js'; + +import withResolvedReverse from './helpers/withResolvedReverse.js'; export default templateCompositeFrom({ annotation: `withUniqueReferencingThing`, inputs: { - data: inputWikiData({allowMixedTypes: false}), - list: input({type: 'string'}), + data: inputWikiData({allowMixedTypes: true}), + reverse: inputSoupyReverse(), }, outputs: ['#uniqueReferencingThing'], steps: () => [ - // Early exit with null (not an empty array) if the data list - // isn't available. - exitWithoutDependency({ - dependency: input('data'), + gobbleSoupyReverse({ + reverse: input('reverse'), }), - withReverseReferenceList({ + withResolvedReverse({ data: input('data'), - list: input('list'), + reverse: '#reverse', + options: input.value({unique: true}), + }).outputs({ + '#resolvedReverse': '#uniqueReferencingThing', }), - - raiseOutputWithoutDependency({ - dependency: '#reverseReferenceList', - mode: input.value('empty'), - output: input.value({'#uniqueReferencingThing': null}), - }), - - { - dependencies: ['#reverseReferenceList'], - compute: (continuation, { - ['#reverseReferenceList']: reverseReferenceList, - }) => continuation({ - ['#uniqueReferencingThing']: - reverseReferenceList[0], - }), - }, ], }); diff --git a/src/data/composite/wiki-properties/annotatedReferenceList.js b/src/data/composite/wiki-properties/annotatedReferenceList.js index d6364475..bb6875f1 100644 --- a/src/data/composite/wiki-properties/annotatedReferenceList.js +++ b/src/data/composite/wiki-properties/annotatedReferenceList.js @@ -1,6 +1,4 @@ import {input, templateCompositeFrom} from '#composite'; -import find from '#find'; -import {combineWikiDataArrays} from '#wiki-data'; import { isContentString, @@ -12,7 +10,7 @@ import { } from '#validators'; import {exposeDependency} from '#composite/control-flow'; -import {inputWikiData, withResolvedAnnotatedReferenceList} +import {inputSoupyFind, inputWikiData, withResolvedAnnotatedReferenceList} from '#composite/wiki-data'; import {referenceListInputDescriptions, referenceListUpdateDescription} @@ -27,7 +25,7 @@ export default templateCompositeFrom({ ...referenceListInputDescriptions(), data: inputWikiData({allowMixedTypes: true}), - find: input({type: 'function'}), + find: inputSoupyFind(), date: input({ validate: isDate, diff --git a/src/data/composite/wiki-properties/index.js b/src/data/composite/wiki-properties/index.js index b55616c0..4aaaeb72 100644 --- a/src/data/composite/wiki-properties/index.js +++ b/src/data/composite/wiki-properties/index.js @@ -21,15 +21,13 @@ export {default as flag} from './flag.js'; export {default as name} from './name.js'; export {default as referenceList} from './referenceList.js'; export {default as referencedArtworkList} from './referencedArtworkList.js'; -export {default as reverseAnnotatedReferenceList} from './reverseAnnotatedReferenceList.js'; -export {default as reverseContributionList} from './reverseContributionList.js'; export {default as reverseReferenceList} from './reverseReferenceList.js'; -export {default as reverseReferencedArtworkList} from './reverseReferencedArtworkList.js'; -export {default as reverseSingleReferenceList} from './reverseSingleReferenceList.js'; export {default as seriesList} from './seriesList.js'; export {default as simpleDate} from './simpleDate.js'; export {default as simpleString} from './simpleString.js'; export {default as singleReference} from './singleReference.js'; +export {default as soupyFind} from './soupyFind.js'; +export {default as soupyReverse} from './soupyReverse.js'; export {default as thing} from './thing.js'; export {default as thingList} from './thingList.js'; export {default as urls} from './urls.js'; diff --git a/src/data/composite/wiki-properties/referenceList.js b/src/data/composite/wiki-properties/referenceList.js index 4d4cb106..4f8207b5 100644 --- a/src/data/composite/wiki-properties/referenceList.js +++ b/src/data/composite/wiki-properties/referenceList.js @@ -11,7 +11,8 @@ import {input, templateCompositeFrom} from '#composite'; import {validateReferenceList} from '#validators'; import {exposeDependency} from '#composite/control-flow'; -import {inputWikiData, withResolvedReferenceList} from '#composite/wiki-data'; +import {inputSoupyFind, inputWikiData, withResolvedReferenceList} + from '#composite/wiki-data'; import {referenceListInputDescriptions, referenceListUpdateDescription} from './helpers/reference-list-helpers.js'; @@ -25,7 +26,7 @@ export default templateCompositeFrom({ ...referenceListInputDescriptions(), data: inputWikiData({allowMixedTypes: true}), - find: input({type: 'function'}), + find: inputSoupyFind(), }, update: diff --git a/src/data/composite/wiki-properties/reverseAnnotatedReferenceList.js b/src/data/composite/wiki-properties/reverseAnnotatedReferenceList.js deleted file mode 100644 index ba7166b9..00000000 --- a/src/data/composite/wiki-properties/reverseAnnotatedReferenceList.js +++ /dev/null @@ -1,33 +0,0 @@ -import {input, templateCompositeFrom} from '#composite'; - -import {exposeDependency} from '#composite/control-flow'; -import {inputWikiData, withReverseAnnotatedReferenceList} - from '#composite/wiki-data'; - -export default templateCompositeFrom({ - annotation: `reverseAnnotatedReferenceList`, - - compose: false, - - inputs: { - data: inputWikiData({allowMixedTypes: false}), - list: input({type: 'string'}), - - forward: input({type: 'string', defaultValue: 'thing'}), - backward: input({type: 'string', defaultValue: 'thing'}), - annotation: input({type: 'string', defaultValue: 'annotation'}), - }, - - steps: () => [ - withReverseAnnotatedReferenceList({ - data: input('data'), - list: input('list'), - - forward: input('forward'), - backward: input('backward'), - annotation: input('annotation'), - }), - - exposeDependency({dependency: '#reverseAnnotatedReferenceList'}), - ], -}); diff --git a/src/data/composite/wiki-properties/reverseContributionList.js b/src/data/composite/wiki-properties/reverseContributionList.js deleted file mode 100644 index 7f3f9c81..00000000 --- a/src/data/composite/wiki-properties/reverseContributionList.js +++ /dev/null @@ -1,24 +0,0 @@ -import {input, templateCompositeFrom} from '#composite'; - -import {exposeDependency} from '#composite/control-flow'; -import {inputWikiData, withReverseContributionList} from '#composite/wiki-data'; - -export default templateCompositeFrom({ - annotation: `reverseContributionList`, - - compose: false, - - inputs: { - data: inputWikiData({allowMixedTypes: false}), - list: input({type: 'string'}), - }, - - steps: () => [ - withReverseContributionList({ - data: input('data'), - list: input('list'), - }), - - exposeDependency({dependency: '#reverseContributionList'}), - ], -}); diff --git a/src/data/composite/wiki-properties/reverseReferenceList.js b/src/data/composite/wiki-properties/reverseReferenceList.js index 84ba67df..6d590a67 100644 --- a/src/data/composite/wiki-properties/reverseReferenceList.js +++ b/src/data/composite/wiki-properties/reverseReferenceList.js @@ -1,13 +1,13 @@ // Neat little shortcut for "reversing" the reference lists stored on other // things - for example, tracks specify a "referenced tracks" property, and // you would use this to compute a corresponding "referenced *by* tracks" -// property. Naturally, the passed ref list property is of the things in the -// wiki data provided, not the requesting Thing itself. +// property. import {input, templateCompositeFrom} from '#composite'; import {exposeDependency} from '#composite/control-flow'; -import {inputWikiData, withReverseReferenceList} from '#composite/wiki-data'; +import {inputSoupyReverse, inputWikiData, withReverseReferenceList} + from '#composite/wiki-data'; export default templateCompositeFrom({ annotation: `reverseReferenceList`, @@ -15,14 +15,14 @@ export default templateCompositeFrom({ compose: false, inputs: { - data: inputWikiData({allowMixedTypes: false}), - list: input({type: 'string'}), + data: inputWikiData({allowMixedTypes: true}), + reverse: inputSoupyReverse(), }, steps: () => [ withReverseReferenceList({ data: input('data'), - list: input('list'), + reverse: input('reverse'), }), exposeDependency({dependency: '#reverseReferenceList'}), diff --git a/src/data/composite/wiki-properties/reverseReferencedArtworkList.js b/src/data/composite/wiki-properties/reverseReferencedArtworkList.js deleted file mode 100644 index 2950bdb9..00000000 --- a/src/data/composite/wiki-properties/reverseReferencedArtworkList.js +++ /dev/null @@ -1,39 +0,0 @@ -import {input, templateCompositeFrom} from '#composite'; -import {combineWikiDataArrays} from '#wiki-data'; - -import {exposeDependency} from '#composite/control-flow'; -import {inputWikiData, withReverseAnnotatedReferenceList} - from '#composite/wiki-data'; - -export default templateCompositeFrom({ - annotation: `reverseReferencedArtworkList`, - - compose: false, - - steps: () => [ - { - dependencies: [ - 'albumData', - 'trackData', - ], - - compute: (continuation, { - albumData, - trackData, - }) => continuation({ - ['#data']: - combineWikiDataArrays([ - albumData, - trackData, - ]), - }), - }, - - withReverseAnnotatedReferenceList({ - data: '#data', - list: input.value('referencedArtworks'), - }), - - exposeDependency({dependency: '#reverseAnnotatedReferenceList'}), - ], -}); diff --git a/src/data/composite/wiki-properties/reverseSingleReferenceList.js b/src/data/composite/wiki-properties/reverseSingleReferenceList.js deleted file mode 100644 index d180b12d..00000000 --- a/src/data/composite/wiki-properties/reverseSingleReferenceList.js +++ /dev/null @@ -1,24 +0,0 @@ -import {input, templateCompositeFrom} from '#composite'; - -import {exposeDependency} from '#composite/control-flow'; -import {inputWikiData, withReverseSingleReferenceList} from '#composite/wiki-data'; - -export default templateCompositeFrom({ - annotation: `reverseSingleReferenceList`, - - compose: false, - - inputs: { - data: inputWikiData({allowMixedTypes: false}), - ref: input({type: 'string'}), - }, - - steps: () => [ - withReverseSingleReferenceList({ - data: input('data'), - ref: input('ref'), - }), - - exposeDependency({dependency: '#reverseSingleReferenceList'}), - ], -}); diff --git a/src/data/composite/wiki-properties/singleReference.js b/src/data/composite/wiki-properties/singleReference.js index db4fc9f9..f532ebbe 100644 --- a/src/data/composite/wiki-properties/singleReference.js +++ b/src/data/composite/wiki-properties/singleReference.js @@ -11,7 +11,8 @@ import {input, templateCompositeFrom} from '#composite'; import {isThingClass, validateReference} from '#validators'; import {exposeDependency} from '#composite/control-flow'; -import {inputWikiData, withResolvedReference} from '#composite/wiki-data'; +import {inputSoupyFind, inputWikiData, withResolvedReference} + from '#composite/wiki-data'; export default templateCompositeFrom({ annotation: `singleReference`, @@ -21,8 +22,7 @@ export default templateCompositeFrom({ inputs: { class: input.staticValue({validate: isThingClass}), - find: input({type: 'function'}), - + find: inputSoupyFind(), data: inputWikiData({allowMixedTypes: false}), }, diff --git a/src/data/composite/wiki-properties/soupyFind.js b/src/data/composite/wiki-properties/soupyFind.js new file mode 100644 index 00000000..0f9a17e3 --- /dev/null +++ b/src/data/composite/wiki-properties/soupyFind.js @@ -0,0 +1,14 @@ +import {isObject} from '#validators'; + +import {inputSoupyFind} from '#composite/wiki-data'; + +function soupyFind() { + return { + flags: {update: true}, + update: {validate: isObject}, + }; +} + +soupyFind.input = inputSoupyFind.input; + +export default soupyFind; diff --git a/src/data/composite/wiki-properties/soupyReverse.js b/src/data/composite/wiki-properties/soupyReverse.js new file mode 100644 index 00000000..269ccd6f --- /dev/null +++ b/src/data/composite/wiki-properties/soupyReverse.js @@ -0,0 +1,22 @@ +import {isObject} from '#validators'; + +import {inputSoupyReverse} from '#composite/wiki-data'; + +function soupyReverse() { + return { + flags: {update: true}, + update: {validate: isObject}, + }; +} + +soupyReverse.input = inputSoupyReverse.input; + +soupyReverse.contributionsBy = + (bindTo, contributionsProperty) => ({ + bindTo, + + referencing: thing => thing[contributionsProperty], + referenced: contrib => [contrib.artist], + }); + +export default soupyReverse; diff --git a/src/data/thing.js b/src/data/thing.js index 78ad3642..c51c5fe5 100644 --- a/src/data/thing.js +++ b/src/data/thing.js @@ -16,6 +16,8 @@ export default class Thing extends CacheableObject { static findSpecs = Symbol.for('Thing.findSpecs'); static findThisThingOnly = Symbol.for('Thing.findThisThingOnly'); + static reverseSpecs = Symbol.for('Thing.reverseSpecs'); + static yamlDocumentSpec = Symbol.for('Thing.yamlDocumentSpec'); static getYamlLoadingSpec = Symbol.for('Thing.getYamlLoadingSpec'); @@ -26,14 +28,13 @@ export default class Thing extends CacheableObject { // Symbol.for('Thing.isThingConstructor') in constructor static [Symbol.for('Thing.isThingConstructor')] = NaN; - static [CacheableObject.propertyDescriptors] = { + constructor() { + super(); + // To detect: // Object.hasOwn(object, Symbol.for('Thing.isThing')) - [Symbol.for('Thing.isThing')]: { - flags: {expose: true}, - expose: {compute: () => NaN}, - }, - }; + this[Symbol.for('Thing.isThing')] = NaN; + } static [Symbol.for('Thing.selectAll')] = _wikiData => []; diff --git a/src/data/things/album.js b/src/data/things/album.js index bd54a356..5f1788f8 100644 --- a/src/data/things/album.js +++ b/src/data/things/album.js @@ -3,15 +3,13 @@ export const DATA_ALBUM_DIRECTORY = 'album'; import * as path from 'node:path'; import {inspect} from 'node:util'; -import CacheableObject from '#cacheable-object'; import {colors} from '#cli'; import {input} from '#composite'; -import find from '#find'; import {traverse} from '#node-utils'; import {sortAlbumsTracksChronologically, sortChronologically} from '#sort'; import {accumulateSum, empty} from '#sugar'; import Thing from '#thing'; -import {isColor, isDate, isDirectory, validateWikiData} from '#validators'; +import {isColor, isDate, isDirectory} from '#validators'; import { parseAdditionalFiles, @@ -27,12 +25,8 @@ import {exitWithoutDependency, exposeDependency, exposeUpdateValueOrContinue} from '#composite/control-flow'; import {withPropertyFromObject} from '#composite/data'; -import { - exitWithoutContribs, - withDirectory, - withResolvedReference, - withCoverArtDate, -} from '#composite/wiki-data'; +import {exitWithoutContribs, withDirectory, withCoverArtDate} + from '#composite/wiki-data'; import { additionalFiles, @@ -50,10 +44,11 @@ import { name, referencedArtworkList, referenceList, - reverseReferencedArtworkList, + reverseReferenceList, simpleDate, simpleString, - singleReference, + soupyFind, + soupyReverse, thing, thingList, urls, @@ -69,7 +64,6 @@ export class Album extends Thing { static [Thing.getPropertyDescriptors] = ({ ArtTag, - Artist, Group, Track, TrackSection, @@ -92,6 +86,7 @@ export class Album extends Thing { }), ], + alwaysReferenceByDirectory: flag(false), alwaysReferenceTracksByDirectory: flag(false), suffixTrackDirectories: flag(false), @@ -229,8 +224,7 @@ export class Album extends Thing { groups: referenceList({ class: input.value(Group), - find: input.value(find.group), - data: 'groupData', + find: soupyFind.input('group'), }), artTags: [ @@ -241,8 +235,7 @@ export class Album extends Thing { referenceList({ class: input.value(ArtTag), - find: input.value(find.artTag), - data: 'artTagData', + find: soupyFind.input('artTag'), }), ], @@ -270,26 +263,20 @@ export class Album extends Thing { // Update only + find: soupyFind(), + reverse: soupyReverse(), + + // used for referencedArtworkList (mixedFind) albumData: wikiData({ class: input.value(Album), }), - artistData: wikiData({ - class: input.value(Artist), - }), - - artTagData: wikiData({ - class: input.value(ArtTag), - }), - - groupData: wikiData({ - class: input.value(Group), - }), - + // used for referencedArtworkList (mixedFind) trackData: wikiData({ class: input.value(Track), }), + // used for withMatchingContributionPresets (indirectly by Contribution) wikiInfo: thing({ class: input.value(WikiInfo), }), @@ -313,7 +300,9 @@ export class Album extends Thing { value: input.value([]), }), - reverseReferencedArtworkList(), + reverseReferenceList({ + reverse: soupyReverse.input('artworksWhichReference'), + }), ], }); @@ -361,6 +350,11 @@ export class Album extends Thing { album: { referenceTypes: ['album', 'album-commentary', 'album-gallery'], bindTo: 'albumData', + + getMatchableNames: album => + (album.alwaysReferenceByDirectory + ? [] + : [album.name]), }, albumWithArtwork: { @@ -369,6 +363,60 @@ export class Album extends Thing { include: album => album.hasCoverArt, + + getMatchableNames: album => + (album.alwaysReferenceByDirectory + ? [] + : [album.name]), + }, + }; + + static [Thing.reverseSpecs] = { + albumsWhoseTracksInclude: { + bindTo: 'albumData', + + referencing: album => [album], + referenced: album => album.tracks, + }, + + albumsWhoseTrackSectionsInclude: { + bindTo: 'albumData', + + referencing: album => [album], + referenced: album => album.trackSections, + }, + + albumsWhoseArtworksFeature: { + bindTo: 'albumData', + + referencing: album => [album], + referenced: album => album.artTags, + }, + + albumsWhoseGroupsInclude: { + bindTo: 'albumData', + + referencing: album => [album], + referenced: album => album.groups, + }, + + albumArtistContributionsBy: + soupyReverse.contributionsBy('albumData', 'artistContribs'), + + albumCoverArtistContributionsBy: + soupyReverse.contributionsBy('albumData', 'coverArtistContribs'), + + albumWallpaperArtistContributionsBy: + soupyReverse.contributionsBy('albumData', 'wallpaperArtistContribs'), + + albumBannerArtistContributionsBy: + soupyReverse.contributionsBy('albumData', 'bannerArtistContribs'), + + albumsWithCommentaryBy: { + bindTo: 'albumData', + + referencing: album => [album], + referenced: album => album.commentatorArtists, }, }; @@ -380,6 +428,7 @@ export class Album extends Thing { 'Directory Suffix': {property: 'directorySuffix'}, 'Suffix Track Directories': {property: 'suffixTrackDirectories'}, + 'Always Reference By Directory': {property: 'alwaysReferenceByDirectory'}, 'Always Reference Tracks By Directory': { property: 'alwaysReferenceTracksByDirectory', }, @@ -516,7 +565,7 @@ export class Album extends Thing { static [Thing.getYamlLoadingSpec] = ({ documentModes: {headerAndEntries}, - thingConstructors: {Album, Track, TrackSectionHelper}, + thingConstructors: {Album, Track}, }) => ({ title: `Process album files`, @@ -640,9 +689,7 @@ export class TrackSection extends Thing { // Update only - albumData: wikiData({ - class: input.value(Album), - }), + reverse: soupyReverse(), // Expose only @@ -727,6 +774,15 @@ export class TrackSection extends Thing { }, }; + static [Thing.reverseSpecs] = { + trackSectionsWhichInclude: { + bindTo: 'trackSectionData', + + referencing: trackSection => [trackSection], + referenced: trackSection => trackSection.tracks, + }, + }; + static [Thing.yamlDocumentSpec] = { fields: { 'Section': {property: 'name'}, diff --git a/src/data/things/art-tag.js b/src/data/things/art-tag.js index 3149b310..9842c887 100644 --- a/src/data/things/art-tag.js +++ b/src/data/things/art-tag.js @@ -12,6 +12,7 @@ import { directory, flag, name, + soupyReverse, wikiData, } from '#composite/wiki-properties'; @@ -41,13 +42,7 @@ export class ArtTag extends Thing { // Update only - albumData: wikiData({ - class: input.value(Album), - }), - - trackData: wikiData({ - class: input.value(Track), - }), + reverse: soupyReverse(), // Expose only @@ -55,11 +50,13 @@ export class ArtTag extends Thing { flags: {expose: true}, expose: { - dependencies: ['this', 'albumData', 'trackData'], - compute: ({this: artTag, albumData, trackData}) => + dependencies: ['this', 'reverse'], + compute: ({this: artTag, reverse}) => sortAlbumsTracksChronologically( - [...albumData, ...trackData] - .filter(({artTags}) => artTags.includes(artTag)), + [ + ...reverse.albumsWhoseArtworksFeature(artTag), + ...reverse.tracksWhoseArtworksFeature(artTag), + ], {getDate: thing => thing.coverArtDate ?? thing.date}), }, }, diff --git a/src/data/things/artist.js b/src/data/things/artist.js index 8fdb8a12..7ed99a8e 100644 --- a/src/data/things/artist.js +++ b/src/data/things/artist.js @@ -5,26 +5,22 @@ import {inspect} from 'node:util'; import CacheableObject from '#cacheable-object'; import {colors} from '#cli'; import {input} from '#composite'; -import find from '#find'; import {sortAlphabetically} from '#sort'; -import {stitchArrays, unique} from '#sugar'; +import {stitchArrays} from '#sugar'; import Thing from '#thing'; import {isName, validateArrayItems} from '#validators'; import {getKebabCase} from '#wiki-data'; -import {exposeDependency} from '#composite/control-flow'; -import {withReverseContributionList} from '#composite/wiki-data'; - import { contentString, directory, fileExtension, flag, name, - reverseAnnotatedReferenceList, - reverseContributionList, reverseReferenceList, singleReference, + soupyFind, + soupyReverse, urls, wikiData, } from '#composite/wiki-properties'; @@ -57,95 +53,62 @@ export class Artist extends Thing { aliasedArtist: singleReference({ class: input.value(Artist), - find: input.value(find.artist), - data: 'artistData', + find: soupyFind.input('artist'), }), // Update only - albumData: wikiData({ - class: input.value(Album), - }), - - artistData: wikiData({ - class: input.value(Artist), - }), - - flashData: wikiData({ - class: input.value(Flash), - }), - - groupData: wikiData({ - class: input.value(Group), - }), - - trackData: wikiData({ - class: input.value(Track), - }), + find: soupyFind(), + reverse: soupyReverse(), // Expose only - trackArtistContributions: reverseContributionList({ - data: 'trackData', - list: input.value('artistContribs'), + trackArtistContributions: reverseReferenceList({ + reverse: soupyReverse.input('trackArtistContributionsBy'), }), - trackContributorContributions: reverseContributionList({ - data: 'trackData', - list: input.value('contributorContribs'), + trackContributorContributions: reverseReferenceList({ + reverse: soupyReverse.input('trackContributorContributionsBy'), }), - trackCoverArtistContributions: reverseContributionList({ - data: 'trackData', - list: input.value('coverArtistContribs'), + trackCoverArtistContributions: reverseReferenceList({ + reverse: soupyReverse.input('trackCoverArtistContributionsBy'), }), tracksAsCommentator: reverseReferenceList({ - data: 'trackData', - list: input.value('commentatorArtists'), + reverse: soupyReverse.input('tracksWithCommentaryBy'), }), - albumArtistContributions: reverseContributionList({ - data: 'albumData', - list: input.value('artistContribs'), + albumArtistContributions: reverseReferenceList({ + reverse: soupyReverse.input('albumArtistContributionsBy'), }), - albumCoverArtistContributions: reverseContributionList({ - data: 'albumData', - list: input.value('coverArtistContribs'), + albumCoverArtistContributions: reverseReferenceList({ + reverse: soupyReverse.input('albumCoverArtistContributionsBy'), }), - albumWallpaperArtistContributions: reverseContributionList({ - data: 'albumData', - list: input.value('wallpaperArtistContribs'), + albumWallpaperArtistContributions: reverseReferenceList({ + reverse: soupyReverse.input('albumWallpaperArtistContributionsBy'), }), - albumBannerArtistContributions: reverseContributionList({ - data: 'albumData', - list: input.value('bannerArtistContribs'), + albumBannerArtistContributions: reverseReferenceList({ + reverse: soupyReverse.input('albumBannerArtistContributionsBy'), }), albumsAsCommentator: reverseReferenceList({ - data: 'albumData', - list: input.value('commentatorArtists'), + reverse: soupyReverse.input('albumsWithCommentaryBy'), }), - flashContributorContributions: reverseContributionList({ - data: 'flashData', - list: input.value('contributorContribs'), + flashContributorContributions: reverseReferenceList({ + reverse: soupyReverse.input('flashContributorContributionsBy'), }), flashesAsCommentator: reverseReferenceList({ - data: 'flashData', - list: input.value('commentatorArtists'), + reverse: soupyReverse.input('flashesWithCommentaryBy'), }), - closelyLinkedGroups: reverseAnnotatedReferenceList({ - data: 'groupData', - list: input.value('closelyLinkedArtists'), - - forward: input.value('artist'), - backward: input.value('group'), + closelyLinkedGroups: reverseReferenceList({ + reverse: soupyReverse.input('groupsCloselyLinkedTo'), }), totalDuration: artistTotalDuration(), diff --git a/src/data/things/contribution.js b/src/data/things/contribution.js index 2712af70..c92fafb4 100644 --- a/src/data/things/contribution.js +++ b/src/data/things/contribution.js @@ -8,8 +8,7 @@ import Thing from '#thing'; import {isStringNonEmpty, isThing, validateReference} from '#validators'; import {exitWithoutDependency, exposeDependency} from '#composite/control-flow'; -import {withResolvedReference} from '#composite/wiki-data'; -import {flag, simpleDate} from '#composite/wiki-properties'; +import {flag, simpleDate, soupyFind} from '#composite/wiki-properties'; import { withFilteredList, @@ -82,6 +81,10 @@ export class Contribution extends Thing { flag(true), ], + // Update only + + find: soupyFind(), + // Expose only context: [ diff --git a/src/data/things/flash.js b/src/data/things/flash.js index aa6b9cd1..b143b560 100644 --- a/src/data/things/flash.js +++ b/src/data/things/flash.js @@ -1,7 +1,6 @@ export const FLASH_DATA_FILE = 'flashes.yaml'; import {input} from '#composite'; -import find from '#find'; import {empty} from '#sugar'; import {sortFlashesChronologically} from '#sort'; import Thing from '#thing'; @@ -30,6 +29,8 @@ import { name, referenceList, simpleDate, + soupyFind, + soupyReverse, thing, urls, wikiData, @@ -42,7 +43,6 @@ export class Flash extends Thing { static [Thing.referenceType] = 'flash'; static [Thing.getPropertyDescriptors] = ({ - Artist, Track, FlashAct, WikiInfo, @@ -105,8 +105,7 @@ export class Flash extends Thing { featuredTracks: referenceList({ class: input.value(Track), - find: input.value(find.track), - data: 'trackData', + find: soupyFind.input('track'), }), urls: urls(), @@ -116,18 +115,10 @@ export class Flash extends Thing { // Update only - artistData: wikiData({ - class: input.value(Artist), - }), - - trackData: wikiData({ - class: input.value(Track), - }), - - flashActData: wikiData({ - class: input.value(FlashAct), - }), + find: soupyFind(), + reverse: soupyReverse(), + // used for withMatchingContributionPresets (indirectly by Contribution) wikiInfo: thing({ class: input.value(WikiInfo), }), @@ -173,6 +164,25 @@ export class Flash extends Thing { }, }; + static [Thing.reverseSpecs] = { + flashesWhichFeature: { + bindTo: 'flashData', + + referencing: flash => [flash], + referenced: flash => flash.featuredTracks, + }, + + flashContributorContributionsBy: + soupyReverse.contributionsBy('flashData', 'contributorContribs'), + + flashesWithCommentaryBy: { + bindTo: 'flashData', + + referencing: flash => [flash], + referenced: flash => flash.commentatorArtists, + }, + }; + static [Thing.yamlDocumentSpec] = { fields: { 'Flash': {property: 'name'}, @@ -242,19 +252,13 @@ export class FlashAct extends Thing { flashes: referenceList({ class: input.value(Flash), - find: input.value(find.flash), - data: 'flashData', + find: soupyFind.input('flash'), }), // Update only - flashData: wikiData({ - class: input.value(Flash), - }), - - flashSideData: wikiData({ - class: input.value(FlashSide), - }), + find: soupyFind(), + reverse: soupyReverse(), // Expose only @@ -271,6 +275,15 @@ export class FlashAct extends Thing { }, }; + static [Thing.reverseSpecs] = { + flashActsWhoseFlashesInclude: { + bindTo: 'flashActData', + + referencing: flashAct => [flashAct], + referenced: flashAct => flashAct.flashes, + }, + }; + static [Thing.yamlDocumentSpec] = { fields: { 'Act': {property: 'name'}, @@ -298,15 +311,12 @@ export class FlashSide extends Thing { acts: referenceList({ class: input.value(FlashAct), - find: input.value(find.flashAct), - data: 'flashActData', + find: soupyFind.input('flashAct'), }), // Update only - flashActData: wikiData({ - class: input.value(FlashAct), - }), + find: soupyFind(), }); static [Thing.yamlDocumentSpec] = { @@ -325,6 +335,15 @@ export class FlashSide extends Thing { }, }; + static [Thing.reverseSpecs] = { + flashSidesWhoseActsInclude: { + bindTo: 'flashSideData', + + referencing: flashSide => [flashSide], + referenced: flashSide => flashSide.acts, + }, + }; + static [Thing.getYamlLoadingSpec] = ({ documentModes: {allInOne}, thingConstructors: {Flash, FlashAct}, diff --git a/src/data/things/group.js b/src/data/things/group.js index 8418cb99..ed3c59bb 100644 --- a/src/data/things/group.js +++ b/src/data/things/group.js @@ -1,7 +1,6 @@ export const GROUP_DATA_FILE = 'groups.yaml'; import {input} from '#composite'; -import find from '#find'; import Thing from '#thing'; import {parseAnnotatedReferences, parseSerieses} from '#yaml'; @@ -13,6 +12,7 @@ import { name, referenceList, seriesList, + soupyFind, urls, wikiData, } from '#composite/wiki-properties'; @@ -32,8 +32,7 @@ export class Group extends Thing { closelyLinkedArtists: annotatedReferenceList({ class: input.value(Artist), - find: input.value(find.artist), - data: 'artistData', + find: soupyFind.input('artist'), date: input.value(null), @@ -43,8 +42,7 @@ export class Group extends Thing { featuredAlbums: referenceList({ class: input.value(Album), - find: input.value(find.album), - data: 'albumData', + find: soupyFind.input('album'), }), serieses: seriesList({ @@ -53,17 +51,8 @@ export class Group extends Thing { // Update only - albumData: wikiData({ - class: input.value(Album), - }), - - artistData: wikiData({ - class: input.value(Artist), - }), - - groupCategoryData: wikiData({ - class: input.value(GroupCategory), - }), + find: soupyFind(), + reverse: soupyFind(), // Expose only @@ -83,9 +72,9 @@ export class Group extends Thing { flags: {expose: true}, expose: { - dependencies: ['this', 'albumData'], - compute: ({this: group, albumData}) => - albumData?.filter((album) => album.groups.includes(group)) ?? [], + dependencies: ['this', 'reverse'], + compute: ({this: group, reverse}) => + reverse.albumsWhoseGroupsInclude(group), }, }, @@ -93,9 +82,9 @@ export class Group extends Thing { flags: {expose: true}, expose: { - dependencies: ['this', 'groupCategoryData'], - compute: ({this: group, groupCategoryData}) => - groupCategoryData.find((category) => category.groups.includes(group)) + dependencies: ['this', 'reverse'], + compute: ({this: group, reverse}) => + reverse.groupCategoriesWhichInclude(group, {unique: true}) ?.color, }, }, @@ -104,9 +93,9 @@ export class Group extends Thing { flags: {expose: true}, expose: { - dependencies: ['this', 'groupCategoryData'], - compute: ({this: group, groupCategoryData}) => - groupCategoryData.find((category) => category.groups.includes(group)) ?? + dependencies: ['this', 'reverse'], + compute: ({this: group, reverse}) => + reverse.groupCategoriesWhichInclude(group, {unique: true}) ?? null, }, }, @@ -119,6 +108,25 @@ export class Group extends Thing { }, }; + static [Thing.reverseSpecs] = { + groupsCloselyLinkedTo: { + bindTo: 'groupData', + + referencing: group => + group.closelyLinkedArtists + .map(({artist, ...referenceDetails}) => ({ + group, + artist, + referenceDetails, + })), + + referenced: ({artist}) => [artist], + + tidy: ({group, referenceDetails}) => + ({group, ...referenceDetails}), + }, + }; + static [Thing.yamlDocumentSpec] = { fields: { 'Group': {property: 'name'}, @@ -210,17 +218,23 @@ export class GroupCategory extends Thing { groups: referenceList({ class: input.value(Group), - find: input.value(find.group), - data: 'groupData', + find: soupyFind.input('group'), }), // Update only - groupData: wikiData({ - class: input.value(Group), - }), + find: soupyFind(), }); + static [Thing.reverseSpecs] = { + groupCategoriesWhichInclude: { + bindTo: 'groupCategoryData', + + referencing: groupCategory => [groupCategory], + referenced: groupCategory => groupCategory.groups, + }, + }; + static [Thing.yamlDocumentSpec] = { fields: { 'Category': {property: 'name'}, diff --git a/src/data/things/homepage-layout.js b/src/data/things/homepage-layout.js index 00d6aef5..47d92471 100644 --- a/src/data/things/homepage-layout.js +++ b/src/data/things/homepage-layout.js @@ -1,7 +1,6 @@ export const HOMEPAGE_LAYOUT_DATA_FILE = 'homepage.yaml'; import {input} from '#composite'; -import find from '#find'; import Thing from '#thing'; import { @@ -17,7 +16,7 @@ import { import {exposeDependency} from '#composite/control-flow'; import {withResolvedReference} from '#composite/wiki-data'; -import {color, contentString, name, referenceList, wikiData} +import {color, contentString, name, referenceList, soupyFind} from '#composite/wiki-properties'; export class HomepageLayout extends Thing { @@ -55,7 +54,7 @@ export class HomepageLayout extends Thing { export class HomepageLayoutRow extends Thing { static [Thing.friendlyName] = `Homepage Row`; - static [Thing.getPropertyDescriptors] = ({Album, Group}) => ({ + static [Thing.getPropertyDescriptors] = () => ({ // Update & expose name: name('Unnamed Homepage Row'), @@ -74,17 +73,7 @@ export class HomepageLayoutRow extends Thing { // Update only - // These wiki data arrays aren't necessarily used by every subclass, but - // to the convenience of providing these, the superclass accepts all wiki - // data arrays depended upon by any subclass. - - albumData: wikiData({ - class: input.value(Album), - }), - - groupData: wikiData({ - class: input.value(Group), - }), + find: soupyFind(), }); static [Thing.yamlDocumentSpec] = { @@ -151,8 +140,7 @@ export class HomepageLayoutAlbumsRow extends HomepageLayoutRow { withResolvedReference({ ref: input.updateValue(), - data: 'groupData', - find: input.value(find.group), + find: soupyFind.input('group'), }), exposeDependency({dependency: '#resolvedReference'}), @@ -160,8 +148,7 @@ export class HomepageLayoutAlbumsRow extends HomepageLayoutRow { sourceAlbums: referenceList({ class: input.value(Album), - find: input.value(find.album), - data: 'albumData', + find: soupyFind.input('album'), }), countAlbumsFromGroup: { diff --git a/src/data/things/index.js b/src/data/things/index.js index f18e283a..9f033c23 100644 --- a/src/data/things/index.js +++ b/src/data/things/index.js @@ -177,6 +177,16 @@ function evaluateSerializeDescriptors() { }); } +function finalizeCacheableObjectPrototypes() { + return descriptorAggregateHelper({ + message: `Errors finalizing Thing class prototypes`, + + op(constructor) { + constructor.finalizeCacheableObjectPrototype(); + }, + }); +} + if (!errorDuplicateClassNames()) process.exit(1); @@ -188,6 +198,9 @@ if (!evaluatePropertyDescriptors()) if (!evaluateSerializeDescriptors()) process.exit(1); +if (!finalizeCacheableObjectPrototypes()) + process.exit(1); + Object.assign(allClasses, {Thing}); export default allClasses; diff --git a/src/data/things/track.js b/src/data/things/track.js index a0d2f641..ff4750db 100644 --- a/src/data/things/track.js +++ b/src/data/things/track.js @@ -3,7 +3,6 @@ import {inspect} from 'node:util'; import CacheableObject from '#cacheable-object'; import {colors} from '#cli'; import {input} from '#composite'; -import find from '#find'; import Thing from '#thing'; import {isBoolean, isColor, isContributionList, isDate, isFileExtension} from '#validators'; @@ -21,7 +20,6 @@ import { import {withPropertyFromObject} from '#composite/data'; import { - exitWithoutDependency, exposeConstant, exposeDependency, exposeDependencyOrContinue, @@ -50,10 +48,11 @@ import { referenceList, referencedArtworkList, reverseReferenceList, - reverseReferencedArtworkList, simpleDate, simpleString, singleReference, + soupyFind, + soupyReverse, thing, urls, wikiData, @@ -63,7 +62,6 @@ import { exitWithoutUniqueCoverArt, inheritContributionListFromOriginalRelease, inheritFromOriginalRelease, - trackReverseReferenceList, withAlbum, withAlwaysReferenceByDirectory, withContainingTrackSection, @@ -83,7 +81,6 @@ export class Track extends Thing { static [Thing.getPropertyDescriptors] = ({ Album, ArtTag, - Artist, Flash, TrackSection, WikiInfo, @@ -221,8 +218,7 @@ export class Track extends Thing { originalReleaseTrack: singleReference({ class: input.value(Track), - find: input.value(find.track), - data: 'trackData', + find: soupyFind.input('track'), }), // Internal use only - for directly identifying an album inside a track's @@ -230,8 +226,7 @@ export class Track extends Thing { // included in an album's track list). dataSourceAlbum: singleReference({ class: input.value(Album), - find: input.value(find.album), - data: 'albumData', + find: soupyFind.input('album'), }), artistContribs: [ @@ -331,8 +326,7 @@ export class Track extends Thing { referenceList({ class: input.value(Track), - find: input.value(find.track), - data: 'trackData', + find: soupyFind.input('track'), }), ], @@ -343,8 +337,7 @@ export class Track extends Thing { referenceList({ class: input.value(Track), - find: input.value(find.track), - data: 'trackData', + find: soupyFind.input('track'), }), ], @@ -355,8 +348,7 @@ export class Track extends Thing { referenceList({ class: input.value(ArtTag), - find: input.value(find.artTag), - data: 'artTagData', + find: soupyFind.input('artTag'), }), ], @@ -376,30 +368,20 @@ export class Track extends Thing { // Update only + find: soupyFind(), + reverse: soupyReverse(), + + // used for referencedArtworkList (mixedFind) albumData: wikiData({ class: input.value(Album), }), - artistData: wikiData({ - class: input.value(Artist), - }), - - artTagData: wikiData({ - class: input.value(ArtTag), - }), - - flashData: wikiData({ - class: input.value(Flash), - }), - + // used for referencedArtworkList (mixedFind) trackData: wikiData({ class: input.value(Track), }), - trackSectionData: wikiData({ - class: input.value(TrackSection), - }), - + // used for withMatchingContributionPresets (indirectly by Contribution) wikiInfo: thing({ class: input.value(WikiInfo), }), @@ -445,17 +427,16 @@ export class Track extends Thing { exposeDependency({dependency: '#otherReleases'}), ], - referencedByTracks: trackReverseReferenceList({ - list: input.value('referencedTracks'), + referencedByTracks: reverseReferenceList({ + reverse: soupyReverse.input('tracksWhichReference'), }), - sampledByTracks: trackReverseReferenceList({ - list: input.value('sampledTracks'), + sampledByTracks: reverseReferenceList({ + reverse: soupyReverse.input('tracksWhichSample'), }), featuredInFlashes: reverseReferenceList({ - data: 'flashData', - list: input.value('featuredTracks'), + reverse: soupyReverse.input('flashesWhichFeature'), }), referencedByArtworks: [ @@ -463,7 +444,9 @@ export class Track extends Thing { value: input.value([]), }), - reverseReferencedArtworkList(), + reverseReferenceList({ + reverse: soupyReverse.input('artworksWhichReference'), + }), ], }); @@ -656,6 +639,45 @@ export class Track extends Thing { }, }; + static [Thing.reverseSpecs] = { + tracksWhichReference: { + bindTo: 'trackData', + + referencing: track => track.isOriginalRelease ? [track] : [], + referenced: track => track.referencedTracks, + }, + + tracksWhichSample: { + bindTo: 'trackData', + + referencing: track => track.isOriginalRelease ? [track] : [], + referenced: track => track.sampledTracks, + }, + + tracksWhoseArtworksFeature: { + bindTo: 'trackData', + + referencing: track => [track], + referenced: track => track.artTags, + }, + + trackArtistContributionsBy: + soupyReverse.contributionsBy('trackData', 'artistContribs'), + + trackContributorContributionsBy: + soupyReverse.contributionsBy('trackData', 'contributorContribs'), + + trackCoverArtistContributionsBy: + soupyReverse.contributionsBy('trackData', 'coverArtistContribs'), + + tracksWithCommentaryBy: { + bindTo: 'trackData', + + referencing: track => [track], + referenced: track => track.commentatorArtists, + }, + }; + // Track YAML loading is handled in album.js. static [Thing.getYamlLoadingSpec] = null; diff --git a/src/data/things/wiki-info.js b/src/data/things/wiki-info.js index ef643681..590598be 100644 --- a/src/data/things/wiki-info.js +++ b/src/data/things/wiki-info.js @@ -1,7 +1,6 @@ export const WIKI_INFO_FILE = 'wiki-info.yaml'; import {input} from '#composite'; -import find from '#find'; import Thing from '#thing'; import {parseContributionPresets} from '#yaml'; @@ -15,7 +14,7 @@ import { } from '#validators'; import {exitWithoutDependency} from '#composite/control-flow'; -import {contentString, flag, name, referenceList, wikiData} +import {contentString, flag, name, referenceList, soupyFind} from '#composite/wiki-properties'; export class WikiInfo extends Thing { @@ -71,8 +70,7 @@ export class WikiInfo extends Thing { divideTrackListsByGroups: referenceList({ class: input.value(Group), - find: input.value(find.group), - data: 'groupData', + find: soupyFind.input('group'), }), contributionPresets: { @@ -99,6 +97,8 @@ export class WikiInfo extends Thing { // Update only + find: soupyFind(), + searchDataAvailable: { flags: {update: true}, update: { @@ -106,10 +106,6 @@ export class WikiInfo extends Thing { default: false, }, }, - - groupData: wikiData({ - class: input.value(Group), - }), }); static [Thing.yamlDocumentSpec] = { diff --git a/src/data/yaml.js b/src/data/yaml.js index 64223662..a1eb73fb 100644 --- a/src/data/yaml.js +++ b/src/data/yaml.js @@ -1225,93 +1225,92 @@ export async function loadAndProcessDataDocuments(dataSteps, {dataPath}) { // Data linking! Basically, provide (portions of) wikiData to the Things which // require it - they'll expose dynamically computed properties as a result (many // of which are required for page HTML generation and other expected behavior). -export function linkWikiDataArrays(wikiData) { +export function linkWikiDataArrays(wikiData, {bindFind, bindReverse}) { const linkWikiDataSpec = new Map([ + // entries must be present here even without any properties to explicitly + // link if the 'find' or 'reverse' properties will be implicitly linked + [wikiData.albumData, [ 'albumData', - 'artTagData', - 'artistData', - 'groupData', 'trackData', 'wikiInfo', ]], - [wikiData.artTagData, [ - 'albumData', - 'trackData', - ]], + [wikiData.artTagData, [/* reverse */]], - [wikiData.artistData, [ - 'albumData', - 'artistData', - 'flashData', - 'groupData', - 'trackData', - ]], + [wikiData.artistData, [/* find, reverse */]], [wikiData.flashData, [ - 'artistData', - 'flashActData', - 'trackData', 'wikiInfo', ]], - [wikiData.flashActData, [ - 'flashData', - 'flashSideData', - ]], + [wikiData.flashActData, [/* find, reverse */]], - [wikiData.flashSideData, [ - 'flashActData', - ]], + [wikiData.flashSideData, [/* find */]], - [wikiData.groupData, [ - 'albumData', - 'artistData', - 'groupCategoryData', - ]], + [wikiData.groupData, [/* find, reverse */]], - [wikiData.groupCategoryData, [ - 'groupData', - ]], + [wikiData.groupCategoryData, [/* find */]], - [wikiData.homepageLayout?.rows, [ - 'albumData', - 'groupData', - ]], + [wikiData.homepageLayout.rows, [/* find */]], [wikiData.trackData, [ 'albumData', - 'artTagData', - 'artistData', - 'flashData', 'trackData', - 'trackSectionData', 'wikiInfo', ]], - [wikiData.trackSectionData, [ - 'albumData', - ]], + [wikiData.trackSectionData, [/* reverse */]], - [[wikiData.wikiInfo], [ - 'groupData', - ]], + [[wikiData.wikiInfo], [/* find */]], ]); + const constructorHasFindMap = new Map(); + const constructorHasReverseMap = new Map(); + + const boundFind = bindFind(wikiData); + const boundReverse = bindReverse(wikiData); + for (const [things, keys] of linkWikiDataSpec.entries()) { if (things === undefined) continue; + for (const thing of things) { if (thing === undefined) continue; + + let hasFind; + if (constructorHasFindMap.has(thing.constructor)) { + hasFind = constructorHasFindMap.get(thing.constructor); + } else { + hasFind = 'find' in thing; + constructorHasFindMap.set(thing.constructor, hasFind); + } + + if (hasFind) { + thing.find = boundFind; + } + + let hasReverse; + if (constructorHasReverseMap.has(thing.constructor)) { + hasReverse = constructorHasReverseMap.get(thing.constructor); + } else { + hasReverse = 'reverse' in thing; + constructorHasReverseMap.set(thing.constructor, hasReverse); + } + + if (hasReverse) { + thing.reverse = boundReverse; + } + for (const key of keys) { if (!(key in wikiData)) continue; + thing[key] = wikiData[key]; } } } } -export function sortWikiDataArrays(dataSteps, wikiData) { +export function sortWikiDataArrays(dataSteps, wikiData, {bindFind, bindReverse}) { for (const [key, value] of Object.entries(wikiData)) { if (!Array.isArray(value)) continue; wikiData[key] = value.slice(); @@ -1327,7 +1326,7 @@ export function sortWikiDataArrays(dataSteps, wikiData) { // slices instead of the original arrays) - this is so that the object // caching system understands that it's working with a new ordering. // We still need to actually provide those updated arrays over again! - linkWikiDataArrays(wikiData); + linkWikiDataArrays(wikiData, {bindFind, bindReverse}); } // Utility function for loading all wiki data from the provided YAML data @@ -1339,6 +1338,7 @@ export function sortWikiDataArrays(dataSteps, wikiData) { export async function quickLoadAllFromYAML(dataPath, { find, bindFind, + bindReverse, getAllFindSpecs, showAggregate: customShowAggregate = showAggregate, @@ -1363,7 +1363,7 @@ export async function quickLoadAllFromYAML(dataPath, { } } - linkWikiDataArrays(wikiData); + linkWikiDataArrays(wikiData, {bindFind, bindReverse}); try { reportDirectoryErrors(wikiData, {getAllFindSpecs}); @@ -1389,7 +1389,7 @@ export async function quickLoadAllFromYAML(dataPath, { logWarn`Content text errors found.`; } - sortWikiDataArrays(dataSteps, wikiData); + sortWikiDataArrays(dataSteps, wikiData, {bindFind, bindReverse}); return wikiData; } diff --git a/src/file-size-preloader.js b/src/file-size-preloader.js index 4eadde7b..b2a55407 100644 --- a/src/file-size-preloader.js +++ b/src/file-size-preloader.js @@ -18,8 +18,10 @@ // are very, very fast. import {stat} from 'node:fs/promises'; +import {relative, resolve, sep} from 'node:path'; import {logWarn} from '#cli'; +import {filterMultipleArrays, transposeArrays} from '#sugar'; export default class FileSizePreloader { #paths = []; @@ -31,6 +33,10 @@ export default class FileSizePreloader { hadErrored = false; + constructor({prefix = ''} = {}) { + this.prefix = prefix; + } + loadPaths(...paths) { this.#paths.push(...paths.filter((p) => !this.#paths.includes(p))); return this.#startLoadingPaths(); @@ -45,9 +51,9 @@ export default class FileSizePreloader { return this.#loadingPromise; } - this.#loadingPromise = new Promise((resolve) => { - this.#resolveLoadingPromise = resolve; - }); + ({promise: this.#loadingPromise, + resolve: this.#resolveLoadingPromise} = + Promise.withResolvers()); this.#loadNextPath(); @@ -96,9 +102,54 @@ export default class FileSizePreloader { } getSizeOfPath(path) { + let size = this.#getSizeOfPath(path); + if (size || !this.prefix) return size; + const path2 = resolve(this.prefix, path); + if (path2 === path) return null; + return this.#getSizeOfPath(path2); + } + + #getSizeOfPath(path) { const index = this.#paths.indexOf(path); if (index === -1) return null; if (index > this.#loadedPathIndex) return null; return this.#sizes[index]; } + + saveAsCache() { + const entries = + transposeArrays([ + this.#paths.slice(0, this.#loadedPathIndex) + .map(path => relative(this.prefix, path)), + + this.#sizes.slice(0, this.#loadedPathIndex), + ]); + + // Do not be alarmed: This cannot be meaningfully moved to + // the top because stringifyCache sorts alphabetically lol + entries.push(['_separator', sep]); + + return Object.fromEntries(entries); + } + + loadFromCache(cache) { + const {_separator: cacheSep, ...rest} = cache; + const entries = Object.entries(rest); + let [newPaths, newSizes] = transposeArrays(entries); + + if (sep !== cacheSep) { + newPaths = newPaths.map(p => p.split(cacheSep).join(sep)); + } + + newPaths = newPaths.map(p => resolve(this.prefix, p)); + + filterMultipleArrays( + newPaths, + newSizes, + path => !this.#paths.includes(path)); + + this.#paths.splice(this.#loadedPathIndex + 1, 0, ...newPaths); + this.#sizes.splice(this.#loadedPathIndex + 1, 0, ...newSizes); + this.#loadedPathIndex += entries.length; + } } diff --git a/src/find-reverse.js b/src/find-reverse.js new file mode 100644 index 00000000..f31d3c45 --- /dev/null +++ b/src/find-reverse.js @@ -0,0 +1,137 @@ +// Helpers common to #find and #reverse logic. + +import thingConstructors from '#things'; + +export function getAllSpecs({ + word, + constructorKey, + + hardcodedSpecs, + postprocessSpec, +}) { + try { + thingConstructors; + } catch (error) { + throw new Error(`Thing constructors aren't ready yet, can't get all ${word} specs`); + } + + const specs = {...hardcodedSpecs}; + + for (const thingConstructor of Object.values(thingConstructors)) { + const thingSpecs = thingConstructor[constructorKey]; + if (!thingSpecs) continue; + + for (const [key, spec] of Object.entries(thingSpecs)) { + specs[key] = + postprocessSpec(spec, { + thingConstructor, + }); + } + } + + return specs; +} + +export function findSpec(key, { + word, + constructorKey, + + hardcodedSpecs, + postprocessSpec, +}) { + if (Object.hasOwn(hardcodedSpecs, key)) { + return hardcodedSpecs[key]; + } + + try { + thingConstructors; + } catch (error) { + throw new Error(`Thing constructors aren't ready yet, can't check if "${word}.${key}" available`); + } + + for (const thingConstructor of Object.values(thingConstructors)) { + const thingSpecs = thingConstructor[constructorKey]; + if (!thingSpecs) continue; + + if (Object.hasOwn(thingSpecs, key)) { + return postprocessSpec(thingSpecs[key], { + thingConstructor, + }); + } + } + + throw new Error(`"${word}.${key}" isn't available`); +} + +export function tokenProxy({ + findSpec, + prepareBehavior, + + handle: customHandle = + (_key) => undefined, +}) { + return new Proxy({}, { + get: (store, key) => { + const custom = customHandle(key); + if (custom !== undefined) { + return custom; + } + + if (!Object.hasOwn(store, key)) { + let behavior = (...args) => { + // This will error if the spec isn't available... + const spec = findSpec(key); + + // ...or, if it is available, replace this function with the + // ready-for-use find function made out of that spec. + return (behavior = prepareBehavior(spec))(...args); + }; + + store[key] = (...args) => behavior(...args); + store[key][tokenKey] = key; + } + + return store[key]; + }, + }); +} + +export function bind(wikiData, opts1, { + getAllSpecs, + prepareBehavior, +}) { + const specs = getAllSpecs(); + + const bound = {}; + + for (const [key, spec] of Object.entries(specs)) { + if (!spec.bindTo) continue; + + const behavior = prepareBehavior(spec); + + const data = + (spec.bindTo === 'wikiData' + ? wikiData + : wikiData[spec.bindTo]); + + bound[key] = + (opts1 + ? (ref, opts2) => + (opts2 + ? behavior(ref, data, {...opts1, ...opts2}) + : behavior(ref, data, opts1)) + : (ref, opts2) => + (opts2 + ? behavior(ref, data, opts2) + : behavior(ref, data))); + + bound[key][boundData] = data; + bound[key][boundOptions] = opts1 ?? {}; + } + + return bound; +} + +export const tokenKey = Symbol.for('find.tokenKey'); +export const boundData = Symbol.for('find.boundData'); +export const boundOptions = Symbol.for('find.boundOptions'); diff --git a/src/find.js b/src/find.js index d647419a..e590bc4f 100644 --- a/src/find.js +++ b/src/find.js @@ -5,6 +5,16 @@ import {compareObjects, stitchArrays, typeAppearance} from '#sugar'; import thingConstructors from '#things'; import {isFunction, validateArrayItems} from '#validators'; +import * as fr from './find-reverse.js'; + +import { + tokenKey as findTokenKey, + boundData as boundFindData, + boundOptions as boundFindOptions, +} from './find-reverse.js'; + +export {findTokenKey, boundFindData, boundFindOptions}; + function warnOrThrow(mode, message) { if (mode === 'error') { throw new Error(message); @@ -24,7 +34,7 @@ export function processAvailableMatchesByName(data, { include = _thing => true, getMatchableNames = thing => - (Object.hasOwn(thing, 'name') + (thing.constructor.hasPropertyDescriptor('name') ? [thing.name] : []), @@ -62,7 +72,7 @@ export function processAvailableMatchesByDirectory(data, { include = _thing => true, getMatchableDirectories = thing => - (Object.hasOwn(thing, 'directory') + (thing.constructor.hasPropertyDescriptor('directory') ? [thing.directory] : [null]), @@ -240,6 +250,14 @@ const hardcodedFindSpecs = { }, }; +const findReverseHelperConfig = { + word: `find`, + constructorKey: Symbol.for('Thing.findSpecs'), + + hardcodedSpecs: hardcodedFindSpecs, + postprocessSpec: postprocessFindSpec, +}; + export function postprocessFindSpec(spec, {thingConstructor}) { const newSpec = {...spec}; @@ -261,58 +279,13 @@ export function postprocessFindSpec(spec, {thingConstructor}) { } export function getAllFindSpecs() { - try { - thingConstructors; - } catch (error) { - throw new Error(`Thing constructors aren't ready yet, can't get all find specs`); - } - - const findSpecs = {...hardcodedFindSpecs}; - - for (const thingConstructor of Object.values(thingConstructors)) { - const thingFindSpecs = thingConstructor[Symbol.for('Thing.findSpecs')]; - if (!thingFindSpecs) continue; - - for (const [key, spec] of Object.entries(thingFindSpecs)) { - findSpecs[key] = - postprocessFindSpec(spec, { - thingConstructor, - }); - } - } - - return findSpecs; + return fr.getAllSpecs(findReverseHelperConfig); } export function findFindSpec(key) { - if (Object.hasOwn(hardcodedFindSpecs, key)) { - return hardcodedFindSpecs[key]; - } - - try { - thingConstructors; - } catch (error) { - throw new Error(`Thing constructors aren't ready yet, can't check if "find.${key}" available`); - } - - for (const thingConstructor of Object.values(thingConstructors)) { - const thingFindSpecs = thingConstructor[Symbol.for('Thing.findSpecs')]; - if (!thingFindSpecs) continue; - - if (Object.hasOwn(thingFindSpecs, key)) { - return postprocessFindSpec(thingFindSpecs[key], { - thingConstructor, - }); - } - } - - throw new Error(`"find.${key}" isn't available`); + return fr.findSpec(key, findReverseHelperConfig); } -export const findTokenKey = Symbol.for('find.findTokenKey'); -export const boundFindData = Symbol.for('find.boundFindData'); -export const boundFindOptions = Symbol.for('find.boundFindOptions'); - function findMixedHelper(config) { const keys = Object.keys(config), @@ -425,27 +398,14 @@ export function findMixed(config) { return findMixedStore.get(config); } -export default new Proxy({}, { - get: (store, key) => { +export default fr.tokenProxy({ + findSpec: findFindSpec, + prepareBehavior: findHelper, + + handle(key) { if (key === 'mixed') { return findMixed; } - - if (!Object.hasOwn(store, key)) { - let behavior = (...args) => { - // This will error if the find spec isn't available... - const findSpec = findFindSpec(key); - - // ...or, if it is available, replace this function with the - // ready-for-use find function made out of that find spec. - return (behavior = findHelper(findSpec))(...args); - }; - - store[key] = (...args) => behavior(...args); - store[key][findTokenKey] = key; - } - - return store[key]; }, }); @@ -454,33 +414,13 @@ export default new Proxy({}, { // function. Note that this caches the arrays read from wikiData right when it's // called, so if their values change, you'll have to continue with a fresh call // to bindFind. -export function bindFind(wikiData, opts1) { - const findSpecs = getAllFindSpecs(); - - const boundFindFns = {}; - - for (const [key, spec] of Object.entries(findSpecs)) { - if (!spec.bindTo) continue; - - const findFn = findHelper(spec); - const thingData = wikiData[spec.bindTo]; - - boundFindFns[key] = - (opts1 - ? (ref, opts2) => - (opts2 - ? findFn(ref, thingData, {...opts1, ...opts2}) - : findFn(ref, thingData, opts1)) - : (ref, opts2) => - (opts2 - ? findFn(ref, thingData, opts2) - : findFn(ref, thingData))); - - boundFindFns[key][boundFindData] = thingData; - boundFindFns[key][boundFindOptions] = opts1 ?? {}; - } +export function bindFind(wikiData, opts) { + const boundFind = fr.bind(wikiData, opts, { + getAllSpecs: getAllFindSpecs, + prepareBehavior: findHelper, + }); - boundFindFns.mixed = findMixed; + boundFind.mixed = findMixed; - return boundFindFns; + return boundFind; } diff --git a/src/gen-thumbs.js b/src/gen-thumbs.js index 6c82761f..3ccd8ce2 100644 --- a/src/gen-thumbs.js +++ b/src/gen-thumbs.js @@ -163,6 +163,7 @@ import { import dimensionsOf from 'image-size'; import CacheableObject from '#cacheable-object'; +import {stringifyCache} from '#cli'; import {commandExists, isMain, promisifyProcess, traverse} from '#node-utils'; import {sortByName} from '#sort'; @@ -346,28 +347,6 @@ export function getThumbnailsAvailableForDimensions([width, height]) { ]; } -function stringifyCache(cache) { - if (Object.keys(cache).length === 0) { - return `{}`; - } - - const entries = Object.entries(cache); - sortByName(entries, {getName: entry => entry[0]}); - - return [ - `{`, - entries - .map(([key, value]) => [JSON.stringify(key), JSON.stringify(value)]) - .map(([key, value]) => `${key}: ${value}`) - .map((line, index, array) => - (index < array.length - 1 - ? `${line},` - : line)) - .map(line => ` ${line}`), - `}`, - ].flat().join('\n'); -} - getThumbnailsAvailableForDimensions.all = Object.entries(thumbnailSpec) .map(([name, {size}]) => [name, size]) diff --git a/src/listing-spec.js b/src/listing-spec.js index bfea397c..749f009a 100644 --- a/src/listing-spec.js +++ b/src/listing-spec.js @@ -238,6 +238,27 @@ listingSpec.push({ groupUnderOther: true, }); +// Dunkass mock. Listings should be Things! In the fuuuuture! +class Listing { + static properties = {}; + + constructor() { + Object.assign(this, this.constructor.properties); + } + + static hasPropertyDescriptor(key) { + return Object.hasOwn(this.properties, key); + } +} + +for (const [index, listing] of listingSpec.entries()) { + class ListingSubclass extends Listing { + static properties = listing; + } + + listingSpec.splice(index, 1, new ListingSubclass); +} + { const errors = []; diff --git a/src/reverse.js b/src/reverse.js new file mode 100644 index 00000000..9ad5c8a7 --- /dev/null +++ b/src/reverse.js @@ -0,0 +1,160 @@ +import * as fr from './find-reverse.js'; + +import {sortByDate} from '#sort'; +import {stitchArrays} from '#sugar'; + +function checkUnique(value) { + if (value.length === 0) { + return null; + } else if (value.length === 1) { + return value[0]; + } else { + throw new Error( + `Requested unique referencing thing, ` + + `but ${value.length} reference this`); + } +} + +function reverseHelper(spec) { + const cache = new WeakMap(); + + return (thing, data, { + unique = false, + } = {}) => { + // Check for an existing cache record which corresponds to this data. + // If it exists, query it for the requested thing, and return that; + // if it doesn't, create it and put it where it needs to be. + + if (cache.has(data)) { + const value = cache.get(data).get(thing) ?? []; + + if (unique) { + return checkUnique(value); + } else { + return value; + } + } + + const cacheRecord = new WeakMap(); + cache.set(data, cacheRecord); + + // Get the referencing and referenced things. This is the meat of how + // one reverse spec is different from another. If the spec includes a + // 'tidy' step, use that to finalize the referencing things, the way + // they'll be recorded as results. + + const interstitialReferencingThings = + (spec.bindTo === 'wikiData' + ? spec.referencing(data) + : data.flatMap(thing => spec.referencing(thing))); + + const referencedThings = + interstitialReferencingThings.map(thing => spec.referenced(thing)); + + const referencingThings = + (spec.tidy + ? interstitialReferencingThings.map(thing => spec.tidy(thing)) + : interstitialReferencingThings); + + // Actually fill in the cache record. Since we're building up a *reverse* + // reference list, track connections in terms of the referenced thing. + // Also gather all referenced things into a set, for sorting purposes. + + const allReferencedThings = new Set(); + + stitchArrays({ + referencingThing: referencingThings, + referencedThings: referencedThings, + }).forEach(({referencingThing, referencedThings}) => { + for (const referencedThing of referencedThings) { + if (cacheRecord.has(referencedThing)) { + cacheRecord.get(referencedThing).push(referencingThing); + } else { + cacheRecord.set(referencedThing, [referencingThing]); + allReferencedThings.add(referencedThing); + } + } + }); + + // Sort the entries in the cache records, too, just by date. The rest of + // sorting should be handled externally - either preceding the reverse + // call (changing the data input) or following (sorting the output). + + for (const referencedThing of allReferencedThings) { + if (cacheRecord.has(referencedThing)) { + const referencingThings = cacheRecord.get(referencedThing); + sortByDate(referencingThings); + } + } + + // Then just pluck out the requested thing from the now-filled + // cache record! + + const value = cacheRecord.get(thing) ?? []; + + if (unique) { + return checkUnique(value); + } else { + return value; + } + }; +} + +const hardcodedReverseSpecs = { + // Artworks aren't Thing objects. + // This spec operates on albums and tracks alike! + artworksWhichReference: { + bindTo: 'wikiData', + + referencing: ({albumData, trackData}) => + [...albumData, ...trackData] + .flatMap(referencingThing => + referencingThing.referencedArtworks + .map(({thing: referencedThing, ...referenceDetails}) => ({ + referencingThing, + referencedThing, + referenceDetails, + }))), + + referenced: ({referencedThing}) => [referencedThing], + + tidy: ({referencingThing, referenceDetails}) => + ({thing: referencingThing, ...referenceDetails}), + }, +}; + +const findReverseHelperConfig = { + word: `reverse`, + constructorKey: Symbol.for('Thing.reverseSpecs'), + + hardcodedSpecs: hardcodedReverseSpecs, + postprocessSpec: postprocessReverseSpec, +}; + +export function postprocessReverseSpec(spec, {thingConstructor}) { + const newSpec = {...spec}; + + void thingConstructor; + + return newSpec; +} + +export function getAllReverseSpecs() { + return fr.getAllSpecs(findReverseHelperConfig); +} + +export function findReverseSpec(key) { + return fr.findSpec(key, findReverseHelperConfig); +} + +export default fr.tokenProxy({ + findSpec: findReverseSpec, + prepareBehavior: reverseHelper, +}); + +export function bindReverse(wikiData, opts) { + return fr.bind(wikiData, opts, { + getAllSpecs: getAllReverseSpecs, + prepareBehavior: reverseHelper, + }); +} diff --git a/src/static/css/site.css b/src/static/css/site.css index 6c853161..514154a5 100644 --- a/src/static/css/site.css +++ b/src/static/css/site.css @@ -228,12 +228,19 @@ body::before, .wallpaper-part { /* Design & Appearance - Layout elements */ +:root { + color-scheme: dark; +} + body { background: black; } body::before { - background-image: url("../../media/bg.jpg"); + /* This is where the basic background-image rule + * gets applied... but the path *to* that media file + * isn't part of the CSS itself anymore! + */ } body::before, .wallpaper-part { diff --git a/src/upd8.js b/src/upd8.js index f4c6326a..045bf139 100755 --- a/src/upd8.js +++ b/src/upd8.js @@ -34,22 +34,23 @@ import '#import-heck'; import {execSync} from 'node:child_process'; -import {readdir, readFile, stat} from 'node:fs/promises'; +import {readdir, readFile, stat, writeFile} from 'node:fs/promises'; import * as path from 'node:path'; import {fileURLToPath} from 'node:url'; import wrap from 'word-wrap'; -import {mapAggregate, showAggregate} from '#aggregate'; +import {mapAggregate, openAggregate, showAggregate} from '#aggregate'; import CacheableObject from '#cacheable-object'; +import {stringifyCache} from '#cli'; import {displayCompositeCacheAnalysis} from '#composite'; import find, {bindFind, getAllFindSpecs} from '#find'; import {processLanguageFile, watchLanguageFile, internalDefaultStringsFile} from '#language'; import {isMain, traverse} from '#node-utils'; +import {bindReverse} from '#reverse'; import {writeSearchData} from '#search'; import {sortByName} from '#sort'; -import {generateURLs, urlSpec} from '#urls'; import {identifyAllWebRoutes} from '#web-routes'; import { @@ -73,6 +74,7 @@ import { import { bindOpts, empty, + filterMultipleArrays, indentWrap as unboundIndentWrap, withEntries, } from '#sugar'; @@ -87,6 +89,15 @@ import genThumbs, { } from '#thumbs'; import { + applyLocalizedWithBaseDirectory, + applyURLSpecOverriding, + generateURLs, + getOrigin, + internalDefaultURLSpecFile, + processURLSpecFromFile, +} from '#urls'; + +import { getAllDataSteps, linkWikiDataArrays, loadYAMLDocumentsFromDataSteps, @@ -123,6 +134,7 @@ const defaultStepStatus = {status: STATUS_NOT_STARTED, annotation: null}; // This will be initialized and mutated over the course of main(). let stepStatusSummary; let showStepStatusSummary = false; +let showStepMemoryInSummary = false; async function main() { Error.stackTraceLimit = Infinity; @@ -138,8 +150,8 @@ async function main() { {...defaultStepStatus, name: `migrate thumbnails`, for: ['thumbs']}, - loadThumbnailCache: - {...defaultStepStatus, name: `load thumbnail cache file`, + loadOfflineThumbnailCache: + {...defaultStepStatus, name: `load offline thumbnail cache file`, for: ['thumbs', 'build']}, generateThumbnails: @@ -178,6 +190,14 @@ async function main() { {...defaultStepStatus, name: `precache nearly all data`, for: ['build']}, + loadURLFiles: + {...defaultStepStatus, name: `load internal & custom url spec files`, + for: ['build']}, + + loadOnlineThumbnailCache: + {...defaultStepStatus, name: `load online thumbnail cache file`, + for: ['thumbs', 'build']}, + // TODO: This should be split into load/watch steps. loadInternalDefaultLanguage: {...defaultStepStatus, name: `load internal default language`, @@ -203,6 +223,10 @@ async function main() { {...defaultStepStatus, name: `preload file sizes`, for: ['build']}, + loadOnlineFileSizeCache: + {...defaultStepStatus, name: `load online file size cache file`, + for: ['build']}, + buildSearchIndex: {...defaultStepStatus, name: `generate search index`, for: ['build', 'search']}, @@ -323,6 +347,16 @@ async function main() { type: 'value', }, + 'urls': { + help: `Specify which optional URL specs to use for this build, customizing where pages are generated or resources are accessed from`, + type: 'value', + }, + + 'show-url-spec': { + help: `Displays the entire computed URL spec, after the data folder's default override and optional specs are applied. This is mostly useful for progammer debugging!`, + type: 'flag', + }, + 'skip-directory-validation': { help: `Skips checking for duplicated directories, which speeds up the build but may cause the wiki to catch on fire`, type: 'flag', @@ -363,11 +397,21 @@ async function main() { type: 'flag', }, + 'refresh-online-thumbs': { + help: `Downloads a fresh copy of the online file size cache, so changes there are immediately reflected`, + type: 'flag', + }, + 'skip-file-sizes': { help: `Skips preloading file sizes for images and additional files, which will be left blank in the build`, type: 'flag', }, + 'refresh-online-file-sizes': { + help: `Downloads a fresh copy of the online file size cache, so changes there are immediately reflected`, + type: 'flag', + }, + 'skip-media-validation': { help: `Skips checking and reporting missing and misplaced media files, which isn't necessary if you aren't adding or removing data or updating directories`, type: 'flag', @@ -417,6 +461,11 @@ async function main() { type: 'flag', }, + 'show-step-memory': { + help: `Include total process memory usage traces at the time each top-level build step ends. Use with --show-step-summary. This is mostly useful for programmer debugging!`, + type: 'flag', + }, + 'queue-size': { help: `Process more or fewer disk files at once to optimize performance or avoid I/O errors, unlimited if set to 0 (between 500 and 700 is usually a safe range for building HSMusic on Windows machines)\nDefaults to ${defaultQueueSize}`, type: 'value', @@ -439,14 +488,6 @@ async function main() { }, magick: {alias: 'magick-threads'}, - // This option is super slow and has the potential for bugs! It puts - // CacheableObject in a mode where every instance is a Proxy which will - // keep track of invalid property accesses. - 'show-invalid-property-accesses': { - help: `Report accesses at runtime to nonexistant properties on wiki data objects, at a dramatic performance cost\n(Internal/development use only)`, - type: 'flag', - }, - 'precache-mode': { help: `Change the way certain runtime-computed values are preemptively evaluated and cached\n\n` + @@ -485,6 +526,7 @@ async function main() { }); showStepStatusSummary = cliOptions['show-step-summary'] ?? false; + showStepMemoryInSummary = cliOptions['show-step-memory'] ?? false; if (cliOptions['help']) { console.log( @@ -565,7 +607,9 @@ async function main() { const showAggregateTraces = cliOptions['show-traces'] ?? false; const precacheMode = cliOptions['precache-mode'] ?? 'common'; - const showInvalidPropertyAccesses = cliOptions['show-invalid-property-accesses'] ?? false; + + const wantedURLSpecKeys = cliOptions['urls'] ?? []; + const showURLSpec = cliOptions['show-url-spec'] ?? false; // Makes writing nicer on the CPU and file I/O parts of the OS, with a // marginal performance deficit while waiting for file writes to finish @@ -888,14 +932,14 @@ async function main() { logInfo`Next scheduled is in ${whenst(delay - delta)}, or by using ${'--refresh-search'}.`; Object.assign(stepStatusSummary.buildSearchIndex, { status: STATUS_NOT_APPLICABLE, - annotation: `earlier than scheduled based on file mtime`, + annotation: `earlier than scheduled`, }); } else { logInfo`Search index hasn't been generated for a little while.`; logInfo`It'll be generated this build, then again in ${whenst(delay)}.`; Object.assign(stepStatusSummary.buildSearchIndex, { status: STATUS_NOT_STARTED, - annotation: `past when shceduled based on file mtime`, + annotation: `past when shceduled`, }); } @@ -929,7 +973,7 @@ async function main() { } if (stepStatusSummary.generateThumbnails.status === STATUS_NOT_STARTED) { - Object.assign(stepStatusSummary.loadThumbnailCache, { + Object.assign(stepStatusSummary.loadOfflineThumbnailCache, { status: STATUS_NOT_APPLICABLE, annotation: `using cache from thumbnail generation`, }); @@ -1081,6 +1125,7 @@ async function main() { status: STATUS_FATAL_ERROR, annotation: `--new-thumbs provided but regeneration not needed`, timeEnd: Date.now(), + memory: process.memoryUsage(), }); return false; @@ -1096,6 +1141,7 @@ async function main() { status: STATUS_FATAL_ERROR, annotation: mediaCachePathAnnotation, timeEnd: Date.now(), + memory: process.memoryUsage(), }); return false; @@ -1162,6 +1208,7 @@ async function main() { status: STATUS_FATAL_ERROR, annotation: mediaCachePathAnnotation, timeEnd: Date.now(), + memory: process.memoryUsage(), }); return false; @@ -1173,6 +1220,7 @@ async function main() { status: STATUS_DONE_CLEAN, annotation: mediaCachePathAnnotation, timeEnd: Date.now(), + memory: process.memoryUsage(), }); if (stepStatusSummary.migrateThumbnails.status === STATUS_NOT_STARTED) { @@ -1192,6 +1240,7 @@ async function main() { status: STATUS_FATAL_ERROR, annotation: `view log for details`, timeEnd: Date.now(), + memory: process.memoryUsage(), }); return false; @@ -1203,6 +1252,7 @@ async function main() { Object.assign(stepStatusSummary.migrateThumbnails, { status: STATUS_DONE_CLEAN, timeEnd: Date.now(), + memory: process.memoryUsage(), }); return true; @@ -1217,16 +1267,17 @@ async function main() { }; if ( - stepStatusSummary.loadThumbnailCache.status === STATUS_NOT_STARTED && + stepStatusSummary.loadOfflineThumbnailCache.status === STATUS_NOT_STARTED && stepStatusSummary.generateThumbnails.status === STATUS_NOT_STARTED ) { - throw new Error(`Unable to continue with both loadThumbnailCache and generateThumbnails`); + throw new Error(`Unable to continue with both loadOfflineThumbnailCache and generateThumbnails`); } let thumbsCache; - if (stepStatusSummary.loadThumbnailCache.status === STATUS_NOT_STARTED) { - Object.assign(stepStatusSummary.loadThumbnailCache, { + // TODO: Skip this step if we're using online thumbs + if (stepStatusSummary.loadOfflineThumbnailCache.status === STATUS_NOT_STARTED) { + Object.assign(stepStatusSummary.loadOfflineThumbnailCache, { status: STATUS_STARTED_NOT_DONE, timeStart: Date.now(), }); @@ -1242,10 +1293,11 @@ async function main() { logError`that you'll be good to go and don't need to process thumbnails` logError`again!`; - Object.assign(stepStatusSummary.loadThumbnailCache, { + Object.assign(stepStatusSummary.loadOfflineThumbnailCache, { status: STATUS_FATAL_ERROR, annotation: `cache does not exist`, timeEnd: Date.now(), + memory: process.memoryUsage(), }); return false; @@ -1259,10 +1311,11 @@ async function main() { logError`to help you out with troubleshooting!`; logError`${'https://hsmusic.wiki/discord/'}`; - Object.assign(stepStatusSummary.loadThumbnailCache, { + Object.assign(stepStatusSummary.loadOfflineThumbnailCache, { status: STATUS_FATAL_ERROR, annotation: `cache malformed or unreadable`, timeEnd: Date.now(), + memory: process.memoryUsage(), }); return false; @@ -1271,9 +1324,10 @@ async function main() { logInfo`Thumbnail cache file successfully read.`; - Object.assign(stepStatusSummary.loadThumbnailCache, { + Object.assign(stepStatusSummary.loadOfflineThumbnailCache, { status: STATUS_DONE_CLEAN, timeEnd: Date.now(), + memory: process.memoryUsage(), }); logInfo`Skipping thumbnail generation.`; @@ -1301,6 +1355,7 @@ async function main() { status: STATUS_FATAL_ERROR, annotation: `view log for details`, timeEnd: Date.now(), + memory: process.memoryUsage(), }); return false; @@ -1309,6 +1364,7 @@ async function main() { Object.assign(stepStatusSummary.generateThumbnails, { status: STATUS_DONE_CLEAN, timeEnd: Date.now(), + memory: process.memoryUsage(), }); if (thumbsOnly) { @@ -1320,10 +1376,6 @@ async function main() { thumbsCache = {}; } - if (showInvalidPropertyAccesses) { - CacheableObject.DEBUG_SLOW_TRACK_INVALID_PROPERTIES = true; - } - Object.assign(stepStatusSummary.loadDataFiles, { status: STATUS_STARTED_NOT_DONE, timeStart: Date.now(), @@ -1346,6 +1398,7 @@ async function main() { status: STATUS_FATAL_ERROR, annotation: `javascript error - view log for details`, timeEnd: Date.now(), + memory: process.memoryUsage(), }); return false; @@ -1385,6 +1438,7 @@ async function main() { status: STATUS_FATAL_ERROR, annotation: `error loading data files`, timeEnd: Date.now(), + memory: process.memoryUsage(), }); return false; @@ -1487,6 +1541,7 @@ async function main() { status: STATUS_FATAL_ERROR, annotation: `wiki info object not available`, timeEnd: Date.now(), + memory: process.memoryUsage(), }); return false; @@ -1499,6 +1554,7 @@ async function main() { Object.assign(stepStatusSummary.loadDataFiles, { status: STATUS_DONE_CLEAN, timeEnd: Date.now(), + memory: process.memoryUsage(), }); } else { logWarn`This might indicate some fields in the YAML data weren't formatted`; @@ -1513,6 +1569,7 @@ async function main() { status: STATUS_HAS_WARNINGS, annotation: `view log for details`, timeEnd: Date.now(), + memory: process.memoryUsage(), }); } } @@ -1526,11 +1583,12 @@ async function main() { timeStart: Date.now(), }); - linkWikiDataArrays(wikiData); + linkWikiDataArrays(wikiData, {bindFind, bindReverse}); Object.assign(stepStatusSummary.linkWikiDataArrays, { status: STATUS_DONE_CLEAN, timeEnd: Date.now(), + memory: process.memoryUsage(), }); if (precacheMode === 'common') { @@ -1602,6 +1660,7 @@ async function main() { status: STATUS_FATAL_ERROR, annotation: `see log for details`, timeEnd: Date.now(), + memory: process.memoryUsage(), }); return false; @@ -1610,11 +1669,10 @@ async function main() { Object.assign(stepStatusSummary.precacheCommonData, { status: STATUS_DONE_CLEAN, timeEnd: Date.now(), + memory: process.memoryUsage(), }); } - const urls = generateURLs(urlSpec); - // Filter out any things with duplicate directories throughout the data, // warning about them too. @@ -1632,6 +1690,7 @@ async function main() { Object.assign(stepStatusSummary.reportDirectoryErrors, { status: STATUS_DONE_CLEAN, timeEnd: Date.now(), + memory: process.memoryUsage(), }); } catch (aggregate) { if (!paragraph) console.log(''); @@ -1649,6 +1708,7 @@ async function main() { status: STATUS_FATAL_ERROR, annotation: `duplicate directories found`, timeEnd: Date.now(), + memory: process.memoryUsage(), }); return false; @@ -1676,6 +1736,7 @@ async function main() { Object.assign(stepStatusSummary.filterReferenceErrors, { status: STATUS_DONE_CLEAN, timeEnd: Date.now(), + memory: process.memoryUsage(), }); } catch (error) { if (!paragraph) console.log(''); @@ -1693,6 +1754,7 @@ async function main() { status: STATUS_HAS_WARNINGS, annotation: `view log for details`, timeEnd: Date.now(), + memory: process.memoryUsage(), }); } } @@ -1712,6 +1774,7 @@ async function main() { Object.assign(stepStatusSummary.reportContentTextErrors, { status: STATUS_DONE_CLEAN, timeEnd: Date.now(), + memory: process.memoryUsage(), }); } catch (error) { if (!paragraph) console.log(''); @@ -1728,6 +1791,7 @@ async function main() { status: STATUS_HAS_WARNINGS, annotation: `view log for details`, timeEnd: Date.now(), + memory: process.memoryUsage(), }); } } @@ -1740,11 +1804,12 @@ async function main() { timeStart: Date.now(), }); - sortWikiDataArrays(yamlDataSteps, wikiData); + sortWikiDataArrays(yamlDataSteps, wikiData, {bindFind, bindReverse}); Object.assign(stepStatusSummary.sortWikiDataArrays, { status: STATUS_DONE_CLEAN, timeEnd: Date.now(), + memory: process.memoryUsage(), }); if (precacheMode === 'all') { @@ -1768,6 +1833,7 @@ async function main() { Object.assign(stepStatusSummary.precacheAllData, { status: STATUS_DONE_CLEAN, timeEnd: Date.now(), + memory: process.memoryUsage(), }); } @@ -1779,6 +1845,354 @@ async function main() { } } + Object.assign(stepStatusSummary.loadURLFiles, { + status: STATUS_STARTED_NOT_DONE, + timeStart: Date.now(), + }); + + let internalURLSpec = {}; + + try { + let aggregate; + ({aggregate, result: internalURLSpec} = + await processURLSpecFromFile(internalDefaultURLSpecFile)); + + aggregate.close(); + } catch (error) { + niceShowAggregate(error); + logError`Couldn't load internal default URL spec.`; + logError`This is required to build the wiki, so stopping here.`; + fileIssue(); + + Object.assign(stepStatusSummary.loadURLFiles, { + status: STATUS_FATAL_ERROR, + annotation: `see log for details`, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + + return false; + } + + // We'll mutate this as we load other url spec files. + const urlSpec = structuredClone(internalURLSpec); + + const allURLSpecDataFiles = + (await readdir(dataPath)) + .filter(name => + name.startsWith('urls') && + ['.json', '.yaml'].includes(path.extname(name))) + .sort() /* Just in case... */ + .map(name => path.join(dataPath, name)); + + const getURLSpecKeyFromFile = file => { + const base = path.basename(file, path.extname(file)); + if (base === 'urls') { + return base; + } else { + return base.replace(/^urls-/, ''); + } + }; + + const isDefaultURLSpecFile = file => + getURLSpecKeyFromFile(file) === 'urls'; + + const overrideDefaultURLSpecFile = + allURLSpecDataFiles.find(file => isDefaultURLSpecFile(file)); + + const optionalURLSpecDataFiles = + allURLSpecDataFiles.filter(file => !isDefaultURLSpecFile(file)); + + const optionalURLSpecDataKeys = + optionalURLSpecDataFiles.map(file => getURLSpecKeyFromFile(file)); + + const selectedURLSpecDataKeys = optionalURLSpecDataKeys.slice(); + const selectedURLSpecDataFiles = optionalURLSpecDataFiles.slice(); + + const {removed: [unusedURLSpecDataKeys]} = + filterMultipleArrays( + selectedURLSpecDataKeys, + selectedURLSpecDataFiles, + (key, _file) => wantedURLSpecKeys.includes(key)); + + if (!empty(selectedURLSpecDataKeys)) { + logInfo`Using these optional URL specs: ${selectedURLSpecDataKeys.join(', ')}`; + if (!empty(unusedURLSpecDataKeys)) { + logInfo`Other available optional URL specs: ${unusedURLSpecDataKeys.join(', ')}`; + } + } else if (!empty(unusedURLSpecDataKeys)) { + logInfo`Not using any optional URL specs.`; + logInfo`These are available with --urls: ${unusedURLSpecDataKeys.join(', ')}`; + } + + if (overrideDefaultURLSpecFile) { + try { + let aggregate; + let overrideDefaultURLSpec; + + ({aggregate, result: overrideDefaultURLSpec} = + await processURLSpecFromFile(overrideDefaultURLSpecFile)); + + aggregate.close(); + + ({aggregate} = + applyURLSpecOverriding(overrideDefaultURLSpec, urlSpec)); + + aggregate.close(); + } catch (error) { + niceShowAggregate(error); + logError`Errors loading this data repo's ${'urls.yaml'} file.`; + logError`This provides essential overrides for this wiki,`; + logError`so stopping here. Debug the errors to continue.`; + + Object.assign(stepStatusSummary.loadURLFiles, { + status: STATUS_FATAL_ERROR, + annotation: `see log for details`, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + + return false; + } + } + + const processURLSpecsAggregate = + openAggregate({message: `Errors processing URL specs`}); + + const selectedURLSpecs = + processURLSpecsAggregate.receive( + await Promise.all( + selectedURLSpecDataFiles + .map(file => processURLSpecFromFile(file)))); + + for (const selectedURLSpec of selectedURLSpecs) { + processURLSpecsAggregate.receive( + applyURLSpecOverriding(selectedURLSpec, urlSpec)); + } + + try { + processURLSpecsAggregate.close(); + } catch (error) { + niceShowAggregate(error); + logWarn`There were errors loading the optional URL specs you`; + logWarn`selected using ${'--urls'}. Since they might misfunction,`; + logWarn`debug the errors or remove the failing ones from ${'--urls'}.`; + + Object.assign(stepStatusSummary.loadURLFiles, { + status: STATUS_FATAL_ERROR, + annotation: `see log for details`, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + + return false; + } + + if (showURLSpec) { + if (!paragraph) console.log(''); + + logInfo`Here's the final URL spec, via ${'--show-url-spec'}:` + console.log(urlSpec); + console.log(''); + + paragraph = true; + } + + Object.assign(stepStatusSummary.loadURLFiles, { + status: STATUS_DONE_CLEAN, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + + if (!getOrigin(urlSpec.thumb.prefix)) { + Object.assign(stepStatusSummary.loadOnlineThumbnailCache, { + status: STATUS_NOT_APPLICABLE, + annotation: `using offline thumbs`, + }); + } + + if (getOrigin(urlSpec.media.prefix)) { + Object.assign(stepStatusSummary.preloadFileSizes, { + status: STATUS_NOT_APPLICABLE, + annotation: `using online media`, + }); + } else { + Object.assign(stepStatusSummary.loadOnlineFileSizeCache, { + status: STATUS_NOT_APPLICABLE, + annotation: `using offline media`, + }); + } + + applyLocalizedWithBaseDirectory(urlSpec); + + const urls = generateURLs(urlSpec); + + if (stepStatusSummary.loadOnlineThumbnailCache.status === STATUS_NOT_STARTED) loadOnlineThumbnailCache: { + Object.assign(stepStatusSummary.loadOnlineThumbnailCache, { + status: STATUS_STARTED_NOT_DONE, + timeStart: Date.now(), + }); + + let onlineThumbsCache = null; + + const cacheFile = path.join(wikiCachePath, 'online-thumbnail-cache.json'); + + let readError = null; + let writeError = null; + + if (!cliOptions['refresh-online-thumbs']) { + try { + onlineThumbsCache = JSON.parse(await readFile(cacheFile)); + } catch (caughtError) { + readError = caughtError; + } + } + + if (onlineThumbsCache) obliterateLocalCopy: { + if (!onlineThumbsCache._urlPrefix) { + // Well, it doesn't even count. + onlineThumbsCache = null; + break obliterateLocalCopy; + } + + if (onlineThumbsCache._urlPrefix !== urlSpec.thumb.prefix) { + logInfo`Local copy of online thumbs cache is for a different prefix.`; + logInfo`It'll be downloaded and replaced, for reuse next time.`; + paragraph = false; + + onlineThumbsCache = null; + break obliterateLocalCopy; + } + + let stats; + try { + stats = await stat(cacheFile); + } catch { + logInfo`Unable to get the stats of local copy of online thumbs cache...`; + logInfo`This is really weird, since we *were* able to read it...`; + logInfo`We're just going to try writing to it and download fresh!`; + paragraph = false; + + onlineThumbsCache = null; + break obliterateLocalCopy; + } + + const delta = Date.now() - stats.mtimeMs; + const minute = 60 * 1000; + const delay = 60 * minute; + + const whenst = duration => `~${Math.ceil(duration / minute)} min`; + + if (delta < delay) { + logInfo`Online thumbs cache was downloaded recently, skipping for this build.`; + logInfo`Next scheduled is in ${whenst(delay - delta)}, or by using ${'--refresh-online-thumbs'}.`; + paragraph = false; + + Object.assign(stepStatusSummary.loadOnlineThumbnailCache, { + status: STATUS_DONE_CLEAN, + annotation: `reusing local copy, earlier than scheduled`, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + + thumbsCache = onlineThumbsCache; + + break loadOnlineThumbnailCache; + } else { + logInfo`Online thumbs cache hasn't been downloaded for a little while.`; + logInfo`It'll be downloaded this build, then again in ${whenst(delay)}.`; + onlineThumbsCache = null; + paragraph = false; + } + } + + try { + await writeFile(cacheFile, stringifyCache(onlineThumbsCache)); + } catch (caughtError) { + writeError = caughtError; + } + + if (readError && writeError && readError.code !== 'ENOENT') { + console.error(readError); + logWarn`Wasn't able to read the local copy of the`; + logWarn`online thumbs cache file...`; + console.error(writeError); + logWarn`...or write to it, either.`; + logWarn`The online thumbs cache will be downloaded`; + logWarn`for every build until you investigate this path:`; + logWarn`${cacheFile}`; + paragraph = false; + } else if (readError && readError.code === 'ENOENT' && !writeError) { + logInfo`No local copy of online thumbs cache.`; + logInfo`It'll be downloaded this time and reused next time.`; + paragraph = false; + } else if (readError && readError.code === 'ENOENT' && writeError) { + console.error(writeError); + logWarn`Doesn't look like we can write a local copy of`; + logWarn`the offline thumbs cache, at this path:`; + logWarn`${cacheFile}`; + logWarn`The online thumbs cache will be downloaded`; + logWarn`for every build until you investigate that.`; + paragraph = false; + } + + const url = new URL(urlSpec.thumb.prefix); + url.pathname = path.posix.join(url.pathname, 'thumbnail-cache.json'); + + try { + onlineThumbsCache = await fetch(url).then(res => res.json()); + } catch (error) { + console.error(error); + logWarn`There was an error downloading the online thumbnail cache.`; + logWarn`The wiki will act as though no thumbs are available at all.`; + paragraph = false; + + Object.assign(stepStatusSummary.loadOnlineThumbnailCache, { + status: STATUS_HAS_WARNINGS, + annotation: `failed to download`, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + + onlineThumbsCache = {}; + thumbsCache = {}; + + break loadOnlineThumbnailCache; + } + + onlineThumbsCache._urlPrefix = urlSpec.thumb.prefix; + + thumbsCache = onlineThumbsCache; + + if (onlineThumbsCache && !writeError) { + try { + await writeFile(cacheFile, stringifyCache(onlineThumbsCache)); + } catch (error) { + console.error(error); + logWarn`There was an error saving a local copy of the`; + logWarn`online thumbnail cache. It'll be fetched again`; + logWarn`next time.`; + paragraph = false; + + Object.assign(stepStatusSummary.loadOnlineThumbnailCache, { + status: STATUS_HAS_WARNINGS, + annotation: `failed to download`, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + + break loadOnlineThumbnailCache; + } + } + + Object.assign(stepStatusSummary.loadOnlineThumbnailCache, { + status: STATUS_DONE_CLEAN, + timeStart: Date.now(), + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + } + const languageReloading = stepStatusSummary.watchLanguageFiles.status === STATUS_NOT_STARTED; @@ -1841,6 +2255,7 @@ async function main() { status: STATUS_FATAL_ERROR, annotation: `see log for details`, timeEnd: Date.now(), + memory: process.memoryUsage(), }); return false; @@ -1854,6 +2269,7 @@ async function main() { Object.assign(stepStatusSummary.loadInternalDefaultLanguage, { status: STATUS_DONE_CLEAN, timeEnd: Date.now(), + memory: process.memoryUsage(), }); let customLanguageWatchers; @@ -1933,6 +2349,7 @@ async function main() { status: STATUS_FATAL_ERROR, annotation: `see log for details`, timeEnd: Date.now(), + memory: process.memoryUsage(), }); errorLoadingCustomLanguages = true; @@ -1964,6 +2381,7 @@ async function main() { Object.assign(stepStatusSummary.watchLanguageFiles, { status: STATUS_DONE_CLEAN, timeEnd: Date.now(), + memory: process.memoryUsage(), }); } else { languages = {}; @@ -1987,11 +2405,13 @@ async function main() { status: STATUS_FATAL_ERROR, annotation: `see log for details`, timeEnd: Date.now(), + memory: process.memoryUsage(), }); } else { Object.assign(stepStatusSummary.loadLanguageFiles, { status: STATUS_DONE_CLEAN, timeEnd: Date.now(), + memory: process.memoryUsage(), }); } } @@ -2029,6 +2449,7 @@ async function main() { status: STATUS_FATAL_ERROR, annotation: `wiki specifies default language whose file is not available`, timeEnd: Date.now(), + memory: process.memoryUsage(), }); return false; @@ -2122,6 +2543,7 @@ async function main() { status: STATUS_DONE_CLEAN, annotation: finalDefaultLanguageAnnotation, timeEnd: Date.now(), + memory: process.memoryUsage(), }); let missingImagePaths; @@ -2144,85 +2566,225 @@ async function main() { Object.assign(stepStatusSummary.verifyImagePaths, { status: STATUS_DONE_CLEAN, timeEnd: Date.now(), + memory: process.memoryUsage(), }); } else if (empty(missingImagePaths)) { Object.assign(stepStatusSummary.verifyImagePaths, { status: STATUS_HAS_WARNINGS, annotation: `misplaced images detected`, timeEnd: Date.now(), + memory: process.memoryUsage(), }); } else if (empty(misplacedImagePaths)) { Object.assign(stepStatusSummary.verifyImagePaths, { status: STATUS_HAS_WARNINGS, annotation: `missing images detected`, timeEnd: Date.now(), + memory: process.memoryUsage(), }); } else { Object.assign(stepStatusSummary.verifyImagePaths, { status: STATUS_HAS_WARNINGS, annotation: `missing and misplaced images detected`, timeEnd: Date.now(), + memory: process.memoryUsage(), }); } } - let getSizeOfAdditionalFile; - let getSizeOfImagePath; + let getSizeOfMediaFile = () => null; + + const fileSizePreloader = + new FileSizePreloader({ + prefix: mediaPath, + }); + + if (stepStatusSummary.loadOnlineFileSizeCache.status === STATUS_NOT_STARTED) loadOnlineFileSizeCache: { + Object.assign(stepStatusSummary.loadOnlineFileSizeCache, { + status: STATUS_STARTED_NOT_DONE, + timeStart: Date.now(), + }); + + let onlineFileSizeCache = null; + + const makeFileSizeCacheAvailable = () => { + fileSizePreloader.loadFromCache(onlineFileSizeCache); + + getSizeOfMediaFile = p => + fileSizePreloader.getSizeOfPath( + path.resolve( + mediaPath, + decodeURIComponent(p).split('/').join(path.sep))); + }; + + const cacheFile = path.join(wikiCachePath, 'online-file-size-cache.json'); + + let readError = null; + let writeError = null; + + if (!cliOptions['refresh-online-file-sizes']) { + try { + onlineFileSizeCache = JSON.parse(await readFile(cacheFile)); + } catch (caughtError) { + readError = caughtError; + } + } + + if (onlineFileSizeCache) obliterateLocalCopy: { + if (!onlineFileSizeCache._urlPrefix) { + // Well, it doesn't even count. + onlineFileSizeCache = null; + break obliterateLocalCopy; + } + + if (onlineFileSizeCache._urlPrefix !== urlSpec.media.prefix) { + logInfo`Local copy of online file size cache is for a different prefix.`; + logInfo`It'll be downloaded and replaced, for reuse next time.`; + paragraph = false; + + onlineFileSizeCache = null; + break obliterateLocalCopy; + } + + let stats; + try { + stats = await stat(cacheFile); + } catch { + logInfo`Unable to get the stats of local copy of online file size cache...`; + logInfo`This is really weird, since we *were* able to read it...`; + logInfo`We're just going to try writing to it and download fresh!`; + paragraph = false; + + onlineFileSizeCache = null; + break obliterateLocalCopy; + } + + const delta = Date.now() - stats.mtimeMs; + const minute = 60 * 1000; + const delay = 60 * minute; + + const whenst = duration => `~${Math.ceil(duration / minute)} min`; + + if (delta < delay) { + logInfo`Online file size cache was downloaded recently, skipping for this build.`; + logInfo`Next scheduled is in ${whenst(delay - delta)}, or by using ${'--refresh-online-file-sizes'}.`; + paragraph = false; + + Object.assign(stepStatusSummary.loadOnlineFileSizeCache, { + status: STATUS_DONE_CLEAN, + annotation: `reusing local copy, earlier than scheduled`, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + + delete onlineFileSizeCache._urlPrefix; + + makeFileSizeCacheAvailable(); + + break loadOnlineFileSizeCache; + } else { + logInfo`Online file size hasn't been downloaded for a little while.`; + logInfo`It'll be downloaded this build, then again in ${whenst(delay)}.`; + onlineFileSizeCache = null; + paragraph = false; + } + } + + try { + await writeFile(cacheFile, stringifyCache(onlineFileSizeCache)); + } catch (caughtError) { + writeError = caughtError; + } + + if (readError && writeError && readError.code !== 'ENOENT') { + console.error(readError); + logWarn`Wasn't able to read the local copy of the`; + logWarn`online file size cache file...`; + console.error(writeError); + logWarn`...or write to it, either.`; + logWarn`The online file size cache will be downloaded`; + logWarn`for every build until you investigate this path:`; + logWarn`${cacheFile}`; + paragraph = false; + } else if (readError && readError.code === 'ENOENT' && !writeError) { + logInfo`No local copy of online file size cache.`; + logInfo`It'll be downloaded this time and reused next time.`; + paragraph = false; + } else if (readError && readError.code === 'ENOENT' && writeError) { + console.error(writeError); + logWarn`Doesn't look like we can write a local copy of`; + logWarn`the offline file size cache, at this path:`; + logWarn`${cacheFile}`; + logWarn`The online file size cache will be downloaded`; + logWarn`for every build until you investigate that.`; + paragraph = false; + } + + const url = new URL(urlSpec.media.prefix); + url.pathname = path.posix.join(url.pathname, 'file-size-cache.json'); + + try { + onlineFileSizeCache = await fetch(url).then(res => res.json()); + } catch (error) { + console.error(error); + logWarn`There was an error downloading the online file size cache.`; + logWarn`The wiki will act as though no file sizes are available at all.`; + paragraph = false; + + Object.assign(stepStatusSummary.loadOnlineFileSizeCache, { + status: STATUS_HAS_WARNINGS, + annotation: `failed to download`, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + + break loadOnlineFileSizeCache; + } + + makeFileSizeCacheAvailable(); + + onlineFileSizeCache._urlPrefix = urlSpec.media.prefix; + + if (onlineFileSizeCache && !writeError) { + try { + await writeFile(cacheFile, stringifyCache(onlineFileSizeCache)); + } catch (error) { + console.error(error); + logWarn`There was an error saving a local copy of the`; + logWarn`online file size cache. It'll be fetched again`; + logWarn`next time.`; + paragraph = false; + + Object.assign(stepStatusSummary.loadOnlineFileSizeCache, { + status: STATUS_HAS_WARNINGS, + annotation: `failed to download`, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + + break loadOnlineFileSizeCache; + } + } + + Object.assign(stepStatusSummary.loadOnlineFileSizeCache, { + status: STATUS_DONE_CLEAN, + timeStart: Date.now(), + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + } - if (stepStatusSummary.preloadFileSizes.status === STATUS_NOT_APPLICABLE) { - getSizeOfAdditionalFile = () => null; - getSizeOfImagePath = () => null; - } else if (stepStatusSummary.preloadFileSizes.status === STATUS_NOT_STARTED) { + if (stepStatusSummary.preloadFileSizes.status === STATUS_NOT_STARTED) { Object.assign(stepStatusSummary.preloadFileSizes, { status: STATUS_STARTED_NOT_DONE, timeStart: Date.now(), }); - const fileSizePreloader = new FileSizePreloader(); - - // File sizes of additional files need to be precalculated before we can - // actually reference 'em in site building, so get those loading right - // away. We actually need to keep track of two things here - the on-device - // file paths we're actually reading, and the corresponding on-site media - // paths that will be exposed in site build code. We'll build a mapping - // function between them so that when site code requests a site path, - // it'll get the size of the file at the corresponding device path. - const additionalFilePaths = [ - ...wikiData.albumData.flatMap((album) => - [ - ...(album.additionalFiles ?? []), - ...album.tracks.flatMap((track) => [ - ...(track.additionalFiles ?? []), - ...(track.sheetMusicFiles ?? []), - ...(track.midiProjectFiles ?? []), - ]), - ] - .flatMap((fileGroup) => fileGroup.files ?? []) - .map((file) => ({ - device: path.join( - mediaPath, - urls - .from('media.root') - .toDevice('media.albumAdditionalFile', album.directory, file) - ), - media: urls - .from('media.root') - .to('media.albumAdditionalFile', album.directory, file), - })) - ), - ]; - - // Same dealio for images. Since just about any image can be embedded and - // we can't super easily know which ones are referenced at runtime, just - // cheat and get file sizes for all images under media. (This includes - // additional files which are images.) - const imageFilePaths = + const mediaFilePaths = await traverse(mediaPath, { pathStyle: 'device', filterDir: dir => dir !== '.git', - filterFile: file => - ['.png', '.gif', '.jpg'].includes(path.extname(file)) && - !isThumb(file), + filterFile: file => !isThumb(file), }).then(files => files .map(file => ({ device: file, @@ -2232,28 +2794,19 @@ async function main() { .to('media.path', path.relative(mediaPath, file).split(path.sep).join('/')), }))); - const getSizeOfMediaFileHelper = paths => (mediaPath) => { - const pair = paths.find(({media}) => media === mediaPath); + getSizeOfMediaFile = mediaPath => { + const pair = mediaFilePaths.find(({media}) => media === mediaPath); if (!pair) return null; return fileSizePreloader.getSizeOfPath(pair.device); }; - getSizeOfAdditionalFile = getSizeOfMediaFileHelper(additionalFilePaths); - getSizeOfImagePath = getSizeOfMediaFileHelper(imageFilePaths); - - logInfo`Preloading filesizes for ${additionalFilePaths.length} additional files...`; + logInfo`Preloading file sizes for ${mediaFilePaths.length} media files...`; - fileSizePreloader.loadPaths(...additionalFilePaths.map((path) => path.device)); - await fileSizePreloader.waitUntilDoneLoading(); - - logInfo`Preloading filesizes for ${imageFilePaths.length} full-resolution images...`; - paragraph = false; - - fileSizePreloader.loadPaths(...imageFilePaths.map((path) => path.device)); + fileSizePreloader.loadPaths(...mediaFilePaths.map(path => path.device)); await fileSizePreloader.waitUntilDoneLoading(); if (fileSizePreloader.hasErrored) { - logWarn`Some media files couldn't be read for preloading filesizes.`; + logWarn`Some media files couldn't be read for preloading file sizes.`; logWarn`This means the wiki won't display file sizes for these files.`; logWarn`Investigate missing or unreadable files to get that fixed!`; @@ -2261,16 +2814,50 @@ async function main() { status: STATUS_HAS_WARNINGS, annotation: `see log for details`, timeEnd: Date.now(), + memory: process.memoryUsage(), }); } else { - logInfo`Done preloading filesizes without any errors - nice!`; + logInfo`Done preloading file sizes without any errors - nice!`; paragraph = false; Object.assign(stepStatusSummary.preloadFileSizes, { status: STATUS_DONE_CLEAN, timeEnd: Date.now(), + memory: process.memoryUsage(), }); } + + // TODO: kinda jank that this is out of band of any particular step, + // even though it's operationally a follow-up to preloadFileSizes + + let oopsCache = false; + saveFileSizeCache: { + let cache; + try { + cache = fileSizePreloader.saveAsCache(); + } catch (error) { + console.error(error); + logWarn`Couldn't compute file size preloader's cache.`; + oopsCache = true; + break saveFileSizeCache; + } + + const cacheFile = path.join(mediaPath, 'file-size-cache.json'); + + try { + await writeFile(cacheFile, stringifyCache(cache)); + } catch (error) { + console.error(error); + logWarn`Couldn't save preloaded file sizes to a cache file:`; + logWarn`${cacheFile}`; + oopsCache = true; + } + } + + if (oopsCache) { + logWarn`This won't affect the build, but this build should not be used`; + logWarn`as a model for another build accessing its media files online.`; + } } if (stepStatusSummary.buildSearchIndex.status === STATUS_NOT_STARTED) { @@ -2293,6 +2880,7 @@ async function main() { Object.assign(stepStatusSummary.buildSearchIndex, { status: STATUS_DONE_CLEAN, timeEnd: Date.now(), + memory: process.memoryUsage(), }); } catch (error) { if (!paragraph) console.log(''); @@ -2310,6 +2898,7 @@ async function main() { status: STATUS_HAS_WARNINGS, annotation: `see log for details`, timeEnd: Date.now(), + memory: process.memoryUsage(), }); } } @@ -2357,6 +2946,7 @@ async function main() { status: STATUS_FATAL_ERROR, message: `JavaScript error - view log for details`, timeEnd: Date.now(), + memory: process.memoryUsage(), }); return false; @@ -2368,6 +2958,7 @@ async function main() { Object.assign(stepStatusSummary.identifyWebRoutes, { status: STATUS_DONE_CLEAN, timeEnd: Date.now(), + memory: process.memoryUsage(), }); } @@ -2422,8 +3013,7 @@ async function main() { console.log(''); const universalUtilities = { - getSizeOfAdditionalFile, - getSizeOfImagePath, + getSizeOfMediaFile, defaultLanguage: finalDefaultLanguage, developersComment, @@ -2464,6 +3054,7 @@ async function main() { status: STATUS_FATAL_ERROR, message: `javascript error - view log for details`, timeEnd: Date.now(), + memory: process.memoryUsage(), }); return false; @@ -2474,6 +3065,7 @@ async function main() { status: STATUS_HAS_WARNINGS, annotation: `may not have completed - view log for details`, timeEnd: Date.now(), + memory: process.memoryUsage(), }); return false; @@ -2482,6 +3074,7 @@ async function main() { Object.assign(stepStatusSummary.performBuild, { status: STATUS_DONE_CLEAN, timeEnd: Date.now(), + memory: process.memoryUsage(), }); return true; @@ -2569,16 +3162,31 @@ if (true || isMain(import.meta.url) || path.basename(process.argv[1]) === 'hsmus const longestDurationLength = Math.max(...stepDurations.map(duration => duration.length)); + const stepMemories = + stepDetails.map(({memory}) => + (memory + ? Math.round(memory["heapUsed"] / 1024 / 1024) + 'MB' + : '-')); + + const longestMemoryLength = + Math.max(...stepMemories.map(memory => memory.length)); + for (let index = 0; index < stepDetails.length; index++) { const {name, status, annotation} = stepDetails[index]; const duration = stepDurations[index]; + const memory = stepMemories[index]; let message = (stepsNotClean[index] ? `!! ` : ` - `); - message += `(${duration})`.padStart(longestDurationLength + 2, ' '); + message += `(${duration} `.padStart(longestDurationLength + 2, ' '); + + if (showStepMemoryInSummary) { + message += ` ${memory})`.padStart(longestMemoryLength + 2, ' '); + } + message += ` `; message += `${name}: `.padEnd(longestNameLength + 4, '.'); message += ` `; @@ -2636,7 +3244,6 @@ if (true || isMain(import.meta.url) || path.basename(process.argv[1]) === 'hsmus } decorateTime.displayTime(); - CacheableObject.showInvalidAccesses(); process.exit(0); })(); diff --git a/src/url-spec.js b/src/url-spec.js index 6ca75e7d..75cd8006 100644 --- a/src/url-spec.js +++ b/src/url-spec.js @@ -1,145 +1,220 @@ -import {withEntries} from '#sugar'; - -// Static files are all grouped under a `static-${STATIC_VERSION}` folder as -// part of a build. This is so that multiple builds of a wiki can coexist -// served from the same server / file system root: older builds' HTML files -// refer to earlier values of STATIC_VERSION, avoiding name collisions. -const STATIC_VERSION = '3p3'; - -const genericPaths = { - root: '', - path: '<>', -}; - -const urlSpec = { - data: { - prefix: 'data/', - - paths: { - ...genericPaths, - - album: 'album/<>', - artist: 'artist/<>', - track: 'track/<>', - }, - }, - - localized: { - // TODO: Implement this. - // prefix: '_languageCode', - - paths: { - ...genericPaths, - page: '<>/', - - home: '', - - album: 'album/<>/', - albumCommentary: 'commentary/album/<>/', - albumGallery: 'album/<>/gallery/', - albumReferencedArtworks: 'album/<>/referenced-art/', - albumReferencingArtworks: 'album/<>/referencing-art/', - - artist: 'artist/<>/', - artistGallery: 'artist/<>/gallery/', - - commentaryIndex: 'commentary/', - - flashIndex: 'flash/', - - flash: 'flash/<>/', - - flashActGallery: 'flash-act/<>/', - - groupInfo: 'group/<>/', - groupGallery: 'group/<>/gallery/', - - listingIndex: 'list/', - - listing: 'list/<>/', - - newsIndex: 'news/', - - newsEntry: 'news/<>/', - - staticPage: '<>/', - - tag: 'tag/<>/', - - track: 'track/<>/', - trackReferencedArtworks: 'track/<>/referenced-art/', - trackReferencingArtworks: 'track/<>/referencing-art/', - }, - }, - - shared: { - paths: genericPaths, - }, - - staticCSS: { - prefix: `static-${STATIC_VERSION}/css/`, - paths: genericPaths, - }, - - staticJS: { - prefix: `static-${STATIC_VERSION}/js/`, - paths: genericPaths, - }, - - staticLib: { - prefix: `static-${STATIC_VERSION}/lib/`, - paths: genericPaths, - }, - - staticMisc: { - prefix: `static-${STATIC_VERSION}/misc/`, - paths: { - ...genericPaths, - icon: 'icons.svg#icon-<>', - }, - }, - - staticSharedUtil: { - prefix: `static-${STATIC_VERSION}/shared-util/`, - paths: genericPaths, - }, - - media: { - prefix: 'media/', - - paths: { - ...genericPaths, - - albumAdditionalFile: 'album-additional/<>/<>', - albumBanner: 'album-art/<>/banner.<>', - albumCover: 'album-art/<>/cover.<>', - albumWallpaper: 'album-art/<>/bg.<>', - albumWallpaperPart: 'album-art/<>/<>', - - artistAvatar: 'artist-avatar/<>.<>', - - flashArt: 'flash-art/<>.<>', - - trackCover: 'album-art/<>/<>.<>', - }, - }, - - thumb: { - prefix: 'thumb/', - paths: genericPaths, - }, - - searchData: { - prefix: 'search-data/', - paths: genericPaths, - }, -}; - -// This gets automatically switched in place when working from a baseDirectory, -// so it should never be referenced manually. -urlSpec.localizedWithBaseDirectory = { - paths: withEntries(urlSpec.localized.paths, (entries) => - entries.map(([key, path]) => [key, '<>/' + path])), -}; - -export default urlSpec; +// Exports defined here are re-exported through urls.js, +// so they're generally imported from '#urls'. + +import {readFile} from 'node:fs/promises'; +import * as path from 'node:path'; +import {fileURLToPath} from 'node:url'; + +import yaml from 'js-yaml'; + +import {annotateError, annotateErrorWithFile, openAggregate} from '#aggregate'; +import {empty, typeAppearance, withEntries} from '#sugar'; + +export const DEFAULT_URL_SPEC_FILE = 'urls-default.yaml'; + +export const internalDefaultURLSpecFile = + path.resolve( + path.dirname(fileURLToPath(import.meta.url)), + DEFAULT_URL_SPEC_FILE); + +function processStringToken(key, token) { + const oops = appearance => + new Error( + `Expected ${key} to be a string or an array of strings, ` + + `got ${appearance}`); + + if (typeof token === 'string') { + return token; + } else if (Array.isArray(token)) { + if (empty(token)) { + throw oops(`empty array`); + } else if (token.every(item => typeof item !== 'string')) { + throw oops(`array of non-strings`); + } else if (token.some(item => typeof item !== 'string')) { + throw oops(`array of mixed strings and non-strings`); + } else { + return token.join(''); + } + } else { + throw oops(typeAppearance(token)); + } +} + +function processObjectToken(key, token) { + const oops = appearance => + new Error( + `Expected ${key} to be an object or an array of objects, ` + + `got ${appearance}`); + + const looksLikeObject = value => + typeof value === 'object' && + value !== null && + !Array.isArray(value); + + if (looksLikeObject(token)) { + return {...token}; + } else if (Array.isArray(token)) { + if (empty(token)) { + throw oops(`empty array`); + } else if (token.every(item => !looksLikeObject(item))) { + throw oops(`array of non-objects`); + } else if (token.some(item => !looksLikeObject(item))) { + throw oops(`array of mixed objects and non-objects`); + } else { + return Object.assign({}, ...token); + } + } +} + +function makeProcessToken(aggregate) { + return (object, key, processFn) => { + if (key in object) { + const value = aggregate.call(processFn, key, object[key]); + if (value === null) { + delete object[key]; + } else { + object[key] = value; + } + } + }; +} + +export function processGroupSpec(groupKey, groupSpec) { + const aggregate = + openAggregate({message: `Errors processing group "${groupKey}"`}); + + const processToken = makeProcessToken(aggregate); + + groupSpec.key = groupKey; + + processToken(groupSpec, 'prefix', processStringToken); + processToken(groupSpec, 'paths', processObjectToken); + + return {aggregate, result: groupSpec}; +} + +export function processURLSpec(sourceSpec) { + const aggregate = + openAggregate({message: `Errors processing URL spec`}); + + sourceSpec ??= {}; + + const urlSpec = structuredClone(sourceSpec); + + delete urlSpec.yamlAliases; + delete urlSpec.localizedWithBaseDirectory; + + aggregate.nest({message: `Errors processing groups`}, groupsAggregate => { + Object.assign(urlSpec, + withEntries(urlSpec, entries => + entries.map(([groupKey, groupSpec]) => [ + groupKey, + groupsAggregate.receive( + processGroupSpec(groupKey, groupSpec)), + ]))); + }); + + switch (sourceSpec.localizedWithBaseDirectory) { + case '<auto>': { + if (!urlSpec.localized) { + aggregate.push(new Error( + `Not ready for 'localizedWithBaseDirectory' group, ` + + `'localized' not available`)); + } else if (!urlSpec.localized.paths) { + aggregate.push(new Error( + `Not ready for 'localizedWithBaseDirectory' group, ` + + `'localized' group's paths not available`)); + } + + break; + } + + case undefined: + break; + + default: + aggregate.push(new Error( + `Expected 'localizedWithBaseDirectory' group to have value '<auto>' ` + + `or not be set`)); + + break; + } + + return {aggregate, result: urlSpec}; +} + +export function applyURLSpecOverriding(overrideSpec, baseSpec) { + const aggregate = openAggregate({message: `Errors applying URL spec`}); + + for (const [groupKey, overrideGroupSpec] of Object.entries(overrideSpec)) { + const baseGroupSpec = baseSpec[groupKey]; + + if (!baseGroupSpec) { + aggregate.push(new Error(`Group key "${groupKey}" not available on base spec`)); + continue; + } + + if (overrideGroupSpec.prefix) { + baseGroupSpec.prefix = overrideGroupSpec.prefix; + } + + if (overrideGroupSpec.paths) { + for (const [pathKey, overridePathValue] of Object.entries(overrideGroupSpec.paths)) { + if (!baseGroupSpec.paths[pathKey]) { + aggregate.push(new Error(`Path key "${groupKey}.${pathKey}" not available on base spec`)); + continue; + } + + baseGroupSpec.paths[pathKey] = overridePathValue; + } + } + } + + return {aggregate}; +} + +export function applyLocalizedWithBaseDirectory(urlSpec) { + const paths = + withEntries(urlSpec.localized.paths, entries => + entries.map(([key, path]) => [key, '<>/' + path])); + + urlSpec.localizedWithBaseDirectory = + Object.assign( + structuredClone(urlSpec.localized), + {paths}); +} + +export async function processURLSpecFromFile(file) { + let contents; + + try { + contents = await readFile(file, 'utf-8'); + } catch (caughtError) { + throw annotateError( + new Error(`Failed to read URL spec file`, {cause: caughtError}), + error => annotateErrorWithFile(error, file)); + } + + let sourceSpec; + let parseLanguage; + + try { + if (path.extname(file) === '.yaml') { + parseLanguage = 'YAML'; + sourceSpec = yaml.load(contents); + } else { + parseLanguage = 'JSON'; + sourceSpec = JSON.parse(contents); + } + } catch (caughtError) { + throw annotateError( + new Error(`Failed to parse URL spec file as valid ${parseLanguage}`, {cause: caughtError}), + error => annotateErrorWithFile(error, file)); + } + + try { + return processURLSpec(sourceSpec); + } catch (caughtError) { + throw annotateErrorWithFile(caughtError, file); + } +} diff --git a/src/urls-default.yaml b/src/urls-default.yaml new file mode 100644 index 00000000..10bc0d23 --- /dev/null +++ b/src/urls-default.yaml @@ -0,0 +1,143 @@ +# These are variables which are used to make expressing this +# YAML file more convenient. They are not exposed externally. +# (Stuff which uses this YAML file can't even see the names +# for each variable!) +yamlAliases: + - &genericPaths + root: '' + path: '<>' + + # Static files are all grouped under a `static-${STATIC_VERSION}` folder as + # part of a build. This is so that multiple builds of a wiki can coexist + # served from the same server / file system root: older builds' HTML files + # refer to earlier values of STATIC_VERSION, avoiding name collisions. + - &staticVersion 3p4 + +data: + prefix: 'data/' + + paths: + - *genericPaths + + - album: 'album/<>' + artist: 'artist/<>' + track: 'track/<>' + +localized: + paths: + - *genericPaths + - page: '<>/' + + home: '' + + album: 'album/<>/' + albumCommentary: 'commentary/album/<>/' + albumGallery: 'album/<>/gallery/' + albumReferencedArtworks: 'album/<>/referenced-art/' + albumReferencingArtworks: 'album/<>/referencing-art/' + + artist: 'artist/<>/' + artistGallery: 'artist/<>/gallery/' + + commentaryIndex: 'commentary/' + + flashIndex: 'flash/' + + flash: 'flash/<>/' + + flashActGallery: 'flash-act/<>/' + + groupInfo: 'group/<>/' + groupGallery: 'group/<>/gallery/' + + listingIndex: 'list/' + + listing: 'list/<>/' + + newsIndex: 'news/' + + newsEntry: 'news/<>/' + + staticPage: '<>/' + + tag: 'tag/<>/' + + track: 'track/<>/' + trackReferencedArtworks: 'track/<>/referenced-art/' + trackReferencingArtworks: 'track/<>/referencing-art/' + +# This gets automatically switched in place when working from +# a baseDirectory, so it should never be referenced manually. +# It's also filled in externally to this YAML spec. +localizedWithBaseDirectory: '<auto>' + +shared: + paths: *genericPaths + +staticCSS: + prefix: + - 'static-' + - *staticVersion + - '/css/' + + paths: *genericPaths + +staticJS: + prefix: + - 'static-' + - *staticVersion + - '/js/' + + paths: *genericPaths + +staticLib: + prefix: + - 'static-' + - *staticVersion + - '/lib/' + + paths: *genericPaths + +staticMisc: + prefix: + - 'static-' + - *staticVersion + - '/misc/' + + paths: + - *genericPaths + - icon: 'icons.svg#icon-<>' + +staticSharedUtil: + prefix: + - 'static-' + - *staticVersion + - '/shared-util/' + + paths: *genericPaths + +media: + prefix: 'media/' + + paths: + - *genericPaths + + - albumAdditionalFile: 'album-additional/<>/<>' + albumBanner: 'album-art/<>/banner.<>' + albumCover: 'album-art/<>/cover.<>' + albumWallpaper: 'album-art/<>/bg.<>' + albumWallpaperPart: 'album-art/<>/<>' + + artistAvatar: 'artist-avatar/<>.<>' + + flashArt: 'flash-art/<>.<>' + + trackCover: 'album-art/<>/<>.<>' + +thumb: + prefix: 'thumb/' + paths: *genericPaths + +searchData: + prefix: 'search-data/' + paths: *genericPaths diff --git a/src/util/urls.js b/src/urls.js index 11b9b8b0..83a8b904 100644 --- a/src/util/urls.js +++ b/src/urls.js @@ -8,17 +8,16 @@ import * as path from 'node:path'; import {withEntries} from '#sugar'; -// This export is only provided for convenience, i.e. to enable the following: -// -// import {urlSpec} from '#urls'; -// -// It's not actually defined in this module's variable scope, and functions -// exported here require a urlSpec (whether this default one or another) to be -// passed directly. -// -export {default as urlSpec} from '../url-spec.js'; +export * from './url-spec.js'; export function generateURLs(urlSpec) { + if ( + typeof urlSpec.localized === 'object' && + typeof urlSpec.localizedWithBaseDirectory !== 'object' + ) { + throw new Error(`Provided urlSpec missing localizedWithBaseDirectory`); + } + const getValueForFullKey = (obj, fullKey) => { const [groupKey, subKey] = fullKey.split('.'); if (!groupKey || !subKey) { @@ -49,8 +48,12 @@ export function generateURLs(urlSpec) { const generateTo = (fromPath, fromGroup) => { const A = trimLeadingSlash(fromPath); - const rebasePrefix = '../' - .repeat((fromGroup.prefix || '').split('/').filter(Boolean).length); + const fromPrefix = fromGroup.prefix || ''; + + const rebasePrefix = + '../'.repeat(fromPrefix.split('/').filter(Boolean).length); + + const fromOrigin = getOrigin(fromPrefix); const pathHelper = (toPath, toGroup) => { let B = trimLeadingSlash(toPath); @@ -58,40 +61,106 @@ export function generateURLs(urlSpec) { let argIndex = 0; B = B.replaceAll('<>', () => `<${argIndex++}>`); - if (toGroup.prefix !== fromGroup.prefix) { - // TODO: Handle differing domains in prefixes. - B = rebasePrefix + (toGroup.prefix || '') + B; - } - const suffix = toPath.endsWith('/') ? '/' : ''; - return { - posix: path.posix.relative(A, B) + suffix, - device: path.relative(A, B) + suffix, - }; - }; + const toPrefix = toGroup.prefix; + + if (toPrefix !== fromPrefix) { + // Compare origins. Note that getOrigin() can + // be null for both prefixes. + const toOrigin = getOrigin(toPrefix); + if (fromOrigin === toOrigin) { + // Go to the root, add the to-group's prefix, then + // continue with normal path.relative() behavior. + B = rebasePrefix + (toGroup.prefix || '') + B; + } else { + // Crossing origins never conceptually represents + // something you can interpret on-`.device()`. + return { + posix: toGroup.prefix + B + suffix, + device: null, + }; + } + } - const groupSymbol = Symbol(); + // If we're coming from a qualified origin (domain), + // then at this point, A and B represent paths on the + // same origin. We can use normal path.relative() behavior. + if (fromOrigin) { + // If we're working on an origin, there's no meaning to + // a `.device()`-local relative path. + return { + posix: path.posix.relative(A, B) + suffix, + device: null, + }; + } else { + return { + posix: path.posix.relative(A, B) + suffix, + device: path.relative(A, B) + suffix, + }; + } + }; - const groupHelper = (urlGroup) => ({ - [groupSymbol]: urlGroup, - ...withEntries(urlGroup.paths, (entries) => - entries.map(([key, path]) => [key, pathHelper(path, urlGroup)]) - ), - }); + const groupHelper = urlGroup => + withEntries(urlGroup.paths, entries => + entries.map(([key, path]) => [ + key, + pathHelper(path, urlGroup), + ])); - const relative = withEntries(urlSpec, (entries) => - entries.map(([key, urlGroup]) => [key, groupHelper(urlGroup)]) - ); + const relative = + withEntries(urlSpec, entries => + entries.map(([key, urlGroup]) => [ + key, + groupHelper(urlGroup), + ])); const toHelper = ({device}) => (key, ...args) => { - const { - value: { - [device ? 'device' : 'posix']: template, - }, - } = getValueForFullKey(relative, key); + const templateKey = (device ? 'device' : 'posix'); + + const {value: {[templateKey]: template}} = + getValueForFullKey(relative, key); + + // If we got past getValueForFullKey(), we've already ruled out + // the common errors, i.e. incorrectly formatted key or invalid + // group key or subkey. + if (template === null) { + // Self-diagnose, brutally. + + const otherTemplateKey = (device ? 'posix' : 'device'); + + const {value: {[templateKey]: otherTemplate}} = + getValueForFullKey(relative, key); + + const effectiveMode = + (otherTemplate + ? `${templateKey} mode` + : `either mode`); + + const toGroupKey = key.split('.')[0]; + + const anyOthers = + Object.values(relative[toGroupKey]) + .find(templates => + (otherTemplate + ? templates[templateKey] + : templates.posix || templates.device)); + + const effectiveTo = + (anyOthers + ? key + : `${toGroupKey}.*`); + + if (anyOthers) { + console.log(relative[toGroupKey]); + } + + throw new Error( + `from(${fromGroup.key}.*).to(${effectiveTo}) ` + + `not available in ${effectiveMode} with this url spec`); + } let missing = 0; let result = template.replaceAll(/<([0-9]+)>/g, (match, n) => { @@ -111,19 +180,31 @@ export function generateURLs(urlSpec) { if (missing) { throw new Error( - `Expected ${missing + args.length} arguments, got ${ - args.length - } (key ${key}, args [${args}])` - ); + `Expected ${missing + args.length} arguments, ` + + `got ${args.length} (key ${key}, args [${args}])`); } return result; }; - return { - to: toHelper({device: false}), - toDevice: toHelper({device: true}), - }; + const toAvailableHelper = + ({device}) => + (key) => { + const templateKey = (device ? 'device' : 'posix'); + + const {value: {[templateKey]: template}} = + getValueForFullKey(relative, key); + + return !!template; + }; + + const to = toHelper({device: false}); + const toDevice = toHelper({device: true}); + + to.available = toAvailableHelper({device: false}); + toDevice.available = toAvailableHelper({device: true}); + + return {to, toDevice}; }; const generateFrom = () => { @@ -144,6 +225,14 @@ export function generateURLs(urlSpec) { return generateFrom(); } +export function getOrigin(prefix) { + try { + return new URL(prefix).origin; + } catch { + return null; + } +} + const thumbnailHelper = (name) => (file) => file.replace(/\.(jpg|png)$/, name + '.jpg'); diff --git a/src/util/aggregate.js b/src/util/aggregate.js index e8f45f3b..c7648c4c 100644 --- a/src/util/aggregate.js +++ b/src/util/aggregate.js @@ -110,7 +110,6 @@ export function openAggregate({ return results.map(({aggregate, result}) => { if (!aggregate) { - console.log('nope:', results); throw new Error(`Expected an array of {aggregate, result} objects`); } diff --git a/src/util/cli.js b/src/util/cli.js index 72979d3f..a40a911f 100644 --- a/src/util/cli.js +++ b/src/util/cli.js @@ -5,6 +5,8 @@ const {process} = globalThis; +import {sortByName} from './sort.js'; + export const ENABLE_COLOR = process && ((process.env.CLICOLOR_FORCE && process.env.CLICOLOR_FORCE === '1') ?? @@ -95,8 +97,12 @@ export async function parseOptions(options, optionDescriptorMap) { // } // // ['--directory', 'apple'] -> {'directory': 'apple'} + // ['--directory=banana'] -> {'directory': 'banana'} // ['--directory', 'artichoke'] -> (error) + // // ['--files', 'a', 'b', 'c', ';'] -> {'files': ['a', 'b', 'c']} + // ['--files=a,b,c'] -> {'files': ['a', 'b', 'c']} + // ['--files', 'a,b,c'] -> {'files': ['a', 'b', 'c']} const handleDashless = optionDescriptorMap[parseOptions.handleDashless]; const handleUnknown = optionDescriptorMap[parseOptions.handleUnknown]; @@ -149,9 +155,27 @@ export async function parseOptions(options, optionDescriptorMap) { } case 'series': { + if (option.includes('=')) { + result[name] = option.split('=')[1].split(','); + break; + } + + // without a semicolon to conclude the series, + // assume the next option expresses the whole series if (!options.slice(i).includes(';')) { - console.error(`Expected a series of values concluding with ; (\\;) for --${name}`); - process.exit(1); + let value = options[++i]; + + if (!value || value.startsWith('-')) { + value = null; + } + + if (!value) { + console.error(`Expected values for --${name}`); + process.exit(1); + } + + result[name] = value.split('=')[1].split(','); + break; } const endIndex = i + options.slice(i).indexOf(';'); @@ -471,3 +495,27 @@ export async function logicalPathTo(target) { const cwd = await logicalCWD(); return relative(cwd, target); } + +export function stringifyCache(cache) { + cache ??= {}; + + if (Object.keys(cache).length === 0) { + return `{}`; + } + + const entries = Object.entries(cache); + sortByName(entries, {getName: entry => entry[0]}); + + return [ + `{`, + entries + .map(([key, value]) => [JSON.stringify(key), JSON.stringify(value)]) + .map(([key, value]) => `${key}: ${value}`) + .map((line, index, array) => + (index < array.length - 1 + ? `${line},` + : line)) + .map(line => ` ${line}`), + `}`, + ].flat().join('\n'); +} diff --git a/src/util/search-spec.js b/src/util/search-spec.js index bc24e1a1..3d05c021 100644 --- a/src/util/search-spec.js +++ b/src/util/search-spec.js @@ -134,14 +134,14 @@ export const searchSpec = { thing.color; fields.artTags = - (Object.hasOwn(thing, 'artTags') + (thing.constructor.hasPropertyDescriptor('artTags') ? thing.artTags.map(artTag => artTag.nameShort) : []); fields.additionalNames = - (Object.hasOwn(thing, 'additionalNames') + (thing.constructor.hasPropertyDescriptor('additionalNames') ? thing.additionalNames.map(entry => entry.name) - : Object.hasOwn(thing, 'aliasNames') + : thing.constructor.hasPropertyDescriptor('aliasNames') ? thing.aliasNames : []); diff --git a/src/write/bind-utilities.js b/src/write/bind-utilities.js index be702c8c..d55ab215 100644 --- a/src/write/bind-utilities.js +++ b/src/write/bind-utilities.js @@ -20,8 +20,7 @@ import { export function bindUtilities({ absoluteTo, defaultLanguage, - getSizeOfAdditionalFile, - getSizeOfImagePath, + getSizeOfMediaFile, language, languages, missingImagePaths, @@ -37,8 +36,7 @@ export function bindUtilities({ Object.assign(bound, { absoluteTo, defaultLanguage, - getSizeOfAdditionalFile, - getSizeOfImagePath, + getSizeOfMediaFile, getThumbnailsAvailableForDimensions, html, language, diff --git a/src/write/build-modes/live-dev-server.js b/src/write/build-modes/live-dev-server.js index f6eec334..dd29c93e 100644 --- a/src/write/build-modes/live-dev-server.js +++ b/src/write/build-modes/live-dev-server.js @@ -5,7 +5,9 @@ import * as path from 'node:path'; import {pipeline} from 'node:stream/promises'; import {inspect as nodeInspect} from 'node:util'; -import {ENABLE_COLOR, colors, logInfo, logWarn, progressCallAll} from '#cli'; +import {openAggregate} from '#aggregate'; +import {ENABLE_COLOR, colors, fileIssue, logInfo, logWarn, progressCallAll} + from '#cli'; import {watchContentDependencies} from '#content-dependencies'; import {quickEvaluate} from '#content-function'; import * as html from '#html'; @@ -165,21 +167,47 @@ export async function go({ const commonUtilities = {...universalUtilities}; + const pathAggregate = openAggregate({message: `Errors computing page paths`}); + let targetSpecPairs = getPageSpecsWithTargets({wikiData}); - const pages = progressCallAll(`Computing page data & paths for ${targetSpecPairs.length} targets.`, + const pages = progressCallAll(`Computing page paths for ${targetSpecPairs.length} targets.`, targetSpecPairs.flatMap(({ pageSpec, target, targetless, }) => () => { - if (targetless) { - const result = pageSpec.pathsTargetless({wikiData}); - return Array.isArray(result) ? result : [result]; - } else { - return pageSpec.pathsForTarget(target); + try { + if (targetless) { + const result = pageSpec.pathsTargetless({wikiData}); + return Array.isArray(result) ? result : [result]; + } else { + return pageSpec.pathsForTarget(target); + } + } catch (caughtError) { + if (targetless) { + pathAggregate.push(new Error( + `Failed to compute targetless paths for ` + + inspect(pageSpec, {compact: true}), + {cause: caughtError})); + } else { + pathAggregate.push(new Error( + `Failed to compute paths for ` + + inspect(target), + {cause: caughtError})); + } + return []; } })).flat(); + try { + pathAggregate.close(); + } catch (error) { + niceShowAggregate(error); + logWarn`Failed to compute page paths for some targets.`; + logWarn`This means some pages that normally exist will be 404s.`; + fileIssue(); + } + logInfo`Will be serving a total of ${pages.length} pages.`; const urlToPageMap = Object.fromEntries(pages diff --git a/src/write/build-modes/repl.js b/src/write/build-modes/repl.js index 957d2c2d..920ad9f7 100644 --- a/src/write/build-modes/repl.js +++ b/src/write/build-modes/repl.js @@ -36,6 +36,7 @@ import * as path from 'node:path'; import * as repl from 'node:repl'; import _find, {bindFind} from '#find'; +import _reverse, {bindReverse} from '#reverse'; import CacheableObject from '#cacheable-object'; import {logWarn} from '#cli'; import {debugComposite} from '#composite'; @@ -66,6 +67,15 @@ export async function getContextAssignments({ logWarn`\`find\` variable will be missing`; } + let reverse; + try { + reverse = bindReverse(wikiData); + } catch (error) { + console.error(error); + logWarn`Failed to prepare wikiData-bound reverse() functions`; + logWarn`\`reverse\` variable will be missing`; + } + const replContext = { universalUtilities, ...universalUtilities, @@ -95,6 +105,10 @@ export async function getContextAssignments({ find, bindFind, + _reverse, + reverse, + bindReverse, + showAggregate, }; diff --git a/src/write/build-modes/static-build.js b/src/write/build-modes/static-build.js index d40e1cb7..3d3c779b 100644 --- a/src/write/build-modes/static-build.js +++ b/src/write/build-modes/static-build.js @@ -27,6 +27,7 @@ import { } from '#cli'; import { + getOrigin, getPagePathname, getURLsFrom, getURLsFromRoot, @@ -436,12 +437,18 @@ async function writePage({ ].filter(Boolean)); } +function filterNoOrigin(route) { + return !getOrigin(route.to); +} + function writeWebRouteSymlinks({ outputPath, webRoutes, }) { const symlinkRoutes = - webRoutes.filter(route => route.statically === 'symlink'); + webRoutes + .filter(route => route.statically === 'symlink') + .filter(filterNoOrigin); const promises = symlinkRoutes.map(async route => { @@ -481,7 +488,9 @@ async function writeWebRouteCopies({ webRoutes, }) { const copyRoutes = - webRoutes.filter(route => route.statically === 'copy'); + webRoutes + .filter(route => route.statically === 'copy') + .filter(filterNoOrigin); const promises = copyRoutes.map(async route => { |