diff options
Diffstat (limited to 'src/data')
102 files changed, 11552 insertions, 3522 deletions
diff --git a/src/data/cacheable-object.js b/src/data/cacheable-object.js index 4afb036..1e7c7aa 100644 --- a/src/data/cacheable-object.js +++ b/src/data/cacheable-object.js @@ -74,236 +74,296 @@ // function, which provides a mapping of exposed property names to whether // or not their dependencies are yet met. -import { color, ENABLE_COLOR } from '../util/cli.js'; +import {inspect as nodeInspect} from 'node:util'; -import { inspect as nodeInspect } from 'util'; +import {colors, ENABLE_COLOR} from '#cli'; function inspect(value) { - return nodeInspect(value, {colors: ENABLE_COLOR}); + return nodeInspect(value, {colors: ENABLE_COLOR}); } export default class CacheableObject { - static instance = Symbol('CacheableObject `this` instance'); - - #propertyUpdateValues = Object.create(null); - #propertyUpdateCacheInvalidators = Object.create(null); - - /* - // Note the constructor doesn't take an initial data source. Due to a quirk - // of JavaScript, private members can't be accessed before the superclass's - // constructor is finished processing - so if we call the overridden - // update() function from inside this constructor, it will error when - // writing to private members. Pretty bad! - // - // That means initial data must be provided by following up with update() - // after constructing the new instance of the Thing (sub)class. - */ - - constructor() { - this.#defineProperties(); - this.#initializeUpdatingPropertyValues(); - - if (CacheableObject.DEBUG_SLOW_TRACK_INVALID_PROPERTIES) { - return new Proxy(this, { - get: (obj, key) => { - if (!Object.hasOwn(obj, key)) { - if (key !== 'constructor') { - CacheableObject._invalidAccesses.add(`(${obj.constructor.name}).${key}`); - } - } - return obj[key]; - } - }); - } + #propertyUpdateValues = Object.create(null); + #propertyUpdateCacheInvalidators = Object.create(null); + + // Note the constructor doesn't take an initial data source. Due to a quirk + // of JavaScript, private members can't be accessed before the superclass's + // constructor is finished processing - so if we call the overridden + // update() function from inside this constructor, it will error when + // writing to private members. Pretty bad! + // + // That means initial data must be provided by following up with update() + // after constructing the new instance of the Thing (sub)class. + + constructor() { + this.#defineProperties(); + this.#initializeUpdatingPropertyValues(); + + if (CacheableObject.DEBUG_SLOW_TRACK_INVALID_PROPERTIES) { + return new Proxy(this, { + get: (obj, key) => { + if (!Object.hasOwn(obj, key)) { + if (key !== 'constructor') { + CacheableObject._invalidAccesses.add(`(${obj.constructor.name}).${key}`); + } + } + return obj[key]; + }, + }); } + } - #initializeUpdatingPropertyValues() { - for (const [ property, descriptor ] of Object.entries(this.constructor.propertyDescriptors)) { - const { flags, update } = descriptor; + #initializeUpdatingPropertyValues() { + for (const [property, descriptor] of Object.entries(this.constructor.propertyDescriptors)) { + const {flags, update} = descriptor; - if (!flags.update) { - continue; - } + if (!flags.update) { + continue; + } - if (update?.default) { - this[property] = update?.default; - } else { - this[property] = null; - } - } + if (update?.default) { + this[property] = update?.default; + } else { + this[property] = null; + } } + } - #defineProperties() { - if (!this.constructor.propertyDescriptors) { - throw new Error(`Expected constructor ${this.constructor.name} to define propertyDescriptors`); - } + #defineProperties() { + if (!this.constructor.propertyDescriptors) { + throw new Error(`Expected constructor ${this.constructor.name} to define propertyDescriptors`); + } - for (const [ property, descriptor ] of Object.entries(this.constructor.propertyDescriptors)) { - const { flags } = descriptor; + for (const [property, descriptor] of Object.entries(this.constructor.propertyDescriptors)) { + const {flags} = descriptor; - const definition = { - configurable: false, - enumerable: true - }; + const definition = { + configurable: false, + enumerable: flags.expose, + }; - if (flags.update) { - definition.set = this.#getUpdateObjectDefinitionSetterFunction(property); - } + if (flags.update) { + definition.set = this.#getUpdateObjectDefinitionSetterFunction(property); + } - if (flags.expose) { - definition.get = this.#getExposeObjectDefinitionGetterFunction(property); - } + if (flags.expose) { + definition.get = this.#getExposeObjectDefinitionGetterFunction(property); + } + + Object.defineProperty(this, property, definition); + } - Object.defineProperty(this, property, definition); + Object.seal(this); + } + + #getUpdateObjectDefinitionSetterFunction(property) { + const {update} = this.#getPropertyDescriptor(property); + const validate = update?.validate; + + return (newValue) => { + const oldValue = this.#propertyUpdateValues[property]; + + if (newValue === undefined) { + throw new TypeError(`Properties cannot be set to undefined`); + } + + if (newValue === oldValue) { + return; + } + + if (newValue !== null && validate) { + try { + const result = validate(newValue); + if (result === undefined) { + throw new TypeError(`Validate function returned undefined`); + } else if (result !== true) { + throw new TypeError(`Validation failed for value ${newValue}`); + } + } catch (caughtError) { + throw new CacheableObjectPropertyValueError( + property, oldValue, newValue, {cause: caughtError}); } + } - Object.seal(this); - } + this.#propertyUpdateValues[property] = newValue; + this.#invalidateCachesDependentUpon(property); + }; + } - #getUpdateObjectDefinitionSetterFunction(property) { - const { update } = this.#getPropertyDescriptor(property); - const validate = update?.validate; - const allowNull = update?.allowNull; + #getPropertyDescriptor(property) { + return this.constructor.propertyDescriptors[property]; + } - return (newValue) => { - const oldValue = this.#propertyUpdateValues[property]; + #invalidateCachesDependentUpon(property) { + const invalidators = this.#propertyUpdateCacheInvalidators[property]; + if (!invalidators) { + return; + } - if (newValue === undefined) { - throw new TypeError(`Properties cannot be set to undefined`); - } + for (const invalidate of invalidators) { + invalidate(); + } + } + + #getExposeObjectDefinitionGetterFunction(property) { + const {flags} = this.#getPropertyDescriptor(property); + const compute = this.#getExposeComputeFunction(property); + + if (compute) { + let cachedValue; + const checkCacheValid = this.#getExposeCheckCacheValidFunction(property); + return () => { + if (checkCacheValid()) { + return cachedValue; + } else { + return (cachedValue = compute()); + } + }; + } else if (!flags.update && !compute) { + throw new Error(`Exposed property ${property} does not update and is missing compute function`); + } else { + return () => this.#propertyUpdateValues[property]; + } + } - if (newValue === oldValue) { - return; - } + #getExposeComputeFunction(property) { + const {flags, expose} = this.#getPropertyDescriptor(property); - if (newValue !== null && validate) { - try { - const result = validate(newValue); - if (result === undefined) { - throw new TypeError(`Validate function returned undefined`); - } else if (result !== true) { - throw new TypeError(`Validation failed for value ${newValue}`); - } - } catch (error) { - error.message = `Property ${color.green(property)} (${inspect(this[property])} -> ${inspect(newValue)}): ${error.message}`; - throw error; - } - } + const compute = expose?.compute; + const transform = expose?.transform; - this.#propertyUpdateValues[property] = newValue; - this.#invalidateCachesDependentUpon(property); - }; + if (flags.update && !transform) { + return null; + } else if (flags.update && compute) { + throw new Error(`Updating property ${property} has compute function, should be formatted as transform`); + } else if (!flags.update && !compute) { + throw new Error(`Exposed property ${property} does not update and is missing compute function`); } - #getUpdatePropertyValidateFunction(property) { - const descriptor = this.#getPropertyDescriptor(property); - } + let getAllDependencies; - #getPropertyDescriptor(property) { - return this.constructor.propertyDescriptors[property]; - } + if (expose.dependencies?.length > 0) { + const dependencyKeys = expose.dependencies.slice(); + const shouldReflect = dependencyKeys.includes('this'); + + getAllDependencies = () => { + const dependencies = Object.create(null); - #invalidateCachesDependentUpon(property) { - for (const invalidate of this.#propertyUpdateCacheInvalidators[property] || []) { - invalidate(); + for (const key of dependencyKeys) { + dependencies[key] = this.#propertyUpdateValues[key]; } - } - #getExposeObjectDefinitionGetterFunction(property) { - const { flags } = this.#getPropertyDescriptor(property); - const compute = this.#getExposeComputeFunction(property); - - if (compute) { - let cachedValue; - const checkCacheValid = this.#getExposeCheckCacheValidFunction(property); - return () => { - if (checkCacheValid()) { - return cachedValue; - } else { - return (cachedValue = compute()); - } - }; - } else if (!flags.update && !compute) { - throw new Error(`Exposed property ${property} does not update and is missing compute function`); - } else { - return () => this.#propertyUpdateValues[property]; + if (shouldReflect) { + dependencies.this = this; } + + return dependencies; + }; + } else { + const dependencies = Object.create(null); + Object.freeze(dependencies); + getAllDependencies = () => dependencies; } - #getExposeComputeFunction(property) { - const { flags, expose } = this.#getPropertyDescriptor(property); + if (flags.update) { + return () => transform(this.#propertyUpdateValues[property], getAllDependencies()); + } else { + return () => compute(getAllDependencies()); + } + } - const compute = expose?.compute; - const transform = expose?.transform; + #getExposeCheckCacheValidFunction(property) { + const {flags, expose} = this.#getPropertyDescriptor(property); - if (flags.update && !transform) { - return null; - } else if (flags.update && compute) { - throw new Error(`Updating property ${property} has compute function, should be formatted as transform`); - } else if (!flags.update && !compute) { - throw new Error(`Exposed property ${property} does not update and is missing compute function`); - } + let valid = false; - const dependencyKeys = expose.dependencies || []; - const dependencyGetters = dependencyKeys.map(key => () => [key, this.#propertyUpdateValues[key]]); - const getAllDependencies = () => Object.fromEntries(dependencyGetters.map(f => f()) - .concat([[this.constructor.instance, this]])); + const invalidate = () => { + valid = false; + }; - if (flags.update) { - return () => transform(this.#propertyUpdateValues[property], getAllDependencies()); - } else { - return () => compute(getAllDependencies()); - } + const dependencyKeys = new Set(expose?.dependencies); + + if (flags.update) { + dependencyKeys.add(property); } - #getExposeCheckCacheValidFunction(property) { - const { flags, expose } = this.#getPropertyDescriptor(property); + for (const key of dependencyKeys) { + if (this.#propertyUpdateCacheInvalidators[key]) { + this.#propertyUpdateCacheInvalidators[key].push(invalidate); + } else { + this.#propertyUpdateCacheInvalidators[key] = [invalidate]; + } + } - let valid = false; + return () => { + if (!valid) { + valid = true; + return false; + } else { + return true; + } + }; + } + + static cacheAllExposedProperties(obj) { + if (!(obj instanceof CacheableObject)) { + console.warn('Not a CacheableObject:', obj); + return; + } - const invalidate = () => { - valid = false; - }; + const {propertyDescriptors} = obj.constructor; - const dependencyKeys = new Set(expose?.dependencies); + if (!propertyDescriptors) { + console.warn('Missing property descriptors:', obj); + return; + } - if (flags.update) { - dependencyKeys.add(property); - } + for (const [property, descriptor] of Object.entries(propertyDescriptors)) { + const {flags} = descriptor; - for (const key of dependencyKeys) { - if (this.#propertyUpdateCacheInvalidators[key]) { - this.#propertyUpdateCacheInvalidators[key].push(invalidate); - } else { - this.#propertyUpdateCacheInvalidators[key] = [invalidate]; - } - } + if (!flags.expose) { + continue; + } - return () => { - if (!valid) { - valid = true; - return false; - } else { - return true; - } - }; + obj[property]; } + } - static DEBUG_SLOW_TRACK_INVALID_PROPERTIES = false; - static _invalidAccesses = new Set(); + static DEBUG_SLOW_TRACK_INVALID_PROPERTIES = false; + static _invalidAccesses = new Set(); - static showInvalidAccesses() { - if (!this.DEBUG_SLOW_TRACK_INVALID_PROPERTIES) { - return; - } + static showInvalidAccesses() { + if (!this.DEBUG_SLOW_TRACK_INVALID_PROPERTIES) { + return; + } - if (!this._invalidAccesses.size) { - return; - } + if (!this._invalidAccesses.size) { + return; + } - console.log(`${this._invalidAccesses.size} unique invalid accesses:`); - for (const line of this._invalidAccesses) { - console.log(` - ${line}`); - } + console.log(`${this._invalidAccesses.size} unique invalid accesses:`); + for (const line of this._invalidAccesses) { + console.log(` - ${line}`); } + } + + static getUpdateValue(object, key) { + if (!Object.hasOwn(object, key)) { + return undefined; + } + + return object.#propertyUpdateValues[key] ?? null; + } +} + +export class CacheableObjectPropertyValueError extends Error { + [Symbol.for('hsmusic.aggregate.translucent')] = true; + + constructor(property, oldValue, newValue, options) { + super( + `Error setting ${colors.green(property)} (${inspect(oldValue)} -> ${inspect(newValue)})`, + options); + + this.property = property; + } } diff --git a/src/data/checks.js b/src/data/checks.js new file mode 100644 index 0000000..44f3efd --- /dev/null +++ b/src/data/checks.js @@ -0,0 +1,704 @@ +// checks.js - general validation and error/warning reporting for data objects + +import {inspect as nodeInspect} from 'node:util'; +import {colors, ENABLE_COLOR} from '#cli'; + +import CacheableObject from '#cacheable-object'; +import {replacerSpec, parseInput} from '#replacer'; +import {compareArrays, cut, cutStart, empty, getNestedProp, iterateMultiline} + from '#sugar'; +import Thing from '#thing'; +import thingConstructors from '#things'; +import {commentaryRegexCaseSensitive} from '#wiki-data'; + +import { + annotateErrorWithIndex, + conditionallySuppressError, + decorateErrorWithIndex, + filterAggregate, + openAggregate, + withAggregate, +} from '#aggregate'; + +function inspect(value, opts = {}) { + return nodeInspect(value, {colors: ENABLE_COLOR, ...opts}); +} + +// Warn about directories which are reused across more than one of the same type +// of Thing. Directories are the unique identifier for most data objects across +// the wiki, so we have to make sure they aren't duplicated! +export function reportDuplicateDirectories(wikiData, { + getAllFindSpecs, +}) { + const duplicateSets = []; + + for (const findSpec of Object.values(getAllFindSpecs())) { + if (!findSpec.bindTo) continue; + + const directoryPlaces = Object.create(null); + const duplicateDirectories = new Set(); + const thingData = wikiData[findSpec.bindTo]; + + for (const thing of thingData) { + if (findSpec.include && !findSpec.include(thing)) { + continue; + } + + const directories = + (findSpec.getMatchableDirectories + ? findSpec.getMatchableDirectories(thing) + : [thing.directory]); + + for (const directory of directories) { + if (directory in directoryPlaces) { + directoryPlaces[directory].push(thing); + duplicateDirectories.add(directory); + } else { + directoryPlaces[directory] = [thing]; + } + } + } + + if (empty(duplicateDirectories)) continue; + + const sortedDuplicateDirectories = + Array.from(duplicateDirectories) + .sort((a, b) => { + const aL = a.toLowerCase(); + const bL = b.toLowerCase(); + return aL < bL ? -1 : aL > bL ? 1 : 0; + }); + + for (const directory of sortedDuplicateDirectories) { + const places = directoryPlaces[directory]; + duplicateSets.push({directory, places}); + } + } + + if (empty(duplicateSets)) return; + + // Multiple find functions may effectively have duplicates across the same + // things. These only need to be reported once, because resolving one of them + // will resolve the rest, so cut out duplicate sets before reporting. + + const seenDuplicateSets = new Map(); + const deduplicateDuplicateSets = []; + + for (const set of duplicateSets) { + if (seenDuplicateSets.has(set.directory)) { + const placeLists = seenDuplicateSets.get(set.directory); + + for (const places of placeLists) { + // We're iterating globally over all duplicate directories, which may + // span multiple kinds of things, but that isn't going to cause an + // issue because we're comparing the contents by identity, anyway. + // Two artists named Foodog aren't going to match two tracks named + // Foodog. + if (compareArrays(places, set.places, {checkOrder: false})) { + continue; + } + } + + placeLists.push(set.places); + } else { + seenDuplicateSets.set(set.directory, [set.places]); + } + + deduplicateDuplicateSets.push(set); + } + + withAggregate({message: `Duplicate directories found`}, ({push}) => { + for (const {directory, places} of deduplicateDuplicateSets) { + push(new Error( + `Duplicate directory ${colors.green(`"${directory}"`)}:\n` + + places.map(thing => ` - ` + inspect(thing)).join('\n'))); + } + }); +} + +function bindFindArtistOrAlias(boundFind) { + return artistRef => { + const alias = boundFind.artistAlias(artistRef, {mode: 'quiet'}); + if (alias) { + // No need to check if the original exists here. Aliases are automatically + // created from a field on the original, so the original certainly exists. + const original = alias.aliasedArtist; + throw new Error(`Reference ${colors.red(artistRef)} is to an alias, should be ${colors.green(original.name)}`); + } + + return boundFind.artist(artistRef); + }; +} + +function getFieldPropertyMessage(yamlDocumentSpec, property) { + const {fields} = yamlDocumentSpec; + + const field = + Object.entries(fields ?? {}) + .find(([field, fieldSpec]) => fieldSpec.property === property) + ?.[0]; + + const fieldPropertyMessage = + (field + ? ` in field ${colors.green(field)}` + : ` in property ${colors.green(property)}`); + + return fieldPropertyMessage; +} + +// Warn about references across data which don't match anything. This involves +// using the find() functions on all references, setting it to 'error' mode, and +// collecting everything in a structured logged (which gets logged if there are +// any errors). At the same time, we remove errored references from the thing's +// data array. +export function filterReferenceErrors(wikiData, { + bindFind, +}) { + const referenceSpec = [ + ['albumData', { + artistContribs: '_contrib', + coverArtistContribs: '_contrib', + trackCoverArtistContribs: '_contrib', + wallpaperArtistContribs: '_contrib', + bannerArtistContribs: '_contrib', + groups: 'group', + artTags: '_artTag', + commentary: '_commentary', + }], + + ['flashData', { + commentary: '_commentary', + }], + + ['groupCategoryData', { + groups: 'group', + }], + + ['homepageLayout.rows', { + sourceGroup: '_homepageSourceGroup', + sourceAlbums: 'album', + }], + + ['flashData', { + contributorContribs: '_contrib', + featuredTracks: 'track', + }], + + ['flashActData', { + flashes: 'flash', + }], + + ['trackData', { + artistContribs: '_contrib', + contributorContribs: '_contrib', + coverArtistContribs: '_contrib', + referencedTracks: '_trackNotRerelease', + sampledTracks: '_trackNotRerelease', + artTags: '_artTag', + originalReleaseTrack: '_trackNotRerelease', + commentary: '_commentary', + }], + + ['wikiInfo', { + divideTrackListsByGroups: 'group', + }], + ]; + + const boundFind = bindFind(wikiData, {mode: 'error'}); + const findArtistOrAlias = bindFindArtistOrAlias(boundFind); + + const aggregate = openAggregate({message: `Errors validating between-thing references in data`}); + for (const [thingDataProp, propSpec] of referenceSpec) { + const thingData = getNestedProp(wikiData, thingDataProp); + const things = Array.isArray(thingData) ? thingData : [thingData]; + aggregate.nest({message: `Reference errors in ${colors.green('wikiData.' + thingDataProp)}`}, ({nest}) => { + for (const thing of things) { + nest({message: `Reference errors in ${inspect(thing)}`}, ({nest, push, filter}) => { + for (const [property, findFnKey] of Object.entries(propSpec)) { + let value = CacheableObject.getUpdateValue(thing, property); + let writeProperty = true; + + switch (findFnKey) { + case '_commentary': + if (value) { + value = + Array.from(value.matchAll(commentaryRegexCaseSensitive)) + .map(({groups}) => groups.artistReferences) + .map(text => text.split(',').map(text => text.trim())); + } + + writeProperty = false; + break; + + case '_contrib': + // Don't write out contributions - these'll be filtered out + // for content and data purposes automatically, and they're + // handy to keep around when update values get checked for + // art tags below. (Possibly no reference-related properties + // need writing, humm...) + writeProperty = false; + break; + } + + if (value === undefined) { + push(new TypeError(`Property ${colors.red(property)} isn't valid for ${colors.green(thing.constructor.name)}`)); + continue; + } + + if (value === null) { + continue; + } + + let findFn; + + switch (findFnKey) { + case '_artTag': + findFn = boundFind.artTag; + break; + + case '_commentary': + findFn = findArtistOrAlias; + break; + + case '_contrib': + findFn = contribRef => findArtistOrAlias(contribRef.who); + break; + + case '_homepageSourceGroup': + findFn = groupRef => { + if (groupRef === 'new-additions' || groupRef === 'new-releases') { + return true; + } + + return boundFind.group(groupRef); + }; + break; + + case '_trackNotRerelease': + findFn = trackRef => { + const track = boundFind.track(trackRef); + const originalRef = track && CacheableObject.getUpdateValue(track, 'originalReleaseTrack'); + + if (originalRef) { + // It's possible for the original to not actually exist, in this case. + // It should still be reported since the 'Originally Released As' field + // was present. + const original = boundFind.track(originalRef, {mode: 'quiet'}); + + // Prefer references by name, but only if it's unambiguous. + const originalByName = + (original + ? boundFind.track(original.name, {mode: 'quiet'}) + : null); + + const shouldBeMessage = + (originalByName + ? colors.green(original.name) + : original + ? colors.green('track:' + original.directory) + : colors.green(originalRef)); + + throw new Error(`Reference ${colors.red(trackRef)} is to a rerelease, should be ${shouldBeMessage}`); + } + + return track; + }; + break; + + default: + findFn = boundFind[findFnKey]; + break; + } + + const suppress = fn => conditionallySuppressError(error => { + if (property === 'sampledTracks') { + // Suppress "didn't match anything" errors in particular, just for samples. + // In hsmusic-data we have a lot of "stub" sample data which don't have + // corresponding tracks yet, so it won't be useful to report such reference + // errors until we take the time to address that. But other errors, like + // malformed reference strings or miscapitalized existing tracks, should + // still be reported, as samples of existing tracks *do* display on the + // website! + if (error.message.includes(`Didn't match anything`)) { + return true; + } + } + + return false; + }, fn); + + const fieldPropertyMessage = + getFieldPropertyMessage( + thing.constructor[Thing.yamlDocumentSpec], + property); + + const findFnMessage = + (findFnKey.startsWith('_') + ? `` + : ` (${colors.green('find.' + findFnKey)})`); + + const errorMessage = + (Array.isArray(value) + ? `Reference errors` + fieldPropertyMessage + findFnMessage + : `Reference error` + fieldPropertyMessage + findFnMessage); + + let newPropertyValue = value; + + determineNewPropertyValue: { + // TODO: The special-casing for artTag is obviously a bit janky. + // It would be nice if this could be moved to processDocument ala + // fieldCombinationErrors, but art tags are only an error if the + // thing doesn't have an artwork - which can't be determined from + // the track document on its own, thanks to inheriting contribs + // from the album. + if (findFnKey === '_artTag') { + let hasCoverArtwork = + !empty(CacheableObject.getUpdateValue(thing, 'coverArtistContribs')); + + if (thing.constructor === thingConstructors.Track) { + if (thing.album) { + hasCoverArtwork ||= + !empty(CacheableObject.getUpdateValue(thing.album, 'trackCoverArtistContribs')); + } + + if (thing.disableUniqueCoverArt) { + hasCoverArtwork = false; + } + } + + if (!hasCoverArtwork) { + nest({message: errorMessage}, ({push}) => { + push(new TypeError(`No cover artwork, so this shouldn't have art tags specified`)); + }); + + newPropertyValue = []; + break determineNewPropertyValue; + } + } + + if (findFnKey === '_commentary') { + filter( + value, {message: errorMessage}, + decorateErrorWithIndex(refs => + (refs.length === 1 + ? suppress(findFn)(refs[0]) + : filterAggregate( + refs, {message: `Errors in entry's artist references`}, + decorateErrorWithIndex(suppress(findFn))) + .aggregate + .close()))); + + // Commentary doesn't write a property value, so no need to set + // anything on `newPropertyValue`. + break determineNewPropertyValue; + } + + if (Array.isArray(value)) { + newPropertyValue = filter( + value, {message: errorMessage}, + decorateErrorWithIndex(suppress(findFn))); + break determineNewPropertyValue; + } + + nest({message: errorMessage}, + suppress(({call}) => { + try { + call(findFn, value); + } catch (error) { + newPropertyValue = null; + throw error; + } + })); + } + + if (writeProperty) { + thing[property] = newPropertyValue; + } + } + }); + } + }); + } + + return aggregate; +} + +export class ContentNodeError extends Error { + constructor({ + length, + columnNumber, + containingLine, + where, + message, + }) { + const headingLine = + `(${where}) ${message}`; + + const textUpToNode = + containingLine.slice(0, columnNumber); + + const nodeText = + containingLine.slice(columnNumber, columnNumber + length); + + const textPastNode = + containingLine.slice(columnNumber + length); + + const containingLines = + containingLine.split('\n'); + + const formattedSourceLines = + containingLines.map((_, index, {length}) => { + let line = ' ⋮ '; + + if (index === 0) { + line += colors.dim(cutStart(textUpToNode, 20)); + } + + line += nodeText; + + if (index === length - 1) { + line += colors.dim(cut(textPastNode, 20)); + } + + return line; + }); + + super([ + headingLine, + ...formattedSourceLines, + ].filter(Boolean).join('\n')); + } +} + +export function reportContentTextErrors(wikiData, { + bindFind, +}) { + const additionalFileShape = { + description: 'description', + }; + + const commentaryShape = { + body: 'commentary body', + artistDisplayText: 'commentary artist display text', + annotation: 'commentary annotation', + }; + + const contentTextSpec = [ + ['albumData', { + additionalFiles: additionalFileShape, + commentary: commentaryShape, + }], + + ['artistData', { + contextNotes: '_content', + }], + + ['flashData', { + commentary: commentaryShape, + }], + + ['flashActData', { + listTerminology: '_content', + }], + + ['flashSideData', { + listTerminology: '_content', + }], + + ['groupData', { + description: '_content', + }], + + ['homepageLayout', { + sidebarContent: '_content', + }], + + ['newsData', { + content: '_content', + }], + + ['staticPageData', { + content: '_content', + }], + + ['trackData', { + additionalFiles: additionalFileShape, + commentary: commentaryShape, + lyrics: '_content', + midiProjectFiles: additionalFileShape, + sheetMusicFiles: additionalFileShape, + }], + + ['wikiInfo', { + description: '_content', + footerContent: '_content', + }], + ]; + + const boundFind = bindFind(wikiData, {mode: 'error'}); + const findArtistOrAlias = bindFindArtistOrAlias(boundFind); + + function* processContent(input) { + const nodes = parseInput(input); + + for (const node of nodes) { + const index = node.i; + const length = node.iEnd - node.i; + + if (node.type === 'tag') { + const replacerKeyImplied = !node.data.replacerKey; + const replacerKey = replacerKeyImplied ? 'track' : node.data.replacerKey.data; + const spec = replacerSpec[replacerKey]; + + if (!spec) { + yield { + index, length, + message: + `Unknown tag key ${colors.red(`"${replacerKey}"`)}`, + }; + + // No spec, no further errors to report. + continue; + } + + const replacerValue = node.data.replacerValue[0].data; + + if (spec.find) { + let findFn; + + switch (spec.find) { + case 'artist': + findFn = findArtistOrAlias; + break; + + default: + findFn = boundFind[spec.find]; + break; + } + + const findRef = + (replacerKeyImplied + ? replacerValue + : replacerKey + `:` + replacerValue); + + try { + findFn(findRef); + } catch (error) { + yield { + index, length, + message: error.message, + }; + + // It's only possible to have one error per node at the moment. + continue; + } + } + } else if (node.type === 'external-link') { + try { + new URL(node.data.href); + } catch (error) { + yield { + index, length, + message: + `Invalid URL ${colors.red(`"${node.data.href}"`)}`, + }; + } + } + } + } + + function callProcessContent({ + nest, + push, + value, + message, + annotateError = error => error, + }) { + const processContentIterator = + nest({message}, ({call}) => + call(processContent, value)); + + if (!processContentIterator) return; + + const multilineIterator = + iterateMultiline(value, processContentIterator, { + formatWhere: true, + getContainingLine: true, + }); + + const errors = []; + + for (const result of multilineIterator) { + errors.push(new ContentNodeError(result)); + } + + if (empty(errors)) return; + + push( + annotateError( + new AggregateError(errors, message))); + } + + withAggregate({message: `Errors validating content text`}, ({nest}) => { + for (const [thingDataProp, propSpec] of contentTextSpec) { + const thingData = getNestedProp(wikiData, thingDataProp); + const things = Array.isArray(thingData) ? thingData : [thingData]; + nest({message: `Content text errors in ${colors.green('wikiData.' + thingDataProp)}`}, ({nest}) => { + for (const thing of things) { + nest({message: `Content text errors in ${inspect(thing)}`}, ({nest, push}) => { + + for (const [property, shape] of Object.entries(propSpec)) { + const value = thing[property]; + + if (value === undefined) { + push(new TypeError(`Property ${colors.red(property)} isn't valid for ${colors.green(thing.constructor.name)}`)); + continue; + } + + if (value === null) { + continue; + } + + const fieldPropertyMessage = + getFieldPropertyMessage( + thing.constructor[Thing.yamlDocumentSpec], + property); + + const topMessage = + `Content text errors` + fieldPropertyMessage; + + if (shape === '_content') { + callProcessContent({ + nest, + push, + value, + message: topMessage, + }); + } else { + nest({message: topMessage}, ({push}) => { + for (const [index, entry] of value.entries()) { + for (const [key, annotation] of Object.entries(shape)) { + const value = entry[key]; + + // TODO: Should this check undefined/null similar to above? + if (!value) continue; + + callProcessContent({ + nest, + push, + value, + message: `Error in ${colors.green(annotation)}`, + annotateError: error => + annotateErrorWithIndex(error, index), + }); + } + } + }); + } + } + }); + } + }); + } + }); +} diff --git a/src/data/composite.js b/src/data/composite.js new file mode 100644 index 0000000..7a98c42 --- /dev/null +++ b/src/data/composite.js @@ -0,0 +1,1297 @@ +import {inspect} from 'node:util'; + +import {decorateErrorWithIndex, openAggregate, withAggregate} + from '#aggregate'; +import {colors} from '#cli'; +import {empty, filterProperties, stitchArrays, typeAppearance, unique} + from '#sugar'; +import {a} from '#validators'; +import {TupleMap} from '#wiki-data'; + +const globalCompositeCache = {}; + +const _valueIntoToken = shape => + (value = null) => + (value === null + ? Symbol.for(`hsmusic.composite.${shape}`) + : typeof value === 'string' + ? Symbol.for(`hsmusic.composite.${shape}:${value}`) + : { + symbol: Symbol.for(`hsmusic.composite.input`), + shape, + value, + }); + +export const input = _valueIntoToken('input'); +input.symbol = Symbol.for('hsmusic.composite.input'); + +input.value = _valueIntoToken('input.value'); +input.dependency = _valueIntoToken('input.dependency'); + +input.myself = () => Symbol.for(`hsmusic.composite.input.myself`); + +input.updateValue = _valueIntoToken('input.updateValue'); + +input.staticDependency = _valueIntoToken('input.staticDependency'); +input.staticValue = _valueIntoToken('input.staticValue'); + +function isInputToken(token) { + if (token === null) { + return false; + } else if (typeof token === 'object') { + return token.symbol === Symbol.for('hsmusic.composite.input'); + } else if (typeof token === 'symbol') { + return token.description.startsWith('hsmusic.composite.input'); + } else { + return false; + } +} + +function getInputTokenShape(token) { + if (!isInputToken(token)) { + throw new TypeError(`Expected an input token, got ${typeAppearance(token)}`); + } + + if (typeof token === 'object') { + return token.shape; + } else { + return token.description.match(/hsmusic\.composite\.(input.*?)(:|$)/)[1]; + } +} + +function getInputTokenValue(token) { + if (!isInputToken(token)) { + throw new TypeError(`Expected an input token, got ${typeAppearance(token)}`); + } + + if (typeof token === 'object') { + return token.value; + } else { + return token.description.match(/hsmusic\.composite\.input.*?:(.*)/)?.[1] ?? null; + } +} + +function getStaticInputMetadata(inputOptions) { + const metadata = {}; + + for (const [name, token] of Object.entries(inputOptions)) { + if (typeof token === 'string') { + metadata[input.staticDependency(name)] = token; + metadata[input.staticValue(name)] = null; + } else if (isInputToken(token)) { + const tokenShape = getInputTokenShape(token); + const tokenValue = getInputTokenValue(token); + + metadata[input.staticDependency(name)] = + (tokenShape === 'input.dependency' + ? tokenValue + : null); + + metadata[input.staticValue(name)] = + (tokenShape === 'input.value' + ? tokenValue + : null); + } else { + metadata[input.staticDependency(name)] = null; + metadata[input.staticValue(name)] = null; + } + } + + return metadata; +} + +function getCompositionName(description) { + return ( + (description.annotation + ? description.annotation + : `unnamed composite`)); +} + +function validateInputValue(value, description) { + const tokenValue = getInputTokenValue(description); + + const {acceptsNull, defaultValue, type, validate} = tokenValue || {}; + + if (value === null || value === undefined) { + if (acceptsNull || defaultValue === null) { + return true; + } else { + throw new TypeError( + (type + ? `Expected ${a(type)}, got ${typeAppearance(value)}` + : `Expected a value, got ${typeAppearance(value)}`)); + } + } + + if (type) { + // Note: null is already handled earlier in this function, so it won't + // cause any trouble here. + const typeofValue = + (typeof value === 'object' + ? Array.isArray(value) ? 'array' : 'object' + : typeof value); + + if (typeofValue !== type) { + throw new TypeError(`Expected ${a(type)}, got ${typeAppearance(value)}`); + } + } + + if (validate) { + validate(value); + } + + return true; +} + +export function templateCompositeFrom(description) { + const compositionName = getCompositionName(description); + + withAggregate({message: `Errors in description for ${compositionName}`}, ({map, nest, push}) => { + if ('steps' in description) { + if (Array.isArray(description.steps)) { + push(new TypeError(`Wrap steps array in a function`)); + } else if (typeof description.steps !== 'function') { + push(new TypeError(`Expected steps to be a function (returning an array)`)); + } + } + + validateInputs: + if ('inputs' in description) { + if ( + Array.isArray(description.inputs) || + typeof description.inputs !== 'object' + ) { + push(new Error(`Expected inputs to be object, got ${typeAppearance(description.inputs)}`)); + break validateInputs; + } + + nest({message: `Errors in static input descriptions for ${compositionName}`}, ({push}) => { + const missingCallsToInput = []; + const wrongCallsToInput = []; + + for (const [name, value] of Object.entries(description.inputs)) { + if (!isInputToken(value)) { + missingCallsToInput.push(name); + continue; + } + + if (!['input', 'input.staticDependency', 'input.staticValue'].includes(getInputTokenShape(value))) { + wrongCallsToInput.push(name); + } + } + + for (const name of missingCallsToInput) { + push(new Error(`${name}: Missing call to input()`)); + } + + for (const name of wrongCallsToInput) { + const shape = getInputTokenShape(description.inputs[name]); + push(new Error(`${name}: Expected call to input, input.staticDependency, or input.staticValue, got ${shape}`)); + } + }); + } + + validateOutputs: + if ('outputs' in description) { + if ( + !Array.isArray(description.outputs) && + typeof description.outputs !== 'function' + ) { + push(new Error(`Expected outputs to be array or function, got ${typeAppearance(description.outputs)}`)); + break validateOutputs; + } + + if (Array.isArray(description.outputs)) { + map( + description.outputs, + decorateErrorWithIndex(value => { + if (typeof value !== 'string') { + throw new Error(`${value}: Expected string, got ${typeAppearance(value)}`) + } else if (!value.startsWith('#')) { + throw new Error(`${value}: Expected "#" at start`); + } + }), + {message: `Errors in output descriptions for ${compositionName}`}); + } + } + }); + + const expectedInputNames = + (description.inputs + ? Object.keys(description.inputs) + : []); + + const instantiate = (inputOptions = {}) => { + withAggregate({message: `Errors in input options passed to ${compositionName}`}, ({push}) => { + const providedInputNames = Object.keys(inputOptions); + + const misplacedInputNames = + providedInputNames + .filter(name => !expectedInputNames.includes(name)); + + const missingInputNames = + expectedInputNames + .filter(name => !providedInputNames.includes(name)) + .filter(name => { + const inputDescription = getInputTokenValue(description.inputs[name]); + if (!inputDescription) return true; + if ('defaultValue' in inputDescription) return false; + if ('defaultDependency' in inputDescription) return false; + return true; + }); + + const wrongTypeInputNames = []; + + const expectedStaticValueInputNames = []; + const expectedStaticDependencyInputNames = []; + const expectedValueProvidingTokenInputNames = []; + + const validateFailedErrors = []; + + for (const [name, value] of Object.entries(inputOptions)) { + if (misplacedInputNames.includes(name)) { + continue; + } + + if (typeof value !== 'string' && !isInputToken(value)) { + wrongTypeInputNames.push(name); + continue; + } + + const descriptionShape = getInputTokenShape(description.inputs[name]); + + const tokenShape = (isInputToken(value) ? getInputTokenShape(value) : null); + const tokenValue = (isInputToken(value) ? getInputTokenValue(value) : null); + + switch (descriptionShape) { + case'input.staticValue': + if (tokenShape !== 'input.value') { + expectedStaticValueInputNames.push(name); + continue; + } + break; + + case 'input.staticDependency': + if (typeof value !== 'string' && tokenShape !== 'input.dependency') { + expectedStaticDependencyInputNames.push(name); + continue; + } + break; + + case 'input': + if (typeof value !== 'string' && ![ + 'input', + 'input.value', + 'input.dependency', + 'input.myself', + 'input.updateValue', + ].includes(tokenShape)) { + expectedValueProvidingTokenInputNames.push(name); + continue; + } + break; + } + + if (tokenShape === 'input.value') { + try { + validateInputValue(tokenValue, description.inputs[name]); + } catch (error) { + error.message = `${name}: ${error.message}`; + validateFailedErrors.push(error); + } + } + } + + if (!empty(misplacedInputNames)) { + push(new Error(`Unexpected input names: ${misplacedInputNames.join(', ')}`)); + } + + if (!empty(missingInputNames)) { + push(new Error(`Required these inputs: ${missingInputNames.join(', ')}`)); + } + + const inputAppearance = name => + (isInputToken(inputOptions[name]) + ? `${getInputTokenShape(inputOptions[name])}() call` + : `dependency name`); + + for (const name of expectedStaticDependencyInputNames) { + const appearance = inputAppearance(name); + push(new Error(`${name}: Expected dependency name, got ${appearance}`)); + } + + for (const name of expectedStaticValueInputNames) { + const appearance = inputAppearance(name) + push(new Error(`${name}: Expected input.value() call, got ${appearance}`)); + } + + for (const name of expectedValueProvidingTokenInputNames) { + const appearance = getInputTokenShape(inputOptions[name]); + push(new Error(`${name}: Expected dependency name or value-providing input() call, got ${appearance}`)); + } + + for (const name of wrongTypeInputNames) { + const type = typeAppearance(inputOptions[name]); + push(new Error(`${name}: Expected dependency name or input() call, got ${type}`)); + } + + for (const error of validateFailedErrors) { + push(error); + } + }); + + const inputMetadata = getStaticInputMetadata(inputOptions); + + const expectedOutputNames = + (Array.isArray(description.outputs) + ? description.outputs + : typeof description.outputs === 'function' + ? description.outputs(inputMetadata) + .map(name => + (name.startsWith('#') + ? name + : '#' + name)) + : []); + + const ownUpdateDescription = + (typeof description.update === 'object' + ? description.update + : typeof description.update === 'function' + ? description.update(inputMetadata) + : null); + + const outputOptions = {}; + + const instantiatedTemplate = { + symbol: templateCompositeFrom.symbol, + + outputs(providedOptions) { + withAggregate({message: `Errors in output options passed to ${compositionName}`}, ({push}) => { + const misplacedOutputNames = []; + const wrongTypeOutputNames = []; + + for (const [name, value] of Object.entries(providedOptions)) { + if (!expectedOutputNames.includes(name)) { + misplacedOutputNames.push(name); + continue; + } + + if (typeof value !== 'string') { + wrongTypeOutputNames.push(name); + continue; + } + } + + if (!empty(misplacedOutputNames)) { + push(new Error(`Unexpected output names: ${misplacedOutputNames.join(', ')}`)); + } + + for (const name of wrongTypeOutputNames) { + const appearance = typeAppearance(providedOptions[name]); + push(new Error(`${name}: Expected string, got ${appearance}`)); + } + }); + + Object.assign(outputOptions, providedOptions); + return instantiatedTemplate; + }, + + toDescription() { + const finalDescription = {}; + + if ('annotation' in description) { + finalDescription.annotation = description.annotation; + } + + if ('compose' in description) { + finalDescription.compose = description.compose; + } + + if (ownUpdateDescription) { + finalDescription.update = ownUpdateDescription; + } + + if ('inputs' in description) { + const inputMapping = {}; + + for (const [name, token] of Object.entries(description.inputs)) { + const tokenValue = getInputTokenValue(token); + if (name in inputOptions) { + if (typeof inputOptions[name] === 'string') { + inputMapping[name] = input.dependency(inputOptions[name]); + } else { + inputMapping[name] = inputOptions[name]; + } + } else if (tokenValue.defaultValue) { + inputMapping[name] = input.value(tokenValue.defaultValue); + } else if (tokenValue.defaultDependency) { + inputMapping[name] = input.dependency(tokenValue.defaultDependency); + } else { + inputMapping[name] = input.value(null); + } + } + + finalDescription.inputMapping = inputMapping; + finalDescription.inputDescriptions = description.inputs; + } + + if ('outputs' in description) { + const finalOutputs = {}; + + for (const name of expectedOutputNames) { + if (name in outputOptions) { + finalOutputs[name] = outputOptions[name]; + } else { + finalOutputs[name] = name; + } + } + + finalDescription.outputs = finalOutputs; + } + + if ('steps' in description) { + finalDescription.steps = description.steps; + } + + return finalDescription; + }, + + toResolvedComposition() { + const ownDescription = instantiatedTemplate.toDescription(); + + const finalDescription = {...ownDescription}; + + const aggregate = openAggregate({message: `Errors resolving ${compositionName}`}); + + const steps = ownDescription.steps(); + + const resolvedSteps = + aggregate.map( + steps, + decorateErrorWithIndex(step => + (step.symbol === templateCompositeFrom.symbol + ? compositeFrom(step.toResolvedComposition()) + : step)), + {message: `Errors resolving steps`}); + + aggregate.close(); + + finalDescription.steps = resolvedSteps; + + return finalDescription; + }, + }; + + return instantiatedTemplate; + }; + + instantiate.inputs = instantiate; + + return instantiate; +} + +templateCompositeFrom.symbol = Symbol(); + +export const continuationSymbol = Symbol.for('compositeFrom: continuation symbol'); +export const noTransformSymbol = Symbol.for('compositeFrom: no-transform symbol'); + +export function compositeFrom(description) { + const {annotation} = description; + const compositionName = getCompositionName(description); + + const debug = fn => { + if (compositeFrom.debug === true) { + const label = + (annotation + ? colors.dim(`[composite: ${annotation}]`) + : colors.dim(`[composite]`)); + const result = fn(); + if (Array.isArray(result)) { + console.log(label, ...result.map(value => + (typeof value === 'object' + ? inspect(value, {depth: 1, colors: true, compact: true, breakLength: Infinity}) + : value))); + } else { + console.log(label, result); + } + } + }; + + if (!Array.isArray(description.steps)) { + throw new TypeError( + `Expected steps to be array, got ${typeAppearance(description.steps)}` + + (annotation ? ` (${annotation})` : '')); + } + + const composition = + description.steps.map(step => + ('toResolvedComposition' in step + ? compositeFrom(step.toResolvedComposition()) + : step)); + + const inputMetadata = getStaticInputMetadata(description.inputMapping ?? {}); + + function _mapDependenciesToOutputs(providedDependencies) { + if (!description.outputs) { + return {}; + } + + if (!providedDependencies) { + return {}; + } + + return ( + Object.fromEntries( + Object.entries(description.outputs) + .map(([continuationName, outputName]) => [ + outputName, + (continuationName in providedDependencies + ? providedDependencies[continuationName] + : providedDependencies[continuationName.replace(/^#/, '')]), + ]))); + } + + // These dependencies were all provided by the composition which this one is + // nested inside, so input('name')-shaped tokens are going to be evaluated + // in the context of the containing composition. + const dependenciesFromInputs = + Object.values(description.inputMapping ?? {}) + .map(token => { + const tokenShape = getInputTokenShape(token); + const tokenValue = getInputTokenValue(token); + switch (tokenShape) { + case 'input.dependency': + return tokenValue; + case 'input': + case 'input.updateValue': + return token; + case 'input.myself': + return 'this'; + default: + return null; + } + }) + .filter(Boolean); + + const anyInputsUseUpdateValue = + dependenciesFromInputs + .filter(dependency => isInputToken(dependency)) + .some(token => getInputTokenShape(token) === 'input.updateValue'); + + const inputNames = + Object.keys(description.inputMapping ?? {}); + + const inputSymbols = + inputNames.map(name => input(name)); + + const inputsMayBeDynamicValue = + stitchArrays({ + mappingToken: Object.values(description.inputMapping ?? {}), + descriptionToken: Object.values(description.inputDescriptions ?? {}), + }).map(({mappingToken, descriptionToken}) => { + if (getInputTokenShape(descriptionToken) === 'input.staticValue') return false; + if (getInputTokenShape(mappingToken) === 'input.value') return false; + return true; + }); + + const inputDescriptions = + Object.values(description.inputDescriptions ?? {}); + + /* + const inputsAcceptNull = + Object.values(description.inputDescriptions ?? {}) + .map(token => { + const tokenValue = getInputTokenValue(token); + if (!tokenValue) return false; + if ('acceptsNull' in tokenValue) return tokenValue.acceptsNull; + if ('defaultValue' in tokenValue) return tokenValue.defaultValue === null; + return false; + }); + */ + + // Update descriptions passed as the value in an input.updateValue() token, + // as provided as inputs for this composition. + const inputUpdateDescriptions = + Object.values(description.inputMapping ?? {}) + .map(token => + (getInputTokenShape(token) === 'input.updateValue' + ? getInputTokenValue(token) + : null)) + .filter(Boolean); + + const base = composition.at(-1); + const steps = composition.slice(); + + const aggregate = openAggregate({ + message: + `Errors preparing composition` + + (annotation ? ` (${annotation})` : ''), + }); + + const compositionNests = description.compose ?? true; + + if (compositionNests && empty(steps)) { + aggregate.push(new TypeError(`Expected at least one step`)); + } + + // Steps default to exposing if using a shorthand syntax where flags aren't + // specified at all. + const stepsExpose = + steps + .map(step => + (step.flags + ? step.flags.expose ?? false + : true)); + + // Steps default to composing if using a shorthand syntax where flags aren't + // specified at all - *and* aren't the base (final step), unless the whole + // composition is nestable. + const stepsCompose = + steps + .map((step, index, {length}) => + (step.flags + ? step.flags.compose ?? false + : (index === length - 1 + ? compositionNests + : true))); + + // Steps update if the corresponding flag is explicitly set, if a transform + // function is provided, or if the dependencies include an input.updateValue + // token. + const stepsUpdate = + steps + .map(step => + (step.flags + ? step.flags.update ?? false + : !!step.transform || + !!step.dependencies?.some(dependency => + isInputToken(dependency) && + getInputTokenShape(dependency) === 'input.updateValue'))); + + // The expose description for a step is just the entire step object, when + // using the shorthand syntax where {flags: {expose: true}} is left implied. + const stepExposeDescriptions = + steps + .map((step, index) => + (stepsExpose[index] + ? (step.flags + ? step.expose ?? null + : step) + : null)); + + // The update description for a step, if present at all, is always set + // explicitly. There may be multiple per step - namely that step's own + // {update} description, and any descriptions passed as the value in an + // input.updateValue({...}) token. + const stepUpdateDescriptions = + steps + .map((step, index) => + (stepsUpdate[index] + ? [ + step.update ?? null, + ...(stepExposeDescriptions[index]?.dependencies ?? []) + .filter(dependency => isInputToken(dependency)) + .filter(token => getInputTokenShape(token) === 'input.updateValue') + .map(token => getInputTokenValue(token)), + ].filter(Boolean) + : [])); + + // Indicates presence of a {compute} function on the expose description. + const stepsCompute = + stepExposeDescriptions + .map(expose => !!expose?.compute); + + // Indicates presence of a {transform} function on the expose description. + const stepsTransform = + stepExposeDescriptions + .map(expose => !!expose?.transform); + + const dependenciesFromSteps = + unique( + stepExposeDescriptions + .flatMap(expose => expose?.dependencies ?? []) + .map(dependency => { + if (typeof dependency === 'string') + return (dependency.startsWith('#') ? null : dependency); + + const tokenShape = getInputTokenShape(dependency); + const tokenValue = getInputTokenValue(dependency); + switch (tokenShape) { + case 'input.dependency': + return (tokenValue.startsWith('#') ? null : tokenValue); + case 'input.myself': + return 'this'; + default: + return null; + } + }) + .filter(Boolean)); + + const anyStepsUseUpdateValue = + stepExposeDescriptions + .some(expose => + (expose?.dependencies + ? expose.dependencies.includes(input.updateValue()) + : false)); + + const anyStepsExpose = + stepsExpose.includes(true); + + const anyStepsUpdate = + stepsUpdate.includes(true); + + const anyStepsCompute = + stepsCompute.includes(true); + + const compositionExposes = + anyStepsExpose; + + const compositionUpdates = + 'update' in description || + anyInputsUseUpdateValue || + anyStepsUseUpdateValue || + anyStepsUpdate; + + const stepEntries = stitchArrays({ + step: steps, + stepComposes: stepsCompose, + stepComputes: stepsCompute, + stepTransforms: stepsTransform, + }); + + for (let i = 0; i < stepEntries.length; i++) { + const { + step, + stepComposes, + stepComputes, + stepTransforms, + } = stepEntries[i]; + + const isBase = i === stepEntries.length - 1; + const message = + `Errors in step #${i + 1}` + + (isBase ? ` (base)` : ``) + + (step.annotation ? ` (${step.annotation})` : ``); + + aggregate.nest({message}, ({push}) => { + if (isBase && stepComposes !== compositionNests) { + return push(new TypeError( + (compositionNests + ? `Base must compose, this composition is nestable` + : `Base must not compose, this composition isn't nestable`))); + } else if (!isBase && !stepComposes) { + return push(new TypeError( + (compositionNests + ? `All steps must compose` + : `All steps (except base) must compose`))); + } + + if ( + !compositionNests && !compositionUpdates && + stepTransforms && !stepComputes + ) { + return push(new TypeError( + `Steps which only transform can't be used in a composition that doesn't update`)); + } + }); + } + + if (!compositionNests && !compositionUpdates && !anyStepsCompute) { + aggregate.push(new TypeError(`Expected at least one step to compute`)); + } + + aggregate.close(); + + function _prepareContinuation(callingTransformForThisStep) { + const continuationStorage = { + returnedWith: null, + providedDependencies: undefined, + providedValue: undefined, + }; + + const continuation = + (callingTransformForThisStep + ? (providedValue, providedDependencies = null) => { + continuationStorage.returnedWith = 'continuation'; + continuationStorage.providedDependencies = providedDependencies; + continuationStorage.providedValue = providedValue; + return continuationSymbol; + } + : (providedDependencies = null) => { + continuationStorage.returnedWith = 'continuation'; + continuationStorage.providedDependencies = providedDependencies; + return continuationSymbol; + }); + + continuation.exit = (providedValue) => { + continuationStorage.returnedWith = 'exit'; + continuationStorage.providedValue = providedValue; + return continuationSymbol; + }; + + if (compositionNests) { + const makeRaiseLike = returnWith => + (callingTransformForThisStep + ? (providedValue, providedDependencies = null) => { + continuationStorage.returnedWith = returnWith; + continuationStorage.providedDependencies = providedDependencies; + continuationStorage.providedValue = providedValue; + return continuationSymbol; + } + : (providedDependencies = null) => { + continuationStorage.returnedWith = returnWith; + continuationStorage.providedDependencies = providedDependencies; + return continuationSymbol; + }); + + continuation.raiseOutput = makeRaiseLike('raiseOutput'); + continuation.raiseOutputAbove = makeRaiseLike('raiseOutputAbove'); + } + + return {continuation, continuationStorage}; + } + + function _computeOrTransform(initialValue, continuationIfApplicable, initialDependencies) { + const expectingTransform = initialValue !== noTransformSymbol; + + let valueSoFar = + (expectingTransform + ? initialValue + : undefined); + + const availableDependencies = {...initialDependencies}; + + const inputValues = + Object.values(description.inputMapping ?? {}) + .map(token => { + const tokenShape = getInputTokenShape(token); + const tokenValue = getInputTokenValue(token); + switch (tokenShape) { + case 'input.dependency': + return initialDependencies[tokenValue]; + case 'input.value': + return tokenValue; + case 'input.updateValue': + if (!expectingTransform) + throw new Error(`Unexpected input.updateValue() accessed on non-transform call`); + return valueSoFar; + case 'input.myself': + return initialDependencies['this']; + case 'input': + return initialDependencies[token]; + default: + throw new TypeError(`Unexpected input shape ${tokenShape}`); + } + }); + + withAggregate({message: `Errors in input values provided to ${compositionName}`}, ({push}) => { + for (const {dynamic, name, value, description} of stitchArrays({ + dynamic: inputsMayBeDynamicValue, + name: inputNames, + value: inputValues, + description: inputDescriptions, + })) { + if (!dynamic) continue; + try { + validateInputValue(value, description); + } catch (error) { + error.message = `${name}: ${error.message}`; + push(error); + } + } + }); + + if (expectingTransform) { + debug(() => [colors.bright(`begin composition - transforming from:`), initialValue]); + } else { + debug(() => colors.bright(`begin composition - not transforming`)); + } + + for (let i = 0; i < steps.length; i++) { + const step = steps[i]; + const isBase = i === steps.length - 1; + + debug(() => [ + `step #${i+1}` + + (isBase + ? ` (base):` + : ` of ${steps.length}:`), + step]); + + const expose = + (step.flags + ? step.expose + : step); + + if (!expose) { + if (!isBase) { + debug(() => `step #${i+1} - no expose description, nothing to do for this step`); + continue; + } + + if (expectingTransform) { + debug(() => `step #${i+1} (base) - no expose description, returning so-far update value:`, valueSoFar); + if (continuationIfApplicable) { + debug(() => colors.bright(`end composition - raise (inferred - composing)`)); + return continuationIfApplicable(valueSoFar); + } else { + debug(() => colors.bright(`end composition - exit (inferred - not composing)`)); + return valueSoFar; + } + } else { + debug(() => `step #${i+1} (base) - no expose description, nothing to continue with`); + if (continuationIfApplicable) { + debug(() => colors.bright(`end composition - raise (inferred - composing)`)); + return continuationIfApplicable(); + } else { + debug(() => colors.bright(`end composition - exit (inferred - not composing)`)); + return null; + } + } + } + + const callingTransformForThisStep = + expectingTransform && expose.transform; + + let continuationStorage; + + const inputDictionary = + Object.fromEntries( + stitchArrays({symbol: inputSymbols, value: inputValues}) + .map(({symbol, value}) => [symbol, value])); + + const filterableDependencies = { + ...availableDependencies, + ...inputMetadata, + ...inputDictionary, + ... + (expectingTransform + ? {[input.updateValue()]: valueSoFar} + : {}), + [input.myself()]: initialDependencies?.['this'] ?? null, + }; + + const selectDependencies = + (expose.dependencies ?? []).map(dependency => { + if (!isInputToken(dependency)) return dependency; + const tokenShape = getInputTokenShape(dependency); + const tokenValue = getInputTokenValue(dependency); + switch (tokenShape) { + case 'input': + case 'input.staticDependency': + case 'input.staticValue': + return dependency; + case 'input.myself': + return input.myself(); + case 'input.dependency': + return tokenValue; + case 'input.updateValue': + return input.updateValue(); + default: + throw new Error(`Unexpected token ${tokenShape} as dependency`); + } + }) + + const filteredDependencies = + filterProperties(filterableDependencies, selectDependencies); + + debug(() => [ + `step #${i+1} - ${callingTransformForThisStep ? 'transform' : 'compute'}`, + `with dependencies:`, filteredDependencies, + `selecting:`, selectDependencies, + `from available:`, filterableDependencies, + ...callingTransformForThisStep ? [`from value:`, valueSoFar] : []]); + + let result; + + const getExpectedEvaluation = () => + (callingTransformForThisStep + ? (filteredDependencies + ? ['transform', valueSoFar, continuationSymbol, filteredDependencies] + : ['transform', valueSoFar, continuationSymbol]) + : (filteredDependencies + ? ['compute', continuationSymbol, filteredDependencies] + : ['compute', continuationSymbol])); + + const naturalEvaluate = () => { + const [name, ...argsLayout] = getExpectedEvaluation(); + + let args; + + if (isBase && !compositionNests) { + args = + argsLayout.filter(arg => arg !== continuationSymbol); + } else { + let continuation; + + ({continuation, continuationStorage} = + _prepareContinuation(callingTransformForThisStep)); + + args = + argsLayout.map(arg => + (arg === continuationSymbol + ? continuation + : arg)); + } + + return expose[name](...args); + } + + switch (step.cache) { + // Warning! Highly WIP! + case 'aggressive': { + const hrnow = () => { + const hrTime = process.hrtime(); + return hrTime[0] * 1000000000 + hrTime[1]; + }; + + const [name, ...args] = getExpectedEvaluation(); + + let cache = globalCompositeCache[step.annotation]; + if (!cache) { + cache = globalCompositeCache[step.annotation] = { + transform: new TupleMap(), + compute: new TupleMap(), + times: { + read: [], + evaluate: [], + }, + }; + } + + const tuplefied = args + .flatMap(arg => [ + Symbol.for('compositeFrom: tuplefied arg divider'), + ...(typeof arg !== 'object' || Array.isArray(arg) + ? [arg] + : Object.entries(arg).flat()), + ]); + + const readTime = hrnow(); + const cacheContents = cache[name].get(tuplefied); + cache.times.read.push(hrnow() - readTime); + + if (cacheContents) { + ({result, continuationStorage} = cacheContents); + } else { + const evaluateTime = hrnow(); + result = naturalEvaluate(); + cache.times.evaluate.push(hrnow() - evaluateTime); + cache[name].set(tuplefied, {result, continuationStorage}); + } + + break; + } + + default: { + result = naturalEvaluate(); + break; + } + } + + if (result !== continuationSymbol) { + debug(() => [`step #${i+1} - result: exit (inferred) ->`, result]); + + if (compositionNests) { + throw new TypeError(`Inferred early-exit is disallowed in nested compositions`); + } + + debug(() => colors.bright(`end composition - exit (inferred)`)); + + return result; + } + + const {returnedWith} = continuationStorage; + + if (returnedWith === 'exit') { + const {providedValue} = continuationStorage; + + debug(() => [`step #${i+1} - result: exit (explicit) ->`, providedValue]); + debug(() => colors.bright(`end composition - exit (explicit)`)); + + if (compositionNests) { + return continuationIfApplicable.exit(providedValue); + } else { + return providedValue; + } + } + + const {providedValue, providedDependencies} = continuationStorage; + + const continuationArgs = []; + if (expectingTransform) { + continuationArgs.push( + (callingTransformForThisStep + ? providedValue ?? null + : valueSoFar ?? null)); + } + + debug(() => { + const base = `step #${i+1} - result: ` + returnedWith; + const parts = []; + + if (callingTransformForThisStep) { + parts.push('value:', providedValue); + } + + if (providedDependencies !== null) { + parts.push(`deps:`, providedDependencies); + } else { + parts.push(`(no deps)`); + } + + if (empty(parts)) { + return base; + } else { + return [base + ' ->', ...parts]; + } + }); + + switch (returnedWith) { + case 'raiseOutput': + debug(() => + (isBase + ? colors.bright(`end composition - raiseOutput (base: explicit)`) + : colors.bright(`end composition - raiseOutput`))); + continuationArgs.push(_mapDependenciesToOutputs(providedDependencies)); + return continuationIfApplicable(...continuationArgs); + + case 'raiseOutputAbove': + debug(() => colors.bright(`end composition - raiseOutputAbove`)); + continuationArgs.push(_mapDependenciesToOutputs(providedDependencies)); + return continuationIfApplicable.raiseOutput(...continuationArgs); + + case 'continuation': + if (isBase) { + debug(() => colors.bright(`end composition - raiseOutput (inferred)`)); + continuationArgs.push(_mapDependenciesToOutputs(providedDependencies)); + return continuationIfApplicable(...continuationArgs); + } else { + Object.assign(availableDependencies, providedDependencies); + if (callingTransformForThisStep && providedValue !== null) { + valueSoFar = providedValue; + } + break; + } + } + } + } + + const constructedDescriptor = {}; + + if (annotation) { + constructedDescriptor.annotation = annotation; + } + + constructedDescriptor.flags = { + update: compositionUpdates, + expose: compositionExposes, + compose: compositionNests, + }; + + if (compositionUpdates) { + // TODO: This is a dumb assign statement, and it could probably do more + // interesting things, like combining validation functions. + constructedDescriptor.update = + Object.assign( + {...description.update ?? {}}, + ...inputUpdateDescriptions, + ...stepUpdateDescriptions.flat()); + } + + if (compositionExposes) { + const expose = constructedDescriptor.expose = {}; + + expose.dependencies = + unique([ + ...dependenciesFromInputs, + ...dependenciesFromSteps, + ]); + + const _wrapper = (...args) => { + try { + return _computeOrTransform(...args); + } catch (thrownError) { + const error = new Error( + `Error computing composition` + + (annotation ? ` ${annotation}` : '')); + error.cause = thrownError; + throw error; + } + }; + + if (compositionNests) { + if (compositionUpdates) { + expose.transform = (value, continuation, dependencies) => + _wrapper(value, continuation, dependencies); + } + + if (anyStepsCompute && !anyStepsUseUpdateValue && !anyInputsUseUpdateValue) { + expose.compute = (continuation, dependencies) => + _wrapper(noTransformSymbol, continuation, dependencies); + } + + if (base.cacheComposition) { + expose.cache = base.cacheComposition; + } + } else if (compositionUpdates) { + if (!empty(steps)) { + expose.transform = (value, dependencies) => + _wrapper(value, null, dependencies); + } + } else { + expose.compute = (dependencies) => + _wrapper(noTransformSymbol, null, dependencies); + } + } + + return constructedDescriptor; +} + +export function displayCompositeCacheAnalysis() { + const showTimes = (cache, key) => { + const times = cache.times[key].slice().sort(); + + const all = times; + const worst10pc = times.slice(-times.length / 10); + const best10pc = times.slice(0, times.length / 10); + const middle50pc = times.slice(times.length / 4, -times.length / 4); + const middle80pc = times.slice(times.length / 10, -times.length / 10); + + const fmt = val => `${(val / 1000).toFixed(2)}ms`.padStart(9); + const avg = times => times.reduce((a, b) => a + b, 0) / times.length; + + const left = ` - ${key}: `; + const indn = ' '.repeat(left.length); + console.log(left + `${fmt(avg(all))} (all ${all.length})`); + console.log(indn + `${fmt(avg(worst10pc))} (worst 10%)`); + console.log(indn + `${fmt(avg(best10pc))} (best 10%)`); + console.log(indn + `${fmt(avg(middle80pc))} (middle 80%)`); + console.log(indn + `${fmt(avg(middle50pc))} (middle 50%)`); + }; + + for (const [annotation, cache] of Object.entries(globalCompositeCache)) { + console.log(`Cached ${annotation}:`); + showTimes(cache, 'evaluate'); + showTimes(cache, 'read'); + } +} + +// Evaluates a function with composite debugging enabled, turns debugging +// off again, and returns the result of the function. This is mostly syntax +// sugar, but also helps avoid unit tests avoid accidentally printing debug +// info for a bunch of unrelated composites (due to property enumeration +// when displaying an unexpected result). Use as so: +// +// Without debugging: +// t.same(thing.someProp, value) +// +// With debugging: +// t.same(debugComposite(() => thing.someProp), value) +// +export function debugComposite(fn) { + compositeFrom.debug = true; + const value = fn(); + compositeFrom.debug = false; + return value; +} diff --git a/src/data/composite/control-flow/exitWithoutDependency.js b/src/data/composite/control-flow/exitWithoutDependency.js new file mode 100644 index 0000000..c660a7e --- /dev/null +++ b/src/data/composite/control-flow/exitWithoutDependency.js @@ -0,0 +1,35 @@ +// Early exits if a dependency isn't available. +// See withResultOfAvailabilityCheck for {mode} options. + +import {input, templateCompositeFrom} from '#composite'; + +import inputAvailabilityCheckMode from './inputAvailabilityCheckMode.js'; +import withResultOfAvailabilityCheck from './withResultOfAvailabilityCheck.js'; + +export default templateCompositeFrom({ + annotation: `exitWithoutDependency`, + + inputs: { + dependency: input({acceptsNull: true}), + mode: inputAvailabilityCheckMode(), + value: input({defaultValue: null}), + }, + + steps: () => [ + withResultOfAvailabilityCheck({ + from: input('dependency'), + mode: input('mode'), + }), + + { + dependencies: ['#availability', input('value')], + compute: (continuation, { + ['#availability']: availability, + [input('value')]: value, + }) => + (availability + ? continuation() + : continuation.exit(value)), + }, + ], +}); diff --git a/src/data/composite/control-flow/exitWithoutUpdateValue.js b/src/data/composite/control-flow/exitWithoutUpdateValue.js new file mode 100644 index 0000000..244b323 --- /dev/null +++ b/src/data/composite/control-flow/exitWithoutUpdateValue.js @@ -0,0 +1,24 @@ +// Early exits if this property's update value isn't available. +// See withResultOfAvailabilityCheck for {mode} options. + +import {input, templateCompositeFrom} from '#composite'; + +import exitWithoutDependency from './exitWithoutDependency.js'; +import inputAvailabilityCheckMode from './inputAvailabilityCheckMode.js'; + +export default templateCompositeFrom({ + annotation: `exitWithoutUpdateValue`, + + inputs: { + mode: inputAvailabilityCheckMode(), + value: input({defaultValue: null}), + }, + + steps: () => [ + exitWithoutDependency({ + dependency: input.updateValue(), + mode: input('mode'), + value: input('value'), + }), + ], +}); diff --git a/src/data/composite/control-flow/exposeConstant.js b/src/data/composite/control-flow/exposeConstant.js new file mode 100644 index 0000000..e76699c --- /dev/null +++ b/src/data/composite/control-flow/exposeConstant.js @@ -0,0 +1,26 @@ +// Exposes a constant value exactly as it is; like exposeDependency, this +// is typically the base of a composition serving as a particular property +// descriptor. It generally follows steps which will conditionally early +// exit with some other value, with the exposeConstant base serving as the +// fallback default value. + +import {input, templateCompositeFrom} from '#composite'; + +export default templateCompositeFrom({ + annotation: `exposeConstant`, + + compose: false, + + inputs: { + value: input.staticValue({acceptsNull: true}), + }, + + steps: () => [ + { + dependencies: [input('value')], + compute: ({ + [input('value')]: value, + }) => value, + }, + ], +}); diff --git a/src/data/composite/control-flow/exposeDependency.js b/src/data/composite/control-flow/exposeDependency.js new file mode 100644 index 0000000..3aa3d03 --- /dev/null +++ b/src/data/composite/control-flow/exposeDependency.js @@ -0,0 +1,28 @@ +// Exposes a dependency exactly as it is; this is typically the base of a +// composition which was created to serve as one property's descriptor. +// +// Please note that this *doesn't* verify that the dependency exists, so +// if you provide the wrong name or it hasn't been set by a previous +// compositional step, the property will be exposed as undefined instead +// of null. + +import {input, templateCompositeFrom} from '#composite'; + +export default templateCompositeFrom({ + annotation: `exposeDependency`, + + compose: false, + + inputs: { + dependency: input.staticDependency({acceptsNull: true}), + }, + + steps: () => [ + { + dependencies: [input('dependency')], + compute: ({ + [input('dependency')]: dependency + }) => dependency, + }, + ], +}); diff --git a/src/data/composite/control-flow/exposeDependencyOrContinue.js b/src/data/composite/control-flow/exposeDependencyOrContinue.js new file mode 100644 index 0000000..0f7f223 --- /dev/null +++ b/src/data/composite/control-flow/exposeDependencyOrContinue.js @@ -0,0 +1,34 @@ +// Exposes a dependency as it is, or continues if it's unavailable. +// See withResultOfAvailabilityCheck for {mode} options. + +import {input, templateCompositeFrom} from '#composite'; + +import inputAvailabilityCheckMode from './inputAvailabilityCheckMode.js'; +import withResultOfAvailabilityCheck from './withResultOfAvailabilityCheck.js'; + +export default templateCompositeFrom({ + annotation: `exposeDependencyOrContinue`, + + inputs: { + dependency: input({acceptsNull: true}), + mode: inputAvailabilityCheckMode(), + }, + + steps: () => [ + withResultOfAvailabilityCheck({ + from: input('dependency'), + mode: input('mode'), + }), + + { + dependencies: ['#availability', input('dependency')], + compute: (continuation, { + ['#availability']: availability, + [input('dependency')]: dependency, + }) => + (availability + ? continuation.exit(dependency) + : continuation()), + }, + ], +}); diff --git a/src/data/composite/control-flow/exposeUpdateValueOrContinue.js b/src/data/composite/control-flow/exposeUpdateValueOrContinue.js new file mode 100644 index 0000000..1f94b33 --- /dev/null +++ b/src/data/composite/control-flow/exposeUpdateValueOrContinue.js @@ -0,0 +1,40 @@ +// Exposes the update value of an {update: true} property as it is, +// or continues if it's unavailable. +// +// See withResultOfAvailabilityCheck for {mode} options. +// +// Provide {validate} here to conveniently set a custom validation check +// for this property's update value. +// + +import {input, templateCompositeFrom} from '#composite'; + +import exposeDependencyOrContinue from './exposeDependencyOrContinue.js'; +import inputAvailabilityCheckMode from './inputAvailabilityCheckMode.js'; + +export default templateCompositeFrom({ + annotation: `exposeUpdateValueOrContinue`, + + inputs: { + mode: inputAvailabilityCheckMode(), + + validate: input({ + type: 'function', + defaultValue: null, + }), + }, + + update: ({ + [input.staticValue('validate')]: validate, + }) => + (validate + ? {validate} + : {}), + + steps: () => [ + exposeDependencyOrContinue({ + dependency: input.updateValue(), + mode: input('mode'), + }), + ], +}); diff --git a/src/data/composite/control-flow/index.js b/src/data/composite/control-flow/index.js new file mode 100644 index 0000000..7fad88b --- /dev/null +++ b/src/data/composite/control-flow/index.js @@ -0,0 +1,14 @@ +// #composite/control-flow +// +// No entries depend on any other entries, except siblings in this directory. +// + +export {default as exitWithoutDependency} from './exitWithoutDependency.js'; +export {default as exitWithoutUpdateValue} from './exitWithoutUpdateValue.js'; +export {default as exposeConstant} from './exposeConstant.js'; +export {default as exposeDependency} from './exposeDependency.js'; +export {default as exposeDependencyOrContinue} from './exposeDependencyOrContinue.js'; +export {default as exposeUpdateValueOrContinue} from './exposeUpdateValueOrContinue.js'; +export {default as raiseOutputWithoutDependency} from './raiseOutputWithoutDependency.js'; +export {default as raiseOutputWithoutUpdateValue} from './raiseOutputWithoutUpdateValue.js'; +export {default as withResultOfAvailabilityCheck} from './withResultOfAvailabilityCheck.js'; diff --git a/src/data/composite/control-flow/inputAvailabilityCheckMode.js b/src/data/composite/control-flow/inputAvailabilityCheckMode.js new file mode 100644 index 0000000..8008fde --- /dev/null +++ b/src/data/composite/control-flow/inputAvailabilityCheckMode.js @@ -0,0 +1,9 @@ +import {input} from '#composite'; +import {is} from '#validators'; + +export default function inputAvailabilityCheckMode() { + return input({ + validate: is('null', 'empty', 'falsy', 'index'), + defaultValue: 'null', + }); +} diff --git a/src/data/composite/control-flow/raiseOutputWithoutDependency.js b/src/data/composite/control-flow/raiseOutputWithoutDependency.js new file mode 100644 index 0000000..3d04f8a --- /dev/null +++ b/src/data/composite/control-flow/raiseOutputWithoutDependency.js @@ -0,0 +1,39 @@ +// Raises if a dependency isn't available. +// See withResultOfAvailabilityCheck for {mode} options. + +import {input, templateCompositeFrom} from '#composite'; + +import inputAvailabilityCheckMode from './inputAvailabilityCheckMode.js'; +import withResultOfAvailabilityCheck from './withResultOfAvailabilityCheck.js'; + +export default templateCompositeFrom({ + annotation: `raiseOutputWithoutDependency`, + + inputs: { + dependency: input({acceptsNull: true}), + mode: inputAvailabilityCheckMode(), + output: input.staticValue({defaultValue: {}}), + }, + + outputs: ({ + [input.staticValue('output')]: output, + }) => Object.keys(output ?? {}), + + steps: () => [ + withResultOfAvailabilityCheck({ + from: input('dependency'), + mode: input('mode'), + }), + + { + dependencies: ['#availability', input('output')], + compute: (continuation, { + ['#availability']: availability, + [input('output')]: output, + }) => + (availability + ? continuation() + : continuation.raiseOutputAbove(output)), + }, + ], +}); diff --git a/src/data/composite/control-flow/raiseOutputWithoutUpdateValue.js b/src/data/composite/control-flow/raiseOutputWithoutUpdateValue.js new file mode 100644 index 0000000..ffa83a9 --- /dev/null +++ b/src/data/composite/control-flow/raiseOutputWithoutUpdateValue.js @@ -0,0 +1,47 @@ +// Raises if this property's update value isn't available. +// See withResultOfAvailabilityCheck for {mode} options! + +import {input, templateCompositeFrom} from '#composite'; + +import inputAvailabilityCheckMode from './inputAvailabilityCheckMode.js'; +import withResultOfAvailabilityCheck from './withResultOfAvailabilityCheck.js'; + +export default templateCompositeFrom({ + annotation: `raiseOutputWithoutUpdateValue`, + + inputs: { + mode: inputAvailabilityCheckMode(), + output: input.staticValue({defaultValue: {}}), + }, + + outputs: ({ + [input.staticValue('output')]: output, + }) => Object.keys(output ?? {}), + + steps: () => [ + withResultOfAvailabilityCheck({ + from: input.updateValue(), + mode: input('mode'), + }), + + // TODO: A bit of a kludge, below. Other "do something with the update + // value" type functions can get by pretty much just passing that value + // as an input (input.updateValue()) into the corresponding "do something + // with a dependency/arbitrary value" function. But we can't do that here, + // because the special behavior, raiseOutputAbove(), only works to raise + // output above the composition it's *directly* nested in. Other languages + // have a throw/catch system that might serve as inspiration for something + // better here. + + { + dependencies: ['#availability', input('output')], + compute: (continuation, { + ['#availability']: availability, + [input('output')]: output, + }) => + (availability + ? continuation() + : continuation.raiseOutputAbove(output)), + }, + ], +}); diff --git a/src/data/composite/control-flow/withResultOfAvailabilityCheck.js b/src/data/composite/control-flow/withResultOfAvailabilityCheck.js new file mode 100644 index 0000000..a694201 --- /dev/null +++ b/src/data/composite/control-flow/withResultOfAvailabilityCheck.js @@ -0,0 +1,71 @@ +// Checks the availability of a dependency and provides the result to later +// steps under '#availability' (by default). This is mainly intended for use +// by the more specific utilities, which you should consider using instead. +// +// Customize {mode} to select one of these modes, or default to 'null': +// +// * 'null': Check that the value isn't null (and not undefined either). +// * 'empty': Check that the value is neither null, undefined, nor an empty +// array. +// * 'falsy': Check that the value isn't false when treated as a boolean +// (nor an empty array). Keep in mind this will also be false +// for values like zero and the empty string! +// * 'index': Check that the value is a number, and is at least zero. +// +// See also: +// - exitWithoutDependency +// - exitWithoutUpdateValue +// - exposeDependencyOrContinue +// - exposeUpdateValueOrContinue +// - raiseOutputWithoutDependency +// - raiseOutputWithoutUpdateValue +// + +import {input, templateCompositeFrom} from '#composite'; +import {empty} from '#sugar'; + +import inputAvailabilityCheckMode from './inputAvailabilityCheckMode.js'; + +export default templateCompositeFrom({ + annotation: `withResultOfAvailabilityCheck`, + + inputs: { + from: input({acceptsNull: true}), + mode: inputAvailabilityCheckMode(), + }, + + outputs: ['#availability'], + + steps: () => [ + { + dependencies: [input('from'), input('mode')], + + compute: (continuation, { + [input('from')]: value, + [input('mode')]: mode, + }) => { + let availability; + + switch (mode) { + case 'null': + availability = value !== undefined && value !== null; + break; + + case 'empty': + availability = value !== undefined && !empty(value); + break; + + case 'falsy': + availability = !!value && (!Array.isArray(value) || !empty(value)); + break; + + case 'index': + availability = typeof value === 'number' && value >= 0; + break; + } + + return continuation({'#availability': availability}); + }, + }, + ], +}); diff --git a/src/data/composite/data/excludeFromList.js b/src/data/composite/data/excludeFromList.js new file mode 100644 index 0000000..d798dcd --- /dev/null +++ b/src/data/composite/data/excludeFromList.js @@ -0,0 +1,55 @@ +// Filters particular values out of a list. Note that this will always +// completely skip over null, but can be used to filter out any other +// primitive or object value. +// +// See also: +// - fillMissingListItems +// +// More list utilities: +// - withFilteredList, withMappedList, withSortedList +// - withFlattenedList, withUnflattenedList +// - withPropertyFromList, withPropertiesFromList +// + +import {input, templateCompositeFrom} from '#composite'; +import {empty} from '#sugar'; + +export default templateCompositeFrom({ + annotation: `excludeFromList`, + + inputs: { + list: input(), + + item: input({defaultValue: null}), + items: input({type: 'array', defaultValue: null}), + }, + + outputs: ({ + [input.staticDependency('list')]: list, + }) => [list ?? '#list'], + + steps: () => [ + { + dependencies: [ + input.staticDependency('list'), + input('list'), + input('item'), + input('items'), + ], + + compute: (continuation, { + [input.staticDependency('list')]: listName, + [input('list')]: listContents, + [input('item')]: excludeItem, + [input('items')]: excludeItems, + }) => continuation({ + [listName ?? '#list']: + listContents.filter(item => { + if (excludeItem !== null && item === excludeItem) return false; + if (!empty(excludeItems) && excludeItems.includes(item)) return false; + return true; + }), + }), + }, + ], +}); diff --git a/src/data/composite/data/fillMissingListItems.js b/src/data/composite/data/fillMissingListItems.js new file mode 100644 index 0000000..4f818a7 --- /dev/null +++ b/src/data/composite/data/fillMissingListItems.js @@ -0,0 +1,50 @@ +// Replaces items of a list, which are null or undefined, with some fallback +// value. By default, this replaces the passed dependency. +// +// See also: +// - excludeFromList +// +// More list utilities: +// - withFilteredList, withMappedList, withSortedList +// - withFlattenedList, withUnflattenedList +// - withPropertyFromList, withPropertiesFromList +// + +import {input, templateCompositeFrom} from '#composite'; + +export default templateCompositeFrom({ + annotation: `fillMissingListItems`, + + inputs: { + list: input({type: 'array'}), + fill: input({acceptsNull: true}), + }, + + outputs: ({ + [input.staticDependency('list')]: list, + }) => [list ?? '#list'], + + steps: () => [ + { + dependencies: [input('list'), input('fill')], + compute: (continuation, { + [input('list')]: list, + [input('fill')]: fill, + }) => continuation({ + ['#filled']: + list.map(item => item ?? fill), + }), + }, + + { + dependencies: [input.staticDependency('list'), '#filled'], + compute: (continuation, { + [input.staticDependency('list')]: list, + ['#filled']: filled, + }) => continuation({ + [list ?? '#list']: + filled, + }), + }, + ], +}); diff --git a/src/data/composite/data/index.js b/src/data/composite/data/index.js new file mode 100644 index 0000000..256c049 --- /dev/null +++ b/src/data/composite/data/index.js @@ -0,0 +1,17 @@ +// #composite/data +// +// Entries here may depend on entries in #composite/control-flow. +// + +export {default as excludeFromList} from './excludeFromList.js'; +export {default as fillMissingListItems} from './fillMissingListItems.js'; +export {default as withFilteredList} from './withFilteredList.js'; +export {default as withFlattenedList} from './withFlattenedList.js'; +export {default as withMappedList} from './withMappedList.js'; +export {default as withPropertiesFromList} from './withPropertiesFromList.js'; +export {default as withPropertiesFromObject} from './withPropertiesFromObject.js'; +export {default as withPropertyFromList} from './withPropertyFromList.js'; +export {default as withPropertyFromObject} from './withPropertyFromObject.js'; +export {default as withSortedList} from './withSortedList.js'; +export {default as withUnflattenedList} from './withUnflattenedList.js'; +export {default as withUniqueItemsOnly} from './withUniqueItemsOnly.js'; diff --git a/src/data/composite/data/withFilteredList.js b/src/data/composite/data/withFilteredList.js new file mode 100644 index 0000000..82e5690 --- /dev/null +++ b/src/data/composite/data/withFilteredList.js @@ -0,0 +1,50 @@ +// Applies a filter - an array of truthy and falsy values - to the index- +// corresponding items in a list. Items which correspond to a truthy value +// are kept, and the rest are excluded from the output list. +// +// TODO: It would be neat to apply an availability check here, e.g. to allow +// not providing a filter at all and performing the check on the contents of +// the list (though on the filter, if present, is fine too). But that's best +// done by some shmancy-fancy mapping support in composite.js, so a bit out +// of reach for now (apart from proving uses built on top of a more boring +// implementation). +// +// TODO: There should be two outputs - one for the items included according to +// the filter, and one for the items excluded. +// +// See also: +// - withMappedList +// - withSortedList +// +// More list utilities: +// - excludeFromList +// - fillMissingListItems +// - withFlattenedList, withUnflattenedList +// - withPropertyFromList, withPropertiesFromList +// + +import {input, templateCompositeFrom} from '#composite'; + +export default templateCompositeFrom({ + annotation: `withFilteredList`, + + inputs: { + list: input({type: 'array'}), + filter: input({type: 'array'}), + }, + + outputs: ['#filteredList'], + + steps: () => [ + { + dependencies: [input('list'), input('filter')], + compute: (continuation, { + [input('list')]: list, + [input('filter')]: filter, + }) => continuation({ + '#filteredList': + list.filter((item, index) => filter[index]), + }), + }, + ], +}); diff --git a/src/data/composite/data/withFlattenedList.js b/src/data/composite/data/withFlattenedList.js new file mode 100644 index 0000000..edfa340 --- /dev/null +++ b/src/data/composite/data/withFlattenedList.js @@ -0,0 +1,47 @@ +// Flattens an array with one level of nested arrays, providing as dependencies +// both the flattened array as well as the original starting indices of each +// successive source array. +// +// See also: +// - withUnflattenedList +// +// More list utilities: +// - excludeFromList +// - fillMissingListItems +// - withFilteredList, withMappedList, withSortedList +// - withPropertyFromList, withPropertiesFromList +// + +import {input, templateCompositeFrom} from '#composite'; + +export default templateCompositeFrom({ + annotation: `withFlattenedList`, + + inputs: { + list: input({type: 'array'}), + }, + + outputs: ['#flattenedList', '#flattenedIndices'], + + steps: () => [ + { + dependencies: [input('list')], + compute(continuation, { + [input('list')]: sourceList, + }) { + const flattenedList = sourceList.flat(); + const indices = []; + let lastEndIndex = 0; + for (const {length} of sourceList) { + indices.push(lastEndIndex); + lastEndIndex += length; + } + + return continuation({ + ['#flattenedList']: flattenedList, + ['#flattenedIndices']: indices, + }); + }, + }, + ], +}); diff --git a/src/data/composite/data/withMappedList.js b/src/data/composite/data/withMappedList.js new file mode 100644 index 0000000..e0a700b --- /dev/null +++ b/src/data/composite/data/withMappedList.js @@ -0,0 +1,39 @@ +// Applies a map function to each item in a list, just like a normal JavaScript +// map. +// +// See also: +// - withFilteredList +// - withSortedList +// +// More list utilities: +// - excludeFromList +// - fillMissingListItems +// - withFlattenedList, withUnflattenedList +// - withPropertyFromList, withPropertiesFromList +// + +import {input, templateCompositeFrom} from '#composite'; + +export default templateCompositeFrom({ + annotation: `withMappedList`, + + inputs: { + list: input({type: 'array'}), + map: input({type: 'function'}), + }, + + outputs: ['#mappedList'], + + steps: () => [ + { + dependencies: [input('list'), input('map')], + compute: (continuation, { + [input('list')]: list, + [input('map')]: mapFn, + }) => continuation({ + ['#mappedList']: + list.map(mapFn), + }), + }, + ], +}); diff --git a/src/data/composite/data/withPropertiesFromList.js b/src/data/composite/data/withPropertiesFromList.js new file mode 100644 index 0000000..08907ba --- /dev/null +++ b/src/data/composite/data/withPropertiesFromList.js @@ -0,0 +1,92 @@ +// Gets the listed properties from each of a list of objects, providing lists +// of property values each into a dependency prefixed with the same name as the +// list (by default). +// +// Like withPropertyFromList, this doesn't alter indices. +// +// See also: +// - withPropertiesFromObject +// - withPropertyFromList +// +// More list utilities: +// - excludeFromList +// - fillMissingListItems +// - withFilteredList, withMappedList, withSortedList +// - withFlattenedList, withUnflattenedList +// + +import {input, templateCompositeFrom} from '#composite'; +import {isString, validateArrayItems} from '#validators'; + +export default templateCompositeFrom({ + annotation: `withPropertiesFromList`, + + inputs: { + list: input({type: 'array'}), + + properties: input({ + validate: validateArrayItems(isString), + }), + + prefix: input.staticValue({type: 'string', defaultValue: null}), + }, + + outputs: ({ + [input.staticDependency('list')]: list, + [input.staticValue('properties')]: properties, + [input.staticValue('prefix')]: prefix, + }) => + (properties + ? properties.map(property => + (prefix + ? `${prefix}.${property}` + : list + ? `${list}.${property}` + : `#list.${property}`)) + : ['#lists']), + + steps: () => [ + { + dependencies: [input('list'), input('properties')], + compute: (continuation, { + [input('list')]: list, + [input('properties')]: properties, + }) => continuation({ + ['#lists']: + Object.fromEntries( + properties.map(property => [ + property, + list.map(item => item[property] ?? null), + ])), + }), + }, + + { + dependencies: [ + input.staticDependency('list'), + input.staticValue('properties'), + input.staticValue('prefix'), + '#lists', + ], + + compute: (continuation, { + [input.staticDependency('list')]: list, + [input.staticValue('properties')]: properties, + [input.staticValue('prefix')]: prefix, + ['#lists']: lists, + }) => + (properties + ? continuation( + Object.fromEntries( + properties.map(property => [ + (prefix + ? `${prefix}.${property}` + : list + ? `${list}.${property}` + : `#list.${property}`), + lists[property], + ]))) + : continuation({'#lists': lists})), + }, + ], +}); diff --git a/src/data/composite/data/withPropertiesFromObject.js b/src/data/composite/data/withPropertiesFromObject.js new file mode 100644 index 0000000..21726b5 --- /dev/null +++ b/src/data/composite/data/withPropertiesFromObject.js @@ -0,0 +1,87 @@ +// Gets the listed properties from some object, providing each property's value +// as a dependency prefixed with the same name as the object (by default). +// If the object itself is null, all provided dependencies will be null; +// if it's missing only select properties, those will be provided as null. +// +// See also: +// - withPropertiesFromList +// - withPropertyFromObject +// + +import {input, templateCompositeFrom} from '#composite'; +import {isString, validateArrayItems} from '#validators'; + +export default templateCompositeFrom({ + annotation: `withPropertiesFromObject`, + + inputs: { + object: input({type: 'object', acceptsNull: true}), + + properties: input({ + type: 'array', + validate: validateArrayItems(isString), + }), + + prefix: input.staticValue({type: 'string', defaultValue: null}), + }, + + outputs: ({ + [input.staticDependency('object')]: object, + [input.staticValue('properties')]: properties, + [input.staticValue('prefix')]: prefix, + }) => + (properties + ? properties.map(property => + (prefix + ? `${prefix}.${property}` + : object + ? `${object}.${property}` + : `#object.${property}`)) + : ['#object']), + + steps: () => [ + { + dependencies: [input('object'), input('properties')], + compute: (continuation, { + [input('object')]: object, + [input('properties')]: properties, + }) => continuation({ + ['#entries']: + (object === null + ? properties.map(property => [property, null]) + : properties.map(property => [property, object[property]])), + }), + }, + + { + dependencies: [ + input.staticDependency('object'), + input.staticValue('properties'), + input.staticValue('prefix'), + '#entries', + ], + + compute: (continuation, { + [input.staticDependency('object')]: object, + [input.staticValue('properties')]: properties, + [input.staticValue('prefix')]: prefix, + ['#entries']: entries, + }) => + (properties + ? continuation( + Object.fromEntries( + entries.map(([property, value]) => [ + (prefix + ? `${prefix}.${property}` + : object + ? `${object}.${property}` + : `#object.${property}`), + value ?? null, + ]))) + : continuation({ + ['#object']: + Object.fromEntries(entries), + })), + }, + ], +}); diff --git a/src/data/composite/data/withPropertyFromList.js b/src/data/composite/data/withPropertyFromList.js new file mode 100644 index 0000000..a2c66d7 --- /dev/null +++ b/src/data/composite/data/withPropertyFromList.js @@ -0,0 +1,82 @@ +// Gets a property from each of a list of objects (in a dependency) and +// provides the results. +// +// This doesn't alter any list indices, so positions which were null in the +// original list are kept null here. Objects which don't have the specified +// property are retained in-place as null. +// +// See also: +// - withPropertiesFromList +// - withPropertyFromObject +// +// More list utilities: +// - excludeFromList +// - fillMissingListItems +// - withFilteredList, withMappedList, withSortedList +// - withFlattenedList, withUnflattenedList +// + +import {input, templateCompositeFrom} from '#composite'; + +function getOutputName({list, property, prefix}) { + if (!property) return `#values`; + if (prefix) return `${prefix}.${property}`; + if (list) return `${list}.${property}`; + return `#list.${property}`; +} + +export default templateCompositeFrom({ + annotation: `withPropertyFromList`, + + inputs: { + list: input({type: 'array'}), + property: input({type: 'string'}), + prefix: input.staticValue({type: 'string', defaultValue: null}), + }, + + outputs: ({ + [input.staticDependency('list')]: list, + [input.staticValue('property')]: property, + [input.staticValue('prefix')]: prefix, + }) => + [getOutputName({list, property, prefix})], + + steps: () => [ + { + dependencies: [input('list'), input('property')], + compute: (continuation, { + [input('list')]: list, + [input('property')]: property, + }) => continuation({ + ['#values']: + list.map(item => item[property] ?? null), + }), + }, + + { + dependencies: [ + input.staticDependency('list'), + input.staticValue('property'), + input.staticValue('prefix'), + ], + + compute: (continuation, { + [input.staticDependency('list')]: list, + [input.staticValue('property')]: property, + [input.staticValue('prefix')]: prefix, + }) => continuation({ + ['#outputName']: + getOutputName({list, property, prefix}), + }), + }, + + { + dependencies: ['#values', '#outputName'], + compute: (continuation, { + ['#values']: values, + ['#outputName']: outputName, + }) => + continuation.raiseOutput({[outputName]: values}), + }, + ], +}); diff --git a/src/data/composite/data/withPropertyFromObject.js b/src/data/composite/data/withPropertyFromObject.js new file mode 100644 index 0000000..b31bab1 --- /dev/null +++ b/src/data/composite/data/withPropertyFromObject.js @@ -0,0 +1,69 @@ +// Gets a property of some object (in a dependency) and provides that value. +// If the object itself is null, or the object doesn't have the listed property, +// the provided dependency will also be null. +// +// See also: +// - withPropertiesFromObject +// - withPropertyFromList +// + +import {input, templateCompositeFrom} from '#composite'; + +export default templateCompositeFrom({ + annotation: `withPropertyFromObject`, + + inputs: { + object: input({type: 'object', acceptsNull: true}), + property: input({type: 'string'}), + }, + + outputs: ({ + [input.staticDependency('object')]: object, + [input.staticValue('property')]: property, + }) => + (object && property + ? (object.startsWith('#') + ? [`${object}.${property}`] + : [`#${object}.${property}`]) + : ['#value']), + + steps: () => [ + { + dependencies: [ + input.staticDependency('object'), + input.staticValue('property'), + ], + + compute: (continuation, { + [input.staticDependency('object')]: object, + [input.staticValue('property')]: property, + }) => continuation({ + '#output': + (object && property + ? (object.startsWith('#') + ? `${object}.${property}` + : `#${object}.${property}`) + : '#value'), + }), + }, + + { + dependencies: [ + '#output', + input('object'), + input('property'), + ], + + compute: (continuation, { + ['#output']: output, + [input('object')]: object, + [input('property')]: property, + }) => continuation({ + [output]: + (object === null + ? null + : object[property] ?? null), + }), + }, + ], +}); diff --git a/src/data/composite/data/withSortedList.js b/src/data/composite/data/withSortedList.js new file mode 100644 index 0000000..dd81078 --- /dev/null +++ b/src/data/composite/data/withSortedList.js @@ -0,0 +1,121 @@ +// Applies a sort function across pairs of items in a list, just like a normal +// JavaScript sort. Alongside the sorted results, so are outputted the indices +// which each item in the unsorted list corresponds to in the sorted one, +// allowing for the results of this sort to be composed in some more involved +// operation. For example, using an alphabetical sort, the list ['banana', +// 'apple', 'pterodactyl'] will output the expected alphabetical items, as well +// as the indices list [1, 0, 2]. +// +// If two items are equal (in the eyes of the sort operation), their placement +// in the sorted list is arbitrary, though every input index will be present in +// '#sortIndices' exactly once (and equal items will be bunched together). +// +// The '#sortIndices' output refers to the "true" index which each source item +// occupies in the sorted list. This sacrifices information about equal items, +// which can be obtained through '#unstableSortIndices' instead: each mapped +// index may appear more than once, and rather than represent exact positions +// in the sorted list, they represent relational values: if items A and B are +// mapped to indices 3 and 5, then A certainly is positioned before B (and vice +// versa); but there may be more than one item in-between. If items C and D are +// both mapped to index 4, then their position relative to each other is +// arbitrary - they are equal - but they both certainly appear after item A and +// before item B. +// +// This implementation is based on the one used for sortMultipleArrays. +// +// See also: +// - withFilteredList +// - withMappedList +// +// More list utilities: +// - excludeFromList +// - fillMissingListItems +// - withFlattenedList, withUnflattenedList +// - withPropertyFromList, withPropertiesFromList +// + +import {input, templateCompositeFrom} from '#composite'; + +export default templateCompositeFrom({ + annotation: `withSortedList`, + + inputs: { + list: input({type: 'array'}), + sort: input({type: 'function'}), + }, + + outputs: ['#sortedList', '#sortIndices', '#unstableSortIndices'], + + steps: () => [ + { + dependencies: [input('list'), input('sort')], + compute(continuation, { + [input('list')]: list, + [input('sort')]: sortFn, + }) { + const symbols = []; + const symbolToIndex = new Map(); + + for (const index of list.keys()) { + const symbol = Symbol(); + symbols.push(symbol); + symbolToIndex.set(symbol, index); + } + + const equalSymbols = new Map(); + + const assertEqual = (symbol1, symbol2) => { + if (equalSymbols.has(symbol1)) { + equalSymbols.get(symbol1).add(symbol2); + } else { + equalSymbols.set(symbol1, new Set([symbol2])); + } + }; + + const isEqual = (symbol1, symbol2) => + !!equalSymbols.get(symbol1)?.has(symbol2); + + symbols.sort((symbol1, symbol2) => { + const comparison = + sortFn( + list[symbolToIndex.get(symbol1)], + list[symbolToIndex.get(symbol2)]); + + if (comparison === 0) { + assertEqual(symbol1, symbol2); + assertEqual(symbol2, symbol1); + } + + return comparison; + }); + + const stableSortIndices = []; + const unstableSortIndices = []; + const sortedList = []; + + let unstableIndex = 0; + + for (const [stableIndex, symbol] of symbols.entries()) { + const sourceIndex = symbolToIndex.get(symbol); + sortedList.push(list[sourceIndex]); + + if (stableIndex > 0) { + const previous = symbols[stableIndex - 1]; + if (!isEqual(symbol, previous)) { + unstableIndex++; + } + } + + stableSortIndices[sourceIndex] = stableIndex; + unstableSortIndices[sourceIndex] = unstableIndex; + } + + return continuation({ + ['#sortedList']: sortedList, + ['#sortIndices']: stableSortIndices, + ['#unstableSortIndices']: unstableSortIndices, + }); + }, + }, + ], +}); diff --git a/src/data/composite/data/withUnflattenedList.js b/src/data/composite/data/withUnflattenedList.js new file mode 100644 index 0000000..39a666d --- /dev/null +++ b/src/data/composite/data/withUnflattenedList.js @@ -0,0 +1,72 @@ +// After mapping the contents of a flattened array in-place (being careful to +// retain the original indices by replacing unmatched results with null instead +// of filtering them out), this function allows for recombining them. It will +// filter out null and undefined items by default (pass {filter: false} to +// disable this). +// +// See also: +// - withFlattenedList +// +// More list utilities: +// - excludeFromList +// - fillMissingListItems +// - withFilteredList, withMappedList, withSortedList +// - withPropertyFromList, withPropertiesFromList +// + +import {input, templateCompositeFrom} from '#composite'; +import {isWholeNumber, validateArrayItems} from '#validators'; + +export default templateCompositeFrom({ + annotation: `withUnflattenedList`, + + inputs: { + list: input({ + type: 'array', + defaultDependency: '#flattenedList', + }), + + indices: input({ + validate: validateArrayItems(isWholeNumber), + defaultDependency: '#flattenedIndices', + }), + + filter: input({ + type: 'boolean', + defaultValue: true, + }), + }, + + outputs: ['#unflattenedList'], + + steps: () => [ + { + dependencies: [input('list'), input('indices'), input('filter')], + compute(continuation, { + [input('list')]: list, + [input('indices')]: indices, + [input('filter')]: filter, + }) { + const unflattenedList = []; + + for (let i = 0; i < indices.length; i++) { + const startIndex = indices[i]; + const endIndex = + (i === indices.length - 1 + ? list.length + : indices[i + 1]); + + const values = list.slice(startIndex, endIndex); + unflattenedList.push( + (filter + ? values.filter(value => value !== null && value !== undefined) + : values)); + } + + return continuation({ + ['#unflattenedList']: unflattenedList, + }); + }, + }, + ], +}); diff --git a/src/data/composite/data/withUniqueItemsOnly.js b/src/data/composite/data/withUniqueItemsOnly.js new file mode 100644 index 0000000..7ee08b0 --- /dev/null +++ b/src/data/composite/data/withUniqueItemsOnly.js @@ -0,0 +1,40 @@ +// Excludes duplicate items from a list and provides the results, overwriting +// the list in-place, if possible. + +import {input, templateCompositeFrom} from '#composite'; +import {unique} from '#sugar'; + +export default templateCompositeFrom({ + annotation: `withUniqueItemsOnly`, + + inputs: { + list: input({type: 'array'}), + }, + + outputs: ({ + [input.staticDependency('list')]: list, + }) => [list ?? '#uniqueItems'], + + steps: () => [ + { + dependencies: [input('list')], + compute: (continuation, { + [input('list')]: list, + }) => continuation({ + ['#values']: + unique(list), + }), + }, + + { + dependencies: ['#values', input.staticDependency('list')], + compute: (continuation, { + '#values': values, + [input.staticDependency('list')]: list, + }) => continuation({ + [list ?? '#uniqueItems']: + values, + }), + }, + ], +}); diff --git a/src/data/composite/things/album/index.js b/src/data/composite/things/album/index.js new file mode 100644 index 0000000..8139f10 --- /dev/null +++ b/src/data/composite/things/album/index.js @@ -0,0 +1,2 @@ +export {default as withTracks} from './withTracks.js'; +export {default as withTrackSections} from './withTrackSections.js'; diff --git a/src/data/composite/things/album/withTrackSections.js b/src/data/composite/things/album/withTrackSections.js new file mode 100644 index 0000000..0a1ebeb --- /dev/null +++ b/src/data/composite/things/album/withTrackSections.js @@ -0,0 +1,127 @@ +import {input, templateCompositeFrom} from '#composite'; +import find from '#find'; +import {empty, filterMultipleArrays, stitchArrays} from '#sugar'; +import {isTrackSectionList} from '#validators'; + +import {exitWithoutDependency, exitWithoutUpdateValue} + from '#composite/control-flow'; +import {withResolvedReferenceList} from '#composite/wiki-data'; + +import { + fillMissingListItems, + withFlattenedList, + withPropertiesFromList, + withUnflattenedList, +} from '#composite/data'; + +export default templateCompositeFrom({ + annotation: `withTrackSections`, + + outputs: ['#trackSections'], + + steps: () => [ + exitWithoutDependency({ + dependency: 'ownTrackData', + value: input.value([]), + }), + + exitWithoutUpdateValue({ + mode: input.value('empty'), + value: input.value([]), + }), + + // TODO: input.updateValue description down here is a kludge. + withPropertiesFromList({ + list: input.updateValue({ + validate: isTrackSectionList, + }), + prefix: input.value('#sections'), + properties: input.value([ + 'tracks', + 'dateOriginallyReleased', + 'isDefaultTrackSection', + 'name', + 'color', + ]), + }), + + fillMissingListItems({ + list: '#sections.tracks', + fill: input.value([]), + }), + + fillMissingListItems({ + list: '#sections.isDefaultTrackSection', + fill: input.value(false), + }), + + fillMissingListItems({ + list: '#sections.name', + fill: input.value('Unnamed Track Section'), + }), + + fillMissingListItems({ + list: '#sections.color', + fill: input.dependency('color'), + }), + + withFlattenedList({ + list: '#sections.tracks', + }).outputs({ + ['#flattenedList']: '#trackRefs', + ['#flattenedIndices']: '#sections.startIndex', + }), + + withResolvedReferenceList({ + list: '#trackRefs', + data: 'ownTrackData', + notFoundMode: input.value('null'), + find: input.value(find.track), + }).outputs({ + ['#resolvedReferenceList']: '#tracks', + }), + + withUnflattenedList({ + list: '#tracks', + indices: '#sections.startIndex', + }).outputs({ + ['#unflattenedList']: '#sections.tracks', + }), + + { + dependencies: [ + '#sections.tracks', + '#sections.name', + '#sections.color', + '#sections.dateOriginallyReleased', + '#sections.isDefaultTrackSection', + '#sections.startIndex', + ], + + compute: (continuation, { + '#sections.tracks': tracks, + '#sections.name': name, + '#sections.color': color, + '#sections.dateOriginallyReleased': dateOriginallyReleased, + '#sections.isDefaultTrackSection': isDefaultTrackSection, + '#sections.startIndex': startIndex, + }) => { + filterMultipleArrays( + tracks, name, color, dateOriginallyReleased, isDefaultTrackSection, startIndex, + tracks => !empty(tracks)); + + return continuation({ + ['#trackSections']: + stitchArrays({ + tracks, + name, + color, + dateOriginallyReleased, + isDefaultTrackSection, + startIndex, + }), + }); + }, + }, + ], +}); diff --git a/src/data/composite/things/album/withTracks.js b/src/data/composite/things/album/withTracks.js new file mode 100644 index 0000000..fff3d5a --- /dev/null +++ b/src/data/composite/things/album/withTracks.js @@ -0,0 +1,51 @@ +import {input, templateCompositeFrom} from '#composite'; +import find from '#find'; + +import {exitWithoutDependency, raiseOutputWithoutDependency} + from '#composite/control-flow'; +import {withResolvedReferenceList} from '#composite/wiki-data'; + +export default templateCompositeFrom({ + annotation: `withTracks`, + + outputs: ['#tracks'], + + steps: () => [ + exitWithoutDependency({ + dependency: 'ownTrackData', + value: input.value([]), + }), + + raiseOutputWithoutDependency({ + dependency: 'trackSections', + mode: input.value('empty'), + output: input.value({ + ['#tracks']: [], + }), + }), + + { + dependencies: ['trackSections'], + compute: (continuation, {trackSections}) => + continuation({ + '#trackRefs': trackSections + .flatMap(section => section.tracks ?? []), + }), + }, + + withResolvedReferenceList({ + list: '#trackRefs', + data: 'ownTrackData', + find: input.value(find.track), + }), + + { + dependencies: ['#resolvedReferenceList'], + compute: (continuation, { + ['#resolvedReferenceList']: resolvedReferenceList, + }) => continuation({ + ['#tracks']: resolvedReferenceList, + }) + }, + ], +}); diff --git a/src/data/composite/things/flash-act/index.js b/src/data/composite/things/flash-act/index.js new file mode 100644 index 0000000..40fecd2 --- /dev/null +++ b/src/data/composite/things/flash-act/index.js @@ -0,0 +1 @@ +export {default as withFlashSide} from './withFlashSide.js'; diff --git a/src/data/composite/things/flash-act/withFlashSide.js b/src/data/composite/things/flash-act/withFlashSide.js new file mode 100644 index 0000000..64daa1f --- /dev/null +++ b/src/data/composite/things/flash-act/withFlashSide.js @@ -0,0 +1,22 @@ +// Gets the flash act's side. This will early exit if flashSideData is missing. +// If there's no side whose list of flash acts includes this act, the output +// dependency will be null. + +import {input, templateCompositeFrom} from '#composite'; + +import {withUniqueReferencingThing} from '#composite/wiki-data'; + +export default templateCompositeFrom({ + annotation: `withFlashSide`, + + outputs: ['#flashSide'], + + steps: () => [ + withUniqueReferencingThing({ + data: 'flashSideData', + list: input.value('acts'), + }).outputs({ + ['#uniqueReferencingThing']: '#flashSide', + }), + ], +}); diff --git a/src/data/composite/things/flash/index.js b/src/data/composite/things/flash/index.js new file mode 100644 index 0000000..63ac13d --- /dev/null +++ b/src/data/composite/things/flash/index.js @@ -0,0 +1 @@ +export {default as withFlashAct} from './withFlashAct.js'; diff --git a/src/data/composite/things/flash/withFlashAct.js b/src/data/composite/things/flash/withFlashAct.js new file mode 100644 index 0000000..652b8bf --- /dev/null +++ b/src/data/composite/things/flash/withFlashAct.js @@ -0,0 +1,22 @@ +// Gets the flash's act. This will early exit if flashActData is missing. +// If there's no flash whose list of flashes includes this flash, the output +// dependency will be null. + +import {input, templateCompositeFrom} from '#composite'; + +import {withUniqueReferencingThing} from '#composite/wiki-data'; + +export default templateCompositeFrom({ + annotation: `withFlashAct`, + + outputs: ['#flashAct'], + + steps: () => [ + withUniqueReferencingThing({ + data: 'flashActData', + list: input.value('flashes'), + }).outputs({ + ['#uniqueReferencingThing']: '#flashAct', + }), + ], +}); diff --git a/src/data/composite/things/track/exitWithoutUniqueCoverArt.js b/src/data/composite/things/track/exitWithoutUniqueCoverArt.js new file mode 100644 index 0000000..f47086d --- /dev/null +++ b/src/data/composite/things/track/exitWithoutUniqueCoverArt.js @@ -0,0 +1,26 @@ +// Shorthand for checking if the track has unique cover art and exposing a +// fallback value if it isn't. + +import {input, templateCompositeFrom} from '#composite'; + +import {exitWithoutDependency} from '#composite/control-flow'; + +import withHasUniqueCoverArt from './withHasUniqueCoverArt.js'; + +export default templateCompositeFrom({ + annotation: `exitWithoutUniqueCoverArt`, + + inputs: { + value: input({defaultValue: null}), + }, + + steps: () => [ + withHasUniqueCoverArt(), + + exitWithoutDependency({ + dependency: '#hasUniqueCoverArt', + mode: input.value('falsy'), + value: input('value'), + }), + ], +}); diff --git a/src/data/composite/things/track/index.js b/src/data/composite/things/track/index.js new file mode 100644 index 0000000..cc723a2 --- /dev/null +++ b/src/data/composite/things/track/index.js @@ -0,0 +1,11 @@ +export {default as exitWithoutUniqueCoverArt} from './exitWithoutUniqueCoverArt.js'; +export {default as inferredAdditionalNameList} from './inferredAdditionalNameList.js'; +export {default as inheritFromOriginalRelease} from './inheritFromOriginalRelease.js'; +export {default as sharedAdditionalNameList} from './sharedAdditionalNameList.js'; +export {default as trackReverseReferenceList} from './trackReverseReferenceList.js'; +export {default as withAlbum} from './withAlbum.js'; +export {default as withAlwaysReferenceByDirectory} from './withAlwaysReferenceByDirectory.js'; +export {default as withContainingTrackSection} from './withContainingTrackSection.js'; +export {default as withHasUniqueCoverArt} from './withHasUniqueCoverArt.js'; +export {default as withOtherReleases} from './withOtherReleases.js'; +export {default as withPropertyFromAlbum} from './withPropertyFromAlbum.js'; diff --git a/src/data/composite/things/track/inferredAdditionalNameList.js b/src/data/composite/things/track/inferredAdditionalNameList.js new file mode 100644 index 0000000..58e8d2a --- /dev/null +++ b/src/data/composite/things/track/inferredAdditionalNameList.js @@ -0,0 +1,67 @@ +// Infers additional name entries from other releases that were titled +// differently; the corresponding releases are stored in eacn entry's "from" +// array, which will include multiple items, if more than one other release +// shares the same name differing from this one's. + +import {input, templateCompositeFrom} from '#composite'; +import {chunkByProperties} from '#sugar'; + +import {exitWithoutDependency} from '#composite/control-flow'; +import {withFilteredList, withPropertyFromList} from '#composite/data'; +import {withThingsSortedAlphabetically} from '#composite/wiki-data'; + +import withOtherReleases from './withOtherReleases.js'; + +export default templateCompositeFrom({ + annotation: `inferredAdditionalNameList`, + + compose: false, + + steps: () => [ + withOtherReleases(), + + exitWithoutDependency({ + dependency: '#otherReleases', + mode: input.value('empty'), + value: input.value([]), + }), + + withPropertyFromList({ + list: '#otherReleases', + property: input.value('name'), + }), + + { + dependencies: ['#otherReleases.name', 'name'], + compute: (continuation, { + ['#otherReleases.name']: releaseNames, + ['name']: ownName, + }) => continuation({ + ['#nameFilter']: + releaseNames.map(name => name !== ownName), + }), + }, + + withFilteredList({ + list: '#otherReleases', + filter: '#nameFilter', + }).outputs({ + '#filteredList': '#differentlyNamedReleases', + }), + + withThingsSortedAlphabetically({ + things: '#differentlyNamedReleases', + }).outputs({ + '#sortedThings': '#differentlyNamedReleases', + }), + + { + dependencies: ['#differentlyNamedReleases'], + compute: ({ + ['#differentlyNamedReleases']: releases, + }) => + chunkByProperties(releases, ['name']) + .map(({name, chunk}) => ({name, from: chunk})), + }, + ], +}); diff --git a/src/data/composite/things/track/inheritFromOriginalRelease.js b/src/data/composite/things/track/inheritFromOriginalRelease.js new file mode 100644 index 0000000..27ed138 --- /dev/null +++ b/src/data/composite/things/track/inheritFromOriginalRelease.js @@ -0,0 +1,50 @@ +// Early exits with a value inherited from the original release, if +// this track is a rerelease, and otherwise continues with no further +// dependencies provided. If allowOverride is true, then the continuation +// will also be called if the original release exposed the requested +// property as null. +// +// Like withOriginalRelease, this will early exit (with notFoundValue) if the +// original release is specified by reference and that reference doesn't +// resolve to anything. + +import {input, templateCompositeFrom} from '#composite'; + +import withOriginalRelease from './withOriginalRelease.js'; + +export default templateCompositeFrom({ + annotation: `inheritFromOriginalRelease`, + + inputs: { + property: input({type: 'string'}), + allowOverride: input({type: 'boolean', defaultValue: false}), + notFoundValue: input({defaultValue: null}), + }, + + steps: () => [ + withOriginalRelease({ + notFoundValue: input('notFoundValue'), + }), + + { + dependencies: [ + '#originalRelease', + input('property'), + input('allowOverride'), + ], + + compute: (continuation, { + ['#originalRelease']: originalRelease, + [input('property')]: originalProperty, + [input('allowOverride')]: allowOverride, + }) => { + if (!originalRelease) return continuation(); + + const value = originalRelease[originalProperty]; + if (allowOverride && value === null) return continuation(); + + return continuation.exit(value); + }, + }, + ], +}); diff --git a/src/data/composite/things/track/sharedAdditionalNameList.js b/src/data/composite/things/track/sharedAdditionalNameList.js new file mode 100644 index 0000000..1806ec8 --- /dev/null +++ b/src/data/composite/things/track/sharedAdditionalNameList.js @@ -0,0 +1,38 @@ +// Compiles additional names directly provided by other releases. + +import {input, templateCompositeFrom} from '#composite'; + +import {exitWithoutDependency, exposeDependency} + from '#composite/control-flow'; +import {withFlattenedList, withPropertyFromList} from '#composite/data'; + +import withOtherReleases from './withOtherReleases.js'; + +export default templateCompositeFrom({ + annotation: `sharedAdditionalNameList`, + + compose: false, + + steps: () => [ + withOtherReleases(), + + exitWithoutDependency({ + dependency: '#otherReleases', + mode: input.value('empty'), + value: input.value([]), + }), + + withPropertyFromList({ + list: '#otherReleases', + property: input.value('additionalNames'), + }), + + withFlattenedList({ + list: '#otherReleases.additionalNames', + }), + + exposeDependency({ + dependency: '#flattenedList', + }), + ], +}); diff --git a/src/data/composite/things/track/trackAdditionalNameList.js b/src/data/composite/things/track/trackAdditionalNameList.js new file mode 100644 index 0000000..65a2263 --- /dev/null +++ b/src/data/composite/things/track/trackAdditionalNameList.js @@ -0,0 +1,38 @@ +// Compiles additional names from various sources. + +import {input, templateCompositeFrom} from '#composite'; +import {isAdditionalNameList} from '#validators'; + +import withInferredAdditionalNames from './withInferredAdditionalNames.js'; +import withSharedAdditionalNames from './withSharedAdditionalNames.js'; + +export default templateCompositeFrom({ + annotation: `trackAdditionalNameList`, + + compose: false, + + update: {validate: isAdditionalNameList}, + + steps: () => [ + withInferredAdditionalNames(), + withSharedAdditionalNames(), + + { + dependencies: [ + '#inferredAdditionalNames', + '#sharedAdditionalNames', + input.updateValue(), + ], + + compute: ({ + ['#inferredAdditionalNames']: inferredAdditionalNames, + ['#sharedAdditionalNames']: sharedAdditionalNames, + [input.updateValue()]: providedAdditionalNames, + }) => [ + ...providedAdditionalNames ?? [], + ...sharedAdditionalNames, + ...inferredAdditionalNames, + ], + }, + ], +}); diff --git a/src/data/composite/things/track/trackReverseReferenceList.js b/src/data/composite/things/track/trackReverseReferenceList.js new file mode 100644 index 0000000..44940ae --- /dev/null +++ b/src/data/composite/things/track/trackReverseReferenceList.js @@ -0,0 +1,38 @@ +// Like a normal reverse reference list ("objects which reference this object +// under a specified property"), only excluding rereleases from the possible +// outputs. While it's useful to travel from a rerelease to the tracks it +// references, rereleases aren't generally relevant from the perspective of +// the tracks *being* referenced. Apart from hiding rereleases from lists on +// the site, it also excludes keeps them from relational data processing, such +// as on the "Tracks - by Times Referenced" listing page. + +import {input, templateCompositeFrom} from '#composite'; +import {withReverseReferenceList} from '#composite/wiki-data'; + +export default templateCompositeFrom({ + annotation: `trackReverseReferenceList`, + + compose: false, + + inputs: { + list: input({type: 'string'}), + }, + + steps: () => [ + withReverseReferenceList({ + data: 'trackData', + list: input('list'), + }), + + { + flags: {expose: true}, + expose: { + dependencies: ['#reverseReferenceList'], + compute: ({ + ['#reverseReferenceList']: reverseReferenceList, + }) => + reverseReferenceList.filter(track => !track.originalReleaseTrack), + }, + }, + ], +}); diff --git a/src/data/composite/things/track/withAlbum.js b/src/data/composite/things/track/withAlbum.js new file mode 100644 index 0000000..03b840d --- /dev/null +++ b/src/data/composite/things/track/withAlbum.js @@ -0,0 +1,22 @@ +// Gets the track's album. This will early exit if albumData is missing. +// If there's no album whose list of tracks includes this track, the output +// dependency will be null. + +import {input, templateCompositeFrom} from '#composite'; + +import {withUniqueReferencingThing} from '#composite/wiki-data'; + +export default templateCompositeFrom({ + annotation: `withAlbum`, + + outputs: ['#album'], + + steps: () => [ + withUniqueReferencingThing({ + data: 'albumData', + list: input.value('tracks'), + }).outputs({ + ['#uniqueReferencingThing']: '#album', + }), + ], +}); diff --git a/src/data/composite/things/track/withAlwaysReferenceByDirectory.js b/src/data/composite/things/track/withAlwaysReferenceByDirectory.js new file mode 100644 index 0000000..fac8e21 --- /dev/null +++ b/src/data/composite/things/track/withAlwaysReferenceByDirectory.js @@ -0,0 +1,78 @@ +// Controls how find.track works - it'll never be matched by a reference +// just to the track's name, which means you don't have to always reference +// some *other* (much more commonly referenced) track by directory instead +// of more naturally by name. + +import {input, templateCompositeFrom} from '#composite'; +import find from '#find'; +import {isBoolean} from '#validators'; + +import {exitWithoutDependency, exposeUpdateValueOrContinue} + from '#composite/control-flow'; +import {withPropertyFromObject} from '#composite/data'; +import {withResolvedReference} from '#composite/wiki-data'; + +export default templateCompositeFrom({ + annotation: `withAlwaysReferenceByDirectory`, + + outputs: ['#alwaysReferenceByDirectory'], + + steps: () => [ + exposeUpdateValueOrContinue({ + validate: input.value(isBoolean), + }), + + // Remaining code is for defaulting to true if this track is a rerelease of + // another with the same name, so everything further depends on access to + // trackData as well as originalReleaseTrack. + + exitWithoutDependency({ + dependency: 'trackData', + mode: input.value('empty'), + value: input.value(false), + }), + + exitWithoutDependency({ + dependency: 'originalReleaseTrack', + value: input.value(false), + }), + + // It's necessary to use the custom trackOriginalReleasesOnly find function + // here, so as to avoid recursion issues - the find.track() function depends + // on accessing each track's alwaysReferenceByDirectory, which means it'll + // hit *this track* - and thus this step - and end up recursing infinitely. + // By definition, find.trackOriginalReleasesOnly excludes tracks which have + // an originalReleaseTrack update value set, which means even though it does + // still access each of tracks' `alwaysReferenceByDirectory` property, it + // won't access that of *this* track - it will never proceed past the + // `exitWithoutDependency` step directly above, so there's no opportunity + // for recursion. + withResolvedReference({ + ref: 'originalReleaseTrack', + data: 'trackData', + find: input.value(find.trackOriginalReleasesOnly), + }).outputs({ + '#resolvedReference': '#originalRelease', + }), + + exitWithoutDependency({ + dependency: '#originalRelease', + value: input.value(false), + }), + + withPropertyFromObject({ + object: '#originalRelease', + property: input.value('name'), + }), + + { + dependencies: ['name', '#originalRelease.name'], + compute: (continuation, { + name, + ['#originalRelease.name']: originalName, + }) => continuation({ + ['#alwaysReferenceByDirectory']: name === originalName, + }), + }, + ], +}); diff --git a/src/data/composite/things/track/withContainingTrackSection.js b/src/data/composite/things/track/withContainingTrackSection.js new file mode 100644 index 0000000..eaac14d --- /dev/null +++ b/src/data/composite/things/track/withContainingTrackSection.js @@ -0,0 +1,42 @@ +// Gets the track section containing this track from its album's track list. + +import {input, templateCompositeFrom} from '#composite'; +import {is} from '#validators'; + +import {raiseOutputWithoutDependency} from '#composite/control-flow'; + +import withPropertyFromAlbum from './withPropertyFromAlbum.js'; + +export default templateCompositeFrom({ + annotation: `withContainingTrackSection`, + + outputs: ['#trackSection'], + + steps: () => [ + withPropertyFromAlbum({ + property: input.value('trackSections'), + }), + + raiseOutputWithoutDependency({ + dependency: '#album.trackSections', + output: input.value({'#trackSection': null}), + }), + + { + dependencies: [ + input.myself(), + '#album.trackSections', + ], + + compute: (continuation, { + [input.myself()]: track, + [input('notFoundMode')]: notFoundMode, + ['#album.trackSections']: trackSections, + }) => continuation({ + ['#trackSection']: + trackSections.find(({tracks}) => tracks.includes(track)) + ?? null, + }), + }, + ], +}); diff --git a/src/data/composite/things/track/withHasUniqueCoverArt.js b/src/data/composite/things/track/withHasUniqueCoverArt.js new file mode 100644 index 0000000..96078d5 --- /dev/null +++ b/src/data/composite/things/track/withHasUniqueCoverArt.js @@ -0,0 +1,61 @@ +// Whether or not the track has "unique" cover artwork - a cover which is +// specifically associated with this track in particular, rather than with +// the track's album as a whole. This is typically used to select between +// displaying the track artwork and a fallback, such as the album artwork +// or a placeholder. (This property is named hasUniqueCoverArt instead of +// the usual hasCoverArt to emphasize that it does not inherit from the +// album.) + +import {input, templateCompositeFrom} from '#composite'; +import {empty} from '#sugar'; + +import {withResolvedContribs} from '#composite/wiki-data'; + +import withPropertyFromAlbum from './withPropertyFromAlbum.js'; + +export default templateCompositeFrom({ + annotation: 'withHasUniqueCoverArt', + + outputs: ['#hasUniqueCoverArt'], + + steps: () => [ + { + dependencies: ['disableUniqueCoverArt'], + compute: (continuation, {disableUniqueCoverArt}) => + (disableUniqueCoverArt + ? continuation.raiseOutput({ + ['#hasUniqueCoverArt']: false, + }) + : continuation()), + }, + + withResolvedContribs({from: 'coverArtistContribs'}), + + { + dependencies: ['#resolvedContribs'], + compute: (continuation, { + ['#resolvedContribs']: contribsFromTrack, + }) => + (empty(contribsFromTrack) + ? continuation() + : continuation.raiseOutput({ + ['#hasUniqueCoverArt']: true, + })), + }, + + withPropertyFromAlbum({ + property: input.value('trackCoverArtistContribs'), + }), + + { + dependencies: ['#album.trackCoverArtistContribs'], + compute: (continuation, { + ['#album.trackCoverArtistContribs']: contribsFromAlbum, + }) => + continuation.raiseOutput({ + ['#hasUniqueCoverArt']: + !empty(contribsFromAlbum), + }), + }, + ], +}); diff --git a/src/data/composite/things/track/withOriginalRelease.js b/src/data/composite/things/track/withOriginalRelease.js new file mode 100644 index 0000000..c7f4965 --- /dev/null +++ b/src/data/composite/things/track/withOriginalRelease.js @@ -0,0 +1,78 @@ +// Just includes the original release of this track as a dependency. +// If this track isn't a rerelease, then it'll provide null, unless the +// {selfIfOriginal} option is set, in which case it'll provide this track +// itself. This will early exit (with notFoundValue) if the original release +// is specified by reference and that reference doesn't resolve to anything. + +import {input, templateCompositeFrom} from '#composite'; +import find from '#find'; +import {validateWikiData} from '#validators'; + +import {exitWithoutDependency, withResultOfAvailabilityCheck} + from '#composite/control-flow'; +import {withResolvedReference} from '#composite/wiki-data'; + +export default templateCompositeFrom({ + annotation: `withOriginalRelease`, + + inputs: { + selfIfOriginal: input({type: 'boolean', defaultValue: false}), + + data: input({ + validate: validateWikiData({referenceType: 'track'}), + defaultDependency: 'trackData', + }), + + notFoundValue: input({defaultValue: null}), + }, + + outputs: ['#originalRelease'], + + steps: () => [ + withResultOfAvailabilityCheck({ + from: 'originalReleaseTrack', + }), + + { + dependencies: [ + input.myself(), + input('selfIfOriginal'), + '#availability', + ], + + compute: (continuation, { + [input.myself()]: track, + [input('selfIfOriginal')]: selfIfOriginal, + '#availability': availability, + }) => + (availability + ? continuation() + : continuation.raiseOutput({ + ['#originalRelease']: + (selfIfOriginal ? track : null), + })), + }, + + withResolvedReference({ + ref: 'originalReleaseTrack', + data: input('data'), + find: input.value(find.track), + }), + + exitWithoutDependency({ + dependency: '#resolvedReference', + value: input('notFoundValue'), + }), + + { + dependencies: ['#resolvedReference'], + + compute: (continuation, { + ['#resolvedReference']: resolvedReference, + }) => + continuation({ + ['#originalRelease']: resolvedReference, + }), + }, + ], +}); diff --git a/src/data/composite/things/track/withOtherReleases.js b/src/data/composite/things/track/withOtherReleases.js new file mode 100644 index 0000000..f8c1c3f --- /dev/null +++ b/src/data/composite/things/track/withOtherReleases.js @@ -0,0 +1,41 @@ +import {input, templateCompositeFrom} from '#composite'; + +import {exitWithoutDependency} from '#composite/control-flow'; + +import withOriginalRelease from './withOriginalRelease.js'; + +export default templateCompositeFrom({ + annotation: `withOtherReleases`, + + outputs: ['#otherReleases'], + + steps: () => [ + exitWithoutDependency({ + dependency: 'trackData', + mode: input.value('empty'), + }), + + withOriginalRelease({ + selfIfOriginal: input.value(true), + notFoundValue: input.value([]), + }), + + { + dependencies: [input.myself(), '#originalRelease', 'trackData'], + compute: (continuation, { + [input.myself()]: thisTrack, + ['#originalRelease']: originalRelease, + trackData, + }) => continuation({ + ['#otherReleases']: + (originalRelease === thisTrack + ? [] + : [originalRelease]) + .concat(trackData.filter(track => + track !== originalRelease && + track !== thisTrack && + track.originalReleaseTrack === originalRelease)), + }), + }, + ], +}); diff --git a/src/data/composite/things/track/withPropertyFromAlbum.js b/src/data/composite/things/track/withPropertyFromAlbum.js new file mode 100644 index 0000000..d41390f --- /dev/null +++ b/src/data/composite/things/track/withPropertyFromAlbum.js @@ -0,0 +1,40 @@ +// Gets a single property from this track's album, providing it as the same +// property name prefixed with '#album.' (by default). + +import {input, templateCompositeFrom} from '#composite'; +import {is} from '#validators'; + +import {withPropertyFromObject} from '#composite/data'; + +import withAlbum from './withAlbum.js'; + +export default templateCompositeFrom({ + annotation: `withPropertyFromAlbum`, + + inputs: { + property: input.staticValue({type: 'string'}), + }, + + outputs: ({ + [input.staticValue('property')]: property, + }) => ['#album.' + property], + + steps: () => [ + withAlbum(), + + withPropertyFromObject({ + object: '#album', + property: input('property'), + }), + + { + dependencies: ['#value', input.staticValue('property')], + compute: (continuation, { + ['#value']: value, + [input.staticValue('property')]: property, + }) => continuation({ + ['#album.' + property]: value, + }), + }, + ], +}); diff --git a/src/data/composite/wiki-data/exitWithoutContribs.js b/src/data/composite/wiki-data/exitWithoutContribs.js new file mode 100644 index 0000000..2c8219f --- /dev/null +++ b/src/data/composite/wiki-data/exitWithoutContribs.js @@ -0,0 +1,47 @@ +// Shorthand for exiting if the contribution list (usually a property's update +// value) resolves to empty - ensuring that the later computed results are only +// returned if these contributions are present. + +import {input, templateCompositeFrom} from '#composite'; +import {isContributionList} from '#validators'; + +import {withResultOfAvailabilityCheck} from '#composite/control-flow'; + +import withResolvedContribs from './withResolvedContribs.js'; + +export default templateCompositeFrom({ + annotation: `exitWithoutContribs`, + + inputs: { + contribs: input({ + validate: isContributionList, + acceptsNull: true, + }), + + value: input({defaultValue: null}), + }, + + steps: () => [ + withResolvedContribs({ + from: input('contribs'), + }), + + // TODO: Fairly certain exitWithoutDependency would be sufficient here. + + withResultOfAvailabilityCheck({ + from: '#resolvedContribs', + mode: input.value('empty'), + }), + + { + dependencies: ['#availability', input('value')], + compute: (continuation, { + ['#availability']: availability, + [input('value')]: value, + }) => + (availability + ? continuation() + : continuation.exit(value)), + }, + ], +}); diff --git a/src/data/composite/wiki-data/index.js b/src/data/composite/wiki-data/index.js new file mode 100644 index 0000000..b4cf6d1 --- /dev/null +++ b/src/data/composite/wiki-data/index.js @@ -0,0 +1,16 @@ +// #composite/wiki-data +// +// Entries here may depend on entries in #composite/control-flow and in +// #composite/data. +// + +export {default as exitWithoutContribs} from './exitWithoutContribs.js'; +export {default as inputWikiData} from './inputWikiData.js'; +export {default as withParsedCommentaryEntries} from './withParsedCommentaryEntries.js'; +export {default as withResolvedContribs} from './withResolvedContribs.js'; +export {default as withResolvedReference} from './withResolvedReference.js'; +export {default as withResolvedReferenceList} from './withResolvedReferenceList.js'; +export {default as withReverseContributionList} from './withReverseContributionList.js'; +export {default as withReverseReferenceList} from './withReverseReferenceList.js'; +export {default as withThingsSortedAlphabetically} from './withThingsSortedAlphabetically.js'; +export {default as withUniqueReferencingThing} from './withUniqueReferencingThing.js'; diff --git a/src/data/composite/wiki-data/inputWikiData.js b/src/data/composite/wiki-data/inputWikiData.js new file mode 100644 index 0000000..cf7a7c2 --- /dev/null +++ b/src/data/composite/wiki-data/inputWikiData.js @@ -0,0 +1,17 @@ +import {input} from '#composite'; +import {validateWikiData} from '#validators'; + +// TODO: This doesn't access a class's own ThingSubclass[Thing.referenceType] +// value because classes aren't initialized by when templateCompositeFrom gets +// called (see: circular imports). So the reference types have to be hard-coded, +// which somewhat defeats the point of storing them on the class in the first +// place... +export default function inputWikiData({ + referenceType = '', + allowMixedTypes = false, +} = {}) { + return input({ + validate: validateWikiData({referenceType, allowMixedTypes}), + acceptsNull: true, + }); +} diff --git a/src/data/composite/wiki-data/withParsedCommentaryEntries.js b/src/data/composite/wiki-data/withParsedCommentaryEntries.js new file mode 100644 index 0000000..f0404a5 --- /dev/null +++ b/src/data/composite/wiki-data/withParsedCommentaryEntries.js @@ -0,0 +1,179 @@ +import {input, templateCompositeFrom} from '#composite'; +import find from '#find'; +import {stitchArrays} from '#sugar'; +import {isCommentary} from '#validators'; +import {commentaryRegexCaseSensitive} from '#wiki-data'; + +import { + fillMissingListItems, + withFlattenedList, + withPropertiesFromList, + withUnflattenedList, +} from '#composite/data'; + +import withResolvedReferenceList from './withResolvedReferenceList.js'; + +export default templateCompositeFrom({ + annotation: `withParsedCommentaryEntries`, + + inputs: { + from: input({validate: isCommentary}), + }, + + outputs: ['#parsedCommentaryEntries'], + + steps: () => [ + { + dependencies: [input('from')], + + compute: (continuation, { + [input('from')]: commentaryText, + }) => continuation({ + ['#rawMatches']: + Array.from(commentaryText.matchAll(commentaryRegexCaseSensitive)), + }), + }, + + withPropertiesFromList({ + list: '#rawMatches', + properties: input.value([ + '0', // The entire match as a string. + 'groups', + 'index', + ]), + }).outputs({ + '#rawMatches.0': '#rawMatches.text', + '#rawMatches.groups': '#rawMatches.groups', + '#rawMatches.index': '#rawMatches.startIndex', + }), + + { + dependencies: [ + '#rawMatches.text', + '#rawMatches.startIndex', + ], + + compute: (continuation, { + ['#rawMatches.text']: text, + ['#rawMatches.startIndex']: startIndex, + }) => continuation({ + ['#rawMatches.endIndex']: + stitchArrays({text, startIndex}) + .map(({text, startIndex}) => startIndex + text.length), + }), + }, + + { + dependencies: [ + input('from'), + '#rawMatches.startIndex', + '#rawMatches.endIndex', + ], + + compute: (continuation, { + [input('from')]: commentaryText, + ['#rawMatches.startIndex']: startIndex, + ['#rawMatches.endIndex']: endIndex, + }) => continuation({ + ['#entries.body']: + stitchArrays({startIndex, endIndex}) + .map(({endIndex}, index, stitched) => + (index === stitched.length - 1 + ? commentaryText.slice(endIndex) + : commentaryText.slice( + endIndex, + stitched[index + 1].startIndex))) + .map(body => body.trim()), + }), + }, + + withPropertiesFromList({ + list: '#rawMatches.groups', + prefix: input.value('#entries'), + properties: input.value([ + 'artistReferences', + 'artistDisplayText', + 'annotation', + 'date', + ]), + }), + + // The artistReferences group will always have a value, since it's required + // for the line to match in the first place. + + { + dependencies: ['#entries.artistReferences'], + compute: (continuation, { + ['#entries.artistReferences']: artistReferenceTexts, + }) => continuation({ + ['#entries.artistReferences']: + artistReferenceTexts + .map(text => text.split(',').map(ref => ref.trim())), + }), + }, + + withFlattenedList({ + list: '#entries.artistReferences', + }), + + withResolvedReferenceList({ + list: '#flattenedList', + data: 'artistData', + find: input.value(find.artist), + notFoundMode: input.value('null'), + }), + + withUnflattenedList({ + list: '#resolvedReferenceList', + }).outputs({ + '#unflattenedList': '#entries.artists', + }), + + fillMissingListItems({ + list: '#entries.artistDisplayText', + fill: input.value(null), + }), + + fillMissingListItems({ + list: '#entries.annotation', + fill: input.value(null), + }), + + { + dependencies: ['#entries.date'], + compute: (continuation, { + ['#entries.date']: date, + }) => continuation({ + ['#entries.date']: + date.map(date => date ? new Date(date) : null), + }), + }, + + { + dependencies: [ + '#entries.artists', + '#entries.artistDisplayText', + '#entries.annotation', + '#entries.date', + '#entries.body', + ], + + compute: (continuation, { + ['#entries.artists']: artists, + ['#entries.artistDisplayText']: artistDisplayText, + ['#entries.annotation']: annotation, + ['#entries.date']: date, + ['#entries.body']: body, + }) => continuation({ + ['#parsedCommentaryEntries']: + stitchArrays({ + artists, + artistDisplayText, + annotation, + date, + body, + }), + }), + }, + ], +}); diff --git a/src/data/composite/wiki-data/withResolvedContribs.js b/src/data/composite/wiki-data/withResolvedContribs.js new file mode 100644 index 0000000..77b0f96 --- /dev/null +++ b/src/data/composite/wiki-data/withResolvedContribs.js @@ -0,0 +1,76 @@ +// Resolves the contribsByRef contained in the provided dependency, +// providing (named by the second argument) the result. "Resolving" +// means mapping the "who" reference of each contribution to an artist +// object, and filtering out those whose "who" doesn't match any artist. + +import {input, templateCompositeFrom} from '#composite'; +import find from '#find'; +import {filterMultipleArrays, stitchArrays} from '#sugar'; +import {is, isContributionList} from '#validators'; + +import { + raiseOutputWithoutDependency, +} from '#composite/control-flow'; + +import { + withPropertiesFromList, +} from '#composite/data'; + +import withResolvedReferenceList from './withResolvedReferenceList.js'; + +export default templateCompositeFrom({ + annotation: `withResolvedContribs`, + + inputs: { + from: input({ + validate: isContributionList, + acceptsNull: true, + }), + + notFoundMode: input({ + validate: is('exit', 'filter', 'null'), + defaultValue: 'null', + }), + }, + + outputs: ['#resolvedContribs'], + + steps: () => [ + raiseOutputWithoutDependency({ + dependency: input('from'), + mode: input.value('empty'), + output: input.value({ + ['#resolvedContribs']: [], + }), + }), + + withPropertiesFromList({ + list: input('from'), + properties: input.value(['who', 'what']), + prefix: input.value('#contribs'), + }), + + withResolvedReferenceList({ + list: '#contribs.who', + data: 'artistData', + find: input.value(find.artist), + notFoundMode: input('notFoundMode'), + }).outputs({ + ['#resolvedReferenceList']: '#contribs.who', + }), + + { + dependencies: ['#contribs.who', '#contribs.what'], + + compute(continuation, { + ['#contribs.who']: who, + ['#contribs.what']: what, + }) { + filterMultipleArrays(who, what, (who, _what) => who); + return continuation({ + ['#resolvedContribs']: stitchArrays({who, what}), + }); + }, + }, + ], +}); diff --git a/src/data/composite/wiki-data/withResolvedReference.js b/src/data/composite/wiki-data/withResolvedReference.js new file mode 100644 index 0000000..ea71707 --- /dev/null +++ b/src/data/composite/wiki-data/withResolvedReference.js @@ -0,0 +1,57 @@ +// Resolves a reference by using the provided find function to match it +// within the provided thingData dependency. This will early exit if the +// data dependency is null. Otherwise, the data object is provided on the +// output dependency, or null, if the reference doesn't match anything or +// itself was null to begin with. + +import {input, templateCompositeFrom} from '#composite'; + +import { + exitWithoutDependency, + raiseOutputWithoutDependency, +} from '#composite/control-flow'; + +import inputWikiData from './inputWikiData.js'; + +export default templateCompositeFrom({ + annotation: `withResolvedReference`, + + inputs: { + ref: input({type: 'string', acceptsNull: true}), + + data: inputWikiData({allowMixedTypes: false}), + find: input({type: 'function'}), + }, + + outputs: ['#resolvedReference'], + + steps: () => [ + raiseOutputWithoutDependency({ + dependency: input('ref'), + output: input.value({ + ['#resolvedReference']: null, + }), + }), + + exitWithoutDependency({ + dependency: input('data'), + }), + + { + dependencies: [ + input('ref'), + input('data'), + input('find'), + ], + + compute: (continuation, { + [input('ref')]: ref, + [input('data')]: data, + [input('find')]: findFunction, + }) => continuation({ + ['#resolvedReference']: + findFunction(ref, data, {mode: 'quiet'}) ?? null, + }), + }, + ], +}); diff --git a/src/data/composite/wiki-data/withResolvedReferenceList.js b/src/data/composite/wiki-data/withResolvedReferenceList.js new file mode 100644 index 0000000..1d39e5b --- /dev/null +++ b/src/data/composite/wiki-data/withResolvedReferenceList.js @@ -0,0 +1,101 @@ +// Resolves a list of references, with each reference matched with provided +// data in the same way as withResolvedReference. This will early exit if the +// data dependency is null (even if the reference list is empty). By default +// it will filter out references which don't match, but this can be changed +// to early exit ({notFoundMode: 'exit'}) or leave null in place ('null'). + +import {input, templateCompositeFrom} from '#composite'; +import {is, isString, validateArrayItems} from '#validators'; + +import { + exitWithoutDependency, + raiseOutputWithoutDependency, +} from '#composite/control-flow'; + +import inputWikiData from './inputWikiData.js'; + +export default templateCompositeFrom({ + annotation: `withResolvedReferenceList`, + + inputs: { + list: input({ + validate: validateArrayItems(isString), + acceptsNull: true, + }), + + data: inputWikiData({allowMixedTypes: false}), + find: input({type: 'function'}), + + notFoundMode: input({ + validate: is('exit', 'filter', 'null'), + defaultValue: 'filter', + }), + }, + + outputs: ['#resolvedReferenceList'], + + steps: () => [ + exitWithoutDependency({ + dependency: input('data'), + value: input.value([]), + }), + + raiseOutputWithoutDependency({ + dependency: input('list'), + mode: input.value('empty'), + output: input.value({ + ['#resolvedReferenceList']: [], + }), + }), + + { + dependencies: [input('list'), input('data'), input('find')], + compute: (continuation, { + [input('list')]: list, + [input('data')]: data, + [input('find')]: findFunction, + }) => + continuation({ + '#matches': list.map(ref => findFunction(ref, data, {mode: 'quiet'})), + }), + }, + + { + dependencies: ['#matches'], + compute: (continuation, {'#matches': matches}) => + (matches.every(match => match) + ? continuation.raiseOutput({ + ['#resolvedReferenceList']: matches, + }) + : continuation()), + }, + + { + dependencies: ['#matches', input('notFoundMode')], + compute(continuation, { + ['#matches']: matches, + [input('notFoundMode')]: notFoundMode, + }) { + switch (notFoundMode) { + case 'exit': + return continuation.exit([]); + + case 'filter': + return continuation.raiseOutput({ + ['#resolvedReferenceList']: + matches.filter(match => match), + }); + + case 'null': + return continuation.raiseOutput({ + ['#resolvedReferenceList']: + matches.map(match => match ?? null), + }); + + default: + throw new TypeError(`Expected notFoundMode to be exit, filter, or null`); + } + }, + }, + ], +}); diff --git a/src/data/composite/wiki-data/withReverseContributionList.js b/src/data/composite/wiki-data/withReverseContributionList.js new file mode 100644 index 0000000..eccb58b --- /dev/null +++ b/src/data/composite/wiki-data/withReverseContributionList.js @@ -0,0 +1,83 @@ +// Analogous implementation for withReverseReferenceList, for contributions. +// This is all duplicate code and both should be ported to the same underlying +// data form later on. +// +// This implementation uses a global cache (via WeakMap) to attempt to speed +// up subsequent similar accesses. +// +// This has absolutely not been rigorously tested with altering properties of +// data objects in a wiki data array which is reused. If a new wiki data array +// is used, a fresh cache will always be created. + +import {input, templateCompositeFrom} from '#composite'; + +import {exitWithoutDependency} from '#composite/control-flow'; + +import inputWikiData from './inputWikiData.js'; + +// Mapping of reference list property to WeakMap. +// Each WeakMap maps a wiki data array to another weak map, +// which in turn maps each referenced thing to an array of +// things referencing it. +const caches = new Map(); + +export default templateCompositeFrom({ + annotation: `withReverseContributionList`, + + inputs: { + data: inputWikiData({allowMixedTypes: false}), + list: input({type: 'string'}), + }, + + outputs: ['#reverseContributionList'], + + steps: () => [ + exitWithoutDependency({ + dependency: input('data'), + value: input.value([]), + mode: input.value('empty'), + }), + + { + dependencies: [input.myself(), input('data'), input('list')], + + compute: (continuation, { + [input.myself()]: myself, + [input('data')]: data, + [input('list')]: list, + }) => { + if (!caches.has(list)) { + caches.set(list, new WeakMap()); + } + + const cache = caches.get(list); + + if (!cache.has(data)) { + const cacheRecord = new WeakMap(); + + for (const referencingThing of data) { + const referenceList = referencingThing[list]; + + // Destructuring {who} is the only unique part of the + // withReverseContributionList implementation, compared to + // withReverseReferneceList. + for (const {who: referencedThing} of referenceList) { + if (cacheRecord.has(referencedThing)) { + cacheRecord.get(referencedThing).push(referencingThing); + } else { + cacheRecord.set(referencedThing, [referencingThing]); + } + } + } + + cache.set(data, cacheRecord); + } + + return continuation({ + ['#reverseContributionList']: + cache.get(data).get(myself) ?? [], + }); + }, + }, + ], +}); diff --git a/src/data/composite/wiki-data/withReverseReferenceList.js b/src/data/composite/wiki-data/withReverseReferenceList.js new file mode 100644 index 0000000..2d7a421 --- /dev/null +++ b/src/data/composite/wiki-data/withReverseReferenceList.js @@ -0,0 +1,81 @@ +// Check out the info on reverseReferenceList! +// This is its composable form. +// +// This implementation uses a global cache (via WeakMap) to attempt to speed +// up subsequent similar accesses. +// +// This has absolutely not been rigorously tested with altering properties of +// data objects in a wiki data array which is reused. If a new wiki data array +// is used, a fresh cache will always be created. +// +// Note that this implementation is mirrored in withReverseContributionList, +// so any changes should be reflected there (until these are combined). + +import {input, templateCompositeFrom} from '#composite'; + +import {exitWithoutDependency} from '#composite/control-flow'; + +import inputWikiData from './inputWikiData.js'; + +// Mapping of reference list property to WeakMap. +// Each WeakMap maps a wiki data array to another weak map, +// which in turn maps each referenced thing to an array of +// things referencing it. +const caches = new Map(); + +export default templateCompositeFrom({ + annotation: `withReverseReferenceList`, + + inputs: { + data: inputWikiData({allowMixedTypes: false}), + list: input({type: 'string'}), + }, + + outputs: ['#reverseReferenceList'], + + steps: () => [ + exitWithoutDependency({ + dependency: input('data'), + value: input.value([]), + mode: input.value('empty'), + }), + + { + dependencies: [input.myself(), input('data'), input('list')], + + compute: (continuation, { + [input.myself()]: myself, + [input('data')]: data, + [input('list')]: list, + }) => { + if (!caches.has(list)) { + caches.set(list, new WeakMap()); + } + + const cache = caches.get(list); + + if (!cache.has(data)) { + const cacheRecord = new WeakMap(); + + for (const referencingThing of data) { + const referenceList = referencingThing[list]; + for (const referencedThing of referenceList) { + if (cacheRecord.has(referencedThing)) { + cacheRecord.get(referencedThing).push(referencingThing); + } else { + cacheRecord.set(referencedThing, [referencingThing]); + } + } + } + + cache.set(data, cacheRecord); + } + + return continuation({ + ['#reverseReferenceList']: + cache.get(data).get(myself) ?? [], + }); + }, + }, + ], +}); diff --git a/src/data/composite/wiki-data/withThingsSortedAlphabetically.js b/src/data/composite/wiki-data/withThingsSortedAlphabetically.js new file mode 100644 index 0000000..5e85fa6 --- /dev/null +++ b/src/data/composite/wiki-data/withThingsSortedAlphabetically.js @@ -0,0 +1,122 @@ +// Sorts a list of live, generic wiki data objects alphabetically. +// Note that this uses localeCompare but isn't specialized to a particular +// language; where localization is concerned (in content), a follow-up, locale- +// specific sort should be performed. But this function does serve to organize +// a list so same-name entries are beside each other. + +import {input, templateCompositeFrom} from '#composite'; +import {compareCaseLessSensitive, normalizeName} from '#sort'; +import {validateWikiData} from '#validators'; + +import {raiseOutputWithoutDependency} from '#composite/control-flow'; +import {withMappedList, withSortedList, withPropertiesFromList} + from '#composite/data'; + +export default templateCompositeFrom({ + annotation: `withThingsSortedAlphabetically`, + + inputs: { + things: input({validate: validateWikiData}), + }, + + outputs: ['#sortedThings'], + + steps: () => [ + raiseOutputWithoutDependency({ + dependency: input('things'), + mode: input.value('empty'), + output: input.value({'#sortedThings': []}), + }), + + withPropertiesFromList({ + list: input('things'), + properties: input.value(['name', 'directory']), + }).outputs({ + '#list.name': '#names', + '#list.directory': '#directories', + }), + + withMappedList({ + list: '#names', + map: input.value(normalizeName), + }).outputs({ + '#mappedList': '#normalizedNames', + }), + + withSortedList({ + list: '#normalizedNames', + sort: input.value(compareCaseLessSensitive), + }).outputs({ + '#unstableSortIndices': '#normalizedNameSortIndices', + }), + + withSortedList({ + list: '#names', + sort: input.value(compareCaseLessSensitive), + }).outputs({ + '#unstableSortIndices': '#nonNormalizedNameSortIndices', + }), + + withSortedList({ + list: '#directories', + sort: input.value(compareCaseLessSensitive), + }).outputs({ + '#unstableSortIndices': '#directorySortIndices', + }), + + // TODO: No primitive for the next two-three steps, yet... + + { + dependencies: [input('things')], + compute: (continuation, { + [input('things')]: things, + }) => continuation({ + ['#combinedSortIndices']: + Array.from( + {length: things.length}, + (_item, index) => index), + }), + }, + + { + dependencies: [ + '#combinedSortIndices', + '#normalizedNameSortIndices', + '#nonNormalizedNameSortIndices', + '#directorySortIndices', + ], + + compute: (continuation, { + ['#combinedSortIndices']: combined, + ['#normalizedNameSortIndices']: normalized, + ['#nonNormalizedNameSortIndices']: nonNormalized, + ['#directorySortIndices']: directory, + }) => continuation({ + ['#combinedSortIndices']: + combined.sort((index1, index2) => { + if (normalized[index1] !== normalized[index2]) + return normalized[index1] - normalized[index2]; + + if (nonNormalized[index1] !== nonNormalized[index2]) + return nonNormalized[index1] - nonNormalized[index2]; + + if (directory[index1] !== directory[index2]) + return directory[index1] - directory[index2]; + + return 0; + }), + }), + }, + + { + dependencies: [input('things'), '#combinedSortIndices'], + compute: (continuation, { + [input('things')]: things, + ['#combinedSortIndices']: combined, + }) => continuation({ + ['#sortedThings']: + combined.map(index => things[index]), + }), + }, + ], +}); diff --git a/src/data/composite/wiki-data/withUniqueReferencingThing.js b/src/data/composite/wiki-data/withUniqueReferencingThing.js new file mode 100644 index 0000000..ce04f83 --- /dev/null +++ b/src/data/composite/wiki-data/withUniqueReferencingThing.js @@ -0,0 +1,52 @@ +// Like withReverseReferenceList, but this is specifically for special "unique" +// references, meaning this thing is referenced by exactly one or zero things +// in the data list. + +import {input, templateCompositeFrom} from '#composite'; + +import {exitWithoutDependency, raiseOutputWithoutDependency} + from '#composite/control-flow'; + +import inputWikiData from './inputWikiData.js'; +import withReverseReferenceList from './withReverseReferenceList.js'; + +export default templateCompositeFrom({ + annotation: `withUniqueReferencingThing`, + + inputs: { + data: inputWikiData({allowMixedTypes: false}), + list: input({type: 'string'}), + }, + + outputs: ['#uniqueReferencingThing'], + + steps: () => [ + // withReverseRefernceList does this check too, but it early exits with + // an empty array. That's no good here! + exitWithoutDependency({ + dependency: input('data'), + mode: input.value('empty'), + }), + + withReverseReferenceList({ + data: input('data'), + list: input('list'), + }), + + raiseOutputWithoutDependency({ + dependency: '#reverseReferenceList', + mode: input.value('empty'), + output: input.value({'#uniqueReferencingThing': null}), + }), + + { + dependencies: ['#reverseReferenceList'], + compute: (continuation, { + ['#reverseReferenceList']: reverseReferenceList, + }) => continuation({ + ['#uniqueReferencingThing']: + reverseReferenceList[0], + }), + }, + ], +}); diff --git a/src/data/composite/wiki-properties/additionalFiles.js b/src/data/composite/wiki-properties/additionalFiles.js new file mode 100644 index 0000000..6760527 --- /dev/null +++ b/src/data/composite/wiki-properties/additionalFiles.js @@ -0,0 +1,30 @@ +// This is a somewhat more involved data structure - it's for additional +// or "bonus" files associated with albums or tracks (or anything else). +// It's got this form: +// +// [ +// {title: 'Booklet', files: ['Booklet.pdf']}, +// { +// title: 'Wallpaper', +// description: 'Cool Wallpaper!', +// files: ['1440x900.png', '1920x1080.png'] +// }, +// {title: 'Alternate Covers', description: null, files: [...]}, +// ... +// ] +// + +import {isAdditionalFileList} from '#validators'; + +// TODO: Not templateCompositeFrom. + +export default function() { + return { + flags: {update: true, expose: true}, + update: {validate: isAdditionalFileList}, + expose: { + transform: (additionalFiles) => + additionalFiles ?? [], + }, + }; +} diff --git a/src/data/composite/wiki-properties/additionalNameList.js b/src/data/composite/wiki-properties/additionalNameList.js new file mode 100644 index 0000000..c5971d4 --- /dev/null +++ b/src/data/composite/wiki-properties/additionalNameList.js @@ -0,0 +1,14 @@ +// A list of additional names! These can be used for a variety of purposes, +// e.g. providing extra searchable titles, localizations, romanizations or +// original titles, and so on. Each item has a name and, optionally, a +// descriptive annotation. + +import {isAdditionalNameList} from '#validators'; + +export default function() { + return { + flags: {update: true, expose: true}, + update: {validate: isAdditionalNameList}, + expose: {transform: value => value ?? []}, + }; +} diff --git a/src/data/composite/wiki-properties/color.js b/src/data/composite/wiki-properties/color.js new file mode 100644 index 0000000..1bc9888 --- /dev/null +++ b/src/data/composite/wiki-properties/color.js @@ -0,0 +1,12 @@ +// A color! This'll be some CSS-ready value. + +import {isColor} from '#validators'; + +// TODO: Not templateCompositeFrom. + +export default function() { + return { + flags: {update: true, expose: true}, + update: {validate: isColor}, + }; +} diff --git a/src/data/composite/wiki-properties/commentary.js b/src/data/composite/wiki-properties/commentary.js new file mode 100644 index 0000000..cd6b7ac --- /dev/null +++ b/src/data/composite/wiki-properties/commentary.js @@ -0,0 +1,30 @@ +// Artist commentary! Generally present on tracks and albums. + +import {input, templateCompositeFrom} from '#composite'; +import {isCommentary} from '#validators'; + +import {exitWithoutDependency, exposeDependency} + from '#composite/control-flow'; +import {withParsedCommentaryEntries} from '#composite/wiki-data'; + +export default templateCompositeFrom({ + annotation: `commentary`, + + compose: false, + + steps: () => [ + exitWithoutDependency({ + dependency: input.updateValue({validate: isCommentary}), + mode: input.value('falsy'), + value: input.value(null), + }), + + withParsedCommentaryEntries({ + from: input.updateValue(), + }), + + exposeDependency({ + dependency: '#parsedCommentaryEntries', + }), + ], +}); diff --git a/src/data/composite/wiki-properties/commentatorArtists.js b/src/data/composite/wiki-properties/commentatorArtists.js new file mode 100644 index 0000000..c5c1476 --- /dev/null +++ b/src/data/composite/wiki-properties/commentatorArtists.js @@ -0,0 +1,49 @@ +// List of artists referenced in commentary entries. +// This is mostly useful for credits and listings on artist pages. + +import {input, templateCompositeFrom} from '#composite'; + +import {exitWithoutDependency, exposeDependency} + from '#composite/control-flow'; +import {withFlattenedList, withPropertyFromList, withUniqueItemsOnly} + from '#composite/data'; +import {withParsedCommentaryEntries} from '#composite/wiki-data'; + +export default templateCompositeFrom({ + annotation: `commentatorArtists`, + + compose: false, + + steps: () => [ + exitWithoutDependency({ + dependency: 'commentary', + mode: input.value('falsy'), + value: input.value([]), + }), + + withParsedCommentaryEntries({ + from: 'commentary', + }), + + withPropertyFromList({ + list: '#parsedCommentaryEntries', + property: input.value('artists'), + }).outputs({ + '#parsedCommentaryEntries.artists': '#artistLists', + }), + + withFlattenedList({ + list: '#artistLists', + }).outputs({ + '#flattenedList': '#artists', + }), + + withUniqueItemsOnly({ + list: '#artists', + }), + + exposeDependency({ + dependency: '#artists', + }), + ], +}); diff --git a/src/data/composite/wiki-properties/contentString.js b/src/data/composite/wiki-properties/contentString.js new file mode 100644 index 0000000..b0e8244 --- /dev/null +++ b/src/data/composite/wiki-properties/contentString.js @@ -0,0 +1,15 @@ +// String type that's slightly more specific than simpleString. If the +// property is a generic piece of human-reading content, this adds some +// useful valiation on top of simpleString - but still check if more +// particular properties like `name` are more appropriate. +// +// This type adapts validation for single- and multiline content. + +import {isContentString} from '#validators'; + +export default function() { + return { + flags: {update: true, expose: true}, + update: {validate: isContentString}, + }; +} diff --git a/src/data/composite/wiki-properties/contribsPresent.js b/src/data/composite/wiki-properties/contribsPresent.js new file mode 100644 index 0000000..24f302a --- /dev/null +++ b/src/data/composite/wiki-properties/contribsPresent.js @@ -0,0 +1,30 @@ +// Nice 'n simple shorthand for an exposed-only flag which is true when any +// contributions are present in the specified property. + +import {input, templateCompositeFrom} from '#composite'; +import {isContributionList} from '#validators'; + +import {exposeDependency, withResultOfAvailabilityCheck} + from '#composite/control-flow'; + +export default templateCompositeFrom({ + annotation: `contribsPresent`, + + compose: false, + + inputs: { + contribs: input.staticDependency({ + validate: isContributionList, + acceptsNull: true, + }), + }, + + steps: () => [ + withResultOfAvailabilityCheck({ + from: input('contribs'), + mode: input.value('empty'), + }), + + exposeDependency({dependency: '#availability'}), + ], +}); diff --git a/src/data/composite/wiki-properties/contributionList.js b/src/data/composite/wiki-properties/contributionList.js new file mode 100644 index 0000000..8fde2ca --- /dev/null +++ b/src/data/composite/wiki-properties/contributionList.js @@ -0,0 +1,35 @@ +// Strong 'n sturdy contribution list, rolling a list of references (provided +// as this property's update value) and the resolved results (as get exposed) +// into one property. Update value will look something like this: +// +// [ +// {who: 'Artist Name', what: 'Viola'}, +// {who: 'artist:john-cena', what: null}, +// ... +// ] +// +// ...typically as processed from YAML, spreadsheet, or elsewhere. +// Exposes as the same, but with the "who" replaced with matches found in +// artistData - which means this always depends on an `artistData` property +// also existing on this object! +// + +import {input, templateCompositeFrom} from '#composite'; +import {isContributionList} from '#validators'; + +import {exposeConstant, exposeDependencyOrContinue} from '#composite/control-flow'; +import {withResolvedContribs} from '#composite/wiki-data'; + +export default templateCompositeFrom({ + annotation: `contributionList`, + + compose: false, + + update: {validate: isContributionList}, + + steps: () => [ + withResolvedContribs({from: input.updateValue()}), + exposeDependencyOrContinue({dependency: '#resolvedContribs'}), + exposeConstant({value: input.value([])}), + ], +}); diff --git a/src/data/composite/wiki-properties/dimensions.js b/src/data/composite/wiki-properties/dimensions.js new file mode 100644 index 0000000..57a0127 --- /dev/null +++ b/src/data/composite/wiki-properties/dimensions.js @@ -0,0 +1,13 @@ +// Plain ol' image dimensions. This is a two-item array of positive integers, +// corresponding to width and height respectively. + +import {isDimensions} from '#validators'; + +// TODO: Not templateCompositeFrom. + +export default function() { + return { + flags: {update: true, expose: true}, + update: {validate: isDimensions}, + }; +} diff --git a/src/data/composite/wiki-properties/directory.js b/src/data/composite/wiki-properties/directory.js new file mode 100644 index 0000000..0b2181c --- /dev/null +++ b/src/data/composite/wiki-properties/directory.js @@ -0,0 +1,23 @@ +// The all-encompassing "directory" property, used as the unique identifier for +// almost any data object. Also corresponds to a part of the URL which pages of +// such objects are visited at. + +import {isDirectory} from '#validators'; +import {getKebabCase} from '#wiki-data'; + +// TODO: Not templateCompositeFrom. + +export default function() { + return { + flags: {update: true, expose: true}, + update: {validate: isDirectory}, + expose: { + dependencies: ['name'], + transform(directory, {name}) { + if (directory === null && name === null) return null; + else if (directory === null) return getKebabCase(name); + else return directory; + }, + }, + }; +} diff --git a/src/data/composite/wiki-properties/duration.js b/src/data/composite/wiki-properties/duration.js new file mode 100644 index 0000000..827f282 --- /dev/null +++ b/src/data/composite/wiki-properties/duration.js @@ -0,0 +1,13 @@ +// Duration! This is a number of seconds, possibly floating point, always +// at minimum zero. + +import {isDuration} from '#validators'; + +// TODO: Not templateCompositeFrom. + +export default function() { + return { + flags: {update: true, expose: true}, + update: {validate: isDuration}, + }; +} diff --git a/src/data/composite/wiki-properties/externalFunction.js b/src/data/composite/wiki-properties/externalFunction.js new file mode 100644 index 0000000..c388da6 --- /dev/null +++ b/src/data/composite/wiki-properties/externalFunction.js @@ -0,0 +1,11 @@ +// External function. These should only be used as dependencies for other +// properties, so they're left unexposed. + +// TODO: Not templateCompositeFrom. + +export default function() { + return { + flags: {update: true}, + update: {validate: (t) => typeof t === 'function'}, + }; +} diff --git a/src/data/composite/wiki-properties/fileExtension.js b/src/data/composite/wiki-properties/fileExtension.js new file mode 100644 index 0000000..c926fa8 --- /dev/null +++ b/src/data/composite/wiki-properties/fileExtension.js @@ -0,0 +1,13 @@ +// A file extension! Or the default, if provided when calling this. + +import {isFileExtension} from '#validators'; + +// TODO: Not templateCompositeFrom. + +export default function(defaultFileExtension = null) { + return { + flags: {update: true, expose: true}, + update: {validate: isFileExtension}, + expose: {transform: (value) => value ?? defaultFileExtension}, + }; +} diff --git a/src/data/composite/wiki-properties/flag.js b/src/data/composite/wiki-properties/flag.js new file mode 100644 index 0000000..076e663 --- /dev/null +++ b/src/data/composite/wiki-properties/flag.js @@ -0,0 +1,19 @@ +// Straightforward flag descriptor for a variety of property purposes. +// Provide a default value, true or false! + +import {isBoolean} from '#validators'; + +// TODO: Not templateCompositeFrom. + +// TODO: The description is a lie. This defaults to false. Bad. + +export default function(defaultValue = false) { + if (typeof defaultValue !== 'boolean') { + throw new TypeError(`Always set explicit defaults for flags!`); + } + + return { + flags: {update: true, expose: true}, + update: {validate: isBoolean, default: defaultValue}, + }; +} diff --git a/src/data/composite/wiki-properties/index.js b/src/data/composite/wiki-properties/index.js new file mode 100644 index 0000000..89cb683 --- /dev/null +++ b/src/data/composite/wiki-properties/index.js @@ -0,0 +1,28 @@ +// #composite/wiki-properties +// +// Entries here may depend on entries in #composite/control-flow, +// #composite/data, and #composite/wiki-data. + +export {default as additionalFiles} from './additionalFiles.js'; +export {default as additionalNameList} from './additionalNameList.js'; +export {default as color} from './color.js'; +export {default as commentary} from './commentary.js'; +export {default as commentatorArtists} from './commentatorArtists.js'; +export {default as contentString} from './contentString.js'; +export {default as contribsPresent} from './contribsPresent.js'; +export {default as contributionList} from './contributionList.js'; +export {default as dimensions} from './dimensions.js'; +export {default as directory} from './directory.js'; +export {default as duration} from './duration.js'; +export {default as externalFunction} from './externalFunction.js'; +export {default as fileExtension} from './fileExtension.js'; +export {default as flag} from './flag.js'; +export {default as name} from './name.js'; +export {default as referenceList} from './referenceList.js'; +export {default as reverseContributionList} from './reverseContributionList.js'; +export {default as reverseReferenceList} from './reverseReferenceList.js'; +export {default as simpleDate} from './simpleDate.js'; +export {default as simpleString} from './simpleString.js'; +export {default as singleReference} from './singleReference.js'; +export {default as urls} from './urls.js'; +export {default as wikiData} from './wikiData.js'; diff --git a/src/data/composite/wiki-properties/name.js b/src/data/composite/wiki-properties/name.js new file mode 100644 index 0000000..5146488 --- /dev/null +++ b/src/data/composite/wiki-properties/name.js @@ -0,0 +1,11 @@ +// A wiki data object's name! Its directory (i.e. unique identifier) will be +// computed based on this value if not otherwise specified. + +import {isName} from '#validators'; + +export default function(defaultName) { + return { + flags: {update: true, expose: true}, + update: {validate: isName, default: defaultName}, + }; +} diff --git a/src/data/composite/wiki-properties/referenceList.js b/src/data/composite/wiki-properties/referenceList.js new file mode 100644 index 0000000..af634a6 --- /dev/null +++ b/src/data/composite/wiki-properties/referenceList.js @@ -0,0 +1,45 @@ +// Stores and exposes a list of references to other data objects; all items +// must be references to the same type, which is specified on the class input. +// +// See also: +// - singleReference +// - withResolvedReferenceList +// + +import {input, templateCompositeFrom} from '#composite'; +import {isThingClass, validateReferenceList} from '#validators'; + +import {exposeDependency} from '#composite/control-flow'; +import {inputWikiData, withResolvedReferenceList} from '#composite/wiki-data'; + +export default templateCompositeFrom({ + annotation: `referenceList`, + + compose: false, + + inputs: { + class: input.staticValue({validate: isThingClass}), + + data: inputWikiData({allowMixedTypes: false}), + + find: input({type: 'function'}), + }, + + update: ({ + [input.staticValue('class')]: thingClass, + }) => ({ + validate: + validateReferenceList( + thingClass[Symbol.for('Thing.referenceType')]), + }), + + steps: () => [ + withResolvedReferenceList({ + list: input.updateValue(), + data: input('data'), + find: input('find'), + }), + + exposeDependency({dependency: '#resolvedReferenceList'}), + ], +}); diff --git a/src/data/composite/wiki-properties/reverseContributionList.js b/src/data/composite/wiki-properties/reverseContributionList.js new file mode 100644 index 0000000..7f3f9c8 --- /dev/null +++ b/src/data/composite/wiki-properties/reverseContributionList.js @@ -0,0 +1,24 @@ +import {input, templateCompositeFrom} from '#composite'; + +import {exposeDependency} from '#composite/control-flow'; +import {inputWikiData, withReverseContributionList} from '#composite/wiki-data'; + +export default templateCompositeFrom({ + annotation: `reverseContributionList`, + + compose: false, + + inputs: { + data: inputWikiData({allowMixedTypes: false}), + list: input({type: 'string'}), + }, + + steps: () => [ + withReverseContributionList({ + data: input('data'), + list: input('list'), + }), + + exposeDependency({dependency: '#reverseContributionList'}), + ], +}); diff --git a/src/data/composite/wiki-properties/reverseReferenceList.js b/src/data/composite/wiki-properties/reverseReferenceList.js new file mode 100644 index 0000000..84ba67d --- /dev/null +++ b/src/data/composite/wiki-properties/reverseReferenceList.js @@ -0,0 +1,30 @@ +// Neat little shortcut for "reversing" the reference lists stored on other +// things - for example, tracks specify a "referenced tracks" property, and +// you would use this to compute a corresponding "referenced *by* tracks" +// property. Naturally, the passed ref list property is of the things in the +// wiki data provided, not the requesting Thing itself. + +import {input, templateCompositeFrom} from '#composite'; + +import {exposeDependency} from '#composite/control-flow'; +import {inputWikiData, withReverseReferenceList} from '#composite/wiki-data'; + +export default templateCompositeFrom({ + annotation: `reverseReferenceList`, + + compose: false, + + inputs: { + data: inputWikiData({allowMixedTypes: false}), + list: input({type: 'string'}), + }, + + steps: () => [ + withReverseReferenceList({ + data: input('data'), + list: input('list'), + }), + + exposeDependency({dependency: '#reverseReferenceList'}), + ], +}); diff --git a/src/data/composite/wiki-properties/simpleDate.js b/src/data/composite/wiki-properties/simpleDate.js new file mode 100644 index 0000000..f08d832 --- /dev/null +++ b/src/data/composite/wiki-properties/simpleDate.js @@ -0,0 +1,14 @@ +// General date type, used as the descriptor for a bunch of properties. +// This isn't dynamic though - it won't inherit from a date stored on +// another object, for example. + +import {isDate} from '#validators'; + +// TODO: Not templateCompositeFrom. + +export default function() { + return { + flags: {update: true, expose: true}, + update: {validate: isDate}, + }; +} diff --git a/src/data/composite/wiki-properties/simpleString.js b/src/data/composite/wiki-properties/simpleString.js new file mode 100644 index 0000000..7bf317a --- /dev/null +++ b/src/data/composite/wiki-properties/simpleString.js @@ -0,0 +1,12 @@ +// General string type. This should probably generally be avoided in favor +// of more specific validation, but using it makes it easy to find where we +// might want to improve later, and it's a useful shorthand meanwhile. + +import {isString} from '#validators'; + +export default function() { + return { + flags: {update: true, expose: true}, + update: {validate: isString}, + }; +} diff --git a/src/data/composite/wiki-properties/singleReference.js b/src/data/composite/wiki-properties/singleReference.js new file mode 100644 index 0000000..db4fc9f --- /dev/null +++ b/src/data/composite/wiki-properties/singleReference.js @@ -0,0 +1,46 @@ +// Stores and exposes one connection, or reference, to another data object. +// The reference must be to a specific type, which is specified on the class +// input. +// +// See also: +// - referenceList +// - withResolvedReference +// + +import {input, templateCompositeFrom} from '#composite'; +import {isThingClass, validateReference} from '#validators'; + +import {exposeDependency} from '#composite/control-flow'; +import {inputWikiData, withResolvedReference} from '#composite/wiki-data'; + +export default templateCompositeFrom({ + annotation: `singleReference`, + + compose: false, + + inputs: { + class: input.staticValue({validate: isThingClass}), + + find: input({type: 'function'}), + + data: inputWikiData({allowMixedTypes: false}), + }, + + update: ({ + [input.staticValue('class')]: thingClass, + }) => ({ + validate: + validateReference( + thingClass[Symbol.for('Thing.referenceType')]), + }), + + steps: () => [ + withResolvedReference({ + ref: input.updateValue(), + data: input('data'), + find: input('find'), + }), + + exposeDependency({dependency: '#resolvedReference'}), + ], +}); diff --git a/src/data/composite/wiki-properties/urls.js b/src/data/composite/wiki-properties/urls.js new file mode 100644 index 0000000..3160a0b --- /dev/null +++ b/src/data/composite/wiki-properties/urls.js @@ -0,0 +1,14 @@ +// A list of URLs! This will always be present on the data object, even if set +// to an empty array or null. + +import {isURL, validateArrayItems} from '#validators'; + +// TODO: Not templateCompositeFrom. + +export default function() { + return { + flags: {update: true, expose: true}, + update: {validate: validateArrayItems(isURL)}, + expose: {transform: value => value ?? []}, + }; +} diff --git a/src/data/composite/wiki-properties/wikiData.js b/src/data/composite/wiki-properties/wikiData.js new file mode 100644 index 0000000..3bebed3 --- /dev/null +++ b/src/data/composite/wiki-properties/wikiData.js @@ -0,0 +1,27 @@ +// General purpose wiki data constructor, for properties like artistData, +// trackData, etc. + +import {input, templateCompositeFrom} from '#composite'; +import {isThingClass, validateWikiData} from '#validators'; + +export default templateCompositeFrom({ + annotation: `wikiData`, + + compose: false, + + inputs: { + class: input.staticValue({validate: isThingClass}), + }, + + update: ({ + [input.staticValue('class')]: thingClass, + }) => ({ + validate: + validateWikiData({ + referenceType: + thingClass[Symbol.for('Thing.referenceType')], + }), + }), + + steps: () => [], +}); diff --git a/src/data/language.js b/src/data/language.js new file mode 100644 index 0000000..a149e19 --- /dev/null +++ b/src/data/language.js @@ -0,0 +1,331 @@ +import EventEmitter from 'node:events'; +import {readFile} from 'node:fs/promises'; +import path from 'node:path'; +import {fileURLToPath} from 'node:url'; + +import chokidar from 'chokidar'; +import he from 'he'; // It stands for "HTML Entities", apparently. Cursed. +import yaml from 'js-yaml'; + +import {annotateError, annotateErrorWithFile, showAggregate, withAggregate} + from '#aggregate'; +import {externalLinkSpec} from '#external-links'; +import {colors, logWarn} from '#cli'; +import {splitKeys, withEntries} from '#sugar'; +import T from '#things'; + +const {Language} = T; + +export const DEFAULT_STRINGS_FILE = 'strings-default.yaml'; + +export const internalDefaultStringsFile = + path.resolve( + path.dirname(fileURLToPath(import.meta.url)), + '../', + DEFAULT_STRINGS_FILE); + +export function processLanguageSpec(spec, {existingCode = null} = {}) { + const { + 'meta.languageCode': code, + 'meta.languageName': name, + + 'meta.languageIntlCode': intlCode = null, + 'meta.hidden': hidden = false, + + ...strings + } = spec; + + withAggregate({message: `Errors validating language spec`}, ({push}) => { + if (!code) { + push(new Error(`Missing language code`)); + } + + if (!name) { + push(new Error(`Missing language name`)); + } + + if (code && existingCode && code !== existingCode) { + push(new Error(`Language code (${code}) doesn't match previous value\n(You'll have to reload hsmusic to load this)`)); + } + }); + + return {code, intlCode, name, hidden, strings}; +} + +export function flattenLanguageSpec(spec) { + const recursive = (keyPath, value) => + (typeof value === 'object' + ? Object.assign({}, ... + Object.entries(value) + .map(([key, value]) => + (key === '_' + ? {[keyPath]: value} + : recursive( + (keyPath ? `${keyPath}.${key}` : key), + value)))) + : {[keyPath]: value}); + + return recursive('', spec); +} + +export function unflattenLanguageSpec(flat, reference) { + const setNestedProp = (obj, key, value) => { + const recursive = (o, k) => { + if (k.length === 1) { + o[k[0]] = value; + return; + } + + if (typeof o[k[0]] === 'undefined') { + o[k[0]] = {}; + } else if (typeof o[k[0]] === 'string') { + o[k[0]] = {_: o[k[0]]}; + } + + recursive(o[k[0]], k.slice(1)); + }; + + return recursive(obj, splitKeys(key)); + }; + + const walkEntries = (ownNode, refNode) => { + const recursive = (refKeys, ownNode, refNode) => { + const [firstKey, ...restKeys] = refKeys; + + if (typeof ownNode[firstKey] === 'undefined') { + return undefined; + } + + const result = + (refKeys.length === 1 + ? walkEntry(ownNode[firstKey], refNode) + : recursive(restKeys, ownNode[firstKey], refNode)); + + if (typeof result === 'undefined') { + return undefined; + } + + if (typeof result === 'string') { + delete ownNode[firstKey]; + return {[firstKey]: result}; + } + + if (refKeys.length > 1) { + return withEntries(result, entries => + entries.map(([key, value]) => [`${firstKey}.${key}`, value])); + } else { + return {[firstKey]: result}; + } + }; + + let mapped; + + for (const [key, value] of Object.entries(refNode)) { + const result = recursive(splitKeys(key), ownNode, refNode[key]); + if (!result) continue; + if (!mapped) mapped = {}; + Object.assign(mapped, result); + } + + return mapped; + }; + + const walkEntry = (ownNode, refNode) => { + if ( + typeof ownNode === 'object' && + typeof refNode === 'object' + ) { + return walkEntries(ownNode, refNode); + } + + if ( + typeof ownNode === 'string' && + typeof refNode === 'object' && + typeof refNode._ === 'string' + ) { + return {_: ownNode}; + } + + if ( + typeof ownNode === 'object' && + typeof refNode === 'string' && + typeof ownNode._ === 'string' + ) { + return ownNode._; + } + + if ( + typeof ownNode === 'string' && + typeof refNode === 'string' + ) { + return ownNode; + } + + return undefined; + }; + + const clean = node => { + if (typeof node === 'string') { + return node; + } + + const entries = Object.entries(node); + if (entries.length === 0) { + return undefined; + } + + let results; + for (const [key, value] of entries) { + const cleanValue = clean(value); + if (typeof cleanValue === 'undefined') continue; + if (!results) results = {}; + results[key] = cleanValue; + } + + return results; + }; + + const storage = {}; + for (const [key, value] of Object.entries(flat)) { + setNestedProp(storage, key, value); + } + + const rootResult = walkEntries(storage, reference); + const spec = rootResult ?? {}; + + const unmapped = clean(storage); + if (unmapped) { + spec['meta.unmapped'] = unmapped; + } + + return spec; +} + +async function processLanguageSpecFromFile(file, processLanguageSpecOpts) { + let contents; + + try { + contents = await readFile(file, 'utf-8'); + } catch (caughtError) { + throw annotateError( + new Error(`Failed to read language file`, {cause: caughtError}), + error => annotateErrorWithFile(error, file)); + } + + let rawSpec; + let parseLanguage; + + try { + if (path.extname(file) === '.yaml') { + parseLanguage = 'YAML'; + rawSpec = yaml.load(contents); + } else { + parseLanguage = 'JSON'; + rawSpec = JSON.parse(contents); + } + } catch (caughtError) { + throw annotateError( + new Error(`Failed to parse language file as valid ${parseLanguage}`, {cause: caughtError}), + error => annotateErrorWithFile(error, file)); + } + + const flattenedSpec = flattenLanguageSpec(rawSpec); + + try { + return processLanguageSpec(flattenedSpec, processLanguageSpecOpts); + } catch (caughtError) { + throw annotateErrorWithFile(caughtError, file); + } +} + +export function initializeLanguageObject() { + const language = new Language(); + + language.escapeHTML = string => + he.encode(string, {useNamedReferences: true}); + + language.externalLinkSpec = externalLinkSpec; + + return language; +} + +export async function processLanguageFile(file) { + const language = initializeLanguageObject(); + const properties = await processLanguageSpecFromFile(file); + return Object.assign(language, properties); +} + +export function watchLanguageFile(file, { + logging = true, +} = {}) { + const basename = path.basename(file); + + const events = new EventEmitter(); + const language = initializeLanguageObject(); + + let emittedReady = false; + let successfullyAppliedLanguage = false; + + Object.assign(events, {language, close}); + + const watcher = chokidar.watch(file); + watcher.on('change', () => handleFileUpdated()); + + setImmediate(handleFileUpdated); + + return events; + + async function close() { + return watcher.close(); + } + + function checkReadyConditions() { + if (emittedReady) return; + if (!successfullyAppliedLanguage) return; + + events.emit('ready'); + emittedReady = true; + } + + async function handleFileUpdated() { + let properties; + + try { + properties = await processLanguageSpecFromFile(file, { + existingCode: + (successfullyAppliedLanguage + ? language.code + : null), + }); + } catch (error) { + events.emit('error', error); + + if (logging) { + const label = + (successfullyAppliedLanguage + ? `${language.name} (${language.code})` + : basename); + + if (successfullyAppliedLanguage) { + logWarn`Failed to load language ${label} - using existing version`; + } else { + logWarn`Failed to load language ${label} - no prior version loaded`; + } + showAggregate(error, {showTraces: false}); + } + + return; + } + + Object.assign(language, properties); + successfullyAppliedLanguage = true; + + if (logging && emittedReady) { + const timestamp = new Date().toLocaleString('en-US', {timeStyle: 'medium'}); + console.log(colors.green(`[${timestamp}] Updated language ${language.name} (${language.code})`)); + } + + events.emit('update'); + checkReadyConditions(); + } +} diff --git a/src/data/patches.js b/src/data/patches.js index 3ed4fad..feeaf39 100644 --- a/src/data/patches.js +++ b/src/data/patches.js @@ -1,370 +1,386 @@ // --> Patch export class Patch { - static INPUT_NONE = 0; - static INPUT_CONSTANT = 1; - static INPUT_DIRECT_CONNECTION = 2; - static INPUT_MANAGED_CONNECTION = 3; + static INPUT_NONE = 0; + static INPUT_CONSTANT = 1; + static INPUT_DIRECT_CONNECTION = 2; + static INPUT_MANAGED_CONNECTION = 3; - static INPUT_UNAVAILABLE = 0; - static INPUT_AVAILABLE = 1; + static INPUT_UNAVAILABLE = 0; + static INPUT_AVAILABLE = 1; - static OUTPUT_UNAVAILABLE = 0; - static OUTPUT_AVAILABLE = 1; + static OUTPUT_UNAVAILABLE = 0; + static OUTPUT_AVAILABLE = 1; - static inputNames = []; inputNames = null; - static outputNames = []; outputNames = null; + static inputNames = []; + inputNames = null; + static outputNames = []; + outputNames = null; - manager = null; - inputs = Object.create(null); + manager = null; + inputs = Object.create(null); - constructor({ - manager, + constructor({ + manager, - inputNames, - outputNames, + inputNames, + outputNames, - inputs, - } = {}) { - this.inputNames = inputNames ?? this.constructor.inputNames; - this.outputNames = outputNames ?? this.constructor.outputNames; + inputs, + } = {}) { + this.inputNames = inputNames ?? this.constructor.inputNames; + this.outputNames = outputNames ?? this.constructor.outputNames; - manager?.addManagedPatch(this); + manager?.addManagedPatch(this); - if (inputs) { - Object.assign(this.inputs, inputs); - } - - this.initializeInputs(); + if (inputs) { + Object.assign(this.inputs, inputs); } - initializeInputs() { - for (const inputName of this.inputNames) { - if (!this.inputs[inputName]) { - this.inputs[inputName] = [Patch.INPUT_NONE]; - } - } - } + this.initializeInputs(); + } - computeInputs() { - const inputs = Object.create(null); - - for (const inputName of this.inputNames) { - const input = this.inputs[inputName]; - switch (input[0]) { - case Patch.INPUT_NONE: - inputs[inputName] = [Patch.INPUT_UNAVAILABLE]; - break; - - case Patch.INPUT_CONSTANT: - inputs[inputName] = [Patch.INPUT_AVAILABLE, input[1]]; - break; - - case Patch.INPUT_DIRECT_CONNECTION: { - const patch = input[1]; - const outputName = input[2]; - const output = patch.computeOutputs()[outputName]; - switch (output[0]) { - case Patch.OUTPUT_UNAVAILABLE: - inputs[inputName] = [Patch.INPUT_UNAVAILABLE]; - break; - case Patch.OUTPUT_AVAILABLE: - inputs[inputName] = [Patch.INPUT_AVAILABLE, output[1]]; - break; - } - throw new Error('Unreachable'); - } - - case Patch.INPUT_MANAGED_CONNECTION: { - if (!this.manager) { - inputs[inputName] = [Patch.INPUT_UNAVAILABLE]; - break; - } - - inputs[inputName] = this.manager.getManagedInput(input[1]); - break; - } - } + initializeInputs() { + for (const inputName of this.inputNames) { + if (!this.inputs[inputName]) { + this.inputs[inputName] = [Patch.INPUT_NONE]; + } + } + } + + computeInputs() { + const inputs = Object.create(null); + + for (const inputName of this.inputNames) { + const input = this.inputs[inputName]; + switch (input[0]) { + case Patch.INPUT_NONE: + inputs[inputName] = [Patch.INPUT_UNAVAILABLE]; + break; + + case Patch.INPUT_CONSTANT: + inputs[inputName] = [Patch.INPUT_AVAILABLE, input[1]]; + break; + + case Patch.INPUT_DIRECT_CONNECTION: { + const patch = input[1]; + const outputName = input[2]; + const output = patch.computeOutputs()[outputName]; + switch (output[0]) { + case Patch.OUTPUT_UNAVAILABLE: + inputs[inputName] = [Patch.INPUT_UNAVAILABLE]; + break; + case Patch.OUTPUT_AVAILABLE: + inputs[inputName] = [Patch.INPUT_AVAILABLE, output[1]]; + break; + } + throw new Error('Unreachable'); } - return inputs; - } + case Patch.INPUT_MANAGED_CONNECTION: { + if (!this.manager) { + inputs[inputName] = [Patch.INPUT_UNAVAILABLE]; + break; + } - computeOutputs() { - const inputs = this.computeInputs(); - const outputs = Object.create(null); - console.log(`Compute: ${this.constructor.name}`); - this.compute(inputs, outputs); - return outputs; + inputs[inputName] = this.manager.getManagedInput(input[1]); + break; + } + } } - compute(inputs, outputs) { - // No-op. Return all outputs as unavailable. This should be overridden - // in subclasses. + return inputs; + } - for (const outputName of this.constructor.outputNames) { - outputs[outputName] = [Patch.OUTPUT_UNAVAILABLE]; - } - } + computeOutputs() { + const inputs = this.computeInputs(); + const outputs = Object.create(null); + console.log(`Compute: ${this.constructor.name}`); + this.compute(inputs, outputs); + return outputs; + } - attachToManager(manager) { - manager.addManagedPatch(this); + compute(inputs, outputs) { + // No-op. Return all outputs as unavailable. This should be overridden + // in subclasses. + + for (const outputName of this.constructor.outputNames) { + outputs[outputName] = [Patch.OUTPUT_UNAVAILABLE]; } + } - detachFromManager() { - if (this.manager) { - this.manager.removeManagedPatch(this); - } + attachToManager(manager) { + manager.addManagedPatch(this); + } + + detachFromManager() { + if (this.manager) { + this.manager.removeManagedPatch(this); } + } } // --> PatchManager export class PatchManager extends Patch { - managedPatches = []; - managedInputs = {}; + managedPatches = []; + managedInputs = {}; - #externalInputPatch = null; - #externalOutputPatch = null; + #externalInputPatch = null; + #externalOutputPatch = null; - constructor(...args) { - super(...args); + constructor(...args) { + super(...args); - this.#externalInputPatch = new PatchManagerExternalInputPatch({manager: this}); - this.#externalOutputPatch = new PatchManagerExternalOutputPatch({manager: this}); - } - - addManagedPatch(patch) { - if (patch.manager === this) { - return false; - } + this.#externalInputPatch = new PatchManagerExternalInputPatch({ + manager: this, + }); - patch.detachFromManager(); - patch.manager = this; + this.#externalOutputPatch = new PatchManagerExternalOutputPatch({ + manager: this, + }); + } - if (patch.manager === this) { - this.managedPatches.push(patch); - return true; - } else { - return false; - } + addManagedPatch(patch) { + if (patch.manager === this) { + return false; } - removeManagedPatch(patch) { - if (patch.manager !== this) { - return false; - } - - patch.manager = null; - - if (patch.manager === this) { - return false; - } + patch.detachFromManager(); + patch.manager = this; - for (const inputNames of patch.inputNames) { - const input = patch.inputs[inputName]; - if (input[0] === Patch.INPUT_MANAGED_CONNECTION) { - this.dropManagedInput(input[1]); - patch.inputs[inputName] = [Patch.INPUT_NONE]; - } - } - - this.managedPatches.splice(this.managedPatches.indexOf(patch), 1); - - return true; + if (patch.manager === this) { + this.managedPatches.push(patch); + return true; + } else { + return false; } + } - addManagedInput(patchWithInput, inputName, patchWithOutput, outputName) { - if (patchWithInput.manager !== this || patchWithOutput.manager !== this) { - throw new Error(`Input and output patches must belong to same manager (this)`); - } - - const input = patchWithInput.inputs[inputName]; - if (input[0] === Patch.INPUT_MANAGED_CONNECTION) { - this.managedInputs[input[1]] = [patchWithOutput, outputName, {}]; - } else { - const key = this.getManagedConnectionIdentifier(); - this.managedInputs[key] = [patchWithOutput, outputName, {}]; - patchWithInput.inputs[inputName] = [Patch.INPUT_MANAGED_CONNECTION, key]; - } - - return true; + removeManagedPatch(patch) { + if (patch.manager !== this) { + return false; } - dropManagedInput(identifier) { - return delete this.managedInputs[key]; + patch.manager = null; + + if (patch.manager === this) { + return false; } - getManagedInput(identifier) { - const connection = this.managedInputs[identifier]; - const patch = connection[0]; - const outputName = connection[1]; - const memory = connection[2]; - return this.computeManagedInput(patch, outputName, memory); + for (const inputName of patch.inputNames) { + const input = patch.inputs[inputName]; + if (input[0] === Patch.INPUT_MANAGED_CONNECTION) { + this.dropManagedInput(input[1]); + patch.inputs[inputName] = [Patch.INPUT_NONE]; + } } - computeManagedInput(patch, outputName, memory) { - // Override this function in subclasses to alter behavior of the "wire" - // used for connecting patches. + this.managedPatches.splice(this.managedPatches.indexOf(patch), 1); - const output = patch.computeOutputs()[outputName]; - switch (output[0]) { - case Patch.OUTPUT_UNAVAILABLE: - return [Patch.INPUT_UNAVAILABLE]; - case Patch.OUTPUT_AVAILABLE: - return [Patch.INPUT_AVAILABLE, output[1]]; - } - } + return true; + } - #managedConnectionIdentifier = 0; - getManagedConnectionIdentifier() { - return this.#managedConnectionIdentifier++; + addManagedInput(patchWithInput, inputName, patchWithOutput, outputName) { + if (patchWithInput.manager !== this || patchWithOutput.manager !== this) { + throw new Error(`Input and output patches must belong to same manager (this)`); } - addExternalInput(patchWithInput, patchInputName, managerInputName) { - return this.addManagedInput(patchWithInput, patchInputName, this.#externalInputPatch, managerInputName); + const input = patchWithInput.inputs[inputName]; + if (input[0] === Patch.INPUT_MANAGED_CONNECTION) { + this.managedInputs[input[1]] = [patchWithOutput, outputName, {}]; + } else { + const key = this.getManagedConnectionIdentifier(); + this.managedInputs[key] = [patchWithOutput, outputName, {}]; + patchWithInput.inputs[inputName] = [Patch.INPUT_MANAGED_CONNECTION, key]; } - setExternalOutput(managerOutputName, patchWithOutput, patchOutputName) { - return this.addManagedInput(this.#externalOutputPatch, managerOutputName, patchWithOutput, patchOutputName); - } - - compute(inputs, outputs) { - Object.assign(outputs, this.#externalOutputPatch.computeOutputs()); + return true; + } + + dropManagedInput(identifier) { + return delete this.managedInputs[identifier]; + } + + getManagedInput(identifier) { + const connection = this.managedInputs[identifier]; + const patch = connection[0]; + const outputName = connection[1]; + const memory = connection[2]; + return this.computeManagedInput(patch, outputName, memory); + } + + computeManagedInput(patch, outputName) { + // Override this function in subclasses to alter behavior of the "wire" + // used for connecting patches. + + const output = patch.computeOutputs()[outputName]; + switch (output[0]) { + case Patch.OUTPUT_UNAVAILABLE: + return [Patch.INPUT_UNAVAILABLE]; + case Patch.OUTPUT_AVAILABLE: + return [Patch.INPUT_AVAILABLE, output[1]]; } + } + + #managedConnectionIdentifier = 0; + getManagedConnectionIdentifier() { + return this.#managedConnectionIdentifier++; + } + + addExternalInput(patchWithInput, patchInputName, managerInputName) { + return this.addManagedInput( + patchWithInput, + patchInputName, + this.#externalInputPatch, + managerInputName + ); + } + + setExternalOutput(managerOutputName, patchWithOutput, patchOutputName) { + return this.addManagedInput( + this.#externalOutputPatch, + managerOutputName, + patchWithOutput, + patchOutputName + ); + } + + compute(inputs, outputs) { + Object.assign(outputs, this.#externalOutputPatch.computeOutputs()); + } } class PatchManagerExternalInputPatch extends Patch { - constructor({manager, ...rest}) { - super({ - manager, - inputNames: manager.inputNames, - outputNames: manager.inputNames, - ...rest - }); - } - - computeInputs() { - return this.manager.computeInputs(); - } - - compute(inputs, outputs) { - for (const name of this.inputNames) { - const input = inputs[name]; - switch (input[0]) { - case Patch.INPUT_UNAVAILABLE: - outputs[name] = [Patch.OUTPUT_UNAVAILABLE]; - break; - case Patch.INPUT_AVAILABLE: - outputs[name] = [Patch.INPUT_AVAILABLE, input[1]]; - break; - } - } + constructor({manager, ...rest}) { + super({ + manager, + inputNames: manager.inputNames, + outputNames: manager.inputNames, + ...rest, + }); + } + + computeInputs() { + return this.manager.computeInputs(); + } + + compute(inputs, outputs) { + for (const name of this.inputNames) { + const input = inputs[name]; + switch (input[0]) { + case Patch.INPUT_UNAVAILABLE: + outputs[name] = [Patch.OUTPUT_UNAVAILABLE]; + break; + case Patch.INPUT_AVAILABLE: + outputs[name] = [Patch.INPUT_AVAILABLE, input[1]]; + break; + } } + } } class PatchManagerExternalOutputPatch extends Patch { - constructor({manager, ...rest}) { - super({ - manager, - inputNames: manager.outputNames, - outputNames: manager.outputNames, - ...rest - }); - } - - compute(inputs, outputs) { - for (const name of this.inputNames) { - const input = inputs[name]; - switch (input[0]) { - case Patch.INPUT_UNAVAILABLE: - outputs[name] = [Patch.OUTPUT_UNAVAILABLE]; - break; - case Patch.INPUT_AVAILABLE: - outputs[name] = [Patch.INPUT_AVAILABLE, input[1]]; - break; - } - } + constructor({manager, ...rest}) { + super({ + manager, + inputNames: manager.outputNames, + outputNames: manager.outputNames, + ...rest, + }); + } + + compute(inputs, outputs) { + for (const name of this.inputNames) { + const input = inputs[name]; + switch (input[0]) { + case Patch.INPUT_UNAVAILABLE: + outputs[name] = [Patch.OUTPUT_UNAVAILABLE]; + break; + case Patch.INPUT_AVAILABLE: + outputs[name] = [Patch.INPUT_AVAILABLE, input[1]]; + break; + } } + } } // --> demo const caches = Symbol(); const common = Symbol(); -const hsmusic = Symbol(); Patch[caches] = { - WireCachedPatchManager: class extends PatchManager { - // "Wire" caching for PatchManager: Remembers the last outputs to come - // from each patch. As long as the inputs for a patch do not change, its - // cached outputs are reused. - - // TODO: This has a unique cache for each managed input. It should - // re-use a cache for the same patch and output name. How can we ensure - // the cache is dropped when the patch is removed, though? (Spoilers: - // probably just override removeManagedPatch) - computeManagedInput(patch, outputName, memory) { - let cache = true; - - const { previousInputs } = memory; - const { inputs } = patch; - if (memory.previousInputs) { - for (const inputName of patch.inputNames) { - // TODO: This doesn't account for connections whose values - // have changed (analogous to bubbling cache invalidation). - if (inputs[inputName] !== previousInputs[inputName]) { - cache = false; - break; - } - } - } else { - cache = false; - } - - if (cache) { - return memory.previousOutputs[outputName]; - } - - const outputs = patch.computeOutputs(); - memory.previousOutputs = outputs; - memory.previousInputs = {...inputs}; - return outputs[outputName]; + WireCachedPatchManager: class extends PatchManager { + // "Wire" caching for PatchManager: Remembers the last outputs to come + // from each patch. As long as the inputs for a patch do not change, its + // cached outputs are reused. + + // TODO: This has a unique cache for each managed input. It should + // re-use a cache for the same patch and output name. How can we ensure + // the cache is dropped when the patch is removed, though? (Spoilers: + // probably just override removeManagedPatch) + computeManagedInput(patch, outputName, memory) { + let cache = true; + + const {previousInputs} = memory; + const {inputs} = patch; + if (memory.previousInputs) { + for (const inputName of patch.inputNames) { + // TODO: This doesn't account for connections whose values + // have changed (analogous to bubbling cache invalidation). + if (inputs[inputName] !== previousInputs[inputName]) { + cache = false; + break; + } } - }, + } else { + cache = false; + } + + if (cache) { + return memory.previousOutputs[outputName]; + } + + const outputs = patch.computeOutputs(); + memory.previousOutputs = outputs; + memory.previousInputs = {...inputs}; + return outputs[outputName]; + } + }, }; Patch[common] = { - Stringify: class extends Patch { - static inputNames = ['value']; - static outputNames = ['value']; - - compute(inputs, outputs) { - if (inputs.value[0] === Patch.INPUT_AVAILABLE) { - outputs.value = [Patch.OUTPUT_AVAILABLE, inputs.value[1].toString()]; - } else { - outputs.value = [Patch.OUTPUT_UNAVAILABLE]; - } - } - }, - - Echo: class extends Patch { - static inputNames = ['value']; - static outputNames = ['value']; - - compute(inputs, outputs) { - if (inputs.value[0] === Patch.INPUT_AVAILABLE) { - outputs.value = [Patch.OUTPUT_AVAILABLE, inputs.value[1]]; - } else { - outputs.value = [Patch.OUTPUT_UNAVAILABLE]; - } - } - }, + Stringify: class extends Patch { + static inputNames = ['value']; + static outputNames = ['value']; + + compute(inputs, outputs) { + if (inputs.value[0] === Patch.INPUT_AVAILABLE) { + outputs.value = [Patch.OUTPUT_AVAILABLE, inputs.value[1].toString()]; + } else { + outputs.value = [Patch.OUTPUT_UNAVAILABLE]; + } + } + }, + + Echo: class extends Patch { + static inputNames = ['value']; + static outputNames = ['value']; + + compute(inputs, outputs) { + if (inputs.value[0] === Patch.INPUT_AVAILABLE) { + outputs.value = [Patch.OUTPUT_AVAILABLE, inputs.value[1]]; + } else { + outputs.value = [Patch.OUTPUT_UNAVAILABLE]; + } + } + }, }; const PM = new Patch[caches].WireCachedPatchManager({ - inputNames: ['externalInput'], - outputNames: ['externalOutput'], + inputNames: ['externalInput'], + outputNames: ['externalOutput'], }); const P1 = new Patch[common].Stringify({manager: PM}); diff --git a/src/data/serialize.js b/src/data/serialize.js index 9d4e888..8cac330 100644 --- a/src/data/serialize.js +++ b/src/data/serialize.js @@ -1,22 +1,26 @@ -// serialize-util.js: simple interface and utility functions for converting +// serialize.js: simple interface and utility functions for converting // Things into a directly serializeable format // Utility functions export function id(x) { - return x; + return x; } export function toRef(thing) { - return thing?.constructor.getReference(thing); + return thing?.constructor.getReference(thing); } export function toRefs(things) { - return things?.map(toRef); + return things?.map(toRef); } export function toContribRefs(contribs) { - return contribs?.map(({ who, what }) => ({who: toRef(who), what})); + return contribs?.map(({who, what}) => ({who: toRef(who), what})); +} + +export function toCommentaryRefs(entries) { + return entries?.map(({artist, ...props}) => ({artist: toRef(artist), ...props})); } // Interface @@ -24,15 +28,18 @@ export function toContribRefs(contribs) { export const serializeDescriptors = Symbol(); export function serializeThing(thing) { - const descriptors = thing.constructor[serializeDescriptors]; - if (!descriptors) { - throw new Error(`Constructor ${thing.constructor.name} does not provide serialize descriptors`); - } + const descriptors = thing.constructor[serializeDescriptors]; + + if (!descriptors) { + throw new Error(`Constructor ${thing.constructor.name} does not provide serialize descriptors`); + } - return Object.fromEntries(Object.entries(descriptors) - .map(([ property, transform ]) => [property, transform(thing[property])])); + return Object.fromEntries( + Object.entries(descriptors) + .map(([property, transform]) => [property, transform(thing[property])]) + ); } export function serializeThings(things) { - return things.map(serializeThing); + return things.map(serializeThing); } diff --git a/src/data/thing.js b/src/data/thing.js new file mode 100644 index 0000000..706e893 --- /dev/null +++ b/src/data/thing.js @@ -0,0 +1,77 @@ +// Thing: base class for wiki data types, providing interfaces generally useful +// to all wiki data objects on top of foundational CacheableObject behavior. + +import {inspect} from 'node:util'; + +import CacheableObject from '#cacheable-object'; +import {colors} from '#cli'; + +export default class Thing extends CacheableObject { + static referenceType = Symbol.for('Thing.referenceType'); + static friendlyName = Symbol.for('Thing.friendlyName'); + + static getPropertyDescriptors = Symbol.for('Thing.getPropertyDescriptors'); + static getSerializeDescriptors = Symbol.for('Thing.getSerializeDescriptors'); + + static findSpecs = Symbol.for('Thing.findSpecs'); + static yamlDocumentSpec = Symbol.for('Thing.yamlDocumentSpec'); + static getYamlLoadingSpec = Symbol.for('Thing.getYamlLoadingSpec'); + + // Default custom inspect function, which may be overridden by Thing + // subclasses. This will be used when displaying aggregate errors and other + // command-line logging - it's the place to provide information useful in + // identifying the Thing being presented. + [inspect.custom]() { + const cname = this.constructor.name; + + return ( + (this.name ? `${cname} ${colors.green(`"${this.name}"`)}` : `${cname}`) + + (this.directory ? ` (${colors.blue(Thing.getReference(this))})` : '') + ); + } + + static getReference(thing) { + if (!thing.constructor[Thing.referenceType]) { + throw TypeError(`Passed Thing is ${thing.constructor.name}, which provides no [Thing.referenceType]`); + } + + if (!thing.directory) { + throw TypeError(`Passed ${thing.constructor.name} is missing its directory`); + } + + return `${thing.constructor[Thing.referenceType]}:${thing.directory}`; + } + + static extendDocumentSpec(thingClass, subspec) { + const superspec = thingClass[Thing.yamlDocumentSpec]; + + const { + fields, + ignoredFields, + invalidFieldCombinations, + ...restOfSubspec + } = subspec; + + const newFields = Object.keys(fields ?? {}); + + return { + ...superspec, + ...restOfSubspec, + + fields: { + ...superspec.fields ?? {}, + ...fields, + }, + + ignoredFields: + (superspec.ignoredFields ?? []) + .filter(field => newFields.includes(field)) + .concat(ignoredFields ?? []), + + invalidFieldCombinations: [ + ...superspec.invalidFieldCombinations ?? [], + ...invalidFieldCombinations ?? [], + ], + }; + } +} diff --git a/src/data/things.js b/src/data/things.js deleted file mode 100644 index daec610..0000000 --- a/src/data/things.js +++ /dev/null @@ -1,1720 +0,0 @@ -// things.js: class definitions for various object types used across the wiki, -// most of which correspond to an output page, such as Track, Album, Artist - -import CacheableObject from './cacheable-object.js'; - -import { - isAdditionalFileList, - isBoolean, - isColor, - isCommentary, - isCountingNumber, - isContributionList, - isDate, - isDimensions, - isDirectory, - isDuration, - isInstance, - isFileExtension, - isLanguageCode, - isName, - isNumber, - isURL, - isString, - isWholeNumber, - oneOf, - validateArrayItems, - validateInstanceOf, - validateReference, - validateReferenceList, -} from './validators.js'; - -import * as S from './serialize.js'; - -import { - getKebabCase, - sortByArtDate, -} from '../util/wiki-data.js'; - -import find from '../util/find.js'; - -import { inspect } from 'util'; -import { color } from '../util/cli.js'; - -// Stub classes (and their exports) at the top of the file - these are -// referenced later when we actually define static class fields. We deliberately -// define the classes and set their static fields in two separate steps so that -// every class coexists from the outset, and can be directly referenced in field -// definitions later. - -// This list also acts as a quick table of contents for this JS file - use -// ctrl+F or similar to skip to a section. - -// -> Thing -export class Thing extends CacheableObject {} - -// -> Album -export class Album extends Thing {} -export class TrackGroup extends CacheableObject {} - -// -> Track -export class Track extends Thing {} - -// -> Artist -export class Artist extends Thing {} - -// -> Group -export class Group extends Thing {} -export class GroupCategory extends CacheableObject {} - -// -> ArtTag -export class ArtTag extends Thing {} - -// -> NewsEntry -export class NewsEntry extends Thing {} - -// -> StaticPage -export class StaticPage extends Thing {} - -// -> HomepageLayout -export class HomepageLayout extends CacheableObject {} -export class HomepageLayoutRow extends CacheableObject {} -export class HomepageLayoutAlbumsRow extends HomepageLayoutRow {} - -// -> Flash -export class Flash extends Thing {} -export class FlashAct extends CacheableObject {} - -// -> WikiInfo -export class WikiInfo extends CacheableObject {} - -// -> Language -export class Language extends CacheableObject {} - -// Before initializing property descriptors, set additional independent -// constants on the classes (which are referenced later). - -Thing.referenceType = Symbol('Thing.referenceType'); - -Album[Thing.referenceType] = 'album'; -Track[Thing.referenceType] = 'track'; -Artist[Thing.referenceType] = 'artist'; -Group[Thing.referenceType] = 'group'; -ArtTag[Thing.referenceType] = 'tag'; -NewsEntry[Thing.referenceType] = 'news-entry'; -StaticPage[Thing.referenceType] = 'static'; -Flash[Thing.referenceType] = 'flash'; - -// -> Thing: base class for wiki data types, providing wiki-specific utility -// functions on top of essential CacheableObject behavior. - -// Regularly reused property descriptors, for ease of access and generally -// duplicating less code across wiki data types. These are specialized utility -// functions, so check each for how its own arguments behave! -Thing.common = { - name: (defaultName) => ({ - flags: {update: true, expose: true}, - update: {validate: isName, default: defaultName} - }), - - color: () => ({ - flags: {update: true, expose: true}, - update: {validate: isColor} - }), - - directory: () => ({ - flags: {update: true, expose: true}, - update: {validate: isDirectory}, - expose: { - dependencies: ['name'], - transform(directory, { name }) { - if (directory === null && name === null) - return null; - else if (directory === null) - return getKebabCase(name); - else - return directory; - } - } - }), - - urls: () => ({ - flags: {update: true, expose: true}, - update: {validate: validateArrayItems(isURL)} - }), - - // A file extension! Or the default, if provided when calling this. - fileExtension: (defaultFileExtension = null) => ({ - flags: {update: true, expose: true}, - update: {validate: isFileExtension}, - expose: {transform: value => value ?? defaultFileExtension} - }), - - // Straightforward flag descriptor for a variety of property purposes. - // Provide a default value, true or false! - flag: (defaultValue = false) => { - if (typeof defaultValue !== 'boolean') { - throw new TypeError(`Always set explicit defaults for flags!`); - } - - return { - flags: {update: true, expose: true}, - update: {validate: isBoolean, default: defaultValue} - }; - }, - - // General date type, used as the descriptor for a bunch of properties. - // This isn't dynamic though - it won't inherit from a date stored on - // another object, for example. - simpleDate: () => ({ - flags: {update: true, expose: true}, - update: {validate: isDate} - }), - - // General string type. This should probably generally be avoided in favor - // of more specific validation, but using it makes it easy to find where we - // might want to improve later, and it's a useful shorthand meanwhile. - simpleString: () => ({ - flags: {update: true, expose: true}, - update: {validate: isString} - }), - - // External function. These should only be used as dependencies for other - // properties, so they're left unexposed. - externalFunction: () => ({ - flags: {update: true}, - update: {validate: t => typeof t === 'function'} - }), - - // Super simple "contributions by reference" list, used for a variety of - // properties (Artists, Cover Artists, etc). This is the property which is - // externally provided, in the form: - // - // [ - // {who: 'Artist Name', what: 'Viola'}, - // {who: 'artist:john-cena', what: null}, - // ... - // ] - // - // ...processed from YAML, spreadsheet, or any other kind of input. - contribsByRef: () => ({ - flags: {update: true, expose: true}, - update: {validate: isContributionList} - }), - - // Artist commentary! Generally present on tracks and albums. - commentary: () => ({ - flags: {update: true, expose: true}, - update: {validate: isCommentary} - }), - - // This is a somewhat more involved data structure - it's for additional - // or "bonus" files associated with albums or tracks (or anything else). - // It's got this form: - // - // [ - // {title: 'Booklet', files: ['Booklet.pdf']}, - // { - // title: 'Wallpaper', - // description: 'Cool Wallpaper!', - // files: ['1440x900.png', '1920x1080.png'] - // }, - // {title: 'Alternate Covers', description: null, files: [...]}, - // ... - // ] - // - additionalFiles: () => ({ - flags: {update: true, expose: true}, - update: {validate: isAdditionalFileList} - }), - - // A reference list! Keep in mind this is for general references to wiki - // objects of (usually) other Thing subclasses, not specifically leitmotif - // references in tracks (although that property uses referenceList too!). - // - // The underlying function validateReferenceList expects a string like - // 'artist' or 'track', but this utility keeps from having to hard-code the - // string in multiple places by referencing the value saved on the class - // instead. - referenceList: thingClass => { - const { [Thing.referenceType]: referenceType } = thingClass; - if (!referenceType) { - throw new Error(`The passed constructor ${thingClass.name} doesn't define Thing.referenceType!`); - } - - return { - flags: {update: true, expose: true}, - update: {validate: validateReferenceList(referenceType)} - }; - }, - - // Corresponding function for a single reference. - singleReference: thingClass => { - const { [Thing.referenceType]: referenceType } = thingClass; - if (!referenceType) { - throw new Error(`The passed constructor ${thingClass.name} doesn't define Thing.referenceType!`); - } - - return { - flags: {update: true, expose: true}, - update: {validate: validateReference(referenceType)} - }; - }, - - // Corresponding dynamic property to referenceList, which takes the values - // in the provided property and searches the specified wiki data for - // matching actual Thing-subclass objects. - dynamicThingsFromReferenceList: ( - referenceListProperty, - thingDataProperty, - findFn - ) => ({ - flags: {expose: true}, - - expose: { - dependencies: [referenceListProperty, thingDataProperty], - compute: ({ [referenceListProperty]: refs, [thingDataProperty]: thingData }) => ( - (refs && thingData - ? (refs - .map(ref => findFn(ref, thingData, {mode: 'quiet'})) - .filter(Boolean)) - : []) - ) - } - }), - - // Corresponding function for a single reference. - dynamicThingFromSingleReference: ( - singleReferenceProperty, - thingDataProperty, - findFn - ) => ({ - flags: {expose: true}, - - expose: { - dependencies: [singleReferenceProperty, thingDataProperty], - compute: ({ [singleReferenceProperty]: ref, [thingDataProperty]: thingData }) => ( - (ref && thingData ? findFn(ref, thingData, {mode: 'quiet'}) : null) - ) - } - }), - - // Corresponding dynamic property to contribsByRef, which takes the values - // in the provided property and searches the object's artistData for - // matching actual Artist objects. The computed structure has the same form - // as contribsByRef, but with Artist objects instead of string references: - // - // [ - // {who: (an Artist), what: 'Viola'}, - // {who: (an Artist), what: null}, - // ... - // ] - // - // Contributions whose "who" values don't match anything in artistData are - // filtered out. (So if the list is all empty, chances are that either the - // reference list is somehow messed up, or artistData isn't being provided - // properly.) - dynamicContribs: (contribsByRefProperty) => ({ - flags: {expose: true}, - expose: { - dependencies: ['artistData', contribsByRefProperty], - compute: ({ artistData, [contribsByRefProperty]: contribsByRef }) => ( - ((contribsByRef && artistData) - ? (contribsByRef - .map(({ who: ref, what }) => ({ - who: find.artist(ref, artistData), - what - })) - .filter(({ who }) => who)) - : []) - ) - } - }), - - // Dynamically inherit a contribution list from some other object, if it - // hasn't been overridden on this object. This is handy for solo albums - // where all tracks have the same artist, for example. - // - // Note: The arguments of this function aren't currently final! The final - // format will look more like (contribsByRef, parentContribsByRef), e.g. - // ('artistContribsByRef', '@album/artistContribsByRef'). - dynamicInheritContribs: ( - contribsByRefProperty, - parentContribsByRefProperty, - thingDataProperty, - findFn - ) => ({ - flags: {expose: true}, - expose: { - dependencies: [contribsByRefProperty, thingDataProperty, 'artistData'], - compute({ - [Thing.instance]: thing, - [contribsByRefProperty]: contribsByRef, - [thingDataProperty]: thingData, - artistData - }) { - if (!artistData) return []; - const refs = (contribsByRef ?? findFn(thing, thingData, {mode: 'quiet'})?.[parentContribsByRefProperty]); - if (!refs) return []; - return (refs - .map(({ who: ref, what }) => ({ - who: find.artist(ref, artistData), - what - })) - .filter(({ who }) => who)); - } - } - }), - - // Neat little shortcut for "reversing" the reference lists stored on other - // things - for example, tracks specify a "referenced tracks" property, and - // you would use this to compute a corresponding "referenced *by* tracks" - // property. Naturally, the passed ref list property is of the things in the - // wiki data provided, not the requesting Thing itself. - reverseReferenceList: (wikiDataProperty, referencerRefListProperty) => ({ - flags: {expose: true}, - - expose: { - dependencies: [wikiDataProperty], - - compute: ({ [wikiDataProperty]: wikiData, [Thing.instance]: thing }) => ( - (wikiData - ? wikiData.filter(t => t[referencerRefListProperty]?.includes(thing)) - : []) - ) - } - }), - - // Corresponding function for single references. Note that the return value - // is still a list - this is for matching all the objects whose single - // reference (in the given property) matches this Thing. - reverseSingleReference: (wikiDataProperty, referencerRefListProperty) => ({ - flags: {expose: true}, - - expose: { - dependencies: [wikiDataProperty], - - compute: ({ [wikiDataProperty]: wikiData, [Thing.instance]: thing }) => ( - wikiData?.filter(t => t[referencerRefListProperty] === thing)) - } - }), - - // General purpose wiki data constructor, for properties like artistData, - // trackData, etc. - wikiData: (thingClass) => ({ - flags: {update: true}, - update: { - validate: validateArrayItems(validateInstanceOf(thingClass)) - } - }), - - // This one's kinda tricky: it parses artist "references" from the - // commentary content, and finds the matching artist for each reference. - // This is mostly useful for credits and listings on artist pages. - commentatorArtists: () => ({ - flags: {expose: true}, - - expose: { - dependencies: ['artistData', 'commentary'], - - compute: ({ artistData, commentary }) => ( - (artistData && commentary - ? Array.from(new Set((Array - .from(commentary - .replace(/<\/?b>/g, '') - .matchAll(/<i>(?<who>.*?):<\/i>/g)) - .map(({ groups: {who} }) => find.artist(who, artistData, {mode: 'quiet'}))))) - : [])) - } - }), -}; - -// Get a reference to a thing (e.g. track:showtime-piano-refrain), using its -// constructor's [Thing.referenceType] as the prefix. This will throw an error -// if the thing's directory isn't yet provided/computable. -Thing.getReference = function(thing) { - if (!thing.constructor[Thing.referenceType]) - throw TypeError(`Passed Thing is ${thing.constructor.name}, which provides no [Thing.referenceType]`); - - if (!thing.directory) - throw TypeError(`Passed ${thing.constructor.name} is missing its directory`); - - return `${thing.constructor[Thing.referenceType]}:${thing.directory}`; -}; - -// Default custom inspect function, which may be overridden by Thing subclasses. -// This will be used when displaying aggregate errors and other in command-line -// logging - it's the place to provide information useful in identifying the -// Thing being presented. -Thing.prototype[inspect.custom] = function() { - const cname = this.constructor.name; - - return (this.name - ? `${cname} ${color.green(`"${this.name}"`)}` - : `${cname}`) + (this.directory - ? ` (${color.blue(Thing.getReference(this))})` - : ''); -}; - -// -> Album - -Album.propertyDescriptors = { - // Update & expose - - name: Thing.common.name('Unnamed Album'), - color: Thing.common.color(), - directory: Thing.common.directory(), - urls: Thing.common.urls(), - - date: Thing.common.simpleDate(), - trackArtDate: Thing.common.simpleDate(), - dateAddedToWiki: Thing.common.simpleDate(), - - coverArtDate: { - flags: {update: true, expose: true}, - - update: {validate: isDate}, - - expose: { - dependencies: ['date'], - transform: (coverArtDate, { date }) => coverArtDate ?? date ?? null - } - }, - - artistContribsByRef: Thing.common.contribsByRef(), - coverArtistContribsByRef: Thing.common.contribsByRef(), - trackCoverArtistContribsByRef: Thing.common.contribsByRef(), - wallpaperArtistContribsByRef: Thing.common.contribsByRef(), - bannerArtistContribsByRef: Thing.common.contribsByRef(), - - groupsByRef: Thing.common.referenceList(Group), - artTagsByRef: Thing.common.referenceList(ArtTag), - - trackGroups: { - flags: {update: true, expose: true}, - - update: { - validate: validateArrayItems(validateInstanceOf(TrackGroup)) - } - }, - - coverArtFileExtension: Thing.common.fileExtension('jpg'), - trackCoverArtFileExtension: Thing.common.fileExtension('jpg'), - - wallpaperStyle: Thing.common.simpleString(), - wallpaperFileExtension: Thing.common.fileExtension('jpg'), - - bannerStyle: Thing.common.simpleString(), - bannerFileExtension: Thing.common.fileExtension('jpg'), - bannerDimensions: { - flags: {update: true, expose: true}, - update: {validate: isDimensions} - }, - - hasCoverArt: Thing.common.flag(true), - hasTrackArt: Thing.common.flag(true), - hasTrackNumbers: Thing.common.flag(true), - isMajorRelease: Thing.common.flag(false), - isListedOnHomepage: Thing.common.flag(true), - - commentary: Thing.common.commentary(), - additionalFiles: Thing.common.additionalFiles(), - - // Update only - - artistData: Thing.common.wikiData(Artist), - artTagData: Thing.common.wikiData(ArtTag), - groupData: Thing.common.wikiData(Group), - trackData: Thing.common.wikiData(Track), - - // Expose only - - artistContribs: Thing.common.dynamicContribs('artistContribsByRef'), - coverArtistContribs: Thing.common.dynamicContribs('coverArtistContribsByRef'), - trackCoverArtistContribs: Thing.common.dynamicContribs('trackCoverArtistContribsByRef'), - wallpaperArtistContribs: Thing.common.dynamicContribs('wallpaperArtistContribsByRef'), - bannerArtistContribs: Thing.common.dynamicContribs('bannerArtistContribsByRef'), - - commentatorArtists: Thing.common.commentatorArtists(), - - tracks: { - flags: {expose: true}, - - expose: { - dependencies: ['trackGroups', 'trackData'], - compute: ({ trackGroups, trackData }) => ( - (trackGroups && trackData - ? (trackGroups - .flatMap(group => group.tracksByRef ?? []) - .map(ref => find.track(ref, trackData, {mode: 'quiet'})) - .filter(Boolean)) - : []) - ) - } - }, - - groups: Thing.common.dynamicThingsFromReferenceList('groupsByRef', 'groupData', find.group), - - artTags: Thing.common.dynamicThingsFromReferenceList('artTagsByRef', 'artTagData', find.artTag), -}; - -Album[S.serializeDescriptors] = { - name: S.id, - color: S.id, - directory: S.id, - urls: S.id, - - date: S.id, - coverArtDate: S.id, - trackArtDate: S.id, - dateAddedToWiki: S.id, - - artistContribs: S.toContribRefs, - coverArtistContribs: S.toContribRefs, - trackCoverArtistContribs: S.toContribRefs, - wallpaperArtistContribs: S.toContribRefs, - bannerArtistContribs: S.toContribRefs, - - coverArtFileExtension: S.id, - trackCoverArtFileExtension: S.id, - wallpaperStyle: S.id, - wallpaperFileExtension: S.id, - bannerStyle: S.id, - bannerFileExtension: S.id, - bannerDimensions: S.id, - - hasTrackArt: S.id, - isMajorRelease: S.id, - isListedOnHomepage: S.id, - - commentary: S.id, - additionalFiles: S.id, - - tracks: S.toRefs, - groups: S.toRefs, - artTags: S.toRefs, - commentatorArtists: S.toRefs, -}; - -TrackGroup.propertyDescriptors = { - // Update & expose - - name: Thing.common.name('Unnamed Track Group'), - - color: { - flags: {update: true, expose: true}, - - update: {validate: isColor}, - - expose: { - dependencies: ['album'], - - transform(color, { album }) { - return color ?? album?.color ?? null; - } - } - }, - - dateOriginallyReleased: Thing.common.simpleDate(), - - tracksByRef: Thing.common.referenceList(Track), - - isDefaultTrackGroup: Thing.common.flag(false), - - // Update only - - album: { - flags: {update: true}, - update: {validate: validateInstanceOf(Album)} - }, - - trackData: Thing.common.wikiData(Track), - - // Expose only - - tracks: { - flags: {expose: true}, - - expose: { - dependencies: ['tracksByRef', 'trackData'], - compute: ({ tracksByRef, trackData }) => ( - (tracksByRef && trackData - ? (tracksByRef - .map(ref => find.track(ref, trackData)) - .filter(Boolean)) - : []) - ) - } - }, - - startIndex: { - flags: {expose: true}, - - expose: { - dependencies: ['album'], - compute: ({ album, [TrackGroup.instance]: trackGroup }) => (album.trackGroups - .slice(0, album.trackGroups.indexOf(trackGroup)) - .reduce((acc, tg) => acc + tg.tracks.length, 0)) - } - }, -}; - -// -> Track - -// This is a quick utility function for now, since the same code is reused in -// several places. Ideally it wouldn't be - we'd just reuse the `album` property -// - but support for that hasn't been coded yet :P -Track.findAlbum = (track, albumData) => { - return albumData?.find(album => album.tracks.includes(track)); -}; - -// Another reused utility function. This one's logic is a bit more complicated. -Track.hasCoverArt = (track, albumData, coverArtistContribsByRef, hasCoverArt) => { - return ( - hasCoverArt ?? - (coverArtistContribsByRef?.length > 0 || null) ?? - Track.findAlbum(track, albumData)?.hasTrackArt ?? - true); -}; - -Track.propertyDescriptors = { - // Update & expose - - name: Thing.common.name('Unnamed Track'), - directory: Thing.common.directory(), - - duration: { - flags: {update: true, expose: true}, - update: {validate: isDuration} - }, - - urls: Thing.common.urls(), - dateFirstReleased: Thing.common.simpleDate(), - - hasURLs: Thing.common.flag(true), - - artistContribsByRef: Thing.common.contribsByRef(), - contributorContribsByRef: Thing.common.contribsByRef(), - coverArtistContribsByRef: Thing.common.contribsByRef(), - - referencedTracksByRef: Thing.common.referenceList(Track), - artTagsByRef: Thing.common.referenceList(ArtTag), - - hasCoverArt: { - flags: {update: true, expose: true}, - - update: {validate: isBoolean}, - - expose: { - dependencies: ['albumData', 'coverArtistContribsByRef'], - transform: (hasCoverArt, { albumData, coverArtistContribsByRef, [Track.instance]: track }) => ( - Track.hasCoverArt(track, albumData, coverArtistContribsByRef, hasCoverArt)) - } - }, - - coverArtFileExtension: { - flags: {update: true, expose: true}, - - update: {validate: isFileExtension}, - - expose: { - dependencies: ['albumData', 'coverArtistContribsByRef'], - transform: (coverArtFileExtension, { albumData, coverArtistContribsByRef, hasCoverArt, [Track.instance]: track }) => ( - coverArtFileExtension ?? - (Track.hasCoverArt(track, albumData, coverArtistContribsByRef, hasCoverArt) - ? Track.findAlbum(track, albumData)?.trackCoverArtFileExtension - : Track.findAlbum(track, albumData)?.coverArtFileExtension) ?? - 'jpg') - } - }, - - // Previously known as: (track).aka - originalReleaseTrackByRef: Thing.common.singleReference(Track), - - dataSourceAlbumByRef: Thing.common.singleReference(Album), - - commentary: Thing.common.commentary(), - lyrics: Thing.common.simpleString(), - additionalFiles: Thing.common.additionalFiles(), - - // Update only - - albumData: Thing.common.wikiData(Album), - artistData: Thing.common.wikiData(Artist), - artTagData: Thing.common.wikiData(ArtTag), - flashData: Thing.common.wikiData(Flash), - trackData: Thing.common.wikiData(Track), - - // Expose only - - commentatorArtists: Thing.common.commentatorArtists(), - - album: { - flags: {expose: true}, - - expose: { - dependencies: ['albumData'], - compute: ({ [Track.instance]: track, albumData }) => ( - albumData?.find(album => album.tracks.includes(track)) ?? null) - } - }, - - // Note - this is an internal property used only to help identify a track. - // It should not be assumed in general that the album and dataSourceAlbum match - // (i.e. a track may dynamically be moved from one album to another, at - // which point dataSourceAlbum refers to where it was originally from, and is - // not generally relevant information). It's also not guaranteed that - // dataSourceAlbum is available (depending on the Track creator to optionally - // provide dataSourceAlbumByRef). - dataSourceAlbum: Thing.common.dynamicThingFromSingleReference('dataSourceAlbumByRef', 'albumData', find.album), - - date: { - flags: {expose: true}, - - expose: { - dependencies: ['albumData', 'dateFirstReleased'], - compute: ({ albumData, dateFirstReleased, [Track.instance]: track }) => ( - dateFirstReleased ?? - Track.findAlbum(track, albumData)?.date ?? - null - ) - } - }, - - color: { - flags: {expose: true}, - - expose: { - dependencies: ['albumData'], - - compute: ({ albumData, [Track.instance]: track }) => ( - (Track.findAlbum(track, albumData)?.trackGroups - .find(tg => tg.tracks.includes(track))?.color) - ?? null - ) - } - }, - - coverArtDate: { - flags: {update: true, expose: true}, - - update: {validate: isDate}, - - expose: { - dependencies: ['albumData', 'dateFirstReleased'], - transform: (coverArtDate, { albumData, dateFirstReleased, [Track.instance]: track }) => ( - coverArtDate ?? - dateFirstReleased ?? - Track.findAlbum(track, albumData)?.trackArtDate ?? - Track.findAlbum(track, albumData)?.date ?? - null - ) - } - }, - - originalReleaseTrack: Thing.common.dynamicThingFromSingleReference('originalReleaseTrackByRef', 'trackData', find.track), - - otherReleases: { - flags: {expose: true}, - - expose: { - dependencies: ['originalReleaseTrackByRef', 'trackData'], - - compute: ({ originalReleaseTrackByRef: t1origRef, trackData, [Track.instance]: t1 }) => { - if (!trackData) { - return []; - } - - const t1orig = find.track(t1origRef, trackData); - - return [ - t1orig, - ...trackData.filter(t2 => { - const { originalReleaseTrack: t2orig } = t2; - return ( - t2 !== t1 && - t2orig && - (t2orig === t1orig || t2orig === t1) - ); - }) - ].filter(Boolean); - } - } - }, - - // Previously known as: (track).artists - artistContribs: Thing.common.dynamicInheritContribs('artistContribsByRef', 'artistContribsByRef', 'albumData', Track.findAlbum), - - // Previously known as: (track).contributors - contributorContribs: Thing.common.dynamicContribs('contributorContribsByRef'), - - // Previously known as: (track).coverArtists - coverArtistContribs: Thing.common.dynamicInheritContribs('coverArtistContribsByRef', 'trackCoverArtistContribsByRef', 'albumData', Track.findAlbum), - - // Previously known as: (track).references - referencedTracks: Thing.common.dynamicThingsFromReferenceList('referencedTracksByRef', 'trackData', find.track), - - // Specifically exclude re-releases from this list - while it's useful to - // get from a re-release to the tracks it references, re-releases aren't - // generally relevant from the perspective of the tracks being referenced. - // Filtering them from data here hides them from the corresponding field - // on the site (obviously), and has the bonus of not counting them when - // counting the number of times a track has been referenced, for use in - // the "Tracks - by Times Referenced" listing page (or other data - // processing). - referencedByTracks: { - flags: {expose: true}, - - expose: { - dependencies: ['trackData'], - - compute: ({ trackData, [Track.instance]: track }) => (trackData - ? (trackData - .filter(t => !t.originalReleaseTrack) - .filter(t => t.referencedTracks?.includes(track))) - : []) - } - }, - - // Previously known as: (track).flashes - featuredInFlashes: Thing.common.reverseReferenceList('flashData', 'featuredTracks'), - - artTags: Thing.common.dynamicThingsFromReferenceList('artTagsByRef', 'artTagData', find.artTag), -}; - -Track.prototype[inspect.custom] = function() { - const base = Thing.prototype[inspect.custom].apply(this); - - const { album, dataSourceAlbum } = this; - const albumName = (album ? album.name : dataSourceAlbum?.name); - const albumIndex = albumName && (album ? album.tracks.indexOf(this) : dataSourceAlbum.tracks.indexOf(this)); - const trackNum = (albumIndex === -1 ? '#?' : `#${albumIndex + 1}`); - - return (albumName - ? base + ` (${color.yellow(trackNum)} in ${color.green(albumName)})` - : base); -}; - -// -> Artist - -Artist.filterByContrib = (thingDataProperty, contribsProperty) => ({ - flags: {expose: true}, - - expose: { - dependencies: [thingDataProperty], - - compute: ({ [thingDataProperty]: thingData, [Artist.instance]: artist }) => ( - thingData?.filter(({ [contribsProperty]: contribs }) => ( - contribs?.some(contrib => contrib.who === artist)))) - } -}); - -Artist.propertyDescriptors = { - // Update & expose - - name: Thing.common.name('Unnamed Artist'), - directory: Thing.common.directory(), - urls: Thing.common.urls(), - contextNotes: Thing.common.simpleString(), - - hasAvatar: Thing.common.flag(false), - avatarFileExtension: Thing.common.fileExtension('jpg'), - - aliasNames: { - flags: {update: true, expose: true}, - update: { - validate: validateArrayItems(isName) - } - }, - - isAlias: Thing.common.flag(), - aliasedArtistRef: Thing.common.singleReference(Artist), - - // Update only - - albumData: Thing.common.wikiData(Album), - artistData: Thing.common.wikiData(Artist), - flashData: Thing.common.wikiData(Flash), - trackData: Thing.common.wikiData(Track), - - // Expose only - - aliasedArtist: { - flags: {expose: true}, - - expose: { - dependencies: ['artistData', 'aliasedArtistRef'], - compute: ({ artistData, aliasedArtistRef }) => ( - (aliasedArtistRef && artistData - ? find.artist(aliasedArtistRef, artistData, {mode: 'quiet'}) - : null) - ) - } - }, - - tracksAsArtist: Artist.filterByContrib('trackData', 'artistContribs'), - tracksAsContributor: Artist.filterByContrib('trackData', 'contributorContribs'), - tracksAsCoverArtist: Artist.filterByContrib('trackData', 'coverArtistContribs'), - - tracksAsAny: { - flags: {expose: true}, - - expose: { - dependencies: ['trackData'], - - compute: ({ trackData, [Artist.instance]: artist }) => ( - trackData?.filter(track => ( - [ - ...track.artistContribs, - ...track.contributorContribs, - ...track.coverArtistContribs - ].some(({ who }) => who === artist)))) - } - }, - - tracksAsCommentator: { - flags: {expose: true}, - - expose: { - dependencies: ['trackData'], - - compute: ({ trackData, [Artist.instance]: artist }) => ( - trackData.filter(({ commentatorArtists }) => commentatorArtists?.includes(artist))) - } - }, - - albumsAsAlbumArtist: Artist.filterByContrib('albumData', 'artistContribs'), - albumsAsCoverArtist: Artist.filterByContrib('albumData', 'coverArtistContribs'), - albumsAsWallpaperArtist: Artist.filterByContrib('albumData', 'wallpaperArtistContribs'), - albumsAsBannerArtist: Artist.filterByContrib('albumData', 'bannerArtistContribs'), - - albumsAsCommentator: { - flags: {expose: true}, - - expose: { - dependencies: ['albumData'], - - compute: ({ albumData, [Artist.instance]: artist }) => ( - albumData.filter(({ commentatorArtists }) => commentatorArtists?.includes(artist))) - } - }, - - flashesAsContributor: Artist.filterByContrib('flashData', 'contributorContribs'), -}; - -Artist[S.serializeDescriptors] = { - name: S.id, - directory: S.id, - urls: S.id, - contextNotes: S.id, - - hasAvatar: S.id, - avatarFileExtension: S.id, - - aliasNames: S.id, - - tracksAsArtist: S.toRefs, - tracksAsContributor: S.toRefs, - tracksAsCoverArtist: S.toRefs, - tracksAsCommentator: S.toRefs, - - albumsAsAlbumArtist: S.toRefs, - albumsAsCoverArtist: S.toRefs, - albumsAsWallpaperArtist: S.toRefs, - albumsAsBannerArtist: S.toRefs, - albumsAsCommentator: S.toRefs, - - flashesAsContributor: S.toRefs, -}; - -// -> Group - -Group.propertyDescriptors = { - // Update & expose - - name: Thing.common.name('Unnamed Group'), - directory: Thing.common.directory(), - - description: Thing.common.simpleString(), - - urls: Thing.common.urls(), - - // Update only - - albumData: Thing.common.wikiData(Album), - groupCategoryData: Thing.common.wikiData(GroupCategory), - - // Expose only - - descriptionShort: { - flags: {expose: true}, - - expose: { - dependencies: ['description'], - compute: ({ description }) => description.split('<hr class="split">')[0] - } - }, - - albums: { - flags: {expose: true}, - - expose: { - dependencies: ['albumData'], - compute: ({ albumData, [Group.instance]: group }) => ( - albumData?.filter(album => album.groups.includes(group)) ?? []) - } - }, - - color: { - flags: {expose: true}, - - expose: { - dependencies: ['groupCategoryData'], - - compute: ({ groupCategoryData, [Group.instance]: group }) => ( - groupCategoryData.find(category => category.groups.includes(group))?.color ?? null) - } - }, - - category: { - flags: {expose: true}, - - expose: { - dependencies: ['groupCategoryData'], - compute: ({ groupCategoryData, [Group.instance]: group }) => ( - groupCategoryData.find(category => category.groups.includes(group)) ?? null) - } - }, -}; - -GroupCategory.propertyDescriptors = { - // Update & expose - - name: Thing.common.name('Unnamed Group Category'), - color: Thing.common.color(), - - groupsByRef: Thing.common.referenceList(Group), - - // Update only - - groupData: Thing.common.wikiData(Group), - - // Expose only - - groups: Thing.common.dynamicThingsFromReferenceList('groupsByRef', 'groupData', find.group), -}; - -// -> ArtTag - -ArtTag.propertyDescriptors = { - // Update & expose - - name: Thing.common.name('Unnamed Art Tag'), - directory: Thing.common.directory(), - color: Thing.common.color(), - isContentWarning: Thing.common.flag(false), - - // Update only - - albumData: Thing.common.wikiData(Album), - trackData: Thing.common.wikiData(Track), - - // Expose only - - // Previously known as: (tag).things - taggedInThings: { - flags: {expose: true}, - - expose: { - dependencies: ['albumData', 'trackData'], - compute: ({ albumData, trackData, [ArtTag.instance]: artTag }) => ( - sortByArtDate([...albumData, ...trackData] - .filter(thing => thing.artTags?.includes(artTag)))) - } - } -}; - -// -> NewsEntry - -NewsEntry.propertyDescriptors = { - // Update & expose - - name: Thing.common.name('Unnamed News Entry'), - directory: Thing.common.directory(), - date: Thing.common.simpleDate(), - - content: Thing.common.simpleString(), - - // Expose only - - contentShort: { - flags: {expose: true}, - - expose: { - dependencies: ['content'], - - compute: ({ content }) => content.split('<hr class="split">')[0] - } - }, -}; - -// -> StaticPage - -StaticPage.propertyDescriptors = { - // Update & expose - - name: Thing.common.name('Unnamed Static Page'), - - nameShort: { - flags: {update: true, expose: true}, - update: {validate: isName}, - - expose: { - dependencies: ['name'], - transform: (value, { name }) => value ?? name - } - }, - - directory: Thing.common.directory(), - content: Thing.common.simpleString(), - stylesheet: Thing.common.simpleString(), - showInNavigationBar: Thing.common.flag(true), -}; - -// -> HomepageLayout - -HomepageLayout.propertyDescriptors = { - // Update & expose - - sidebarContent: Thing.common.simpleString(), - - rows: { - flags: {update: true, expose: true}, - - update: { - validate: validateArrayItems(validateInstanceOf(HomepageLayoutRow)) - } - }, -}; - -HomepageLayoutRow.propertyDescriptors = { - // Update & expose - - name: Thing.common.name('Unnamed Homepage Row'), - - type: { - flags: {update: true, expose: true}, - - update: { - validate(value) { - throw new Error(`'type' property validator must be overridden`); - } - } - }, - - color: Thing.common.color(), - - // Update only - - // These aren't necessarily used by every HomepageLayoutRow subclass, but - // for convenience of providing this data, every row accepts all wiki data - // arrays depended upon by any subclass's behavior. - albumData: Thing.common.wikiData(Album), - groupData: Thing.common.wikiData(Group), -}; - -HomepageLayoutAlbumsRow.propertyDescriptors = { - ...HomepageLayoutRow.propertyDescriptors, - - // Update & expose - - type: { - flags: {update: true, expose: true}, - update: { - validate(value) { - if (value !== 'albums') { - throw new TypeError(`Expected 'albums'`); - } - - return true; - } - } - }, - - sourceGroupByRef: Thing.common.singleReference(Group), - sourceAlbumsByRef: Thing.common.referenceList(Album), - - countAlbumsFromGroup: { - flags: {update: true, expose: true}, - update: {validate: isCountingNumber} - }, - - actionLinks: { - flags: {update: true, expose: true}, - update: {validate: validateArrayItems(isString)} - }, - - // Expose only - - sourceGroup: Thing.common.dynamicThingFromSingleReference('sourceGroupByRef', 'groupData', find.group), - sourceAlbums: Thing.common.dynamicThingsFromReferenceList('sourceAlbumsByRef', 'albumData', find.album), -}; - -// -> Flash - -Flash.propertyDescriptors = { - // Update & expose - - name: Thing.common.name('Unnamed Flash'), - - directory: { - flags: {update: true, expose: true}, - update: {validate: isDirectory}, - - // Flashes expose directory differently from other Things! Their - // default directory is dependent on the page number (or ID), not - // the name. - expose: { - dependencies: ['page'], - transform(directory, { page }) { - if (directory === null && page === null) - return null; - else if (directory === null) - return page; - else - return directory; - } - } - }, - - page: { - flags: {update: true, expose: true}, - update: {validate: oneOf(isString, isNumber)}, - - expose: { - transform: value => (value === null ? null : value.toString()) - } - }, - - date: Thing.common.simpleDate(), - - coverArtFileExtension: Thing.common.fileExtension('jpg'), - - contributorContribsByRef: Thing.common.contribsByRef(), - - featuredTracksByRef: Thing.common.referenceList(Track), - - urls: Thing.common.urls(), - - // Update only - - artistData: Thing.common.wikiData(Artist), - trackData: Thing.common.wikiData(Track), - flashActData: Thing.common.wikiData(FlashAct), - - // Expose only - - contributorContribs: Thing.common.dynamicContribs('contributorContribsByRef'), - - featuredTracks: Thing.common.dynamicThingsFromReferenceList('featuredTracksByRef', 'trackData', find.track), - - act: { - flags: {expose: true}, - - expose: { - dependencies: ['flashActData'], - - compute: ({ flashActData, [Flash.instance]: flash }) => ( - flashActData.find(act => act.flashes.includes(flash)) ?? null) - } - }, - - color: { - flags: {expose: true}, - - expose: { - dependencies: ['flashActData'], - - compute: ({ flashActData, [Flash.instance]: flash }) => ( - flashActData.find(act => act.flashes.includes(flash))?.color ?? null) - } - }, -}; - -Flash[S.serializeDescriptors] = { - name: S.id, - page: S.id, - directory: S.id, - date: S.id, - contributors: S.toContribRefs, - tracks: S.toRefs, - urls: S.id, - color: S.id, -}; - -FlashAct.propertyDescriptors = { - // Update & expose - - name: Thing.common.name('Unnamed Flash Act'), - color: Thing.common.color(), - anchor: Thing.common.simpleString(), - jump: Thing.common.simpleString(), - jumpColor: Thing.common.color(), - - flashesByRef: Thing.common.referenceList(Flash), - - // Update only - - flashData: Thing.common.wikiData(Flash), - - // Expose only - - flashes: Thing.common.dynamicThingsFromReferenceList('flashesByRef', 'flashData', find.flash), -}; - -// -> WikiInfo - -WikiInfo.propertyDescriptors = { - // Update & expose - - name: Thing.common.name('Unnamed Wiki'), - - // Displayed in nav bar. - nameShort: { - flags: {update: true, expose: true}, - update: {validate: isName}, - - expose: { - dependencies: ['name'], - transform: (value, { name }) => value ?? name - } - }, - - color: Thing.common.color(), - - // One-line description used for <meta rel="description"> tag. - description: Thing.common.simpleString(), - - footerContent: Thing.common.simpleString(), - - defaultLanguage: { - flags: {update: true, expose: true}, - update: {validate: isLanguageCode} - }, - - canonicalBase: { - flags: {update: true, expose: true}, - update: {validate: isURL} - }, - - divideTrackListsByGroupsByRef: Thing.common.referenceList(Group), - - // Feature toggles - enableFlashesAndGames: Thing.common.flag(false), - enableListings: Thing.common.flag(false), - enableNews: Thing.common.flag(false), - enableArtTagUI: Thing.common.flag(false), - enableGroupUI: Thing.common.flag(false), - - // Update only - - groupData: Thing.common.wikiData(Group), - - // Expose only - - divideTrackListsByGroups: Thing.common.dynamicThingsFromReferenceList('divideTrackListsByGroupsByRef', 'groupData', find.group), -}; - -// -> Language - -const intlHelper = (constructor, opts) => ({ - flags: {expose: true}, - expose: { - dependencies: ['code', 'intlCode'], - compute: ({ code, intlCode }) => { - const constructCode = intlCode ?? code; - if (!constructCode) return null; - return Reflect.construct(constructor, [constructCode, opts]); - } - } -}); - -Language.propertyDescriptors = { - // Update & expose - - // General language code. This is used to identify the language distinctly - // from other languages (similar to how "Directory" operates in many data - // objects). - code: { - flags: {update: true, expose: true}, - update: {validate: isLanguageCode} - }, - - // Human-readable name. This should be the language's own native name, not - // localized to any other language. - name: Thing.common.simpleString(), - - // Language code specific to JavaScript's Internationalization (Intl) API. - // Usually this will be the same as the language's general code, but it - // may be overridden to provide Intl constructors an alternative value. - intlCode: { - flags: {update: true, expose: true}, - update: {validate: isLanguageCode}, - expose: { - dependencies: ['code'], - transform: (intlCode, { code }) => intlCode ?? code - } - }, - - // Flag which represents whether or not to hide a language from general - // access. If a language is hidden, its portion of the website will still - // be built (with all strings localized to the language), but it won't be - // included in controls for switching languages or the <link rel=alternate> - // tags used for search engine optimization. This flag is intended for use - // with languages that are currently in development and not ready for - // formal release, or which are just kept hidden as "experimental zones" - // for wiki development or content testing. - hidden: Thing.common.flag(false), - - // Mapping of translation keys to values (strings). Generally, don't - // access this object directly - use methods instead. - strings: { - flags: {update: true, expose: true}, - update: {validate: t => typeof t === 'object'}, - expose: { - dependencies: ['inheritedStrings'], - transform(strings, { inheritedStrings }) { - if (strings || inheritedStrings) { - return {...inheritedStrings ?? {}, ...strings ?? {}}; - } else { - return null; - } - } - } - }, - - // May be provided to specify "default" strings, generally (but not - // necessarily) inherited from another Language object. - inheritedStrings: { - flags: {update: true, expose: true}, - update: {validate: t => typeof t === 'object'} - }, - - // Update only - - escapeHTML: Thing.common.externalFunction(), - - // Expose only - - intl_date: intlHelper(Intl.DateTimeFormat, {full: true}), - intl_number: intlHelper(Intl.NumberFormat), - intl_listConjunction: intlHelper(Intl.ListFormat, {type: 'conjunction'}), - intl_listDisjunction: intlHelper(Intl.ListFormat, {type: 'disjunction'}), - intl_listUnit: intlHelper(Intl.ListFormat, {type: 'unit'}), - intl_pluralCardinal: intlHelper(Intl.PluralRules, {type: 'cardinal'}), - intl_pluralOrdinal: intlHelper(Intl.PluralRules, {type: 'ordinal'}), - - validKeys: { - flags: {expose: true}, - - expose: { - dependencies: ['strings', 'inheritedStrings'], - compute: ({ strings, inheritedStrings }) => Array.from(new Set([ - ...Object.keys(inheritedStrings ?? {}), - ...Object.keys(strings ?? {}) - ])) - } - }, - - strings_htmlEscaped: { - flags: {expose: true}, - expose: { - dependencies: ['strings', 'inheritedStrings', 'escapeHTML'], - compute({ strings, inheritedStrings, escapeHTML }) { - if (!(strings || inheritedStrings) || !escapeHTML) return null; - const allStrings = {...inheritedStrings ?? {}, ...strings ?? {}}; - return Object.fromEntries(Object.entries(allStrings) - .map(([ k, v ]) => [k, escapeHTML(v)])); - } - } - }, -}; - -const countHelper = (stringKey, argName = stringKey) => function(value, {unit = false} = {}) { - return this.$( - (unit - ? `count.${stringKey}.withUnit.` + this.getUnitForm(value) - : `count.${stringKey}`), - {[argName]: this.formatNumber(value)}); -}; - -Object.assign(Language.prototype, { - $(key, args = {}) { - return this.formatString(key, args); - }, - - assertIntlAvailable(property) { - if (!this[property]) { - throw new Error(`Intl API ${property} unavailable`); - } - }, - - getUnitForm(value) { - this.assertIntlAvailable('intl_pluralCardinal'); - return this.intl_pluralCardinal.select(value); - }, - - formatString(key, args = {}) { - if (this.strings && !this.strings_htmlEscaped) { - throw new Error(`HTML-escaped strings unavailable - please ensure escapeHTML function is provided`); - } - - return this.formatStringHelper(this.strings_htmlEscaped, key, args); - }, - - formatStringNoHTMLEscape(key, args = {}) { - return this.formatStringHelper(this.strings, key, args); - }, - - formatStringHelper(strings, key, args = {}) { - if (!strings) { - throw new Error(`Strings unavailable`); - } - - if (!this.validKeys.includes(key)) { - throw new Error(`Invalid key ${key} accessed`); - } - - const template = strings[key]; - - // Convert the keys on the args dict from camelCase to CONSTANT_CASE. - // (This isn't an OUTRAGEOUSLY versatile algorithm for doing that, 8ut - // like, who cares, dude?) Also, this is an array, 8ecause it's handy - // for the iterating we're a8out to do. - const processedArgs = Object.entries(args) - .map(([ k, v ]) => [k.replace(/[A-Z]/g, '_$&').toUpperCase(), v]); - - // Replacement time! Woot. Reduce comes in handy here! - const output = processedArgs.reduce( - (x, [ k, v ]) => x.replaceAll(`{${k}}`, v), - template); - - // Post-processing: if any expected arguments *weren't* replaced, that - // is almost definitely an error. - if (output.match(/\{[A-Z_]+\}/)) { - throw new Error(`Args in ${key} were missing - output: ${output}`); - } - - return output; - }, - - formatDate(date) { - this.assertIntlAvailable('intl_date'); - return this.intl_date.format(date); - }, - - formatDateRange(startDate, endDate) { - this.assertIntlAvailable('intl_date'); - return this.intl_date.formatRange(startDate, endDate); - }, - - formatDuration(secTotal, {approximate = false, unit = false} = {}) { - if (secTotal === 0) { - return this.formatString('count.duration.missing'); - } - - const hour = Math.floor(secTotal / 3600); - const min = Math.floor((secTotal - hour * 3600) / 60); - const sec = Math.floor(secTotal - hour * 3600 - min * 60); - - const pad = val => val.toString().padStart(2, '0'); - - const stringSubkey = unit ? '.withUnit' : ''; - - const duration = (hour > 0 - ? this.formatString('count.duration.hours' + stringSubkey, { - hours: hour, - minutes: pad(min), - seconds: pad(sec) - }) - : this.formatString('count.duration.minutes' + stringSubkey, { - minutes: min, - seconds: pad(sec) - })); - - return (approximate - ? this.formatString('count.duration.approximate', {duration}) - : duration); - }, - - formatIndex(value) { - this.assertIntlAvailable('intl_pluralOrdinal'); - return this.formatString('count.index.' + this.intl_pluralOrdinal.select(value), {index: value}); - }, - - formatNumber(value) { - this.assertIntlAvailable('intl_number'); - return this.intl_number.format(value); - }, - - formatWordCount(value) { - const num = this.formatNumber(value > 1000 - ? Math.floor(value / 100) / 10 - : value); - - const words = (value > 1000 - ? this.formatString('count.words.thousand', {words: num}) - : this.formatString('count.words', {words: num})); - - return this.formatString('count.words.withUnit.' + this.getUnitForm(value), {words}); - }, - - // Conjunction list: A, B, and C - formatConjunctionList(array) { - this.assertIntlAvailable('intl_listConjunction'); - return this.intl_listConjunction.format(array); - }, - - // Disjunction lists: A, B, or C - formatDisjunctionList(array) { - this.assertIntlAvailable('intl_listDisjunction'); - return this.intl_listDisjunction.format(array); - }, - - // Unit lists: A, B, C - formatUnitList(array) { - this.assertIntlAvailable('intl_listUnit'); - return this.intl_listUnit.format(array); - }, - - // File sizes: 42.5 kB, 127.2 MB, 4.13 GB, 998.82 TB - formatFileSize(bytes) { - if (!bytes) return ''; - - bytes = parseInt(bytes); - if (isNaN(bytes)) return ''; - - const round = exp => Math.round(bytes / 10 ** (exp - 1)) / 10; - - if (bytes >= 10 ** 12) { - return this.formatString('count.fileSize.terabytes', {terabytes: round(12)}); - } else if (bytes >= 10 ** 9) { - return this.formatString('count.fileSize.gigabytes', {gigabytes: round(9)}); - } else if (bytes >= 10 ** 6) { - return this.formatString('count.fileSize.megabytes', {megabytes: round(6)}); - } else if (bytes >= 10 ** 3) { - return this.formatString('count.fileSize.kilobytes', {kilobytes: round(3)}); - } else { - return this.formatString('count.fileSize.bytes', {bytes}); - } - }, - - // TODO: These are hard-coded. Is there a better way? - countAlbums: countHelper('albums'), - countCommentaryEntries: countHelper('commentaryEntries', 'entries'), - countContributions: countHelper('contributions'), - countCoverArts: countHelper('coverArts'), - countTimesReferenced: countHelper('timesReferenced'), - countTimesUsed: countHelper('timesUsed'), - countTracks: countHelper('tracks'), -}); diff --git a/src/data/things/album.js b/src/data/things/album.js new file mode 100644 index 0000000..40cd463 --- /dev/null +++ b/src/data/things/album.js @@ -0,0 +1,434 @@ +export const DATA_ALBUM_DIRECTORY = 'album'; + +import * as path from 'node:path'; + +import {input} from '#composite'; +import find from '#find'; +import {traverse} from '#node-utils'; +import {sortAlbumsTracksChronologically, sortChronologically} from '#sort'; +import {empty} from '#sugar'; +import Thing from '#thing'; +import {isDate} from '#validators'; +import {parseAdditionalFiles, parseContributors, parseDate, parseDimensions} + from '#yaml'; + +import {exposeDependency, exposeUpdateValueOrContinue} + from '#composite/control-flow'; +import {exitWithoutContribs} from '#composite/wiki-data'; + +import { + additionalFiles, + commentary, + color, + commentatorArtists, + contribsPresent, + contributionList, + dimensions, + directory, + fileExtension, + flag, + name, + referenceList, + simpleDate, + simpleString, + urls, + wikiData, +} from '#composite/wiki-properties'; + +import {withTracks, withTrackSections} from '#composite/things/album'; + +export class Album extends Thing { + static [Thing.referenceType] = 'album'; + + static [Thing.getPropertyDescriptors] = ({ArtTag, Artist, Group, Track}) => ({ + // Update & expose + + name: name('Unnamed Album'), + color: color(), + directory: directory(), + urls: urls(), + + bandcampAlbumIdentifier: simpleString(), + bandcampArtworkIdentifier: simpleString(), + + date: simpleDate(), + trackArtDate: simpleDate(), + dateAddedToWiki: simpleDate(), + + coverArtDate: [ + exitWithoutContribs({contribs: 'coverArtistContribs'}), + + exposeUpdateValueOrContinue({ + validate: input.value(isDate), + }), + + exposeDependency({dependency: 'date'}), + ], + + coverArtFileExtension: [ + exitWithoutContribs({contribs: 'coverArtistContribs'}), + fileExtension('jpg'), + ], + + trackCoverArtFileExtension: fileExtension('jpg'), + + wallpaperFileExtension: [ + exitWithoutContribs({contribs: 'wallpaperArtistContribs'}), + fileExtension('jpg'), + ], + + bannerFileExtension: [ + exitWithoutContribs({contribs: 'bannerArtistContribs'}), + fileExtension('jpg'), + ], + + wallpaperStyle: [ + exitWithoutContribs({contribs: 'wallpaperArtistContribs'}), + simpleString(), + ], + + bannerStyle: [ + exitWithoutContribs({contribs: 'bannerArtistContribs'}), + simpleString(), + ], + + coverArtDimensions: [ + exitWithoutContribs({contribs: 'coverArtistContribs'}), + dimensions(), + ], + + bannerDimensions: [ + exitWithoutContribs({contribs: 'bannerArtistContribs'}), + dimensions(), + ], + + hasTrackNumbers: flag(true), + isListedOnHomepage: flag(true), + isListedInGalleries: flag(true), + + commentary: commentary(), + additionalFiles: additionalFiles(), + + trackSections: [ + withTrackSections(), + exposeDependency({dependency: '#trackSections'}), + ], + + artistContribs: contributionList(), + coverArtistContribs: contributionList(), + trackCoverArtistContribs: contributionList(), + wallpaperArtistContribs: contributionList(), + bannerArtistContribs: contributionList(), + + groups: referenceList({ + class: input.value(Group), + find: input.value(find.group), + data: 'groupData', + }), + + artTags: [ + exitWithoutContribs({ + contribs: 'coverArtistContribs', + value: input.value([]), + }), + + referenceList({ + class: input.value(ArtTag), + find: input.value(find.artTag), + data: 'artTagData', + }), + ], + + // Update only + + artistData: wikiData({ + class: input.value(Artist), + }), + + artTagData: wikiData({ + class: input.value(ArtTag), + }), + + groupData: wikiData({ + class: input.value(Group), + }), + + // Only the tracks which belong to this album. + // Necessary for computing the track list, so provide this statically + // or keep it updated. + ownTrackData: wikiData({ + class: input.value(Track), + }), + + // Expose only + + commentatorArtists: commentatorArtists(), + + hasCoverArt: contribsPresent({contribs: 'coverArtistContribs'}), + hasWallpaperArt: contribsPresent({contribs: 'wallpaperArtistContribs'}), + hasBannerArt: contribsPresent({contribs: 'bannerArtistContribs'}), + + tracks: [ + withTracks(), + exposeDependency({dependency: '#tracks'}), + ], + }); + + static [Thing.getSerializeDescriptors] = ({ + serialize: S, + }) => ({ + name: S.id, + color: S.id, + directory: S.id, + urls: S.id, + + date: S.id, + coverArtDate: S.id, + trackArtDate: S.id, + dateAddedToWiki: S.id, + + artistContribs: S.toContribRefs, + coverArtistContribs: S.toContribRefs, + trackCoverArtistContribs: S.toContribRefs, + wallpaperArtistContribs: S.toContribRefs, + bannerArtistContribs: S.toContribRefs, + + coverArtFileExtension: S.id, + trackCoverArtFileExtension: S.id, + wallpaperStyle: S.id, + wallpaperFileExtension: S.id, + bannerStyle: S.id, + bannerFileExtension: S.id, + bannerDimensions: S.id, + + hasTrackArt: S.id, + isListedOnHomepage: S.id, + + commentary: S.toCommentaryRefs, + + additionalFiles: S.id, + + tracks: S.toRefs, + groups: S.toRefs, + artTags: S.toRefs, + commentatorArtists: S.toRefs, + }); + + static [Thing.findSpecs] = { + album: { + referenceTypes: ['album', 'album-commentary', 'album-gallery'], + bindTo: 'albumData', + }, + }; + + static [Thing.yamlDocumentSpec] = { + fields: { + 'Album': {property: 'name'}, + 'Directory': {property: 'directory'}, + + 'Bandcamp Album ID': { + property: 'bandcampAlbumIdentifier', + transform: String, + }, + + 'Bandcamp Artwork ID': { + property: 'bandcampArtworkIdentifier', + transform: String, + }, + + 'Date': { + property: 'date', + transform: parseDate, + }, + + 'Color': {property: 'color'}, + 'URLs': {property: 'urls'}, + + 'Has Track Numbers': {property: 'hasTrackNumbers'}, + 'Listed on Homepage': {property: 'isListedOnHomepage'}, + 'Listed in Galleries': {property: 'isListedInGalleries'}, + + 'Cover Art Date': { + property: 'coverArtDate', + transform: parseDate, + }, + + 'Default Track Cover Art Date': { + property: 'trackArtDate', + transform: parseDate, + }, + + 'Date Added': { + property: 'dateAddedToWiki', + transform: parseDate, + }, + + 'Cover Art File Extension': {property: 'coverArtFileExtension'}, + 'Track Art File Extension': {property: 'trackCoverArtFileExtension'}, + + 'Cover Art Dimensions': { + property: 'coverArtDimensions', + transform: parseDimensions, + }, + + 'Wallpaper Artists': { + property: 'wallpaperArtistContribs', + transform: parseContributors, + }, + + 'Wallpaper Style': {property: 'wallpaperStyle'}, + 'Wallpaper File Extension': {property: 'wallpaperFileExtension'}, + + 'Banner Artists': { + property: 'bannerArtistContribs', + transform: parseContributors, + }, + + 'Banner Style': {property: 'bannerStyle'}, + 'Banner File Extension': {property: 'bannerFileExtension'}, + + 'Banner Dimensions': { + property: 'bannerDimensions', + transform: parseDimensions, + }, + + 'Commentary': {property: 'commentary'}, + + 'Additional Files': { + property: 'additionalFiles', + transform: parseAdditionalFiles, + }, + + 'Franchises': {ignore: true}, + + 'Artists': { + property: 'artistContribs', + transform: parseContributors, + }, + + 'Cover Artists': { + property: 'coverArtistContribs', + transform: parseContributors, + }, + + 'Default Track Cover Artists': { + property: 'trackCoverArtistContribs', + transform: parseContributors, + }, + + 'Groups': {property: 'groups'}, + 'Art Tags': {property: 'artTags'}, + + 'Review Points': {ignore: true}, + }, + }; + + static [Thing.getYamlLoadingSpec] = ({ + documentModes: {headerAndEntries}, + thingConstructors: {Album, Track, TrackSectionHelper}, + }) => ({ + title: `Process album files`, + + files: dataPath => + traverse(path.join(dataPath, DATA_ALBUM_DIRECTORY), { + filterFile: name => path.extname(name) === '.yaml', + prefixPath: DATA_ALBUM_DIRECTORY, + }), + + documentMode: headerAndEntries, + headerDocumentThing: Album, + entryDocumentThing: document => + ('Section' in document + ? TrackSectionHelper + : Track), + + save(results) { + const albumData = []; + const trackData = []; + + for (const {header: album, entries} of results) { + // We can't mutate an array once it's set as a property value, + // so prepare the track sections that will show up in a track list + // all the way before actually applying them. (It's okay to mutate + // an individual section before applying it, since those are just + // generic objects; they aren't Things in and of themselves.) + const trackSections = []; + const ownTrackData = []; + + let currentTrackSection = { + name: `Default Track Section`, + isDefaultTrackSection: true, + tracks: [], + }; + + const albumRef = Thing.getReference(album); + + const closeCurrentTrackSection = () => { + if (!empty(currentTrackSection.tracks)) { + trackSections.push(currentTrackSection); + } + }; + + for (const entry of entries) { + if (entry instanceof TrackSectionHelper) { + closeCurrentTrackSection(); + + currentTrackSection = { + name: entry.name, + color: entry.color, + dateOriginallyReleased: entry.dateOriginallyReleased, + isDefaultTrackSection: false, + tracks: [], + }; + + continue; + } + + trackData.push(entry); + + entry.dataSourceAlbum = albumRef; + + ownTrackData.push(entry); + currentTrackSection.tracks.push(Thing.getReference(entry)); + } + + closeCurrentTrackSection(); + + albumData.push(album); + + album.trackSections = trackSections; + album.ownTrackData = ownTrackData; + } + + return {albumData, trackData}; + }, + + sort({albumData, trackData}) { + sortChronologically(albumData); + sortAlbumsTracksChronologically(trackData); + }, + }); +} + +export class TrackSectionHelper extends Thing { + static [Thing.friendlyName] = `Track Section`; + + static [Thing.getPropertyDescriptors] = () => ({ + name: name('Unnamed Track Section'), + color: color(), + dateOriginallyReleased: simpleDate(), + isDefaultTrackGroup: flag(false), + }) + + static [Thing.yamlDocumentSpec] = { + fields: { + 'Section': {property: 'name'}, + 'Color': {property: 'color'}, + + 'Date Originally Released': { + property: 'dateOriginallyReleased', + transform: parseDate, + }, + }, + }; +} diff --git a/src/data/things/art-tag.js b/src/data/things/art-tag.js new file mode 100644 index 0000000..3149b31 --- /dev/null +++ b/src/data/things/art-tag.js @@ -0,0 +1,107 @@ +export const ART_TAG_DATA_FILE = 'tags.yaml'; + +import {input} from '#composite'; +import {sortAlphabetically, sortAlbumsTracksChronologically} from '#sort'; +import Thing from '#thing'; +import {isName} from '#validators'; + +import {exposeUpdateValueOrContinue} from '#composite/control-flow'; + +import { + color, + directory, + flag, + name, + wikiData, +} from '#composite/wiki-properties'; + +export class ArtTag extends Thing { + static [Thing.referenceType] = 'tag'; + static [Thing.friendlyName] = `Art Tag`; + + static [Thing.getPropertyDescriptors] = ({Album, Track}) => ({ + // Update & expose + + name: name('Unnamed Art Tag'), + directory: directory(), + color: color(), + isContentWarning: flag(false), + + nameShort: [ + exposeUpdateValueOrContinue({ + validate: input.value(isName), + }), + + { + dependencies: ['name'], + compute: ({name}) => + name.replace(/ \([^)]*?\)$/, ''), + }, + ], + + // Update only + + albumData: wikiData({ + class: input.value(Album), + }), + + trackData: wikiData({ + class: input.value(Track), + }), + + // Expose only + + taggedInThings: { + flags: {expose: true}, + + expose: { + dependencies: ['this', 'albumData', 'trackData'], + compute: ({this: artTag, albumData, trackData}) => + sortAlbumsTracksChronologically( + [...albumData, ...trackData] + .filter(({artTags}) => artTags.includes(artTag)), + {getDate: thing => thing.coverArtDate ?? thing.date}), + }, + }, + }); + + static [Thing.findSpecs] = { + artTag: { + referenceTypes: ['tag'], + bindTo: 'artTagData', + + getMatchableNames: tag => + (tag.isContentWarning + ? [`cw: ${tag.name}`] + : [tag.name]), + }, + }; + + static [Thing.yamlDocumentSpec] = { + fields: { + 'Tag': {property: 'name'}, + 'Short Name': {property: 'nameShort'}, + 'Directory': {property: 'directory'}, + + 'Color': {property: 'color'}, + 'Is CW': {property: 'isContentWarning'}, + }, + }; + + static [Thing.getYamlLoadingSpec] = ({ + documentModes: {allInOne}, + thingConstructors: {ArtTag}, + }) => ({ + title: `Process art tags file`, + file: ART_TAG_DATA_FILE, + + documentMode: allInOne, + documentThing: ArtTag, + + save: (results) => ({artTagData: results}), + + sort({artTagData}) { + sortAlphabetically(artTagData); + }, + }); +} diff --git a/src/data/things/artist.js b/src/data/things/artist.js new file mode 100644 index 0000000..841d652 --- /dev/null +++ b/src/data/things/artist.js @@ -0,0 +1,392 @@ +export const ARTIST_DATA_FILE = 'artists.yaml'; + +import {inspect} from 'node:util'; + +import CacheableObject from '#cacheable-object'; +import {colors} from '#cli'; +import {input} from '#composite'; +import find from '#find'; +import {sortAlphabetically} from '#sort'; +import {stitchArrays, unique} from '#sugar'; +import Thing from '#thing'; +import {isName, validateArrayItems} from '#validators'; +import {getKebabCase} from '#wiki-data'; + +import {withReverseContributionList} from '#composite/wiki-data'; + +import { + contentString, + directory, + fileExtension, + flag, + name, + reverseContributionList, + reverseReferenceList, + singleReference, + urls, + wikiData, +} from '#composite/wiki-properties'; + +export class Artist extends Thing { + static [Thing.referenceType] = 'artist'; + static [Thing.wikiDataArray] = 'artistData'; + + static [Thing.getPropertyDescriptors] = ({Album, Flash, Track}) => ({ + // Update & expose + + name: name('Unnamed Artist'), + directory: directory(), + urls: urls(), + + contextNotes: contentString(), + + hasAvatar: flag(false), + avatarFileExtension: fileExtension('jpg'), + + aliasNames: { + flags: {update: true, expose: true}, + update: {validate: validateArrayItems(isName)}, + expose: {transform: (names) => names ?? []}, + }, + + isAlias: flag(), + + aliasedArtist: singleReference({ + class: input.value(Artist), + find: input.value(find.artist), + data: 'artistData', + }), + + // Update only + + albumData: wikiData({ + class: input.value(Album), + }), + + artistData: wikiData({ + class: input.value(Artist), + }), + + flashData: wikiData({ + class: input.value(Flash), + }), + + trackData: wikiData({ + class: input.value(Track), + }), + + // Expose only + + tracksAsArtist: reverseContributionList({ + data: 'trackData', + list: input.value('artistContribs'), + }), + + tracksAsContributor: reverseContributionList({ + data: 'trackData', + list: input.value('contributorContribs'), + }), + + tracksAsCoverArtist: reverseContributionList({ + data: 'trackData', + list: input.value('coverArtistContribs'), + }), + + tracksAsAny: [ + withReverseContributionList({ + data: 'trackData', + list: input.value('artistContribs'), + }).outputs({ + '#reverseContributionList': '#tracksAsArtist', + }), + + withReverseContributionList({ + data: 'trackData', + list: input.value('contributorContribs'), + }).outputs({ + '#reverseContributionList': '#tracksAsContributor', + }), + + withReverseContributionList({ + data: 'trackData', + list: input.value('coverArtistContribs'), + }).outputs({ + '#reverseContributionList': '#tracksAsCoverArtist', + }), + + { + dependencies: [ + '#tracksAsArtist', + '#tracksAsContributor', + '#tracksAsCoverArtist', + ], + + compute: ({ + ['#tracksAsArtist']: tracksAsArtist, + ['#tracksAsContributor']: tracksAsContributor, + ['#tracksAsCoverArtist']: tracksAsCoverArtist, + }) => + unique([ + ...tracksAsArtist, + ...tracksAsContributor, + ...tracksAsCoverArtist, + ]), + }, + ], + + tracksAsCommentator: reverseReferenceList({ + data: 'trackData', + list: input.value('commentatorArtists'), + }), + + albumsAsAlbumArtist: reverseContributionList({ + data: 'albumData', + list: input.value('artistContribs'), + }), + + albumsAsCoverArtist: reverseContributionList({ + data: 'albumData', + list: input.value('coverArtistContribs'), + }), + + albumsAsWallpaperArtist: reverseContributionList({ + data: 'albumData', + list: input.value('wallpaperArtistContribs'), + }), + + albumsAsBannerArtist: reverseContributionList({ + data: 'albumData', + list: input.value('bannerArtistContribs'), + }), + + albumsAsAny: [ + withReverseContributionList({ + data: 'albumData', + list: input.value('artistContribs'), + }).outputs({ + '#reverseContributionList': '#albumsAsArtist', + }), + + withReverseContributionList({ + data: 'albumData', + list: input.value('coverArtistContribs'), + }).outputs({ + '#reverseContributionList': '#albumsAsCoverArtist', + }), + + withReverseContributionList({ + data: 'albumData', + list: input.value('wallpaperArtistContribs'), + }).outputs({ + '#reverseContributionList': '#albumsAsWallpaperArtist', + }), + + withReverseContributionList({ + data: 'albumData', + list: input.value('bannerArtistContribs'), + }).outputs({ + '#reverseContributionList': '#albumsAsBannerArtist', + }), + + { + dependencies: [ + '#albumsAsArtist', + '#albumsAsCoverArtist', + '#albumsAsWallpaperArtist', + '#albumsAsBannerArtist', + ], + + compute: ({ + ['#albumsAsArtist']: albumsAsArtist, + ['#albumsAsCoverArtist']: albumsAsCoverArtist, + ['#albumsAsWallpaperArtist']: albumsAsWallpaperArtist, + ['#albumsAsBannerArtist']: albumsAsBannerArtist, + }) => + unique([ + ...albumsAsArtist, + ...albumsAsCoverArtist, + ...albumsAsWallpaperArtist, + ...albumsAsBannerArtist, + ]), + }, + ], + + albumsAsCommentator: reverseReferenceList({ + data: 'albumData', + list: input.value('commentatorArtists'), + }), + + flashesAsContributor: reverseContributionList({ + data: 'flashData', + list: input.value('contributorContribs'), + }), + + flashesAsCommentator: reverseReferenceList({ + data: 'flashData', + list: input.value('commentatorArtists'), + }), + }); + + static [Thing.getSerializeDescriptors] = ({ + serialize: S, + }) => ({ + name: S.id, + directory: S.id, + urls: S.id, + contextNotes: S.id, + + hasAvatar: S.id, + avatarFileExtension: S.id, + + aliasNames: S.id, + + tracksAsArtist: S.toRefs, + tracksAsContributor: S.toRefs, + tracksAsCoverArtist: S.toRefs, + tracksAsCommentator: S.toRefs, + + albumsAsAlbumArtist: S.toRefs, + albumsAsCoverArtist: S.toRefs, + albumsAsWallpaperArtist: S.toRefs, + albumsAsBannerArtist: S.toRefs, + albumsAsCommentator: S.toRefs, + + flashesAsContributor: S.toRefs, + }); + + static [Thing.findSpecs] = { + artist: { + referenceTypes: ['artist', 'artist-gallery'], + bindTo: 'artistData', + + include: artist => !artist.isAlias, + }, + + artistAlias: { + referenceTypes: ['artist', 'artist-gallery'], + bindTo: 'artistData', + + include: artist => artist.isAlias, + + getMatchableDirectories(artist) { + const originalArtist = artist.aliasedArtist; + + // Aliases never match by the same directory as the original. + if (artist.directory === originalArtist.directory) { + return []; + } + + // Aliases never match by the same directory as some *previous* alias + // in the original's alias list. This is honestly a bit awkward, but it + // avoids artist aliases conflicting with each other when checking for + // duplicate directories. + for (const aliasName of originalArtist.aliasNames) { + // These are trouble. We should be accessing aliases' directories + // directly, but artists currently don't expose a reverse reference + // list for aliases. (This is pending a cleanup of "reverse reference" + // behavior in general.) It doesn't actually cause problems *here* + // because alias directories are computed from their names 100% of the + // time, but that *is* an assumption this code makes. + if (aliasName === artist.name) continue; + if (artist.directory === getKebabCase(aliasName)) { + return []; + } + } + + // And, aliases never return just a blank string. This part is pretty + // spooky because it doesn't handle two differently named aliases, on + // different artists, who have names that are similar *apart* from a + // character that's shortened. But that's also so fundamentally scary + // that we can't support it properly with existing code, anyway - we + // would need to be able to specifically set a directory *on an alias,* + // which currently can't be done in YAML data files. + if (artist.directory === '') { + return []; + } + + return [artist.directory]; + }, + }, + }; + + static [Thing.yamlDocumentSpec] = { + fields: { + 'Artist': {property: 'name'}, + 'Directory': {property: 'directory'}, + 'URLs': {property: 'urls'}, + 'Context Notes': {property: 'contextNotes'}, + + 'Has Avatar': {property: 'hasAvatar'}, + 'Avatar File Extension': {property: 'avatarFileExtension'}, + + 'Aliases': {property: 'aliasNames'}, + + 'Dead URLs': {ignore: true}, + + 'Review Points': {ignore: true}, + }, + }; + + static [Thing.getYamlLoadingSpec] = ({ + documentModes: {allInOne}, + thingConstructors: {Artist}, + }) => ({ + title: `Process artists file`, + file: ARTIST_DATA_FILE, + + documentMode: allInOne, + documentThing: Artist, + + save(results) { + const artists = results; + + const artistRefs = + artists.map(artist => Thing.getReference(artist)); + + const artistAliasNames = + artists.map(artist => artist.aliasNames); + + const artistAliases = + stitchArrays({ + originalArtistRef: artistRefs, + aliasNames: artistAliasNames, + }).flatMap(({originalArtistRef, aliasNames}) => + aliasNames.map(name => { + const alias = new Artist(); + alias.name = name; + alias.isAlias = true; + alias.aliasedArtist = originalArtistRef; + return alias; + })); + + const artistData = [...artists, ...artistAliases]; + + return {artistData}; + }, + + sort({artistData}) { + sortAlphabetically(artistData); + }, + }); + + [inspect.custom]() { + const parts = []; + + parts.push(Thing.prototype[inspect.custom].apply(this)); + + if (CacheableObject.getUpdateValue(this, 'isAlias')) { + parts.unshift(`${colors.yellow('[alias]')} `); + + let aliasedArtist; + try { + aliasedArtist = this.aliasedArtist.name; + } catch (_error) { + aliasedArtist = CacheableObject.getUpdateValue(this, 'aliasedArtist'); + } + + parts.push(` ${colors.yellow(`[of ${aliasedArtist}]`)}`); + } + + return parts.join(''); + } +} diff --git a/src/data/things/flash.js b/src/data/things/flash.js new file mode 100644 index 0000000..ceed79f --- /dev/null +++ b/src/data/things/flash.js @@ -0,0 +1,370 @@ +export const FLASH_DATA_FILE = 'flashes.yaml'; + +import {input} from '#composite'; +import find from '#find'; +import {empty} from '#sugar'; +import {sortFlashesChronologically} from '#sort'; +import Thing from '#thing'; +import {anyOf, isColor, isContentString, isDirectory, isNumber, isString} + from '#validators'; +import {parseDate, parseContributors} from '#yaml'; + +import {withPropertyFromObject} from '#composite/data'; + +import { + exposeConstant, + exposeDependency, + exposeDependencyOrContinue, + exposeUpdateValueOrContinue, +} from '#composite/control-flow'; + +import { + color, + commentary, + commentatorArtists, + contentString, + contributionList, + directory, + fileExtension, + name, + referenceList, + simpleDate, + urls, + wikiData, +} from '#composite/wiki-properties'; + +import {withFlashAct} from '#composite/things/flash'; +import {withFlashSide} from '#composite/things/flash-act'; + +export class Flash extends Thing { + static [Thing.referenceType] = 'flash'; + + static [Thing.getPropertyDescriptors] = ({Artist, Track, FlashAct}) => ({ + // Update & expose + + name: name('Unnamed Flash'), + + directory: { + flags: {update: true, expose: true}, + update: {validate: isDirectory}, + + // Flashes expose directory differently from other Things! Their + // default directory is dependent on the page number (or ID), not + // the name. + expose: { + dependencies: ['page'], + transform(directory, {page}) { + if (directory === null && page === null) return null; + else if (directory === null) return page; + else return directory; + }, + }, + }, + + page: { + flags: {update: true, expose: true}, + update: {validate: anyOf(isString, isNumber)}, + + expose: { + transform: (value) => (value === null ? null : value.toString()), + }, + }, + + color: [ + exposeUpdateValueOrContinue({ + validate: input.value(isColor), + }), + + withFlashAct(), + + withPropertyFromObject({ + object: '#flashAct', + property: input.value('color'), + }), + + exposeDependency({dependency: '#flashAct.color'}), + ], + + date: simpleDate(), + + coverArtFileExtension: fileExtension('jpg'), + + contributorContribs: contributionList(), + + featuredTracks: referenceList({ + class: input.value(Track), + find: input.value(find.track), + data: 'trackData', + }), + + urls: urls(), + + commentary: commentary(), + + // Update only + + artistData: wikiData({ + class: input.value(Artist), + }), + + trackData: wikiData({ + class: input.value(Track), + }), + + flashActData: wikiData({ + class: input.value(FlashAct), + }), + + // Expose only + + commentatorArtists: commentatorArtists(), + + act: [ + withFlashAct(), + exposeDependency({dependency: '#flashAct'}), + ], + + side: [ + withFlashAct(), + + withPropertyFromObject({ + object: '#flashAct', + property: input.value('side'), + }), + + exposeDependency({dependency: '#flashAct.side'}), + ], + }); + + static [Thing.getSerializeDescriptors] = ({ + serialize: S, + }) => ({ + name: S.id, + page: S.id, + directory: S.id, + date: S.id, + contributors: S.toContribRefs, + tracks: S.toRefs, + urls: S.id, + color: S.id, + }); + + static [Thing.findSpecs] = { + flash: { + referenceTypes: ['flash'], + bindTo: 'flashData', + }, + }; + + static [Thing.yamlDocumentSpec] = { + fields: { + 'Flash': {property: 'name'}, + 'Directory': {property: 'directory'}, + 'Page': {property: 'page'}, + 'Color': {property: 'color'}, + 'URLs': {property: 'urls'}, + + 'Date': { + property: 'date', + transform: parseDate, + }, + + 'Cover Art File Extension': {property: 'coverArtFileExtension'}, + + 'Featured Tracks': {property: 'featuredTracks'}, + + 'Contributors': { + property: 'contributorContribs', + transform: parseContributors, + }, + + 'Commentary': {property: 'commentary'}, + + 'Review Points': {ignore: true}, + }, + }; +} + +export class FlashAct extends Thing { + static [Thing.referenceType] = 'flash-act'; + static [Thing.friendlyName] = `Flash Act`; + + static [Thing.getPropertyDescriptors] = () => ({ + // Update & expose + + name: name('Unnamed Flash Act'), + directory: directory(), + color: color(), + + listTerminology: [ + exposeUpdateValueOrContinue({ + validate: input.value(isContentString), + }), + + withFlashSide(), + + withPropertyFromObject({ + object: '#flashSide', + property: input.value('listTerminology'), + }), + + exposeDependencyOrContinue({ + dependency: '#flashSide.listTerminology', + }), + + exposeConstant({ + value: input.value(null), + }), + ], + + flashes: referenceList({ + class: input.value(Flash), + find: input.value(find.flash), + data: 'flashData', + }), + + // Update only + + flashData: wikiData({ + class: input.value(Flash), + }), + + flashSideData: wikiData({ + class: input.value(FlashSide), + }), + + // Expose only + + side: [ + withFlashSide(), + exposeDependency({dependency: '#flashSide'}), + ], + }); + + static [Thing.findSpecs] = { + flashAct: { + referenceTypes: ['flash-act'], + bindTo: 'flashActData', + }, + }; + + static [Thing.yamlDocumentSpec] = { + fields: { + 'Act': {property: 'name'}, + 'Directory': {property: 'directory'}, + + 'Color': {property: 'color'}, + 'List Terminology': {property: 'listTerminology'}, + + 'Review Points': {ignore: true}, + }, + }; +} + +export class FlashSide extends Thing { + static [Thing.referenceType] = 'flash-side'; + static [Thing.friendlyName] = `Flash Side`; + + static [Thing.getPropertyDescriptors] = () => ({ + // Update & expose + + name: name('Unnamed Flash Side'), + directory: directory(), + color: color(), + listTerminology: contentString(), + + acts: referenceList({ + class: input.value(FlashAct), + find: input.value(find.flashAct), + data: 'flashActData', + }), + + // Update only + + flashActData: wikiData({ + class: input.value(FlashAct), + }), + }); + + static [Thing.yamlDocumentSpec] = { + fields: { + 'Side': {property: 'name'}, + 'Directory': {property: 'directory'}, + 'Color': {property: 'color'}, + 'List Terminology': {property: 'listTerminology'}, + }, + }; + + static [Thing.findSpecs] = { + flashSide: { + referenceTypes: ['flash-side'], + bindTo: 'flashSideData', + }, + }; + + static [Thing.getYamlLoadingSpec] = ({ + documentModes: {allInOne}, + thingConstructors: {Flash, FlashAct}, + }) => ({ + title: `Process flashes file`, + file: FLASH_DATA_FILE, + + documentMode: allInOne, + documentThing: document => + ('Side' in document + ? FlashSide + : 'Act' in document + ? FlashAct + : Flash), + + save(results) { + // JavaScript likes you. + + if (!empty(results) && !(results[0] instanceof FlashSide)) { + throw new Error(`Expected a side at top of flash data file`); + } + + let index = 0; + let thing; + for (; thing = results[index]; index++) { + const flashSide = thing; + const flashActRefs = []; + + if (results[index + 1] instanceof Flash) { + throw new Error(`Expected an act to immediately follow a side`); + } + + for ( + index++; + (thing = results[index]) && thing instanceof FlashAct; + index++ + ) { + const flashAct = thing; + const flashRefs = []; + for ( + index++; + (thing = results[index]) && thing instanceof Flash; + index++ + ) { + flashRefs.push(Thing.getReference(thing)); + } + index--; + flashAct.flashes = flashRefs; + flashActRefs.push(Thing.getReference(flashAct)); + } + index--; + flashSide.acts = flashActRefs; + } + + const flashData = results.filter(x => x instanceof Flash); + const flashActData = results.filter(x => x instanceof FlashAct); + const flashSideData = results.filter(x => x instanceof FlashSide); + + return {flashData, flashActData, flashSideData}; + }, + + sort({flashData}) { + sortFlashesChronologically(flashData); + }, + }); +} diff --git a/src/data/things/group.js b/src/data/things/group.js new file mode 100644 index 0000000..0dbbbb7 --- /dev/null +++ b/src/data/things/group.js @@ -0,0 +1,194 @@ +export const GROUP_DATA_FILE = 'groups.yaml'; + +import {input} from '#composite'; +import find from '#find'; +import Thing from '#thing'; + +import { + color, + contentString, + directory, + name, + referenceList, + urls, + wikiData, +} from '#composite/wiki-properties'; + +export class Group extends Thing { + static [Thing.referenceType] = 'group'; + + static [Thing.getPropertyDescriptors] = ({Album}) => ({ + // Update & expose + + name: name('Unnamed Group'), + directory: directory(), + + description: contentString(), + + urls: urls(), + + featuredAlbums: referenceList({ + class: input.value(Album), + find: input.value(find.album), + data: 'albumData', + }), + + // Update only + + albumData: wikiData({ + class: input.value(Album), + }), + + groupCategoryData: wikiData({ + class: input.value(GroupCategory), + }), + + // Expose only + + descriptionShort: { + flags: {expose: true}, + + expose: { + dependencies: ['description'], + compute: ({description}) => + (description + ? description.split('<hr class="split">')[0] + : null), + }, + }, + + albums: { + flags: {expose: true}, + + expose: { + dependencies: ['this', 'albumData'], + compute: ({this: group, albumData}) => + albumData?.filter((album) => album.groups.includes(group)) ?? [], + }, + }, + + color: { + flags: {expose: true}, + + expose: { + dependencies: ['this', 'groupCategoryData'], + compute: ({this: group, groupCategoryData}) => + groupCategoryData.find((category) => category.groups.includes(group)) + ?.color, + }, + }, + + category: { + flags: {expose: true}, + + expose: { + dependencies: ['this', 'groupCategoryData'], + compute: ({this: group, groupCategoryData}) => + groupCategoryData.find((category) => category.groups.includes(group)) ?? + null, + }, + }, + }); + + static [Thing.findSpecs] = { + group: { + referenceTypes: ['group', 'group-gallery'], + bindTo: 'groupData', + }, + }; + + static [Thing.yamlDocumentSpec] = { + fields: { + 'Group': {property: 'name'}, + 'Directory': {property: 'directory'}, + 'Description': {property: 'description'}, + 'URLs': {property: 'urls'}, + + 'Featured Albums': {property: 'featuredAlbums'}, + + 'Review Points': {ignore: true}, + }, + }; + + static [Thing.getYamlLoadingSpec] = ({ + documentModes: {allInOne}, + thingConstructors: {Group, GroupCategory}, + }) => ({ + title: `Process groups file`, + file: GROUP_DATA_FILE, + + documentMode: allInOne, + documentThing: document => + ('Category' in document + ? GroupCategory + : Group), + + save(results) { + let groupCategory; + let groupRefs = []; + + if (results[0] && !(results[0] instanceof GroupCategory)) { + throw new Error(`Expected a category at top of group data file`); + } + + for (const thing of results) { + if (thing instanceof GroupCategory) { + if (groupCategory) { + Object.assign(groupCategory, {groups: groupRefs}); + } + + groupCategory = thing; + groupRefs = []; + } else { + groupRefs.push(Thing.getReference(thing)); + } + } + + if (groupCategory) { + Object.assign(groupCategory, {groups: groupRefs}); + } + + const groupData = results.filter(x => x instanceof Group); + const groupCategoryData = results.filter(x => x instanceof GroupCategory); + + return {groupData, groupCategoryData}; + }, + + // Groups aren't sorted at all, always preserving the order in the data + // file as-is. + sort: null, + }); +} + +export class GroupCategory extends Thing { + static [Thing.referenceType] = 'group-category'; + static [Thing.friendlyName] = `Group Category`; + + static [Thing.getPropertyDescriptors] = ({Group}) => ({ + // Update & expose + + name: name('Unnamed Group Category'), + directory: directory(), + + color: color(), + + groups: referenceList({ + class: input.value(Group), + find: input.value(find.group), + data: 'groupData', + }), + + // Update only + + groupData: wikiData({ + class: input.value(Group), + }), + }); + + static [Thing.yamlDocumentSpec] = { + fields: { + 'Category': {property: 'name'}, + 'Color': {property: 'color'}, + }, + }; +} diff --git a/src/data/things/homepage-layout.js b/src/data/things/homepage-layout.js new file mode 100644 index 0000000..00d6aef --- /dev/null +++ b/src/data/things/homepage-layout.js @@ -0,0 +1,223 @@ +export const HOMEPAGE_LAYOUT_DATA_FILE = 'homepage.yaml'; + +import {input} from '#composite'; +import find from '#find'; +import Thing from '#thing'; + +import { + anyOf, + is, + isCountingNumber, + isString, + isStringNonEmpty, + validateArrayItems, + validateInstanceOf, + validateReference, +} from '#validators'; + +import {exposeDependency} from '#composite/control-flow'; +import {withResolvedReference} from '#composite/wiki-data'; +import {color, contentString, name, referenceList, wikiData} + from '#composite/wiki-properties'; + +export class HomepageLayout extends Thing { + static [Thing.friendlyName] = `Homepage Layout`; + + static [Thing.getPropertyDescriptors] = ({HomepageLayoutRow}) => ({ + // Update & expose + + sidebarContent: contentString(), + + navbarLinks: { + flags: {update: true, expose: true}, + update: {validate: validateArrayItems(isStringNonEmpty)}, + }, + + rows: { + flags: {update: true, expose: true}, + + update: { + validate: validateArrayItems(validateInstanceOf(HomepageLayoutRow)), + }, + }, + }); + + static [Thing.yamlDocumentSpec] = { + fields: { + 'Homepage': {ignore: true}, + + 'Sidebar Content': {property: 'sidebarContent'}, + 'Navbar Links': {property: 'navbarLinks'}, + }, + }; +} + +export class HomepageLayoutRow extends Thing { + static [Thing.friendlyName] = `Homepage Row`; + + static [Thing.getPropertyDescriptors] = ({Album, Group}) => ({ + // Update & expose + + name: name('Unnamed Homepage Row'), + + type: { + flags: {update: true, expose: true}, + + update: { + validate() { + throw new Error(`'type' property validator must be overridden`); + }, + }, + }, + + color: color(), + + // Update only + + // These wiki data arrays aren't necessarily used by every subclass, but + // to the convenience of providing these, the superclass accepts all wiki + // data arrays depended upon by any subclass. + + albumData: wikiData({ + class: input.value(Album), + }), + + groupData: wikiData({ + class: input.value(Group), + }), + }); + + static [Thing.yamlDocumentSpec] = { + fields: { + 'Row': {property: 'name'}, + 'Color': {property: 'color'}, + 'Type': {property: 'type'}, + }, + }; +} + +export class HomepageLayoutAlbumsRow extends HomepageLayoutRow { + static [Thing.friendlyName] = `Homepage Albums Row`; + + static [Thing.getPropertyDescriptors] = (opts, {Album, Group} = opts) => ({ + ...HomepageLayoutRow[Thing.getPropertyDescriptors](opts), + + // Update & expose + + type: { + flags: {update: true, expose: true}, + update: { + validate(value) { + if (value !== 'albums') { + throw new TypeError(`Expected 'albums'`); + } + + return true; + }, + }, + }, + + displayStyle: { + flags: {update: true, expose: true}, + + update: { + validate: is('grid', 'carousel'), + }, + + expose: { + transform: (displayStyle) => + displayStyle ?? 'grid', + }, + }, + + sourceGroup: [ + { + flags: {expose: true, update: true, compose: true}, + + update: { + validate: + anyOf( + is('new-releases', 'new-additions'), + validateReference(Group[Thing.referenceType])), + }, + + expose: { + transform: (value, continuation) => + (value === 'new-releases' || value === 'new-additions' + ? value + : continuation(value)), + }, + }, + + withResolvedReference({ + ref: input.updateValue(), + data: 'groupData', + find: input.value(find.group), + }), + + exposeDependency({dependency: '#resolvedReference'}), + ], + + sourceAlbums: referenceList({ + class: input.value(Album), + find: input.value(find.album), + data: 'albumData', + }), + + countAlbumsFromGroup: { + flags: {update: true, expose: true}, + update: {validate: isCountingNumber}, + }, + + actionLinks: { + flags: {update: true, expose: true}, + update: {validate: validateArrayItems(isString)}, + }, + }); + + static [Thing.yamlDocumentSpec] = Thing.extendDocumentSpec(HomepageLayoutRow, { + fields: { + 'Display Style': {property: 'displayStyle'}, + 'Group': {property: 'sourceGroup'}, + 'Count': {property: 'countAlbumsFromGroup'}, + 'Albums': {property: 'sourceAlbums'}, + 'Actions': {property: 'actionLinks'}, + }, + }); + + static [Thing.getYamlLoadingSpec] = ({ + documentModes: {headerAndEntries}, // Kludge, see below + thingConstructors: { + HomepageLayout, + HomepageLayoutAlbumsRow, + }, + }) => ({ + title: `Process homepage layout file`, + + // Kludge: This benefits from the same headerAndEntries style messaging as + // albums and tracks (for example), but that document mode is designed to + // support multiple files, and only one is actually getting processed here. + files: [HOMEPAGE_LAYOUT_DATA_FILE], + + documentMode: headerAndEntries, + headerDocumentThing: HomepageLayout, + entryDocumentThing: document => { + switch (document['Type']) { + case 'albums': + return HomepageLayoutAlbumsRow; + default: + throw new TypeError(`No processDocument function for row type ${document['Type']}!`); + } + }, + + save(results) { + if (!results[0]) { + return; + } + + const {header: homepageLayout, entries: rows} = results[0]; + Object.assign(homepageLayout, {rows}); + return {homepageLayout}; + }, + }); +} diff --git a/src/data/things/index.js b/src/data/things/index.js new file mode 100644 index 0000000..3bf8409 --- /dev/null +++ b/src/data/things/index.js @@ -0,0 +1,188 @@ +import * as path from 'node:path'; +import {fileURLToPath} from 'node:url'; + +import {openAggregate, showAggregate} from '#aggregate'; +import {logError} from '#cli'; +import {compositeFrom} from '#composite'; +import * as serialize from '#serialize'; + +import Thing from '#thing'; + +import * as albumClasses from './album.js'; +import * as artTagClasses from './art-tag.js'; +import * as artistClasses from './artist.js'; +import * as flashClasses from './flash.js'; +import * as groupClasses from './group.js'; +import * as homepageLayoutClasses from './homepage-layout.js'; +import * as languageClasses from './language.js'; +import * as newsEntryClasses from './news-entry.js'; +import * as staticPageClasses from './static-page.js'; +import * as trackClasses from './track.js'; +import * as wikiInfoClasses from './wiki-info.js'; + +const allClassLists = { + 'album.js': albumClasses, + 'art-tag.js': artTagClasses, + 'artist.js': artistClasses, + 'flash.js': flashClasses, + 'group.js': groupClasses, + 'homepage-layout.js': homepageLayoutClasses, + 'language.js': languageClasses, + 'news-entry.js': newsEntryClasses, + 'static-page.js': staticPageClasses, + 'track.js': trackClasses, + 'wiki-info.js': wikiInfoClasses, +}; + +let allClasses = Object.create(null); + +// src/data/things/index.js -> src/ +const __dirname = path.dirname( + path.resolve( + fileURLToPath(import.meta.url), + '../..')); + +function niceShowAggregate(error, ...opts) { + showAggregate(error, { + pathToFileURL: (f) => path.relative(__dirname, fileURLToPath(f)), + ...opts, + }); +} + +function errorDuplicateClassNames() { + const locationDict = Object.create(null); + + for (const [location, classes] of Object.entries(allClassLists)) { + for (const className of Object.keys(classes)) { + if (className in locationDict) { + locationDict[className].push(location); + } else { + locationDict[className] = [location]; + } + } + } + + let success = true; + + for (const [className, locations] of Object.entries(locationDict)) { + if (locations.length === 1) { + continue; + } + + logError`Thing class name ${`"${className}"`} is defined more than once: ${locations.join(', ')}`; + success = false; + } + + return success; +} + +function flattenClassLists() { + for (const classes of Object.values(allClassLists)) { + for (const [name, constructor] of Object.entries(classes)) { + if (typeof constructor !== 'function') continue; + if (!(constructor.prototype instanceof Thing)) continue; + allClasses[name] = constructor; + } + } +} + +function descriptorAggregateHelper({ + showFailedClasses, + message, + op, +}) { + const failureSymbol = Symbol(); + const aggregate = openAggregate({ + message, + returnOnFail: failureSymbol, + }); + + const failedClasses = []; + + for (const [name, constructor] of Object.entries(allClasses)) { + const result = aggregate.call(op, constructor); + + if (result === failureSymbol) { + failedClasses.push(name); + } + } + + try { + aggregate.close(); + return true; + } catch (error) { + niceShowAggregate(error); + showFailedClasses(failedClasses); + return false; + } +} + +function evaluatePropertyDescriptors() { + const opts = {...allClasses}; + + return descriptorAggregateHelper({ + message: `Errors evaluating Thing class property descriptors`, + + op(constructor) { + if (!constructor[Thing.getPropertyDescriptors]) { + throw new Error(`Missing [Thing.getPropertyDescriptors] function`); + } + + const results = constructor[Thing.getPropertyDescriptors](opts); + + for (const [key, value] of Object.entries(results)) { + if (Array.isArray(value)) { + results[key] = compositeFrom({ + annotation: `${constructor.name}.${key}`, + compose: false, + steps: value, + }); + } else if (value.toResolvedComposition) { + results[key] = compositeFrom(value.toResolvedComposition()); + } + } + + constructor.propertyDescriptors = results; + }, + + showFailedClasses(failedClasses) { + logError`Failed to evaluate property descriptors for classes: ${failedClasses.join(', ')}`; + }, + }); +} + +function evaluateSerializeDescriptors() { + const opts = {...allClasses, serialize}; + + return descriptorAggregateHelper({ + message: `Errors evaluating Thing class serialize descriptors`, + + op(constructor) { + if (!constructor[Thing.getSerializeDescriptors]) { + return; + } + + constructor[serialize.serializeDescriptors] = + constructor[Thing.getSerializeDescriptors](opts); + }, + + showFailedClasses(failedClasses) { + logError`Failed to evaluate serialize descriptors for classes: ${failedClasses.join(', ')}`; + }, + }); +} + +if (!errorDuplicateClassNames()) + process.exit(1); + +flattenClassLists(); + +if (!evaluatePropertyDescriptors()) + process.exit(1); + +if (!evaluateSerializeDescriptors()) + process.exit(1); + +Object.assign(allClasses, {Thing}); + +export default allClasses; diff --git a/src/data/things/language.js b/src/data/things/language.js new file mode 100644 index 0000000..dbe1ff3 --- /dev/null +++ b/src/data/things/language.js @@ -0,0 +1,730 @@ +import { Temporal, toTemporalInstant } from '@js-temporal/polyfill'; + +import {withAggregate} from '#aggregate'; +import CacheableObject from '#cacheable-object'; +import {logWarn} from '#cli'; +import * as html from '#html'; +import {empty} from '#sugar'; +import {isLanguageCode} from '#validators'; +import Thing from '#thing'; + +import { + getExternalLinkStringOfStyleFromDescriptors, + getExternalLinkStringsFromDescriptors, + isExternalLinkContext, + isExternalLinkSpec, + isExternalLinkStyle, +} from '#external-links'; + +import {externalFunction, flag, name} from '#composite/wiki-properties'; + +export const languageOptionRegex = /{(?<name>[A-Z0-9_]+)}/g; + +export class Language extends Thing { + static [Thing.getPropertyDescriptors] = () => ({ + // Update & expose + + // General language code. This is used to identify the language distinctly + // from other languages (similar to how "Directory" operates in many data + // objects). + code: { + flags: {update: true, expose: true}, + update: {validate: isLanguageCode}, + }, + + // Human-readable name. This should be the language's own native name, not + // localized to any other language. + name: name(`Unnamed Language`), + + // Language code specific to JavaScript's Internationalization (Intl) API. + // Usually this will be the same as the language's general code, but it + // may be overridden to provide Intl constructors an alternative value. + intlCode: { + flags: {update: true, expose: true}, + update: {validate: isLanguageCode}, + expose: { + dependencies: ['code'], + transform: (intlCode, {code}) => intlCode ?? code, + }, + }, + + // Flag which represents whether or not to hide a language from general + // access. If a language is hidden, its portion of the website will still + // be built (with all strings localized to the language), but it won't be + // included in controls for switching languages or the <link rel=alternate> + // tags used for search engine optimization. This flag is intended for use + // with languages that are currently in development and not ready for + // formal release, or which are just kept hidden as "experimental zones" + // for wiki development or content testing. + hidden: flag(false), + + // Mapping of translation keys to values (strings). Generally, don't + // access this object directly - use methods instead. + strings: { + flags: {update: true, expose: true}, + update: {validate: (t) => typeof t === 'object'}, + + expose: { + dependencies: ['inheritedStrings', 'code'], + transform(strings, {inheritedStrings, code}) { + if (!strings && !inheritedStrings) return null; + if (!inheritedStrings) return strings; + + const validStrings = { + ...inheritedStrings, + ...strings, + }; + + const optionsFromTemplate = template => + Array.from(template.matchAll(languageOptionRegex)) + .map(({groups}) => groups.name); + + for (const [key, providedTemplate] of Object.entries(strings)) { + const inheritedTemplate = inheritedStrings[key]; + if (!inheritedTemplate) continue; + + const providedOptions = optionsFromTemplate(providedTemplate); + const inheritedOptions = optionsFromTemplate(inheritedTemplate); + + const missingOptionNames = + inheritedOptions.filter(name => !providedOptions.includes(name)); + + const misplacedOptionNames = + providedOptions.filter(name => !inheritedOptions.includes(name)); + + if (!empty(missingOptionNames) || !empty(misplacedOptionNames)) { + logWarn`Not using ${code ?? '(no code)'} string ${key}:`; + if (!empty(missingOptionNames)) + logWarn`- Missing options: ${missingOptionNames.join(', ')}`; + if (!empty(misplacedOptionNames)) + logWarn`- Unexpected options: ${misplacedOptionNames.join(', ')}`; + validStrings[key] = inheritedStrings[key]; + } + } + + return validStrings; + }, + }, + }, + + // May be provided to specify "default" strings, generally (but not + // necessarily) inherited from another Language object. + inheritedStrings: { + flags: {update: true, expose: true}, + update: {validate: (t) => typeof t === 'object'}, + }, + + // List of descriptors for providing to external link utilities when using + // language.formatExternalLink - refer to util/external-links.js for info. + externalLinkSpec: { + flags: {update: true, expose: true}, + update: {validate: isExternalLinkSpec}, + }, + + // Update only + + escapeHTML: externalFunction(), + + // Expose only + + intl_date: this.#intlHelper(Intl.DateTimeFormat, {full: true}), + intl_number: this.#intlHelper(Intl.NumberFormat), + intl_listConjunction: this.#intlHelper(Intl.ListFormat, {type: 'conjunction'}), + intl_listDisjunction: this.#intlHelper(Intl.ListFormat, {type: 'disjunction'}), + intl_listUnit: this.#intlHelper(Intl.ListFormat, {type: 'unit'}), + intl_pluralCardinal: this.#intlHelper(Intl.PluralRules, {type: 'cardinal'}), + intl_pluralOrdinal: this.#intlHelper(Intl.PluralRules, {type: 'ordinal'}), + + validKeys: { + flags: {expose: true}, + + expose: { + dependencies: ['strings', 'inheritedStrings'], + compute: ({strings, inheritedStrings}) => + Array.from( + new Set([ + ...Object.keys(inheritedStrings ?? {}), + ...Object.keys(strings ?? {}), + ]) + ), + }, + }, + + // TODO: This currently isn't used. Is it still needed? + strings_htmlEscaped: { + flags: {expose: true}, + expose: { + dependencies: ['strings', 'inheritedStrings', 'escapeHTML'], + compute({strings, inheritedStrings, escapeHTML}) { + if (!(strings || inheritedStrings) || !escapeHTML) return null; + const allStrings = {...inheritedStrings, ...strings}; + return Object.fromEntries( + Object.entries(allStrings).map(([k, v]) => [k, escapeHTML(v)]) + ); + }, + }, + }, + }); + + static #intlHelper (constructor, opts) { + return { + flags: {expose: true}, + expose: { + dependencies: ['code', 'intlCode'], + compute: ({code, intlCode}) => { + const constructCode = intlCode ?? code; + if (!constructCode) return null; + return Reflect.construct(constructor, [constructCode, opts]); + }, + }, + }; + } + + $(...args) { + return this.formatString(...args); + } + + assertIntlAvailable(property) { + if (!this[property]) { + throw new Error(`Intl API ${property} unavailable`); + } + } + + getUnitForm(value) { + this.assertIntlAvailable('intl_pluralCardinal'); + return this.intl_pluralCardinal.select(value); + } + + formatString(...args) { + const hasOptions = + typeof args.at(-1) === 'object' && + args.at(-1) !== null; + + const key = + (hasOptions ? args.slice(0, -1) : args) + .filter(Boolean) + .join('.'); + + const options = + (hasOptions + ? args.at(-1) + : {}); + + if (!this.strings) { + throw new Error(`Strings unavailable`); + } + + if (!this.validKeys.includes(key)) { + throw new Error(`Invalid key ${key} accessed`); + } + + // These will be filled up as we iterate over the template, slotting in + // each option (if it's present). + const missingOptionNames = new Set(); + + // And this will have entries deleted as they're encountered in the + // template. Leftover entries are misplaced. + const optionsMap = + new Map( + Object.entries(options).map(([name, value]) => [ + name + .replace(/[A-Z]/g, '_$&') + .toUpperCase(), + value, + ])); + + const output = this.#iterateOverTemplate({ + template: this.strings[key], + + match: languageOptionRegex, + + insert: ({name: optionName}, canceledForming) => { + if (optionsMap.has(optionName)) { + let optionValue; + + // We'll only need the option's value if we're going to use it as + // part of the formed output (see below). + if (!canceledForming) { + optionValue = optionsMap.get(optionName); + } + + // But we always have to delete expected options off the provided + // option map, since the leftovers are what will be used to tell + // which are misplaced. + optionsMap.delete(optionName); + + if (canceledForming) { + return undefined; + } else { + return optionValue; + } + } else { + // We don't need to continue forming the output if we've hit a + // missing option name, since the end result of this formatString + // call will be a thrown error, and formed output won't be needed. + missingOptionNames.add(optionName); + return undefined; + } + }, + }); + + const misplacedOptionNames = + Array.from(optionsMap.keys()); + + withAggregate({message: `Errors in options for string "${key}"`}, ({push}) => { + if (!empty(missingOptionNames)) { + const names = Array.from(missingOptionNames).join(`, `); + push(new Error(`Missing options: ${names}`)); + } + + if (!empty(misplacedOptionNames)) { + const names = Array.from(misplacedOptionNames).join(`, `); + push(new Error(`Unexpected options: ${names}`)); + } + }); + + return output; + } + + #iterateOverTemplate({ + template, + match: regexp, + insert: insertFn, + }) { + const outputParts = []; + + let canceledForming = false; + + let lastIndex = 0; + let partInProgress = ''; + + for (const match of template.matchAll(regexp)) { + const insertion = + insertFn(match.groups, canceledForming); + + if (insertion === undefined) { + canceledForming = true; + } + + // Don't proceed with forming logic if the insertion function has + // indicated that's not needed anymore - but continue iterating over + // the rest of the template's matches, so other iteration logic (with + // side effects) gets to process everything. + if (canceledForming) { + continue; + } + + partInProgress += template.slice(lastIndex, match.index); + + // Sanitize string arguments in particular. These are taken to come from + // (raw) data and may include special characters that aren't meant to be + // rendered as HTML markup. + const sanitizedInsertion = + this.#sanitizeValueForInsertion(insertion); + + if (typeof sanitizedInsertion === 'string') { + // Join consecutive strings together. + partInProgress += sanitizedInsertion; + } else if ( + sanitizedInsertion instanceof html.Tag && + sanitizedInsertion.contentOnly + ) { + // Collapse string-only tag contents onto the current string part. + partInProgress += sanitizedInsertion.toString(); + } else { + // Push the string part in progress, then the insertion as-is. + outputParts.push(partInProgress); + outputParts.push(sanitizedInsertion); + partInProgress = ''; + } + + lastIndex = match.index + match[0].length; + } + + if (canceledForming) { + return undefined; + } + + // Tack onto the final partInProgress, which may still have a value by this + // point, if the final inserted value was a string. (Otherwise, it'll just + // be equal to the remaining template text.) + if (lastIndex < template.length) { + partInProgress += template.slice(lastIndex); + } + + if (partInProgress) { + outputParts.push(partInProgress); + } + + return this.#wrapSanitized(outputParts); + } + + // Processes a value so that it's suitable to be inserted into a template. + // For strings, this escapes HTML special characters, displaying them as-are + // instead of representing HTML markup. For numbers and booleans, this turns + // them into string values, so they never accidentally get caught as falsy + // by #html stringification. Everything else - most importantly including + // html.Tag objects - gets left as-is, preserving the value exactly as it's + // provided. + #sanitizeValueForInsertion(value) { + const escapeHTML = CacheableObject.getUpdateValue(this, 'escapeHTML'); + if (!escapeHTML) { + throw new Error(`escapeHTML unavailable`); + } + + switch (typeof value) { + case 'string': + return escapeHTML(value); + + case 'number': + case 'boolean': + return value.toString(); + + default: + return value; + } + } + + // Wraps the output of a formatting function in a no-name-nor-attributes + // HTML tag, which will indicate to other calls to formatString that this + // content is a string *that may contain HTML* and doesn't need to + // sanitized any further. It'll still .toString() to just the string + // contents, if needed. + #wrapSanitized(content) { + return html.tags(content, { + [html.blessAttributes]: true, + [html.joinChildren]: '', + [html.noEdgeWhitespace]: true, + }); + } + + // Similar to the above internal methods, but this one is public. + // It should be used when embedding content that may not have previously + // been sanitized directly into an HTML tag or template's contents. + // The templating engine usually handles this on its own, as does passing + // a value (sanitized or not) directly for inserting into formatting + // functions, but if you used a custom slot validation function (for example, + // {validate: v => v.isHTML} instead of {type: 'string'} / {type: 'html'}) + // and are embedding the contents of the slot as a direct child of another + // tag, you should manually sanitize those contents with this function. + sanitize(value) { + if (typeof value === 'string') { + return this.#wrapSanitized(this.#sanitizeValueForInsertion(value)); + } else { + return value; + } + } + + formatDate(date) { + this.assertIntlAvailable('intl_date'); + return this.intl_date.format(date); + } + + formatDateRange(startDate, endDate) { + this.assertIntlAvailable('intl_date'); + return this.intl_date.formatRange(startDate, endDate); + } + + formatDateDuration({ + years: numYears = 0, + months: numMonths = 0, + days: numDays = 0, + approximate = false, + }) { + let basis; + + const years = this.countYears(numYears, {unit: true}); + const months = this.countMonths(numMonths, {unit: true}); + const days = this.countDays(numDays, {unit: true}); + + if (numYears && numMonths && numDays) + basis = this.formatString('count.dateDuration.yearsMonthsDays', {years, months, days}); + else if (numYears && numMonths) + basis = this.formatString('count.dateDuration.yearsMonths', {years, months}); + else if (numYears && numDays) + basis = this.formatString('count.dateDuration.yearsDays', {years, days}); + else if (numYears) + basis = this.formatString('count.dateDuration.years', {years}); + else if (numMonths && numDays) + basis = this.formatString('count.dateDuration.monthsDays', {months, days}); + else if (numMonths) + basis = this.formatString('count.dateDuration.months', {months}); + else if (numDays) + basis = this.formatString('count.dateDuration.days', {days}); + else + return this.formatString('count.dateDuration.zero'); + + if (approximate) { + return this.formatString('count.dateDuration.approximate', { + duration: basis, + }); + } else { + return basis; + } + } + + formatRelativeDate(currentDate, referenceDate, { + considerRoundingDays = false, + approximate = true, + absolute = true, + } = {}) { + const currentInstant = toTemporalInstant.apply(currentDate); + const referenceInstant = toTemporalInstant.apply(referenceDate); + + const comparison = + Temporal.Instant.compare(currentInstant, referenceInstant); + + if (comparison === 0) { + return this.formatString('count.dateDuration.same'); + } + + const currentTDZ = currentInstant.toZonedDateTimeISO('Etc/UTC'); + const referenceTDZ = referenceInstant.toZonedDateTimeISO('Etc/UTC'); + + const earlierTDZ = (comparison === -1 ? currentTDZ : referenceTDZ); + const laterTDZ = (comparison === 1 ? currentTDZ : referenceTDZ); + + const {years, months, days} = + laterTDZ.since(earlierTDZ, { + largestUnit: 'year', + smallestUnit: + (considerRoundingDays + ? (laterTDZ.since(earlierTDZ, { + largestUnit: 'year', + smallestUnit: 'day', + }).years + ? 'month' + : 'day') + : 'day'), + roundingMode: 'halfCeil', + }); + + const duration = + this.formatDateDuration({ + years, months, days, + approximate: false, + }); + + const relative = + this.formatString( + 'count.dateDuration', + (approximate && (years || months || days) + ? (comparison === -1 + ? 'approximateEarlier' + : 'approximateLater') + : (comparison === -1 + ? 'earlier' + : 'later')), + {duration}); + + if (absolute) { + return this.formatString('count.dateDuration.relativeAbsolute', { + relative, + absolute: this.formatDate(currentDate), + }); + } else { + return relative; + } + } + + formatDuration(secTotal, {approximate = false, unit = false} = {}) { + if (secTotal === 0) { + return this.formatString('count.duration.missing'); + } + + const hour = Math.floor(secTotal / 3600); + const min = Math.floor((secTotal - hour * 3600) / 60); + const sec = Math.floor(secTotal - hour * 3600 - min * 60); + + const pad = (val) => val.toString().padStart(2, '0'); + + const stringSubkey = unit ? '.withUnit' : ''; + + const duration = + hour > 0 + ? this.formatString('count.duration.hours' + stringSubkey, { + hours: hour, + minutes: pad(min), + seconds: pad(sec), + }) + : this.formatString('count.duration.minutes' + stringSubkey, { + minutes: min, + seconds: pad(sec), + }); + + return approximate + ? this.formatString('count.duration.approximate', {duration}) + : duration; + } + + formatExternalLink(url, { + style = 'platform', + context = 'generic', + } = {}) { + if (!this.externalLinkSpec) { + throw new TypeError(`externalLinkSpec unavailable`); + } + + isExternalLinkContext(context); + + if (style === 'all') { + return getExternalLinkStringsFromDescriptors(url, this.externalLinkSpec, { + language: this, + context, + }); + } + + isExternalLinkStyle(style); + + const result = + getExternalLinkStringOfStyleFromDescriptors(url, style, this.externalLinkSpec, { + language: this, + context, + }); + + // It's possible for there to not actually be any string available for the + // given URL, style, and context, and we want this to be detectable via + // html.blank(). + return result ?? html.blank(); + } + + formatIndex(value) { + this.assertIntlAvailable('intl_pluralOrdinal'); + return this.formatString('count.index.' + this.intl_pluralOrdinal.select(value), {index: value}); + } + + formatNumber(value) { + this.assertIntlAvailable('intl_number'); + return this.intl_number.format(value); + } + + formatWordCount(value) { + const num = this.formatNumber( + value > 1000 ? Math.floor(value / 100) / 10 : value + ); + + const words = + value > 1000 + ? this.formatString('count.words.thousand', {words: num}) + : this.formatString('count.words', {words: num}); + + return this.formatString('count.words.withUnit.' + this.getUnitForm(value), {words}); + } + + #formatListHelper(array, processFn) { + // Operate on "insertion markers" instead of the actual contents of the + // array, because the process function (likely an Intl operation) is taken + // to only operate on strings. We'll insert the contents of the array back + // at these points afterwards. + + const insertionMarkers = + Array.from( + {length: array.length}, + (_item, index) => `<::insertion_${index}>`); + + // Basically the same insertion logic as in formatString. Like there, we + // can't assume that insertion markers were kept in the same order as they + // were provided, so we'll refer to the marked index. But we don't need to + // worry about some of the indices *not* corresponding to a provided source + // item, like we do in formatString, so that cuts out a lot of the + // validation logic. + + return this.#iterateOverTemplate({ + template: processFn(insertionMarkers), + + match: /<::insertion_(?<index>[0-9]+)>/g, + + insert: ({index: markerIndex}) => { + return array[markerIndex]; + }, + }); + } + + // Conjunction list: A, B, and C + formatConjunctionList(array) { + this.assertIntlAvailable('intl_listConjunction'); + return this.#formatListHelper( + array, + array => this.intl_listConjunction.format(array)); + } + + // Disjunction lists: A, B, or C + formatDisjunctionList(array) { + this.assertIntlAvailable('intl_listDisjunction'); + return this.#formatListHelper( + array, + array => this.intl_listDisjunction.format(array)); + } + + // Unit lists: A, B, C + formatUnitList(array) { + this.assertIntlAvailable('intl_listUnit'); + return this.#formatListHelper( + array, + array => this.intl_listUnit.format(array)); + } + + // Lists without separator: A B C + formatListWithoutSeparator(array) { + return this.#formatListHelper( + array, + array => array.join(' ')); + } + + // File sizes: 42.5 kB, 127.2 MB, 4.13 GB, 998.82 TB + formatFileSize(bytes) { + if (!bytes) return ''; + + bytes = parseInt(bytes); + if (isNaN(bytes)) return ''; + + const round = (exp) => Math.round(bytes / 10 ** (exp - 1)) / 10; + + if (bytes >= 10 ** 12) { + return this.formatString('count.fileSize.terabytes', { + terabytes: round(12), + }); + } else if (bytes >= 10 ** 9) { + return this.formatString('count.fileSize.gigabytes', { + gigabytes: round(9), + }); + } else if (bytes >= 10 ** 6) { + return this.formatString('count.fileSize.megabytes', { + megabytes: round(6), + }); + } else if (bytes >= 10 ** 3) { + return this.formatString('count.fileSize.kilobytes', { + kilobytes: round(3), + }); + } else { + return this.formatString('count.fileSize.bytes', {bytes}); + } + } +} + +const countHelper = (stringKey, optionName = stringKey) => + function(value, {unit = false} = {}) { + return this.formatString( + unit + ? `count.${stringKey}.withUnit.` + this.getUnitForm(value) + : `count.${stringKey}`, + {[optionName]: this.formatNumber(value)}); + }; + +// TODO: These are hard-coded. Is there a better way? +Object.assign(Language.prototype, { + countAdditionalFiles: countHelper('additionalFiles', 'files'), + countAlbums: countHelper('albums'), + countArtworks: countHelper('artworks'), + countCommentaryEntries: countHelper('commentaryEntries', 'entries'), + countContributions: countHelper('contributions'), + countCoverArts: countHelper('coverArts'), + countDays: countHelper('days'), + countFlashes: countHelper('flashes'), + countMonths: countHelper('months'), + countTimesReferenced: countHelper('timesReferenced'), + countTimesUsed: countHelper('timesUsed'), + countTracks: countHelper('tracks'), + countWeeks: countHelper('weeks'), + countYears: countHelper('years'), +}); diff --git a/src/data/things/news-entry.js b/src/data/things/news-entry.js new file mode 100644 index 0000000..43d1638 --- /dev/null +++ b/src/data/things/news-entry.js @@ -0,0 +1,73 @@ +export const NEWS_DATA_FILE = 'news.yaml'; + +import {sortChronologically} from '#sort'; +import Thing from '#thing'; +import {parseDate} from '#yaml'; + +import {contentString, directory, name, simpleDate} + from '#composite/wiki-properties'; + +export class NewsEntry extends Thing { + static [Thing.referenceType] = 'news-entry'; + static [Thing.friendlyName] = `News Entry`; + + static [Thing.getPropertyDescriptors] = () => ({ + // Update & expose + + name: name('Unnamed News Entry'), + directory: directory(), + date: simpleDate(), + + content: contentString(), + + // Expose only + + contentShort: { + flags: {expose: true}, + + expose: { + dependencies: ['content'], + + compute: ({content}) => content.split('<hr class="split">')[0], + }, + }, + }); + + static [Thing.findSpecs] = { + newsEntry: { + referenceTypes: ['news-entry'], + bindTo: 'newsData', + }, + }; + + static [Thing.yamlDocumentSpec] = { + fields: { + 'Name': {property: 'name'}, + 'Directory': {property: 'directory'}, + + 'Date': { + property: 'date', + transform: parseDate, + }, + + 'Content': {property: 'content'}, + }, + }; + + static [Thing.getYamlLoadingSpec] = ({ + documentModes: {allInOne}, + thingConstructors: {NewsEntry}, + }) => ({ + title: `Process news data file`, + file: NEWS_DATA_FILE, + + documentMode: allInOne, + documentThing: NewsEntry, + + save: (results) => ({newsData: results}), + + sort({newsData}) { + sortChronologically(newsData, {latestFirst: true}); + }, + }); +} diff --git a/src/data/things/static-page.js b/src/data/things/static-page.js new file mode 100644 index 0000000..0327497 --- /dev/null +++ b/src/data/things/static-page.js @@ -0,0 +1,80 @@ +export const DATA_STATIC_PAGE_DIRECTORY = 'static-page'; + +import * as path from 'node:path'; + +import {traverse} from '#node-utils'; +import {sortAlphabetically} from '#sort'; +import Thing from '#thing'; +import {isName} from '#validators'; + +import {contentString, directory, name, simpleString} + from '#composite/wiki-properties'; + +export class StaticPage extends Thing { + static [Thing.referenceType] = 'static'; + static [Thing.friendlyName] = `Static Page`; + + static [Thing.getPropertyDescriptors] = () => ({ + // Update & expose + + name: name('Unnamed Static Page'), + + nameShort: { + flags: {update: true, expose: true}, + update: {validate: isName}, + + expose: { + dependencies: ['name'], + transform: (value, {name}) => value ?? name, + }, + }, + + directory: directory(), + content: contentString(), + stylesheet: simpleString(), + script: simpleString(), + }); + + static [Thing.findSpecs] = { + staticPage: { + referenceTypes: ['static'], + bindTo: 'staticPageData', + }, + }; + + static [Thing.yamlDocumentSpec] = { + fields: { + 'Name': {property: 'name'}, + 'Short Name': {property: 'nameShort'}, + 'Directory': {property: 'directory'}, + + 'Style': {property: 'stylesheet'}, + 'Script': {property: 'script'}, + 'Content': {property: 'content'}, + + 'Review Points': {ignore: true}, + }, + }; + + static [Thing.getYamlLoadingSpec] = ({ + documentModes: {onePerFile}, + thingConstructors: {StaticPage}, + }) => ({ + title: `Process static page files`, + + files: dataPath => + traverse(path.join(dataPath, DATA_STATIC_PAGE_DIRECTORY), { + filterFile: name => path.extname(name) === '.yaml', + prefixPath: DATA_STATIC_PAGE_DIRECTORY, + }), + + documentMode: onePerFile, + documentThing: StaticPage, + + save: (results) => ({staticPageData: results}), + + sort({staticPageData}) { + sortAlphabetically(staticPageData); + }, + }); +} diff --git a/src/data/things/track.js b/src/data/things/track.js new file mode 100644 index 0000000..cc49fc2 --- /dev/null +++ b/src/data/things/track.js @@ -0,0 +1,566 @@ +import {inspect} from 'node:util'; + +import CacheableObject from '#cacheable-object'; +import {colors} from '#cli'; +import {input} from '#composite'; +import find from '#find'; +import Thing from '#thing'; +import {isColor, isContributionList, isDate, isFileExtension} + from '#validators'; + +import { + parseAdditionalFiles, + parseAdditionalNames, + parseContributors, + parseDate, + parseDimensions, + parseDuration, +} from '#yaml'; + +import {withPropertyFromObject} from '#composite/data'; +import {withResolvedContribs} from '#composite/wiki-data'; + +import { + exitWithoutDependency, + exposeConstant, + exposeDependency, + exposeDependencyOrContinue, + exposeUpdateValueOrContinue, +} from '#composite/control-flow'; + +import { + additionalFiles, + additionalNameList, + commentary, + commentatorArtists, + contentString, + contributionList, + dimensions, + directory, + duration, + flag, + name, + referenceList, + reverseReferenceList, + simpleDate, + singleReference, + simpleString, + urls, + wikiData, +} from '#composite/wiki-properties'; + +import { + exitWithoutUniqueCoverArt, + inferredAdditionalNameList, + inheritFromOriginalRelease, + sharedAdditionalNameList, + trackReverseReferenceList, + withAlbum, + withAlwaysReferenceByDirectory, + withContainingTrackSection, + withHasUniqueCoverArt, + withOtherReleases, + withPropertyFromAlbum, +} from '#composite/things/track'; + +export class Track extends Thing { + static [Thing.referenceType] = 'track'; + + static [Thing.getPropertyDescriptors] = ({Album, ArtTag, Artist, Flash}) => ({ + // Update & expose + + name: name('Unnamed Track'), + directory: directory(), + + additionalNames: additionalNameList(), + sharedAdditionalNames: sharedAdditionalNameList(), + inferredAdditionalNames: inferredAdditionalNameList(), + + bandcampTrackIdentifier: simpleString(), + bandcampArtworkIdentifier: simpleString(), + + duration: duration(), + urls: urls(), + dateFirstReleased: simpleDate(), + + color: [ + exposeUpdateValueOrContinue({ + validate: input.value(isColor), + }), + + withContainingTrackSection(), + + withPropertyFromObject({ + object: '#trackSection', + property: input.value('color'), + }), + + exposeDependencyOrContinue({dependency: '#trackSection.color'}), + + withPropertyFromAlbum({ + property: input.value('color'), + }), + + exposeDependency({dependency: '#album.color'}), + ], + + alwaysReferenceByDirectory: [ + withAlwaysReferenceByDirectory(), + exposeDependency({dependency: '#alwaysReferenceByDirectory'}), + ], + + // Disables presenting the track as though it has its own unique artwork. + // This flag should only be used in select circumstances, i.e. to override + // an album's trackCoverArtists. This flag supercedes that property, as well + // as the track's own coverArtists. + disableUniqueCoverArt: flag(), + + // File extension for track's corresponding media file. This represents the + // track's unique cover artwork, if any, and does not inherit the extension + // of the album's main artwork. It does inherit trackCoverArtFileExtension, + // if present on the album. + coverArtFileExtension: [ + exitWithoutUniqueCoverArt(), + + exposeUpdateValueOrContinue({ + validate: input.value(isFileExtension), + }), + + withPropertyFromAlbum({ + property: input.value('trackCoverArtFileExtension'), + }), + + exposeDependencyOrContinue({dependency: '#album.trackCoverArtFileExtension'}), + + exposeConstant({ + value: input.value('jpg'), + }), + ], + + // Date of cover art release. Like coverArtFileExtension, this represents + // only the track's own unique cover artwork, if any. This exposes only as + // the track's own coverArtDate or its album's trackArtDate, so if neither + // is specified, this value is null. + coverArtDate: [ + withHasUniqueCoverArt(), + + exitWithoutDependency({ + dependency: '#hasUniqueCoverArt', + mode: input.value('falsy'), + }), + + exposeUpdateValueOrContinue({ + validate: input.value(isDate), + }), + + withPropertyFromAlbum({ + property: input.value('trackArtDate'), + }), + + exposeDependency({dependency: '#album.trackArtDate'}), + ], + + coverArtDimensions: [ + exitWithoutUniqueCoverArt(), + dimensions(), + ], + + commentary: commentary(), + + lyrics: [ + inheritFromOriginalRelease({ + property: input.value('lyrics'), + }), + + contentString(), + ], + + additionalFiles: additionalFiles(), + sheetMusicFiles: additionalFiles(), + midiProjectFiles: additionalFiles(), + + originalReleaseTrack: singleReference({ + class: input.value(Track), + find: input.value(find.track), + data: 'trackData', + }), + + // Internal use only - for directly identifying an album inside a track's + // util.inspect display, if it isn't indirectly available (by way of being + // included in an album's track list). + dataSourceAlbum: singleReference({ + class: input.value(Album), + find: input.value(find.album), + data: 'albumData', + }), + + artistContribs: [ + inheritFromOriginalRelease({ + property: input.value('artistContribs'), + notFoundValue: input.value([]), + }), + + withResolvedContribs({ + from: input.updateValue({validate: isContributionList}), + }).outputs({ + '#resolvedContribs': '#artistContribs', + }), + + exposeDependencyOrContinue({ + dependency: '#artistContribs', + mode: input.value('empty'), + }), + + withPropertyFromAlbum({ + property: input.value('artistContribs'), + }), + + exposeDependency({dependency: '#album.artistContribs'}), + ], + + contributorContribs: [ + inheritFromOriginalRelease({ + property: input.value('contributorContribs'), + notFoundValue: input.value([]), + }), + + contributionList(), + ], + + // Cover artists aren't inherited from the original release, since it + // typically varies by release and isn't defined by the musical qualities + // of the track. + coverArtistContribs: [ + exitWithoutUniqueCoverArt({ + value: input.value([]), + }), + + withResolvedContribs({ + from: input.updateValue({validate: isContributionList}), + }).outputs({ + '#resolvedContribs': '#coverArtistContribs', + }), + + exposeDependencyOrContinue({ + dependency: '#coverArtistContribs', + mode: input.value('empty'), + }), + + withPropertyFromAlbum({ + property: input.value('trackCoverArtistContribs'), + }), + + exposeDependency({dependency: '#album.trackCoverArtistContribs'}), + ], + + referencedTracks: [ + inheritFromOriginalRelease({ + property: input.value('referencedTracks'), + notFoundValue: input.value([]), + }), + + referenceList({ + class: input.value(Track), + find: input.value(find.track), + data: 'trackData', + }), + ], + + sampledTracks: [ + inheritFromOriginalRelease({ + property: input.value('sampledTracks'), + notFoundValue: input.value([]), + }), + + referenceList({ + class: input.value(Track), + find: input.value(find.track), + data: 'trackData', + }), + ], + + artTags: [ + exitWithoutUniqueCoverArt({ + value: input.value([]), + }), + + referenceList({ + class: input.value(ArtTag), + find: input.value(find.artTag), + data: 'artTagData', + }), + ], + + // Update only + + albumData: wikiData({ + class: input.value(Album), + }), + + artistData: wikiData({ + class: input.value(Artist), + }), + + artTagData: wikiData({ + class: input.value(ArtTag), + }), + + flashData: wikiData({ + class: input.value(Flash), + }), + + trackData: wikiData({ + class: input.value(Track), + }), + + // Expose only + + commentatorArtists: commentatorArtists(), + + album: [ + withAlbum(), + exposeDependency({dependency: '#album'}), + ], + + date: [ + exposeDependencyOrContinue({dependency: 'dateFirstReleased'}), + + withPropertyFromAlbum({ + property: input.value('date'), + }), + + exposeDependency({dependency: '#album.date'}), + ], + + hasUniqueCoverArt: [ + withHasUniqueCoverArt(), + exposeDependency({dependency: '#hasUniqueCoverArt'}), + ], + + otherReleases: [ + withOtherReleases(), + exposeDependency({dependency: '#otherReleases'}), + ], + + referencedByTracks: trackReverseReferenceList({ + list: input.value('referencedTracks'), + }), + + sampledByTracks: trackReverseReferenceList({ + list: input.value('sampledTracks'), + }), + + featuredInFlashes: reverseReferenceList({ + data: 'flashData', + list: input.value('featuredTracks'), + }), + }); + + static [Thing.yamlDocumentSpec] = { + fields: { + 'Track': {property: 'name'}, + 'Directory': {property: 'directory'}, + + 'Additional Names': { + property: 'additionalNames', + transform: parseAdditionalNames, + }, + + 'Bandcamp Track ID': { + property: 'bandcampTrackIdentifier', + transform: String, + }, + + 'Bandcamp Artwork ID': { + property: 'bandcampArtworkIdentifier', + transform: String, + }, + + 'Duration': { + property: 'duration', + transform: parseDuration, + }, + + 'Color': {property: 'color'}, + 'URLs': {property: 'urls'}, + + 'Date First Released': { + property: 'dateFirstReleased', + transform: parseDate, + }, + + 'Cover Art Date': { + property: 'coverArtDate', + transform: parseDate, + }, + + 'Cover Art File Extension': {property: 'coverArtFileExtension'}, + + 'Cover Art Dimensions': { + property: 'coverArtDimensions', + transform: parseDimensions, + }, + + 'Has Cover Art': { + property: 'disableUniqueCoverArt', + transform: value => + (typeof value === 'boolean' + ? !value + : value), + }, + + 'Always Reference By Directory': {property: 'alwaysReferenceByDirectory'}, + + 'Lyrics': {property: 'lyrics'}, + 'Commentary': {property: 'commentary'}, + + 'Additional Files': { + property: 'additionalFiles', + transform: parseAdditionalFiles, + }, + + 'Sheet Music Files': { + property: 'sheetMusicFiles', + transform: parseAdditionalFiles, + }, + + 'MIDI Project Files': { + property: 'midiProjectFiles', + transform: parseAdditionalFiles, + }, + + 'Originally Released As': {property: 'originalReleaseTrack'}, + 'Referenced Tracks': {property: 'referencedTracks'}, + 'Sampled Tracks': {property: 'sampledTracks'}, + + 'Franchises': {ignore: true}, + 'Inherit Franchises': {ignore: true}, + + 'Artists': { + property: 'artistContribs', + transform: parseContributors, + }, + + 'Contributors': { + property: 'contributorContribs', + transform: parseContributors, + }, + + 'Cover Artists': { + property: 'coverArtistContribs', + transform: parseContributors, + }, + + 'Art Tags': {property: 'artTags'}, + + 'Review Points': {ignore: true}, + }, + + invalidFieldCombinations: [ + {message: `Rereleases inherit references from the original`, fields: [ + 'Originally Released As', + 'Referenced Tracks', + ]}, + + {message: `Rereleases inherit samples from the original`, fields: [ + 'Originally Released As', + 'Sampled Tracks', + ]}, + + {message: `Rereleases inherit artists from the original`, fields: [ + 'Originally Released As', + 'Artists', + ]}, + + {message: `Rereleases inherit contributors from the original`, fields: [ + 'Originally Released As', + 'Contributors', + ]}, + + {message: `Rereleases inherit lyrics from the original`, fields: [ + 'Originally Released As', + 'Lyrics', + ]}, + + { + message: ({'Has Cover Art': hasCoverArt}) => + (hasCoverArt + ? `"Has Cover Art: true" is inferred from cover artist credits` + : `Tracks without cover art must not have cover artist credits`), + + fields: [ + 'Has Cover Art', + 'Cover Artists', + ], + }, + ], + }; + + static [Thing.findSpecs] = { + track: { + referenceTypes: ['track'], + bindTo: 'trackData', + + getMatchableNames: track => + (track.alwaysReferenceByDirectory + ? [] + : [track.name]), + }, + + trackOriginalReleasesOnly: { + referenceTypes: ['track'], + bindTo: 'trackData', + + include: track => + !CacheableObject.getUpdateValue(track, 'originalReleaseTrack'), + + // It's still necessary to check alwaysReferenceByDirectory here, since + // it may be set manually (with `Always Reference By Directory: true`), + // and these shouldn't be matched by name (as per usual). + // See the definition for that property for more information. + getMatchableNames: track => + (track.alwaysReferenceByDirectory + ? [] + : [track.name]), + }, + }; + + // Track YAML loading is handled in album.js. + static [Thing.getYamlLoadingSpec] = null; + + [inspect.custom](depth) { + const parts = []; + + parts.push(Thing.prototype[inspect.custom].apply(this)); + + if (CacheableObject.getUpdateValue(this, 'originalReleaseTrack')) { + parts.unshift(`${colors.yellow('[rerelease]')} `); + } + + let album; + + if (depth >= 0) { + try { + album = this.album; + } catch (_error) { + // Computing album might crash for any reason, which we don't want to + // distract from another error we might be trying to work out at the + // moment (for which debugging might involve inspecting this track!). + } + + album ??= this.dataSourceAlbum; + } + + if (album) { + const albumName = album.name; + const albumIndex = album.tracks.indexOf(this); + const trackNum = + (albumIndex === -1 + ? 'indeterminate position' + : `#${albumIndex + 1}`); + parts.push(` (${colors.yellow(trackNum)} in ${colors.green(albumName)})`); + } + + return parts.join(''); + } +} diff --git a/src/data/things/wiki-info.js b/src/data/things/wiki-info.js new file mode 100644 index 0000000..316bd3b --- /dev/null +++ b/src/data/things/wiki-info.js @@ -0,0 +1,110 @@ +export const WIKI_INFO_FILE = 'wiki-info.yaml'; + +import {input} from '#composite'; +import find from '#find'; +import Thing from '#thing'; +import {isColor, isLanguageCode, isName, isURL} from '#validators'; + +import {contentString, flag, name, referenceList, wikiData} + from '#composite/wiki-properties'; + +export class WikiInfo extends Thing { + static [Thing.friendlyName] = `Wiki Info`; + + static [Thing.getPropertyDescriptors] = ({Group}) => ({ + // Update & expose + + name: name('Unnamed Wiki'), + + // Displayed in nav bar. + nameShort: { + flags: {update: true, expose: true}, + update: {validate: isName}, + + expose: { + dependencies: ['name'], + transform: (value, {name}) => value ?? name, + }, + }, + + color: { + flags: {update: true, expose: true}, + update: {validate: isColor}, + + expose: { + transform: color => color ?? '#0088ff', + }, + }, + + // One-line description used for <meta rel="description"> tag. + description: contentString(), + + footerContent: contentString(), + + defaultLanguage: { + flags: {update: true, expose: true}, + update: {validate: isLanguageCode}, + }, + + canonicalBase: { + flags: {update: true, expose: true}, + update: {validate: isURL}, + }, + + divideTrackListsByGroups: referenceList({ + class: input.value(Group), + find: input.value(find.group), + data: 'groupData', + }), + + // Feature toggles + enableFlashesAndGames: flag(false), + enableListings: flag(false), + enableNews: flag(false), + enableArtTagUI: flag(false), + enableGroupUI: flag(false), + + // Update only + + groupData: wikiData({ + class: input.value(Group), + }), + }); + + static [Thing.yamlDocumentSpec] = { + fields: { + 'Name': {property: 'name'}, + 'Short Name': {property: 'nameShort'}, + 'Color': {property: 'color'}, + 'Description': {property: 'description'}, + 'Footer Content': {property: 'footerContent'}, + 'Default Language': {property: 'defaultLanguage'}, + 'Canonical Base': {property: 'canonicalBase'}, + 'Divide Track Lists By Groups': {property: 'divideTrackListsByGroups'}, + 'Enable Flashes & Games': {property: 'enableFlashesAndGames'}, + 'Enable Listings': {property: 'enableListings'}, + 'Enable News': {property: 'enableNews'}, + 'Enable Art Tag UI': {property: 'enableArtTagUI'}, + 'Enable Group UI': {property: 'enableGroupUI'}, + }, + }; + + static [Thing.getYamlLoadingSpec] = ({ + documentModes: {oneDocumentTotal}, + thingConstructors: {WikiInfo}, + }) => ({ + title: `Process wiki info file`, + file: WIKI_INFO_FILE, + + documentMode: oneDocumentTotal, + documentThing: WikiInfo, + + save(wikiInfo) { + if (!wikiInfo) { + return; + } + + return {wikiInfo}; + }, + }); +} diff --git a/src/data/validators.js b/src/data/validators.js index 0d325ae..987f806 100644 --- a/src/data/validators.js +++ b/src/data/validators.js @@ -1,367 +1,997 @@ -import { withAggregate } from '../util/sugar.js'; +import {inspect as nodeInspect} from 'node:util'; -import { color, ENABLE_COLOR, decorateTime } from '../util/cli.js'; - -import { inspect as nodeInspect } from 'util'; +import {openAggregate, withAggregate} from '#aggregate'; +import {colors, ENABLE_COLOR} from '#cli'; +import {cut, empty, matchMultiline, typeAppearance} from '#sugar'; +import {commentaryRegexCaseInsensitive, commentaryRegexCaseSensitiveOneShot} + from '#wiki-data'; function inspect(value) { - return nodeInspect(value, {colors: ENABLE_COLOR}); + return nodeInspect(value, {colors: ENABLE_COLOR}); +} + +export function getValidatorCreator(validator) { + return validator[Symbol.for(`hsmusic.validator.creator`)] ?? null; +} + +export function getValidatorCreatorMeta(validator) { + return validator[Symbol.for(`hsmusic.validator.creatorMeta`)] ?? null; +} + +export function setValidatorCreatorMeta(validator, creator, meta) { + validator[Symbol.for(`hsmusic.validator.creator`)] = creator; + validator[Symbol.for(`hsmusic.validator.creatorMeta`)] = meta; + return validator; } // Basic types (primitives) -function a(noun) { - return (/[aeiou]/.test(noun[0]) ? `an ${noun}` : `a ${noun}`); +export function a(noun) { + return /[aeiou]/.test(noun[0]) ? `an ${noun}` : `a ${noun}`; } -function isType(value, type) { +export function validateType(type) { + const fn = value => { if (typeof value !== type) - throw new TypeError(`Expected ${a(type)}, got ${typeof value}`); + throw new TypeError(`Expected ${a(type)}, got ${typeAppearance(value)}`); return true; -} + }; -export function isBoolean(value) { - return isType(value, 'boolean'); -} + setValidatorCreatorMeta(fn, validateType, {type}); -export function isNumber(value) { - return isType(value, 'number'); + return fn; } +export const isBoolean = + validateType('boolean'); + +export const isFunction = + validateType('function'); + +export const isNumber = + validateType('number'); + +export const isString = + validateType('string'); + +export const isSymbol = + validateType('symbol'); + +// Use isObject instead, which disallows null. +export const isTypeofObject = + validateType('object'); + export function isPositive(number) { - isNumber(number); + isNumber(number); - if (number <= 0) - throw new TypeError(`Expected positive number`); + if (number <= 0) throw new TypeError(`Expected positive number`); - return true; + return true; } export function isNegative(number) { - isNumber(number); + isNumber(number); - if (number >= 0) - throw new TypeError(`Expected negative number`); + if (number >= 0) throw new TypeError(`Expected negative number`); - return true; + return true; } export function isPositiveOrZero(number) { - isNumber(number); + isNumber(number); - if (number < 0) - throw new TypeError(`Expected positive number or zero`); + if (number < 0) throw new TypeError(`Expected positive number or zero`); - return true; + return true; } export function isNegativeOrZero(number) { - isNumber(number); + isNumber(number); - if (number > 0) - throw new TypeError(`Expected negative number or zero`); + if (number > 0) throw new TypeError(`Expected negative number or zero`); - return true; + return true; } export function isInteger(number) { - isNumber(number); + isNumber(number); - if (number % 1 !== 0) - throw new TypeError(`Expected integer`); + if (number % 1 !== 0) throw new TypeError(`Expected integer`); - return true; + return true; } export function isCountingNumber(number) { - isInteger(number); - isPositive(number); + isInteger(number); + isPositive(number); - return true; + return true; } export function isWholeNumber(number) { - isInteger(number); - isPositiveOrZero(number); - - return true; -} + isInteger(number); + isPositiveOrZero(number); -export function isString(value) { - return isType(value, 'string'); + return true; } export function isStringNonEmpty(value) { - isString(value); + isString(value); - if (value.trim().length === 0) - throw new TypeError(`Expected non-empty string`); + if (value.trim().length === 0) + throw new TypeError(`Expected non-empty string`); - return true; + return true; +} + +export function optional(validator) { + return value => + value === null || + value === undefined || + validator(value); } // Complex types (non-primitives) export function isInstance(value, constructor) { - isObject(value); + isObject(value); - if (!(value instanceof constructor)) - throw new TypeError(`Expected ${constructor.name}, got ${value.constructor.name}`); + if (!(value instanceof constructor)) + throw new TypeError(`Expected ${constructor.name}, got ${value.constructor.name}`); - return true; + return true; } export function isDate(value) { - return isInstance(value, Date); + isInstance(value, Date); + + if (isNaN(value)) + throw new TypeError(`Expected valid date`); + + return true; } export function isObject(value) { - isType(value, 'object'); + isTypeofObject(value); - // Note: Please remember that null is always a valid value for properties - // held by a CacheableObject. This assertion is exclusively for use in other - // contexts. - if (value === null) - throw new TypeError(`Expected an object, got null`); + // Note: Please remember that null is always a valid value for properties + // held by a CacheableObject. This assertion is exclusively for use in other + // contexts. + if (value === null) + throw new TypeError(`Expected an object, got null`); - return true; + return true; } export function isArray(value) { - if (typeof value !== 'object' || value === null || !Array.isArray(value)) - throw new TypeError(`Expected an array, got ${value}`); + if (typeof value !== 'object' || value === null || !Array.isArray(value)) + throw new TypeError(`Expected an array, got ${typeAppearance(value)}`); + + return true; +} + +// This one's shaped a bit different from other "is" functions. +// More like validate functions, it returns a function. +export function is(...values) { + if (Array.isArray(values)) { + values = new Set(values); + } + + if (values.size === 1) { + const expected = Array.from(values)[0]; + + return (value) => { + if (value !== expected) { + throw new TypeError(`Expected ${expected}, got ${value}`); + } + + return true; + }; + } + + const fn = (value) => { + if (!values.has(value)) { + throw new TypeError(`Expected one of ${Array.from(values).join(' ')}, got ${value}`); + } return true; + }; + + setValidatorCreatorMeta(fn, is, {values}); + + return fn; } function validateArrayItemsHelper(itemValidator) { - return (item, index) => { - try { - const value = itemValidator(item); - - if (value !== true) { - throw new Error(`Expected validator to return true`); - } - } catch (error) { - error.message = `(index: ${color.green(index)}, item: ${inspect(item)}) ${error.message}`; - throw error; - } - }; + return (item, index, array) => { + try { + const value = itemValidator(item, index, array); + + if (value !== true) { + throw new Error(`Expected validator to return true`); + } + } catch (caughtError) { + const indexPart = colors.yellow(`zero-index ${index}`) + const itemPart = inspect(item); + const message = `Error at ${indexPart}: ${itemPart}`; + const error = new Error(message, {cause: caughtError}); + error[Symbol.for('hsmusic.annotateError.indexInSourceArray')] = index; + throw error; + } + }; } export function validateArrayItems(itemValidator) { - const fn = validateArrayItemsHelper(itemValidator); + const helper = validateArrayItemsHelper(itemValidator); - return array => { - isArray(array); + return (array) => { + isArray(array); - withAggregate({message: 'Errors validating array items'}, ({ wrap }) => { - array.forEach(wrap(fn)); - }); + withAggregate({message: 'Errors validating array items'}, ({call}) => { + for (let index = 0; index < array.length; index++) { + call(helper, array[index], index, array); + } + }); - return true; - }; + return true; + }; +} + +export function strictArrayOf(itemValidator) { + return validateArrayItems(itemValidator); +} + +export function sparseArrayOf(itemValidator) { + return validateArrayItems((item, index, array) => { + if (item === false || item === null) { + return true; + } + + return itemValidator(item, index, array); + }); +} + +export function looseArrayOf(itemValidator) { + return validateArrayItems((item, index, array) => { + if (item === false || item === null || item === undefined) { + return true; + } + + return itemValidator(item, index, array); + }); } export function validateInstanceOf(constructor) { - return object => isInstance(object, constructor); + const fn = (object) => isInstance(object, constructor); + + setValidatorCreatorMeta(fn, validateInstanceOf, {constructor}); + + return fn; } // Wiki data (primitives & non-primitives) export function isColor(color) { - isStringNonEmpty(color); + isStringNonEmpty(color); - if (color.startsWith('#')) { - if (![1 + 3, 1 + 4, 1 + 6, 1 + 8].includes(color.length)) - throw new TypeError(`Expected #rgb, #rgba, #rrggbb, or #rrggbbaa, got length ${color.length}`); + if (color.startsWith('#')) { + if (![4, 5, 7, 9].includes(color.length)) + throw new TypeError(`Expected #rgb, #rgba, #rrggbb, or #rrggbbaa, got length ${color.length}`); - if (/[^0-9a-fA-F]/.test(color.slice(1))) - throw new TypeError(`Expected hexadecimal digits`); + if (/[^0-9a-fA-F]/.test(color.slice(1))) + throw new TypeError(`Expected hexadecimal digits`); - return true; - } + return true; + } - throw new TypeError(`Unknown color format`); + throw new TypeError(`Unknown color format`); } -export function isCommentary(commentary) { - return isString(commentary); +export function isCommentary(commentaryText) { + isContentString(commentaryText); + + const rawMatches = + Array.from(commentaryText.matchAll(commentaryRegexCaseInsensitive)); + + if (empty(rawMatches)) { + throw new TypeError(`Expected at least one commentary heading`); + } + + const niceMatches = + rawMatches.map(match => ({ + position: match.index, + length: match[0].length, + })); + + validateArrayItems(({position, length}, index) => { + if (index === 0 && position > 0) { + throw new TypeError(`Expected first commentary heading to be at top`); + } + + const ownInput = commentaryText.slice(position, position + length); + const restOfInput = commentaryText.slice(position + length); + + const upToNextLineBreak = + (restOfInput.includes('\n') + ? restOfInput.slice(0, restOfInput.indexOf('\n')) + : restOfInput); + + if (/\S/.test(upToNextLineBreak)) { + throw new TypeError( + `Expected commentary heading to occupy entire line, got extra text:\n` + + `${colors.green(`"${cut(ownInput, 40)}"`)} (<- heading)\n` + + `(extra on same line ->) ${colors.red(`"${cut(upToNextLineBreak, 30)}"`)}\n` + + `(Check for missing "|-" in YAML, or a misshapen annotation)`); + } + + if (!commentaryRegexCaseSensitiveOneShot.test(ownInput)) { + throw new TypeError( + `Miscapitalization in commentary heading:\n` + + `${colors.red(`"${cut(ownInput, 60)}"`)}\n` + + `(Check for ${colors.red(`"<I>"`)} instead of ${colors.green(`"<i>"`)})`); + } + + const nextHeading = + (index === niceMatches.length - 1 + ? commentaryText.length + : niceMatches[index + 1].position); + + const upToNextHeading = + commentaryText.slice(position + length, nextHeading); + + if (!/\S/.test(upToNextHeading)) { + throw new TypeError( + `Expected commentary entry to have body text, only got a heading`); + } + + return true; + })(niceMatches); + + return true; } const isArtistRef = validateReference('artist'); export function validateProperties(spec) { - const specEntries = Object.entries(spec); - const specKeys = Object.keys(spec); - - return object => { - isObject(object); - - if (Array.isArray(object)) - throw new TypeError(`Expected an object, got array`); - - withAggregate({message: `Errors validating object properties`}, ({ call }) => { - for (const [ specKey, specValidator ] of specEntries) { - call(() => { - const value = object[specKey]; - try { - specValidator(value); - } catch (error) { - error.message = `(key: ${color.green(specKey)}, value: ${inspect(value)}) ${error.message}`; - throw error; - } - }); - } - - const unknownKeys = Object.keys(object).filter(key => !specKeys.includes(key)); - if (unknownKeys.length > 0) { - call(() => { - throw new Error(`Unknown keys present (${unknownKeys.length}): [${unknownKeys.join(', ')}]`); - }); - } - }); + const { + [validateProperties.validateOtherKeys]: validateOtherKeys = null, + [validateProperties.allowOtherKeys]: allowOtherKeys = false, + } = spec; - return true; - }; -} + const specEntries = Object.entries(spec); + const specKeys = Object.keys(spec); + + return (object) => { + isObject(object); + if (Array.isArray(object)) + throw new TypeError(`Expected an object, got array`); + + withAggregate({message: `Errors validating object properties`}, ({push}) => { + const testEntries = specEntries.slice(); + + const unknownKeys = Object.keys(object).filter((key) => !specKeys.includes(key)); + if (validateOtherKeys) { + for (const key of unknownKeys) { + testEntries.push([key, validateOtherKeys]); + } + } + + for (const [specKey, specValidator] of testEntries) { + const value = object[specKey]; + try { + specValidator(value); + } catch (caughtError) { + const keyPart = colors.green(specKey); + const valuePart = inspect(value); + const message = `Error for key ${keyPart}: ${valuePart}`; + push(new Error(message, {cause: caughtError})); + } + } + + if (!validateOtherKeys && !allowOtherKeys && !empty(unknownKeys)) { + push(new Error( + `Unknown keys present (${unknownKeys.length}): [${unknownKeys.join(', ')}]`)); + } + }); + + return true; + }; +} + +validateProperties.validateOtherKeys = Symbol(); +validateProperties.allowOtherKeys = Symbol(); + +export const validateAllPropertyValues = (validator) => + validateProperties({ + [validateProperties.validateOtherKeys]: validator, + }); + +const illeaglInvisibleSpace = { + action: 'delete', +}; + +const illegalVisibleSpace = { + action: 'replace', + with: ' ', + withAnnotation: `normal space`, +}; + +const illegalContentSpec = [ + {illegal: '\u200b', annotation: `zero-width space`, ...illeaglInvisibleSpace}, + {illegal: '\u2005', annotation: `four-per-em space`, ...illegalVisibleSpace}, + {illegal: '\u205f', annotation: `medium mathematical space`, ...illegalVisibleSpace}, + {illegal: '\xa0', annotation: `non-breaking space`, ...illegalVisibleSpace}, + + { + action: 'replace', + illegal: '<a href', + annotation: `HTML-style link`, + with: '[...](...)', + withAnnotation: `markdown`, + }, +]; + +for (const entry of illegalContentSpec) { + entry.test = string => + string.startsWith(entry.illegal); + + if (entry.action === 'replace') { + entry.enact = string => + string.replaceAll(entry.illegal, entry.with); + } +} + +const illegalContentRegexp = + new RegExp( + illegalContentSpec + .map(entry => entry.illegal) + .map(illegal => `${illegal}+`) + .join('|'), + 'g'); + +const illegalCharactersInContent = + illegalContentSpec + .map(entry => entry.illegal) + .join(''); + +const legalContentNearEndRegexp = + new RegExp(`[^\n${illegalCharactersInContent}]+$`); + +const legalContentNearStartRegexp = + new RegExp(`^[^\n${illegalCharactersInContent}]+`); + +const trimWhitespaceNearBothSidesRegexp = + /^ +| +$/gm; + +const trimWhitespaceNearEndRegexp = + / +$/gm; + +export function isContentString(content) { + isStringNonEmpty(content); + + const mainAggregate = openAggregate({ + message: `Errors validating content string`, + translucent: 'single', + }); + + const illegalAggregate = openAggregate({ + message: `Illegal characters found in content string`, + }); + + for (const {match, where} of matchMultiline(content, illegalContentRegexp)) { + const {annotation, action, ...options} = + illegalContentSpec + .find(entry => entry.test(match[0])); + + const matchStart = match.index; + const matchEnd = match.index + match[0].length; + + const before = + content + .slice(Math.max(0, matchStart - 3), matchStart) + .match(legalContentNearEndRegexp) + ?.[0]; + + const after = + content + .slice(matchEnd, Math.min(content.length, matchEnd + 3)) + .match(legalContentNearStartRegexp) + ?.[0]; + + const beforePart = + before && `"${before}"`; + + const afterPart = + after && `"${after}"`; + + const surroundings = + (before && after + ? `between ${beforePart} and ${afterPart}` + : before + ? `after ${beforePart}` + : after + ? `before ${afterPart}` + : ``); + + const illegalPart = + colors.red( + (annotation + ? `"${match[0]}" (${annotation})` + : `"${match[0]}"`)); + + const replacement = + (action === 'replace' + ? options.enact(match[0]) + : null); + + const replaceWithPart = + (action === 'replace' + ? colors.green( + (options.withAnnotation + ? `"${replacement}" (${options.withAnnotation})` + : `"${replacement}"`)) + : null); + + const actionPart = + (action === `delete` + ? `Delete ${illegalPart}` + : action === 'replace' + ? `Replace ${illegalPart} with ${replaceWithPart}` + : `Matched ${illegalPart}`); + + const parts = [ + actionPart, + surroundings, + `(${where})`, + ].filter(Boolean); + + illegalAggregate.push(new TypeError(parts.join(` `))); + } + + const isMultiline = content.includes('\n'); + + const trimWhitespaceAggregate = openAggregate({ + message: + (isMultiline + ? `Whitespace found at end of line` + : `Whitespace found at start or end`), + }); + + const trimWhitespaceRegexp = + (isMultiline + ? trimWhitespaceNearEndRegexp + : trimWhitespaceNearBothSidesRegexp); + + for ( + const {match, lineNumber, columnNumber, containingLine} of + matchMultiline(content, trimWhitespaceRegexp, { + formatWhere: false, + getContainingLine: true, + }) + ) { + const linePart = + colors.yellow(`line ${lineNumber + 1}`); + + const where = + (match[0].length === containingLine.length + ? `as all of ${linePart}` + : columnNumber === 0 + ? (isMultiline + ? `at start of ${linePart}` + : `at start`) + : (isMultiline + ? `at end of ${linePart}` + : `at end`)); + + const whitespacePart = + colors.red(`"${match[0]}"`); + + const parts = [ + `Matched ${whitespacePart}`, + where, + ]; + + trimWhitespaceAggregate.push(new TypeError(parts.join(` `))); + } + + mainAggregate.call(() => illegalAggregate.close()); + mainAggregate.call(() => trimWhitespaceAggregate.close()); + mainAggregate.close(); + + return true; +} + +export function isThingClass(thingClass) { + isFunction(thingClass); + + if (!Object.hasOwn(thingClass, Symbol.for('Thing.referenceType'))) { + throw new TypeError(`Expected a Thing constructor, missing Thing.referenceType`); + } + + return true; +} export const isContribution = validateProperties({ - who: isArtistRef, - what: value => value === undefined || value === null || isStringNonEmpty(value), + who: isArtistRef, + what: optional(isStringNonEmpty), }); export const isContributionList = validateArrayItems(isContribution); export const isAdditionalFile = validateProperties({ - title: isString, - description: value => (value === undefined || value === null || isString(value)), - files: validateArrayItems(isString) + title: isName, + description: optional(isContentString), + files: optional(validateArrayItems(isString)), }); export const isAdditionalFileList = validateArrayItems(isAdditionalFile); +export const isTrackSection = validateProperties({ + name: optional(isName), + color: optional(isColor), + dateOriginallyReleased: optional(isDate), + isDefaultTrackSection: optional(isBoolean), + tracks: optional(validateReferenceList('track')), +}); + +export const isTrackSectionList = validateArrayItems(isTrackSection); + export function isDimensions(dimensions) { - isArray(dimensions); + isArray(dimensions); - if (dimensions.length !== 2) - throw new TypeError(`Expected 2 item array`); + if (dimensions.length !== 2) throw new TypeError(`Expected 2 item array`); + if (dimensions[0] !== null) { isPositive(dimensions[0]); isInteger(dimensions[0]); + } + + if (dimensions[1] !== null) { isPositive(dimensions[1]); isInteger(dimensions[1]); + } - return true; + return true; } export function isDirectory(directory) { - isStringNonEmpty(directory); + isStringNonEmpty(directory); - if (directory.match(/[^a-zA-Z0-9_\-]/)) - throw new TypeError(`Expected only letters, numbers, dash, and underscore, got "${directory}"`); + if (directory.match(/[^a-zA-Z0-9_-]/)) + throw new TypeError(`Expected only letters, numbers, dash, and underscore, got "${directory}"`); - return true; + return true; } export function isDuration(duration) { - isNumber(duration); - isPositiveOrZero(duration); + isNumber(duration); + isPositiveOrZero(duration); - return true; + return true; } export function isFileExtension(string) { - isStringNonEmpty(string); + isStringNonEmpty(string); - if (string[0] === '.') - throw new TypeError(`Expected no dot (.) at the start of file extension`); + if (string[0] === '.') + throw new TypeError(`Expected no dot (.) at the start of file extension`); - if (string.match(/[^a-zA-Z0-9_]/)) - throw new TypeError(`Expected only alphanumeric and underscore`); + if (string.match(/[^a-zA-Z0-9_]/)) + throw new TypeError(`Expected only alphanumeric and underscore`); - return true; + return true; } export function isLanguageCode(string) { - // TODO: This is a stub function because really we don't need a detailed - // is-language-code parser right now. + // TODO: This is a stub function because really we don't need a detailed + // is-language-code parser right now. - isString(string); + isString(string); - return true; + return true; } export function isName(name) { - return isString(name); + return isContentString(name); } export function isURL(string) { - isStringNonEmpty(string); + isStringNonEmpty(string); - new URL(string); + new URL(string); - return true; + return true; } export function validateReference(type = 'track') { - return ref => { - isStringNonEmpty(ref); + return (ref) => { + isStringNonEmpty(ref); - const match = ref.trim().match(/^(?:(?<typePart>\S+):(?=\S))?(?<directoryPart>.+)(?<!:)$/); + const match = ref + .trim() + .match(/^(?:(?<typePart>\S+):(?=\S))?(?<directoryPart>.+)(?<!:)$/); - if (!match) - throw new TypeError(`Malformed reference`); + if (!match) throw new TypeError(`Malformed reference`); - const { groups: { typePart, directoryPart } } = match; + const {groups: {typePart, directoryPart}} = match; - if (typePart && typePart !== type) - throw new TypeError(`Expected ref to begin with "${type}:", got "${typePart}:"`); + if (typePart) { + if (typePart !== type) + throw new TypeError(`Expected ref to begin with "${type}:", got "${typePart}:"`); - if (typePart) - isDirectory(directoryPart); + isDirectory(directoryPart); + } - isName(ref); + isName(ref); - return true; - }; + return true; + }; } export function validateReferenceList(type = '') { - return validateArrayItems(validateReference(type)); + return validateArrayItems(validateReference(type)); } +const validateWikiData_cache = {}; + +export function validateWikiData({ + referenceType = '', + allowMixedTypes = false, +}) { + if (referenceType && allowMixedTypes) { + throw new TypeError(`Don't specify both referenceType and allowMixedTypes`); + } + + validateWikiData_cache[referenceType] ??= {}; + validateWikiData_cache[referenceType][allowMixedTypes] ??= new WeakMap(); + + const isArrayOfObjects = validateArrayItems(isObject); + + return (array) => { + const subcache = validateWikiData_cache[referenceType][allowMixedTypes]; + if (subcache.has(array)) return subcache.get(array); + + let OK = false; + + try { + isArrayOfObjects(array); + + if (empty(array)) { + OK = true; return true; + } + + const allRefTypes = new Set(); + + let foundThing = false; + let foundOtherObject = false; + + for (const object of array) { + const {[Symbol.for('Thing.referenceType')]: referenceType} = object.constructor; + + if (referenceType === undefined) { + foundOtherObject = true; + + // Early-exit if a Thing has been found - nothing more can be learned. + if (foundThing) { + throw new TypeError(`Expected array of wiki data objects, got mixed items`); + } + } else { + foundThing = true; + + // Early-exit if a non-Thing object has been found - nothing more can + // be learned. + if (foundOtherObject) { + throw new TypeError(`Expected array of wiki data objects, got mixed items`); + } + + allRefTypes.add(referenceType); + } + } + + if (foundOtherObject && !foundThing) { + throw new TypeError(`Expected array of wiki data objects, got array of other objects`); + } + + if (allRefTypes.size > 1) { + if (allowMixedTypes) { + OK = true; return true; + } + + const types = () => Array.from(allRefTypes).join(', '); + + if (referenceType) { + if (allRefTypes.has(referenceType)) { + allRefTypes.remove(referenceType); + throw new TypeError(`Expected array of only ${referenceType}, also got other types: ${types()}`) + } else { + throw new TypeError(`Expected array of only ${referenceType}, got other types: ${types()}`); + } + } + + throw new TypeError(`Expected array of unmixed reference types, got multiple: ${types()}`); + } + + const onlyRefType = Array.from(allRefTypes)[0]; + + if (referenceType && onlyRefType !== referenceType) { + throw new TypeError(`Expected array of ${referenceType}, got array of ${onlyRefType}`) + } + + OK = true; return true; + } finally { + subcache.set(array, OK); + } + }; +} + +export const isAdditionalName = validateProperties({ + name: isName, + annotation: optional(isContentString), + + // TODO: This only allows indicating sourcing from a track. + // That's okay for the current limited use of "from", but + // could be expanded later. + from: + // Double TODO: Explicitly allowing both references and + // live objects to co-exist is definitely weird, and + // altogether questions the way we define validators... + optional(anyOf( + validateReferenceList('track'), + validateWikiData({referenceType: 'track'}))), +}); + +export const isAdditionalNameList = validateArrayItems(isAdditionalName); + // Compositional utilities -export function oneOf(...checks) { - return value => { - const errorMeta = []; +export function anyOf(...validators) { + const validConstants = new Set(); + const validConstructors = new Set(); + const validTypes = new Set(); - for (let i = 0, check; check = checks[i]; i++) { - try { - const result = check(value); + const constantValidators = []; + const constructorValidators = []; + const typeValidators = []; - if (result !== true) { - throw new Error(`Check returned false`); - } + const leftoverValidators = []; - return true; - } catch (error) { - errorMeta.push([check, i, error]); - } + for (const validator of validators) { + const creator = getValidatorCreator(validator); + const creatorMeta = getValidatorCreatorMeta(validator); + + switch (creator) { + case is: + for (const value of creatorMeta.values) { + validConstants.add(value); } - // Don't process error messages until every check has failed. - const errors = []; - for (const [ check, i, error ] of errorMeta) { - error.message = (check.name - ? `(#${i} "${check.name}") ${error.message}` - : `(#${i}) ${error.message}`); - error.check = check; - errors.push(error); + constantValidators.push(validator); + break; + + case validateInstanceOf: + validConstructors.add(creatorMeta.constructor); + constructorValidators.push(validator); + break; + + case validateType: + validTypes.add(creatorMeta.type); + typeValidators.push(validator); + break; + + default: + leftoverValidators.push(validator); + break; + } + } + + return (value) => { + const errorInfo = []; + + if (validConstants.has(value)) { + return true; + } + + if (!empty(validTypes)) { + if (validTypes.has(typeof value)) { + return true; + } + } + + for (const constructor of validConstructors) { + if (value instanceof constructor) { + return true; + } + } + + for (const [i, validator] of leftoverValidators.entries()) { + try { + const result = validator(value); + + if (result !== true) { + throw new Error(`Check returned false`); } - throw new AggregateError(errors, `Expected one of ${checks.length} possible checks, but none were true`); - }; + + return true; + } catch (error) { + errorInfo.push([validator, i, error]); + } + } + + // Don't process error messages until every validator has failed. + + const errors = []; + const prefaceErrorInfo = []; + + let offset = 0; + + if (!empty(validConstants)) { + const constants = + Array.from(validConstants); + + const gotPart = `, got ${value}`; + + prefaceErrorInfo.push([ + constantValidators, + offset++, + new TypeError( + `Expected any of ${constants.join(' ')}` + gotPart), + ]); + } + + if (!empty(validTypes)) { + const types = + Array.from(validTypes); + + const gotType = typeAppearance(value); + const gotPart = `, got ${gotType}`; + + prefaceErrorInfo.push([ + typeValidators, + offset++, + new TypeError( + `Expected any of ${types.join(', ')}` + gotPart), + ]); + } + + if (!empty(validConstructors)) { + const names = + Array.from(validConstructors) + .map(constructor => constructor.name); + + const gotName = value?.constructor?.name; + const gotPart = (gotName ? `, got ${gotName}` : ``); + + prefaceErrorInfo.push([ + constructorValidators, + offset++, + new TypeError( + `Expected any of ${names.join(', ')}` + gotPart), + ]); + } + + for (const info of errorInfo) { + info[1] += offset; + } + + for (const [validator, i, error] of prefaceErrorInfo.concat(errorInfo)) { + error.message = + (validator?.name + ? `${i + 1}. "${validator.name}": ${error.message}` + : `${i + 1}. ${error.message}`); + + error.check = + (Array.isArray(validator) && validator.length === 1 + ? validator[0] + : validator); + + errors.push(error); + } + + const total = offset + leftoverValidators.length; + throw new AggregateError(errors, + `Expected any of ${total} possible checks, ` + + `but none were true`); + }; } diff --git a/src/data/yaml.js b/src/data/yaml.js index 32cf729..86f3014 100644 --- a/src/data/yaml.js +++ b/src/data/yaml.js @@ -1,595 +1,494 @@ // yaml.js - specification for HSMusic YAML data file format and utilities for -// loading and processing YAML files and documents +// loading, processing, and validating YAML files and documents -import * as path from 'path'; -import yaml from 'js-yaml'; +import {readFile, stat} from 'node:fs/promises'; +import * as path from 'node:path'; +import {inspect as nodeInspect} from 'node:util'; -import { readFile } from 'fs/promises'; -import { inspect as nodeInspect } from 'util'; +import yaml from 'js-yaml'; -import { - Album, - Artist, - ArtTag, - Flash, - FlashAct, - Group, - GroupCategory, - HomepageLayout, - HomepageLayoutAlbumsRow, - HomepageLayoutRow, - NewsEntry, - StaticPage, - Thing, - Track, - TrackGroup, - WikiInfo, -} from './things.js'; +import {colors, ENABLE_COLOR, logInfo, logWarn} from '#cli'; +import {sortByName} from '#sort'; +import {atOffset, empty, filterProperties, typeAppearance, withEntries} + from '#sugar'; +import Thing from '#thing'; +import thingConstructors from '#things'; import { - color, - ENABLE_COLOR, - logInfo, - logWarn, -} from '../util/cli.js'; + filterReferenceErrors, + reportContentTextErrors, + reportDuplicateDirectories, +} from '#data-checks'; import { - decorateErrorWithIndex, - mapAggregate, - openAggregate, - showAggregate, - withAggregate, -} from '../util/sugar.js'; - -import { - sortByDate, - sortByName, -} from '../util/wiki-data.js'; - -import find, { bindFind } from '../util/find.js'; -import { findFiles } from '../util/io.js'; - -// --> General supporting stuff - -function inspect(value) { - return nodeInspect(value, {colors: ENABLE_COLOR}); + annotateErrorWithFile, + decorateErrorWithIndex, + decorateErrorWithAnnotation, + openAggregate, + showAggregate, + withAggregate, +} from '#aggregate'; + +function inspect(value, opts = {}) { + return nodeInspect(value, {colors: ENABLE_COLOR, ...opts}); } -// --> YAML data repository structure constants - -export const WIKI_INFO_FILE = 'wiki-info.yaml'; -export const BUILD_DIRECTIVE_DATA_FILE = 'build-directives.yaml'; -export const HOMEPAGE_LAYOUT_DATA_FILE = 'homepage.yaml'; -export const ARTIST_DATA_FILE = 'artists.yaml'; -export const FLASH_DATA_FILE = 'flashes.yaml'; -export const NEWS_DATA_FILE = 'news.yaml'; -export const ART_TAG_DATA_FILE = 'tags.yaml'; -export const GROUP_DATA_FILE = 'groups.yaml'; -export const STATIC_PAGE_DATA_FILE = 'static-pages.yaml'; - -export const DATA_ALBUM_DIRECTORY = 'album'; - -// --> Document processing functions - // General function for inputting a single document (usually loaded from YAML) // and outputting an instance of a provided Thing subclass. // // makeProcessDocument is a factory function: the returned function will take a // document and apply the configuration passed to makeProcessDocument in order // to construct a Thing subclass. -function makeProcessDocument(thingClass, { - // Optional early step for transforming field values before providing them - // to the Thing's update() method. This is useful when the input format - // (i.e. values in the document) differ from the format the actual Thing - // expects. - // - // Each key and value are a field name (not an update() property) and a - // function which takes the value for that field and returns the value which - // will be passed on to update(). - fieldTransformations = {}, - - // Mapping of Thing.update() source properties to field names. - // - // Note this is property -> field, not field -> property. This is a - // shorthand convenience because properties are generally typical - // camel-cased JS properties, while fields may contain whitespace and be - // more easily represented as quoted strings. - propertyFieldMapping, - - // Completely ignored fields. These won't throw an unknown field error if - // they're present in a document, but they won't be used for Thing property - // generation, either. Useful for stuff that's present in data files but not - // yet implemented as part of a Thing's data model! - ignoredFields = [] +// +function makeProcessDocument(thingConstructor, { + // The bulk of configuration happens here in the spec's `fields` property. + // Each key is a field that's expected on the source document; fields that + // don't match one of these keys will cause an error. Values are object + // entries describing what to do with the field. + // + // A field entry's `property` tells what property the value for this field + // will be put into, on the respective Thing (subclass) instance. + // + // A field entry's `transform` optionally allows converting the raw value in + // YAML into some other format before providing setting it on the Thing + // instance. + // + // If a field entry has `ignore: true`, it will be completely skipped by the + // YAML parser - it won't be validated, read, or loaded into data objects. + // This is mainly useful for fields that are purely annotational or are + // currently placeholders. + // + fields: fieldSpecs = {}, + + // List of fields which are invalid when coexisting in a document. + // Data objects are generally allowing with regards to what properties go + // together, allowing for properties to be set separately from each other + // instead of complaining about invalid or unused-data cases. But it's + // useful to see these kinds of errors when actually validating YAML files! + // + // Each item of this array should itself be an object with a descriptive + // message and a list of fields. Of those fields, none should ever coexist + // with any other. For example: + // + // [ + // {message: '...', fields: ['A', 'B', 'C']}, + // {message: '...', fields: ['C', 'D']}, + // ] + // + // ...means A can't coexist with B or C, B can't coexist with A or C, and + // C can't coexist iwth A, B, or D - but it's okay for D to coexist with + // A or B. + // + invalidFieldCombinations = [], }) { - if (!propertyFieldMapping) { - throw new Error(`Expected propertyFieldMapping to be provided`); - } - - const knownFields = Object.values(propertyFieldMapping); - - // Invert the property-field mapping, since it'll come in handy for - // assigning update() source values later. - const fieldPropertyMapping = Object.fromEntries( - (Object.entries(propertyFieldMapping) - .map(([ property, field ]) => [field, property]))); - - const decorateErrorWithName = fn => { - const nameField = propertyFieldMapping['name']; - if (!nameField) return fn; - - return document => { - try { - return fn(document); - } catch (error) { - const name = document[nameField]; - error.message = (name - ? `(name: ${inspect(name)}) ${error.message}` - : `(${color.dim(`no name found`)}) ${error.message}`); - throw error; - } - }; + if (!thingConstructor) { + throw new Error(`Missing Thing class`); + } + + if (!fieldSpecs) { + throw new Error(`Expected fields to be provided`); + } + + const knownFields = Object.keys(fieldSpecs); + + const ignoredFields = + Object.entries(fieldSpecs) + .filter(([, {ignore}]) => ignore) + .map(([field]) => field); + + const propertyToField = + withEntries(fieldSpecs, entries => entries + .map(([field, {property}]) => [property, field])); + + // TODO: Is this function even necessary?? + // Aren't we doing basically the same work in the function it's decorating??? + const decorateErrorWithName = (fn) => { + const nameField = propertyToField.name; + if (!nameField) return fn; + + return (document) => { + try { + return fn(document); + } catch (error) { + const name = document[nameField]; + error.message = name + ? `(name: ${inspect(name)}) ${error.message}` + : `(${colors.dim(`no name found`)}) ${error.message}`; + throw error; + } }; - - return decorateErrorWithName(document => { - const documentEntries = Object.entries(document) - .filter(([ field ]) => !ignoredFields.includes(field)); - - const unknownFields = documentEntries - .map(([ field ]) => field) - .filter(field => !knownFields.includes(field)); - - if (unknownFields.length) { - throw new makeProcessDocument.UnknownFieldsError(unknownFields); - } - - const fieldValues = {}; - - for (const [ field, value ] of documentEntries) { - if (Object.hasOwn(fieldTransformations, field)) { - fieldValues[field] = fieldTransformations[field](value); - } else { - fieldValues[field] = value; - } - } - - const sourceProperties = {}; - - for (const [ field, value ] of Object.entries(fieldValues)) { - const property = fieldPropertyMapping[field]; - sourceProperties[property] = value; - } - - const thing = Reflect.construct(thingClass, []); - - withAggregate({message: `Errors applying ${color.green(thingClass.name)} properties`}, ({ call }) => { - for (const [ property, value ] of Object.entries(sourceProperties)) { - call(() => (thing[property] = value)); - } - }); - - return thing; + }; + + return decorateErrorWithName((document) => { + const nameField = propertyToField.name; + const namePart = + (nameField + ? (document[nameField] + ? ` named ${colors.green(`"${document[nameField]}"`)}` + : ` (name field, "${nameField}", not specified)`) + : ``); + + const constructorPart = + (thingConstructor[Thing.friendlyName] + ? thingConstructor[Thing.friendlyName] + : thingConstructor.name + ? thingConstructor.name + : `document`); + + const aggregate = openAggregate({ + message: `Errors processing ${constructorPart}` + namePart, }); -} - -makeProcessDocument.UnknownFieldsError = class UnknownFieldsError extends Error { - constructor(fields) { - super(`Unknown fields present: ${fields.join(', ')}`); - this.fields = fields; - } -}; -export const processAlbumDocument = makeProcessDocument(Album, { - fieldTransformations: { - 'Artists': parseContributors, - 'Cover Artists': parseContributors, - 'Default Track Cover Artists': parseContributors, - 'Wallpaper Artists': parseContributors, - 'Banner Artists': parseContributors, + const documentEntries = Object.entries(document) + .filter(([field]) => !ignoredFields.includes(field)); - 'Date': value => new Date(value), - 'Date Added': value => new Date(value), - 'Cover Art Date': value => new Date(value), - 'Default Track Cover Art Date': value => new Date(value), + const skippedFields = new Set(); - 'Banner Dimensions': parseDimensions, + const unknownFields = documentEntries + .map(([field]) => field) + .filter((field) => !knownFields.includes(field)); - 'Additional Files': parseAdditionalFiles, - }, + if (!empty(unknownFields)) { + aggregate.push(new UnknownFieldsError(unknownFields)); - propertyFieldMapping: { - name: 'Album', - - color: 'Color', - directory: 'Directory', - urls: 'URLs', - - artistContribsByRef: 'Artists', - coverArtistContribsByRef: 'Cover Artists', - trackCoverArtistContribsByRef: 'Default Track Cover Artists', - - coverArtFileExtension: 'Cover Art File Extension', - trackCoverArtFileExtension: 'Track Art File Extension', + for (const field of unknownFields) { + skippedFields.add(field); + } + } - wallpaperArtistContribsByRef: 'Wallpaper Artists', - wallpaperStyle: 'Wallpaper Style', - wallpaperFileExtension: 'Wallpaper File Extension', + const presentFields = Object.keys(document); - bannerArtistContribsByRef: 'Banner Artists', - bannerStyle: 'Banner Style', - bannerFileExtension: 'Banner File Extension', - bannerDimensions: 'Banner Dimensions', + const fieldCombinationErrors = []; - date: 'Date', - trackArtDate: 'Default Track Cover Art Date', - coverArtDate: 'Cover Art Date', - dateAddedToWiki: 'Date Added', + for (const {message, fields} of invalidFieldCombinations) { + const fieldsPresent = + presentFields.filter(field => fields.includes(field)); - hasCoverArt: 'Has Cover Art', - hasTrackArt: 'Has Track Art', - hasTrackNumbers: 'Has Track Numbers', - isMajorRelease: 'Major Release', - isListedOnHomepage: 'Listed on Homepage', + if (fieldsPresent.length >= 2) { + const filteredDocument = + filterProperties( + document, + fieldsPresent, + {preserveOriginalOrder: true}); - groupsByRef: 'Groups', - artTagsByRef: 'Art Tags', - commentary: 'Commentary', + fieldCombinationErrors.push( + new FieldCombinationError(filteredDocument, message)); - additionalFiles: 'Additional Files', + for (const field of Object.keys(filteredDocument)) { + skippedFields.add(field); + } + } } -}); - -export const processTrackGroupDocument = makeProcessDocument(TrackGroup, { - fieldTransformations: { - 'Date Originally Released': value => new Date(value), - }, - propertyFieldMapping: { - name: 'Group', - color: 'Color', - dateOriginallyReleased: 'Date Originally Released', + if (!empty(fieldCombinationErrors)) { + aggregate.push(new FieldCombinationAggregateError(fieldCombinationErrors)); } -}); - -export const processTrackDocument = makeProcessDocument(Track, { - fieldTransformations: { - 'Duration': getDurationInSeconds, - 'Date First Released': value => new Date(value), - 'Cover Art Date': value => new Date(value), + const fieldValues = {}; - 'Artists': parseContributors, - 'Contributors': parseContributors, - 'Cover Artists': parseContributors, + for (const [field, documentValue] of documentEntries) { + if (skippedFields.has(field)) continue; - 'Additional Files': parseAdditionalFiles, - }, + // This variable would like to certify itself as "not into capitalism". + let propertyValue = + (fieldSpecs[field].transform + ? fieldSpecs[field].transform(documentValue) + : documentValue); - propertyFieldMapping: { - name: 'Track', + // Completely blank items in a YAML list are read as null. + // They're handy to have around when filling out a document and shouldn't + // be considered an error (or data at all). + if (Array.isArray(propertyValue)) { + const wasEmpty = empty(propertyValue); - directory: 'Directory', - duration: 'Duration', - urls: 'URLs', + propertyValue = + propertyValue.filter(item => item !== null); - coverArtDate: 'Cover Art Date', - coverArtFileExtension: 'Cover Art File Extension', - dateFirstReleased: 'Date First Released', - hasCoverArt: 'Has Cover Art', - hasURLs: 'Has URLs', + const isEmpty = empty(propertyValue); - referencedTracksByRef: 'Referenced Tracks', - artistContribsByRef: 'Artists', - contributorContribsByRef: 'Contributors', - coverArtistContribsByRef: 'Cover Artists', - artTagsByRef: 'Art Tags', - originalReleaseTrackByRef: 'Originally Released As', - - commentary: 'Commentary', - lyrics: 'Lyrics', - - additionalFiles: 'Additional Files', - }, + // Don't set arrays which are empty as a result of the above filter. + // Arrays which were originally empty, i.e. `Field: []`, are still + // valid data, but if it's just an array not containing any filled out + // items, it should be treated as a placeholder and skipped over. + if (isEmpty && !wasEmpty) { + propertyValue = null; + } + } - ignoredFields: ['Sampled Tracks'] -}); + fieldValues[field] = propertyValue; + } -export const processArtistDocument = makeProcessDocument(Artist, { - propertyFieldMapping: { - name: 'Artist', + const thing = Reflect.construct(thingConstructor, []); - directory: 'Directory', - urls: 'URLs', - hasAvatar: 'Has Avatar', - avatarFileExtension: 'Avatar File Extension', + const fieldValueErrors = []; - aliasNames: 'Aliases', + for (const [field, value] of Object.entries(fieldValues)) { + const {property} = fieldSpecs[field]; - contextNotes: 'Context Notes' - }, + try { + thing[property] = value; + } catch (caughtError) { + skippedFields.add(field); + fieldValueErrors.push(new FieldValueError( + field, value, {cause: caughtError})); + } + } - ignoredFields: ['Dead URLs'] -}); + if (!empty(fieldValueErrors)) { + aggregate.push(new FieldValueAggregateError( + fieldValueErrors, thingConstructor)); + } -export const processFlashDocument = makeProcessDocument(Flash, { - fieldTransformations: { - 'Date': value => new Date(value), + if (skippedFields.size >= 1) { + aggregate.push( + new SkippedFieldsSummaryError( + filterProperties( + document, + Array.from(skippedFields), + {preserveOriginalOrder: true}))); + } - 'Contributors': parseContributors, - }, + return {thing, aggregate}; + }); +} - propertyFieldMapping: { - name: 'Flash', +export class UnknownFieldsError extends Error { + constructor(fields) { + super(`Unknown fields ignored: ${fields.map(field => colors.red(field)).join(', ')}`); + this.fields = fields; + } +} - directory: 'Directory', - page: 'Page', - date: 'Date', - coverArtFileExtension: 'Cover Art File Extension', +export class FieldCombinationAggregateError extends AggregateError { + constructor(errors) { + super(errors, `Invalid field combinations - all involved fields ignored`); + } +} - featuredTracksByRef: 'Featured Tracks', - contributorContribsByRef: 'Contributors', - urls: 'URLs' - }, -}); +export class FieldCombinationError extends Error { + constructor(fields, message) { + const fieldNames = Object.keys(fields); + + const fieldNamesText = + fieldNames + .map(field => colors.red(field)) + .join(', '); + + const mainMessage = `Don't combine ${fieldNamesText}`; + + const causeMessage = + (typeof message === 'function' + ? message(fields) + : typeof message === 'string' + ? message + : null); + + super(mainMessage, { + cause: + (causeMessage + ? new Error(causeMessage) + : null), + }); -export const processFlashActDocument = makeProcessDocument(FlashAct, { - propertyFieldMapping: { - name: 'Act', - color: 'Color', - anchor: 'Anchor', - jump: 'Jump', - jumpColor: 'Jump Color' - } -}); - -export const processNewsEntryDocument = makeProcessDocument(NewsEntry, { - fieldTransformations: { - 'Date': value => new Date(value) - }, - - propertyFieldMapping: { - name: 'Name', - directory: 'Directory', - date: 'Date', - content: 'Content', - } -}); - -export const processArtTagDocument = makeProcessDocument(ArtTag, { - propertyFieldMapping: { - name: 'Tag', - directory: 'Directory', - color: 'Color', - isContentWarning: 'Is CW' - } -}); - -export const processGroupDocument = makeProcessDocument(Group, { - propertyFieldMapping: { - name: 'Group', - directory: 'Directory', - description: 'Description', - urls: 'URLs', - } -}); + this.fields = fields; + } +} -export const processGroupCategoryDocument = makeProcessDocument(GroupCategory, { - propertyFieldMapping: { - name: 'Category', - color: 'Color', - } -}); +export class FieldValueAggregateError extends AggregateError { + [Symbol.for('hsmusic.aggregate.translucent')] = true; -export const processStaticPageDocument = makeProcessDocument(StaticPage, { - propertyFieldMapping: { - name: 'Name', - nameShort: 'Short Name', - directory: 'Directory', + constructor(errors, thingConstructor) { + const constructorText = + colors.green(thingConstructor.name); - content: 'Content', - stylesheet: 'Style', + super( + errors, + `Errors processing field values for ${constructorText}`); + } +} - showInNavigationBar: 'Show in Navigation Bar' - } -}); - -export const processWikiInfoDocument = makeProcessDocument(WikiInfo, { - propertyFieldMapping: { - name: 'Name', - nameShort: 'Short Name', - color: 'Color', - description: 'Description', - footerContent: 'Footer Content', - defaultLanguage: 'Default Language', - canonicalBase: 'Canonical Base', - divideTrackListsByGroupsByRef: 'Divide Track Lists By Groups', - enableFlashesAndGames: 'Enable Flashes & Games', - enableListings: 'Enable Listings', - enableNews: 'Enable News', - enableArtTagUI: 'Enable Art Tag UI', - enableGroupUI: 'Enable Group UI', - } -}); +export class FieldValueError extends Error { + constructor(field, value, options) { + const fieldText = + colors.green(`"${field}"`); -export const processHomepageLayoutDocument = makeProcessDocument(HomepageLayout, { - propertyFieldMapping: { - sidebarContent: 'Sidebar Content' - }, + const valueText = + inspect(value, {maxStringLength: 40}); - ignoredFields: ['Homepage'] -}); + super( + `Failed to set ${fieldText} field to ${valueText}`, + options); + } +} -export function makeProcessHomepageLayoutRowDocument(rowClass, spec) { - return makeProcessDocument(rowClass, { - ...spec, +export class SkippedFieldsSummaryError extends Error { + constructor(filteredDocument) { + const entries = Object.entries(filteredDocument); + + const lines = + entries.map(([field, value]) => + ` - ${field}: ` + + inspect(value, {maxStringLength: 70}) + .split('\n') + .map((line, index) => index === 0 ? line : ` ${line}`) + .join('\n')); + + const numFieldsText = + (entries.length === 1 + ? `1 field` + : `${entries.length} fields`); + + super( + colors.bright(colors.yellow(`Altogether, skipped ${numFieldsText}:\n`)) + + lines.join('\n') + '\n' + + colors.bright(colors.yellow(`See above errors for details.`))); + } +} - propertyFieldMapping: { - name: 'Row', - color: 'Color', - type: 'Type', - ...spec.propertyFieldMapping, - } - }); +export function parseDate(date) { + return new Date(date); } -export const homepageLayoutRowTypeProcessMapping = { - albums: makeProcessHomepageLayoutRowDocument(HomepageLayoutAlbumsRow, { - propertyFieldMapping: { - sourceGroupByRef: 'Group', - countAlbumsFromGroup: 'Count', - sourceAlbumsByRef: 'Albums', - actionLinks: 'Actions' - } - }) -}; +export function parseDuration(string) { + if (typeof string !== 'string') { + return string; + } + + const parts = string.split(':').map((n) => parseInt(n)); + if (parts.length === 3) { + return parts[0] * 3600 + parts[1] * 60 + parts[2]; + } else if (parts.length === 2) { + return parts[0] * 60 + parts[1]; + } else { + return 0; + } +} -export function processHomepageLayoutRowDocument(document) { - const type = document['Type']; +export function parseAdditionalFiles(array) { + if (!Array.isArray(array)) { + // Error will be caught when validating against whatever this value is + return array; + } + + return array.map((item) => ({ + title: item['Title'], + description: item['Description'] ?? null, + files: item['Files'], + })); +} - const match = Object.entries(homepageLayoutRowTypeProcessMapping) - .find(([ key ]) => key === type); +export const extractAccentRegex = + /^(?<main>.*?)(?: \((?<accent>.*)\))?$/; - if (!match) { - throw new TypeError(`No processDocument function for row type ${type}!`); - } +export const extractPrefixAccentRegex = + /^(?:\((?<accent>.*)\) )?(?<main>.*?)$/; - return match[1](document); -} +export function parseContributors(contributionStrings) { + // If this isn't something we can parse, just return it as-is. + // The Thing object's validators will handle the data error better + // than we're able to here. + if (!Array.isArray(contributionStrings)) { + return contributionStrings; + } -// --> Utilities shared across document parsing functions + return contributionStrings.map(item => { + if (typeof item === 'object' && item['Who']) + return {who: item['Who'], what: item['What'] ?? null}; -export function getDurationInSeconds(string) { - if (typeof string === 'number') { - return string; - } + if (typeof item !== 'string') return item; - if (typeof string !== 'string') { - throw new TypeError(`Expected a string or number, got ${string}`); - } + const match = item.match(extractAccentRegex); + if (!match) return item; - const parts = string.split(':').map(n => parseInt(n)) - if (parts.length === 3) { - return parts[0] * 3600 + parts[1] * 60 + parts[2] - } else if (parts.length === 2) { - return parts[0] * 60 + parts[1] - } else { - return 0 - } + return { + who: match.groups.main, + what: match.groups.accent ?? null, + }; + }); } -export function parseAdditionalFiles(array) { - if (!array) return null; - if (!Array.isArray(array)) { - // Error will be caught when validating against whatever this value is - return array; - } +export function parseAdditionalNames(additionalNameStrings) { + if (!Array.isArray(additionalNameStrings)) { + return additionalNameStrings; + } - return array.map(item => ({ - title: item['Title'], - description: item['Description'] ?? null, - files: item['Files'] - })); -} + return additionalNameStrings.map(item => { + if (typeof item === 'object' && item['Name']) + return {name: item['Name'], annotation: item['Annotation'] ?? null}; -export function parseCommentary(text) { - if (text) { - const lines = String(text).split('\n'); - if (!lines[0].replace(/<\/b>/g, '').includes(':</i>')) { - return {error: `An entry is missing commentary citation: "${lines[0].slice(0, 40)}..."`}; - } - return text; - } else { - return null; - } -} + if (typeof item !== 'string') return item; -export function parseContributors(contributors) { - if (!contributors) { - return null; - } + const match = item.match(extractAccentRegex); + if (!match) return item; - if (contributors.length === 1 && contributors[0].startsWith('<i>')) { - const arr = []; - arr.textContent = contributors[0]; - return arr; - } + return { + name: match.groups.main, + annotation: match.groups.accent ?? null, + }; + }); +} - contributors = contributors.map(contrib => { - // 8asically, the format is "Who (What)", or just "Who". 8e sure to - // keep in mind that "what" doesn't necessarily have a value! - const match = contrib.match(/^(.*?)( \((.*)\))?$/); - if (!match) { - return contrib; - } - const who = match[1]; - const what = match[3] || null; - return {who, what}; - }); +export function parseDimensions(string) { + // It's technically possible to pass an array like [30, 40] through here. + // That's not really an issue because if it isn't of the appropriate shape, + // the Thing object's validators will handle the error. + if (typeof string !== 'string') { + return string; + } - const badContributor = contributors.find(val => typeof val === 'string'); - if (badContributor) { - return {error: `An entry has an incorrectly formatted contributor, "${badContributor}".`}; - } + const parts = string.split(/[x,* ]+/g); - if (contributors.length === 1 && contributors[0].who === 'none') { - return null; - } + if (parts.length !== 2) { + throw new Error(`Invalid dimensions: ${string} (expected "width & height")`); + } - return contributors; -} + const nums = parts.map((part) => Number(part.trim())); -function parseDimensions(string) { - if (!string) { - return null; - } + if (nums.includes(NaN)) { + throw new Error(`Invalid dimensions: ${string} (couldn't parse as numbers)`); + } - const parts = string.split(/[x,* ]+/g); - if (parts.length !== 2) throw new Error(`Invalid dimensions: ${string} (expected width & height)`); - const nums = parts.map(part => Number(part.trim())); - if (nums.includes(NaN)) throw new Error(`Invalid dimensions: ${string} (couldn't parse as numbers)`); - return nums; + return nums; } -// --> Data repository loading functions and descriptors - // documentModes: Symbols indicating sets of behavior for loading and processing // data files. export const documentModes = { - // onePerFile: One document per file. Expects files array (or function) and - // processDocument function. Obviously, each specified data file should only - // contain one YAML document (an error will be thrown otherwise). Calls save - // with an array of processed documents (wiki objects). - onePerFile: Symbol('Document mode: onePerFile'), - - // headerAndEntries: One or more documents per file; the first document is - // treated as a "header" and represents data which pertains to all following - // "entry" documents. Expects files array (or function) and - // processHeaderDocument and processEntryDocument functions. Calls save with - // an array of {header, entries} objects. - // - // Please note that the final results loaded from each file may be "missing" - // data objects corresponding to entry documents if the processEntryDocument - // function throws on any entries, resulting in partial data provided to - // save() - errors will be caught and thrown in the final buildSteps - // aggregate. However, if the processHeaderDocument function fails, all - // following documents in the same file will be ignored as well (i.e. an - // entire file will be excempt from the save() function's input). - headerAndEntries: Symbol('Document mode: headerAndEntries'), - - // allInOne: One or more documents, all contained in one file. Expects file - // string (or function) and processDocument function. Calls save with an - // array of processed documents (wiki objects). - allInOne: Symbol('Document mode: allInOne'), - - // oneDocumentTotal: Just a single document, represented in one file. - // Expects file string (or function) and processDocument function. Calls - // save with the single processed wiki document (data object). - // - // Please note that if the single document fails to process, the save() - // function won't be called at all, generally resulting in an altogether - // missing property from the global wikiData object. This should be caught - // and handled externally. - oneDocumentTotal: Symbol('Document mode: oneDocumentTotal'), + // onePerFile: One document per file. Expects files array (or function) and + // processDocument function. Obviously, each specified data file should only + // contain one YAML document (an error will be thrown otherwise). Calls save + // with an array of processed documents (wiki objects). + onePerFile: Symbol('Document mode: onePerFile'), + + // headerAndEntries: One or more documents per file; the first document is + // treated as a "header" and represents data which pertains to all following + // "entry" documents. Expects files array (or function) and + // processHeaderDocument and processEntryDocument functions. Calls save with + // an array of {header, entries} objects. + // + // Please note that the final results loaded from each file may be "missing" + // data objects corresponding to entry documents if the processEntryDocument + // function throws on any entries, resulting in partial data provided to + // save() - errors will be caught and thrown in the final buildSteps + // aggregate. However, if the processHeaderDocument function fails, all + // following documents in the same file will be ignored as well (i.e. an + // entire file will be excempt from the save() function's input). + headerAndEntries: Symbol('Document mode: headerAndEntries'), + + // allInOne: One or more documents, all contained in one file. Expects file + // string (or function) and processDocument function. Calls save with an + // array of processed documents (wiki objects). + allInOne: Symbol('Document mode: allInOne'), + + // oneDocumentTotal: Just a single document, represented in one file. + // Expects file string (or function) and processDocument function. Calls + // save with the single processed wiki document (data object). + // + // Please note that if the single document fails to process, the save() + // function won't be called at all, generally resulting in an altogether + // missing property from the global wikiData object. This should be caught + // and handled externally. + oneDocumentTotal: Symbol('Document mode: oneDocumentTotal'), }; // dataSteps: Top-level array of "steps" for loading YAML document files. @@ -624,670 +523,490 @@ export const documentModes = { // them to each other, setting additional properties, etc). Input argument // format depends on documentMode. // -export const dataSteps = [ - { - title: `Process wiki info file`, - file: WIKI_INFO_FILE, +export const getDataSteps = () => { + const steps = []; - documentMode: documentModes.oneDocumentTotal, - processDocument: processWikiInfoDocument, + for (const thingConstructor of Object.values(thingConstructors)) { + const getSpecFn = thingConstructor[Thing.getYamlLoadingSpec]; + if (!getSpecFn) continue; - save(wikiInfo) { - if (!wikiInfo) { - return; - } + steps.push(getSpecFn({ + documentModes, + thingConstructors, + })); + } - return {wikiInfo}; + sortByName(steps, {getName: step => step.title}); + + return steps; +}; + +export async function loadAndProcessDataDocuments({dataPath}) { + const processDataAggregate = openAggregate({ + message: `Errors processing data files`, + }); + const wikiDataResult = {}; + + function decorateErrorWithFile(fn) { + return decorateErrorWithAnnotation(fn, + (caughtError, firstArg) => + annotateErrorWithFile( + caughtError, + path.relative( + dataPath, + (typeof firstArg === 'object' + ? firstArg.file + : firstArg)))); + } + + function asyncDecorateErrorWithFile(fn) { + return decorateErrorWithFile(fn).async; + } + + for (const dataStep of getDataSteps()) { + await processDataAggregate.nestAsync( + { + message: `Errors during data step: ${colors.bright(dataStep.title)}`, + translucent: true, + }, + async ({call, callAsync, map, mapAsync, push}) => { + const {documentMode} = dataStep; + + if (!Object.values(documentModes).includes(documentMode)) { + throw new Error(`Invalid documentMode: ${documentMode.toString()}`); } - }, - - { - title: `Process album files`, - files: async dataPath => ( - (await findFiles(path.join(dataPath, DATA_ALBUM_DIRECTORY), { - filter: f => path.extname(f) === '.yaml', - joinParentDirectory: false - })).map(file => path.join(DATA_ALBUM_DIRECTORY, file))), - - documentMode: documentModes.headerAndEntries, - processHeaderDocument: processAlbumDocument, - processEntryDocument(document) { - return ('Group' in document - ? processTrackGroupDocument(document) - : processTrackDocument(document)); - }, - - save(results) { - const albumData = []; - const trackData = []; - - for (const { header: album, entries } of results) { - // We can't mutate an array once it's set as a property - // value, so prepare the tracks and track groups that will - // show up in a track list all the way before actually - // applying them. - const trackGroups = []; - let currentTracksByRef = null; - let currentTrackGroup = null; - - const albumRef = Thing.getReference(album); - - function closeCurrentTrackGroup() { - if (currentTracksByRef) { - let trackGroup; - - if (currentTrackGroup) { - trackGroup = currentTrackGroup; - } else { - trackGroup = new TrackGroup(); - trackGroup.name = `Default Track Group`; - trackGroup.isDefaultTrackGroup = true; - } - - trackGroup.album = album; - trackGroup.tracksByRef = currentTracksByRef; - trackGroups.push(trackGroup); - } - } - for (const entry of entries) { - if (entry instanceof TrackGroup) { - closeCurrentTrackGroup(); - currentTracksByRef = []; - currentTrackGroup = entry; - continue; - } + // Hear me out, it's been like 1200 years since I wrote the rest of + // this beautifully error-containing code and I don't know how to + // integrate this nicely. So I'm just returning the result and the + // error that should be thrown. Yes, we're back in callback hell, + // just without the callbacks. Thank you. + const filterBlankDocuments = documents => { + const aggregate = openAggregate({ + message: `Found blank documents - check for extra '${colors.cyan(`---`)}'`, + }); + + const filteredDocuments = + documents + .filter(doc => doc !== null); + + if (filteredDocuments.length !== documents.length) { + const blankIndexRangeInfo = + documents + .map((doc, index) => [doc, index]) + .filter(([doc]) => doc === null) + .map(([doc, index]) => index) + .reduce((accumulator, index) => { + if (accumulator.length === 0) { + return [[index, index]]; + } + const current = accumulator.at(-1); + const rest = accumulator.slice(0, -1); + if (current[1] === index - 1) { + return rest.concat([[current[0], index]]); + } else { + return accumulator.concat([[index, index]]); + } + }, []) + .map(([start, end]) => ({ + start, + end, + count: end - start + 1, + previous: atOffset(documents, start, -1), + next: atOffset(documents, end, +1), + })); + + for (const {start, end, count, previous, next} of blankIndexRangeInfo) { + const parts = []; + + if (count === 1) { + const range = `#${start + 1}`; + parts.push(`${count} document (${colors.yellow(range)}), `); + } else { + const range = `#${start + 1}-${end + 1}`; + parts.push(`${count} documents (${colors.yellow(range)}), `); + } + + if (previous === null) { + parts.push(`at start of file`); + } else if (next === null) { + parts.push(`at end of file`); + } else { + const previousDescription = Object.entries(previous).at(0).join(': '); + const nextDescription = Object.entries(next).at(0).join(': '); + parts.push(`between "${colors.cyan(previousDescription)}" and "${colors.cyan(nextDescription)}"`); + } + + aggregate.push(new Error(parts.join(''))); + } + } - trackData.push(entry); + return {documents: filteredDocuments, aggregate}; + }; - entry.dataSourceAlbumByRef = albumRef; + const processDocument = (document, thingClassOrFn) => { + const thingClass = + (thingClassOrFn.prototype instanceof Thing + ? thingClassOrFn + : thingClassOrFn(document)); - const trackRef = Thing.getReference(entry); - if (currentTracksByRef) { - currentTracksByRef.push(trackRef); - } else { - currentTracksByRef = [trackRef]; - } - } + if (typeof thingClass !== 'function') { + throw new Error(`Expected a thing class, got ${typeAppearance(thingClass)}`); + } - closeCurrentTrackGroup(); + if (!(thingClass.prototype instanceof Thing)) { + throw new Error(`Expected a thing class, got ${thingClass.name}`); + } - album.trackGroups = trackGroups; - albumData.push(album); - } + const spec = thingClass[Thing.yamlDocumentSpec]; - return {albumData, trackData}; - } - }, - - { - title: `Process artists file`, - file: ARTIST_DATA_FILE, - - documentMode: documentModes.allInOne, - processDocument: processArtistDocument, - - save(results) { - const artistData = results; - - const artistAliasData = results.flatMap(artist => { - const origRef = Thing.getReference(artist); - return (artist.aliasNames?.map(name => { - const alias = new Artist(); - alias.name = name; - alias.isAlias = true; - alias.aliasedArtistRef = origRef; - alias.artistData = artistData; - return alias; - }) ?? []); - }); - - return {artistData, artistAliasData}; - } - }, - - // TODO: WD.wikiInfo.enableFlashesAndGames && - { - title: `Process flashes file`, - file: FLASH_DATA_FILE, - - documentMode: documentModes.allInOne, - processDocument(document) { - return ('Act' in document - ? processFlashActDocument(document) - : processFlashDocument(document)); - }, - - save(results) { - let flashAct; - let flashesByRef = []; - - if (results[0] && !(results[0] instanceof FlashAct)) { - throw new Error(`Expected an act at top of flash data file`); - } + if (!spec) { + throw new Error(`Class "${thingClass.name}" doesn't specify Thing.yamlDocumentSpec`); + } - for (const thing of results) { - if (thing instanceof FlashAct) { - if (flashAct) { - Object.assign(flashAct, {flashesByRef}); - } + // TODO: Making a function to only call it just like that is + // obviously pretty jank! It should be created once per data step. + const fn = makeProcessDocument(thingClass, spec); + return fn(document); + }; - flashAct = thing; - flashesByRef = []; + if ( + documentMode === documentModes.allInOne || + documentMode === documentModes.oneDocumentTotal + ) { + if (!dataStep.file) { + throw new Error(`Expected 'file' property for ${documentMode.toString()}`); + } + + const file = path.join( + dataPath, + typeof dataStep.file === 'function' + ? await callAsync(dataStep.file, dataPath) + : dataStep.file); + + const statResult = await callAsync(() => + stat(file).then( + () => true, + error => { + if (error.code === 'ENOENT') { + return false; } else { - flashesByRef.push(Thing.getReference(thing)); + throw error; } - } + })); - if (flashAct) { - Object.assign(flashAct, {flashesByRef}); - } + if (statResult === false) { + const saveResult = call(dataStep.save, { + [documentModes.allInOne]: [], + [documentModes.oneDocumentTotal]: {}, + }[documentMode]); - const flashData = results.filter(x => x instanceof Flash); - const flashActData = results.filter(x => x instanceof FlashAct); + if (!saveResult) return; - return {flashData, flashActData}; - } - }, + Object.assign(wikiDataResult, saveResult); - { - title: `Process groups file`, - file: GROUP_DATA_FILE, + return; + } - documentMode: documentModes.allInOne, - processDocument(document) { - return ('Category' in document - ? processGroupCategoryDocument(document) - : processGroupDocument(document)); - }, + const readResult = await callAsync(readFile, file, 'utf-8'); - save(results) { - let groupCategory; - let groupsByRef = []; + if (!readResult) { + return; + } - if (results[0] && !(results[0] instanceof GroupCategory)) { - throw new Error(`Expected a category at top of group data file`); - } + let processResults; - for (const thing of results) { - if (thing instanceof GroupCategory) { - if (groupCategory) { - Object.assign(groupCategory, {groupsByRef}); - } + switch (documentMode) { + case documentModes.oneDocumentTotal: { + const yamlResult = call(yaml.load, readResult); - groupCategory = thing; - groupsByRef = []; - } else { - groupsByRef.push(Thing.getReference(thing)); - } - } + if (!yamlResult) { + processResults = null; + break; + } - if (groupCategory) { - Object.assign(groupCategory, {groupsByRef}); - } + const {thing, aggregate} = + processDocument(yamlResult, dataStep.documentThing); - const groupData = results.filter(x => x instanceof Group); - const groupCategoryData = results.filter(x => x instanceof GroupCategory); + processResults = thing; - return {groupData, groupCategoryData}; - } - }, + call(() => aggregate.close()); - { - title: `Process homepage layout file`, - files: [HOMEPAGE_LAYOUT_DATA_FILE], + break; + } - documentMode: documentModes.headerAndEntries, - processHeaderDocument: processHomepageLayoutDocument, - processEntryDocument: processHomepageLayoutRowDocument, + case documentModes.allInOne: { + const yamlResults = call(yaml.loadAll, readResult); - save(results) { - if (!results[0]) { + if (!yamlResults) { + processResults = []; return; - } + } - const { header: homepageLayout, entries: rows } = results[0]; - Object.assign(homepageLayout, {rows}); - return {homepageLayout}; - } - }, + const {documents, aggregate: filterAggregate} = + filterBlankDocuments(yamlResults); - // TODO: WD.wikiInfo.enableNews && - { - title: `Process news data file`, - file: NEWS_DATA_FILE, + call(filterAggregate.close); - documentMode: documentModes.allInOne, - processDocument: processNewsEntryDocument, + processResults = []; - save(newsData) { - sortByDate(newsData); - newsData.reverse(); + map(documents, decorateErrorWithIndex(document => { + const {thing, aggregate} = + processDocument(document, dataStep.documentThing); - return {newsData}; - } - }, + processResults.push(thing); + aggregate.close(); + }), {message: `Errors processing documents`}); - { - title: `Process art tags file`, - file: ART_TAG_DATA_FILE, + break; + } + } - documentMode: documentModes.allInOne, - processDocument: processArtTagDocument, + if (!processResults) return; - save(artTagData) { - artTagData.sort(sortByName); + const saveResult = call(dataStep.save, processResults); - return {artTagData}; - } - }, + if (!saveResult) return; - { - title: `Process static pages file`, - file: STATIC_PAGE_DATA_FILE, + Object.assign(wikiDataResult, saveResult); - documentMode: documentModes.allInOne, - processDocument: processStaticPageDocument, + return; + } - save(staticPageData) { - return {staticPageData}; + if (!dataStep.files) { + throw new Error(`Expected 'files' property for ${documentMode.toString()}`); } - }, -]; -export async function loadAndProcessDataDocuments({ - dataPath, -}) { - const processDataAggregate = openAggregate({message: `Errors processing data files`}); - const wikiDataResult = {}; + const filesFromDataStep = + (typeof dataStep.files === 'function' + ? await callAsync(() => + dataStep.files(dataPath).then( + files => files, + error => { + if (error.code === 'ENOENT') { + return []; + } else { + throw error; + } + })) + : dataStep.files); - function decorateErrorWithFile(fn) { - return (x, index, array) => { + const filesUnderDataPath = + filesFromDataStep + .map(file => path.join(dataPath, file)); + + const yamlResults = []; + + await mapAsync(filesUnderDataPath, {message: `Errors loading data files`}, + asyncDecorateErrorWithFile(async file => { + let contents; try { - return fn(x, index, array); - } catch (error) { - error.message += ( - (error.message.includes('\n') ? '\n' : ' ') + - `(file: ${color.bright(color.blue(path.relative(dataPath, x.file)))})` - ); - throw error; + contents = await readFile(file, 'utf-8'); + } catch (caughtError) { + throw new Error(`Failed to read data file`, {cause: caughtError}); } - }; - } - for (const dataStep of dataSteps) { - await processDataAggregate.nestAsync( - {message: `Errors during data step: ${dataStep.title}`}, - async ({call, callAsync, map, mapAsync, nest}) => { - const { documentMode } = dataStep; + let documents; + try { + documents = yaml.loadAll(contents); + } catch (caughtError) { + throw new Error(`Failed to parse valid YAML`, {cause: caughtError}); + } - if (!(Object.values(documentModes).includes(documentMode))) { - throw new Error(`Invalid documentMode: ${documentMode.toString()}`); - } + const {documents: filteredDocuments, aggregate: filterAggregate} = + filterBlankDocuments(documents); - if (documentMode === documentModes.allInOne || documentMode === documentModes.oneDocumentTotal) { - if (!dataStep.file) { - throw new Error(`Expected 'file' property for ${documentMode.toString()}`); - } + try { + filterAggregate.close(); + } catch (caughtError) { + // Blank documents aren't a critical error, they're just something + // that should be noted - the (filtered) documents still get pushed. + const pathToFile = path.relative(dataPath, file); + annotateErrorWithFile(caughtError, pathToFile); + push(caughtError); + } - const file = path.join(dataPath, - (typeof dataStep.file === 'function' - ? await callAsync(dataStep.file, dataPath) - : dataStep.file)); + yamlResults.push({file, documents: filteredDocuments}); + })); - const readResult = await callAsync(readFile, file, 'utf-8'); + const processResults = []; - if (!readResult) { - return; - } + switch (documentMode) { + case documentModes.headerAndEntries: + map(yamlResults, {message: `Errors processing documents in data files`, translucent: true}, + decorateErrorWithFile(({documents}) => { + const headerDocument = documents[0]; + const entryDocuments = documents.slice(1).filter(Boolean); - const yamlResult = (documentMode === documentModes.oneDocumentTotal - ? call(yaml.load, readResult) - : call(yaml.loadAll, readResult)); + if (!headerDocument) + throw new Error(`Missing header document (empty file or erroneously starting with "---"?)`); - if (!yamlResult) { - return; - } + withAggregate({message: `Errors processing documents`}, ({push}) => { + const {thing: headerObject, aggregate: headerAggregate} = + processDocument(headerDocument, dataStep.headerDocumentThing); - let processResults; + try { + headerAggregate.close(); + } catch (caughtError) { + caughtError.message = `(${colors.yellow(`header`)}) ${caughtError.message}`; + push(caughtError); + } - if (documentMode === documentModes.oneDocumentTotal) { - nest({message: `Errors processing document`}, ({ call }) => { - processResults = call(dataStep.processDocument, yamlResult); - }); - } else { - const { result, aggregate } = mapAggregate( - yamlResult, - decorateErrorWithIndex(dataStep.processDocument), - {message: `Errors processing documents`} - ); - processResults = result; - call(aggregate.close); - } + const entryObjects = []; - if (!processResults) return; + for (let index = 0; index < entryDocuments.length; index++) { + const entryDocument = entryDocuments[index]; - const saveResult = call(dataStep.save, processResults); + const {thing: entryObject, aggregate: entryAggregate} = + processDocument(entryDocument, dataStep.entryDocumentThing); - if (!saveResult) return; + entryObjects.push(entryObject); - Object.assign(wikiDataResult, saveResult); + try { + entryAggregate.close(); + } catch (caughtError) { + caughtError.message = `(${colors.yellow(`entry #${index + 1}`)}) ${caughtError.message}`; + push(caughtError); + } + } - return; - } + processResults.push({ + header: headerObject, + entries: entryObjects, + }); + }); + })); + break; - if (!dataStep.files) { - throw new Error(`Expected 'files' property for ${documentMode.toString()}`); - } + case documentModes.onePerFile: + map(yamlResults, {message: `Errors processing data files as valid documents`}, + decorateErrorWithFile(({documents}) => { + if (documents.length > 1) + throw new Error(`Only expected one document to be present per file, got ${documents.length} here`); - const files = ( - (typeof dataStep.files === 'function' - ? await callAsync(dataStep.files, dataPath) - : dataStep.files) - .map(file => path.join(dataPath, file))); - - const readResults = await mapAsync( - files, - file => (readFile(file, 'utf-8') - .then(contents => ({file, contents}))), - {message: `Errors reading data files`}); - - const yamlResults = map( - readResults, - decorateErrorWithFile( - ({ file, contents }) => ({file, documents: yaml.loadAll(contents)})), - {message: `Errors parsing data files as valid YAML`}); - - let processResults; - - if (documentMode === documentModes.headerAndEntries) { - nest({message: `Errors processing data files as valid documents`}, ({ call, map }) => { - processResults = []; - - yamlResults.forEach(({ file, documents }) => { - const [ headerDocument, ...entryDocuments ] = documents; - - const header = call( - decorateErrorWithFile( - ({ document }) => dataStep.processHeaderDocument(document)), - {file, document: headerDocument}); - - // Don't continue processing files whose header - // document is invalid - the entire file is excempt - // from data in this case. - if (!header) { - return; - } - - const entries = map( - entryDocuments.map(document => ({file, document})), - decorateErrorWithFile( - decorateErrorWithIndex( - ({ document }) => dataStep.processEntryDocument(document))), - {message: `Errors processing entry documents`}); - - // Entries may be incomplete (i.e. any errored - // documents won't have a processed output - // represented here) - this is intentional! By - // principle, partial output is preferred over - // erroring an entire file. - processResults.push({header, entries}); - }); - }); - } + if (empty(documents) || !documents[0]) + throw new Error(`Expected a document, this file is empty`); - if (documentMode === documentModes.onePerFile) { - nest({message: `Errors processing data files as valid documents`}, ({ call, map }) => { - processResults = []; - - yamlResults.forEach(({ file, documents }) => { - if (documents.length > 1) { - call(decorateErrorWithFile(() => { - throw new Error(`Only expected one document to be present per file`); - })); - return; - } - - const result = call( - decorateErrorWithFile( - ({ document }) => dataStep.processDocument(document)), - {file, document: documents[0]}); - - if (!result) { - return; - } - - processResults.push(result); - }); - }); - } + const {thing, aggregate} = + processDocument(documents[0], dataStep.documentThing); - const saveResult = call(dataStep.save, processResults); + processResults.push(thing); + aggregate.close(); + })); + break; + } - if (!saveResult) return; + const saveResult = call(dataStep.save, processResults); - Object.assign(wikiDataResult, saveResult); - }); - } + if (!saveResult) return; - return { - aggregate: processDataAggregate, - result: wikiDataResult - }; + Object.assign(wikiDataResult, saveResult); + } + ); + } + + return { + aggregate: processDataAggregate, + result: wikiDataResult, + }; } // Data linking! Basically, provide (portions of) wikiData to the Things which // require it - they'll expose dynamically computed properties as a result (many -// of which are required for page HTML generation). +// of which are required for page HTML generation and other expected behavior). export function linkWikiDataArrays(wikiData) { - function assignWikiData(things, ...keys) { - for (let i = 0; i < things.length; i++) { - for (let j = 0; j < keys.length; j++) { - const key = keys[j]; - things[i][key] = wikiData[key]; - } - } + const linkWikiDataSpec = new Map([ + [wikiData.albumData, [ + 'artTagData', + 'artistData', + 'groupData', + ]], + + [wikiData.artTagData, [ + 'albumData', + 'trackData', + ]], + + [wikiData.artistData, [ + 'albumData', + 'artistData', + 'flashData', + 'trackData', + ]], + + [wikiData.flashData, [ + 'artistData', + 'flashActData', + 'trackData', + ]], + + [wikiData.flashActData, [ + 'flashData', + 'flashSideData', + ]], + + [wikiData.flashSideData, [ + 'flashActData', + ]], + + [wikiData.groupData, [ + 'albumData', + 'groupCategoryData', + ]], + + [wikiData.groupCategoryData, [ + 'groupData', + ]], + + [wikiData.homepageLayout?.rows, [ + 'albumData', + 'groupData', + ]], + + [wikiData.trackData, [ + 'albumData', + 'artTagData', + 'artistData', + 'flashData', + 'trackData', + ]], + + [[wikiData.wikiInfo], [ + 'groupData', + ]], + ]); + + for (const [things, keys] of linkWikiDataSpec.entries()) { + if (things === undefined) continue; + for (const thing of things) { + if (thing === undefined) continue; + for (const key of keys) { + if (!(key in wikiData)) continue; + thing[key] = wikiData[key]; + } } - - const WD = wikiData; - - assignWikiData([WD.wikiInfo], 'groupData'); - - assignWikiData(WD.albumData, 'artistData', 'artTagData', 'groupData', 'trackData'); - WD.albumData.forEach(album => assignWikiData(album.trackGroups, 'trackData')); - - assignWikiData(WD.trackData, 'albumData', 'artistData', 'artTagData', 'flashData', 'trackData'); - assignWikiData(WD.artistData, 'albumData', 'artistData', 'flashData', 'trackData'); - assignWikiData(WD.groupData, 'albumData', 'groupCategoryData'); - assignWikiData(WD.groupCategoryData, 'groupData'); - assignWikiData(WD.flashData, 'artistData', 'flashActData', 'trackData'); - assignWikiData(WD.flashActData, 'flashData'); - assignWikiData(WD.artTagData, 'albumData', 'trackData'); - assignWikiData(WD.homepageLayout.rows, 'albumData', 'groupData'); + } } export function sortWikiDataArrays(wikiData) { - Object.assign(wikiData, { - albumData: sortByDate(wikiData.albumData.slice()), - trackData: sortByDate(wikiData.trackData.slice()) - }); - - // Re-link data arrays, so that every object has the new, sorted versions. - // Note that the sorting step deliberately creates new arrays (mutating - // slices instead of the original arrays) - this is so that the object - // caching system understands that it's working with a new ordering. - // We still need to actually provide those updated arrays over again! - linkWikiDataArrays(wikiData); -} - -// Warn about directories which are reused across more than one of the same type -// of Thing. Directories are the unique identifier for most data objects across -// the wiki, so we have to make sure they aren't duplicated! This also -// altogether filters out instances of things with duplicate directories (so if -// two tracks share the directory "megalovania", they'll both be skipped for the -// build, for example). -export function filterDuplicateDirectories(wikiData) { - const deduplicateSpec = [ - 'albumData', - 'artTagData', - 'flashData', - 'groupData', - 'newsData', - 'trackData', - ]; - - const aggregate = openAggregate({message: `Duplicate directories found`}); - for (const thingDataProp of deduplicateSpec) { - const thingData = wikiData[thingDataProp]; - aggregate.nest({message: `Duplicate directories found in ${color.green('wikiData.' + thingDataProp)}`}, ({ call }) => { - const directoryPlaces = Object.create(null); - const duplicateDirectories = []; - for (const thing of thingData) { - const { directory } = thing; - if (directory in directoryPlaces) { - directoryPlaces[directory].push(thing); - duplicateDirectories.push(directory); - } else { - directoryPlaces[directory] = [thing]; - } - } - if (!duplicateDirectories.length) return; - duplicateDirectories.sort((a, b) => { - const aL = a.toLowerCase(); - const bL = b.toLowerCase(); - return aL < bL ? -1 : aL > bL ? 1 : 0; - }); - for (const directory of duplicateDirectories) { - const places = directoryPlaces[directory]; - call(() => { - throw new Error(`Duplicate directory ${color.green(directory)}:\n` + - places.map(thing => ` - ` + inspect(thing)).join('\n')); - }); - } - const allDuplicatedThings = Object.values(directoryPlaces).filter(arr => arr.length > 1).flat(); - const filteredThings = thingData.filter(thing => !allDuplicatedThings.includes(thing)); - wikiData[thingDataProp] = filteredThings; - }); - } - - // TODO: This code closes the aggregate but it generally gets closed again - // by the caller. This works but it might be weird to assume closing an - // aggregate twice is okay, maybe there's a better solution? Expose a new - // function on aggregates for checking if it *would* error? - // (i.e: errors.length > 0) - try { - aggregate.close(); - } catch (error) { - // Duplicate entries were found and filtered out, resulting in altered - // wikiData arrays. These must be re-linked so objects receive the new - // data. - linkWikiDataArrays(wikiData); - } - return aggregate; -} - -// Warn about references across data which don't match anything. This involves -// using the find() functions on all references, setting it to 'error' mode, and -// collecting everything in a structured logged (which gets logged if there are -// any errors). At the same time, we remove errored references from the thing's -// data array. -export function filterReferenceErrors(wikiData) { - const referenceSpec = [ - ['wikiInfo', { - divideTrackListsByGroupsByRef: 'group', - }], - - ['albumData', { - artistContribsByRef: '_contrib', - coverArtistContribsByRef: '_contrib', - trackCoverArtistContribsByRef: '_contrib', - wallpaperArtistContribsByRef: '_contrib', - bannerArtistContribsByRef: '_contrib', - groupsByRef: 'group', - artTagsByRef: 'artTag', - }], - - ['trackData', { - artistContribsByRef: '_contrib', - contributorContribsByRef: '_contrib', - coverArtistContribsByRef: '_contrib', - referencedTracksByRef: 'track', - artTagsByRef: 'artTag', - originalReleaseTrackByRef: 'track', - }], - - ['groupCategoryData', { - groupsByRef: 'group', - }], - - ['homepageLayout.rows', { - sourceGroupsByRef: 'group', - sourceAlbumsByRef: 'album', - }], - - ['flashData', { - contributorContribsByRef: '_contrib', - featuredTracksByRef: 'track', - }], - - ['flashActData', { - flashesByRef: 'flash', - }], - ]; - - function getNestedProp(obj, key) { - const recursive = (o, k) => (k.length === 1 - ? o[k[0]] - : recursive(o[k[0]], k.slice(1))); - const keys = key.split(/(?<=(?<!\\)(?:\\\\)*)\./); - return recursive(obj, keys); - } - - const aggregate = openAggregate({message: `Errors validating between-thing references in data`}); - const boundFind = bindFind(wikiData, {mode: 'error'}); - for (const [ thingDataProp, propSpec ] of referenceSpec) { - const thingData = getNestedProp(wikiData, thingDataProp); - aggregate.nest({message: `Reference errors in ${color.green('wikiData.' + thingDataProp)}`}, ({ nest }) => { - const things = Array.isArray(thingData) ? thingData : [thingData]; - for (const thing of things) { - nest({message: `Reference errors in ${inspect(thing)}`}, ({ filter }) => { - for (const [ property, findFnKey ] of Object.entries(propSpec)) { - if (!thing[property]) continue; - if (findFnKey === '_contrib') { - thing[property] = filter(thing[property], - decorateErrorWithIndex(({ who }) => { - const alias = find.artist(who, wikiData.artistAliasData, {mode: 'quiet'}); - if (alias) { - const original = find.artist(alias.aliasedArtistRef, wikiData.artistData, {mode: 'quiet'}); - throw new Error(`Reference ${color.red(who)} is to an alias, should be ${color.green(original.name)}`); - } - return boundFind.artist(who); - }), - {message: `Reference errors in contributions ${color.green(property)} (${color.green('find.artist')})`}); - continue; - } - const findFn = boundFind[findFnKey]; - const value = thing[property]; - if (Array.isArray(value)) { - thing[property] = filter(value, decorateErrorWithIndex(findFn), - {message: `Reference errors in property ${color.green(property)} (${color.green('find.' + findFnKey)})`}); - } else { - nest({message: `Reference error in property ${color.green(property)} (${color.green('find.' + findFnKey)})`}, ({ call }) => { - try { - call(findFn, value); - } catch (error) { - thing[property] = null; - throw error; - } - }); - } - } - }); - } - }); - } - - return aggregate; + for (const [key, value] of Object.entries(wikiData)) { + if (!Array.isArray(value)) continue; + wikiData[key] = value.slice(); + } + + const steps = getDataSteps(); + + for (const step of steps) { + if (!step.sort) continue; + step.sort(wikiData); + } + + // Re-link data arrays, so that every object has the new, sorted versions. + // Note that the sorting step deliberately creates new arrays (mutating + // slices instead of the original arrays) - this is so that the object + // caching system understands that it's working with a new ordering. + // We still need to actually provide those updated arrays over again! + linkWikiDataArrays(wikiData); } // Utility function for loading all wiki data from the provided YAML data @@ -1297,47 +1016,56 @@ export function filterReferenceErrors(wikiData) { // where reporting info about data loading isn't as relevant as during the // main wiki build process. export async function quickLoadAllFromYAML(dataPath, { - showAggregate: customShowAggregate = showAggregate, -} = {}) { - const showAggregate = customShowAggregate; + bindFind, + getAllFindSpecs, - let wikiData; - - { - const { aggregate, result } = await loadAndProcessDataDocuments({ - dataPath, - }); + showAggregate: customShowAggregate = showAggregate, +}) { + const showAggregate = customShowAggregate; - wikiData = result; + let wikiData; - try { - aggregate.close(); - logInfo`Loaded data without errors. (complete data)`; - } catch (error) { - showAggregate(error); - logWarn`Loaded data with errors. (partial data)`; - } - } + { + const {aggregate, result} = await loadAndProcessDataDocuments({dataPath}); - linkWikiDataArrays(wikiData); + wikiData = result; try { - filterDuplicateDirectories(wikiData).close(); - logInfo`No duplicate directories found. (complete data)`; + aggregate.close(); + logInfo`Loaded data without errors. (complete data)`; } catch (error) { - showAggregate(error); - logWarn`Duplicate directories found. (partial data)`; + showAggregate(error); + logWarn`Loaded data with errors. (partial data)`; } - - try { - filterReferenceErrors(wikiData).close(); - logInfo`No reference errors found. (complete data)`; - } catch (error) { - showAggregate(error); - logWarn`Duplicate directories found. (partial data)`; - } - - sortWikiDataArrays(wikiData); - - return wikiData; + } + + linkWikiDataArrays(wikiData); + + try { + reportDuplicateDirectories(wikiData, {getAllFindSpecs}); + logInfo`No duplicate directories found. (complete data)`; + } catch (error) { + showAggregate(error); + logWarn`Duplicate directories found. (partial data)`; + } + + try { + filterReferenceErrors(wikiData, {bindFind}).close(); + logInfo`No reference errors found. (complete data)`; + } catch (error) { + showAggregate(error); + logWarn`Reference errors found. (partial data)`; + } + + try { + reportContentTextErrors(wikiData, {bindFind}); + logInfo`No content text errors found.`; + } catch (error) { + showAggregate(error); + logWarn`Content text errors found.`; + } + + sortWikiDataArrays(wikiData); + + return wikiData; } |