diff options
Diffstat (limited to 'src/data/yaml.js')
| -rw-r--r-- | src/data/yaml.js | 2683 |
1 files changed, 1597 insertions, 1086 deletions
diff --git a/src/data/yaml.js b/src/data/yaml.js index 32cf7292..af1d5740 100644 --- a/src/data/yaml.js +++ b/src/data/yaml.js @@ -1,595 +1,867 @@ // yaml.js - specification for HSMusic YAML data file format and utilities for -// loading and processing YAML files and documents +// loading, processing, and validating YAML files and documents -import * as path from 'path'; -import yaml from 'js-yaml'; +import {readFile, stat} from 'node:fs/promises'; +import * as path from 'node:path'; +import {inspect as nodeInspect} from 'node:util'; -import { readFile } from 'fs/promises'; -import { inspect as nodeInspect } from 'util'; +import yaml from 'js-yaml'; -import { - Album, - Artist, - ArtTag, - Flash, - FlashAct, - Group, - GroupCategory, - HomepageLayout, - HomepageLayoutAlbumsRow, - HomepageLayoutRow, - NewsEntry, - StaticPage, - Thing, - Track, - TrackGroup, - WikiInfo, -} from './things.js'; +import {colors, ENABLE_COLOR, logInfo, logWarn} from '#cli'; +import {sortByName} from '#sort'; +import Thing from '#thing'; +import thingConstructors from '#things'; import { - color, - ENABLE_COLOR, - logInfo, - logWarn, -} from '../util/cli.js'; + aggregateThrows, + annotateErrorWithFile, + decorateErrorWithIndex, + decorateErrorWithAnnotation, + openAggregate, + showAggregate, +} from '#aggregate'; import { - decorateErrorWithIndex, - mapAggregate, - openAggregate, - showAggregate, - withAggregate, -} from '../util/sugar.js'; + filterReferenceErrors, + reportContentTextErrors, + reportDirectoryErrors, +} from '#data-checks'; import { - sortByDate, - sortByName, -} from '../util/wiki-data.js'; - -import find, { bindFind } from '../util/find.js'; -import { findFiles } from '../util/io.js'; - -// --> General supporting stuff - -function inspect(value) { - return nodeInspect(value, {colors: ENABLE_COLOR}); + atOffset, + empty, + filterProperties, + getNestedProp, + stitchArrays, + typeAppearance, + unique, + withEntries, +} from '#sugar'; + +function inspect(value, opts = {}) { + return nodeInspect(value, {colors: ENABLE_COLOR, ...opts}); } -// --> YAML data repository structure constants - -export const WIKI_INFO_FILE = 'wiki-info.yaml'; -export const BUILD_DIRECTIVE_DATA_FILE = 'build-directives.yaml'; -export const HOMEPAGE_LAYOUT_DATA_FILE = 'homepage.yaml'; -export const ARTIST_DATA_FILE = 'artists.yaml'; -export const FLASH_DATA_FILE = 'flashes.yaml'; -export const NEWS_DATA_FILE = 'news.yaml'; -export const ART_TAG_DATA_FILE = 'tags.yaml'; -export const GROUP_DATA_FILE = 'groups.yaml'; -export const STATIC_PAGE_DATA_FILE = 'static-pages.yaml'; - -export const DATA_ALBUM_DIRECTORY = 'album'; - -// --> Document processing functions - // General function for inputting a single document (usually loaded from YAML) // and outputting an instance of a provided Thing subclass. // // makeProcessDocument is a factory function: the returned function will take a // document and apply the configuration passed to makeProcessDocument in order // to construct a Thing subclass. -function makeProcessDocument(thingClass, { - // Optional early step for transforming field values before providing them - // to the Thing's update() method. This is useful when the input format - // (i.e. values in the document) differ from the format the actual Thing - // expects. - // - // Each key and value are a field name (not an update() property) and a - // function which takes the value for that field and returns the value which - // will be passed on to update(). - fieldTransformations = {}, - - // Mapping of Thing.update() source properties to field names. - // - // Note this is property -> field, not field -> property. This is a - // shorthand convenience because properties are generally typical - // camel-cased JS properties, while fields may contain whitespace and be - // more easily represented as quoted strings. - propertyFieldMapping, - - // Completely ignored fields. These won't throw an unknown field error if - // they're present in a document, but they won't be used for Thing property - // generation, either. Useful for stuff that's present in data files but not - // yet implemented as part of a Thing's data model! - ignoredFields = [] +// +function makeProcessDocument(thingConstructor, { + // The bulk of configuration happens here in the spec's `fields` property. + // Each key is a field that's expected on the source document; fields that + // don't match one of these keys will cause an error. Values are object + // entries describing what to do with the field. + // + // A field entry's `property` tells what property the value for this field + // will be put into, on the respective Thing (subclass) instance. + // + // A field entry's `transform` optionally allows converting the raw value in + // YAML into some other format before providing setting it on the Thing + // instance. + // + // If a field entry has `ignore: true`, it will be completely skipped by the + // YAML parser - it won't be validated, read, or loaded into data objects. + // This is mainly useful for fields that are purely annotational or are + // currently placeholders. + // + fields: fieldSpecs = {}, + + // List of fields which are invalid when coexisting in a document. + // Data objects are generally allowing with regards to what properties go + // together, allowing for properties to be set separately from each other + // instead of complaining about invalid or unused-data cases. But it's + // useful to see these kinds of errors when actually validating YAML files! + // + // Each item of this array should itself be an object with a descriptive + // message and a list of fields. Of those fields, none should ever coexist + // with any other. For example: + // + // [ + // {message: '...', fields: ['A', 'B', 'C']}, + // {message: '...', fields: ['C', 'D']}, + // ] + // + // ...means A can't coexist with B or C, B can't coexist with A or C, and + // C can't coexist iwth A, B, or D - but it's okay for D to coexist with + // A or B. + // + invalidFieldCombinations = [], + + // Bouncing function used to process subdocuments: this is a function which + // in turn calls the appropriate *result of* makeProcessDocument. + processDocument: bouncer, }) { - if (!propertyFieldMapping) { - throw new Error(`Expected propertyFieldMapping to be provided`); + if (!thingConstructor) { + throw new Error(`Missing Thing class`); + } + + if (!fieldSpecs) { + throw new Error(`Expected fields to be provided`); + } + + if (!bouncer) { + throw new Error(`Missing processDocument bouncer`); + } + + const knownFields = Object.keys(fieldSpecs); + + const ignoredFields = + Object.entries(fieldSpecs) + .filter(([, {ignore}]) => ignore) + .map(([field]) => field); + + const propertyToField = + withEntries(fieldSpecs, entries => entries + .map(([field, {property}]) => [property, field])); + + // TODO: Is this function even necessary?? + // Aren't we doing basically the same work in the function it's decorating??? + const decorateErrorWithName = (fn) => { + const nameField = propertyToField.name; + if (!nameField) return fn; + + return (document) => { + try { + return fn(document); + } catch (error) { + const name = document[nameField]; + error.message = name + ? `(name: ${inspect(name)}) ${error.message}` + : `(${colors.dim(`no name found`)}) ${error.message}`; + throw error; + } + }; + }; + + return decorateErrorWithName((document) => { + const nameField = propertyToField.name; + const namePart = + (nameField + ? (document[nameField] + ? ` named ${colors.green(`"${document[nameField]}"`)}` + : ` (name field, "${nameField}", not specified)`) + : ``); + + const constructorPart = + (thingConstructor[Thing.friendlyName] + ? thingConstructor[Thing.friendlyName] + : thingConstructor.name + ? thingConstructor.name + : `document`); + + const aggregate = openAggregate({ + ...aggregateThrows(ProcessDocumentError), + message: `Errors processing ${constructorPart}` + namePart, + }); + + const thing = Reflect.construct(thingConstructor, []); + + const documentEntries = Object.entries(document) + .filter(([field]) => !ignoredFields.includes(field)); + + const skippedFields = new Set(); + + const unknownFields = documentEntries + .map(([field]) => field) + .filter((field) => !knownFields.includes(field)); + + if (!empty(unknownFields)) { + aggregate.push(new UnknownFieldsError(unknownFields)); + + for (const field of unknownFields) { + skippedFields.add(field); + } } - const knownFields = Object.values(propertyFieldMapping); - - // Invert the property-field mapping, since it'll come in handy for - // assigning update() source values later. - const fieldPropertyMapping = Object.fromEntries( - (Object.entries(propertyFieldMapping) - .map(([ property, field ]) => [field, property]))); - - const decorateErrorWithName = fn => { - const nameField = propertyFieldMapping['name']; - if (!nameField) return fn; - - return document => { - try { - return fn(document); - } catch (error) { - const name = document[nameField]; - error.message = (name - ? `(name: ${inspect(name)}) ${error.message}` - : `(${color.dim(`no name found`)}) ${error.message}`); - throw error; - } - }; - }; + const presentFields = Object.keys(document); - return decorateErrorWithName(document => { - const documentEntries = Object.entries(document) - .filter(([ field ]) => !ignoredFields.includes(field)); + const fieldCombinationErrors = []; - const unknownFields = documentEntries - .map(([ field ]) => field) - .filter(field => !knownFields.includes(field)); + for (const {message, fields} of invalidFieldCombinations) { + const fieldsPresent = + presentFields.filter(field => fields.includes(field)); - if (unknownFields.length) { - throw new makeProcessDocument.UnknownFieldsError(unknownFields); - } + if (fieldsPresent.length >= 2) { + const filteredDocument = + filterProperties( + document, + fieldsPresent, + {preserveOriginalOrder: true}); - const fieldValues = {}; + fieldCombinationErrors.push( + new FieldCombinationError(filteredDocument, message)); - for (const [ field, value ] of documentEntries) { - if (Object.hasOwn(fieldTransformations, field)) { - fieldValues[field] = fieldTransformations[field](value); - } else { - fieldValues[field] = value; - } + for (const field of Object.keys(filteredDocument)) { + skippedFields.add(field); } + } + } - const sourceProperties = {}; + if (!empty(fieldCombinationErrors)) { + aggregate.push(new FieldCombinationAggregateError(fieldCombinationErrors)); + } - for (const [ field, value ] of Object.entries(fieldValues)) { - const property = fieldPropertyMapping[field]; - sourceProperties[property] = value; - } + const fieldValues = {}; + + const subdocSymbol = Symbol('subdoc'); + const subdocLayouts = {}; + + const isSubdocToken = value => + typeof value === 'object' && + value !== null && + Object.hasOwn(value, subdocSymbol); + + const transformUtilities = { + ...thingConstructors, + + subdoc(documentType, data, { + bindInto = null, + provide = null, + } = {}) { + if (!documentType) + throw new Error(`Expected document type, got ${typeAppearance(documentType)}`); + if (!data) + throw new Error(`Expected data, got ${typeAppearance(data)}`); + if (typeof data !== 'object' || data === null) + throw new Error(`Expected data to be an object, got ${typeAppearance(data)}`); + if (typeof bindInto !== 'string' && bindInto !== null) + throw new Error(`Expected bindInto to be a string, got ${typeAppearance(bindInto)}`); + if (typeof provide !== 'object' && provide !== null) + throw new Error(`Expected provide to be an object, got ${typeAppearance(provide)}`); + + return { + [subdocSymbol]: { + documentType, + data, + bindInto, + provide, + }, + }; + }, + }; - const thing = Reflect.construct(thingClass, []); + for (const [field, documentValue] of documentEntries) { + if (skippedFields.has(field)) continue; - withAggregate({message: `Errors applying ${color.green(thingClass.name)} properties`}, ({ call }) => { - for (const [ property, value ] of Object.entries(sourceProperties)) { - call(() => (thing[property] = value)); - } - }); + // This variable would like to certify itself as "not into capitalism". + let propertyValue = + (fieldSpecs[field].transform + ? fieldSpecs[field].transform(documentValue, transformUtilities) + : documentValue); - return thing; - }); -} + // Completely blank items in a YAML list are read as null. + // They're handy to have around when filling out a document and shouldn't + // be considered an error (or data at all). + if (Array.isArray(propertyValue)) { + const wasEmpty = empty(propertyValue); -makeProcessDocument.UnknownFieldsError = class UnknownFieldsError extends Error { - constructor(fields) { - super(`Unknown fields present: ${fields.join(', ')}`); - this.fields = fields; - } -}; + propertyValue = + propertyValue.filter(item => item !== null); -export const processAlbumDocument = makeProcessDocument(Album, { - fieldTransformations: { - 'Artists': parseContributors, - 'Cover Artists': parseContributors, - 'Default Track Cover Artists': parseContributors, - 'Wallpaper Artists': parseContributors, - 'Banner Artists': parseContributors, + const isEmpty = empty(propertyValue); - 'Date': value => new Date(value), - 'Date Added': value => new Date(value), - 'Cover Art Date': value => new Date(value), - 'Default Track Cover Art Date': value => new Date(value), + // Don't set arrays which are empty as a result of the above filter. + // Arrays which were originally empty, i.e. `Field: []`, are still + // valid data, but if it's just an array not containing any filled out + // items, it should be treated as a placeholder and skipped over. + if (isEmpty && !wasEmpty) { + propertyValue = null; + } + } - 'Banner Dimensions': parseDimensions, + if (isSubdocToken(propertyValue)) { + subdocLayouts[field] = propertyValue[subdocSymbol]; + continue; + } - 'Additional Files': parseAdditionalFiles, - }, + if (Array.isArray(propertyValue) && propertyValue.every(isSubdocToken)) { + subdocLayouts[field] = + propertyValue + .map(token => token[subdocSymbol]); + continue; + } - propertyFieldMapping: { - name: 'Album', + fieldValues[field] = propertyValue; + } - color: 'Color', - directory: 'Directory', - urls: 'URLs', + const subdocErrors = []; - artistContribsByRef: 'Artists', - coverArtistContribsByRef: 'Cover Artists', - trackCoverArtistContribsByRef: 'Default Track Cover Artists', + const followSubdocSetup = setup => { + let error = null; - coverArtFileExtension: 'Cover Art File Extension', - trackCoverArtFileExtension: 'Track Art File Extension', + let subthing; + try { + const result = bouncer(setup.data, setup.documentType); + subthing = result.thing; + result.aggregate.close(); + } catch (caughtError) { + error = caughtError; + } - wallpaperArtistContribsByRef: 'Wallpaper Artists', - wallpaperStyle: 'Wallpaper Style', - wallpaperFileExtension: 'Wallpaper File Extension', + if (subthing) { + if (setup.bindInto) { + subthing[setup.bindInto] = thing; + } - bannerArtistContribsByRef: 'Banner Artists', - bannerStyle: 'Banner Style', - bannerFileExtension: 'Banner File Extension', - bannerDimensions: 'Banner Dimensions', + if (setup.provide) { + Object.assign(subthing, setup.provide); + } + } - date: 'Date', - trackArtDate: 'Default Track Cover Art Date', - coverArtDate: 'Cover Art Date', - dateAddedToWiki: 'Date Added', + return {error, subthing}; + }; + + for (const [field, layout] of Object.entries(subdocLayouts)) { + if (Array.isArray(layout)) { + const subthings = []; + let anySucceeded = false; + let anyFailed = false; + + for (const [index, setup] of layout.entries()) { + const {subthing, error} = followSubdocSetup(setup); + if (error) { + subdocErrors.push(new SubdocError( + {field, index}, + setup, + {cause: error})); + } + + if (subthing) { + subthings.push(subthing); + anySucceeded = true; + } else { + anyFailed = true; + } + } - hasCoverArt: 'Has Cover Art', - hasTrackArt: 'Has Track Art', - hasTrackNumbers: 'Has Track Numbers', - isMajorRelease: 'Major Release', - isListedOnHomepage: 'Listed on Homepage', + if (anySucceeded) { + fieldValues[field] = subthings; + } else if (anyFailed) { + skippedFields.add(field); + } + } else { + const setup = layout; + const {subthing, error} = followSubdocSetup(setup); + + if (error) { + subdocErrors.push(new SubdocError( + {field}, + setup, + {cause: error})); + } - groupsByRef: 'Groups', - artTagsByRef: 'Art Tags', - commentary: 'Commentary', + if (subthing) { + fieldValues[field] = subthing; + } else { + skippedFields.add(field); + } + } + } - additionalFiles: 'Additional Files', + if (!empty(subdocErrors)) { + aggregate.push(new SubdocAggregateError( + subdocErrors, thingConstructor)); } -}); -export const processTrackGroupDocument = makeProcessDocument(TrackGroup, { - fieldTransformations: { - 'Date Originally Released': value => new Date(value), - }, + const fieldValueErrors = []; + + for (const [field, value] of Object.entries(fieldValues)) { + const {property} = fieldSpecs[field]; - propertyFieldMapping: { - name: 'Group', - color: 'Color', - dateOriginallyReleased: 'Date Originally Released', + try { + thing[property] = value; + } catch (caughtError) { + skippedFields.add(field); + fieldValueErrors.push(new FieldValueError( + field, value, {cause: caughtError})); + } } -}); -export const processTrackDocument = makeProcessDocument(Track, { - fieldTransformations: { - 'Duration': getDurationInSeconds, + if (!empty(fieldValueErrors)) { + aggregate.push(new FieldValueAggregateError( + fieldValueErrors, thingConstructor)); + } - 'Date First Released': value => new Date(value), - 'Cover Art Date': value => new Date(value), + if (skippedFields.size >= 1) { + aggregate.push( + new SkippedFieldsSummaryError( + filterProperties( + document, + Array.from(skippedFields), + {preserveOriginalOrder: true}))); + } - 'Artists': parseContributors, - 'Contributors': parseContributors, - 'Cover Artists': parseContributors, + return {thing, aggregate}; + }); +} - 'Additional Files': parseAdditionalFiles, - }, +export class ProcessDocumentError extends AggregateError {} - propertyFieldMapping: { - name: 'Track', +export class UnknownFieldsError extends Error { + constructor(fields) { + super(`Unknown fields ignored: ${fields.map(field => colors.red(field)).join(', ')}`); + this.fields = fields; + } +} - directory: 'Directory', - duration: 'Duration', - urls: 'URLs', +export class FieldCombinationAggregateError extends AggregateError { + constructor(errors) { + super(errors, `Invalid field combinations - all involved fields ignored`); + } +} - coverArtDate: 'Cover Art Date', - coverArtFileExtension: 'Cover Art File Extension', - dateFirstReleased: 'Date First Released', - hasCoverArt: 'Has Cover Art', - hasURLs: 'Has URLs', +export class FieldCombinationError extends Error { + constructor(fields, message) { + const fieldNames = Object.keys(fields); + + const fieldNamesText = + fieldNames + .map(field => colors.red(field)) + .join(', '); + + const mainMessage = `Don't combine ${fieldNamesText}`; + + const causeMessage = + (typeof message === 'function' + ? message(fields) + : typeof message === 'string' + ? message + : null); + + super(mainMessage, { + cause: + (causeMessage + ? new Error(causeMessage) + : null), + }); - referencedTracksByRef: 'Referenced Tracks', - artistContribsByRef: 'Artists', - contributorContribsByRef: 'Contributors', - coverArtistContribsByRef: 'Cover Artists', - artTagsByRef: 'Art Tags', - originalReleaseTrackByRef: 'Originally Released As', + this.fields = fields; + } +} - commentary: 'Commentary', - lyrics: 'Lyrics', +export class FieldValueAggregateError extends AggregateError { + [Symbol.for('hsmusic.aggregate.translucent')] = true; - additionalFiles: 'Additional Files', - }, + constructor(errors, thingConstructor) { + const constructorText = + colors.green(thingConstructor.name); - ignoredFields: ['Sampled Tracks'] -}); + super( + errors, + `Errors processing field values for ${constructorText}`); + } +} -export const processArtistDocument = makeProcessDocument(Artist, { - propertyFieldMapping: { - name: 'Artist', +export class FieldValueError extends Error { + constructor(field, value, options) { + const fieldText = + colors.green(`"${field}"`); - directory: 'Directory', - urls: 'URLs', - hasAvatar: 'Has Avatar', - avatarFileExtension: 'Avatar File Extension', + const valueText = + inspect(value, {maxStringLength: 40}); - aliasNames: 'Aliases', + super( + `Failed to set ${fieldText} field to ${valueText}`, + options); + } +} - contextNotes: 'Context Notes' - }, +export class SkippedFieldsSummaryError extends Error { + constructor(filteredDocument) { + const entries = Object.entries(filteredDocument); + + const lines = + entries.map(([field, value]) => + ` - ${field}: ` + + inspect(value, {maxStringLength: 70}) + .split('\n') + .map((line, index) => index === 0 ? line : ` ${line}`) + .join('\n')); + + const numFieldsText = + (entries.length === 1 + ? `1 field` + : `${entries.length} fields`); + + super( + colors.bright(colors.yellow(`Altogether, skipped ${numFieldsText}:`)) + '\n' + + lines.join('\n') + '\n' + + colors.bright(colors.yellow(`See above errors for details.`))); + } +} - ignoredFields: ['Dead URLs'] -}); +export class SubdocError extends Error { + constructor({field, index = null}, setup, options) { + const fieldText = + (index === null + ? colors.green(`"${field}"`) + : colors.yellow(`#${index + 1}`) + ' in ' + + colors.green(`"${field}"`)); -export const processFlashDocument = makeProcessDocument(Flash, { - fieldTransformations: { - 'Date': value => new Date(value), + const constructorText = + setup.documentType.name; - 'Contributors': parseContributors, - }, + if (options.cause instanceof ProcessDocumentError) { + options.cause[Symbol.for('hsmusic.aggregate.translucent')] = true; + } - propertyFieldMapping: { - name: 'Flash', + super( + `Errors processing ${constructorText} for ${fieldText} field`, + options); + } +} - directory: 'Directory', - page: 'Page', - date: 'Date', - coverArtFileExtension: 'Cover Art File Extension', +export class SubdocAggregateError extends AggregateError { + [Symbol.for('hsmusic.aggregate.translucent')] = true; - featuredTracksByRef: 'Featured Tracks', - contributorContribsByRef: 'Contributors', - urls: 'URLs' - }, -}); + constructor(errors, thingConstructor) { + const constructorText = + colors.green(thingConstructor.name); -export const processFlashActDocument = makeProcessDocument(FlashAct, { - propertyFieldMapping: { - name: 'Act', - color: 'Color', - anchor: 'Anchor', - jump: 'Jump', - jumpColor: 'Jump Color' - } -}); - -export const processNewsEntryDocument = makeProcessDocument(NewsEntry, { - fieldTransformations: { - 'Date': value => new Date(value) - }, - - propertyFieldMapping: { - name: 'Name', - directory: 'Directory', - date: 'Date', - content: 'Content', - } -}); - -export const processArtTagDocument = makeProcessDocument(ArtTag, { - propertyFieldMapping: { - name: 'Tag', - directory: 'Directory', - color: 'Color', - isContentWarning: 'Is CW' - } -}); - -export const processGroupDocument = makeProcessDocument(Group, { - propertyFieldMapping: { - name: 'Group', - directory: 'Directory', - description: 'Description', - urls: 'URLs', - } -}); + super( + errors, + `Errors processing subdocuments for ${constructorText}`); + } +} -export const processGroupCategoryDocument = makeProcessDocument(GroupCategory, { - propertyFieldMapping: { - name: 'Category', - color: 'Color', - } -}); +export function parseDate(date) { + return new Date(date); +} -export const processStaticPageDocument = makeProcessDocument(StaticPage, { - propertyFieldMapping: { - name: 'Name', - nameShort: 'Short Name', - directory: 'Directory', +export function parseDuration(string) { + if (typeof string !== 'string') { + return string; + } + + const parts = string.split(':').map((n) => parseInt(n)); + if (parts.length === 3) { + return parts[0] * 3600 + parts[1] * 60 + parts[2]; + } else if (parts.length === 2) { + return parts[0] * 60 + parts[1]; + } else { + return 0; + } +} - content: 'Content', - stylesheet: 'Style', +export const extractAccentRegex = + /^(?<main>.*?)(?: \((?<accent>.*)\))?$/; + +export const extractPrefixAccentRegex = + /^(?:\((?<accent>.*)\) )?(?<main>.*?)$/; + +// TODO: Should this fit better within actual YAML loading infrastructure?? +export function parseArrayEntries(entries, mapFn) { + // If this isn't something we can parse, just return it as-is. + // The Thing object's validators will handle the data error better + // than we're able to here. + if (!Array.isArray(entries)) { + return entries; + } + + // If the array is REALLY ACTUALLY empty (it's represented in YAML + // as literally an empty []), that's something we want to reflect. + if (empty(entries)) { + return entries; + } + + const nonNullEntries = + entries.filter(value => value !== null); + + // On the other hand, if the array only contains null, it's just + // a placeholder, so skip over the field like it's not actually + // been put there yet. + if (empty(nonNullEntries)) { + return null; + } + + return entries.map(mapFn); +} - showInNavigationBar: 'Show in Navigation Bar' - } -}); - -export const processWikiInfoDocument = makeProcessDocument(WikiInfo, { - propertyFieldMapping: { - name: 'Name', - nameShort: 'Short Name', - color: 'Color', - description: 'Description', - footerContent: 'Footer Content', - defaultLanguage: 'Default Language', - canonicalBase: 'Canonical Base', - divideTrackListsByGroupsByRef: 'Divide Track Lists By Groups', - enableFlashesAndGames: 'Enable Flashes & Games', - enableListings: 'Enable Listings', - enableNews: 'Enable News', - enableArtTagUI: 'Enable Art Tag UI', - enableGroupUI: 'Enable Group UI', - } -}); +export function parseContributors(entries) { + return parseArrayEntries(entries, item => { + if (typeof item === 'object' && item['Who']) + return { + artist: item['Who'], + annotation: item['What'] ?? null, + }; -export const processHomepageLayoutDocument = makeProcessDocument(HomepageLayout, { - propertyFieldMapping: { - sidebarContent: 'Sidebar Content' - }, + if (typeof item === 'object' && item['Artist']) + return { + artist: item['Artist'], + annotation: item['Annotation'] ?? null, - ignoredFields: ['Homepage'] -}); + countInContributionTotals: item['Count In Contribution Totals'] ?? null, + countInDurationTotals: item['Count In Duration Totals'] ?? null, + }; -export function makeProcessHomepageLayoutRowDocument(rowClass, spec) { - return makeProcessDocument(rowClass, { - ...spec, + if (typeof item !== 'string') return item; - propertyFieldMapping: { - name: 'Row', - color: 'Color', - type: 'Type', - ...spec.propertyFieldMapping, - } - }); + const match = item.match(extractAccentRegex); + if (!match) return item; + + return { + artist: match.groups.main, + annotation: match.groups.accent ?? null, + }; + }); } -export const homepageLayoutRowTypeProcessMapping = { - albums: makeProcessHomepageLayoutRowDocument(HomepageLayoutAlbumsRow, { - propertyFieldMapping: { - sourceGroupByRef: 'Group', - countAlbumsFromGroup: 'Count', - sourceAlbumsByRef: 'Albums', - actionLinks: 'Actions' - } - }) -}; +export function parseAdditionalFiles(entries) { + return parseArrayEntries(entries, item => { + if (typeof item !== 'object') return item; -export function processHomepageLayoutRowDocument(document) { - const type = document['Type']; + return { + title: item['Title'], + description: item['Description'] ?? null, + files: item['Files'], + }; + }); +} - const match = Object.entries(homepageLayoutRowTypeProcessMapping) - .find(([ key ]) => key === type); +export function parseAdditionalNames(entries) { + return parseArrayEntries(entries, item => { + if (typeof item === 'object' && typeof item['Name'] === 'string') + return { + name: item['Name'], + annotation: item['Annotation'] ?? null, + }; - if (!match) { - throw new TypeError(`No processDocument function for row type ${type}!`); - } + if (typeof item !== 'string') return item; - return match[1](document); -} + const match = item.match(extractAccentRegex); + if (!match) return item; -// --> Utilities shared across document parsing functions + return { + name: match.groups.main, + annotation: match.groups.accent ?? null, + }; + }); +} -export function getDurationInSeconds(string) { - if (typeof string === 'number') { - return string; - } +export function parseSerieses(entries) { + return parseArrayEntries(entries, item => { + if (typeof item !== 'object') return item; - if (typeof string !== 'string') { - throw new TypeError(`Expected a string or number, got ${string}`); - } + return { + name: item['Name'], + description: item['Description'] ?? null, + albums: item['Albums'] ?? null, - const parts = string.split(':').map(n => parseInt(n)) - if (parts.length === 3) { - return parts[0] * 3600 + parts[1] * 60 + parts[2] - } else if (parts.length === 2) { - return parts[0] * 60 + parts[1] - } else { - return 0 - } + showAlbumArtists: item['Show Album Artists'] ?? null, + }; + }); } -export function parseAdditionalFiles(array) { - if (!array) return null; - if (!Array.isArray(array)) { - // Error will be caught when validating against whatever this value is - return array; - } +export function parseWallpaperParts(entries) { + return parseArrayEntries(entries, item => { + if (typeof item !== 'object') return item; - return array.map(item => ({ - title: item['Title'], - description: item['Description'] ?? null, - files: item['Files'] - })); + return { + asset: + (item['Asset'] === 'none' + ? null + : item['Asset'] ?? null), + + style: item['Style'] ?? null, + }; + }); } -export function parseCommentary(text) { - if (text) { - const lines = String(text).split('\n'); - if (!lines[0].replace(/<\/b>/g, '').includes(':</i>')) { - return {error: `An entry is missing commentary citation: "${lines[0].slice(0, 40)}..."`}; - } - return text; - } else { - return null; - } +export function parseDimensions(string) { + // It's technically possible to pass an array like [30, 40] through here. + // That's not really an issue because if it isn't of the appropriate shape, + // the Thing object's validators will handle the error. + if (typeof string !== 'string') { + return string; + } + + const parts = string.split(/[x,* ]+/g); + + if (parts.length !== 2) { + throw new Error(`Invalid dimensions: ${string} (expected "width & height")`); + } + + const nums = parts.map((part) => Number(part.trim())); + + if (nums.includes(NaN)) { + throw new Error(`Invalid dimensions: ${string} (couldn't parse as numbers)`); + } + + return nums; } -export function parseContributors(contributors) { - if (!contributors) { - return null; - } +export const contributionPresetYAMLSpec = [ + {from: 'Album', to: 'album', fields: [ + {from: 'Artists', to: 'artistContribs'}, + ]}, - if (contributors.length === 1 && contributors[0].startsWith('<i>')) { - const arr = []; - arr.textContent = contributors[0]; - return arr; - } + {from: 'Flash', to: 'flash', fields: [ + {from: 'Contributors', to: 'contributorContribs'}, + ]}, - contributors = contributors.map(contrib => { - // 8asically, the format is "Who (What)", or just "Who". 8e sure to - // keep in mind that "what" doesn't necessarily have a value! - const match = contrib.match(/^(.*?)( \((.*)\))?$/); - if (!match) { - return contrib; - } - const who = match[1]; - const what = match[3] || null; - return {who, what}; + {from: 'Track', to: 'track', fields: [ + {from: 'Artists', to: 'artistContribs'}, + {from: 'Contributors', to: 'contributorContribs'}, + ]}, +]; + +export function parseContributionPresetContext(context) { + if (!Array.isArray(context)) { + return context; + } + + const [target, ...fields] = context; + + const targetEntry = + contributionPresetYAMLSpec + .find(({from}) => from === target); + + if (!targetEntry) { + return context; + } + + const properties = + fields.map(field => { + const fieldEntry = + targetEntry.fields + .find(({from}) => from === field); + + if (!fieldEntry) return field; + + return fieldEntry.to; }); - const badContributor = contributors.find(val => typeof val === 'string'); - if (badContributor) { - return {error: `An entry has an incorrectly formatted contributor, "${badContributor}".`}; - } + return [targetEntry.to, ...properties]; +} - if (contributors.length === 1 && contributors[0].who === 'none') { - return null; - } +export function parseContributionPresets(list) { + if (!Array.isArray(list)) return list; + + return list.map(item => { + if (typeof item !== 'object') return item; + + return { + annotation: + item['Annotation'] ?? null, + + context: + parseContributionPresetContext( + item['Context'] ?? null), + + countInContributionTotals: + item['Count In Contribution Totals'] ?? null, - return contributors; + countInDurationTotals: + item['Count In Duration Totals'] ?? null, + }; + }); } -function parseDimensions(string) { - if (!string) { - return null; - } +export function parseAnnotatedReferences(entries, { + referenceField = 'References', + annotationField = 'Annotation', + referenceProperty = 'reference', + annotationProperty = 'annotation', +} = {}) { + return parseArrayEntries(entries, item => { + if (typeof item === 'object' && item[referenceField]) + return { + [referenceProperty]: item[referenceField], + [annotationProperty]: item[annotationField] ?? null, + }; + + if (typeof item !== 'string') return item; + + const match = item.match(extractAccentRegex); + if (!match) + return { + [referenceProperty]: item, + [annotationProperty]: null, + }; - const parts = string.split(/[x,* ]+/g); - if (parts.length !== 2) throw new Error(`Invalid dimensions: ${string} (expected width & height)`); - const nums = parts.map(part => Number(part.trim())); - if (nums.includes(NaN)) throw new Error(`Invalid dimensions: ${string} (couldn't parse as numbers)`); - return nums; + return { + [referenceProperty]: match.groups.main, + [annotationProperty]: match.groups.accent ?? null, + }; + }); } -// --> Data repository loading functions and descriptors +export function parseArtwork({ + single = false, + dimensionsFromThingProperty = null, + fileExtensionFromThingProperty = null, + dateFromThingProperty = null, + artistContribsFromThingProperty = null, + artistContribsArtistProperty = null, + artTagsFromThingProperty = null, + referencedArtworksFromThingProperty = null, +}) { + const provide = { + dimensionsFromThingProperty, + fileExtensionFromThingProperty, + dateFromThingProperty, + artistContribsFromThingProperty, + artistContribsArtistProperty, + artTagsFromThingProperty, + referencedArtworksFromThingProperty, + }; + + const parseSingleEntry = (entry, {subdoc, Artwork}) => + subdoc(Artwork, entry, {bindInto: 'thing', provide}); + + const transform = (value, ...args) => + (Array.isArray(value) + ? value.map(entry => parseSingleEntry(entry, ...args)) + : single + ? parseSingleEntry(value, ...args) + : [parseSingleEntry(value, ...args)]); + + transform.provide = provide; + + return transform; +} // documentModes: Symbols indicating sets of behavior for loading and processing // data files. export const documentModes = { - // onePerFile: One document per file. Expects files array (or function) and - // processDocument function. Obviously, each specified data file should only - // contain one YAML document (an error will be thrown otherwise). Calls save - // with an array of processed documents (wiki objects). - onePerFile: Symbol('Document mode: onePerFile'), - - // headerAndEntries: One or more documents per file; the first document is - // treated as a "header" and represents data which pertains to all following - // "entry" documents. Expects files array (or function) and - // processHeaderDocument and processEntryDocument functions. Calls save with - // an array of {header, entries} objects. - // - // Please note that the final results loaded from each file may be "missing" - // data objects corresponding to entry documents if the processEntryDocument - // function throws on any entries, resulting in partial data provided to - // save() - errors will be caught and thrown in the final buildSteps - // aggregate. However, if the processHeaderDocument function fails, all - // following documents in the same file will be ignored as well (i.e. an - // entire file will be excempt from the save() function's input). - headerAndEntries: Symbol('Document mode: headerAndEntries'), - - // allInOne: One or more documents, all contained in one file. Expects file - // string (or function) and processDocument function. Calls save with an - // array of processed documents (wiki objects). - allInOne: Symbol('Document mode: allInOne'), - - // oneDocumentTotal: Just a single document, represented in one file. - // Expects file string (or function) and processDocument function. Calls - // save with the single processed wiki document (data object). - // - // Please note that if the single document fails to process, the save() - // function won't be called at all, generally resulting in an altogether - // missing property from the global wikiData object. This should be caught - // and handled externally. - oneDocumentTotal: Symbol('Document mode: oneDocumentTotal'), + // onePerFile: One document per file. Expects files array (or function) and + // processDocument function. Obviously, each specified data file should only + // contain one YAML document (an error will be thrown otherwise). Calls save + // with an array of processed documents (wiki objects). + onePerFile: Symbol('Document mode: onePerFile'), + + // headerAndEntries: One or more documents per file; the first document is + // treated as a "header" and represents data which pertains to all following + // "entry" documents. Expects files array (or function) and + // processHeaderDocument and processEntryDocument functions. Calls save with + // an array of {header, entries} objects. + // + // Please note that the final results loaded from each file may be "missing" + // data objects corresponding to entry documents if the processEntryDocument + // function throws on any entries, resulting in partial data provided to + // save() - errors will be caught and thrown in the final buildSteps + // aggregate. However, if the processHeaderDocument function fails, all + // following documents in the same file will be ignored as well (i.e. an + // entire file will be excempt from the save() function's input). + headerAndEntries: Symbol('Document mode: headerAndEntries'), + + // allInOne: One or more documents, all contained in one file. Expects file + // string (or function) and processDocument function. Calls save with an + // array of processed documents (wiki objects). + allInOne: Symbol('Document mode: allInOne'), + + // oneDocumentTotal: Just a single document, represented in one file. + // Expects file string (or function) and processDocument function. Calls + // save with the single processed wiki document (data object). + // + // Please note that if the single document fails to process, the save() + // function won't be called at all, generally resulting in an altogether + // missing property from the global wikiData object. This should be caught + // and handled externally. + oneDocumentTotal: Symbol('Document mode: oneDocumentTotal'), }; // dataSteps: Top-level array of "steps" for loading YAML document files. @@ -624,670 +896,700 @@ export const documentModes = { // them to each other, setting additional properties, etc). Input argument // format depends on documentMode. // -export const dataSteps = [ - { - title: `Process wiki info file`, - file: WIKI_INFO_FILE, +export function getAllDataSteps() { + try { + thingConstructors; + } catch (error) { + throw new Error(`Thing constructors aren't ready yet, can't get all data steps`); + } - documentMode: documentModes.oneDocumentTotal, - processDocument: processWikiInfoDocument, + const steps = []; - save(wikiInfo) { - if (!wikiInfo) { - return; - } + const seenLoadingFns = new Set(); - return {wikiInfo}; - } - }, - - { - title: `Process album files`, - files: async dataPath => ( - (await findFiles(path.join(dataPath, DATA_ALBUM_DIRECTORY), { - filter: f => path.extname(f) === '.yaml', - joinParentDirectory: false - })).map(file => path.join(DATA_ALBUM_DIRECTORY, file))), - - documentMode: documentModes.headerAndEntries, - processHeaderDocument: processAlbumDocument, - processEntryDocument(document) { - return ('Group' in document - ? processTrackGroupDocument(document) - : processTrackDocument(document)); - }, + for (const thingConstructor of Object.values(thingConstructors)) { + const getSpecFn = thingConstructor[Thing.getYamlLoadingSpec]; + if (!getSpecFn) continue; - save(results) { - const albumData = []; - const trackData = []; - - for (const { header: album, entries } of results) { - // We can't mutate an array once it's set as a property - // value, so prepare the tracks and track groups that will - // show up in a track list all the way before actually - // applying them. - const trackGroups = []; - let currentTracksByRef = null; - let currentTrackGroup = null; - - const albumRef = Thing.getReference(album); - - function closeCurrentTrackGroup() { - if (currentTracksByRef) { - let trackGroup; - - if (currentTrackGroup) { - trackGroup = currentTrackGroup; - } else { - trackGroup = new TrackGroup(); - trackGroup.name = `Default Track Group`; - trackGroup.isDefaultTrackGroup = true; - } - - trackGroup.album = album; - trackGroup.tracksByRef = currentTracksByRef; - trackGroups.push(trackGroup); - } - } + // Subclasses can expose literally the same static properties + // by inheritence. We don't want to double-count those! + if (seenLoadingFns.has(getSpecFn)) continue; + seenLoadingFns.add(getSpecFn); - for (const entry of entries) { - if (entry instanceof TrackGroup) { - closeCurrentTrackGroup(); - currentTracksByRef = []; - currentTrackGroup = entry; - continue; - } + steps.push(getSpecFn({ + documentModes, + thingConstructors, + })); + } - trackData.push(entry); + sortByName(steps, {getName: step => step.title}); - entry.dataSourceAlbumByRef = albumRef; + return steps; +} - const trackRef = Thing.getReference(entry); - if (currentTracksByRef) { - currentTracksByRef.push(trackRef); - } else { - currentTracksByRef = [trackRef]; - } - } +export async function getFilesFromDataStep(dataStep, {dataPath}) { + const {documentMode} = dataStep; + + switch (documentMode) { + case documentModes.allInOne: + case documentModes.oneDocumentTotal: { + if (!dataStep.file) { + throw new Error(`Expected 'file' property for ${documentMode.toString()}`); + } + + const localFile = + (typeof dataStep.file === 'function' + ? await dataStep.file(dataPath) + : dataStep.file); + + const fileUnderDataPath = + path.join(dataPath, localFile); + + const statResult = + await stat(fileUnderDataPath).then( + () => true, + error => { + if (error.code === 'ENOENT') { + return false; + } else { + throw error; + } + }); - closeCurrentTrackGroup(); + if (statResult) { + return [fileUnderDataPath]; + } else { + return []; + } + } - album.trackGroups = trackGroups; - albumData.push(album); - } + case documentModes.headerAndEntries: + case documentModes.onePerFile: { + if (!dataStep.files) { + throw new Error(`Expected 'files' property for ${documentMode.toString()}`); + } + + const localFiles = + (typeof dataStep.files === 'function' + ? await dataStep.files(dataPath).then( + files => files, + error => { + if (error.code === 'ENOENT') { + return []; + } else { + throw error; + } + }) + : dataStep.files); - return {albumData, trackData}; - } - }, - - { - title: `Process artists file`, - file: ARTIST_DATA_FILE, - - documentMode: documentModes.allInOne, - processDocument: processArtistDocument, - - save(results) { - const artistData = results; - - const artistAliasData = results.flatMap(artist => { - const origRef = Thing.getReference(artist); - return (artist.aliasNames?.map(name => { - const alias = new Artist(); - alias.name = name; - alias.isAlias = true; - alias.aliasedArtistRef = origRef; - alias.artistData = artistData; - return alias; - }) ?? []); - }); - - return {artistData, artistAliasData}; - } - }, - - // TODO: WD.wikiInfo.enableFlashesAndGames && - { - title: `Process flashes file`, - file: FLASH_DATA_FILE, - - documentMode: documentModes.allInOne, - processDocument(document) { - return ('Act' in document - ? processFlashActDocument(document) - : processFlashDocument(document)); - }, + const filesUnderDataPath = + localFiles + .map(file => path.join(dataPath, file)); - save(results) { - let flashAct; - let flashesByRef = []; + return filesUnderDataPath; + } - if (results[0] && !(results[0] instanceof FlashAct)) { - throw new Error(`Expected an act at top of flash data file`); - } + default: + throw new Error(`Unknown document mode ${documentMode.toString()}`); + } +} - for (const thing of results) { - if (thing instanceof FlashAct) { - if (flashAct) { - Object.assign(flashAct, {flashesByRef}); - } +export async function loadYAMLDocumentsFromFile(file) { + let contents; + try { + contents = await readFile(file, 'utf-8'); + } catch (caughtError) { + throw new Error(`Failed to read data file`, {cause: caughtError}); + } + + let documents; + try { + documents = yaml.loadAll(contents); + } catch (caughtError) { + throw new Error(`Failed to parse valid YAML`, {cause: caughtError}); + } + + const aggregate = openAggregate({ + message: `Found blank documents - check for extra '${colors.cyan(`---`)}'`, + }); + + const filteredDocuments = + documents + .filter(doc => doc !== null); + + if (filteredDocuments.length !== documents.length) { + const blankIndexRangeInfo = + documents + .map((doc, index) => [doc, index]) + .filter(([doc]) => doc === null) + .map(([doc, index]) => index) + .reduce((accumulator, index) => { + if (accumulator.length === 0) { + return [[index, index]]; + } + const current = accumulator.at(-1); + const rest = accumulator.slice(0, -1); + if (current[1] === index - 1) { + return rest.concat([[current[0], index]]); + } else { + return accumulator.concat([[index, index]]); + } + }, []) + .map(([start, end]) => ({ + start, + end, + count: end - start + 1, + previous: atOffset(documents, start, -1), + next: atOffset(documents, end, +1), + })); + + for (const {start, end, count, previous, next} of blankIndexRangeInfo) { + const parts = []; + + if (count === 1) { + const range = `#${start + 1}`; + parts.push(`${count} document (${colors.yellow(range)}), `); + } else { + const range = `#${start + 1}-${end + 1}`; + parts.push(`${count} documents (${colors.yellow(range)}), `); + } + + if (previous === null) { + parts.push(`at start of file`); + } else if (next === null) { + parts.push(`at end of file`); + } else { + const previousDescription = Object.entries(previous).at(0).join(': '); + const nextDescription = Object.entries(next).at(0).join(': '); + parts.push(`between "${colors.cyan(previousDescription)}" and "${colors.cyan(nextDescription)}"`); + } + + aggregate.push(new Error(parts.join(''))); + } + } - flashAct = thing; - flashesByRef = []; - } else { - flashesByRef.push(Thing.getReference(thing)); - } - } + return {result: filteredDocuments, aggregate}; +} - if (flashAct) { - Object.assign(flashAct, {flashesByRef}); - } +// Mapping from dataStep (spec) object each to a sub-map, from thing class to +// processDocument function. +const processDocumentFns = new WeakMap(); + +export function processThingsFromDataStep(documents, dataStep) { + let submap; + if (processDocumentFns.has(dataStep)) { + submap = processDocumentFns.get(dataStep); + } else { + submap = new Map(); + processDocumentFns.set(dataStep, submap); + } + + function processDocument(document, thingClassOrFn) { + const thingClass = + (thingClassOrFn.prototype instanceof Thing + ? thingClassOrFn + : thingClassOrFn(document)); + + let fn; + if (submap.has(thingClass)) { + fn = submap.get(thingClass); + } else { + if (typeof thingClass !== 'function') { + throw new Error(`Expected a thing class, got ${typeAppearance(thingClass)}`); + } - const flashData = results.filter(x => x instanceof Flash); - const flashActData = results.filter(x => x instanceof FlashAct); + if (!(thingClass.prototype instanceof Thing)) { + throw new Error(`Expected a thing class, got ${thingClass.name}`); + } - return {flashData, flashActData}; - } - }, + const spec = thingClass[Thing.yamlDocumentSpec]; - { - title: `Process groups file`, - file: GROUP_DATA_FILE, + if (!spec) { + throw new Error(`Class "${thingClass.name}" doesn't specify Thing.yamlDocumentSpec`); + } - documentMode: documentModes.allInOne, - processDocument(document) { - return ('Category' in document - ? processGroupCategoryDocument(document) - : processGroupDocument(document)); - }, + fn = makeProcessDocument(thingClass, {...spec, processDocument}); + submap.set(thingClass, fn); + } - save(results) { - let groupCategory; - let groupsByRef = []; + return fn(document); + } - if (results[0] && !(results[0] instanceof GroupCategory)) { - throw new Error(`Expected a category at top of group data file`); - } + const {documentMode} = dataStep; - for (const thing of results) { - if (thing instanceof GroupCategory) { - if (groupCategory) { - Object.assign(groupCategory, {groupsByRef}); - } + switch (documentMode) { + case documentModes.allInOne: { + const result = []; + const aggregate = openAggregate({message: `Errors processing documents`}); - groupCategory = thing; - groupsByRef = []; - } else { - groupsByRef.push(Thing.getReference(thing)); - } - } + documents.forEach( + decorateErrorWithIndex((document, index) => { + const {thing, aggregate: subAggregate} = + processDocument(document, dataStep.documentThing); - if (groupCategory) { - Object.assign(groupCategory, {groupsByRef}); - } + thing[Thing.yamlSourceDocument] = document; + thing[Thing.yamlSourceDocumentPlacement] = + [documentModes.allInOne, index]; - const groupData = results.filter(x => x instanceof Group); - const groupCategoryData = results.filter(x => x instanceof GroupCategory); + result.push(thing); + aggregate.call(subAggregate.close); + })); - return {groupData, groupCategoryData}; - } - }, + return { + aggregate, + result, + things: result, + }; + } - { - title: `Process homepage layout file`, - files: [HOMEPAGE_LAYOUT_DATA_FILE], + case documentModes.oneDocumentTotal: { + if (documents.length > 1) + throw new Error(`Only expected one document to be present, got ${documents.length}`); - documentMode: documentModes.headerAndEntries, - processHeaderDocument: processHomepageLayoutDocument, - processEntryDocument: processHomepageLayoutRowDocument, + const {thing, aggregate} = + processDocument(documents[0], dataStep.documentThing); - save(results) { - if (!results[0]) { - return; - } + thing[Thing.yamlSourceDocument] = documents[0]; + thing[Thing.yamlSourceDocumentPlacement] = + [documentModes.oneDocumentTotal]; - const { header: homepageLayout, entries: rows } = results[0]; - Object.assign(homepageLayout, {rows}); - return {homepageLayout}; - } - }, + return { + aggregate, + result: thing, + things: [thing], + }; + } - // TODO: WD.wikiInfo.enableNews && - { - title: `Process news data file`, - file: NEWS_DATA_FILE, + case documentModes.headerAndEntries: { + const headerDocument = documents[0]; + const entryDocuments = documents.slice(1).filter(Boolean); - documentMode: documentModes.allInOne, - processDocument: processNewsEntryDocument, + if (!headerDocument) + throw new Error(`Missing header document (empty file or erroneously starting with "---"?)`); - save(newsData) { - sortByDate(newsData); - newsData.reverse(); + const aggregate = openAggregate({message: `Errors processing documents`}); - return {newsData}; - } - }, + const {thing: headerThing, aggregate: headerAggregate} = + processDocument(headerDocument, dataStep.headerDocumentThing); - { - title: `Process art tags file`, - file: ART_TAG_DATA_FILE, + headerThing[Thing.yamlSourceDocument] = headerDocument; + headerThing[Thing.yamlSourceDocumentPlacement] = + [documentModes.headerAndEntries, 'header']; - documentMode: documentModes.allInOne, - processDocument: processArtTagDocument, + try { + headerAggregate.close(); + } catch (caughtError) { + caughtError.message = `(${colors.yellow(`header`)}) ${caughtError.message}`; + aggregate.push(caughtError); + } - save(artTagData) { - artTagData.sort(sortByName); + const entryThings = []; - return {artTagData}; - } - }, + for (const [index, entryDocument] of entryDocuments.entries()) { + const {thing: entryThing, aggregate: entryAggregate} = + processDocument(entryDocument, dataStep.entryDocumentThing); - { - title: `Process static pages file`, - file: STATIC_PAGE_DATA_FILE, + entryThing[Thing.yamlSourceDocument] = entryDocument; + entryThing[Thing.yamlSourceDocumentPlacement] = + [documentModes.headerAndEntries, 'entry', index]; - documentMode: documentModes.allInOne, - processDocument: processStaticPageDocument, + entryThings.push(entryThing); - save(staticPageData) { - return {staticPageData}; + try { + entryAggregate.close(); + } catch (caughtError) { + caughtError.message = `(${colors.yellow(`entry #${index + 1}`)}) ${caughtError.message}`; + aggregate.push(caughtError); } - }, -]; + } -export async function loadAndProcessDataDocuments({ - dataPath, -}) { - const processDataAggregate = openAggregate({message: `Errors processing data files`}); - const wikiDataResult = {}; - - function decorateErrorWithFile(fn) { - return (x, index, array) => { - try { - return fn(x, index, array); - } catch (error) { - error.message += ( - (error.message.includes('\n') ? '\n' : ' ') + - `(file: ${color.bright(color.blue(path.relative(dataPath, x.file)))})` - ); - throw error; - } - }; + return { + aggregate, + result: { + header: headerThing, + entries: entryThings, + }, + things: [headerThing, ...entryThings], + }; } - for (const dataStep of dataSteps) { - await processDataAggregate.nestAsync( - {message: `Errors during data step: ${dataStep.title}`}, - async ({call, callAsync, map, mapAsync, nest}) => { - const { documentMode } = dataStep; + case documentModes.onePerFile: { + if (documents.length > 1) + throw new Error(`Only expected one document to be present per file, got ${documents.length} here`); - if (!(Object.values(documentModes).includes(documentMode))) { - throw new Error(`Invalid documentMode: ${documentMode.toString()}`); - } + if (empty(documents) || !documents[0]) + throw new Error(`Expected a document, this file is empty`); - if (documentMode === documentModes.allInOne || documentMode === documentModes.oneDocumentTotal) { - if (!dataStep.file) { - throw new Error(`Expected 'file' property for ${documentMode.toString()}`); - } + const {thing, aggregate} = + processDocument(documents[0], dataStep.documentThing); - const file = path.join(dataPath, - (typeof dataStep.file === 'function' - ? await callAsync(dataStep.file, dataPath) - : dataStep.file)); + thing[Thing.yamlSourceDocument] = documents[0]; + thing[Thing.yamlSourceDocumentPlacement] = + [documentModes.onePerFile]; - const readResult = await callAsync(readFile, file, 'utf-8'); + return { + aggregate, + result: thing, + things: [thing], + }; + } - if (!readResult) { - return; - } + default: + throw new Error(`Unknown document mode ${documentMode.toString()}`); + } +} - const yamlResult = (documentMode === documentModes.oneDocumentTotal - ? call(yaml.load, readResult) - : call(yaml.loadAll, readResult)); +export function decorateErrorWithFileFromDataPath(fn, {dataPath}) { + return decorateErrorWithAnnotation(fn, + (caughtError, firstArg) => + annotateErrorWithFile( + caughtError, + path.relative( + dataPath, + (typeof firstArg === 'object' + ? firstArg.file + : firstArg)))); +} - if (!yamlResult) { - return; - } +// Loads a list of files for each data step, and a list of documents +// for each file. +export async function loadYAMLDocumentsFromDataSteps(dataSteps, {dataPath}) { + const aggregate = + openAggregate({ + message: `Errors loading data files`, + translucent: true, + }); - let processResults; + const fileLists = + await Promise.all( + dataSteps.map(dataStep => + getFilesFromDataStep(dataStep, {dataPath}))); - if (documentMode === documentModes.oneDocumentTotal) { - nest({message: `Errors processing document`}, ({ call }) => { - processResults = call(dataStep.processDocument, yamlResult); - }); - } else { - const { result, aggregate } = mapAggregate( - yamlResult, - decorateErrorWithIndex(dataStep.processDocument), - {message: `Errors processing documents`} - ); - processResults = result; - call(aggregate.close); - } + const filePromises = + fileLists + .map(files => files + .map(file => + loadYAMLDocumentsFromFile(file).then( + ({result, aggregate}) => { + const close = + decorateErrorWithFileFromDataPath(aggregate.close, {dataPath}); - if (!processResults) return; + aggregate.close = () => + close({file}); - const saveResult = call(dataStep.save, processResults); + return {result, aggregate}; + }, + (error) => { + const aggregate = {}; - if (!saveResult) return; + annotateErrorWithFile(error, path.relative(dataPath, file)); - Object.assign(wikiDataResult, saveResult); + aggregate.close = () => { + throw error; + }; + + return {result: [], aggregate}; + }))); + + const fileListPromises = + filePromises + .map(filePromises => Promise.all(filePromises)); + + const dataStepPromises = + stitchArrays({ + dataStep: dataSteps, + fileListPromise: fileListPromises, + }).map(async ({dataStep, fileListPromise}) => + openAggregate({ + message: `Errors loading data files for data step: ${colors.bright(dataStep.title)}`, + translucent: true, + }).contain(await fileListPromise)); + + const documentLists = + aggregate + .receive(await Promise.all(dataStepPromises)); + + return {aggregate, result: {documentLists, fileLists}}; +} - return; - } +// Loads a list of things from a list of documents for each file +// for each data step. Nesting! +export async function processThingsFromDataSteps(documentLists, fileLists, dataSteps, {dataPath}) { + const aggregate = + openAggregate({ + message: `Errors processing documents in data files`, + translucent: true, + }); - if (!dataStep.files) { - throw new Error(`Expected 'files' property for ${documentMode.toString()}`); - } + const filePromises = + stitchArrays({ + dataStep: dataSteps, + files: fileLists, + documentLists: documentLists, + }).map(({dataStep, files, documentLists}) => + stitchArrays({ + file: files, + documents: documentLists, + }).map(({file, documents}) => { + const {result, aggregate, things} = + processThingsFromDataStep(documents, dataStep); - const files = ( - (typeof dataStep.files === 'function' - ? await callAsync(dataStep.files, dataPath) - : dataStep.files) - .map(file => path.join(dataPath, file))); - - const readResults = await mapAsync( - files, - file => (readFile(file, 'utf-8') - .then(contents => ({file, contents}))), - {message: `Errors reading data files`}); - - const yamlResults = map( - readResults, - decorateErrorWithFile( - ({ file, contents }) => ({file, documents: yaml.loadAll(contents)})), - {message: `Errors parsing data files as valid YAML`}); - - let processResults; - - if (documentMode === documentModes.headerAndEntries) { - nest({message: `Errors processing data files as valid documents`}, ({ call, map }) => { - processResults = []; - - yamlResults.forEach(({ file, documents }) => { - const [ headerDocument, ...entryDocuments ] = documents; - - const header = call( - decorateErrorWithFile( - ({ document }) => dataStep.processHeaderDocument(document)), - {file, document: headerDocument}); - - // Don't continue processing files whose header - // document is invalid - the entire file is excempt - // from data in this case. - if (!header) { - return; - } - - const entries = map( - entryDocuments.map(document => ({file, document})), - decorateErrorWithFile( - decorateErrorWithIndex( - ({ document }) => dataStep.processEntryDocument(document))), - {message: `Errors processing entry documents`}); - - // Entries may be incomplete (i.e. any errored - // documents won't have a processed output - // represented here) - this is intentional! By - // principle, partial output is preferred over - // erroring an entire file. - processResults.push({header, entries}); - }); - }); - } + for (const thing of things) { + thing[Thing.yamlSourceFilename] = + path.relative(dataPath, file) + .split(path.sep) + .join(path.posix.sep); + } - if (documentMode === documentModes.onePerFile) { - nest({message: `Errors processing data files as valid documents`}, ({ call, map }) => { - processResults = []; - - yamlResults.forEach(({ file, documents }) => { - if (documents.length > 1) { - call(decorateErrorWithFile(() => { - throw new Error(`Only expected one document to be present per file`); - })); - return; - } - - const result = call( - decorateErrorWithFile( - ({ document }) => dataStep.processDocument(document)), - {file, document: documents[0]}); - - if (!result) { - return; - } - - processResults.push(result); - }); - }); - } + const close = decorateErrorWithFileFromDataPath(aggregate.close, {dataPath}); + aggregate.close = () => close({file}); - const saveResult = call(dataStep.save, processResults); + return {result, aggregate}; + })); - if (!saveResult) return; + const fileListPromises = + filePromises + .map(filePromises => Promise.all(filePromises)); - Object.assign(wikiDataResult, saveResult); - }); - } + const dataStepPromises = + stitchArrays({ + dataStep: dataSteps, + fileListPromise: fileListPromises, + }).map(async ({dataStep, fileListPromise}) => + openAggregate({ + message: `Errors loading data files for data step: ${colors.bright(dataStep.title)}`, + translucent: true, + }).contain(await fileListPromise)); - return { - aggregate: processDataAggregate, - result: wikiDataResult - }; + const thingLists = + aggregate + .receive(await Promise.all(dataStepPromises)); + + return {aggregate, result: thingLists}; } -// Data linking! Basically, provide (portions of) wikiData to the Things which -// require it - they'll expose dynamically computed properties as a result (many -// of which are required for page HTML generation). -export function linkWikiDataArrays(wikiData) { - function assignWikiData(things, ...keys) { - for (let i = 0; i < things.length; i++) { - for (let j = 0; j < keys.length; j++) { - const key = keys[j]; - things[i][key] = wikiData[key]; - } - } +// Flattens a list of *lists* of things for a given data step (each list +// corresponding to one YAML file) into results to be saved on the final +// wikiData object, routing thing lists into the step's save() function. +export function saveThingsFromDataStep(thingLists, dataStep) { + const {documentMode} = dataStep; + + switch (documentMode) { + case documentModes.allInOne: { + const things = + (empty(thingLists) + ? [] + : thingLists[0]); + + return dataStep.save(things); } - const WD = wikiData; + case documentModes.oneDocumentTotal: { + const thing = + (empty(thingLists) + ? {} + : thingLists[0]); - assignWikiData([WD.wikiInfo], 'groupData'); + return dataStep.save(thing); + } - assignWikiData(WD.albumData, 'artistData', 'artTagData', 'groupData', 'trackData'); - WD.albumData.forEach(album => assignWikiData(album.trackGroups, 'trackData')); + case documentModes.headerAndEntries: + case documentModes.onePerFile: { + return dataStep.save(thingLists); + } - assignWikiData(WD.trackData, 'albumData', 'artistData', 'artTagData', 'flashData', 'trackData'); - assignWikiData(WD.artistData, 'albumData', 'artistData', 'flashData', 'trackData'); - assignWikiData(WD.groupData, 'albumData', 'groupCategoryData'); - assignWikiData(WD.groupCategoryData, 'groupData'); - assignWikiData(WD.flashData, 'artistData', 'flashActData', 'trackData'); - assignWikiData(WD.flashActData, 'flashData'); - assignWikiData(WD.artTagData, 'albumData', 'trackData'); - assignWikiData(WD.homepageLayout.rows, 'albumData', 'groupData'); + default: + throw new Error(`Invalid documentMode: ${documentMode.toString()}`); + } } -export function sortWikiDataArrays(wikiData) { - Object.assign(wikiData, { - albumData: sortByDate(wikiData.albumData.slice()), - trackData: sortByDate(wikiData.trackData.slice()) +// Flattens a list of *lists* of things for each data step (each list +// corresponding to one YAML file) into the final wikiData object, +// routing thing lists into each step's save() function. +export function saveThingsFromDataSteps(thingLists, dataSteps) { + const aggregate = + openAggregate({ + message: `Errors finalizing things from data files`, + translucent: true, }); - // Re-link data arrays, so that every object has the new, sorted versions. - // Note that the sorting step deliberately creates new arrays (mutating - // slices instead of the original arrays) - this is so that the object - // caching system understands that it's working with a new ordering. - // We still need to actually provide those updated arrays over again! - linkWikiDataArrays(wikiData); -} + const wikiData = {}; -// Warn about directories which are reused across more than one of the same type -// of Thing. Directories are the unique identifier for most data objects across -// the wiki, so we have to make sure they aren't duplicated! This also -// altogether filters out instances of things with duplicate directories (so if -// two tracks share the directory "megalovania", they'll both be skipped for the -// build, for example). -export function filterDuplicateDirectories(wikiData) { - const deduplicateSpec = [ - 'albumData', - 'artTagData', - 'flashData', - 'groupData', - 'newsData', - 'trackData', - ]; - - const aggregate = openAggregate({message: `Duplicate directories found`}); - for (const thingDataProp of deduplicateSpec) { - const thingData = wikiData[thingDataProp]; - aggregate.nest({message: `Duplicate directories found in ${color.green('wikiData.' + thingDataProp)}`}, ({ call }) => { - const directoryPlaces = Object.create(null); - const duplicateDirectories = []; - for (const thing of thingData) { - const { directory } = thing; - if (directory in directoryPlaces) { - directoryPlaces[directory].push(thing); - duplicateDirectories.push(directory); - } else { - directoryPlaces[directory] = [thing]; - } + stitchArrays({ + dataStep: dataSteps, + thingLists: thingLists, + }).map(({dataStep, thingLists}) => { + try { + return saveThingsFromDataStep(thingLists, dataStep); + } catch (caughtError) { + const error = new Error( + `Error finalizing things for data step: ${colors.bright(dataStep.title)}`, + {cause: caughtError}); + + error[Symbol.for('hsmusic.aggregate.translucent')] = true; + + aggregate.push(error); + + return null; + } + }) + .filter(Boolean) + .forEach(saveResult => { + for (const [saveKey, saveValue] of Object.entries(saveResult)) { + if (Object.hasOwn(wikiData, saveKey)) { + if (Array.isArray(wikiData[saveKey])) { + if (Array.isArray(saveValue)) { + wikiData[saveKey].push(...saveValue); + } else { + throw new Error(`${saveKey} already present, expected array of items to push`); } - if (!duplicateDirectories.length) return; - duplicateDirectories.sort((a, b) => { - const aL = a.toLowerCase(); - const bL = b.toLowerCase(); - return aL < bL ? -1 : aL > bL ? 1 : 0; - }); - for (const directory of duplicateDirectories) { - const places = directoryPlaces[directory]; - call(() => { - throw new Error(`Duplicate directory ${color.green(directory)}:\n` + - places.map(thing => ` - ` + inspect(thing)).join('\n')); - }); + } else { + if (Array.isArray(saveValue)) { + throw new Error(`${saveKey} already present and not an array, refusing to overwrite`); + } else { + throw new Error(`${saveKey} already present, refusing to overwrite`); } - const allDuplicatedThings = Object.values(directoryPlaces).filter(arr => arr.length > 1).flat(); - const filteredThings = thingData.filter(thing => !allDuplicatedThings.includes(thing)); - wikiData[thingDataProp] = filteredThings; - }); - } + } + } else { + wikiData[saveKey] = saveValue; + } + } + }); - // TODO: This code closes the aggregate but it generally gets closed again - // by the caller. This works but it might be weird to assume closing an - // aggregate twice is okay, maybe there's a better solution? Expose a new - // function on aggregates for checking if it *would* error? - // (i.e: errors.length > 0) - try { - aggregate.close(); - } catch (error) { - // Duplicate entries were found and filtered out, resulting in altered - // wikiData arrays. These must be re-linked so objects receive the new - // data. - linkWikiDataArrays(wikiData); - } - return aggregate; + return {aggregate, result: wikiData}; } -// Warn about references across data which don't match anything. This involves -// using the find() functions on all references, setting it to 'error' mode, and -// collecting everything in a structured logged (which gets logged if there are -// any errors). At the same time, we remove errored references from the thing's -// data array. -export function filterReferenceErrors(wikiData) { - const referenceSpec = [ - ['wikiInfo', { - divideTrackListsByGroupsByRef: 'group', - }], - - ['albumData', { - artistContribsByRef: '_contrib', - coverArtistContribsByRef: '_contrib', - trackCoverArtistContribsByRef: '_contrib', - wallpaperArtistContribsByRef: '_contrib', - bannerArtistContribsByRef: '_contrib', - groupsByRef: 'group', - artTagsByRef: 'artTag', - }], - - ['trackData', { - artistContribsByRef: '_contrib', - contributorContribsByRef: '_contrib', - coverArtistContribsByRef: '_contrib', - referencedTracksByRef: 'track', - artTagsByRef: 'artTag', - originalReleaseTrackByRef: 'track', - }], - - ['groupCategoryData', { - groupsByRef: 'group', - }], - - ['homepageLayout.rows', { - sourceGroupsByRef: 'group', - sourceAlbumsByRef: 'album', - }], - - ['flashData', { - contributorContribsByRef: '_contrib', - featuredTracksByRef: 'track', - }], - - ['flashActData', { - flashesByRef: 'flash', - }], - ]; - - function getNestedProp(obj, key) { - const recursive = (o, k) => (k.length === 1 - ? o[k[0]] - : recursive(o[k[0]], k.slice(1))); - const keys = key.split(/(?<=(?<!\\)(?:\\\\)*)\./); - return recursive(obj, keys); - } +export async function loadAndProcessDataDocuments(dataSteps, {dataPath}) { + const aggregate = + openAggregate({ + message: `Errors processing data files`, + }); - const aggregate = openAggregate({message: `Errors validating between-thing references in data`}); - const boundFind = bindFind(wikiData, {mode: 'error'}); - for (const [ thingDataProp, propSpec ] of referenceSpec) { - const thingData = getNestedProp(wikiData, thingDataProp); - aggregate.nest({message: `Reference errors in ${color.green('wikiData.' + thingDataProp)}`}, ({ nest }) => { - const things = Array.isArray(thingData) ? thingData : [thingData]; - for (const thing of things) { - nest({message: `Reference errors in ${inspect(thing)}`}, ({ filter }) => { - for (const [ property, findFnKey ] of Object.entries(propSpec)) { - if (!thing[property]) continue; - if (findFnKey === '_contrib') { - thing[property] = filter(thing[property], - decorateErrorWithIndex(({ who }) => { - const alias = find.artist(who, wikiData.artistAliasData, {mode: 'quiet'}); - if (alias) { - const original = find.artist(alias.aliasedArtistRef, wikiData.artistData, {mode: 'quiet'}); - throw new Error(`Reference ${color.red(who)} is to an alias, should be ${color.green(original.name)}`); - } - return boundFind.artist(who); - }), - {message: `Reference errors in contributions ${color.green(property)} (${color.green('find.artist')})`}); - continue; - } - const findFn = boundFind[findFnKey]; - const value = thing[property]; - if (Array.isArray(value)) { - thing[property] = filter(value, decorateErrorWithIndex(findFn), - {message: `Reference errors in property ${color.green(property)} (${color.green('find.' + findFnKey)})`}); - } else { - nest({message: `Reference error in property ${color.green(property)} (${color.green('find.' + findFnKey)})`}, ({ call }) => { - try { - call(findFn, value); - } catch (error) { - thing[property] = null; - throw error; - } - }); - } - } - }); - } - }); + const {documentLists, fileLists} = + aggregate.receive( + await loadYAMLDocumentsFromDataSteps(dataSteps, {dataPath})); + + const thingLists = + aggregate.receive( + await processThingsFromDataSteps(documentLists, fileLists, dataSteps, {dataPath})); + + const wikiData = + aggregate.receive( + saveThingsFromDataSteps(thingLists, dataSteps)); + + return {aggregate, result: wikiData}; +} + +// Data linking! Basically, provide (portions of) wikiData to the Things which +// require it - they'll expose dynamically computed properties as a result (many +// of which are required for page HTML generation and other expected behavior). +export function linkWikiDataArrays(wikiData, {bindFind, bindReverse}) { + const linkWikiDataSpec = new Map([ + // entries must be present here even without any properties to explicitly + // link if the 'find' or 'reverse' properties will be implicitly linked + + ['albumData', [ + 'artworkData', + 'wikiInfo', + ]], + + ['artTagData', [/* reverse */]], + + ['artistData', [/* find, reverse */]], + + ['artworkData', ['artworkData']], + + ['flashData', [ + 'wikiInfo', + ]], + + ['flashActData', [/* find, reverse */]], + + ['flashSideData', [/* find */]], + + ['groupData', [/* find, reverse */]], + + ['groupCategoryData', [/* find */]], + + ['homepageLayout.sections.rows', [/* find */]], + + ['trackData', [ + 'artworkData', + 'trackData', + 'wikiInfo', + ]], + + ['trackSectionData', [/* reverse */]], + + ['wikiInfo', [/* find */]], + ]); + + const constructorHasFindMap = new Map(); + const constructorHasReverseMap = new Map(); + + const boundFind = bindFind(wikiData); + const boundReverse = bindReverse(wikiData); + + for (const [thingDataProp, keys] of linkWikiDataSpec.entries()) { + const thingData = getNestedProp(wikiData, thingDataProp); + const things = + (Array.isArray(thingData) + ? thingData.flat(Infinity) + : [thingData]); + + for (const thing of things) { + if (thing === undefined) continue; + + let hasFind; + if (constructorHasFindMap.has(thing.constructor)) { + hasFind = constructorHasFindMap.get(thing.constructor); + } else { + hasFind = 'find' in thing; + constructorHasFindMap.set(thing.constructor, hasFind); + } + + if (hasFind) { + thing.find = boundFind; + } + + let hasReverse; + if (constructorHasReverseMap.has(thing.constructor)) { + hasReverse = constructorHasReverseMap.get(thing.constructor); + } else { + hasReverse = 'reverse' in thing; + constructorHasReverseMap.set(thing.constructor, hasReverse); + } + + if (hasReverse) { + thing.reverse = boundReverse; + } + + for (const key of keys) { + if (!(key in wikiData)) continue; + + thing[key] = wikiData[key]; + } } + } +} - return aggregate; +export function sortWikiDataArrays(dataSteps, wikiData, {bindFind, bindReverse}) { + for (const [key, value] of Object.entries(wikiData)) { + if (!Array.isArray(value)) continue; + wikiData[key] = value.slice(); + } + + for (const step of dataSteps) { + if (!step.sort) continue; + step.sort(wikiData); + } + + // Re-link data arrays, so that every object has the new, sorted versions. + // Note that the sorting step deliberately creates new arrays (mutating + // slices instead of the original arrays) - this is so that the object + // caching system understands that it's working with a new ordering. + // We still need to actually provide those updated arrays over again! + linkWikiDataArrays(wikiData, {bindFind, bindReverse}); } // Utility function for loading all wiki data from the provided YAML data @@ -1297,47 +1599,256 @@ export function filterReferenceErrors(wikiData) { // where reporting info about data loading isn't as relevant as during the // main wiki build process. export async function quickLoadAllFromYAML(dataPath, { - showAggregate: customShowAggregate = showAggregate, -} = {}) { - const showAggregate = customShowAggregate; + find, + bindFind, + bindReverse, + getAllFindSpecs, - let wikiData; + showAggregate: customShowAggregate = showAggregate, +}) { + const showAggregate = customShowAggregate; - { - const { aggregate, result } = await loadAndProcessDataDocuments({ - dataPath, - }); + const dataSteps = getAllDataSteps(); - wikiData = result; + let wikiData; - try { - aggregate.close(); - logInfo`Loaded data without errors. (complete data)`; - } catch (error) { - showAggregate(error); - logWarn`Loaded data with errors. (partial data)`; - } - } + { + const {aggregate, result} = await loadAndProcessDataDocuments(dataSteps, {dataPath}); - linkWikiDataArrays(wikiData); + wikiData = result; try { - filterDuplicateDirectories(wikiData).close(); - logInfo`No duplicate directories found. (complete data)`; + aggregate.close(); + logInfo`Loaded data without errors. (complete data)`; } catch (error) { - showAggregate(error); - logWarn`Duplicate directories found. (partial data)`; + showAggregate(error); + logWarn`Loaded data with errors. (partial data)`; } + } + + linkWikiDataArrays(wikiData, {bindFind, bindReverse}); + + try { + reportDirectoryErrors(wikiData, {getAllFindSpecs}); + logInfo`No duplicate directories found. (complete data)`; + } catch (error) { + showAggregate(error); + logWarn`Duplicate directories found. (partial data)`; + } + + try { + filterReferenceErrors(wikiData, {find, bindFind}).close(); + logInfo`No reference errors found. (complete data)`; + } catch (error) { + showAggregate(error); + logWarn`Reference errors found. (partial data)`; + } + + try { + reportContentTextErrors(wikiData, {bindFind}); + logInfo`No content text errors found.`; + } catch (error) { + showAggregate(error); + logWarn`Content text errors found.`; + } + + sortWikiDataArrays(dataSteps, wikiData, {bindFind, bindReverse}); + + return wikiData; +} - try { - filterReferenceErrors(wikiData).close(); - logInfo`No reference errors found. (complete data)`; - } catch (error) { - showAggregate(error); - logWarn`Duplicate directories found. (partial data)`; +export function cruddilyGetAllThings(wikiData) { + const allThings = []; + + for (const v of Object.values(wikiData)) { + if (Array.isArray(v)) { + allThings.push(...v); + } else { + allThings.push(v); + } + } + + return allThings; +} + +export function getThingLayoutForFilename(filename, wikiData) { + const things = + cruddilyGetAllThings(wikiData) + .filter(thing => + thing[Thing.yamlSourceFilename] === filename); + + if (empty(things)) { + return null; + } + + const allDocumentModes = + unique(things.map(thing => + thing[Thing.yamlSourceDocumentPlacement][0])); + + if (allDocumentModes.length > 1) { + throw new Error(`More than one document mode for documents from ${filename}`); + } + + const documentMode = allDocumentModes[0]; + + switch (documentMode) { + case documentModes.allInOne: { + return { + documentMode, + things: + things.sort((a, b) => + a[Thing.yamlSourceDocumentPlacement][1] - + b[Thing.yamlSourceDocumentPlacement][1]), + }; + } + + case documentModes.oneDocumentTotal: + case documentModes.onePerFile: { + if (things.length > 1) { + throw new Error(`More than one document for ${filename}`); + } + + return { + documentMode, + thing: things[0], + }; + } + + case documentModes.headerAndEntries: { + const headerThings = + things.filter(thing => + thing[Thing.yamlSourceDocumentPlacement][1] === 'header'); + + if (headerThings.length > 1) { + throw new Error(`More than one header document for ${filename}`); + } + + return { + documentMode, + headerThing: headerThings[0] ?? null, + entryThings: + things + .filter(thing => + thing[Thing.yamlSourceDocumentPlacement][1] === 'entry') + .sort((a, b) => + a[Thing.yamlSourceDocumentPlacement][2] - + b[Thing.yamlSourceDocumentPlacement][2]), + }; + } + + default: { + return {documentMode}; + } + } +} + +export function flattenThingLayoutToDocumentOrder(layout) { + switch (layout.documentMode) { + case documentModes.oneDocumentTotal: + case documentModes.onePerFile: { + if (layout.thing) { + return [0]; + } else { + return []; + } + } + + case documentModes.allInOne: { + const indices = + layout.things + .map(thing => thing[Thing.yamlSourceDocumentPlacement][1]); + + return indices; + } + + case documentModes.headerAndEntries: { + const entryIndices = + layout.entryThings + .map(thing => thing[Thing.yamlSourceDocumentPlacement][2]) + .map(index => index + 1); + + if (layout.headerThing) { + return [0, ...entryIndices]; + } else { + return entryIndices; + } + } + + default: { + throw new Error(`Unknown document mode`); } + } +} + +export function* splitDocumentsInYAMLSourceText(sourceText) { + // Not multiline! + const dividerRegex = /(?:\r\n|\n|^)-{3,}(?:\r\n|\n|$)/g; + + let previousDivider = ''; + + while (true) { + const {lastIndex} = dividerRegex; + const match = dividerRegex.exec(sourceText); + if (match) { + const nextDivider = match[0]; + + yield { + previousDivider, + nextDivider, + text: sourceText.slice(lastIndex, match.index), + }; + + previousDivider = nextDivider; + } else { + const nextDivider = ''; + const lineBreak = previousDivider.match(/\r?\n/)?.[0] ?? ''; + + yield { + previousDivider, + nextDivider, + text: sourceText.slice(lastIndex).replace(/(?<!\n)$/, lineBreak), + }; + + return; + } + } +} + +export function recombineDocumentsIntoYAMLSourceText(documents) { + const dividers = + unique( + documents + .flatMap(d => [d.previousDivider, d.nextDivider]) + .filter(Boolean)); + + const divider = dividers[0]; + + if (dividers.length > 1) { + // TODO: Accommodate mixed dividers as best as we can lol + logWarn`Found multiple dividers in this file, using only ${divider}`; + } + + let sourceText = ''; + + for (const document of documents) { + if (sourceText) { + sourceText += divider; + } + + sourceText += document.text; + } + + return sourceText; +} + +export function reorderDocumentsInYAMLSourceText(sourceText, order) { + const sourceDocuments = + Array.from(splitDocumentsInYAMLSourceText(sourceText)); - sortWikiDataArrays(wikiData); + const sortedDocuments = + Array.from( + order, + sourceIndex => sourceDocuments[sourceIndex]); - return wikiData; + return recombineDocumentsIntoYAMLSourceText(sortedDocuments); } |