diff options
Diffstat (limited to 'src/data/yaml.js')
| -rw-r--r-- | src/data/yaml.js | 1737 |
1 files changed, 1329 insertions, 408 deletions
diff --git a/src/data/yaml.js b/src/data/yaml.js index f84f0378..9a0295b8 100644 --- a/src/data/yaml.js +++ b/src/data/yaml.js @@ -8,27 +8,38 @@ import {inspect as nodeInspect} from 'node:util'; import yaml from 'js-yaml'; import {colors, ENABLE_COLOR, logInfo, logWarn} from '#cli'; +import {parseContentNodes, splitContentNodesAround} from '#replacer'; import {sortByName} from '#sort'; -import {atOffset, empty, filterProperties, typeAppearance, withEntries} - from '#sugar'; import Thing from '#thing'; import thingConstructors from '#things'; +import {matchContentEntries, multipleLyricsDetectionRegex} from '#wiki-data'; import { - filterReferenceErrors, - reportContentTextErrors, - reportDuplicateDirectories, -} from '#data-checks'; - -import { + aggregateThrows, annotateErrorWithFile, decorateErrorWithIndex, decorateErrorWithAnnotation, openAggregate, showAggregate, - withAggregate, } from '#aggregate'; +import { + filterReferenceErrors, + reportContentTextErrors, + reportDirectoryErrors, +} from '#data-checks'; + +import { + atOffset, + empty, + filterProperties, + getNestedProp, + stitchArrays, + typeAppearance, + unique, + withEntries, +} from '#sugar'; + function inspect(value, opts = {}) { return nodeInspect(value, {colors: ENABLE_COLOR, ...opts}); } @@ -80,6 +91,10 @@ function makeProcessDocument(thingConstructor, { // A or B. // invalidFieldCombinations = [], + + // Bouncing function used to process subdocuments: this is a function which + // in turn calls the appropriate *result of* makeProcessDocument. + processDocument: bouncer, }) { if (!thingConstructor) { throw new Error(`Missing Thing class`); @@ -89,6 +104,10 @@ function makeProcessDocument(thingConstructor, { throw new Error(`Expected fields to be provided`); } + if (!bouncer) { + throw new Error(`Missing processDocument bouncer`); + } + const knownFields = Object.keys(fieldSpecs); const ignoredFields = @@ -136,9 +155,12 @@ function makeProcessDocument(thingConstructor, { : `document`); const aggregate = openAggregate({ + ...aggregateThrows(ProcessDocumentError), message: `Errors processing ${constructorPart}` + namePart, }); + const thing = Reflect.construct(thingConstructor, []); + const documentEntries = Object.entries(document) .filter(([field]) => !ignoredFields.includes(field)); @@ -186,13 +208,50 @@ function makeProcessDocument(thingConstructor, { const fieldValues = {}; + const subdocSymbol = Symbol('subdoc'); + const subdocLayouts = {}; + + const isSubdocToken = value => + typeof value === 'object' && + value !== null && + Object.hasOwn(value, subdocSymbol); + + const transformUtilities = { + ...thingConstructors, + + subdoc(documentType, data, { + bindInto = null, + provide = null, + } = {}) { + if (!documentType) + throw new Error(`Expected document type, got ${typeAppearance(documentType)}`); + if (!data) + throw new Error(`Expected data, got ${typeAppearance(data)}`); + if (typeof data !== 'object' || data === null) + throw new Error(`Expected data to be an object, got ${typeAppearance(data)}`); + if (typeof bindInto !== 'string' && bindInto !== null) + throw new Error(`Expected bindInto to be a string, got ${typeAppearance(bindInto)}`); + if (typeof provide !== 'object' && provide !== null) + throw new Error(`Expected provide to be an object, got ${typeAppearance(provide)}`); + + return { + [subdocSymbol]: { + documentType, + data, + bindInto, + provide, + }, + }; + }, + }; + for (const [field, documentValue] of documentEntries) { if (skippedFields.has(field)) continue; // This variable would like to certify itself as "not into capitalism". let propertyValue = (fieldSpecs[field].transform - ? fieldSpecs[field].transform(documentValue) + ? fieldSpecs[field].transform(documentValue, transformUtilities) : documentValue); // Completely blank items in a YAML list are read as null. @@ -215,10 +274,99 @@ function makeProcessDocument(thingConstructor, { } } + if (isSubdocToken(propertyValue)) { + subdocLayouts[field] = propertyValue[subdocSymbol]; + continue; + } + + if (Array.isArray(propertyValue) && propertyValue.every(isSubdocToken)) { + subdocLayouts[field] = + propertyValue + .map(token => token[subdocSymbol]); + continue; + } + fieldValues[field] = propertyValue; } - const thing = Reflect.construct(thingConstructor, []); + const subdocErrors = []; + + const followSubdocSetup = setup => { + let error = null; + + let subthing; + try { + const result = bouncer(setup.data, setup.documentType); + subthing = result.thing; + result.aggregate.close(); + } catch (caughtError) { + error = caughtError; + } + + if (subthing) { + if (setup.bindInto) { + subthing[setup.bindInto] = thing; + } + + if (setup.provide) { + Object.assign(subthing, setup.provide); + } + } + + return {error, subthing}; + }; + + for (const [field, layout] of Object.entries(subdocLayouts)) { + if (Array.isArray(layout)) { + const subthings = []; + let anySucceeded = false; + let anyFailed = false; + + for (const [index, setup] of layout.entries()) { + const {subthing, error} = followSubdocSetup(setup); + if (error) { + subdocErrors.push(new SubdocError( + {field, index}, + setup, + {cause: error})); + } + + if (subthing) { + subthings.push(subthing); + anySucceeded = true; + } else { + anyFailed = true; + } + } + + if (anySucceeded) { + fieldValues[field] = subthings; + } else if (anyFailed) { + skippedFields.add(field); + } + } else { + const setup = layout; + const {subthing, error} = followSubdocSetup(setup); + + if (error) { + subdocErrors.push(new SubdocError( + {field}, + setup, + {cause: error})); + } + + if (subthing) { + fieldValues[field] = subthing; + } else { + skippedFields.add(field); + } + } + } + + if (!empty(subdocErrors)) { + aggregate.push(new SubdocAggregateError( + subdocErrors, thingConstructor)); + } const fieldValueErrors = []; @@ -252,6 +400,8 @@ function makeProcessDocument(thingConstructor, { }); } +export class ProcessDocumentError extends AggregateError {} + export class UnknownFieldsError extends Error { constructor(fields) { super(`Unknown fields ignored: ${fields.map(field => colors.red(field)).join(', ')}`); @@ -339,12 +489,46 @@ export class SkippedFieldsSummaryError extends Error { : `${entries.length} fields`); super( - colors.bright(colors.yellow(`Altogether, skipped ${numFieldsText}:\n`)) + + colors.bright(colors.yellow(`Altogether, skipped ${numFieldsText}:`)) + '\n' + lines.join('\n') + '\n' + colors.bright(colors.yellow(`See above errors for details.`))); } } +export class SubdocError extends Error { + constructor({field, index = null}, setup, options) { + const fieldText = + (index === null + ? colors.green(`"${field}"`) + : colors.yellow(`#${index + 1}`) + ' in ' + + colors.green(`"${field}"`)); + + const constructorText = + setup.documentType.name; + + if (options.cause instanceof ProcessDocumentError) { + options.cause[Symbol.for('hsmusic.aggregate.translucent')] = true; + } + + super( + `Errors processing ${constructorText} for ${fieldText} field`, + options); + } +} + +export class SubdocAggregateError extends AggregateError { + [Symbol.for('hsmusic.aggregate.translucent')] = true; + + constructor(errors, thingConstructor) { + const constructorText = + colors.green(thingConstructor.name); + + super( + errors, + `Errors processing subdocuments for ${constructorText}`); + } +} + export function parseDate(date) { return new Date(date); } @@ -364,36 +548,56 @@ export function parseDuration(string) { } } -export function parseAdditionalFiles(array) { - if (!Array.isArray(array)) { - // Error will be caught when validating against whatever this value is - return array; - } - - return array.map((item) => ({ - title: item['Title'], - description: item['Description'] ?? null, - files: item['Files'], - })); -} - export const extractAccentRegex = /^(?<main>.*?)(?: \((?<accent>.*)\))?$/; export const extractPrefixAccentRegex = /^(?:\((?<accent>.*)\) )?(?<main>.*?)$/; -export function parseContributors(contributionStrings) { +// TODO: Should this fit better within actual YAML loading infrastructure?? +export function parseArrayEntries(entries, mapFn) { // If this isn't something we can parse, just return it as-is. // The Thing object's validators will handle the data error better // than we're able to here. - if (!Array.isArray(contributionStrings)) { - return contributionStrings; + if (!Array.isArray(entries)) { + return entries; } - return contributionStrings.map(item => { + // If the array is REALLY ACTUALLY empty (it's represented in YAML + // as literally an empty []), that's something we want to reflect. + if (empty(entries)) { + return entries; + } + + const nonNullEntries = + entries.filter(value => value !== null); + + // On the other hand, if the array only contains null, it's just + // a placeholder, so skip over the field like it's not actually + // been put there yet. + if (empty(nonNullEntries)) { + return null; + } + + return entries.map(mapFn); +} + +export function parseContributors(entries) { + return parseArrayEntries(entries, item => { if (typeof item === 'object' && item['Who']) - return {who: item['Who'], what: item['What'] ?? null}; + return { + artist: item['Who'], + annotation: item['What'] ?? null, + }; + + if (typeof item === 'object' && item['Artist']) + return { + artist: item['Artist'], + annotation: item['Annotation'] ?? null, + + countInContributionTotals: item['Count In Contribution Totals'] ?? null, + countInDurationTotals: item['Count In Duration Totals'] ?? null, + }; if (typeof item !== 'string') return item; @@ -401,29 +605,59 @@ export function parseContributors(contributionStrings) { if (!match) return item; return { - who: match.groups.main, - what: match.groups.accent ?? null, + artist: match.groups.main, + annotation: match.groups.accent ?? null, }; }); } -export function parseAdditionalNames(additionalNameStrings) { - if (!Array.isArray(additionalNameStrings)) { - return additionalNameStrings; - } +export function parseAdditionalFiles(entries, {subdoc, AdditionalFile}) { + return parseArrayEntries(entries, item => { + if (typeof item !== 'object') return item; + + return subdoc(AdditionalFile, item, {bindInto: 'thing'}); + }); +} - return additionalNameStrings.map(item => { - if (typeof item === 'object' && item['Name']) - return {name: item['Name'], annotation: item['Annotation'] ?? null}; +export function parseAdditionalNames(entries, {subdoc, AdditionalName}) { + return parseArrayEntries(entries, item => { + if (typeof item === 'object') { + return subdoc(AdditionalName, item, {bindInto: 'thing'}); + } if (typeof item !== 'string') return item; const match = item.match(extractAccentRegex); if (!match) return item; + const document = { + ['Name']: match.groups.main, + ['Annotation']: match.groups.accent ?? null, + }; + + return subdoc(AdditionalName, document, {bindInto: 'thing'}); + }); +} + +export function parseSerieses(entries, {subdoc, Series}) { + return parseArrayEntries(entries, item => { + if (typeof item !== 'object') return item; + + return subdoc(Series, item, {bindInto: 'group'}); + }); +} + +export function parseWallpaperParts(entries) { + return parseArrayEntries(entries, item => { + if (typeof item !== 'object') return item; + return { - name: match.groups.main, - annotation: match.groups.accent ?? null, + asset: + (item['Asset'] === 'none' + ? null + : item['Asset'] ?? null), + + style: item['Style'] ?? null, }; }); } @@ -451,6 +685,268 @@ export function parseDimensions(string) { return nums; } +export const contributionPresetYAMLSpec = [ + {from: 'Album', to: 'album', fields: [ + {from: 'Artists', to: 'artistContribs'}, + ]}, + + {from: 'Flash', to: 'flash', fields: [ + {from: 'Contributors', to: 'contributorContribs'}, + ]}, + + {from: 'Track', to: 'track', fields: [ + {from: 'Artists', to: 'artistContribs'}, + {from: 'Contributors', to: 'contributorContribs'}, + ]}, +]; + +export function parseContributionPresetContext(context) { + if (!Array.isArray(context)) { + return context; + } + + const [target, ...fields] = context; + + const targetEntry = + contributionPresetYAMLSpec + .find(({from}) => from === target); + + if (!targetEntry) { + return context; + } + + const properties = + fields.map(field => { + const fieldEntry = + targetEntry.fields + .find(({from}) => from === field); + + if (!fieldEntry) return field; + + return fieldEntry.to; + }); + + return [targetEntry.to, ...properties]; +} + +export function parseContributionPresets(list) { + if (!Array.isArray(list)) return list; + + return list.map(item => { + if (typeof item !== 'object') return item; + + return { + annotation: + item['Annotation'] ?? null, + + context: + parseContributionPresetContext( + item['Context'] ?? null), + + countInContributionTotals: + item['Count In Contribution Totals'] ?? null, + + countInDurationTotals: + item['Count In Duration Totals'] ?? null, + }; + }); +} + +export function parseAnnotatedReferences(entries, { + referenceField = 'References', + annotationField = 'Annotation', + referenceProperty = 'reference', + annotationProperty = 'annotation', +} = {}) { + return parseArrayEntries(entries, item => { + if (typeof item === 'object' && item[referenceField]) + return { + [referenceProperty]: item[referenceField], + [annotationProperty]: item[annotationField] ?? null, + }; + + if (typeof item !== 'string') return item; + + const match = item.match(extractAccentRegex); + if (!match) + return { + [referenceProperty]: item, + [annotationProperty]: null, + }; + + return { + [referenceProperty]: match.groups.main, + [annotationProperty]: match.groups.accent ?? null, + }; + }); +} + +export function parseArtwork({ + single = false, + thingProperty = null, + dimensionsFromThingProperty = null, + fileExtensionFromThingProperty = null, + dateFromThingProperty = null, + artistContribsFromThingProperty = null, + artistContribsArtistProperty = null, + artTagsFromThingProperty = null, + referencedArtworksFromThingProperty = null, +}) { + const provide = { + thingProperty, + dimensionsFromThingProperty, + fileExtensionFromThingProperty, + dateFromThingProperty, + artistContribsFromThingProperty, + artistContribsArtistProperty, + artTagsFromThingProperty, + referencedArtworksFromThingProperty, + }; + + const parseSingleEntry = (entry, {subdoc, Artwork}) => + subdoc(Artwork, entry, {bindInto: 'thing', provide}); + + const transform = (value, ...args) => + (Array.isArray(value) + ? value.map(entry => parseSingleEntry(entry, ...args)) + : single + ? parseSingleEntry(value, ...args) + : [parseSingleEntry(value, ...args)]); + + transform.provide = provide; + + return transform; +} + +export function parseContentEntriesFromSourceText(thingClass, sourceText, {subdoc}) { + function map(matchEntry) { + let artistText = null, artistReferences = null; + + const artistTextNodes = + Array.from( + splitContentNodesAround( + parseContentNodes(matchEntry.artistText), + /\|/g)); + + const separatorIndices = + artistTextNodes + .filter(node => node.type === 'separator') + .map(node => artistTextNodes.indexOf(node)); + + if (empty(separatorIndices)) { + if (artistTextNodes.length === 1 && artistTextNodes[0].type === 'text') { + artistReferences = matchEntry.artistText; + } else { + artistText = matchEntry.artistText; + } + } else { + const firstSeparatorIndex = + separatorIndices.at(0); + + const secondSeparatorIndex = + separatorIndices.at(1) ?? + artistTextNodes.length; + + artistReferences = + matchEntry.artistText.slice( + artistTextNodes.at(0).i, + artistTextNodes.at(firstSeparatorIndex - 1).iEnd); + + artistText = + matchEntry.artistText.slice( + artistTextNodes.at(firstSeparatorIndex).iEnd, + artistTextNodes.at(secondSeparatorIndex - 1).iEnd); + } + + if (artistReferences) { + artistReferences = + artistReferences + .split(',') + .map(ref => ref.trim()); + } + + return { + 'Artists': + artistReferences, + + 'Artist Text': + artistText, + + 'Annotation': + matchEntry.annotation, + + 'Date': + matchEntry.date, + + 'Second Date': + matchEntry.secondDate, + + 'Date Kind': + matchEntry.dateKind, + + 'Access Date': + matchEntry.accessDate, + + 'Access Kind': + matchEntry.accessKind, + + 'Body': + matchEntry.body, + }; + } + + const documents = + matchContentEntries(sourceText) + .map(matchEntry => + withEntries( + map(matchEntry), + entries => entries + .filter(([key, value]) => + value !== undefined && + value !== null))); + + const subdocs = + documents.map(document => + subdoc(thingClass, document, {bindInto: 'thing'})); + + return subdocs; +} + +export function parseContentEntries(thingClass, value, {subdoc}) { + if (typeof value === 'string') { + return parseContentEntriesFromSourceText(thingClass, value, {subdoc}); + } else if (Array.isArray(value)) { + return value.map(doc => subdoc(thingClass, doc, {bindInto: 'thing'})); + } else { + return value; + } +} + +export function parseCommentary(value, {subdoc, CommentaryEntry}) { + return parseContentEntries(CommentaryEntry, value, {subdoc}); +} + +export function parseCreditingSources(value, {subdoc, CreditingSourcesEntry}) { + return parseContentEntries(CreditingSourcesEntry, value, {subdoc}); +} + +export function parseReferencingSources(value, {subdoc, ReferencingSourcesEntry}) { + return parseContentEntries(ReferencingSourcesEntry, value, {subdoc}); +} + +export function parseLyrics(value, {subdoc, LyricsEntry}) { + if ( + typeof value === 'string' && + !multipleLyricsDetectionRegex.test(value) + ) { + const document = {'Body': value}; + + return [subdoc(LyricsEntry, document, {bindInto: 'thing'})]; + } + + return parseContentEntries(LyricsEntry, value, {subdoc}); +} + // documentModes: Symbols indicating sets of behavior for loading and processing // data files. export const documentModes = { @@ -523,13 +1019,26 @@ export const documentModes = { // them to each other, setting additional properties, etc). Input argument // format depends on documentMode. // -export const getDataSteps = () => { +export function getAllDataSteps() { + try { + thingConstructors; + } catch { + throw new Error(`Thing constructors aren't ready yet, can't get all data steps`); + } + const steps = []; + const seenLoadingFns = new Set(); + for (const thingConstructor of Object.values(thingConstructors)) { const getSpecFn = thingConstructor[Thing.getYamlLoadingSpec]; if (!getSpecFn) continue; + // Subclasses can expose literally the same static properties + // by inheritence. We don't want to double-count those! + if (seenLoadingFns.has(getSpecFn)) continue; + seenLoadingFns.add(getSpecFn); + steps.push(getSpecFn({ documentModes, thingConstructors, @@ -539,459 +1048,671 @@ export const getDataSteps = () => { sortByName(steps, {getName: step => step.title}); return steps; -}; +} -export async function loadAndProcessDataDocuments({dataPath}) { - const processDataAggregate = openAggregate({ - message: `Errors processing data files`, - }); - const wikiDataResult = {}; - - function decorateErrorWithFile(fn) { - return decorateErrorWithAnnotation(fn, - (caughtError, firstArg) => - annotateErrorWithFile( - caughtError, - path.relative( - dataPath, - (typeof firstArg === 'object' - ? firstArg.file - : firstArg)))); - } +export async function getFilesFromDataStep(dataStep, {dataPath}) { + const {documentMode} = dataStep; - function asyncDecorateErrorWithFile(fn) { - return decorateErrorWithFile(fn).async; - } + switch (documentMode) { + case documentModes.allInOne: + case documentModes.oneDocumentTotal: { + if (!dataStep.file) { + throw new Error(`Expected 'file' property for ${documentMode.toString()}`); + } - for (const dataStep of getDataSteps()) { - await processDataAggregate.nestAsync( - { - message: `Errors during data step: ${colors.bright(dataStep.title)}`, - translucent: true, - }, - async ({call, callAsync, map, mapAsync, push}) => { - const {documentMode} = dataStep; + const localFile = + (typeof dataStep.file === 'function' + ? await dataStep.file(dataPath) + : dataStep.file); + + const fileUnderDataPath = + path.join(dataPath, localFile); + + const statResult = + await stat(fileUnderDataPath).then( + () => true, + error => { + if (error.code === 'ENOENT') { + return false; + } else { + throw error; + } + }); - if (!Object.values(documentModes).includes(documentMode)) { - throw new Error(`Invalid documentMode: ${documentMode.toString()}`); - } + if (statResult) { + return [fileUnderDataPath]; + } else { + return []; + } + } - // Hear me out, it's been like 1200 years since I wrote the rest of - // this beautifully error-containing code and I don't know how to - // integrate this nicely. So I'm just returning the result and the - // error that should be thrown. Yes, we're back in callback hell, - // just without the callbacks. Thank you. - const filterBlankDocuments = documents => { - const aggregate = openAggregate({ - message: `Found blank documents - check for extra '${colors.cyan(`---`)}'`, - }); + case documentModes.headerAndEntries: + case documentModes.onePerFile: { + if (!dataStep.files) { + throw new Error(`Expected 'files' property for ${documentMode.toString()}`); + } - const filteredDocuments = - documents - .filter(doc => doc !== null); - - if (filteredDocuments.length !== documents.length) { - const blankIndexRangeInfo = - documents - .map((doc, index) => [doc, index]) - .filter(([doc]) => doc === null) - .map(([doc, index]) => index) - .reduce((accumulator, index) => { - if (accumulator.length === 0) { - return [[index, index]]; - } - const current = accumulator.at(-1); - const rest = accumulator.slice(0, -1); - if (current[1] === index - 1) { - return rest.concat([[current[0], index]]); - } else { - return accumulator.concat([[index, index]]); - } - }, []) - .map(([start, end]) => ({ - start, - end, - count: end - start + 1, - previous: atOffset(documents, start, -1), - next: atOffset(documents, end, +1), - })); - - for (const {start, end, count, previous, next} of blankIndexRangeInfo) { - const parts = []; - - if (count === 1) { - const range = `#${start + 1}`; - parts.push(`${count} document (${colors.yellow(range)}), `); - } else { - const range = `#${start + 1}-${end + 1}`; - parts.push(`${count} documents (${colors.yellow(range)}), `); - } - - if (previous === null) { - parts.push(`at start of file`); - } else if (next === null) { - parts.push(`at end of file`); - } else { - const previousDescription = Object.entries(previous).at(0).join(': '); - const nextDescription = Object.entries(next).at(0).join(': '); - parts.push(`between "${colors.cyan(previousDescription)}" and "${colors.cyan(nextDescription)}"`); - } - - aggregate.push(new Error(parts.join(''))); - } - } + const localFiles = + (typeof dataStep.files === 'function' + ? await dataStep.files(dataPath).then( + files => files, + error => { + if (error.code === 'ENOENT') { + return []; + } else { + throw error; + } + }) + : dataStep.files); - return {documents: filteredDocuments, aggregate}; - }; + const filesUnderDataPath = + localFiles + .map(file => path.join(dataPath, file)); - const processDocument = (document, thingClassOrFn) => { - const thingClass = - (thingClassOrFn.prototype instanceof Thing - ? thingClassOrFn - : thingClassOrFn(document)); + return filesUnderDataPath; + } - if (typeof thingClass !== 'function') { - throw new Error(`Expected a thing class, got ${typeAppearance(thingClass)}`); - } + default: + throw new Error(`Unknown document mode ${documentMode.toString()}`); + } +} - if (!(thingClass.prototype instanceof Thing)) { - throw new Error(`Expected a thing class, got ${thingClass.name}`); - } +export async function loadYAMLDocumentsFromFile(file) { + let contents; + try { + contents = await readFile(file, 'utf-8'); + } catch (caughtError) { + throw new Error(`Failed to read data file`, {cause: caughtError}); + } - const spec = thingClass[Thing.yamlDocumentSpec]; + let documents; + try { + documents = yaml.loadAll(contents); + } catch (caughtError) { + throw new Error(`Failed to parse valid YAML`, {cause: caughtError}); + } + + const aggregate = openAggregate({ + message: `Found blank documents - check for extra '${colors.cyan(`---`)}'`, + }); - if (!spec) { - throw new Error(`Class "${thingClass.name}" doesn't specify Thing.yamlDocumentSpec`); + const filteredDocuments = + documents + .filter(doc => doc !== null); + + if (filteredDocuments.length !== documents.length) { + const blankIndexRangeInfo = + documents + .map((doc, index) => [doc, index]) + .filter(([doc]) => doc === null) + .map(([doc, index]) => index) + .reduce((accumulator, index) => { + if (accumulator.length === 0) { + return [[index, index]]; } + const current = accumulator.at(-1); + const rest = accumulator.slice(0, -1); + if (current[1] === index - 1) { + return rest.concat([[current[0], index]]); + } else { + return accumulator.concat([[index, index]]); + } + }, []) + .map(([start, end]) => ({ + start, + end, + count: end - start + 1, + previous: atOffset(documents, start, -1), + next: atOffset(documents, end, +1), + })); + + for (const {start, end, count, previous, next} of blankIndexRangeInfo) { + const parts = []; + + if (count === 1) { + const range = `#${start + 1}`; + parts.push(`${count} document (${colors.yellow(range)}), `); + } else { + const range = `#${start + 1}-${end + 1}`; + parts.push(`${count} documents (${colors.yellow(range)}), `); + } - // TODO: Making a function to only call it just like that is - // obviously pretty jank! It should be created once per data step. - const fn = makeProcessDocument(thingClass, spec); - return fn(document); - }; + if (previous === null) { + parts.push(`at start of file`); + } else if (next === null) { + parts.push(`at end of file`); + } else { + const previousDescription = Object.entries(previous).at(0).join(': '); + const nextDescription = Object.entries(next).at(0).join(': '); + parts.push(`between "${colors.cyan(previousDescription)}" and "${colors.cyan(nextDescription)}"`); + } - if ( - documentMode === documentModes.allInOne || - documentMode === documentModes.oneDocumentTotal - ) { - if (!dataStep.file) { - throw new Error(`Expected 'file' property for ${documentMode.toString()}`); - } + aggregate.push(new Error(parts.join(''))); + } + } - const file = path.join( - dataPath, - typeof dataStep.file === 'function' - ? await callAsync(dataStep.file, dataPath) - : dataStep.file); + return {result: filteredDocuments, aggregate}; +} - const statResult = await callAsync(() => - stat(file).then( - () => true, - error => { - if (error.code === 'ENOENT') { - return false; - } else { - throw error; - } - })); +// Mapping from dataStep (spec) object each to a sub-map, from thing class to +// processDocument function. +const processDocumentFns = new WeakMap(); - if (statResult === false) { - const saveResult = call(dataStep.save, { - [documentModes.allInOne]: [], - [documentModes.oneDocumentTotal]: {}, - }[documentMode]); +export function processThingsFromDataStep(documents, dataStep) { + let submap; + if (processDocumentFns.has(dataStep)) { + submap = processDocumentFns.get(dataStep); + } else { + submap = new Map(); + processDocumentFns.set(dataStep, submap); + } - if (!saveResult) return; + function processDocument(document, thingClassOrFn) { + const thingClass = + (thingClassOrFn.prototype instanceof Thing + ? thingClassOrFn + : thingClassOrFn(document)); + + let fn; + if (submap.has(thingClass)) { + fn = submap.get(thingClass); + } else { + if (typeof thingClass !== 'function') { + throw new Error(`Expected a thing class, got ${typeAppearance(thingClass)}`); + } - Object.assign(wikiDataResult, saveResult); + if (!(thingClass.prototype instanceof Thing)) { + throw new Error(`Expected a thing class, got ${thingClass.name}`); + } - return; - } + const spec = thingClass[Thing.yamlDocumentSpec]; + + if (!spec) { + throw new Error(`Class "${thingClass.name}" doesn't specify Thing.yamlDocumentSpec`); + } - const readResult = await callAsync(readFile, file, 'utf-8'); + fn = makeProcessDocument(thingClass, {...spec, processDocument}); + submap.set(thingClass, fn); + } - if (!readResult) { - return; - } + return fn(document); + } - let processResults; + const {documentMode} = dataStep; - switch (documentMode) { - case documentModes.oneDocumentTotal: { - const yamlResult = call(yaml.load, readResult); + switch (documentMode) { + case documentModes.allInOne: { + const result = []; + const aggregate = openAggregate({message: `Errors processing documents`}); - if (!yamlResult) { - processResults = null; - break; - } + documents.forEach( + decorateErrorWithIndex((document, index) => { + const {thing, aggregate: subAggregate} = + processDocument(document, dataStep.documentThing); - const {thing, aggregate} = - processDocument(yamlResult, dataStep.documentThing); + thing[Thing.yamlSourceDocument] = document; + thing[Thing.yamlSourceDocumentPlacement] = + [documentModes.allInOne, index]; - processResults = thing; + result.push(thing); + aggregate.call(subAggregate.close); + })); - call(() => aggregate.close()); + return { + aggregate, + result, + things: result, + }; + } - break; - } + case documentModes.oneDocumentTotal: { + if (documents.length > 1) + throw new Error(`Only expected one document to be present, got ${documents.length}`); - case documentModes.allInOne: { - const yamlResults = call(yaml.loadAll, readResult); + const {thing, aggregate} = + processDocument(documents[0], dataStep.documentThing); - if (!yamlResults) { - processResults = []; - return; - } + thing[Thing.yamlSourceDocument] = documents[0]; + thing[Thing.yamlSourceDocumentPlacement] = + [documentModes.oneDocumentTotal]; - const {documents, aggregate: filterAggregate} = - filterBlankDocuments(yamlResults); + return { + aggregate, + result: thing, + things: [thing], + }; + } - call(filterAggregate.close); + case documentModes.headerAndEntries: { + const headerDocument = documents[0]; + const entryDocuments = documents.slice(1).filter(Boolean); - processResults = []; + if (!headerDocument) + throw new Error(`Missing header document (empty file or erroneously starting with "---"?)`); - map(documents, decorateErrorWithIndex(document => { - const {thing, aggregate} = - processDocument(document, dataStep.documentThing); + const aggregate = openAggregate({message: `Errors processing documents`}); - processResults.push(thing); - aggregate.close(); - }), {message: `Errors processing documents`}); + const {thing: headerThing, aggregate: headerAggregate} = + processDocument(headerDocument, dataStep.headerDocumentThing); - break; - } - } + headerThing[Thing.yamlSourceDocument] = headerDocument; + headerThing[Thing.yamlSourceDocumentPlacement] = + [documentModes.headerAndEntries, 'header']; - if (!processResults) return; + try { + headerAggregate.close(); + } catch (caughtError) { + caughtError.message = `(${colors.yellow(`header`)}) ${caughtError.message}`; + aggregate.push(caughtError); + } - const saveResult = call(dataStep.save, processResults); + const entryThings = []; - if (!saveResult) return; + for (const [index, entryDocument] of entryDocuments.entries()) { + const {thing: entryThing, aggregate: entryAggregate} = + processDocument(entryDocument, dataStep.entryDocumentThing); - Object.assign(wikiDataResult, saveResult); + entryThing[Thing.yamlSourceDocument] = entryDocument; + entryThing[Thing.yamlSourceDocumentPlacement] = + [documentModes.headerAndEntries, 'entry', index]; - return; - } + entryThings.push(entryThing); - if (!dataStep.files) { - throw new Error(`Expected 'files' property for ${documentMode.toString()}`); + try { + entryAggregate.close(); + } catch (caughtError) { + caughtError.message = `(${colors.yellow(`entry #${index + 1}`)}) ${caughtError.message}`; + aggregate.push(caughtError); } + } - const filesFromDataStep = - (typeof dataStep.files === 'function' - ? await callAsync(() => - dataStep.files(dataPath).then( - files => files, - error => { - if (error.code === 'ENOENT') { - return []; - } else { - throw error; - } - })) - : dataStep.files); - - const filesUnderDataPath = - filesFromDataStep - .map(file => path.join(dataPath, file)); - - const yamlResults = []; - - await mapAsync(filesUnderDataPath, {message: `Errors loading data files`}, - asyncDecorateErrorWithFile(async file => { - let contents; - try { - contents = await readFile(file, 'utf-8'); - } catch (caughtError) { - throw new Error(`Failed to read data file`, {cause: caughtError}); - } + return { + aggregate, + result: { + header: headerThing, + entries: entryThings, + }, + things: [headerThing, ...entryThings], + }; + } - let documents; - try { - documents = yaml.loadAll(contents); - } catch (caughtError) { - throw new Error(`Failed to parse valid YAML`, {cause: caughtError}); - } + case documentModes.onePerFile: { + if (documents.length > 1) + throw new Error(`Only expected one document to be present per file, got ${documents.length} here`); + + if (empty(documents) || !documents[0]) + throw new Error(`Expected a document, this file is empty`); + + const {thing, aggregate} = + processDocument(documents[0], dataStep.documentThing); + + thing[Thing.yamlSourceDocument] = documents[0]; + thing[Thing.yamlSourceDocumentPlacement] = + [documentModes.onePerFile]; + + return { + aggregate, + result: thing, + things: [thing], + }; + } + + default: + throw new Error(`Unknown document mode ${documentMode.toString()}`); + } +} + +export function decorateErrorWithFileFromDataPath(fn, {dataPath}) { + return decorateErrorWithAnnotation(fn, + (caughtError, firstArg) => + annotateErrorWithFile( + caughtError, + path.relative( + dataPath, + (typeof firstArg === 'object' + ? firstArg.file + : firstArg)))); +} + +// Loads a list of files for each data step, and a list of documents +// for each file. +export async function loadYAMLDocumentsFromDataSteps(dataSteps, {dataPath}) { + const aggregate = + openAggregate({ + message: `Errors loading data files`, + translucent: true, + }); - const {documents: filteredDocuments, aggregate: filterAggregate} = - filterBlankDocuments(documents); - - try { - filterAggregate.close(); - } catch (caughtError) { - // Blank documents aren't a critical error, they're just something - // that should be noted - the (filtered) documents still get pushed. - const pathToFile = path.relative(dataPath, file); - annotateErrorWithFile(caughtError, pathToFile); - push(caughtError); + const fileLists = + await Promise.all( + dataSteps.map(dataStep => + getFilesFromDataStep(dataStep, {dataPath}))); + + const filePromises = + fileLists + .map(files => files + .map(file => + loadYAMLDocumentsFromFile(file).then( + ({result, aggregate}) => { + const close = + decorateErrorWithFileFromDataPath(aggregate.close, {dataPath}); + + aggregate.close = () => + close({file}); + + return {result, aggregate}; + }, + (error) => { + const aggregate = {}; + + annotateErrorWithFile(error, path.relative(dataPath, file)); + + aggregate.close = () => { + throw error; + }; + + return {result: [], aggregate}; + }))); + + const fileListPromises = + filePromises + .map(filePromises => Promise.all(filePromises)); + + const dataStepPromises = + stitchArrays({ + dataStep: dataSteps, + fileListPromise: fileListPromises, + }).map(async ({dataStep, fileListPromise}) => + openAggregate({ + message: `Errors loading data files for data step: ${colors.bright(dataStep.title)}`, + translucent: true, + }).contain(await fileListPromise)); + + const documentLists = + aggregate + .receive(await Promise.all(dataStepPromises)); + + return {aggregate, result: {documentLists, fileLists}}; +} + +// Loads a list of things from a list of documents for each file +// for each data step. Nesting! +export async function processThingsFromDataSteps(documentLists, fileLists, dataSteps, {dataPath}) { + const aggregate = + openAggregate({ + message: `Errors processing documents in data files`, + translucent: true, + }); + + const filePromises = + stitchArrays({ + dataStep: dataSteps, + files: fileLists, + documentLists: documentLists, + }).map(({dataStep, files, documentLists}) => + stitchArrays({ + file: files, + documents: documentLists, + }).map(({file, documents}) => { + const {result, aggregate, things} = + processThingsFromDataStep(documents, dataStep); + + for (const thing of things) { + thing[Thing.yamlSourceFilename] = + path.relative(dataPath, file) + .split(path.sep) + .join(path.posix.sep); } - yamlResults.push({file, documents: filteredDocuments}); + const close = decorateErrorWithFileFromDataPath(aggregate.close, {dataPath}); + aggregate.close = () => close({file}); + + return {result, aggregate}; })); - const processResults = []; - - switch (documentMode) { - case documentModes.headerAndEntries: - map(yamlResults, {message: `Errors processing documents in data files`, translucent: true}, - decorateErrorWithFile(({documents}) => { - const headerDocument = documents[0]; - const entryDocuments = documents.slice(1).filter(Boolean); - - if (!headerDocument) - throw new Error(`Missing header document (empty file or erroneously starting with "---"?)`); - - withAggregate({message: `Errors processing documents`}, ({push}) => { - const {thing: headerObject, aggregate: headerAggregate} = - processDocument(headerDocument, dataStep.headerDocumentThing); - - try { - headerAggregate.close(); - } catch (caughtError) { - caughtError.message = `(${colors.yellow(`header`)}) ${caughtError.message}`; - push(caughtError); - } - - const entryObjects = []; - - for (let index = 0; index < entryDocuments.length; index++) { - const entryDocument = entryDocuments[index]; - - const {thing: entryObject, aggregate: entryAggregate} = - processDocument(entryDocument, dataStep.entryDocumentThing); - - entryObjects.push(entryObject); - - try { - entryAggregate.close(); - } catch (caughtError) { - caughtError.message = `(${colors.yellow(`entry #${index + 1}`)}) ${caughtError.message}`; - push(caughtError); - } - } - - processResults.push({ - header: headerObject, - entries: entryObjects, - }); - }); - })); - break; - - case documentModes.onePerFile: - map(yamlResults, {message: `Errors processing data files as valid documents`}, - decorateErrorWithFile(({documents}) => { - if (documents.length > 1) - throw new Error(`Only expected one document to be present per file, got ${documents.length} here`); - - if (empty(documents) || !documents[0]) - throw new Error(`Expected a document, this file is empty`); - - const {thing, aggregate} = - processDocument(documents[0], dataStep.documentThing); - - processResults.push(thing); - aggregate.close(); - })); - break; - } + const fileListPromises = + filePromises + .map(filePromises => Promise.all(filePromises)); + + const dataStepPromises = + stitchArrays({ + dataStep: dataSteps, + fileListPromise: fileListPromises, + }).map(async ({dataStep, fileListPromise}) => + openAggregate({ + message: `Errors loading data files for data step: ${colors.bright(dataStep.title)}`, + translucent: true, + }).contain(await fileListPromise)); + + const thingLists = + aggregate + .receive(await Promise.all(dataStepPromises)); + + return {aggregate, result: thingLists}; +} - const saveResult = call(dataStep.save, processResults); +// Flattens a list of *lists* of things for a given data step (each list +// corresponding to one YAML file) into results to be saved on the final +// wikiData object, routing thing lists into the step's save() function. +export function saveThingsFromDataStep(thingLists, dataStep) { + const {documentMode} = dataStep; - if (!saveResult) return; + switch (documentMode) { + case documentModes.allInOne: { + const things = + (empty(thingLists) + ? [] + : thingLists[0]); - Object.assign(wikiDataResult, saveResult); - } - ); + return dataStep.save(things); + } + + case documentModes.oneDocumentTotal: { + const thing = + (empty(thingLists) + ? {} + : thingLists[0]); + + return dataStep.save(thing); + } + + case documentModes.headerAndEntries: + case documentModes.onePerFile: { + return dataStep.save(thingLists); + } + + default: + throw new Error(`Invalid documentMode: ${documentMode.toString()}`); } +} - return { - aggregate: processDataAggregate, - result: wikiDataResult, - }; +// Flattens a list of *lists* of things for each data step (each list +// corresponding to one YAML file) into the final wikiData object, +// routing thing lists into each step's save() function. +export function saveThingsFromDataSteps(thingLists, dataSteps) { + const aggregate = + openAggregate({ + message: `Errors finalizing things from data files`, + translucent: true, + }); + + const wikiData = {}; + + stitchArrays({ + dataStep: dataSteps, + thingLists: thingLists, + }).map(({dataStep, thingLists}) => { + try { + return saveThingsFromDataStep(thingLists, dataStep); + } catch (caughtError) { + const error = new Error( + `Error finalizing things for data step: ${colors.bright(dataStep.title)}`, + {cause: caughtError}); + + error[Symbol.for('hsmusic.aggregate.translucent')] = true; + + aggregate.push(error); + + return null; + } + }) + .filter(Boolean) + .forEach(saveResult => { + for (const [saveKey, saveValue] of Object.entries(saveResult)) { + if (Object.hasOwn(wikiData, saveKey)) { + if (Array.isArray(wikiData[saveKey])) { + if (Array.isArray(saveValue)) { + wikiData[saveKey].push(...saveValue); + } else { + throw new Error(`${saveKey} already present, expected array of items to push`); + } + } else { + if (Array.isArray(saveValue)) { + throw new Error(`${saveKey} already present and not an array, refusing to overwrite`); + } else { + throw new Error(`${saveKey} already present, refusing to overwrite`); + } + } + } else { + wikiData[saveKey] = saveValue; + } + } + }); + + return {aggregate, result: wikiData}; +} + +export async function loadAndProcessDataDocuments(dataSteps, {dataPath}) { + const aggregate = + openAggregate({ + message: `Errors processing data files`, + }); + + const {documentLists, fileLists} = + aggregate.receive( + await loadYAMLDocumentsFromDataSteps(dataSteps, {dataPath})); + + const thingLists = + aggregate.receive( + await processThingsFromDataSteps(documentLists, fileLists, dataSteps, {dataPath})); + + const wikiData = + aggregate.receive( + saveThingsFromDataSteps(thingLists, dataSteps)); + + return {aggregate, result: wikiData}; } // Data linking! Basically, provide (portions of) wikiData to the Things which // require it - they'll expose dynamically computed properties as a result (many // of which are required for page HTML generation and other expected behavior). -export function linkWikiDataArrays(wikiData) { +export function linkWikiDataArrays(wikiData, {bindFind, bindReverse}) { const linkWikiDataSpec = new Map([ - [wikiData.albumData, [ - 'artTagData', - 'artistData', - 'groupData', - ]], + // entries must be present here even without any properties to explicitly + // link if the 'find' or 'reverse' properties will be implicitly linked - [wikiData.artTagData, [ - 'albumData', - 'trackData', + ['albumData', [ + 'artworkData', + 'wikiInfo', ]], - [wikiData.artistData, [ - 'albumData', - 'artistData', - 'flashData', - 'trackData', - ]], + ['artTagData', [/* reverse */]], - [wikiData.flashData, [ - 'artistData', - 'flashActData', - 'trackData', - ]], + ['artistData', [/* find, reverse */]], - [wikiData.flashActData, [ - 'flashData', - ]], + ['artworkData', ['artworkData']], - [wikiData.groupData, [ - 'albumData', - 'groupCategoryData', - ]], + ['commentaryData', [/* find */]], - [wikiData.groupCategoryData, [ - 'groupData', - ]], + ['creditingSourceData', [/* find */]], - [wikiData.homepageLayout?.rows, [ - 'albumData', - 'groupData', + ['flashData', [ + 'wikiInfo', ]], - [wikiData.trackData, [ - 'albumData', - 'artTagData', - 'artistData', - 'flashData', + ['flashActData', [/* find, reverse */]], + + ['flashSideData', [/* find */]], + + ['groupData', [/* find, reverse */]], + + ['groupCategoryData', [/* find */]], + + ['homepageLayout.sections.rows', [/* find */]], + + ['lyricsData', [/* find */]], + + ['referencingSourceData', [/* find */]], + + ['seriesData', [/* find */]], + + ['trackData', [ + 'artworkData', 'trackData', + 'wikiInfo', ]], - [[wikiData.wikiInfo], [ - 'groupData', - ]], + ['trackSectionData', [/* reverse */]], + + ['wikiInfo', [/* find */]], ]); - for (const [things, keys] of linkWikiDataSpec.entries()) { - if (things === undefined) continue; + const constructorHasFindMap = new Map(); + const constructorHasReverseMap = new Map(); + + const boundFind = bindFind(wikiData); + const boundReverse = bindReverse(wikiData); + + for (const [thingDataProp, keys] of linkWikiDataSpec.entries()) { + const thingData = getNestedProp(wikiData, thingDataProp); + const things = + (Array.isArray(thingData) + ? thingData.flat(Infinity) + : [thingData]); + for (const thing of things) { if (thing === undefined) continue; + + let hasFind; + if (constructorHasFindMap.has(thing.constructor)) { + hasFind = constructorHasFindMap.get(thing.constructor); + } else { + hasFind = 'find' in thing; + constructorHasFindMap.set(thing.constructor, hasFind); + } + + if (hasFind) { + thing.find = boundFind; + } + + let hasReverse; + if (constructorHasReverseMap.has(thing.constructor)) { + hasReverse = constructorHasReverseMap.get(thing.constructor); + } else { + hasReverse = 'reverse' in thing; + constructorHasReverseMap.set(thing.constructor, hasReverse); + } + + if (hasReverse) { + thing.reverse = boundReverse; + } + for (const key of keys) { if (!(key in wikiData)) continue; + thing[key] = wikiData[key]; } } } } -export function sortWikiDataArrays(wikiData) { +export function sortWikiDataArrays(dataSteps, wikiData, {bindFind, bindReverse}) { for (const [key, value] of Object.entries(wikiData)) { if (!Array.isArray(value)) continue; wikiData[key] = value.slice(); } - const steps = getDataSteps(); - - for (const step of steps) { + for (const step of dataSteps) { if (!step.sort) continue; step.sort(wikiData); } @@ -1001,7 +1722,7 @@ export function sortWikiDataArrays(wikiData) { // slices instead of the original arrays) - this is so that the object // caching system understands that it's working with a new ordering. // We still need to actually provide those updated arrays over again! - linkWikiDataArrays(wikiData); + linkWikiDataArrays(wikiData, {bindFind, bindReverse}); } // Utility function for loading all wiki data from the provided YAML data @@ -1011,17 +1732,21 @@ export function sortWikiDataArrays(wikiData) { // where reporting info about data loading isn't as relevant as during the // main wiki build process. export async function quickLoadAllFromYAML(dataPath, { + find, bindFind, + bindReverse, getAllFindSpecs, showAggregate: customShowAggregate = showAggregate, }) { const showAggregate = customShowAggregate; + const dataSteps = getAllDataSteps(); + let wikiData; { - const {aggregate, result} = await loadAndProcessDataDocuments({dataPath}); + const {aggregate, result} = await loadAndProcessDataDocuments(dataSteps, {dataPath}); wikiData = result; @@ -1034,10 +1759,10 @@ export async function quickLoadAllFromYAML(dataPath, { } } - linkWikiDataArrays(wikiData); + linkWikiDataArrays(wikiData, {bindFind, bindReverse}); try { - reportDuplicateDirectories(wikiData, {getAllFindSpecs}); + reportDirectoryErrors(wikiData, {getAllFindSpecs}); logInfo`No duplicate directories found. (complete data)`; } catch (error) { showAggregate(error); @@ -1045,7 +1770,7 @@ export async function quickLoadAllFromYAML(dataPath, { } try { - filterReferenceErrors(wikiData, {bindFind}).close(); + filterReferenceErrors(wikiData, {find, bindFind}).close(); logInfo`No reference errors found. (complete data)`; } catch (error) { showAggregate(error); @@ -1060,7 +1785,203 @@ export async function quickLoadAllFromYAML(dataPath, { logWarn`Content text errors found.`; } - sortWikiDataArrays(wikiData); + sortWikiDataArrays(dataSteps, wikiData, {bindFind, bindReverse}); return wikiData; } + +export function cruddilyGetAllThings(wikiData) { + const allThings = []; + + for (const v of Object.values(wikiData)) { + if (Array.isArray(v)) { + allThings.push(...v); + } else { + allThings.push(v); + } + } + + return allThings; +} + +export function getThingLayoutForFilename(filename, wikiData) { + const things = + cruddilyGetAllThings(wikiData) + .filter(thing => + thing[Thing.yamlSourceFilename] === filename); + + if (empty(things)) { + return null; + } + + const allDocumentModes = + unique(things.map(thing => + thing[Thing.yamlSourceDocumentPlacement][0])); + + if (allDocumentModes.length > 1) { + throw new Error(`More than one document mode for documents from ${filename}`); + } + + const documentMode = allDocumentModes[0]; + + switch (documentMode) { + case documentModes.allInOne: { + return { + documentMode, + things: + things.sort((a, b) => + a[Thing.yamlSourceDocumentPlacement][1] - + b[Thing.yamlSourceDocumentPlacement][1]), + }; + } + + case documentModes.oneDocumentTotal: + case documentModes.onePerFile: { + if (things.length > 1) { + throw new Error(`More than one document for ${filename}`); + } + + return { + documentMode, + thing: things[0], + }; + } + + case documentModes.headerAndEntries: { + const headerThings = + things.filter(thing => + thing[Thing.yamlSourceDocumentPlacement][1] === 'header'); + + if (headerThings.length > 1) { + throw new Error(`More than one header document for ${filename}`); + } + + return { + documentMode, + headerThing: headerThings[0] ?? null, + entryThings: + things + .filter(thing => + thing[Thing.yamlSourceDocumentPlacement][1] === 'entry') + .sort((a, b) => + a[Thing.yamlSourceDocumentPlacement][2] - + b[Thing.yamlSourceDocumentPlacement][2]), + }; + } + + default: { + return {documentMode}; + } + } +} + +export function flattenThingLayoutToDocumentOrder(layout) { + switch (layout.documentMode) { + case documentModes.oneDocumentTotal: + case documentModes.onePerFile: { + if (layout.thing) { + return [0]; + } else { + return []; + } + } + + case documentModes.allInOne: { + const indices = + layout.things + .map(thing => thing[Thing.yamlSourceDocumentPlacement][1]); + + return indices; + } + + case documentModes.headerAndEntries: { + const entryIndices = + layout.entryThings + .map(thing => thing[Thing.yamlSourceDocumentPlacement][2]) + .map(index => index + 1); + + if (layout.headerThing) { + return [0, ...entryIndices]; + } else { + return entryIndices; + } + } + + default: { + throw new Error(`Unknown document mode`); + } + } +} + +export function* splitDocumentsInYAMLSourceText(sourceText) { + // Not multiline! + const dividerRegex = /(?:\r\n|\n|^)-{3,}(?:\r\n|\n|$)/g; + + let previousDivider = ''; + + while (true) { + const {lastIndex} = dividerRegex; + const match = dividerRegex.exec(sourceText); + if (match) { + const nextDivider = match[0]; + + yield { + previousDivider, + nextDivider, + text: sourceText.slice(lastIndex, match.index), + }; + + previousDivider = nextDivider; + } else { + const nextDivider = ''; + const lineBreak = previousDivider.match(/\r?\n/)?.[0] ?? ''; + + yield { + previousDivider, + nextDivider, + text: sourceText.slice(lastIndex).replace(/(?<!\n)$/, lineBreak), + }; + + return; + } + } +} + +export function recombineDocumentsIntoYAMLSourceText(documents) { + const dividers = + unique( + documents + .flatMap(d => [d.previousDivider, d.nextDivider]) + .filter(Boolean)); + + const divider = dividers[0]; + + if (dividers.length > 1) { + // TODO: Accommodate mixed dividers as best as we can lol + logWarn`Found multiple dividers in this file, using only ${divider}`; + } + + let sourceText = ''; + + for (const document of documents) { + if (sourceText) { + sourceText += divider; + } + + sourceText += document.text; + } + + return sourceText; +} + +export function reorderDocumentsInYAMLSourceText(sourceText, order) { + const sourceDocuments = + Array.from(splitDocumentsInYAMLSourceText(sourceText)); + + const sortedDocuments = + Array.from( + order, + sourceIndex => sourceDocuments[sourceIndex]); + + return recombineDocumentsIntoYAMLSourceText(sortedDocuments); +} |