From c1f93e2d270292bca7e991c180db618f4902a3f9 Mon Sep 17 00:00:00 2001 From: "(quasar) nebula" Date: Mon, 30 Oct 2023 22:35:18 -0300 Subject: content, data: fix places that assume coverArtDate defaults to date --- src/data/things/art-tag.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/data') diff --git a/src/data/things/art-tag.js b/src/data/things/art-tag.js index 6503beec..8901ab39 100644 --- a/src/data/things/art-tag.js +++ b/src/data/things/art-tag.js @@ -54,7 +54,7 @@ export class ArtTag extends Thing { sortAlbumsTracksChronologically( [...albumData, ...trackData] .filter(({artTags}) => artTags.includes(artTag)), - {getDate: o => o.coverArtDate}), + {getDate: thing => thing.coverArtDate ?? thing.date}), }, }, }); -- cgit 1.3.0-6-gf8a5 From 8e174abde6a6b9b46e2cf885115c58bedcfd0802 Mon Sep 17 00:00:00 2001 From: "(quasar) nebula" Date: Thu, 2 Nov 2023 13:58:34 -0300 Subject: yaml: fix mis-nested errors in non-array reference fields --- src/data/yaml.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/data') diff --git a/src/data/yaml.js b/src/data/yaml.js index f7856cb7..0ffe9682 100644 --- a/src/data/yaml.js +++ b/src/data/yaml.js @@ -1662,7 +1662,7 @@ export function filterReferenceErrors(wikiData) { } } - nest({message: `Reference errors in ${inspect(thing)}`}, ({push, filter}) => { + nest({message: `Reference errors in ${inspect(thing)}`}, ({nest, push, filter}) => { for (const [property, findFnKey] of Object.entries(propSpec)) { const value = CacheableObject.getUpdateValue(thing, property); -- cgit 1.3.0-6-gf8a5 From e42e5527b99426c1b74fea150cf62214de73087e Mon Sep 17 00:00:00 2001 From: "(quasar) nebula" Date: Sat, 4 Nov 2023 20:20:27 -0300 Subject: group: add GroupCategory.directory, referenceType group-category --- src/data/things/group.js | 3 +++ 1 file changed, 3 insertions(+) (limited to 'src/data') diff --git a/src/data/things/group.js b/src/data/things/group.js index 8764a9db..7bb917ad 100644 --- a/src/data/things/group.js +++ b/src/data/things/group.js @@ -83,12 +83,15 @@ export class Group extends Thing { } export class GroupCategory extends Thing { + static [Thing.referenceType] = 'group-category'; static [Thing.friendlyName] = `Group Category`; static [Thing.getPropertyDescriptors] = ({Group}) => ({ // Update & expose name: name('Unnamed Group Category'), + directory: directory(), + color: color(), groups: referenceList({ -- cgit 1.3.0-6-gf8a5 From 9e42c9f3773d431bc62fcf76f0da2cc852dfc329 Mon Sep 17 00:00:00 2001 From: "(quasar) nebula" Date: Sat, 4 Nov 2023 20:22:00 -0300 Subject: data: wikiData: use validateWikiData instead of instance checks --- src/data/composite/wiki-properties/wikiData.js | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) (limited to 'src/data') diff --git a/src/data/composite/wiki-properties/wikiData.js b/src/data/composite/wiki-properties/wikiData.js index 4ea47785..5965b949 100644 --- a/src/data/composite/wiki-properties/wikiData.js +++ b/src/data/composite/wiki-properties/wikiData.js @@ -1,17 +1,20 @@ // General purpose wiki data constructor, for properties like artistData, // trackData, etc. -import {validateArrayItems, validateInstanceOf} from '#validators'; +import {validateWikiData} from '#validators'; -// TODO: Not templateCompositeFrom. +// TODO: Kludge. +import Thing from '../../things/thing.js'; -// TODO: This should validate with validateWikiData. +// TODO: Not templateCompositeFrom. export default function(thingClass) { + const referenceType = thingClass[Thing.referenceType]; + return { flags: {update: true}, update: { - validate: validateArrayItems(validateInstanceOf(thingClass)), + validate: validateWikiData({referenceType}), }, }; } -- cgit 1.3.0-6-gf8a5 From e4974b2af9419acd644497274bd49f39880b2282 Mon Sep 17 00:00:00 2001 From: "(quasar) nebula" Date: Sat, 4 Nov 2023 20:40:25 -0300 Subject: data: support stepless updating compositions --- src/data/things/composite.js | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) (limited to 'src/data') diff --git a/src/data/things/composite.js b/src/data/things/composite.js index 51525bc1..c3b08f86 100644 --- a/src/data/things/composite.js +++ b/src/data/things/composite.js @@ -802,8 +802,8 @@ export function compositeFrom(description) { }); } - if (!compositionNests && !anyStepsCompute && !anyStepsTransform) { - aggregate.push(new TypeError(`Expected at least one step to compute or transform`)); + if (!compositionNests && !compositionUpdates && !anyStepsCompute) { + aggregate.push(new TypeError(`Expected at least one step to compute`)); } aggregate.close(); @@ -1241,8 +1241,10 @@ export function compositeFrom(description) { expose.cache = base.cacheComposition; } } else if (compositionUpdates) { - expose.transform = (value, dependencies) => - _wrapper(value, null, dependencies); + if (!empty(steps)) { + expose.transform = (value, dependencies) => + _wrapper(value, null, dependencies); + } } else { expose.compute = (dependencies) => _wrapper(noTransformSymbol, null, dependencies); -- cgit 1.3.0-6-gf8a5 From c4eba52cd5023e3b503937b47e2bc6f77527d5c3 Mon Sep 17 00:00:00 2001 From: "(quasar) nebula" Date: Sat, 4 Nov 2023 20:43:29 -0300 Subject: data: wikiData: port to templateCompositeFrom syntax --- src/data/composite/wiki-properties/wikiData.js | 29 +++++++++++++++++--------- src/data/things/album.js | 19 +++++++++++++---- src/data/things/art-tag.js | 9 ++++++-- src/data/things/artist.js | 19 +++++++++++++---- src/data/things/flash.js | 18 ++++++++++++---- src/data/things/group.js | 13 +++++++++--- src/data/things/homepage-layout.js | 16 +++++++++----- src/data/things/track.js | 24 ++++++++++++++++----- src/data/things/wiki-info.js | 4 +++- 9 files changed, 113 insertions(+), 38 deletions(-) (limited to 'src/data') diff --git a/src/data/composite/wiki-properties/wikiData.js b/src/data/composite/wiki-properties/wikiData.js index 5965b949..5cea49a0 100644 --- a/src/data/composite/wiki-properties/wikiData.js +++ b/src/data/composite/wiki-properties/wikiData.js @@ -1,20 +1,29 @@ // General purpose wiki data constructor, for properties like artistData, // trackData, etc. +import {input, templateCompositeFrom} from '#composite'; import {validateWikiData} from '#validators'; +import {inputThingClass} from '#composite/wiki-data'; + // TODO: Kludge. import Thing from '../../things/thing.js'; -// TODO: Not templateCompositeFrom. +export default templateCompositeFrom({ + annotation: `wikiData`, + + compose: false, + + inputs: { + class: inputThingClass(), + }, -export default function(thingClass) { - const referenceType = thingClass[Thing.referenceType]; + update: ({ + [input.staticValue('class')]: thingClass, + }) => { + const referenceType = thingClass[Thing.referenceType]; + return {validate: validateWikiData({referenceType})}; + }, - return { - flags: {update: true}, - update: { - validate: validateWikiData({referenceType}), - }, - }; -} + steps: () => [], +}); diff --git a/src/data/things/album.js b/src/data/things/album.js index 546fda3b..af3eb042 100644 --- a/src/data/things/album.js +++ b/src/data/things/album.js @@ -121,10 +121,21 @@ export class Album extends Thing { // Update only - artistData: wikiData(Artist), - artTagData: wikiData(ArtTag), - groupData: wikiData(Group), - trackData: wikiData(Track), + artistData: wikiData({ + class: input.value(Artist), + }), + + artTagData: wikiData({ + class: input.value(ArtTag), + }), + + groupData: wikiData({ + class: input.value(Group), + }), + + trackData: wikiData({ + class: input.value(Track), + }), // Expose only diff --git a/src/data/things/art-tag.js b/src/data/things/art-tag.js index 8901ab39..f9e5f0f3 100644 --- a/src/data/things/art-tag.js +++ b/src/data/things/art-tag.js @@ -40,8 +40,13 @@ export class ArtTag extends Thing { // Update only - albumData: wikiData(Album), - trackData: wikiData(Track), + albumData: wikiData({ + class: input.value(Album), + }), + + trackData: wikiData({ + class: input.value(Track), + }), // Expose only diff --git a/src/data/things/artist.js b/src/data/things/artist.js index ea19d2ba..e0350b86 100644 --- a/src/data/things/artist.js +++ b/src/data/things/artist.js @@ -45,10 +45,21 @@ export class Artist extends Thing { // Update only - albumData: wikiData(Album), - artistData: wikiData(Artist), - flashData: wikiData(Flash), - trackData: wikiData(Track), + albumData: wikiData({ + class: input.value(Album), + }), + + artistData: wikiData({ + class: input.value(Artist), + }), + + flashData: wikiData({ + class: input.value(Flash), + }), + + trackData: wikiData({ + class: input.value(Track), + }), // Expose only diff --git a/src/data/things/flash.js b/src/data/things/flash.js index e2afcef4..1bdda6c8 100644 --- a/src/data/things/flash.js +++ b/src/data/things/flash.js @@ -95,9 +95,17 @@ export class Flash extends Thing { // Update only - artistData: wikiData(Artist), - trackData: wikiData(Track), - flashActData: wikiData(FlashAct), + artistData: wikiData({ + class: input.value(Artist), + }), + + trackData: wikiData({ + class: input.value(Track), + }), + + flashActData: wikiData({ + class: input.value(FlashAct), + }), // Expose only @@ -159,6 +167,8 @@ export class FlashAct extends Thing { // Update only - flashData: wikiData(Flash), + flashData: wikiData({ + class: input.value(Flash), + }), }) } diff --git a/src/data/things/group.js b/src/data/things/group.js index 7bb917ad..75469bbd 100644 --- a/src/data/things/group.js +++ b/src/data/things/group.js @@ -34,8 +34,13 @@ export class Group extends Thing { // Update only - albumData: wikiData(Album), - groupCategoryData: wikiData(GroupCategory), + albumData: wikiData({ + class: input.value(Album), + }), + + groupCategoryData: wikiData({ + class: input.value(GroupCategory), + }), // Expose only @@ -102,6 +107,8 @@ export class GroupCategory extends Thing { // Update only - groupData: wikiData(Group), + groupData: wikiData({ + class: input.value(Group), + }), }); } diff --git a/src/data/things/homepage-layout.js b/src/data/things/homepage-layout.js index bfa971ca..59c069bd 100644 --- a/src/data/things/homepage-layout.js +++ b/src/data/things/homepage-layout.js @@ -70,11 +70,17 @@ export class HomepageLayoutRow extends Thing { // Update only - // These aren't necessarily used by every HomepageLayoutRow subclass, but - // for convenience of providing this data, every row accepts all wiki data - // arrays depended upon by any subclass's behavior. - albumData: wikiData(Album), - groupData: wikiData(Group), + // These wiki data arrays aren't necessarily used by every subclass, but + // to the convenience of providing these, the superclass accepts all wiki + // data arrays depended upon by any subclass. + + albumData: wikiData({ + class: input.value(Album), + }), + + groupData: wikiData({ + class: input.value(Group), + }), }); } diff --git a/src/data/things/track.js b/src/data/things/track.js index db325a17..8d310611 100644 --- a/src/data/things/track.js +++ b/src/data/things/track.js @@ -256,11 +256,25 @@ export class Track extends Thing { // Update only - albumData: wikiData(Album), - artistData: wikiData(Artist), - artTagData: wikiData(ArtTag), - flashData: wikiData(Flash), - trackData: wikiData(Track), + albumData: wikiData({ + class: input.value(Album), + }), + + artistData: wikiData({ + class: input.value(Artist), + }), + + artTagData: wikiData({ + class: input.value(ArtTag), + }), + + flashData: wikiData({ + class: input.value(Flash), + }), + + trackData: wikiData({ + class: input.value(Track), + }), // Expose only diff --git a/src/data/things/wiki-info.js b/src/data/things/wiki-info.js index 6286a267..89053d62 100644 --- a/src/data/things/wiki-info.js +++ b/src/data/things/wiki-info.js @@ -64,6 +64,8 @@ export class WikiInfo extends Thing { // Update only - groupData: wikiData(Group), + groupData: wikiData({ + class: input.value(Group), + }), }); } -- cgit 1.3.0-6-gf8a5 From 9f4c3b913fa6b12a236cedf76abe120f0321f53e Mon Sep 17 00:00:00 2001 From: "(quasar) nebula" Date: Sat, 4 Nov 2023 21:01:23 -0300 Subject: data: validateWikiData: early exit for mixed items --- src/data/things/validators.js | 34 +++++++++++++++++++++++++++------- 1 file changed, 27 insertions(+), 7 deletions(-) (limited to 'src/data') diff --git a/src/data/things/validators.js b/src/data/things/validators.js index ee301f15..ea4303fc 100644 --- a/src/data/things/validators.js +++ b/src/data/things/validators.js @@ -433,18 +433,38 @@ export function validateWikiData({ OK = true; return true; } - const allRefTypes = - new Set(array.map(object => - object.constructor[Symbol.for('Thing.referenceType')])); + const allRefTypes = new Set(); - if (allRefTypes.has(undefined)) { - if (allRefTypes.size === 1) { - throw new TypeError(`Expected array of wiki data objects, got array of other objects`); + let foundThing = false; + let foundOtherObject = false; + + for (const object of array) { + const {[Symbol.for('Thing.referenceType')]: referenceType} = object.constructor; + + if (referenceType === undefined) { + foundOtherObject = true; + + // Early-exit if a Thing has been found - nothing more can be learned. + if (foundThing) { + throw new TypeError(`Expected array of wiki data objects, got mixed items`); + } } else { - throw new TypeError(`Expected array of wiki data objects, got mixed items`); + foundThing = true; + + // Early-exit if a non-Thing object has been found - nothing more can + // be learned. + if (foundOtherObject) { + throw new TypeError(`Expected array of wiki data objects, got mixed items`); + } + + allRefTypes.add(referenceType); } } + if (foundOtherObject && !foundThing) { + throw new TypeError(`Expected array of wiki data objects, got array of other objects`); + } + if (allRefTypes.size > 1) { if (allowMixedTypes) { OK = true; return true; -- cgit 1.3.0-6-gf8a5 From 5aad4eae6629eaa1e4dd849b03abff8888afdb4d Mon Sep 17 00:00:00 2001 From: "(quasar) nebula" Date: Sat, 4 Nov 2023 21:01:48 -0300 Subject: data: validateWikiData: fix messaging for mismatch one-ref-type --- src/data/things/validators.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'src/data') diff --git a/src/data/things/validators.js b/src/data/things/validators.js index ea4303fc..f60c363c 100644 --- a/src/data/things/validators.js +++ b/src/data/things/validators.js @@ -484,8 +484,10 @@ export function validateWikiData({ throw new TypeError(`Expected array of unmixed reference types, got multiple: ${types()}`); } - if (referenceType && !allRefTypes.has(referenceType)) { - throw new TypeError(`Expected array of ${referenceType}, got array of ${allRefTypes[0]}`) + const onlyRefType = Array.from(allRefTypes)[0]; + + if (referenceType && onlyRefType !== referenceType) { + throw new TypeError(`Expected array of ${referenceType}, got array of ${onlyRefType}`) } OK = true; return true; -- cgit 1.3.0-6-gf8a5 From 527e4618fdc57d80ac79ca9ceb3eed60fca90d6b Mon Sep 17 00:00:00 2001 From: "(quasar) nebula" Date: Sat, 4 Nov 2023 21:19:44 -0300 Subject: data: always require at least one step for nesting compositions --- src/data/things/composite.js | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'src/data') diff --git a/src/data/things/composite.js b/src/data/things/composite.js index c3b08f86..113f0a4f 100644 --- a/src/data/things/composite.js +++ b/src/data/things/composite.js @@ -637,6 +637,10 @@ export function compositeFrom(description) { const compositionNests = description.compose ?? true; + if (compositionNests && empty(steps)) { + aggregate.push(new TypeError(`Expected at least one step`)); + } + // Steps default to exposing if using a shorthand syntax where flags aren't // specified at all. const stepsExpose = -- cgit 1.3.0-6-gf8a5 From d6672865a59a94a2acdd3ec7e827f96f8c3e67e4 Mon Sep 17 00:00:00 2001 From: "(quasar) nebula" Date: Sun, 5 Nov 2023 17:52:25 -0400 Subject: data: withAlwaysReferenceByDirectory: micro-optimizations --- .../things/track/withAlwaysReferenceByDirectory.js | 29 ++++++++++++++-------- 1 file changed, 19 insertions(+), 10 deletions(-) (limited to 'src/data') diff --git a/src/data/composite/things/track/withAlwaysReferenceByDirectory.js b/src/data/composite/things/track/withAlwaysReferenceByDirectory.js index d27f7b23..52d72124 100644 --- a/src/data/composite/things/track/withAlwaysReferenceByDirectory.js +++ b/src/data/composite/things/track/withAlwaysReferenceByDirectory.js @@ -53,19 +53,28 @@ export default templateCompositeFrom({ // logic on a completely unrelated context. { dependencies: [input.myself(), 'trackData', 'originalReleaseTrack'], - compute: (continuation, { + compute(continuation, { [input.myself()]: thisTrack, ['trackData']: trackData, ['originalReleaseTrack']: ref, - }) => continuation({ - ['#originalRelease']: - (ref.startsWith('track:') - ? trackData.find(track => track.directory === ref.slice('track:'.length)) - : trackData.find(track => - track !== thisTrack && - !CacheableObject.getUpdateValue(track, 'originalReleaseTrack') && - track.name.toLowerCase() === ref.toLowerCase())), - }) + }) { + let originalRelease; + + if (ref.startsWith('track:')) { + const refDirectory = ref.slice('track:'.length); + originalRelease = + trackData.find(track => track.directory === refDirectory); + } else { + const refName = ref.toLowerCase(); + originalRelease = + trackData.find(track => + track.name.toLowerCase() === refName && + track !== thisTrack && + !CacheableObject.getUpdateValue(track, 'originalReleaseTrack')); + } + + return continuation({['#originalRelease']: originalRelease}); + }, }, exitWithoutDependency({ -- cgit 1.3.0-6-gf8a5 From 7841f97cde6182359bb3f2b0f6117c6379ef18dc Mon Sep 17 00:00:00 2001 From: "(quasar) nebula" Date: Sun, 5 Nov 2023 18:29:22 -0400 Subject: data, find: use clean-logic, cached find.trackOriginalReleasesOnly --- .../things/track/withAlwaysReferenceByDirectory.js | 60 +++++++--------------- 1 file changed, 19 insertions(+), 41 deletions(-) (limited to 'src/data') diff --git a/src/data/composite/things/track/withAlwaysReferenceByDirectory.js b/src/data/composite/things/track/withAlwaysReferenceByDirectory.js index 52d72124..fac8e213 100644 --- a/src/data/composite/things/track/withAlwaysReferenceByDirectory.js +++ b/src/data/composite/things/track/withAlwaysReferenceByDirectory.js @@ -2,22 +2,15 @@ // just to the track's name, which means you don't have to always reference // some *other* (much more commonly referenced) track by directory instead // of more naturally by name. -// -// See the implementation for an important caveat about matching the original -// track against other tracks, which uses a custom implementation pulling (and -// duplicating) details from #find instead of using withOriginalRelease and the -// usual withResolvedReference / find.track() utilities. -// import {input, templateCompositeFrom} from '#composite'; +import find from '#find'; import {isBoolean} from '#validators'; import {exitWithoutDependency, exposeUpdateValueOrContinue} from '#composite/control-flow'; import {withPropertyFromObject} from '#composite/data'; - -// TODO: Kludge. (The usage of this, not so much the import.) -import CacheableObject from '../../../things/cacheable-object.js'; +import {withResolvedReference} from '#composite/wiki-data'; export default templateCompositeFrom({ annotation: `withAlwaysReferenceByDirectory`, @@ -44,38 +37,23 @@ export default templateCompositeFrom({ value: input.value(false), }), - // "Slow" / uncached, manual search from trackData (with this track - // excluded). Otherwise there end up being pretty bad recursion issues - // (track1.alwaysReferencedByDirectory depends on searching through data - // including track2, which depends on evaluating track2.alwaysReferenced- - // ByDirectory, which depends on searcing through data including track1...) - // That said, this is 100% a kludge, since it involves duplicating find - // logic on a completely unrelated context. - { - dependencies: [input.myself(), 'trackData', 'originalReleaseTrack'], - compute(continuation, { - [input.myself()]: thisTrack, - ['trackData']: trackData, - ['originalReleaseTrack']: ref, - }) { - let originalRelease; - - if (ref.startsWith('track:')) { - const refDirectory = ref.slice('track:'.length); - originalRelease = - trackData.find(track => track.directory === refDirectory); - } else { - const refName = ref.toLowerCase(); - originalRelease = - trackData.find(track => - track.name.toLowerCase() === refName && - track !== thisTrack && - !CacheableObject.getUpdateValue(track, 'originalReleaseTrack')); - } - - return continuation({['#originalRelease']: originalRelease}); - }, - }, + // It's necessary to use the custom trackOriginalReleasesOnly find function + // here, so as to avoid recursion issues - the find.track() function depends + // on accessing each track's alwaysReferenceByDirectory, which means it'll + // hit *this track* - and thus this step - and end up recursing infinitely. + // By definition, find.trackOriginalReleasesOnly excludes tracks which have + // an originalReleaseTrack update value set, which means even though it does + // still access each of tracks' `alwaysReferenceByDirectory` property, it + // won't access that of *this* track - it will never proceed past the + // `exitWithoutDependency` step directly above, so there's no opportunity + // for recursion. + withResolvedReference({ + ref: 'originalReleaseTrack', + data: 'trackData', + find: input.value(find.trackOriginalReleasesOnly), + }).outputs({ + '#resolvedReference': '#originalRelease', + }), exitWithoutDependency({ dependency: '#originalRelease', -- cgit 1.3.0-6-gf8a5 From ab306affb06b9f94a2f6b8dc8607614949b3ab0e Mon Sep 17 00:00:00 2001 From: "(quasar) nebula" Date: Mon, 6 Nov 2023 15:36:57 -0400 Subject: yaml: tidy aggregate nesting and error syntax --- src/data/yaml.js | 196 +++++++++++++++++++++++++++++++------------------------ 1 file changed, 110 insertions(+), 86 deletions(-) (limited to 'src/data') diff --git a/src/data/yaml.js b/src/data/yaml.js index 0ffe9682..37f31800 100644 --- a/src/data/yaml.js +++ b/src/data/yaml.js @@ -18,11 +18,11 @@ import T, { } from '#things'; import { + annotateErrorWithFile, conditionallySuppressError, decorateErrorWithIndex, empty, filterProperties, - mapAggregate, openAggregate, showAggregate, withAggregate, @@ -1119,15 +1119,30 @@ export async function loadAndProcessDataDocuments({dataPath}) { }); const wikiDataResult = {}; + const _getFileFromArgument = arg => + (typeof arg === 'object' + ? arg.file + : arg); + function decorateErrorWithFile(fn) { - return (x, index, array) => { + return (...args) => { try { - return fn(x, index, array); - } catch (error) { - error.message += - (error.message.includes('\n') ? '\n' : ' ') + - `(file: ${colors.bright(colors.blue(path.relative(dataPath, x.file)))})`; - throw error; + return fn(...args); + } catch (caughtError) { + const file = _getFileFromArgument(args[0]); + throw annotateErrorWithFile(caughtError, path.relative(dataPath, file)); + } + }; + } + + // Certified gensync moment. + function asyncDecorateErrorWithFile(fn) { + return async (...args) => { + try { + return await fn(...args); + } catch (caughtError) { + const file = _getFileFromArgument(args[0]); + throw annotateErrorWithFile(caughtError, path.relative(dataPath, file)); } }; } @@ -1135,7 +1150,7 @@ export async function loadAndProcessDataDocuments({dataPath}) { for (const dataStep of dataSteps) { await processDataAggregate.nestAsync( {message: `Errors during data step: ${colors.bright(dataStep.title)}`}, - async ({call, callAsync, map, mapAsync, push, nest}) => { + async ({call, callAsync, map, mapAsync, push}) => { const {documentMode} = dataStep; if (!Object.values(documentModes).includes(documentMode)) { @@ -1323,8 +1338,8 @@ export async function loadAndProcessDataDocuments({dataPath}) { throw new Error(`Expected 'files' property for ${documentMode.toString()}`); } - let files = ( - typeof dataStep.files === 'function' + const filesFromDataStep = + (typeof dataStep.files === 'function' ? await callAsync(() => dataStep.files(dataPath).then( files => files, @@ -1335,101 +1350,110 @@ export async function loadAndProcessDataDocuments({dataPath}) { throw error; } })) - : dataStep.files - ); + : dataStep.files); - if (!files) { - return; - } + const filesUnderDataPath = + filesFromDataStep + .map(file => path.join(dataPath, file)); - files = files.map((file) => path.join(dataPath, file)); - - const readResults = await mapAsync( - files, - (file) => readFile(file, 'utf-8').then((contents) => ({file, contents})), - {message: `Errors reading data files`}); - - let yamlResults = map( - readResults, - decorateErrorWithFile(({file, contents}) => ({ - file, - documents: yaml.loadAll(contents), - })), - {message: `Errors parsing data files as valid YAML`}); - - yamlResults = yamlResults.map(({file, documents}) => { - const {documents: filteredDocuments, aggregate} = filterBlankDocuments(documents); - call(decorateErrorWithFile(aggregate.close), {file}); - return {file, documents: filteredDocuments}; - }); + const yamlResults = []; + + await mapAsync(filesUnderDataPath, {message: `Errors loading data files`}, + asyncDecorateErrorWithFile(async file => { + let contents; + try { + contents = await readFile(file, 'utf-8'); + } catch (caughtError) { + throw new Error(`Failed to read data file`, {cause: caughtError}); + } + + let documents; + try { + documents = yaml.loadAll(contents); + } catch (caughtError) { + throw new Error(`Failed to parse valid YAML`, {cause: caughtError}); + } + + const {documents: filteredDocuments, aggregate: filterAggregate} = + filterBlankDocuments(documents); + + try { + filterAggregate.close(); + } catch (caughtError) { + // Blank documents aren't a critical error, they're just something + // that should be noted - the (filtered) documents still get pushed. + const pathToFile = path.relative(dataPath, file); + annotateErrorWithFile(caughtError, pathToFile); + push(caughtError); + } + + yamlResults.push({file, documents: filteredDocuments}); + })); const processResults = []; switch (documentMode) { case documentModes.headerAndEntries: - map(yamlResults, decorateErrorWithFile(({documents}) => { - const headerDocument = documents[0]; - const entryDocuments = documents.slice(1).filter(Boolean); - - if (!headerDocument) - throw new Error(`Missing header document (empty file or erroneously starting with "---"?)`); - - // This'll be decorated with the file, and groups together any - // errors from processing the header and entry documents. - const fileAggregate = - openAggregate({message: `Errors processing documents`}); - - const {thing: headerObject, aggregate: headerAggregate} = - dataStep.processHeaderDocument(headerDocument); - - try { - headerAggregate.close() - } catch (caughtError) { - caughtError.message = `(${colors.yellow(`header`)}) ${caughtError.message}`; - fileAggregate.push(caughtError); - } + map(yamlResults, {message: `Errors processing documents in data files`}, + decorateErrorWithFile(({documents}) => { + const headerDocument = documents[0]; + const entryDocuments = documents.slice(1).filter(Boolean); - const entryObjects = []; + if (!headerDocument) + throw new Error(`Missing header document (empty file or erroneously starting with "---"?)`); - for (let index = 0; index < entryDocuments.length; index++) { - const entryDocument = entryDocuments[index]; + withAggregate({message: `Errors processing documents`}, ({push}) => { + const {thing: headerObject, aggregate: headerAggregate} = + dataStep.processHeaderDocument(headerDocument); - const {thing: entryObject, aggregate: entryAggregate} = - dataStep.processEntryDocument(entryDocument); + try { + headerAggregate.close(); + } catch (caughtError) { + caughtError.message = `(${colors.yellow(`header`)}) ${caughtError.message}`; + push(caughtError); + } - entryObjects.push(entryObject); + const entryObjects = []; - try { - entryAggregate.close(); - } catch (caughtError) { - caughtError.message = `(${colors.yellow(`entry #${index + 1}`)}) ${caughtError.message}`; - fileAggregate.push(caughtError); - } - } + for (let index = 0; index < entryDocuments.length; index++) { + const entryDocument = entryDocuments[index]; - processResults.push({ - header: headerObject, - entries: entryObjects, - }); + const {thing: entryObject, aggregate: entryAggregate} = + dataStep.processEntryDocument(entryDocument); - fileAggregate.close(); - }), {message: `Errors processing documents in data files`}); + entryObjects.push(entryObject); + + try { + entryAggregate.close(); + } catch (caughtError) { + caughtError.message = `(${colors.yellow(`entry #${index + 1}`)}) ${caughtError.message}`; + push(caughtError); + } + } + + processResults.push({ + header: headerObject, + entries: entryObjects, + }); + }); + })); break; case documentModes.onePerFile: - map(yamlResults, decorateErrorWithFile(({documents}) => { - if (documents.length > 1) - throw new Error(`Only expected one document to be present per file, got ${documents.length} here`); + map(yamlResults, {message: `Errors processing data files as valid documents`}, + decorateErrorWithFile(({documents}) => { + if (documents.length > 1) + throw new Error(`Only expected one document to be present per file, got ${documents.length} here`); - if (empty(documents) || !documents[0]) - throw new Error(`Expected a document, this file is empty`); + if (empty(documents) || !documents[0]) + throw new Error(`Expected a document, this file is empty`); - const {thing, aggregate} = - dataStep.processDocument(documents[0]); + const {thing, aggregate} = + dataStep.processDocument(documents[0]); - processResults.push(thing); - aggregate.close(); - }), {message: `Errors processing data files as valid documents`}); + processResults.push(thing); + aggregate.close(); + })); break; } -- cgit 1.3.0-6-gf8a5 From 682b62b33aa6e5a4c512343d0355d32cb1c67c17 Mon Sep 17 00:00:00 2001 From: "(quasar) nebula" Date: Mon, 6 Nov 2023 16:29:16 -0400 Subject: yaml: consolidate logic in async-adaptive decorateErrorWithFile --- src/data/yaml.js | 33 +++++++++++---------------------- 1 file changed, 11 insertions(+), 22 deletions(-) (limited to 'src/data') diff --git a/src/data/yaml.js b/src/data/yaml.js index 37f31800..1d35bae8 100644 --- a/src/data/yaml.js +++ b/src/data/yaml.js @@ -21,6 +21,7 @@ import { annotateErrorWithFile, conditionallySuppressError, decorateErrorWithIndex, + decorateErrorWithAnnotation, empty, filterProperties, openAggregate, @@ -1119,32 +1120,20 @@ export async function loadAndProcessDataDocuments({dataPath}) { }); const wikiDataResult = {}; - const _getFileFromArgument = arg => - (typeof arg === 'object' - ? arg.file - : arg); - function decorateErrorWithFile(fn) { - return (...args) => { - try { - return fn(...args); - } catch (caughtError) { - const file = _getFileFromArgument(args[0]); - throw annotateErrorWithFile(caughtError, path.relative(dataPath, file)); - } - }; + return decorateErrorWithAnnotation(fn, + (caughtError, firstArg) => + annotateErrorWithFile( + caughtError, + path.relative( + dataPath, + (typeof firstArg === 'object' + ? firstArg.file + : firstArg)))); } - // Certified gensync moment. function asyncDecorateErrorWithFile(fn) { - return async (...args) => { - try { - return await fn(...args); - } catch (caughtError) { - const file = _getFileFromArgument(args[0]); - throw annotateErrorWithFile(caughtError, path.relative(dataPath, file)); - } - }; + return decorateErrorWithFile(fn).async; } for (const dataStep of dataSteps) { -- cgit 1.3.0-6-gf8a5 From 0768953f9538f0bbd65835b0a4293e2ba438ce52 Mon Sep 17 00:00:00 2001 From: "(quasar) nebula" Date: Wed, 1 Nov 2023 09:00:55 -0300 Subject: data: tidy language loading code, add processLanguageSpec --- src/data/language.js | 46 +++++++++++++++++++++++++++------------------- 1 file changed, 27 insertions(+), 19 deletions(-) (limited to 'src/data') diff --git a/src/data/language.js b/src/data/language.js index 09466907..34de8779 100644 --- a/src/data/language.js +++ b/src/data/language.js @@ -5,35 +5,43 @@ import he from 'he'; import T from '#things'; -export async function processLanguageFile(file) { - const contents = await readFile(file, 'utf-8'); - const json = JSON.parse(contents); +export function processLanguageSpec(spec) { + const { + 'meta.languageCode': code, + 'meta.languageName': name, + + 'meta.languageIntlCode': intlCode = null, + 'meta.hidden': hidden = false, + + ...strings + } = spec; - const code = json['meta.languageCode']; if (!code) { throw new Error(`Missing language code (file: ${file})`); } - delete json['meta.languageCode']; - const intlCode = json['meta.languageIntlCode'] ?? null; - delete json['meta.languageIntlCode']; - - const name = json['meta.languageName']; if (!name) { throw new Error(`Missing language name (${code})`); } - delete json['meta.languageName']; - - const hidden = json['meta.hidden'] ?? false; - delete json['meta.hidden']; const language = new T.Language(); - language.code = code; - language.intlCode = intlCode; - language.name = name; - language.hidden = hidden; - language.escapeHTML = (string) => + + Object.assign(language, { + code, + intlCode, + name, + hidden, + strings, + }); + + language.escapeHTML = string => he.encode(string, {useNamedReferences: true}); - language.strings = json; + return language; } + +export async function processLanguageFile(file) { + const contents = await readFile(file, 'utf-8'); + const spec = JSON.parse(contents); + return processLanguageSpec(spec); +} -- cgit 1.3.0-6-gf8a5 From cc3a6e32b957c60aa29027fa575e4b3ca0c05c64 Mon Sep 17 00:00:00 2001 From: "(quasar) nebula" Date: Wed, 1 Nov 2023 09:12:23 -0300 Subject: data: more language loading refactoring --- src/data/language.js | 40 +++++++++++++++++++++++----------------- 1 file changed, 23 insertions(+), 17 deletions(-) (limited to 'src/data') diff --git a/src/data/language.js b/src/data/language.js index 34de8779..b71e55a2 100644 --- a/src/data/language.js +++ b/src/data/language.js @@ -1,10 +1,13 @@ import {readFile} from 'node:fs/promises'; -// It stands for "HTML Entities", apparently. Cursed. -import he from 'he'; +import chokidar from 'chokidar'; +import he from 'he'; // It stands for "HTML Entities", apparently. Cursed. +import {withAggregate} from '#sugar'; import T from '#things'; +const {Language} = T; + export function processLanguageSpec(spec) { const { 'meta.languageCode': code, @@ -16,23 +19,21 @@ export function processLanguageSpec(spec) { ...strings } = spec; - if (!code) { - throw new Error(`Missing language code (file: ${file})`); - } + withAggregate({message: `Errors validating language spec`}, ({push}) => { + if (!code) { + push(new Error(`Missing language code (file: ${file})`)); + } - if (!name) { - throw new Error(`Missing language name (${code})`); - } + if (!name) { + push(new Error(`Missing language name (${code})`)); + } + }); - const language = new T.Language(); + return {code, intlCode, name, hidden, strings}; +} - Object.assign(language, { - code, - intlCode, - name, - hidden, - strings, - }); +export function initializeLanguageObject() { + const language = new Language(); language.escapeHTML = string => he.encode(string, {useNamedReferences: true}); @@ -43,5 +44,10 @@ export function processLanguageSpec(spec) { export async function processLanguageFile(file) { const contents = await readFile(file, 'utf-8'); const spec = JSON.parse(contents); - return processLanguageSpec(spec); + + const language = initializeLanguageObject(); + const properties = processLanguageSpec(spec); + Object.assign(language, properties); + + return language; } -- cgit 1.3.0-6-gf8a5 From d497be7b5e1e4d9f9a8ca71de0a82def384467f8 Mon Sep 17 00:00:00 2001 From: "(quasar) nebula" Date: Mon, 6 Nov 2023 17:42:35 -0400 Subject: data: language: basic watchLanguageFile implementation --- src/data/language.js | 108 +++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 100 insertions(+), 8 deletions(-) (limited to 'src/data') diff --git a/src/data/language.js b/src/data/language.js index b71e55a2..ec38cbde 100644 --- a/src/data/language.js +++ b/src/data/language.js @@ -1,10 +1,19 @@ +import EventEmitter from 'node:events'; import {readFile} from 'node:fs/promises'; +import path from 'node:path'; import chokidar from 'chokidar'; import he from 'he'; // It stands for "HTML Entities", apparently. Cursed. -import {withAggregate} from '#sugar'; import T from '#things'; +import {colors, logWarn} from '#cli'; + +import { + annotateError, + annotateErrorWithFile, + showAggregate, + withAggregate, +} from '#sugar'; const {Language} = T; @@ -21,17 +30,43 @@ export function processLanguageSpec(spec) { withAggregate({message: `Errors validating language spec`}, ({push}) => { if (!code) { - push(new Error(`Missing language code (file: ${file})`)); + push(new Error(`Missing language code`)); } if (!name) { - push(new Error(`Missing language name (${code})`)); + push(new Error(`Missing language name`)); } }); return {code, intlCode, name, hidden, strings}; } +async function processLanguageSpecFromFile(file) { + let contents, spec; + + try { + contents = await readFile(file, 'utf-8'); + } catch (caughtError) { + throw annotateError( + new Error(`Failed to read language file`, {cause: caughtError}), + error => annotateErrorWithFile(error, file)); + } + + try { + spec = JSON.parse(contents); + } catch (caughtError) { + throw annotateError( + new Error(`Failed to parse language file as valid JSON`, {cause: caughtError}), + error => annotateErrorWithFile(error, file)); + } + + try { + return processLanguageSpec(spec); + } catch (caughtError) { + throw annotateErrorWithFile(caughtError, file); + } +} + export function initializeLanguageObject() { const language = new Language(); @@ -42,12 +77,69 @@ export function initializeLanguageObject() { } export async function processLanguageFile(file) { - const contents = await readFile(file, 'utf-8'); - const spec = JSON.parse(contents); + const language = initializeLanguageObject(); + const properties = await processLanguageSpecFromFile(file); + return Object.assign(language, properties); +} + +export function watchLanguageFile(file, { + logging = true, +} = {}) { + const basename = path.basename(file); + const events = new EventEmitter(); const language = initializeLanguageObject(); - const properties = processLanguageSpec(spec); - Object.assign(language, properties); - return language; + let emittedReady = false; + let successfullyAppliedLanguage = false; + + Object.assign(events, {language, close}); + + const watcher = chokidar.watch(file); + watcher.on('change', () => handleFileUpdated()); + + setImmediate(handleFileUpdated); + + return events; + + async function close() { + return watcher.close(); + } + + function checkReadyConditions() { + if (emittedReady) return; + if (!successfullyAppliedLanguage) return; + + events.emit('ready'); + emittedReady = true; + } + + async function handleFileUpdated() { + let properties; + + try { + properties = await processLanguageSpecFromFile(file); + } catch (error) { + if (logging) { + if (successfullyAppliedLanguage) { + logWarn`Failed to load language ${basename} - using existing version`; + } else { + logWarn`Failed to load language ${basename} - no prior version loaded`; + } + showAggregate(error, {showTraces: false}); + } + return; + } + + Object.assign(language, properties); + successfullyAppliedLanguage = true; + + if (logging && emittedReady) { + const timestamp = new Date().toLocaleString('en-US', {timeStyle: 'medium'}); + console.log(colors.green(`[${timestamp}] Updated language ${language.name} (${language.code})`)); + } + + events.emit('update'); + checkReadyConditions(); + } } -- cgit 1.3.0-6-gf8a5 From 06949e1d20d38d38eb05999ca236f2c7d150691e Mon Sep 17 00:00:00 2001 From: "(quasar) nebula" Date: Mon, 6 Nov 2023 17:55:25 -0400 Subject: upd8: basic watchLanguageFile integration for internal language --- src/data/language.js | 3 +++ 1 file changed, 3 insertions(+) (limited to 'src/data') diff --git a/src/data/language.js b/src/data/language.js index ec38cbde..5ab3936e 100644 --- a/src/data/language.js +++ b/src/data/language.js @@ -120,6 +120,8 @@ export function watchLanguageFile(file, { try { properties = await processLanguageSpecFromFile(file); } catch (error) { + events.emit('error', error); + if (logging) { if (successfullyAppliedLanguage) { logWarn`Failed to load language ${basename} - using existing version`; @@ -128,6 +130,7 @@ export function watchLanguageFile(file, { } showAggregate(error, {showTraces: false}); } + return; } -- cgit 1.3.0-6-gf8a5 From 9f3a1f476752059681fbe21f8a1f7bf11dd73c9b Mon Sep 17 00:00:00 2001 From: "(quasar) nebula" Date: Mon, 6 Nov 2023 18:43:49 -0400 Subject: data: language: nicer language labelling for successive errors --- src/data/language.js | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) (limited to 'src/data') diff --git a/src/data/language.js b/src/data/language.js index 5ab3936e..99eaa58f 100644 --- a/src/data/language.js +++ b/src/data/language.js @@ -17,7 +17,7 @@ import { const {Language} = T; -export function processLanguageSpec(spec) { +export function processLanguageSpec(spec, {existingCode = null}) { const { 'meta.languageCode': code, 'meta.languageName': name, @@ -36,12 +36,16 @@ export function processLanguageSpec(spec) { if (!name) { push(new Error(`Missing language name`)); } + + if (code && existingCode && code !== existingCode) { + push(new Error(`Language code (${code}) doesn't match previous value\n(You'll have to reload hsmusic to load this)`)); + } }); return {code, intlCode, name, hidden, strings}; } -async function processLanguageSpecFromFile(file) { +async function processLanguageSpecFromFile(file, processLanguageSpecOpts) { let contents, spec; try { @@ -61,7 +65,7 @@ async function processLanguageSpecFromFile(file) { } try { - return processLanguageSpec(spec); + return processLanguageSpec(spec, processLanguageSpecOpts); } catch (caughtError) { throw annotateErrorWithFile(caughtError, file); } @@ -118,15 +122,25 @@ export function watchLanguageFile(file, { let properties; try { - properties = await processLanguageSpecFromFile(file); + properties = await processLanguageSpecFromFile(file, { + existingCode: + (successfullyAppliedLanguage + ? language.code + : null), + }); } catch (error) { events.emit('error', error); if (logging) { + const label = + (successfullyAppliedLanguage + ? `${language.name} (${language.code})` + : basename); + if (successfullyAppliedLanguage) { - logWarn`Failed to load language ${basename} - using existing version`; + logWarn`Failed to load language ${label} - using existing version`; } else { - logWarn`Failed to load language ${basename} - no prior version loaded`; + logWarn`Failed to load language ${label} - no prior version loaded`; } showAggregate(error, {showTraces: false}); } -- cgit 1.3.0-6-gf8a5 From bd3affb31b6b2e5cb0667c550bcdbde8af51a392 Mon Sep 17 00:00:00 2001 From: "(quasar) nebula" Date: Mon, 6 Nov 2023 19:06:27 -0400 Subject: data: language: basic support for loading language from YAML --- src/data/language.js | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) (limited to 'src/data') diff --git a/src/data/language.js b/src/data/language.js index 99eaa58f..aed16057 100644 --- a/src/data/language.js +++ b/src/data/language.js @@ -4,6 +4,7 @@ import path from 'node:path'; import chokidar from 'chokidar'; import he from 'he'; // It stands for "HTML Entities", apparently. Cursed. +import yaml from 'js-yaml'; import T from '#things'; import {colors, logWarn} from '#cli'; @@ -56,11 +57,18 @@ async function processLanguageSpecFromFile(file, processLanguageSpecOpts) { error => annotateErrorWithFile(error, file)); } + let parseLanguage; try { - spec = JSON.parse(contents); + if (path.extname(file) === '.yaml') { + parseLanguage = 'YAML'; + spec = yaml.load(contents); + } else { + parseLanguage = 'JSON'; + spec = JSON.parse(contents); + } } catch (caughtError) { throw annotateError( - new Error(`Failed to parse language file as valid JSON`, {cause: caughtError}), + new Error(`Failed to parse language file as valid ${parseLanguage}`, {cause: caughtError}), error => annotateErrorWithFile(error, file)); } -- cgit 1.3.0-6-gf8a5 From 8d24f17f729c7da550824ab4134b89757754fb9c Mon Sep 17 00:00:00 2001 From: "(quasar) nebula" Date: Mon, 6 Nov 2023 20:08:15 -0400 Subject: data: language: flatten language spec, allow for structuring --- src/data/language.js | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) (limited to 'src/data') diff --git a/src/data/language.js b/src/data/language.js index aed16057..99efc03d 100644 --- a/src/data/language.js +++ b/src/data/language.js @@ -46,8 +46,24 @@ export function processLanguageSpec(spec, {existingCode = null}) { return {code, intlCode, name, hidden, strings}; } +function flattenLanguageSpec(spec) { + const recursive = (keyPath, value) => + (typeof value === 'object' + ? Object.assign({}, ... + Object.entries(value) + .map(([key, value]) => + (key === '_' + ? {[keyPath]: value} + : recursive( + (keyPath ? `${keyPath}.${key}` : key), + value)))) + : {[keyPath]: value}); + + return recursive('', spec); +} + async function processLanguageSpecFromFile(file, processLanguageSpecOpts) { - let contents, spec; + let contents; try { contents = await readFile(file, 'utf-8'); @@ -57,14 +73,16 @@ async function processLanguageSpecFromFile(file, processLanguageSpecOpts) { error => annotateErrorWithFile(error, file)); } + let rawSpec; let parseLanguage; + try { if (path.extname(file) === '.yaml') { parseLanguage = 'YAML'; - spec = yaml.load(contents); + rawSpec = yaml.load(contents); } else { parseLanguage = 'JSON'; - spec = JSON.parse(contents); + rawSpec = JSON.parse(contents); } } catch (caughtError) { throw annotateError( @@ -72,8 +90,10 @@ async function processLanguageSpecFromFile(file, processLanguageSpecOpts) { error => annotateErrorWithFile(error, file)); } + const flattenedSpec = flattenLanguageSpec(rawSpec); + try { - return processLanguageSpec(spec, processLanguageSpecOpts); + return processLanguageSpec(flattenedSpec, processLanguageSpecOpts); } catch (caughtError) { throw annotateErrorWithFile(caughtError, file); } -- cgit 1.3.0-6-gf8a5 From 7fa4f92c8a41754e198ade96a7d5d0dd5b0aa59e Mon Sep 17 00:00:00 2001 From: "(quasar) nebula" Date: Tue, 7 Nov 2023 09:12:47 -0400 Subject: upd8: add --no-language-reloading option, default for static-build --- src/data/language.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/data') diff --git a/src/data/language.js b/src/data/language.js index 99eaa58f..15c11933 100644 --- a/src/data/language.js +++ b/src/data/language.js @@ -17,7 +17,7 @@ import { const {Language} = T; -export function processLanguageSpec(spec, {existingCode = null}) { +export function processLanguageSpec(spec, {existingCode = null} = {}) { const { 'meta.languageCode': code, 'meta.languageName': name, -- cgit 1.3.0-6-gf8a5