Merge branch 'preview' into listing-tweaks

14 files changed, 459 insertions, 199 deletions

diff --git a/src/data/composite/things/track/withAlwaysReferenceByDirectory.js b/src/data/composite/things/track/withAlwaysReferenceByDirectory.js
index d27f7b23..fac8e213 100644
--- a/src/data/composite/things/track/withAlwaysReferenceByDirectory.js
+++ b/src/data/composite/things/track/withAlwaysReferenceByDirectory.js
@@ -2,22 +2,15 @@
 // just to the track's name, which means you don't have to always reference
 // some *other* (much more commonly referenced) track by directory instead
 // of more naturally by name.
-//
-// See the implementation for an important caveat about matching the original
-// track against other tracks, which uses a custom implementation pulling (and
-// duplicating) details from #find instead of using withOriginalRelease and the
-// usual withResolvedReference / find.track() utilities.
-//
 
 import {input, templateCompositeFrom} from '#composite';
+import find from '#find';
 import {isBoolean} from '#validators';
 
 import {exitWithoutDependency, exposeUpdateValueOrContinue}
   from '#composite/control-flow';
 import {withPropertyFromObject} from '#composite/data';
-
-// TODO: Kludge. (The usage of this, not so much the import.)
-import CacheableObject from '../../../things/cacheable-object.js';
+import {withResolvedReference} from '#composite/wiki-data';
 
 export default templateCompositeFrom({
   annotation: `withAlwaysReferenceByDirectory`,
@@ -44,29 +37,23 @@ export default templateCompositeFrom({
       value: input.value(false),
     }),
 
-    // "Slow" / uncached, manual search from trackData (with this track
-    // excluded). Otherwise there end up being pretty bad recursion issues
-    // (track1.alwaysReferencedByDirectory depends on searching through data
-    // including track2, which depends on evaluating track2.alwaysReferenced-
-    // ByDirectory, which depends on searcing through data including track1...)
-    // That said, this is 100% a kludge, since it involves duplicating find
-    // logic on a completely unrelated context.
-    {
-      dependencies: [input.myself(), 'trackData', 'originalReleaseTrack'],
-      compute: (continuation, {
-        [input.myself()]: thisTrack,
-        ['trackData']: trackData,
-        ['originalReleaseTrack']: ref,
-      }) => continuation({
-        ['#originalRelease']:
-          (ref.startsWith('track:')
-            ? trackData.find(track => track.directory === ref.slice('track:'.length))
-            : trackData.find(track =>
-                track !== thisTrack &&
-                !CacheableObject.getUpdateValue(track, 'originalReleaseTrack') &&
-                track.name.toLowerCase() === ref.toLowerCase())),
-      })
-    },
+    // It's necessary to use the custom trackOriginalReleasesOnly find function
+    // here, so as to avoid recursion issues - the find.track() function depends
+    // on accessing each track's alwaysReferenceByDirectory, which means it'll
+    // hit *this track* - and thus this step - and end up recursing infinitely.
+    // By definition, find.trackOriginalReleasesOnly excludes tracks which have
+    // an originalReleaseTrack update value set, which means even though it does
+    // still access each of tracks' `alwaysReferenceByDirectory` property, it
+    // won't access that of *this* track - it will never proceed past the
+    // `exitWithoutDependency` step directly above, so there's no opportunity
+    // for recursion.
+    withResolvedReference({
+      ref: 'originalReleaseTrack',
+      data: 'trackData',
+      find: input.value(find.trackOriginalReleasesOnly),
+    }).outputs({
+      '#resolvedReference': '#originalRelease',
+    }),
 
     exitWithoutDependency({
       dependency: '#originalRelease',
diff --git a/src/data/composite/wiki-properties/wikiData.js b/src/data/composite/wiki-properties/wikiData.js
index 4ea47785..5cea49a0 100644
--- a/src/data/composite/wiki-properties/wikiData.js
+++ b/src/data/composite/wiki-properties/wikiData.js
@@ -1,17 +1,29 @@
 // General purpose wiki data constructor, for properties like artistData,
 // trackData, etc.
 
-import {validateArrayItems, validateInstanceOf} from '#validators';
+import {input, templateCompositeFrom} from '#composite';
+import {validateWikiData} from '#validators';
 
-// TODO: Not templateCompositeFrom.
+import {inputThingClass} from '#composite/wiki-data';
 
-// TODO: This should validate with validateWikiData.
+// TODO: Kludge.
+import Thing from '../../things/thing.js';
 
-export default function(thingClass) {
-  return {
-    flags: {update: true},
-    update: {
-      validate: validateArrayItems(validateInstanceOf(thingClass)),
-    },
-  };
-}
+export default templateCompositeFrom({
+  annotation: `wikiData`,
+
+  compose: false,
+
+  inputs: {
+    class: inputThingClass(),
+  },
+
+  update: ({
+    [input.staticValue('class')]: thingClass,
+  }) => {
+    const referenceType = thingClass[Thing.referenceType];
+    return {validate: validateWikiData({referenceType})};
+  },
+
+  steps: () => [],
+});
diff --git a/src/data/language.js b/src/data/language.js
index 09466907..6ffc31e0 100644
--- a/src/data/language.js
+++ b/src/data/language.js
@@ -1,39 +1,190 @@
+import EventEmitter from 'node:events';
 import {readFile} from 'node:fs/promises';
+import path from 'node:path';
 
-// It stands for "HTML Entities", apparently. Cursed.
-import he from 'he';
+import chokidar from 'chokidar';
+import he from 'he'; // It stands for "HTML Entities", apparently. Cursed.
+import yaml from 'js-yaml';
 
 import T from '#things';
+import {colors, logWarn} from '#cli';
 
-export async function processLanguageFile(file) {
-  const contents = await readFile(file, 'utf-8');
-  const json = JSON.parse(contents);
+import {
+  annotateError,
+  annotateErrorWithFile,
+  showAggregate,
+  withAggregate,
+} from '#sugar';
+
+const {Language} = T;
+
+export function processLanguageSpec(spec, {existingCode = null} = {}) {
+  const {
+    'meta.languageCode': code,
+    'meta.languageName': name,
+
+    'meta.languageIntlCode': intlCode = null,
+    'meta.hidden': hidden = false,
+
+    ...strings
+  } = spec;
+
+  withAggregate({message: `Errors validating language spec`}, ({push}) => {
+    if (!code) {
+      push(new Error(`Missing language code`));
+    }
+
+    if (!name) {
+      push(new Error(`Missing language name`));
+    }
+
+    if (code && existingCode && code !== existingCode) {
+      push(new Error(`Language code (${code}) doesn't match previous value\n(You'll have to reload hsmusic to load this)`));
+    }
+  });
+
+  return {code, intlCode, name, hidden, strings};
+}
+
+function flattenLanguageSpec(spec) {
+  const recursive = (keyPath, value) =>
+    (typeof value === 'object'
+      ? Object.assign({}, ...
+          Object.entries(value)
+            .map(([key, value]) =>
+              (key === '_'
+                ? {[keyPath]: value}
+                : recursive(
+                    (keyPath ? `${keyPath}.${key}` : key),
+                    value))))
+      : {[keyPath]: value});
 
-  const code = json['meta.languageCode'];
-  if (!code) {
-    throw new Error(`Missing language code (file: ${file})`);
+  return recursive('', spec);
+}
+
+async function processLanguageSpecFromFile(file, processLanguageSpecOpts) {
+  let contents;
+
+  try {
+    contents = await readFile(file, 'utf-8');
+  } catch (caughtError) {
+    throw annotateError(
+      new Error(`Failed to read language file`, {cause: caughtError}),
+      error => annotateErrorWithFile(error, file));
   }
-  delete json['meta.languageCode'];
 
-  const intlCode = json['meta.languageIntlCode'] ?? null;
-  delete json['meta.languageIntlCode'];
+  let rawSpec;
+  let parseLanguage;
 
-  const name = json['meta.languageName'];
-  if (!name) {
-    throw new Error(`Missing language name (${code})`);
+  try {
+    if (path.extname(file) === '.yaml') {
+      parseLanguage = 'YAML';
+      rawSpec = yaml.load(contents);
+    } else {
+      parseLanguage = 'JSON';
+      rawSpec = JSON.parse(contents);
+    }
+  } catch (caughtError) {
+    throw annotateError(
+      new Error(`Failed to parse language file as valid ${parseLanguage}`, {cause: caughtError}),
+      error => annotateErrorWithFile(error, file));
   }
-  delete json['meta.languageName'];
 
-  const hidden = json['meta.hidden'] ?? false;
-  delete json['meta.hidden'];
+  const flattenedSpec = flattenLanguageSpec(rawSpec);
+
+  try {
+    return processLanguageSpec(flattenedSpec, processLanguageSpecOpts);
+  } catch (caughtError) {
+    throw annotateErrorWithFile(caughtError, file);
+  }
+}
 
-  const language = new T.Language();
-  language.code = code;
-  language.intlCode = intlCode;
-  language.name = name;
-  language.hidden = hidden;
-  language.escapeHTML = (string) =>
+export function initializeLanguageObject() {
+  const language = new Language();
+
+  language.escapeHTML = string =>
     he.encode(string, {useNamedReferences: true});
-  language.strings = json;
+
   return language;
 }
+
+export async function processLanguageFile(file) {
+  const language = initializeLanguageObject();
+  const properties = await processLanguageSpecFromFile(file);
+  return Object.assign(language, properties);
+}
+
+export function watchLanguageFile(file, {
+  logging = true,
+} = {}) {
+  const basename = path.basename(file);
+
+  const events = new EventEmitter();
+  const language = initializeLanguageObject();
+
+  let emittedReady = false;
+  let successfullyAppliedLanguage = false;
+
+  Object.assign(events, {language, close});
+
+  const watcher = chokidar.watch(file);
+  watcher.on('change', () => handleFileUpdated());
+
+  setImmediate(handleFileUpdated);
+
+  return events;
+
+  async function close() {
+    return watcher.close();
+  }
+
+  function checkReadyConditions() {
+    if (emittedReady) return;
+    if (!successfullyAppliedLanguage) return;
+
+    events.emit('ready');
+    emittedReady = true;
+  }
+
+  async function handleFileUpdated() {
+    let properties;
+
+    try {
+      properties = await processLanguageSpecFromFile(file, {
+        existingCode:
+          (successfullyAppliedLanguage
+            ? language.code
+            : null),
+      });
+    } catch (error) {
+      events.emit('error', error);
+
+      if (logging) {
+        const label =
+          (successfullyAppliedLanguage
+            ? `${language.name} (${language.code})`
+            : basename);
+
+        if (successfullyAppliedLanguage) {
+          logWarn`Failed to load language ${label} - using existing version`;
+        } else {
+          logWarn`Failed to load language ${label} - no prior version loaded`;
+        }
+        showAggregate(error, {showTraces: false});
+      }
+
+      return;
+    }
+
+    Object.assign(language, properties);
+    successfullyAppliedLanguage = true;
+
+    if (logging && emittedReady) {
+      const timestamp = new Date().toLocaleString('en-US', {timeStyle: 'medium'});
+      console.log(colors.green(`[${timestamp}] Updated language ${language.name} (${language.code})`));
+    }
+
+    events.emit('update');
+    checkReadyConditions();
+  }
+}
diff --git a/src/data/things/album.js b/src/data/things/album.js
index 546fda3b..af3eb042 100644
--- a/src/data/things/album.js
+++ b/src/data/things/album.js
@@ -121,10 +121,21 @@ export class Album extends Thing {
 
     // Update only
 
-    artistData: wikiData(Artist),
-    artTagData: wikiData(ArtTag),
-    groupData: wikiData(Group),
-    trackData: wikiData(Track),
+    artistData: wikiData({
+      class: input.value(Artist),
+    }),
+
+    artTagData: wikiData({
+      class: input.value(ArtTag),
+    }),
+
+    groupData: wikiData({
+      class: input.value(Group),
+    }),
+
+    trackData: wikiData({
+      class: input.value(Track),
+    }),
 
     // Expose only
 
diff --git a/src/data/things/art-tag.js b/src/data/things/art-tag.js
index 6503beec..f9e5f0f3 100644
--- a/src/data/things/art-tag.js
+++ b/src/data/things/art-tag.js
@@ -40,8 +40,13 @@ export class ArtTag extends Thing {
 
     // Update only
 
-    albumData: wikiData(Album),
-    trackData: wikiData(Track),
+    albumData: wikiData({
+      class: input.value(Album),
+    }),
+
+    trackData: wikiData({
+      class: input.value(Track),
+    }),
 
     // Expose only
 
@@ -54,7 +59,7 @@ export class ArtTag extends Thing {
           sortAlbumsTracksChronologically(
             [...albumData, ...trackData]
               .filter(({artTags}) => artTags.includes(artTag)),
-            {getDate: o => o.coverArtDate}),
+            {getDate: thing => thing.coverArtDate ?? thing.date}),
       },
     },
   });
diff --git a/src/data/things/artist.js b/src/data/things/artist.js
index 1b313db6..a51723c4 100644
--- a/src/data/things/artist.js
+++ b/src/data/things/artist.js
@@ -45,10 +45,21 @@ export class Artist extends Thing {
 
     // Update only
 
-    albumData: wikiData(Album),
-    artistData: wikiData(Artist),
-    flashData: wikiData(Flash),
-    trackData: wikiData(Track),
+    albumData: wikiData({
+      class: input.value(Album),
+    }),
+
+    artistData: wikiData({
+      class: input.value(Artist),
+    }),
+
+    flashData: wikiData({
+      class: input.value(Flash),
+    }),
+
+    trackData: wikiData({
+      class: input.value(Track),
+    }),
 
     // Expose only
 
diff --git a/src/data/things/composite.js b/src/data/things/composite.js
index 51525bc1..113f0a4f 100644
--- a/src/data/things/composite.js
+++ b/src/data/things/composite.js
@@ -637,6 +637,10 @@ export function compositeFrom(description) {
 
   const compositionNests = description.compose ?? true;
 
+  if (compositionNests && empty(steps)) {
+    aggregate.push(new TypeError(`Expected at least one step`));
+  }
+
   // Steps default to exposing if using a shorthand syntax where flags aren't
   // specified at all.
   const stepsExpose =
@@ -802,8 +806,8 @@ export function compositeFrom(description) {
     });
   }
 
-  if (!compositionNests && !anyStepsCompute && !anyStepsTransform) {
-    aggregate.push(new TypeError(`Expected at least one step to compute or transform`));
+  if (!compositionNests && !compositionUpdates && !anyStepsCompute) {
+    aggregate.push(new TypeError(`Expected at least one step to compute`));
   }
 
   aggregate.close();
@@ -1241,8 +1245,10 @@ export function compositeFrom(description) {
         expose.cache = base.cacheComposition;
       }
     } else if (compositionUpdates) {
-      expose.transform = (value, dependencies) =>
-        _wrapper(value, null, dependencies);
+      if (!empty(steps)) {
+        expose.transform = (value, dependencies) =>
+          _wrapper(value, null, dependencies);
+      }
     } else {
       expose.compute = (dependencies) =>
         _wrapper(noTransformSymbol, null, dependencies);
diff --git a/src/data/things/flash.js b/src/data/things/flash.js
index e2afcef4..1bdda6c8 100644
--- a/src/data/things/flash.js
+++ b/src/data/things/flash.js
@@ -95,9 +95,17 @@ export class Flash extends Thing {
 
     // Update only
 
-    artistData: wikiData(Artist),
-    trackData: wikiData(Track),
-    flashActData: wikiData(FlashAct),
+    artistData: wikiData({
+      class: input.value(Artist),
+    }),
+
+    trackData: wikiData({
+      class: input.value(Track),
+    }),
+
+    flashActData: wikiData({
+      class: input.value(FlashAct),
+    }),
 
     // Expose only
 
@@ -159,6 +167,8 @@ export class FlashAct extends Thing {
 
     // Update only
 
-    flashData: wikiData(Flash),
+    flashData: wikiData({
+      class: input.value(Flash),
+    }),
   })
 }
diff --git a/src/data/things/group.js b/src/data/things/group.js
index 8764a9db..75469bbd 100644
--- a/src/data/things/group.js
+++ b/src/data/things/group.js
@@ -34,8 +34,13 @@ export class Group extends Thing {
 
     // Update only
 
-    albumData: wikiData(Album),
-    groupCategoryData: wikiData(GroupCategory),
+    albumData: wikiData({
+      class: input.value(Album),
+    }),
+
+    groupCategoryData: wikiData({
+      class: input.value(GroupCategory),
+    }),
 
     // Expose only
 
@@ -83,12 +88,15 @@ export class Group extends Thing {
 }
 
 export class GroupCategory extends Thing {
+  static [Thing.referenceType] = 'group-category';
   static [Thing.friendlyName] = `Group Category`;
 
   static [Thing.getPropertyDescriptors] = ({Group}) => ({
     // Update & expose
 
     name: name('Unnamed Group Category'),
+    directory: directory(),
+
     color: color(),
 
     groups: referenceList({
@@ -99,6 +107,8 @@ export class GroupCategory extends Thing {
 
     // Update only
 
-    groupData: wikiData(Group),
+    groupData: wikiData({
+      class: input.value(Group),
+    }),
   });
 }
diff --git a/src/data/things/homepage-layout.js b/src/data/things/homepage-layout.js
index bfa971ca..59c069bd 100644
--- a/src/data/things/homepage-layout.js
+++ b/src/data/things/homepage-layout.js
@@ -70,11 +70,17 @@ export class HomepageLayoutRow extends Thing {
 
     // Update only
 
-    // These aren't necessarily used by every HomepageLayoutRow subclass, but
-    // for convenience of providing this data, every row accepts all wiki data
-    // arrays depended upon by any subclass's behavior.
-    albumData: wikiData(Album),
-    groupData: wikiData(Group),
+    // These wiki data arrays aren't necessarily used by every subclass, but
+    // to the convenience of providing these, the superclass accepts all wiki
+    // data arrays depended upon by any subclass.
+
+    albumData: wikiData({
+      class: input.value(Album),
+    }),
+
+    groupData: wikiData({
+      class: input.value(Group),
+    }),
   });
 }
 
diff --git a/src/data/things/track.js b/src/data/things/track.js
index db325a17..8d310611 100644
--- a/src/data/things/track.js
+++ b/src/data/things/track.js
@@ -256,11 +256,25 @@ export class Track extends Thing {
 
     // Update only
 
-    albumData: wikiData(Album),
-    artistData: wikiData(Artist),
-    artTagData: wikiData(ArtTag),
-    flashData: wikiData(Flash),
-    trackData: wikiData(Track),
+    albumData: wikiData({
+      class: input.value(Album),
+    }),
+
+    artistData: wikiData({
+      class: input.value(Artist),
+    }),
+
+    artTagData: wikiData({
+      class: input.value(ArtTag),
+    }),
+
+    flashData: wikiData({
+      class: input.value(Flash),
+    }),
+
+    trackData: wikiData({
+      class: input.value(Track),
+    }),
 
     // Expose only
 
diff --git a/src/data/things/validators.js b/src/data/things/validators.js
index ee301f15..f60c363c 100644
--- a/src/data/things/validators.js
+++ b/src/data/things/validators.js
@@ -433,18 +433,38 @@ export function validateWikiData({
         OK = true; return true;
       }
 
-      const allRefTypes =
-        new Set(array.map(object =>
-          object.constructor[Symbol.for('Thing.referenceType')]));
+      const allRefTypes = new Set();
 
-      if (allRefTypes.has(undefined)) {
-        if (allRefTypes.size === 1) {
-          throw new TypeError(`Expected array of wiki data objects, got array of other objects`);
+      let foundThing = false;
+      let foundOtherObject = false;
+
+      for (const object of array) {
+        const {[Symbol.for('Thing.referenceType')]: referenceType} = object.constructor;
+
+        if (referenceType === undefined) {
+          foundOtherObject = true;
+
+          // Early-exit if a Thing has been found - nothing more can be learned.
+          if (foundThing) {
+            throw new TypeError(`Expected array of wiki data objects, got mixed items`);
+          }
         } else {
-          throw new TypeError(`Expected array of wiki data objects, got mixed items`);
+          foundThing = true;
+
+          // Early-exit if a non-Thing object has been found - nothing more can
+          // be learned.
+          if (foundOtherObject) {
+            throw new TypeError(`Expected array of wiki data objects, got mixed items`);
+          }
+
+          allRefTypes.add(referenceType);
         }
       }
 
+      if (foundOtherObject && !foundThing) {
+        throw new TypeError(`Expected array of wiki data objects, got array of other objects`);
+      }
+
       if (allRefTypes.size > 1) {
         if (allowMixedTypes) {
           OK = true; return true;
@@ -464,8 +484,10 @@ export function validateWikiData({
         throw new TypeError(`Expected array of unmixed reference types, got multiple: ${types()}`);
       }
 
-      if (referenceType && !allRefTypes.has(referenceType)) {
-        throw new TypeError(`Expected array of ${referenceType}, got array of ${allRefTypes[0]}`)
+      const onlyRefType = Array.from(allRefTypes)[0];
+
+      if (referenceType && onlyRefType !== referenceType) {
+        throw new TypeError(`Expected array of ${referenceType}, got array of ${onlyRefType}`)
       }
 
       OK = true; return true;
diff --git a/src/data/things/wiki-info.js b/src/data/things/wiki-info.js
index 6286a267..89053d62 100644
--- a/src/data/things/wiki-info.js
+++ b/src/data/things/wiki-info.js
@@ -64,6 +64,8 @@ export class WikiInfo extends Thing {
 
     // Update only
 
-    groupData: wikiData(Group),
+    groupData: wikiData({
+      class: input.value(Group),
+    }),
   });
 }
diff --git a/src/data/yaml.js b/src/data/yaml.js
index f7856cb7..1d35bae8 100644
--- a/src/data/yaml.js
+++ b/src/data/yaml.js
@@ -18,11 +18,12 @@ import T, {
 } from '#things';
 
 import {
+  annotateErrorWithFile,
   conditionallySuppressError,
   decorateErrorWithIndex,
+  decorateErrorWithAnnotation,
   empty,
   filterProperties,
-  mapAggregate,
   openAggregate,
   showAggregate,
   withAggregate,
@@ -1120,22 +1121,25 @@ export async function loadAndProcessDataDocuments({dataPath}) {
   const wikiDataResult = {};
 
   function decorateErrorWithFile(fn) {
-    return (x, index, array) => {
-      try {
-        return fn(x, index, array);
-      } catch (error) {
-        error.message +=
-          (error.message.includes('\n') ? '\n' : ' ') +
-          `(file: ${colors.bright(colors.blue(path.relative(dataPath, x.file)))})`;
-        throw error;
-      }
-    };
+    return decorateErrorWithAnnotation(fn,
+      (caughtError, firstArg) =>
+        annotateErrorWithFile(
+          caughtError,
+          path.relative(
+            dataPath,
+            (typeof firstArg === 'object'
+              ? firstArg.file
+              : firstArg))));
+  }
+
+  function asyncDecorateErrorWithFile(fn) {
+    return decorateErrorWithFile(fn).async;
   }
 
   for (const dataStep of dataSteps) {
     await processDataAggregate.nestAsync(
       {message: `Errors during data step: ${colors.bright(dataStep.title)}`},
-      async ({call, callAsync, map, mapAsync, push, nest}) => {
+      async ({call, callAsync, map, mapAsync, push}) => {
         const {documentMode} = dataStep;
 
         if (!Object.values(documentModes).includes(documentMode)) {
@@ -1323,8 +1327,8 @@ export async function loadAndProcessDataDocuments({dataPath}) {
           throw new Error(`Expected 'files' property for ${documentMode.toString()}`);
         }
 
-        let files = (
-          typeof dataStep.files === 'function'
+        const filesFromDataStep =
+          (typeof dataStep.files === 'function'
             ? await callAsync(() =>
                 dataStep.files(dataPath).then(
                   files => files,
@@ -1335,101 +1339,110 @@ export async function loadAndProcessDataDocuments({dataPath}) {
                       throw error;
                     }
                   }))
-            : dataStep.files
-        );
+            : dataStep.files);
 
-        if (!files) {
-          return;
-        }
+        const filesUnderDataPath =
+          filesFromDataStep
+            .map(file => path.join(dataPath, file));
 
-        files = files.map((file) => path.join(dataPath, file));
-
-        const readResults = await mapAsync(
-          files,
-          (file) => readFile(file, 'utf-8').then((contents) => ({file, contents})),
-          {message: `Errors reading data files`});
-
-        let yamlResults = map(
-          readResults,
-          decorateErrorWithFile(({file, contents}) => ({
-            file,
-            documents: yaml.loadAll(contents),
-          })),
-          {message: `Errors parsing data files as valid YAML`});
-
-        yamlResults = yamlResults.map(({file, documents}) => {
-          const {documents: filteredDocuments, aggregate} = filterBlankDocuments(documents);
-          call(decorateErrorWithFile(aggregate.close), {file});
-          return {file, documents: filteredDocuments};
-        });
+        const yamlResults = [];
+
+        await mapAsync(filesUnderDataPath, {message: `Errors loading data files`},
+          asyncDecorateErrorWithFile(async file => {
+            let contents;
+            try {
+              contents = await readFile(file, 'utf-8');
+            } catch (caughtError) {
+              throw new Error(`Failed to read data file`, {cause: caughtError});
+            }
+
+            let documents;
+            try {
+              documents = yaml.loadAll(contents);
+            } catch (caughtError) {
+              throw new Error(`Failed to parse valid YAML`, {cause: caughtError});
+            }
+
+            const {documents: filteredDocuments, aggregate: filterAggregate} =
+              filterBlankDocuments(documents);
+
+            try {
+              filterAggregate.close();
+            } catch (caughtError) {
+              // Blank documents aren't a critical error, they're just something
+              // that should be noted - the (filtered) documents still get pushed.
+              const pathToFile = path.relative(dataPath, file);
+              annotateErrorWithFile(caughtError, pathToFile);
+              push(caughtError);
+            }
+
+            yamlResults.push({file, documents: filteredDocuments});
+          }));
 
         const processResults = [];
 
         switch (documentMode) {
           case documentModes.headerAndEntries:
-            map(yamlResults, decorateErrorWithFile(({documents}) => {
-              const headerDocument = documents[0];
-              const entryDocuments = documents.slice(1).filter(Boolean);
-
-              if (!headerDocument)
-                throw new Error(`Missing header document (empty file or erroneously starting with "---"?)`);
-
-              // This'll be decorated with the file, and groups together any
-              // errors from processing the header and entry documents.
-              const fileAggregate =
-                openAggregate({message: `Errors processing documents`});
-
-              const {thing: headerObject, aggregate: headerAggregate} =
-                dataStep.processHeaderDocument(headerDocument);
-
-              try {
-                headerAggregate.close()
-              } catch (caughtError) {
-                caughtError.message = `(${colors.yellow(`header`)}) ${caughtError.message}`;
-                fileAggregate.push(caughtError);
-              }
+            map(yamlResults, {message: `Errors processing documents in data files`},
+              decorateErrorWithFile(({documents}) => {
+                const headerDocument = documents[0];
+                const entryDocuments = documents.slice(1).filter(Boolean);
+
+                if (!headerDocument)
+                  throw new Error(`Missing header document (empty file or erroneously starting with "---"?)`);
+
+                withAggregate({message: `Errors processing documents`}, ({push}) => {
+                  const {thing: headerObject, aggregate: headerAggregate} =
+                    dataStep.processHeaderDocument(headerDocument);
 
-              const entryObjects = [];
+                  try {
+                    headerAggregate.close();
+                  } catch (caughtError) {
+                    caughtError.message = `(${colors.yellow(`header`)}) ${caughtError.message}`;
+                    push(caughtError);
+                  }
 
-              for (let index = 0; index < entryDocuments.length; index++) {
-                const entryDocument = entryDocuments[index];
+                  const entryObjects = [];
 
-                const {thing: entryObject, aggregate: entryAggregate} =
-                  dataStep.processEntryDocument(entryDocument);
+                  for (let index = 0; index < entryDocuments.length; index++) {
+                    const entryDocument = entryDocuments[index];
 
-                entryObjects.push(entryObject);
+                    const {thing: entryObject, aggregate: entryAggregate} =
+                      dataStep.processEntryDocument(entryDocument);
 
-                try {
-                  entryAggregate.close();
-                } catch (caughtError) {
-                  caughtError.message = `(${colors.yellow(`entry #${index + 1}`)}) ${caughtError.message}`;
-                  fileAggregate.push(caughtError);
-                }
-              }
+                    entryObjects.push(entryObject);
 
-              processResults.push({
-                header: headerObject,
-                entries: entryObjects,
-              });
+                    try {
+                      entryAggregate.close();
+                    } catch (caughtError) {
+                      caughtError.message = `(${colors.yellow(`entry #${index + 1}`)}) ${caughtError.message}`;
+                      push(caughtError);
+                    }
+                  }
 
-              fileAggregate.close();
-            }), {message: `Errors processing documents in data files`});
+                  processResults.push({
+                    header: headerObject,
+                    entries: entryObjects,
+                  });
+                });
+              }));
             break;
 
           case documentModes.onePerFile:
-            map(yamlResults, decorateErrorWithFile(({documents}) => {
-              if (documents.length > 1)
-                throw new Error(`Only expected one document to be present per file, got ${documents.length} here`);
+            map(yamlResults, {message: `Errors processing data files as valid documents`},
+              decorateErrorWithFile(({documents}) => {
+                if (documents.length > 1)
+                  throw new Error(`Only expected one document to be present per file, got ${documents.length} here`);
 
-              if (empty(documents) || !documents[0])
-                throw new Error(`Expected a document, this file is empty`);
+                if (empty(documents) || !documents[0])
+                  throw new Error(`Expected a document, this file is empty`);
 
-              const {thing, aggregate} =
-                dataStep.processDocument(documents[0]);
+                const {thing, aggregate} =
+                  dataStep.processDocument(documents[0]);
 
-              processResults.push(thing);
-              aggregate.close();
-            }), {message: `Errors processing data files as valid documents`});
+                processResults.push(thing);
+                aggregate.close();
+              }));
             break;
         }
 
@@ -1662,7 +1675,7 @@ export function filterReferenceErrors(wikiData) {
           }
         }
 
-        nest({message: `Reference errors in ${inspect(thing)}`}, ({push, filter}) => {
+        nest({message: `Reference errors in ${inspect(thing)}`}, ({nest, push, filter}) => {
           for (const [property, findFnKey] of Object.entries(propSpec)) {
             const value = CacheableObject.getUpdateValue(thing, property);