« get me outta code hell

hsmusic-wiki - HSMusic - static wiki software cataloguing collaborative creation
about summary refs log tree commit diff
path: root/src/upd8.js
diff options
context:
space:
mode:
Diffstat (limited to 'src/upd8.js')
-rwxr-xr-xsrc/upd8.js1473
1 files changed, 839 insertions, 634 deletions
diff --git a/src/upd8.js b/src/upd8.js
index f78a21c4..0ea998a2 100755
--- a/src/upd8.js
+++ b/src/upd8.js
@@ -17,6 +17,9 @@
 //      going to 8e in. May8e JSON, 8ut more likely some weird custom format
 //      which will 8e a lot easier to edit.
 //
+//      Like three years later oh god: SURPISE! We went with the latter, but
+//      they're YAML now. Probably. Assuming that hasn't changed, yet.
+//
 //   3. Generate the page files! They're just static index.html files, and are
 //      what gh-pages (or wherever this is hosted) will show to clients.
 //      Hopefully pretty minimalistic HTML, 8ut like, shrug. They'll reference
@@ -28,40 +31,6 @@
 // Oh yeah, like. Just run this through some relatively recent version of
 // node.js and you'll 8e fine. ...Within the project root. O8viously.
 
-// HEY FUTURE ME!!!!!!!! Don't forget to implement artist pages! Those are,
-// like, the coolest idea you've had yet, so DO NOT FORGET. (Remem8er, link
-// from track listings, etc!) --- Thanks, past me. To futurerer me: an al8um
-// listing page (a list of all the al8ums)! Make sure to sort these 8y date -
-// we'll need a new field for al8ums.
-
-// ^^^^^^^^ DID THAT! 8ut also, artist images. Pro8a8ly stolen from the fandom
-// wiki (I found half those images anywayz).
-
-// TRACK ART CREDITS. This is a must.
-
-// 2020-08-23
-// ATTENTION ALL 8*TCHES AND OTHER GENDER TRUCKERS: AS IT TURNS OUT, THIS CODE
-// ****SUCKS****. I DON'T THINK ANYTHING WILL EVER REDEEM IT, 8UT THAT DOESN'T
-// MEAN WE CAN'T TAKE SOME ACTION TO MAKE WRITING IT A LITTLE LESS TERRI8LE.
-// We're gonna start defining STRUCTURES to make things suck less!!!!!!!!
-// No classes 8ecause those are a huge pain and like, pro8a8ly 8ad performance
-// or whatever -- just some standard structures that should 8e followed
-// wherever reasona8le. Only one I need today is the contri8 one 8ut let's put
-// any new general-purpose structures here too, ok?
-//
-// Contri8ution: {who, what, date, thing}. D8 and thing are the new fields.
-//
-// Use these wisely, which is to say all the time and instead of whatever
-// terri8le new pseudo structure you're trying to invent!!!!!!!!
-//
-// Upd8 2021-01-03: Soooooooo we didn't actually really end up using these,
-// lol? Well there's still only one anyway. Kinda ended up doing a 8ig refactor
-// of all the o8ject structures today. It's not *especially* relevant 8ut feels
-// worth mentioning? I'd get rid of this comment 8lock 8ut I like it too much!
-// Even though I haven't actually reread it, lol. 8ut yeah, hopefully in the
-// spirit of this "make things more consistent" attitude I 8rought up 8ack in
-// August, stuff's lookin' 8etter than ever now. W00t!
-
 import * as path from 'path';
 import { promisify } from 'util';
 import { fileURLToPath } from 'url';
@@ -75,6 +44,8 @@ import fixWS from 'fix-whitespace';
 // It stands for "HTML Entities", apparently. Cursed.
 import he from 'he';
 
+import yaml from 'js-yaml';
+
 import {
     // This is the dum8est name for a function possi8le. Like, SURE, fine, may8e
     // the UNIX people had some valid reason to go with the weird truncated
@@ -109,6 +80,8 @@ import {
     unlink
 } from 'fs/promises';
 
+import { inspect as nodeInspect } from 'util';
+
 import genThumbs from './gen-thumbs.js';
 import { listingSpec, listingTargetSpec } from './listing-spec.js';
 import urlSpec from './url-spec.js';
@@ -118,6 +91,18 @@ import find from './util/find.js';
 import * as html from './util/html.js';
 import unbound_link, {getLinkThemeString} from './util/link.js';
 
+import Album, { TrackGroup } from './thing/album.js';
+import Artist from './thing/artist.js';
+import ArtTag from './thing/art-tag.js';
+import Flash, { FlashAct } from './thing/flash.js';
+import Group, { GroupCategory } from './thing/group.js';
+import HomepageLayout, {
+    HomepageLayoutAlbumsRow,
+} from './thing/homepage-layout.js';
+import NewsEntry from './thing/news-entry.js';
+import Thing from './thing/thing.js';
+import Track from './thing/track.js';
+
 import {
     fancifyFlashURL,
     fancifyURL,
@@ -129,6 +114,7 @@ import {
     getAlbumStylesheet,
     getArtistString,
     getFlashGridHTML,
+    getFooterLocalizationLinks,
     getGridHTML,
     getRevealStringFromTags,
     getRevealStringFromWarnings,
@@ -137,12 +123,14 @@ import {
 } from './misc-templates.js';
 
 import {
+    color,
     decorateTime,
     logWarn,
     logInfo,
     logError,
     parseOptions,
-    progressPromiseAll
+    progressPromiseAll,
+    ENABLE_COLOR
 } from './util/cli.js';
 
 import {
@@ -184,11 +172,16 @@ import {
 
 import {
     bindOpts,
-    call,
+    filterAggregateAsync,
     filterEmptyLines,
+    mapAggregate,
+    mapAggregateAsync,
+    openAggregate,
     queue,
+    showAggregate,
     splitArray,
     unique,
+    withAggregate,
     withEntries
 } from './util/sugar.js';
 
@@ -208,13 +201,14 @@ const __dirname = path.dirname(fileURLToPath(import.meta.url));
 
 const CACHEBUST = 7;
 
+// MAKE THESE END IN YAML
 const WIKI_INFO_FILE = 'wiki-info.txt';
-const HOMEPAGE_INFO_FILE = 'homepage.txt';
-const ARTIST_DATA_FILE = 'artists.txt';
-const FLASH_DATA_FILE = 'flashes.txt';
-const NEWS_DATA_FILE = 'news.txt';
-const TAG_DATA_FILE = 'tags.txt';
-const GROUP_DATA_FILE = 'groups.txt';
+const HOMEPAGE_LAYOUT_DATA_FILE = 'homepage.yaml';
+const ARTIST_DATA_FILE = 'artists.yaml';
+const FLASH_DATA_FILE = 'flashes.yaml';
+const NEWS_DATA_FILE = 'news.yaml';
+const ART_TAG_DATA_FILE = 'tags.yaml';
+const GROUP_DATA_FILE = 'groups.yaml';
 const STATIC_PAGE_DATA_FILE = 'static-pages.txt';
 const DEFAULT_STRINGS_FILE = 'strings-default.json';
 
@@ -235,6 +229,10 @@ const STATIC_DIRECTORY = 'static';
 // read from and processed to compose the majority of album and track data.
 const DATA_ALBUM_DIRECTORY = 'album';
 
+function inspect(value) {
+    return nodeInspect(value, {colors: ENABLE_COLOR});
+}
+
 // Shared varia8les! These are more efficient to access than a shared varia8le
 // (or at least I h8pe so), and are easier to pass across functions than a
 // 8unch of specific arguments.
@@ -274,20 +272,11 @@ function splitLines(text) {
     return text.split(/\r\n|\r|\n/);
 }
 
-function* getSections(lines) {
-    // ::::)
-    const isSeparatorLine = line => /^-{8,}/.test(line);
-    yield* splitArray(lines, isSeparatorLine);
-}
-
-function getBasicField(lines, name) {
-    const line = lines.find(line => line.startsWith(name + ':'));
-    return line && line.slice(name.length + 1).trim();
-}
+function parseDimensions(string) {
+    if (!string) {
+        return null;
+    }
 
-function getDimensionsField(lines, name) {
-    const string = getBasicField(lines, name);
-    if (!string) return string;
     const parts = string.split(/[x,* ]+/g);
     if (parts.length !== 2) throw new Error(`Invalid dimensions: ${string} (expected width & height)`);
     const nums = parts.map(part => Number(part.trim()));
@@ -295,52 +284,7 @@ function getDimensionsField(lines, name) {
     return nums;
 }
 
-function getBooleanField(lines, name) {
-    // The ?? oper8tor (which is just, hilariously named, lol) can 8e used to
-    // specify a default!
-    const value = getBasicField(lines, name);
-    switch (value) {
-        case 'yes':
-        case 'true':
-            return true;
-        case 'no':
-        case 'false':
-            return false;
-        default:
-            return null;
-    }
-}
-
-function getListField(lines, name) {
-    let startIndex = lines.findIndex(line => line.startsWith(name + ':'));
-    // If callers want to default to an empty array, they should stick
-    // "|| []" after the call.
-    if (startIndex === -1) {
-        return null;
-    }
-    // We increment startIndex 8ecause we don't want to include the
-    // "heading" line (e.g. "URLs:") in the actual data.
-    startIndex++;
-    let endIndex = lines.findIndex((line, index) => index >= startIndex && !line.startsWith('- '));
-    if (endIndex === -1) {
-        endIndex = lines.length;
-    }
-    if (endIndex === startIndex) {
-        // If there is no list that comes after the heading line, treat the
-        // heading line itself as the comma-separ8ted array value, using
-        // the 8asic field function to do that. (It's l8 and my 8rain is
-        // sleepy. Please excuse any unhelpful comments I may write, or may
-        // have already written, in this st8. Thanks!)
-        const value = getBasicField(lines, name);
-        return value && value.split(',').map(val => val.trim());
-    }
-    const listLines = lines.slice(startIndex, endIndex);
-    return listLines.map(line => line.slice(2));
-};
-
-function getContributionField(section, name) {
-    let contributors = getListField(section, name);
-
+function parseContributors(contributors) {
     if (!contributors) {
         return null;
     }
@@ -375,27 +319,6 @@ function getContributionField(section, name) {
     return contributors;
 };
 
-function getMultilineField(lines, name) {
-    // All this code is 8asically the same as the getListText - just with a
-    // different line prefix (four spaces instead of a dash and a space).
-    let startIndex = lines.findIndex(line => line.startsWith(name + ':'));
-    if (startIndex === -1) {
-        return null;
-    }
-    startIndex++;
-    let endIndex = lines.findIndex((line, index) => index >= startIndex && line.length && !line.startsWith('    '));
-    if (endIndex === -1) {
-        endIndex = lines.length;
-    }
-    // If there aren't any content lines, don't return anything!
-    if (endIndex === startIndex) {
-        return null;
-    }
-    // We also join the lines instead of returning an array.
-    const listLines = lines.slice(startIndex, endIndex);
-    return listLines.map(line => line.slice(4)).join('\n');
-};
-
 const replacerSpec = {
     'album': {
         find: 'album',
@@ -737,321 +660,380 @@ function transformLyrics(text, {
     return outLines.join('\n');
 }
 
-function getCommentaryField(lines) {
-    const text = getMultilineField(lines, 'Commentary');
-    if (text) {
-        const lines = text.split('\n');
-        if (!lines[0].replace(/<\/b>/g, '').includes(':</i>')) {
-            return {error: `An entry is missing commentary citation: "${lines[0].slice(0, 40)}..."`};
-        }
-        return text;
-    } else {
-        return null;
-    }
-};
-
-async function processAlbumDataFile(file) {
-    let contents;
-    try {
-        contents = await readFile(file, 'utf-8');
-    } catch (error) {
-        // This function can return "error o8jects," which are really just
-        // ordinary o8jects with an error message attached. I'm not 8othering
-        // with error codes here or anywhere in this function; while this would
-        // normally 8e 8ad coding practice, it doesn't really matter here,
-        // 8ecause this isn't an API getting consumed 8y other services (e.g.
-        // translaction functions). If we return an error, the caller will just
-        // print the attached message in the output summary.
-        return {error: `Could not read ${file} (${error.code}).`};
-    }
-
-    // We're pro8a8ly supposed to, like, search for a header somewhere in the
-    // al8um contents, to make sure it's trying to 8e the intended structure
-    // and is a valid utf-8 (or at least ASCII) file. 8ut like, whatever.
-    // We'll just return more specific errors if it's missing necessary data
-    // fields.
+// Use parseErrorFactory to declare different "types" of errors. By storing the
+// factory itself in an accessible location, the type of error may be detected
+// by comparing against its own factory property.
+function parseErrorFactory(annotation) {
+    return function factory(data = null) {
+        return {
+            error: true,
+            annotation,
+            data,
+            factory
+        };
+    };
+}
 
-    const contentLines = contents.split(/\r\n|\r|\n/);
+function parseField(object, key, steps) {
+    let value = object[key];
 
-    // In this line of code I defeat the purpose of using a generator in the
-    // first place. Sorry!!!!!!!!
-    const sections = Array.from(getSections(contentLines));
+    for (const step of steps) {
+        try {
+            value = step(value);
+        } catch (error) {
+            throw parseField.stepError({
+                stepName: step.name,
+                stepError: error
+            });
+        }
+    }
 
-    const albumSection = sections[0];
-    const album = {};
+    return value;
+}
 
-    album.name = getBasicField(albumSection, 'Album');
+parseField.stepError = parseErrorFactory('step failed');
 
-    if (!album.name) {
-        return {error: `The file "${path.relative(dataPath, file)}" is missing the "Album" field - maybe this is a misplaced file instead of album data?`};
+function assertFieldPresent(value) {
+    if (value === undefined || value === null) {
+        throw assertFieldPresent.missingField();
+    } else {
+        return value;
     }
+}
 
-    album.artists = getContributionField(albumSection, 'Artists') || getContributionField(albumSection, 'Artist');
-    album.wallpaperArtists = getContributionField(albumSection, 'Wallpaper Art');
-    album.wallpaperStyle = getMultilineField(albumSection, 'Wallpaper Style');
-    album.wallpaperFileExtension = getBasicField(albumSection, 'Wallpaper File Extension') || 'jpg';
-    album.bannerArtists = getContributionField(albumSection, 'Banner Art');
-    album.bannerStyle = getMultilineField(albumSection, 'Banner Style');
-    album.bannerFileExtension = getBasicField(albumSection, 'Banner File Extension') || 'jpg';
-    album.bannerDimensions = getDimensionsField(albumSection, 'Banner Dimensions');
-    album.date = getBasicField(albumSection, 'Date');
-    album.trackArtDate = getBasicField(albumSection, 'Track Art Date') || album.date;
-    album.coverArtDate = getBasicField(albumSection, 'Cover Art Date') || album.date;
-    album.dateAdded = getBasicField(albumSection, 'Date Added');
-    album.coverArtists = getContributionField(albumSection, 'Cover Art');
-    album.hasTrackArt = getBooleanField(albumSection, 'Has Track Art') ?? true;
-    album.trackCoverArtists = getContributionField(albumSection, 'Track Art');
-    album.artTags = getListField(albumSection, 'Art Tags') || [];
-    album.commentary = getCommentaryField(albumSection);
-    album.urls = getListField(albumSection, 'URLs') || [];
-    album.groups = getListField(albumSection, 'Groups') || [];
-    album.directory = getBasicField(albumSection, 'Directory');
-    album.isMajorRelease = getBooleanField(albumSection, 'Major Release') ?? false;
-    album.isListedOnHomepage = getBooleanField(albumSection, 'Listed on Homepage') ?? true;
+assertFieldPresent.missingField = parseErrorFactory('missing field');
 
-    if (album.artists && album.artists.error) {
-        return {error: `${album.artists.error} (in ${album.name})`};
+function assertValidDate(dateString, {optional = false} = {}) {
+    if (dateString && isNaN(Date.parse(dateString))) {
+        throw assertValidDate.invalidDate();
     }
+    return value;
+}
 
-    if (album.coverArtists && album.coverArtists.error) {
-        return {error: `${album.coverArtists.error} (in ${album.name})`};
-    }
+assertValidDate.invalidDate = parseErrorFactory('invalid date');
 
-    if (album.commentary && album.commentary.error) {
-        return {error: `${album.commentary.error} (in ${album.name})`};
+function parseCommentary(text) {
+    if (text) {
+        const lines = String(text).split('\n');
+        if (!lines[0].replace(/<\/b>/g, '').includes(':</i>')) {
+            return {error: `An entry is missing commentary citation: "${lines[0].slice(0, 40)}..."`};
+        }
+        return text;
+    } else {
+        return null;
     }
+}
 
-    if (album.trackCoverArtists && album.trackCoverArtists.error) {
-        return {error: `${album.trackCoverArtists.error} (in ${album.name})`};
-    }
+// General function for inputting a single document (usually loaded from YAML)
+// and outputting an instance of a provided Thing subclass.
+//
+// makeProcessDocument is a factory function: the returned function will take a
+// document and apply the configuration passed to makeProcessDocument in order
+// to construct a Thing subclass.
+function makeProcessDocument(thingClass, {
+    // Optional early step for transforming field values before providing them
+    // to the Thing's update() method. This is useful when the input format
+    // (i.e. values in the document) differ from the format the actual Thing
+    // expects.
+    //
+    // Each key and value are a field name (not an update() property) and a
+    // function which takes the value for that field and returns the value which
+    // will be passed on to update().
+    fieldTransformations = {},
 
-    if (!album.coverArtists) {
-        return {error: `The album "${album.name}" is missing the "Cover Art" field.`};
-    }
+    // Mapping of Thing.update() source properties to field names.
+    //
+    // Note this is property -> field, not field -> property. This is a
+    // shorthand convenience because properties are generally typical
+    // camel-cased JS properties, while fields may contain whitespace and be
+    // more easily represented as quoted strings.
+    propertyFieldMapping,
+
+    // Completely ignored fields. These won't throw an unknown field error if
+    // they're present in a document, but they won't be used for Thing property
+    // generation, either. Useful for stuff that's present in data files but not
+    // yet implemented as part of a Thing's data model!
+    ignoredFields = []
+}) {
+    if (!propertyFieldMapping) {
+        throw new Error(`Expected propertyFieldMapping to be provided`);
+    }
+
+    const knownFields = Object.values(propertyFieldMapping);
+
+    // Invert the property-field mapping, since it'll come in handy for
+    // assigning update() source values later.
+    const fieldPropertyMapping = Object.fromEntries(
+        (Object.entries(propertyFieldMapping)
+            .map(([ property, field ]) => [field, property])));
+
+    const decorateErrorWithName = fn => {
+        const nameField = propertyFieldMapping['name'];
+        if (!nameField) return fn;
+
+        return document => {
+            try {
+                return fn(document);
+            } catch (error) {
+                const name = document[nameField];
+                error.message = (name
+                    ? `(name: ${inspect(name)}) ${error.message}`
+                    : `(${color.dim(`no name found`)}) ${error.message}`);
+                throw error;
+            }
+        };
+    };
 
-    album.color = (
-        getBasicField(albumSection, 'Color') ||
-        getBasicField(albumSection, 'FG')
-    );
+    return decorateErrorWithName(document => {
+        const documentEntries = Object.entries(document)
+            .filter(([ field ]) => !ignoredFields.includes(field));
 
-    if (!album.name) {
-        return {error: `Expected "Album" (name) field!`};
-    }
+        const unknownFields = documentEntries
+            .map(([ field ]) => field)
+            .filter(field => !knownFields.includes(field));
 
-    if (!album.date) {
-        return {error: `Expected "Date" field! (in ${album.name})`};
-    }
+        if (unknownFields.length) {
+            throw new makeProcessDocument.UnknownFieldsError(unknownFields);
+        }
 
-    if (!album.dateAdded) {
-        return {error: `Expected "Date Added" field! (in ${album.name})`};
-    }
+        const fieldValues = {};
 
-    if (isNaN(Date.parse(album.date))) {
-        return {error: `Invalid Date field: "${album.date}" (in ${album.name})`};
-    }
+        for (const [ field, value ] of documentEntries) {
+            if (Object.hasOwn(fieldTransformations, field)) {
+                fieldValues[field] = fieldTransformations[field](value);
+            } else {
+                fieldValues[field] = value;
+            }
+        }
 
-    if (isNaN(Date.parse(album.trackArtDate))) {
-        return {error: `Invalid Track Art Date field: "${album.trackArtDate}" (in ${album.name})`};
-    }
+        const sourceProperties = {};
 
-    if (isNaN(Date.parse(album.coverArtDate))) {
-        return {error: `Invalid Cover Art Date field: "${album.coverArtDate}" (in ${album.name})`};
-    }
+        for (const [ field, value ] of Object.entries(fieldValues)) {
+            const property = fieldPropertyMapping[field];
+            sourceProperties[property] = value;
+        }
 
-    if (isNaN(Date.parse(album.dateAdded))) {
-        return {error: `Invalid Date Added field: "${album.dateAdded}" (in ${album.name})`};
-    }
+        const thing = Reflect.construct(thingClass, []);
 
-    album.date = new Date(album.date);
-    album.trackArtDate = new Date(album.trackArtDate);
-    album.coverArtDate = new Date(album.coverArtDate);
-    album.dateAdded = new Date(album.dateAdded);
+        withAggregate({message: `Errors applying ${color.green(thingClass.name)} properties`}, ({ call }) => {
+            for (const [ property, value ] of Object.entries(sourceProperties)) {
+                call(() => (thing[property] = value));
+            }
+        });
 
-    if (!album.directory) {
-        album.directory = getKebabCase(album.name);
-    }
+        return thing;
+    });
+}
 
-    album.tracks = [];
+makeProcessDocument.UnknownFieldsError = class UnknownFieldsError extends Error {
+    constructor(fields) {
+        super(`Unknown fields present: ${fields.join(', ')}`);
+        this.fields = fields;
+    }
+};
 
-    // will be overwritten if a group section is found!
-    album.trackGroups = null;
+const processAlbumDocument = makeProcessDocument(Album, {
+    fieldTransformations: {
+        'Artists': parseContributors,
+        'Cover Artists': parseContributors,
+        'Default Track Cover Artists': parseContributors,
+        'Wallpaper Artists': parseContributors,
+        'Banner Artists': parseContributors,
 
-    let group = null;
-    let trackIndex = 0;
+        'Date': value => new Date(value),
+        'Date Added': value => new Date(value),
+        'Cover Art Date': value => new Date(value),
+        'Default Track Cover Art Date': value => new Date(value),
 
-    for (const section of sections.slice(1)) {
-        // Just skip empty sections. Sometimes I paste a 8unch of dividers,
-        // and this lets the empty sections doing that creates (temporarily)
-        // exist without raising an error.
-        if (!section.filter(Boolean).length) {
-            continue;
-        }
+        'Banner Dimensions': parseDimensions,
+    },
 
-        const groupName = getBasicField(section, 'Group');
-        if (groupName) {
-            group = {
-                name: groupName,
-                color: (
-                    getBasicField(section, 'Color') ||
-                    getBasicField(section, 'FG') ||
-                    album.color
-                ),
-                originalDate: getBasicField(section, 'Original Date'),
-                startIndex: trackIndex,
-                tracks: []
-            };
-            if (group.originalDate) {
-                if (isNaN(Date.parse(group.originalDate))) {
-                    return {error: `The track group "${group.name}" has an invalid "Original Date" field: "${group.originalDate}"`};
-                }
-                group.originalDate = new Date(group.originalDate);
-                group.date = group.originalDate;
+    propertyFieldMapping: {
+        name: 'Album',
+
+        color: 'Color',
+        directory: 'Directory',
+        urls: 'URLs',
+
+        artistContribsByRef: 'Artists',
+        coverArtistContribsByRef: 'Cover Artists',
+        trackCoverArtistContribsByRef: 'Default Track Cover Artists',
+
+        wallpaperArtistContribsByRef: 'Wallpaper Artists',
+        wallpaperStyle: 'Wallpaper Style',
+        wallpaperFileExtension: 'Wallpaper File Extension',
+
+        bannerArtistContribsByRef: 'Banner Artists',
+        bannerStyle: 'Banner Style',
+        bannerFileExtension: 'Banner File Extension',
+        bannerDimensions: 'Banner Dimensions',
+
+        date: 'Date',
+        trackArtDate: 'Default Track Cover Art Date',
+        coverArtDate: 'Cover Art Date',
+        dateAddedToWiki: 'Date Added',
+
+        hasTrackArt: 'Has Track Art',
+        isMajorRelease: 'Major Release',
+        isListedOnHomepage: 'Listed on Homepage',
+
+        aka: 'Also Released As',
+        groupsByRef: 'Groups',
+        artTagsByRef: 'Art Tags',
+        commentary: 'Commentary',
+    }
+});
+
+function processAlbumEntryDocuments(documents) {
+    // Slightly separate meanings: tracks is the array of Track objects (and
+    // only Track objects); trackGroups is the array of TrackGroup objects,
+    // organizing (by string reference) the Track objects within the Album.
+    // tracks is returned for collating with the rest of wiki data; trackGroups
+    // is directly set on the album object.
+    const tracks = [];
+    const trackGroups = [];
+
+    // We can't mutate an array once it's set as a property value, so prepare
+    // the tracks that will show up in a track list all the way before actually
+    // applying it.
+    let currentTracksByRef = null;
+    let currentTrackGroupDoc = null;
+
+    function closeCurrentTrackGroup() {
+        if (currentTracksByRef) {
+            let trackGroup;
+
+            if (currentTrackGroupDoc) {
+                trackGroup = processTrackGroupDocument(currentTrackGroupDoc);
             } else {
-                group.date = album.date;
-            }
-            if (album.trackGroups) {
-                album.trackGroups.push(group);
-            } else {
-                album.trackGroups = [group];
+                trackGroup = new TrackGroup();
+                trackGroup.isDefaultTrackGroup = true;
             }
+
+            trackGroup.tracksByRef = currentTracksByRef;
+            trackGroups.push(trackGroup);
+        }
+    }
+
+    for (const doc of documents) {
+        if (doc['Group']) {
+            closeCurrentTrackGroup();
+            currentTracksByRef = [];
+            currentTrackGroupDoc = doc;
             continue;
         }
 
-        trackIndex++;
+        const track = processTrackDocument(doc);
+        tracks.push(track);
 
-        const track = {};
+        const ref = Thing.getReference(track);
+        if (currentTracksByRef) {
+            currentTracksByRef.push(ref);
+        } else {
+            currentTracksByRef = [ref];
+        }
+    }
 
-        track.name = getBasicField(section, 'Track');
-        track.commentary = getCommentaryField(section);
-        track.lyrics = getMultilineField(section, 'Lyrics');
-        track.originalDate = getBasicField(section, 'Original Date');
-        track.coverArtDate = getBasicField(section, 'Cover Art Date') || track.originalDate || album.trackArtDate;
-        track.references = getListField(section, 'References') || [];
-        track.artists = getContributionField(section, 'Artists') || getContributionField(section, 'Artist');
-        track.coverArtists = getContributionField(section, 'Track Art');
-        track.artTags = getListField(section, 'Art Tags') || [];
-        track.contributors = getContributionField(section, 'Contributors') || [];
-        track.directory = getBasicField(section, 'Directory');
-        track.aka = getBasicField(section, 'AKA');
+    closeCurrentTrackGroup();
 
-        if (!track.name) {
-            return {error: `A track section is missing the "Track" (name) field (in ${album.name}, previous: ${album.tracks[album.tracks.length - 1]?.name}).`};
-        }
+    return {tracks, trackGroups};
+}
 
-        let durationString = getBasicField(section, 'Duration') || '0:00';
-        track.duration = getDurationInSeconds(durationString);
+const processTrackGroupDocument = makeProcessDocument(TrackGroup, {
+    fieldTransformations: {
+        'Date Originally Released': value => new Date(value),
+    },
 
-        if (track.contributors.error) {
-            return {error: `${track.contributors.error} (in ${track.name}, ${album.name})`};
-        }
+    propertyFieldMapping: {
+        name: 'Group',
+        color: 'Color',
+        dateOriginallyReleased: 'Date Originally Released',
+    }
+});
 
-        if (track.commentary && track.commentary.error) {
-            return {error: `${track.commentary.error} (in ${track.name}, ${album.name})`};
-        }
+const processTrackDocument = makeProcessDocument(Track, {
+    fieldTransformations: {
+        'Duration': getDurationInSeconds,
 
-        if (!track.artists) {
-            // If an al8um has an artist specified (usually 8ecause it's a solo
-            // al8um), let tracks inherit that artist. We won't display the
-            // "8y <artist>" string on the al8um listing.
-            if (album.artists) {
-                track.artists = album.artists;
-            } else {
-                return {error: `The track "${track.name}" is missing the "Artist" field (in ${album.name}).`};
-            }
-        }
+        'Date First Released': value => new Date(value),
+        'Cover Art Date': value => new Date(value),
 
-        if (!track.coverArtists) {
-            if (getBasicField(section, 'Track Art') !== 'none' && album.hasTrackArt) {
-                if (album.trackCoverArtists) {
-                    track.coverArtists = album.trackCoverArtists;
-                } else {
-                    return {error: `The track "${track.name}" is missing the "Track Art" field (in ${album.name}).`};
-                }
-            }
-        }
+        'Artists': parseContributors,
+        'Contributors': parseContributors,
+        'Cover Artists': parseContributors,
+    },
 
-        if (track.coverArtists && track.coverArtists.length && track.coverArtists[0] === 'none') {
-            track.coverArtists = null;
-        }
+    propertyFieldMapping: {
+        name: 'Track',
 
-        if (!track.directory) {
-            track.directory = getKebabCase(track.name);
-        }
+        directory: 'Directory',
+        duration: 'Duration',
+        urls: 'URLs',
 
-        if (track.originalDate) {
-            if (isNaN(Date.parse(track.originalDate))) {
-                return {error: `The track "${track.name}"'s has an invalid "Original Date" field: "${track.originalDate}"`};
-            }
-            track.originalDate = new Date(track.originalDate);
-            track.date = new Date(track.originalDate);
-        } else if (group && group.originalDate) {
-            track.originalDate = group.originalDate;
-            track.date = group.originalDate;
-        } else {
-            track.date = album.date;
-        }
+        coverArtDate: 'Cover Art Date',
+        dateFirstReleased: 'Date First Released',
+        hasCoverArt: 'Has Cover Art',
+        hasURLs: 'Has URLs',
 
-        track.coverArtDate = new Date(track.coverArtDate);
+        referencedTracksByRef: 'Referenced Tracks',
+        artistContribsByRef: 'Artists',
+        contributorContribsByRef: 'Contributors',
+        coverArtistContribsByRef: 'Cover Artists',
+        artTagsByRef: 'Art Tags',
+        originalReleaseTrackByRef: 'Originally Released As',
 
-        const hasURLs = getBooleanField(section, 'Has URLs') ?? true;
+        commentary: 'Commentary',
+        lyrics: 'Lyrics'
+    },
 
-        track.urls = hasURLs && (getListField(section, 'URLs') || []).filter(Boolean);
+    ignoredFields: ['Sampled Tracks']
+});
 
-        if (hasURLs && !track.urls.length) {
-            return {error: `The track "${track.name}" should have at least one URL specified.`};
-        }
+const processArtistDocument = makeProcessDocument(Artist, {
+    propertyFieldMapping: {
+        name: 'Artist',
 
-        // 8ack-reference the al8um o8ject! This is very useful for when
-        // we're outputting the track pages.
-        track.album = album;
+        directory: 'Directory',
+        urls: 'URLs',
 
-        if (group) {
-            track.color = group.color;
-            group.tracks.push(track);
-        } else {
-            track.color = album.color;
-        }
+        aliasRefs: 'Aliases',
 
-        album.tracks.push(track);
-    }
+        contextNotes: 'Context Notes'
+    },
 
-    return album;
-}
+    ignoredFields: ['Dead URLs']
+});
 
-async function processArtistDataFile(file) {
-    let contents;
-    try {
-        contents = await readFile(file, 'utf-8');
-    } catch (error) {
-        return {error: `Could not read ${file} (${error.code}).`};
-    }
+const processFlashDocument = makeProcessDocument(Flash, {
+    fieldTransformations: {
+        'Date': value => new Date(value),
 
-    const contentLines = splitLines(contents);
-    const sections = Array.from(getSections(contentLines));
+        'Contributors': parseContributors,
+    },
 
-    return sections.filter(s => s.filter(Boolean).length).map(section => {
-        const name = getBasicField(section, 'Artist');
-        const urls = (getListField(section, 'URLs') || []).filter(Boolean);
-        const alias = getBasicField(section, 'Alias');
-        const hasAvatar = getBooleanField(section, 'Has Avatar') ?? false;
-        const note = getMultilineField(section, 'Note');
-        let directory = getBasicField(section, 'Directory');
+    propertyFieldMapping: {
+        name: 'Flash',
 
-        if (!name) {
-            return {error: 'Expected "Artist" (name) field!'};
-        }
+        directory: 'Directory',
+        page: 'Page',
+        date: 'Date',
+        coverArtFileExtension: 'Cover Art File Extension',
 
-        if (!directory) {
-            directory = getKebabCase(name);
-        }
+        featuredTracksByRef: 'Featured Tracks',
+        contributorContribsByRef: 'Contributors',
+        urls: 'URLs'
+    },
+});
 
-        if (alias) {
-            return {name, directory, alias};
-        } else {
-            return {name, directory, urls, note, hasAvatar};
-        }
-    });
-}
+const processFlashActDocument = makeProcessDocument(FlashAct, {
+    propertyFieldMapping: {
+        name: 'Act',
+        color: 'Color',
+        anchor: 'Anchor',
+        jump: 'Jump',
+        jumpColor: 'Jump Color'
+    }
+});
 
 async function processFlashDataFile(file) {
     let contents;
@@ -1113,101 +1095,43 @@ async function processFlashDataFile(file) {
     });
 }
 
-async function processNewsDataFile(file) {
-    let contents;
-    try {
-        contents = await readFile(file, 'utf-8');
-    } catch (error) {
-        return {error: `Could not read ${file} (${error.code}).`};
-    }
-
-    const contentLines = splitLines(contents);
-    const sections = Array.from(getSections(contentLines));
-
-    return sections.map(section => {
-        const name = getBasicField(section, 'Name');
-        if (!name) {
-            return {error: 'Expected "Name" field!'};
-        }
-
-        const directory = getBasicField(section, 'Directory') || getBasicField(section, 'ID');
-        if (!directory) {
-            return {error: 'Expected "Directory" field!'};
-        }
-
-        let body = getMultilineField(section, 'Body');
-        if (!body) {
-            return {error: 'Expected "Body" field!'};
-        }
-
-        let date = getBasicField(section, 'Date');
-        if (!date) {
-            return {error: 'Expected "Date" field!'};
-        }
-
-        if (isNaN(Date.parse(date))) {
-            return {error: `Invalid date field: "${date}"`};
-        }
-
-        date = new Date(date);
-
-        let bodyShort = body.split('<hr class="split">')[0];
-
-        return {
-            name,
-            directory,
-            body,
-            bodyShort,
-            date
-        };
-    });
-}
+const processNewsEntryDocument = makeProcessDocument(NewsEntry, {
+    fieldTransformations: {
+        'Date': value => new Date(value)
+    },
 
-async function processTagDataFile(file) {
-    let contents;
-    try {
-        contents = await readFile(file, 'utf-8');
-    } catch (error) {
-        if (error.code === 'ENOENT') {
-            return [];
-        } else {
-            return {error: `Could not read ${file} (${error.code}).`};
-        }
+    propertyFieldMapping: {
+        name: 'Name',
+        directory: 'Directory',
+        date: 'Date',
+        content: 'Content',
     }
+});
 
-    const contentLines = splitLines(contents);
-    const sections = Array.from(getSections(contentLines));
-
-    return sections.map(section => {
-        let isCW = false;
-
-        let name = getBasicField(section, 'Tag');
-        if (!name) {
-            name = getBasicField(section, 'CW');
-            isCW = true;
-            if (!name) {
-                return {error: 'Expected "Tag" or "CW" field!'};
-            }
-        }
-
-        let color;
-        if (!isCW) {
-            color = getBasicField(section, 'Color');
-            if (!color) {
-                return {error: 'Expected "Color" field!'};
-            }
-        }
+const processArtTagDocument = makeProcessDocument(ArtTag, {
+    propertyFieldMapping: {
+        name: 'Tag',
+        directory: 'Directory',
+        color: 'Color',
+        isContentWarning: 'Is CW'
+    }
+});
 
-        const directory = getKebabCase(name);
+const processGroupDocument = makeProcessDocument(Group, {
+    propertyFieldMapping: {
+        name: 'Group',
+        directory: 'Directory',
+        description: 'Description',
+        urls: 'URLs',
+    }
+});
 
-        return {
-            name,
-            directory,
-            isCW,
-            color
-        };
-    });
-}
+const processGroupCategoryDocument = makeProcessDocument(GroupCategory, {
+    propertyFieldMapping: {
+        name: 'Category',
+        color: 'Color',
+    }
+});
 
 async function processGroupDataFile(file) {
     let contents;
@@ -1378,75 +1302,61 @@ async function processWikiInfoFile(file) {
     };
 }
 
-async function processHomepageInfoFile(file) {
-    let contents;
-    try {
-        contents = await readFile(file, 'utf-8');
-    } catch (error) {
-        return {error: `Could not read ${file} (${error.code}).`};
-    }
-
-    const contentLines = splitLines(contents);
-    const sections = Array.from(getSections(contentLines));
-
-    const [ firstSection, ...rowSections ] = sections;
-
-    const sidebar = getMultilineField(firstSection, 'Sidebar');
+const processHomepageLayoutDocument = makeProcessDocument(HomepageLayout, {
+    propertyFieldMapping: {
+        sidebarContent: 'Sidebar Content'
+    },
 
-    const validRowTypes = ['albums'];
+    ignoredFields: ['Homepage']
+});
 
-    const rows = rowSections.map(section => {
-        const name = getBasicField(section, 'Row');
-        if (!name) {
-            return {error: 'Expected "Row" (name) field!'};
-        }
+const homepageLayoutRowBaseSpec = {
+};
 
-        const color = getBasicField(section, 'Color');
+const makeProcessHomepageLayoutRowDocument = (rowClass, spec) => makeProcessDocument(rowClass, {
+    ...spec,
 
-        const type = getBasicField(section, 'Type');
-        if (!type) {
-            return {error: 'Expected "Type" field!'};
-        }
+    propertyFieldMapping: {
+        name: 'Row',
+        color: 'Color',
+        type: 'Type',
+        ...spec.propertyFieldMapping,
+    }
+});
 
-        if (!validRowTypes.includes(type)) {
-            return {error: `Expected "Type" field to be one of: ${validRowTypes.join(', ')}`};
+const homepageLayoutRowTypeProcessMapping = {
+    albums: makeProcessHomepageLayoutRowDocument(HomepageLayoutAlbumsRow, {
+        propertyFieldMapping: {
+            sourceGroupByRef: 'Group',
+            countAlbumsFromGroup: 'Count',
+            sourceAlbumsByRef: 'Albums',
+            actionLinks: 'Actions'
         }
+    })
+};
 
-        const row = {name, color, type};
-
-        switch (type) {
-            case 'albums': {
-                const group = getBasicField(section, 'Group') || null;
-                const albums = getListField(section, 'Albums') || [];
-
-                if (!group && !albums) {
-                    return {error: 'Expected "Group" and/or "Albums" field!'};
-                }
-
-                let groupCount = getBasicField(section, 'Count');
-                if (group && !groupCount) {
-                    return {error: 'Expected "Count" field!'};
-                }
-
-                if (groupCount) {
-                    if (isNaN(parseInt(groupCount))) {
-                        return {error: `Invalid Count field: "${groupCount}"`};
-                    }
+function processHomepageLayoutRowDocument(document) {
+    const type = document['Type'];
 
-                    groupCount = parseInt(groupCount);
-                }
+    const match = Object.entries(homepageLayoutRowTypeProcessMapping)
+        .find(([ key ]) => key === type);
 
-                const actions = getListField(section, 'Actions') || [];
-
-                return {...row, group, groupCount, albums, actions};
-            }
-        }
-    });
+    if (!match) {
+        throw new TypeError(`No processDocument function for row type ${type}!`);
+    }
 
-    return {sidebar, rows};
+    return match[1](document);
 }
 
 function getDurationInSeconds(string) {
+    if (typeof string === 'number') {
+        return string;
+    }
+
+    if (typeof string !== 'string') {
+        throw new TypeError(`Expected a string or number, got ${string}`);
+    }
+
     const parts = string.split(':').map(n => parseInt(n))
     if (parts.length === 3) {
         return parts[0] * 3600 + parts[1] * 60 + parts[2]
@@ -1726,15 +1636,35 @@ writePage.to = ({
 }) => (targetFullKey, ...args) => {
     const [ groupKey, subKey ] = targetFullKey.split('.');
     let path = paths.subdirectoryPrefix;
+
+    let from;
+    let to;
+
     // When linking to *outside* the localized area of the site, we need to
     // make sure the result is correctly relative to the 8ase directory.
-    if (groupKey !== 'localized' && baseDirectory) {
-        path += urls.from('localizedWithBaseDirectory.' + pageSubKey).to(targetFullKey, ...args);
+    if (groupKey !== 'localized' && groupKey !== 'localizedDefaultLanguage' && baseDirectory) {
+        from = 'localizedWithBaseDirectory.' + pageSubKey;
+        to = targetFullKey;
+    } else if (groupKey === 'localizedDefaultLanguage' && baseDirectory) {
+        // Special case for specifically linking *from* a page with base
+        // directory *to* a page without! Used for the language switcher and
+        // hopefully nothing else oh god.
+        from = 'localizedWithBaseDirectory.' + pageSubKey;
+        to = 'localized.' + subKey;
+    } else if (groupKey === 'localizedDefaultLanguage') {
+        // Linking to the default, except surprise, we're already IN the default
+        // (no baseDirectory set).
+        from = 'localized.' + pageSubKey;
+        to = 'localized.' + subKey;
     } else {
         // If we're linking inside the localized area (or there just is no
         // 8ase directory), the 8ase directory doesn't matter.
-        path += urls.from('localized.' + pageSubKey).to(targetFullKey, ...args);
+        from = 'localized.' + pageSubKey;
+        to = targetFullKey;
     }
+
+    path += urls.from(from).to(to, ...args);
+
     return path;
 };
 
@@ -1792,8 +1722,12 @@ writePage.html = (pageFn, {
     footer.classes ??= [];
     footer.content ??= (wikiInfo.footer ? transformMultiline(wikiInfo.footer) : '');
 
+    footer.content += '\n' + getFooterLocalizationLinks(paths.pathname, {
+        languages, paths, strings, to
+    });
+
     const canonical = (wikiInfo.canonicalBase
-        ? wikiInfo.canonicalBase + (paths.pathname === '/' ? '' : paths.pathanme)
+        ? wikiInfo.canonicalBase + (paths.pathname === '/' ? '' : paths.pathname)
         : '');
 
     const collapseSidebars = (sidebarLeft.collapse !== false) && (sidebarRight.collapse !== false);
@@ -1876,10 +1810,10 @@ writePage.html = (pageFn, {
                     cur.toCurrentPage ? '' :
                     cur.toHome ? to('localized.home') :
                     cur.path ? to(...cur.path) :
-                    cur.href ? call(() => {
+                    cur.href ? (() => {
                         logWarn`Using legacy href format nav link in ${paths.pathname}`;
                         return cur.href;
-                    }) :
+                    })() :
                     null)
             };
             if (attributes.href === null) {
@@ -2035,6 +1969,7 @@ writePage.paths = (baseDirectory, fullKey, directory = '', {
     const outputFile = path.join(outputDirectory, file);
 
     return {
+        toPath: [fullKey, directory],
         pathname,
         subdirectoryPrefix,
         outputDirectory, outputFile
@@ -2353,6 +2288,7 @@ async function main() {
         logInfo`Writing all languages.`;
     }
 
+    /*
     WD.wikiInfo = await processWikiInfoFile(path.join(dataPath, WIKI_INFO_FILE));
     if (WD.wikiInfo.error) {
         console.log(`\x1b[31;1m${WD.wikiInfo.error}\x1b[0m`);
@@ -2377,23 +2313,7 @@ async function main() {
     } else {
         languages.default = defaultStrings;
     }
-
-    WD.homepageInfo = await processHomepageInfoFile(path.join(dataPath, HOMEPAGE_INFO_FILE));
-
-    if (WD.homepageInfo.error) {
-        console.log(`\x1b[31;1m${WD.homepageInfo.error}\x1b[0m`);
-        return;
-    }
-
-    {
-        const errors = WD.homepageInfo.rows.filter(obj => obj.error);
-        if (errors.length) {
-            for (const error of errors) {
-                console.log(`\x1b[31;1m${error.error}\x1b[0m`);
-            }
-            return;
-        }
-    }
+    */
 
     // 8ut wait, you might say, how do we know which al8um these data files
     // correspond to???????? You wouldn't dare suggest we parse the actual
@@ -2419,101 +2339,405 @@ async function main() {
     // avoiding that in our code 8ecause, again, we want to avoid assuming the
     // format of the returned paths here - they're only meant to 8e used for
     // reading as-is.
-    const albumDataFiles = await findFiles(path.join(dataPath, DATA_ALBUM_DIRECTORY));
+    const albumDataFiles = await findFiles(path.join(dataPath, DATA_ALBUM_DIRECTORY), f => path.extname(f) === '.yaml');
 
-    // Technically, we could do the data file reading and output writing at the
-    // same time, 8ut that kinda makes the code messy, so I'm not 8othering
-    // with it.
-    WD.albumData = await progressPromiseAll(`Reading & processing album files.`, albumDataFiles.map(processAlbumDataFile));
+    const documentModes = {
+        onePerFile: Symbol('Document mode: One per file'),
+        headerAndEntries: Symbol('Document mode: Header and entries'),
+        allInOne: Symbol('Document mode: All in one')
+    };
 
-    {
-        const errors = WD.albumData.filter(obj => obj.error);
-        if (errors.length) {
-            for (const error of errors) {
-                console.log(`\x1b[31;1m${error.error}\x1b[0m`);
+    const dataSteps = [
+        {
+            title: `Process album files`,
+            files: albumDataFiles,
+
+            documentMode: documentModes.headerAndEntries,
+            processHeaderDocument: processAlbumDocument,
+            processEntryDocument(document) {
+                return ('Group' in document
+                    ? processTrackGroupDocument(document)
+                    : processTrackDocument(document));
+            },
+
+            // processEntryDocuments: processAlbumEntryDocuments,
+
+            save(results) {
+                const albumData = [];
+                const trackData = [];
+
+                for (const { header: album, entries } of results) {
+                    // We can't mutate an array once it's set as a property
+                    // value, so prepare the tracks and track groups that will
+                    // show up in a track list all the way before actually
+                    // applying them.
+                    const trackGroups = [];
+                    let currentTracksByRef = null;
+                    let currentTrackGroup = null;
+
+                    function closeCurrentTrackGroup() {
+                        if (currentTracksByRef) {
+                            let trackGroup;
+
+                            if (currentTrackGroup) {
+                                trackGroup = currentTrackGroup;
+                            } else {
+                                trackGroup = new TrackGroup();
+                                trackGroup.name = `Default Track Group`;
+                                trackGroup.isDefaultTrackGroup = true;
+                            }
+
+                            trackGroup.tracksByRef = currentTracksByRef;
+                            trackGroups.push(trackGroup);
+                        }
+                    }
+
+                    for (const entry of entries) {
+                        if (entry instanceof TrackGroup) {
+                            closeCurrentTrackGroup();
+                            currentTracksByRef = [];
+                            currentTrackGroup = entry;
+                            continue;
+                        }
+
+                        trackData.push(entry);
+
+                        const ref = Thing.getReference(entry);
+                        if (currentTracksByRef) {
+                            currentTracksByRef.push(ref);
+                        } else {
+                            currentTracksByRef = [ref];
+                        }
+                    }
+
+                    closeCurrentTrackGroup();
+
+                    album.trackGroups = trackGroups;
+                    albumData.push(album);
+                }
+
+                sortByDate(albumData);
+                sortByDate(trackData);
+
+                Object.assign(wikiData, {albumData, trackData});
             }
-            return;
-        }
-    }
+        },
 
-    sortByDate(WD.albumData);
+        {
+            title: `Process artists file`,
+            files: [path.join(dataPath, ARTIST_DATA_FILE)],
 
-    WD.artistData = await processArtistDataFile(path.join(dataPath, ARTIST_DATA_FILE));
-    if (WD.artistData.error) {
-        console.log(`\x1b[31;1m${WD.artistData.error}\x1b[0m`);
-        return;
-    }
+            documentMode: documentModes.allInOne,
+            processDocument: processArtistDocument,
 
-    {
-        const errors = WD.artistData.filter(obj => obj.error);
-        if (errors.length) {
-            for (const error of errors) {
-                console.log(`\x1b[31;1m${error.error}\x1b[0m`);
+            save(results) {
+                wikiData.artistData = results;
             }
-            return;
-        }
-    }
+        },
+
+        // TODO: WD.wikiInfo.features.flashesAndGames &&
+        {
+            title: `Process flashes file`,
+            files: [path.join(dataPath, FLASH_DATA_FILE)],
+
+            documentMode: documentModes.allInOne,
+            processDocument(document) {
+                return ('Act' in document
+                    ? processFlashActDocument(document)
+                    : processFlashDocument(document));
+            },
+
+            save(results) {
+                let flashAct;
+                let flashesByRef = [];
+
+                if (results[0] && !(results[0] instanceof FlashAct)) {
+                    throw new Error(`Expected an act at top of flash data file`);
+                }
 
-    WD.artistAliasData = WD.artistData.filter(x => x.alias);
-    WD.artistData = WD.artistData.filter(x => !x.alias);
+                for (const thing of results) {
+                    if (thing instanceof FlashAct) {
+                        if (flashAct) {
+                            Object.assign(flashAct, {flashesByRef});
+                        }
 
-    WD.trackData = getAllTracks(WD.albumData);
+                        flashAct = thing;
+                        flashesByRef = [];
+                    } else {
+                        flashesByRef.push(Thing.getReference(thing));
+                    }
+                }
 
-    if (WD.wikiInfo.features.flashesAndGames) {
-        WD.flashData = await processFlashDataFile(path.join(dataPath, FLASH_DATA_FILE));
-        if (WD.flashData.error) {
-            console.log(`\x1b[31;1m${WD.flashData.error}\x1b[0m`);
-            return;
-        }
+                if (flashAct) {
+                    Object.assign(flashAct, {flashesByRef});
+                }
 
-        const errors = WD.flashData.filter(obj => obj.error);
-        if (errors.length) {
-            for (const error of errors) {
-                console.log(`\x1b[31;1m${error.error}\x1b[0m`);
+                wikiData.flashData = results.filter(x => x instanceof Flash);
+                wikiData.flashActData = results.filter(x => x instanceof FlashAct);
             }
-            return;
-        }
-    }
+        },
 
-    WD.flashActData = WD.flashData?.filter(x => x.act8r8k);
-    WD.flashData = WD.flashData?.filter(x => !x.act8r8k);
+        {
+            title: `Process groups file`,
+            files: [path.join(dataPath, GROUP_DATA_FILE)],
+
+            documentMode: documentModes.allInOne,
+            processDocument(document) {
+                return ('Category' in document
+                    ? processGroupCategoryDocument(document)
+                    : processGroupDocument(document));
+            },
+
+            save(results) {
+                let groupCategory;
+                let groupsByRef = [];
+
+                if (results[0] && !(results[0] instanceof GroupCategory)) {
+                    throw new Error(`Expected a category at top of group data file`);
+                }
 
-    WD.tagData = await processTagDataFile(path.join(dataPath, TAG_DATA_FILE));
-    if (WD.tagData.error) {
-        console.log(`\x1b[31;1m${WD.tagData.error}\x1b[0m`);
-        return;
+                for (const thing of results) {
+                    if (thing instanceof GroupCategory) {
+                        if (groupCategory) {
+                            Object.assign(groupCategory, {groupsByRef});
+                        }
+
+                        groupCategory = thing;
+                        groupsByRef = [];
+                    } else {
+                        groupsByRef.push(Thing.getReference(thing));
+                    }
+                }
+
+                if (groupCategory) {
+                    Object.assign(groupCategory, {groupsByRef});
+                }
+
+                wikiData.groupData = results.filter(x => x instanceof Group);
+                wikiData.groupCategoryData = results.filter(x => x instanceof GroupCategory);
+            }
+        },
+
+        {
+            title: `Process homepage layout file`,
+            files: [path.join(dataPath, HOMEPAGE_LAYOUT_DATA_FILE)],
+
+            documentMode: documentModes.headerAndEntries,
+            processHeaderDocument: processHomepageLayoutDocument,
+            processEntryDocument: processHomepageLayoutRowDocument,
+
+            save(results) {
+                if (!results[0]) {
+                    return;
+                }
+
+                const { header: homepageLayout, entries: rows } = results[0];
+                Object.assign(homepageLayout, {rows});
+                Object.assign(wikiData, {homepageLayout});
+            }
+        },
+
+        // TODO: WD.wikiInfo.features.news &&
+        {
+            title: `Process news data file`,
+            files: [path.join(dataPath, NEWS_DATA_FILE)],
+
+            documentMode: documentModes.allInOne,
+            processDocument: processNewsEntryDocument,
+
+            save(results) {
+                sortByDate(results);
+                results.reverse();
+
+                wikiData.newsData = results;
+            }
+        },
+
+        {
+            title: `Process art tags file`,
+            files: [path.join(dataPath, ART_TAG_DATA_FILE)],
+
+            documentMode: documentModes.allInOne,
+            processDocument: processArtTagDocument,
+
+            save(results) {
+                results.sort(sortByName);
+
+                wikiData.tagData = results;
+            }
+        },
+    ];
+
+    const processDataAggregate = openAggregate({message: `Errors processing data files`});
+
+    function decorateErrorWithFile(fn) {
+        return (x, index, array) => {
+            try {
+                return fn(x, index, array);
+            } catch (error) {
+                error.message += (
+                    (error.message.includes('\n') ? '\n' : ' ') +
+                    `(file: ${color.bright(color.blue(path.relative(dataPath, x.file)))})`
+                );
+                throw error;
+            }
+        };
     }
 
-    {
-        const errors = WD.tagData.filter(obj => obj.error);
-        if (errors.length) {
-            for (const error of errors) {
-                console.log(`\x1b[31;1m${error.error}\x1b[0m`);
+    function decorateErrorWithIndex(fn) {
+        return (x, index, array) => {
+            try {
+                return fn(x, index, array);
+            } catch (error) {
+                error.message = `(${color.yellow(`#${index + 1}`)}) ${error.message}`;
+                throw error;
             }
-            return;
         }
     }
 
-    WD.tagData.sort(sortByName);
+    for (const dataStep of dataSteps) {
+        await processDataAggregate.nestAsync(
+            {message: `Errors during data step: ${dataStep.title}`},
+            async ({call, callAsync, map, mapAsync, nest}) => {
+                const { documentMode } = dataStep;
 
-    WD.groupData = await processGroupDataFile(path.join(dataPath, GROUP_DATA_FILE));
-    if (WD.groupData.error) {
-        console.log(`\x1b[31;1m${WD.groupData.error}\x1b[0m`);
-        return;
+                if (!(Object.values(documentModes).includes(documentMode))) {
+                    throw new Error(`Invalid documentMode: ${documentMode.toString()}`);
+                }
+
+                if (documentMode === documentModes.allInOne) {
+                    if (dataStep.files.length !== 1) {
+                        throw new Error(`Expected 1 file for all-in-one documentMode, not ${files.length}`);
+                    }
+
+                    const file = dataStep.files[0];
+
+                    const readResult = await callAsync(readFile, file);
+
+                    if (!readResult) {
+                        return;
+                    }
+
+                    const yamlResult = call(yaml.loadAll, readResult);
+
+                    if (!yamlResult) {
+                        return;
+                    }
+
+                    const {
+                        result: processResults,
+                        aggregate: processAggregate
+                    } = mapAggregate(
+                        yamlResult,
+                        decorateErrorWithIndex(dataStep.processDocument),
+                        {message: `Errors processing documents`}
+                    );
+
+                    call(processAggregate.close);
+
+                    dataStep.save(processResults);
+
+                    return;
+                }
+
+                const readResults = await mapAsync(
+                    dataStep.files,
+                    file => (readFile(file, 'utf-8')
+                        .then(contents => ({file, contents}))),
+                    {
+                        message: `Errors reading data files`,
+                        promiseAll: array => progressPromiseAll(`Data step: ${dataStep.title} (reading data files)`, array)
+                    });
+
+                const yamlResults = map(
+                    readResults,
+                    decorateErrorWithFile(
+                        ({ file, contents }) => ({file, documents: yaml.loadAll(contents)})),
+                    {message: `Errors parsing data files as valid YAML`});
+
+                let processResults;
+
+                if (documentMode === documentModes.headerAndEntries) {
+                    nest({message: `Errors processing data files as valid documents`}, ({ call, map }) => {
+                        processResults = [];
+
+                        yamlResults.forEach(({ file, documents }) => {
+                            const [ headerDocument, ...entryDocuments ] = documents;
+
+                            const header = call(
+                                decorateErrorWithFile(
+                                    ({ document }) => dataStep.processHeaderDocument(document)),
+                                {file, document: headerDocument});
+
+                            // Don't continue processing files whose header
+                            // document is invalid - the entire file is excempt
+                            // from data in this case.
+                            if (!header) {
+                                return;
+                            }
+
+                            const entries = map(
+                                entryDocuments.map(document => ({file, document})),
+                                decorateErrorWithFile(
+                                    decorateErrorWithIndex(
+                                        ({ document }) => dataStep.processEntryDocument(document))),
+                                {message: `Errors processing entry documents`});
+
+                            // Entries may be incomplete (i.e. any errored
+                            // documents won't have a processed output
+                            // represented here) - this is intentional! By
+                            // principle, partial output is preferred over
+                            // erroring an entire file.
+                            processResults.push({header, entries});
+                        });
+                    });
+                }
+
+                if (documentMode === documentModes.onePerFile) {
+                    throw new Error('TODO: onePerFile not yet implemented');
+                }
+
+                dataStep.save(processResults);
+            });
     }
 
     {
-        const errors = WD.groupData.filter(obj => obj.error);
-        if (errors.length) {
-            for (const error of errors) {
-                console.log(`\x1b[31;1m${error.error}\x1b[0m`);
-            }
-            return;
+        try {
+            logInfo`Loaded data and processed objects:`;
+            logInfo` - ${wikiData.albumData.length} albums`;
+            logInfo` - ${wikiData.trackData.length} tracks`;
+            logInfo` - ${wikiData.artistData.length} artists`;
+            if (wikiData.flashData)
+                logInfo` - ${wikiData.flashData.length} flashes (${wikiData.flashActData.length} acts)`;
+            logInfo` - ${wikiData.groupData.length} groups (${wikiData.groupCategoryData.length} categories)`;
+            logInfo` - ${wikiData.tagData.length} art tags`;
+            if (wikiData.newsData)
+                logInfo` - ${wikiData.newsData.length} news entries`;
+            if (wikiData.homepageLayout)
+                logInfo` - ${1} homepage layout (${wikiData.homepageLayout.rows.length} rows)`;
+        } catch (error) {
+            console.error(`Error showing data summary:`, error);
+        }
+
+        let errorless = true;
+        try {
+            processDataAggregate.close();
+        } catch (error) {
+            showAggregate(error, {pathToFile: f => path.relative(__dirname, f)});
+            logWarn`The above errors were detected while processing data files.`;
+            logWarn`If the remaining valid data is complete enough, the wiki will`;
+            logWarn`still build - but all errored data will be skipped.`;
+            logWarn`(Resolve errors for more complete output!)`;
+            errorless = false;
+        }
+
+        if (errorless) {
+            logInfo`All data processed without any errors - nice!`;
+            logInfo`(This means all source files will be fully accounted for during page generation.)`;
         }
     }
 
-    WD.groupCategoryData = WD.groupData.filter(x => x.isCategory);
-    WD.groupData = WD.groupData.filter(x => x.isGroup);
+    process.exit();
 
     WD.staticPageData = await processStaticPageDataFile(path.join(dataPath, STATIC_PAGE_DATA_FILE));
     if (WD.staticPageData.error) {
@@ -2531,25 +2755,6 @@ async function main() {
         }
     }
 
-    if (WD.wikiInfo.features.news) {
-        WD.newsData = await processNewsDataFile(path.join(dataPath, NEWS_DATA_FILE));
-        if (WD.newsData.error) {
-            console.log(`\x1b[31;1m${WD.newsData.error}\x1b[0m`);
-            return;
-        }
-
-        const errors = WD.newsData.filter(obj => obj.error);
-        if (errors.length) {
-            for (const error of errors) {
-                console.log(`\x1b[31;1m${error.error}\x1b[0m`);
-            }
-            return;
-        }
-
-        sortByDate(WD.newsData);
-        WD.newsData.reverse();
-    }
-
     {
         const tagNames = new Set([...WD.trackData, ...WD.albumData].flatMap(thing => thing.artTags));