flash data

7 files changed, 385 insertions, 115 deletions

diff --git a/src/thing/album.js b/src/thing/album.js
index 11af801..8a9fde2 100644
--- a/src/thing/album.js
+++ b/src/thing/album.js
@@ -10,6 +10,7 @@ import {
     isDate,
     isDimensions,
     isDirectory,
+    isFileExtension,
     isName,
     isURL,
     isString,
@@ -176,7 +177,7 @@ export default class Album extends Thing {
 
         wallpaperFileExtension: {
             flags: {update: true, expose: true},
-            update: {validate: isString}
+            update: {validate: isFileExtension}
         },
 
         bannerStyle: {
@@ -186,7 +187,7 @@ export default class Album extends Thing {
 
         bannerFileExtension: {
             flags: {update: true, expose: true},
-            update: {validate: isString}
+            update: {validate: isFileExtension}
         },
 
         bannerDimensions: {
diff --git a/src/thing/cacheable-object.js b/src/thing/cacheable-object.js
index f478fd2..3c14101 100644
--- a/src/thing/cacheable-object.js
+++ b/src/thing/cacheable-object.js
@@ -214,11 +214,13 @@ export default class CacheableObject {
     #getExposeComputeFunction(property) {
         const { flags, expose } = this.#getPropertyDescriptor(property);
 
-        const compute = (!flags.update && expose?.compute);
-        const transform = (flags.update && expose?.transform);
+        const compute = expose?.compute;
+        const transform = expose?.transform;
 
         if (flags.update && !transform) {
             return null;
+        } else if (flags.update && compute) {
+            throw new Error(`Updating property ${property} has compute function, should be formatted as transform`);
         } else if (!flags.update && !compute) {
             throw new Error(`Exposed property ${property} does not update and is missing compute function`);
         }
diff --git a/src/thing/flash.js b/src/thing/flash.js
new file mode 100644
index 0000000..4eac65a
--- /dev/null
+++ b/src/thing/flash.js
@@ -0,0 +1,129 @@
+import Thing from './thing.js';
+
+import {
+    isColor,
+    isContributionList,
+    isDate,
+    isDirectory,
+    isFileExtension,
+    isName,
+    isNumber,
+    isString,
+    isURL,
+    oneOf,
+    validateArrayItems,
+    validateReferenceList,
+} from './validators.js';
+
+export default class Flash extends Thing {
+    static [Thing.referenceType] = 'flash';
+
+    static propertyDescriptors = {
+        // Update & expose
+
+        name: {
+            flags: {update: true, expose: true},
+
+            update: {
+                default: 'Unnamed Flash',
+                validate: isName
+            }
+        },
+
+        directory: {
+            flags: {update: true, expose: true},
+            update: {validate: isDirectory},
+
+            // Flashes expose directory differently from other Things! Their
+            // default directory is dependent on the page number (or ID), not
+            // the name.
+            expose: {
+                dependencies: ['page'],
+                transform(directory, { page }) {
+                    if (directory === null && page === null)
+                        return null;
+                    else if (directory === null)
+                        return page;
+                    else
+                        return directory;
+                }
+            }
+        },
+
+        page: {
+            flags: {update: true, expose: true},
+            update: {validate: oneOf(isString, isNumber)},
+
+            expose: {
+                transform: value => value.toString()
+            }
+        },
+
+        date: {
+            flags: {update: true, expose: true},
+            update: {validate: isDate}
+        },
+
+        coverArtFileExtension: {
+            flags: {update: true, expose: true},
+            update: {validate: isFileExtension}
+        },
+
+        featuredTracksByRef: {
+            flags: {update: true, expose: true},
+            update: {validate: validateReferenceList('track')}
+        },
+
+        contributorContribsByRef: {
+            flags: {update: true, expose: true},
+            update: {validate: isContributionList}
+        },
+
+        urls: {
+            flags: {update: true, expose: true},
+            update: {validate: validateArrayItems(isURL)}
+        },
+    };
+}
+
+export class FlashAct extends Thing {
+    static [Thing.referenceType] = 'flash-act';
+
+    static propertyDescriptors = {
+        // Update & expose
+
+        name: {
+            flags: {update: true, expose: true},
+
+            update: {
+                default: 'Unnamed Flash Act',
+                validate: isName
+            }
+        },
+
+        color: {
+            flags: {update: true, expose: true},
+            update: {validate: isColor}
+        },
+
+        anchor: {
+            flags: {update: true, expose: true},
+            update: {validate: isString}
+        },
+
+        jump: {
+            flags: {update: true, expose: true},
+            update: {validate: isString}
+        },
+
+        jumpColor: {
+            flags: {update: true, expose: true},
+            update: {validate: isColor}
+        },
+
+        flashesByRef: {
+            flags: {update: true, expose: true},
+            update: {validate: validateReferenceList('flash')}
+        },
+    };
+}
diff --git a/src/thing/validators.js b/src/thing/validators.js
index e745771..a465e9d 100644
--- a/src/thing/validators.js
+++ b/src/thing/validators.js
@@ -222,6 +222,18 @@ export function isDuration(duration) {
     return true;
 }
 
+export function isFileExtension(string) {
+    isStringNonEmpty(string);
+
+    if (string[0] === '.')
+        throw new TypeError(`Expected no dot (.) at the start of file extension`);
+
+    if (string.match(/[^a-zA-Z0-9_]/))
+        throw new TypeError(`Expected only alphanumeric and underscore`);
+
+    return true;
+}
+
 export function isName(name) {
     return isString(name);
 }
@@ -260,3 +272,36 @@ export function validateReference(type = 'track') {
 export function validateReferenceList(type = '') {
     return validateArrayItems(validateReference(type));
 }
+
+// Compositional utilities
+
+export function oneOf(...checks) {
+    return value => {
+        const errorMeta = [];
+
+        for (let i = 0, check; check = checks[i]; i++) {
+            try {
+                const result = check(value);
+
+                if (result !== true) {
+                    throw new Error(`Check returned false`);
+                }
+
+                return true;
+            } catch (error) {
+                errorMeta.push([check, i, error]);
+            }
+        }
+
+        // Don't process error messages until every check has failed.
+        const errors = [];
+        for (const [ check, i, error ] of errorMeta) {
+            error.message = (check.name
+                ? `(#${i} "${check.name}") ${error.message}`
+                : `(#${i}) ${error.message}`);
+            error.check = check;
+            errors.push(error);
+        }
+        throw new AggregateError(errors, `Expected one of ${checks.length} possible checks, but none were true`);
+    };
+}
diff --git a/src/upd8.js b/src/upd8.js
index 6df3cc2..0292e60 100755
--- a/src/upd8.js
+++ b/src/upd8.js
@@ -80,6 +80,8 @@ import {
     unlink
 } from 'fs/promises';
 
+import { inspect as nodeInspect } from 'util';
+
 import genThumbs from './gen-thumbs.js';
 import { listingSpec, listingTargetSpec } from './listing-spec.js';
 import urlSpec from './url-spec.js';
@@ -91,6 +93,7 @@ import unbound_link, {getLinkThemeString} from './util/link.js';
 
 import Album, { TrackGroup } from './thing/album.js';
 import Artist from './thing/artist.js';
+import Flash, { FlashAct } from './thing/flash.js';
 import Thing from './thing/thing.js';
 import Track from './thing/track.js';
 
@@ -120,7 +123,8 @@ import {
     logInfo,
     logError,
     parseOptions,
-    progressPromiseAll
+    progressPromiseAll,
+    ENABLE_COLOR
 } from './util/cli.js';
 
 import {
@@ -195,7 +199,7 @@ const CACHEBUST = 7;
 const WIKI_INFO_FILE = 'wiki-info.txt';
 const HOMEPAGE_INFO_FILE = 'homepage.txt';
 const ARTIST_DATA_FILE = 'artists.yaml';
-const FLASH_DATA_FILE = 'flashes.txt';
+const FLASH_DATA_FILE = 'flashes.yaml';
 const NEWS_DATA_FILE = 'news.txt';
 const TAG_DATA_FILE = 'tags.txt';
 const GROUP_DATA_FILE = 'groups.txt';
@@ -219,6 +223,10 @@ const STATIC_DIRECTORY = 'static';
 // read from and processed to compose the majority of album and track data.
 const DATA_ALBUM_DIRECTORY = 'album';
 
+function inspect(value) {
+    return nodeInspect(value, {colors: ENABLE_COLOR});
+}
+
 // Shared varia8les! These are more efficient to access than a shared varia8le
 // (or at least I h8pe so), and are easier to pass across functions than a
 // 8unch of specific arguments.
@@ -713,10 +721,10 @@ function parseCommentary(text) {
 // General function for inputting a single document (usually loaded from YAML)
 // and outputting an instance of a provided Thing subclass.
 //
-// makeParseDocument is a factory function: the returned function will take a
-// document and apply the configuration passed to makeParseDocument in order to
-// construct a Thing subclass.
-function makeParseDocument(thingClass, {
+// makeProcessDocument is a factory function: the returned function will take a
+// document and apply the configuration passed to makeProcessDocument in order
+// to construct a Thing subclass.
+function makeProcessDocument(thingClass, {
     // Optional early step for transforming field values before providing them
     // to the Thing's update() method. This is useful when the input format
     // (i.e. values in the document) differ from the format the actual Thing
@@ -749,7 +757,24 @@ function makeParseDocument(thingClass, {
         (Object.entries(propertyFieldMapping)
             .map(([ property, field ]) => [field, property])));
 
-    return function(document) {
+    const decorateErrorWithName = fn => {
+        const nameField = propertyFieldMapping['name'];
+        if (!nameField) return fn;
+
+        return document => {
+            try {
+                return fn(document);
+            } catch (error) {
+                const name = document[nameField];
+                error.message = (name
+                    ? `(name: ${inspect(name)}) ${error.message}`
+                    : `(${color.dim(`no name found`)}) ${error.message}`);
+                throw error;
+            }
+        };
+    };
+
+    return decorateErrorWithName(document => {
         const documentEntries = Object.entries(document)
             .filter(([ field ]) => !ignoredFields.includes(field));
 
@@ -758,7 +783,7 @@ function makeParseDocument(thingClass, {
             .filter(field => !knownFields.includes(field));
 
         if (unknownFields.length) {
-            throw new makeParseDocument.UnknownFieldsError(unknownFields);
+            throw new makeProcessDocument.UnknownFieldsError(unknownFields);
         }
 
         const fieldValues = {};
@@ -789,17 +814,17 @@ function makeParseDocument(thingClass, {
         });
 
         return thing;
-    };
+    });
 }
 
-makeParseDocument.UnknownFieldsError = class UnknownFieldsError extends Error {
+makeProcessDocument.UnknownFieldsError = class UnknownFieldsError extends Error {
     constructor(fields) {
         super(`Unknown fields present: ${fields.join(', ')}`);
         this.fields = fields;
     }
 };
 
-const parseAlbumDocument = makeParseDocument(Album, {
+const processAlbumDocument = makeProcessDocument(Album, {
     fieldTransformations: {
         'Artists': parseContributors,
         'Cover Artists': parseContributors,
@@ -851,7 +876,7 @@ const parseAlbumDocument = makeParseDocument(Album, {
     }
 });
 
-function parseAlbumEntryDocuments(documents) {
+function processAlbumEntryDocuments(documents) {
     // Slightly separate meanings: tracks is the array of Track objects (and
     // only Track objects); trackGroups is the array of TrackGroup objects,
     // organizing (by string reference) the Track objects within the Album.
@@ -906,7 +931,7 @@ function parseAlbumEntryDocuments(documents) {
     return {tracks, trackGroups};
 }
 
-const parseTrackGroupDocument = makeParseDocument(TrackGroup, {
+const parseTrackGroupDocument = makeProcessDocument(TrackGroup, {
     fieldTransformations: {
         'Date Originally Released': value => new Date(value),
     },
@@ -918,7 +943,7 @@ const parseTrackGroupDocument = makeParseDocument(TrackGroup, {
     }
 });
 
-const parseTrackDocument = makeParseDocument(Track, {
+const parseTrackDocument = makeProcessDocument(Track, {
     fieldTransformations: {
         'Duration': getDurationInSeconds,
 
@@ -956,7 +981,7 @@ const parseTrackDocument = makeParseDocument(Track, {
     ignoredFields: ['Sampled Tracks']
 });
 
-const processArtistDocument = makeParseDocument(Artist, {
+const processArtistDocument = makeProcessDocument(Artist, {
     propertyFieldMapping: {
         name: 'Artist',
 
@@ -971,40 +996,36 @@ const processArtistDocument = makeParseDocument(Artist, {
     ignoredFields: ['Dead URLs']
 });
 
-async function processArtistDataFile(file) {
-    let contents;
-    try {
-        contents = await readFile(file, 'utf-8');
-    } catch (error) {
-        return {error: `Could not read ${file} (${error.code}).`};
-    }
+const processFlashDocument = makeProcessDocument(Flash, {
+    fieldTransformations: {
+        'Date': value => new Date(value),
 
-    const contentLines = splitLines(contents);
-    const sections = Array.from(getSections(contentLines));
+        'Contributors': parseContributors,
+    },
 
-    return sections.filter(s => s.filter(Boolean).length).map(section => {
-        const name = getBasicField(section, 'Artist');
-        const urls = (getListField(section, 'URLs') || []).filter(Boolean);
-        const alias = getBasicField(section, 'Alias');
-        const hasAvatar = getBooleanField(section, 'Has Avatar') ?? false;
-        const note = getMultilineField(section, 'Note');
-        let directory = getBasicField(section, 'Directory');
+    propertyFieldMapping: {
+        name: 'Flash',
 
-        if (!name) {
-            return {error: 'Expected "Artist" (name) field!'};
-        }
+        directory: 'Directory',
+        page: 'Page',
+        date: 'Date',
+        coverArtFileExtension: 'Cover Art File Extension',
 
-        if (!directory) {
-            directory = getKebabCase(name);
-        }
+        featuredTracksByRef: 'Featured Tracks',
+        contributorContribsByRef: 'Contributors',
+        urls: 'URLs'
+    },
+});
 
-        if (alias) {
-            return {name, directory, alias};
-        } else {
-            return {name, directory, urls, note, hasAvatar};
-        }
-    });
-}
+const processFlashActDocument = makeProcessDocument(FlashAct, {
+    propertyFieldMapping: {
+        name: 'Act',
+        color: 'Color',
+        anchor: 'Anchor',
+        jump: 'Jump',
+        jumpColor: 'Jump Color'
+    }
+});
 
 async function processFlashDataFile(file) {
     let contents;
@@ -2421,8 +2442,8 @@ async function main() {
             files: albumDataFiles,
 
             documentMode: documentModes.headerAndEntries,
-            processHeaderDocument: parseAlbumDocument,
-            processEntryDocuments: parseAlbumEntryDocuments,
+            processHeaderDocument: processAlbumDocument,
+            processEntryDocuments: processAlbumEntryDocuments,
 
             save(results) {
                 const albumData = [];
@@ -2437,6 +2458,9 @@ async function main() {
                     trackData.push(...tracks);
                 }
 
+                sortByDate(albumData);
+                sortByDate(trackData);
+
                 Object.assign(wikiData, {albumData, trackData});
             }
         },
@@ -2451,7 +2475,49 @@ async function main() {
             save(results) {
                 wikiData.artistData = results;
             }
-        }
+        },
+
+        // TODO: WD.wikiInfo.features.flashesAndGames &&
+        {
+            title: `Process flash file`,
+            files: [path.join(dataPath, FLASH_DATA_FILE)],
+
+            documentMode: documentModes.allInOne,
+            processDocument(document) {
+                return ('Act' in document
+                    ? processFlashActDocument(document)
+                    : processFlashDocument(document));
+            },
+
+            save(results) {
+                let flashAct;
+                let flashesByRef = [];
+
+                if (results[0] && !(results[0] instanceof FlashAct)) {
+                    throw new Error(`Expected an act at top of flash data file`);
+                }
+
+                for (const thing of results) {
+                    if (thing instanceof FlashAct) {
+                        if (flashAct) {
+                            Object.assign(flashAct, {flashesByRef});
+                        }
+
+                        flashAct = thing;
+                        flashesByRef = [];
+                    } else {
+                        flashesByRef.push(Thing.getReference(thing));
+                    }
+                }
+
+                if (flashAct) {
+                    Object.assign(flashAct, {flashesByRef});
+                }
+
+                wikiData.flashData = results.filter(x => x instanceof Flash);
+                wikiData.flashActData = results.filter(x => x instanceof FlashAct);
+            }
+        },
     ];
 
     const processDataAggregate = openAggregate({message: `Errors processing data files`});
@@ -2487,7 +2553,7 @@ async function main() {
     for (const dataStep of dataSteps) {
         await processDataAggregate.nestAsync(
             {message: `Errors during data step: ${dataStep.title}`},
-            async ({call, map, mapAsync}) => {
+            async ({call, callAsync, map, mapAsync}) => {
                 const processDocuments = documentModeFunctions[dataStep.documentMode];
 
                 if (!processDocuments) {
@@ -2500,15 +2566,32 @@ async function main() {
                     }
 
                     const file = dataStep.files[0];
-                    const readResult = await readFile(file);
-                    const yamlResult = yaml.loadAll(readResult);
+
+                    const readResult = await callAsync(readFile, file);
+
+                    if (!readResult) {
+                        return;
+                    }
+
+                    const yamlResult = call(yaml.loadAll, readResult);
+
+                    if (!yamlResult) {
+                        return;
+                    }
 
                     const {
                         result: processResults,
                         aggregate: processAggregate
                     } = mapAggregate(
                         yamlResult,
-                        dataStep.processDocument,
+                        (document, i) => {
+                            try {
+                                return dataStep.processDocument(document);
+                            } catch (error) {
+                                error.message = `(${color.yellow(`#${i + 1}`)}) ${error.message}`;
+                                throw error;
+                            }
+                        },
                         {message: `Errors processing documents`}
                     );
 
@@ -2544,71 +2627,33 @@ async function main() {
             });
     }
 
-    logInfo`Loaded data and processed objects:`;
-    logInfo` - ${wikiData.albumData.length} albums`;
-    logInfo` - ${wikiData.trackData.length} tracks`;
-    logInfo` - ${wikiData.artistData.length} artists`;
-
-    try {
-        processDataAggregate.close();
-    } catch (error) {
-        showAggregate(error, {pathToFile: f => path.relative(__dirname, f)});
-        logWarn`The above errors were detected while processing data files.`;
-        logWarn`If the remaining valid data is complete enough, the wiki will`;
-        logWarn`still build - but all errored data will be skipped.`;
-        logWarn`(Resolve errors for more complete output!)`;
-    }
-
-    process.exit();
-
     {
-        const errors = WD.albumData.filter(obj => obj.error);
-        if (errors.length) {
-            for (const error of errors) {
-                console.log(`\x1b[31;1m${error.error}\x1b[0m`);
-            }
-            return;
+        logInfo`Loaded data and processed objects:`;
+        logInfo` - ${wikiData.albumData.length} albums`;
+        logInfo` - ${wikiData.trackData.length} tracks`;
+        logInfo` - ${wikiData.artistData.length} artists`;
+        if (wikiData.flashData)
+            logInfo` - ${wikiData.flashData.length} flashes (${wikiData.flashActData.length} acts)`;
+
+        let errorless = true;
+        try {
+            processDataAggregate.close();
+        } catch (error) {
+            showAggregate(error, {pathToFile: f => path.relative(__dirname, f)});
+            logWarn`The above errors were detected while processing data files.`;
+            logWarn`If the remaining valid data is complete enough, the wiki will`;
+            logWarn`still build - but all errored data will be skipped.`;
+            logWarn`(Resolve errors for more complete output!)`;
+            errorless = false;
         }
-    }
-
-    sortByDate(WD.albumData);
-
-    WD.artistData = await processArtistDataFile(path.join(dataPath, ARTIST_DATA_FILE));
-    if (WD.artistData.error) {
-        console.log(`\x1b[31;1m${WD.artistData.error}\x1b[0m`);
-        return;
-    }
 
-    {
-        const errors = WD.artistData.filter(obj => obj.error);
-        if (errors.length) {
-            for (const error of errors) {
-                console.log(`\x1b[31;1m${error.error}\x1b[0m`);
-            }
-            return;
+        if (errorless) {
+            logInfo`All data processed without any errors - nice!`;
+            logInfo`(This means all source files will be fully accounted for during page generation.)`;
         }
     }
 
-    WD.artistAliasData = WD.artistData.filter(x => x.alias);
-    WD.artistData = WD.artistData.filter(x => !x.alias);
-
-    WD.trackData = getAllTracks(WD.albumData);
-
-    if (WD.wikiInfo.features.flashesAndGames) {
-        WD.flashData = await processFlashDataFile(path.join(dataPath, FLASH_DATA_FILE));
-        if (WD.flashData.error) {
-            console.log(`\x1b[31;1m${WD.flashData.error}\x1b[0m`);
-            return;
-        }
-
-        const errors = WD.flashData.filter(obj => obj.error);
-        if (errors.length) {
-            for (const error of errors) {
-                console.log(`\x1b[31;1m${error.error}\x1b[0m`);
-            }
-            return;
-        }
-    }
+    process.exit();
 
     WD.flashActData = WD.flashData?.filter(x => x.act8r8k);
     WD.flashData = WD.flashData?.filter(x => !x.act8r8k);
diff --git a/src/util/sugar.js b/src/util/sugar.js
index 774534c..219c3ee 100644
--- a/src/util/sugar.js
+++ b/src/util/sugar.js
@@ -350,7 +350,7 @@ export function _withAggregate(mode, aggregateOpts, fn) {
     }
 }
 
-export function showAggregate(topError, {pathToFile = null} = {}) {
+export function showAggregate(topError, {pathToFile = p => p} = {}) {
     const recursive = (error, {level}) => {
         const stackLines = error.stack?.split('\n');
         const stackLine = stackLines?.find(line =>
diff --git a/test/data-validators.js b/test/data-validators.js
index 867068c..feec127 100644
--- a/test/data-validators.js
+++ b/test/data-validators.js
@@ -17,8 +17,12 @@ import {
     isDimensions,
     isDirectory,
     isDuration,
+    isFileExtension,
     validateReference,
     validateReferenceList,
+
+    // Compositional utilities
+    oneOf,
 } from '../src/thing/validators.js';
 
 function test(msg, fn) {
@@ -157,6 +161,16 @@ test('isDuration', t => {
     t.throws(() => isDuration('10:25'), TypeError);
 });
 
+test('isFileExtension', t => {
+    t.plan(6);
+    t.ok(isFileExtension('png'));
+    t.ok(isFileExtension('jpg'));
+    t.ok(isFileExtension('sub_loc'));
+    t.throws(() => isFileExtension(''), TypeError);
+    t.throws(() => isFileExtension('.jpg'), TypeError);
+    t.throws(() => isFileExtension('just an image bro!!!!'), TypeError);
+});
+
 test.skip('isName', t => {
     // TODO
 });
@@ -216,3 +230,37 @@ test('validateReferenceList', t => {
     t.true(caughtError.errors[0] instanceof TypeError);
     t.true(caughtError.errors[1] instanceof TypeError);
 });
+
+test('oneOf', t => {
+    t.plan(11);
+
+    const isStringOrNumber = oneOf(isString, isNumber);
+
+    t.ok(isStringOrNumber('hello world'));
+    t.ok(isStringOrNumber(42));
+    t.throws(() => isStringOrNumber(false));
+
+    const mockError = new Error();
+    const neverSucceeds = () => {
+        throw mockError;
+    };
+
+    const isStringOrGetRekt = oneOf(isString, neverSucceeds);
+
+    t.ok(isStringOrGetRekt('phew!'));
+
+    let caughtError = null;
+    try {
+        isStringOrGetRekt(0xdeadbeef);
+    } catch (err) {
+        caughtError = err;
+    }
+
+    t.isNot(caughtError, null);
+    t.true(caughtError instanceof AggregateError);
+    t.is(caughtError.errors.length, 2);
+    t.true(caughtError.errors[0] instanceof TypeError);
+    t.is(caughtError.errors[0].check, isString);
+    t.is(caughtError.errors[1], mockError);
+    t.is(caughtError.errors[1].check, neverSucceeds);
+});