track data

8 files changed, 368 insertions, 43 deletions

diff --git a/src/thing/album.js b/src/thing/album.js
index 9899b6af..11af8019 100644
--- a/src/thing/album.js
+++ b/src/thing/album.js
@@ -1,4 +1,6 @@
+import CacheableObject from './cacheable-object.js';
 import Thing from './thing.js';
+import find from '../util/find.js';
 
 import {
     isBoolean,
@@ -12,21 +14,70 @@ import {
     isURL,
     isString,
     validateArrayItems,
+    validateInstanceOf,
     validateReference,
     validateReferenceList,
 } from './validators.js';
 
-export default class Album extends Thing {
+export class TrackGroup extends CacheableObject {
     static propertyDescriptors = {
         // Update & expose
 
         name: {
             flags: {update: true, expose: true},
+            update: {default: 'Unnamed Track Group', validate: isName}
+        },
 
-            update: {
-                default: 'Unnamed Album',
-                validate: isName
+        color: {
+            flags: {update: true, expose: true},
+            update: {validate: isColor}
+        },
+
+        dateOriginallyReleased: {
+            flags: {update: true, expose: true},
+            update: {validate: isDate}
+        },
+
+        tracksByRef: {
+            flags: {update: true, expose: true},
+            update: {validate: validateReferenceList('track')}
+        },
+
+        isDefaultTrackGroup: {
+            flags: {update: true, expose: true},
+            update: {validate: isBoolean}
+        },
+
+        // Update only
+
+        trackData: {
+            flags: {update: true},
+            update: {validate: validateArrayItems(item => isInstance(item, Track))}
+        },
+
+        // Expose only
+
+        tracks: {
+            flags: {expose: true},
+
+            expose: {
+                dependencies: ['tracksByRef', 'trackData'],
+                compute: ({ tracksByRef, trackData }) => (
+                    tracksByRef.map(ref => find.track(ref, {wikiData: {trackData}})))
             }
+        }
+    };
+}
+
+export default class Album extends Thing {
+    static [Thing.referenceType] = 'album';
+
+    static propertyDescriptors = {
+        // Update & expose
+
+        name: {
+            flags: {update: true, expose: true},
+            update: {default: 'Unnamed Album', validate: isName}
         },
 
         color: {
@@ -36,7 +87,8 @@ export default class Album extends Thing {
 
         directory: {
             flags: {update: true, expose: true},
-            update: {validate: isDirectory}
+            update: {validate: isDirectory},
+            expose: Thing.directoryExpose
         },
 
         urls: {
@@ -109,11 +161,11 @@ export default class Album extends Thing {
             }
         },
 
-        tracksByRef: {
+        trackGroups: {
             flags: {update: true, expose: true},
 
             update: {
-                validate: validateReferenceList('track')
+                validate: validateArrayItems(validateInstanceOf(TrackGroup))
             }
         },
 
@@ -176,6 +228,7 @@ export default class Album extends Thing {
 
         // Expose only
 
+        /*
         tracks: {
             flags: {expose: true},
 
@@ -185,11 +238,14 @@ export default class Album extends Thing {
                     trackReferences.map(ref => find.track(ref, {wikiData})))
             }
         },
+        */
 
         // Update only
 
+        /*
         wikiData: {
             flags: {update: true}
         }
+        */
     };
 }
diff --git a/src/thing/thing.js b/src/thing/thing.js
index dd3126c1..54a278d1 100644
--- a/src/thing/thing.js
+++ b/src/thing/thing.js
@@ -1,30 +1,32 @@
 // Base class for Things. No, we will not come up with a better name.
 // Sorry not sorry! :)
-//
-// NB: Since these methods all involve processing a variety of input data, some
-// of which will pass and some of which may fail, any failures should be thrown
-// together as an AggregateError. See util/sugar.js for utility functions to
-// make writing code around this easier!
 
 import CacheableObject from './cacheable-object.js';
 
+import { getKebabCase } from '../util/wiki-data.js';
+
 export default class Thing extends CacheableObject {
-    static propertyDescriptors = Symbol('Thing property descriptors');
+    static referenceType = Symbol('Thing.referenceType');
+
+    static directoryExpose = {
+        dependencies: ['name'],
+        transform(directory, { name }) {
+            if (directory === null && name === null)
+                return null;
+            else if (directory === null)
+                return getKebabCase(name);
+            else
+                return directory;
+        }
+    };
+
+    static getReference(thing) {
+        if (!thing.constructor[Thing.referenceType])
+            throw TypeError(`Passed Thing is ${thing.constructor.name}, which provides no [Thing.referenceType]`);
 
-    // Called when collecting the full list of available things of that type
-    // for wiki data; this method determine whether or not to include it.
-    //
-    // This should return whether or not the object is complete enough to be
-    // used across the wiki - not whether every optional attribute is provided!
-    // (That is, attributes required for postprocessing & basic page generation
-    // are all present.)
-    checkComplete() {}
+        if (!thing.directory)
+            throw TypeError(`Passed ${thing.constructor.name} is missing its directory`);
 
-    // Called when adding the thing to the wiki data list, and when its source
-    // data is updated (provided checkComplete() passes).
-    //
-    // This should generate any cached object references, across other wiki
-    // data; for example, building an array of actual track objects
-    // corresponding to an album's track list ('track:cool-track' strings).
-    postprocess({wikiData}) {}
+        return `${thing.constructor[Thing.referenceType]}:${thing.directory}`;
+    }
 }
diff --git a/src/thing/track.js b/src/thing/track.js
new file mode 100644
index 00000000..3174cabb
--- /dev/null
+++ b/src/thing/track.js
@@ -0,0 +1,112 @@
+import Thing from './thing.js';
+
+import {
+    isBoolean,
+    isColor,
+    isCommentary,
+    isContributionList,
+    isDate,
+    isDirectory,
+    isDuration,
+    isName,
+    isURL,
+    isString,
+    validateArrayItems,
+    validateReference,
+    validateReferenceList,
+} from './validators.js';
+
+export default class Track extends Thing {
+    static [Thing.referenceType] = 'track';
+
+    static propertyDescriptors = {
+        // Update & expose
+
+        name: {
+            flags: {update: true, expose: true},
+
+            update: {
+                default: 'Unnamed Track',
+                validate: isName
+            }
+        },
+
+        directory: {
+            flags: {update: true, expose: true},
+            update: {validate: isDirectory},
+            expose: Thing.directoryExpose
+        },
+
+        duration: {
+            flags: {update: true, expose: true},
+            update: {validate: isDuration}
+        },
+
+        urls: {
+            flags: {update: true, expose: true},
+
+            update: {
+                validate: validateArrayItems(isURL)
+            }
+        },
+
+        dateFirstReleased: {
+            flags: {update: true, expose: true},
+            update: {validate: isDate}
+        },
+
+        hasCoverArt: {
+            flags: {update: true, expose: true},
+            update: {default: true, validate: isBoolean}
+        },
+
+        hasURLs: {
+            flags: {update: true, expose: true},
+            update: {default: true, validate: isBoolean}
+        },
+
+        referencedTracksByRef: {
+            flags: {update: true, expose: true},
+            update: {validate: validateReferenceList('track')}
+        },
+
+        artistContribsByRef: {
+            flags: {update: true, expose: true},
+            update: {validate: isContributionList}
+        },
+
+        contributorContribsByRef: {
+            flags: {update: true, expose: true},
+            update: {validate: isContributionList}
+        },
+
+        coverArtistContribsByRef: {
+            flags: {update: true, expose: true},
+            update: {validate: isContributionList}
+        },
+
+        artTagsByRef: {
+            flags: {update: true, expose: true},
+            update: {validate: validateReferenceList('tag')}
+        },
+
+        originalReleaseTrackByRef: {
+            flags: {update: true, expose: true},
+            update: {validate: validateReference('track')}
+        },
+
+        commentary: {
+            flags: {update: true, expose: true},
+            update: {validate: isCommentary}
+        },
+
+        lyrics: {
+            flags: {update: true, expose: true},
+            update: {validate: isString}
+        },
+
+        // Update only
+
+        // Expose only
+    };
+}
diff --git a/src/thing/validators.js b/src/thing/validators.js
index 2bdb2995..e745771a 100644
--- a/src/thing/validators.js
+++ b/src/thing/validators.js
@@ -47,6 +47,24 @@ export function isNegative(number) {
     return true;
 }
 
+export function isPositiveOrZero(number) {
+    isNumber(number);
+
+    if (number < 0)
+        throw new TypeError(`Expected positive number or zero`);
+
+    return true;
+}
+
+export function isNegativeOrZero(number) {
+    isNumber(number);
+
+    if (number > 0)
+        throw new TypeError(`Expected negative number or zero`);
+
+    return true;
+}
+
 export function isInteger(number) {
     isNumber(number);
 
@@ -130,6 +148,10 @@ export function validateArrayItems(itemValidator) {
     };
 }
 
+export function validateInstanceOf(constructor) {
+    return object => isInstance(object, constructor);
+}
+
 // Wiki data (primitives & non-primitives)
 
 export function isColor(color) {
@@ -187,8 +209,15 @@ export function isDimensions(dimensions) {
 export function isDirectory(directory) {
     isStringNonEmpty(directory);
 
-    if (directory.match(/[^a-zA-Z0-9\-]/))
-        throw new TypeError(`Expected only letters, numbers, and dash, got "${directory}"`);
+    if (directory.match(/[^a-zA-Z0-9_\-]/))
+        throw new TypeError(`Expected only letters, numbers, dash, and underscore, got "${directory}"`);
+
+    return true;
+}
+
+export function isDuration(duration) {
+    isNumber(duration);
+    isPositiveOrZero(duration);
 
     return true;
 }
@@ -209,13 +238,17 @@ export function validateReference(type = 'track') {
     return ref => {
         isStringNonEmpty(ref);
 
-        const hasTwoParts = ref.includes(':');
-        const [ typePart, directoryPart ] = ref.split(':');
+        const match = ref.trim().match(/^(?:(?<typePart>\S+):(?=\S))?(?<directoryPart>.+)(?<!:)$/);
+
+        if (!match)
+            throw new TypeError(`Malformed reference`);
+
+        const { groups: { typePart, directoryPart } } = match;
 
-        if (hasTwoParts && typePart !== type)
-            throw new TypeError(`Expected ref to begin with "${type}:", got "${typePart}:" (ref: ${ref})`);
+        if (typePart && typePart !== type)
+            throw new TypeError(`Expected ref to begin with "${type}:", got "${typePart}:"`);
 
-        if (hasTwoParts)
+        if (typePart)
             isDirectory(directoryPart);
 
         isName(ref);
diff --git a/src/upd8.js b/src/upd8.js
index fd5a21ca..60e0c0b4 100755
--- a/src/upd8.js
+++ b/src/upd8.js
@@ -89,7 +89,9 @@ import find from './util/find.js';
 import * as html from './util/html.js';
 import unbound_link, {getLinkThemeString} from './util/link.js';
 
-import Album from './thing/album.js';
+import Album, { TrackGroup } from './thing/album.js';
+import Thing from './thing/thing.js';
+import Track from './thing/track.js';
 
 import {
     fancifyFlashURL,
@@ -868,7 +870,7 @@ makeParseDocument.UnknownFieldsError = class UnknownFieldsError extends Error {
     }
 };
 
-processAlbumDataFile.parseDocument = makeParseDocument(Album, {
+const parseAlbumDocument = makeParseDocument(Album, {
     fieldTransformations: {
         'Artists': parseContributors,
         'Cover Artists': parseContributors,
@@ -945,7 +947,62 @@ async function processAlbumDataFile(file) {
 
     const albumDoc = documents[0];
 
-    return processAlbumDataFile.parseDocument(albumDoc, {file});
+    const album = parseAlbumDocument(albumDoc, {file});
+
+    // Slightly separate meanings: tracks is the array of Track objects (and
+    // only Track objects); trackGroups is the array of TrackGroup objects,
+    // organizing (by string reference) the Track objects within the Album.
+    // tracks is returned for collating with the rest of wiki data; trackGroups
+    // is directly set on the album object.
+    const tracks = [];
+    const trackGroups = [];
+
+    // We can't mutate an array once it's set as a property value, so prepare
+    // the tracks that will show up in a track list all the way before actually
+    // applying it.
+    let currentTracksByRef = null;
+    let currentTrackGroupDoc = null;
+
+    function closeCurrentTrackGroup() {
+        if (currentTracksByRef) {
+            let trackGroup;
+
+            if (currentTrackGroupDoc) {
+                trackGroup = parseTrackGroupDocument(currentTrackGroupDoc, {file});
+            } else {
+                trackGroup = new TrackGroup();
+                trackGroup.isDefaultTrackGroup = true;
+            }
+
+            trackGroup.tracksByRef = currentTracksByRef;
+            trackGroups.push(trackGroup);
+        }
+    }
+
+    for (const doc of documents.slice(1)) {
+        if (doc['Group']) {
+            closeCurrentTrackGroup();
+            currentTracksByRef = [];
+            currentTrackGroupDoc = doc;
+            continue;
+        }
+
+        const track = parseTrackDocument(doc, {file});
+        tracks.push(track);
+
+        const ref = Thing.getReference(track);
+        if (currentTracksByRef) {
+            currentTracksByRef.push(ref);
+        } else {
+            currentTracksByRef = [ref];
+        }
+    }
+
+    closeCurrentTrackGroup();
+
+    album.trackGroups = trackGroups;
+
+    return {album, tracks};
 
     // --------------------------------------------------------------
 
@@ -1197,6 +1254,52 @@ async function processAlbumDataFile(file) {
     return album;
 }
 
+const parseTrackGroupDocument = makeParseDocument(TrackGroup, {
+    fieldTransformations: {
+        'Date Originally Released': value => new Date(value),
+    },
+
+    propertyFieldMapping: {
+        name: 'Group',
+        color: 'Color',
+        dateOriginallyReleased: 'Date Originally Released',
+    }
+});
+
+const parseTrackDocument = makeParseDocument(Track, {
+    fieldTransformations: {
+        'Duration': getDurationInSeconds,
+
+        'Date First Released': value => new Date(value),
+
+        'Artists': parseContributors,
+        'Contributors': parseContributors,
+        'Cover Artists': parseContributors,
+    },
+
+    propertyFieldMapping: {
+        name: 'Track',
+
+        directory: 'Directory',
+        duration: 'Duration',
+        urls: 'URLs',
+
+        dateFirstReleased: 'Date First Released',
+        hasCoverArt: 'Has Cover Art',
+        hasURLs: 'Has URLs',
+
+        referencedTracksByRef: 'Referenced Tracks',
+        artistContribsByRef: 'Artists',
+        contributorContribsByRef: 'Contributors',
+        coverArtistContribsByRef: 'Cover Artists',
+        artTagsByRef: 'Art Tags',
+        originalReleaseTrackByRef: 'Originally Released As',
+
+        commentary: 'Commentary',
+        lyrics: 'Lyrics'
+    }
+});
+
 async function processArtistDataFile(file) {
     let contents;
     try {
@@ -1626,6 +1729,14 @@ async function processHomepageInfoFile(file) {
 }
 
 function getDurationInSeconds(string) {
+    if (typeof string === 'number') {
+        return string;
+    }
+
+    if (typeof string !== 'string') {
+        throw new TypeError(`Expected a string or number, got ${string}`);
+    }
+
     const parts = string.split(':').map(n => parseInt(n))
     if (parts.length === 3) {
         return parts[0] * 3600 + parts[1] * 60 + parts[2]
@@ -2645,7 +2756,7 @@ async function main() {
     try {
         processDataAggregate.close();
     } catch (error) {
-        showAggregate(error);
+        showAggregate(error, {pathToFile: f => path.relative(__dirname, f)});
     }
 
     process.exit();
diff --git a/src/util/find.js b/src/util/find.js
index 5f69bbec..423046b3 100644
--- a/src/util/find.js
+++ b/src/util/find.js
@@ -7,7 +7,7 @@ function findHelper(keys, dataProp, findFns = {}) {
     const byDirectory = findFns.byDirectory || matchDirectory;
     const byName = findFns.byName || matchName;
 
-    const keyRefRegex = new RegExp(`^((${keys.join('|')}):)?(.*)$`);
+    const keyRefRegex = new RegExp(`^((${keys.join('|')}):(?:\S))?(.*)$`);
 
     return (fullRef, {wikiData}) => {
         if (!fullRef) return null;
diff --git a/src/util/sugar.js b/src/util/sugar.js
index 64291f36..075aa190 100644
--- a/src/util/sugar.js
+++ b/src/util/sugar.js
@@ -328,7 +328,7 @@ export function withAggregate(aggregateOpts, fn) {
     return result;
 }
 
-export function showAggregate(topError) {
+export function showAggregate(topError, {pathToFile = null} = {}) {
     const recursive = error => {
         const stackLines = error.stack?.split('\n');
         const stackLine = stackLines?.find(line =>
@@ -336,7 +336,7 @@ export function showAggregate(topError) {
             && !line.includes('sugar')
             && !line.includes('node:internal'));
         const tracePart = (stackLine
-            ? '- ' + stackLine.trim()
+            ? '- ' + stackLine.trim().replace(/file:\/\/(.*\.js)/, (match, pathname) => pathToFile(pathname))
             : '(no stack trace)');
 
         const header = `[${error.constructor.name || 'unnamed'}] ${error.message || '(no message)'} ${color.dim(tracePart)}`;
diff --git a/test/data-validators.js b/test/data-validators.js
index e6b8b43e..867068c7 100644
--- a/test/data-validators.js
+++ b/test/data-validators.js
@@ -16,6 +16,7 @@ import {
     // Wiki data
     isDimensions,
     isDirectory,
+    isDuration,
     validateReference,
     validateReferenceList,
 } from '../src/thing/validators.js';
@@ -138,14 +139,24 @@ test('isDimensions', t => {
 });
 
 test('isDirectory', t => {
-    t.plan(5);
+    t.plan(6);
     t.ok(isDirectory('savior-of-the-waking-world'));
     t.ok(isDirectory('MeGaLoVania'));
+    t.ok(isDirectory('ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_'));
     t.throws(() => isDirectory(123), TypeError);
     t.throws(() => isDirectory(''), TypeError);
     t.throws(() => isDirectory('troll saint nicholas and the quest for the holy pail'), TypeError);
 });
 
+test('isDuration', t => {
+    t.plan(5);
+    t.ok(isDuration(60));
+    t.ok(isDuration(0.02));
+    t.ok(isDuration(0));
+    t.throws(() => isDuration(-1), TypeError);
+    t.throws(() => isDuration('10:25'), TypeError);
+});
+
 test.skip('isName', t => {
     // TODO
 });