initial working changes for big data restructure

5 files changed, 729 insertions, 76 deletions

diff --git a/src/thing/album.js b/src/thing/album.js
index e99cfc36..7be092e0 100644
--- a/src/thing/album.js
+++ b/src/thing/album.js
@@ -1,28 +1,267 @@
 import Thing from './thing.js';
 
 import {
-    validateDirectory,
-    validateReference
-} from './structures.js';
+    isBoolean,
+    isColor,
+    isCommentary,
+    isContributionList,
+    isDate,
+    isDimensions,
+    isDirectory,
+    isName,
+    isURL,
+    isString,
+    validateArrayItems,
+    validateReference,
+    validateReferenceList,
+} from './validators.js';
 
 import {
+    aggregateThrows,
     showAggregate,
     withAggregate
 } from '../util/sugar.js';
 
 export default class Album extends Thing {
+    /*
+    #name = 'Unnamed Album';
+
+    #color = null;
     #directory = null;
+    #urls = [];
+
+    #artists = [];
+    #coverArtists = [];
+    #trackCoverArtists = [];
+
+    #wallpaperArtists = [];
+    #wallpaperStyle = '';
+    #wallpaperFileExtension = 'jpg';
+
+    #bannerArtists = [];
+    #bannerStyle = '';
+    #bannerFileExtension = 'jpg';
+    #bannerDimensions = [0, 0];
+
+    #date = null;
+    #trackArtDate = null;
+    #coverArtDate = null;
+    #dateAddedToWiki = null;
+
+    #hasTrackArt = true;
+    #isMajorRelease = false;
+    #isListedOnHomepage = true;
+
+    #aka = '';
+    #groups = [];
+    #artTags = [];
+    #commentary = '';
+
     #tracks = [];
 
-    static updateError = {
+    static propertyError = {
+        name: Thing.extendPropertyError('name'),
         directory: Thing.extendPropertyError('directory'),
         tracks: Thing.extendPropertyError('tracks')
     };
+    */
+
+    static propertyDescriptors = {
+        // Update & expose
+
+        name: {
+            flags: {update: true, expose: true},
+
+            update: {
+                default: 'Unnamed Album',
+                validate: isName
+            }
+        },
+
+        color: {
+            flags: {update: true, expose: true},
+            update: {validate: isColor}
+        },
+
+        directory: {
+            flags: {update: true, expose: true},
+            update: {validate: isDirectory}
+        },
+
+        urls: {
+            flags: {update: true, expose: true},
+
+            update: {
+                validate: validateArrayItems(isURL)
+            }
+        },
+
+        date: {
+            flags: {update: true, expose: true},
+            update: {validate: isDate}
+        },
+
+        coverArtDate: {
+            flags: {update: true, expose: true},
+            update: {validate: isDate}
+        },
+
+        trackArtDate: {
+            flags: {update: true, expose: true},
+            update: {validate: isDate}
+        },
+
+        dateAddedToWiki: {
+            flags: {update: true, expose: true},
+
+            update: {validate: isDate}
+        },
+
+        artistContribsByRef: {
+            flags: {update: true, expose: true},
+            update: {validate: isContributionList}
+        },
+
+        coverArtistContribsByRef: {
+            flags: {update: true, expose: true},
+            update: {validate: isContributionList}
+        },
+
+        trackCoverArtistContribsByRef: {
+            flags: {update: true, expose: true},
+            update: {validate: isContributionList}
+        },
+
+        wallpaperArtistContribsByRef: {
+            flags: {update: true, expose: true},
+            update: {validate: isContributionList}
+        },
+
+        bannerArtistContribsByRef: {
+            flags: {update: true, expose: true},
+            update: {validate: isContributionList}
+        },
+
+        groupsByRef: {
+            flags: {update: true, expose: true},
+
+            update: {
+                validate: validateReferenceList('group')
+            }
+        },
+
+        artTagsByRef: {
+            flags: {update: true, expose: true},
+
+            update: {
+                validate: validateReferenceList('tag')
+            }
+        },
 
+        tracksByRef: {
+            flags: {update: true, expose: true},
+
+            update: {
+                validate: validateReferenceList('track')
+            }
+        },
+
+        wallpaperStyle: {
+            flags: {update: true, expose: true},
+            update: {validate: isString}
+        },
+
+        wallpaperFileExtension: {
+            flags: {update: true, expose: true},
+            update: {validate: isString}
+        },
+
+        bannerStyle: {
+            flags: {update: true, expose: true},
+            update: {validate: isString}
+        },
+
+        bannerFileExtension: {
+            flags: {update: true, expose: true},
+            update: {validate: isString}
+        },
+
+        bannerDimensions: {
+            flags: {update: true, expose: true},
+            update: {validate: isDimensions}
+        },
+
+        hasTrackArt: {
+            flags: {update: true, expose: true},
+
+            update: {
+                default: true,
+                validate: isBoolean
+            }
+        },
+
+        isMajorRelease: {
+            flags: {update: true, expose: true},
+
+            update: {
+                default: false,
+                validate: isBoolean
+            }
+        },
+
+        isListedOnHomepage: {
+            flags: {update: true, expose: true},
+
+            update: {
+                default: true,
+                validate: isBoolean
+            }
+        },
+
+        commentary: {
+            flags: {update: true, expose: true},
+            update: {validate: isCommentary}
+        },
+
+        // Expose only
+
+        tracks: {
+            flags: {expose: true},
+
+            expose: {
+                dependencies: ['trackReferences', 'wikiData'],
+                compute: ({trackReferences, wikiData}) => (
+                    trackReferences.map(ref => find.track(ref, {wikiData})))
+            }
+        },
+
+        // Update only
+
+        wikiData: {
+            flags: {update: true}
+        }
+    };
+
+    /*
     update(source) {
-        const err = this.constructor.updateError;
+        const err = this.constructor.propertyError;
+
+        withAggregate(aggregateThrows(Thing.UpdateError), ({ nest, filter, throws }) => {
+            if (source.name) {
+                nest(throws(err.name), ({ call }) => {
+                    if (call(validateName, source.name)) {
+                        this.#name = source.name;
+                    }
+                });
+            }
 
-        withAggregate(({ nest, filter, throws }) => {
+            if (source.color) {
+                nest(throws(err.color), ({ call }) => {
+                    if (call(validateColor, source.color)) {
+                        this.#color = source.color;
+                    }
+                });
+            }
 
             if (source.directory) {
                 nest(throws(err.directory), ({ call }) => {
@@ -37,10 +276,13 @@ export default class Album extends Thing {
         });
     }
 
+    get name() { return this.#name; }
     get directory() { return this.#directory; }
     get tracks() { return this.#tracks; }
+    */
 }
 
+/*
 const album = new Album();
 
 console.log('tracks (before):', album.tracks);
@@ -60,3 +302,4 @@ try {
 }
 
 console.log('tracks (after):', album.tracks);
+*/
diff --git a/src/thing/cacheable-object.js b/src/thing/cacheable-object.js
new file mode 100644
index 00000000..f478fd23
--- /dev/null
+++ b/src/thing/cacheable-object.js
@@ -0,0 +1,269 @@
+// Generally extendable class for caching properties and handling dependencies,
+// with a few key properties:
+//
+// 1) The behavior of every property is defined by its descriptor, which is a
+//    static value stored on the subclass (all instances share the same property
+//    descriptors).
+//
+//  1a) Additional properties may not be added past the time of object
+//      construction, and attempts to do so (including externally setting a
+//      property name which has no corresponding descriptor) will throw a
+//      TypeError. (This is done via an Object.seal(this) call after a newly
+//      created instance defines its own properties according to the descriptor
+//      on its constructor class.)
+//
+// 2) Properties may have two flags set: update and expose. Properties which
+//    update are provided values from the external. Properties which expose
+//    provide values to the external, generally dependent on other update
+//    properties (within the same object).
+//
+//  2a) Properties may be flagged as both updating and exposing. This is so
+//      that the same name may be used for both "output" and "input".
+//
+// 3) Exposed properties have values which are computations dependent on other
+//    properties, as described by a `compute` function on the descriptor.
+//    Depended-upon properties are explicitly listed on the descriptor next to
+//    this function, and are only provided as arguments to the function once
+//    listed.
+//
+//  3a) An exposed property may depend only upon updating properties, not other
+//      exposed properties (within the same object). This is to force the
+//      general complexity of a single object to be fairly simple: inputs
+//      directly determine outputs, with the only in-between step being the
+//      `compute` function, no multiple-layer dependencies. Note that this is
+//      only true within a given object - externally, values provided to one
+//      object's `update` may be (and regularly are) the exposed values of
+//      another object.
+//
+//  3b) If a property both updates and exposes, it is automatically regarded as
+//      a dependancy. (That is, its exposed value will depend on the value it is
+//      updated with.) Rather than a required `compute` function, these have an
+//      optional `transform` function, which takes the update value as its first
+//      argument and then the usual key-value dependencies as its second. If no
+//      `transform` function is provided, the expose value is the same as the
+//      update value.
+//
+// 4) Exposed properties are cached; that is, if no depended-upon properties are
+//    updated, the value of an exposed property is not recomputed.
+//
+//  4a) The cache for an exposed property is invalidated as soon as any of its
+//      dependencies are updated, but the cache itself is lazy: the exposed
+//      value will not be recomputed until it is again accessed. (Likewise, an
+//      exposed value won't be computed for the first time until it is first
+//      accessed.)
+//
+// 5) Updating a property may optionally apply validation checks before passing,
+//    declared by a `validate` function on the `update` block. This function
+//    should either throw an error (e.g. TypeError) or return false if the value
+//    is invalid.
+//
+// 6) Objects do not expect all updating properties to be provided at once.
+//    Incomplete objects are deliberately supported and enabled.
+//
+//  6a) The default value for every updating property is null; undefined is not
+//      accepted as a property value under any circumstances (it always errors).
+//      However, this default may be overridden by specifying a `default` value
+//      on a property's `update` block. (This value will be checked against
+//      the property's validate function.) Note that a property may always be
+//      updated to null, even if the default is non-null. (Null always bypasses
+//      the validate check.)
+//
+//  6b) It's required by the external consumer of an object to determine whether
+//      or not the object is ready for use (within the larger program). This is
+//      convenienced by the static CacheableObject.listAccessibleProperties()
+//      function, which provides a mapping of exposed property names to whether
+//      or not their dependencies are yet met.
+
+import { color, ENABLE_COLOR } from '../util/cli.js';
+
+import { inspect as nodeInspect } from 'util';
+
+function inspect(value) {
+    return nodeInspect(value, {colors: ENABLE_COLOR});
+}
+
+export default class CacheableObject {
+    #propertyUpdateValues = Object.create(null);
+    #propertyUpdateCacheInvalidators = Object.create(null);
+
+    /*
+    // Note the constructor doesn't take an initial data source. Due to a quirk
+    // of JavaScript, private members can't be accessed before the superclass's
+    // constructor is finished processing - so if we call the overridden
+    // update() function from inside this constructor, it will error when
+    // writing to private members. Pretty bad!
+    //
+    // That means initial data must be provided by following up with update()
+    // after constructing the new instance of the Thing (sub)class.
+    */
+
+    constructor() {
+        this.#defineProperties();
+        this.#initializeUpdatingPropertyValues();
+    }
+
+    #initializeUpdatingPropertyValues() {
+        for (const [ property, descriptor ] of Object.entries(this.constructor.propertyDescriptors)) {
+            const { flags, update } = descriptor;
+
+            if (!flags.update) {
+                continue;
+            }
+
+            if (update?.default) {
+                this[property] = update?.default;
+            } else {
+                this[property] = null;
+            }
+        }
+    }
+
+    #defineProperties() {
+        for (const [ property, descriptor ] of Object.entries(this.constructor.propertyDescriptors)) {
+            const { flags } = descriptor;
+
+            const definition = {
+                configurable: false,
+                enumerable: true
+            };
+
+            if (flags.update) {
+                definition.set = this.#getUpdateObjectDefinitionSetterFunction(property);
+            }
+
+            if (flags.expose) {
+                definition.get = this.#getExposeObjectDefinitionGetterFunction(property);
+            }
+
+            Object.defineProperty(this, property, definition);
+        }
+
+        Object.seal(this);
+    }
+
+    #getUpdateObjectDefinitionSetterFunction(property) {
+        const { update } = this.#getPropertyDescriptor(property);
+        const validate = update?.validate;
+        const allowNull = update?.allowNull;
+
+        return (newValue) => {
+            const oldValue = this.#propertyUpdateValues[property];
+
+            if (newValue === undefined) {
+                throw new ValueError(`Properties cannot be set to undefined`);
+            }
+
+            if (newValue === oldValue) {
+                return;
+            }
+
+            if (newValue !== null && validate) {
+                try {
+                    const result = validate(newValue);
+                    if (result === undefined) {
+                        throw new TypeError(`Validate function returned undefined`);
+                    } else if (result !== true) {
+                        throw new TypeError(`Validation failed for value ${newValue}`);
+                    }
+                } catch (error) {
+                    error.message = `Property ${color.green(property)} (${inspect(this[property])} -> ${inspect(newValue)}): ${error.message}`;
+                    throw error;
+                }
+            }
+
+            this.#propertyUpdateValues[property] = newValue;
+            this.#invalidateCachesDependentUpon(property);
+        };
+    }
+
+    #getUpdatePropertyValidateFunction(property) {
+        const descriptor = this.#getPropertyDescriptor(property);
+    }
+
+    #getPropertyDescriptor(property) {
+        return this.constructor.propertyDescriptors[property];
+    }
+
+    #invalidateCachesDependentUpon(property) {
+        for (const invalidate of this.#propertyUpdateCacheInvalidators[property] || []) {
+            invalidate();
+        }
+    }
+
+    #getExposeObjectDefinitionGetterFunction(property) {
+        const { flags } = this.#getPropertyDescriptor(property);
+        const compute = this.#getExposeComputeFunction(property);
+
+        if (compute) {
+            let cachedValue;
+            const checkCacheValid = this.#getExposeCheckCacheValidFunction(property);
+            return () => {
+                if (checkCacheValid()) {
+                    return cachedValue;
+                } else {
+                    return (cachedValue = compute());
+                }
+            };
+        } else if (!flags.update && !compute) {
+            throw new Error(`Exposed property ${property} does not update and is missing compute function`);
+        } else {
+            return () => this.#propertyUpdateValues[property];
+        }
+    }
+
+    #getExposeComputeFunction(property) {
+        const { flags, expose } = this.#getPropertyDescriptor(property);
+
+        const compute = (!flags.update && expose?.compute);
+        const transform = (flags.update && expose?.transform);
+
+        if (flags.update && !transform) {
+            return null;
+        } else if (!flags.update && !compute) {
+            throw new Error(`Exposed property ${property} does not update and is missing compute function`);
+        }
+
+        const dependencyKeys = expose.dependencies || [];
+        const dependencyGetters = dependencyKeys.map(key => () => [key, this.#propertyUpdateValues[key]]);
+        const getAllDependencies = () => Object.fromEntries(dependencyGetters.map(f => f()));
+
+        if (flags.update) {
+            return () => transform(this.#propertyUpdateValues[property], getAllDependencies());
+        } else {
+            return () => compute(getAllDependencies());
+        }
+    }
+
+    #getExposeCheckCacheValidFunction(property) {
+        const { flags, expose } = this.#getPropertyDescriptor(property);
+
+        let valid = false;
+
+        const invalidate = () => {
+            valid = false;
+        };
+
+        const dependencyKeys = new Set(expose?.dependencies);
+
+        if (flags.update) {
+            dependencyKeys.add(property);
+        }
+
+        for (const key of dependencyKeys) {
+            if (this.#propertyUpdateCacheInvalidators[key]) {
+                this.#propertyUpdateCacheInvalidators[key].push(invalidate);
+            } else {
+                this.#propertyUpdateCacheInvalidators[key] = [invalidate];
+            }
+        }
+
+        return () => {
+            if (!valid) {
+                valid = true;
+                return false;
+            } else {
+                return true;
+            }
+        };
+    }
+}
diff --git a/src/thing/structures.js b/src/thing/structures.js
index 89c9bd39..364ba149 100644
--- a/src/thing/structures.js
+++ b/src/thing/structures.js
@@ -1,32 +1 @@
 // Generic structure utilities common across various Thing types.
-
-export function validateDirectory(directory) {
-    if (typeof directory !== 'string')
-        throw new TypeError(`Expected a string, got ${directory}`);
-
-    if (directory.length === 0)
-        throw new TypeError(`Expected directory to be non-zero length`);
-
-    if (directory.match(/[^a-zA-Z0-9\-]/))
-        throw new TypeError(`Expected only letters, numbers, and dash, got "${directory}"`);
-
-    return true;
-}
-
-export function validateReference(type = '') {
-    return ref => {
-        if (typeof ref !== 'string')
-            throw new TypeError(`Expected a string, got ${ref}`);
-
-        if (type) {
-            if (!ref.includes(':'))
-                throw new TypeError(`Expected ref to begin with "${type}:", but no type specified (ref: ${ref})`);
-
-            const typePart = ref.split(':')[0];
-            if (typePart !== type)
-                throw new TypeError(`Expected ref to begin with "${type}:", got "${typePart}:" (ref: ${ref})`);
-        }
-
-        return true;
-    };
-}
diff --git a/src/thing/thing.js b/src/thing/thing.js
index c2465e32..dd3126c1 100644
--- a/src/thing/thing.js
+++ b/src/thing/thing.js
@@ -6,46 +6,10 @@
 // together as an AggregateError. See util/sugar.js for utility functions to
 // make writing code around this easier!
 
-export default class Thing {
-    constructor(source, {
-        wikiData
-    } = {}) {
-        if (source) {
-            this.update(source);
-        }
+import CacheableObject from './cacheable-object.js';
 
-        if (wikiData && this.checkComplete()) {
-            this.postprocess({wikiData});
-        }
-    }
-
-    static PropertyError = class extends AggregateError {
-        #key = this.constructor.key;
-        get key() { return this.#key; }
-
-        constructor(errors) {
-            super(errors, '');
-            this.message = `${errors.length} error(s) in property "${this.#key}"`;
-        }
-    };
-
-    static extendPropertyError(key) {
-        const cls = class extends this.PropertyError {
-            static #key = key;
-            static get key() { return this.#key; }
-        };
-
-        Object.defineProperty(cls, 'name', {value: `PropertyError:${key}`});
-        return cls;
-    }
-
-    // Called when instantiating a thing, and when its data is updated for any
-    // reason. (Which currently includes no reasons, but hey, future-proofing!)
-    //
-    // Don't expect source to be a complete object, even on the first call - the
-    // method checkComplete() will prevent incomplete resources from being mixed
-    // with the rest.
-    update(source) {}
+export default class Thing extends CacheableObject {
+    static propertyDescriptors = Symbol('Thing property descriptors');
 
     // Called when collecting the full list of available things of that type
     // for wiki data; this method determine whether or not to include it.
diff --git a/src/thing/validators.js b/src/thing/validators.js
new file mode 100644
index 00000000..05736914
--- /dev/null
+++ b/src/thing/validators.js
@@ -0,0 +1,208 @@
+import { withAggregate } from '../util/sugar.js';
+
+// Basic types (primitives)
+
+function a(noun) {
+    return (/[aeiou]/.test(noun[0]) ? `an ${noun}` : `a ${noun}`);
+}
+
+function isType(value, type) {
+    if (typeof value !== type)
+        throw new TypeError(`Expected ${a(type)}, got ${typeof value}`);
+
+    return true;
+}
+
+export function isBoolean(value) {
+    return isType(value, 'boolean');
+}
+
+export function isNumber(value) {
+    return isType(value, 'number');
+}
+
+export function isPositive(number) {
+    isNumber(number);
+
+    if (number <= 0)
+        throw new TypeError(`Expected positive number`);
+
+    return true;
+}
+
+export function isNegative(number) {
+    isNumber(number);
+
+    if (number >= 0)
+        throw new TypeError(`Expected negative number`);
+
+    return true;
+}
+
+export function isInteger(number) {
+    isNumber(number);
+
+    if (number % 1 !== 0)
+        throw new TypeError(`Expected integer`);
+
+    return true;
+}
+
+export function isString(value) {
+    return isType(value, 'string');
+}
+
+export function isStringNonEmpty(value) {
+    isString(value);
+
+    if (value.trim().length === 0)
+        throw new TypeError(`Expected non-empty string`);
+
+    return true;
+}
+
+// Complex types (non-primitives)
+
+function isInstance(value, constructor) {
+    isObject(value);
+
+    if (!(value instanceof constructor))
+        throw new TypeError(`Expected ${constructor.name}, got ${value.constructor.name}`);
+
+    return true;
+}
+
+export function isDate(value) {
+    return isInstance(value, Date);
+}
+
+export function isObject(value) {
+    isType(value, 'object');
+
+    // Note: Please remember that null is always a valid value for properties
+    // held by a CacheableObject. This assertion is exclusively for use in other
+    // contexts.
+    if (value === null)
+        throw new TypeError(`Expected an object, got null`);
+
+    return true;
+}
+
+export function isArray(value) {
+    isObject(value);
+
+    if (!Array.isArray(value))
+        throw new TypeError(`Expected an array, got ${value}`);
+
+    return true;
+}
+
+export function validateArrayItems(itemValidator) {
+    return array => {
+        isArray(array);
+
+        withAggregate({message: 'Errors validating array items'}, ({ wrap }) => {
+            array.forEach(wrap(itemValidator));
+        });
+
+        return true;
+    };
+}
+
+// Wiki data (primitives & non-primitives)
+
+export function isColor(color) {
+    isStringNonEmpty(color);
+
+    if (color.startsWith('#')) {
+        if (![1 + 3, 1 + 4, 1 + 6, 1 + 8].includes(color.length))
+            throw new TypeError(`Expected #rgb, #rgba, #rrggbb, or #rrggbbaa, got length ${color.length}`);
+
+        if (/[^0-9a-fA-F]/.test(color.slice(1)))
+            throw new TypeError(`Expected hexadecimal digits`);
+
+        return true;
+    }
+
+    throw new TypeError(`Unknown color format`);
+}
+
+export function isCommentary(commentary) {
+    return isString(commentary);
+}
+
+const isArtistRef = validateReference('artist');
+
+export function isContribution(contrib) {
+    // TODO: Use better object validation for this (supporting aggregates etc)
+
+    isObject(contrib);
+
+    isArtistRef(contrib.who);
+
+    if (contrib.what !== null) {
+        isStringNonEmpty(contrib.what);
+    }
+
+    return true;
+}
+
+export const isContributionList = validateArrayItems(isContribution);
+
+export function isDimensions(dimensions) {
+    isArray(dimensions);
+
+    if (dimensions.length !== 2)
+        throw new TypeError(`Expected 2 item array`);
+
+    isPositive(dimensions[0]);
+    isInteger(dimensions[0]);
+    isPositive(dimensions[1]);
+    isInteger(dimensions[1]);
+
+    return true;
+}
+
+export function isDirectory(directory) {
+    isStringNonEmpty(directory);
+
+    if (directory.match(/[^a-zA-Z0-9\-]/))
+        throw new TypeError(`Expected only letters, numbers, and dash, got "${directory}"`);
+
+    return true;
+}
+
+export function isName(name) {
+    return isString(name);
+}
+
+export function isURL(string) {
+    isStringNonEmpty(string);
+
+    new URL(string);
+
+    return true;
+}
+
+export function validateReference(type = 'track') {
+    return ref => {
+        isStringNonEmpty(ref);
+
+        const hasTwoParts = ref.includes(':');
+        const [ typePart, directoryPart ] = ref.split(':');
+
+        if (hasTwoParts && typePart !== type)
+            throw new TypeError(`Expected ref to begin with "${type}:", got "${typePart}:" (ref: ${ref})`);
+
+        if (hasTwoParts)
+            isDirectory(directoryPart);
+
+        isName(ref);
+
+        return true;
+    };
+}
+
+export function validateReferenceList(type = '') {
+    return validateArrayItems(validateReference(type));
+}