diff options
| -rw-r--r-- | src/thing/album.js | 5 | ||||
| -rw-r--r-- | src/thing/cacheable-object.js | 6 | ||||
| -rw-r--r-- | src/thing/flash.js | 129 | ||||
| -rw-r--r-- | src/thing/validators.js | 45 | ||||
| -rwxr-xr-x | src/upd8.js | 265 | ||||
| -rw-r--r-- | src/util/sugar.js | 2 | ||||
| -rw-r--r-- | test/data-validators.js | 48 |
7 files changed, 385 insertions, 115 deletions
diff --git a/src/thing/album.js b/src/thing/album.js index 11af8019..8a9fde2c 100644 --- a/src/thing/album.js +++ b/src/thing/album.js @@ -10,6 +10,7 @@ import { isDate, isDimensions, isDirectory, + isFileExtension, isName, isURL, isString, @@ -176,7 +177,7 @@ export default class Album extends Thing { wallpaperFileExtension: { flags: {update: true, expose: true}, - update: {validate: isString} + update: {validate: isFileExtension} }, bannerStyle: { @@ -186,7 +187,7 @@ export default class Album extends Thing { bannerFileExtension: { flags: {update: true, expose: true}, - update: {validate: isString} + update: {validate: isFileExtension} }, bannerDimensions: { diff --git a/src/thing/cacheable-object.js b/src/thing/cacheable-object.js index f478fd23..3c14101c 100644 --- a/src/thing/cacheable-object.js +++ b/src/thing/cacheable-object.js @@ -214,11 +214,13 @@ export default class CacheableObject { #getExposeComputeFunction(property) { const { flags, expose } = this.#getPropertyDescriptor(property); - const compute = (!flags.update && expose?.compute); - const transform = (flags.update && expose?.transform); + const compute = expose?.compute; + const transform = expose?.transform; if (flags.update && !transform) { return null; + } else if (flags.update && compute) { + throw new Error(`Updating property ${property} has compute function, should be formatted as transform`); } else if (!flags.update && !compute) { throw new Error(`Exposed property ${property} does not update and is missing compute function`); } diff --git a/src/thing/flash.js b/src/thing/flash.js new file mode 100644 index 00000000..4eac65ad --- /dev/null +++ b/src/thing/flash.js @@ -0,0 +1,129 @@ +import Thing from './thing.js'; + +import { + isColor, + isContributionList, + isDate, + isDirectory, + isFileExtension, + isName, + isNumber, + isString, + isURL, + oneOf, + validateArrayItems, + validateReferenceList, +} from './validators.js'; + +export default class Flash extends Thing { + static [Thing.referenceType] = 'flash'; + + static propertyDescriptors = { + // Update & expose + + name: { + flags: {update: true, expose: true}, + + update: { + default: 'Unnamed Flash', + validate: isName + } + }, + + directory: { + flags: {update: true, expose: true}, + update: {validate: isDirectory}, + + // Flashes expose directory differently from other Things! Their + // default directory is dependent on the page number (or ID), not + // the name. + expose: { + dependencies: ['page'], + transform(directory, { page }) { + if (directory === null && page === null) + return null; + else if (directory === null) + return page; + else + return directory; + } + } + }, + + page: { + flags: {update: true, expose: true}, + update: {validate: oneOf(isString, isNumber)}, + + expose: { + transform: value => value.toString() + } + }, + + date: { + flags: {update: true, expose: true}, + update: {validate: isDate} + }, + + coverArtFileExtension: { + flags: {update: true, expose: true}, + update: {validate: isFileExtension} + }, + + featuredTracksByRef: { + flags: {update: true, expose: true}, + update: {validate: validateReferenceList('track')} + }, + + contributorContribsByRef: { + flags: {update: true, expose: true}, + update: {validate: isContributionList} + }, + + urls: { + flags: {update: true, expose: true}, + update: {validate: validateArrayItems(isURL)} + }, + }; +} + +export class FlashAct extends Thing { + static [Thing.referenceType] = 'flash-act'; + + static propertyDescriptors = { + // Update & expose + + name: { + flags: {update: true, expose: true}, + + update: { + default: 'Unnamed Flash Act', + validate: isName + } + }, + + color: { + flags: {update: true, expose: true}, + update: {validate: isColor} + }, + + anchor: { + flags: {update: true, expose: true}, + update: {validate: isString} + }, + + jump: { + flags: {update: true, expose: true}, + update: {validate: isString} + }, + + jumpColor: { + flags: {update: true, expose: true}, + update: {validate: isColor} + }, + + flashesByRef: { + flags: {update: true, expose: true}, + update: {validate: validateReferenceList('flash')} + }, + }; +} diff --git a/src/thing/validators.js b/src/thing/validators.js index e745771a..a465e9d1 100644 --- a/src/thing/validators.js +++ b/src/thing/validators.js @@ -222,6 +222,18 @@ export function isDuration(duration) { return true; } +export function isFileExtension(string) { + isStringNonEmpty(string); + + if (string[0] === '.') + throw new TypeError(`Expected no dot (.) at the start of file extension`); + + if (string.match(/[^a-zA-Z0-9_]/)) + throw new TypeError(`Expected only alphanumeric and underscore`); + + return true; +} + export function isName(name) { return isString(name); } @@ -260,3 +272,36 @@ export function validateReference(type = 'track') { export function validateReferenceList(type = '') { return validateArrayItems(validateReference(type)); } + +// Compositional utilities + +export function oneOf(...checks) { + return value => { + const errorMeta = []; + + for (let i = 0, check; check = checks[i]; i++) { + try { + const result = check(value); + + if (result !== true) { + throw new Error(`Check returned false`); + } + + return true; + } catch (error) { + errorMeta.push([check, i, error]); + } + } + + // Don't process error messages until every check has failed. + const errors = []; + for (const [ check, i, error ] of errorMeta) { + error.message = (check.name + ? `(#${i} "${check.name}") ${error.message}` + : `(#${i}) ${error.message}`); + error.check = check; + errors.push(error); + } + throw new AggregateError(errors, `Expected one of ${checks.length} possible checks, but none were true`); + }; +} diff --git a/src/upd8.js b/src/upd8.js index 6df3cc2c..0292e609 100755 --- a/src/upd8.js +++ b/src/upd8.js @@ -80,6 +80,8 @@ import { unlink } from 'fs/promises'; +import { inspect as nodeInspect } from 'util'; + import genThumbs from './gen-thumbs.js'; import { listingSpec, listingTargetSpec } from './listing-spec.js'; import urlSpec from './url-spec.js'; @@ -91,6 +93,7 @@ import unbound_link, {getLinkThemeString} from './util/link.js'; import Album, { TrackGroup } from './thing/album.js'; import Artist from './thing/artist.js'; +import Flash, { FlashAct } from './thing/flash.js'; import Thing from './thing/thing.js'; import Track from './thing/track.js'; @@ -120,7 +123,8 @@ import { logInfo, logError, parseOptions, - progressPromiseAll + progressPromiseAll, + ENABLE_COLOR } from './util/cli.js'; import { @@ -195,7 +199,7 @@ const CACHEBUST = 7; const WIKI_INFO_FILE = 'wiki-info.txt'; const HOMEPAGE_INFO_FILE = 'homepage.txt'; const ARTIST_DATA_FILE = 'artists.yaml'; -const FLASH_DATA_FILE = 'flashes.txt'; +const FLASH_DATA_FILE = 'flashes.yaml'; const NEWS_DATA_FILE = 'news.txt'; const TAG_DATA_FILE = 'tags.txt'; const GROUP_DATA_FILE = 'groups.txt'; @@ -219,6 +223,10 @@ const STATIC_DIRECTORY = 'static'; // read from and processed to compose the majority of album and track data. const DATA_ALBUM_DIRECTORY = 'album'; +function inspect(value) { + return nodeInspect(value, {colors: ENABLE_COLOR}); +} + // Shared varia8les! These are more efficient to access than a shared varia8le // (or at least I h8pe so), and are easier to pass across functions than a // 8unch of specific arguments. @@ -713,10 +721,10 @@ function parseCommentary(text) { // General function for inputting a single document (usually loaded from YAML) // and outputting an instance of a provided Thing subclass. // -// makeParseDocument is a factory function: the returned function will take a -// document and apply the configuration passed to makeParseDocument in order to -// construct a Thing subclass. -function makeParseDocument(thingClass, { +// makeProcessDocument is a factory function: the returned function will take a +// document and apply the configuration passed to makeProcessDocument in order +// to construct a Thing subclass. +function makeProcessDocument(thingClass, { // Optional early step for transforming field values before providing them // to the Thing's update() method. This is useful when the input format // (i.e. values in the document) differ from the format the actual Thing @@ -749,7 +757,24 @@ function makeParseDocument(thingClass, { (Object.entries(propertyFieldMapping) .map(([ property, field ]) => [field, property]))); - return function(document) { + const decorateErrorWithName = fn => { + const nameField = propertyFieldMapping['name']; + if (!nameField) return fn; + + return document => { + try { + return fn(document); + } catch (error) { + const name = document[nameField]; + error.message = (name + ? `(name: ${inspect(name)}) ${error.message}` + : `(${color.dim(`no name found`)}) ${error.message}`); + throw error; + } + }; + }; + + return decorateErrorWithName(document => { const documentEntries = Object.entries(document) .filter(([ field ]) => !ignoredFields.includes(field)); @@ -758,7 +783,7 @@ function makeParseDocument(thingClass, { .filter(field => !knownFields.includes(field)); if (unknownFields.length) { - throw new makeParseDocument.UnknownFieldsError(unknownFields); + throw new makeProcessDocument.UnknownFieldsError(unknownFields); } const fieldValues = {}; @@ -789,17 +814,17 @@ function makeParseDocument(thingClass, { }); return thing; - }; + }); } -makeParseDocument.UnknownFieldsError = class UnknownFieldsError extends Error { +makeProcessDocument.UnknownFieldsError = class UnknownFieldsError extends Error { constructor(fields) { super(`Unknown fields present: ${fields.join(', ')}`); this.fields = fields; } }; -const parseAlbumDocument = makeParseDocument(Album, { +const processAlbumDocument = makeProcessDocument(Album, { fieldTransformations: { 'Artists': parseContributors, 'Cover Artists': parseContributors, @@ -851,7 +876,7 @@ const parseAlbumDocument = makeParseDocument(Album, { } }); -function parseAlbumEntryDocuments(documents) { +function processAlbumEntryDocuments(documents) { // Slightly separate meanings: tracks is the array of Track objects (and // only Track objects); trackGroups is the array of TrackGroup objects, // organizing (by string reference) the Track objects within the Album. @@ -906,7 +931,7 @@ function parseAlbumEntryDocuments(documents) { return {tracks, trackGroups}; } -const parseTrackGroupDocument = makeParseDocument(TrackGroup, { +const parseTrackGroupDocument = makeProcessDocument(TrackGroup, { fieldTransformations: { 'Date Originally Released': value => new Date(value), }, @@ -918,7 +943,7 @@ const parseTrackGroupDocument = makeParseDocument(TrackGroup, { } }); -const parseTrackDocument = makeParseDocument(Track, { +const parseTrackDocument = makeProcessDocument(Track, { fieldTransformations: { 'Duration': getDurationInSeconds, @@ -956,7 +981,7 @@ const parseTrackDocument = makeParseDocument(Track, { ignoredFields: ['Sampled Tracks'] }); -const processArtistDocument = makeParseDocument(Artist, { +const processArtistDocument = makeProcessDocument(Artist, { propertyFieldMapping: { name: 'Artist', @@ -971,40 +996,36 @@ const processArtistDocument = makeParseDocument(Artist, { ignoredFields: ['Dead URLs'] }); -async function processArtistDataFile(file) { - let contents; - try { - contents = await readFile(file, 'utf-8'); - } catch (error) { - return {error: `Could not read ${file} (${error.code}).`}; - } +const processFlashDocument = makeProcessDocument(Flash, { + fieldTransformations: { + 'Date': value => new Date(value), - const contentLines = splitLines(contents); - const sections = Array.from(getSections(contentLines)); + 'Contributors': parseContributors, + }, - return sections.filter(s => s.filter(Boolean).length).map(section => { - const name = getBasicField(section, 'Artist'); - const urls = (getListField(section, 'URLs') || []).filter(Boolean); - const alias = getBasicField(section, 'Alias'); - const hasAvatar = getBooleanField(section, 'Has Avatar') ?? false; - const note = getMultilineField(section, 'Note'); - let directory = getBasicField(section, 'Directory'); + propertyFieldMapping: { + name: 'Flash', - if (!name) { - return {error: 'Expected "Artist" (name) field!'}; - } + directory: 'Directory', + page: 'Page', + date: 'Date', + coverArtFileExtension: 'Cover Art File Extension', - if (!directory) { - directory = getKebabCase(name); - } + featuredTracksByRef: 'Featured Tracks', + contributorContribsByRef: 'Contributors', + urls: 'URLs' + }, +}); - if (alias) { - return {name, directory, alias}; - } else { - return {name, directory, urls, note, hasAvatar}; - } - }); -} +const processFlashActDocument = makeProcessDocument(FlashAct, { + propertyFieldMapping: { + name: 'Act', + color: 'Color', + anchor: 'Anchor', + jump: 'Jump', + jumpColor: 'Jump Color' + } +}); async function processFlashDataFile(file) { let contents; @@ -2421,8 +2442,8 @@ async function main() { files: albumDataFiles, documentMode: documentModes.headerAndEntries, - processHeaderDocument: parseAlbumDocument, - processEntryDocuments: parseAlbumEntryDocuments, + processHeaderDocument: processAlbumDocument, + processEntryDocuments: processAlbumEntryDocuments, save(results) { const albumData = []; @@ -2437,6 +2458,9 @@ async function main() { trackData.push(...tracks); } + sortByDate(albumData); + sortByDate(trackData); + Object.assign(wikiData, {albumData, trackData}); } }, @@ -2451,7 +2475,49 @@ async function main() { save(results) { wikiData.artistData = results; } - } + }, + + // TODO: WD.wikiInfo.features.flashesAndGames && + { + title: `Process flash file`, + files: [path.join(dataPath, FLASH_DATA_FILE)], + + documentMode: documentModes.allInOne, + processDocument(document) { + return ('Act' in document + ? processFlashActDocument(document) + : processFlashDocument(document)); + }, + + save(results) { + let flashAct; + let flashesByRef = []; + + if (results[0] && !(results[0] instanceof FlashAct)) { + throw new Error(`Expected an act at top of flash data file`); + } + + for (const thing of results) { + if (thing instanceof FlashAct) { + if (flashAct) { + Object.assign(flashAct, {flashesByRef}); + } + + flashAct = thing; + flashesByRef = []; + } else { + flashesByRef.push(Thing.getReference(thing)); + } + } + + if (flashAct) { + Object.assign(flashAct, {flashesByRef}); + } + + wikiData.flashData = results.filter(x => x instanceof Flash); + wikiData.flashActData = results.filter(x => x instanceof FlashAct); + } + }, ]; const processDataAggregate = openAggregate({message: `Errors processing data files`}); @@ -2487,7 +2553,7 @@ async function main() { for (const dataStep of dataSteps) { await processDataAggregate.nestAsync( {message: `Errors during data step: ${dataStep.title}`}, - async ({call, map, mapAsync}) => { + async ({call, callAsync, map, mapAsync}) => { const processDocuments = documentModeFunctions[dataStep.documentMode]; if (!processDocuments) { @@ -2500,15 +2566,32 @@ async function main() { } const file = dataStep.files[0]; - const readResult = await readFile(file); - const yamlResult = yaml.loadAll(readResult); + + const readResult = await callAsync(readFile, file); + + if (!readResult) { + return; + } + + const yamlResult = call(yaml.loadAll, readResult); + + if (!yamlResult) { + return; + } const { result: processResults, aggregate: processAggregate } = mapAggregate( yamlResult, - dataStep.processDocument, + (document, i) => { + try { + return dataStep.processDocument(document); + } catch (error) { + error.message = `(${color.yellow(`#${i + 1}`)}) ${error.message}`; + throw error; + } + }, {message: `Errors processing documents`} ); @@ -2544,71 +2627,33 @@ async function main() { }); } - logInfo`Loaded data and processed objects:`; - logInfo` - ${wikiData.albumData.length} albums`; - logInfo` - ${wikiData.trackData.length} tracks`; - logInfo` - ${wikiData.artistData.length} artists`; - - try { - processDataAggregate.close(); - } catch (error) { - showAggregate(error, {pathToFile: f => path.relative(__dirname, f)}); - logWarn`The above errors were detected while processing data files.`; - logWarn`If the remaining valid data is complete enough, the wiki will`; - logWarn`still build - but all errored data will be skipped.`; - logWarn`(Resolve errors for more complete output!)`; - } - - process.exit(); - { - const errors = WD.albumData.filter(obj => obj.error); - if (errors.length) { - for (const error of errors) { - console.log(`\x1b[31;1m${error.error}\x1b[0m`); - } - return; + logInfo`Loaded data and processed objects:`; + logInfo` - ${wikiData.albumData.length} albums`; + logInfo` - ${wikiData.trackData.length} tracks`; + logInfo` - ${wikiData.artistData.length} artists`; + if (wikiData.flashData) + logInfo` - ${wikiData.flashData.length} flashes (${wikiData.flashActData.length} acts)`; + + let errorless = true; + try { + processDataAggregate.close(); + } catch (error) { + showAggregate(error, {pathToFile: f => path.relative(__dirname, f)}); + logWarn`The above errors were detected while processing data files.`; + logWarn`If the remaining valid data is complete enough, the wiki will`; + logWarn`still build - but all errored data will be skipped.`; + logWarn`(Resolve errors for more complete output!)`; + errorless = false; } - } - - sortByDate(WD.albumData); - - WD.artistData = await processArtistDataFile(path.join(dataPath, ARTIST_DATA_FILE)); - if (WD.artistData.error) { - console.log(`\x1b[31;1m${WD.artistData.error}\x1b[0m`); - return; - } - { - const errors = WD.artistData.filter(obj => obj.error); - if (errors.length) { - for (const error of errors) { - console.log(`\x1b[31;1m${error.error}\x1b[0m`); - } - return; + if (errorless) { + logInfo`All data processed without any errors - nice!`; + logInfo`(This means all source files will be fully accounted for during page generation.)`; } } - WD.artistAliasData = WD.artistData.filter(x => x.alias); - WD.artistData = WD.artistData.filter(x => !x.alias); - - WD.trackData = getAllTracks(WD.albumData); - - if (WD.wikiInfo.features.flashesAndGames) { - WD.flashData = await processFlashDataFile(path.join(dataPath, FLASH_DATA_FILE)); - if (WD.flashData.error) { - console.log(`\x1b[31;1m${WD.flashData.error}\x1b[0m`); - return; - } - - const errors = WD.flashData.filter(obj => obj.error); - if (errors.length) { - for (const error of errors) { - console.log(`\x1b[31;1m${error.error}\x1b[0m`); - } - return; - } - } + process.exit(); WD.flashActData = WD.flashData?.filter(x => x.act8r8k); WD.flashData = WD.flashData?.filter(x => !x.act8r8k); diff --git a/src/util/sugar.js b/src/util/sugar.js index 774534c2..219c3eec 100644 --- a/src/util/sugar.js +++ b/src/util/sugar.js @@ -350,7 +350,7 @@ export function _withAggregate(mode, aggregateOpts, fn) { } } -export function showAggregate(topError, {pathToFile = null} = {}) { +export function showAggregate(topError, {pathToFile = p => p} = {}) { const recursive = (error, {level}) => { const stackLines = error.stack?.split('\n'); const stackLine = stackLines?.find(line => diff --git a/test/data-validators.js b/test/data-validators.js index 867068c7..feec127a 100644 --- a/test/data-validators.js +++ b/test/data-validators.js @@ -17,8 +17,12 @@ import { isDimensions, isDirectory, isDuration, + isFileExtension, validateReference, validateReferenceList, + + // Compositional utilities + oneOf, } from '../src/thing/validators.js'; function test(msg, fn) { @@ -157,6 +161,16 @@ test('isDuration', t => { t.throws(() => isDuration('10:25'), TypeError); }); +test('isFileExtension', t => { + t.plan(6); + t.ok(isFileExtension('png')); + t.ok(isFileExtension('jpg')); + t.ok(isFileExtension('sub_loc')); + t.throws(() => isFileExtension(''), TypeError); + t.throws(() => isFileExtension('.jpg'), TypeError); + t.throws(() => isFileExtension('just an image bro!!!!'), TypeError); +}); + test.skip('isName', t => { // TODO }); @@ -216,3 +230,37 @@ test('validateReferenceList', t => { t.true(caughtError.errors[0] instanceof TypeError); t.true(caughtError.errors[1] instanceof TypeError); }); + +test('oneOf', t => { + t.plan(11); + + const isStringOrNumber = oneOf(isString, isNumber); + + t.ok(isStringOrNumber('hello world')); + t.ok(isStringOrNumber(42)); + t.throws(() => isStringOrNumber(false)); + + const mockError = new Error(); + const neverSucceeds = () => { + throw mockError; + }; + + const isStringOrGetRekt = oneOf(isString, neverSucceeds); + + t.ok(isStringOrGetRekt('phew!')); + + let caughtError = null; + try { + isStringOrGetRekt(0xdeadbeef); + } catch (err) { + caughtError = err; + } + + t.isNot(caughtError, null); + t.true(caughtError instanceof AggregateError); + t.is(caughtError.errors.length, 2); + t.true(caughtError.errors[0] instanceof TypeError); + t.is(caughtError.errors[0].check, isString); + t.is(caughtError.errors[1], mockError); + t.is(caughtError.errors[1].check, neverSucceeds); +}); |