From eb8b316bb9af7d34720de1fa8f8dbd4b81513b32 Mon Sep 17 00:00:00 2001 From: "(quasar) nebula" Date: Mon, 29 Jan 2024 08:10:30 -0400 Subject: data, yaml: store data step info on Thing constructors --- src/data/thing.js | 1 + src/data/things/album.js | 87 ++++++++++ src/data/things/art-tag.js | 21 ++- src/data/things/artist.js | 31 ++++ src/data/things/flash.js | 47 ++++++ src/data/things/group.js | 47 ++++++ src/data/things/homepage-layout.js | 38 +++++ src/data/things/news-entry.js | 21 +++ src/data/things/static-page.js | 28 ++++ src/data/things/track.js | 3 + src/data/things/wiki-info.js | 21 +++ src/data/yaml.js | 325 ++----------------------------------- 12 files changed, 361 insertions(+), 309 deletions(-) (limited to 'src/data') diff --git a/src/data/thing.js b/src/data/thing.js index ae8e71af..028c0dcf 100644 --- a/src/data/thing.js +++ b/src/data/thing.js @@ -14,6 +14,7 @@ export default class Thing extends CacheableObject { static getSerializeDescriptors = Symbol.for('Thing.getSerializeDescriptors'); static yamlDocumentSpec = Symbol.for('Thing.yamlDocumentSpec'); + static getYamlLoadingSpec = Symbol.for('Thing.getYamlLoadingSpec'); // Default custom inspect function, which may be overridden by Thing // subclasses. This will be used when displaying aggregate errors and other diff --git a/src/data/things/album.js b/src/data/things/album.js index 9e6d28d9..9d4729e4 100644 --- a/src/data/things/album.js +++ b/src/data/things/album.js @@ -1,5 +1,11 @@ +export const DATA_ALBUM_DIRECTORY = 'album'; + +import * as path from 'node:path'; + import {input} from '#composite'; import find from '#find'; +import {traverse} from '#node-utils'; +import {empty} from '#sugar'; import Thing from '#thing'; import {isDate} from '#validators'; import {parseAdditionalFiles, parseContributors, parseDate, parseDimensions} @@ -283,6 +289,87 @@ export class Album extends Thing { 'Review Points': {ignore: true}, }, }; + + static [Thing.getYamlLoadingSpec] = ({ + documentModes: {headerAndEntries}, + thingConstructors: {Album, Track, TrackSectionHelper}, + }) => ({ + title: `Process album files`, + + files: dataPath => + traverse(path.join(dataPath, DATA_ALBUM_DIRECTORY), { + filterFile: name => path.extname(name) === '.yaml', + prefixPath: DATA_ALBUM_DIRECTORY, + }), + + documentMode: headerAndEntries, + headerDocumentThing: Album, + entryDocumentThing: document => + ('Section' in document + ? TrackSectionHelper + : Track), + + save(results) { + const albumData = []; + const trackData = []; + + for (const {header: album, entries} of results) { + // We can't mutate an array once it's set as a property value, + // so prepare the track sections that will show up in a track list + // all the way before actually applying them. (It's okay to mutate + // an individual section before applying it, since those are just + // generic objects; they aren't Things in and of themselves.) + const trackSections = []; + const ownTrackData = []; + + let currentTrackSection = { + name: `Default Track Section`, + isDefaultTrackSection: true, + tracks: [], + }; + + const albumRef = Thing.getReference(album); + + const closeCurrentTrackSection = () => { + if (!empty(currentTrackSection.tracks)) { + trackSections.push(currentTrackSection); + } + }; + + for (const entry of entries) { + if (entry instanceof TrackSectionHelper) { + closeCurrentTrackSection(); + + currentTrackSection = { + name: entry.name, + color: entry.color, + dateOriginallyReleased: entry.dateOriginallyReleased, + isDefaultTrackSection: false, + tracks: [], + }; + + continue; + } + + trackData.push(entry); + + entry.dataSourceAlbum = albumRef; + + ownTrackData.push(entry); + currentTrackSection.tracks.push(Thing.getReference(entry)); + } + + closeCurrentTrackSection(); + + albumData.push(album); + + album.trackSections = trackSections; + album.ownTrackData = ownTrackData; + } + + return {albumData, trackData}; + }, + }); } export class TrackSectionHelper extends Thing { diff --git a/src/data/things/art-tag.js b/src/data/things/art-tag.js index af6677f0..29cd2990 100644 --- a/src/data/things/art-tag.js +++ b/src/data/things/art-tag.js @@ -1,7 +1,9 @@ +export const ART_TAG_DATA_FILE = 'tags.yaml'; + import {input} from '#composite'; import Thing from '#thing'; import {isName} from '#validators'; -import {sortAlbumsTracksChronologically} from '#wiki-data'; +import {sortAlphabetically, sortAlbumsTracksChronologically} from '#wiki-data'; import {exposeUpdateValueOrContinue} from '#composite/control-flow'; @@ -73,4 +75,21 @@ export class ArtTag extends Thing { 'Is CW': {property: 'isContentWarning'}, }, }; + + static [Thing.getYamlLoadingSpec] = ({ + documentModes: {allInOne}, + thingConstructors: {ArtTag}, + }) => ({ + title: `Process art tags file`, + file: ART_TAG_DATA_FILE, + + documentMode: allInOne, + documentThing: ArtTag, + + save(artTagData) { + sortAlphabetically(artTagData); + + return {artTagData}; + }, + }); } diff --git a/src/data/things/artist.js b/src/data/things/artist.js index ab08f522..8f32afd7 100644 --- a/src/data/things/artist.js +++ b/src/data/things/artist.js @@ -1,3 +1,5 @@ +export const ARTIST_DATA_FILE = 'artists.yaml'; + import {input} from '#composite'; import find from '#find'; import {unique} from '#sugar'; @@ -257,4 +259,33 @@ export class Artist extends Thing { 'Review Points': {ignore: true}, }, }; + + static [Thing.getYamlLoadingSpec] = ({ + documentModes: {allInOne}, + thingConstructors: {Artist}, + }) => ({ + title: `Process artists file`, + file: ARTIST_DATA_FILE, + + documentMode: allInOne, + documentThing: Artist, + + save(results) { + const artistData = results; + + const artistAliasData = results.flatMap((artist) => { + const origRef = Thing.getReference(artist); + return artist.aliasNames?.map((name) => { + const alias = new Artist(); + alias.name = name; + alias.isAlias = true; + alias.aliasedArtist = origRef; + alias.artistData = artistData; + return alias; + }) ?? []; + }); + + return {artistData, artistAliasData}; + }, + }); } diff --git a/src/data/things/flash.js b/src/data/things/flash.js index 945d80dd..ad80cbf2 100644 --- a/src/data/things/flash.js +++ b/src/data/things/flash.js @@ -1,3 +1,5 @@ +export const FLASH_DATA_FILE = 'flashes.yaml'; + import {input} from '#composite'; import find from '#find'; import Thing from '#thing'; @@ -204,4 +206,49 @@ export class FlashAct extends Thing { 'Review Points': {ignore: true}, }, }; + + static [Thing.getYamlLoadingSpec] = ({ + documentModes: {allInOne}, + thingConstructors: {Flash, FlashAct}, + }) => ({ + title: `Process flashes file`, + file: FLASH_DATA_FILE, + + documentMode: allInOne, + documentThing: document => + ('Act' in document + ? FlashAct + : Flash), + + save(results) { + let flashAct; + let flashRefs = []; + + if (results[0] && !(results[0] instanceof FlashAct)) { + throw new Error(`Expected an act at top of flash data file`); + } + + for (const thing of results) { + if (thing instanceof FlashAct) { + if (flashAct) { + Object.assign(flashAct, {flashes: flashRefs}); + } + + flashAct = thing; + flashRefs = []; + } else { + flashRefs.push(Thing.getReference(thing)); + } + } + + if (flashAct) { + Object.assign(flashAct, {flashes: flashRefs}); + } + + const flashData = results.filter(x => x instanceof Flash); + const flashActData = results.filter(x => x instanceof FlashAct); + + return {flashData, flashActData}; + }, + }); } diff --git a/src/data/things/group.js b/src/data/things/group.js index adcd6ad1..a32cd64d 100644 --- a/src/data/things/group.js +++ b/src/data/things/group.js @@ -1,3 +1,5 @@ +export const GROUP_DATA_FILE = 'groups.yaml'; + import {input} from '#composite'; import find from '#find'; import Thing from '#thing'; @@ -97,6 +99,51 @@ export class Group extends Thing { 'Review Points': {ignore: true}, }, }; + + static [Thing.getYamlLoadingSpec] = ({ + documentModes: {allInOne}, + thingConstructors: {Group, GroupCategory}, + }) => ({ + title: `Process groups file`, + file: GROUP_DATA_FILE, + + documentMode: allInOne, + documentThing: document => + ('Category' in document + ? GroupCategory + : Group), + + save(results) { + let groupCategory; + let groupRefs = []; + + if (results[0] && !(results[0] instanceof GroupCategory)) { + throw new Error(`Expected a category at top of group data file`); + } + + for (const thing of results) { + if (thing instanceof GroupCategory) { + if (groupCategory) { + Object.assign(groupCategory, {groups: groupRefs}); + } + + groupCategory = thing; + groupRefs = []; + } else { + groupRefs.push(Thing.getReference(thing)); + } + } + + if (groupCategory) { + Object.assign(groupCategory, {groups: groupRefs}); + } + + const groupData = results.filter(x => x instanceof Group); + const groupCategoryData = results.filter(x => x instanceof GroupCategory); + + return {groupData, groupCategoryData}; + }, + }); } export class GroupCategory extends Thing { diff --git a/src/data/things/homepage-layout.js b/src/data/things/homepage-layout.js index 38fd5a7a..00d6aef5 100644 --- a/src/data/things/homepage-layout.js +++ b/src/data/things/homepage-layout.js @@ -1,3 +1,5 @@ +export const HOMEPAGE_LAYOUT_DATA_FILE = 'homepage.yaml'; + import {input} from '#composite'; import find from '#find'; import Thing from '#thing'; @@ -182,4 +184,40 @@ export class HomepageLayoutAlbumsRow extends HomepageLayoutRow { 'Actions': {property: 'actionLinks'}, }, }); + + static [Thing.getYamlLoadingSpec] = ({ + documentModes: {headerAndEntries}, // Kludge, see below + thingConstructors: { + HomepageLayout, + HomepageLayoutAlbumsRow, + }, + }) => ({ + title: `Process homepage layout file`, + + // Kludge: This benefits from the same headerAndEntries style messaging as + // albums and tracks (for example), but that document mode is designed to + // support multiple files, and only one is actually getting processed here. + files: [HOMEPAGE_LAYOUT_DATA_FILE], + + documentMode: headerAndEntries, + headerDocumentThing: HomepageLayout, + entryDocumentThing: document => { + switch (document['Type']) { + case 'albums': + return HomepageLayoutAlbumsRow; + default: + throw new TypeError(`No processDocument function for row type ${document['Type']}!`); + } + }, + + save(results) { + if (!results[0]) { + return; + } + + const {header: homepageLayout, entries: rows} = results[0]; + Object.assign(homepageLayout, {rows}); + return {homepageLayout}; + }, + }); } diff --git a/src/data/things/news-entry.js b/src/data/things/news-entry.js index 5a022449..658453b0 100644 --- a/src/data/things/news-entry.js +++ b/src/data/things/news-entry.js @@ -1,4 +1,7 @@ +export const NEWS_DATA_FILE = 'news.yaml'; + import Thing from '#thing'; +import {sortChronologically} from '#wiki-data'; import {parseDate} from '#yaml'; import {contentString, directory, name, simpleDate} @@ -43,4 +46,22 @@ export class NewsEntry extends Thing { 'Content': {property: 'content'}, }, }; + + static [Thing.getYamlLoadingSpec] = ({ + documentModes: {allInOne}, + thingConstructors: {NewsEntry}, + }) => ({ + title: `Process news data file`, + file: NEWS_DATA_FILE, + + documentMode: allInOne, + documentThing: NewsEntry, + + save(newsData) { + sortChronologically(newsData); + newsData.reverse(); + + return {newsData}; + }, + }); } diff --git a/src/data/things/static-page.js b/src/data/things/static-page.js index 2da7312b..1e8cb7c6 100644 --- a/src/data/things/static-page.js +++ b/src/data/things/static-page.js @@ -1,5 +1,11 @@ +export const DATA_STATIC_PAGE_DIRECTORY = 'static-page'; + +import * as path from 'node:path'; + +import {traverse} from '#node-utils'; import Thing from '#thing'; import {isName} from '#validators'; +import {sortAlphabetically} from '#wiki-data'; import {contentString, directory, name, simpleString} from '#composite/wiki-properties'; @@ -42,4 +48,26 @@ export class StaticPage extends Thing { 'Review Points': {ignore: true}, }, }; + + static [Thing.getYamlLoadingSpec] = ({ + documentModes: {onePerFile}, + thingConstructors: {StaticPage}, + }) => ({ + title: `Process static page files`, + + files: dataPath => + traverse(path.join(dataPath, DATA_STATIC_PAGE_DIRECTORY), { + filterFile: name => path.extname(name) === '.yaml', + prefixPath: DATA_STATIC_PAGE_DIRECTORY, + }), + + documentMode: onePerFile, + documentThing: StaticPage, + + save(staticPageData) { + sortAlphabetically(staticPageData); + + return {staticPageData}; + }, + }); } diff --git a/src/data/things/track.js b/src/data/things/track.js index dd102683..d1a12aac 100644 --- a/src/data/things/track.js +++ b/src/data/things/track.js @@ -451,6 +451,9 @@ export class Track extends Thing { ], }; + // Track YAML loading is handled in album.js. + static [Thing.getYamlLoadingSpec] = null; + [inspect.custom](depth) { const parts = []; diff --git a/src/data/things/wiki-info.js b/src/data/things/wiki-info.js index fd6c239c..316bd3bb 100644 --- a/src/data/things/wiki-info.js +++ b/src/data/things/wiki-info.js @@ -1,3 +1,5 @@ +export const WIKI_INFO_FILE = 'wiki-info.yaml'; + import {input} from '#composite'; import find from '#find'; import Thing from '#thing'; @@ -86,4 +88,23 @@ export class WikiInfo extends Thing { 'Enable Group UI': {property: 'enableGroupUI'}, }, }; + + static [Thing.getYamlLoadingSpec] = ({ + documentModes: {oneDocumentTotal}, + thingConstructors: {WikiInfo}, + }) => ({ + title: `Process wiki info file`, + file: WIKI_INFO_FILE, + + documentMode: oneDocumentTotal, + documentThing: WikiInfo, + + save(wikiInfo) { + if (!wikiInfo) { + return; + } + + return {wikiInfo}; + }, + }); } diff --git a/src/data/yaml.js b/src/data/yaml.js index fe8bfdc0..b0ec3c9c 100644 --- a/src/data/yaml.js +++ b/src/data/yaml.js @@ -7,13 +7,11 @@ import {inspect as nodeInspect} from 'node:util'; import yaml from 'js-yaml'; -import CacheableObject, {CacheableObjectPropertyValueError} - from '#cacheable-object'; +import CacheableObject from '#cacheable-object'; import {colors, ENABLE_COLOR, logInfo, logWarn} from '#cli'; import find, {bindFind} from '#find'; -import {traverse} from '#node-utils'; import Thing from '#thing'; -import T from '#things'; +import thingConstructors from '#things'; import { annotateErrorWithFile, @@ -34,32 +32,15 @@ import { import { commentaryRegex, sortAlbumsTracksChronologically, - sortAlphabetically, + sortByName, sortChronologically, sortFlashesChronologically, } from '#wiki-data'; -// --> General supporting stuff - function inspect(value, opts = {}) { return nodeInspect(value, {colors: ENABLE_COLOR, ...opts}); } -// --> YAML data repository structure constants - -export const ART_TAG_DATA_FILE = 'tags.yaml'; -export const ARTIST_DATA_FILE = 'artists.yaml'; -export const FLASH_DATA_FILE = 'flashes.yaml'; -export const GROUP_DATA_FILE = 'groups.yaml'; -export const HOMEPAGE_LAYOUT_DATA_FILE = 'homepage.yaml'; -export const NEWS_DATA_FILE = 'news.yaml'; -export const WIKI_INFO_FILE = 'wiki-info.yaml'; - -export const DATA_ALBUM_DIRECTORY = 'album'; -export const DATA_STATIC_PAGE_DIRECTORY = 'static-page'; - -// --> Document processing functions - // General function for inputting a single document (usually loaded from YAML) // and outputting an instance of a provided Thing subclass. // @@ -372,8 +353,6 @@ export class SkippedFieldsSummaryError extends Error { } } -// --> Utilities shared across document parsing functions - export function parseDate(date) { return new Date(date); } @@ -480,8 +459,6 @@ export function parseDimensions(string) { return nums; } -// --> Data repository loading functions and descriptors - // documentModes: Symbols indicating sets of behavior for loading and processing // data files. export const documentModes = { @@ -554,291 +531,23 @@ export const documentModes = { // them to each other, setting additional properties, etc). Input argument // format depends on documentMode. // -export const getDataSteps = () => [ - { - title: `Process wiki info file`, - file: WIKI_INFO_FILE, - - documentMode: documentModes.oneDocumentTotal, - documentThing: T.WikiInfo, - - save(wikiInfo) { - if (!wikiInfo) { - return; - } - - return {wikiInfo}; - }, - }, +export const getDataSteps = () => { + const steps = []; - { - title: `Process album files`, - - files: dataPath => - traverse(path.join(dataPath, DATA_ALBUM_DIRECTORY), { - filterFile: name => path.extname(name) === '.yaml', - prefixPath: DATA_ALBUM_DIRECTORY, - }), - - documentMode: documentModes.headerAndEntries, - headerDocumentThing: T.Album, - entryDocumentThing: document => - ('Section' in document - ? T.TrackSectionHelper - : T.Track), - - save(results) { - const albumData = []; - const trackData = []; - - for (const {header: album, entries} of results) { - // We can't mutate an array once it's set as a property value, - // so prepare the track sections that will show up in a track list - // all the way before actually applying them. (It's okay to mutate - // an individual section before applying it, since those are just - // generic objects; they aren't Things in and of themselves.) - const trackSections = []; - const ownTrackData = []; - - let currentTrackSection = { - name: `Default Track Section`, - isDefaultTrackSection: true, - tracks: [], - }; - - const albumRef = Thing.getReference(album); - - const closeCurrentTrackSection = () => { - if (!empty(currentTrackSection.tracks)) { - trackSections.push(currentTrackSection); - } - }; - - for (const entry of entries) { - if (entry instanceof T.TrackSectionHelper) { - closeCurrentTrackSection(); - - currentTrackSection = { - name: entry.name, - color: entry.color, - dateOriginallyReleased: entry.dateOriginallyReleased, - isDefaultTrackSection: false, - tracks: [], - }; - - continue; - } + for (const thing of Object.values(thingConstructors)) { + const getSpecFn = thing[Thing.getYamlLoadingSpec]; + if (!getSpecFn) continue; - trackData.push(entry); - - entry.dataSourceAlbum = albumRef; - - ownTrackData.push(entry); - currentTrackSection.tracks.push(Thing.getReference(entry)); - } - - closeCurrentTrackSection(); - - albumData.push(album); - - album.trackSections = trackSections; - album.ownTrackData = ownTrackData; - } - - return {albumData, trackData}; - }, - }, - - { - title: `Process artists file`, - file: ARTIST_DATA_FILE, - - documentMode: documentModes.allInOne, - documentThing: T.Artist, - - save(results) { - const artistData = results; - - const artistAliasData = results.flatMap((artist) => { - const origRef = Thing.getReference(artist); - return artist.aliasNames?.map((name) => { - const alias = new T.Artist(); - alias.name = name; - alias.isAlias = true; - alias.aliasedArtist = origRef; - alias.artistData = artistData; - return alias; - }) ?? []; - }); - - return {artistData, artistAliasData}; - }, - }, - - // TODO: WD.wikiInfo.enableFlashesAndGames && - { - title: `Process flashes file`, - file: FLASH_DATA_FILE, - - documentMode: documentModes.allInOne, - documentThing: document => - ('Act' in document - ? T.FlashAct - : T.Flash), - - save(results) { - let flashAct; - let flashRefs = []; - - if (results[0] && !(results[0] instanceof T.FlashAct)) { - throw new Error(`Expected an act at top of flash data file`); - } - - for (const thing of results) { - if (thing instanceof T.FlashAct) { - if (flashAct) { - Object.assign(flashAct, {flashes: flashRefs}); - } - - flashAct = thing; - flashRefs = []; - } else { - flashRefs.push(Thing.getReference(thing)); - } - } - - if (flashAct) { - Object.assign(flashAct, {flashes: flashRefs}); - } - - const flashData = results.filter((x) => x instanceof T.Flash); - const flashActData = results.filter((x) => x instanceof T.FlashAct); - - return {flashData, flashActData}; - }, - }, - - { - title: `Process groups file`, - file: GROUP_DATA_FILE, - - documentMode: documentModes.allInOne, - documentThing: document => - ('Category' in document - ? T.GroupCategory - : T.Group), - - save(results) { - let groupCategory; - let groupRefs = []; - - if (results[0] && !(results[0] instanceof T.GroupCategory)) { - throw new Error(`Expected a category at top of group data file`); - } - - for (const thing of results) { - if (thing instanceof T.GroupCategory) { - if (groupCategory) { - Object.assign(groupCategory, {groups: groupRefs}); - } - - groupCategory = thing; - groupRefs = []; - } else { - groupRefs.push(Thing.getReference(thing)); - } - } - - if (groupCategory) { - Object.assign(groupCategory, {groups: groupRefs}); - } - - const groupData = results.filter((x) => x instanceof T.Group); - const groupCategoryData = results.filter((x) => x instanceof T.GroupCategory); - - return {groupData, groupCategoryData}; - }, - }, - - { - title: `Process homepage layout file`, - - // Kludge: This benefits from the same headerAndEntries style messaging as - // albums and tracks (for example), but that document mode is designed to - // support multiple files, and only one is actually getting processed here. - files: [HOMEPAGE_LAYOUT_DATA_FILE], - - documentMode: documentModes.headerAndEntries, - headerDocumentThing: T.HomepageLayout, - entryDocumentThing: document => { - switch (document['Type']) { - case 'albums': - return T.HomepageLayoutAlbumsRow; - default: - throw new TypeError(`No processDocument function for row type ${document['Type']}!`); - } - }, - - save(results) { - if (!results[0]) { - return; - } - - const {header: homepageLayout, entries: rows} = results[0]; - Object.assign(homepageLayout, {rows}); - return {homepageLayout}; - }, - }, - - // TODO: WD.wikiInfo.enableNews && - { - title: `Process news data file`, - file: NEWS_DATA_FILE, - - documentMode: documentModes.allInOne, - documentThing: T.NewsEntry, - - save(newsData) { - sortChronologically(newsData); - newsData.reverse(); - - return {newsData}; - }, - }, - - { - title: `Process art tags file`, - file: ART_TAG_DATA_FILE, - - documentMode: documentModes.allInOne, - documentThing: T.ArtTag, - - save(artTagData) { - sortAlphabetically(artTagData); - - return {artTagData}; - }, - }, - - { - title: `Process static page files`, - - files: dataPath => - traverse(path.join(dataPath, DATA_STATIC_PAGE_DIRECTORY), { - filterFile: name => path.extname(name) === '.yaml', - prefixPath: DATA_STATIC_PAGE_DIRECTORY, - }), - - documentMode: documentModes.onePerFile, - documentThing: T.StaticPage, + steps.push(getSpecFn({ + documentModes, + thingConstructors, + })); + } - save(staticPageData) { - sortAlphabetically(staticPageData); + sortByName(steps, {getName: step => step.title}); - return {staticPageData}; - }, - }, -]; + return steps; +}; export async function loadAndProcessDataDocuments({dataPath}) { const processDataAggregate = openAggregate({ @@ -1605,7 +1314,7 @@ export function filterReferenceErrors(wikiData) { let hasCoverArtwork = !empty(CacheableObject.getUpdateValue(thing, 'coverArtistContribs')); - if (thing.constructor === T.Track) { + if (thing.constructor === thingConstructors.Track) { if (thing.album) { hasCoverArtwork ||= !empty(CacheableObject.getUpdateValue(thing.album, 'trackCoverArtistContribs')); -- cgit 1.3.0-6-gf8a5