diff options
Diffstat (limited to 'src/data/yaml.js')
| -rw-r--r-- | src/data/yaml.js | 209 |
1 files changed, 119 insertions, 90 deletions
diff --git a/src/data/yaml.js b/src/data/yaml.js index 07e0a3d2..c799be5f 100644 --- a/src/data/yaml.js +++ b/src/data/yaml.js @@ -7,10 +7,10 @@ import {inspect as nodeInspect} from 'node:util'; import yaml from 'js-yaml'; -import {color, ENABLE_COLOR, logInfo, logWarn} from '#cli'; +import {colors, ENABLE_COLOR, logInfo, logWarn} from '#cli'; import find, {bindFind} from '#find'; import {traverse} from '#node-utils'; -import T from '#things'; +import T, {CacheableObject, Thing} from '#things'; import { conditionallySuppressError, @@ -137,7 +137,7 @@ function makeProcessDocument( const name = document[nameField]; error.message = name ? `(name: ${inspect(name)}) ${error.message}` - : `(${color.dim(`no name found`)}) ${error.message}`; + : `(${colors.dim(`no name found`)}) ${error.message}`; throw error; } }; @@ -195,7 +195,7 @@ function makeProcessDocument( const thing = Reflect.construct(thingClass, []); - withAggregate({message: `Errors applying ${color.green(thingClass.name)} properties`}, ({call}) => { + withAggregate({message: `Errors applying ${colors.green(thingClass.name)} properties`}, ({call}) => { for (const [property, value] of Object.entries(sourceProperties)) { call(() => (thing[property] = value)); } @@ -228,7 +228,7 @@ makeProcessDocument.FieldCombinationsError = class FieldCombinationsError extend makeProcessDocument.FieldCombinationError = class FieldCombinationError extends Error { constructor(fields, message) { const fieldNames = Object.keys(fields); - const combinePart = `Don't combine ${fieldNames.map(field => color.red(field)).join(', ')}`; + const combinePart = `Don't combine ${fieldNames.map(field => colors.red(field)).join(', ')}`; const messagePart = (typeof message === 'function' @@ -278,11 +278,11 @@ export const processAlbumDocument = makeProcessDocument(T.Album, { coverArtFileExtension: 'Cover Art File Extension', trackCoverArtFileExtension: 'Track Art File Extension', - wallpaperArtistContribsByRef: 'Wallpaper Artists', + wallpaperArtistContribs: 'Wallpaper Artists', wallpaperStyle: 'Wallpaper Style', wallpaperFileExtension: 'Wallpaper File Extension', - bannerArtistContribsByRef: 'Banner Artists', + bannerArtistContribs: 'Banner Artists', bannerStyle: 'Banner Style', bannerFileExtension: 'Banner File Extension', bannerDimensions: 'Banner Dimensions', @@ -290,11 +290,11 @@ export const processAlbumDocument = makeProcessDocument(T.Album, { commentary: 'Commentary', additionalFiles: 'Additional Files', - artistContribsByRef: 'Artists', - coverArtistContribsByRef: 'Cover Artists', - trackCoverArtistContribsByRef: 'Default Track Cover Artists', - groupsByRef: 'Groups', - artTagsByRef: 'Art Tags', + artistContribs: 'Artists', + coverArtistContribs: 'Cover Artists', + trackCoverArtistContribs: 'Default Track Cover Artists', + groups: 'Groups', + artTags: 'Art Tags', }, }); @@ -316,6 +316,10 @@ export const processTrackDocument = makeProcessDocument(T.Track, { 'Date First Released': (value) => new Date(value), 'Cover Art Date': (value) => new Date(value), + 'Has Cover Art': (value) => + (value === true ? false : + value === false ? true : + value), 'Artists': parseContributors, 'Contributors': parseContributors, @@ -336,7 +340,7 @@ export const processTrackDocument = makeProcessDocument(T.Track, { dateFirstReleased: 'Date First Released', coverArtDate: 'Cover Art Date', coverArtFileExtension: 'Cover Art File Extension', - hasCoverArt: 'Has Cover Art', + disableUniqueCoverArt: 'Has Cover Art', // This gets transformed to flip true/false. alwaysReferenceByDirectory: 'Always Reference By Directory', @@ -346,13 +350,13 @@ export const processTrackDocument = makeProcessDocument(T.Track, { sheetMusicFiles: 'Sheet Music Files', midiProjectFiles: 'MIDI Project Files', - originalReleaseTrackByRef: 'Originally Released As', - referencedTracksByRef: 'Referenced Tracks', - sampledTracksByRef: 'Sampled Tracks', - artistContribsByRef: 'Artists', - contributorContribsByRef: 'Contributors', - coverArtistContribsByRef: 'Cover Artists', - artTagsByRef: 'Art Tags', + originalReleaseTrack: 'Originally Released As', + referencedTracks: 'Referenced Tracks', + sampledTracks: 'Sampled Tracks', + artistContribs: 'Artists', + contributorContribs: 'Contributors', + coverArtistContribs: 'Cover Artists', + artTags: 'Art Tags', }, invalidFieldCombinations: [ @@ -422,8 +426,8 @@ export const processFlashDocument = makeProcessDocument(T.Flash, { date: 'Date', coverArtFileExtension: 'Cover Art File Extension', - featuredTracksByRef: 'Featured Tracks', - contributorContribsByRef: 'Contributors', + featuredTracks: 'Featured Tracks', + contributorContribs: 'Contributors', }, }); @@ -468,7 +472,7 @@ export const processGroupDocument = makeProcessDocument(T.Group, { description: 'Description', urls: 'URLs', - featuredAlbumsByRef: 'Featured Albums', + featuredAlbums: 'Featured Albums', }, }); @@ -499,7 +503,7 @@ export const processWikiInfoDocument = makeProcessDocument(T.WikiInfo, { footerContent: 'Footer Content', defaultLanguage: 'Default Language', canonicalBase: 'Canonical Base', - divideTrackListsByGroupsByRef: 'Divide Track Lists By Groups', + divideTrackListsByGroups: 'Divide Track Lists By Groups', enableFlashesAndGames: 'Enable Flashes & Games', enableListings: 'Enable Listings', enableNews: 'Enable News', @@ -534,9 +538,9 @@ export const homepageLayoutRowTypeProcessMapping = { albums: makeProcessHomepageLayoutRowDocument(T.HomepageLayoutAlbumsRow, { propertyFieldMapping: { displayStyle: 'Display Style', - sourceGroupByRef: 'Group', + sourceGroup: 'Group', countAlbumsFromGroup: 'Count', - sourceAlbumsByRef: 'Albums', + sourceAlbums: 'Albums', actionLinks: 'Actions', }, }), @@ -769,13 +773,13 @@ export const dataSteps = [ let currentTrackSection = { name: `Default Track Section`, isDefaultTrackSection: true, - tracksByRef: [], + tracks: [], }; - const albumRef = T.Thing.getReference(album); + const albumRef = Thing.getReference(album); const closeCurrentTrackSection = () => { - if (!empty(currentTrackSection.tracksByRef)) { + if (!empty(currentTrackSection.tracks)) { trackSections.push(currentTrackSection); } }; @@ -789,7 +793,7 @@ export const dataSteps = [ color: entry.color, dateOriginallyReleased: entry.dateOriginallyReleased, isDefaultTrackSection: false, - tracksByRef: [], + tracks: [], }; continue; @@ -797,9 +801,9 @@ export const dataSteps = [ trackData.push(entry); - entry.dataSourceAlbumByRef = albumRef; + entry.dataSourceAlbum = albumRef; - currentTrackSection.tracksByRef.push(T.Thing.getReference(entry)); + currentTrackSection.tracks.push(Thing.getReference(entry)); } closeCurrentTrackSection(); @@ -823,12 +827,12 @@ export const dataSteps = [ const artistData = results; const artistAliasData = results.flatMap((artist) => { - const origRef = T.Thing.getReference(artist); + const origRef = Thing.getReference(artist); return artist.aliasNames?.map((name) => { const alias = new T.Artist(); alias.name = name; alias.isAlias = true; - alias.aliasedArtistRef = origRef; + alias.aliasedArtist = origRef; alias.artistData = artistData; return alias; }) ?? []; @@ -852,7 +856,7 @@ export const dataSteps = [ save(results) { let flashAct; - let flashesByRef = []; + let flashRefs = []; if (results[0] && !(results[0] instanceof T.FlashAct)) { throw new Error(`Expected an act at top of flash data file`); @@ -861,18 +865,18 @@ export const dataSteps = [ for (const thing of results) { if (thing instanceof T.FlashAct) { if (flashAct) { - Object.assign(flashAct, {flashesByRef}); + Object.assign(flashAct, {flashes: flashRefs}); } flashAct = thing; - flashesByRef = []; + flashRefs = []; } else { - flashesByRef.push(T.Thing.getReference(thing)); + flashRefs.push(Thing.getReference(thing)); } } if (flashAct) { - Object.assign(flashAct, {flashesByRef}); + Object.assign(flashAct, {flashes: flashRefs}); } const flashData = results.filter((x) => x instanceof T.Flash); @@ -895,7 +899,7 @@ export const dataSteps = [ save(results) { let groupCategory; - let groupsByRef = []; + let groupRefs = []; if (results[0] && !(results[0] instanceof T.GroupCategory)) { throw new Error(`Expected a category at top of group data file`); @@ -904,18 +908,18 @@ export const dataSteps = [ for (const thing of results) { if (thing instanceof T.GroupCategory) { if (groupCategory) { - Object.assign(groupCategory, {groupsByRef}); + Object.assign(groupCategory, {groups: groupRefs}); } groupCategory = thing; - groupsByRef = []; + groupRefs = []; } else { - groupsByRef.push(T.Thing.getReference(thing)); + groupRefs.push(Thing.getReference(thing)); } } if (groupCategory) { - Object.assign(groupCategory, {groupsByRef}); + Object.assign(groupCategory, {groups: groupRefs}); } const groupData = results.filter((x) => x instanceof T.Group); @@ -1007,7 +1011,7 @@ export async function loadAndProcessDataDocuments({dataPath}) { } catch (error) { error.message += (error.message.includes('\n') ? '\n' : ' ') + - `(file: ${color.bright(color.blue(path.relative(dataPath, x.file)))})`; + `(file: ${colors.bright(colors.blue(path.relative(dataPath, x.file)))})`; throw error; } }; @@ -1030,7 +1034,7 @@ export async function loadAndProcessDataDocuments({dataPath}) { // just without the callbacks. Thank you. const filterBlankDocuments = documents => { const aggregate = openAggregate({ - message: `Found blank documents - check for extra '${color.cyan(`---`)}'`, + message: `Found blank documents - check for extra '${colors.cyan(`---`)}'`, }); const filteredDocuments = @@ -1074,10 +1078,10 @@ export async function loadAndProcessDataDocuments({dataPath}) { if (count === 1) { const range = `#${start + 1}`; - parts.push(`${count} document (${color.yellow(range)}), `); + parts.push(`${count} document (${colors.yellow(range)}), `); } else { const range = `#${start + 1}-${end + 1}`; - parts.push(`${count} documents (${color.yellow(range)}), `); + parts.push(`${count} documents (${colors.yellow(range)}), `); } if (previous === null) { @@ -1087,7 +1091,7 @@ export async function loadAndProcessDataDocuments({dataPath}) { } else { const previousDescription = Object.entries(previous).at(0).join(': '); const nextDescription = Object.entries(next).at(0).join(': '); - parts.push(`between "${color.cyan(previousDescription)}" and "${color.cyan(nextDescription)}"`); + parts.push(`between "${colors.cyan(previousDescription)}" and "${colors.cyan(nextDescription)}"`); } aggregate.push(new Error(parts.join(''))); @@ -1318,13 +1322,27 @@ export async function loadAndProcessDataDocuments({dataPath}) { // Data linking! Basically, provide (portions of) wikiData to the Things which // require it - they'll expose dynamically computed properties as a result (many -// of which are required for page HTML generation). -export function linkWikiDataArrays(wikiData) { +// of which are required for page HTML generation and other expected behavior). +// +// The XXX_decacheWikiData option should be used specifically to mark +// points where you *aren't* replacing any of the arrays under wikiData with +// new values, and are using linkWikiDataArrays to instead "decache" data +// properties which depend on any of them. It's currently not possible for +// a CacheableObject to depend directly on the value of a property exposed +// on some other CacheableObject, so when those values change, you have to +// manually decache before the object will realize its cache isn't valid +// anymore. +export function linkWikiDataArrays(wikiData, { + XXX_decacheWikiData = false, +} = {}) { function assignWikiData(things, ...keys) { + if (things === undefined) return; for (let i = 0; i < things.length; i++) { const thing = things[i]; for (let j = 0; j < keys.length; j++) { const key = keys[j]; + if (!(key in wikiData)) continue; + if (XXX_decacheWikiData) thing[key] = []; thing[key] = wikiData[key]; } } @@ -1342,7 +1360,7 @@ export function linkWikiDataArrays(wikiData) { assignWikiData(WD.flashData, 'artistData', 'flashActData', 'trackData'); assignWikiData(WD.flashActData, 'flashData'); assignWikiData(WD.artTagData, 'albumData', 'trackData'); - assignWikiData(WD.homepageLayout.rows, 'albumData', 'groupData'); + assignWikiData(WD.homepageLayout?.rows, 'albumData', 'groupData'); } export function sortWikiDataArrays(wikiData) { @@ -1379,7 +1397,7 @@ export function filterDuplicateDirectories(wikiData) { const aggregate = openAggregate({message: `Duplicate directories found`}); for (const thingDataProp of deduplicateSpec) { const thingData = wikiData[thingDataProp]; - aggregate.nest({message: `Duplicate directories found in ${color.green('wikiData.' + thingDataProp)}`}, ({call}) => { + aggregate.nest({message: `Duplicate directories found in ${colors.green('wikiData.' + thingDataProp)}`}, ({call}) => { const directoryPlaces = Object.create(null); const duplicateDirectories = []; @@ -1405,7 +1423,7 @@ export function filterDuplicateDirectories(wikiData) { const places = directoryPlaces[directory]; call(() => { throw new Error( - `Duplicate directory ${color.green(directory)}:\n` + + `Duplicate directory ${colors.green(directory)}:\n` + places.map((thing) => ` - ` + inspect(thing)).join('\n') ); }); @@ -1446,45 +1464,45 @@ export function filterDuplicateDirectories(wikiData) { export function filterReferenceErrors(wikiData) { const referenceSpec = [ ['wikiInfo', processWikiInfoDocument, { - divideTrackListsByGroupsByRef: 'group', + divideTrackListsByGroups: 'group', }], ['albumData', processAlbumDocument, { - artistContribsByRef: '_contrib', - coverArtistContribsByRef: '_contrib', - trackCoverArtistContribsByRef: '_contrib', - wallpaperArtistContribsByRef: '_contrib', - bannerArtistContribsByRef: '_contrib', - groupsByRef: 'group', - artTagsByRef: 'artTag', + artistContribs: '_contrib', + coverArtistContribs: '_contrib', + trackCoverArtistContribs: '_contrib', + wallpaperArtistContribs: '_contrib', + bannerArtistContribs: '_contrib', + groups: 'group', + artTags: 'artTag', }], ['trackData', processTrackDocument, { - artistContribsByRef: '_contrib', - contributorContribsByRef: '_contrib', - coverArtistContribsByRef: '_contrib', - referencedTracksByRef: '_trackNotRerelease', - sampledTracksByRef: '_trackNotRerelease', - artTagsByRef: 'artTag', - originalReleaseTrackByRef: '_trackNotRerelease', + artistContribs: '_contrib', + contributorContribs: '_contrib', + coverArtistContribs: '_contrib', + referencedTracks: '_trackNotRerelease', + sampledTracks: '_trackNotRerelease', + artTags: 'artTag', + originalReleaseTrack: '_trackNotRerelease', }], ['groupCategoryData', processGroupCategoryDocument, { - groupsByRef: 'group', + groups: 'group', }], ['homepageLayout.rows', undefined, { - sourceGroupByRef: 'group', - sourceAlbumsByRef: 'album', + sourceGroup: '_homepageSourceGroup', + sourceAlbums: 'album', }], ['flashData', processFlashDocument, { - contributorContribsByRef: '_contrib', - featuredTracksByRef: 'track', + contributorContribs: '_contrib', + featuredTracks: 'track', }], ['flashActData', processFlashActDocument, { - flashesByRef: 'flash', + flashes: 'flash', }], ]; @@ -1500,7 +1518,7 @@ export function filterReferenceErrors(wikiData) { for (const [thingDataProp, providedProcessDocumentFn, propSpec] of referenceSpec) { const thingData = getNestedProp(wikiData, thingDataProp); - aggregate.nest({message: `Reference errors in ${color.green('wikiData.' + thingDataProp)}`}, ({nest}) => { + aggregate.nest({message: `Reference errors in ${colors.green('wikiData.' + thingDataProp)}`}, ({nest}) => { const things = Array.isArray(thingData) ? thingData : [thingData]; for (const thing of things) { @@ -1516,10 +1534,10 @@ export function filterReferenceErrors(wikiData) { nest({message: `Reference errors in ${inspect(thing)}`}, ({push, filter}) => { for (const [property, findFnKey] of Object.entries(propSpec)) { - const value = thing[property]; + const value = CacheableObject.getUpdateValue(thing, property); if (value === undefined) { - push(new TypeError(`Property ${color.red(property)} isn't valid for ${color.green(thing.constructor.name)}`)); + push(new TypeError(`Property ${colors.red(property)} isn't valid for ${colors.green(thing.constructor.name)}`)); continue; } @@ -1536,23 +1554,34 @@ export function filterReferenceErrors(wikiData) { if (alias) { // No need to check if the original exists here. Aliases are automatically // created from a field on the original, so the original certainly exists. - const original = find.artist(alias.aliasedArtistRef, wikiData.artistData, {mode: 'quiet'}); - throw new Error(`Reference ${color.red(contribRef.who)} is to an alias, should be ${color.green(original.name)}`); + const original = alias.aliasedArtist; + throw new Error(`Reference ${colors.red(contribRef.who)} is to an alias, should be ${colors.green(original.name)}`); } return boundFind.artist(contribRef.who); }; break; + case '_homepageSourceGroup': + findFn = groupRef => { + if (groupRef === 'new-additions' || groupRef === 'new-releases') { + return true; + } + + return boundFind.group(groupRef); + }; + break; + case '_trackNotRerelease': findFn = trackRef => { const track = find.track(trackRef, wikiData.trackData, {mode: 'error'}); + const originalRef = track && CacheableObject.getUpdateValue(track, 'originalReleaseTrack'); - if (track?.originalReleaseTrackByRef) { + if (originalRef) { // It's possible for the original to not actually exist, in this case. // It should still be reported since the 'Originally Released As' field // was present. - const original = find.track(track.originalReleaseTrackByRef, wikiData.trackData, {mode: 'quiet'}); + const original = find.track(originalRef, wikiData.trackData, {mode: 'quiet'}); // Prefer references by name, but only if it's unambiguous. const originalByName = @@ -1562,12 +1591,12 @@ export function filterReferenceErrors(wikiData) { const shouldBeMessage = (originalByName - ? color.green(original.name) + ? colors.green(original.name) : original - ? color.green('track:' + original.directory) - : color.green(track.originalReleaseTrackByRef)); + ? colors.green('track:' + original.directory) + : colors.green(originalRef)); - throw new Error(`Reference ${color.red(trackRef)} is to a rerelease, should be ${shouldBeMessage}`); + throw new Error(`Reference ${colors.red(trackRef)} is to a rerelease, should be ${shouldBeMessage}`); } return track; @@ -1580,7 +1609,7 @@ export function filterReferenceErrors(wikiData) { } const suppress = fn => conditionallySuppressError(error => { - if (property === 'sampledTracksByRef') { + if (property === 'sampledTracks') { // Suppress "didn't match anything" errors in particular, just for samples. // In hsmusic-data we have a lot of "stub" sample data which don't have // corresponding tracks yet, so it won't be useful to report such reference @@ -1598,13 +1627,13 @@ export function filterReferenceErrors(wikiData) { const fieldPropertyMessage = (processDocumentFn?.propertyFieldMapping?.[property] - ? ` in field ${color.green(processDocumentFn.propertyFieldMapping[property])}` - : ` in property ${color.green(property)}`); + ? ` in field ${colors.green(processDocumentFn.propertyFieldMapping[property])}` + : ` in property ${colors.green(property)}`); const findFnMessage = (findFnKey.startsWith('_') ? `` - : ` (${color.green('find.' + findFnKey)})`); + : ` (${colors.green('find.' + findFnKey)})`); const errorMessage = (Array.isArray(value) |