From ad079e0c9e6ab32203bc684851e586ed10c3378c Mon Sep 17 00:00:00 2001 From: "(quasar) nebula" Date: Wed, 16 Feb 2022 21:48:11 -0400 Subject: artist ref validation, nicer showAggregate output --- src/data/things.js | 7 ++- src/upd8.js | 171 +++++++++++++++++++++++++++++------------------------ src/util/find.js | 29 +++++---- src/util/sugar.js | 33 +++++++---- 4 files changed, 136 insertions(+), 104 deletions(-) diff --git a/src/data/things.js b/src/data/things.js index 6119f2d..dd85e4a 100644 --- a/src/data/things.js +++ b/src/data/things.js @@ -548,7 +548,12 @@ Artist.propertyDescriptors = { urls: Thing.common.urls(), contextNotes: Thing.common.simpleString(), - aliasRefs: Thing.common.referenceList(Artist), + aliasNames: { + flags: {update: true, expose: true}, + update: { + validate: validateArrayItems(isName) + } + }, }; // -> Group diff --git a/src/upd8.js b/src/upd8.js index 69d45ec..56fbd64 100755 --- a/src/upd8.js +++ b/src/upd8.js @@ -1003,7 +1003,7 @@ const processArtistDocument = makeProcessDocument(Artist, { directory: 'Directory', urls: 'URLs', - aliasRefs: 'Aliases', + aliasNames: 'Aliases', contextNotes: 'Context Notes' }, @@ -2160,6 +2160,14 @@ async function main() { type: 'flag' }, + // Want sweet, sweet trace8ack info in aggreg8te error messages? This + // will print all the juicy details (or at least the first relevant + // line) right to your output, 8ut also pro8a8ly give you a headache + // 8ecause wow that is a lot of visual noise. + 'show-traces': { + type: 'flag' + }, + 'queue-size': { type: 'value', validate(size) { @@ -2211,6 +2219,15 @@ async function main() { const skipThumbs = miscOptions['skip-thumbs'] ?? false; const thumbsOnly = miscOptions['thumbs-only'] ?? false; + const showAggregateTraces = miscOptions['show-traces'] ?? false; + + const niceShowAggregate = (error, ...opts) => { + showAggregate(error, { + showTraces: showAggregateTraces, + pathToFile: f => path.relative(__dirname, f), + ...opts + }); + }; if (skipThumbs && thumbsOnly) { logInfo`Well, you've put yourself rather between a roc and a hard place, hmmmm?`; @@ -2731,7 +2748,7 @@ async function main() { try { processDataAggregate.close(); } catch (error) { - showAggregate(error, {pathToFile: f => path.relative(__dirname, f)}); + niceShowAggregate(error); logWarn`The above errors were detected while processing data files.`; logWarn`If the remaining valid data is complete enough, the wiki will`; logWarn`still build - but all errored data will be skipped.`; @@ -2777,20 +2794,6 @@ async function main() { trackData: sortByDate(WD.trackData.slice()) }); - console.log(WD.trackData[0].name, WD.trackData[0].album.name); - console.log(WD.albumData[0].name, WD.albumData[0].tracks[0].name); - console.log(WD.trackData[0].artistContribs[0].who.name); - const demoAlbum1 = WD.albumData.find(album => album.name === 'Alternia'); - const demoAlbum2 = WD.albumData.find(album => album.name === 'Homestuck Vol. 5'); - const demoAlbum3 = WD.albumData.find(album => album.name === 'Homestuck Vol. 1'); - console.log(demoAlbum1.artistContribs[0]?.who.name); - console.log(demoAlbum2.tracks[0].name, - demoAlbum2.tracks[0].date, - demoAlbum2.tracks[0].coverArtDate); - console.log(demoAlbum3.tracks[0].coverArtistContribs[0]?.who.name); - - return; - // Update languages o8ject with the wiki-specified default language! // This will make page files for that language 8e gener8ted at the root // directory, instead of the language-specific su8directory. @@ -2831,75 +2834,87 @@ async function main() { WD.justEverythingSortedByArtDateMan = sortByArtDate(WD.justEverythingMan.slice()); // console.log(JSON.stringify(justEverythingSortedByArtDateMan.map(toAnythingMan), null, 2)); - return; - - const artistRefs = Array.from(new Set([ - ...WD.artistData.filter(artist => !artist.alias).map(artist => artist.name), - ...[ - ...WD.albumData.flatMap(album => [ - ...album.artists || [], - ...album.coverArtists || [], - ...album.wallpaperArtists || [], - ...album.tracks.flatMap(track => [ - ...track.artists, - ...track.coverArtists || [], - ...track.contributors || [] - ]) - ]), - ...(WD.flashData?.flatMap(flash => [ - ...flash.contributors || [] - ]) || []) - ].map(contribution => contribution.who) - ])); - - artistNames.sort((a, b) => a.toLowerCase() < b.toLowerCase() ? -1 : a.toLowerCase() > b.toLowerCase() ? 1 : 0); - + // TODO: this should probably be some kinda generalized function lol { - let buffer = []; - const clearBuffer = function() { - if (buffer.length) { - for (const entry of buffer.slice(0, -1)) { - console.log(`\x1b[2m... ${entry.name} ...\x1b[0m`); - } - const lastEntry = buffer[buffer.length - 1]; - console.log(`\x1b[2m... \x1b[0m${lastEntry.name}\x1b[0;2m ...\x1b[0m`); - buffer = []; - } - }; - const showWhere = (name, color) => { - const where = WD.justEverythingMan.filter(thing => [ - ...thing.coverArtists || [], - ...thing.contributors || [], - ...thing.artists || [] - ].some(({ who }) => who === name)); - for (const thing of where) { - console.log(`\x1b[${color}m- ` + (thing.album ? `(\x1b[1m${thing.album.name}\x1b[0;${color}m)` : '') + ` \x1b[1m${thing.name}\x1b[0m`); + const aggregate = openAggregate({message: `Errors validating artist references in data`}); + + const aliasToOrigMap = new Map(); + const aliasArtistData = wikiData.artistData.flatMap(artist => { + return (artist.aliasNames?.map(ref => { + const alias = new Artist(); + alias.name = ref; + aliasToOrigMap.set(alias, artist); + return alias; + }) ?? []); + }); + + const sources = [ + [WD.albumData, [ + 'artistContribsByRef', + 'coverArtistContribsByRef', + 'trackCoverArtistContribsByRef', + 'wallpaperArtistContribsByRef', + 'bannerArtistContribsByRef' + ]], + [WD.trackData, [ + 'artistContribsByRef', + 'contributorContribsByRef', + 'coverArtistContribsByRef' + ]], + [WD.flashData, [ + 'contributorContribsByRef' + ]] + ] + + for (const [ things, properties ] of sources) { + if (!things) { + continue; } - }; - let CR4SH = false; - for (let name of artistNames) { - const entry = find.artist(name, {wikiData}); - if (!entry) { - clearBuffer(); - console.log(`\x1b[31mMissing entry for artist "\x1b[1m${name}\x1b[0;31m"\x1b[0m`); - showWhere(name, 31); - CR4SH = true; - } else if (entry.alias) { - console.log(`\x1b[33mArtist "\x1b[1m${name}\x1b[0;33m" should be named "\x1b[1m${entry.alias}\x1b[0;33m"\x1b[0m`); - showWhere(name, 33); - CR4SH = true; - } else { - buffer.push(entry); - if (buffer.length > 3) { - buffer.shift(); - } + + for (const thing of things) { + aggregate.nest({message: `Errors for ${thing.constructor.name} ${color.green(thing.name)} (${color.green(Thing.getReference(thing))})`}, thingAgg => { + for (const prop of properties) { + const contribs = thing[prop]; + if (!contribs) { + continue; + } + thingAgg.nest({message: `Errors for property ${color.green(prop)}`}, propAgg => { + for (const { who: ref } of contribs) { + propAgg.call(() => { + const entryAlias = find.artist(ref, {wikiData: {artistData: aliasArtistData}, quiet: true}); + if (entryAlias) { + const orig = aliasToOrigMap.get(entryAlias); + throw new Error(`Reference ${color.red(ref)} is to an alias, reference ${color.green(orig.name)} instead`); + } + const entry = find.artist(ref, {wikiData: {artistData: wikiData.artistData}, quiet: true}); + if (!entry) { + throw new Error(`No entry found for reference ${color.red(ref)}`); + } + if ( + ref.toLowerCase() === entry.name.toLowerCase() && + ref !== entry.name + ) { + throw new Error(`Miscapitalized name ${color.red(ref)}, reference ${color.green(entry.name)} instead`); + } + }); + } + }); + } + }); } } - if (CR4SH) { + + try { + aggregate.close(); + } catch (error) { + niceShowAggregate(error); + // TODO: more graceful auto-resolve, filter out invalid references return; } } + return; + { const directories = []; for (const { directory, name } of WD.albumData) { @@ -2930,7 +2945,7 @@ async function main() { { const artists = []; const artistsLC = []; - for (const name of artistNames) { + for (const name of artistRefs) { if (!artists.includes(name) && artistsLC.includes(name.toLowerCase())) { const other = artists.find(oth => oth.toLowerCase() === name.toLowerCase()); console.log(`\x1b[31;1mMiscapitalized artist name: ${name}, ${other}\x1b[0m`); diff --git a/src/util/find.js b/src/util/find.js index 0e22063..a9af238 100644 --- a/src/util/find.js +++ b/src/util/find.js @@ -9,7 +9,7 @@ function findHelper(keys, dataProp, findFns = {}) { const keyRefRegex = new RegExp(String.raw`^(?:(${keys.join('|')}):(?=\S))?(.*)$`); - return (fullRef, {wikiData}) => { + return (fullRef, {wikiData, quiet = false}) => { if (!fullRef) return null; if (typeof fullRef !== 'string') { throw new Error(`Got a reference that is ${typeof fullRef}, not string: ${fullRef}`); @@ -26,10 +26,10 @@ function findHelper(keys, dataProp, findFns = {}) { const data = wikiData[dataProp]; const found = (key - ? byDirectory(ref, data) - : byName(ref, data)); + ? byDirectory(ref, data, quiet) + : byName(ref, data, quiet)); - if (!found) { + if (!found && !quiet) { logWarn`Didn't match anything for ${fullRef}!`; } @@ -37,19 +37,22 @@ function findHelper(keys, dataProp, findFns = {}) { }; } -function matchDirectory(ref, data) { +function matchDirectory(ref, data, quiet) { return data.find(({ directory }) => directory === ref); } -function matchName(ref, data) { +function matchName(ref, data, quiet) { const matches = data.filter(({ name }) => name.toLowerCase() === ref.toLowerCase()); if (matches.length > 1) { - logError`Multiple matches for reference "${ref}". Please resolve:`; - for (const match of matches) { - logError`- ${match.name} (${match.directory})`; + // TODO: This should definitely be a thrown error. + if (!quiet) { + logError`Multiple matches for reference "${ref}". Please resolve:`; + for (const match of matches) { + logError`- ${match.name} (${match.directory})`; + } + logError`Returning null for this reference.`; } - logError`Returning null for this reference.`; return null; } @@ -59,15 +62,15 @@ function matchName(ref, data) { const thing = matches[0]; - if (ref !== thing.name) { + if (ref !== thing.name && !quiet) { logWarn`Bad capitalization: ${'\x1b[31m' + ref} -> ${'\x1b[32m' + thing.name}`; } return thing; } -function matchTagName(ref, data) { - return matchName(ref.startsWith('cw: ') ? ref.slice(4) : ref, data); +function matchTagName(ref, data, quiet) { + return matchName(ref.startsWith('cw: ') ? ref.slice(4) : ref, data, quiet); } const find = { diff --git a/src/util/sugar.js b/src/util/sugar.js index d6bc3df..c8f1706 100644 --- a/src/util/sugar.js +++ b/src/util/sugar.js @@ -350,19 +350,28 @@ export function _withAggregate(mode, aggregateOpts, fn) { } } -export function showAggregate(topError, {pathToFile = p => p} = {}) { +export function showAggregate(topError, { + pathToFile = p => p, + showTraces = true +} = {}) { const recursive = (error, {level}) => { - const stackLines = error.stack?.split('\n'); - const stackLine = stackLines?.find(line => - line.trim().startsWith('at') - && !line.includes('sugar') - && !line.includes('node:internal') - && !line.includes('')); - const tracePart = (stackLine - ? '- ' + stackLine.trim().replace(/file:\/\/(.*\.js)/, (match, pathname) => pathToFile(pathname)) - : '(no stack trace)'); - - const header = `[${error.constructor.name || 'unnamed'}] ${error.message || '(no message)'} ${color.dim(tracePart)}`; + let header = (showTraces + ? `[${error.constructor.name || 'unnamed'}] ${error.message || '(no message)'}` + : (error instanceof AggregateError + ? `[${error.message || '(no message)'}]` + : error.message || '(no message)')); + if (showTraces) { + const stackLines = error.stack?.split('\n'); + const stackLine = stackLines?.find(line => + line.trim().startsWith('at') + && !line.includes('sugar') + && !line.includes('node:internal') + && !line.includes('')); + const tracePart = (stackLine + ? '- ' + stackLine.trim().replace(/file:\/\/(.*\.js)/, (match, pathname) => pathToFile(pathname)) + : '(no stack trace)'); + header += ` ${color.dim(tracePart)}`; + } const bar = (level % 2 === 0 ? '\u2502' : color.dim('\u254e')); -- cgit 1.3.0-6-gf8a5