diff options
Diffstat (limited to 'src/data/checks.js')
| -rw-r--r-- | src/data/checks.js | 358 |
1 files changed, 272 insertions, 86 deletions
diff --git a/src/data/checks.js b/src/data/checks.js index b11b5d55..fd2c4931 100644 --- a/src/data/checks.js +++ b/src/data/checks.js @@ -4,14 +4,14 @@ import {inspect as nodeInspect} from 'node:util'; import {colors, ENABLE_COLOR} from '#cli'; import CacheableObject from '#cacheable-object'; -import {replacerSpec, parseInput} from '#replacer'; +import {replacerSpec, parseContentNodes} from '#replacer'; import {compareArrays, cut, cutStart, empty, getNestedProp, iterateMultiline} from '#sugar'; import Thing from '#thing'; import thingConstructors from '#things'; -import {combineWikiDataArrays, commentaryRegexCaseSensitive} from '#wiki-data'; import { + annotateError, annotateErrorWithIndex, conditionallySuppressError, decorateErrorWithIndex, @@ -60,7 +60,7 @@ export function reportDirectoryErrors(wikiData, { : [thing.directory]); for (const directory of directories) { - if (directory === null || directory === undefined) { + if (directory === '' || directory === null || directory === undefined) { missingDirectoryThings.add(thing); continue; } @@ -166,6 +166,69 @@ function getFieldPropertyMessage(yamlDocumentSpec, property) { return fieldPropertyMessage; } +function decoAnnotateFindErrors(findFn) { + function annotateMultipleNameMatchesIncludingUnfortunatelyUnsecondary(error) { + const matches = error[Symbol.for('hsmusic.find.multipleNameMatches')]; + if (!matches) return; + + const notSoSecondary = + matches + .map(match => match.thing ?? match) + .filter(match => + match.isTrack && + match.isMainRelease && + CacheableObject.getUpdateValue(match, 'mainRelease')); + + if (empty(notSoSecondary)) return; + + let {message} = error; + message += (message.includes('\n') ? '\n\n' : '\n'); + message += colors.bright(colors.yellow('<!>')) + ' '; + message += colors.yellow(`Some of these tracks are meant to be secondary releases,`) + '\n'; + message += ' '.repeat(4); + message += colors.yellow(`but another error is keeping that from processing correctly!`) + '\n'; + message += ' '.repeat(4); + message += colors.yellow(`Probably look for an error to do with "Main Release", first.`); + Object.assign(error, {message}); + } + + return (...args) => { + try { + return findFn(...args); + } catch (caughtError) { + throw annotateError(caughtError, ...[ + annotateMultipleNameMatchesIncludingUnfortunatelyUnsecondary, + ]); + } + }; +} + +function decoSuppressFindErrors(findFn, {property}) { + void property; + + return conditionallySuppressError(_error => { + // We're not suppressing any errors at the moment. + // An old suppression is kept below for reference. + + /* + if (property === 'sampledTracks') { + // Suppress "didn't match anything" errors in particular, just for samples. + // In hsmusic-data we have a lot of "stub" sample data which don't have + // corresponding tracks yet, so it won't be useful to report such reference + // errors until we take the time to address that. But other errors, like + // malformed reference strings or miscapitalized existing tracks, should + // still be reported, as samples of existing tracks *do* display on the + // website! + if (error.message.includes(`Didn't match anything`)) { + return true; + } + } + */ + + return false; + }, findFn); +} + // Warn about references across data which don't match anything. This involves // using the find() functions on all references, setting it to 'error' mode, and // collecting everything in a structured logged (which gets logged if there are @@ -185,15 +248,21 @@ export function filterReferenceErrors(wikiData, { groups: 'group', artTags: '_artTag', referencedArtworks: '_artwork', - commentary: '_commentary', + commentary: '_content', + creditingSources: '_content', }], ['artTagData', { directDescendantArtTags: 'artTag', }], + ['artworkData', { + referencedArtworks: '_artwork', + }], + ['flashData', { - commentary: '_commentary', + commentary: '_content', + creditingSources: '_content', }], ['groupCategoryData', { @@ -220,20 +289,24 @@ export function filterReferenceErrors(wikiData, { flashes: 'flash', }], - ['groupData', { - serieses: '_serieses', + ['seriesData', { + albums: 'album', }], ['trackData', { artistContribs: '_contrib', contributorContribs: '_contrib', coverArtistContribs: '_contrib', + previousProductionTracks: '_trackMainReleasesOnly', referencedTracks: '_trackMainReleasesOnly', sampledTracks: '_trackMainReleasesOnly', artTags: '_artTag', referencedArtworks: '_artwork', - mainReleaseTrack: '_trackMainReleasesOnly', - commentary: '_commentary', + mainRelease: '_mainRelease', + commentary: '_content', + creditingSources: '_content', + referencingSources: '_content', + lyrics: '_content', }], ['wikiInfo', { @@ -268,12 +341,12 @@ export function filterReferenceErrors(wikiData, { let writeProperty = true; switch (findFnKey) { - case '_commentary': + case '_content': if (value) { value = - Array.from(value.matchAll(commentaryRegexCaseSensitive)) - .map(({groups}) => groups.artistReferences) - .map(text => text.split(',').map(text => text.trim())); + value.map(entry => + CacheableObject.getUpdateValue(entry, 'artists') ?? + []); } writeProperty = false; @@ -287,15 +360,6 @@ export function filterReferenceErrors(wikiData, { // need writing, humm...) writeProperty = false; break; - - case '_serieses': - if (value) { - // Doesn't report on which series has the error, but... - value = value.flatMap(series => series.albums); - } - - writeProperty = false; - break; } if (value === undefined) { @@ -329,7 +393,7 @@ export function filterReferenceErrors(wikiData, { findFn = boundFind.artTag; break; - case '_commentary': + case '_content': findFn = findArtistOrAlias; break; @@ -347,8 +411,94 @@ export function filterReferenceErrors(wikiData, { }; break; - case '_serieses': - findFn = boundFind.album; + case '_mainRelease': + findFn = ref => { + // Mocking what's going on in `withMainRelease`. + + if (ref === 'same name single') { + // Accessing the current thing here. + try { + return boundFind.albumSinglesOnly(thing.name, { + fuzz: { + capitalization: true, + kebab: true, + }, + }); + } catch (caughtError) { + throw new Error( + `Didn't match a single with the same name`, + {cause: caughtError}); + } + } + + let track, trackError; + let album, albumError; + + try { + track = boundFind.trackMainReleasesOnly(ref); + } catch (caughtError) { + trackError = new Error( + `Didn't match a track`, {cause: caughtError}); + } + + try { + album = boundFind.album(ref); + } catch (caughtError) { + albumError = new Error( + `Didn't match an album`, {cause: caughtError}); + } + + if (track && album) { + if (album.tracks.includes(track)) { + return track; + } else { + throw new Error( + `Unrelated album and track matched for reference "${ref}". Please resolve:\n` + + `- ${inspect(track)}\n` + + `- ${inspect(album)}\n` + + `Returning null for this reference.`); + } + } + + if (track) { + return track; + } + + if (album) { + // At this point verification depends on the thing itself, + // which is currently in lexical scope, but if this code + // gets refactored, there might be trouble here... + + if (thing.mainReleaseTrack === null) { + if (album === thing.album) { + throw new Error( + `Matched album for reference "${ref}":\n` + + `- ` + inspect(album) + `\n` + + `...but this is the album that includes this secondary release, itself.\n` + + `Please resolve by pointing to aonther album here, or by removing this\n` + + `Main Release field, if this track is meant to be the main release.`); + } else { + throw new Error( + `Matched album for reference "${ref}":\n` + + `- ` + inspect(album) + `\n` + + `...but none of its tracks automatically match this secondary release.\n` + + `Please resolve by specifying the track here, instead of the album.`); + } + } else { + return album; + } + } + + const aggregateCause = + new AggregateError([albumError, trackError]); + + aggregateCause[Symbol.for('hsmusic.aggregate.translucent')] = true; + + throw new Error(`Trouble matching "${ref}"`, { + cause: aggregateCause, + }); + } + break; case '_trackArtwork': @@ -357,9 +507,16 @@ export function filterReferenceErrors(wikiData, { case '_trackMainReleasesOnly': findFn = trackRef => { - const track = boundFind.track(trackRef); - const mainRef = track && CacheableObject.getUpdateValue(track, 'mainReleaseTrack'); + let track = boundFind.trackMainReleasesOnly(trackRef, {mode: 'quiet'}); + if (track) { + return track; + } + // Will error normally, if this can't unambiguously resolve + // or doesn't match any track. + track = boundFind.track(trackRef); + + const mainRef = CacheableObject.getUpdateValue(track, 'mainRelease'); if (mainRef) { // It's possible for the main release to not actually exist, in this case. // It should still be reported since the 'Main Release' field was present. @@ -390,27 +547,8 @@ export function filterReferenceErrors(wikiData, { break; } - const suppress = fn => conditionallySuppressError(error => { - // We're not suppressing any errors at the moment. - // An old suppression is kept below for reference. - - /* - if (property === 'sampledTracks') { - // Suppress "didn't match anything" errors in particular, just for samples. - // In hsmusic-data we have a lot of "stub" sample data which don't have - // corresponding tracks yet, so it won't be useful to report such reference - // errors until we take the time to address that. But other errors, like - // malformed reference strings or miscapitalized existing tracks, should - // still be reported, as samples of existing tracks *do* display on the - // website! - if (error.message.includes(`Didn't match anything`)) { - return true; - } - } - */ - - return false; - }, fn); + findFn = decoSuppressFindErrors(findFn, {property}); + findFn = decoAnnotateFindErrors(findFn); const fieldPropertyMessage = getFieldPropertyMessage( @@ -461,15 +599,15 @@ export function filterReferenceErrors(wikiData, { } } - if (findFnKey === '_commentary') { + if (findFnKey === '_content') { filter( value, {message: errorMessage}, decorateErrorWithIndex(refs => (refs.length === 1 - ? suppress(findFn)(refs[0]) + ? findFn(refs[0]) : filterAggregate( refs, {message: `Errors in entry's artist references`}, - decorateErrorWithIndex(suppress(findFn))) + decorateErrorWithIndex(findFn)) .aggregate .close()))); @@ -481,19 +619,18 @@ export function filterReferenceErrors(wikiData, { if (Array.isArray(value)) { newPropertyValue = filter( value, {message: errorMessage}, - decorateErrorWithIndex(suppress(findFn))); + decorateErrorWithIndex(findFn)); break determineNewPropertyValue; } - nest({message: errorMessage}, - suppress(({call}) => { - try { - call(findFn, value); - } catch (error) { - newPropertyValue = null; - throw error; - } - })); + nest({message: errorMessage}, ({call}) => { + try { + call(findFn, value); + } catch (error) { + newPropertyValue = null; + throw error; + } + }); } if (writeProperty) { @@ -517,7 +654,11 @@ export class ContentNodeError extends Error { message, }) { const headingLine = - `(${where}) ${message}`; + (message.includes('\n\n') + ? `(${where})\n\n` + message + '\n' + : message.includes('\n') + ? `(${where})\n` + message + : `(${where}) ${message}`); const textUpToNode = containingLine.slice(0, columnNumber); @@ -562,16 +703,29 @@ export function reportContentTextErrors(wikiData, { description: 'description', }; + const artworkShape = { + source: 'artwork source', + originDetails: 'artwork origin details', + }; + const commentaryShape = { body: 'commentary body', - artistDisplayText: 'commentary artist display text', + artistText: 'commentary artist text', annotation: 'commentary annotation', }; + const lyricsShape = { + body: 'lyrics body', + artistText: 'lyrics artist text', + annotation: 'lyrics annotation', + }; + const contentTextSpec = [ ['albumData', { additionalFiles: additionalFileShape, commentary: commentaryShape, + creditingSources: commentaryShape, + coverArtworks: artworkShape, }], ['artTagData', { @@ -584,6 +738,8 @@ export function reportContentTextErrors(wikiData, { ['flashData', { commentary: commentaryShape, + creditingSources: commentaryShape, + coverArtwork: artworkShape, }], ['flashActData', { @@ -613,10 +769,12 @@ export function reportContentTextErrors(wikiData, { ['trackData', { additionalFiles: additionalFileShape, commentary: commentaryShape, - creditSources: commentaryShape, - lyrics: '_content', + creditingSources: commentaryShape, + referencingSources: commentaryShape, + lyrics: lyricsShape, midiProjectFiles: additionalFileShape, sheetMusicFiles: additionalFileShape, + trackArtworks: artworkShape, }], ['wikiInfo', { @@ -625,11 +783,19 @@ export function reportContentTextErrors(wikiData, { }], ]; - const boundFind = bindFind(wikiData, {mode: 'error'}); + const boundFind = + bindFind(wikiData, { + mode: 'error', + fuzz: { + capitalization: true, + kebab: true, + }, + }); + const findArtistOrAlias = bindFindArtistOrAlias(boundFind); function* processContent(input) { - const nodes = parseInput(input); + const nodes = parseContentNodes(input); for (const node of nodes) { const index = node.i; @@ -666,6 +832,9 @@ export function reportContentTextErrors(wikiData, { break; } + findFn = decoSuppressFindErrors(findFn, {property: null}); + findFn = decoAnnotateFindErrors(findFn); + const findRef = (replacerKeyImplied ? replacerValue @@ -686,7 +855,7 @@ export function reportContentTextErrors(wikiData, { } else if (node.type === 'external-link') { try { new URL(node.data.href); - } catch (error) { + } catch { yield { index, length, message: @@ -737,8 +906,8 @@ export function reportContentTextErrors(wikiData, { for (const thing of things) { nest({message: `Content text errors in ${inspect(thing)}`}, ({nest, push}) => { - for (const [property, shape] of Object.entries(propSpec)) { - const value = thing[property]; + for (let [property, shape] of Object.entries(propSpec)) { + let value = thing[property]; if (value === undefined) { push(new TypeError(`Property ${colors.red(property)} isn't valid for ${colors.green(thing.constructor.name)}`)); @@ -757,6 +926,31 @@ export function reportContentTextErrors(wikiData, { const topMessage = `Content text errors` + fieldPropertyMessage; + const checkShapeEntries = (entry, callProcessContentOpts) => { + for (const [key, annotation] of Object.entries(shape)) { + const value = entry[key]; + + // TODO: This should be an undefined/null check, like above, + // but it's not, because sometimes the stuff we're checking + // here isn't actually coded as a Thing - so the properties + // might really be undefined instead of null. Terrifying and + // awful. And most of all, citation needed. + if (!value) continue; + + callProcessContent({ + ...callProcessContentOpts, + + // TODO: `nest` isn't provided by `callProcessContentOpts` + //`but `push` is - this is to match the old code, but + // what's the deal here? + nest, + + value, + message: `Error in ${colors.green(annotation)}`, + }); + } + }; + if (shape === '_content') { callProcessContent({ nest, @@ -764,26 +958,18 @@ export function reportContentTextErrors(wikiData, { value, message: topMessage, }); - } else { + } else if (Array.isArray(value)) { nest({message: topMessage}, ({push}) => { for (const [index, entry] of value.entries()) { - for (const [key, annotation] of Object.entries(shape)) { - const value = entry[key]; - - // TODO: Should this check undefined/null similar to above? - if (!value) continue; - - callProcessContent({ - nest, - push, - value, - message: `Error in ${colors.green(annotation)}`, - annotateError: error => - annotateErrorWithIndex(error, index), - }); - } + checkShapeEntries(entry, { + push, + annotateError: error => + annotateErrorWithIndex(error, index), + }); } }); + } else { + checkShapeEntries(value, {push}); } } }); |