diff options
Diffstat (limited to 'src/data/checks.js')
| -rw-r--r-- | src/data/checks.js | 255 |
1 files changed, 200 insertions, 55 deletions
diff --git a/src/data/checks.js b/src/data/checks.js index ad621bab..3fcb6d3b 100644 --- a/src/data/checks.js +++ b/src/data/checks.js @@ -4,12 +4,11 @@ import {inspect as nodeInspect} from 'node:util'; import {colors, ENABLE_COLOR} from '#cli'; import CacheableObject from '#cacheable-object'; -import {replacerSpec, parseInput} from '#replacer'; +import {replacerSpec, parseContentNodes} from '#replacer'; import {compareArrays, cut, cutStart, empty, getNestedProp, iterateMultiline} from '#sugar'; import Thing from '#thing'; import thingConstructors from '#things'; -import {commentaryRegexCaseSensitive} from '#wiki-data'; import { annotateErrorWithIndex, @@ -50,7 +49,7 @@ export function reportDirectoryErrors(wikiData, { if (!thingData) continue; for (const thing of thingData) { - if (findSpec.include && !findSpec.include(thing)) { + if (findSpec.include && !findSpec.include(thing, thingConstructors)) { continue; } @@ -172,6 +171,7 @@ function getFieldPropertyMessage(yamlDocumentSpec, property) { // any errors). At the same time, we remove errored references from the thing's // data array. export function filterReferenceErrors(wikiData, { + find, bindFind, }) { const referenceSpec = [ @@ -183,18 +183,35 @@ export function filterReferenceErrors(wikiData, { bannerArtistContribs: '_contrib', groups: 'group', artTags: '_artTag', - commentary: '_commentary', + referencedArtworks: '_artwork', + commentary: '_content', + creditingSources: '_content', + }], + + ['artTagData', { + directDescendantArtTags: 'artTag', + }], + + ['artworkData', { + referencedArtworks: '_artwork', }], ['flashData', { - commentary: '_commentary', + commentary: '_content', + creditingSources: '_content', }], ['groupCategoryData', { groups: 'group', }], - ['homepageLayout.rows', { + ['homepageLayout.sections.rows', { + _include: row => row.type === 'album carousel', + albums: 'album', + }], + + ['homepageLayout.sections.rows', { + _include: row => row.type === 'album grid', sourceGroup: '_homepageSourceGroup', sourceAlbums: 'album', }], @@ -208,15 +225,23 @@ export function filterReferenceErrors(wikiData, { flashes: 'flash', }], + ['seriesData', { + albums: 'album', + }], + ['trackData', { artistContribs: '_contrib', contributorContribs: '_contrib', coverArtistContribs: '_contrib', - referencedTracks: '_trackNotRerelease', - sampledTracks: '_trackNotRerelease', + referencedTracks: '_trackMainReleasesOnly', + sampledTracks: '_trackMainReleasesOnly', artTags: '_artTag', - originalReleaseTrack: '_trackNotRerelease', - commentary: '_commentary', + referencedArtworks: '_artwork', + mainReleaseTrack: '_trackMainReleasesOnly', + commentary: '_content', + creditingSources: '_content', + referencingSources: '_content', + lyrics: '_content', }], ['wikiInfo', { @@ -230,21 +255,33 @@ export function filterReferenceErrors(wikiData, { const aggregate = openAggregate({message: `Errors validating between-thing references in data`}); for (const [thingDataProp, propSpec] of referenceSpec) { const thingData = getNestedProp(wikiData, thingDataProp); - const things = Array.isArray(thingData) ? thingData : [thingData]; + const things = + (Array.isArray(thingData) + ? thingData.flat(Infinity) + : [thingData]); + aggregate.nest({message: `Reference errors in ${colors.green('wikiData.' + thingDataProp)}`}, ({nest}) => { for (const thing of things) { + if (propSpec._include && !propSpec._include(thing)) { + continue; + } + nest({message: `Reference errors in ${inspect(thing)}`}, ({nest, push, filter}) => { for (const [property, findFnKey] of Object.entries(propSpec)) { + if (property === '_include') { + continue; + } + let value = CacheableObject.getUpdateValue(thing, property); let writeProperty = true; switch (findFnKey) { - case '_commentary': + case '_content': if (value) { value = - Array.from(value.matchAll(commentaryRegexCaseSensitive)) - .map(({groups}) => groups.artistReferences) - .map(text => text.split(',').map(text => text.trim())); + value.map(entry => + CacheableObject.getUpdateValue(entry, 'artists') ?? + []); } writeProperty = false; @@ -272,11 +309,26 @@ export function filterReferenceErrors(wikiData, { let findFn; switch (findFnKey) { + case '_artwork': { + const mixed = + find.mixed({ + album: find.albumPrimaryArtwork, + track: find.trackPrimaryArtwork, + }); + + const data = + wikiData.artworkData; + + findFn = ref => mixed(ref.reference, data, {mode: 'error'}); + + break; + } + case '_artTag': findFn = boundFind.artTag; break; - case '_commentary': + case '_content': findFn = findArtistOrAlias; break; @@ -294,29 +346,32 @@ export function filterReferenceErrors(wikiData, { }; break; - case '_trackNotRerelease': + case '_trackArtwork': + findFn = ref => boundFind.track(ref.reference); + break; + + case '_trackMainReleasesOnly': findFn = trackRef => { const track = boundFind.track(trackRef); - const originalRef = track && CacheableObject.getUpdateValue(track, 'originalReleaseTrack'); + const mainRef = track && CacheableObject.getUpdateValue(track, 'mainReleaseTrack'); - if (originalRef) { - // It's possible for the original to not actually exist, in this case. - // It should still be reported since the 'Originally Released As' field - // was present. - const original = boundFind.track(originalRef, {mode: 'quiet'}); + if (mainRef) { + // It's possible for the main release to not actually exist, in this case. + // It should still be reported since the 'Main Release' field was present. + const main = boundFind.track(mainRef, {mode: 'quiet'}); // Prefer references by name, but only if it's unambiguous. - const originalByName = - (original - ? boundFind.track(original.name, {mode: 'quiet'}) + const mainByName = + (main + ? boundFind.track(main.name, {mode: 'quiet'}) : null); const shouldBeMessage = - (originalByName - ? colors.green(original.name) - : original - ? colors.green('track:' + original.directory) - : colors.green(originalRef)); + (mainByName + ? colors.green(main.name) + : main + ? colors.green('track:' + main.directory) + : colors.green(mainRef)); throw new Error(`Reference ${colors.red(trackRef)} is to a rerelease, should be ${shouldBeMessage}`); } @@ -330,7 +385,11 @@ export function filterReferenceErrors(wikiData, { break; } - const suppress = fn => conditionallySuppressError(error => { + const suppress = fn => conditionallySuppressError(_error => { + // We're not suppressing any errors at the moment. + // An old suppression is kept below for reference. + + /* if (property === 'sampledTracks') { // Suppress "didn't match anything" errors in particular, just for samples. // In hsmusic-data we have a lot of "stub" sample data which don't have @@ -343,6 +402,7 @@ export function filterReferenceErrors(wikiData, { return true; } } + */ return false; }, fn); @@ -396,7 +456,7 @@ export function filterReferenceErrors(wikiData, { } } - if (findFnKey === '_commentary') { + if (findFnKey === '_content') { filter( value, {message: errorMessage}, decorateErrorWithIndex(refs => @@ -497,16 +557,33 @@ export function reportContentTextErrors(wikiData, { description: 'description', }; + const artworkShape = { + source: 'artwork source', + originDetails: 'artwork origin details', + }; + const commentaryShape = { body: 'commentary body', - artistDisplayText: 'commentary artist display text', + artistText: 'commentary artist text', annotation: 'commentary annotation', }; + const lyricsShape = { + body: 'lyrics body', + artistText: 'lyrics artist text', + annotation: 'lyrics annotation', + }; + const contentTextSpec = [ ['albumData', { additionalFiles: additionalFileShape, commentary: commentaryShape, + creditingSources: commentaryShape, + coverArtworks: artworkShape, + }], + + ['artTagData', { + description: '_content', }], ['artistData', { @@ -515,6 +592,8 @@ export function reportContentTextErrors(wikiData, { ['flashData', { commentary: commentaryShape, + creditingSources: commentaryShape, + coverArtwork: artworkShape, }], ['flashActData', { @@ -544,9 +623,12 @@ export function reportContentTextErrors(wikiData, { ['trackData', { additionalFiles: additionalFileShape, commentary: commentaryShape, - lyrics: '_content', + creditingSources: commentaryShape, + referencingSources: commentaryShape, + lyrics: lyricsShape, midiProjectFiles: additionalFileShape, sheetMusicFiles: additionalFileShape, + trackArtworks: artworkShape, }], ['wikiInfo', { @@ -559,7 +641,7 @@ export function reportContentTextErrors(wikiData, { const findArtistOrAlias = bindFindArtistOrAlias(boundFind); function* processContent(input) { - const nodes = parseInput(input); + const nodes = parseContentNodes(input); for (const node of nodes) { const index = node.i; @@ -616,7 +698,7 @@ export function reportContentTextErrors(wikiData, { } else if (node.type === 'external-link') { try { new URL(node.data.href); - } catch (error) { + } catch { yield { index, length, message: @@ -667,8 +749,8 @@ export function reportContentTextErrors(wikiData, { for (const thing of things) { nest({message: `Content text errors in ${inspect(thing)}`}, ({nest, push}) => { - for (const [property, shape] of Object.entries(propSpec)) { - const value = thing[property]; + for (let [property, shape] of Object.entries(propSpec)) { + let value = thing[property]; if (value === undefined) { push(new TypeError(`Property ${colors.red(property)} isn't valid for ${colors.green(thing.constructor.name)}`)); @@ -687,6 +769,31 @@ export function reportContentTextErrors(wikiData, { const topMessage = `Content text errors` + fieldPropertyMessage; + const checkShapeEntries = (entry, callProcessContentOpts) => { + for (const [key, annotation] of Object.entries(shape)) { + const value = entry[key]; + + // TODO: This should be an undefined/null check, like above, + // but it's not, because sometimes the stuff we're checking + // here isn't actually coded as a Thing - so the properties + // might really be undefined instead of null. Terrifying and + // awful. And most of all, citation needed. + if (!value) continue; + + callProcessContent({ + ...callProcessContentOpts, + + // TODO: `nest` isn't provided by `callProcessContentOpts` + //`but `push` is - this is to match the old code, but + // what's the deal here? + nest, + + value, + message: `Error in ${colors.green(annotation)}`, + }); + } + }; + if (shape === '_content') { callProcessContent({ nest, @@ -694,26 +801,18 @@ export function reportContentTextErrors(wikiData, { value, message: topMessage, }); - } else { + } else if (Array.isArray(value)) { nest({message: topMessage}, ({push}) => { for (const [index, entry] of value.entries()) { - for (const [key, annotation] of Object.entries(shape)) { - const value = entry[key]; - - // TODO: Should this check undefined/null similar to above? - if (!value) continue; - - callProcessContent({ - nest, - push, - value, - message: `Error in ${colors.green(annotation)}`, - annotateError: error => - annotateErrorWithIndex(error, index), - }); - } + checkShapeEntries(entry, { + push, + annotateError: error => + annotateErrorWithIndex(error, index), + }); } }); + } else { + checkShapeEntries(value, {push}); } } }); @@ -722,3 +821,49 @@ export function reportContentTextErrors(wikiData, { } }); } + +export function reportOrphanedArtworks(wikiData) { + const aggregate = + openAggregate({message: `Artwork objects are orphaned`}); + + const assess = ({ + message, + filterThing, + filterContribs, + link, + }) => { + aggregate.nest({message: `Orphaned ${message}`}, ({push}) => { + const ostensibleArtworks = + wikiData.artworkData + .filter(artwork => + artwork.thing instanceof filterThing && + artwork.artistContribsFromThingProperty === filterContribs); + + const orphanedArtworks = + ostensibleArtworks + .filter(artwork => !artwork.thing[link].includes(artwork)); + + for (const artwork of orphanedArtworks) { + push(new Error(`Orphaned: ${inspect(artwork)}`)); + } + }); + }; + + const {Album, Track} = thingConstructors; + + assess({ + message: `album cover artworks`, + filterThing: Album, + filterContribs: 'coverArtistContribs', + link: 'coverArtworks', + }); + + assess({ + message: `track artworks`, + filterThing: Track, + filterContribs: 'coverArtistContribs', + link: 'trackArtworks', + }); + + aggregate.close(); +} |