diff options
Diffstat (limited to 'src/data')
22 files changed, 554 insertions, 385 deletions
diff --git a/src/data/checks.js b/src/data/checks.js index b11b5d55..52024144 100644 --- a/src/data/checks.js +++ b/src/data/checks.js @@ -9,7 +9,6 @@ import {compareArrays, cut, cutStart, empty, getNestedProp, iterateMultiline} from '#sugar'; import Thing from '#thing'; import thingConstructors from '#things'; -import {combineWikiDataArrays, commentaryRegexCaseSensitive} from '#wiki-data'; import { annotateErrorWithIndex, @@ -185,7 +184,8 @@ export function filterReferenceErrors(wikiData, { groups: 'group', artTags: '_artTag', referencedArtworks: '_artwork', - commentary: '_commentary', + commentary: '_content', + creditSources: '_content', }], ['artTagData', { @@ -193,7 +193,8 @@ export function filterReferenceErrors(wikiData, { }], ['flashData', { - commentary: '_commentary', + commentary: '_content', + creditSources: '_content', }], ['groupCategoryData', { @@ -233,7 +234,9 @@ export function filterReferenceErrors(wikiData, { artTags: '_artTag', referencedArtworks: '_artwork', mainReleaseTrack: '_trackMainReleasesOnly', - commentary: '_commentary', + commentary: '_content', + creditSources: '_content', + lyrics: '_content', }], ['wikiInfo', { @@ -268,12 +271,12 @@ export function filterReferenceErrors(wikiData, { let writeProperty = true; switch (findFnKey) { - case '_commentary': + case '_content': if (value) { value = - Array.from(value.matchAll(commentaryRegexCaseSensitive)) - .map(({groups}) => groups.artistReferences) - .map(text => text.split(',').map(text => text.trim())); + value.map(entry => + CacheableObject.getUpdateValue(entry, 'artists') ?? + []); } writeProperty = false; @@ -329,7 +332,7 @@ export function filterReferenceErrors(wikiData, { findFn = boundFind.artTag; break; - case '_commentary': + case '_content': findFn = findArtistOrAlias; break; @@ -461,7 +464,7 @@ export function filterReferenceErrors(wikiData, { } } - if (findFnKey === '_commentary') { + if (findFnKey === '_content') { filter( value, {message: errorMessage}, decorateErrorWithIndex(refs => @@ -568,6 +571,12 @@ export function reportContentTextErrors(wikiData, { annotation: 'commentary annotation', }; + const lyricsShape = { + body: 'lyrics body', + artistDisplayText: 'lyrics artist display text', + annotation: 'lyrics annotation', + }; + const contentTextSpec = [ ['albumData', { additionalFiles: additionalFileShape, @@ -614,7 +623,7 @@ export function reportContentTextErrors(wikiData, { additionalFiles: additionalFileShape, commentary: commentaryShape, creditSources: commentaryShape, - lyrics: '_content', + lyrics: lyricsShape, midiProjectFiles: additionalFileShape, sheetMusicFiles: additionalFileShape, }], @@ -737,8 +746,8 @@ export function reportContentTextErrors(wikiData, { for (const thing of things) { nest({message: `Content text errors in ${inspect(thing)}`}, ({nest, push}) => { - for (const [property, shape] of Object.entries(propSpec)) { - const value = thing[property]; + for (let [property, shape] of Object.entries(propSpec)) { + let value = thing[property]; if (value === undefined) { push(new TypeError(`Property ${colors.red(property)} isn't valid for ${colors.green(thing.constructor.name)}`)); diff --git a/src/data/composite/things/album/index.js b/src/data/composite/things/album/index.js index 8b5098f0..dfc6864f 100644 --- a/src/data/composite/things/album/index.js +++ b/src/data/composite/things/album/index.js @@ -1 +1,2 @@ +export {default as withHasCoverArt} from './withHasCoverArt.js'; export {default as withTracks} from './withTracks.js'; diff --git a/src/data/composite/things/album/withHasCoverArt.js b/src/data/composite/things/album/withHasCoverArt.js new file mode 100644 index 00000000..fd3f2894 --- /dev/null +++ b/src/data/composite/things/album/withHasCoverArt.js @@ -0,0 +1,64 @@ +// TODO: This shouldn't be coded as an Album-specific thing, +// or even really to do with cover artworks in particular, either. + +import {input, templateCompositeFrom} from '#composite'; + +import {raiseOutputWithoutDependency, withResultOfAvailabilityCheck} + from '#composite/control-flow'; +import {fillMissingListItems, withFlattenedList, withPropertyFromList} + from '#composite/data'; + +export default templateCompositeFrom({ + annotation: 'withHasCoverArt', + + outputs: ['#hasCoverArt'], + + steps: () => [ + withResultOfAvailabilityCheck({ + from: 'coverArtistContribs', + mode: input.value('empty'), + }), + + { + dependencies: ['#availability'], + compute: (continuation, { + ['#availability']: availability, + }) => + (availability + ? continuation.raiseOutput({ + ['#hasCoverArt']: true, + }) + : continuation()), + }, + + raiseOutputWithoutDependency({ + dependency: 'coverArtworks', + mode: input.value('empty'), + output: input.value({'#hasCoverArt': false}), + }), + + withPropertyFromList({ + list: 'coverArtworks', + property: input.value('artistContribs'), + internal: input.value(true), + }), + + // Since we're getting the update value for each artwork's artistContribs, + // it may not be set at all, and in that case won't be exposing as []. + fillMissingListItems({ + list: '#coverArtworks.artistContribs', + fill: input.value([]), + }), + + withFlattenedList({ + list: '#coverArtworks.artistContribs', + }), + + withResultOfAvailabilityCheck({ + from: '#flattenedList', + mode: input.value('empty'), + }).outputs({ + '#availability': '#hasCoverArt', + }), + ], +}); diff --git a/src/data/composite/things/commentary-entry/index.js b/src/data/composite/things/commentary-entry/index.js new file mode 100644 index 00000000..091bae1a --- /dev/null +++ b/src/data/composite/things/commentary-entry/index.js @@ -0,0 +1 @@ +export {default as withWebArchiveDate} from './withWebArchiveDate.js'; diff --git a/src/data/composite/things/commentary-entry/withWebArchiveDate.js b/src/data/composite/things/commentary-entry/withWebArchiveDate.js new file mode 100644 index 00000000..3aaa4f64 --- /dev/null +++ b/src/data/composite/things/commentary-entry/withWebArchiveDate.js @@ -0,0 +1,41 @@ +import {input, templateCompositeFrom} from '#composite'; + +import {raiseOutputWithoutDependency} from '#composite/control-flow'; + +export default templateCompositeFrom({ + annotation: `withWebArchiveDate`, + + outputs: ['#webArchiveDate'], + + steps: () => [ + { + dependencies: ['annotation'], + + compute: (continuation, {annotation}) => + continuation({ + ['#dateText']: + annotation + ?.match(/https?:\/\/web.archive.org\/web\/([0-9]{8,8})[0-9]*\//) + ?.[1] ?? + null, + }), + }, + + raiseOutputWithoutDependency({ + dependency: '#dateText', + output: input.value({['#webArchiveDate']: null}), + }), + + { + dependencies: ['#dateText'], + compute: (continuation, {['#dateText']: dateText}) => + continuation({ + ['#webArchiveDate']: + new Date( + dateText.slice(0, 4) + '/' + + dateText.slice(4, 6) + '/' + + dateText.slice(6, 8)), + }), + }, + ], +}); diff --git a/src/data/composite/things/track/withHasUniqueCoverArt.js b/src/data/composite/things/track/withHasUniqueCoverArt.js index af68073e..85d3b92a 100644 --- a/src/data/composite/things/track/withHasUniqueCoverArt.js +++ b/src/data/composite/things/track/withHasUniqueCoverArt.js @@ -15,7 +15,8 @@ import {input, templateCompositeFrom} from '#composite'; import {raiseOutputWithoutDependency, withResultOfAvailabilityCheck} from '#composite/control-flow'; -import {withFlattenedList, withPropertyFromList} from '#composite/data'; +import {fillMissingListItems, withFlattenedList, withPropertyFromList} + from '#composite/data'; import withPropertyFromAlbum from './withPropertyFromAlbum.js'; @@ -86,6 +87,13 @@ export default templateCompositeFrom({ internal: input.value(true), }), + // Since we're getting the update value for each artwork's artistContribs, + // it may not be set at all, and in that case won't be exposing as []. + fillMissingListItems({ + list: '#trackArtworks.artistContribs', + fill: input.value([]), + }), + withFlattenedList({ list: '#trackArtworks.artistContribs', }), @@ -93,17 +101,8 @@ export default templateCompositeFrom({ withResultOfAvailabilityCheck({ from: '#flattenedList', mode: input.value('empty'), + }).outputs({ + '#availability': '#hasUniqueCoverArt', }), - - { - dependencies: ['#availability'], - compute: (continuation, { - ['#availability']: availability, - }) => - continuation({ - ['#hasUniqueCoverArt']: - availability, - }), - }, ], }); diff --git a/src/data/composite/wiki-data/index.js b/src/data/composite/wiki-data/index.js index ee7411f2..005c68c0 100644 --- a/src/data/composite/wiki-data/index.js +++ b/src/data/composite/wiki-data/index.js @@ -16,7 +16,6 @@ export {default as withConstitutedArtwork} from './withConstitutedArtwork.js'; export {default as withContributionListSums} from './withContributionListSums.js'; export {default as withCoverArtDate} from './withCoverArtDate.js'; export {default as withDirectory} from './withDirectory.js'; -export {default as withParsedCommentaryEntries} from './withParsedCommentaryEntries.js'; export {default as withRecontextualizedContributionList} from './withRecontextualizedContributionList.js'; export {default as withRedatedContributionList} from './withRedatedContributionList.js'; export {default as withResolvedAnnotatedReferenceList} from './withResolvedAnnotatedReferenceList.js'; diff --git a/src/data/composite/wiki-data/withConstitutedArtwork.js b/src/data/composite/wiki-data/withConstitutedArtwork.js index 44623450..9e260abf 100644 --- a/src/data/composite/wiki-data/withConstitutedArtwork.js +++ b/src/data/composite/wiki-data/withConstitutedArtwork.js @@ -8,9 +8,11 @@ export default templateCompositeFrom({ inputs: { dimensionsFromThingProperty: input({type: 'string', acceptsNull: true}), fileExtensionFromThingProperty: input({type: 'string', acceptsNull: true}), + dateFromThingProperty: input({type: 'string', acceptsNull: true}), artistContribsFromThingProperty: input({type: 'string', acceptsNull: true}), artistContribsArtistProperty: input({type: 'string', acceptsNull: true}), - dateFromThingProperty: input({type: 'string', acceptsNull: true}), + artTagsFromThingProperty: input({type: 'string', acceptsNull: true}), + referencedArtworksFromThingProperty: input({type: 'string', acceptsNull: true}), }, outputs: ['#constitutedArtwork'], @@ -21,18 +23,22 @@ export default templateCompositeFrom({ input.myself(), input('dimensionsFromThingProperty'), input('fileExtensionFromThingProperty'), + input('dateFromThingProperty'), input('artistContribsFromThingProperty'), input('artistContribsArtistProperty'), - input('dateFromThingProperty'), + input('artTagsFromThingProperty'), + input('referencedArtworksFromThingProperty'), ], compute: (continuation, { [input.myself()]: myself, [input('dimensionsFromThingProperty')]: dimensionsFromThingProperty, [input('fileExtensionFromThingProperty')]: fileExtensionFromThingProperty, + [input('dateFromThingProperty')]: dateFromThingProperty, [input('artistContribsFromThingProperty')]: artistContribsFromThingProperty, [input('artistContribsArtistProperty')]: artistContribsArtistProperty, - [input('dateFromThingProperty')]: dateFromThingProperty, + [input('artTagsFromThingProperty')]: artTagsFromThingProperty, + [input('referencedArtworksFromThingProperty')]: referencedArtworksFromThingProperty, }) => continuation({ ['#constitutedArtwork']: Object.assign(new thingConstructors.Artwork, { @@ -41,7 +47,9 @@ export default templateCompositeFrom({ fileExtensionFromThingProperty, artistContribsFromThingProperty, artistContribsArtistProperty, + artTagsFromThingProperty, dateFromThingProperty, + referencedArtworksFromThingProperty, }), }), }, diff --git a/src/data/composite/wiki-data/withParsedCommentaryEntries.js b/src/data/composite/wiki-data/withParsedCommentaryEntries.js deleted file mode 100644 index 9bf4278c..00000000 --- a/src/data/composite/wiki-data/withParsedCommentaryEntries.js +++ /dev/null @@ -1,260 +0,0 @@ -import {input, templateCompositeFrom} from '#composite'; -import {stitchArrays} from '#sugar'; -import {isCommentary} from '#validators'; -import {commentaryRegexCaseSensitive} from '#wiki-data'; - -import { - fillMissingListItems, - withFlattenedList, - withPropertiesFromList, - withUnflattenedList, -} from '#composite/data'; - -import inputSoupyFind from './inputSoupyFind.js'; -import withResolvedReferenceList from './withResolvedReferenceList.js'; - -export default templateCompositeFrom({ - annotation: `withParsedCommentaryEntries`, - - inputs: { - from: input({validate: isCommentary}), - }, - - outputs: ['#parsedCommentaryEntries'], - - steps: () => [ - { - dependencies: [input('from')], - - compute: (continuation, { - [input('from')]: commentaryText, - }) => continuation({ - ['#rawMatches']: - Array.from(commentaryText.matchAll(commentaryRegexCaseSensitive)), - }), - }, - - withPropertiesFromList({ - list: '#rawMatches', - properties: input.value([ - '0', // The entire match as a string. - 'groups', - 'index', - ]), - }).outputs({ - '#rawMatches.0': '#rawMatches.text', - '#rawMatches.groups': '#rawMatches.groups', - '#rawMatches.index': '#rawMatches.startIndex', - }), - - { - dependencies: [ - '#rawMatches.text', - '#rawMatches.startIndex', - ], - - compute: (continuation, { - ['#rawMatches.text']: text, - ['#rawMatches.startIndex']: startIndex, - }) => continuation({ - ['#rawMatches.endIndex']: - stitchArrays({text, startIndex}) - .map(({text, startIndex}) => startIndex + text.length), - }), - }, - - { - dependencies: [ - input('from'), - '#rawMatches.startIndex', - '#rawMatches.endIndex', - ], - - compute: (continuation, { - [input('from')]: commentaryText, - ['#rawMatches.startIndex']: startIndex, - ['#rawMatches.endIndex']: endIndex, - }) => continuation({ - ['#entries.body']: - stitchArrays({startIndex, endIndex}) - .map(({endIndex}, index, stitched) => - (index === stitched.length - 1 - ? commentaryText.slice(endIndex) - : commentaryText.slice( - endIndex, - stitched[index + 1].startIndex))) - .map(body => body.trim()), - }), - }, - - withPropertiesFromList({ - list: '#rawMatches.groups', - prefix: input.value('#entries'), - properties: input.value([ - 'artistReferences', - 'artistDisplayText', - 'annotation', - 'date', - 'secondDate', - 'dateKind', - 'accessDate', - 'accessKind', - ]), - }), - - // The artistReferences group will always have a value, since it's required - // for the line to match in the first place. - - { - dependencies: ['#entries.artistReferences'], - compute: (continuation, { - ['#entries.artistReferences']: artistReferenceTexts, - }) => continuation({ - ['#entries.artistReferences']: - artistReferenceTexts - .map(text => text.split(',').map(ref => ref.trim())), - }), - }, - - withFlattenedList({ - list: '#entries.artistReferences', - }), - - withResolvedReferenceList({ - list: '#flattenedList', - find: inputSoupyFind.input('artist'), - notFoundMode: input.value('null'), - }), - - withUnflattenedList({ - list: '#resolvedReferenceList', - }).outputs({ - '#unflattenedList': '#entries.artists', - }), - - fillMissingListItems({ - list: '#entries.artistDisplayText', - fill: input.value(null), - }), - - fillMissingListItems({ - list: '#entries.annotation', - fill: input.value(null), - }), - - { - dependencies: ['#entries.annotation'], - compute: (continuation, { - ['#entries.annotation']: annotation, - }) => continuation({ - ['#entries.webArchiveDate']: - annotation - .map(text => text?.match(/https?:\/\/web.archive.org\/web\/([0-9]{8,8})[0-9]*\//)) - .map(match => match?.[1]) - .map(dateText => - (dateText - ? dateText.slice(0, 4) + '/' + - dateText.slice(4, 6) + '/' + - dateText.slice(6, 8) - : null)), - }), - }, - - { - dependencies: ['#entries.date'], - compute: (continuation, { - ['#entries.date']: date, - }) => continuation({ - ['#entries.date']: - date - .map(date => date ? new Date(date) : null), - }), - }, - - { - dependencies: ['#entries.secondDate'], - compute: (continuation, { - ['#entries.secondDate']: secondDate, - }) => continuation({ - ['#entries.secondDate']: - secondDate - .map(date => date ? new Date(date) : null), - }), - }, - - fillMissingListItems({ - list: '#entries.dateKind', - fill: input.value(null), - }), - - { - dependencies: ['#entries.accessDate', '#entries.webArchiveDate'], - compute: (continuation, { - ['#entries.accessDate']: accessDate, - ['#entries.webArchiveDate']: webArchiveDate, - }) => continuation({ - ['#entries.accessDate']: - stitchArrays({accessDate, webArchiveDate}) - .map(({accessDate, webArchiveDate}) => - accessDate ?? - webArchiveDate ?? - null) - .map(date => date ? new Date(date) : date), - }), - }, - - { - dependencies: ['#entries.accessKind', '#entries.webArchiveDate'], - compute: (continuation, { - ['#entries.accessKind']: accessKind, - ['#entries.webArchiveDate']: webArchiveDate, - }) => continuation({ - ['#entries.accessKind']: - stitchArrays({accessKind, webArchiveDate}) - .map(({accessKind, webArchiveDate}) => - accessKind ?? - (webArchiveDate && 'captured') ?? - null), - }), - }, - - { - dependencies: [ - '#entries.artists', - '#entries.artistDisplayText', - '#entries.annotation', - '#entries.date', - '#entries.secondDate', - '#entries.dateKind', - '#entries.accessDate', - '#entries.accessKind', - '#entries.body', - ], - - compute: (continuation, { - ['#entries.artists']: artists, - ['#entries.artistDisplayText']: artistDisplayText, - ['#entries.annotation']: annotation, - ['#entries.date']: date, - ['#entries.secondDate']: secondDate, - ['#entries.dateKind']: dateKind, - ['#entries.accessDate']: accessDate, - ['#entries.accessKind']: accessKind, - ['#entries.body']: body, - }) => continuation({ - ['#parsedCommentaryEntries']: - stitchArrays({ - artists, - artistDisplayText, - annotation, - date, - secondDate, - dateKind, - accessDate, - accessKind, - body, - }), - }), - }, - ], -}); diff --git a/src/data/composite/wiki-properties/commentary.js b/src/data/composite/wiki-properties/commentary.js deleted file mode 100644 index 9625278d..00000000 --- a/src/data/composite/wiki-properties/commentary.js +++ /dev/null @@ -1,30 +0,0 @@ -// Artist commentary! Generally present on tracks and albums. - -import {input, templateCompositeFrom} from '#composite'; -import {isCommentary} from '#validators'; - -import {exitWithoutDependency, exposeDependency} - from '#composite/control-flow'; -import {withParsedCommentaryEntries} from '#composite/wiki-data'; - -export default templateCompositeFrom({ - annotation: `commentary`, - - compose: false, - - steps: () => [ - exitWithoutDependency({ - dependency: input.updateValue({validate: isCommentary}), - mode: input.value('falsy'), - value: input.value([]), - }), - - withParsedCommentaryEntries({ - from: input.updateValue(), - }), - - exposeDependency({ - dependency: '#parsedCommentaryEntries', - }), - ], -}); diff --git a/src/data/composite/wiki-properties/commentatorArtists.js b/src/data/composite/wiki-properties/commentatorArtists.js index c5c14769..54d3e1a5 100644 --- a/src/data/composite/wiki-properties/commentatorArtists.js +++ b/src/data/composite/wiki-properties/commentatorArtists.js @@ -7,7 +7,6 @@ import {exitWithoutDependency, exposeDependency} from '#composite/control-flow'; import {withFlattenedList, withPropertyFromList, withUniqueItemsOnly} from '#composite/data'; -import {withParsedCommentaryEntries} from '#composite/wiki-data'; export default templateCompositeFrom({ annotation: `commentatorArtists`, @@ -21,19 +20,13 @@ export default templateCompositeFrom({ value: input.value([]), }), - withParsedCommentaryEntries({ - from: 'commentary', - }), - withPropertyFromList({ - list: '#parsedCommentaryEntries', + list: 'commentary', property: input.value('artists'), - }).outputs({ - '#parsedCommentaryEntries.artists': '#artistLists', }), withFlattenedList({ - list: '#artistLists', + list: '#commentary.artists', }).outputs({ '#flattenedList': '#artists', }), diff --git a/src/data/composite/wiki-properties/constitutibleArtwork.js b/src/data/composite/wiki-properties/constitutibleArtwork.js index 9f7ba13e..0ee3bfcd 100644 --- a/src/data/composite/wiki-properties/constitutibleArtwork.js +++ b/src/data/composite/wiki-properties/constitutibleArtwork.js @@ -19,9 +19,11 @@ const template = templateCompositeFrom({ inputs: { dimensionsFromThingProperty: input({type: 'string', acceptsNull: true}), fileExtensionFromThingProperty: input({type: 'string', acceptsNull: true}), + dateFromThingProperty: input({type: 'string', acceptsNull: true}), artistContribsFromThingProperty: input({type: 'string', acceptsNull: true}), artistContribsArtistProperty: input({type: 'string', acceptsNull: true}), - dateFromThingProperty: input({type: 'string', acceptsNull: true}), + artTagsFromThingProperty: input({type: 'string', acceptsNull: true}), + referencedArtworksFromThingProperty: input({type: 'string', acceptsNull: true}), }, steps: () => [ @@ -35,9 +37,11 @@ const template = templateCompositeFrom({ withConstitutedArtwork({ dimensionsFromThingProperty: input('dimensionsFromThingProperty'), fileExtensionFromThingProperty: input('fileExtensionFromThingProperty'), + dateFromThingProperty: input('dateFromThingProperty'), artistContribsFromThingProperty: input('artistContribsFromThingProperty'), artistContribsArtistProperty: input('artistContribsArtistProperty'), - dateFromThingProperty: input('dateFromThingProperty'), + artTagsFromThingProperty: input('artTagsFromThingProperty'), + referencedArtworksFromThingProperty: input('referencedArtworksFromThingProperty'), }), exposeDependency({ diff --git a/src/data/composite/wiki-properties/constitutibleArtworkList.js b/src/data/composite/wiki-properties/constitutibleArtworkList.js index 29e6c774..246c08b5 100644 --- a/src/data/composite/wiki-properties/constitutibleArtworkList.js +++ b/src/data/composite/wiki-properties/constitutibleArtworkList.js @@ -18,9 +18,11 @@ const template = templateCompositeFrom({ inputs: { dimensionsFromThingProperty: input({type: 'string', acceptsNull: true}), fileExtensionFromThingProperty: input({type: 'string', acceptsNull: true}), + dateFromThingProperty: input({type: 'string', acceptsNull: true}), artistContribsFromThingProperty: input({type: 'string', acceptsNull: true}), artistContribsArtistProperty: input({type: 'string', acceptsNull: true}), - dateFromThingProperty: input({type: 'string', acceptsNull: true}), + artTagsFromThingProperty: input({type: 'string', acceptsNull: true}), + referencedArtworksFromThingProperty: input({type: 'string', acceptsNull: true}), }, steps: () => [ @@ -34,9 +36,11 @@ const template = templateCompositeFrom({ withConstitutedArtwork({ dimensionsFromThingProperty: input('dimensionsFromThingProperty'), fileExtensionFromThingProperty: input('fileExtensionFromThingProperty'), + dateFromThingProperty: input('dateFromThingProperty'), artistContribsFromThingProperty: input('artistContribsFromThingProperty'), artistContribsArtistProperty: input('artistContribsArtistProperty'), - dateFromThingProperty: input('dateFromThingProperty'), + artTagsFromThingProperty: input('artTagsFromThingProperty'), + referencedArtworksFromThingProperty: input('referencedArtworksFromThingProperty'), }), { diff --git a/src/data/composite/wiki-properties/index.js b/src/data/composite/wiki-properties/index.js index 06a627ec..d5e7657e 100644 --- a/src/data/composite/wiki-properties/index.js +++ b/src/data/composite/wiki-properties/index.js @@ -7,7 +7,6 @@ export {default as additionalFiles} from './additionalFiles.js'; export {default as additionalNameList} from './additionalNameList.js'; export {default as annotatedReferenceList} from './annotatedReferenceList.js'; export {default as color} from './color.js'; -export {default as commentary} from './commentary.js'; export {default as commentatorArtists} from './commentatorArtists.js'; export {default as constitutibleArtwork} from './constitutibleArtwork.js'; export {default as constitutibleArtworkList} from './constitutibleArtworkList.js'; diff --git a/src/data/things/album.js b/src/data/things/album.js index 7c85366a..8a25a8ac 100644 --- a/src/data/things/album.js +++ b/src/data/things/album.js @@ -3,6 +3,7 @@ export const DATA_ALBUM_DIRECTORY = 'album'; import * as path from 'node:path'; import {inspect} from 'node:util'; +import CacheableObject from '#cacheable-object'; import {colors} from '#cli'; import {input} from '#composite'; import {traverse} from '#node-utils'; @@ -16,7 +17,9 @@ import { parseAdditionalNames, parseAnnotatedReferences, parseArtwork, + parseCommentary, parseContributors, + parseCreditingSources, parseDate, parseDimensions, parseWallpaperParts, @@ -32,7 +35,6 @@ import {exitWithoutContribs, withDirectory, withCoverArtDate} import { additionalFiles, additionalNameList, - commentary, color, commentatorArtists, constitutibleArtwork, @@ -59,7 +61,7 @@ import { wikiData, } from '#composite/wiki-properties'; -import {withTracks} from '#composite/things/album'; +import {withHasCoverArt, withTracks} from '#composite/things/album'; import {withAlbum, withContinueCountingFrom, withStartCountingFrom} from '#composite/things/track-section'; @@ -69,6 +71,8 @@ export class Album extends Thing { static [Thing.getPropertyDescriptors] = ({ ArtTag, Artwork, + CommentaryEntry, + CreditingSourcesEntry, Group, Track, TrackSection, @@ -188,9 +192,11 @@ export class Album extends Thing { ], coverArtworks: [ + withHasCoverArt(), + exitWithoutDependency({ - dependency: 'coverArtistContribs', - mode: input.value('empty'), + dependency: '#hasCoverArt', + mode: input.value('falsy'), value: input.value([]), }), @@ -202,8 +208,14 @@ export class Album extends Thing { isListedOnHomepage: flag(true), isListedInGalleries: flag(true), - commentary: commentary(), - creditSources: commentary(), + commentary: thingList({ + class: input.value(CommentaryEntry), + }), + + creditSources: thingList({ + class: input.value(CreditingSourcesEntry), + }), + additionalFiles: additionalFiles(), trackSections: thingList({ @@ -297,7 +309,11 @@ export class Album extends Thing { commentatorArtists: commentatorArtists(), - hasCoverArt: contribsPresent({contribs: 'coverArtistContribs'}), + hasCoverArt: [ + withHasCoverArt(), + exposeDependency({dependency: '#hasCoverArt'}), + ], + hasWallpaperArt: contribsPresent({contribs: 'wallpaperArtistContribs'}), hasBannerArt: contribsPresent({contribs: 'bannerArtistContribs'}), @@ -505,6 +521,8 @@ export class Album extends Thing { dateFromThingProperty: 'coverArtDate', artistContribsFromThingProperty: 'coverArtistContribs', artistContribsArtistProperty: 'albumCoverArtistContributions', + artTagsFromThingProperty: 'artTags', + referencedArtworksFromThingProperty: 'referencedArtworks', }), }, @@ -588,8 +606,15 @@ export class Album extends Thing { transform: parseDimensions, }, - 'Commentary': {property: 'commentary'}, - 'Credit Sources': {property: 'creditSources'}, + 'Commentary': { + property: 'commentary', + transform: parseCommentary, + }, + + 'Credit Sources': { + property: 'creditSources', + transform: parseCreditingSources, + }, 'Additional Files': { property: 'additionalFiles', @@ -660,7 +685,11 @@ export class Album extends Thing { const albumData = []; const trackSectionData = []; const trackData = []; + const artworkData = []; + const commentaryData = []; + const creditingSourceData = []; + const lyricsData = []; for (const {header: album, entries} of results) { const trackSections = []; @@ -707,6 +736,13 @@ export class Album extends Thing { entry.album = album; artworkData.push(...entry.trackArtworks); + commentaryData.push(...entry.commentary); + creditingSourceData.push(...entry.creditSources); + + // TODO: As exposed, Track.lyrics tries to inherit from the main + // release, which is impossible before the data's been linked. + // We just use the update value here. But it's icky! + lyricsData.push(...CacheableObject.getUpdateValue(entry, 'lyrics') ?? []); } closeCurrentTrackSection(); @@ -723,6 +759,9 @@ export class Album extends Thing { artworkData.push(album.wallpaperArtwork); } + commentaryData.push(...album.commentary); + creditingSourceData.push(...album.creditSources); + album.trackSections = trackSections; } @@ -730,7 +769,11 @@ export class Album extends Thing { albumData, trackSectionData, trackData, + artworkData, + commentaryData, + creditingSourceData, + lyricsData, }; }, @@ -761,12 +804,18 @@ export class Album extends Thing { ]; } + // TODO: using trackCover here is obviously, badly wrong + // but we ought to refactor banners and wallpapers similarly + // (i.e. depend on those intrinsic artwork paths rather than + // accessing media.{albumBanner,albumWallpaper} from content + // or other code directly) return [ - 'media.albumCover', + 'media.trackCover', + this.directory, (artwork.unqualifiedDirectory - ? this.directory + '-' + artwork.unqualifiedDirectory - : this.directory), + ? 'cover-' + artwork.unqualifiedDirectory + : 'cover'), artwork.fileExtension, ]; diff --git a/src/data/things/art-tag.js b/src/data/things/art-tag.js index 7944beb0..57e156ee 100644 --- a/src/data/things/art-tag.js +++ b/src/data/things/art-tag.js @@ -92,22 +92,11 @@ export class ArtTag extends Thing { }, ], - directlyTaggedInThings: { - flags: {expose: true}, - - expose: { - dependencies: ['this', 'reverse'], - compute: ({this: artTag, reverse}) => - sortAlbumsTracksChronologically( - [ - ...reverse.albumsWhoseArtworksFeature(artTag), - ...reverse.tracksWhoseArtworksFeature(artTag), - ], - {getDate: thing => thing.coverArtDate}), - }, - }, + directlyFeaturedInArtworks: reverseReferenceList({ + reverse: soupyReverse.input('artworksWhichFeature'), + }), - indirectlyTaggedInThings: [ + indirectlyFeaturedInArtworks: [ withAllDescendantArtTags(), { @@ -115,7 +104,7 @@ export class ArtTag extends Thing { compute: ({'#allDescendantArtTags': allDescendantArtTags}) => unique( allDescendantArtTags - .flatMap(artTag => artTag.directlyTaggedInThings)), + .flatMap(artTag => artTag.directlyFeaturedInArtworks)), }, ], diff --git a/src/data/things/artwork.js b/src/data/things/artwork.js index 65032d86..2a97fd6d 100644 --- a/src/data/things/artwork.js +++ b/src/data/things/artwork.js @@ -3,7 +3,6 @@ import {inspect} from 'node:util'; import {input} from '#composite'; import find from '#find'; import Thing from '#thing'; -import {parseAnnotatedReferences, parseContributors, parseDate} from '#yaml'; import { isContentString, @@ -18,6 +17,13 @@ import { validateReferenceList, } from '#validators'; +import { + parseAnnotatedReferences, + parseContributors, + parseDate, + parseDimensions, +} from '#yaml'; + import {withPropertyFromObject} from '#composite/data'; import { @@ -132,6 +138,11 @@ export class Artwork extends Thing { ['#value']: '#dimensionsFromThing', }), + exitWithoutDependency({ + dependency: 'dimensionsFromThingProperty', + value: input.value(null), + }), + exposeDependencyOrContinue({ dependency: '#dimensionsFromThing', }), @@ -179,6 +190,8 @@ export class Artwork extends Thing { }), ], + artTagsFromThingProperty: simpleString(), + artTags: [ withResolvedReferenceList({ list: input.updateValue({ @@ -194,13 +207,20 @@ export class Artwork extends Thing { mode: input.value('empty'), }), + exitWithoutDependency({ + dependency: 'artTagsFromThingProperty', + value: input.value([]), + }), + withPropertyFromObject({ object: 'thing', - property: input.value('artTags'), + property: 'artTagsFromThingProperty', + }).outputs({ + ['#value']: '#artTags', }), exposeDependencyOrContinue({ - dependency: '#thing.artTags', + dependency: '#artTags', }), exposeConstant({ @@ -208,6 +228,8 @@ export class Artwork extends Thing { }), ], + referencedArtworksFromThingProperty: simpleString(), + referencedArtworks: [ { compute: (continuation) => continuation({ @@ -244,13 +266,20 @@ export class Artwork extends Thing { mode: input.value('empty'), }), + exitWithoutDependency({ + dependency: 'referencedArtworksFromThingProperty', + value: input.value([]), + }), + withPropertyFromObject({ object: 'thing', - property: input.value('referencedArtworks'), + property: 'referencedArtworksFromThingProperty', + }).outputs({ + ['#value']: '#referencedArtworks', }), exposeDependencyOrContinue({ - dependency: '#thing.referencedArtworks', + dependency: '#referencedArtworks', }), exposeConstant({ @@ -280,6 +309,11 @@ export class Artwork extends Thing { 'Directory': {property: 'unqualifiedDirectory'}, 'File Extension': {property: 'fileExtension'}, + 'Dimensions': { + property: 'dimensions', + transform: parseDimensions, + }, + 'Label': {property: 'label'}, 'Source': {property: 'source'}, @@ -323,6 +357,13 @@ export class Artwork extends Thing { date: ({artwork}) => artwork.date, }, + + artworksWhichFeature: { + bindTo: 'artworkData', + + referencing: artwork => [artwork], + referenced: artwork => artwork.artTags, + }, }; get path() { diff --git a/src/data/things/content.js b/src/data/things/content.js new file mode 100644 index 00000000..7f352795 --- /dev/null +++ b/src/data/things/content.js @@ -0,0 +1,122 @@ +import {input} from '#composite'; +import find from '#find'; +import Thing from '#thing'; +import {is, isDate} from '#validators'; +import {parseDate} from '#yaml'; + +import {contentString, referenceList, simpleDate, soupyFind, thing} + from '#composite/wiki-properties'; + +import { + exposeConstant, + exposeDependencyOrContinue, + exposeUpdateValueOrContinue, + withResultOfAvailabilityCheck, +} from '#composite/control-flow'; + +import {withWebArchiveDate} from '#composite/things/commentary-entry'; + +export class ContentEntry extends Thing { + static [Thing.getPropertyDescriptors] = ({Artist}) => ({ + // Update & expose + + thing: thing(), + + artists: referenceList({ + class: input.value(Artist), + find: soupyFind.input('artist'), + }), + + artistText: contentString(), + + annotation: contentString(), + + dateKind: { + flags: {update: true, expose: true}, + + update: { + validate: is(...[ + 'sometime', + 'throughout', + 'around', + ]), + }, + }, + + accessKind: [ + exposeUpdateValueOrContinue({ + validate: input.value( + is(...[ + 'captured', + 'accessed', + ])), + }), + + withWebArchiveDate(), + + withResultOfAvailabilityCheck({ + from: '#webArchiveDate', + }), + + { + dependencies: ['#availability'], + compute: (continuation, {['#availability']: availability}) => + (availability + ? continuation.exit('captured') + : continuation()), + }, + + exposeConstant({ + value: input.value(null), + }), + ], + + date: simpleDate(), + + secondDate: simpleDate(), + + accessDate: [ + exposeUpdateValueOrContinue({ + validate: input.value(isDate), + }), + + withWebArchiveDate(), + + exposeDependencyOrContinue({ + dependency: '#webArchiveDate', + }), + + exposeConstant({ + value: input.value(null), + }), + ], + + body: contentString(), + + // Update only + + find: soupyFind(), + }); + + static [Thing.yamlDocumentSpec] = { + fields: { + 'Artists': {property: 'artists'}, + 'Artist Text': {property: 'artistText'}, + + 'Annotation': {property: 'annotation'}, + + 'Date Kind': {property: 'dateKind'}, + 'Access Kind': {property: 'accessKind'}, + + 'Date': {property: 'date', transform: parseDate}, + 'Second Date': {property: 'secondDate', transform: parseDate}, + 'Access Date': {property: 'accessDate', transform: parseDate}, + + 'Body': {property: 'body'}, + }, + }; +} + +export class CommentaryEntry extends ContentEntry {} +export class LyricsEntry extends ContentEntry {} +export class CreditingSourcesEntry extends ContentEntry {} diff --git a/src/data/things/flash.js b/src/data/things/flash.js index ace18af9..dac674dd 100644 --- a/src/data/things/flash.js +++ b/src/data/things/flash.js @@ -10,7 +10,9 @@ import {anyOf, isColor, isContentString, isDirectory, isNumber, isString} import { parseArtwork, parseAdditionalNames, + parseCommentary, parseContributors, + parseCreditingSources, parseDate, parseDimensions, } from '#yaml'; @@ -27,7 +29,6 @@ import { import { additionalNameList, color, - commentary, commentatorArtists, constitutibleArtwork, contentString, @@ -41,6 +42,7 @@ import { soupyFind, soupyReverse, thing, + thingList, urls, wikiData, } from '#composite/wiki-properties'; @@ -52,6 +54,8 @@ export class Flash extends Thing { static [Thing.referenceType] = 'flash'; static [Thing.getPropertyDescriptors] = ({ + CommentaryEntry, + CreditingSourcesEntry, Track, FlashAct, WikiInfo, @@ -125,8 +129,13 @@ export class Flash extends Thing { additionalNames: additionalNameList(), - commentary: commentary(), - creditSources: commentary(), + commentary: thingList({ + class: input.value(CommentaryEntry), + }), + + creditSources: thingList({ + class: input.value(CreditingSourcesEntry), + }), // Update only @@ -240,8 +249,15 @@ export class Flash extends Thing { transform: parseContributors, }, - 'Commentary': {property: 'commentary'}, - 'Credit Sources': {property: 'creditSources'}, + 'Commentary': { + property: 'commentary', + transform: parseCommentary, + }, + + 'Credit Sources': { + property: 'creditSources', + transform: parseCreditingSources, + }, 'Review Points': {ignore: true}, }, @@ -441,8 +457,18 @@ export class FlashSide extends Thing { const flashSideData = results.filter(x => x instanceof FlashSide); const artworkData = flashData.map(flash => flash.coverArtwork); - - return {flashData, flashActData, flashSideData, artworkData}; + const commentaryData = flashData.flatMap(flash => flash.commentary); + const creditingSourceData = flashData.flatMap(flash => flash.creditSources); + + return { + flashData, + flashActData, + flashSideData, + + artworkData, + commentaryData, + creditingSourceData, + }; }, sort({flashData}) { diff --git a/src/data/things/index.js b/src/data/things/index.js index 96cec88e..b832ab75 100644 --- a/src/data/things/index.js +++ b/src/data/things/index.js @@ -13,6 +13,7 @@ import * as albumClasses from './album.js'; import * as artTagClasses from './art-tag.js'; import * as artistClasses from './artist.js'; import * as artworkClasses from './artwork.js'; +import * as contentClasses from './content.js'; import * as contributionClasses from './contribution.js'; import * as flashClasses from './flash.js'; import * as groupClasses from './group.js'; @@ -29,6 +30,7 @@ const allClassLists = { 'art-tag.js': artTagClasses, 'artist.js': artistClasses, 'artwork.js': artworkClasses, + 'content.js': contentClasses, 'contribution.js': contributionClasses, 'flash.js': flashClasses, 'group.js': groupClasses, diff --git a/src/data/things/track.js b/src/data/things/track.js index 2d2cc002..ae7be170 100644 --- a/src/data/things/track.js +++ b/src/data/things/track.js @@ -12,10 +12,13 @@ import { parseAdditionalNames, parseAnnotatedReferences, parseArtwork, + parseCommentary, parseContributors, + parseCreditingSources, parseDate, parseDimensions, parseDuration, + parseLyrics, } from '#yaml'; import {withPropertyFromObject} from '#composite/data'; @@ -37,7 +40,6 @@ import { import { additionalFiles, additionalNameList, - commentary, commentatorArtists, constitutibleArtworkList, contentString, @@ -56,6 +58,7 @@ import { soupyFind, soupyReverse, thing, + thingList, urls, wikiData, } from '#composite/wiki-properties'; @@ -86,7 +89,10 @@ export class Track extends Thing { Album, ArtTag, Artwork, + CommentaryEntry, + CreditingSourcesEntry, Flash, + LyricsEntry, TrackSection, WikiInfo, }) => ({ @@ -215,12 +221,23 @@ export class Track extends Thing { dimensions(), ], - commentary: commentary(), - creditSources: commentary(), + commentary: thingList({ + class: input.value(CommentaryEntry), + }), + + creditSources: thingList({ + class: input.value(CreditingSourcesEntry), + }), lyrics: [ + // TODO: Inherited lyrics are literally the same objects, so of course + // their .thing properties aren't going to point back to this one, and + // certainly couldn't be recontextualized... inheritFromMainRelease(), - contentString(), + + thingList({ + class: input.value(LyricsEntry), + }), ], additionalFiles: additionalFiles(), @@ -480,9 +497,20 @@ export class Track extends Thing { 'Always Reference By Directory': {property: 'alwaysReferenceByDirectory'}, - 'Lyrics': {property: 'lyrics'}, - 'Commentary': {property: 'commentary'}, - 'Credit Sources': {property: 'creditSources'}, + 'Lyrics': { + property: 'lyrics', + transform: parseLyrics, + }, + + 'Commentary': { + property: 'commentary', + transform: parseCommentary, + }, + + 'Credit Sources': { + property: 'creditSources', + transform: parseCreditingSources, + }, 'Additional Files': { property: 'additionalFiles', @@ -533,6 +561,8 @@ export class Track extends Thing { dimensionsFromThingProperty: 'coverArtDimensions', fileExtensionFromThingProperty: 'coverArtFileExtension', dateFromThingProperty: 'coverArtDate', + artTagsFromThingProperty: 'artTags', + referencedArtworksFromThingProperty: 'referencedArtworks', artistContribsFromThingProperty: 'coverArtistContribs', artistContribsArtistProperty: 'trackCoverArtistContributions', }), diff --git a/src/data/yaml.js b/src/data/yaml.js index 07dbe882..a2c76b45 100644 --- a/src/data/yaml.js +++ b/src/data/yaml.js @@ -11,6 +11,7 @@ import {colors, ENABLE_COLOR, logInfo, logWarn} from '#cli'; import {sortByName} from '#sort'; import Thing from '#thing'; import thingConstructors from '#things'; +import {matchContentEntries, multipleLyricsDetectionRegex} from '#wiki-data'; import { aggregateThrows, @@ -791,11 +792,13 @@ export function parseAnnotatedReferences(entries, { export function parseArtwork({ single = false, - dimensionsFromThingProperty, - fileExtensionFromThingProperty, - dateFromThingProperty, - artistContribsFromThingProperty, - artistContribsArtistProperty, + dimensionsFromThingProperty = null, + fileExtensionFromThingProperty = null, + dateFromThingProperty = null, + artistContribsFromThingProperty = null, + artistContribsArtistProperty = null, + artTagsFromThingProperty = null, + referencedArtworksFromThingProperty = null, }) { const provide = { dimensionsFromThingProperty, @@ -803,6 +806,8 @@ export function parseArtwork({ dateFromThingProperty, artistContribsFromThingProperty, artistContribsArtistProperty, + artTagsFromThingProperty, + referencedArtworksFromThingProperty, }; const parseSingleEntry = (entry, {subdoc, Artwork}) => @@ -820,6 +825,73 @@ export function parseArtwork({ return transform; } +export function parseContentEntries(thingClass, sourceText, {subdoc}) { + const map = matchEntry => ({ + 'Artists': + matchEntry.artistReferences + .split(',') + .map(ref => ref.trim()), + + 'Artist Text': + matchEntry.artistDisplayText, + + 'Annotation': + matchEntry.annotation, + + 'Date': + matchEntry.date, + + 'Second Date': + matchEntry.secondDate, + + 'Date Kind': + matchEntry.dateKind, + + 'Access Date': + matchEntry.accessDate, + + 'Access Kind': + matchEntry.accessKind, + + 'Body': + matchEntry.body, + }); + + const documents = + matchContentEntries(sourceText) + .map(matchEntry => + withEntries( + map(matchEntry), + entries => entries + .filter(([key, value]) => + value !== undefined && + value !== null))); + + const subdocs = + documents.map(document => + subdoc(thingClass, document, {bindInto: 'thing'})); + + return subdocs; +} + +export function parseCommentary(sourceText, {subdoc, CommentaryEntry}) { + return parseContentEntries(CommentaryEntry, sourceText, {subdoc}); +} + +export function parseCreditingSources(sourceText, {subdoc, CreditingSourcesEntry}) { + return parseContentEntries(CreditingSourcesEntry, sourceText, {subdoc}); +} + +export function parseLyrics(sourceText, {subdoc, LyricsEntry}) { + if (!multipleLyricsDetectionRegex.test(sourceText)) { + const document = {'Body': sourceText}; + + return [subdoc(LyricsEntry, document, {bindInto: 'thing'})]; + } + + return parseContentEntries(LyricsEntry, sourceText, {subdoc}); +} + // documentModes: Symbols indicating sets of behavior for loading and processing // data files. export const documentModes = { @@ -1495,6 +1567,10 @@ export function linkWikiDataArrays(wikiData, {bindFind, bindReverse}) { ['artworkData', ['artworkData']], + ['commentaryData', [/* find */]], + + ['creditingSourceData', [/* find */]], + ['flashData', [ 'wikiInfo', ]], @@ -1509,6 +1585,8 @@ export function linkWikiDataArrays(wikiData, {bindFind, bindReverse}) { ['homepageLayout.sections.rows', [/* find */]], + ['lyricsData', [/* find */]], + ['trackData', [ 'artworkData', 'trackData', |