diff options
Diffstat (limited to 'src/data')
34 files changed, 759 insertions, 760 deletions
diff --git a/src/data/checks.js b/src/data/checks.js index 25863d2d..52024144 100644 --- a/src/data/checks.js +++ b/src/data/checks.js @@ -19,12 +19,6 @@ import { withAggregate, } from '#aggregate'; -import { - combineWikiDataArrays, - commentaryRegexCaseSensitive, - oldStyleLyricsDetectionRegex, -} from '#wiki-data'; - function inspect(value, opts = {}) { return nodeInspect(value, {colors: ENABLE_COLOR, ...opts}); } @@ -190,7 +184,8 @@ export function filterReferenceErrors(wikiData, { groups: 'group', artTags: '_artTag', referencedArtworks: '_artwork', - commentary: '_commentary', + commentary: '_content', + creditSources: '_content', }], ['artTagData', { @@ -198,7 +193,8 @@ export function filterReferenceErrors(wikiData, { }], ['flashData', { - commentary: '_commentary', + commentary: '_content', + creditSources: '_content', }], ['groupCategoryData', { @@ -238,7 +234,9 @@ export function filterReferenceErrors(wikiData, { artTags: '_artTag', referencedArtworks: '_artwork', mainReleaseTrack: '_trackMainReleasesOnly', - commentary: '_commentary', + commentary: '_content', + creditSources: '_content', + lyrics: '_content', }], ['wikiInfo', { @@ -273,12 +271,12 @@ export function filterReferenceErrors(wikiData, { let writeProperty = true; switch (findFnKey) { - case '_commentary': + case '_content': if (value) { value = - Array.from(value.matchAll(commentaryRegexCaseSensitive)) - .map(({groups}) => groups.artistReferences) - .map(text => text.split(',').map(text => text.trim())); + value.map(entry => + CacheableObject.getUpdateValue(entry, 'artists') ?? + []); } writeProperty = false; @@ -334,7 +332,7 @@ export function filterReferenceErrors(wikiData, { findFn = boundFind.artTag; break; - case '_commentary': + case '_content': findFn = findArtistOrAlias; break; @@ -466,7 +464,7 @@ export function filterReferenceErrors(wikiData, { } } - if (findFnKey === '_commentary') { + if (findFnKey === '_content') { filter( value, {message: errorMessage}, decorateErrorWithIndex(refs => @@ -573,7 +571,7 @@ export function reportContentTextErrors(wikiData, { annotation: 'commentary annotation', }; - const newStyleLyricsShape = { + const lyricsShape = { body: 'lyrics body', artistDisplayText: 'lyrics artist display text', annotation: 'lyrics annotation', @@ -625,7 +623,7 @@ export function reportContentTextErrors(wikiData, { additionalFiles: additionalFileShape, commentary: commentaryShape, creditSources: commentaryShape, - lyrics: '_lyrics', + lyrics: lyricsShape, midiProjectFiles: additionalFileShape, sheetMusicFiles: additionalFileShape, }], @@ -749,7 +747,6 @@ export function reportContentTextErrors(wikiData, { nest({message: `Content text errors in ${inspect(thing)}`}, ({nest, push}) => { for (let [property, shape] of Object.entries(propSpec)) { - const rawValue = CacheableObject.getUpdateValue(thing, property); let value = thing[property]; if (value === undefined) { @@ -761,15 +758,6 @@ export function reportContentTextErrors(wikiData, { continue; } - if (shape === '_lyrics') { - if (oldStyleLyricsDetectionRegex.test(rawValue)) { - value = rawValue; - shape = '_content'; - } else { - shape = newStyleLyricsShape; - } - } - const fieldPropertyMessage = getFieldPropertyMessage( thing.constructor[Thing.yamlDocumentSpec], diff --git a/src/data/composite/control-flow/flipFilter.js b/src/data/composite/control-flow/flipFilter.js new file mode 100644 index 00000000..995bacad --- /dev/null +++ b/src/data/composite/control-flow/flipFilter.js @@ -0,0 +1,36 @@ +// Flips a filter, so that each true item becomes false, and vice versa. +// Overwrites the provided dependency. +// +// See also: +// - withAvailabilityFilter + +import {input, templateCompositeFrom} from '#composite'; + +export default templateCompositeFrom({ + annotation: `flipFilter`, + + inputs: { + filter: input({type: 'array'}), + }, + + outputs: ({ + [input.staticDependency('filter')]: filterDependency, + }) => [filterDependency ?? '#flippedFilter'], + + steps: () => [ + { + dependencies: [ + input('filter'), + input.staticDependency('filter'), + ], + + compute: (continuation, { + [input('filter')]: filter, + [input.staticDependency('filter')]: filterDependency, + }) => continuation({ + [filterDependency ?? '#flippedFilter']: + filter.map(item => !item), + }), + }, + ], +}); diff --git a/src/data/composite/control-flow/index.js b/src/data/composite/control-flow/index.js index 7e137a14..778dc66b 100644 --- a/src/data/composite/control-flow/index.js +++ b/src/data/composite/control-flow/index.js @@ -10,6 +10,7 @@ export {default as exposeDependency} from './exposeDependency.js'; export {default as exposeDependencyOrContinue} from './exposeDependencyOrContinue.js'; export {default as exposeUpdateValueOrContinue} from './exposeUpdateValueOrContinue.js'; export {default as exposeWhetherDependencyAvailable} from './exposeWhetherDependencyAvailable.js'; +export {default as flipFilter} from './flipFilter.js'; export {default as raiseOutputWithoutDependency} from './raiseOutputWithoutDependency.js'; export {default as raiseOutputWithoutUpdateValue} from './raiseOutputWithoutUpdateValue.js'; export {default as withAvailabilityFilter} from './withAvailabilityFilter.js'; diff --git a/src/data/composite/control-flow/withAvailabilityFilter.js b/src/data/composite/control-flow/withAvailabilityFilter.js index cfea998e..fd93af71 100644 --- a/src/data/composite/control-flow/withAvailabilityFilter.js +++ b/src/data/composite/control-flow/withAvailabilityFilter.js @@ -4,6 +4,7 @@ // Accepts the same mode options as withResultOfAvailabilityCheck. // // See also: +// - flipFilter // - withFilteredList // - withResultOfAvailabilityCheck // diff --git a/src/data/composite/data/withFilteredList.js b/src/data/composite/data/withFilteredList.js index 44c1661d..15ee3373 100644 --- a/src/data/composite/data/withFilteredList.js +++ b/src/data/composite/data/withFilteredList.js @@ -2,9 +2,6 @@ // corresponding items in a list. Items which correspond to a truthy value // are kept, and the rest are excluded from the output list. // -// If the flip option is set, only items corresponding with a *falsy* value in -// the filter are kept. -// // TODO: There should be two outputs - one for the items included according to // the filter, and one for the items excluded. // @@ -22,28 +19,19 @@ export default templateCompositeFrom({ inputs: { list: input({type: 'array'}), filter: input({type: 'array'}), - - flip: input({ - type: 'boolean', - defaultValue: false, - }), }, outputs: ['#filteredList'], steps: () => [ { - dependencies: [input('list'), input('filter'), input('flip')], + dependencies: [input('list'), input('filter')], compute: (continuation, { [input('list')]: list, [input('filter')]: filter, - [input('flip')]: flip, }) => continuation({ '#filteredList': - list.filter((_item, index) => - (flip - ? !filter[index] - : filter[index])), + list.filter((_item, index) => filter[index]), }), }, ], diff --git a/src/data/composite/data/withNearbyItemFromList.js b/src/data/composite/data/withNearbyItemFromList.js index 83a8cc21..5e165219 100644 --- a/src/data/composite/data/withNearbyItemFromList.js +++ b/src/data/composite/data/withNearbyItemFromList.js @@ -9,6 +9,10 @@ // - If the 'valuePastEdge' input is provided, that value will be output // instead of null. // +// - If the 'filter' input is provided, corresponding items will be skipped, +// and only (repeating `offset`) the next included in the filter will be +// returned. +// // Both the list and item must be provided. // // See also: @@ -16,7 +20,6 @@ // import {input, templateCompositeFrom} from '#composite'; -import {atOffset} from '#sugar'; import {raiseOutputWithoutDependency} from '#composite/control-flow'; @@ -28,9 +31,12 @@ export default templateCompositeFrom({ inputs: { list: input({acceptsNull: false, type: 'array'}), item: input({acceptsNull: false}), - offset: input({type: 'number'}), + wrap: input({type: 'boolean', defaultValue: false}), + valuePastEdge: input({defaultValue: null}), + + filter: input({defaultValue: null, type: 'array'}), }, outputs: ['#nearbyItem'], @@ -45,29 +51,55 @@ export default templateCompositeFrom({ dependency: '#index', mode: input.value('index'), - output: input.value({ - ['#nearbyItem']: - null, - }), + output: input.value({'#nearbyItem': null}), }), { dependencies: [ input('list'), input('offset'), + input('wrap'), + input('valuePastEdge'), + + input('filter'), + '#index', ], compute: (continuation, { [input('list')]: list, [input('offset')]: offset, + [input('wrap')]: wrap, + [input('valuePastEdge')]: valuePastEdge, + + [input('filter')]: filter, + ['#index']: index, - }) => continuation({ - ['#nearbyItem']: - atOffset(list, index, offset, {wrap}), - }), + }) => { + const startIndex = index; + + do { + index += offset; + + if (wrap) { + index = index % list.length; + } else if (index < 0) { + return continuation({'#nearbyItem': valuePastEdge}); + } else if (index >= list.length) { + return continuation({'#nearbyItem': valuePastEdge}); + } + + if (filter && !filter[index]) { + continue; + } + + return continuation({'#nearbyItem': list[index]}); + } while (index !== startIndex); + + return continuation({'#nearbyItem': null}); + }, }, ], }); diff --git a/src/data/composite/data/withPropertyFromObject.js b/src/data/composite/data/withPropertyFromObject.js index 4f240506..7b452b99 100644 --- a/src/data/composite/data/withPropertyFromObject.js +++ b/src/data/composite/data/withPropertyFromObject.js @@ -13,6 +13,21 @@ import CacheableObject from '#cacheable-object'; import {input, templateCompositeFrom} from '#composite'; +function getOutputName({ + [input.staticDependency('object')]: object, + [input.staticValue('property')]: property, +}) { + if (object && property) { + if (object.startsWith('#')) { + return `${object}.${property}`; + } else { + return `#${object}.${property}`; + } + } else { + return '#value'; + } +} + export default templateCompositeFrom({ annotation: `withPropertyFromObject`, @@ -22,15 +37,7 @@ export default templateCompositeFrom({ internal: input({type: 'boolean', defaultValue: false}), }, - outputs: ({ - [input.staticDependency('object')]: object, - [input.staticValue('property')]: property, - }) => - (object && property - ? (object.startsWith('#') - ? [`${object}.${property}`] - : [`#${object}.${property}`]) - : ['#value']), + outputs: inputs => [getOutputName(inputs)], steps: () => [ { @@ -39,17 +46,8 @@ export default templateCompositeFrom({ input.staticValue('property'), ], - compute: (continuation, { - [input.staticDependency('object')]: object, - [input.staticValue('property')]: property, - }) => continuation({ - '#output': - (object && property - ? (object.startsWith('#') - ? `${object}.${property}` - : `#${object}.${property}`) - : '#value'), - }), + compute: (continuation, inputs) => + continuation({'#output': getOutputName(inputs)}), }, { diff --git a/src/data/composite/things/artwork/index.js b/src/data/composite/things/artwork/index.js index b92bff72..3693c10f 100644 --- a/src/data/composite/things/artwork/index.js +++ b/src/data/composite/things/artwork/index.js @@ -1 +1,5 @@ +export {default as withAttachedArtwork} from './withAttachedArtwork.js'; +export {default as withContainingArtworkList} from './withContainingArtworkList.js'; +export {default as withContribsFromAttachedArtwork} from './withContribsFromAttachedArtwork.js'; export {default as withDate} from './withDate.js'; +export {default as withPropertyFromAttachedArtwork} from './withPropertyFromAttachedArtwork.js'; diff --git a/src/data/composite/things/artwork/withAttachedArtwork.js b/src/data/composite/things/artwork/withAttachedArtwork.js new file mode 100644 index 00000000..d7c0d87b --- /dev/null +++ b/src/data/composite/things/artwork/withAttachedArtwork.js @@ -0,0 +1,43 @@ +import {input, templateCompositeFrom} from '#composite'; + +import {flipFilter, raiseOutputWithoutDependency} + from '#composite/control-flow'; +import {withNearbyItemFromList, withPropertyFromList} from '#composite/data'; + +import withContainingArtworkList from './withContainingArtworkList.js'; + +export default templateCompositeFrom({ + annotaion: `withContribsFromMainArtwork`, + + outputs: ['#attachedArtwork'], + + steps: () => [ + raiseOutputWithoutDependency({ + dependency: 'attachAbove', + mode: input.value('falsy'), + output: input.value({'#attachedArtwork': null}), + }), + + withContainingArtworkList(), + + withPropertyFromList({ + list: '#containingArtworkList', + property: input.value('attachAbove'), + }), + + flipFilter({ + filter: '#containingArtworkList.attachAbove', + }).outputs({ + '#containingArtworkList.attachAbove': '#filterNotAttached', + }), + + withNearbyItemFromList({ + list: '#containingArtworkList', + item: input.myself(), + offset: input.value(-1), + filter: '#filterNotAttached', + }).outputs({ + '#nearbyItem': '#attachedArtwork', + }), + ], +}); diff --git a/src/data/composite/things/artwork/withContainingArtworkList.js b/src/data/composite/things/artwork/withContainingArtworkList.js new file mode 100644 index 00000000..9c928ffd --- /dev/null +++ b/src/data/composite/things/artwork/withContainingArtworkList.js @@ -0,0 +1,46 @@ +// Gets the list of artworks which contains this one, which is functionally +// equivalent to `this.thing[this.thingProperty]`. If the exposed value is not +// a list at all (i.e. the property holds a single artwork), this composition +// outputs null. + +import {input, templateCompositeFrom} from '#composite'; + +import {raiseOutputWithoutDependency} from '#composite/control-flow'; +import {withPropertyFromObject} from '#composite/data'; + +export default templateCompositeFrom({ + annotation: `withContainingArtworkList`, + + outputs: ['#containingArtworkList'], + + steps: () => [ + raiseOutputWithoutDependency({ + dependency: 'thing', + output: input.value({'#containingArtworkList': null}), + }), + + raiseOutputWithoutDependency({ + dependency: 'thingProperty', + output: input.value({'#containingArtworkList': null}), + }), + + withPropertyFromObject({ + object: 'thing', + property: 'thingProperty', + }).outputs({ + '#value': '#containingValue', + }), + + { + dependencies: ['#containingValue'], + compute: (continuation, { + ['#containingValue']: containingValue, + }) => continuation({ + ['#containingArtworkList']: + (Array.isArray(containingValue) + ? containingValue + : null), + }), + }, + ], +}); diff --git a/src/data/composite/things/artwork/withContribsFromAttachedArtwork.js b/src/data/composite/things/artwork/withContribsFromAttachedArtwork.js new file mode 100644 index 00000000..36abb3fe --- /dev/null +++ b/src/data/composite/things/artwork/withContribsFromAttachedArtwork.js @@ -0,0 +1,28 @@ +import {input, templateCompositeFrom} from '#composite'; + +import {raiseOutputWithoutDependency} from '#composite/control-flow'; +import {withPropertyFromObject} from '#composite/data'; +import {withRecontextualizedContributionList} from '#composite/wiki-data'; + +import withPropertyFromAttachedArtwork from './withPropertyFromAttachedArtwork.js'; + +export default templateCompositeFrom({ + annotaion: `withContribsFromAttachedArtwork`, + + outputs: ['#attachedArtwork.artistContribs'], + + steps: () => [ + withPropertyFromAttachedArtwork({ + property: input.value('artistContribs'), + }), + + raiseOutputWithoutDependency({ + dependency: '#attachedArtwork.artistContribs', + output: input.value({'#attachedArtwork.artistContribs': null}), + }), + + withRecontextualizedContributionList({ + list: '#attachedArtwork.artistContribs', + }), + ], +}); diff --git a/src/data/composite/things/artwork/withPropertyFromAttachedArtwork.js b/src/data/composite/things/artwork/withPropertyFromAttachedArtwork.js new file mode 100644 index 00000000..a2f954b9 --- /dev/null +++ b/src/data/composite/things/artwork/withPropertyFromAttachedArtwork.js @@ -0,0 +1,65 @@ +import {input, templateCompositeFrom} from '#composite'; + +import {withResultOfAvailabilityCheck} from '#composite/control-flow'; +import {withPropertyFromObject} from '#composite/data'; + +import withAttachedArtwork from './withAttachedArtwork.js'; + +function getOutputName({ + [input.staticValue('property')]: property, +}) { + if (property) { + return `#attachedArtwork.${property}`; + } else { + return '#value'; + } +} + +export default templateCompositeFrom({ + annotation: `withPropertyFromAttachedArtwork`, + + inputs: { + property: input({type: 'string'}), + }, + + outputs: inputs => [getOutputName(inputs)], + + steps: () => [ + { + dependencies: [input.staticValue('property')], + compute: (continuation, inputs) => + continuation({'#output': getOutputName(inputs)}), + }, + + withAttachedArtwork(), + + withResultOfAvailabilityCheck({ + from: '#attachedArtwork', + }), + + { + dependencies: ['#availability', '#output'], + compute: (continuation, { + ['#availability']: availability, + ['#output']: output, + }) => + (availability + ? continuation() + : continuation.raiseOutput({[output]: null})), + }, + + withPropertyFromObject({ + object: '#attachedArtwork', + property: input('property'), + }), + + { + dependencies: ['#value', '#output'], + compute: (continuation, { + ['#value']: value, + ['#output']: output, + }) => + continuation.raiseOutput({[output]: value}), + }, + ], +}); diff --git a/src/data/composite/things/commentary-entry/index.js b/src/data/composite/things/commentary-entry/index.js new file mode 100644 index 00000000..091bae1a --- /dev/null +++ b/src/data/composite/things/commentary-entry/index.js @@ -0,0 +1 @@ +export {default as withWebArchiveDate} from './withWebArchiveDate.js'; diff --git a/src/data/composite/things/commentary-entry/withWebArchiveDate.js b/src/data/composite/things/commentary-entry/withWebArchiveDate.js new file mode 100644 index 00000000..3aaa4f64 --- /dev/null +++ b/src/data/composite/things/commentary-entry/withWebArchiveDate.js @@ -0,0 +1,41 @@ +import {input, templateCompositeFrom} from '#composite'; + +import {raiseOutputWithoutDependency} from '#composite/control-flow'; + +export default templateCompositeFrom({ + annotation: `withWebArchiveDate`, + + outputs: ['#webArchiveDate'], + + steps: () => [ + { + dependencies: ['annotation'], + + compute: (continuation, {annotation}) => + continuation({ + ['#dateText']: + annotation + ?.match(/https?:\/\/web.archive.org\/web\/([0-9]{8,8})[0-9]*\//) + ?.[1] ?? + null, + }), + }, + + raiseOutputWithoutDependency({ + dependency: '#dateText', + output: input.value({['#webArchiveDate']: null}), + }), + + { + dependencies: ['#dateText'], + compute: (continuation, {['#dateText']: dateText}) => + continuation({ + ['#webArchiveDate']: + new Date( + dateText.slice(0, 4) + '/' + + dateText.slice(4, 6) + '/' + + dateText.slice(6, 8)), + }), + }, + ], +}); diff --git a/src/data/composite/wiki-data/index.js b/src/data/composite/wiki-data/index.js index 1d94f74b..005c68c0 100644 --- a/src/data/composite/wiki-data/index.js +++ b/src/data/composite/wiki-data/index.js @@ -11,15 +11,11 @@ export {default as inputNotFoundMode} from './inputNotFoundMode.js'; export {default as inputSoupyFind} from './inputSoupyFind.js'; export {default as inputSoupyReverse} from './inputSoupyReverse.js'; export {default as inputWikiData} from './inputWikiData.js'; -export {default as processContentEntryDates} from './processContentEntryDates.js'; export {default as withClonedThings} from './withClonedThings.js'; export {default as withConstitutedArtwork} from './withConstitutedArtwork.js'; export {default as withContributionListSums} from './withContributionListSums.js'; export {default as withCoverArtDate} from './withCoverArtDate.js'; export {default as withDirectory} from './withDirectory.js'; -export {default as withParsedCommentaryEntries} from './withParsedCommentaryEntries.js'; -export {default as withParsedContentEntries} from './withParsedContentEntries.js'; -export {default as withParsedLyricsEntries} from './withParsedLyricsEntries.js'; export {default as withRecontextualizedContributionList} from './withRecontextualizedContributionList.js'; export {default as withRedatedContributionList} from './withRedatedContributionList.js'; export {default as withResolvedAnnotatedReferenceList} from './withResolvedAnnotatedReferenceList.js'; diff --git a/src/data/composite/wiki-data/processContentEntryDates.js b/src/data/composite/wiki-data/processContentEntryDates.js deleted file mode 100644 index e418a121..00000000 --- a/src/data/composite/wiki-data/processContentEntryDates.js +++ /dev/null @@ -1,181 +0,0 @@ -import {input, templateCompositeFrom} from '#composite'; -import {stitchArrays} from '#sugar'; -import {isContentString, isString, looseArrayOf} from '#validators'; - -import {fillMissingListItems} from '#composite/data'; - -// Important note: These two kinds of inputs have the exact same shape!! -// This isn't on purpose (besides that they *are* both supposed to be strings). -// They just don't have any more particular validation, yet. - -const inputDateList = defaultDependency => - input({ - validate: looseArrayOf(isString), - defaultDependency, - }); - -const inputKindList = defaultDependency => - input.staticDependency({ - validate: looseArrayOf(isString), - defaultDependency: defaultDependency, - }); - -export default templateCompositeFrom({ - annotation: `processContentEntryDates`, - - inputs: { - annotations: input({ - validate: looseArrayOf(isContentString), - defaultDependency: '#entries.annotation', - }), - - dates: inputDateList('#entries.date'), - secondDates: inputDateList('#entries.secondDate'), - accessDates: inputDateList('#entries.accessDate'), - - dateKinds: inputKindList('#entries.dateKind'), - accessKinds: inputKindList('#entries.accessKind'), - }, - - outputs: ({ - [input.staticDependency('dates')]: dates, - [input.staticDependency('secondDates')]: secondDates, - [input.staticDependency('accessDates')]: accessDates, - [input.staticDependency('dateKinds')]: dateKinds, - [input.staticDependency('accessKinds')]: accessKinds, - }) => [ - dates ?? '#processedContentEntryDates', - secondDates ?? '#processedContentEntrySecondDates', - accessDates ?? '#processedContentEntryAccessDates', - dateKinds ?? '#processedContentEntryDateKinds', - accessKinds ?? '#processedContentEntryAccessKinds', - ], - - steps: () => [ - { - dependencies: [input('annotations')], - compute: (continuation, { - [input('annotations')]: annotations, - }) => continuation({ - ['#webArchiveDates']: - annotations - .map(text => text?.match(/https?:\/\/web.archive.org\/web\/([0-9]{8,8})[0-9]*\//)) - .map(match => match?.[1]) - .map(dateText => - (dateText - ? dateText.slice(0, 4) + '/' + - dateText.slice(4, 6) + '/' + - dateText.slice(6, 8) - : null)), - }), - }, - - { - dependencies: [input('dates')], - compute: (continuation, { - [input('dates')]: dates, - }) => continuation({ - ['#processedContentEntryDates']: - dates - .map(date => date ? new Date(date) : null), - }), - }, - - { - dependencies: [input('secondDates')], - compute: (continuation, { - [input('secondDates')]: secondDates, - }) => continuation({ - ['#processedContentEntrySecondDates']: - secondDates - .map(date => date ? new Date(date) : null), - }), - }, - - fillMissingListItems({ - list: input('dateKinds'), - fill: input.value(null), - }).outputs({ - '#list': '#processedContentEntryDateKinds', - }), - - { - dependencies: [input('accessDates'), '#webArchiveDates'], - compute: (continuation, { - [input('accessDates')]: accessDates, - ['#webArchiveDates']: webArchiveDates, - }) => continuation({ - ['#processedContentEntryAccessDates']: - stitchArrays({ - accessDate: accessDates, - webArchiveDate: webArchiveDates - }).map(({accessDate, webArchiveDate}) => - accessDate ?? - webArchiveDate ?? - null) - .map(date => date ? new Date(date) : date), - }), - }, - - { - dependencies: [input('accessKinds'), '#webArchiveDates'], - compute: (continuation, { - [input('accessKinds')]: accessKinds, - ['#webArchiveDates']: webArchiveDates, - }) => continuation({ - ['#processedContentEntryAccessKinds']: - stitchArrays({ - accessKind: accessKinds, - webArchiveDate: webArchiveDates, - }).map(({accessKind, webArchiveDate}) => - accessKind ?? - (webArchiveDate && 'captured') ?? - null), - }), - }, - - // TODO: Annoying conversion step for outputs, would be nice to avoid. - { - dependencies: [ - '#processedContentEntryDates', - '#processedContentEntrySecondDates', - '#processedContentEntryAccessDates', - '#processedContentEntryDateKinds', - '#processedContentEntryAccessKinds', - input.staticDependency('dates'), - input.staticDependency('secondDates'), - input.staticDependency('accessDates'), - input.staticDependency('dateKinds'), - input.staticDependency('accessKinds'), - ], - - compute: (continuation, { - ['#processedContentEntryDates']: processedContentEntryDates, - ['#processedContentEntrySecondDates']: processedContentEntrySecondDates, - ['#processedContentEntryAccessDates']: processedContentEntryAccessDates, - ['#processedContentEntryDateKinds']: processedContentEntryDateKinds, - ['#processedContentEntryAccessKinds']: processedContentEntryAccessKinds, - [input.staticDependency('dates')]: dates, - [input.staticDependency('secondDates')]: secondDates, - [input.staticDependency('accessDates')]: accessDates, - [input.staticDependency('dateKinds')]: dateKinds, - [input.staticDependency('accessKinds')]: accessKinds, - }) => continuation({ - [dates ?? '#processedContentEntryDates']: - processedContentEntryDates, - - [secondDates ?? '#processedContentEntrySecondDates']: - processedContentEntrySecondDates, - - [accessDates ?? '#processedContentEntryAccessDates']: - processedContentEntryAccessDates, - - [dateKinds ?? '#processedContentEntryDateKinds']: - processedContentEntryDateKinds, - - [accessKinds ?? '#processedContentEntryAccessKinds']: - processedContentEntryAccessKinds, - }), - }, - ], -}); diff --git a/src/data/composite/wiki-data/withConstitutedArtwork.js b/src/data/composite/wiki-data/withConstitutedArtwork.js index 9e260abf..6187d55b 100644 --- a/src/data/composite/wiki-data/withConstitutedArtwork.js +++ b/src/data/composite/wiki-data/withConstitutedArtwork.js @@ -6,6 +6,7 @@ export default templateCompositeFrom({ annotation: `withConstitutedArtwork`, inputs: { + thingProperty: input({type: 'string', acceptsNull: true}), dimensionsFromThingProperty: input({type: 'string', acceptsNull: true}), fileExtensionFromThingProperty: input({type: 'string', acceptsNull: true}), dateFromThingProperty: input({type: 'string', acceptsNull: true}), @@ -21,6 +22,7 @@ export default templateCompositeFrom({ { dependencies: [ input.myself(), + input('thingProperty'), input('dimensionsFromThingProperty'), input('fileExtensionFromThingProperty'), input('dateFromThingProperty'), @@ -32,6 +34,7 @@ export default templateCompositeFrom({ compute: (continuation, { [input.myself()]: myself, + [input('thingProperty')]: thingProperty, [input('dimensionsFromThingProperty')]: dimensionsFromThingProperty, [input('fileExtensionFromThingProperty')]: fileExtensionFromThingProperty, [input('dateFromThingProperty')]: dateFromThingProperty, @@ -43,6 +46,7 @@ export default templateCompositeFrom({ ['#constitutedArtwork']: Object.assign(new thingConstructors.Artwork, { thing: myself, + thingProperty, dimensionsFromThingProperty, fileExtensionFromThingProperty, artistContribsFromThingProperty, diff --git a/src/data/composite/wiki-data/withParsedCommentaryEntries.js b/src/data/composite/wiki-data/withParsedCommentaryEntries.js deleted file mode 100644 index 6794c479..00000000 --- a/src/data/composite/wiki-data/withParsedCommentaryEntries.js +++ /dev/null @@ -1,129 +0,0 @@ -import {input, templateCompositeFrom} from '#composite'; -import {stitchArrays} from '#sugar'; -import {isCommentary} from '#validators'; -import {commentaryRegexCaseSensitive} from '#wiki-data'; - -import { - fillMissingListItems, - withFlattenedList, - withPropertiesFromList, - withUnflattenedList, -} from '#composite/data'; - -import inputSoupyFind from './inputSoupyFind.js'; -import processContentEntryDates from './processContentEntryDates.js'; -import withParsedContentEntries from './withParsedContentEntries.js'; -import withResolvedReferenceList from './withResolvedReferenceList.js'; - -export default templateCompositeFrom({ - annotation: `withParsedCommentaryEntries`, - - inputs: { - from: input({validate: isCommentary}), - }, - - outputs: ['#parsedCommentaryEntries'], - - steps: () => [ - withParsedContentEntries({ - from: input('from'), - caseSensitiveRegex: input.value(commentaryRegexCaseSensitive), - }), - - withPropertiesFromList({ - list: '#parsedContentEntryHeadings', - prefix: input.value('#entries'), - properties: input.value([ - 'artistReferences', - 'artistDisplayText', - 'annotation', - 'date', - 'secondDate', - 'dateKind', - 'accessDate', - 'accessKind', - ]), - }), - - // The artistReferences group will always have a value, since it's required - // for the line to match in the first place. - - { - dependencies: ['#entries.artistReferences'], - compute: (continuation, { - ['#entries.artistReferences']: artistReferenceTexts, - }) => continuation({ - ['#entries.artistReferences']: - artistReferenceTexts - .map(text => text.split(',').map(ref => ref.trim())), - }), - }, - - withFlattenedList({ - list: '#entries.artistReferences', - }), - - withResolvedReferenceList({ - list: '#flattenedList', - find: inputSoupyFind.input('artist'), - notFoundMode: input.value('null'), - }), - - withUnflattenedList({ - list: '#resolvedReferenceList', - }).outputs({ - '#unflattenedList': '#entries.artists', - }), - - fillMissingListItems({ - list: '#entries.artistDisplayText', - fill: input.value(null), - }), - - fillMissingListItems({ - list: '#entries.annotation', - fill: input.value(null), - }), - - processContentEntryDates(), - - { - dependencies: [ - '#entries.artists', - '#entries.artistDisplayText', - '#entries.annotation', - '#entries.date', - '#entries.secondDate', - '#entries.dateKind', - '#entries.accessDate', - '#entries.accessKind', - '#parsedContentEntryBodies', - ], - - compute: (continuation, { - ['#entries.artists']: artists, - ['#entries.artistDisplayText']: artistDisplayText, - ['#entries.annotation']: annotation, - ['#entries.date']: date, - ['#entries.secondDate']: secondDate, - ['#entries.dateKind']: dateKind, - ['#entries.accessDate']: accessDate, - ['#entries.accessKind']: accessKind, - ['#parsedContentEntryBodies']: body, - }) => continuation({ - ['#parsedCommentaryEntries']: - stitchArrays({ - artists, - artistDisplayText, - annotation, - date, - secondDate, - dateKind, - accessDate, - accessKind, - body, - }), - }), - }, - ], -}); diff --git a/src/data/composite/wiki-data/withParsedContentEntries.js b/src/data/composite/wiki-data/withParsedContentEntries.js deleted file mode 100644 index 2a9b3f6a..00000000 --- a/src/data/composite/wiki-data/withParsedContentEntries.js +++ /dev/null @@ -1,111 +0,0 @@ -import {input, templateCompositeFrom} from '#composite'; -import {stitchArrays} from '#sugar'; -import {isContentString, validateInstanceOf} from '#validators'; - -import {withPropertiesFromList} from '#composite/data'; - -export default templateCompositeFrom({ - annotation: `withParsedContentEntries`, - - inputs: { - // TODO: Is there any way to validate this input based on the *other* - // inputs proivded, i.e. regexes? This kind of just assumes the string - // has already been validated according to the form the regex expects, - // which *is* always the case (as used), but it seems a bit awkward. - from: input({validate: isContentString}), - - caseSensitiveRegex: input({ - validate: validateInstanceOf(RegExp), - }), - }, - - outputs: [ - '#parsedContentEntryHeadings', - '#parsedContentEntryBodies', - ], - - steps: () => [ - { - dependencies: [ - input('from'), - input('caseSensitiveRegex'), - ], - - compute: (continuation, { - [input('from')]: commentaryText, - [input('caseSensitiveRegex')]: caseSensitiveRegex, - }) => continuation({ - ['#rawMatches']: - Array.from(commentaryText.matchAll(caseSensitiveRegex)), - }), - }, - - withPropertiesFromList({ - list: '#rawMatches', - properties: input.value([ - '0', // The entire match as a string. - 'groups', - 'index', - ]), - }).outputs({ - '#rawMatches.0': '#rawMatches.text', - '#rawMatches.groups': '#parsedContentEntryHeadings', - '#rawMatches.index': '#rawMatches.startIndex', - }), - - { - dependencies: [ - '#rawMatches.text', - '#rawMatches.startIndex', - ], - - compute: (continuation, { - ['#rawMatches.text']: text, - ['#rawMatches.startIndex']: startIndex, - }) => continuation({ - ['#rawMatches.endIndex']: - stitchArrays({text, startIndex}) - .map(({text, startIndex}) => startIndex + text.length), - }), - }, - - { - dependencies: [ - input('from'), - '#rawMatches.startIndex', - '#rawMatches.endIndex', - ], - - compute: (continuation, { - [input('from')]: commentaryText, - ['#rawMatches.startIndex']: startIndex, - ['#rawMatches.endIndex']: endIndex, - }) => continuation({ - ['#parsedContentEntryBodies']: - stitchArrays({startIndex, endIndex}) - .map(({endIndex}, index, stitched) => - (index === stitched.length - 1 - ? commentaryText.slice(endIndex) - : commentaryText.slice( - endIndex, - stitched[index + 1].startIndex))) - .map(body => body.trim()), - }), - }, - - { - dependencies: [ - '#parsedContentEntryHeadings', - '#parsedContentEntryBodies', - ], - - compute: (continuation, { - ['#parsedContentEntryHeadings']: parsedContentEntryHeadings, - ['#parsedContentEntryBodies']: parsedContentEntryBodies, - }) => continuation({ - ['#parsedContentEntryHeadings']: parsedContentEntryHeadings, - ['#parsedContentEntryBodies']: parsedContentEntryBodies, - }) - } - ], -}); diff --git a/src/data/composite/wiki-data/withParsedLyricsEntries.js b/src/data/composite/wiki-data/withParsedLyricsEntries.js deleted file mode 100644 index d13bfbaa..00000000 --- a/src/data/composite/wiki-data/withParsedLyricsEntries.js +++ /dev/null @@ -1,157 +0,0 @@ -import {input, templateCompositeFrom} from '#composite'; -import {stitchArrays} from '#sugar'; -import {isLyrics} from '#validators'; -import {commentaryRegexCaseSensitive, oldStyleLyricsDetectionRegex} - from '#wiki-data'; - -import { - fillMissingListItems, - withFlattenedList, - withPropertiesFromList, - withUnflattenedList, -} from '#composite/data'; - -import inputSoupyFind from './inputSoupyFind.js'; -import processContentEntryDates from './processContentEntryDates.js'; -import withParsedContentEntries from './withParsedContentEntries.js'; -import withResolvedReferenceList from './withResolvedReferenceList.js'; - -function constituteLyricsEntry(text) { - return { - artists: [], - artistDisplayText: null, - annotation: null, - date: null, - secondDate: null, - dateKind: null, - accessDate: null, - accessKind: null, - body: text, - }; -} - -export default templateCompositeFrom({ - annotation: `withParsedLyricsEntries`, - - inputs: { - from: input({validate: isLyrics}), - }, - - outputs: ['#parsedLyricsEntries'], - - steps: () => [ - { - dependencies: [input('from')], - compute: (continuation, { - [input('from')]: lyrics, - }) => - (oldStyleLyricsDetectionRegex.test(lyrics) - ? continuation() - : continuation.raiseOutput({ - ['#parsedLyricsEntries']: - [constituteLyricsEntry(lyrics)], - })), - }, - - withParsedContentEntries({ - from: input('from'), - caseSensitiveRegex: input.value(commentaryRegexCaseSensitive), - }), - - withPropertiesFromList({ - list: '#parsedContentEntryHeadings', - prefix: input.value('#entries'), - properties: input.value([ - 'artistReferences', - 'artistDisplayText', - 'annotation', - 'date', - 'secondDate', - 'dateKind', - 'accessDate', - 'accessKind', - ]), - }), - - // The artistReferences group will always have a value, since it's required - // for the line to match in the first place. - - { - dependencies: ['#entries.artistReferences'], - compute: (continuation, { - ['#entries.artistReferences']: artistReferenceTexts, - }) => continuation({ - ['#entries.artistReferences']: - artistReferenceTexts - .map(text => text.split(',').map(ref => ref.trim())), - }), - }, - - withFlattenedList({ - list: '#entries.artistReferences', - }), - - withResolvedReferenceList({ - list: '#flattenedList', - find: inputSoupyFind.input('artist'), - notFoundMode: input.value('null'), - }), - - withUnflattenedList({ - list: '#resolvedReferenceList', - }).outputs({ - '#unflattenedList': '#entries.artists', - }), - - fillMissingListItems({ - list: '#entries.artistDisplayText', - fill: input.value(null), - }), - - fillMissingListItems({ - list: '#entries.annotation', - fill: input.value(null), - }), - - processContentEntryDates(), - - { - dependencies: [ - '#entries.artists', - '#entries.artistDisplayText', - '#entries.annotation', - '#entries.date', - '#entries.secondDate', - '#entries.dateKind', - '#entries.accessDate', - '#entries.accessKind', - '#parsedContentEntryBodies', - ], - - compute: (continuation, { - ['#entries.artists']: artists, - ['#entries.artistDisplayText']: artistDisplayText, - ['#entries.annotation']: annotation, - ['#entries.date']: date, - ['#entries.secondDate']: secondDate, - ['#entries.dateKind']: dateKind, - ['#entries.accessDate']: accessDate, - ['#entries.accessKind']: accessKind, - ['#parsedContentEntryBodies']: body, - }) => continuation({ - ['#parsedLyricsEntries']: - stitchArrays({ - artists, - artistDisplayText, - annotation, - date, - secondDate, - dateKind, - accessDate, - accessKind, - body, - }), - }), - }, - ], -}); diff --git a/src/data/composite/wiki-properties/commentary.js b/src/data/composite/wiki-properties/commentary.js deleted file mode 100644 index 928bbd1b..00000000 --- a/src/data/composite/wiki-properties/commentary.js +++ /dev/null @@ -1,34 +0,0 @@ -// Artist commentary! Generally present on tracks and albums. - -import {input, templateCompositeFrom} from '#composite'; -import {isCommentary} from '#validators'; - -import {exitWithoutDependency, exposeDependency} - from '#composite/control-flow'; -import {withParsedCommentaryEntries} from '#composite/wiki-data'; - -export default templateCompositeFrom({ - annotation: `commentary`, - - compose: false, - - update: { - validate: isCommentary, - }, - - steps: () => [ - exitWithoutDependency({ - dependency: input.updateValue(), - mode: input.value('falsy'), - value: input.value([]), - }), - - withParsedCommentaryEntries({ - from: input.updateValue(), - }), - - exposeDependency({ - dependency: '#parsedCommentaryEntries', - }), - ], -}); diff --git a/src/data/composite/wiki-properties/commentatorArtists.js b/src/data/composite/wiki-properties/commentatorArtists.js index c5c14769..54d3e1a5 100644 --- a/src/data/composite/wiki-properties/commentatorArtists.js +++ b/src/data/composite/wiki-properties/commentatorArtists.js @@ -7,7 +7,6 @@ import {exitWithoutDependency, exposeDependency} from '#composite/control-flow'; import {withFlattenedList, withPropertyFromList, withUniqueItemsOnly} from '#composite/data'; -import {withParsedCommentaryEntries} from '#composite/wiki-data'; export default templateCompositeFrom({ annotation: `commentatorArtists`, @@ -21,19 +20,13 @@ export default templateCompositeFrom({ value: input.value([]), }), - withParsedCommentaryEntries({ - from: 'commentary', - }), - withPropertyFromList({ - list: '#parsedCommentaryEntries', + list: 'commentary', property: input.value('artists'), - }).outputs({ - '#parsedCommentaryEntries.artists': '#artistLists', }), withFlattenedList({ - list: '#artistLists', + list: '#commentary.artists', }).outputs({ '#flattenedList': '#artists', }), diff --git a/src/data/composite/wiki-properties/constitutibleArtwork.js b/src/data/composite/wiki-properties/constitutibleArtwork.js index 0ee3bfcd..48f4211a 100644 --- a/src/data/composite/wiki-properties/constitutibleArtwork.js +++ b/src/data/composite/wiki-properties/constitutibleArtwork.js @@ -17,6 +17,7 @@ const template = templateCompositeFrom({ compose: false, inputs: { + thingProperty: input({type: 'string', acceptsNull: true}), dimensionsFromThingProperty: input({type: 'string', acceptsNull: true}), fileExtensionFromThingProperty: input({type: 'string', acceptsNull: true}), dateFromThingProperty: input({type: 'string', acceptsNull: true}), @@ -35,6 +36,7 @@ const template = templateCompositeFrom({ }), withConstitutedArtwork({ + thingProperty: input('thingProperty'), dimensionsFromThingProperty: input('dimensionsFromThingProperty'), fileExtensionFromThingProperty: input('fileExtensionFromThingProperty'), dateFromThingProperty: input('dateFromThingProperty'), diff --git a/src/data/composite/wiki-properties/constitutibleArtworkList.js b/src/data/composite/wiki-properties/constitutibleArtworkList.js index 246c08b5..dad3a957 100644 --- a/src/data/composite/wiki-properties/constitutibleArtworkList.js +++ b/src/data/composite/wiki-properties/constitutibleArtworkList.js @@ -16,6 +16,7 @@ const template = templateCompositeFrom({ compose: false, inputs: { + thingProperty: input({type: 'string', acceptsNull: true}), dimensionsFromThingProperty: input({type: 'string', acceptsNull: true}), fileExtensionFromThingProperty: input({type: 'string', acceptsNull: true}), dateFromThingProperty: input({type: 'string', acceptsNull: true}), @@ -34,6 +35,7 @@ const template = templateCompositeFrom({ }), withConstitutedArtwork({ + thingProperty: input('thingProperty'), dimensionsFromThingProperty: input('dimensionsFromThingProperty'), fileExtensionFromThingProperty: input('fileExtensionFromThingProperty'), dateFromThingProperty: input('dateFromThingProperty'), diff --git a/src/data/composite/wiki-properties/index.js b/src/data/composite/wiki-properties/index.js index 892fc44a..d5e7657e 100644 --- a/src/data/composite/wiki-properties/index.js +++ b/src/data/composite/wiki-properties/index.js @@ -7,7 +7,6 @@ export {default as additionalFiles} from './additionalFiles.js'; export {default as additionalNameList} from './additionalNameList.js'; export {default as annotatedReferenceList} from './annotatedReferenceList.js'; export {default as color} from './color.js'; -export {default as commentary} from './commentary.js'; export {default as commentatorArtists} from './commentatorArtists.js'; export {default as constitutibleArtwork} from './constitutibleArtwork.js'; export {default as constitutibleArtworkList} from './constitutibleArtworkList.js'; @@ -20,7 +19,6 @@ export {default as duration} from './duration.js'; export {default as externalFunction} from './externalFunction.js'; export {default as fileExtension} from './fileExtension.js'; export {default as flag} from './flag.js'; -export {default as lyrics} from './lyrics.js'; export {default as name} from './name.js'; export {default as referenceList} from './referenceList.js'; export {default as referencedArtworkList} from './referencedArtworkList.js'; diff --git a/src/data/composite/wiki-properties/lyrics.js b/src/data/composite/wiki-properties/lyrics.js deleted file mode 100644 index eb5e524a..00000000 --- a/src/data/composite/wiki-properties/lyrics.js +++ /dev/null @@ -1,36 +0,0 @@ -// Lyrics! This comes in two styles - "old", where there's just one set of -// lyrics, or the newer/standard one, with multiple sets that are each -// annotated, credited, etc. - -import {input, templateCompositeFrom} from '#composite'; -import {isLyrics} from '#validators'; - -import {exitWithoutDependency, exposeDependency} - from '#composite/control-flow'; -import {withParsedLyricsEntries} from '#composite/wiki-data'; - -export default templateCompositeFrom({ - annotation: `lyrics`, - - compose: false, - - update: { - validate: isLyrics, - }, - - steps: () => [ - exitWithoutDependency({ - dependency: input.updateValue(), - mode: input.value('falsy'), - value: input.value([]), - }), - - withParsedLyricsEntries({ - from: input.updateValue(), - }), - - exposeDependency({ - dependency: '#parsedLyricsEntries', - }), - ], -}); diff --git a/src/data/things/album.js b/src/data/things/album.js index 4c85ddfa..c71b9820 100644 --- a/src/data/things/album.js +++ b/src/data/things/album.js @@ -3,6 +3,7 @@ export const DATA_ALBUM_DIRECTORY = 'album'; import * as path from 'node:path'; import {inspect} from 'node:util'; +import CacheableObject from '#cacheable-object'; import {colors} from '#cli'; import {input} from '#composite'; import {traverse} from '#node-utils'; @@ -16,7 +17,9 @@ import { parseAdditionalNames, parseAnnotatedReferences, parseArtwork, + parseCommentary, parseContributors, + parseCreditingSources, parseDate, parseDimensions, parseWallpaperParts, @@ -32,7 +35,6 @@ import {exitWithoutContribs, withDirectory, withCoverArtDate} import { additionalFiles, additionalNameList, - commentary, color, commentatorArtists, constitutibleArtwork, @@ -69,6 +71,8 @@ export class Album extends Thing { static [Thing.getPropertyDescriptors] = ({ ArtTag, Artwork, + CommentaryEntry, + CreditingSourcesEntry, Group, Track, TrackSection, @@ -204,8 +208,14 @@ export class Album extends Thing { isListedOnHomepage: flag(true), isListedInGalleries: flag(true), - commentary: commentary(), - creditSources: commentary(), + commentary: thingList({ + class: input.value(CommentaryEntry), + }), + + creditSources: thingList({ + class: input.value(CreditingSourcesEntry), + }), + additionalFiles: additionalFiles(), trackSections: thingList({ @@ -506,6 +516,7 @@ export class Album extends Thing { property: 'coverArtworks', transform: parseArtwork({ + thingProperty: 'coverArtworks', dimensionsFromThingProperty: 'coverArtDimensions', fileExtensionFromThingProperty: 'coverArtFileExtension', dateFromThingProperty: 'coverArtDate', @@ -521,6 +532,7 @@ export class Album extends Thing { transform: parseArtwork({ single: true, + thingProperty: 'bannerArtwork', dimensionsFromThingProperty: 'bannerDimensions', fileExtensionFromThingProperty: 'bannerFileExtension', dateFromThingProperty: 'date', @@ -534,6 +546,7 @@ export class Album extends Thing { transform: parseArtwork({ single: true, + thingProperty: 'wallpaperArtwork', dimensionsFromThingProperty: null, fileExtensionFromThingProperty: 'wallpaperFileExtension', dateFromThingProperty: 'date', @@ -596,8 +609,15 @@ export class Album extends Thing { transform: parseDimensions, }, - 'Commentary': {property: 'commentary'}, - 'Credit Sources': {property: 'creditSources'}, + 'Commentary': { + property: 'commentary', + transform: parseCommentary, + }, + + 'Credit Sources': { + property: 'creditSources', + transform: parseCreditingSources, + }, 'Additional Files': { property: 'additionalFiles', @@ -668,7 +688,11 @@ export class Album extends Thing { const albumData = []; const trackSectionData = []; const trackData = []; + const artworkData = []; + const commentaryData = []; + const creditingSourceData = []; + const lyricsData = []; for (const {header: album, entries} of results) { const trackSections = []; @@ -715,6 +739,13 @@ export class Album extends Thing { entry.album = album; artworkData.push(...entry.trackArtworks); + commentaryData.push(...entry.commentary); + creditingSourceData.push(...entry.creditSources); + + // TODO: As exposed, Track.lyrics tries to inherit from the main + // release, which is impossible before the data's been linked. + // We just use the update value here. But it's icky! + lyricsData.push(...CacheableObject.getUpdateValue(entry, 'lyrics') ?? []); } closeCurrentTrackSection(); @@ -731,6 +762,9 @@ export class Album extends Thing { artworkData.push(album.wallpaperArtwork); } + commentaryData.push(...album.commentary); + creditingSourceData.push(...album.creditSources); + album.trackSections = trackSections; } @@ -738,7 +772,11 @@ export class Album extends Thing { albumData, trackSectionData, trackData, + artworkData, + commentaryData, + creditingSourceData, + lyricsData, }; }, diff --git a/src/data/things/artist.js b/src/data/things/artist.js index 87e1c563..9e329c74 100644 --- a/src/data/things/artist.js +++ b/src/data/things/artist.js @@ -213,6 +213,7 @@ export class Artist extends Thing { transform: parseArtwork({ single: true, + thingProperty: 'avatarArtwork', fileExtensionFromThingProperty: 'avatarFileExtension', }), }, diff --git a/src/data/things/artwork.js b/src/data/things/artwork.js index 2a97fd6d..3cdb07d0 100644 --- a/src/data/things/artwork.js +++ b/src/data/things/artwork.js @@ -24,7 +24,7 @@ import { parseDimensions, } from '#yaml'; -import {withPropertyFromObject} from '#composite/data'; +import {withIndexInList, withPropertyFromObject} from '#composite/data'; import { exitWithoutDependency, @@ -44,6 +44,7 @@ import { import { contentString, directory, + flag, reverseReferenceList, simpleString, soupyFind, @@ -52,7 +53,13 @@ import { wikiData, } from '#composite/wiki-properties'; -import {withDate} from '#composite/things/artwork'; +import { + withAttachedArtwork, + withContainingArtworkList, + withContribsFromAttachedArtwork, + withPropertyFromAttachedArtwork, + withDate, +} from '#composite/things/artwork'; export class Artwork extends Thing { static [Thing.referenceType] = 'artwork'; @@ -68,6 +75,7 @@ export class Artwork extends Thing { }), thing: thing(), + thingProperty: simpleString(), label: simpleString(), source: contentString(), @@ -152,6 +160,8 @@ export class Artwork extends Thing { }), ], + attachAbove: flag(false), + artistContribsFromThingProperty: simpleString(), artistContribsArtistProperty: simpleString(), @@ -169,6 +179,12 @@ export class Artwork extends Thing { mode: input.value('empty'), }), + withContribsFromAttachedArtwork(), + + exposeDependencyOrContinue({ + dependency: '#attachedArtwork.artistContribs', + }), + exitWithoutDependency({ dependency: 'artistContribsFromThingProperty', value: input.value([]), @@ -207,6 +223,14 @@ export class Artwork extends Thing { mode: input.value('empty'), }), + withPropertyFromAttachedArtwork({ + property: input.value('artTags'), + }), + + exposeDependencyOrContinue({ + dependency: '#attachedArtwork.artTags', + }), + exitWithoutDependency({ dependency: 'artTagsFromThingProperty', value: input.value([]), @@ -302,6 +326,51 @@ export class Artwork extends Thing { referencedByArtworks: reverseReferenceList({ reverse: soupyReverse.input('artworksWhichReference'), }), + + isMainArtwork: [ + withContainingArtworkList(), + + exitWithoutDependency({ + dependency: '#containingArtworkList', + value: input.value(null), + }), + + { + dependencies: [input.myself(), '#containingArtworkList'], + compute: ({ + [input.myself()]: myself, + ['#containingArtworkList']: list, + }) => + list[0] === myself, + }, + ], + + mainArtwork: [ + withContainingArtworkList(), + + exitWithoutDependency({ + dependency: '#containingArtworkList', + value: input.value(null), + }), + + { + dependencies: ['#containingArtworkList'], + compute: ({'#containingArtworkList': list}) => + list[0], + }, + ], + + attachedArtwork: [ + withAttachedArtwork(), + + exposeDependency({ + dependency: '#attachedArtwork', + }), + ], + + attachingArtworks: reverseReferenceList({ + reverse: soupyReverse.input('artworksWhichAttach'), + }), }); static [Thing.yamlDocumentSpec] = { @@ -322,6 +391,8 @@ export class Artwork extends Thing { transform: parseDate, }, + 'Attach Above': {property: 'attachAbove'}, + 'Artists': { property: 'artistContribs', transform: parseContributors, @@ -358,6 +429,18 @@ export class Artwork extends Thing { date: ({artwork}) => artwork.date, }, + artworksWhichAttach: { + bindTo: 'artworkData', + + referencing: referencingArtwork => + (referencingArtwork.attachAbove + ? [referencingArtwork] + : []), + + referenced: referencingArtwork => + [referencingArtwork.attachedArtwork], + }, + artworksWhichFeature: { bindTo: 'artworkData', diff --git a/src/data/things/content.js b/src/data/things/content.js new file mode 100644 index 00000000..7f352795 --- /dev/null +++ b/src/data/things/content.js @@ -0,0 +1,122 @@ +import {input} from '#composite'; +import find from '#find'; +import Thing from '#thing'; +import {is, isDate} from '#validators'; +import {parseDate} from '#yaml'; + +import {contentString, referenceList, simpleDate, soupyFind, thing} + from '#composite/wiki-properties'; + +import { + exposeConstant, + exposeDependencyOrContinue, + exposeUpdateValueOrContinue, + withResultOfAvailabilityCheck, +} from '#composite/control-flow'; + +import {withWebArchiveDate} from '#composite/things/commentary-entry'; + +export class ContentEntry extends Thing { + static [Thing.getPropertyDescriptors] = ({Artist}) => ({ + // Update & expose + + thing: thing(), + + artists: referenceList({ + class: input.value(Artist), + find: soupyFind.input('artist'), + }), + + artistText: contentString(), + + annotation: contentString(), + + dateKind: { + flags: {update: true, expose: true}, + + update: { + validate: is(...[ + 'sometime', + 'throughout', + 'around', + ]), + }, + }, + + accessKind: [ + exposeUpdateValueOrContinue({ + validate: input.value( + is(...[ + 'captured', + 'accessed', + ])), + }), + + withWebArchiveDate(), + + withResultOfAvailabilityCheck({ + from: '#webArchiveDate', + }), + + { + dependencies: ['#availability'], + compute: (continuation, {['#availability']: availability}) => + (availability + ? continuation.exit('captured') + : continuation()), + }, + + exposeConstant({ + value: input.value(null), + }), + ], + + date: simpleDate(), + + secondDate: simpleDate(), + + accessDate: [ + exposeUpdateValueOrContinue({ + validate: input.value(isDate), + }), + + withWebArchiveDate(), + + exposeDependencyOrContinue({ + dependency: '#webArchiveDate', + }), + + exposeConstant({ + value: input.value(null), + }), + ], + + body: contentString(), + + // Update only + + find: soupyFind(), + }); + + static [Thing.yamlDocumentSpec] = { + fields: { + 'Artists': {property: 'artists'}, + 'Artist Text': {property: 'artistText'}, + + 'Annotation': {property: 'annotation'}, + + 'Date Kind': {property: 'dateKind'}, + 'Access Kind': {property: 'accessKind'}, + + 'Date': {property: 'date', transform: parseDate}, + 'Second Date': {property: 'secondDate', transform: parseDate}, + 'Access Date': {property: 'accessDate', transform: parseDate}, + + 'Body': {property: 'body'}, + }, + }; +} + +export class CommentaryEntry extends ContentEntry {} +export class LyricsEntry extends ContentEntry {} +export class CreditingSourcesEntry extends ContentEntry {} diff --git a/src/data/things/flash.js b/src/data/things/flash.js index ace18af9..a0bcb523 100644 --- a/src/data/things/flash.js +++ b/src/data/things/flash.js @@ -10,7 +10,9 @@ import {anyOf, isColor, isContentString, isDirectory, isNumber, isString} import { parseArtwork, parseAdditionalNames, + parseCommentary, parseContributors, + parseCreditingSources, parseDate, parseDimensions, } from '#yaml'; @@ -27,7 +29,6 @@ import { import { additionalNameList, color, - commentary, commentatorArtists, constitutibleArtwork, contentString, @@ -41,6 +42,7 @@ import { soupyFind, soupyReverse, thing, + thingList, urls, wikiData, } from '#composite/wiki-properties'; @@ -52,6 +54,8 @@ export class Flash extends Thing { static [Thing.referenceType] = 'flash'; static [Thing.getPropertyDescriptors] = ({ + CommentaryEntry, + CreditingSourcesEntry, Track, FlashAct, WikiInfo, @@ -125,8 +129,13 @@ export class Flash extends Thing { additionalNames: additionalNameList(), - commentary: commentary(), - creditSources: commentary(), + commentary: thingList({ + class: input.value(CommentaryEntry), + }), + + creditSources: thingList({ + class: input.value(CreditingSourcesEntry), + }), // Update only @@ -221,6 +230,7 @@ export class Flash extends Thing { transform: parseArtwork({ single: true, + thingProperty: 'coverArtwork', fileExtensionFromThingProperty: 'coverArtFileExtension', dimensionsFromThingProperty: 'coverArtDimensions', }), @@ -240,8 +250,15 @@ export class Flash extends Thing { transform: parseContributors, }, - 'Commentary': {property: 'commentary'}, - 'Credit Sources': {property: 'creditSources'}, + 'Commentary': { + property: 'commentary', + transform: parseCommentary, + }, + + 'Credit Sources': { + property: 'creditSources', + transform: parseCreditingSources, + }, 'Review Points': {ignore: true}, }, @@ -441,8 +458,18 @@ export class FlashSide extends Thing { const flashSideData = results.filter(x => x instanceof FlashSide); const artworkData = flashData.map(flash => flash.coverArtwork); - - return {flashData, flashActData, flashSideData, artworkData}; + const commentaryData = flashData.flatMap(flash => flash.commentary); + const creditingSourceData = flashData.flatMap(flash => flash.creditSources); + + return { + flashData, + flashActData, + flashSideData, + + artworkData, + commentaryData, + creditingSourceData, + }; }, sort({flashData}) { diff --git a/src/data/things/index.js b/src/data/things/index.js index 96cec88e..b832ab75 100644 --- a/src/data/things/index.js +++ b/src/data/things/index.js @@ -13,6 +13,7 @@ import * as albumClasses from './album.js'; import * as artTagClasses from './art-tag.js'; import * as artistClasses from './artist.js'; import * as artworkClasses from './artwork.js'; +import * as contentClasses from './content.js'; import * as contributionClasses from './contribution.js'; import * as flashClasses from './flash.js'; import * as groupClasses from './group.js'; @@ -29,6 +30,7 @@ const allClassLists = { 'art-tag.js': artTagClasses, 'artist.js': artistClasses, 'artwork.js': artworkClasses, + 'content.js': contentClasses, 'contribution.js': contributionClasses, 'flash.js': flashClasses, 'group.js': groupClasses, diff --git a/src/data/things/track.js b/src/data/things/track.js index bcf84aa8..57aaa90d 100644 --- a/src/data/things/track.js +++ b/src/data/things/track.js @@ -12,10 +12,13 @@ import { parseAdditionalNames, parseAnnotatedReferences, parseArtwork, + parseCommentary, parseContributors, + parseCreditingSources, parseDate, parseDimensions, parseDuration, + parseLyrics, } from '#yaml'; import {withPropertyFromObject} from '#composite/data'; @@ -37,7 +40,6 @@ import { import { additionalFiles, additionalNameList, - commentary, commentatorArtists, constitutibleArtworkList, contentString, @@ -46,7 +48,6 @@ import { directory, duration, flag, - lyrics, name, referenceList, referencedArtworkList, @@ -57,6 +58,7 @@ import { soupyFind, soupyReverse, thing, + thingList, urls, wikiData, } from '#composite/wiki-properties'; @@ -87,7 +89,10 @@ export class Track extends Thing { Album, ArtTag, Artwork, + CommentaryEntry, + CreditingSourcesEntry, Flash, + LyricsEntry, TrackSection, WikiInfo, }) => ({ @@ -216,12 +221,23 @@ export class Track extends Thing { dimensions(), ], - commentary: commentary(), - creditSources: commentary(), + commentary: thingList({ + class: input.value(CommentaryEntry), + }), + + creditSources: thingList({ + class: input.value(CreditingSourcesEntry), + }), lyrics: [ + // TODO: Inherited lyrics are literally the same objects, so of course + // their .thing properties aren't going to point back to this one, and + // certainly couldn't be recontextualized... inheritFromMainRelease(), - lyrics(), + + thingList({ + class: input.value(LyricsEntry), + }), ], additionalFiles: additionalFiles(), @@ -481,9 +497,20 @@ export class Track extends Thing { 'Always Reference By Directory': {property: 'alwaysReferenceByDirectory'}, - 'Lyrics': {property: 'lyrics'}, - 'Commentary': {property: 'commentary'}, - 'Credit Sources': {property: 'creditSources'}, + 'Lyrics': { + property: 'lyrics', + transform: parseLyrics, + }, + + 'Commentary': { + property: 'commentary', + transform: parseCommentary, + }, + + 'Credit Sources': { + property: 'creditSources', + transform: parseCreditingSources, + }, 'Additional Files': { property: 'additionalFiles', @@ -531,6 +558,7 @@ export class Track extends Thing { property: 'trackArtworks', transform: parseArtwork({ + thingProperty: 'trackArtworks', dimensionsFromThingProperty: 'coverArtDimensions', fileExtensionFromThingProperty: 'coverArtFileExtension', dateFromThingProperty: 'coverArtDate', diff --git a/src/data/yaml.js b/src/data/yaml.js index 50317238..036fe8a7 100644 --- a/src/data/yaml.js +++ b/src/data/yaml.js @@ -11,6 +11,7 @@ import {colors, ENABLE_COLOR, logInfo, logWarn} from '#cli'; import {sortByName} from '#sort'; import Thing from '#thing'; import thingConstructors from '#things'; +import {matchContentEntries, multipleLyricsDetectionRegex} from '#wiki-data'; import { aggregateThrows, @@ -791,6 +792,7 @@ export function parseAnnotatedReferences(entries, { export function parseArtwork({ single = false, + thingProperty = null, dimensionsFromThingProperty = null, fileExtensionFromThingProperty = null, dateFromThingProperty = null, @@ -800,6 +802,7 @@ export function parseArtwork({ referencedArtworksFromThingProperty = null, }) { const provide = { + thingProperty, dimensionsFromThingProperty, fileExtensionFromThingProperty, dateFromThingProperty, @@ -824,6 +827,73 @@ export function parseArtwork({ return transform; } +export function parseContentEntries(thingClass, sourceText, {subdoc}) { + const map = matchEntry => ({ + 'Artists': + matchEntry.artistReferences + .split(',') + .map(ref => ref.trim()), + + 'Artist Text': + matchEntry.artistDisplayText, + + 'Annotation': + matchEntry.annotation, + + 'Date': + matchEntry.date, + + 'Second Date': + matchEntry.secondDate, + + 'Date Kind': + matchEntry.dateKind, + + 'Access Date': + matchEntry.accessDate, + + 'Access Kind': + matchEntry.accessKind, + + 'Body': + matchEntry.body, + }); + + const documents = + matchContentEntries(sourceText) + .map(matchEntry => + withEntries( + map(matchEntry), + entries => entries + .filter(([key, value]) => + value !== undefined && + value !== null))); + + const subdocs = + documents.map(document => + subdoc(thingClass, document, {bindInto: 'thing'})); + + return subdocs; +} + +export function parseCommentary(sourceText, {subdoc, CommentaryEntry}) { + return parseContentEntries(CommentaryEntry, sourceText, {subdoc}); +} + +export function parseCreditingSources(sourceText, {subdoc, CreditingSourcesEntry}) { + return parseContentEntries(CreditingSourcesEntry, sourceText, {subdoc}); +} + +export function parseLyrics(sourceText, {subdoc, LyricsEntry}) { + if (!multipleLyricsDetectionRegex.test(sourceText)) { + const document = {'Body': sourceText}; + + return [subdoc(LyricsEntry, document, {bindInto: 'thing'})]; + } + + return parseContentEntries(LyricsEntry, sourceText, {subdoc}); +} + // documentModes: Symbols indicating sets of behavior for loading and processing // data files. export const documentModes = { @@ -1499,6 +1569,10 @@ export function linkWikiDataArrays(wikiData, {bindFind, bindReverse}) { ['artworkData', ['artworkData']], + ['commentaryData', [/* find */]], + + ['creditingSourceData', [/* find */]], + ['flashData', [ 'wikiInfo', ]], @@ -1513,6 +1587,8 @@ export function linkWikiDataArrays(wikiData, {bindFind, bindReverse}) { ['homepageLayout.sections.rows', [/* find */]], + ['lyricsData', [/* find */]], + ['trackData', [ 'artworkData', 'trackData', @@ -1781,14 +1857,16 @@ export function flattenThingLayoutToDocumentOrder(layout) { } export function* splitDocumentsInYAMLSourceText(sourceText) { - const dividerRegex = /^-{3,}\n?/gm; + // Not multiline! + const dividerRegex = /(?:\r\n|\n|^)-{3,}(?:\r\n|\n|$)/g; + let previousDivider = ''; while (true) { const {lastIndex} = dividerRegex; const match = dividerRegex.exec(sourceText); if (match) { - const nextDivider = match[0].trim(); + const nextDivider = match[0]; yield { previousDivider, @@ -1799,11 +1877,12 @@ export function* splitDocumentsInYAMLSourceText(sourceText) { previousDivider = nextDivider; } else { const nextDivider = ''; + const lineBreak = previousDivider.match(/\r?\n/)?.[0] ?? ''; yield { previousDivider, nextDivider, - text: sourceText.slice(lastIndex).replace(/(?<!\n)$/, '\n'), + text: sourceText.slice(lastIndex).replace(/(?<!\n)$/, lineBreak), }; return; @@ -1829,7 +1908,7 @@ export function recombineDocumentsIntoYAMLSourceText(documents) { for (const document of documents) { if (sourceText) { - sourceText += divider + '\n'; + sourceText += divider; } sourceText += document.text; |