From 7342b35d0da518fa5559dadc3239fd574a105432 Mon Sep 17 00:00:00 2001 From: "(quasar) nebula" Date: Mon, 14 Apr 2025 20:59:17 -0300 Subject: data: CommentaryEntry Fully integrated, all in one commit! Wow! --- src/data/checks.js | 5 +- .../composite/things/commentary-entry/index.js | 1 + .../things/commentary-entry/withWebArchiveDate.js | 41 +++++++ src/data/composite/wiki-data/index.js | 1 - .../wiki-data/withParsedCommentaryEntries.js | 129 --------------------- src/data/composite/wiki-properties/commentary.js | 34 ------ .../wiki-properties/commentatorArtists.js | 11 +- src/data/composite/wiki-properties/index.js | 1 - src/data/things/album.js | 35 +++++- src/data/things/commentary-entry.js | 118 +++++++++++++++++++ src/data/things/flash.js | 38 ++++-- src/data/things/index.js | 2 + src/data/things/track.js | 25 +++- src/data/yaml.js | 54 +++++++++ 14 files changed, 301 insertions(+), 194 deletions(-) create mode 100644 src/data/composite/things/commentary-entry/index.js create mode 100644 src/data/composite/things/commentary-entry/withWebArchiveDate.js delete mode 100644 src/data/composite/wiki-data/withParsedCommentaryEntries.js delete mode 100644 src/data/composite/wiki-properties/commentary.js create mode 100644 src/data/things/commentary-entry.js (limited to 'src/data') diff --git a/src/data/checks.js b/src/data/checks.js index 25863d2d..510bbb30 100644 --- a/src/data/checks.js +++ b/src/data/checks.js @@ -276,9 +276,8 @@ export function filterReferenceErrors(wikiData, { case '_commentary': if (value) { value = - Array.from(value.matchAll(commentaryRegexCaseSensitive)) - .map(({groups}) => groups.artistReferences) - .map(text => text.split(',').map(text => text.trim())); + value + .map(entry => CacheableObject.getUpdateValue(entry, 'artists')); } writeProperty = false; diff --git a/src/data/composite/things/commentary-entry/index.js b/src/data/composite/things/commentary-entry/index.js new file mode 100644 index 00000000..091bae1a --- /dev/null +++ b/src/data/composite/things/commentary-entry/index.js @@ -0,0 +1 @@ +export {default as withWebArchiveDate} from './withWebArchiveDate.js'; diff --git a/src/data/composite/things/commentary-entry/withWebArchiveDate.js b/src/data/composite/things/commentary-entry/withWebArchiveDate.js new file mode 100644 index 00000000..3aaa4f64 --- /dev/null +++ b/src/data/composite/things/commentary-entry/withWebArchiveDate.js @@ -0,0 +1,41 @@ +import {input, templateCompositeFrom} from '#composite'; + +import {raiseOutputWithoutDependency} from '#composite/control-flow'; + +export default templateCompositeFrom({ + annotation: `withWebArchiveDate`, + + outputs: ['#webArchiveDate'], + + steps: () => [ + { + dependencies: ['annotation'], + + compute: (continuation, {annotation}) => + continuation({ + ['#dateText']: + annotation + ?.match(/https?:\/\/web.archive.org\/web\/([0-9]{8,8})[0-9]*\//) + ?.[1] ?? + null, + }), + }, + + raiseOutputWithoutDependency({ + dependency: '#dateText', + output: input.value({['#webArchiveDate']: null}), + }), + + { + dependencies: ['#dateText'], + compute: (continuation, {['#dateText']: dateText}) => + continuation({ + ['#webArchiveDate']: + new Date( + dateText.slice(0, 4) + '/' + + dateText.slice(4, 6) + '/' + + dateText.slice(6, 8)), + }), + }, + ], +}); diff --git a/src/data/composite/wiki-data/index.js b/src/data/composite/wiki-data/index.js index 1d94f74b..d2b82a2e 100644 --- a/src/data/composite/wiki-data/index.js +++ b/src/data/composite/wiki-data/index.js @@ -17,7 +17,6 @@ export {default as withConstitutedArtwork} from './withConstitutedArtwork.js'; export {default as withContributionListSums} from './withContributionListSums.js'; export {default as withCoverArtDate} from './withCoverArtDate.js'; export {default as withDirectory} from './withDirectory.js'; -export {default as withParsedCommentaryEntries} from './withParsedCommentaryEntries.js'; export {default as withParsedContentEntries} from './withParsedContentEntries.js'; export {default as withParsedLyricsEntries} from './withParsedLyricsEntries.js'; export {default as withRecontextualizedContributionList} from './withRecontextualizedContributionList.js'; diff --git a/src/data/composite/wiki-data/withParsedCommentaryEntries.js b/src/data/composite/wiki-data/withParsedCommentaryEntries.js deleted file mode 100644 index 6794c479..00000000 --- a/src/data/composite/wiki-data/withParsedCommentaryEntries.js +++ /dev/null @@ -1,129 +0,0 @@ -import {input, templateCompositeFrom} from '#composite'; -import {stitchArrays} from '#sugar'; -import {isCommentary} from '#validators'; -import {commentaryRegexCaseSensitive} from '#wiki-data'; - -import { - fillMissingListItems, - withFlattenedList, - withPropertiesFromList, - withUnflattenedList, -} from '#composite/data'; - -import inputSoupyFind from './inputSoupyFind.js'; -import processContentEntryDates from './processContentEntryDates.js'; -import withParsedContentEntries from './withParsedContentEntries.js'; -import withResolvedReferenceList from './withResolvedReferenceList.js'; - -export default templateCompositeFrom({ - annotation: `withParsedCommentaryEntries`, - - inputs: { - from: input({validate: isCommentary}), - }, - - outputs: ['#parsedCommentaryEntries'], - - steps: () => [ - withParsedContentEntries({ - from: input('from'), - caseSensitiveRegex: input.value(commentaryRegexCaseSensitive), - }), - - withPropertiesFromList({ - list: '#parsedContentEntryHeadings', - prefix: input.value('#entries'), - properties: input.value([ - 'artistReferences', - 'artistDisplayText', - 'annotation', - 'date', - 'secondDate', - 'dateKind', - 'accessDate', - 'accessKind', - ]), - }), - - // The artistReferences group will always have a value, since it's required - // for the line to match in the first place. - - { - dependencies: ['#entries.artistReferences'], - compute: (continuation, { - ['#entries.artistReferences']: artistReferenceTexts, - }) => continuation({ - ['#entries.artistReferences']: - artistReferenceTexts - .map(text => text.split(',').map(ref => ref.trim())), - }), - }, - - withFlattenedList({ - list: '#entries.artistReferences', - }), - - withResolvedReferenceList({ - list: '#flattenedList', - find: inputSoupyFind.input('artist'), - notFoundMode: input.value('null'), - }), - - withUnflattenedList({ - list: '#resolvedReferenceList', - }).outputs({ - '#unflattenedList': '#entries.artists', - }), - - fillMissingListItems({ - list: '#entries.artistDisplayText', - fill: input.value(null), - }), - - fillMissingListItems({ - list: '#entries.annotation', - fill: input.value(null), - }), - - processContentEntryDates(), - - { - dependencies: [ - '#entries.artists', - '#entries.artistDisplayText', - '#entries.annotation', - '#entries.date', - '#entries.secondDate', - '#entries.dateKind', - '#entries.accessDate', - '#entries.accessKind', - '#parsedContentEntryBodies', - ], - - compute: (continuation, { - ['#entries.artists']: artists, - ['#entries.artistDisplayText']: artistDisplayText, - ['#entries.annotation']: annotation, - ['#entries.date']: date, - ['#entries.secondDate']: secondDate, - ['#entries.dateKind']: dateKind, - ['#entries.accessDate']: accessDate, - ['#entries.accessKind']: accessKind, - ['#parsedContentEntryBodies']: body, - }) => continuation({ - ['#parsedCommentaryEntries']: - stitchArrays({ - artists, - artistDisplayText, - annotation, - date, - secondDate, - dateKind, - accessDate, - accessKind, - body, - }), - }), - }, - ], -}); diff --git a/src/data/composite/wiki-properties/commentary.js b/src/data/composite/wiki-properties/commentary.js deleted file mode 100644 index 928bbd1b..00000000 --- a/src/data/composite/wiki-properties/commentary.js +++ /dev/null @@ -1,34 +0,0 @@ -// Artist commentary! Generally present on tracks and albums. - -import {input, templateCompositeFrom} from '#composite'; -import {isCommentary} from '#validators'; - -import {exitWithoutDependency, exposeDependency} - from '#composite/control-flow'; -import {withParsedCommentaryEntries} from '#composite/wiki-data'; - -export default templateCompositeFrom({ - annotation: `commentary`, - - compose: false, - - update: { - validate: isCommentary, - }, - - steps: () => [ - exitWithoutDependency({ - dependency: input.updateValue(), - mode: input.value('falsy'), - value: input.value([]), - }), - - withParsedCommentaryEntries({ - from: input.updateValue(), - }), - - exposeDependency({ - dependency: '#parsedCommentaryEntries', - }), - ], -}); diff --git a/src/data/composite/wiki-properties/commentatorArtists.js b/src/data/composite/wiki-properties/commentatorArtists.js index c5c14769..54d3e1a5 100644 --- a/src/data/composite/wiki-properties/commentatorArtists.js +++ b/src/data/composite/wiki-properties/commentatorArtists.js @@ -7,7 +7,6 @@ import {exitWithoutDependency, exposeDependency} from '#composite/control-flow'; import {withFlattenedList, withPropertyFromList, withUniqueItemsOnly} from '#composite/data'; -import {withParsedCommentaryEntries} from '#composite/wiki-data'; export default templateCompositeFrom({ annotation: `commentatorArtists`, @@ -21,19 +20,13 @@ export default templateCompositeFrom({ value: input.value([]), }), - withParsedCommentaryEntries({ - from: 'commentary', - }), - withPropertyFromList({ - list: '#parsedCommentaryEntries', + list: 'commentary', property: input.value('artists'), - }).outputs({ - '#parsedCommentaryEntries.artists': '#artistLists', }), withFlattenedList({ - list: '#artistLists', + list: '#commentary.artists', }).outputs({ '#flattenedList': '#artists', }), diff --git a/src/data/composite/wiki-properties/index.js b/src/data/composite/wiki-properties/index.js index 892fc44a..1a4360a3 100644 --- a/src/data/composite/wiki-properties/index.js +++ b/src/data/composite/wiki-properties/index.js @@ -7,7 +7,6 @@ export {default as additionalFiles} from './additionalFiles.js'; export {default as additionalNameList} from './additionalNameList.js'; export {default as annotatedReferenceList} from './annotatedReferenceList.js'; export {default as color} from './color.js'; -export {default as commentary} from './commentary.js'; export {default as commentatorArtists} from './commentatorArtists.js'; export {default as constitutibleArtwork} from './constitutibleArtwork.js'; export {default as constitutibleArtworkList} from './constitutibleArtworkList.js'; diff --git a/src/data/things/album.js b/src/data/things/album.js index 4c85ddfa..2f5e1093 100644 --- a/src/data/things/album.js +++ b/src/data/things/album.js @@ -16,6 +16,7 @@ import { parseAdditionalNames, parseAnnotatedReferences, parseArtwork, + parseCommentary, parseContributors, parseDate, parseDimensions, @@ -32,7 +33,6 @@ import {exitWithoutContribs, withDirectory, withCoverArtDate} import { additionalFiles, additionalNameList, - commentary, color, commentatorArtists, constitutibleArtwork, @@ -69,6 +69,7 @@ export class Album extends Thing { static [Thing.getPropertyDescriptors] = ({ ArtTag, Artwork, + CommentaryEntry, Group, Track, TrackSection, @@ -204,8 +205,14 @@ export class Album extends Thing { isListedOnHomepage: flag(true), isListedInGalleries: flag(true), - commentary: commentary(), - creditSources: commentary(), + commentary: thingList({ + class: input.value(CommentaryEntry), + }), + + creditSources: thingList({ + class: input.value(CommentaryEntry), + }), + additionalFiles: additionalFiles(), trackSections: thingList({ @@ -596,8 +603,15 @@ export class Album extends Thing { transform: parseDimensions, }, - 'Commentary': {property: 'commentary'}, - 'Credit Sources': {property: 'creditSources'}, + 'Commentary': { + property: 'commentary', + transform: parseCommentary, + }, + + 'Credit Sources': { + property: 'creditSources', + transform: parseCommentary, + }, 'Additional Files': { property: 'additionalFiles', @@ -668,7 +682,10 @@ export class Album extends Thing { const albumData = []; const trackSectionData = []; const trackData = []; + const artworkData = []; + const commentaryData = []; + const creditingSourceData = []; for (const {header: album, entries} of results) { const trackSections = []; @@ -715,6 +732,8 @@ export class Album extends Thing { entry.album = album; artworkData.push(...entry.trackArtworks); + commentaryData.push(...entry.commentary); + creditingSourceData.push(...entry.creditSources); } closeCurrentTrackSection(); @@ -731,6 +750,9 @@ export class Album extends Thing { artworkData.push(album.wallpaperArtwork); } + commentaryData.push(...album.commentary); + creditingSourceData.push(...album.creditSources); + album.trackSections = trackSections; } @@ -738,7 +760,10 @@ export class Album extends Thing { albumData, trackSectionData, trackData, + artworkData, + commentaryData, + creditingSourceData, }; }, diff --git a/src/data/things/commentary-entry.js b/src/data/things/commentary-entry.js new file mode 100644 index 00000000..3cc53d85 --- /dev/null +++ b/src/data/things/commentary-entry.js @@ -0,0 +1,118 @@ +import {input} from '#composite'; +import find from '#find'; +import Thing from '#thing'; +import {is, isDate} from '#validators'; +import {parseDate} from '#yaml'; + +import {contentString, referenceList, simpleDate, soupyFind, thing} + from '#composite/wiki-properties'; + +import { + exposeConstant, + exposeDependencyOrContinue, + exposeUpdateValueOrContinue, + withResultOfAvailabilityCheck, +} from '#composite/control-flow'; + +import {withWebArchiveDate} from '#composite/things/commentary-entry'; + +export class CommentaryEntry extends Thing { + static [Thing.getPropertyDescriptors] = ({Artist}) => ({ + // Update & expose + + thing: thing(), + + artists: referenceList({ + class: input.value(Artist), + find: soupyFind.input('artist'), + }), + + artistText: contentString(), + + annotation: contentString(), + + dateKind: { + flags: {update: true, expose: true}, + + update: { + validate: is(...[ + 'sometime', + 'throughout', + 'around', + ]), + }, + }, + + accessKind: [ + exposeUpdateValueOrContinue({ + validate: input.value( + is(...[ + 'captured', + 'accessed', + ])), + }), + + withWebArchiveDate(), + + withResultOfAvailabilityCheck({ + from: '#webArchiveDate', + }), + + { + dependencies: ['#availability'], + compute: (continuation, {['#availability']: availability}) => + (availability + ? continuation.exit('captured') + : continuation()), + }, + + exposeConstant({ + value: input.value(null), + }), + ], + + date: simpleDate(), + + secondDate: simpleDate(), + + accessDate: [ + exposeUpdateValueOrContinue({ + validate: input.value(isDate), + }), + + withWebArchiveDate(), + + exposeDependencyOrContinue({ + dependency: '#webArchiveDate', + }), + + exposeConstant({ + value: input.value(null), + }), + ], + + body: contentString(), + + // Update only + + find: soupyFind(), + }); + + static [Thing.yamlDocumentSpec] = { + fields: { + 'Artists': {property: 'artists'}, + 'Artist Text': {property: 'artistText'}, + + 'Annotation': {property: 'annotation'}, + + 'Date Kind': {property: 'dateKind'}, + 'Access Kind': {property: 'accessKind'}, + + 'Date': {property: 'date', transform: parseDate}, + 'Second Date': {property: 'secondDate', transform: parseDate}, + 'Access Date': {property: 'accessDate', transform: parseDate}, + + 'Body': {property: 'body'}, + }, + }; +} diff --git a/src/data/things/flash.js b/src/data/things/flash.js index ace18af9..d115db9f 100644 --- a/src/data/things/flash.js +++ b/src/data/things/flash.js @@ -10,6 +10,7 @@ import {anyOf, isColor, isContentString, isDirectory, isNumber, isString} import { parseArtwork, parseAdditionalNames, + parseCommentary, parseContributors, parseDate, parseDimensions, @@ -27,7 +28,6 @@ import { import { additionalNameList, color, - commentary, commentatorArtists, constitutibleArtwork, contentString, @@ -41,6 +41,7 @@ import { soupyFind, soupyReverse, thing, + thingList, urls, wikiData, } from '#composite/wiki-properties'; @@ -52,6 +53,7 @@ export class Flash extends Thing { static [Thing.referenceType] = 'flash'; static [Thing.getPropertyDescriptors] = ({ + CommentaryEntry, Track, FlashAct, WikiInfo, @@ -125,8 +127,13 @@ export class Flash extends Thing { additionalNames: additionalNameList(), - commentary: commentary(), - creditSources: commentary(), + commentary: thingList({ + class: input.value(CommentaryEntry), + }), + + creditSources: thingList({ + class: input.value(CommentaryEntry), + }), // Update only @@ -240,8 +247,15 @@ export class Flash extends Thing { transform: parseContributors, }, - 'Commentary': {property: 'commentary'}, - 'Credit Sources': {property: 'creditSources'}, + 'Commentary': { + property: 'commentary', + transform: parseCommentary, + }, + + 'Credit Sources': { + property: 'creditSources', + transform: parseCommentary, + }, 'Review Points': {ignore: true}, }, @@ -441,8 +455,18 @@ export class FlashSide extends Thing { const flashSideData = results.filter(x => x instanceof FlashSide); const artworkData = flashData.map(flash => flash.coverArtwork); - - return {flashData, flashActData, flashSideData, artworkData}; + const commentaryData = flashData.flatMap(flash => flash.commentary); + const creditingSourceData = flashData.flatMap(flash => flash.creditSources); + + return { + flashData, + flashActData, + flashSideData, + + artworkData, + commentaryData, + creditingSourceData, + }; }, sort({flashData}) { diff --git a/src/data/things/index.js b/src/data/things/index.js index 96cec88e..59d8a490 100644 --- a/src/data/things/index.js +++ b/src/data/things/index.js @@ -13,6 +13,7 @@ import * as albumClasses from './album.js'; import * as artTagClasses from './art-tag.js'; import * as artistClasses from './artist.js'; import * as artworkClasses from './artwork.js'; +import * as commentaryEntryClasses from './commentary-entry.js'; import * as contributionClasses from './contribution.js'; import * as flashClasses from './flash.js'; import * as groupClasses from './group.js'; @@ -29,6 +30,7 @@ const allClassLists = { 'art-tag.js': artTagClasses, 'artist.js': artistClasses, 'artwork.js': artworkClasses, + 'commentary-entry.js': commentaryEntryClasses, 'contribution.js': contributionClasses, 'flash.js': flashClasses, 'group.js': groupClasses, diff --git a/src/data/things/track.js b/src/data/things/track.js index bcf84aa8..4a30433c 100644 --- a/src/data/things/track.js +++ b/src/data/things/track.js @@ -12,6 +12,7 @@ import { parseAdditionalNames, parseAnnotatedReferences, parseArtwork, + parseCommentary, parseContributors, parseDate, parseDimensions, @@ -37,7 +38,6 @@ import { import { additionalFiles, additionalNameList, - commentary, commentatorArtists, constitutibleArtworkList, contentString, @@ -57,6 +57,7 @@ import { soupyFind, soupyReverse, thing, + thingList, urls, wikiData, } from '#composite/wiki-properties'; @@ -87,6 +88,7 @@ export class Track extends Thing { Album, ArtTag, Artwork, + CommentaryEntry, Flash, TrackSection, WikiInfo, @@ -216,8 +218,13 @@ export class Track extends Thing { dimensions(), ], - commentary: commentary(), - creditSources: commentary(), + commentary: thingList({ + class: input.value(CommentaryEntry), + }), + + creditSources: thingList({ + class: input.value(CommentaryEntry), + }), lyrics: [ inheritFromMainRelease(), @@ -482,8 +489,16 @@ export class Track extends Thing { 'Always Reference By Directory': {property: 'alwaysReferenceByDirectory'}, 'Lyrics': {property: 'lyrics'}, - 'Commentary': {property: 'commentary'}, - 'Credit Sources': {property: 'creditSources'}, + + 'Commentary': { + property: 'commentary', + transform: parseCommentary, + }, + + 'Credit Sources': { + property: 'creditSources', + transform: parseCommentary, + }, 'Additional Files': { property: 'additionalFiles', diff --git a/src/data/yaml.js b/src/data/yaml.js index 50317238..e6b3fa71 100644 --- a/src/data/yaml.js +++ b/src/data/yaml.js @@ -11,6 +11,7 @@ import {colors, ENABLE_COLOR, logInfo, logWarn} from '#cli'; import {sortByName} from '#sort'; import Thing from '#thing'; import thingConstructors from '#things'; +import {matchCommentaryEntries} from '#wiki-data'; import { aggregateThrows, @@ -824,6 +825,55 @@ export function parseArtwork({ return transform; } +export function parseCommentary(sourceText, {subdoc, CommentaryEntry}) { + const map = matchEntry => ({ + 'Artists': + matchEntry.artistReferences + .split(',') + .map(ref => ref.trim()), + + 'Artist Text': + matchEntry.artistDisplayText, + + 'Annotation': + matchEntry.annotation, + + 'Date': + matchEntry.date, + + 'Second Date': + matchEntry.secondDate, + + 'Date Kind': + matchEntry.dateKind, + + 'Access Date': + matchEntry.accessDate, + + 'Access Kind': + matchEntry.accessKind, + + 'Body': + matchEntry.body, + }); + + const documents = + matchCommentaryEntries(sourceText) + .map(matchEntry => + withEntries( + map(matchEntry), + entries => entries + .filter(([key, value]) => + value !== undefined && + value !== null))); + + const subdocs = + documents.map(document => + subdoc(CommentaryEntry, document, {bindInto: 'thing'})); + + return subdocs; +} + // documentModes: Symbols indicating sets of behavior for loading and processing // data files. export const documentModes = { @@ -1499,6 +1549,10 @@ export function linkWikiDataArrays(wikiData, {bindFind, bindReverse}) { ['artworkData', ['artworkData']], + ['commentaryData', [/* find */]], + + ['creditingSourceData', [/* find */]], + ['flashData', [ 'wikiInfo', ]], -- cgit 1.3.0-6-gf8a5