From a34b8d027866fbe858a4d2ff3543bc84c9d5983a Mon Sep 17 00:00:00 2001 From: "(quasar) nebula" Date: Fri, 17 Nov 2023 06:53:34 -0400 Subject: data, yaml, content: support multiple artists per commentary entry --- .../dependencies/generateCommentaryEntry.js | 13 ++++--- .../wiki-data/withParsedCommentaryEntries.js | 41 +++++++++++++++++----- .../wiki-properties/commentatorArtists.js | 23 +++++++----- src/data/yaml.js | 24 ++++++++++--- src/util/wiki-data.js | 2 +- 5 files changed, 75 insertions(+), 28 deletions(-) (limited to 'src') diff --git a/src/content/dependencies/generateCommentaryEntry.js b/src/content/dependencies/generateCommentaryEntry.js index b265ed41..0b2b2558 100644 --- a/src/content/dependencies/generateCommentaryEntry.js +++ b/src/content/dependencies/generateCommentaryEntry.js @@ -1,3 +1,5 @@ +import {empty} from '#sugar'; + export default { contentDependencies: [ 'generateColorStyleVariables', @@ -8,9 +10,10 @@ export default { extraDependencies: ['html', 'language'], relations: (relation, entry) => ({ - artistLink: - (entry.artist && !entry.artistDisplayText - ? relation('linkArtist', entry.artist) + artistLinks: + (!empty(entry.artists) && !entry.artistDisplayText + ? entry.artists + .map(artist => relation('linkArtist', artist)) : null), artistsContent: @@ -45,8 +48,8 @@ export default { html.tag('span', {class: 'commentary-entry-artists'}, (relations.artistsContent ? relations.artistsContent.slot('mode', 'inline') - : relations.artistLink - ? relations.artistLink + : relations.artistLinks + ? language.formatConjunctionList(relations.artistLinks) : language.$('misc.artistCommentary.entry.title.noArtists'))); const accentParts = ['misc.artistCommentary.entry.title.accent']; diff --git a/src/data/composite/wiki-data/withParsedCommentaryEntries.js b/src/data/composite/wiki-data/withParsedCommentaryEntries.js index 7b1c9484..25c07a37 100644 --- a/src/data/composite/wiki-data/withParsedCommentaryEntries.js +++ b/src/data/composite/wiki-data/withParsedCommentaryEntries.js @@ -4,7 +4,12 @@ import {stitchArrays} from '#sugar'; import {isCommentary} from '#validators'; import {commentaryRegex} from '#wiki-data'; -import {fillMissingListItems, withPropertiesFromList} from '#composite/data'; +import { + fillMissingListItems, + withFlattenedList, + withPropertiesFromList, + withUnflattenedList, +} from '#composite/data'; import withResolvedReferenceList from './withResolvedReferenceList.js'; @@ -86,23 +91,43 @@ export default templateCompositeFrom({ list: '#rawMatches.groups', prefix: input.value('#entries'), properties: input.value([ - 'artistReference', + 'artistReferences', 'artistDisplayText', 'annotation', 'date', ]), }), - // The artistReference group will always have a value, since it's required + // The artistReferences group will always have a value, since it's required // for the line to match in the first place. + { + dependencies: ['#entries.artistReferences'], + compute: (continuation, { + ['#entries.artistReferences']: artistReferenceTexts, + }) => continuation({ + ['#entries.artistReferences']: + artistReferenceTexts + .map(text => text.split(',').map(ref => ref.trim())), + }), + }, + + withFlattenedList({ + list: '#entries.artistReferences', + }), + withResolvedReferenceList({ - list: '#entries.artistReference', + list: '#flattenedList', data: 'artistData', find: input.value(find.artist), notFoundMode: input.value('null'), + }), + + withUnflattenedList({ + list: '#resolvedReferenceList', + filter: input.value(false), }).outputs({ - '#resolvedReferenceList': '#entries.artist', + '#unflattenedList': '#entries.artists', }), fillMissingListItems({ @@ -127,7 +152,7 @@ export default templateCompositeFrom({ { dependencies: [ - '#entries.artist', + '#entries.artists', '#entries.artistDisplayText', '#entries.annotation', '#entries.date', @@ -135,7 +160,7 @@ export default templateCompositeFrom({ ], compute: (continuation, { - ['#entries.artist']: artist, + ['#entries.artists']: artists, ['#entries.artistDisplayText']: artistDisplayText, ['#entries.annotation']: annotation, ['#entries.date']: date, @@ -143,7 +168,7 @@ export default templateCompositeFrom({ }) => continuation({ ['#parsedCommentaryEntries']: stitchArrays({ - artist, + artists, artistDisplayText, annotation, date, diff --git a/src/data/composite/wiki-properties/commentatorArtists.js b/src/data/composite/wiki-properties/commentatorArtists.js index 8720e66d..65ab1466 100644 --- a/src/data/composite/wiki-properties/commentatorArtists.js +++ b/src/data/composite/wiki-properties/commentatorArtists.js @@ -4,8 +4,9 @@ import {input, templateCompositeFrom} from '#composite'; import {unique} from '#sugar'; -import {exitWithoutDependency} from '#composite/control-flow'; -import {withPropertyFromList} from '#composite/data'; +import {exitWithoutDependency, exposeDependency} + from '#composite/control-flow'; +import {withFlattenedList, withPropertyFromList} from '#composite/data'; import {withParsedCommentaryEntries} from '#composite/wiki-data'; export default templateCompositeFrom({ @@ -26,15 +27,19 @@ export default templateCompositeFrom({ withPropertyFromList({ list: '#parsedCommentaryEntries', - property: input.value('artist'), + property: input.value('artists'), }).outputs({ - '#parsedCommentaryEntries.artist': '#artists', + '#parsedCommentaryEntries.artists': '#artistLists', }), - { - dependencies: ['#artists'], - compute: ({'#artists': artists}) => - unique(artists.filter(artist => artist !== null)), - }, + withFlattenedList({ + list: '#artistLists', + }).outputs({ + '#flattenedList': '#artists', + }), + + exposeDependency({ + dependency: '#artists', + }), ], }); diff --git a/src/data/yaml.js b/src/data/yaml.js index 843e70b3..0734d539 100644 --- a/src/data/yaml.js +++ b/src/data/yaml.js @@ -21,6 +21,7 @@ import { decorateErrorWithIndex, decorateErrorWithAnnotation, empty, + filterAggregate, filterProperties, openAggregate, showAggregate, @@ -1686,8 +1687,10 @@ export function filterReferenceErrors(wikiData) { if (value) { value = Array.from(value.matchAll(commentaryRegex)) - .map(({groups}) => groups.artistReference); + .map(({groups}) => groups.artistReferences) + .map(text => text.split(',').map(text => text.trim())); } + writeProperty = false; break; } @@ -1804,11 +1807,22 @@ export function filterReferenceErrors(wikiData) { let newPropertyValue = value; - if (Array.isArray(value)) { + if (findFnKey === '_commentary') { + // Commentary doesn't write a property value, so no need to set. + filter( + value, {message: errorMessage}, + decorateErrorWithIndex(refs => + (refs.length === 1 + ? suppress(findFn)(refs[0]) + : filterAggregate( + refs, {message: `Errors in entry's artist references`}, + decorateErrorWithIndex(suppress(findFn))) + .aggregate + .close()))); + } else if (Array.isArray(value)) { newPropertyValue = filter( - value, - decorateErrorWithIndex(suppress(findFn)), - {message: errorMessage}); + value, {message: errorMessage}, + decorateErrorWithIndex(suppress(findFn))); } else { nest({message: errorMessage}, suppress(({call}) => { diff --git a/src/util/wiki-data.js b/src/util/wiki-data.js index 75a141d3..4c7ec043 100644 --- a/src/util/wiki-data.js +++ b/src/util/wiki-data.js @@ -652,7 +652,7 @@ export function sortFlashesChronologically(data, { // out of the original string based on the indices matched using this. // export const commentaryRegex = - /^(?.+?)(?:\|(?.+))?:<\/i>(?: \((?(?:.*?(?=[,)]))*?)(?:,? ?(?[a-zA-Z]+ [0-9]{1,2}, [0-9]{4,4}|[0-9]{1,2} [^,]*[0-9]{4,4}|[0-9]{1,2}\/[0-9]{1,2}\/[0-9]{2,4}))?\))?/gm; + /^(?.+?)(?:\|(?.+))?:<\/i>(?: \((?(?:.*?(?=[,)]))*?)(?:,? ?(?[a-zA-Z]+ [0-9]{1,2}, [0-9]{4,4}|[0-9]{1,2} [^,]*[0-9]{4,4}|[0-9]{1,2}\/[0-9]{1,2}\/[0-9]{2,4}))?\))?/gm; export function filterAlbumsByCommentary(albums) { return albums -- cgit 1.3.0-6-gf8a5 From 7da15227b8623a2fcef28c4f7988a2f89e5ab8b3 Mon Sep 17 00:00:00 2001 From: "(quasar) nebula" Date: Sat, 18 Nov 2023 19:22:08 -0400 Subject: content: transformContent: use marked for inline + own instances There's obviously lots of room to do more here, but this is a simple way to get typical inline contents passing through marked. --- src/content/dependencies/transformContent.js | 57 +++++++++++++++++++--------- 1 file changed, 39 insertions(+), 18 deletions(-) (limited to 'src') diff --git a/src/content/dependencies/transformContent.js b/src/content/dependencies/transformContent.js index dab89630..7b2d0573 100644 --- a/src/content/dependencies/transformContent.js +++ b/src/content/dependencies/transformContent.js @@ -1,7 +1,7 @@ import {bindFind} from '#find'; import {parseInput} from '#replacer'; -import {marked} from 'marked'; +import {Marked} from 'marked'; export const replacerSpec = { album: { @@ -147,6 +147,29 @@ const linkIndexRelationMap = { newsIndex: 'linkNewsIndex', }; +const commonMarkedOptions = { + headerIds: false, + mangle: false, +}; + +const multilineMarked = new Marked({ + ...commonMarkedOptions, +}); + +const inlineMarked = new Marked({ + ...commonMarkedOptions, + + renderer: { + paragraph(text) { + return text; + }, + }, +}); + +const lyricsMarked = new Marked({ + ...commonMarkedOptions, +}); + function getPlaceholder(node, content) { return {type: 'text', data: content.slice(node.i, node.iEnd)}; } @@ -447,21 +470,9 @@ export default { return link.data; } - // In inline mode, no further processing is needed! - - if (slots.mode === 'inline') { - return html.tags( - contentFromNodes.map(node => node.data), - {[html.joinChildren]: ''}); - } - - // Multiline mode has a secondary processing stage where it's passed... - // through marked! Rolling your own Markdown only gets you so far :D - - const markedOptions = { - headerIds: false, - mangle: false, - }; + // Content always goes through marked (i.e. parsing as Markdown). + // This does require some attention to detail, mostly to do with line + // breaks (in multiline mode) and extracting/re-inserting non-text nodes. // The content of non-text nodes can end up getting mangled by marked. // To avoid this, we replace them with mundane placeholders, then @@ -536,6 +547,16 @@ export default { return html.tags(tags, {[html.joinChildren]: ''}); }; + if (slots.mode === 'inline') { + const markedInput = + extractNonTextNodes(); + + const markedOutput = + inlineMarked.parse(markedInput); + + return reinsertNonTextNodes(markedOutput); + } + // This is separated into its own function just since we're gonna reuse // it in a minute if everything goes to heck in lyrics mode. const transformMultiline = () => { @@ -552,7 +573,7 @@ export default { .replace(/(?<=^>.*)\n+(?!^>)/gm, '\n\n'); const markedOutput = - marked.parse(markedInput, markedOptions); + multilineMarked.parse(markedInput); return reinsertNonTextNodes(markedOutput); } @@ -602,7 +623,7 @@ export default { }); const markedOutput = - marked.parse(markedInput, markedOptions); + lyricsMarked.parse(markedInput); return reinsertNonTextNodes(markedOutput); } -- cgit 1.3.0-6-gf8a5 From 4b7da4c1f8c359e5c82c4cc5e0cfb78f8204850f Mon Sep 17 00:00:00 2001 From: "(quasar) nebula" Date: Sat, 18 Nov 2023 19:21:31 -0400 Subject: data: parse commentary heading contents to end of line --- src/util/wiki-data.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/util/wiki-data.js b/src/util/wiki-data.js index 4c7ec043..8a6d4345 100644 --- a/src/util/wiki-data.js +++ b/src/util/wiki-data.js @@ -652,7 +652,7 @@ export function sortFlashesChronologically(data, { // out of the original string based on the indices matched using this. // export const commentaryRegex = - /^(?.+?)(?:\|(?.+))?:<\/i>(?: \((?(?:.*?(?=[,)]))*?)(?:,? ?(?[a-zA-Z]+ [0-9]{1,2}, [0-9]{4,4}|[0-9]{1,2} [^,]*[0-9]{4,4}|[0-9]{1,2}\/[0-9]{1,2}\/[0-9]{2,4}))?\))?/gm; + /^(?.+?)(?:\|(?.+))?:<\/i>(?: \((?(?:.*?(?=,|\)$))*?)(?:,? ?(?[a-zA-Z]+ [0-9]{1,2}, [0-9]{4,4}|[0-9]{1,2} [^,]*[0-9]{4,4}|[0-9]{1,2}\/[0-9]{1,2}\/[0-9]{2,4}))?\))?$/gm; export function filterAlbumsByCommentary(albums) { return albums -- cgit 1.3.0-6-gf8a5 From 522c982bf0b5a0bd39512eb56a9d0d8d8feea44e Mon Sep 17 00:00:00 2001 From: "(quasar) nebula" Date: Sat, 18 Nov 2023 19:30:27 -0400 Subject: data: looser commentary date parsing + clearer regex explanation --- src/util/wiki-data.js | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'src') diff --git a/src/util/wiki-data.js b/src/util/wiki-data.js index 8a6d4345..09b3623e 100644 --- a/src/util/wiki-data.js +++ b/src/util/wiki-data.js @@ -641,8 +641,10 @@ export function sortFlashesChronologically(data, { // // * "25 December 2019" - one or two number digits, followed by any text, // followed by four number digits -// * "12/25/2019" - one or two number digits, a slash, one or two number -// digits, a slash, and two to four number digits +// * "December 25, 2019" - one all-letters word, a space, one or two number +// digits, a comma, and four number digits +// * "12/25/2019" etc - three sets of one to four number digits, separated +// by slashes (only valid formats are MM/DD/YYYY and YYYY/MM/DD) // // Capturing group "artistReference" is all the characters between and // (apart from the pipe and "artistDisplayText" text, if present), and is either @@ -652,7 +654,7 @@ export function sortFlashesChronologically(data, { // out of the original string based on the indices matched using this. // export const commentaryRegex = - /^(?.+?)(?:\|(?.+))?:<\/i>(?: \((?(?:.*?(?=,|\)$))*?)(?:,? ?(?[a-zA-Z]+ [0-9]{1,2}, [0-9]{4,4}|[0-9]{1,2} [^,]*[0-9]{4,4}|[0-9]{1,2}\/[0-9]{1,2}\/[0-9]{2,4}))?\))?$/gm; + /^(?.+?)(?:\|(?.+))?:<\/i>(?: \((?(?:.*?(?=,|\)$))*?)(?:,? ?(?[a-zA-Z]+ [0-9]{1,2}, [0-9]{4,4}|[0-9]{1,2} [^,]*[0-9]{4,4}|[0-9]{1,4}\/[0-9]{1,4}\/[0-9]{1,4}))?\))?$/gm; export function filterAlbumsByCommentary(albums) { return albums -- cgit 1.3.0-6-gf8a5 From e874f7e4d0df5fed25d4c359c8cb403e67061e59 Mon Sep 17 00:00:00 2001 From: "(quasar) nebula" Date: Sat, 18 Nov 2023 19:33:38 -0400 Subject: data: support dash-style short dates in commentary dates --- src/util/wiki-data.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/util/wiki-data.js b/src/util/wiki-data.js index 09b3623e..5e3182a9 100644 --- a/src/util/wiki-data.js +++ b/src/util/wiki-data.js @@ -644,7 +644,7 @@ export function sortFlashesChronologically(data, { // * "December 25, 2019" - one all-letters word, a space, one or two number // digits, a comma, and four number digits // * "12/25/2019" etc - three sets of one to four number digits, separated -// by slashes (only valid formats are MM/DD/YYYY and YYYY/MM/DD) +// by slashes or dashes (only valid orders are MM/DD/YYYY and YYYY/MM/DD) // // Capturing group "artistReference" is all the characters between and // (apart from the pipe and "artistDisplayText" text, if present), and is either @@ -654,7 +654,7 @@ export function sortFlashesChronologically(data, { // out of the original string based on the indices matched using this. // export const commentaryRegex = - /^(?.+?)(?:\|(?.+))?:<\/i>(?: \((?(?:.*?(?=,|\)$))*?)(?:,? ?(?[a-zA-Z]+ [0-9]{1,2}, [0-9]{4,4}|[0-9]{1,2} [^,]*[0-9]{4,4}|[0-9]{1,4}\/[0-9]{1,4}\/[0-9]{1,4}))?\))?$/gm; + /^(?.+?)(?:\|(?.+))?:<\/i>(?: \((?(?:.*?(?=,|\)$))*?)(?:,? ?(?[a-zA-Z]+ [0-9]{1,2}, [0-9]{4,4}|[0-9]{1,2} [^,]*[0-9]{4,4}|[0-9]{1,4}[-/][0-9]{1,4}[-/][0-9]{1,4}))?\))?$/gm; export function filterAlbumsByCommentary(albums) { return albums -- cgit 1.3.0-6-gf8a5 From f82b74f62595c4def21b176c366c703da02c331e Mon Sep 17 00:00:00 2001 From: "(quasar) nebula" Date: Sat, 18 Nov 2023 20:15:07 -0400 Subject: data, test: withUniqueItemsOnly (#composite/data) --- src/data/composite/data/index.js | 1 + src/data/composite/data/withUniqueItemsOnly.js | 40 ++++++++++++++++++++++++++ 2 files changed, 41 insertions(+) create mode 100644 src/data/composite/data/withUniqueItemsOnly.js (limited to 'src') diff --git a/src/data/composite/data/index.js b/src/data/composite/data/index.js index db1c37cc..e2927afd 100644 --- a/src/data/composite/data/index.js +++ b/src/data/composite/data/index.js @@ -11,3 +11,4 @@ export {default as withPropertiesFromObject} from './withPropertiesFromObject.js export {default as withPropertyFromList} from './withPropertyFromList.js'; export {default as withPropertyFromObject} from './withPropertyFromObject.js'; export {default as withUnflattenedList} from './withUnflattenedList.js'; +export {default as withUniqueItemsOnly} from './withUniqueItemsOnly.js'; diff --git a/src/data/composite/data/withUniqueItemsOnly.js b/src/data/composite/data/withUniqueItemsOnly.js new file mode 100644 index 00000000..7ee08b08 --- /dev/null +++ b/src/data/composite/data/withUniqueItemsOnly.js @@ -0,0 +1,40 @@ +// Excludes duplicate items from a list and provides the results, overwriting +// the list in-place, if possible. + +import {input, templateCompositeFrom} from '#composite'; +import {unique} from '#sugar'; + +export default templateCompositeFrom({ + annotation: `withUniqueItemsOnly`, + + inputs: { + list: input({type: 'array'}), + }, + + outputs: ({ + [input.staticDependency('list')]: list, + }) => [list ?? '#uniqueItems'], + + steps: () => [ + { + dependencies: [input('list')], + compute: (continuation, { + [input('list')]: list, + }) => continuation({ + ['#values']: + unique(list), + }), + }, + + { + dependencies: ['#values', input.staticDependency('list')], + compute: (continuation, { + '#values': values, + [input.staticDependency('list')]: list, + }) => continuation({ + [list ?? '#uniqueItems']: + values, + }), + }, + ], +}); -- cgit 1.3.0-6-gf8a5 From 5b8060bb86d457a0d23b607aa866c4d7d6eb6f0f Mon Sep 17 00:00:00 2001 From: "(quasar) nebula" Date: Sat, 18 Nov 2023 20:16:40 -0400 Subject: data: withParsedCommentaryEntries: filter out null artists --- src/data/composite/wiki-data/withParsedCommentaryEntries.js | 1 - 1 file changed, 1 deletion(-) (limited to 'src') diff --git a/src/data/composite/wiki-data/withParsedCommentaryEntries.js b/src/data/composite/wiki-data/withParsedCommentaryEntries.js index 25c07a37..edfc9e3c 100644 --- a/src/data/composite/wiki-data/withParsedCommentaryEntries.js +++ b/src/data/composite/wiki-data/withParsedCommentaryEntries.js @@ -125,7 +125,6 @@ export default templateCompositeFrom({ withUnflattenedList({ list: '#resolvedReferenceList', - filter: input.value(false), }).outputs({ '#unflattenedList': '#entries.artists', }), -- cgit 1.3.0-6-gf8a5 From e35d23f4e9492b497138dce3f21382872e329e71 Mon Sep 17 00:00:00 2001 From: "(quasar) nebula" Date: Sat, 18 Nov 2023 20:16:57 -0400 Subject: data: commentatorArtists: filter out duplicate artists --- src/data/composite/wiki-properties/commentatorArtists.js | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/data/composite/wiki-properties/commentatorArtists.js b/src/data/composite/wiki-properties/commentatorArtists.js index 65ab1466..f400bbfc 100644 --- a/src/data/composite/wiki-properties/commentatorArtists.js +++ b/src/data/composite/wiki-properties/commentatorArtists.js @@ -6,7 +6,8 @@ import {unique} from '#sugar'; import {exitWithoutDependency, exposeDependency} from '#composite/control-flow'; -import {withFlattenedList, withPropertyFromList} from '#composite/data'; +import {withFlattenedList, withPropertyFromList, withUniqueItemsOnly} + from '#composite/data'; import {withParsedCommentaryEntries} from '#composite/wiki-data'; export default templateCompositeFrom({ @@ -38,6 +39,10 @@ export default templateCompositeFrom({ '#flattenedList': '#artists', }), + withUniqueItemsOnly({ + list: '#artists', + }), + exposeDependency({ dependency: '#artists', }), -- cgit 1.3.0-6-gf8a5