diff options
Diffstat (limited to 'src/common-util')
| -rw-r--r-- | src/common-util/search-spec.js | 217 | ||||
| -rw-r--r-- | src/common-util/sort.js | 16 | ||||
| -rw-r--r-- | src/common-util/sugar.js | 112 | ||||
| -rw-r--r-- | src/common-util/wiki-data.js | 69 |
4 files changed, 282 insertions, 132 deletions
diff --git a/src/common-util/search-spec.js b/src/common-util/search-spec.js index 75de0d16..af5ec201 100644 --- a/src/common-util/search-spec.js +++ b/src/common-util/search-spec.js @@ -85,107 +85,116 @@ function prepareArtwork(thing, { return serializeSrc; } -export const searchSpec = { - generic: { - query: ({ - albumData, - artTagData, - artistData, - flashData, - groupData, - trackData, - }) => [ - albumData, - - artTagData, - - artistData - .filter(artist => !artist.isAlias), - - flashData, - - groupData, - - trackData - // Exclude rereleases - there's no reasonable way to differentiate - // them from the main release as part of this query. - .filter(track => !track.mainReleaseTrack), - ].flat(), - - process(thing, opts) { - const fields = {}; - - fields.primaryName = - thing.name; - - const kind = - thing.constructor[Symbol.for('Thing.referenceType')]; - - fields.parentName = - (kind === 'track' - ? thing.album.name - : kind === 'group' - ? thing.category.name - : kind === 'flash' - ? thing.act.name - : null); - - fields.color = - thing.color; - - fields.artTags = - (thing.constructor.hasPropertyDescriptor('artTags') - ? thing.artTags.map(artTag => artTag.nameShort) - : []); - - fields.additionalNames = - (thing.constructor.hasPropertyDescriptor('additionalNames') - ? thing.additionalNames.map(entry => entry.name) - : thing.constructor.hasPropertyDescriptor('aliasNames') - ? thing.aliasNames - : []); - - const contribKeys = [ - 'artistContribs', - 'bannerArtistContribs', - 'contributorContribs', - 'coverArtistContribs', - 'wallpaperArtistContribs', - ]; +function baselineProcess(thing, opts) { + const fields = {}; + + fields.primaryName = + thing.name; - const contributions = - contribKeys - .filter(key => Object.hasOwn(thing, key)) - .flatMap(key => thing[key]); + fields.artwork = + prepareArtwork(thing, opts); + + fields.color = + thing.color; + + return fields; +} - fields.contributors = - contributions - .flatMap(({artist}) => [ - artist.name, - ...artist.aliasNames, - ]); +const baselineStore = [ + 'primaryName', + 'artwork', + 'color', +]; - const groups = - (Object.hasOwn(thing, 'groups') - ? thing.groups - : Object.hasOwn(thing, 'album') - ? thing.album.groups - : []); +function genericQuery(wikiData) { + return [ + wikiData.albumData, - const mainContributorNames = - contributions - .map(({artist}) => artist.name); + wikiData.artTagData, - fields.groups = - groups - .filter(group => !mainContributorNames.includes(group.name)) - .map(group => group.name); + wikiData.artistData + .filter(artist => !artist.isAlias), + + wikiData.flashData, + + wikiData.groupData, + + wikiData.trackData + // Exclude rereleases - there's no reasonable way to differentiate + // them from the main release as part of this query. + .filter(track => !track.mainReleaseTrack), + ].flat(); +} + +function genericProcess(thing, opts) { + const fields = baselineProcess(thing, opts); + + const kind = + thing.constructor[Symbol.for('Thing.referenceType')]; + + fields.parentName = + (kind === 'track' + ? thing.album.name + : kind === 'group' + ? thing.category.name + : kind === 'flash' + ? thing.act.name + : null); + + fields.artTags = + (thing.constructor.hasPropertyDescriptor('artTags') + ? thing.artTags.map(artTag => artTag.nameShort) + : []); + + fields.additionalNames = + (thing.constructor.hasPropertyDescriptor('additionalNames') + ? thing.additionalNames.map(entry => entry.name) + : thing.constructor.hasPropertyDescriptor('aliasNames') + ? thing.aliasNames + : []); + + const contribKeys = [ + 'artistContribs', + 'contributorContribs', + ]; + + const contributions = + contribKeys + .filter(key => Object.hasOwn(thing, key)) + .flatMap(key => thing[key]); + + fields.contributors = + contributions + .flatMap(({artist}) => [ + artist.name, + ...artist.aliasNames, + ]); + + const groups = + (Object.hasOwn(thing, 'groups') + ? thing.groups + : Object.hasOwn(thing, 'album') + ? thing.album.groups + : []); + + const mainContributorNames = + contributions + .map(({artist}) => artist.name); + + fields.groups = + groups + .filter(group => !mainContributorNames.includes(group.name)) + .map(group => group.name); + + return fields; +} - fields.artwork = - prepareArtwork(thing, opts); +const genericStore = baselineStore; - return fields; - }, +export const searchSpec = { + generic: { + query: genericQuery, + process: genericProcess, index: [ 'primaryName', @@ -194,13 +203,25 @@ export const searchSpec = { 'additionalNames', 'contributors', 'groups', - ], + ].map(field => ({field, tokenize: 'forward'})), - store: [ + store: genericStore, + }, + + verbatim: { + query: genericQuery, + process: genericProcess, + + index: [ 'primaryName', - 'artwork', - 'color', + 'parentName', + 'artTags', + 'additionalNames', + 'contributors', + 'groups', ], + + store: genericStore, }, }; diff --git a/src/common-util/sort.js b/src/common-util/sort.js index 3cfe8f70..d93d94c1 100644 --- a/src/common-util/sort.js +++ b/src/common-util/sort.js @@ -389,6 +389,22 @@ export function sortAlbumsTracksChronologically(data, { return data; } +export function sortArtworksChronologically(data, { + latestFirst = false, +} = {}) { + // Artworks conveniently describe their things as artwork.thing, so they + // work in sortEntryThingPairs. (Yes, this is just assuming the artworks + // are only for albums and tracks... sorry... TODO...) + sortEntryThingPairs(data, things => + sortAlbumsTracksChronologically(things, {latestFirst})); + + // Artworks' own dates always matter before however the thing places itself, + // and accommodate per-thing properties like coverArtDate anyway. + sortByDate(data, {latestFirst}); + + return data; +} + export function sortFlashesChronologically(data, { latestFirst = false, getDate, diff --git a/src/common-util/sugar.js b/src/common-util/sugar.js index 66e160aa..e931ad59 100644 --- a/src/common-util/sugar.js +++ b/src/common-util/sugar.js @@ -116,10 +116,14 @@ export function findIndexOrEnd(array, fn) { // returns null (or values in the array are nullish), they'll just be skipped in // the sum. export function accumulateSum(array, fn = x => x) { + if (!Array.isArray(array)) { + return accumulateSum(Array.from(array, fn)); + } + return array.reduce( (accumulator, value, index, array) => accumulator + - fn(value, index, array) ?? 0, + (fn(value, index, array) ?? 0), 0); } @@ -221,6 +225,9 @@ export const compareArrays = (arr1, arr2, {checkOrder = true} = {}) => ? arr1.every((x, i) => arr2[i] === x) : arr1.every((x) => arr2.includes(x))); +export const exhaust = (generatorFunction) => + Array.from(generatorFunction()); + export function compareObjects(obj1, obj2, { checkOrder = false, checkSymbols = true, @@ -251,11 +258,20 @@ export function compareObjects(obj1, obj2, { // Stolen from jq! Which pro8a8ly stole the concept from other places. Nice. export const withEntries = (obj, fn) => { - const result = fn(Object.entries(obj)); - if (result instanceof Promise) { - return result.then(entries => Object.fromEntries(entries)); + if (obj instanceof Map) { + const result = fn(Array.from(obj.entries())); + if (result instanceof Promise) { + return result.then(entries => new map(entries)); + } else { + return new Map(result); + } } else { - return Object.fromEntries(result); + const result = fn(Object.entries(obj)); + if (result instanceof Promise) { + return result.then(entries => Object.fromEntries(entries)); + } else { + return Object.fromEntries(result); + } } } @@ -299,34 +315,74 @@ export function filterProperties(object, properties, { return filteredObject; } -export function queue(array, max = 50) { - if (max === 0) { - return array.map((fn) => fn()); +export function queue(functionList, queueSize = 50) { + if (queueSize === 0) { + return functionList.map(fn => fn()); } - const begin = []; - let current = 0; - const ret = array.map( - (fn) => - new Promise((resolve, reject) => { - begin.push(() => { - current++; - Promise.resolve(fn()).then((value) => { - current--; - if (current < max && begin.length) { - begin.shift()(); - } - resolve(value); - }, reject); - }); - }) - ); + const promiseList = []; + const resolveList = []; + const rejectList = []; - for (let i = 0; i < max && begin.length; i++) { - begin.shift()(); + for (let i = 0; i < functionList.length; i++) { + const promiseWithResolvers = Promise.withResolvers(); + promiseList.push(promiseWithResolvers.promise); + resolveList.push(promiseWithResolvers.resolve); + rejectList.push(promiseWithResolvers.reject); } - return ret; + let cursor = 0; + let running = 0; + + const next = async () => { + if (running >= queueSize) { + return; + } + + if (cursor === functionList.length) { + return; + } + + const thisFunction = functionList[cursor]; + const thisResolve = resolveList[cursor]; + const thisReject = rejectList[cursor]; + + delete functionList[cursor]; + delete resolveList[cursor]; + delete rejectList[cursor]; + + cursor++; + running++; + + try { + thisResolve(await thisFunction()); + } catch (error) { + thisReject(error); + } finally { + running--; + + // If the cursor is at 1, this is the first promise that resolved, + // so we're now done the "kick start", and can start the remaining + // promises (up to queueSize). + if (cursor === 1) { + // Since only one promise is used for the "kick start", and that one + // has just resolved, we know there's none running at all right now, + // and can start as many as specified in the queueSize right away. + for (let i = 0; i < queueSize; i++) { + next(); + } + } else { + next(); + } + } + }; + + // Only start a single promise, as a "kick start", so that it resolves as + // early as possible (it will resolve before we use CPU to start the rest + // of the promises, up to queueSize). + next(); + + return promiseList; } export function delay(ms) { diff --git a/src/common-util/wiki-data.js b/src/common-util/wiki-data.js index 4bbef8ab..546f1ad9 100644 --- a/src/common-util/wiki-data.js +++ b/src/common-util/wiki-data.js @@ -57,10 +57,9 @@ export function getKebabCase(name) { // Specific data utilities -// Matches heading details from commentary data in roughly the formats: +// Matches heading details from commentary data in roughly the format: // -// <i>artistReference:</i> (annotation, date) -// <i>artistReference|artistDisplayText:</i> (annotation, date) +// <i>artistText:</i> (annotation, date) // // where capturing group "annotation" can be any text at all, except that the // last entry (past a comma or the only content within parentheses), if parsed @@ -83,8 +82,9 @@ export function getKebabCase(name) { // parentheses can be part of the actual annotation content. // // Capturing group "artistReference" is all the characters between <i> and </i> -// (apart from the pipe and "artistDisplayText" text, if present), and is either -// the name of an artist or an "artist:directory"-style reference. +// (apart from the pipe and the "artistText" group, if present), and is either +// the name of one or more artist or "artist:directory"-style references, +// joined by commas, if multiple. // // This regular expression *doesn't* match bodies, which will need to be parsed // out of the original string based on the indices matched using this. @@ -94,7 +94,7 @@ const dateRegex = groupName => String.raw`(?<${groupName}>[a-zA-Z]+ [0-9]{1,2}, [0-9]{4,4}|[0-9]{1,2} [^,]*[0-9]{4,4}|[0-9]{1,4}[-/][0-9]{1,4}[-/][0-9]{1,4})`; const commentaryRegexRaw = - String.raw`^<i>(?<artistReferences>.+?)(?:\|(?<artistDisplayText>.+))?:<\/i>(?: \((?<annotation>(?:.*?(?=,|\)[^)]*$))*?)(?:,? ?(?:(?<dateKind>sometime|throughout|around) )?${dateRegex('date')}(?: ?- ?${dateRegex('secondDate')})?(?: (?<accessKind>captured|accessed) ${dateRegex('accessDate')})?)?\))?`; + String.raw`^<i>(?<artistText>.+?):<\/i>(?: \((?<annotation>(?:.*?(?=,|\)[^)]*$))*?)(?:,? ?(?:(?<dateKind>sometime|throughout|around) )?${dateRegex('date')}(?: ?- ?${dateRegex('secondDate')})?(?: (?<accessKind>captured|accessed) ${dateRegex('accessDate')})?)?\))?`; export const commentaryRegexCaseInsensitive = new RegExp(commentaryRegexRaw, 'gmi'); export const commentaryRegexCaseSensitive = @@ -102,6 +102,36 @@ export const commentaryRegexCaseSensitive = export const commentaryRegexCaseSensitiveOneShot = new RegExp(commentaryRegexRaw); +// The #validators function isOldStyleLyrics() describes +// what this regular expression detects against. +export const multipleLyricsDetectionRegex = + /^<i>.*:<\/i>/m; + +export function matchContentEntries(sourceText) { + const matchEntries = []; + + let previousMatchEntry = null; + let previousEndIndex = null; + + for (const {0: matchText, index: startIndex, groups: matchEntry} + of sourceText.matchAll(commentaryRegexCaseSensitive)) { + if (previousMatchEntry) { + previousMatchEntry.body = sourceText.slice(previousEndIndex, startIndex); + } + + matchEntries.push(matchEntry); + + previousMatchEntry = matchEntry; + previousEndIndex = startIndex + matchText.length; + } + + if (previousMatchEntry) { + previousMatchEntry.body = sourceText.slice(previousEndIndex); + } + + return matchEntries; +} + export function filterAlbumsByCommentary(albums) { return albums .filter((album) => [album, ...album.tracks].some((x) => x.commentary)); @@ -492,3 +522,30 @@ export function combineWikiDataArrays(arrays) { return combined; } } + +// Markdown stuff + +export function* matchMarkdownLinks(markdownSource, {marked}) { + const plausibleLinkRegexp = /\[.*?\)/g; + + let plausibleMatch = null; + while (plausibleMatch = plausibleLinkRegexp.exec(markdownSource)) { + // Pedantic rules use more particular parentheses detection in link + // destinations - they allow one level of balanced parentheses, and + // otherwise, parentheses must be escaped. This allows for entire links + // to be wrapped in parentheses, e.g below: + // + // This is so cool. ([You know??](https://example.com)) + // + const definiteMatch = + marked.Lexer.rules.inline.pedantic.link + .exec(markdownSource.slice(plausibleMatch.index)); + + if (definiteMatch) { + const [{length}, label, href] = definiteMatch; + const index = plausibleMatch.index + definiteMatch.index; + + yield {label, href, index, length}; + } + } +} |