diff options
Diffstat (limited to 'src/common-util')
| -rw-r--r-- | src/common-util/search-spec.js | 217 | ||||
| -rw-r--r-- | src/common-util/sort.js | 29 | ||||
| -rw-r--r-- | src/common-util/sugar.js | 154 | ||||
| -rw-r--r-- | src/common-util/wiki-data.js | 98 |
4 files changed, 357 insertions, 141 deletions
diff --git a/src/common-util/search-spec.js b/src/common-util/search-spec.js index 3d05c021..af5ec201 100644 --- a/src/common-util/search-spec.js +++ b/src/common-util/search-spec.js @@ -85,107 +85,116 @@ function prepareArtwork(thing, { return serializeSrc; } -export const searchSpec = { - generic: { - query: ({ - albumData, - artTagData, - artistData, - flashData, - groupData, - trackData, - }) => [ - albumData, - - artTagData, - - artistData - .filter(artist => !artist.isAlias), - - flashData, - - groupData, - - trackData - // Exclude rereleases - there's no reasonable way to differentiate - // them from the main release as part of this query. - .filter(track => !track.originalReleaseTrack), - ].flat(), - - process(thing, opts) { - const fields = {}; - - fields.primaryName = - thing.name; - - const kind = - thing.constructor[Symbol.for('Thing.referenceType')]; - - fields.parentName = - (kind === 'track' - ? thing.album.name - : kind === 'group' - ? thing.category.name - : kind === 'flash' - ? thing.act.name - : null); - - fields.color = - thing.color; - - fields.artTags = - (thing.constructor.hasPropertyDescriptor('artTags') - ? thing.artTags.map(artTag => artTag.nameShort) - : []); - - fields.additionalNames = - (thing.constructor.hasPropertyDescriptor('additionalNames') - ? thing.additionalNames.map(entry => entry.name) - : thing.constructor.hasPropertyDescriptor('aliasNames') - ? thing.aliasNames - : []); - - const contribKeys = [ - 'artistContribs', - 'bannerArtistContribs', - 'contributorContribs', - 'coverArtistContribs', - 'wallpaperArtistContribs', - ]; +function baselineProcess(thing, opts) { + const fields = {}; + + fields.primaryName = + thing.name; - const contributions = - contribKeys - .filter(key => Object.hasOwn(thing, key)) - .flatMap(key => thing[key]); + fields.artwork = + prepareArtwork(thing, opts); + + fields.color = + thing.color; + + return fields; +} - fields.contributors = - contributions - .flatMap(({artist}) => [ - artist.name, - ...artist.aliasNames, - ]); +const baselineStore = [ + 'primaryName', + 'artwork', + 'color', +]; - const groups = - (Object.hasOwn(thing, 'groups') - ? thing.groups - : Object.hasOwn(thing, 'album') - ? thing.album.groups - : []); +function genericQuery(wikiData) { + return [ + wikiData.albumData, - const mainContributorNames = - contributions - .map(({artist}) => artist.name); + wikiData.artTagData, - fields.groups = - groups - .filter(group => !mainContributorNames.includes(group.name)) - .map(group => group.name); + wikiData.artistData + .filter(artist => !artist.isAlias), + + wikiData.flashData, + + wikiData.groupData, + + wikiData.trackData + // Exclude rereleases - there's no reasonable way to differentiate + // them from the main release as part of this query. + .filter(track => !track.mainReleaseTrack), + ].flat(); +} + +function genericProcess(thing, opts) { + const fields = baselineProcess(thing, opts); + + const kind = + thing.constructor[Symbol.for('Thing.referenceType')]; + + fields.parentName = + (kind === 'track' + ? thing.album.name + : kind === 'group' + ? thing.category.name + : kind === 'flash' + ? thing.act.name + : null); + + fields.artTags = + (thing.constructor.hasPropertyDescriptor('artTags') + ? thing.artTags.map(artTag => artTag.nameShort) + : []); + + fields.additionalNames = + (thing.constructor.hasPropertyDescriptor('additionalNames') + ? thing.additionalNames.map(entry => entry.name) + : thing.constructor.hasPropertyDescriptor('aliasNames') + ? thing.aliasNames + : []); + + const contribKeys = [ + 'artistContribs', + 'contributorContribs', + ]; + + const contributions = + contribKeys + .filter(key => Object.hasOwn(thing, key)) + .flatMap(key => thing[key]); + + fields.contributors = + contributions + .flatMap(({artist}) => [ + artist.name, + ...artist.aliasNames, + ]); + + const groups = + (Object.hasOwn(thing, 'groups') + ? thing.groups + : Object.hasOwn(thing, 'album') + ? thing.album.groups + : []); + + const mainContributorNames = + contributions + .map(({artist}) => artist.name); + + fields.groups = + groups + .filter(group => !mainContributorNames.includes(group.name)) + .map(group => group.name); + + return fields; +} - fields.artwork = - prepareArtwork(thing, opts); +const genericStore = baselineStore; - return fields; - }, +export const searchSpec = { + generic: { + query: genericQuery, + process: genericProcess, index: [ 'primaryName', @@ -194,13 +203,25 @@ export const searchSpec = { 'additionalNames', 'contributors', 'groups', - ], + ].map(field => ({field, tokenize: 'forward'})), - store: [ + store: genericStore, + }, + + verbatim: { + query: genericQuery, + process: genericProcess, + + index: [ 'primaryName', - 'artwork', - 'color', + 'parentName', + 'artTags', + 'additionalNames', + 'contributors', + 'groups', ], + + store: genericStore, }, }; diff --git a/src/common-util/sort.js b/src/common-util/sort.js index ea1e024a..d93d94c1 100644 --- a/src/common-util/sort.js +++ b/src/common-util/sort.js @@ -3,6 +3,12 @@ // initial sort matters! (Spoilers: If what you're doing involves any kind of // parallelization, it definitely matters.) +// TODO: This is obviously limiting. It does describe the behavior +// we've been *assuming* for the entire time the wiki is around, +// but it would be nice to support sorting in different locales +// somehow. +export const SORTING_LOCALE = 'en'; + import {empty, sortMultipleArrays, unique} from './sugar.js'; @@ -17,8 +23,8 @@ export function compareCaseLessSensitive(a, b) { const bl = b.toLowerCase(); return al === bl - ? a.localeCompare(b, undefined, {numeric: true}) - : al.localeCompare(bl, undefined, {numeric: true}); + ? a.localeCompare(b, SORTING_LOCALE, {numeric: true}) + : al.localeCompare(bl, SORTING_LOCALE, {numeric: true}); } // Subtract common prefixes and other characters which some people don't like @@ -383,6 +389,22 @@ export function sortAlbumsTracksChronologically(data, { return data; } +export function sortArtworksChronologically(data, { + latestFirst = false, +} = {}) { + // Artworks conveniently describe their things as artwork.thing, so they + // work in sortEntryThingPairs. (Yes, this is just assuming the artworks + // are only for albums and tracks... sorry... TODO...) + sortEntryThingPairs(data, things => + sortAlbumsTracksChronologically(things, {latestFirst})); + + // Artworks' own dates always matter before however the thing places itself, + // and accommodate per-thing properties like coverArtDate anyway. + sortByDate(data, {latestFirst}); + + return data; +} + export function sortFlashesChronologically(data, { latestFirst = false, getDate, @@ -407,6 +429,7 @@ export function sortFlashesChronologically(data, { export function sortContributionsChronologically(data, sortThings, { latestFirst = false, + getThing = contrib => contrib.thing, } = {}) { // Contributions only have one date property (which is provided when // the contribution is created). They're sorted by this most primarily, @@ -415,7 +438,7 @@ export function sortContributionsChronologically(data, sortThings, { const entries = data.map(contrib => ({ entry: contrib, - thing: contrib.thing, + thing: getThing(contrib), })); sortEntryThingPairs( diff --git a/src/common-util/sugar.js b/src/common-util/sugar.js index 90d47b7c..e931ad59 100644 --- a/src/common-util/sugar.js +++ b/src/common-util/sugar.js @@ -116,10 +116,14 @@ export function findIndexOrEnd(array, fn) { // returns null (or values in the array are nullish), they'll just be skipped in // the sum. export function accumulateSum(array, fn = x => x) { + if (!Array.isArray(array)) { + return accumulateSum(Array.from(array, fn)); + } + return array.reduce( (accumulator, value, index, array) => accumulator + - fn(value, index, array) ?? 0, + (fn(value, index, array) ?? 0), 0); } @@ -221,6 +225,9 @@ export const compareArrays = (arr1, arr2, {checkOrder = true} = {}) => ? arr1.every((x, i) => arr2[i] === x) : arr1.every((x) => arr2.includes(x))); +export const exhaust = (generatorFunction) => + Array.from(generatorFunction()); + export function compareObjects(obj1, obj2, { checkOrder = false, checkSymbols = true, @@ -251,11 +258,20 @@ export function compareObjects(obj1, obj2, { // Stolen from jq! Which pro8a8ly stole the concept from other places. Nice. export const withEntries = (obj, fn) => { - const result = fn(Object.entries(obj)); - if (result instanceof Promise) { - return result.then(entries => Object.fromEntries(entries)); + if (obj instanceof Map) { + const result = fn(Array.from(obj.entries())); + if (result instanceof Promise) { + return result.then(entries => new map(entries)); + } else { + return new Map(result); + } } else { - return Object.fromEntries(result); + const result = fn(Object.entries(obj)); + if (result instanceof Promise) { + return result.then(entries => Object.fromEntries(entries)); + } else { + return Object.fromEntries(result); + } } } @@ -299,34 +315,74 @@ export function filterProperties(object, properties, { return filteredObject; } -export function queue(array, max = 50) { - if (max === 0) { - return array.map((fn) => fn()); +export function queue(functionList, queueSize = 50) { + if (queueSize === 0) { + return functionList.map(fn => fn()); } - const begin = []; - let current = 0; - const ret = array.map( - (fn) => - new Promise((resolve, reject) => { - begin.push(() => { - current++; - Promise.resolve(fn()).then((value) => { - current--; - if (current < max && begin.length) { - begin.shift()(); - } - resolve(value); - }, reject); - }); - }) - ); + const promiseList = []; + const resolveList = []; + const rejectList = []; - for (let i = 0; i < max && begin.length; i++) { - begin.shift()(); + for (let i = 0; i < functionList.length; i++) { + const promiseWithResolvers = Promise.withResolvers(); + promiseList.push(promiseWithResolvers.promise); + resolveList.push(promiseWithResolvers.resolve); + rejectList.push(promiseWithResolvers.reject); } - return ret; + let cursor = 0; + let running = 0; + + const next = async () => { + if (running >= queueSize) { + return; + } + + if (cursor === functionList.length) { + return; + } + + const thisFunction = functionList[cursor]; + const thisResolve = resolveList[cursor]; + const thisReject = rejectList[cursor]; + + delete functionList[cursor]; + delete resolveList[cursor]; + delete rejectList[cursor]; + + cursor++; + running++; + + try { + thisResolve(await thisFunction()); + } catch (error) { + thisReject(error); + } finally { + running--; + + // If the cursor is at 1, this is the first promise that resolved, + // so we're now done the "kick start", and can start the remaining + // promises (up to queueSize). + if (cursor === 1) { + // Since only one promise is used for the "kick start", and that one + // has just resolved, we know there's none running at all right now, + // and can start as many as specified in the queueSize right away. + for (let i = 0; i < queueSize; i++) { + next(); + } + } else { + next(); + } + } + }; + + // Only start a single promise, as a "kick start", so that it resolves as + // early as possible (it will resolve before we use CPU to start the rest + // of the promises, up to queueSize). + next(); + + return promiseList; } export function delay(ms) { @@ -356,11 +412,19 @@ export function splitKeys(key) { } // Follows a key path like 'foo.bar.baz' to get an item nested deeply inside -// an object. +// an object. If a value partway through the chain is an array, the values +// down the rest of the chain are gotten for each item in the array. +// +// obj: {x: [{y: ['a']}, {y: ['b', 'c']}]} +// key: 'x.y' +// -> [['a'], ['b', 'c']] +// export function getNestedProp(obj, key) { const recursive = (o, k) => (k.length === 1 ? o[k[0]] + : Array.isArray(o[k[0]]) + ? o[k[0]].map(v => recursive(v, k.slice(1))) : recursive(o[k[0]], k.slice(1))); return recursive(obj, splitKeys(key)); @@ -773,6 +837,38 @@ export function chunkMultipleArrays(...args) { return results; } +// This (or its helper function) should probably be a generator, but generators +// are scary... Note that the root node is never considered a leaf, even if it +// doesn't have any branches. It does NOT pay attention to the *values* of the +// leaf nodes - it's suited to handle this kind of form: +// +// { +// foo: { +// bar: {}, +// baz: {}, +// qux: { +// woz: {}, +// }, +// }, +// } +// +// for which it outputs ['bar', 'baz', 'woz']. +// +export function collectTreeLeaves(tree) { + const recursive = ([key, value]) => + (value instanceof Map + ? (value.size === 0 + ? [key] + : Array.from(value.entries()).flatMap(recursive)) + : (empty(Object.keys(value)) + ? [key] + : Object.entries(value).flatMap(recursive))); + + const root = Symbol(); + const leaves = recursive([root, tree]); + return (leaves[0] === root ? [] : leaves); +} + // Delicious function annotations, such as: // // (*bound) soWeAreBackInTheMine diff --git a/src/common-util/wiki-data.js b/src/common-util/wiki-data.js index f97ecd63..546f1ad9 100644 --- a/src/common-util/wiki-data.js +++ b/src/common-util/wiki-data.js @@ -1,6 +1,6 @@ // Utility functions for interacting with wiki data. -import {accumulateSum, empty, unique} from './sugar.js'; +import {accumulateSum, chunkByConditions, empty, unique} from './sugar.js'; import {sortByDate} from './sort.js'; // This is a duplicate binding of filterMultipleArrays that's included purely @@ -57,10 +57,9 @@ export function getKebabCase(name) { // Specific data utilities -// Matches heading details from commentary data in roughly the formats: +// Matches heading details from commentary data in roughly the format: // -// <i>artistReference:</i> (annotation, date) -// <i>artistReference|artistDisplayText:</i> (annotation, date) +// <i>artistText:</i> (annotation, date) // // where capturing group "annotation" can be any text at all, except that the // last entry (past a comma or the only content within parentheses), if parsed @@ -83,8 +82,9 @@ export function getKebabCase(name) { // parentheses can be part of the actual annotation content. // // Capturing group "artistReference" is all the characters between <i> and </i> -// (apart from the pipe and "artistDisplayText" text, if present), and is either -// the name of an artist or an "artist:directory"-style reference. +// (apart from the pipe and the "artistText" group, if present), and is either +// the name of one or more artist or "artist:directory"-style references, +// joined by commas, if multiple. // // This regular expression *doesn't* match bodies, which will need to be parsed // out of the original string based on the indices matched using this. @@ -94,7 +94,7 @@ const dateRegex = groupName => String.raw`(?<${groupName}>[a-zA-Z]+ [0-9]{1,2}, [0-9]{4,4}|[0-9]{1,2} [^,]*[0-9]{4,4}|[0-9]{1,4}[-/][0-9]{1,4}[-/][0-9]{1,4})`; const commentaryRegexRaw = - String.raw`^<i>(?<artistReferences>.+?)(?:\|(?<artistDisplayText>.+))?:<\/i>(?: \((?<annotation>(?:.*?(?=,|\)[^)]*$))*?)(?:,? ?(?:(?<dateKind>sometime|throughout|around) )?${dateRegex('date')}(?: ?- ?${dateRegex('secondDate')})?(?: (?<accessKind>captured|accessed) ${dateRegex('accessDate')})?)?\))?`; + String.raw`^<i>(?<artistText>.+?):<\/i>(?: \((?<annotation>(?:.*?(?=,|\)[^)]*$))*?)(?:,? ?(?:(?<dateKind>sometime|throughout|around) )?${dateRegex('date')}(?: ?- ?${dateRegex('secondDate')})?(?: (?<accessKind>captured|accessed) ${dateRegex('accessDate')})?)?\))?`; export const commentaryRegexCaseInsensitive = new RegExp(commentaryRegexRaw, 'gmi'); export const commentaryRegexCaseSensitive = @@ -102,6 +102,36 @@ export const commentaryRegexCaseSensitive = export const commentaryRegexCaseSensitiveOneShot = new RegExp(commentaryRegexRaw); +// The #validators function isOldStyleLyrics() describes +// what this regular expression detects against. +export const multipleLyricsDetectionRegex = + /^<i>.*:<\/i>/m; + +export function matchContentEntries(sourceText) { + const matchEntries = []; + + let previousMatchEntry = null; + let previousEndIndex = null; + + for (const {0: matchText, index: startIndex, groups: matchEntry} + of sourceText.matchAll(commentaryRegexCaseSensitive)) { + if (previousMatchEntry) { + previousMatchEntry.body = sourceText.slice(previousEndIndex, startIndex); + } + + matchEntries.push(matchEntry); + + previousMatchEntry = matchEntry; + previousEndIndex = startIndex + matchText.length; + } + + if (previousMatchEntry) { + previousMatchEntry.body = sourceText.slice(previousEndIndex); + } + + return matchEntries; +} + export function filterAlbumsByCommentary(albums) { return albums .filter((album) => [album, ...album.tracks].some((x) => x.commentary)); @@ -167,10 +197,10 @@ export function getFlashLink(flash) { } export function getTotalDuration(tracks, { - originalReleasesOnly = false, + mainReleasesOnly = false, } = {}) { - if (originalReleasesOnly) { - tracks = tracks.filter(t => !t.originalReleaseTrack); + if (mainReleasesOnly) { + tracks = tracks.filter(t => !t.mainReleaseTrack); } return accumulateSum(tracks, track => track.duration); @@ -192,6 +222,25 @@ export function getArtistAvatar(artist, {to}) { return to('media.artistAvatar', artist.directory, artist.avatarFileExtension); } +// Used in multiple content functions for the artist info page, +// because shared logic is torture oooooooooooooooo. +export function chunkArtistTrackContributions(contributions) { + return ( + // First chunk by (contribution) date and album. + chunkByConditions(contributions, [ + ({date: date1}, {date: date2}) => + +date1 !== +date2, + ({thing: track1}, {thing: track2}) => + track1.album !== track2.album, + ]).map(contribs => + // Then, *within* the boundaries of the existing chunks, + // chunk contributions to the same thing together. + chunkByConditions(contribs, [ + ({thing: thing1}, {thing: thing2}) => + thing1 !== thing2, + ]))); +} + // Big-ass homepage row functions export function getNewAdditions(numAlbums, {albumData}) { @@ -342,7 +391,7 @@ export function filterItemsForCarousel(items) { return items .filter(item => item.hasCoverArt) - .filter(item => item.artTags.every(tag => !tag.isContentWarning)) + .filter(item => item.artTags.every(artTag => !artTag.isContentWarning)) .slice(0, maxCarouselLayoutItems + 1); } @@ -473,3 +522,30 @@ export function combineWikiDataArrays(arrays) { return combined; } } + +// Markdown stuff + +export function* matchMarkdownLinks(markdownSource, {marked}) { + const plausibleLinkRegexp = /\[.*?\)/g; + + let plausibleMatch = null; + while (plausibleMatch = plausibleLinkRegexp.exec(markdownSource)) { + // Pedantic rules use more particular parentheses detection in link + // destinations - they allow one level of balanced parentheses, and + // otherwise, parentheses must be escaped. This allows for entire links + // to be wrapped in parentheses, e.g below: + // + // This is so cool. ([You know??](https://example.com)) + // + const definiteMatch = + marked.Lexer.rules.inline.pedantic.link + .exec(markdownSource.slice(plausibleMatch.index)); + + if (definiteMatch) { + const [{length}, label, href] = definiteMatch; + const index = plausibleMatch.index + definiteMatch.index; + + yield {label, href, index, length}; + } + } +} |