diff options
Diffstat (limited to 'src/common-util')
| -rw-r--r-- | src/common-util/search-spec.js | 329 | ||||
| -rw-r--r-- | src/common-util/sort.js | 9 | ||||
| -rw-r--r-- | src/common-util/sugar.js | 112 | ||||
| -rw-r--r-- | src/common-util/wiki-data.js | 75 |
4 files changed, 337 insertions, 188 deletions
diff --git a/src/common-util/search-spec.js b/src/common-util/search-spec.js index 75de0d16..731e5495 100644 --- a/src/common-util/search-spec.js +++ b/src/common-util/search-spec.js @@ -1,57 +1,19 @@ // Index structures shared by client and server, and relevant interfaces. -function getArtworkPath(thing) { - switch (thing.constructor[Symbol.for('Thing.referenceType')]) { - case 'album': { - return [ - 'media.albumCover', - thing.directory, - thing.coverArtFileExtension, - ]; - } - - case 'flash': { - return [ - 'media.flashArt', - thing.directory, - thing.coverArtFileExtension, - ]; - } - - case 'track': { - if (thing.hasUniqueCoverArt) { - return [ - 'media.trackCover', - thing.album.directory, - thing.directory, - thing.coverArtFileExtension, - ]; - } else if (thing.album.hasCoverArt) { - return [ - 'media.albumCover', - thing.album.directory, - thing.album.coverArtFileExtension, - ]; - } else { - return null; - } - } - - default: - return null; - } -} - -function prepareArtwork(thing, { +function prepareArtwork(artwork, thing, { checkIfImagePathHasCachedThumbnails, getThumbnailEqualOrSmaller, urls, }) { + if (!artwork) { + return undefined; + } + const hasWarnings = - thing.artTags?.some(artTag => artTag.isContentWarning); + artwork.artTags?.some(artTag => artTag.isContentWarning); const artworkPath = - getArtworkPath(thing); + artwork.path; if (!artworkPath) { return undefined; @@ -85,107 +47,162 @@ function prepareArtwork(thing, { return serializeSrc; } +function baselineProcess(thing, opts) { + const fields = {}; + + fields.primaryName = + thing.name; + + fields.artwork = + null; + + fields.color = + thing.color; + + fields.disambiguator = + null; + + return fields; +} + +const baselineStore = [ + 'primaryName', + 'disambiguator', + 'artwork', + 'color', +]; + +function genericQuery(wikiData) { + const groupOrder = + wikiData.wikiInfo.divideTrackListsByGroups; + + const getGroupRank = thing => { + const relevantRanks = + Array.from(groupOrder.entries()) + .filter(({1: group}) => thing.groups.includes(group)) + .map(({0: index}) => index); + + if (relevantRanks.length === 0) { + return Infinity; + } else if (relevantRanks.length === 1) { + return relevantRanks[0]; + } else { + return relevantRanks[0] + 0.5; + } + } + + const sortByGroupRank = things => + things.sort((a, b) => getGroupRank(a) - getGroupRank(b)); + + return [ + sortByGroupRank(wikiData.albumData.slice()), + + wikiData.artTagData, + + wikiData.artistData + .filter(artist => !artist.isAlias), + + wikiData.flashData, + + wikiData.groupData, + + sortByGroupRank( + wikiData.trackData + .filter(track => !track.mainReleaseTrack)), + ].flat(); +} + +function genericProcess(thing, opts) { + const fields = baselineProcess(thing, opts); + + const kind = + thing.constructor[Symbol.for('Thing.referenceType')]; + + const boundPrepareArtwork = artwork => + prepareArtwork(artwork, thing, opts); + + fields.artwork = + (kind === 'track' && thing.hasUniqueCoverArt + ? boundPrepareArtwork(thing.trackArtworks[0]) + : kind === 'track' + ? boundPrepareArtwork(thing.album.coverArtworks[0]) + : kind === 'album' + ? boundPrepareArtwork(thing.coverArtworks[0]) + : kind === 'flash' + ? boundPrepareArtwork(thing.coverArtwork) + : null); + + fields.parentName = + (kind === 'track' + ? thing.album.name + : kind === 'group' + ? thing.category.name + : kind === 'flash' + ? thing.act.name + : null); + + fields.disambiguator = + fields.parentName; + + fields.artTags = + (Array.from(new Set( + (kind === 'track' + ? thing.trackArtworks.flatMap(artwork => artwork.artTags) + : kind === 'album' + ? thing.coverArtworks.flatMap(artwork => artwork.artTags) + : [])))) + + .map(artTag => artTag.nameShort); + + fields.additionalNames = + (thing.constructor.hasPropertyDescriptor('additionalNames') + ? thing.additionalNames.map(entry => entry.name) + : thing.constructor.hasPropertyDescriptor('aliasNames') + ? thing.aliasNames + : []); + + const contribKeys = [ + 'artistContribs', + 'contributorContribs', + ]; + + const contributions = + contribKeys + .filter(key => Object.hasOwn(thing, key)) + .flatMap(key => thing[key]); + + fields.contributors = + contributions + .flatMap(({artist}) => [ + artist.name, + ...artist.aliasNames, + ]); + + const groups = + (Object.hasOwn(thing, 'groups') + ? thing.groups + : Object.hasOwn(thing, 'album') + ? thing.album.groups + : []); + + const mainContributorNames = + contributions + .map(({artist}) => artist.name); + + fields.groups = + groups + .filter(group => !mainContributorNames.includes(group.name)) + .map(group => group.name); + + return fields; +} + +const genericStore = baselineStore; + export const searchSpec = { generic: { - query: ({ - albumData, - artTagData, - artistData, - flashData, - groupData, - trackData, - }) => [ - albumData, - - artTagData, - - artistData - .filter(artist => !artist.isAlias), - - flashData, - - groupData, - - trackData - // Exclude rereleases - there's no reasonable way to differentiate - // them from the main release as part of this query. - .filter(track => !track.mainReleaseTrack), - ].flat(), - - process(thing, opts) { - const fields = {}; - - fields.primaryName = - thing.name; - - const kind = - thing.constructor[Symbol.for('Thing.referenceType')]; - - fields.parentName = - (kind === 'track' - ? thing.album.name - : kind === 'group' - ? thing.category.name - : kind === 'flash' - ? thing.act.name - : null); - - fields.color = - thing.color; - - fields.artTags = - (thing.constructor.hasPropertyDescriptor('artTags') - ? thing.artTags.map(artTag => artTag.nameShort) - : []); - - fields.additionalNames = - (thing.constructor.hasPropertyDescriptor('additionalNames') - ? thing.additionalNames.map(entry => entry.name) - : thing.constructor.hasPropertyDescriptor('aliasNames') - ? thing.aliasNames - : []); - - const contribKeys = [ - 'artistContribs', - 'bannerArtistContribs', - 'contributorContribs', - 'coverArtistContribs', - 'wallpaperArtistContribs', - ]; - - const contributions = - contribKeys - .filter(key => Object.hasOwn(thing, key)) - .flatMap(key => thing[key]); - - fields.contributors = - contributions - .flatMap(({artist}) => [ - artist.name, - ...artist.aliasNames, - ]); - - const groups = - (Object.hasOwn(thing, 'groups') - ? thing.groups - : Object.hasOwn(thing, 'album') - ? thing.album.groups - : []); - - const mainContributorNames = - contributions - .map(({artist}) => artist.name); - - fields.groups = - groups - .filter(group => !mainContributorNames.includes(group.name)) - .map(group => group.name); - - fields.artwork = - prepareArtwork(thing, opts); - - return fields; - }, + query: genericQuery, + process: genericProcess, index: [ 'primaryName', @@ -194,13 +211,25 @@ export const searchSpec = { 'additionalNames', 'contributors', 'groups', - ], + ].map(field => ({field, tokenize: 'forward'})), - store: [ + store: genericStore, + }, + + verbatim: { + query: genericQuery, + process: genericProcess, + + index: [ 'primaryName', - 'artwork', - 'color', + 'parentName', + 'artTags', + 'additionalNames', + 'contributors', + 'groups', ], + + store: genericStore, }, }; @@ -209,6 +238,10 @@ export function makeSearchIndex(descriptor, {FlexSearch}) { id: 'reference', index: descriptor.index, store: descriptor.store, + + // Disable scoring, always return results according to provided order + // (specified above in `genericQuery`, etc). + resolution: 1, }); } diff --git a/src/common-util/sort.js b/src/common-util/sort.js index d93d94c1..bbe4e551 100644 --- a/src/common-util/sort.js +++ b/src/common-util/sort.js @@ -370,11 +370,12 @@ export function sortAlbumsTracksChronologically(data, { getDate, } = {}) { // Sort albums before tracks... - sortByConditions(data, [(t) => t.album === undefined]); + sortByConditions(data, [t => t.isAlbum]); - // Group tracks by album... - sortByDirectory(data, { - getDirectory: (t) => (t.album ? t.album.directory : t.directory), + // Put albums alphabetically, and group with them... + sortAlphabetically(data, { + getDirectory: t => t.isTrack ? t.album.directory : t.directory, + getName: t => t.isTrack ? t.album.name : t.name, }); // Sort tracks by position in album... diff --git a/src/common-util/sugar.js b/src/common-util/sugar.js index 66e160aa..e931ad59 100644 --- a/src/common-util/sugar.js +++ b/src/common-util/sugar.js @@ -116,10 +116,14 @@ export function findIndexOrEnd(array, fn) { // returns null (or values in the array are nullish), they'll just be skipped in // the sum. export function accumulateSum(array, fn = x => x) { + if (!Array.isArray(array)) { + return accumulateSum(Array.from(array, fn)); + } + return array.reduce( (accumulator, value, index, array) => accumulator + - fn(value, index, array) ?? 0, + (fn(value, index, array) ?? 0), 0); } @@ -221,6 +225,9 @@ export const compareArrays = (arr1, arr2, {checkOrder = true} = {}) => ? arr1.every((x, i) => arr2[i] === x) : arr1.every((x) => arr2.includes(x))); +export const exhaust = (generatorFunction) => + Array.from(generatorFunction()); + export function compareObjects(obj1, obj2, { checkOrder = false, checkSymbols = true, @@ -251,11 +258,20 @@ export function compareObjects(obj1, obj2, { // Stolen from jq! Which pro8a8ly stole the concept from other places. Nice. export const withEntries = (obj, fn) => { - const result = fn(Object.entries(obj)); - if (result instanceof Promise) { - return result.then(entries => Object.fromEntries(entries)); + if (obj instanceof Map) { + const result = fn(Array.from(obj.entries())); + if (result instanceof Promise) { + return result.then(entries => new map(entries)); + } else { + return new Map(result); + } } else { - return Object.fromEntries(result); + const result = fn(Object.entries(obj)); + if (result instanceof Promise) { + return result.then(entries => Object.fromEntries(entries)); + } else { + return Object.fromEntries(result); + } } } @@ -299,34 +315,74 @@ export function filterProperties(object, properties, { return filteredObject; } -export function queue(array, max = 50) { - if (max === 0) { - return array.map((fn) => fn()); +export function queue(functionList, queueSize = 50) { + if (queueSize === 0) { + return functionList.map(fn => fn()); } - const begin = []; - let current = 0; - const ret = array.map( - (fn) => - new Promise((resolve, reject) => { - begin.push(() => { - current++; - Promise.resolve(fn()).then((value) => { - current--; - if (current < max && begin.length) { - begin.shift()(); - } - resolve(value); - }, reject); - }); - }) - ); + const promiseList = []; + const resolveList = []; + const rejectList = []; - for (let i = 0; i < max && begin.length; i++) { - begin.shift()(); + for (let i = 0; i < functionList.length; i++) { + const promiseWithResolvers = Promise.withResolvers(); + promiseList.push(promiseWithResolvers.promise); + resolveList.push(promiseWithResolvers.resolve); + rejectList.push(promiseWithResolvers.reject); } - return ret; + let cursor = 0; + let running = 0; + + const next = async () => { + if (running >= queueSize) { + return; + } + + if (cursor === functionList.length) { + return; + } + + const thisFunction = functionList[cursor]; + const thisResolve = resolveList[cursor]; + const thisReject = rejectList[cursor]; + + delete functionList[cursor]; + delete resolveList[cursor]; + delete rejectList[cursor]; + + cursor++; + running++; + + try { + thisResolve(await thisFunction()); + } catch (error) { + thisReject(error); + } finally { + running--; + + // If the cursor is at 1, this is the first promise that resolved, + // so we're now done the "kick start", and can start the remaining + // promises (up to queueSize). + if (cursor === 1) { + // Since only one promise is used for the "kick start", and that one + // has just resolved, we know there's none running at all right now, + // and can start as many as specified in the queueSize right away. + for (let i = 0; i < queueSize; i++) { + next(); + } + } else { + next(); + } + } + }; + + // Only start a single promise, as a "kick start", so that it resolves as + // early as possible (it will resolve before we use CPU to start the rest + // of the promises, up to queueSize). + next(); + + return promiseList; } export function delay(ms) { diff --git a/src/common-util/wiki-data.js b/src/common-util/wiki-data.js index a4c6b3bd..0f6591c1 100644 --- a/src/common-util/wiki-data.js +++ b/src/common-util/wiki-data.js @@ -34,6 +34,9 @@ export function getKebabCase(name) { // General punctuation which always separates surrounding words .replace(/[/@#$%*()_=,[\]{}|\\;:<>?`~]/g, '-') + // More punctuation which always separates surrounding words + .replace(/[\u{2013}-\u{2014}]/u, '-') // En Dash, Em Dash + // Accented characters .replace(/[áâäàå]/gi, 'a') .replace(/[çč]/gi, 'c') @@ -57,10 +60,9 @@ export function getKebabCase(name) { // Specific data utilities -// Matches heading details from commentary data in roughly the formats: +// Matches heading details from commentary data in roughly the format: // -// <i>artistReference:</i> (annotation, date) -// <i>artistReference|artistDisplayText:</i> (annotation, date) +// <i>artistText:</i> (annotation, date) // // where capturing group "annotation" can be any text at all, except that the // last entry (past a comma or the only content within parentheses), if parsed @@ -83,8 +85,9 @@ export function getKebabCase(name) { // parentheses can be part of the actual annotation content. // // Capturing group "artistReference" is all the characters between <i> and </i> -// (apart from the pipe and "artistDisplayText" text, if present), and is either -// the name of an artist or an "artist:directory"-style reference. +// (apart from the pipe and the "artistText" group, if present), and is either +// the name of one or more artist or "artist:directory"-style references, +// joined by commas, if multiple. // // This regular expression *doesn't* match bodies, which will need to be parsed // out of the original string based on the indices matched using this. @@ -94,7 +97,7 @@ const dateRegex = groupName => String.raw`(?<${groupName}>[a-zA-Z]+ [0-9]{1,2}, [0-9]{4,4}|[0-9]{1,2} [^,]*[0-9]{4,4}|[0-9]{1,4}[-/][0-9]{1,4}[-/][0-9]{1,4})`; const commentaryRegexRaw = - String.raw`^<i>(?<artistReferences>.+?)(?:\|(?<artistDisplayText>.+))?:<\/i>(?: \((?<annotation>(?:.*?(?=,|\)[^)]*$))*?)(?:,? ?(?:(?<dateKind>sometime|throughout|around) )?${dateRegex('date')}(?: ?- ?${dateRegex('secondDate')})?(?: (?<accessKind>captured|accessed) ${dateRegex('accessDate')})?)?\))?`; + String.raw`^<i>(?<artistText>.+?):<\/i>(?: \((?<annotation>(?:.*?(?=,|\)[^)]*$))*?)(?:,? ?(?:(?<dateKind>sometime|throughout|around) )?${dateRegex('date')}(?: ?- ?${dateRegex('secondDate')})?(?: (?<accessKind>captured|accessed) ${dateRegex('accessDate')})?)?\))?`; export const commentaryRegexCaseInsensitive = new RegExp(commentaryRegexRaw, 'gmi'); export const commentaryRegexCaseSensitive = @@ -113,10 +116,16 @@ export function matchContentEntries(sourceText) { let previousMatchEntry = null; let previousEndIndex = null; + const trimBody = body => + body + .replace(/^\n*/, '') + .replace(/\n*$/, ''); + for (const {0: matchText, index: startIndex, groups: matchEntry} of sourceText.matchAll(commentaryRegexCaseSensitive)) { if (previousMatchEntry) { - previousMatchEntry.body = sourceText.slice(previousEndIndex, startIndex); + previousMatchEntry.body = + trimBody(sourceText.slice(previousEndIndex, startIndex)); } matchEntries.push(matchEntry); @@ -126,7 +135,8 @@ export function matchContentEntries(sourceText) { } if (previousMatchEntry) { - previousMatchEntry.body = sourceText.slice(previousEndIndex); + previousMatchEntry.body = + trimBody(sourceText.slice(previousEndIndex)); } return matchEntries; @@ -522,3 +532,52 @@ export function combineWikiDataArrays(arrays) { return combined; } } + +// Markdown stuff + +export function* matchMarkdownLinks(markdownSource, {marked}) { + const plausibleLinkRegexp = /\[(?=.*?\))/g; + + // Pedantic rules use more particular parentheses detection in link + // destinations - they allow one level of balanced parentheses, and + // otherwise, parentheses must be escaped. This allows for entire links + // to be wrapped in parentheses, e.g below: + // + // This is so cool. ([You know??](https://example.com)) + // + const definiteLinkRegexp = marked.Lexer.rules.inline.pedantic.link; + + let plausibleMatch = null; + while (plausibleMatch = plausibleLinkRegexp.exec(markdownSource)) { + const definiteMatch = + definiteLinkRegexp.exec(markdownSource.slice(plausibleMatch.index)); + + if (!definiteMatch) { + continue; + } + + const [{length}, label, href] = definiteMatch; + const index = plausibleMatch.index + definiteMatch.index; + + yield {label, href, index, length}; + } +} + +export function* matchInlineLinks(source) { + const plausibleLinkRegexp = /\b[a-z]*:\/\/[^ ]*?(?=(?:[,.!?]*)(?:\s|$))/gm; + + let plausibleMatch = null; + while (plausibleMatch = plausibleLinkRegexp.exec(source)) { + const [href] = plausibleMatch; + const {index} = plausibleMatch; + const [{length}] = plausibleMatch; + + try { + new URL(href); + } catch { + continue; + } + + yield {href, length, index}; + } +} |