From 9be32e448e65efeef59fa1ed6c2f4190c86d83d4 Mon Sep 17 00:00:00 2001 From: "(quasar) nebula" Date: Wed, 8 Oct 2025 20:52:28 -0300 Subject: search: query -> select, factor out backend parts of searchSpec --- package.json | 3 +- src/common-util/search-shape.js | 104 ++++++++++++++ src/common-util/search-spec.js | 292 ---------------------------------------- src/search-select.js | 213 +++++++++++++++++++++++++++++ src/search.js | 3 +- src/static/js/search-worker.js | 3 +- 6 files changed, 323 insertions(+), 295 deletions(-) create mode 100644 src/common-util/search-shape.js delete mode 100644 src/common-util/search-spec.js create mode 100644 src/search-select.js diff --git a/package.json b/package.json index 7ad8fd07..7ec7fafb 100644 --- a/package.json +++ b/package.json @@ -45,7 +45,8 @@ "#replacer": "./src/replacer.js", "#reverse": "./src/reverse.js", "#search": "./src/search.js", - "#search-spec": "./src/common-util/search-spec.js", + "#search-shape": "./src/common-util/search-shape.js", + "#search-select": "./src/search-select.js", "#serialize": "./src/data/serialize.js", "#sort": "./src/common-util/sort.js", "#sugar": "./src/common-util/sugar.js", diff --git a/src/common-util/search-shape.js b/src/common-util/search-shape.js new file mode 100644 index 00000000..7f81a089 --- /dev/null +++ b/src/common-util/search-shape.js @@ -0,0 +1,104 @@ +// Index structures shared by client and server, and relevant interfaces. +// First and foremost, this is complemented by src/search-select.js, which +// actually fills the search indexes up with stuff. During build this all +// gets consumed by src/search.js to make an index, fill it with stuff +// (as described by search-select.js), and export it to disk; then on +// the client that export is consumed by src/static/js/search-worker.js, +// which builds an index in the same shape and imports the data for query. + +const baselineStore = [ + 'primaryName', + 'disambiguator', + 'artwork', + 'color', +]; + +const genericStore = baselineStore; + +const searchShape = { + generic: { + index: [ + 'primaryName', + 'parentName', + 'artTags', + 'additionalNames', + 'contributors', + 'groups', + ].map(field => ({field, tokenize: 'forward'})), + + store: genericStore, + }, + + verbatim: { + index: [ + 'primaryName', + 'parentName', + 'artTags', + 'additionalNames', + 'contributors', + 'groups', + ], + + store: genericStore, + }, +}; + +export default searchShape; + +export function makeSearchIndex(descriptor, {FlexSearch}) { + return new FlexSearch.Document({ + id: 'reference', + index: descriptor.index, + store: descriptor.store, + + // Disable scoring, always return results according to provided order + // (specified above in `genericQuery`, etc). + resolution: 1, + }); +} + +// TODO: This function basically mirrors bind-utilities.js, which isn't +// exactly robust, but... binding might need some more thought across the +// codebase in *general.* +function bindSearchUtilities({ + checkIfImagePathHasCachedThumbnails, + getThumbnailEqualOrSmaller, + thumbsCache, + urls, +}) { + // TODO: :boom: + + const bound = { + urls, + }; + + bound.checkIfImagePathHasCachedThumbnails = + (imagePath) => + checkIfImagePathHasCachedThumbnails(imagePath, thumbsCache); + + bound.getThumbnailEqualOrSmaller = + (preferred, imagePath) => + getThumbnailEqualOrSmaller(preferred, imagePath, thumbsCache); + + return bound; +} + +export function populateSearchIndex(index, descriptor, opts) { + const {wikiData} = opts; + const bound = bindSearchUtilities(opts); + + for (const thing of descriptor.select(wikiData)) { + const reference = thing.constructor.getReference(thing); + + let processed; + try { + processed = descriptor.process(thing, bound); + } catch (caughtError) { + throw new Error( + `Failed to process searchable thing ${reference}`, + {cause: caughtError}); + } + + index.add({reference, ...processed}); + } +} diff --git a/src/common-util/search-spec.js b/src/common-util/search-spec.js deleted file mode 100644 index 731e5495..00000000 --- a/src/common-util/search-spec.js +++ /dev/null @@ -1,292 +0,0 @@ -// Index structures shared by client and server, and relevant interfaces. - -function prepareArtwork(artwork, thing, { - checkIfImagePathHasCachedThumbnails, - getThumbnailEqualOrSmaller, - urls, -}) { - if (!artwork) { - return undefined; - } - - const hasWarnings = - artwork.artTags?.some(artTag => artTag.isContentWarning); - - const artworkPath = - artwork.path; - - if (!artworkPath) { - return undefined; - } - - const mediaSrc = - urls - .from('media.root') - .to(...artworkPath); - - if (!checkIfImagePathHasCachedThumbnails(mediaSrc)) { - return undefined; - } - - const selectedSize = - getThumbnailEqualOrSmaller( - (hasWarnings ? 'mini' : 'adorb'), - mediaSrc); - - const mediaSrcJpeg = - mediaSrc.replace(/\.(png|jpg)$/, `.${selectedSize}.jpg`); - - const displaySrc = - urls - .from('thumb.root') - .to('thumb.path', mediaSrcJpeg); - - const serializeSrc = - displaySrc.replace(thing.directory, '<>'); - - return serializeSrc; -} - -function baselineProcess(thing, opts) { - const fields = {}; - - fields.primaryName = - thing.name; - - fields.artwork = - null; - - fields.color = - thing.color; - - fields.disambiguator = - null; - - return fields; -} - -const baselineStore = [ - 'primaryName', - 'disambiguator', - 'artwork', - 'color', -]; - -function genericQuery(wikiData) { - const groupOrder = - wikiData.wikiInfo.divideTrackListsByGroups; - - const getGroupRank = thing => { - const relevantRanks = - Array.from(groupOrder.entries()) - .filter(({1: group}) => thing.groups.includes(group)) - .map(({0: index}) => index); - - if (relevantRanks.length === 0) { - return Infinity; - } else if (relevantRanks.length === 1) { - return relevantRanks[0]; - } else { - return relevantRanks[0] + 0.5; - } - } - - const sortByGroupRank = things => - things.sort((a, b) => getGroupRank(a) - getGroupRank(b)); - - return [ - sortByGroupRank(wikiData.albumData.slice()), - - wikiData.artTagData, - - wikiData.artistData - .filter(artist => !artist.isAlias), - - wikiData.flashData, - - wikiData.groupData, - - sortByGroupRank( - wikiData.trackData - .filter(track => !track.mainReleaseTrack)), - ].flat(); -} - -function genericProcess(thing, opts) { - const fields = baselineProcess(thing, opts); - - const kind = - thing.constructor[Symbol.for('Thing.referenceType')]; - - const boundPrepareArtwork = artwork => - prepareArtwork(artwork, thing, opts); - - fields.artwork = - (kind === 'track' && thing.hasUniqueCoverArt - ? boundPrepareArtwork(thing.trackArtworks[0]) - : kind === 'track' - ? boundPrepareArtwork(thing.album.coverArtworks[0]) - : kind === 'album' - ? boundPrepareArtwork(thing.coverArtworks[0]) - : kind === 'flash' - ? boundPrepareArtwork(thing.coverArtwork) - : null); - - fields.parentName = - (kind === 'track' - ? thing.album.name - : kind === 'group' - ? thing.category.name - : kind === 'flash' - ? thing.act.name - : null); - - fields.disambiguator = - fields.parentName; - - fields.artTags = - (Array.from(new Set( - (kind === 'track' - ? thing.trackArtworks.flatMap(artwork => artwork.artTags) - : kind === 'album' - ? thing.coverArtworks.flatMap(artwork => artwork.artTags) - : [])))) - - .map(artTag => artTag.nameShort); - - fields.additionalNames = - (thing.constructor.hasPropertyDescriptor('additionalNames') - ? thing.additionalNames.map(entry => entry.name) - : thing.constructor.hasPropertyDescriptor('aliasNames') - ? thing.aliasNames - : []); - - const contribKeys = [ - 'artistContribs', - 'contributorContribs', - ]; - - const contributions = - contribKeys - .filter(key => Object.hasOwn(thing, key)) - .flatMap(key => thing[key]); - - fields.contributors = - contributions - .flatMap(({artist}) => [ - artist.name, - ...artist.aliasNames, - ]); - - const groups = - (Object.hasOwn(thing, 'groups') - ? thing.groups - : Object.hasOwn(thing, 'album') - ? thing.album.groups - : []); - - const mainContributorNames = - contributions - .map(({artist}) => artist.name); - - fields.groups = - groups - .filter(group => !mainContributorNames.includes(group.name)) - .map(group => group.name); - - return fields; -} - -const genericStore = baselineStore; - -export const searchSpec = { - generic: { - query: genericQuery, - process: genericProcess, - - index: [ - 'primaryName', - 'parentName', - 'artTags', - 'additionalNames', - 'contributors', - 'groups', - ].map(field => ({field, tokenize: 'forward'})), - - store: genericStore, - }, - - verbatim: { - query: genericQuery, - process: genericProcess, - - index: [ - 'primaryName', - 'parentName', - 'artTags', - 'additionalNames', - 'contributors', - 'groups', - ], - - store: genericStore, - }, -}; - -export function makeSearchIndex(descriptor, {FlexSearch}) { - return new FlexSearch.Document({ - id: 'reference', - index: descriptor.index, - store: descriptor.store, - - // Disable scoring, always return results according to provided order - // (specified above in `genericQuery`, etc). - resolution: 1, - }); -} - -// TODO: This function basically mirrors bind-utilities.js, which isn't -// exactly robust, but... binding might need some more thought across the -// codebase in *general.* -function bindSearchUtilities({ - checkIfImagePathHasCachedThumbnails, - getThumbnailEqualOrSmaller, - thumbsCache, - urls, -}) { - const bound = { - urls, - }; - - bound.checkIfImagePathHasCachedThumbnails = - (imagePath) => - checkIfImagePathHasCachedThumbnails(imagePath, thumbsCache); - - bound.getThumbnailEqualOrSmaller = - (preferred, imagePath) => - getThumbnailEqualOrSmaller(preferred, imagePath, thumbsCache); - - return bound; -} - -export function populateSearchIndex(index, descriptor, opts) { - const {wikiData} = opts; - const bound = bindSearchUtilities(opts); - - const collection = descriptor.query(wikiData); - - for (const thing of collection) { - const reference = thing.constructor.getReference(thing); - - let processed; - try { - processed = descriptor.process(thing, bound); - } catch (caughtError) { - throw new Error( - `Failed to process searchable thing ${reference}`, - {cause: caughtError}); - } - - index.add({reference, ...processed}); - } -} diff --git a/src/search-select.js b/src/search-select.js new file mode 100644 index 00000000..e7372ad4 --- /dev/null +++ b/src/search-select.js @@ -0,0 +1,213 @@ +// Complements the specs in search-shape.js with the functions that actually +// process live wiki data into records that are appropriate for storage. +// These files totally go together, so read them side by side, okay? + +import baseSearchSpec from '#search-shape'; + +function prepareArtwork(artwork, thing, { + checkIfImagePathHasCachedThumbnails, + getThumbnailEqualOrSmaller, + urls, +}) { + if (!artwork) { + return undefined; + } + + const hasWarnings = + artwork.artTags?.some(artTag => artTag.isContentWarning); + + const artworkPath = + artwork.path; + + if (!artworkPath) { + return undefined; + } + + const mediaSrc = + urls + .from('media.root') + .to(...artworkPath); + + if (!checkIfImagePathHasCachedThumbnails(mediaSrc)) { + return undefined; + } + + const selectedSize = + getThumbnailEqualOrSmaller( + (hasWarnings ? 'mini' : 'adorb'), + mediaSrc); + + const mediaSrcJpeg = + mediaSrc.replace(/\.(png|jpg)$/, `.${selectedSize}.jpg`); + + const displaySrc = + urls + .from('thumb.root') + .to('thumb.path', mediaSrcJpeg); + + const serializeSrc = + displaySrc.replace(thing.directory, '<>'); + + return serializeSrc; +} + +function baselineProcess(thing, opts) { + const fields = {}; + + fields.primaryName = + thing.name; + + fields.artwork = + null; + + fields.color = + thing.color; + + fields.disambiguator = + null; + + return fields; +} + +function genericSelect(wikiData) { + const groupOrder = + wikiData.wikiInfo.divideTrackListsByGroups; + + const getGroupRank = thing => { + const relevantRanks = + Array.from(groupOrder.entries()) + .filter(({1: group}) => thing.groups.includes(group)) + .map(({0: index}) => index); + + if (relevantRanks.length === 0) { + return Infinity; + } else if (relevantRanks.length === 1) { + return relevantRanks[0]; + } else { + return relevantRanks[0] + 0.5; + } + } + + const sortByGroupRank = things => + things.sort((a, b) => getGroupRank(a) - getGroupRank(b)); + + return [ + sortByGroupRank(wikiData.albumData.slice()), + + wikiData.artTagData, + + wikiData.artistData + .filter(artist => !artist.isAlias), + + wikiData.flashData, + + wikiData.groupData, + + sortByGroupRank( + wikiData.trackData + .filter(track => track.isMainRelease)), + ].flat(); +} + +function genericProcess(thing, opts) { + const fields = baselineProcess(thing, opts); + + const kind = + thing.constructor[Symbol.for('Thing.referenceType')]; + + const boundPrepareArtwork = artwork => + prepareArtwork(artwork, thing, opts); + + fields.artwork = + (kind === 'track' && thing.hasUniqueCoverArt + ? boundPrepareArtwork(thing.trackArtworks[0]) + : kind === 'track' + ? boundPrepareArtwork(thing.album.coverArtworks[0]) + : kind === 'album' + ? boundPrepareArtwork(thing.coverArtworks[0]) + : kind === 'flash' + ? boundPrepareArtwork(thing.coverArtwork) + : null); + + fields.parentName = + (kind === 'track' + ? thing.album.name + : kind === 'group' + ? thing.category.name + : kind === 'flash' + ? thing.act.name + : null); + + fields.disambiguator = + fields.parentName; + + fields.artTags = + (Array.from(new Set( + (kind === 'track' + ? thing.trackArtworks.flatMap(artwork => artwork.artTags) + : kind === 'album' + ? thing.coverArtworks.flatMap(artwork => artwork.artTags) + : [])))) + + .map(artTag => artTag.nameShort); + + fields.additionalNames = + (thing.constructor.hasPropertyDescriptor('additionalNames') + ? thing.additionalNames.map(entry => entry.name) + : thing.constructor.hasPropertyDescriptor('aliasNames') + ? thing.aliasNames + : []); + + const contribKeys = [ + 'artistContribs', + 'contributorContribs', + ]; + + const contributions = + contribKeys + .filter(key => Object.hasOwn(thing, key)) + .flatMap(key => thing[key]); + + fields.contributors = + contributions + .flatMap(({artist}) => [ + artist.name, + ...artist.aliasNames, + ]); + + const groups = + (Object.hasOwn(thing, 'groups') + ? thing.groups + : Object.hasOwn(thing, 'album') + ? thing.album.groups + : []); + + const mainContributorNames = + contributions + .map(({artist}) => artist.name); + + fields.groups = + groups + .filter(group => !mainContributorNames.includes(group.name)) + .map(group => group.name); + + return fields; +} + +const spiffySearchSpec = { + generic: { + ...baseSearchSpec.generic, + + select: genericSelect, + process: genericProcess, + }, + + verbatim: { + ...baseSearchSpec.verbatim, + + select: genericSelect, + process: genericProcess, + }, +}; + +export default spiffySearchSpec; diff --git a/src/search.js b/src/search.js index a2dae9e1..5d4c7331 100644 --- a/src/search.js +++ b/src/search.js @@ -9,7 +9,8 @@ import FlexSearch from 'flexsearch'; import {pack} from 'msgpackr'; import {logWarn} from '#cli'; -import {makeSearchIndex, populateSearchIndex, searchSpec} from '#search-spec'; +import {makeSearchIndex, populateSearchIndex} from '#search-shape'; +import searchSpec from '#search-select'; import {stitchArrays} from '#sugar'; import {checkIfImagePathHasCachedThumbnails, getThumbnailEqualOrSmaller} from '#thumbs'; diff --git a/src/static/js/search-worker.js b/src/static/js/search-worker.js index 3e9fbfca..387cbca0 100644 --- a/src/static/js/search-worker.js +++ b/src/static/js/search-worker.js @@ -2,7 +2,8 @@ import FlexSearch from '../lib/flexsearch/flexsearch.bundle.module.min.js'; -import {makeSearchIndex, searchSpec} from '../shared-util/search-spec.js'; +import {default as searchSpec, makeSearchIndex} + from '../shared-util/search-shape.js'; import { empty, -- cgit 1.3.0-6-gf8a5