diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/search.js | 169 | ||||
| -rw-r--r-- | src/static/js/search-worker.js | 8 | ||||
| -rw-r--r-- | src/util/search-spec.js | 150 | ||||
| -rw-r--r-- | src/util/searchSchema.js | 46 |
4 files changed, 188 insertions, 185 deletions
diff --git a/src/search.js b/src/search.js index dd9c0b2f..5524344f 100644 --- a/src/search.js +++ b/src/search.js @@ -6,143 +6,17 @@ import * as path from 'node:path'; import FlexSearch from 'flexsearch'; import {logError, logInfo, logWarn} from '#cli'; -import Thing from '#thing'; - -import {makeSearchIndexes} from './util/searchSchema.js'; - -const DEBUG_DOC_GEN = true; - -async function populateSearchIndexes(indexes, wikiData) { - - const haveLoggedDocOfThing = {}; // debugging only - - function readCollectionIntoIndex( - collection, - index, - mapper - ) { - // Add a doc for mapper(thing) to index for each thing in collection. - for (const thing of collection) { - const reference = Thing.getReference(thing); - - // Get mapped fields from thing - let mappedResult; - try { - mappedResult = mapper(thing); - } catch (e) { - // Enrich error context - logError`Failed to write searchable doc for thing ${reference}`; - const thingSchemaSummary = Object.fromEntries( - Object.entries(thing) - .map(([k, v]) => [k, v ? (v.constructor.name || typeof v) : v]) - ); - logError("Availible properties: " + JSON.stringify(thingSchemaSummary, null, 2)); - throw e; - } - - // Build doc and add to index - const doc = { - reference, - ...mappedResult - } - // Print description of an output doc, if debugging enabled. - if (DEBUG_DOC_GEN && !haveLoggedDocOfThing[thing.constructor.name]) { - logInfo(JSON.stringify(doc, null, 2)); - haveLoggedDocOfThing[thing.constructor.name] = true; - } - index.add(doc); - } - } - - // Albums - readCollectionIntoIndex( - wikiData.albumData, - indexes.albums, - album => ({ - name: album.name, - groups: album.groups.map(group => group.name), - }) - ); - - // Tracks - readCollectionIntoIndex( - wikiData.trackData, - indexes.tracks, - track => ({ - name: track.name, - color: track.color, - album: track.album.name, - albumDirectory: track.album.directory, - - artists: [ - track.artistContribs.map(contrib => contrib.artist.name), - ...track.artistContribs.map(contrib => contrib.artist.aliasNames) - ].flat(), - - additionalNames: track.additionalNames.map(entry => entry.name), - - artworkKind: - (track.hasUniqueCoverArt - ? 'track' - : track.album.hasCoverArt - ? 'album' - : 'none'), - }) - ); - - // Artists - const realArtists = - wikiData.artistData - .filter(artist => !artist.isAlias); - - readCollectionIntoIndex( - realArtists, - indexes.artists, - artist => ({ - names: [artist.name, ...artist.aliasNames], - }) - ); - - // Groups - readCollectionIntoIndex( - wikiData.groupData, - indexes.groups, - group => ({ - names: group.name, - description: group.description, - // category: group.category - }) - ); - - // Flashes - readCollectionIntoIndex( - wikiData.flashData, - indexes.flashes, - flash => ({ - name: flash.name, - tracks: flash.featuredTracks.map(track => track.name), - contributors: [ - flash.contributorContribs.map(contrib => contrib.artist.name), - ...flash.contributorContribs.map(contrib => contrib.artist.aliasNames) - ].flat() - }) - ); -} +import {makeSearchIndex, populateSearchIndex, searchSpec} from '#search-spec'; +import {stitchArrays} from '#sugar'; -async function exportIndexesToJson(indexes) { - const searchData = {}; +async function exportIndexToJSON(index) { + const results = {}; - // Map each index to an export promise, and await all. - await Promise.all( - Object.entries(indexes) - .map(([indexName, index]) => { - searchData[indexName] = {}; - return index.export((key, data) => { - searchData[indexName][key] = data; - }); - })); + await index.export((key, data) => { + results[key] = data; + }) - return searchData; + return results; } export async function writeSearchData({ @@ -158,11 +32,32 @@ export async function writeSearchData({ // 2. Add documents to index // 3. Save index to exportable json - const indexes = makeSearchIndexes(FlexSearch); + const keys = + Object.keys(searchSpec); + + const descriptors = + Object.values(searchSpec); + + const indexes = + descriptors + .map(descriptor => + makeSearchIndex(descriptor, {FlexSearch})); + + stitchArrays({ + index: indexes, + descriptor: descriptors, + }).forEach(({index, descriptor}) => + populateSearchIndex(index, descriptor, {wikiData})); - await populateSearchIndexes(indexes, wikiData); + const jsonIndexes = + await Promise.all(indexes.map(exportIndexToJSON)); - const searchData = await exportIndexesToJson(indexes); + const searchData = + Object.fromEntries( + stitchArrays({ + key: keys, + value: jsonIndexes, + }).map(({key, value}) => [key, value])); const outputDirectory = path.join(wikiCachePath, 'search'); diff --git a/src/static/js/search-worker.js b/src/static/js/search-worker.js index 166be2a2..0b3c8cc5 100644 --- a/src/static/js/search-worker.js +++ b/src/static/js/search-worker.js @@ -1,4 +1,4 @@ -import {makeSearchIndexes} from '../shared-util/searchSchema.js'; +import {makeSearchIndex, searchSpec} from '../shared-util/search-spec.js'; import {withEntries} from '../shared-util/sugar.js'; import FlexSearch from '../lib/flexsearch/flexsearch.bundle.module.min.js'; @@ -21,7 +21,11 @@ main().then( async function main() { indexes = - makeSearchIndexes(FlexSearch); + withEntries(searchSpec, entries => entries + .map(([key, descriptor]) => [ + key, + makeSearchIndex(descriptor, {FlexSearch}), + ])); searchData = await fetch('/search-data/index.json') diff --git a/src/util/search-spec.js b/src/util/search-spec.js new file mode 100644 index 00000000..b26869a2 --- /dev/null +++ b/src/util/search-spec.js @@ -0,0 +1,150 @@ +// Index structures shared by client and server, and relevant interfaces. + +export const searchSpec = { + albums: { + query: ({albumData}) => albumData, + + process: (album) => ({ + name: + album.name, + + groups: + album.groups.map(group => group.name), + }), + + index: [ + 'name', + 'groups', + ], + }, + + tracks: { + query: ({trackData}) => trackData, + + process: (track) => ({ + name: + track.name, + + color: + track.color, + + album: + track.album.name, + + albumDirectory: + track.album.directory, + + artists: + track.artistContribs + .map(contrib => contrib.artist) + .flatMap(artist => [artist.name, ...artist.aliasNames]), + + additionalNames: + track.additionalNames + .map(entry => entry.name), + + artworkKind: + (track.hasUniqueCoverArt + ? 'track' + : track.album.hasCoverArt + ? 'album' + : 'none'), + }), + + index: [ + 'name', + 'album', + 'artists', + 'additionalNames', + ], + + store: [ + 'color', + 'name', + 'albumDirectory', + 'artworkKind', + ], + }, + + artists: { + query: ({artistData}) => + artistData + .filter(artist => !artist.isAlias), + + process: (artist) => ({ + names: + [artist.name, ...artist.aliasNames], + }), + + index: [ + 'names', + ], + }, + + groups: { + query: ({groupData}) => groupData, + + process: (group) => ({ + names: group.name, + description: group.description, + // category: group.category + }), + + index: [ + 'name', + 'description', + // 'category', + ], + }, + + flashes: { + query: ({flashData}) => flashData, + + process: (flash) => ({ + name: + flash.name, + + tracks: + flash.featuredTracks + .map(track => track.name), + + contributors: + flash.contributorContribs + .map(contrib => contrib.artist) + .flatMap(artist => [artist.name, ...artist.aliasNames]), + }), + + index: [ + 'name', + 'tracks', + 'contributors', + ], + }, +}; + +export function makeSearchIndex(descriptor, {FlexSearch}) { + return new FlexSearch.Document({ + id: 'reference', + index: descriptor.index, + store: descriptor.store, + }); +} + +export function populateSearchIndex(index, descriptor, {wikiData}) { + const collection = descriptor.query(wikiData); + + for (const thing of collection) { + const reference = thing.constructor.getReference(thing); + + let processed; + try { + processed = descriptor.process(thing); + } catch (caughtError) { + throw new Error( + `Failed to process searchable thing ${reference}`, + {cause: caughtError}); + } + + index.add({reference, ...processed}); + } +} diff --git a/src/util/searchSchema.js b/src/util/searchSchema.js deleted file mode 100644 index dffd1c1f..00000000 --- a/src/util/searchSchema.js +++ /dev/null @@ -1,46 +0,0 @@ -// Index structures shared by client and server. - -export function makeSearchIndexes(FlexSearch, documentOptions = {}) { - const doc = documentSchema => - new FlexSearch.Document({ - id: 'reference', - ...documentOptions, - ...documentSchema, - }); - - const indexes = { - albums: doc({ - index: ['name', 'groups'], - }), - - tracks: doc({ - index: [ - 'name', - 'album', - 'artists', - 'additionalNames', - ], - - store: [ - 'color', - 'name', - 'albumDirectory', - 'artworkKind', - ], - }), - - artists: doc({ - index: ['names'], - }), - - groups: doc({ - index: ['name', 'description', 'category'], - }), - - flashes: doc({ - index: ['name', 'tracks', 'contributors'], - }), - }; - - return indexes; -} |