search: refactor search spec definition & interfaces

5 files changed, 189 insertions, 185 deletions

diff --git a/package.json b/package.json
index a08be2dd..c8f96e98 100644
--- a/package.json
+++ b/package.json
@@ -39,6 +39,7 @@
         "#repl": "./src/write/build-modes/repl.js",
         "#replacer": "./src/util/replacer.js",
         "#search": "./src/search.js",
+        "#search-spec": "./src/util/search-spec.js",
         "#serialize": "./src/data/serialize.js",
         "#sort": "./src/util/sort.js",
         "#sugar": "./src/util/sugar.js",
diff --git a/src/search.js b/src/search.js
index dd9c0b2f..5524344f 100644
--- a/src/search.js
+++ b/src/search.js
@@ -6,143 +6,17 @@ import * as path from 'node:path';
 import FlexSearch from 'flexsearch';
 
 import {logError, logInfo, logWarn} from '#cli';
-import Thing from '#thing';
-
-import {makeSearchIndexes} from './util/searchSchema.js';
-
-const DEBUG_DOC_GEN = true;
-
-async function populateSearchIndexes(indexes, wikiData) {
-
-  const haveLoggedDocOfThing = {}; // debugging only
-
-  function readCollectionIntoIndex(
-    collection,
-    index,
-    mapper
-  ) {
-    // Add a doc for mapper(thing) to index for each thing in collection.
-    for (const thing of collection) {
-      const reference = Thing.getReference(thing);
-
-      // Get mapped fields from thing
-      let mappedResult;
-      try {
-        mappedResult = mapper(thing);
-      } catch (e) {
-        // Enrich error context
-        logError`Failed to write searchable doc for thing ${reference}`;
-        const thingSchemaSummary = Object.fromEntries(
-          Object.entries(thing)
-          .map(([k, v]) => [k, v ? (v.constructor.name || typeof v) : v])
-        );
-        logError("Availible properties: " + JSON.stringify(thingSchemaSummary, null, 2));
-        throw e;
-      }
-
-      // Build doc and add to index
-      const doc = {
-        reference,
-        ...mappedResult
-      }
-      // Print description of an output doc, if debugging enabled.
-      if (DEBUG_DOC_GEN && !haveLoggedDocOfThing[thing.constructor.name]) {
-        logInfo(JSON.stringify(doc, null, 2));
-        haveLoggedDocOfThing[thing.constructor.name] = true;
-      }
-      index.add(doc);
-    }
-  }
-
-  // Albums
-  readCollectionIntoIndex(
-    wikiData.albumData,
-    indexes.albums,
-    album => ({
-      name: album.name,
-      groups: album.groups.map(group => group.name),
-    })
-  );
-
-  // Tracks
-  readCollectionIntoIndex(
-    wikiData.trackData,
-    indexes.tracks,
-    track => ({
-      name: track.name,
-      color: track.color,
-      album: track.album.name,
-      albumDirectory: track.album.directory,
-
-      artists: [
-        track.artistContribs.map(contrib => contrib.artist.name),
-        ...track.artistContribs.map(contrib => contrib.artist.aliasNames)
-      ].flat(),
-
-      additionalNames: track.additionalNames.map(entry => entry.name),
-
-      artworkKind:
-        (track.hasUniqueCoverArt
-          ? 'track'
-       : track.album.hasCoverArt
-          ? 'album'
-          : 'none'),
-    })
-  );
-
-  // Artists
-  const realArtists =
-    wikiData.artistData
-      .filter(artist => !artist.isAlias);
-
-  readCollectionIntoIndex(
-    realArtists,
-    indexes.artists,
-    artist => ({
-      names: [artist.name, ...artist.aliasNames],
-    })
-  );
-
-  // Groups
-  readCollectionIntoIndex(
-    wikiData.groupData,
-    indexes.groups,
-    group => ({
-      names: group.name,
-      description: group.description,
-      // category: group.category
-    })
-  );
-
-  // Flashes
-  readCollectionIntoIndex(
-    wikiData.flashData,
-    indexes.flashes,
-    flash => ({
-      name: flash.name,
-      tracks: flash.featuredTracks.map(track => track.name),
-      contributors: [
-        flash.contributorContribs.map(contrib => contrib.artist.name),
-        ...flash.contributorContribs.map(contrib => contrib.artist.aliasNames)
-      ].flat()
-    })
-  );
-}
+import {makeSearchIndex, populateSearchIndex, searchSpec} from '#search-spec';
+import {stitchArrays} from '#sugar';
 
-async function exportIndexesToJson(indexes) {
-  const searchData = {};
+async function exportIndexToJSON(index) {
+  const results = {};
 
-  // Map each index to an export promise, and await all.
-  await Promise.all(
-    Object.entries(indexes)
-      .map(([indexName, index]) => {
-        searchData[indexName] = {};
-        return index.export((key, data) => {
-          searchData[indexName][key] = data;
-        });
-      }));
+  await index.export((key, data) => {
+    results[key] = data;
+  })
 
-  return searchData;
+  return results;
 }
 
 export async function writeSearchData({
@@ -158,11 +32,32 @@ export async function writeSearchData({
   // 2. Add documents to index
   // 3. Save index to exportable json
 
-  const indexes = makeSearchIndexes(FlexSearch);
+  const keys =
+    Object.keys(searchSpec);
+
+  const descriptors =
+    Object.values(searchSpec);
+
+  const indexes =
+    descriptors
+      .map(descriptor =>
+        makeSearchIndex(descriptor, {FlexSearch}));
+
+  stitchArrays({
+    index: indexes,
+    descriptor: descriptors,
+  }).forEach(({index, descriptor}) =>
+      populateSearchIndex(index, descriptor, {wikiData}));
 
-  await populateSearchIndexes(indexes, wikiData);
+  const jsonIndexes =
+    await Promise.all(indexes.map(exportIndexToJSON));
 
-  const searchData = await exportIndexesToJson(indexes);
+  const searchData =
+    Object.fromEntries(
+      stitchArrays({
+        key: keys,
+        value: jsonIndexes,
+      }).map(({key, value}) => [key, value]));
 
   const outputDirectory =
     path.join(wikiCachePath, 'search');
diff --git a/src/static/js/search-worker.js b/src/static/js/search-worker.js
index 166be2a2..0b3c8cc5 100644
--- a/src/static/js/search-worker.js
+++ b/src/static/js/search-worker.js
@@ -1,4 +1,4 @@
-import {makeSearchIndexes} from '../shared-util/searchSchema.js';
+import {makeSearchIndex, searchSpec} from '../shared-util/search-spec.js';
 import {withEntries} from '../shared-util/sugar.js';
 
 import FlexSearch from '../lib/flexsearch/flexsearch.bundle.module.min.js';
@@ -21,7 +21,11 @@ main().then(
 
 async function main() {
   indexes =
-    makeSearchIndexes(FlexSearch);
+    withEntries(searchSpec, entries => entries
+      .map(([key, descriptor]) => [
+        key,
+        makeSearchIndex(descriptor, {FlexSearch}),
+      ]));
 
   searchData =
     await fetch('/search-data/index.json')
diff --git a/src/util/search-spec.js b/src/util/search-spec.js
new file mode 100644
index 00000000..b26869a2
--- /dev/null
+++ b/src/util/search-spec.js
@@ -0,0 +1,150 @@
+// Index structures shared by client and server, and relevant interfaces.
+
+export const searchSpec = {
+  albums: {
+    query: ({albumData}) => albumData,
+
+    process: (album) => ({
+      name:
+        album.name,
+
+      groups:
+        album.groups.map(group => group.name),
+    }),
+
+    index: [
+      'name',
+      'groups',
+    ],
+  },
+
+  tracks: {
+    query: ({trackData}) => trackData,
+
+    process: (track) => ({
+      name:
+        track.name,
+
+      color:
+        track.color,
+
+      album:
+        track.album.name,
+
+      albumDirectory:
+        track.album.directory,
+
+      artists:
+        track.artistContribs
+          .map(contrib => contrib.artist)
+          .flatMap(artist => [artist.name, ...artist.aliasNames]),
+
+      additionalNames:
+        track.additionalNames
+          .map(entry => entry.name),
+
+      artworkKind:
+        (track.hasUniqueCoverArt
+          ? 'track'
+       : track.album.hasCoverArt
+          ? 'album'
+          : 'none'),
+    }),
+
+    index: [
+      'name',
+      'album',
+      'artists',
+      'additionalNames',
+    ],
+
+    store: [
+      'color',
+      'name',
+      'albumDirectory',
+      'artworkKind',
+    ],
+  },
+
+  artists: {
+    query: ({artistData}) =>
+      artistData
+        .filter(artist => !artist.isAlias),
+
+    process: (artist) => ({
+      names:
+        [artist.name, ...artist.aliasNames],
+    }),
+
+    index: [
+      'names',
+    ],
+  },
+
+  groups: {
+    query: ({groupData}) => groupData,
+
+    process: (group) => ({
+      names: group.name,
+      description: group.description,
+      // category: group.category
+    }),
+
+    index: [
+      'name',
+      'description',
+      // 'category',
+    ],
+  },
+
+  flashes: {
+    query: ({flashData}) => flashData,
+
+    process: (flash) => ({
+      name:
+        flash.name,
+
+      tracks:
+        flash.featuredTracks
+          .map(track => track.name),
+
+      contributors:
+        flash.contributorContribs
+          .map(contrib => contrib.artist)
+          .flatMap(artist => [artist.name, ...artist.aliasNames]),
+    }),
+
+    index: [
+      'name',
+      'tracks',
+      'contributors',
+    ],
+  },
+};
+
+export function makeSearchIndex(descriptor, {FlexSearch}) {
+  return new FlexSearch.Document({
+    id: 'reference',
+    index: descriptor.index,
+    store: descriptor.store,
+  });
+}
+
+export function populateSearchIndex(index, descriptor, {wikiData}) {
+  const collection = descriptor.query(wikiData);
+
+  for (const thing of collection) {
+    const reference = thing.constructor.getReference(thing);
+
+    let processed;
+    try {
+      processed = descriptor.process(thing);
+    } catch (caughtError) {
+      throw new Error(
+        `Failed to process searchable thing ${reference}`,
+        {cause: caughtError});
+    }
+
+    index.add({reference, ...processed});
+  }
+}
diff --git a/src/util/searchSchema.js b/src/util/searchSchema.js
deleted file mode 100644
index dffd1c1f..00000000
--- a/src/util/searchSchema.js
+++ /dev/null
@@ -1,46 +0,0 @@
-// Index structures shared by client and server.
-
-export function makeSearchIndexes(FlexSearch, documentOptions = {}) {
-  const doc = documentSchema =>
-    new FlexSearch.Document({
-      id: 'reference',
-      ...documentOptions,
-      ...documentSchema,
-    });
-
-  const indexes = {
-    albums: doc({
-      index: ['name', 'groups'],
-    }),
-
-    tracks: doc({
-      index: [
-        'name',
-        'album',
-        'artists',
-        'additionalNames',
-      ],
-
-      store: [
-        'color',
-        'name',
-        'albumDirectory',
-        'artworkKind',
-      ],
-    }),
-
-    artists: doc({
-      index: ['names'],
-    }),
-
-    groups: doc({
-      index: ['name', 'description', 'category'],
-    }),
-
-    flashes: doc({
-      index: ['name', 'tracks', 'contributors'],
-    }),
-  };
-
-  return indexes;
-}