search, client: write indexes to individual files

3 files changed, 65 insertions, 21 deletions

diff --git a/src/search.js b/src/search.js
index 33d5d838..687aa215 100644
--- a/src/search.js
+++ b/src/search.js
@@ -1,11 +1,12 @@
 'use strict';
 
+import {createHash} from 'node:crypto';
 import {mkdir, writeFile} from 'node:fs/promises';
 import * as path from 'node:path';
 
 import FlexSearch from 'flexsearch';
 
-import {logError, logInfo, logWarn} from '#cli';
+import {logWarn} from '#cli';
 import {makeSearchIndex, populateSearchIndex, searchSpec} from '#search-spec';
 import {stitchArrays} from '#sugar';
 import {checkIfImagePathHasCachedThumbnails, getThumbnailEqualOrSmaller}
@@ -15,10 +16,20 @@ async function exportIndexToJSON(index) {
   const results = {};
 
   await index.export((key, data) => {
-    results[key] = data;
-  })
+    if (data === undefined) {
+      return;
+    }
 
-  return results;
+    if (typeof data !== 'string') {
+      logWarn`Got something besides a string from index.export(), skipping:`;
+      console.warn(key, data);
+      return;
+    }
+
+    results[key] = JSON.parse(data);
+  });
+
+  return JSON.stringify(results);
 }
 
 export async function writeSearchData({
@@ -62,21 +73,40 @@ export async function writeSearchData({
   const jsonIndexes =
     await Promise.all(indexes.map(exportIndexToJSON));
 
-  const searchData =
-    Object.fromEntries(
-      stitchArrays({
-        key: keys,
-        value: jsonIndexes,
-      }).map(({key, value}) => [key, value]));
-
   const outputDirectory =
     path.join(wikiCachePath, 'search');
 
-  const outputFile =
+  const mainIndexFile =
     path.join(outputDirectory, 'index.json');
 
+  const mainIndexJSON =
+    JSON.stringify(
+      Object.fromEntries(
+        stitchArrays({
+          key: keys,
+          json: jsonIndexes,
+        }).map(({key, json}) => {
+          const md5 = createHash('md5');
+          md5.write(json);
+
+          const value = {
+            md5: md5.digest('hex'),
+          };
+
+          return [key, value];
+        })));
+
+
   await mkdir(outputDirectory, {recursive: true});
-  await writeFile(outputFile, JSON.stringify(searchData));
 
-  logInfo`Search index successfully written.`;
+  await Promise.all(
+    stitchArrays({
+      key: keys,
+      json: jsonIndexes,
+    }).map(({key, json}) =>
+        writeFile(
+          path.join(outputDirectory, key + '.json'),
+          json)));
+
+  await writeFile(mainIndexFile, mainIndexJSON);
 }
diff --git a/src/static/js/search-worker.js b/src/static/js/search-worker.js
index a6ecd3ab..c3975380 100644
--- a/src/static/js/search-worker.js
+++ b/src/static/js/search-worker.js
@@ -6,7 +6,6 @@ import FlexSearch from '../lib/flexsearch/flexsearch.bundle.module.min.js';
 
 let status = null;
 let indexes = null;
-let searchData = null;
 
 onmessage = handleWindowMessage;
 postStatus('alive');
@@ -20,6 +19,10 @@ main().then(
     postStatus('setup-error');
   });
 
+function rebase(path) {
+  return `/search-data/` + path;
+}
+
 async function main() {
   indexes =
     withEntries(searchSpec, entries => entries
@@ -28,16 +31,25 @@ async function main() {
         makeSearchIndex(descriptor, {FlexSearch}),
       ]));
 
-  searchData =
-    await fetch('/search-data/index.json')
+  const indexData =
+    await fetch(rebase('index.json'))
       .then(resp => resp.json());
 
+  await Promise.all(
+    Object.entries(indexData)
+      .map(([key, _info]) =>
+        fetch(rebase(key + '.json'))
+          .then(res => res.json())
+          .then(data => {
+            importIndex(key, data);
+          })));
+}
+
+function importIndex(indexKey, indexData) {
   // If this fails, it's because an outdated index was cached.
   // TODO: If this fails, try again once with a cache busting url.
-  for (const [indexName, indexData] of Object.entries(searchData)) {
-    for (const [key, value] of Object.entries(indexData)) {
-      indexes[indexName].import(key, value);
-    }
+  for (const [key, value] of Object.entries(indexData)) {
+    indexes[indexKey].import(key, JSON.stringify(value));
   }
 }
 
diff --git a/src/upd8.js b/src/upd8.js
index de4b51e8..06b8b64c 100755
--- a/src/upd8.js
+++ b/src/upd8.js
@@ -1498,6 +1498,8 @@ async function main() {
       wikiData,
     });
 
+    logInfo`Search data successfully written.`;
+
     Object.assign(stepStatusSummary.buildSearchIndex, {
       status: STATUS_DONE_CLEAN,
       timeEnd: Date.now(),