From 30914eb56468b388e4b5cb2090292c5932171eb3 Mon Sep 17 00:00:00 2001
From: "(quasar) nebula" <qznebula@protonmail.com>
Date: Mon, 13 May 2024 14:03:03 -0300
Subject: search, client: use json-compress and msgpackr for search indexes

---
 src/search.js                  | 29 +++++++++++++++++-----------
 src/static/js/search-worker.js | 44 +++++++++++++++++++++++++++++++-----------
 src/web-routes.js              | 10 ++++++++++
 3 files changed, 61 insertions(+), 22 deletions(-)

(limited to 'src')

diff --git a/src/search.js b/src/search.js
index 687aa215..a2dae9e1 100644
--- a/src/search.js
+++ b/src/search.js
@@ -4,7 +4,9 @@ import {createHash} from 'node:crypto';
 import {mkdir, writeFile} from 'node:fs/promises';
 import * as path from 'node:path';
 
+import {compress} from 'compress-json';
 import FlexSearch from 'flexsearch';
+import {pack} from 'msgpackr';
 
 import {logWarn} from '#cli';
 import {makeSearchIndex, populateSearchIndex, searchSpec} from '#search-spec';
@@ -12,7 +14,7 @@ import {stitchArrays} from '#sugar';
 import {checkIfImagePathHasCachedThumbnails, getThumbnailEqualOrSmaller}
   from '#thumbs';
 
-async function exportIndexToJSON(index) {
+async function serializeIndex(index) {
   const results = {};
 
   await index.export((key, data) => {
@@ -29,7 +31,7 @@ async function exportIndexToJSON(index) {
     results[key] = JSON.parse(data);
   });
 
-  return JSON.stringify(results);
+  return results;
 }
 
 export async function writeSearchData({
@@ -70,8 +72,13 @@ export async function writeSearchData({
         wikiData,
       }));
 
-  const jsonIndexes =
-    await Promise.all(indexes.map(exportIndexToJSON));
+  const serializedIndexes =
+    await Promise.all(indexes.map(serializeIndex));
+
+  const packedIndexes =
+    serializedIndexes
+      .map(data => compress(data))
+      .map(data => pack(data));
 
   const outputDirectory =
     path.join(wikiCachePath, 'search');
@@ -84,10 +91,10 @@ export async function writeSearchData({
       Object.fromEntries(
         stitchArrays({
           key: keys,
-          json: jsonIndexes,
-        }).map(({key, json}) => {
+          buffer: packedIndexes,
+        }).map(({key, buffer}) => {
           const md5 = createHash('md5');
-          md5.write(json);
+          md5.write(buffer);
 
           const value = {
             md5: md5.digest('hex'),
@@ -102,11 +109,11 @@ export async function writeSearchData({
   await Promise.all(
     stitchArrays({
       key: keys,
-      json: jsonIndexes,
-    }).map(({key, json}) =>
+      buffer: packedIndexes,
+    }).map(({key, buffer}) =>
         writeFile(
-          path.join(outputDirectory, key + '.json'),
-          json)));
+          path.join(outputDirectory, key + '.json.msgpack'),
+          buffer)));
 
   await writeFile(mainIndexFile, mainIndexJSON);
 }
diff --git a/src/static/js/search-worker.js b/src/static/js/search-worker.js
index c3975380..b8ab4a63 100644
--- a/src/static/js/search-worker.js
+++ b/src/static/js/search-worker.js
@@ -1,8 +1,14 @@
+import FlexSearch from '../lib/flexsearch/flexsearch.bundle.module.min.js';
+
 import {makeSearchIndex, searchSpec} from '../shared-util/search-spec.js';
 import {empty, groupArray, stitchArrays, unique, withEntries}
   from '../shared-util/sugar.js';
 
-import FlexSearch from '../lib/flexsearch/flexsearch.bundle.module.min.js';
+import {loadDependency} from './module-import-shims.js';
+
+// Will be loaded from dependencies.
+let decompress;
+let unpack;
 
 let status = null;
 let indexes = null;
@@ -10,14 +16,27 @@ let indexes = null;
 onmessage = handleWindowMessage;
 postStatus('alive');
 
-main().then(
-  () => {
-    postStatus('ready');
-  },
-  error => {
-    console.error(`Search worker setup error:`, error);
-    postStatus('setup-error');
-  });
+loadDependencies()
+  .then(main)
+  .then(
+    () => {
+      postStatus('ready');
+    },
+    error => {
+      console.error(`Search worker setup error:`, error);
+      postStatus('setup-error');
+    });
+
+async function loadDependencies() {
+  const {compressJSON} =
+    await loadDependency.fromWindow('../lib/compress-json/bundle.min.js');
+
+  const msgpackr =
+    await loadDependency.fromModuleExports('../lib/msgpackr/index.js');
+
+  ({decompress} = compressJSON);
+  ({unpack} = msgpackr);
+}
 
 function rebase(path) {
   return `/search-data/` + path;
@@ -38,8 +57,11 @@ async function main() {
   await Promise.all(
     Object.entries(indexData)
       .map(([key, _info]) =>
-        fetch(rebase(key + '.json'))
-          .then(res => res.json())
+        fetch(rebase(key + '.json.msgpack'))
+          .then(res => res.arrayBuffer())
+          .then(buffer => new Uint8Array(buffer))
+          .then(data => unpack(data))
+          .then(data => decompress(data))
           .then(data => {
             importIndex(key, data);
           })));
diff --git a/src/web-routes.js b/src/web-routes.js
index 8bb2fba3..7e08d06f 100644
--- a/src/web-routes.js
+++ b/src/web-routes.js
@@ -59,9 +59,19 @@ export const dependencyRoutes = [
     name: 'chroma-js',
   }),
 
+  quickNodeDependency({
+    name: 'compress-json',
+    path: '..', // exit dist, access bundle.js
+  }),
+
   quickNodeDependency({
     name: 'flexsearch',
   }),
+
+  quickNodeDependency({
+    name: 'msgpackr',
+    path: 'dist',
+  }),
 ].flat();
 
 export const allStaticWebRoutes = [
-- 
cgit 1.3.0-6-gf8a5