diff options
author | (quasar) nebula <qznebula@protonmail.com> | 2024-05-13 14:03:03 -0300 |
---|---|---|
committer | (quasar) nebula <qznebula@protonmail.com> | 2024-05-31 12:11:53 -0300 |
commit | 30914eb56468b388e4b5cb2090292c5932171eb3 (patch) | |
tree | 6419b871ac9b8523a97b5d171e4e3cfa3c694d42 /src | |
parent | 3b9f95a125d28b8cda3ebca1fe248e2026d2e56d (diff) |
search, client: use json-compress and msgpackr for search indexes
Diffstat (limited to 'src')
-rw-r--r-- | src/search.js | 29 | ||||
-rw-r--r-- | src/static/js/search-worker.js | 44 | ||||
-rw-r--r-- | src/web-routes.js | 10 |
3 files changed, 61 insertions, 22 deletions
diff --git a/src/search.js b/src/search.js index 687aa215..a2dae9e1 100644 --- a/src/search.js +++ b/src/search.js @@ -4,7 +4,9 @@ import {createHash} from 'node:crypto'; import {mkdir, writeFile} from 'node:fs/promises'; import * as path from 'node:path'; +import {compress} from 'compress-json'; import FlexSearch from 'flexsearch'; +import {pack} from 'msgpackr'; import {logWarn} from '#cli'; import {makeSearchIndex, populateSearchIndex, searchSpec} from '#search-spec'; @@ -12,7 +14,7 @@ import {stitchArrays} from '#sugar'; import {checkIfImagePathHasCachedThumbnails, getThumbnailEqualOrSmaller} from '#thumbs'; -async function exportIndexToJSON(index) { +async function serializeIndex(index) { const results = {}; await index.export((key, data) => { @@ -29,7 +31,7 @@ async function exportIndexToJSON(index) { results[key] = JSON.parse(data); }); - return JSON.stringify(results); + return results; } export async function writeSearchData({ @@ -70,8 +72,13 @@ export async function writeSearchData({ wikiData, })); - const jsonIndexes = - await Promise.all(indexes.map(exportIndexToJSON)); + const serializedIndexes = + await Promise.all(indexes.map(serializeIndex)); + + const packedIndexes = + serializedIndexes + .map(data => compress(data)) + .map(data => pack(data)); const outputDirectory = path.join(wikiCachePath, 'search'); @@ -84,10 +91,10 @@ export async function writeSearchData({ Object.fromEntries( stitchArrays({ key: keys, - json: jsonIndexes, - }).map(({key, json}) => { + buffer: packedIndexes, + }).map(({key, buffer}) => { const md5 = createHash('md5'); - md5.write(json); + md5.write(buffer); const value = { md5: md5.digest('hex'), @@ -102,11 +109,11 @@ export async function writeSearchData({ await Promise.all( stitchArrays({ key: keys, - json: jsonIndexes, - }).map(({key, json}) => + buffer: packedIndexes, + }).map(({key, buffer}) => writeFile( - path.join(outputDirectory, key + '.json'), - json))); + path.join(outputDirectory, key + '.json.msgpack'), + buffer))); await writeFile(mainIndexFile, mainIndexJSON); } diff --git a/src/static/js/search-worker.js b/src/static/js/search-worker.js index c3975380..b8ab4a63 100644 --- a/src/static/js/search-worker.js +++ b/src/static/js/search-worker.js @@ -1,8 +1,14 @@ +import FlexSearch from '../lib/flexsearch/flexsearch.bundle.module.min.js'; + import {makeSearchIndex, searchSpec} from '../shared-util/search-spec.js'; import {empty, groupArray, stitchArrays, unique, withEntries} from '../shared-util/sugar.js'; -import FlexSearch from '../lib/flexsearch/flexsearch.bundle.module.min.js'; +import {loadDependency} from './module-import-shims.js'; + +// Will be loaded from dependencies. +let decompress; +let unpack; let status = null; let indexes = null; @@ -10,14 +16,27 @@ let indexes = null; onmessage = handleWindowMessage; postStatus('alive'); -main().then( - () => { - postStatus('ready'); - }, - error => { - console.error(`Search worker setup error:`, error); - postStatus('setup-error'); - }); +loadDependencies() + .then(main) + .then( + () => { + postStatus('ready'); + }, + error => { + console.error(`Search worker setup error:`, error); + postStatus('setup-error'); + }); + +async function loadDependencies() { + const {compressJSON} = + await loadDependency.fromWindow('../lib/compress-json/bundle.min.js'); + + const msgpackr = + await loadDependency.fromModuleExports('../lib/msgpackr/index.js'); + + ({decompress} = compressJSON); + ({unpack} = msgpackr); +} function rebase(path) { return `/search-data/` + path; @@ -38,8 +57,11 @@ async function main() { await Promise.all( Object.entries(indexData) .map(([key, _info]) => - fetch(rebase(key + '.json')) - .then(res => res.json()) + fetch(rebase(key + '.json.msgpack')) + .then(res => res.arrayBuffer()) + .then(buffer => new Uint8Array(buffer)) + .then(data => unpack(data)) + .then(data => decompress(data)) .then(data => { importIndex(key, data); }))); diff --git a/src/web-routes.js b/src/web-routes.js index 8bb2fba3..7e08d06f 100644 --- a/src/web-routes.js +++ b/src/web-routes.js @@ -60,8 +60,18 @@ export const dependencyRoutes = [ }), quickNodeDependency({ + name: 'compress-json', + path: '..', // exit dist, access bundle.js + }), + + quickNodeDependency({ name: 'flexsearch', }), + + quickNodeDependency({ + name: 'msgpackr', + path: 'dist', + }), ].flat(); export const allStaticWebRoutes = [ |