diff options
| -rw-r--r-- | package-lock.json | 201 | ||||
| -rw-r--r-- | package.json | 2 | ||||
| -rw-r--r-- | src/search.js | 29 | ||||
| -rw-r--r-- | src/static/js/search-worker.js | 44 | ||||
| -rw-r--r-- | src/web-routes.js | 10 |
5 files changed, 258 insertions, 28 deletions
diff --git a/package-lock.json b/package-lock.json index 8caa8aaa..3b85faa9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,12 +12,14 @@ "@js-temporal/polyfill": "^0.4.4", "chroma-js": "^2.4.2", "command-exists": "^1.2.9", + "compress-json": "^3.0.5", "eslint": "^8.37.0", "flexsearch": "^0.7.43", "he": "^1.2.0", "image-size": "^1.0.2", "js-yaml": "^4.1.0", "marked": "^10.0.0", + "msgpackr": "^1.10.2", "striptags": "^4.0.0-alpha.4", "word-wrap": "^1.2.3" }, @@ -303,6 +305,78 @@ "node": ">=12" } }, + "node_modules/@msgpackr-extract/msgpackr-extract-darwin-arm64": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-arm64/-/msgpackr-extract-darwin-arm64-3.0.2.tgz", + "integrity": "sha512-9bfjwDxIDWmmOKusUcqdS4Rw+SETlp9Dy39Xui9BEGEk19dDwH0jhipwFzEff/pFg95NKymc6TOTbRKcWeRqyQ==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-darwin-x64": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-x64/-/msgpackr-extract-darwin-x64-3.0.2.tgz", + "integrity": "sha512-lwriRAHm1Yg4iDf23Oxm9n/t5Zpw1lVnxYU3HnJPTi2lJRkKTrps1KVgvL6m7WvmhYVt/FIsssWay+k45QHeuw==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-linux-arm": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm/-/msgpackr-extract-linux-arm-3.0.2.tgz", + "integrity": "sha512-MOI9Dlfrpi2Cuc7i5dXdxPbFIgbDBGgKR5F2yWEa6FVEtSWncfVNKW5AKjImAQ6CZlBK9tympdsZJ2xThBiWWA==", + "cpu": [ + "arm" + ], + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-linux-arm64": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm64/-/msgpackr-extract-linux-arm64-3.0.2.tgz", + "integrity": "sha512-FU20Bo66/f7He9Fp9sP2zaJ1Q8L9uLPZQDub/WlUip78JlPeMbVL8546HbZfcW9LNciEXc8d+tThSJjSC+tmsg==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-linux-x64": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-x64/-/msgpackr-extract-linux-x64-3.0.2.tgz", + "integrity": "sha512-gsWNDCklNy7Ajk0vBBf9jEx04RUxuDQfBse918Ww+Qb9HCPoGzS+XJTLe96iN3BVK7grnLiYghP/M4L8VsaHeA==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-win32-x64": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-win32-x64/-/msgpackr-extract-win32-x64-3.0.2.tgz", + "integrity": "sha512-O+6Gs8UeDbyFpbSh2CPEz/UOrrdWPTBYNblZK5CxxLisYt4kGX3Sc+czffFonyjiGSq3jWLwJS/CCJc7tBr4sQ==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "win32" + ] + }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", @@ -1962,6 +2036,11 @@ "resolved": "https://registry.npmjs.org/command-exists/-/command-exists-1.2.9.tgz", "integrity": "sha512-LTQ/SGc+s0Xc0Fu5WaKnR0YiygZkm9eKFvyS+fRsU7/ZWFF8ykFM6Pc9aCVf1+xasOOZpO3BAVgVrKvsqKHV7w==" }, + "node_modules/compress-json": { + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/compress-json/-/compress-json-3.0.5.tgz", + "integrity": "sha512-HYiJvE0cTIygI9zXqY5fkRr7H3NV3UAME0enzwN5M0JkzMOtUcjSyaH7HxVRzXsn7IIXD0STA9M5jyWkxERSLg==" + }, "node_modules/concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", @@ -2074,9 +2153,9 @@ "dev": true }, "node_modules/escalade": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz", - "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==", + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.2.tgz", + "integrity": "sha512-ErCHMCae19vR8vQGe50xIsVomy19rg6gFu3+r3jkEO46suLMWBksvVyoGgQV+jOfl84ZSOSlmv6Gxa89PmTGmA==", "dev": true, "engines": { "node": ">=6" @@ -3306,6 +3385,35 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" }, + "node_modules/msgpackr": { + "version": "1.10.2", + "resolved": "https://registry.npmjs.org/msgpackr/-/msgpackr-1.10.2.tgz", + "integrity": "sha512-L60rsPynBvNE+8BWipKKZ9jHcSGbtyJYIwjRq0VrIvQ08cRjntGXJYW/tmciZ2IHWIY8WEW32Qa2xbh5+SKBZA==", + "optionalDependencies": { + "msgpackr-extract": "^3.0.2" + } + }, + "node_modules/msgpackr-extract": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/msgpackr-extract/-/msgpackr-extract-3.0.2.tgz", + "integrity": "sha512-SdzXp4kD/Qf8agZ9+iTu6eql0m3kWm1A2y1hkpTeVNENutaB0BwHlSvAIaMxwntmRUAUjon2V4L8Z/njd0Ct8A==", + "hasInstallScript": true, + "optional": true, + "dependencies": { + "node-gyp-build-optional-packages": "5.0.7" + }, + "bin": { + "download-msgpackr-prebuilds": "bin/download-prebuilds.js" + }, + "optionalDependencies": { + "@msgpackr-extract/msgpackr-extract-darwin-arm64": "3.0.2", + "@msgpackr-extract/msgpackr-extract-darwin-x64": "3.0.2", + "@msgpackr-extract/msgpackr-extract-linux-arm": "3.0.2", + "@msgpackr-extract/msgpackr-extract-linux-arm64": "3.0.2", + "@msgpackr-extract/msgpackr-extract-linux-x64": "3.0.2", + "@msgpackr-extract/msgpackr-extract-win32-x64": "3.0.2" + } + }, "node_modules/natural-compare": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", @@ -3344,6 +3452,17 @@ "node": "^16.14.0 || >=18.0.0" } }, + "node_modules/node-gyp-build-optional-packages": { + "version": "5.0.7", + "resolved": "https://registry.npmjs.org/node-gyp-build-optional-packages/-/node-gyp-build-optional-packages-5.0.7.tgz", + "integrity": "sha512-YlCCc6Wffkx0kHkmam79GKvDQ6x+QZkMjFGrIMxgFNILFvGSbCp2fCBC55pGTT9gVaz8Na5CLmxt/urtzRv36w==", + "optional": true, + "bin": { + "node-gyp-build-optional-packages": "bin.js", + "node-gyp-build-optional-packages-optional": "optional.js", + "node-gyp-build-optional-packages-test": "build-test.js" + } + }, "node_modules/node-gyp/node_modules/brace-expansion": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", @@ -5564,6 +5683,42 @@ "tslib": "^2.4.1" } }, + "@msgpackr-extract/msgpackr-extract-darwin-arm64": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-arm64/-/msgpackr-extract-darwin-arm64-3.0.2.tgz", + "integrity": "sha512-9bfjwDxIDWmmOKusUcqdS4Rw+SETlp9Dy39Xui9BEGEk19dDwH0jhipwFzEff/pFg95NKymc6TOTbRKcWeRqyQ==", + "optional": true + }, + "@msgpackr-extract/msgpackr-extract-darwin-x64": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-x64/-/msgpackr-extract-darwin-x64-3.0.2.tgz", + "integrity": "sha512-lwriRAHm1Yg4iDf23Oxm9n/t5Zpw1lVnxYU3HnJPTi2lJRkKTrps1KVgvL6m7WvmhYVt/FIsssWay+k45QHeuw==", + "optional": true + }, + "@msgpackr-extract/msgpackr-extract-linux-arm": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm/-/msgpackr-extract-linux-arm-3.0.2.tgz", + "integrity": "sha512-MOI9Dlfrpi2Cuc7i5dXdxPbFIgbDBGgKR5F2yWEa6FVEtSWncfVNKW5AKjImAQ6CZlBK9tympdsZJ2xThBiWWA==", + "optional": true + }, + "@msgpackr-extract/msgpackr-extract-linux-arm64": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm64/-/msgpackr-extract-linux-arm64-3.0.2.tgz", + "integrity": "sha512-FU20Bo66/f7He9Fp9sP2zaJ1Q8L9uLPZQDub/WlUip78JlPeMbVL8546HbZfcW9LNciEXc8d+tThSJjSC+tmsg==", + "optional": true + }, + "@msgpackr-extract/msgpackr-extract-linux-x64": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-x64/-/msgpackr-extract-linux-x64-3.0.2.tgz", + "integrity": "sha512-gsWNDCklNy7Ajk0vBBf9jEx04RUxuDQfBse918Ww+Qb9HCPoGzS+XJTLe96iN3BVK7grnLiYghP/M4L8VsaHeA==", + "optional": true + }, + "@msgpackr-extract/msgpackr-extract-win32-x64": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-win32-x64/-/msgpackr-extract-win32-x64-3.0.2.tgz", + "integrity": "sha512-O+6Gs8UeDbyFpbSh2CPEz/UOrrdWPTBYNblZK5CxxLisYt4kGX3Sc+czffFonyjiGSq3jWLwJS/CCJc7tBr4sQ==", + "optional": true + }, "@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", @@ -6725,6 +6880,11 @@ "resolved": "https://registry.npmjs.org/command-exists/-/command-exists-1.2.9.tgz", "integrity": "sha512-LTQ/SGc+s0Xc0Fu5WaKnR0YiygZkm9eKFvyS+fRsU7/ZWFF8ykFM6Pc9aCVf1+xasOOZpO3BAVgVrKvsqKHV7w==" }, + "compress-json": { + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/compress-json/-/compress-json-3.0.5.tgz", + "integrity": "sha512-HYiJvE0cTIygI9zXqY5fkRr7H3NV3UAME0enzwN5M0JkzMOtUcjSyaH7HxVRzXsn7IIXD0STA9M5jyWkxERSLg==" + }, "concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", @@ -6814,9 +6974,9 @@ "dev": true }, "escalade": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz", - "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==", + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.2.tgz", + "integrity": "sha512-ErCHMCae19vR8vQGe50xIsVomy19rg6gFu3+r3jkEO46suLMWBksvVyoGgQV+jOfl84ZSOSlmv6Gxa89PmTGmA==", "dev": true }, "escape-string-regexp": { @@ -7721,6 +7881,29 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" }, + "msgpackr": { + "version": "1.10.2", + "resolved": "https://registry.npmjs.org/msgpackr/-/msgpackr-1.10.2.tgz", + "integrity": "sha512-L60rsPynBvNE+8BWipKKZ9jHcSGbtyJYIwjRq0VrIvQ08cRjntGXJYW/tmciZ2IHWIY8WEW32Qa2xbh5+SKBZA==", + "requires": { + "msgpackr-extract": "^3.0.2" + } + }, + "msgpackr-extract": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/msgpackr-extract/-/msgpackr-extract-3.0.2.tgz", + "integrity": "sha512-SdzXp4kD/Qf8agZ9+iTu6eql0m3kWm1A2y1hkpTeVNENutaB0BwHlSvAIaMxwntmRUAUjon2V4L8Z/njd0Ct8A==", + "optional": true, + "requires": { + "@msgpackr-extract/msgpackr-extract-darwin-arm64": "3.0.2", + "@msgpackr-extract/msgpackr-extract-darwin-x64": "3.0.2", + "@msgpackr-extract/msgpackr-extract-linux-arm": "3.0.2", + "@msgpackr-extract/msgpackr-extract-linux-arm64": "3.0.2", + "@msgpackr-extract/msgpackr-extract-linux-x64": "3.0.2", + "@msgpackr-extract/msgpackr-extract-win32-x64": "3.0.2", + "node-gyp-build-optional-packages": "5.0.7" + } + }, "natural-compare": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", @@ -7798,6 +7981,12 @@ } } }, + "node-gyp-build-optional-packages": { + "version": "5.0.7", + "resolved": "https://registry.npmjs.org/node-gyp-build-optional-packages/-/node-gyp-build-optional-packages-5.0.7.tgz", + "integrity": "sha512-YlCCc6Wffkx0kHkmam79GKvDQ6x+QZkMjFGrIMxgFNILFvGSbCp2fCBC55pGTT9gVaz8Na5CLmxt/urtzRv36w==", + "optional": true + }, "nopt": { "version": "7.2.0", "resolved": "https://registry.npmjs.org/nopt/-/nopt-7.2.0.tgz", diff --git a/package.json b/package.json index c8f96e98..fd8ac9cf 100644 --- a/package.json +++ b/package.json @@ -60,12 +60,14 @@ "@js-temporal/polyfill": "^0.4.4", "chroma-js": "^2.4.2", "command-exists": "^1.2.9", + "compress-json": "^3.0.5", "eslint": "^8.37.0", "flexsearch": "^0.7.43", "he": "^1.2.0", "image-size": "^1.0.2", "js-yaml": "^4.1.0", "marked": "^10.0.0", + "msgpackr": "^1.10.2", "striptags": "^4.0.0-alpha.4", "word-wrap": "^1.2.3" }, diff --git a/src/search.js b/src/search.js index 687aa215..a2dae9e1 100644 --- a/src/search.js +++ b/src/search.js @@ -4,7 +4,9 @@ import {createHash} from 'node:crypto'; import {mkdir, writeFile} from 'node:fs/promises'; import * as path from 'node:path'; +import {compress} from 'compress-json'; import FlexSearch from 'flexsearch'; +import {pack} from 'msgpackr'; import {logWarn} from '#cli'; import {makeSearchIndex, populateSearchIndex, searchSpec} from '#search-spec'; @@ -12,7 +14,7 @@ import {stitchArrays} from '#sugar'; import {checkIfImagePathHasCachedThumbnails, getThumbnailEqualOrSmaller} from '#thumbs'; -async function exportIndexToJSON(index) { +async function serializeIndex(index) { const results = {}; await index.export((key, data) => { @@ -29,7 +31,7 @@ async function exportIndexToJSON(index) { results[key] = JSON.parse(data); }); - return JSON.stringify(results); + return results; } export async function writeSearchData({ @@ -70,8 +72,13 @@ export async function writeSearchData({ wikiData, })); - const jsonIndexes = - await Promise.all(indexes.map(exportIndexToJSON)); + const serializedIndexes = + await Promise.all(indexes.map(serializeIndex)); + + const packedIndexes = + serializedIndexes + .map(data => compress(data)) + .map(data => pack(data)); const outputDirectory = path.join(wikiCachePath, 'search'); @@ -84,10 +91,10 @@ export async function writeSearchData({ Object.fromEntries( stitchArrays({ key: keys, - json: jsonIndexes, - }).map(({key, json}) => { + buffer: packedIndexes, + }).map(({key, buffer}) => { const md5 = createHash('md5'); - md5.write(json); + md5.write(buffer); const value = { md5: md5.digest('hex'), @@ -102,11 +109,11 @@ export async function writeSearchData({ await Promise.all( stitchArrays({ key: keys, - json: jsonIndexes, - }).map(({key, json}) => + buffer: packedIndexes, + }).map(({key, buffer}) => writeFile( - path.join(outputDirectory, key + '.json'), - json))); + path.join(outputDirectory, key + '.json.msgpack'), + buffer))); await writeFile(mainIndexFile, mainIndexJSON); } diff --git a/src/static/js/search-worker.js b/src/static/js/search-worker.js index c3975380..b8ab4a63 100644 --- a/src/static/js/search-worker.js +++ b/src/static/js/search-worker.js @@ -1,8 +1,14 @@ +import FlexSearch from '../lib/flexsearch/flexsearch.bundle.module.min.js'; + import {makeSearchIndex, searchSpec} from '../shared-util/search-spec.js'; import {empty, groupArray, stitchArrays, unique, withEntries} from '../shared-util/sugar.js'; -import FlexSearch from '../lib/flexsearch/flexsearch.bundle.module.min.js'; +import {loadDependency} from './module-import-shims.js'; + +// Will be loaded from dependencies. +let decompress; +let unpack; let status = null; let indexes = null; @@ -10,14 +16,27 @@ let indexes = null; onmessage = handleWindowMessage; postStatus('alive'); -main().then( - () => { - postStatus('ready'); - }, - error => { - console.error(`Search worker setup error:`, error); - postStatus('setup-error'); - }); +loadDependencies() + .then(main) + .then( + () => { + postStatus('ready'); + }, + error => { + console.error(`Search worker setup error:`, error); + postStatus('setup-error'); + }); + +async function loadDependencies() { + const {compressJSON} = + await loadDependency.fromWindow('../lib/compress-json/bundle.min.js'); + + const msgpackr = + await loadDependency.fromModuleExports('../lib/msgpackr/index.js'); + + ({decompress} = compressJSON); + ({unpack} = msgpackr); +} function rebase(path) { return `/search-data/` + path; @@ -38,8 +57,11 @@ async function main() { await Promise.all( Object.entries(indexData) .map(([key, _info]) => - fetch(rebase(key + '.json')) - .then(res => res.json()) + fetch(rebase(key + '.json.msgpack')) + .then(res => res.arrayBuffer()) + .then(buffer => new Uint8Array(buffer)) + .then(data => unpack(data)) + .then(data => decompress(data)) .then(data => { importIndex(key, data); }))); diff --git a/src/web-routes.js b/src/web-routes.js index 8bb2fba3..7e08d06f 100644 --- a/src/web-routes.js +++ b/src/web-routes.js @@ -60,8 +60,18 @@ export const dependencyRoutes = [ }), quickNodeDependency({ + name: 'compress-json', + path: '..', // exit dist, access bundle.js + }), + + quickNodeDependency({ name: 'flexsearch', }), + + quickNodeDependency({ + name: 'msgpackr', + path: 'dist', + }), ].flat(); export const allStaticWebRoutes = [ |