diff options
Diffstat (limited to 'src')
-rw-r--r-- | src/data/things/album.js | 8 | ||||
-rw-r--r-- | src/data/things/composite.js | 111 | ||||
-rw-r--r-- | src/data/things/thing.js | 25 | ||||
-rwxr-xr-x | src/upd8.js | 14 | ||||
-rw-r--r-- | src/util/wiki-data.js | 68 |
5 files changed, 209 insertions, 17 deletions
diff --git a/src/data/things/album.js b/src/data/things/album.js index 7569eb80..b134b78d 100644 --- a/src/data/things/album.js +++ b/src/data/things/album.js @@ -125,6 +125,14 @@ export class Album extends Thing { intoIndices: '#sections.startIndex', }), + { + dependencies: ['#trackRefs'], + compute: ({'#trackRefs': tracks}, continuation) => { + console.log(tracks); + return continuation(); + } + }, + withResolvedReferenceList({ list: '#trackRefs', data: 'trackData', diff --git a/src/data/things/composite.js b/src/data/things/composite.js index 3a63f22d..26124b56 100644 --- a/src/data/things/composite.js +++ b/src/data/things/composite.js @@ -1,6 +1,7 @@ import {inspect} from 'node:util'; import {colors} from '#cli'; +import {TupleMap} from '#wiki-data'; import { empty, @@ -341,6 +342,8 @@ import { // syntax as for other compositional steps, and it'll work out cleanly! // +const globalCompositeCache = {}; + export function compositeFrom(firstArg, secondArg) { const debug = fn => { if (compositeFrom.debug === true) { @@ -567,8 +570,8 @@ export function compositeFrom(firstArg, secondArg) { return {continuation, continuationStorage}; } - const continuationSymbol = Symbol('continuation symbol'); - const noTransformSymbol = Symbol('no-transform symbol'); + const continuationSymbol = Symbol.for('compositeFrom: continuation symbol'); + const noTransformSymbol = Symbol.for('compositeFrom: no-transform symbol'); function _computeOrTransform(initialValue, initialDependencies, continuationIfApplicable) { const expectingTransform = initialValue !== noTransformSymbol; @@ -634,21 +637,83 @@ export function compositeFrom(firstArg, secondArg) { const callingTransformForThisStep = expectingTransform && expose.transform; + let continuationStorage; + const filteredDependencies = _filterDependencies(availableDependencies, expose); - const {continuation, continuationStorage} = _prepareContinuation(callingTransformForThisStep); debug(() => [ `step #${i+1} - ${callingTransformForThisStep ? 'transform' : 'compute'}`, `with dependencies:`, filteredDependencies]); - const result = + let result; + + const getExpectedEvaluation = () => (callingTransformForThisStep ? (filteredDependencies - ? expose.transform(valueSoFar, filteredDependencies, continuation) - : expose.transform(valueSoFar, continuation)) + ? ['transform', valueSoFar, filteredDependencies] + : ['transform', valueSoFar]) : (filteredDependencies - ? expose.compute(filteredDependencies, continuation) - : expose.compute(continuation))); + ? ['compute', filteredDependencies] + : ['compute'])); + + const naturalEvaluate = () => { + const [name, ...args] = getExpectedEvaluation(); + let continuation; + ({continuation, continuationStorage} = _prepareContinuation(callingTransformForThisStep)); + return expose[name](...args, continuation); + } + + switch (step.cache) { + // Warning! Highly WIP! + case 'aggressive': { + const hrnow = () => { + const hrTime = process.hrtime(); + return hrTime[0] * 1000000000 + hrTime[1]; + }; + + const [name, ...args] = getExpectedEvaluation(); + + let cache = globalCompositeCache[step.annotation]; + if (!cache) { + cache = globalCompositeCache[step.annotation] = { + transform: new TupleMap(), + compute: new TupleMap(), + times: { + read: [], + evaluate: [], + }, + }; + } + + const tuplefied = args + .flatMap(arg => [ + Symbol.for('compositeFrom: tuplefied arg divider'), + ...(typeof arg !== 'object' || Array.isArray(arg) + ? [arg] + : Object.entries(arg).flat()), + ]); + + const readTime = hrnow(); + const cacheContents = cache[name].get(tuplefied); + cache.times.read.push(hrnow() - readTime); + + if (cacheContents) { + ({result, continuationStorage} = cacheContents); + } else { + const evaluateTime = hrnow(); + result = naturalEvaluate(); + cache.times.evaluate.push(hrnow() - evaluateTime); + cache[name].set(tuplefied, {result, continuationStorage}); + } + + break; + } + + default: { + result = naturalEvaluate(); + break; + } + } if (result !== continuationSymbol) { debug(() => [`step #${i+1} - result: exit (inferred) ->`, result]); @@ -775,6 +840,7 @@ export function compositeFrom(firstArg, secondArg) { if (baseComposes) { if (anyStepsTransform) expose.transform = transformFn; if (anyStepsCompute) expose.compute = computeFn; + if (base.cacheComposition) expose.cache = base.cacheComposition; } else if (baseUpdates) { expose.transform = transformFn; } else { @@ -785,6 +851,35 @@ export function compositeFrom(firstArg, secondArg) { return constructedDescriptor; } +export function displayCompositeCacheAnalysis() { + const showTimes = (cache, key) => { + const times = cache.times[key].slice().sort(); + + const all = times; + const worst10pc = times.slice(-times.length / 10); + const best10pc = times.slice(0, times.length / 10); + const middle50pc = times.slice(times.length / 4, -times.length / 4); + const middle80pc = times.slice(times.length / 10, -times.length / 10); + + const fmt = val => `${(val / 1000).toFixed(2)}ms`.padStart(9); + const avg = times => times.reduce((a, b) => a + b, 0) / times.length; + + const left = ` - ${key}: `; + const indn = ' '.repeat(left.length); + console.log(left + `${fmt(avg(all))} (all ${all.length})`); + console.log(indn + `${fmt(avg(worst10pc))} (worst 10%)`); + console.log(indn + `${fmt(avg(best10pc))} (best 10%)`); + console.log(indn + `${fmt(avg(middle80pc))} (middle 80%)`); + console.log(indn + `${fmt(avg(middle50pc))} (middle 50%)`); + }; + + for (const [annotation, cache] of Object.entries(globalCompositeCache)) { + console.log(`Cached ${annotation}:`); + showTimes(cache, 'evaluate'); + showTimes(cache, 'read'); + } +} + // Evaluates a function with composite debugging enabled, turns debugging // off again, and returns the result of the function. This is mostly syntax // sugar, but also helps avoid unit tests avoid accidentally printing debug diff --git a/src/data/things/thing.js b/src/data/things/thing.js index b1a9a802..19954b19 100644 --- a/src/data/things/thing.js +++ b/src/data/things/thing.js @@ -512,7 +512,7 @@ export function withResolvedReferenceList({ throw new TypeError(`Expected notFoundMode to be filter, exit, or null`); } - return compositeFrom(`withResolvedReferenceList`, [ + const composite = compositeFrom(`withResolvedReferenceList`, [ exitWithoutDependency({ dependency: data, value: [], @@ -526,13 +526,19 @@ export function withResolvedReferenceList({ }), { - mapDependencies: {list, data}, - options: {findFunction}, - - compute: ({list, data, '#options': {findFunction}}, continuation) => - continuation({ - '#matches': list.map(ref => findFunction(ref, data, {mode: 'quiet'})), - }), + cache: 'aggressive', + annotation: `withResolvedReferenceList.getMatches`, + flags: {expose: true, compose: true}, + + compute: { + mapDependencies: {list, data}, + options: {findFunction}, + + compute: ({list, data, '#options': {findFunction}}, continuation) => + continuation({ + '#matches': list.map(ref => findFunction(ref, data, {mode: 'quiet'})), + }), + }, }, { @@ -569,6 +575,9 @@ export function withResolvedReferenceList({ }, }, ]); + + console.log(composite.expose); + return composite; } // Check out the info on reverseReferenceList! diff --git a/src/upd8.js b/src/upd8.js index f6091ca2..7f423271 100755 --- a/src/upd8.js +++ b/src/upd8.js @@ -38,6 +38,7 @@ import {fileURLToPath} from 'node:url'; import wrap from 'word-wrap'; +import {displayCompositeCacheAnalysis} from '#composite'; import {processLanguageFile} from '#language'; import {isMain, traverse} from '#node-utils'; import bootRepl from '#repl'; @@ -612,6 +613,10 @@ async function main() { // which are only available after the initial linking. sortWikiDataArrays(wikiData); + console.log( + CacheableObject.getUpdateValue(wikiData.albumData[0], 'trackSections'), + wikiData.albumData[0].trackSections); + if (precacheData) { progressCallAll('Caching all data values', Object.entries(wikiData) .filter(([key]) => @@ -625,6 +630,11 @@ async function main() { .map(thing => () => CacheableObject.cacheAllExposedProperties(thing))); } + if (noBuild) { + displayCompositeCacheAnalysis(); + if (precacheData) return; + } + const internalDefaultLanguage = await processLanguageFile( path.join(__dirname, DEFAULT_STRINGS_FILE)); @@ -754,7 +764,9 @@ async function main() { logInfo`Done preloading filesizes!`; - if (noBuild) return; + if (noBuild) { + return; + } const developersComment = `<!--\n` + [ diff --git a/src/util/wiki-data.js b/src/util/wiki-data.js index 0eab2204..ac652b27 100644 --- a/src/util/wiki-data.js +++ b/src/util/wiki-data.js @@ -874,3 +874,71 @@ export function filterItemsForCarousel(items) { .filter(item => item.artTags.every(tag => !tag.isContentWarning)) .slice(0, maxCarouselLayoutItems + 1); } + +// Ridiculous caching support nonsense + +export class TupleMap { + static maxNestedTupleLength = 25; + + #store = [undefined, null, null, null]; + + #lifetime(value) { + if (Array.isArray(value) && value.length <= TupleMap.maxNestedTupleLength) { + return 'tuple'; + } else if ( + typeof value === 'object' && value !== null || + typeof value === 'function' + ) { + return 'weak'; + } else { + return 'strong'; + } + } + + #getSubstoreShallow(value, store) { + const lifetime = this.#lifetime(value); + const mapIndex = {weak: 1, strong: 2, tuple: 3}[lifetime]; + + let map = store[mapIndex]; + if (map === null) { + map = store[mapIndex] = + (lifetime === 'weak' ? new WeakMap() + : lifetime === 'strong' ? new Map() + : lifetime === 'tuple' ? new TupleMap() + : null); + } + + if (map.has(value)) { + return map.get(value); + } else { + const substore = [undefined, null, null, null]; + map.set(value, substore); + return substore; + } + } + + #getSubstoreDeep(tuple, store = this.#store) { + if (tuple.length === 0) { + return store; + } else { + const [first, ...rest] = tuple; + return this.#getSubstoreDeep(rest, this.#getSubstoreShallow(first, store)); + } + } + + get(tuple) { + const store = this.#getSubstoreDeep(tuple); + return store[0]; + } + + has(tuple) { + const store = this.#getSubstoreDeep(tuple); + return store[0] !== undefined; + } + + set(tuple, value) { + const store = this.#getSubstoreDeep(tuple); + store[0] = value; + return value; + } +} |