diff options
Diffstat (limited to 'src/upd8.js')
| -rwxr-xr-x | src/upd8.js | 805 |
1 files changed, 706 insertions, 99 deletions
diff --git a/src/upd8.js b/src/upd8.js index f4c6326a..045bf139 100755 --- a/src/upd8.js +++ b/src/upd8.js @@ -34,22 +34,23 @@ import '#import-heck'; import {execSync} from 'node:child_process'; -import {readdir, readFile, stat} from 'node:fs/promises'; +import {readdir, readFile, stat, writeFile} from 'node:fs/promises'; import * as path from 'node:path'; import {fileURLToPath} from 'node:url'; import wrap from 'word-wrap'; -import {mapAggregate, showAggregate} from '#aggregate'; +import {mapAggregate, openAggregate, showAggregate} from '#aggregate'; import CacheableObject from '#cacheable-object'; +import {stringifyCache} from '#cli'; import {displayCompositeCacheAnalysis} from '#composite'; import find, {bindFind, getAllFindSpecs} from '#find'; import {processLanguageFile, watchLanguageFile, internalDefaultStringsFile} from '#language'; import {isMain, traverse} from '#node-utils'; +import {bindReverse} from '#reverse'; import {writeSearchData} from '#search'; import {sortByName} from '#sort'; -import {generateURLs, urlSpec} from '#urls'; import {identifyAllWebRoutes} from '#web-routes'; import { @@ -73,6 +74,7 @@ import { import { bindOpts, empty, + filterMultipleArrays, indentWrap as unboundIndentWrap, withEntries, } from '#sugar'; @@ -87,6 +89,15 @@ import genThumbs, { } from '#thumbs'; import { + applyLocalizedWithBaseDirectory, + applyURLSpecOverriding, + generateURLs, + getOrigin, + internalDefaultURLSpecFile, + processURLSpecFromFile, +} from '#urls'; + +import { getAllDataSteps, linkWikiDataArrays, loadYAMLDocumentsFromDataSteps, @@ -123,6 +134,7 @@ const defaultStepStatus = {status: STATUS_NOT_STARTED, annotation: null}; // This will be initialized and mutated over the course of main(). let stepStatusSummary; let showStepStatusSummary = false; +let showStepMemoryInSummary = false; async function main() { Error.stackTraceLimit = Infinity; @@ -138,8 +150,8 @@ async function main() { {...defaultStepStatus, name: `migrate thumbnails`, for: ['thumbs']}, - loadThumbnailCache: - {...defaultStepStatus, name: `load thumbnail cache file`, + loadOfflineThumbnailCache: + {...defaultStepStatus, name: `load offline thumbnail cache file`, for: ['thumbs', 'build']}, generateThumbnails: @@ -178,6 +190,14 @@ async function main() { {...defaultStepStatus, name: `precache nearly all data`, for: ['build']}, + loadURLFiles: + {...defaultStepStatus, name: `load internal & custom url spec files`, + for: ['build']}, + + loadOnlineThumbnailCache: + {...defaultStepStatus, name: `load online thumbnail cache file`, + for: ['thumbs', 'build']}, + // TODO: This should be split into load/watch steps. loadInternalDefaultLanguage: {...defaultStepStatus, name: `load internal default language`, @@ -203,6 +223,10 @@ async function main() { {...defaultStepStatus, name: `preload file sizes`, for: ['build']}, + loadOnlineFileSizeCache: + {...defaultStepStatus, name: `load online file size cache file`, + for: ['build']}, + buildSearchIndex: {...defaultStepStatus, name: `generate search index`, for: ['build', 'search']}, @@ -323,6 +347,16 @@ async function main() { type: 'value', }, + 'urls': { + help: `Specify which optional URL specs to use for this build, customizing where pages are generated or resources are accessed from`, + type: 'value', + }, + + 'show-url-spec': { + help: `Displays the entire computed URL spec, after the data folder's default override and optional specs are applied. This is mostly useful for progammer debugging!`, + type: 'flag', + }, + 'skip-directory-validation': { help: `Skips checking for duplicated directories, which speeds up the build but may cause the wiki to catch on fire`, type: 'flag', @@ -363,11 +397,21 @@ async function main() { type: 'flag', }, + 'refresh-online-thumbs': { + help: `Downloads a fresh copy of the online file size cache, so changes there are immediately reflected`, + type: 'flag', + }, + 'skip-file-sizes': { help: `Skips preloading file sizes for images and additional files, which will be left blank in the build`, type: 'flag', }, + 'refresh-online-file-sizes': { + help: `Downloads a fresh copy of the online file size cache, so changes there are immediately reflected`, + type: 'flag', + }, + 'skip-media-validation': { help: `Skips checking and reporting missing and misplaced media files, which isn't necessary if you aren't adding or removing data or updating directories`, type: 'flag', @@ -417,6 +461,11 @@ async function main() { type: 'flag', }, + 'show-step-memory': { + help: `Include total process memory usage traces at the time each top-level build step ends. Use with --show-step-summary. This is mostly useful for programmer debugging!`, + type: 'flag', + }, + 'queue-size': { help: `Process more or fewer disk files at once to optimize performance or avoid I/O errors, unlimited if set to 0 (between 500 and 700 is usually a safe range for building HSMusic on Windows machines)\nDefaults to ${defaultQueueSize}`, type: 'value', @@ -439,14 +488,6 @@ async function main() { }, magick: {alias: 'magick-threads'}, - // This option is super slow and has the potential for bugs! It puts - // CacheableObject in a mode where every instance is a Proxy which will - // keep track of invalid property accesses. - 'show-invalid-property-accesses': { - help: `Report accesses at runtime to nonexistant properties on wiki data objects, at a dramatic performance cost\n(Internal/development use only)`, - type: 'flag', - }, - 'precache-mode': { help: `Change the way certain runtime-computed values are preemptively evaluated and cached\n\n` + @@ -485,6 +526,7 @@ async function main() { }); showStepStatusSummary = cliOptions['show-step-summary'] ?? false; + showStepMemoryInSummary = cliOptions['show-step-memory'] ?? false; if (cliOptions['help']) { console.log( @@ -565,7 +607,9 @@ async function main() { const showAggregateTraces = cliOptions['show-traces'] ?? false; const precacheMode = cliOptions['precache-mode'] ?? 'common'; - const showInvalidPropertyAccesses = cliOptions['show-invalid-property-accesses'] ?? false; + + const wantedURLSpecKeys = cliOptions['urls'] ?? []; + const showURLSpec = cliOptions['show-url-spec'] ?? false; // Makes writing nicer on the CPU and file I/O parts of the OS, with a // marginal performance deficit while waiting for file writes to finish @@ -888,14 +932,14 @@ async function main() { logInfo`Next scheduled is in ${whenst(delay - delta)}, or by using ${'--refresh-search'}.`; Object.assign(stepStatusSummary.buildSearchIndex, { status: STATUS_NOT_APPLICABLE, - annotation: `earlier than scheduled based on file mtime`, + annotation: `earlier than scheduled`, }); } else { logInfo`Search index hasn't been generated for a little while.`; logInfo`It'll be generated this build, then again in ${whenst(delay)}.`; Object.assign(stepStatusSummary.buildSearchIndex, { status: STATUS_NOT_STARTED, - annotation: `past when shceduled based on file mtime`, + annotation: `past when shceduled`, }); } @@ -929,7 +973,7 @@ async function main() { } if (stepStatusSummary.generateThumbnails.status === STATUS_NOT_STARTED) { - Object.assign(stepStatusSummary.loadThumbnailCache, { + Object.assign(stepStatusSummary.loadOfflineThumbnailCache, { status: STATUS_NOT_APPLICABLE, annotation: `using cache from thumbnail generation`, }); @@ -1081,6 +1125,7 @@ async function main() { status: STATUS_FATAL_ERROR, annotation: `--new-thumbs provided but regeneration not needed`, timeEnd: Date.now(), + memory: process.memoryUsage(), }); return false; @@ -1096,6 +1141,7 @@ async function main() { status: STATUS_FATAL_ERROR, annotation: mediaCachePathAnnotation, timeEnd: Date.now(), + memory: process.memoryUsage(), }); return false; @@ -1162,6 +1208,7 @@ async function main() { status: STATUS_FATAL_ERROR, annotation: mediaCachePathAnnotation, timeEnd: Date.now(), + memory: process.memoryUsage(), }); return false; @@ -1173,6 +1220,7 @@ async function main() { status: STATUS_DONE_CLEAN, annotation: mediaCachePathAnnotation, timeEnd: Date.now(), + memory: process.memoryUsage(), }); if (stepStatusSummary.migrateThumbnails.status === STATUS_NOT_STARTED) { @@ -1192,6 +1240,7 @@ async function main() { status: STATUS_FATAL_ERROR, annotation: `view log for details`, timeEnd: Date.now(), + memory: process.memoryUsage(), }); return false; @@ -1203,6 +1252,7 @@ async function main() { Object.assign(stepStatusSummary.migrateThumbnails, { status: STATUS_DONE_CLEAN, timeEnd: Date.now(), + memory: process.memoryUsage(), }); return true; @@ -1217,16 +1267,17 @@ async function main() { }; if ( - stepStatusSummary.loadThumbnailCache.status === STATUS_NOT_STARTED && + stepStatusSummary.loadOfflineThumbnailCache.status === STATUS_NOT_STARTED && stepStatusSummary.generateThumbnails.status === STATUS_NOT_STARTED ) { - throw new Error(`Unable to continue with both loadThumbnailCache and generateThumbnails`); + throw new Error(`Unable to continue with both loadOfflineThumbnailCache and generateThumbnails`); } let thumbsCache; - if (stepStatusSummary.loadThumbnailCache.status === STATUS_NOT_STARTED) { - Object.assign(stepStatusSummary.loadThumbnailCache, { + // TODO: Skip this step if we're using online thumbs + if (stepStatusSummary.loadOfflineThumbnailCache.status === STATUS_NOT_STARTED) { + Object.assign(stepStatusSummary.loadOfflineThumbnailCache, { status: STATUS_STARTED_NOT_DONE, timeStart: Date.now(), }); @@ -1242,10 +1293,11 @@ async function main() { logError`that you'll be good to go and don't need to process thumbnails` logError`again!`; - Object.assign(stepStatusSummary.loadThumbnailCache, { + Object.assign(stepStatusSummary.loadOfflineThumbnailCache, { status: STATUS_FATAL_ERROR, annotation: `cache does not exist`, timeEnd: Date.now(), + memory: process.memoryUsage(), }); return false; @@ -1259,10 +1311,11 @@ async function main() { logError`to help you out with troubleshooting!`; logError`${'https://hsmusic.wiki/discord/'}`; - Object.assign(stepStatusSummary.loadThumbnailCache, { + Object.assign(stepStatusSummary.loadOfflineThumbnailCache, { status: STATUS_FATAL_ERROR, annotation: `cache malformed or unreadable`, timeEnd: Date.now(), + memory: process.memoryUsage(), }); return false; @@ -1271,9 +1324,10 @@ async function main() { logInfo`Thumbnail cache file successfully read.`; - Object.assign(stepStatusSummary.loadThumbnailCache, { + Object.assign(stepStatusSummary.loadOfflineThumbnailCache, { status: STATUS_DONE_CLEAN, timeEnd: Date.now(), + memory: process.memoryUsage(), }); logInfo`Skipping thumbnail generation.`; @@ -1301,6 +1355,7 @@ async function main() { status: STATUS_FATAL_ERROR, annotation: `view log for details`, timeEnd: Date.now(), + memory: process.memoryUsage(), }); return false; @@ -1309,6 +1364,7 @@ async function main() { Object.assign(stepStatusSummary.generateThumbnails, { status: STATUS_DONE_CLEAN, timeEnd: Date.now(), + memory: process.memoryUsage(), }); if (thumbsOnly) { @@ -1320,10 +1376,6 @@ async function main() { thumbsCache = {}; } - if (showInvalidPropertyAccesses) { - CacheableObject.DEBUG_SLOW_TRACK_INVALID_PROPERTIES = true; - } - Object.assign(stepStatusSummary.loadDataFiles, { status: STATUS_STARTED_NOT_DONE, timeStart: Date.now(), @@ -1346,6 +1398,7 @@ async function main() { status: STATUS_FATAL_ERROR, annotation: `javascript error - view log for details`, timeEnd: Date.now(), + memory: process.memoryUsage(), }); return false; @@ -1385,6 +1438,7 @@ async function main() { status: STATUS_FATAL_ERROR, annotation: `error loading data files`, timeEnd: Date.now(), + memory: process.memoryUsage(), }); return false; @@ -1487,6 +1541,7 @@ async function main() { status: STATUS_FATAL_ERROR, annotation: `wiki info object not available`, timeEnd: Date.now(), + memory: process.memoryUsage(), }); return false; @@ -1499,6 +1554,7 @@ async function main() { Object.assign(stepStatusSummary.loadDataFiles, { status: STATUS_DONE_CLEAN, timeEnd: Date.now(), + memory: process.memoryUsage(), }); } else { logWarn`This might indicate some fields in the YAML data weren't formatted`; @@ -1513,6 +1569,7 @@ async function main() { status: STATUS_HAS_WARNINGS, annotation: `view log for details`, timeEnd: Date.now(), + memory: process.memoryUsage(), }); } } @@ -1526,11 +1583,12 @@ async function main() { timeStart: Date.now(), }); - linkWikiDataArrays(wikiData); + linkWikiDataArrays(wikiData, {bindFind, bindReverse}); Object.assign(stepStatusSummary.linkWikiDataArrays, { status: STATUS_DONE_CLEAN, timeEnd: Date.now(), + memory: process.memoryUsage(), }); if (precacheMode === 'common') { @@ -1602,6 +1660,7 @@ async function main() { status: STATUS_FATAL_ERROR, annotation: `see log for details`, timeEnd: Date.now(), + memory: process.memoryUsage(), }); return false; @@ -1610,11 +1669,10 @@ async function main() { Object.assign(stepStatusSummary.precacheCommonData, { status: STATUS_DONE_CLEAN, timeEnd: Date.now(), + memory: process.memoryUsage(), }); } - const urls = generateURLs(urlSpec); - // Filter out any things with duplicate directories throughout the data, // warning about them too. @@ -1632,6 +1690,7 @@ async function main() { Object.assign(stepStatusSummary.reportDirectoryErrors, { status: STATUS_DONE_CLEAN, timeEnd: Date.now(), + memory: process.memoryUsage(), }); } catch (aggregate) { if (!paragraph) console.log(''); @@ -1649,6 +1708,7 @@ async function main() { status: STATUS_FATAL_ERROR, annotation: `duplicate directories found`, timeEnd: Date.now(), + memory: process.memoryUsage(), }); return false; @@ -1676,6 +1736,7 @@ async function main() { Object.assign(stepStatusSummary.filterReferenceErrors, { status: STATUS_DONE_CLEAN, timeEnd: Date.now(), + memory: process.memoryUsage(), }); } catch (error) { if (!paragraph) console.log(''); @@ -1693,6 +1754,7 @@ async function main() { status: STATUS_HAS_WARNINGS, annotation: `view log for details`, timeEnd: Date.now(), + memory: process.memoryUsage(), }); } } @@ -1712,6 +1774,7 @@ async function main() { Object.assign(stepStatusSummary.reportContentTextErrors, { status: STATUS_DONE_CLEAN, timeEnd: Date.now(), + memory: process.memoryUsage(), }); } catch (error) { if (!paragraph) console.log(''); @@ -1728,6 +1791,7 @@ async function main() { status: STATUS_HAS_WARNINGS, annotation: `view log for details`, timeEnd: Date.now(), + memory: process.memoryUsage(), }); } } @@ -1740,11 +1804,12 @@ async function main() { timeStart: Date.now(), }); - sortWikiDataArrays(yamlDataSteps, wikiData); + sortWikiDataArrays(yamlDataSteps, wikiData, {bindFind, bindReverse}); Object.assign(stepStatusSummary.sortWikiDataArrays, { status: STATUS_DONE_CLEAN, timeEnd: Date.now(), + memory: process.memoryUsage(), }); if (precacheMode === 'all') { @@ -1768,6 +1833,7 @@ async function main() { Object.assign(stepStatusSummary.precacheAllData, { status: STATUS_DONE_CLEAN, timeEnd: Date.now(), + memory: process.memoryUsage(), }); } @@ -1779,6 +1845,354 @@ async function main() { } } + Object.assign(stepStatusSummary.loadURLFiles, { + status: STATUS_STARTED_NOT_DONE, + timeStart: Date.now(), + }); + + let internalURLSpec = {}; + + try { + let aggregate; + ({aggregate, result: internalURLSpec} = + await processURLSpecFromFile(internalDefaultURLSpecFile)); + + aggregate.close(); + } catch (error) { + niceShowAggregate(error); + logError`Couldn't load internal default URL spec.`; + logError`This is required to build the wiki, so stopping here.`; + fileIssue(); + + Object.assign(stepStatusSummary.loadURLFiles, { + status: STATUS_FATAL_ERROR, + annotation: `see log for details`, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + + return false; + } + + // We'll mutate this as we load other url spec files. + const urlSpec = structuredClone(internalURLSpec); + + const allURLSpecDataFiles = + (await readdir(dataPath)) + .filter(name => + name.startsWith('urls') && + ['.json', '.yaml'].includes(path.extname(name))) + .sort() /* Just in case... */ + .map(name => path.join(dataPath, name)); + + const getURLSpecKeyFromFile = file => { + const base = path.basename(file, path.extname(file)); + if (base === 'urls') { + return base; + } else { + return base.replace(/^urls-/, ''); + } + }; + + const isDefaultURLSpecFile = file => + getURLSpecKeyFromFile(file) === 'urls'; + + const overrideDefaultURLSpecFile = + allURLSpecDataFiles.find(file => isDefaultURLSpecFile(file)); + + const optionalURLSpecDataFiles = + allURLSpecDataFiles.filter(file => !isDefaultURLSpecFile(file)); + + const optionalURLSpecDataKeys = + optionalURLSpecDataFiles.map(file => getURLSpecKeyFromFile(file)); + + const selectedURLSpecDataKeys = optionalURLSpecDataKeys.slice(); + const selectedURLSpecDataFiles = optionalURLSpecDataFiles.slice(); + + const {removed: [unusedURLSpecDataKeys]} = + filterMultipleArrays( + selectedURLSpecDataKeys, + selectedURLSpecDataFiles, + (key, _file) => wantedURLSpecKeys.includes(key)); + + if (!empty(selectedURLSpecDataKeys)) { + logInfo`Using these optional URL specs: ${selectedURLSpecDataKeys.join(', ')}`; + if (!empty(unusedURLSpecDataKeys)) { + logInfo`Other available optional URL specs: ${unusedURLSpecDataKeys.join(', ')}`; + } + } else if (!empty(unusedURLSpecDataKeys)) { + logInfo`Not using any optional URL specs.`; + logInfo`These are available with --urls: ${unusedURLSpecDataKeys.join(', ')}`; + } + + if (overrideDefaultURLSpecFile) { + try { + let aggregate; + let overrideDefaultURLSpec; + + ({aggregate, result: overrideDefaultURLSpec} = + await processURLSpecFromFile(overrideDefaultURLSpecFile)); + + aggregate.close(); + + ({aggregate} = + applyURLSpecOverriding(overrideDefaultURLSpec, urlSpec)); + + aggregate.close(); + } catch (error) { + niceShowAggregate(error); + logError`Errors loading this data repo's ${'urls.yaml'} file.`; + logError`This provides essential overrides for this wiki,`; + logError`so stopping here. Debug the errors to continue.`; + + Object.assign(stepStatusSummary.loadURLFiles, { + status: STATUS_FATAL_ERROR, + annotation: `see log for details`, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + + return false; + } + } + + const processURLSpecsAggregate = + openAggregate({message: `Errors processing URL specs`}); + + const selectedURLSpecs = + processURLSpecsAggregate.receive( + await Promise.all( + selectedURLSpecDataFiles + .map(file => processURLSpecFromFile(file)))); + + for (const selectedURLSpec of selectedURLSpecs) { + processURLSpecsAggregate.receive( + applyURLSpecOverriding(selectedURLSpec, urlSpec)); + } + + try { + processURLSpecsAggregate.close(); + } catch (error) { + niceShowAggregate(error); + logWarn`There were errors loading the optional URL specs you`; + logWarn`selected using ${'--urls'}. Since they might misfunction,`; + logWarn`debug the errors or remove the failing ones from ${'--urls'}.`; + + Object.assign(stepStatusSummary.loadURLFiles, { + status: STATUS_FATAL_ERROR, + annotation: `see log for details`, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + + return false; + } + + if (showURLSpec) { + if (!paragraph) console.log(''); + + logInfo`Here's the final URL spec, via ${'--show-url-spec'}:` + console.log(urlSpec); + console.log(''); + + paragraph = true; + } + + Object.assign(stepStatusSummary.loadURLFiles, { + status: STATUS_DONE_CLEAN, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + + if (!getOrigin(urlSpec.thumb.prefix)) { + Object.assign(stepStatusSummary.loadOnlineThumbnailCache, { + status: STATUS_NOT_APPLICABLE, + annotation: `using offline thumbs`, + }); + } + + if (getOrigin(urlSpec.media.prefix)) { + Object.assign(stepStatusSummary.preloadFileSizes, { + status: STATUS_NOT_APPLICABLE, + annotation: `using online media`, + }); + } else { + Object.assign(stepStatusSummary.loadOnlineFileSizeCache, { + status: STATUS_NOT_APPLICABLE, + annotation: `using offline media`, + }); + } + + applyLocalizedWithBaseDirectory(urlSpec); + + const urls = generateURLs(urlSpec); + + if (stepStatusSummary.loadOnlineThumbnailCache.status === STATUS_NOT_STARTED) loadOnlineThumbnailCache: { + Object.assign(stepStatusSummary.loadOnlineThumbnailCache, { + status: STATUS_STARTED_NOT_DONE, + timeStart: Date.now(), + }); + + let onlineThumbsCache = null; + + const cacheFile = path.join(wikiCachePath, 'online-thumbnail-cache.json'); + + let readError = null; + let writeError = null; + + if (!cliOptions['refresh-online-thumbs']) { + try { + onlineThumbsCache = JSON.parse(await readFile(cacheFile)); + } catch (caughtError) { + readError = caughtError; + } + } + + if (onlineThumbsCache) obliterateLocalCopy: { + if (!onlineThumbsCache._urlPrefix) { + // Well, it doesn't even count. + onlineThumbsCache = null; + break obliterateLocalCopy; + } + + if (onlineThumbsCache._urlPrefix !== urlSpec.thumb.prefix) { + logInfo`Local copy of online thumbs cache is for a different prefix.`; + logInfo`It'll be downloaded and replaced, for reuse next time.`; + paragraph = false; + + onlineThumbsCache = null; + break obliterateLocalCopy; + } + + let stats; + try { + stats = await stat(cacheFile); + } catch { + logInfo`Unable to get the stats of local copy of online thumbs cache...`; + logInfo`This is really weird, since we *were* able to read it...`; + logInfo`We're just going to try writing to it and download fresh!`; + paragraph = false; + + onlineThumbsCache = null; + break obliterateLocalCopy; + } + + const delta = Date.now() - stats.mtimeMs; + const minute = 60 * 1000; + const delay = 60 * minute; + + const whenst = duration => `~${Math.ceil(duration / minute)} min`; + + if (delta < delay) { + logInfo`Online thumbs cache was downloaded recently, skipping for this build.`; + logInfo`Next scheduled is in ${whenst(delay - delta)}, or by using ${'--refresh-online-thumbs'}.`; + paragraph = false; + + Object.assign(stepStatusSummary.loadOnlineThumbnailCache, { + status: STATUS_DONE_CLEAN, + annotation: `reusing local copy, earlier than scheduled`, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + + thumbsCache = onlineThumbsCache; + + break loadOnlineThumbnailCache; + } else { + logInfo`Online thumbs cache hasn't been downloaded for a little while.`; + logInfo`It'll be downloaded this build, then again in ${whenst(delay)}.`; + onlineThumbsCache = null; + paragraph = false; + } + } + + try { + await writeFile(cacheFile, stringifyCache(onlineThumbsCache)); + } catch (caughtError) { + writeError = caughtError; + } + + if (readError && writeError && readError.code !== 'ENOENT') { + console.error(readError); + logWarn`Wasn't able to read the local copy of the`; + logWarn`online thumbs cache file...`; + console.error(writeError); + logWarn`...or write to it, either.`; + logWarn`The online thumbs cache will be downloaded`; + logWarn`for every build until you investigate this path:`; + logWarn`${cacheFile}`; + paragraph = false; + } else if (readError && readError.code === 'ENOENT' && !writeError) { + logInfo`No local copy of online thumbs cache.`; + logInfo`It'll be downloaded this time and reused next time.`; + paragraph = false; + } else if (readError && readError.code === 'ENOENT' && writeError) { + console.error(writeError); + logWarn`Doesn't look like we can write a local copy of`; + logWarn`the offline thumbs cache, at this path:`; + logWarn`${cacheFile}`; + logWarn`The online thumbs cache will be downloaded`; + logWarn`for every build until you investigate that.`; + paragraph = false; + } + + const url = new URL(urlSpec.thumb.prefix); + url.pathname = path.posix.join(url.pathname, 'thumbnail-cache.json'); + + try { + onlineThumbsCache = await fetch(url).then(res => res.json()); + } catch (error) { + console.error(error); + logWarn`There was an error downloading the online thumbnail cache.`; + logWarn`The wiki will act as though no thumbs are available at all.`; + paragraph = false; + + Object.assign(stepStatusSummary.loadOnlineThumbnailCache, { + status: STATUS_HAS_WARNINGS, + annotation: `failed to download`, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + + onlineThumbsCache = {}; + thumbsCache = {}; + + break loadOnlineThumbnailCache; + } + + onlineThumbsCache._urlPrefix = urlSpec.thumb.prefix; + + thumbsCache = onlineThumbsCache; + + if (onlineThumbsCache && !writeError) { + try { + await writeFile(cacheFile, stringifyCache(onlineThumbsCache)); + } catch (error) { + console.error(error); + logWarn`There was an error saving a local copy of the`; + logWarn`online thumbnail cache. It'll be fetched again`; + logWarn`next time.`; + paragraph = false; + + Object.assign(stepStatusSummary.loadOnlineThumbnailCache, { + status: STATUS_HAS_WARNINGS, + annotation: `failed to download`, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + + break loadOnlineThumbnailCache; + } + } + + Object.assign(stepStatusSummary.loadOnlineThumbnailCache, { + status: STATUS_DONE_CLEAN, + timeStart: Date.now(), + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + } + const languageReloading = stepStatusSummary.watchLanguageFiles.status === STATUS_NOT_STARTED; @@ -1841,6 +2255,7 @@ async function main() { status: STATUS_FATAL_ERROR, annotation: `see log for details`, timeEnd: Date.now(), + memory: process.memoryUsage(), }); return false; @@ -1854,6 +2269,7 @@ async function main() { Object.assign(stepStatusSummary.loadInternalDefaultLanguage, { status: STATUS_DONE_CLEAN, timeEnd: Date.now(), + memory: process.memoryUsage(), }); let customLanguageWatchers; @@ -1933,6 +2349,7 @@ async function main() { status: STATUS_FATAL_ERROR, annotation: `see log for details`, timeEnd: Date.now(), + memory: process.memoryUsage(), }); errorLoadingCustomLanguages = true; @@ -1964,6 +2381,7 @@ async function main() { Object.assign(stepStatusSummary.watchLanguageFiles, { status: STATUS_DONE_CLEAN, timeEnd: Date.now(), + memory: process.memoryUsage(), }); } else { languages = {}; @@ -1987,11 +2405,13 @@ async function main() { status: STATUS_FATAL_ERROR, annotation: `see log for details`, timeEnd: Date.now(), + memory: process.memoryUsage(), }); } else { Object.assign(stepStatusSummary.loadLanguageFiles, { status: STATUS_DONE_CLEAN, timeEnd: Date.now(), + memory: process.memoryUsage(), }); } } @@ -2029,6 +2449,7 @@ async function main() { status: STATUS_FATAL_ERROR, annotation: `wiki specifies default language whose file is not available`, timeEnd: Date.now(), + memory: process.memoryUsage(), }); return false; @@ -2122,6 +2543,7 @@ async function main() { status: STATUS_DONE_CLEAN, annotation: finalDefaultLanguageAnnotation, timeEnd: Date.now(), + memory: process.memoryUsage(), }); let missingImagePaths; @@ -2144,85 +2566,225 @@ async function main() { Object.assign(stepStatusSummary.verifyImagePaths, { status: STATUS_DONE_CLEAN, timeEnd: Date.now(), + memory: process.memoryUsage(), }); } else if (empty(missingImagePaths)) { Object.assign(stepStatusSummary.verifyImagePaths, { status: STATUS_HAS_WARNINGS, annotation: `misplaced images detected`, timeEnd: Date.now(), + memory: process.memoryUsage(), }); } else if (empty(misplacedImagePaths)) { Object.assign(stepStatusSummary.verifyImagePaths, { status: STATUS_HAS_WARNINGS, annotation: `missing images detected`, timeEnd: Date.now(), + memory: process.memoryUsage(), }); } else { Object.assign(stepStatusSummary.verifyImagePaths, { status: STATUS_HAS_WARNINGS, annotation: `missing and misplaced images detected`, timeEnd: Date.now(), + memory: process.memoryUsage(), }); } } - let getSizeOfAdditionalFile; - let getSizeOfImagePath; + let getSizeOfMediaFile = () => null; + + const fileSizePreloader = + new FileSizePreloader({ + prefix: mediaPath, + }); + + if (stepStatusSummary.loadOnlineFileSizeCache.status === STATUS_NOT_STARTED) loadOnlineFileSizeCache: { + Object.assign(stepStatusSummary.loadOnlineFileSizeCache, { + status: STATUS_STARTED_NOT_DONE, + timeStart: Date.now(), + }); + + let onlineFileSizeCache = null; + + const makeFileSizeCacheAvailable = () => { + fileSizePreloader.loadFromCache(onlineFileSizeCache); + + getSizeOfMediaFile = p => + fileSizePreloader.getSizeOfPath( + path.resolve( + mediaPath, + decodeURIComponent(p).split('/').join(path.sep))); + }; + + const cacheFile = path.join(wikiCachePath, 'online-file-size-cache.json'); + + let readError = null; + let writeError = null; + + if (!cliOptions['refresh-online-file-sizes']) { + try { + onlineFileSizeCache = JSON.parse(await readFile(cacheFile)); + } catch (caughtError) { + readError = caughtError; + } + } + + if (onlineFileSizeCache) obliterateLocalCopy: { + if (!onlineFileSizeCache._urlPrefix) { + // Well, it doesn't even count. + onlineFileSizeCache = null; + break obliterateLocalCopy; + } + + if (onlineFileSizeCache._urlPrefix !== urlSpec.media.prefix) { + logInfo`Local copy of online file size cache is for a different prefix.`; + logInfo`It'll be downloaded and replaced, for reuse next time.`; + paragraph = false; + + onlineFileSizeCache = null; + break obliterateLocalCopy; + } + + let stats; + try { + stats = await stat(cacheFile); + } catch { + logInfo`Unable to get the stats of local copy of online file size cache...`; + logInfo`This is really weird, since we *were* able to read it...`; + logInfo`We're just going to try writing to it and download fresh!`; + paragraph = false; + + onlineFileSizeCache = null; + break obliterateLocalCopy; + } + + const delta = Date.now() - stats.mtimeMs; + const minute = 60 * 1000; + const delay = 60 * minute; + + const whenst = duration => `~${Math.ceil(duration / minute)} min`; + + if (delta < delay) { + logInfo`Online file size cache was downloaded recently, skipping for this build.`; + logInfo`Next scheduled is in ${whenst(delay - delta)}, or by using ${'--refresh-online-file-sizes'}.`; + paragraph = false; + + Object.assign(stepStatusSummary.loadOnlineFileSizeCache, { + status: STATUS_DONE_CLEAN, + annotation: `reusing local copy, earlier than scheduled`, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + + delete onlineFileSizeCache._urlPrefix; + + makeFileSizeCacheAvailable(); + + break loadOnlineFileSizeCache; + } else { + logInfo`Online file size hasn't been downloaded for a little while.`; + logInfo`It'll be downloaded this build, then again in ${whenst(delay)}.`; + onlineFileSizeCache = null; + paragraph = false; + } + } + + try { + await writeFile(cacheFile, stringifyCache(onlineFileSizeCache)); + } catch (caughtError) { + writeError = caughtError; + } + + if (readError && writeError && readError.code !== 'ENOENT') { + console.error(readError); + logWarn`Wasn't able to read the local copy of the`; + logWarn`online file size cache file...`; + console.error(writeError); + logWarn`...or write to it, either.`; + logWarn`The online file size cache will be downloaded`; + logWarn`for every build until you investigate this path:`; + logWarn`${cacheFile}`; + paragraph = false; + } else if (readError && readError.code === 'ENOENT' && !writeError) { + logInfo`No local copy of online file size cache.`; + logInfo`It'll be downloaded this time and reused next time.`; + paragraph = false; + } else if (readError && readError.code === 'ENOENT' && writeError) { + console.error(writeError); + logWarn`Doesn't look like we can write a local copy of`; + logWarn`the offline file size cache, at this path:`; + logWarn`${cacheFile}`; + logWarn`The online file size cache will be downloaded`; + logWarn`for every build until you investigate that.`; + paragraph = false; + } + + const url = new URL(urlSpec.media.prefix); + url.pathname = path.posix.join(url.pathname, 'file-size-cache.json'); + + try { + onlineFileSizeCache = await fetch(url).then(res => res.json()); + } catch (error) { + console.error(error); + logWarn`There was an error downloading the online file size cache.`; + logWarn`The wiki will act as though no file sizes are available at all.`; + paragraph = false; + + Object.assign(stepStatusSummary.loadOnlineFileSizeCache, { + status: STATUS_HAS_WARNINGS, + annotation: `failed to download`, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + + break loadOnlineFileSizeCache; + } + + makeFileSizeCacheAvailable(); + + onlineFileSizeCache._urlPrefix = urlSpec.media.prefix; + + if (onlineFileSizeCache && !writeError) { + try { + await writeFile(cacheFile, stringifyCache(onlineFileSizeCache)); + } catch (error) { + console.error(error); + logWarn`There was an error saving a local copy of the`; + logWarn`online file size cache. It'll be fetched again`; + logWarn`next time.`; + paragraph = false; + + Object.assign(stepStatusSummary.loadOnlineFileSizeCache, { + status: STATUS_HAS_WARNINGS, + annotation: `failed to download`, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + + break loadOnlineFileSizeCache; + } + } + + Object.assign(stepStatusSummary.loadOnlineFileSizeCache, { + status: STATUS_DONE_CLEAN, + timeStart: Date.now(), + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + } - if (stepStatusSummary.preloadFileSizes.status === STATUS_NOT_APPLICABLE) { - getSizeOfAdditionalFile = () => null; - getSizeOfImagePath = () => null; - } else if (stepStatusSummary.preloadFileSizes.status === STATUS_NOT_STARTED) { + if (stepStatusSummary.preloadFileSizes.status === STATUS_NOT_STARTED) { Object.assign(stepStatusSummary.preloadFileSizes, { status: STATUS_STARTED_NOT_DONE, timeStart: Date.now(), }); - const fileSizePreloader = new FileSizePreloader(); - - // File sizes of additional files need to be precalculated before we can - // actually reference 'em in site building, so get those loading right - // away. We actually need to keep track of two things here - the on-device - // file paths we're actually reading, and the corresponding on-site media - // paths that will be exposed in site build code. We'll build a mapping - // function between them so that when site code requests a site path, - // it'll get the size of the file at the corresponding device path. - const additionalFilePaths = [ - ...wikiData.albumData.flatMap((album) => - [ - ...(album.additionalFiles ?? []), - ...album.tracks.flatMap((track) => [ - ...(track.additionalFiles ?? []), - ...(track.sheetMusicFiles ?? []), - ...(track.midiProjectFiles ?? []), - ]), - ] - .flatMap((fileGroup) => fileGroup.files ?? []) - .map((file) => ({ - device: path.join( - mediaPath, - urls - .from('media.root') - .toDevice('media.albumAdditionalFile', album.directory, file) - ), - media: urls - .from('media.root') - .to('media.albumAdditionalFile', album.directory, file), - })) - ), - ]; - - // Same dealio for images. Since just about any image can be embedded and - // we can't super easily know which ones are referenced at runtime, just - // cheat and get file sizes for all images under media. (This includes - // additional files which are images.) - const imageFilePaths = + const mediaFilePaths = await traverse(mediaPath, { pathStyle: 'device', filterDir: dir => dir !== '.git', - filterFile: file => - ['.png', '.gif', '.jpg'].includes(path.extname(file)) && - !isThumb(file), + filterFile: file => !isThumb(file), }).then(files => files .map(file => ({ device: file, @@ -2232,28 +2794,19 @@ async function main() { .to('media.path', path.relative(mediaPath, file).split(path.sep).join('/')), }))); - const getSizeOfMediaFileHelper = paths => (mediaPath) => { - const pair = paths.find(({media}) => media === mediaPath); + getSizeOfMediaFile = mediaPath => { + const pair = mediaFilePaths.find(({media}) => media === mediaPath); if (!pair) return null; return fileSizePreloader.getSizeOfPath(pair.device); }; - getSizeOfAdditionalFile = getSizeOfMediaFileHelper(additionalFilePaths); - getSizeOfImagePath = getSizeOfMediaFileHelper(imageFilePaths); - - logInfo`Preloading filesizes for ${additionalFilePaths.length} additional files...`; + logInfo`Preloading file sizes for ${mediaFilePaths.length} media files...`; - fileSizePreloader.loadPaths(...additionalFilePaths.map((path) => path.device)); - await fileSizePreloader.waitUntilDoneLoading(); - - logInfo`Preloading filesizes for ${imageFilePaths.length} full-resolution images...`; - paragraph = false; - - fileSizePreloader.loadPaths(...imageFilePaths.map((path) => path.device)); + fileSizePreloader.loadPaths(...mediaFilePaths.map(path => path.device)); await fileSizePreloader.waitUntilDoneLoading(); if (fileSizePreloader.hasErrored) { - logWarn`Some media files couldn't be read for preloading filesizes.`; + logWarn`Some media files couldn't be read for preloading file sizes.`; logWarn`This means the wiki won't display file sizes for these files.`; logWarn`Investigate missing or unreadable files to get that fixed!`; @@ -2261,16 +2814,50 @@ async function main() { status: STATUS_HAS_WARNINGS, annotation: `see log for details`, timeEnd: Date.now(), + memory: process.memoryUsage(), }); } else { - logInfo`Done preloading filesizes without any errors - nice!`; + logInfo`Done preloading file sizes without any errors - nice!`; paragraph = false; Object.assign(stepStatusSummary.preloadFileSizes, { status: STATUS_DONE_CLEAN, timeEnd: Date.now(), + memory: process.memoryUsage(), }); } + + // TODO: kinda jank that this is out of band of any particular step, + // even though it's operationally a follow-up to preloadFileSizes + + let oopsCache = false; + saveFileSizeCache: { + let cache; + try { + cache = fileSizePreloader.saveAsCache(); + } catch (error) { + console.error(error); + logWarn`Couldn't compute file size preloader's cache.`; + oopsCache = true; + break saveFileSizeCache; + } + + const cacheFile = path.join(mediaPath, 'file-size-cache.json'); + + try { + await writeFile(cacheFile, stringifyCache(cache)); + } catch (error) { + console.error(error); + logWarn`Couldn't save preloaded file sizes to a cache file:`; + logWarn`${cacheFile}`; + oopsCache = true; + } + } + + if (oopsCache) { + logWarn`This won't affect the build, but this build should not be used`; + logWarn`as a model for another build accessing its media files online.`; + } } if (stepStatusSummary.buildSearchIndex.status === STATUS_NOT_STARTED) { @@ -2293,6 +2880,7 @@ async function main() { Object.assign(stepStatusSummary.buildSearchIndex, { status: STATUS_DONE_CLEAN, timeEnd: Date.now(), + memory: process.memoryUsage(), }); } catch (error) { if (!paragraph) console.log(''); @@ -2310,6 +2898,7 @@ async function main() { status: STATUS_HAS_WARNINGS, annotation: `see log for details`, timeEnd: Date.now(), + memory: process.memoryUsage(), }); } } @@ -2357,6 +2946,7 @@ async function main() { status: STATUS_FATAL_ERROR, message: `JavaScript error - view log for details`, timeEnd: Date.now(), + memory: process.memoryUsage(), }); return false; @@ -2368,6 +2958,7 @@ async function main() { Object.assign(stepStatusSummary.identifyWebRoutes, { status: STATUS_DONE_CLEAN, timeEnd: Date.now(), + memory: process.memoryUsage(), }); } @@ -2422,8 +3013,7 @@ async function main() { console.log(''); const universalUtilities = { - getSizeOfAdditionalFile, - getSizeOfImagePath, + getSizeOfMediaFile, defaultLanguage: finalDefaultLanguage, developersComment, @@ -2464,6 +3054,7 @@ async function main() { status: STATUS_FATAL_ERROR, message: `javascript error - view log for details`, timeEnd: Date.now(), + memory: process.memoryUsage(), }); return false; @@ -2474,6 +3065,7 @@ async function main() { status: STATUS_HAS_WARNINGS, annotation: `may not have completed - view log for details`, timeEnd: Date.now(), + memory: process.memoryUsage(), }); return false; @@ -2482,6 +3074,7 @@ async function main() { Object.assign(stepStatusSummary.performBuild, { status: STATUS_DONE_CLEAN, timeEnd: Date.now(), + memory: process.memoryUsage(), }); return true; @@ -2569,16 +3162,31 @@ if (true || isMain(import.meta.url) || path.basename(process.argv[1]) === 'hsmus const longestDurationLength = Math.max(...stepDurations.map(duration => duration.length)); + const stepMemories = + stepDetails.map(({memory}) => + (memory + ? Math.round(memory["heapUsed"] / 1024 / 1024) + 'MB' + : '-')); + + const longestMemoryLength = + Math.max(...stepMemories.map(memory => memory.length)); + for (let index = 0; index < stepDetails.length; index++) { const {name, status, annotation} = stepDetails[index]; const duration = stepDurations[index]; + const memory = stepMemories[index]; let message = (stepsNotClean[index] ? `!! ` : ` - `); - message += `(${duration})`.padStart(longestDurationLength + 2, ' '); + message += `(${duration} `.padStart(longestDurationLength + 2, ' '); + + if (showStepMemoryInSummary) { + message += ` ${memory})`.padStart(longestMemoryLength + 2, ' '); + } + message += ` `; message += `${name}: `.padEnd(longestNameLength + 4, '.'); message += ` `; @@ -2636,7 +3244,6 @@ if (true || isMain(import.meta.url) || path.basename(process.argv[1]) === 'hsmus } decorateTime.displayTime(); - CacheableObject.showInvalidAccesses(); process.exit(0); })(); |