diff options
Diffstat (limited to 'src/upd8.js')
| -rwxr-xr-x | src/upd8.js | 1385 |
1 files changed, 1135 insertions, 250 deletions
diff --git a/src/upd8.js b/src/upd8.js index 9e4ef4fb..86ecab69 100755 --- a/src/upd8.js +++ b/src/upd8.js @@ -34,22 +34,24 @@ import '#import-heck'; import {execSync} from 'node:child_process'; -import {readdir, readFile, stat} from 'node:fs/promises'; +import {readdir, readFile, stat, writeFile} from 'node:fs/promises'; import * as path from 'node:path'; import {fileURLToPath} from 'node:url'; import wrap from 'word-wrap'; -import {mapAggregate, showAggregate} from '#aggregate'; +import {mapAggregate, openAggregate, showAggregate} from '#aggregate'; import CacheableObject from '#cacheable-object'; +import {stringifyCache} from '#cli'; import {displayCompositeCacheAnalysis} from '#composite'; -import {bindFind, getAllFindSpecs} from '#find'; +import find, {bindFind, getAllFindSpecs} from '#find'; import {processLanguageFile, watchLanguageFile, internalDefaultStringsFile} from '#language'; import {isMain, traverse} from '#node-utils'; +import {bindReverse} from '#reverse'; import {writeSearchData} from '#search'; import {sortByName} from '#sort'; -import {generateURLs, urlSpec} from '#urls'; +import thingConstructors from '#things'; import {identifyAllWebRoutes} from '#web-routes'; import { @@ -66,13 +68,15 @@ import { import { filterReferenceErrors, - reportDirectoryErrors, reportContentTextErrors, + reportDirectoryErrors, + reportOrphanedArtworks, } from '#data-checks'; import { bindOpts, empty, + filterMultipleArrays, indentWrap as unboundIndentWrap, withEntries, } from '#sugar'; @@ -87,6 +91,15 @@ import genThumbs, { } from '#thumbs'; import { + applyLocalizedWithBaseDirectory, + applyURLSpecOverriding, + generateURLs, + getOrigin, + internalDefaultURLSpecFile, + processURLSpecFromFile, +} from '#urls'; + +import { getAllDataSteps, linkWikiDataArrays, loadYAMLDocumentsFromDataSteps, @@ -122,7 +135,8 @@ const defaultStepStatus = {status: STATUS_NOT_STARTED, annotation: null}; // Defined globally for quick access outside the main() function's contents. // This will be initialized and mutated over the course of main(). let stepStatusSummary; -let showStepStatusSummary = false; +let shouldShowStepStatusSummary = false; +let shouldShowStepMemoryInSummary = false; async function main() { Error.stackTraceLimit = Infinity; @@ -138,8 +152,8 @@ async function main() { {...defaultStepStatus, name: `migrate thumbnails`, for: ['thumbs']}, - loadThumbnailCache: - {...defaultStepStatus, name: `load thumbnail cache file`, + loadOfflineThumbnailCache: + {...defaultStepStatus, name: `load offline thumbnail cache file`, for: ['thumbs', 'build']}, generateThumbnails: @@ -162,6 +176,10 @@ async function main() { {...defaultStepStatus, name: `report directory errors`, for: ['verify']}, + reportOrphanedArtworks: + {...defaultStepStatus, name: `report orphaned artworks`, + for: ['verify']}, + filterReferenceErrors: {...defaultStepStatus, name: `filter reference errors`, for: ['verify']}, @@ -178,6 +196,21 @@ async function main() { {...defaultStepStatus, name: `precache nearly all data`, for: ['build']}, + sortWikiDataSourceFiles: + {...defaultStepStatus, name: `apply sorting rules to wiki data files`, + for: ['build']}, + + checkWikiDataSourceFileSorting: + {...defaultStepStatus, name: `check sorting rules against wiki data files`}, + + loadURLFiles: + {...defaultStepStatus, name: `load internal & custom url spec files`, + for: ['build']}, + + loadOnlineThumbnailCache: + {...defaultStepStatus, name: `load online thumbnail cache file`, + for: ['thumbs', 'build']}, + // TODO: This should be split into load/watch steps. loadInternalDefaultLanguage: {...defaultStepStatus, name: `load internal default language`, @@ -203,6 +236,10 @@ async function main() { {...defaultStepStatus, name: `preload file sizes`, for: ['build']}, + loadOnlineFileSizeCache: + {...defaultStepStatus, name: `load online file size cache file`, + for: ['build']}, + buildSearchIndex: {...defaultStepStatus, name: `generate search index`, for: ['build', 'search']}, @@ -246,6 +283,35 @@ async function main() { })); let selectedBuildModeFlag; + let sortInAdditionToBuild = false; + + // As an exception, --sort can be combined with another build mode. + if (selectedBuildModeFlags.length >= 2 && selectedBuildModeFlags.includes('sort')) { + sortInAdditionToBuild = true; + selectedBuildModeFlags.splice(selectedBuildModeFlags.indexOf('sort'), 1); + } + + if (sortInAdditionToBuild) { + Object.assign(stepStatusSummary.sortWikiDataSourceFiles, { + status: STATUS_NOT_STARTED, + annotation: `--sort provided with another build mode`, + }); + + Object.assign(stepStatusSummary.checkWikiDataSourceFileSorting, { + status: STATUS_NOT_APPLICABLE, + annotation: `--sort provided, dry run not applicable`, + }); + } else { + Object.assign(stepStatusSummary.sortWikiDataSourceFiles, { + status: STATUS_NOT_APPLICABLE, + annotation: `--sort not provided, dry run only`, + }); + + Object.assign(stepStatusSummary.checkWikiDataSourceFileSorting, { + status: STATUS_NOT_STARTED, + annotation: `--sort not provided, dry run applicable`, + }); + } if (empty(selectedBuildModeFlags)) { // No build mode selected. This is not a valid state for building the wiki, @@ -323,11 +389,36 @@ async function main() { type: 'value', }, + 'urls': { + help: `Specify which optional URL specs to use for this build, customizing where pages are generated or resources are accessed from`, + type: 'value', + }, + + 'show-url-spec': { + help: `Displays the entire computed URL spec, after the data folder's default override and optional specs are applied. This is mostly useful for progammer debugging!`, + type: 'flag', + }, + + 'skip-directory-validation': { + help: `Skips checking for duplicated directories, which speeds up the build but may cause the wiki to catch on fire`, + type: 'flag', + }, + + 'skip-orphaned-artwork-validation': { + help: `Skips checking for internally orphaned artworks, which is a bad idea, unless you're debugging those in particular`, + type: 'flag', + }, + 'skip-reference-validation': { help: `Skips checking and reporting reference errors, which speeds up the build but may silently allow erroneous data to pass through`, type: 'flag', }, + 'skip-content-text-validation': { + help: `Skips checking and reporting content text errors, which speeds up the build but may silently allow misformatted or mislinked content to pass through`, + type: 'flag', + }, + // Thum8nail gener8tion is *usually* something you want, 8ut it can 8e // kinda a pain to run every time, since it does necessit8te reading // every media file at run time. Pass this to skip it. @@ -353,11 +444,26 @@ async function main() { type: 'flag', }, + 'refresh-online-thumbs': { + help: `Downloads a fresh copy of the online file size cache, so changes there are immediately reflected`, + type: 'flag', + }, + 'skip-file-sizes': { help: `Skips preloading file sizes for images and additional files, which will be left blank in the build`, type: 'flag', }, + 'refresh-online-file-sizes': { + help: `Downloads a fresh copy of the online file size cache, so changes there are immediately reflected`, + type: 'flag', + }, + + 'skip-sorting-validation': { + help: `Skips checking the if custom sorting rules for this wiki are satisfied`, + type: 'flag', + }, + 'skip-media-validation': { help: `Skips checking and reporting missing and misplaced media files, which isn't necessary if you aren't adding or removing data or updating directories`, type: 'flag', @@ -407,6 +513,11 @@ async function main() { type: 'flag', }, + 'show-step-memory': { + help: `Include total process memory usage traces at the time each top-level build step ends. Use with --show-step-summary. This is mostly useful for programmer debugging!`, + type: 'flag', + }, + 'queue-size': { help: `Process more or fewer disk files at once to optimize performance or avoid I/O errors, unlimited if set to 0 (between 500 and 700 is usually a safe range for building HSMusic on Windows machines)\nDefaults to ${defaultQueueSize}`, type: 'value', @@ -429,14 +540,6 @@ async function main() { }, magick: {alias: 'magick-threads'}, - // This option is super slow and has the potential for bugs! It puts - // CacheableObject in a mode where every instance is a Proxy which will - // keep track of invalid property accesses. - 'show-invalid-property-accesses': { - help: `Report accesses at runtime to nonexistant properties on wiki data objects, at a dramatic performance cost\n(Internal/development use only)`, - type: 'flag', - }, - 'precache-mode': { help: `Change the way certain runtime-computed values are preemptively evaluated and cached\n\n` + @@ -474,7 +577,8 @@ async function main() { ...buildOptions, }); - showStepStatusSummary = cliOptions['show-step-summary'] ?? false; + shouldShowStepStatusSummary = cliOptions['show-step-summary'] ?? false; + shouldShowStepMemoryInSummary = cliOptions['show-step-memory'] ?? false; if (cliOptions['help']) { console.log( @@ -555,7 +659,9 @@ async function main() { const showAggregateTraces = cliOptions['show-traces'] ?? false; const precacheMode = cliOptions['precache-mode'] ?? 'common'; - const showInvalidPropertyAccesses = cliOptions['show-invalid-property-accesses'] ?? false; + + const wantedURLSpecKeys = cliOptions['urls'] ?? []; + const showURLSpec = cliOptions['show-url-spec'] ?? false; // Makes writing nicer on the CPU and file I/O parts of the OS, with a // marginal performance deficit while waiting for file writes to finish @@ -651,9 +757,13 @@ async function main() { step.annotation = `--${cliFlag} provided`; if (cliFlagWarning) { + if (!paragraph) console.log(''); + for (const line of cliFlagWarning.split('\n')) { logWarn(line); } + + paragraph = false; } for (const step of cliFlagDisablesSteps) { @@ -732,6 +842,27 @@ async function main() { } }; + fallbackStep('reportDirectoryErrors', { + default: 'perform', + cli: { + flag: 'skip-directory-validation', + negate: true, + warn: + `Skipping directory validation. If any directories are duplicated\n` + + `in data, the build will probably fail in unpredictable ways.`, + }, + }); + + fallbackStep('reportOrphanedArtworks', { + default: 'perform', + cli: { + flag: 'skip-orphaned-artwork-validation', + negate: true, + warn: + `Skipping orphaned artwork validation. Hopefully you're debugging!`, + }, + }); + fallbackStep('filterReferenceErrors', { default: 'perform', cli: { @@ -740,7 +871,19 @@ async function main() { warn: `Skipping reference validation. If any reference errors are present\n` + `in data, they will be silently passed along to the build.`, - } + }, + }); + + fallbackStep('reportContentTextErrors', { + default: 'perform', + cli: { + flag: 'skip-content-text-validation', + negate: true, + warn: + `Skipping content text validation. If any commentary or other content\n` + + `is misformatted or has bad links, it will be silently passed along\n` + + `to the build.`, + }, }); fallbackStep('generateThumbnails', { @@ -851,20 +994,32 @@ async function main() { logInfo`Next scheduled is in ${whenst(delay - delta)}, or by using ${'--refresh-search'}.`; Object.assign(stepStatusSummary.buildSearchIndex, { status: STATUS_NOT_APPLICABLE, - annotation: `earlier than scheduled based on file mtime`, + annotation: `earlier than scheduled`, }); } else { logInfo`Search index hasn't been generated for a little while.`; logInfo`It'll be generated this build, then again in ${whenst(delay)}.`; Object.assign(stepStatusSummary.buildSearchIndex, { status: STATUS_NOT_STARTED, - annotation: `past when shceduled based on file mtime`, + annotation: `past when shceduled`, }); } paragraph = false; } + fallbackStep('checkWikiDataSourceFileSorting', { + default: 'perform', + buildConfig: 'sort', + cli: { + flag: 'skip-sorting-validation', + negate: true, + warning: + `Skipping sorting validation. If any of this wiki's sorting rules are not\n` + + `satisfied, those errors will be silently passed along to the build.`, + }, + }); + fallbackStep('verifyImagePaths', { default: 'perform', buildConfig: 'mediaValidation', @@ -892,7 +1047,7 @@ async function main() { } if (stepStatusSummary.generateThumbnails.status === STATUS_NOT_STARTED) { - Object.assign(stepStatusSummary.loadThumbnailCache, { + Object.assign(stepStatusSummary.loadOfflineThumbnailCache, { status: STATUS_NOT_APPLICABLE, annotation: `using cache from thumbnail generation`, }); @@ -1044,6 +1199,7 @@ async function main() { status: STATUS_FATAL_ERROR, annotation: `--new-thumbs provided but regeneration not needed`, timeEnd: Date.now(), + memory: process.memoryUsage(), }); return false; @@ -1059,6 +1215,7 @@ async function main() { status: STATUS_FATAL_ERROR, annotation: mediaCachePathAnnotation, timeEnd: Date.now(), + memory: process.memoryUsage(), }); return false; @@ -1125,6 +1282,7 @@ async function main() { status: STATUS_FATAL_ERROR, annotation: mediaCachePathAnnotation, timeEnd: Date.now(), + memory: process.memoryUsage(), }); return false; @@ -1136,6 +1294,7 @@ async function main() { status: STATUS_DONE_CLEAN, annotation: mediaCachePathAnnotation, timeEnd: Date.now(), + memory: process.memoryUsage(), }); if (stepStatusSummary.migrateThumbnails.status === STATUS_NOT_STARTED) { @@ -1155,6 +1314,7 @@ async function main() { status: STATUS_FATAL_ERROR, annotation: `view log for details`, timeEnd: Date.now(), + memory: process.memoryUsage(), }); return false; @@ -1166,6 +1326,7 @@ async function main() { Object.assign(stepStatusSummary.migrateThumbnails, { status: STATUS_DONE_CLEAN, timeEnd: Date.now(), + memory: process.memoryUsage(), }); return true; @@ -1180,16 +1341,17 @@ async function main() { }; if ( - stepStatusSummary.loadThumbnailCache.status === STATUS_NOT_STARTED && + stepStatusSummary.loadOfflineThumbnailCache.status === STATUS_NOT_STARTED && stepStatusSummary.generateThumbnails.status === STATUS_NOT_STARTED ) { - throw new Error(`Unable to continue with both loadThumbnailCache and generateThumbnails`); + throw new Error(`Unable to continue with both loadOfflineThumbnailCache and generateThumbnails`); } let thumbsCache; - if (stepStatusSummary.loadThumbnailCache.status === STATUS_NOT_STARTED) { - Object.assign(stepStatusSummary.loadThumbnailCache, { + // TODO: Skip this step if we're using online thumbs + if (stepStatusSummary.loadOfflineThumbnailCache.status === STATUS_NOT_STARTED) { + Object.assign(stepStatusSummary.loadOfflineThumbnailCache, { status: STATUS_STARTED_NOT_DONE, timeStart: Date.now(), }); @@ -1205,10 +1367,11 @@ async function main() { logError`that you'll be good to go and don't need to process thumbnails` logError`again!`; - Object.assign(stepStatusSummary.loadThumbnailCache, { + Object.assign(stepStatusSummary.loadOfflineThumbnailCache, { status: STATUS_FATAL_ERROR, annotation: `cache does not exist`, timeEnd: Date.now(), + memory: process.memoryUsage(), }); return false; @@ -1222,10 +1385,11 @@ async function main() { logError`to help you out with troubleshooting!`; logError`${'https://hsmusic.wiki/discord/'}`; - Object.assign(stepStatusSummary.loadThumbnailCache, { + Object.assign(stepStatusSummary.loadOfflineThumbnailCache, { status: STATUS_FATAL_ERROR, annotation: `cache malformed or unreadable`, timeEnd: Date.now(), + memory: process.memoryUsage(), }); return false; @@ -1234,9 +1398,10 @@ async function main() { logInfo`Thumbnail cache file successfully read.`; - Object.assign(stepStatusSummary.loadThumbnailCache, { + Object.assign(stepStatusSummary.loadOfflineThumbnailCache, { status: STATUS_DONE_CLEAN, timeEnd: Date.now(), + memory: process.memoryUsage(), }); logInfo`Skipping thumbnail generation.`; @@ -1264,6 +1429,7 @@ async function main() { status: STATUS_FATAL_ERROR, annotation: `view log for details`, timeEnd: Date.now(), + memory: process.memoryUsage(), }); return false; @@ -1272,6 +1438,7 @@ async function main() { Object.assign(stepStatusSummary.generateThumbnails, { status: STATUS_DONE_CLEAN, timeEnd: Date.now(), + memory: process.memoryUsage(), }); if (thumbsOnly) { @@ -1283,10 +1450,6 @@ async function main() { thumbsCache = {}; } - if (showInvalidPropertyAccesses) { - CacheableObject.DEBUG_SLOW_TRACK_INVALID_PROPERTIES = true; - } - Object.assign(stepStatusSummary.loadDataFiles, { status: STATUS_STARTED_NOT_DONE, timeStart: Date.now(), @@ -1309,6 +1472,7 @@ async function main() { status: STATUS_FATAL_ERROR, annotation: `javascript error - view log for details`, timeEnd: Date.now(), + memory: process.memoryUsage(), }); return false; @@ -1348,6 +1512,7 @@ async function main() { status: STATUS_FATAL_ERROR, annotation: `error loading data files`, timeEnd: Date.now(), + memory: process.memoryUsage(), }); return false; @@ -1389,6 +1554,10 @@ async function main() { ? prop : wikiData[prop]); + if (array && empty(array)) { + return; + } + logInfo` - ${array?.length ?? colors.red('(Missing!)')} ${colors.normal(colors.dim(label))}`; } @@ -1415,9 +1584,14 @@ async function main() { logThings('newsData', 'news entries'); } logThings('staticPageData', 'static pages'); + logThings('sortingRules', 'sorting rules'); if (wikiData.homepageLayout) { logInfo` - ${1} homepage layout (${ - wikiData.homepageLayout.rows.length + wikiData.homepageLayout.sections.length + } sections, ${ + wikiData.homepageLayout.sections + .flatMap(section => section.rows) + .length } rows)`; } if (wikiData.wikiInfo) { @@ -1450,6 +1624,7 @@ async function main() { status: STATUS_FATAL_ERROR, annotation: `wiki info object not available`, timeEnd: Date.now(), + memory: process.memoryUsage(), }); return false; @@ -1462,6 +1637,7 @@ async function main() { Object.assign(stepStatusSummary.loadDataFiles, { status: STATUS_DONE_CLEAN, timeEnd: Date.now(), + memory: process.memoryUsage(), }); } else { logWarn`This might indicate some fields in the YAML data weren't formatted`; @@ -1476,6 +1652,7 @@ async function main() { status: STATUS_HAS_WARNINGS, annotation: `view log for details`, timeEnd: Date.now(), + memory: process.memoryUsage(), }); } } @@ -1489,11 +1666,12 @@ async function main() { timeStart: Date.now(), }); - linkWikiDataArrays(wikiData); + linkWikiDataArrays(wikiData, {bindFind, bindReverse}); Object.assign(stepStatusSummary.linkWikiDataArrays, { status: STATUS_DONE_CLEAN, timeEnd: Date.now(), + memory: process.memoryUsage(), }); if (precacheMode === 'common') { @@ -1565,6 +1743,7 @@ async function main() { status: STATUS_FATAL_ERROR, annotation: `see log for details`, timeEnd: Date.now(), + memory: process.memoryUsage(), }); return false; @@ -1573,51 +1752,88 @@ async function main() { Object.assign(stepStatusSummary.precacheCommonData, { status: STATUS_DONE_CLEAN, timeEnd: Date.now(), + memory: process.memoryUsage(), }); } - const urls = generateURLs(urlSpec); + // Check for things with duplicate directories throughout the data, + // and halt if any are found. - // Filter out any things with duplicate directories throughout the data, - // warning about them too. + if (stepStatusSummary.reportDirectoryErrors.status === STATUS_NOT_STARTED) { + Object.assign(stepStatusSummary.reportDirectoryErrors, { + status: STATUS_STARTED_NOT_DONE, + timeStart: Date.now(), + }); - Object.assign(stepStatusSummary.reportDirectoryErrors, { - status: STATUS_STARTED_NOT_DONE, - timeStart: Date.now(), - }); + try { + reportDirectoryErrors(wikiData, {getAllFindSpecs}); + logInfo`No duplicate directories found - nice!`; + paragraph = false; - try { - reportDirectoryErrors(wikiData, {getAllFindSpecs}); - logInfo`No duplicate directories found - nice!`; - paragraph = false; + Object.assign(stepStatusSummary.reportDirectoryErrors, { + status: STATUS_DONE_CLEAN, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + } catch (aggregate) { + if (!paragraph) console.log(''); + niceShowAggregate(aggregate); - Object.assign(stepStatusSummary.reportDirectoryErrors, { - status: STATUS_DONE_CLEAN, - timeEnd: Date.now(), - }); - } catch (aggregate) { - if (!paragraph) console.log(''); - niceShowAggregate(aggregate); + logWarn`The above duplicate directories were detected while reviewing data files.`; + logWarn`Since it's impossible to automatically determine which one's directory is`; + logWarn`correct, the build can't continue. Specify unique 'Directory' fields in`; + logWarn`some or all of these data entries to resolve the errors.`; - logWarn`The above duplicate directories were detected while reviewing data files.`; - logWarn`Since it's impossible to automatically determine which one's directory is`; - logWarn`correct, the build can't continue. Specify unique 'Directory' fields in`; - logWarn`some or all of these data entries to resolve the errors.`; + console.log(''); + paragraph = true; - console.log(''); - paragraph = true; + Object.assign(stepStatusSummary.reportDirectoryErrors, { + status: STATUS_FATAL_ERROR, + annotation: `duplicate directories found`, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); - Object.assign(stepStatusSummary.reportDirectoryErrors, { - status: STATUS_FATAL_ERROR, - annotation: `duplicate directories found`, - timeEnd: Date.now(), + return false; + } + } + + // Check for artwork objects which have been orphaned from their things, + // and halt if any are found. + + if (stepStatusSummary.reportOrphanedArtworks.status === STATUS_NOT_STARTED) { + Object.assign(stepStatusSummary.reportOrphanedArtworks, { + status: STATUS_STARTED_NOT_DONE, + timeStart: Date.now(), }); - return false; + try { + reportOrphanedArtworks(wikiData, {getAllFindSpecs}); + + Object.assign(stepStatusSummary.reportOrphanedArtworks, { + status: STATUS_DONE_CLEAN, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + } catch (aggregate) { + if (!paragraph) console.log(''); + niceShowAggregate(aggregate); + + logError`Failed to initialize artwork data connections properly.`; + fileIssue(); + + Object.assign(stepStatusSummary.reportOrphanedArtworks, { + status: STATUS_FATAL_ERROR, + annotation: `orphaned artworks found`, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + + return false; + } } - // Filter out any reference errors throughout the data, warning about them - // too. + // Filter out any reference errors throughout the data, warning about these. if (stepStatusSummary.filterReferenceErrors.status === STATUS_NOT_STARTED) { Object.assign(stepStatusSummary.filterReferenceErrors, { @@ -1626,7 +1842,7 @@ async function main() { }); const filterReferenceErrorsAggregate = - filterReferenceErrors(wikiData, {bindFind}); + filterReferenceErrors(wikiData, {find, bindFind}); try { filterReferenceErrorsAggregate.close(); @@ -1637,6 +1853,7 @@ async function main() { Object.assign(stepStatusSummary.filterReferenceErrors, { status: STATUS_DONE_CLEAN, timeEnd: Date.now(), + memory: process.memoryUsage(), }); } catch (error) { if (!paragraph) console.log(''); @@ -1654,6 +1871,7 @@ async function main() { status: STATUS_HAS_WARNINGS, annotation: `view log for details`, timeEnd: Date.now(), + memory: process.memoryUsage(), }); } } @@ -1673,6 +1891,7 @@ async function main() { Object.assign(stepStatusSummary.reportContentTextErrors, { status: STATUS_DONE_CLEAN, timeEnd: Date.now(), + memory: process.memoryUsage(), }); } catch (error) { if (!paragraph) console.log(''); @@ -1689,6 +1908,7 @@ async function main() { status: STATUS_HAS_WARNINGS, annotation: `view log for details`, timeEnd: Date.now(), + memory: process.memoryUsage(), }); } } @@ -1701,11 +1921,12 @@ async function main() { timeStart: Date.now(), }); - sortWikiDataArrays(yamlDataSteps, wikiData); + sortWikiDataArrays(yamlDataSteps, wikiData, {bindFind, bindReverse}); Object.assign(stepStatusSummary.sortWikiDataArrays, { status: STATUS_DONE_CLEAN, timeEnd: Date.now(), + memory: process.memoryUsage(), }); if (precacheMode === 'all') { @@ -1729,9 +1950,81 @@ async function main() { Object.assign(stepStatusSummary.precacheAllData, { status: STATUS_DONE_CLEAN, timeEnd: Date.now(), + memory: process.memoryUsage(), }); } + if (stepStatusSummary.sortWikiDataSourceFiles.status === STATUS_NOT_STARTED) { + Object.assign(stepStatusSummary.sortWikiDataSourceFiles, { + status: STATUS_STARTED_NOT_DONE, + timeStart: Date.now(), + }); + + const {SortingRule} = thingConstructors; + const results = + await Array.fromAsync(SortingRule.go({dataPath, wikiData})); + + if (results.some(result => result.changed)) { + logInfo`Updated data files to satisfy sorting.`; + logInfo`Restarting automatically, since that's now needed!`; + + Object.assign(stepStatusSummary.sortWikiDataSourceFiles, { + status: STATUS_DONE_CLEAN, + annotation: `changes cueing restart`, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + + return 'restart'; + } else { + logInfo`All sorting rules are satisfied. Nice!`; + paragraph = false; + + Object.assign(stepStatusSummary.sortWikiDataSourceFiles, { + status: STATUS_DONE_CLEAN, + annotation: `no changes needed`, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + } + } else if (stepStatusSummary.checkWikiDataSourceFileSorting.status === STATUS_NOT_STARTED) { + Object.assign(stepStatusSummary.checkWikiDataSourceFileSorting, { + status: STATUS_STARTED_NOT_DONE, + timeStart: Date.now(), + }); + + const {SortingRule} = thingConstructors; + const results = + await Array.fromAsync(SortingRule.go({dataPath, wikiData, dry: true})); + + const needed = results.filter(result => result.changed); + + if (empty(needed)) { + logInfo`All sorting rules are satisfied. Nice!`; + paragraph = false; + + Object.assign(stepStatusSummary.checkWikiDataSourceFileSorting, { + status: STATUS_DONE_CLEAN, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + } else { + logWarn`Some of this wiki's sorting rules currently aren't satisfied:`; + for (const {rule} of needed) { + logWarn`- ${rule.message}`; + } + logWarn`Run ${'hsmusic --sort'} to automatically update data files.`; + paragraph = false; + + Object.assign(stepStatusSummary.checkWikiDataSourceFileSorting, { + status: STATUS_HAS_WARNINGS, + annotation: `not all rules satisfied`, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + } + } + if (stepStatusSummary.performBuild.status === STATUS_NOT_APPLICABLE) { displayCompositeCacheAnalysis(); @@ -1740,6 +2033,354 @@ async function main() { } } + Object.assign(stepStatusSummary.loadURLFiles, { + status: STATUS_STARTED_NOT_DONE, + timeStart: Date.now(), + }); + + let internalURLSpec = {}; + + try { + let aggregate; + ({aggregate, result: internalURLSpec} = + await processURLSpecFromFile(internalDefaultURLSpecFile)); + + aggregate.close(); + } catch (error) { + niceShowAggregate(error); + logError`Couldn't load internal default URL spec.`; + logError`This is required to build the wiki, so stopping here.`; + fileIssue(); + + Object.assign(stepStatusSummary.loadURLFiles, { + status: STATUS_FATAL_ERROR, + annotation: `see log for details`, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + + return false; + } + + // We'll mutate this as we load other url spec files. + const urlSpec = structuredClone(internalURLSpec); + + const allURLSpecDataFiles = + (await readdir(dataPath)) + .filter(name => + name.startsWith('urls') && + ['.json', '.yaml'].includes(path.extname(name))) + .sort() /* Just in case... */ + .map(name => path.join(dataPath, name)); + + const getURLSpecKeyFromFile = file => { + const base = path.basename(file, path.extname(file)); + if (base === 'urls') { + return base; + } else { + return base.replace(/^urls-/, ''); + } + }; + + const isDefaultURLSpecFile = file => + getURLSpecKeyFromFile(file) === 'urls'; + + const overrideDefaultURLSpecFile = + allURLSpecDataFiles.find(file => isDefaultURLSpecFile(file)); + + const optionalURLSpecDataFiles = + allURLSpecDataFiles.filter(file => !isDefaultURLSpecFile(file)); + + const optionalURLSpecDataKeys = + optionalURLSpecDataFiles.map(file => getURLSpecKeyFromFile(file)); + + const selectedURLSpecDataKeys = optionalURLSpecDataKeys.slice(); + const selectedURLSpecDataFiles = optionalURLSpecDataFiles.slice(); + + const {removed: [unusedURLSpecDataKeys]} = + filterMultipleArrays( + selectedURLSpecDataKeys, + selectedURLSpecDataFiles, + (key, _file) => wantedURLSpecKeys.includes(key)); + + if (!empty(selectedURLSpecDataKeys)) { + logInfo`Using these optional URL specs: ${selectedURLSpecDataKeys.join(', ')}`; + if (!empty(unusedURLSpecDataKeys)) { + logInfo`Other available optional URL specs: ${unusedURLSpecDataKeys.join(', ')}`; + } + } else if (!empty(unusedURLSpecDataKeys)) { + logInfo`Not using any optional URL specs.`; + logInfo`These are available with --urls: ${unusedURLSpecDataKeys.join(', ')}`; + } + + if (overrideDefaultURLSpecFile) { + try { + let aggregate; + let overrideDefaultURLSpec; + + ({aggregate, result: overrideDefaultURLSpec} = + await processURLSpecFromFile(overrideDefaultURLSpecFile)); + + aggregate.close(); + + ({aggregate} = + applyURLSpecOverriding(overrideDefaultURLSpec, urlSpec)); + + aggregate.close(); + } catch (error) { + niceShowAggregate(error); + logError`Errors loading this data repo's ${'urls.yaml'} file.`; + logError`This provides essential overrides for this wiki,`; + logError`so stopping here. Debug the errors to continue.`; + + Object.assign(stepStatusSummary.loadURLFiles, { + status: STATUS_FATAL_ERROR, + annotation: `see log for details`, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + + return false; + } + } + + const processURLSpecsAggregate = + openAggregate({message: `Errors processing URL specs`}); + + const selectedURLSpecs = + processURLSpecsAggregate.receive( + await Promise.all( + selectedURLSpecDataFiles + .map(file => processURLSpecFromFile(file)))); + + for (const selectedURLSpec of selectedURLSpecs) { + processURLSpecsAggregate.receive( + applyURLSpecOverriding(selectedURLSpec, urlSpec)); + } + + try { + processURLSpecsAggregate.close(); + } catch (error) { + niceShowAggregate(error); + logWarn`There were errors loading the optional URL specs you`; + logWarn`selected using ${'--urls'}. Since they might misfunction,`; + logWarn`debug the errors or remove the failing ones from ${'--urls'}.`; + + Object.assign(stepStatusSummary.loadURLFiles, { + status: STATUS_FATAL_ERROR, + annotation: `see log for details`, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + + return false; + } + + if (showURLSpec) { + if (!paragraph) console.log(''); + + logInfo`Here's the final URL spec, via ${'--show-url-spec'}:` + console.log(urlSpec); + console.log(''); + + paragraph = true; + } + + Object.assign(stepStatusSummary.loadURLFiles, { + status: STATUS_DONE_CLEAN, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + + if (!getOrigin(urlSpec.thumb.prefix)) { + Object.assign(stepStatusSummary.loadOnlineThumbnailCache, { + status: STATUS_NOT_APPLICABLE, + annotation: `using offline thumbs`, + }); + } + + if (getOrigin(urlSpec.media.prefix)) { + Object.assign(stepStatusSummary.preloadFileSizes, { + status: STATUS_NOT_APPLICABLE, + annotation: `using online media`, + }); + } else { + Object.assign(stepStatusSummary.loadOnlineFileSizeCache, { + status: STATUS_NOT_APPLICABLE, + annotation: `using offline media`, + }); + } + + applyLocalizedWithBaseDirectory(urlSpec); + + const urls = generateURLs(urlSpec); + + if (stepStatusSummary.loadOnlineThumbnailCache.status === STATUS_NOT_STARTED) loadOnlineThumbnailCache: { + Object.assign(stepStatusSummary.loadOnlineThumbnailCache, { + status: STATUS_STARTED_NOT_DONE, + timeStart: Date.now(), + }); + + let onlineThumbsCache = null; + + const cacheFile = path.join(wikiCachePath, 'online-thumbnail-cache.json'); + + let readError = null; + let writeError = null; + + if (!cliOptions['refresh-online-thumbs']) { + try { + onlineThumbsCache = JSON.parse(await readFile(cacheFile)); + } catch (caughtError) { + readError = caughtError; + } + } + + if (onlineThumbsCache) obliterateLocalCopy: { + if (!onlineThumbsCache._urlPrefix) { + // Well, it doesn't even count. + onlineThumbsCache = null; + break obliterateLocalCopy; + } + + if (onlineThumbsCache._urlPrefix !== urlSpec.thumb.prefix) { + logInfo`Local copy of online thumbs cache is for a different prefix.`; + logInfo`It'll be downloaded and replaced, for reuse next time.`; + paragraph = false; + + onlineThumbsCache = null; + break obliterateLocalCopy; + } + + let stats; + try { + stats = await stat(cacheFile); + } catch { + logInfo`Unable to get the stats of local copy of online thumbs cache...`; + logInfo`This is really weird, since we *were* able to read it...`; + logInfo`We're just going to try writing to it and download fresh!`; + paragraph = false; + + onlineThumbsCache = null; + break obliterateLocalCopy; + } + + const delta = Date.now() - stats.mtimeMs; + const minute = 60 * 1000; + const delay = 60 * minute; + + const whenst = duration => `~${Math.ceil(duration / minute)} min`; + + if (delta < delay) { + logInfo`Online thumbs cache was downloaded recently, skipping for this build.`; + logInfo`Next scheduled is in ${whenst(delay - delta)}, or by using ${'--refresh-online-thumbs'}.`; + paragraph = false; + + Object.assign(stepStatusSummary.loadOnlineThumbnailCache, { + status: STATUS_DONE_CLEAN, + annotation: `reusing local copy, earlier than scheduled`, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + + thumbsCache = onlineThumbsCache; + + break loadOnlineThumbnailCache; + } else { + logInfo`Online thumbs cache hasn't been downloaded for a little while.`; + logInfo`It'll be downloaded this build, then again in ${whenst(delay)}.`; + onlineThumbsCache = null; + paragraph = false; + } + } + + try { + await writeFile(cacheFile, stringifyCache(onlineThumbsCache)); + } catch (caughtError) { + writeError = caughtError; + } + + if (readError && writeError && readError.code !== 'ENOENT') { + console.error(readError); + logWarn`Wasn't able to read the local copy of the`; + logWarn`online thumbs cache file...`; + console.error(writeError); + logWarn`...or write to it, either.`; + logWarn`The online thumbs cache will be downloaded`; + logWarn`for every build until you investigate this path:`; + logWarn`${cacheFile}`; + paragraph = false; + } else if (readError && readError.code === 'ENOENT' && !writeError) { + logInfo`No local copy of online thumbs cache.`; + logInfo`It'll be downloaded this time and reused next time.`; + paragraph = false; + } else if (readError && readError.code === 'ENOENT' && writeError) { + console.error(writeError); + logWarn`Doesn't look like we can write a local copy of`; + logWarn`the offline thumbs cache, at this path:`; + logWarn`${cacheFile}`; + logWarn`The online thumbs cache will be downloaded`; + logWarn`for every build until you investigate that.`; + paragraph = false; + } + + const url = new URL(urlSpec.thumb.prefix); + url.pathname = path.posix.join(url.pathname, 'thumbnail-cache.json'); + + try { + onlineThumbsCache = await fetch(url).then(res => res.json()); + } catch (error) { + console.error(error); + logWarn`There was an error downloading the online thumbnail cache.`; + logWarn`The wiki will act as though no thumbs are available at all.`; + paragraph = false; + + Object.assign(stepStatusSummary.loadOnlineThumbnailCache, { + status: STATUS_HAS_WARNINGS, + annotation: `failed to download`, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + + onlineThumbsCache = {}; + thumbsCache = {}; + + break loadOnlineThumbnailCache; + } + + onlineThumbsCache._urlPrefix = urlSpec.thumb.prefix; + + thumbsCache = onlineThumbsCache; + + if (onlineThumbsCache && !writeError) { + try { + await writeFile(cacheFile, stringifyCache(onlineThumbsCache)); + } catch (error) { + console.error(error); + logWarn`There was an error saving a local copy of the`; + logWarn`online thumbnail cache. It'll be fetched again`; + logWarn`next time.`; + paragraph = false; + + Object.assign(stepStatusSummary.loadOnlineThumbnailCache, { + status: STATUS_HAS_WARNINGS, + annotation: `failed to download`, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + + break loadOnlineThumbnailCache; + } + } + + Object.assign(stepStatusSummary.loadOnlineThumbnailCache, { + status: STATUS_DONE_CLEAN, + timeStart: Date.now(), + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + } + const languageReloading = stepStatusSummary.watchLanguageFiles.status === STATUS_NOT_STARTED; @@ -1802,6 +2443,7 @@ async function main() { status: STATUS_FATAL_ERROR, annotation: `see log for details`, timeEnd: Date.now(), + memory: process.memoryUsage(), }); return false; @@ -1815,6 +2457,7 @@ async function main() { Object.assign(stepStatusSummary.loadInternalDefaultLanguage, { status: STATUS_DONE_CLEAN, timeEnd: Date.now(), + memory: process.memoryUsage(), }); let customLanguageWatchers; @@ -1894,6 +2537,7 @@ async function main() { status: STATUS_FATAL_ERROR, annotation: `see log for details`, timeEnd: Date.now(), + memory: process.memoryUsage(), }); errorLoadingCustomLanguages = true; @@ -1925,6 +2569,7 @@ async function main() { Object.assign(stepStatusSummary.watchLanguageFiles, { status: STATUS_DONE_CLEAN, timeEnd: Date.now(), + memory: process.memoryUsage(), }); } else { languages = {}; @@ -1948,11 +2593,13 @@ async function main() { status: STATUS_FATAL_ERROR, annotation: `see log for details`, timeEnd: Date.now(), + memory: process.memoryUsage(), }); } else { Object.assign(stepStatusSummary.loadLanguageFiles, { status: STATUS_DONE_CLEAN, timeEnd: Date.now(), + memory: process.memoryUsage(), }); } } @@ -1990,6 +2637,7 @@ async function main() { status: STATUS_FATAL_ERROR, annotation: `wiki specifies default language whose file is not available`, timeEnd: Date.now(), + memory: process.memoryUsage(), }); return false; @@ -2083,6 +2731,7 @@ async function main() { status: STATUS_DONE_CLEAN, annotation: finalDefaultLanguageAnnotation, timeEnd: Date.now(), + memory: process.memoryUsage(), }); let missingImagePaths; @@ -2105,85 +2754,225 @@ async function main() { Object.assign(stepStatusSummary.verifyImagePaths, { status: STATUS_DONE_CLEAN, timeEnd: Date.now(), + memory: process.memoryUsage(), }); } else if (empty(missingImagePaths)) { Object.assign(stepStatusSummary.verifyImagePaths, { status: STATUS_HAS_WARNINGS, annotation: `misplaced images detected`, timeEnd: Date.now(), + memory: process.memoryUsage(), }); } else if (empty(misplacedImagePaths)) { Object.assign(stepStatusSummary.verifyImagePaths, { status: STATUS_HAS_WARNINGS, annotation: `missing images detected`, timeEnd: Date.now(), + memory: process.memoryUsage(), }); } else { Object.assign(stepStatusSummary.verifyImagePaths, { status: STATUS_HAS_WARNINGS, annotation: `missing and misplaced images detected`, timeEnd: Date.now(), + memory: process.memoryUsage(), }); } } - let getSizeOfAdditionalFile; - let getSizeOfImagePath; + let getSizeOfMediaFile = () => null; + + const fileSizePreloader = + new FileSizePreloader({ + prefix: mediaPath, + }); + + if (stepStatusSummary.loadOnlineFileSizeCache.status === STATUS_NOT_STARTED) loadOnlineFileSizeCache: { + Object.assign(stepStatusSummary.loadOnlineFileSizeCache, { + status: STATUS_STARTED_NOT_DONE, + timeStart: Date.now(), + }); + + let onlineFileSizeCache = null; + + const makeFileSizeCacheAvailable = () => { + fileSizePreloader.loadFromCache(onlineFileSizeCache); + + getSizeOfMediaFile = p => + fileSizePreloader.getSizeOfPath( + path.resolve( + mediaPath, + decodeURIComponent(p).split('/').join(path.sep))); + }; + + const cacheFile = path.join(wikiCachePath, 'online-file-size-cache.json'); + + let readError = null; + let writeError = null; + + if (!cliOptions['refresh-online-file-sizes']) { + try { + onlineFileSizeCache = JSON.parse(await readFile(cacheFile)); + } catch (caughtError) { + readError = caughtError; + } + } + + if (onlineFileSizeCache) obliterateLocalCopy: { + if (!onlineFileSizeCache._urlPrefix) { + // Well, it doesn't even count. + onlineFileSizeCache = null; + break obliterateLocalCopy; + } + + if (onlineFileSizeCache._urlPrefix !== urlSpec.media.prefix) { + logInfo`Local copy of online file size cache is for a different prefix.`; + logInfo`It'll be downloaded and replaced, for reuse next time.`; + paragraph = false; + + onlineFileSizeCache = null; + break obliterateLocalCopy; + } + + let stats; + try { + stats = await stat(cacheFile); + } catch { + logInfo`Unable to get the stats of local copy of online file size cache...`; + logInfo`This is really weird, since we *were* able to read it...`; + logInfo`We're just going to try writing to it and download fresh!`; + paragraph = false; + + onlineFileSizeCache = null; + break obliterateLocalCopy; + } + + const delta = Date.now() - stats.mtimeMs; + const minute = 60 * 1000; + const delay = 60 * minute; + + const whenst = duration => `~${Math.ceil(duration / minute)} min`; + + if (delta < delay) { + logInfo`Online file size cache was downloaded recently, skipping for this build.`; + logInfo`Next scheduled is in ${whenst(delay - delta)}, or by using ${'--refresh-online-file-sizes'}.`; + paragraph = false; + + Object.assign(stepStatusSummary.loadOnlineFileSizeCache, { + status: STATUS_DONE_CLEAN, + annotation: `reusing local copy, earlier than scheduled`, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); - if (stepStatusSummary.preloadFileSizes.status === STATUS_NOT_APPLICABLE) { - getSizeOfAdditionalFile = () => null; - getSizeOfImagePath = () => null; - } else if (stepStatusSummary.preloadFileSizes.status === STATUS_NOT_STARTED) { + delete onlineFileSizeCache._urlPrefix; + + makeFileSizeCacheAvailable(); + + break loadOnlineFileSizeCache; + } else { + logInfo`Online file size hasn't been downloaded for a little while.`; + logInfo`It'll be downloaded this build, then again in ${whenst(delay)}.`; + onlineFileSizeCache = null; + paragraph = false; + } + } + + try { + await writeFile(cacheFile, stringifyCache(onlineFileSizeCache)); + } catch (caughtError) { + writeError = caughtError; + } + + if (readError && writeError && readError.code !== 'ENOENT') { + console.error(readError); + logWarn`Wasn't able to read the local copy of the`; + logWarn`online file size cache file...`; + console.error(writeError); + logWarn`...or write to it, either.`; + logWarn`The online file size cache will be downloaded`; + logWarn`for every build until you investigate this path:`; + logWarn`${cacheFile}`; + paragraph = false; + } else if (readError && readError.code === 'ENOENT' && !writeError) { + logInfo`No local copy of online file size cache.`; + logInfo`It'll be downloaded this time and reused next time.`; + paragraph = false; + } else if (readError && readError.code === 'ENOENT' && writeError) { + console.error(writeError); + logWarn`Doesn't look like we can write a local copy of`; + logWarn`the offline file size cache, at this path:`; + logWarn`${cacheFile}`; + logWarn`The online file size cache will be downloaded`; + logWarn`for every build until you investigate that.`; + paragraph = false; + } + + const url = new URL(urlSpec.media.prefix); + url.pathname = path.posix.join(url.pathname, 'file-size-cache.json'); + + try { + onlineFileSizeCache = await fetch(url).then(res => res.json()); + } catch (error) { + console.error(error); + logWarn`There was an error downloading the online file size cache.`; + logWarn`The wiki will act as though no file sizes are available at all.`; + paragraph = false; + + Object.assign(stepStatusSummary.loadOnlineFileSizeCache, { + status: STATUS_HAS_WARNINGS, + annotation: `failed to download`, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + + break loadOnlineFileSizeCache; + } + + makeFileSizeCacheAvailable(); + + onlineFileSizeCache._urlPrefix = urlSpec.media.prefix; + + if (onlineFileSizeCache && !writeError) { + try { + await writeFile(cacheFile, stringifyCache(onlineFileSizeCache)); + } catch (error) { + console.error(error); + logWarn`There was an error saving a local copy of the`; + logWarn`online file size cache. It'll be fetched again`; + logWarn`next time.`; + paragraph = false; + + Object.assign(stepStatusSummary.loadOnlineFileSizeCache, { + status: STATUS_HAS_WARNINGS, + annotation: `failed to download`, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + + break loadOnlineFileSizeCache; + } + } + + Object.assign(stepStatusSummary.loadOnlineFileSizeCache, { + status: STATUS_DONE_CLEAN, + timeStart: Date.now(), + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + } + + if (stepStatusSummary.preloadFileSizes.status === STATUS_NOT_STARTED) { Object.assign(stepStatusSummary.preloadFileSizes, { status: STATUS_STARTED_NOT_DONE, timeStart: Date.now(), }); - const fileSizePreloader = new FileSizePreloader(); - - // File sizes of additional files need to be precalculated before we can - // actually reference 'em in site building, so get those loading right - // away. We actually need to keep track of two things here - the on-device - // file paths we're actually reading, and the corresponding on-site media - // paths that will be exposed in site build code. We'll build a mapping - // function between them so that when site code requests a site path, - // it'll get the size of the file at the corresponding device path. - const additionalFilePaths = [ - ...wikiData.albumData.flatMap((album) => - [ - ...(album.additionalFiles ?? []), - ...album.tracks.flatMap((track) => [ - ...(track.additionalFiles ?? []), - ...(track.sheetMusicFiles ?? []), - ...(track.midiProjectFiles ?? []), - ]), - ] - .flatMap((fileGroup) => fileGroup.files ?? []) - .map((file) => ({ - device: path.join( - mediaPath, - urls - .from('media.root') - .toDevice('media.albumAdditionalFile', album.directory, file) - ), - media: urls - .from('media.root') - .to('media.albumAdditionalFile', album.directory, file), - })) - ), - ]; - - // Same dealio for images. Since just about any image can be embedded and - // we can't super easily know which ones are referenced at runtime, just - // cheat and get file sizes for all images under media. (This includes - // additional files which are images.) - const imageFilePaths = + const mediaFilePaths = await traverse(mediaPath, { pathStyle: 'device', filterDir: dir => dir !== '.git', - filterFile: file => - ['.png', '.gif', '.jpg'].includes(path.extname(file)) && - !isThumb(file), + filterFile: file => !isThumb(file), }).then(files => files .map(file => ({ device: file, @@ -2193,28 +2982,19 @@ async function main() { .to('media.path', path.relative(mediaPath, file).split(path.sep).join('/')), }))); - const getSizeOfMediaFileHelper = paths => (mediaPath) => { - const pair = paths.find(({media}) => media === mediaPath); + getSizeOfMediaFile = mediaPath => { + const pair = mediaFilePaths.find(({media}) => media === mediaPath); if (!pair) return null; return fileSizePreloader.getSizeOfPath(pair.device); }; - getSizeOfAdditionalFile = getSizeOfMediaFileHelper(additionalFilePaths); - getSizeOfImagePath = getSizeOfMediaFileHelper(imageFilePaths); - - logInfo`Preloading filesizes for ${additionalFilePaths.length} additional files...`; - - fileSizePreloader.loadPaths(...additionalFilePaths.map((path) => path.device)); - await fileSizePreloader.waitUntilDoneLoading(); - - logInfo`Preloading filesizes for ${imageFilePaths.length} full-resolution images...`; - paragraph = false; + logInfo`Preloading file sizes for ${mediaFilePaths.length} media files...`; - fileSizePreloader.loadPaths(...imageFilePaths.map((path) => path.device)); + fileSizePreloader.loadPaths(...mediaFilePaths.map(path => path.device)); await fileSizePreloader.waitUntilDoneLoading(); if (fileSizePreloader.hasErrored) { - logWarn`Some media files couldn't be read for preloading filesizes.`; + logWarn`Some media files couldn't be read for preloading file sizes.`; logWarn`This means the wiki won't display file sizes for these files.`; logWarn`Investigate missing or unreadable files to get that fixed!`; @@ -2222,16 +3002,50 @@ async function main() { status: STATUS_HAS_WARNINGS, annotation: `see log for details`, timeEnd: Date.now(), + memory: process.memoryUsage(), }); } else { - logInfo`Done preloading filesizes without any errors - nice!`; + logInfo`Done preloading file sizes without any errors - nice!`; paragraph = false; Object.assign(stepStatusSummary.preloadFileSizes, { status: STATUS_DONE_CLEAN, timeEnd: Date.now(), + memory: process.memoryUsage(), }); } + + // TODO: kinda jank that this is out of band of any particular step, + // even though it's operationally a follow-up to preloadFileSizes + + let oopsCache = false; + saveFileSizeCache: { + let cache; + try { + cache = fileSizePreloader.saveAsCache(); + } catch (error) { + console.error(error); + logWarn`Couldn't compute file size preloader's cache.`; + oopsCache = true; + break saveFileSizeCache; + } + + const cacheFile = path.join(mediaPath, 'file-size-cache.json'); + + try { + await writeFile(cacheFile, stringifyCache(cache)); + } catch (error) { + console.error(error); + logWarn`Couldn't save preloaded file sizes to a cache file:`; + logWarn`${cacheFile}`; + oopsCache = true; + } + } + + if (oopsCache) { + logWarn`This won't affect the build, but this build should not be used`; + logWarn`as a model for another build accessing its media files online.`; + } } if (stepStatusSummary.buildSearchIndex.status === STATUS_NOT_STARTED) { @@ -2254,6 +3068,7 @@ async function main() { Object.assign(stepStatusSummary.buildSearchIndex, { status: STATUS_DONE_CLEAN, timeEnd: Date.now(), + memory: process.memoryUsage(), }); } catch (error) { if (!paragraph) console.log(''); @@ -2271,6 +3086,7 @@ async function main() { status: STATUS_HAS_WARNINGS, annotation: `see log for details`, timeEnd: Date.now(), + memory: process.memoryUsage(), }); } } @@ -2318,6 +3134,7 @@ async function main() { status: STATUS_FATAL_ERROR, message: `JavaScript error - view log for details`, timeEnd: Date.now(), + memory: process.memoryUsage(), }); return false; @@ -2329,6 +3146,7 @@ async function main() { Object.assign(stepStatusSummary.identifyWebRoutes, { status: STATUS_DONE_CLEAN, timeEnd: Date.now(), + memory: process.memoryUsage(), }); } @@ -2382,29 +3200,36 @@ async function main() { logInfo`Passing control over to build mode: ${selectedBuildModeFlag}`; console.log(''); + const universalUtilities = { + getSizeOfMediaFile, + + defaultLanguage: finalDefaultLanguage, + developersComment, + languages, + missingImagePaths, + thumbsCache, + urlSpec, + urls, + wikiData, + }; + try { buildModeResult = await selectedBuildMode.go({ cliOptions, + queueSize, + + universalUtilities, + ...universalUtilities, + dataPath, mediaPath, mediaCachePath, wikiCachePath, - queueSize, srcRootPath: __dirname, - defaultLanguage: finalDefaultLanguage, - languages, - missingImagePaths, - thumbsCache, - urls, - urlSpec, webRoutes: preparedWebRoutes, - wikiData, closeLanguageWatchers, - developersComment, - getSizeOfAdditionalFile, - getSizeOfImagePath, niceShowAggregate, }); } catch (error) { @@ -2417,6 +3242,7 @@ async function main() { status: STATUS_FATAL_ERROR, message: `javascript error - view log for details`, timeEnd: Date.now(), + memory: process.memoryUsage(), }); return false; @@ -2427,6 +3253,7 @@ async function main() { status: STATUS_HAS_WARNINGS, annotation: `may not have completed - view log for details`, timeEnd: Date.now(), + memory: process.memoryUsage(), }); return false; @@ -2435,6 +3262,7 @@ async function main() { Object.assign(stepStatusSummary.performBuild, { status: STATUS_DONE_CLEAN, timeEnd: Date.now(), + memory: process.memoryUsage(), }); return true; @@ -2445,126 +3273,67 @@ async function main() { if (true || isMain(import.meta.url) || path.basename(process.argv[1]) === 'hsmusic') { (async () => { let result; + let numRestarts = 0; const totalTimeStart = Date.now(); - try { - result = await main(); - } catch (error) { - if (error instanceof AggregateError) { - showAggregate(error); - } else if (error.cause) { - console.error(error); - showAggregate(error); - } else { - console.error(error); - } - } - - const totalTimeEnd = Date.now(); - - const formatDuration = timeDelta => { - const seconds = timeDelta / 1000; - - if (seconds > 90) { - const modSeconds = Math.floor(seconds % 60); - const minutes = Math.floor(seconds - seconds % 60) / 60; - return `${minutes}m${modSeconds}s`; - } - - if (seconds < 0.1) { - return 'instant'; + while (true) { + try { + result = await main(); + } catch (error) { + if (error instanceof AggregateError) { + showAggregate(error); + } else if (error.cause) { + console.error(error); + showAggregate(error); + } else { + console.error(error); + } } - const precision = (seconds > 1 ? 3 : 2); - return `${seconds.toPrecision(precision)}s`; - }; - - if (showStepStatusSummary) { - const totalDuration = formatDuration(totalTimeEnd - totalTimeStart); - - console.error(colors.bright(`Step summary:`)); - - const longestNameLength = - Math.max(... - Object.values(stepStatusSummary) - .map(({name}) => name.length)); - - const stepsNotClean = - Object.values(stepStatusSummary) - .map(({status}) => - status === STATUS_HAS_WARNINGS || - status === STATUS_FATAL_ERROR || - status === STATUS_STARTED_NOT_DONE); - - const anyStepsNotClean = - stepsNotClean.includes(true); - - const stepDetails = Object.values(stepStatusSummary); - - const stepDurations = - stepDetails.map(({status, timeStart, timeEnd}) => { - if ( - status === STATUS_NOT_APPLICABLE || - status === STATUS_NOT_STARTED || - status === STATUS_STARTED_NOT_DONE - ) { - return '-'; - } + if (result === 'restart') { + console.log(''); - if (typeof timeStart !== 'number' || typeof timeEnd !== 'number') { - return 'unknown'; + if (shouldShowStepStatusSummary) { + if (numRestarts >= 1) { + console.error(colors.bright(`Step summary since latest restart:`)); + } else { + console.error(colors.bright(`Step summary before restart:`)); } - return formatDuration(timeEnd - timeStart); - }); - - const longestDurationLength = - Math.max(...stepDurations.map(duration => duration.length)); - - for (let index = 0; index < stepDetails.length; index++) { - const {name, status, annotation} = stepDetails[index]; - const duration = stepDurations[index]; - - let message = - (stepsNotClean[index] - ? `!! ` - : ` - `); - - message += `(${duration})`.padStart(longestDurationLength + 2, ' '); - message += ` `; - message += `${name}: `.padEnd(longestNameLength + 4, '.'); - message += ` `; - message += status; - - if (annotation) { - message += ` (${annotation})`; + showStepStatusSummary(); + console.log(''); } - switch (status) { - case STATUS_DONE_CLEAN: - console.error(colors.green(message)); - break; - - case STATUS_NOT_STARTED: - case STATUS_NOT_APPLICABLE: - console.error(colors.dim(message)); - break; + if (numRestarts > 5) { + logError`A restart was cued, but we've restarted a bunch already.`; + logError`Exiting because this is probably a bug!`; + console.log(''); + break; + } else { + console.log(''); + logInfo`A restart was cued. This is probably normal, and required`; + logInfo`to load updated data files. Restarting automatically now!`; + console.log(''); + numRestarts++; + } + } else { + break; + } + } - case STATUS_HAS_WARNINGS: - case STATUS_STARTED_NOT_DONE: - console.error(colors.yellow(message)); - break; + if (shouldShowStepStatusSummary) { + if (numRestarts >= 1) { + console.error(colors.bright(`Step summary after final restart:`)); + } else { + console.error(colors.bright(`Step summary:`)); + } - case STATUS_FATAL_ERROR: - console.error(colors.red(message)); - break; + const {anyStepsNotClean} = + showStepStatusSummary(); - default: - console.error(message); - break; - } - } + const totalTimeEnd = Date.now(); + const totalDuration = formatDuration(totalTimeEnd - totalTimeStart); console.error(colors.bright(`Done in ${totalDuration}.`)); @@ -2589,8 +3358,124 @@ if (true || isMain(import.meta.url) || path.basename(process.argv[1]) === 'hsmus } decorateTime.displayTime(); - CacheableObject.showInvalidAccesses(); process.exit(0); })(); } + +function formatDuration(timeDelta) { + const seconds = timeDelta / 1000; + + if (seconds > 90) { + const modSeconds = Math.floor(seconds % 60); + const minutes = Math.floor(seconds - seconds % 60) / 60; + return `${minutes}m${modSeconds}s`; + } + + if (seconds < 0.1) { + return 'instant'; + } + + const precision = (seconds > 1 ? 3 : 2); + return `${seconds.toPrecision(precision)}s`; +} + +function showStepStatusSummary() { + const longestNameLength = + Math.max(... + Object.values(stepStatusSummary) + .map(({name}) => name.length)); + + const stepsNotClean = + Object.values(stepStatusSummary) + .map(({status}) => + status === STATUS_HAS_WARNINGS || + status === STATUS_FATAL_ERROR || + status === STATUS_STARTED_NOT_DONE); + + const anyStepsNotClean = + stepsNotClean.includes(true); + + const stepDetails = Object.values(stepStatusSummary); + + const stepDurations = + stepDetails.map(({status, timeStart, timeEnd}) => { + if ( + status === STATUS_NOT_APPLICABLE || + status === STATUS_NOT_STARTED || + status === STATUS_STARTED_NOT_DONE + ) { + return '-'; + } + + if (typeof timeStart !== 'number' || typeof timeEnd !== 'number') { + return 'unknown'; + } + + return formatDuration(timeEnd - timeStart); + }); + + const longestDurationLength = + Math.max(...stepDurations.map(duration => duration.length)); + + const stepMemories = + stepDetails.map(({memory}) => + (memory + ? Math.round(memory["heapUsed"] / 1024 / 1024) + 'MB' + : '-')); + + const longestMemoryLength = + Math.max(...stepMemories.map(memory => memory.length)); + + for (let index = 0; index < stepDetails.length; index++) { + const {name, status, annotation} = stepDetails[index]; + const duration = stepDurations[index]; + const memory = stepMemories[index]; + + let message = + (stepsNotClean[index] + ? `!! ` + : ` - `); + + message += `(${duration} `.padStart(longestDurationLength + 2, ' '); + + if (shouldShowStepMemoryInSummary) { + message += ` ${memory})`.padStart(longestMemoryLength + 2, ' '); + } + + message += ` `; + message += `${name}: `.padEnd(longestNameLength + 4, '.'); + message += ` `; + message += status; + + if (annotation) { + message += ` (${annotation})`; + } + + switch (status) { + case STATUS_DONE_CLEAN: + console.error(colors.green(message)); + break; + + case STATUS_NOT_STARTED: + case STATUS_NOT_APPLICABLE: + console.error(colors.dim(message)); + break; + + case STATUS_HAS_WARNINGS: + case STATUS_STARTED_NOT_DONE: + console.error(colors.yellow(message)); + break; + + case STATUS_FATAL_ERROR: + console.error(colors.red(message)); + break; + + default: + console.error(message); + break; + } + } + + return {anyStepsNotClean}; +} |