diff options
Diffstat (limited to 'src/upd8.js')
| -rwxr-xr-x | src/upd8.js | 2044 |
1 files changed, 1608 insertions, 436 deletions
diff --git a/src/upd8.js b/src/upd8.js index f35f9e5f..ae072d5a 100755 --- a/src/upd8.js +++ b/src/upd8.js @@ -34,22 +34,25 @@ import '#import-heck'; import {execSync} from 'node:child_process'; -import {readdir, readFile} from 'node:fs/promises'; +import {readdir, readFile, stat, writeFile} from 'node:fs/promises'; import * as path from 'node:path'; import {fileURLToPath} from 'node:url'; import wrap from 'word-wrap'; -import {mapAggregate, showAggregate} from '#aggregate'; +import {mapAggregate, openAggregate, showAggregate} from '#aggregate'; import CacheableObject from '#cacheable-object'; +import {formatDuration, stringifyCache} from '#cli'; import {displayCompositeCacheAnalysis} from '#composite'; -import {bindFind, getAllFindSpecs} from '#find'; +import * as html from '#html'; +import find, {bindFind, getAllFindSpecs} from '#find'; import {processLanguageFile, watchLanguageFile, internalDefaultStringsFile} from '#language'; import {isMain, traverse} from '#node-utils'; +import {bindReverse} from '#reverse'; +import {writeSearchData} from '#search'; import {sortByName} from '#sort'; -import {empty, withEntries} from '#sugar'; -import {generateURLs, urlSpec} from '#urls'; +import thingConstructors from '#things'; import {identifyAllWebRoutes} from '#web-routes'; import { @@ -61,14 +64,24 @@ import { logError, parseOptions, progressCallAll, + showHelpForOptions as unboundShowHelpForOptions, } from '#cli'; import { filterReferenceErrors, - reportDirectoryErrors, reportContentTextErrors, + reportDirectoryErrors, + reportOrphanedArtworks, } from '#data-checks'; +import { + bindOpts, + empty, + filterMultipleArrays, + indentWrap as unboundIndentWrap, + withEntries, +} from '#sugar'; + import genThumbs, { CACHE_FILE as thumbsCacheFile, defaultMagickThreads, @@ -79,6 +92,15 @@ import genThumbs, { } from '#thumbs'; import { + applyLocalizedWithBaseDirectory, + applyURLSpecOverriding, + generateURLs, + getOrigin, + internalDefaultURLSpecFile, + processURLSpecFromFile, +} from '#urls'; + +import { getAllDataSteps, linkWikiDataArrays, loadYAMLDocumentsFromDataSteps, @@ -93,12 +115,10 @@ import * as buildModes from './write/build-modes/index.js'; const __dirname = path.dirname(fileURLToPath(import.meta.url)); -const CACHEBUST = 23; - let COMMIT; try { COMMIT = execSync('git log --format="%h %B" -n 1 HEAD', {cwd: __dirname}).toString().trim(); -} catch (error) { +} catch { COMMIT = '(failed to detect)'; } @@ -116,74 +136,138 @@ const defaultStepStatus = {status: STATUS_NOT_STARTED, annotation: null}; // Defined globally for quick access outside the main() function's contents. // This will be initialized and mutated over the course of main(). let stepStatusSummary; -let showStepStatusSummary = false; +let shouldShowStepStatusSummary = false; +let shouldShowStepMemoryInSummary = false; async function main() { Error.stackTraceLimit = Infinity; + let paragraph = true; + stepStatusSummary = { determineMediaCachePath: - {...defaultStepStatus, name: `determine media cache path`}, + {...defaultStepStatus, name: `determine media cache path`, + for: ['thumbs', 'build']}, migrateThumbnails: - {...defaultStepStatus, name: `migrate thumbnails`}, + {...defaultStepStatus, name: `migrate thumbnails`, + for: ['thumbs']}, - loadThumbnailCache: - {...defaultStepStatus, name: `load thumbnail cache file`}, + loadOfflineThumbnailCache: + {...defaultStepStatus, name: `load offline thumbnail cache file`, + for: ['thumbs', 'build']}, generateThumbnails: - {...defaultStepStatus, name: `generate thumbnails`}, + {...defaultStepStatus, name: `generate thumbnails`, + for: ['thumbs']}, loadDataFiles: - {...defaultStepStatus, name: `load and process data files`}, + {...defaultStepStatus, name: `load and process data files`, + for: ['build']}, linkWikiDataArrays: - {...defaultStepStatus, name: `link wiki data arrays`}, + {...defaultStepStatus, name: `link wiki data arrays`, + for: ['build']}, precacheCommonData: - {...defaultStepStatus, name: `precache common data`}, + {...defaultStepStatus, name: `precache common data`, + for: ['build']}, reportDirectoryErrors: - {...defaultStepStatus, name: `report directory errors`}, + {...defaultStepStatus, name: `report directory errors`, + for: ['verify']}, + + reportOrphanedArtworks: + {...defaultStepStatus, name: `report orphaned artworks`, + for: ['verify']}, filterReferenceErrors: - {...defaultStepStatus, name: `filter reference errors`}, + {...defaultStepStatus, name: `filter reference errors`, + for: ['verify']}, reportContentTextErrors: - {...defaultStepStatus, name: `report content text errors`}, + {...defaultStepStatus, name: `report content text errors`, + for: ['verify']}, sortWikiDataArrays: - {...defaultStepStatus, name: `sort wiki data arrays`}, + {...defaultStepStatus, name: `sort wiki data arrays`, + for: ['build']}, precacheAllData: - {...defaultStepStatus, name: `precache nearly all data`}, + {...defaultStepStatus, name: `precache nearly all data`, + for: ['build']}, + + sortWikiDataSourceFiles: + {...defaultStepStatus, name: `apply sorting rules to wiki data files`, + for: ['build']}, + + checkWikiDataSourceFileSorting: + {...defaultStepStatus, name: `check sorting rules against wiki data files`}, + + loadURLFiles: + {...defaultStepStatus, name: `load internal & custom url spec files`, + for: ['build']}, + + loadOnlineThumbnailCache: + {...defaultStepStatus, name: `load online thumbnail cache file`, + for: ['thumbs', 'build']}, // TODO: This should be split into load/watch steps. loadInternalDefaultLanguage: - {...defaultStepStatus, name: `load internal default language`}, + {...defaultStepStatus, name: `load internal default language`, + for: ['build']}, loadLanguageFiles: - {...defaultStepStatus, name: `statically load custom language files`}, + {...defaultStepStatus, name: `statically load custom language files`, + for: ['build']}, watchLanguageFiles: - {...defaultStepStatus, name: `watch custom language files`}, + {...defaultStepStatus, name: `watch custom language files`, + for: ['build']}, initializeDefaultLanguage: - {...defaultStepStatus, name: `initialize default language`}, + {...defaultStepStatus, name: `initialize default language`, + for: ['build']}, verifyImagePaths: - {...defaultStepStatus, name: `verify missing/misplaced image paths`}, + {...defaultStepStatus, name: `verify missing/misplaced image paths`, + for: ['verify']}, preloadFileSizes: - {...defaultStepStatus, name: `preload file sizes`}, + {...defaultStepStatus, name: `preload file sizes`, + for: ['build']}, + + loadOnlineFileSizeCache: + {...defaultStepStatus, name: `load online file size cache file`, + for: ['build']}, + + buildSearchIndex: + {...defaultStepStatus, name: `generate search index`, + for: ['build', 'search']}, identifyWebRoutes: - {...defaultStepStatus, name: `identify web routes`}, + {...defaultStepStatus, name: `identify web routes`, + for: ['build']}, performBuild: - {...defaultStepStatus, name: `perform selected build mode`}, + {...defaultStepStatus, name: `perform selected build mode`, + for: ['build']}, }; + const stepsWhich = condition => + Object.entries(stepStatusSummary) + .filter(([_key, value]) => condition(value)) + .map(([key]) => key); + + /* eslint-disable-next-line no-unused-vars */ + const stepsFor = (...which) => + stepsWhich(step => + which.some(w => step.for?.includes(w))); + + const stepsNotFor = (...which) => + stepsWhich(step => + which.every(w => !step.for?.includes(w))); + const defaultQueueSize = 500; const buildModeFlagOptions = ( @@ -200,21 +284,53 @@ async function main() { })); let selectedBuildModeFlag; - let usingDefaultBuildMode; + let sortInAdditionToBuild = false; + + // As an exception, --sort can be combined with another build mode. + if (selectedBuildModeFlags.length >= 2 && selectedBuildModeFlags.includes('sort')) { + sortInAdditionToBuild = true; + selectedBuildModeFlags.splice(selectedBuildModeFlags.indexOf('sort'), 1); + } + + if (sortInAdditionToBuild) { + Object.assign(stepStatusSummary.sortWikiDataSourceFiles, { + status: STATUS_NOT_STARTED, + annotation: `--sort provided with another build mode`, + }); + + Object.assign(stepStatusSummary.checkWikiDataSourceFileSorting, { + status: STATUS_NOT_APPLICABLE, + annotation: `--sort provided, dry run not applicable`, + }); + } else { + Object.assign(stepStatusSummary.sortWikiDataSourceFiles, { + status: STATUS_NOT_APPLICABLE, + annotation: `--sort not provided, dry run only`, + }); + + Object.assign(stepStatusSummary.checkWikiDataSourceFileSorting, { + status: STATUS_NOT_STARTED, + annotation: `--sort not provided, dry run applicable`, + }); + } if (empty(selectedBuildModeFlags)) { - selectedBuildModeFlag = 'static-build'; - usingDefaultBuildMode = true; + // No build mode selected. This is not a valid state for building the wiki, + // but we want to let access to --help, so we'll show a message about what + // to do later. + selectedBuildModeFlag = null; } else if (selectedBuildModeFlags.length > 1) { logError`Building multiple modes (${selectedBuildModeFlags.join(', ')}) at once not supported.`; - logError`Please specify a maximum of one build mode.`; + logError`Please specify one build mode.`; return false; } else { selectedBuildModeFlag = selectedBuildModeFlags[0]; - usingDefaultBuildMode = false; } - const selectedBuildMode = buildModes[selectedBuildModeFlag]; + const selectedBuildMode = + (selectedBuildModeFlag + ? buildModes[selectedBuildModeFlag] + : null); // This is about to get a whole lot more stuff put in it. const wikiData = { @@ -222,7 +338,10 @@ async function main() { listingTargetSpec, }; - const buildOptions = selectedBuildMode.getCLIOptions(); + const buildOptions = + (selectedBuildMode + ? selectedBuildMode.getCLIOptions() + : {}); const commonOptions = { 'help': { @@ -234,7 +353,7 @@ async function main() { // and like a jillion other things too. Pretty much everything which // makes an individual wiki what it is goes here! 'data-path': { - help: `Specify path to data directory, including YAML files that cover all info about wiki content, layout, and structure\n\nAlways required for wiki building, but may be provided via the HSMUSIC_DATA environment variable instead`, + help: `Specify path to data directory, including YAML files that cover all info about wiki content, layout, and structure\n\nAlways required for wiki building; may be provided via the HSMUSIC_DATA environment variable`, type: 'value', }, @@ -242,17 +361,17 @@ async function main() { // categorized; check out MEDIA_ALBUM_ART_DIRECTORY and other constants // near the top of this file (upd8.js). 'media-path': { - help: `Specify path to media directory, including album artwork and additional files, as well as custom site layout media and other media files for reference or linking in wiki content\n\nAlways required for wiki building, but may be provided via the HSMUSIC_MEDIA environment variable instead`, + help: `Specify path to media directory, including album artwork and additional files, as well as custom site layout media and other media files for reference or linking in wiki content\n\nAlways required for wiki building; may be provided via the HSMUSIC_MEDIA environment variable`, type: 'value', }, 'media-cache-path': { - help: `Specify path to media cache directory, including automatically generated thumbnails\n\nThis usually doesn't need to be provided, and will be inferred either by loading "media-cache" from --cache-path, or by adding "-cache" to the end of the media directory\n\nAlso may be provided via the HSMUSIC_MEDIA_CACHE environment variable`, + help: `Specify path to media cache directory, including automatically generated thumbnails\n\nThis usually doesn't need to be provided, and will be inferred either by loading "media-cache" from --cache-path, or by adding "-cache" to the end of the media directory\n\nMay be provided via the HSMUSIC_MEDIA_CACHE environment variable`, type: 'value', }, 'cache-path': { - help: `Specify path to general cache directory, usually containing generated thumbnails and assorted files reused between builds\n\nRequired for some features and may always be required if you're starting a new workspace\n\nAlso may be provided via the HSMUSIC_CACHE environment varaible`, + help: `Specify path to general cache directory, usually containing generated thumbnails and assorted files reused between builds\n\nAlways required for wiki building; may be provided via the HSMUSIC_CACHE environment varaible`, type: 'value', }, @@ -271,11 +390,36 @@ async function main() { type: 'value', }, + 'urls': { + help: `Specify which optional URL specs to use for this build, customizing where pages are generated or resources are accessed from`, + type: 'value', + }, + + 'show-url-spec': { + help: `Displays the entire computed URL spec, after the data folder's default override and optional specs are applied. This is mostly useful for progammer debugging!`, + type: 'flag', + }, + + 'skip-directory-validation': { + help: `Skips checking for duplicated directories, which speeds up the build but may cause the wiki to catch on fire`, + type: 'flag', + }, + + 'skip-orphaned-artwork-validation': { + help: `Skips checking for internally orphaned artworks, which is a bad idea, unless you're debugging those in particular`, + type: 'flag', + }, + 'skip-reference-validation': { help: `Skips checking and reporting reference errors, which speeds up the build but may silently allow erroneous data to pass through`, type: 'flag', }, + 'skip-content-text-validation': { + help: `Skips checking and reporting content text errors, which speeds up the build but may silently allow misformatted or mislinked content to pass through`, + type: 'flag', + }, + // Thum8nail gener8tion is *usually* something you want, 8ut it can 8e // kinda a pain to run every time, since it does necessit8te reading // every media file at run time. Pass this to skip it. @@ -301,16 +445,41 @@ async function main() { type: 'flag', }, + 'refresh-online-thumbs': { + help: `Downloads a fresh copy of the online file size cache, so changes there are immediately reflected`, + type: 'flag', + }, + 'skip-file-sizes': { help: `Skips preloading file sizes for images and additional files, which will be left blank in the build`, type: 'flag', }, + 'refresh-online-file-sizes': { + help: `Downloads a fresh copy of the online file size cache, so changes there are immediately reflected`, + type: 'flag', + }, + + 'skip-sorting-validation': { + help: `Skips checking the if custom sorting rules for this wiki are satisfied`, + type: 'flag', + }, + 'skip-media-validation': { help: `Skips checking and reporting missing and misplaced media files, which isn't necessary if you aren't adding or removing data or updating directories`, type: 'flag', }, + 'refresh-search': { + help: `Generate the text search index this build, instead of waiting for the automatic delay`, + type: 'flag', + }, + + 'skip-search': { + help: `Skip creation of the text search index no matter what, even if it'd normally be scheduled for now`, + type: 'flag', + }, + // Just working on data entries and not interested in actually // generating site HTML yet? This flag will cut execution off right // 8efore any site 8uilding actually happens. @@ -345,6 +514,16 @@ async function main() { type: 'flag', }, + 'show-step-memory': { + help: `Include total process memory usage traces at the time each top-level build step ends. Use with --show-step-summary. This is mostly useful for programmer debugging!`, + type: 'flag', + }, + + 'skip-self-diagnosis': { + help: `Disable some runtime validation for the wiki's own code, which speeds up long builds, but may allow unpredicted corner cases to fail strangely and silently`, + type: 'flag', + }, + 'queue-size': { help: `Process more or fewer disk files at once to optimize performance or avoid I/O errors, unlimited if set to 0 (between 500 and 700 is usually a safe range for building HSMusic on Windows machines)\nDefaults to ${defaultQueueSize}`, type: 'value', @@ -367,14 +546,6 @@ async function main() { }, magick: {alias: 'magick-threads'}, - // This option is super slow and has the potential for bugs! It puts - // CacheableObject in a mode where every instance is a Proxy which will - // keep track of invalid property accesses. - 'show-invalid-property-accesses': { - help: `Report accesses at runtime to nonexistant properties on wiki data objects, at a dramatic performance cost\n(Internal/development use only)`, - type: 'flag', - }, - 'precache-mode': { help: `Change the way certain runtime-computed values are preemptively evaluated and cached\n\n` + @@ -390,6 +561,18 @@ async function main() { }, }; + const indentWrap = + bindOpts(unboundIndentWrap, { + wrap, + }); + + const showHelpForOptions = + bindOpts(unboundShowHelpForOptions, { + [bindOpts.bindIndex]: 0, + indentWrap, + sort: sortByName, + }); + const cliOptions = await parseOptions(process.argv.slice(2), { // We don't want to error when we receive these options, so specify them // here, even though we won't be doing anything with them later. @@ -400,89 +583,72 @@ async function main() { ...buildOptions, }); - if (cliOptions['help']) { - const indentWrap = (spaces, str) => wrap(str, {width: 60 - spaces, indent: ' '.repeat(spaces)}); - - const showOptions = (msg, options) => { - console.log(colors.bright(msg)); - - const entries = Object.entries(options); - const sortedOptions = sortByName(entries - .map(([name, descriptor]) => ({name, descriptor}))); - - if (!sortedOptions.length) { - console.log(`(No options available)`) - } - - let justInsertedPaddingLine = false; - - for (const {name, descriptor} of sortedOptions) { - if (descriptor.alias) { - continue; - } - - const aliases = entries - .filter(([_name, {alias}]) => alias === name) - .map(([name]) => name); - - let wrappedHelp, wrappedHelpLines = 0; - if (descriptor.help) { - wrappedHelp = indentWrap(4, descriptor.help); - wrappedHelpLines = wrappedHelp.split('\n').length; - } - - if (wrappedHelpLines > 0 && !justInsertedPaddingLine) { - console.log(''); - } - - console.log(colors.bright(` --` + name) + - (aliases.length - ? ` (or: ${aliases.map(alias => colors.bright(`--` + alias)).join(', ')})` - : '') + - (descriptor.help - ? '' - : colors.dim(' (no help provided)'))); - - if (wrappedHelp) { - console.log(wrappedHelp); - } - - if (wrappedHelpLines > 1) { - console.log(''); - justInsertedPaddingLine = true; - } else { - justInsertedPaddingLine = false; - } - } - - if (!justInsertedPaddingLine) { - console.log(``); - } - }; + shouldShowStepStatusSummary = cliOptions['show-step-summary'] ?? false; + shouldShowStepMemoryInSummary = cliOptions['show-step-memory'] ?? false; + if (cliOptions['help']) { console.log( - colors.bright(`hsmusic (aka. Homestuck Music Wiki)\n`) + + colors.bright(`hsmusic (aka. Homestuck Music Wiki, HSMusic Wiki)\n`) + `static wiki software cataloguing collaborative creation\n`); - console.log(indentWrap(0, - `The \`hsmusic\` command provides basic control over all parts of generating user-visible HTML pages and website content/structure from provided data, media, and language directories.\n` + + console.log(indentWrap( + `The \`hsmusic\` command provides basic control over ` + + `all parts of generating user-visible HTML pages ` + + `and website content/structure ` + + `from provided data, media, and language directories.\n` + `\n` + `CLI options are divided into three groups:\n`)); - console.log(` 1) ` + indentWrap(4, - `Common options: These are shared by all build modes and always have the same essential behavior`).trim()); - console.log(` 2) ` + indentWrap(4, - `Build mode selection: One build mode may be selected (or else the default, --static-build, is used), and it decides which entire set of behavior to use for providing site content to the user`).trim()); - console.log(` 3) ` + indentWrap(4, - `Build options: Each build mode has a set of unique options which customize behavior for that build mode`).trim()); + + console.log(` 1) ` + indentWrap( + `Common options: ` + + `These are shared by all build modes ` + + `and always have the same essential behavior`, + {spaces: 4, bullet: true})); + + console.log(` 2) ` + indentWrap( + `Build mode selection: ` + + `One build mode should be selected, ` + + `and it decides the main set of behavior to use ` + + `for presenting or interacting with site content`, + {spaces: 4, bullet: true})); + + console.log(` 3) ` + indentWrap( + `Build options: ` + + `Each build mode has a set of unique options ` + + `which customize behavior for that build mode`, + {spaces: 4, bullet: true})); + console.log(``); - showOptions(`Common options`, commonOptions); - showOptions(`Build mode selection`, buildModeFlagOptions); + showHelpForOptions({ + heading: `Common options`, + options: commonOptions, + wrap, + }); + + showHelpForOptions({ + heading: `Build mode selection`, + options: buildModeFlagOptions, + wrap, + }); + + if (selectedBuildMode) { + showHelpForOptions({ + heading: `Build options for --${selectedBuildModeFlag}`, + options: buildOptions, + wrap, + }); + } else { + console.log( + `Specify a build mode and run with ${colors.bright('--help')} again for info\n` + + `about the options for that build mode.`); + } - if (buildOptions) { - showOptions(`Build options for --${selectedBuildModeFlag} (${ - usingDefaultBuildMode ? 'default' : 'selected' - })`, buildOptions); + for (const step of Object.values(stepStatusSummary)) { + Object.assign(step, { + status: STATUS_NOT_APPLICABLE, + annotation: `--help provided`, + }); } return true; @@ -496,12 +662,13 @@ async function main() { const thumbsOnly = cliOptions['thumbs-only'] ?? false; const noInput = cliOptions['no-input'] ?? false; - showStepStatusSummary = cliOptions['show-step-summary'] ?? false; - - const showAggregateTraces = cliOptions['show-traces'] ?? false; + const skipSelfDiagnosis = cliOptions['skip-self-diagnosis'] ?? false; + const showTraces = cliOptions['show-traces'] ?? false; const precacheMode = cliOptions['precache-mode'] ?? 'common'; - const showInvalidPropertyAccesses = cliOptions['show-invalid-property-accesses'] ?? false; + + const wantedURLSpecKeys = cliOptions['urls'] ?? []; + const showURLSpec = cliOptions['show-url-spec'] ?? false; // Makes writing nicer on the CPU and file I/O parts of the OS, with a // marginal performance deficit while waiting for file writes to finish @@ -519,10 +686,10 @@ async function main() { } if (!wikiCachePath) { - logWarn`No --cache-path option nor HSMUSIC_CACHE set; provide for more features`; + logError`${`Expected --cache-path option or HSMUSIC_CACHE to be set`}`; } - if (!dataPath || !mediaPath) { + if (!dataPath || !mediaPath || !wikiCachePath) { return false; } @@ -533,65 +700,90 @@ async function main() { status: STATUS_NOT_APPLICABLE, annotation: `--no-build provided`, }); - } else { - if (usingDefaultBuildMode) { - logInfo`No build mode specified, will use default: ${selectedBuildModeFlag}`; - } else { - logInfo`Will use specified build mode: ${selectedBuildModeFlag}`; - } } // Finish setting up defaults by combining information from all options. const _fallbackStep = (stepKey, { default: defaultValue, + cli: cliArg, + buildConfig: buildConfigKey = null, + }) => { + const buildConfig = selectedBuildMode?.config?.[buildConfigKey]; + const {[stepKey]: step} = stepStatusSummary; + + const cliEntries = + (cliArg === null || cliArg === undefined + ? [] + : Array.isArray(cliArg) + ? cliArg + : [cliArg]); - cli: { + for (const { flag: cliFlag = null, negate: cliFlagNegates = false, warn: cliFlagWarning = null, - } = {}, - - buildConfig: buildConfigKey, - }) => { - const {[buildConfigKey]: buildConfig} = selectedBuildMode.config; - const {[stepKey]: step} = stepStatusSummary; + disable: cliFlagDisablesSteps = [], + } of cliEntries) { + if (!cliOptions[cliFlag]) { + continue; + } - if (cliFlag && cliOptions[cliFlag]) { const cliPart = `--` + cliFlag; const modePart = `--` + selectedBuildModeFlag; + if (buildConfig?.applicable === false) { if (cliFlagNegates) { logWarn`${cliPart} provided, but ${modePart} already skips this step`; logWarn`Redundant option ${cliPart}`; + continue; } else { logWarn`${cliPart} provided, but this step isn't applicable for ${modePart}`; logWarn`Ignoring option ${cliPart}`; + continue; } - } else if (buildConfig?.required === true) { + } + + if (buildConfig?.required === true) { if (cliFlagNegates) { logWarn`${cliPart} provided, but ${modePart} requires this step`; logWarn`Ignoring option ${cliPart}`; + continue; } else { logWarn`${cliPart} provided, but ${modePart} already requires this step`; logWarn`Redundant option ${cliPart}`; + continue; } - } else { - step.status = - (cliFlagNegates - ? STATUS_NOT_APPLICABLE - : STATUS_NOT_STARTED); + } - step.annotation = `--${cliFlag} provided`; + step.status = + (cliFlagNegates + ? STATUS_NOT_APPLICABLE + : STATUS_NOT_STARTED); - if (cliFlagWarning) { - for (const line of cliFlagWarning.split('\n')) { - logWarn(line); - } + step.annotation = `--${cliFlag} provided`; + + if (cliFlagWarning) { + if (!paragraph) console.log(''); + + for (const line of cliFlagWarning.split('\n')) { + logWarn(line); } - return; + paragraph = false; } + + for (const step of cliFlagDisablesSteps) { + const summary = stepStatusSummary[step]; + if (summary.status === STATUS_NOT_APPLICABLE && summary.annotation) { + stepStatusSummary.performBuild.annotation += `; --${cliFlag} provided`; + } else { + summary.status = STATUS_NOT_APPLICABLE; + summary.annotation = `--${cliFlag} provided`; + } + } + + return; } if (buildConfig?.required === true) { @@ -619,12 +811,22 @@ async function main() { } switch (defaultValue) { - case 'skip': + case 'skip': { step.status = STATUS_NOT_APPLICABLE; - if (cliFlag && !cliFlagNegates) { - step.annotation = `--${cliFlag} not provided`; + + const enablingFlags = + cliEntries + .filter(({negate}) => !negate) + .map(({flag}) => flag); + + if (!empty(enablingFlags)) { + step.annotation = + enablingFlags.map(flag => `--${flag}`).join(', ') + + ` not provided`; } + break; + } case 'perform': break; @@ -647,32 +849,67 @@ async function main() { } }; + fallbackStep('reportDirectoryErrors', { + default: 'perform', + cli: { + flag: 'skip-directory-validation', + negate: true, + warn: + `Skipping directory validation. If any directories are duplicated\n` + + `in data, the build will probably fail in unpredictable ways.`, + }, + }); + + fallbackStep('reportOrphanedArtworks', { + default: 'perform', + cli: { + flag: 'skip-orphaned-artwork-validation', + negate: true, + warn: + `Skipping orphaned artwork validation. Hopefully you're debugging!`, + }, + }); + fallbackStep('filterReferenceErrors', { default: 'perform', - buildConfig: null, cli: { flag: 'skip-reference-validation', negate: true, warn: `Skipping reference validation. If any reference errors are present\n` + `in data, they will be silently passed along to the build.`, - } + }, }); - fallbackStep('generateThumbnails', { + fallbackStep('reportContentTextErrors', { default: 'perform', - buildConfig: 'thumbs', cli: { - flag: 'skip-thumbs', + flag: 'skip-content-text-validation', negate: true, + warn: + `Skipping content text validation. If any commentary or other content\n` + + `is misformatted or has bad links, it will be silently passed along\n` + + `to the build.`, }, }); + fallbackStep('generateThumbnails', { + default: 'perform', + buildConfig: 'thumbs', + cli: [ + {flag: 'thumbs-only', disable: stepsNotFor('thumbs')}, + {flag: 'skip-thumbs', negate: true}, + ], + }); + fallbackStep('migrateThumbnails', { default: 'skip', - buildConfig: null, cli: { flag: 'migrate-thumbs', + disable: [ + ...stepsNotFor('thumbs'), + 'generateThumbnails', + ], }, }); @@ -686,10 +923,110 @@ async function main() { }); fallbackStep('identifyWebRoutes', { - default: 'skip', + default: 'perform', buildConfig: 'webRoutes', }); + decideBuildSearchIndex: { + fallbackStep('buildSearchIndex', { + default: 'skip', + buildConfig: 'search', + cli: [ + {flag: 'refresh-search'}, + {flag: 'skip-search', negate: true}, + ], + }); + + if (cliOptions['refresh-search'] || cliOptions['skip-search']) { + if (cliOptions['refresh-search']) { + logInfo`${'--refresh-search'} provided, will generate search fresh this build.`; + } + + break decideBuildSearchIndex; + } + + if (stepStatusSummary.buildSearchIndex.status !== STATUS_NOT_APPLICABLE) { + break decideBuildSearchIndex; + } + + if (selectedBuildMode?.config?.search?.default === 'skip') { + break decideBuildSearchIndex; + } + + // TODO: OK this is a little silly. + if (stepStatusSummary.buildSearchIndex.annotation?.startsWith('N/A')) { + break decideBuildSearchIndex; + } + + const indexFile = path.join(wikiCachePath, 'search', 'index.json') + let stats; + try { + stats = await stat(indexFile); + } catch (error) { + if (error.code === 'ENOENT') { + Object.assign(stepStatusSummary.buildSearchIndex, { + status: STATUS_NOT_STARTED, + annotation: `search/index.json not present, will create`, + }); + + logInfo`Looks like the search cache doesn't exist.`; + logInfo`It'll be generated fresh, this build!`; + } else { + Object.assign(stepStatusSummary.buildSearchIndex, { + status: STATUS_NOT_APPLICABLE, + annotation: `error getting search index stats`, + }); + + if (!paragraph) console.log(''); + console.error(error); + + logWarn`There was an error checking the search index file, located at:`; + logWarn`${indexFile}`; + logWarn`You may want to toss out the "search" folder; it'll be generated`; + logWarn`anew, if you do, and may fix this error.`; + } + + paragraph = false; + break decideBuildSearchIndex; + } + + const delta = Date.now() - stats.mtimeMs; + const minute = 60 * 1000; + const delay = 45 * minute; + + const whenst = duration => `~${Math.ceil(duration / minute)} min`; + + if (delta < delay) { + logInfo`Search index was generated recently, skipping for this build.`; + logInfo`Next scheduled is in ${whenst(delay - delta)}, or by using ${'--refresh-search'}.`; + Object.assign(stepStatusSummary.buildSearchIndex, { + status: STATUS_NOT_APPLICABLE, + annotation: `earlier than scheduled`, + }); + } else { + logInfo`Search index hasn't been generated for a little while.`; + logInfo`It'll be generated this build, then again in ${whenst(delay)}.`; + Object.assign(stepStatusSummary.buildSearchIndex, { + status: STATUS_NOT_STARTED, + annotation: `past when shceduled`, + }); + } + + paragraph = false; + } + + fallbackStep('checkWikiDataSourceFileSorting', { + default: 'perform', + buildConfig: 'sort', + cli: { + flag: 'skip-sorting-validation', + negate: true, + warning: + `Skipping sorting validation. If any of this wiki's sorting rules are not\n` + + `satisfied, those errors will be silently passed along to the build.`, + }, + }); + fallbackStep('verifyImagePaths', { default: 'perform', buildConfig: 'mediaValidation', @@ -717,7 +1054,7 @@ async function main() { } if (stepStatusSummary.generateThumbnails.status === STATUS_NOT_STARTED) { - Object.assign(stepStatusSummary.loadThumbnailCache, { + Object.assign(stepStatusSummary.loadOfflineThumbnailCache, { status: STATUS_NOT_APPLICABLE, annotation: `using cache from thumbnail generation`, }); @@ -730,33 +1067,43 @@ async function main() { }); } + // TODO: These should error if the option was actually provided but + // the relevant steps were already disabled for some other reason. switch (precacheMode) { case 'common': - Object.assign(stepStatusSummary.precacheAllData, { - status: STATUS_NOT_APPLICABLE, - annotation: `--precache-mode is common, not all`, - }); + if (stepStatusSummary.precacheAllData.status === STATUS_NOT_STARTED) { + Object.assign(stepStatusSummary.precacheAllData, { + status: STATUS_NOT_APPLICABLE, + annotation: `--precache-mode is common, not all`, + }); + } break; case 'all': - Object.assign(stepStatusSummary.precacheCommonData, { - status: STATUS_NOT_APPLICABLE, - annotation: `--precache-mode is all, not common`, - }); + if (stepStatusSummary.precacheCommonData.status === STATUS_NOT_STARTED) { + Object.assign(stepStatusSummary.precacheCommonData, { + status: STATUS_NOT_APPLICABLE, + annotation: `--precache-mode is all, not common`, + }); + } break; case 'none': - Object.assign(stepStatusSummary.precacheCommonData, { - status: STATUS_NOT_APPLICABLE, - annotation: `--precache-mode is none`, - }); + if (stepStatusSummary.precacheCommonData.status === STATUS_NOT_STARTED) { + Object.assign(stepStatusSummary.precacheCommonData, { + status: STATUS_NOT_APPLICABLE, + annotation: `--precache-mode is none`, + }); + } - Object.assign(stepStatusSummary.precacheAllData, { - status: STATUS_NOT_APPLICABLE, - annotation: `--precache-mode is none`, - }); + if (stepStatusSummary.precacheAllData.status === STATUS_NOT_STARTED) { + Object.assign(stepStatusSummary.precacheAllData, { + status: STATUS_NOT_APPLICABLE, + annotation: `--precache-mode is none`, + }); + } break; } @@ -778,6 +1125,56 @@ async function main() { return false; } + // If we're going to require a build mode and none is specified, + // exit and show what to do. This must not precede anything that might + // disable the build (e.g. changing its status to STATUS_NOT_APPLICABLE). + + if (stepStatusSummary.performBuild.status === STATUS_NOT_STARTED) { + if (selectedBuildMode) { + logInfo`Will use specified build mode: ${selectedBuildModeFlag}`; + } else { + showHelpForOptions({ + heading: `Please specify a build mode:`, + options: buildModeFlagOptions, + }); + + console.log( + `(Use ${colors.bright('--help')} for general info and all options, or specify\n` + + ` a build mode alongside ${colors.bright('--help')} for that mode's options!`); + + for (const step of Object.values(stepStatusSummary)) { + Object.assign(step, { + status: STATUS_NOT_APPLICABLE, + annotation: `no build mode provided`, + }); + } + + return false; + } + } else if (selectedBuildMode) { + if (stepStatusSummary.performBuild.annotation) { + logError`You've specified a build mode, ${selectedBuildModeFlag}, but it won't be used,`; + logError`according to the message: ${`"${stepStatusSummary.performBuild.annotation}"`}`; + } else { + logError`You've specified a build mode, ${selectedBuildModeFlag}, but it won't be used,`; + logError`probably because of another option you've provided.`; + } + logError`Please remove ${'--' + selectedBuildModeFlag} or the conflicting option.`; + return false; + } + + if (skipSelfDiagnosis) { + logWarn`${'Skipping code self-diagnosis.'} (--skip-self-diagnosis provided)`; + logWarn`This build should run substantially faster, but corner cases`; + logWarn`not previously predicted may fail strangely and silently.`; + + html.disableSlotValidation(); + } + + if (!showTraces) { + html.disableTagTracing(); + } + Object.assign(stepStatusSummary.determineMediaCachePath, { status: STATUS_STARTED_NOT_DONE, timeStart: Date.now(), @@ -821,6 +1218,7 @@ async function main() { status: STATUS_FATAL_ERROR, annotation: `--new-thumbs provided but regeneration not needed`, timeEnd: Date.now(), + memory: process.memoryUsage(), }); return false; @@ -836,6 +1234,7 @@ async function main() { status: STATUS_FATAL_ERROR, annotation: mediaCachePathAnnotation, timeEnd: Date.now(), + memory: process.memoryUsage(), }); return false; @@ -885,17 +1284,15 @@ async function main() { logError`a new path with ${'--media-cache-path'} or ${'HSMUSIC_MEDIA_CACHE'}.`; break; - case `missing wiki cache to create media cache inside`: + case `media path not provided`: /* unreachable */ console.error(''); - logError`It looks like you're starting totally fresh, so please`; - logError`create a ${'cache'} folder and provide it with ${'--cache-path'}`; - logError`or ${'HSMUSIC_CACHE'}. The media cache will automatically be`; - logError`generated inside of this folder!`; + logError`It seems a ${'--media-path'} (or ${'HSMUSIC_MEDIA'}) wasn't provided.`; + logError`Make sure one of these is actually pointing to a path that exists.`; break; - case `media path not provided`: /* unreachable */ + case `cache path not provided`: /* unreachable */ console.error(''); - logError`It seems a ${'--media-path'} (or ${'HSMUSIC_MEDIA'}) wasn't provided.`; + logError`It seems a ${'--cache-path'} (or ${'HSMUSIC_CACHE'}) wasn't provided.`; logError`Make sure one of these is actually pointing to a path that exists.`; break; } @@ -904,6 +1301,7 @@ async function main() { status: STATUS_FATAL_ERROR, annotation: mediaCachePathAnnotation, timeEnd: Date.now(), + memory: process.memoryUsage(), }); return false; @@ -915,6 +1313,7 @@ async function main() { status: STATUS_DONE_CLEAN, annotation: mediaCachePathAnnotation, timeEnd: Date.now(), + memory: process.memoryUsage(), }); if (stepStatusSummary.migrateThumbnails.status === STATUS_NOT_STARTED) { @@ -934,6 +1333,7 @@ async function main() { status: STATUS_FATAL_ERROR, annotation: `view log for details`, timeEnd: Date.now(), + memory: process.memoryUsage(), }); return false; @@ -945,6 +1345,7 @@ async function main() { Object.assign(stepStatusSummary.migrateThumbnails, { status: STATUS_DONE_CLEAN, timeEnd: Date.now(), + memory: process.memoryUsage(), }); return true; @@ -952,23 +1353,24 @@ async function main() { const niceShowAggregate = (error, ...opts) => { showAggregate(error, { - showTraces: showAggregateTraces, + showTraces, pathToFileURL: (f) => path.relative(__dirname, fileURLToPath(f)), ...opts, }); }; if ( - stepStatusSummary.loadThumbnailCache.status === STATUS_NOT_STARTED && + stepStatusSummary.loadOfflineThumbnailCache.status === STATUS_NOT_STARTED && stepStatusSummary.generateThumbnails.status === STATUS_NOT_STARTED ) { - throw new Error(`Unable to continue with both loadThumbnailCache and generateThumbnails`); + throw new Error(`Unable to continue with both loadOfflineThumbnailCache and generateThumbnails`); } let thumbsCache; - if (stepStatusSummary.loadThumbnailCache.status === STATUS_NOT_STARTED) { - Object.assign(stepStatusSummary.loadThumbnailCache, { + // TODO: Skip this step if we're using online thumbs + if (stepStatusSummary.loadOfflineThumbnailCache.status === STATUS_NOT_STARTED) { + Object.assign(stepStatusSummary.loadOfflineThumbnailCache, { status: STATUS_STARTED_NOT_DONE, timeStart: Date.now(), }); @@ -984,10 +1386,11 @@ async function main() { logError`that you'll be good to go and don't need to process thumbnails` logError`again!`; - Object.assign(stepStatusSummary.loadThumbnailCache, { + Object.assign(stepStatusSummary.loadOfflineThumbnailCache, { status: STATUS_FATAL_ERROR, annotation: `cache does not exist`, timeEnd: Date.now(), + memory: process.memoryUsage(), }); return false; @@ -1001,10 +1404,11 @@ async function main() { logError`to help you out with troubleshooting!`; logError`${'https://hsmusic.wiki/discord/'}`; - Object.assign(stepStatusSummary.loadThumbnailCache, { + Object.assign(stepStatusSummary.loadOfflineThumbnailCache, { status: STATUS_FATAL_ERROR, annotation: `cache malformed or unreadable`, timeEnd: Date.now(), + memory: process.memoryUsage(), }); return false; @@ -1013,9 +1417,10 @@ async function main() { logInfo`Thumbnail cache file successfully read.`; - Object.assign(stepStatusSummary.loadThumbnailCache, { + Object.assign(stepStatusSummary.loadOfflineThumbnailCache, { status: STATUS_DONE_CLEAN, timeEnd: Date.now(), + memory: process.memoryUsage(), }); logInfo`Skipping thumbnail generation.`; @@ -1043,6 +1448,7 @@ async function main() { status: STATUS_FATAL_ERROR, annotation: `view log for details`, timeEnd: Date.now(), + memory: process.memoryUsage(), }); return false; @@ -1051,6 +1457,7 @@ async function main() { Object.assign(stepStatusSummary.generateThumbnails, { status: STATUS_DONE_CLEAN, timeEnd: Date.now(), + memory: process.memoryUsage(), }); if (thumbsOnly) { @@ -1062,12 +1469,6 @@ async function main() { thumbsCache = {}; } - if (showInvalidPropertyAccesses) { - CacheableObject.DEBUG_SLOW_TRACK_INVALID_PROPERTIES = true; - } - - let paragraph = false; - Object.assign(stepStatusSummary.loadDataFiles, { status: STATUS_STARTED_NOT_DONE, timeStart: Date.now(), @@ -1090,6 +1491,7 @@ async function main() { status: STATUS_FATAL_ERROR, annotation: `javascript error - view log for details`, timeEnd: Date.now(), + memory: process.memoryUsage(), }); return false; @@ -1129,6 +1531,7 @@ async function main() { status: STATUS_FATAL_ERROR, annotation: `error loading data files`, timeEnd: Date.now(), + memory: process.memoryUsage(), }); return false; @@ -1170,6 +1573,10 @@ async function main() { ? prop : wikiData[prop]); + if (array && empty(array)) { + return; + } + logInfo` - ${array?.length ?? colors.red('(Missing!)')} ${colors.normal(colors.dim(label))}`; } @@ -1196,9 +1603,14 @@ async function main() { logThings('newsData', 'news entries'); } logThings('staticPageData', 'static pages'); + logThings('sortingRules', 'sorting rules'); if (wikiData.homepageLayout) { logInfo` - ${1} homepage layout (${ - wikiData.homepageLayout.rows.length + wikiData.homepageLayout.sections.length + } sections, ${ + wikiData.homepageLayout.sections + .flatMap(section => section.rows) + .length } rows)`; } if (wikiData.wikiInfo) { @@ -1231,6 +1643,7 @@ async function main() { status: STATUS_FATAL_ERROR, annotation: `wiki info object not available`, timeEnd: Date.now(), + memory: process.memoryUsage(), }); return false; @@ -1243,6 +1656,7 @@ async function main() { Object.assign(stepStatusSummary.loadDataFiles, { status: STATUS_DONE_CLEAN, timeEnd: Date.now(), + memory: process.memoryUsage(), }); } else { logWarn`This might indicate some fields in the YAML data weren't formatted`; @@ -1257,6 +1671,7 @@ async function main() { status: STATUS_HAS_WARNINGS, annotation: `view log for details`, timeEnd: Date.now(), + memory: process.memoryUsage(), }); } } @@ -1270,11 +1685,12 @@ async function main() { timeStart: Date.now(), }); - linkWikiDataArrays(wikiData); + linkWikiDataArrays(wikiData, {bindFind, bindReverse}); Object.assign(stepStatusSummary.linkWikiDataArrays, { status: STATUS_DONE_CLEAN, timeEnd: Date.now(), + memory: process.memoryUsage(), }); if (precacheMode === 'common') { @@ -1324,62 +1740,119 @@ async function main() { ]), }; - for (const [wikiDataKey, properties] of Object.entries(commonDataMap)) { - const thingData = wikiData[wikiDataKey]; - const allProperties = new Set(['name', 'directory', ...properties]); - for (const thing of thingData) { - for (const property of allProperties) { - void thing[property]; + try { + for (const [wikiDataKey, properties] of Object.entries(commonDataMap)) { + const thingData = wikiData[wikiDataKey]; + const allProperties = new Set(['name', 'directory', ...properties]); + for (const thing of thingData) { + for (const property of allProperties) { + void thing[property]; + } } } + } catch (error) { + if (!paragraph) console.log(''); + niceShowAggregate(error); + console.log(''); + + logError`There was an error precaching internal data objects.`; + fileIssue(); + + Object.assign(stepStatusSummary.precacheCommonData, { + status: STATUS_FATAL_ERROR, + annotation: `see log for details`, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + + return false; } Object.assign(stepStatusSummary.precacheCommonData, { status: STATUS_DONE_CLEAN, timeEnd: Date.now(), + memory: process.memoryUsage(), }); } - // Filter out any things with duplicate directories throughout the data, - // warning about them too. - - Object.assign(stepStatusSummary.reportDirectoryErrors, { - status: STATUS_STARTED_NOT_DONE, - timeStart: Date.now(), - }); - - try { - reportDirectoryErrors(wikiData, {getAllFindSpecs}); - logInfo`No duplicate directories found - nice!`; - paragraph = false; + // Check for things with duplicate directories throughout the data, + // and halt if any are found. + if (stepStatusSummary.reportDirectoryErrors.status === STATUS_NOT_STARTED) { Object.assign(stepStatusSummary.reportDirectoryErrors, { - status: STATUS_DONE_CLEAN, - timeEnd: Date.now(), + status: STATUS_STARTED_NOT_DONE, + timeStart: Date.now(), }); - } catch (aggregate) { - if (!paragraph) console.log(''); - niceShowAggregate(aggregate); - logWarn`The above duplicate directories were detected while reviewing data files.`; - logWarn`Since it's impossible to automatically determine which one's directory is`; - logWarn`correct, the build can't continue. Specify unique 'Directory' fields in`; - logWarn`some or all of these data entries to resolve the errors.`; + try { + reportDirectoryErrors(wikiData, {getAllFindSpecs}); + logInfo`No duplicate directories found - nice!`; + paragraph = false; - console.log(''); - paragraph = true; + Object.assign(stepStatusSummary.reportDirectoryErrors, { + status: STATUS_DONE_CLEAN, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + } catch (aggregate) { + if (!paragraph) console.log(''); + niceShowAggregate(aggregate); - Object.assign(stepStatusSummary.reportDirectoryErrors, { - status: STATUS_FATAL_ERROR, - annotation: `duplicate directories found`, - timeEnd: Date.now(), + logWarn`The above duplicate directories were detected while reviewing data files.`; + logWarn`Since it's impossible to automatically determine which one's directory is`; + logWarn`correct, the build can't continue. Specify unique 'Directory' fields in`; + logWarn`some or all of these data entries to resolve the errors.`; + + console.log(''); + paragraph = true; + + Object.assign(stepStatusSummary.reportDirectoryErrors, { + status: STATUS_FATAL_ERROR, + annotation: `duplicate directories found`, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + + return false; + } + } + + // Check for artwork objects which have been orphaned from their things, + // and halt if any are found. + + if (stepStatusSummary.reportOrphanedArtworks.status === STATUS_NOT_STARTED) { + Object.assign(stepStatusSummary.reportOrphanedArtworks, { + status: STATUS_STARTED_NOT_DONE, + timeStart: Date.now(), }); - return false; + try { + reportOrphanedArtworks(wikiData, {getAllFindSpecs}); + + Object.assign(stepStatusSummary.reportOrphanedArtworks, { + status: STATUS_DONE_CLEAN, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + } catch (aggregate) { + if (!paragraph) console.log(''); + niceShowAggregate(aggregate); + + logError`Failed to initialize artwork data connections properly.`; + fileIssue(); + + Object.assign(stepStatusSummary.reportOrphanedArtworks, { + status: STATUS_FATAL_ERROR, + annotation: `orphaned artworks found`, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + + return false; + } } - // Filter out any reference errors throughout the data, warning about them - // too. + // Filter out any reference errors throughout the data, warning about these. if (stepStatusSummary.filterReferenceErrors.status === STATUS_NOT_STARTED) { Object.assign(stepStatusSummary.filterReferenceErrors, { @@ -1388,7 +1861,7 @@ async function main() { }); const filterReferenceErrorsAggregate = - filterReferenceErrors(wikiData, {bindFind}); + filterReferenceErrors(wikiData, {find, bindFind}); try { filterReferenceErrorsAggregate.close(); @@ -1399,6 +1872,7 @@ async function main() { Object.assign(stepStatusSummary.filterReferenceErrors, { status: STATUS_DONE_CLEAN, timeEnd: Date.now(), + memory: process.memoryUsage(), }); } catch (error) { if (!paragraph) console.log(''); @@ -1416,6 +1890,7 @@ async function main() { status: STATUS_HAS_WARNINGS, annotation: `view log for details`, timeEnd: Date.now(), + memory: process.memoryUsage(), }); } } @@ -1435,6 +1910,7 @@ async function main() { Object.assign(stepStatusSummary.reportContentTextErrors, { status: STATUS_DONE_CLEAN, timeEnd: Date.now(), + memory: process.memoryUsage(), }); } catch (error) { if (!paragraph) console.log(''); @@ -1451,6 +1927,7 @@ async function main() { status: STATUS_HAS_WARNINGS, annotation: `view log for details`, timeEnd: Date.now(), + memory: process.memoryUsage(), }); } } @@ -1463,11 +1940,12 @@ async function main() { timeStart: Date.now(), }); - sortWikiDataArrays(yamlDataSteps, wikiData); + sortWikiDataArrays(yamlDataSteps, wikiData, {bindFind, bindReverse}); Object.assign(stepStatusSummary.sortWikiDataArrays, { status: STATUS_DONE_CLEAN, timeEnd: Date.now(), + memory: process.memoryUsage(), }); if (precacheMode === 'all') { @@ -1491,7 +1969,79 @@ async function main() { Object.assign(stepStatusSummary.precacheAllData, { status: STATUS_DONE_CLEAN, timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + } + + if (stepStatusSummary.sortWikiDataSourceFiles.status === STATUS_NOT_STARTED) { + Object.assign(stepStatusSummary.sortWikiDataSourceFiles, { + status: STATUS_STARTED_NOT_DONE, + timeStart: Date.now(), + }); + + const {SortingRule} = thingConstructors; + const results = + await Array.fromAsync(SortingRule.go({dataPath, wikiData})); + + if (results.some(result => result.changed)) { + logInfo`Updated data files to satisfy sorting.`; + logInfo`Restarting automatically, since that's now needed!`; + + Object.assign(stepStatusSummary.sortWikiDataSourceFiles, { + status: STATUS_DONE_CLEAN, + annotation: `changes cueing restart`, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + + return 'restart'; + } else { + logInfo`All sorting rules are satisfied. Nice!`; + paragraph = false; + + Object.assign(stepStatusSummary.sortWikiDataSourceFiles, { + status: STATUS_DONE_CLEAN, + annotation: `no changes needed`, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + } + } else if (stepStatusSummary.checkWikiDataSourceFileSorting.status === STATUS_NOT_STARTED) { + Object.assign(stepStatusSummary.checkWikiDataSourceFileSorting, { + status: STATUS_STARTED_NOT_DONE, + timeStart: Date.now(), }); + + const {SortingRule} = thingConstructors; + const results = + await Array.fromAsync(SortingRule.go({dataPath, wikiData, dry: true})); + + const needed = results.filter(result => result.changed); + + if (empty(needed)) { + logInfo`All sorting rules are satisfied. Nice!`; + paragraph = false; + + Object.assign(stepStatusSummary.checkWikiDataSourceFileSorting, { + status: STATUS_DONE_CLEAN, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + } else { + logWarn`Some of this wiki's sorting rules currently aren't satisfied:`; + for (const {rule} of needed) { + logWarn`- ${rule.message}`; + } + logWarn`Run ${'hsmusic --sort'} to automatically update data files.`; + paragraph = false; + + Object.assign(stepStatusSummary.checkWikiDataSourceFileSorting, { + status: STATUS_HAS_WARNINGS, + annotation: `not all rules satisfied`, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + } } if (stepStatusSummary.performBuild.status === STATUS_NOT_APPLICABLE) { @@ -1502,6 +2052,354 @@ async function main() { } } + Object.assign(stepStatusSummary.loadURLFiles, { + status: STATUS_STARTED_NOT_DONE, + timeStart: Date.now(), + }); + + let internalURLSpec = {}; + + try { + let aggregate; + ({aggregate, result: internalURLSpec} = + await processURLSpecFromFile(internalDefaultURLSpecFile)); + + aggregate.close(); + } catch (error) { + niceShowAggregate(error); + logError`Couldn't load internal default URL spec.`; + logError`This is required to build the wiki, so stopping here.`; + fileIssue(); + + Object.assign(stepStatusSummary.loadURLFiles, { + status: STATUS_FATAL_ERROR, + annotation: `see log for details`, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + + return false; + } + + // We'll mutate this as we load other url spec files. + const urlSpec = structuredClone(internalURLSpec); + + const allURLSpecDataFiles = + (await readdir(dataPath)) + .filter(name => + name.startsWith('urls') && + ['.json', '.yaml'].includes(path.extname(name))) + .sort() /* Just in case... */ + .map(name => path.join(dataPath, name)); + + const getURLSpecKeyFromFile = file => { + const base = path.basename(file, path.extname(file)); + if (base === 'urls') { + return base; + } else { + return base.replace(/^urls-/, ''); + } + }; + + const isDefaultURLSpecFile = file => + getURLSpecKeyFromFile(file) === 'urls'; + + const overrideDefaultURLSpecFile = + allURLSpecDataFiles.find(file => isDefaultURLSpecFile(file)); + + const optionalURLSpecDataFiles = + allURLSpecDataFiles.filter(file => !isDefaultURLSpecFile(file)); + + const optionalURLSpecDataKeys = + optionalURLSpecDataFiles.map(file => getURLSpecKeyFromFile(file)); + + const selectedURLSpecDataKeys = optionalURLSpecDataKeys.slice(); + const selectedURLSpecDataFiles = optionalURLSpecDataFiles.slice(); + + const {removed: [unusedURLSpecDataKeys]} = + filterMultipleArrays( + selectedURLSpecDataKeys, + selectedURLSpecDataFiles, + (key, _file) => wantedURLSpecKeys.includes(key)); + + if (!empty(selectedURLSpecDataKeys)) { + logInfo`Using these optional URL specs: ${selectedURLSpecDataKeys.join(', ')}`; + if (!empty(unusedURLSpecDataKeys)) { + logInfo`Other available optional URL specs: ${unusedURLSpecDataKeys.join(', ')}`; + } + } else if (!empty(unusedURLSpecDataKeys)) { + logInfo`Not using any optional URL specs.`; + logInfo`These are available with --urls: ${unusedURLSpecDataKeys.join(', ')}`; + } + + if (overrideDefaultURLSpecFile) { + try { + let aggregate; + let overrideDefaultURLSpec; + + ({aggregate, result: overrideDefaultURLSpec} = + await processURLSpecFromFile(overrideDefaultURLSpecFile)); + + aggregate.close(); + + ({aggregate} = + applyURLSpecOverriding(overrideDefaultURLSpec, urlSpec)); + + aggregate.close(); + } catch (error) { + niceShowAggregate(error); + logError`Errors loading this data repo's ${'urls.yaml'} file.`; + logError`This provides essential overrides for this wiki,`; + logError`so stopping here. Debug the errors to continue.`; + + Object.assign(stepStatusSummary.loadURLFiles, { + status: STATUS_FATAL_ERROR, + annotation: `see log for details`, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + + return false; + } + } + + const processURLSpecsAggregate = + openAggregate({message: `Errors processing URL specs`}); + + const selectedURLSpecs = + processURLSpecsAggregate.receive( + await Promise.all( + selectedURLSpecDataFiles + .map(file => processURLSpecFromFile(file)))); + + for (const selectedURLSpec of selectedURLSpecs) { + processURLSpecsAggregate.receive( + applyURLSpecOverriding(selectedURLSpec, urlSpec)); + } + + try { + processURLSpecsAggregate.close(); + } catch (error) { + niceShowAggregate(error); + logWarn`There were errors loading the optional URL specs you`; + logWarn`selected using ${'--urls'}. Since they might misfunction,`; + logWarn`debug the errors or remove the failing ones from ${'--urls'}.`; + + Object.assign(stepStatusSummary.loadURLFiles, { + status: STATUS_FATAL_ERROR, + annotation: `see log for details`, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + + return false; + } + + if (showURLSpec) { + if (!paragraph) console.log(''); + + logInfo`Here's the final URL spec, via ${'--show-url-spec'}:` + console.log(urlSpec); + console.log(''); + + paragraph = true; + } + + Object.assign(stepStatusSummary.loadURLFiles, { + status: STATUS_DONE_CLEAN, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + + if (!getOrigin(urlSpec.thumb.prefix)) { + Object.assign(stepStatusSummary.loadOnlineThumbnailCache, { + status: STATUS_NOT_APPLICABLE, + annotation: `using offline thumbs`, + }); + } + + if (getOrigin(urlSpec.media.prefix)) { + Object.assign(stepStatusSummary.preloadFileSizes, { + status: STATUS_NOT_APPLICABLE, + annotation: `using online media`, + }); + } else { + Object.assign(stepStatusSummary.loadOnlineFileSizeCache, { + status: STATUS_NOT_APPLICABLE, + annotation: `using offline media`, + }); + } + + applyLocalizedWithBaseDirectory(urlSpec); + + const urls = generateURLs(urlSpec); + + if (stepStatusSummary.loadOnlineThumbnailCache.status === STATUS_NOT_STARTED) loadOnlineThumbnailCache: { + Object.assign(stepStatusSummary.loadOnlineThumbnailCache, { + status: STATUS_STARTED_NOT_DONE, + timeStart: Date.now(), + }); + + let onlineThumbsCache = null; + + const cacheFile = path.join(wikiCachePath, 'online-thumbnail-cache.json'); + + let readError = null; + let writeError = null; + + if (!cliOptions['refresh-online-thumbs']) { + try { + onlineThumbsCache = JSON.parse(await readFile(cacheFile)); + } catch (caughtError) { + readError = caughtError; + } + } + + if (onlineThumbsCache) obliterateLocalCopy: { + if (!onlineThumbsCache._urlPrefix) { + // Well, it doesn't even count. + onlineThumbsCache = null; + break obliterateLocalCopy; + } + + if (onlineThumbsCache._urlPrefix !== urlSpec.thumb.prefix) { + logInfo`Local copy of online thumbs cache is for a different prefix.`; + logInfo`It'll be downloaded and replaced, for reuse next time.`; + paragraph = false; + + onlineThumbsCache = null; + break obliterateLocalCopy; + } + + let stats; + try { + stats = await stat(cacheFile); + } catch { + logInfo`Unable to get the stats of local copy of online thumbs cache...`; + logInfo`This is really weird, since we *were* able to read it...`; + logInfo`We're just going to try writing to it and download fresh!`; + paragraph = false; + + onlineThumbsCache = null; + break obliterateLocalCopy; + } + + const delta = Date.now() - stats.mtimeMs; + const minute = 60 * 1000; + const delay = 60 * minute; + + const whenst = duration => `~${Math.ceil(duration / minute)} min`; + + if (delta < delay) { + logInfo`Online thumbs cache was downloaded recently, skipping for this build.`; + logInfo`Next scheduled is in ${whenst(delay - delta)}, or by using ${'--refresh-online-thumbs'}.`; + paragraph = false; + + Object.assign(stepStatusSummary.loadOnlineThumbnailCache, { + status: STATUS_DONE_CLEAN, + annotation: `reusing local copy, earlier than scheduled`, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + + thumbsCache = onlineThumbsCache; + + break loadOnlineThumbnailCache; + } else { + logInfo`Online thumbs cache hasn't been downloaded for a little while.`; + logInfo`It'll be downloaded this build, then again in ${whenst(delay)}.`; + onlineThumbsCache = null; + paragraph = false; + } + } + + try { + await writeFile(cacheFile, stringifyCache(onlineThumbsCache)); + } catch (caughtError) { + writeError = caughtError; + } + + if (readError && writeError && readError.code !== 'ENOENT') { + console.error(readError); + logWarn`Wasn't able to read the local copy of the`; + logWarn`online thumbs cache file...`; + console.error(writeError); + logWarn`...or write to it, either.`; + logWarn`The online thumbs cache will be downloaded`; + logWarn`for every build until you investigate this path:`; + logWarn`${cacheFile}`; + paragraph = false; + } else if (readError && readError.code === 'ENOENT' && !writeError) { + logInfo`No local copy of online thumbs cache.`; + logInfo`It'll be downloaded this time and reused next time.`; + paragraph = false; + } else if (readError && readError.code === 'ENOENT' && writeError) { + console.error(writeError); + logWarn`Doesn't look like we can write a local copy of`; + logWarn`the offline thumbs cache, at this path:`; + logWarn`${cacheFile}`; + logWarn`The online thumbs cache will be downloaded`; + logWarn`for every build until you investigate that.`; + paragraph = false; + } + + const url = new URL(urlSpec.thumb.prefix); + url.pathname = path.posix.join(url.pathname, 'thumbnail-cache.json'); + + try { + onlineThumbsCache = await fetch(url).then(res => res.json()); + } catch (error) { + console.error(error); + logWarn`There was an error downloading the online thumbnail cache.`; + logWarn`The wiki will act as though no thumbs are available at all.`; + paragraph = false; + + Object.assign(stepStatusSummary.loadOnlineThumbnailCache, { + status: STATUS_HAS_WARNINGS, + annotation: `failed to download`, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + + onlineThumbsCache = {}; + thumbsCache = {}; + + break loadOnlineThumbnailCache; + } + + onlineThumbsCache._urlPrefix = urlSpec.thumb.prefix; + + thumbsCache = onlineThumbsCache; + + if (onlineThumbsCache && !writeError) { + try { + await writeFile(cacheFile, stringifyCache(onlineThumbsCache)); + } catch (error) { + console.error(error); + logWarn`There was an error saving a local copy of the`; + logWarn`online thumbnail cache. It'll be fetched again`; + logWarn`next time.`; + paragraph = false; + + Object.assign(stepStatusSummary.loadOnlineThumbnailCache, { + status: STATUS_HAS_WARNINGS, + annotation: `failed to download`, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + + break loadOnlineThumbnailCache; + } + } + + Object.assign(stepStatusSummary.loadOnlineThumbnailCache, { + status: STATUS_DONE_CLEAN, + timeStart: Date.now(), + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + } + const languageReloading = stepStatusSummary.watchLanguageFiles.status === STATUS_NOT_STARTED; @@ -1540,7 +2438,7 @@ async function main() { }); internalDefaultLanguage = internalDefaultLanguageWatcher.language; - } catch (_error) { + } catch { // No need to display the error here - it's already printed by // watchLanguageFile. errorLoadingInternalDefaultLanguage = true; @@ -1564,6 +2462,7 @@ async function main() { status: STATUS_FATAL_ERROR, annotation: `see log for details`, timeEnd: Date.now(), + memory: process.memoryUsage(), }); return false; @@ -1577,6 +2476,7 @@ async function main() { Object.assign(stepStatusSummary.loadInternalDefaultLanguage, { status: STATUS_DONE_CLEAN, timeEnd: Date.now(), + memory: process.memoryUsage(), }); let customLanguageWatchers; @@ -1656,6 +2556,7 @@ async function main() { status: STATUS_FATAL_ERROR, annotation: `see log for details`, timeEnd: Date.now(), + memory: process.memoryUsage(), }); errorLoadingCustomLanguages = true; @@ -1687,6 +2588,7 @@ async function main() { Object.assign(stepStatusSummary.watchLanguageFiles, { status: STATUS_DONE_CLEAN, timeEnd: Date.now(), + memory: process.memoryUsage(), }); } else { languages = {}; @@ -1710,11 +2612,13 @@ async function main() { status: STATUS_FATAL_ERROR, annotation: `see log for details`, timeEnd: Date.now(), + memory: process.memoryUsage(), }); } else { Object.assign(stepStatusSummary.loadLanguageFiles, { status: STATUS_DONE_CLEAN, timeEnd: Date.now(), + memory: process.memoryUsage(), }); } } @@ -1752,6 +2656,7 @@ async function main() { status: STATUS_FATAL_ERROR, annotation: `wiki specifies default language whose file is not available`, timeEnd: Date.now(), + memory: process.memoryUsage(), }); return false; @@ -1845,10 +2750,9 @@ async function main() { status: STATUS_DONE_CLEAN, annotation: finalDefaultLanguageAnnotation, timeEnd: Date.now(), + memory: process.memoryUsage(), }); - const urls = generateURLs(urlSpec); - let missingImagePaths; if (stepStatusSummary.verifyImagePaths.status === STATUS_NOT_APPLICABLE) { @@ -1869,85 +2773,225 @@ async function main() { Object.assign(stepStatusSummary.verifyImagePaths, { status: STATUS_DONE_CLEAN, timeEnd: Date.now(), + memory: process.memoryUsage(), }); } else if (empty(missingImagePaths)) { Object.assign(stepStatusSummary.verifyImagePaths, { status: STATUS_HAS_WARNINGS, annotation: `misplaced images detected`, timeEnd: Date.now(), + memory: process.memoryUsage(), }); } else if (empty(misplacedImagePaths)) { Object.assign(stepStatusSummary.verifyImagePaths, { status: STATUS_HAS_WARNINGS, annotation: `missing images detected`, timeEnd: Date.now(), + memory: process.memoryUsage(), }); } else { Object.assign(stepStatusSummary.verifyImagePaths, { status: STATUS_HAS_WARNINGS, annotation: `missing and misplaced images detected`, timeEnd: Date.now(), + memory: process.memoryUsage(), }); } } - let getSizeOfAdditionalFile; - let getSizeOfImagePath; + let getSizeOfMediaFile = () => null; + + const fileSizePreloader = + new FileSizePreloader({ + prefix: mediaPath, + }); + + if (stepStatusSummary.loadOnlineFileSizeCache.status === STATUS_NOT_STARTED) loadOnlineFileSizeCache: { + Object.assign(stepStatusSummary.loadOnlineFileSizeCache, { + status: STATUS_STARTED_NOT_DONE, + timeStart: Date.now(), + }); + + let onlineFileSizeCache = null; + + const makeFileSizeCacheAvailable = () => { + fileSizePreloader.loadFromCache(onlineFileSizeCache); + + getSizeOfMediaFile = p => + fileSizePreloader.getSizeOfPath( + path.resolve( + mediaPath, + decodeURIComponent(p).split('/').join(path.sep))); + }; + + const cacheFile = path.join(wikiCachePath, 'online-file-size-cache.json'); + + let readError = null; + let writeError = null; + + if (!cliOptions['refresh-online-file-sizes']) { + try { + onlineFileSizeCache = JSON.parse(await readFile(cacheFile)); + } catch (caughtError) { + readError = caughtError; + } + } + + if (onlineFileSizeCache) obliterateLocalCopy: { + if (!onlineFileSizeCache._urlPrefix) { + // Well, it doesn't even count. + onlineFileSizeCache = null; + break obliterateLocalCopy; + } + + if (onlineFileSizeCache._urlPrefix !== urlSpec.media.prefix) { + logInfo`Local copy of online file size cache is for a different prefix.`; + logInfo`It'll be downloaded and replaced, for reuse next time.`; + paragraph = false; + + onlineFileSizeCache = null; + break obliterateLocalCopy; + } + + let stats; + try { + stats = await stat(cacheFile); + } catch { + logInfo`Unable to get the stats of local copy of online file size cache...`; + logInfo`This is really weird, since we *were* able to read it...`; + logInfo`We're just going to try writing to it and download fresh!`; + paragraph = false; + + onlineFileSizeCache = null; + break obliterateLocalCopy; + } + + const delta = Date.now() - stats.mtimeMs; + const minute = 60 * 1000; + const delay = 60 * minute; + + const whenst = duration => `~${Math.ceil(duration / minute)} min`; + + if (delta < delay) { + logInfo`Online file size cache was downloaded recently, skipping for this build.`; + logInfo`Next scheduled is in ${whenst(delay - delta)}, or by using ${'--refresh-online-file-sizes'}.`; + paragraph = false; + + Object.assign(stepStatusSummary.loadOnlineFileSizeCache, { + status: STATUS_DONE_CLEAN, + annotation: `reusing local copy, earlier than scheduled`, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + + delete onlineFileSizeCache._urlPrefix; + + makeFileSizeCacheAvailable(); + + break loadOnlineFileSizeCache; + } else { + logInfo`Online file size hasn't been downloaded for a little while.`; + logInfo`It'll be downloaded this build, then again in ${whenst(delay)}.`; + onlineFileSizeCache = null; + paragraph = false; + } + } + + try { + await writeFile(cacheFile, stringifyCache(onlineFileSizeCache)); + } catch (caughtError) { + writeError = caughtError; + } + + if (readError && writeError && readError.code !== 'ENOENT') { + console.error(readError); + logWarn`Wasn't able to read the local copy of the`; + logWarn`online file size cache file...`; + console.error(writeError); + logWarn`...or write to it, either.`; + logWarn`The online file size cache will be downloaded`; + logWarn`for every build until you investigate this path:`; + logWarn`${cacheFile}`; + paragraph = false; + } else if (readError && readError.code === 'ENOENT' && !writeError) { + logInfo`No local copy of online file size cache.`; + logInfo`It'll be downloaded this time and reused next time.`; + paragraph = false; + } else if (readError && readError.code === 'ENOENT' && writeError) { + console.error(writeError); + logWarn`Doesn't look like we can write a local copy of`; + logWarn`the offline file size cache, at this path:`; + logWarn`${cacheFile}`; + logWarn`The online file size cache will be downloaded`; + logWarn`for every build until you investigate that.`; + paragraph = false; + } + + const url = new URL(urlSpec.media.prefix); + url.pathname = path.posix.join(url.pathname, 'file-size-cache.json'); + + try { + onlineFileSizeCache = await fetch(url).then(res => res.json()); + } catch (error) { + console.error(error); + logWarn`There was an error downloading the online file size cache.`; + logWarn`The wiki will act as though no file sizes are available at all.`; + paragraph = false; + + Object.assign(stepStatusSummary.loadOnlineFileSizeCache, { + status: STATUS_HAS_WARNINGS, + annotation: `failed to download`, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + + break loadOnlineFileSizeCache; + } + + makeFileSizeCacheAvailable(); + + onlineFileSizeCache._urlPrefix = urlSpec.media.prefix; + + if (onlineFileSizeCache && !writeError) { + try { + await writeFile(cacheFile, stringifyCache(onlineFileSizeCache)); + } catch (error) { + console.error(error); + logWarn`There was an error saving a local copy of the`; + logWarn`online file size cache. It'll be fetched again`; + logWarn`next time.`; + paragraph = false; + + Object.assign(stepStatusSummary.loadOnlineFileSizeCache, { + status: STATUS_HAS_WARNINGS, + annotation: `failed to download`, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + + break loadOnlineFileSizeCache; + } + } + + Object.assign(stepStatusSummary.loadOnlineFileSizeCache, { + status: STATUS_DONE_CLEAN, + timeStart: Date.now(), + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + } - if (stepStatusSummary.preloadFileSizes.status === STATUS_NOT_APPLICABLE) { - getSizeOfAdditionalFile = () => null; - getSizeOfImagePath = () => null; - } else if (stepStatusSummary.preloadFileSizes.status === STATUS_NOT_STARTED) { + if (stepStatusSummary.preloadFileSizes.status === STATUS_NOT_STARTED) { Object.assign(stepStatusSummary.preloadFileSizes, { status: STATUS_STARTED_NOT_DONE, timeStart: Date.now(), }); - const fileSizePreloader = new FileSizePreloader(); - - // File sizes of additional files need to be precalculated before we can - // actually reference 'em in site building, so get those loading right - // away. We actually need to keep track of two things here - the on-device - // file paths we're actually reading, and the corresponding on-site media - // paths that will be exposed in site build code. We'll build a mapping - // function between them so that when site code requests a site path, - // it'll get the size of the file at the corresponding device path. - const additionalFilePaths = [ - ...wikiData.albumData.flatMap((album) => - [ - ...(album.additionalFiles ?? []), - ...album.tracks.flatMap((track) => [ - ...(track.additionalFiles ?? []), - ...(track.sheetMusicFiles ?? []), - ...(track.midiProjectFiles ?? []), - ]), - ] - .flatMap((fileGroup) => fileGroup.files ?? []) - .map((file) => ({ - device: path.join( - mediaPath, - urls - .from('media.root') - .toDevice('media.albumAdditionalFile', album.directory, file) - ), - media: urls - .from('media.root') - .to('media.albumAdditionalFile', album.directory, file), - })) - ), - ]; - - // Same dealio for images. Since just about any image can be embedded and - // we can't super easily know which ones are referenced at runtime, just - // cheat and get file sizes for all images under media. (This includes - // additional files which are images.) - const imageFilePaths = + const mediaFilePaths = await traverse(mediaPath, { pathStyle: 'device', filterDir: dir => dir !== '.git', - filterFile: file => - ['.png', '.gif', '.jpg'].includes(path.extname(file)) && - !isThumb(file), + filterFile: file => !isThumb(file), }).then(files => files .map(file => ({ device: file, @@ -1957,28 +3001,19 @@ async function main() { .to('media.path', path.relative(mediaPath, file).split(path.sep).join('/')), }))); - const getSizeOfMediaFileHelper = paths => (mediaPath) => { - const pair = paths.find(({media}) => media === mediaPath); + getSizeOfMediaFile = mediaPath => { + const pair = mediaFilePaths.find(({media}) => media === mediaPath); if (!pair) return null; return fileSizePreloader.getSizeOfPath(pair.device); }; - getSizeOfAdditionalFile = getSizeOfMediaFileHelper(additionalFilePaths); - getSizeOfImagePath = getSizeOfMediaFileHelper(imageFilePaths); + logInfo`Preloading file sizes for ${mediaFilePaths.length} media files...`; - logInfo`Preloading filesizes for ${additionalFilePaths.length} additional files...`; - - fileSizePreloader.loadPaths(...additionalFilePaths.map((path) => path.device)); - await fileSizePreloader.waitUntilDoneLoading(); - - logInfo`Preloading filesizes for ${imageFilePaths.length} full-resolution images...`; - paragraph = false; - - fileSizePreloader.loadPaths(...imageFilePaths.map((path) => path.device)); + fileSizePreloader.loadPaths(...mediaFilePaths.map(path => path.device)); await fileSizePreloader.waitUntilDoneLoading(); if (fileSizePreloader.hasErrored) { - logWarn`Some media files couldn't be read for preloading filesizes.`; + logWarn`Some media files couldn't be read for preloading file sizes.`; logWarn`This means the wiki won't display file sizes for these files.`; logWarn`Investigate missing or unreadable files to get that fixed!`; @@ -1986,19 +3021,97 @@ async function main() { status: STATUS_HAS_WARNINGS, annotation: `see log for details`, timeEnd: Date.now(), + memory: process.memoryUsage(), }); } else { - logInfo`Done preloading filesizes without any errors - nice!`; + logInfo`Done preloading file sizes without any errors - nice!`; paragraph = false; Object.assign(stepStatusSummary.preloadFileSizes, { status: STATUS_DONE_CLEAN, timeEnd: Date.now(), + memory: process.memoryUsage(), }); } + + // TODO: kinda jank that this is out of band of any particular step, + // even though it's operationally a follow-up to preloadFileSizes + + let oopsCache = false; + saveFileSizeCache: { + let cache; + try { + cache = fileSizePreloader.saveAsCache(); + } catch (error) { + console.error(error); + logWarn`Couldn't compute file size preloader's cache.`; + oopsCache = true; + break saveFileSizeCache; + } + + const cacheFile = path.join(mediaPath, 'file-size-cache.json'); + + try { + await writeFile(cacheFile, stringifyCache(cache)); + } catch (error) { + console.error(error); + logWarn`Couldn't save preloaded file sizes to a cache file:`; + logWarn`${cacheFile}`; + oopsCache = true; + } + } + + if (oopsCache) { + logWarn`This won't affect the build, but this build should not be used`; + logWarn`as a model for another build accessing its media files online.`; + } } - let webRoutes = null; + if (stepStatusSummary.buildSearchIndex.status === STATUS_NOT_STARTED) { + Object.assign(stepStatusSummary.buildSearchIndex, { + status: STATUS_STARTED_NOT_DONE, + timeStart: Date.now(), + }); + + try { + await writeSearchData({ + thumbsCache, + urls, + wikiCachePath, + wikiData, + }); + + logInfo`Search data successfully written - nice!`; + paragraph = false; + + Object.assign(stepStatusSummary.buildSearchIndex, { + status: STATUS_DONE_CLEAN, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + } catch (error) { + if (!paragraph) console.log(''); + niceShowAggregate(error); + + logError`There was an error preparing or writing search data.`; + fileIssue(); + logWarn`Any existing search data will be reused, and search may be`; + logWarn`generally dysfunctional. The site should work otherwise, though!`; + + console.log(''); + paragraph = true; + + Object.assign(stepStatusSummary.buildSearchIndex, { + status: STATUS_HAS_WARNINGS, + annotation: `see log for details`, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + } + } + + let webRouteSources = null; + let preparedWebRoutes = null; if (stepStatusSummary.identifyWebRoutes.status === STATUS_NOT_STARTED) { Object.assign(stepStatusSummary.identifyWebRoutes, { @@ -2009,7 +3122,7 @@ async function main() { const fromRoot = urls.from('shared.root'); try { - const webRouteSources = await identifyAllWebRoutes({ + webRouteSources = await identifyAllWebRoutes({ mediaCachePath, mediaPath, wikiCachePath, @@ -2025,7 +3138,7 @@ async function main() { {message: `Errors computing effective web route paths`},); aggregate.close(); - webRoutes = result; + preparedWebRoutes = result; } catch (error) { if (!paragraph) console.log(''); niceShowAggregate(error); @@ -2040,20 +3153,28 @@ async function main() { status: STATUS_FATAL_ERROR, message: `JavaScript error - view log for details`, timeEnd: Date.now(), + memory: process.memoryUsage(), }); return false; } - logInfo`Successfully determined web routes.`; + logInfo`Successfully determined web routes - nice!`; paragraph = false; Object.assign(stepStatusSummary.identifyWebRoutes, { status: STATUS_DONE_CLEAN, timeEnd: Date.now(), + memory: process.memoryUsage(), }); } + wikiData.wikiInfo.searchDataAvailable = + (webRouteSources + ? webRouteSources + .some(({to}) => to[0].startsWith('searchData')) + : null); + if (stepStatusSummary.performBuild.status === STATUS_NOT_APPLICABLE) { return true; } @@ -2098,29 +3219,37 @@ async function main() { logInfo`Passing control over to build mode: ${selectedBuildModeFlag}`; console.log(''); + const universalUtilities = { + getSizeOfMediaFile, + + defaultLanguage: finalDefaultLanguage, + developersComment, + languages, + missingImagePaths, + niceShowAggregate, + thumbsCache, + urlSpec, + urls, + wikiData, + }; + try { buildModeResult = await selectedBuildMode.go({ cliOptions, + queueSize, + + universalUtilities, + ...universalUtilities, + dataPath, mediaPath, mediaCachePath, - queueSize, + wikiCachePath, srcRootPath: __dirname, - defaultLanguage: finalDefaultLanguage, - languages, - missingImagePaths, - thumbsCache, - urls, - urlSpec, - webRoutes, - wikiData, + webRoutes: preparedWebRoutes, - cachebust: '?' + CACHEBUST, closeLanguageWatchers, - developersComment, - getSizeOfAdditionalFile, - getSizeOfImagePath, niceShowAggregate, }); } catch (error) { @@ -2133,6 +3262,7 @@ async function main() { status: STATUS_FATAL_ERROR, message: `javascript error - view log for details`, timeEnd: Date.now(), + memory: process.memoryUsage(), }); return false; @@ -2143,6 +3273,7 @@ async function main() { status: STATUS_HAS_WARNINGS, annotation: `may not have completed - view log for details`, timeEnd: Date.now(), + memory: process.memoryUsage(), }); return false; @@ -2151,136 +3282,78 @@ async function main() { Object.assign(stepStatusSummary.performBuild, { status: STATUS_DONE_CLEAN, timeEnd: Date.now(), + memory: process.memoryUsage(), }); return true; } // TODO: isMain detection isn't consistent across platforms here -/* eslint-disable-next-line no-constant-condition */ +// eslint-disable-next-line no-constant-binary-expression if (true || isMain(import.meta.url) || path.basename(process.argv[1]) === 'hsmusic') { (async () => { let result; + let numRestarts = 0; const totalTimeStart = Date.now(); - try { - result = await main(); - } catch (error) { - if (error instanceof AggregateError) { - showAggregate(error); - } else if (error.cause) { - console.error(error); - showAggregate(error); - } else { - console.error(error); - } - } - - const totalTimeEnd = Date.now(); - - const formatDuration = timeDelta => { - const seconds = timeDelta / 1000; - - if (seconds > 90) { - const modSeconds = Math.floor(seconds % 60); - const minutes = Math.floor(seconds - seconds % 60) / 60; - return `${minutes}m${modSeconds}s`; - } - - if (seconds < 0.1) { - return 'instant'; + while (true) { + try { + result = await main(); + } catch (error) { + if (error instanceof AggregateError) { + showAggregate(error); + } else if (error.cause) { + console.error(error); + showAggregate(error); + } else { + console.error(error); + } } - const precision = (seconds > 1 ? 3 : 2); - return `${seconds.toPrecision(precision)}s`; - }; + if (result === 'restart') { + console.log(''); - if (showStepStatusSummary) { - const totalDuration = formatDuration(totalTimeEnd - totalTimeStart); - - console.error(colors.bright(`Step summary:`)); - - const longestNameLength = - Math.max(... - Object.values(stepStatusSummary) - .map(({name}) => name.length)); - - const stepsNotClean = - Object.values(stepStatusSummary) - .map(({status}) => - status === STATUS_HAS_WARNINGS || - status === STATUS_FATAL_ERROR || - status === STATUS_STARTED_NOT_DONE); - - const anyStepsNotClean = - stepsNotClean.includes(true); - - const stepDetails = Object.values(stepStatusSummary); - - const stepDurations = - stepDetails.map(({status, timeStart, timeEnd}) => { - if ( - status === STATUS_NOT_APPLICABLE || - status === STATUS_NOT_STARTED || - status === STATUS_STARTED_NOT_DONE - ) { - return '-'; - } - - if (typeof timeStart !== 'number' || typeof timeEnd !== 'number') { - return 'unknown'; + if (shouldShowStepStatusSummary) { + if (numRestarts >= 1) { + console.error(colors.bright(`Step summary since latest restart:`)); + } else { + console.error(colors.bright(`Step summary before restart:`)); } - return formatDuration(timeEnd - timeStart); - }); - - const longestDurationLength = - Math.max(...stepDurations.map(duration => duration.length)); - - for (let index = 0; index < stepDetails.length; index++) { - const {name, status, annotation} = stepDetails[index]; - const duration = stepDurations[index]; - - let message = - (stepsNotClean[index] - ? `!! ` - : ` - `); - - message += `(${duration})`.padStart(longestDurationLength + 2, ' '); - message += ` `; - message += `${name}: `.padEnd(longestNameLength + 4, '.'); - message += ` `; - message += status; - - if (annotation) { - message += ` (${annotation})`; + showStepStatusSummary(); + console.log(''); } - switch (status) { - case STATUS_DONE_CLEAN: - console.error(colors.green(message)); - break; - - case STATUS_NOT_STARTED: - case STATUS_NOT_APPLICABLE: - console.error(colors.dim(message)); - break; + if (numRestarts > 5) { + logError`A restart was cued, but we've restarted a bunch already.`; + logError`Exiting because this is probably a bug!`; + console.log(''); + break; + } else { + console.log(''); + logInfo`A restart was cued. This is probably normal, and required`; + logInfo`to load updated data files. Restarting automatically now!`; + console.log(''); + numRestarts++; + } + } else { + break; + } + } - case STATUS_HAS_WARNINGS: - case STATUS_STARTED_NOT_DONE: - console.error(colors.yellow(message)); - break; + if (shouldShowStepStatusSummary) { + if (numRestarts >= 1) { + console.error(colors.bright(`Step summary after final restart:`)); + } else { + console.error(colors.bright(`Step summary:`)); + } - case STATUS_FATAL_ERROR: - console.error(colors.red(message)); - break; + const {anyStepsNotClean} = + showStepStatusSummary(); - default: - console.error(message); - break; - } - } + const totalTimeEnd = Date.now(); + const totalDuration = formatDuration(totalTimeEnd - totalTimeStart); console.error(colors.bright(`Done in ${totalDuration}.`)); @@ -2305,8 +3378,107 @@ if (true || isMain(import.meta.url) || path.basename(process.argv[1]) === 'hsmus } decorateTime.displayTime(); - CacheableObject.showInvalidAccesses(); process.exit(0); })(); } + +function showStepStatusSummary() { + const longestNameLength = + Math.max(... + Object.values(stepStatusSummary) + .map(({name}) => name.length)); + + const stepsNotClean = + Object.values(stepStatusSummary) + .map(({status}) => + status === STATUS_HAS_WARNINGS || + status === STATUS_FATAL_ERROR || + status === STATUS_STARTED_NOT_DONE); + + const anyStepsNotClean = + stepsNotClean.includes(true); + + const stepDetails = Object.values(stepStatusSummary); + + const stepDurations = + stepDetails.map(({status, timeStart, timeEnd}) => { + if ( + status === STATUS_NOT_APPLICABLE || + status === STATUS_NOT_STARTED || + status === STATUS_STARTED_NOT_DONE + ) { + return '-'; + } + + if (typeof timeStart !== 'number' || typeof timeEnd !== 'number') { + return 'unknown'; + } + + return formatDuration(timeEnd - timeStart); + }); + + const longestDurationLength = + Math.max(...stepDurations.map(duration => duration.length)); + + const stepMemories = + stepDetails.map(({memory}) => + (memory + ? Math.round(memory["heapUsed"] / 1024 / 1024) + 'MB' + : '-')); + + const longestMemoryLength = + Math.max(...stepMemories.map(memory => memory.length)); + + for (let index = 0; index < stepDetails.length; index++) { + const {name, status, annotation} = stepDetails[index]; + const duration = stepDurations[index]; + const memory = stepMemories[index]; + + let message = + (stepsNotClean[index] + ? `!! ` + : ` - `); + + message += `(${duration} `.padStart(longestDurationLength + 2, ' '); + + if (shouldShowStepMemoryInSummary) { + message += ` ${memory})`.padStart(longestMemoryLength + 2, ' '); + } + + message += ` `; + message += `${name}: `.padEnd(longestNameLength + 4, '.'); + message += ` `; + message += status; + + if (annotation) { + message += ` (${annotation})`; + } + + switch (status) { + case STATUS_DONE_CLEAN: + console.error(colors.green(message)); + break; + + case STATUS_NOT_STARTED: + case STATUS_NOT_APPLICABLE: + console.error(colors.dim(message)); + break; + + case STATUS_HAS_WARNINGS: + case STATUS_STARTED_NOT_DONE: + console.error(colors.yellow(message)); + break; + + case STATUS_FATAL_ERROR: + console.error(colors.red(message)); + break; + + default: + console.error(message); + break; + } + } + + return {anyStepsNotClean}; +} |