diff options
Diffstat (limited to 'src/upd8.js')
| -rwxr-xr-x | src/upd8.js | 508 |
1 files changed, 371 insertions, 137 deletions
diff --git a/src/upd8.js b/src/upd8.js index 045bf139..ae072d5a 100755 --- a/src/upd8.js +++ b/src/upd8.js @@ -42,8 +42,9 @@ import wrap from 'word-wrap'; import {mapAggregate, openAggregate, showAggregate} from '#aggregate'; import CacheableObject from '#cacheable-object'; -import {stringifyCache} from '#cli'; +import {formatDuration, stringifyCache} from '#cli'; import {displayCompositeCacheAnalysis} from '#composite'; +import * as html from '#html'; import find, {bindFind, getAllFindSpecs} from '#find'; import {processLanguageFile, watchLanguageFile, internalDefaultStringsFile} from '#language'; @@ -51,6 +52,7 @@ import {isMain, traverse} from '#node-utils'; import {bindReverse} from '#reverse'; import {writeSearchData} from '#search'; import {sortByName} from '#sort'; +import thingConstructors from '#things'; import {identifyAllWebRoutes} from '#web-routes'; import { @@ -67,8 +69,9 @@ import { import { filterReferenceErrors, - reportDirectoryErrors, reportContentTextErrors, + reportDirectoryErrors, + reportOrphanedArtworks, } from '#data-checks'; import { @@ -115,7 +118,7 @@ const __dirname = path.dirname(fileURLToPath(import.meta.url)); let COMMIT; try { COMMIT = execSync('git log --format="%h %B" -n 1 HEAD', {cwd: __dirname}).toString().trim(); -} catch (error) { +} catch { COMMIT = '(failed to detect)'; } @@ -133,8 +136,8 @@ const defaultStepStatus = {status: STATUS_NOT_STARTED, annotation: null}; // Defined globally for quick access outside the main() function's contents. // This will be initialized and mutated over the course of main(). let stepStatusSummary; -let showStepStatusSummary = false; -let showStepMemoryInSummary = false; +let shouldShowStepStatusSummary = false; +let shouldShowStepMemoryInSummary = false; async function main() { Error.stackTraceLimit = Infinity; @@ -174,6 +177,10 @@ async function main() { {...defaultStepStatus, name: `report directory errors`, for: ['verify']}, + reportOrphanedArtworks: + {...defaultStepStatus, name: `report orphaned artworks`, + for: ['verify']}, + filterReferenceErrors: {...defaultStepStatus, name: `filter reference errors`, for: ['verify']}, @@ -190,6 +197,13 @@ async function main() { {...defaultStepStatus, name: `precache nearly all data`, for: ['build']}, + sortWikiDataSourceFiles: + {...defaultStepStatus, name: `apply sorting rules to wiki data files`, + for: ['build']}, + + checkWikiDataSourceFileSorting: + {...defaultStepStatus, name: `check sorting rules against wiki data files`}, + loadURLFiles: {...defaultStepStatus, name: `load internal & custom url spec files`, for: ['build']}, @@ -270,6 +284,35 @@ async function main() { })); let selectedBuildModeFlag; + let sortInAdditionToBuild = false; + + // As an exception, --sort can be combined with another build mode. + if (selectedBuildModeFlags.length >= 2 && selectedBuildModeFlags.includes('sort')) { + sortInAdditionToBuild = true; + selectedBuildModeFlags.splice(selectedBuildModeFlags.indexOf('sort'), 1); + } + + if (sortInAdditionToBuild) { + Object.assign(stepStatusSummary.sortWikiDataSourceFiles, { + status: STATUS_NOT_STARTED, + annotation: `--sort provided with another build mode`, + }); + + Object.assign(stepStatusSummary.checkWikiDataSourceFileSorting, { + status: STATUS_NOT_APPLICABLE, + annotation: `--sort provided, dry run not applicable`, + }); + } else { + Object.assign(stepStatusSummary.sortWikiDataSourceFiles, { + status: STATUS_NOT_APPLICABLE, + annotation: `--sort not provided, dry run only`, + }); + + Object.assign(stepStatusSummary.checkWikiDataSourceFileSorting, { + status: STATUS_NOT_STARTED, + annotation: `--sort not provided, dry run applicable`, + }); + } if (empty(selectedBuildModeFlags)) { // No build mode selected. This is not a valid state for building the wiki, @@ -362,6 +405,11 @@ async function main() { type: 'flag', }, + 'skip-orphaned-artwork-validation': { + help: `Skips checking for internally orphaned artworks, which is a bad idea, unless you're debugging those in particular`, + type: 'flag', + }, + 'skip-reference-validation': { help: `Skips checking and reporting reference errors, which speeds up the build but may silently allow erroneous data to pass through`, type: 'flag', @@ -412,6 +460,11 @@ async function main() { type: 'flag', }, + 'skip-sorting-validation': { + help: `Skips checking the if custom sorting rules for this wiki are satisfied`, + type: 'flag', + }, + 'skip-media-validation': { help: `Skips checking and reporting missing and misplaced media files, which isn't necessary if you aren't adding or removing data or updating directories`, type: 'flag', @@ -466,6 +519,11 @@ async function main() { type: 'flag', }, + 'skip-self-diagnosis': { + help: `Disable some runtime validation for the wiki's own code, which speeds up long builds, but may allow unpredicted corner cases to fail strangely and silently`, + type: 'flag', + }, + 'queue-size': { help: `Process more or fewer disk files at once to optimize performance or avoid I/O errors, unlimited if set to 0 (between 500 and 700 is usually a safe range for building HSMusic on Windows machines)\nDefaults to ${defaultQueueSize}`, type: 'value', @@ -525,8 +583,8 @@ async function main() { ...buildOptions, }); - showStepStatusSummary = cliOptions['show-step-summary'] ?? false; - showStepMemoryInSummary = cliOptions['show-step-memory'] ?? false; + shouldShowStepStatusSummary = cliOptions['show-step-summary'] ?? false; + shouldShowStepMemoryInSummary = cliOptions['show-step-memory'] ?? false; if (cliOptions['help']) { console.log( @@ -604,7 +662,8 @@ async function main() { const thumbsOnly = cliOptions['thumbs-only'] ?? false; const noInput = cliOptions['no-input'] ?? false; - const showAggregateTraces = cliOptions['show-traces'] ?? false; + const skipSelfDiagnosis = cliOptions['skip-self-diagnosis'] ?? false; + const showTraces = cliOptions['show-traces'] ?? false; const precacheMode = cliOptions['precache-mode'] ?? 'common'; @@ -801,6 +860,16 @@ async function main() { }, }); + fallbackStep('reportOrphanedArtworks', { + default: 'perform', + cli: { + flag: 'skip-orphaned-artwork-validation', + negate: true, + warn: + `Skipping orphaned artwork validation. Hopefully you're debugging!`, + }, + }); + fallbackStep('filterReferenceErrors', { default: 'perform', cli: { @@ -946,6 +1015,18 @@ async function main() { paragraph = false; } + fallbackStep('checkWikiDataSourceFileSorting', { + default: 'perform', + buildConfig: 'sort', + cli: { + flag: 'skip-sorting-validation', + negate: true, + warning: + `Skipping sorting validation. If any of this wiki's sorting rules are not\n` + + `satisfied, those errors will be silently passed along to the build.`, + }, + }); + fallbackStep('verifyImagePaths', { default: 'perform', buildConfig: 'mediaValidation', @@ -1082,6 +1163,18 @@ async function main() { return false; } + if (skipSelfDiagnosis) { + logWarn`${'Skipping code self-diagnosis.'} (--skip-self-diagnosis provided)`; + logWarn`This build should run substantially faster, but corner cases`; + logWarn`not previously predicted may fail strangely and silently.`; + + html.disableSlotValidation(); + } + + if (!showTraces) { + html.disableTagTracing(); + } + Object.assign(stepStatusSummary.determineMediaCachePath, { status: STATUS_STARTED_NOT_DONE, timeStart: Date.now(), @@ -1260,7 +1353,7 @@ async function main() { const niceShowAggregate = (error, ...opts) => { showAggregate(error, { - showTraces: showAggregateTraces, + showTraces, pathToFileURL: (f) => path.relative(__dirname, fileURLToPath(f)), ...opts, }); @@ -1480,6 +1573,10 @@ async function main() { ? prop : wikiData[prop]); + if (array && empty(array)) { + return; + } + logInfo` - ${array?.length ?? colors.red('(Missing!)')} ${colors.normal(colors.dim(label))}`; } @@ -1506,9 +1603,14 @@ async function main() { logThings('newsData', 'news entries'); } logThings('staticPageData', 'static pages'); + logThings('sortingRules', 'sorting rules'); if (wikiData.homepageLayout) { logInfo` - ${1} homepage layout (${ - wikiData.homepageLayout.rows.length + wikiData.homepageLayout.sections.length + } sections, ${ + wikiData.homepageLayout.sections + .flatMap(section => section.rows) + .length } rows)`; } if (wikiData.wikiInfo) { @@ -1673,8 +1775,8 @@ async function main() { }); } - // Filter out any things with duplicate directories throughout the data, - // warning about them too. + // Check for things with duplicate directories throughout the data, + // and halt if any are found. if (stepStatusSummary.reportDirectoryErrors.status === STATUS_NOT_STARTED) { Object.assign(stepStatusSummary.reportDirectoryErrors, { @@ -1715,8 +1817,42 @@ async function main() { } } - // Filter out any reference errors throughout the data, warning about them - // too. + // Check for artwork objects which have been orphaned from their things, + // and halt if any are found. + + if (stepStatusSummary.reportOrphanedArtworks.status === STATUS_NOT_STARTED) { + Object.assign(stepStatusSummary.reportOrphanedArtworks, { + status: STATUS_STARTED_NOT_DONE, + timeStart: Date.now(), + }); + + try { + reportOrphanedArtworks(wikiData, {getAllFindSpecs}); + + Object.assign(stepStatusSummary.reportOrphanedArtworks, { + status: STATUS_DONE_CLEAN, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + } catch (aggregate) { + if (!paragraph) console.log(''); + niceShowAggregate(aggregate); + + logError`Failed to initialize artwork data connections properly.`; + fileIssue(); + + Object.assign(stepStatusSummary.reportOrphanedArtworks, { + status: STATUS_FATAL_ERROR, + annotation: `orphaned artworks found`, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + + return false; + } + } + + // Filter out any reference errors throughout the data, warning about these. if (stepStatusSummary.filterReferenceErrors.status === STATUS_NOT_STARTED) { Object.assign(stepStatusSummary.filterReferenceErrors, { @@ -1837,6 +1973,77 @@ async function main() { }); } + if (stepStatusSummary.sortWikiDataSourceFiles.status === STATUS_NOT_STARTED) { + Object.assign(stepStatusSummary.sortWikiDataSourceFiles, { + status: STATUS_STARTED_NOT_DONE, + timeStart: Date.now(), + }); + + const {SortingRule} = thingConstructors; + const results = + await Array.fromAsync(SortingRule.go({dataPath, wikiData})); + + if (results.some(result => result.changed)) { + logInfo`Updated data files to satisfy sorting.`; + logInfo`Restarting automatically, since that's now needed!`; + + Object.assign(stepStatusSummary.sortWikiDataSourceFiles, { + status: STATUS_DONE_CLEAN, + annotation: `changes cueing restart`, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + + return 'restart'; + } else { + logInfo`All sorting rules are satisfied. Nice!`; + paragraph = false; + + Object.assign(stepStatusSummary.sortWikiDataSourceFiles, { + status: STATUS_DONE_CLEAN, + annotation: `no changes needed`, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + } + } else if (stepStatusSummary.checkWikiDataSourceFileSorting.status === STATUS_NOT_STARTED) { + Object.assign(stepStatusSummary.checkWikiDataSourceFileSorting, { + status: STATUS_STARTED_NOT_DONE, + timeStart: Date.now(), + }); + + const {SortingRule} = thingConstructors; + const results = + await Array.fromAsync(SortingRule.go({dataPath, wikiData, dry: true})); + + const needed = results.filter(result => result.changed); + + if (empty(needed)) { + logInfo`All sorting rules are satisfied. Nice!`; + paragraph = false; + + Object.assign(stepStatusSummary.checkWikiDataSourceFileSorting, { + status: STATUS_DONE_CLEAN, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + } else { + logWarn`Some of this wiki's sorting rules currently aren't satisfied:`; + for (const {rule} of needed) { + logWarn`- ${rule.message}`; + } + logWarn`Run ${'hsmusic --sort'} to automatically update data files.`; + paragraph = false; + + Object.assign(stepStatusSummary.checkWikiDataSourceFileSorting, { + status: STATUS_HAS_WARNINGS, + annotation: `not all rules satisfied`, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + } + } + if (stepStatusSummary.performBuild.status === STATUS_NOT_APPLICABLE) { displayCompositeCacheAnalysis(); @@ -2231,7 +2438,7 @@ async function main() { }); internalDefaultLanguage = internalDefaultLanguageWatcher.language; - } catch (_error) { + } catch { // No need to display the error here - it's already printed by // watchLanguageFile. errorLoadingInternalDefaultLanguage = true; @@ -3019,6 +3226,7 @@ async function main() { developersComment, languages, missingImagePaths, + niceShowAggregate, thumbsCache, urlSpec, urls, @@ -3081,145 +3289,71 @@ async function main() { } // TODO: isMain detection isn't consistent across platforms here -/* eslint-disable-next-line no-constant-condition */ +// eslint-disable-next-line no-constant-binary-expression if (true || isMain(import.meta.url) || path.basename(process.argv[1]) === 'hsmusic') { (async () => { let result; + let numRestarts = 0; const totalTimeStart = Date.now(); - try { - result = await main(); - } catch (error) { - if (error instanceof AggregateError) { - showAggregate(error); - } else if (error.cause) { - console.error(error); - showAggregate(error); - } else { - console.error(error); - } - } - - const totalTimeEnd = Date.now(); - - const formatDuration = timeDelta => { - const seconds = timeDelta / 1000; - - if (seconds > 90) { - const modSeconds = Math.floor(seconds % 60); - const minutes = Math.floor(seconds - seconds % 60) / 60; - return `${minutes}m${modSeconds}s`; - } - - if (seconds < 0.1) { - return 'instant'; + while (true) { + try { + result = await main(); + } catch (error) { + if (error instanceof AggregateError) { + showAggregate(error); + } else if (error.cause) { + console.error(error); + showAggregate(error); + } else { + console.error(error); + } } - const precision = (seconds > 1 ? 3 : 2); - return `${seconds.toPrecision(precision)}s`; - }; - - if (showStepStatusSummary) { - const totalDuration = formatDuration(totalTimeEnd - totalTimeStart); - - console.error(colors.bright(`Step summary:`)); - - const longestNameLength = - Math.max(... - Object.values(stepStatusSummary) - .map(({name}) => name.length)); - - const stepsNotClean = - Object.values(stepStatusSummary) - .map(({status}) => - status === STATUS_HAS_WARNINGS || - status === STATUS_FATAL_ERROR || - status === STATUS_STARTED_NOT_DONE); - - const anyStepsNotClean = - stepsNotClean.includes(true); - - const stepDetails = Object.values(stepStatusSummary); - - const stepDurations = - stepDetails.map(({status, timeStart, timeEnd}) => { - if ( - status === STATUS_NOT_APPLICABLE || - status === STATUS_NOT_STARTED || - status === STATUS_STARTED_NOT_DONE - ) { - return '-'; - } + if (result === 'restart') { + console.log(''); - if (typeof timeStart !== 'number' || typeof timeEnd !== 'number') { - return 'unknown'; + if (shouldShowStepStatusSummary) { + if (numRestarts >= 1) { + console.error(colors.bright(`Step summary since latest restart:`)); + } else { + console.error(colors.bright(`Step summary before restart:`)); } - return formatDuration(timeEnd - timeStart); - }); - - const longestDurationLength = - Math.max(...stepDurations.map(duration => duration.length)); - - const stepMemories = - stepDetails.map(({memory}) => - (memory - ? Math.round(memory["heapUsed"] / 1024 / 1024) + 'MB' - : '-')); - - const longestMemoryLength = - Math.max(...stepMemories.map(memory => memory.length)); - - for (let index = 0; index < stepDetails.length; index++) { - const {name, status, annotation} = stepDetails[index]; - const duration = stepDurations[index]; - const memory = stepMemories[index]; - - let message = - (stepsNotClean[index] - ? `!! ` - : ` - `); - - message += `(${duration} `.padStart(longestDurationLength + 2, ' '); - - if (showStepMemoryInSummary) { - message += ` ${memory})`.padStart(longestMemoryLength + 2, ' '); + showStepStatusSummary(); + console.log(''); } - message += ` `; - message += `${name}: `.padEnd(longestNameLength + 4, '.'); - message += ` `; - message += status; - - if (annotation) { - message += ` (${annotation})`; + if (numRestarts > 5) { + logError`A restart was cued, but we've restarted a bunch already.`; + logError`Exiting because this is probably a bug!`; + console.log(''); + break; + } else { + console.log(''); + logInfo`A restart was cued. This is probably normal, and required`; + logInfo`to load updated data files. Restarting automatically now!`; + console.log(''); + numRestarts++; } + } else { + break; + } + } - switch (status) { - case STATUS_DONE_CLEAN: - console.error(colors.green(message)); - break; - - case STATUS_NOT_STARTED: - case STATUS_NOT_APPLICABLE: - console.error(colors.dim(message)); - break; - - case STATUS_HAS_WARNINGS: - case STATUS_STARTED_NOT_DONE: - console.error(colors.yellow(message)); - break; + if (shouldShowStepStatusSummary) { + if (numRestarts >= 1) { + console.error(colors.bright(`Step summary after final restart:`)); + } else { + console.error(colors.bright(`Step summary:`)); + } - case STATUS_FATAL_ERROR: - console.error(colors.red(message)); - break; + const {anyStepsNotClean} = + showStepStatusSummary(); - default: - console.error(message); - break; - } - } + const totalTimeEnd = Date.now(); + const totalDuration = formatDuration(totalTimeEnd - totalTimeStart); console.error(colors.bright(`Done in ${totalDuration}.`)); @@ -3248,3 +3382,103 @@ if (true || isMain(import.meta.url) || path.basename(process.argv[1]) === 'hsmus process.exit(0); })(); } + +function showStepStatusSummary() { + const longestNameLength = + Math.max(... + Object.values(stepStatusSummary) + .map(({name}) => name.length)); + + const stepsNotClean = + Object.values(stepStatusSummary) + .map(({status}) => + status === STATUS_HAS_WARNINGS || + status === STATUS_FATAL_ERROR || + status === STATUS_STARTED_NOT_DONE); + + const anyStepsNotClean = + stepsNotClean.includes(true); + + const stepDetails = Object.values(stepStatusSummary); + + const stepDurations = + stepDetails.map(({status, timeStart, timeEnd}) => { + if ( + status === STATUS_NOT_APPLICABLE || + status === STATUS_NOT_STARTED || + status === STATUS_STARTED_NOT_DONE + ) { + return '-'; + } + + if (typeof timeStart !== 'number' || typeof timeEnd !== 'number') { + return 'unknown'; + } + + return formatDuration(timeEnd - timeStart); + }); + + const longestDurationLength = + Math.max(...stepDurations.map(duration => duration.length)); + + const stepMemories = + stepDetails.map(({memory}) => + (memory + ? Math.round(memory["heapUsed"] / 1024 / 1024) + 'MB' + : '-')); + + const longestMemoryLength = + Math.max(...stepMemories.map(memory => memory.length)); + + for (let index = 0; index < stepDetails.length; index++) { + const {name, status, annotation} = stepDetails[index]; + const duration = stepDurations[index]; + const memory = stepMemories[index]; + + let message = + (stepsNotClean[index] + ? `!! ` + : ` - `); + + message += `(${duration} `.padStart(longestDurationLength + 2, ' '); + + if (shouldShowStepMemoryInSummary) { + message += ` ${memory})`.padStart(longestMemoryLength + 2, ' '); + } + + message += ` `; + message += `${name}: `.padEnd(longestNameLength + 4, '.'); + message += ` `; + message += status; + + if (annotation) { + message += ` (${annotation})`; + } + + switch (status) { + case STATUS_DONE_CLEAN: + console.error(colors.green(message)); + break; + + case STATUS_NOT_STARTED: + case STATUS_NOT_APPLICABLE: + console.error(colors.dim(message)); + break; + + case STATUS_HAS_WARNINGS: + case STATUS_STARTED_NOT_DONE: + console.error(colors.yellow(message)); + break; + + case STATUS_FATAL_ERROR: + console.error(colors.red(message)); + break; + + default: + console.error(message); + break; + } + } + + return {anyStepsNotClean}; +} |