diff options
Diffstat (limited to 'src/upd8.js')
| -rwxr-xr-x | src/upd8.js | 782 |
1 files changed, 591 insertions, 191 deletions
diff --git a/src/upd8.js b/src/upd8.js index 5589877c..b2f6e638 100755 --- a/src/upd8.js +++ b/src/upd8.js @@ -42,15 +42,19 @@ import wrap from 'word-wrap'; import {mapAggregate, openAggregate, showAggregate} from '#aggregate'; import CacheableObject from '#cacheable-object'; -import {stringifyCache} from '#cli'; +import {formatDuration, stringifyCache} from '#cli'; import {displayCompositeCacheAnalysis} from '#composite'; +import * as html from '#html'; import find, {bindFind, getAllFindSpecs} from '#find'; import {processLanguageFile, watchLanguageFile, internalDefaultStringsFile} from '#language'; import {isMain, traverse} from '#node-utils'; +import * as quickstat from '#quickstat'; import {bindReverse} from '#reverse'; import {writeSearchData} from '#search'; import {sortByName} from '#sort'; +import thingConstructors from '#things'; +import {disableCuratedURLValidation} from '#validators'; import {identifyAllWebRoutes} from '#web-routes'; import { @@ -67,8 +71,9 @@ import { import { filterReferenceErrors, - reportDirectoryErrors, reportContentTextErrors, + reportDirectoryErrors, + reportOrphanedArtworks, } from '#data-checks'; import { @@ -76,6 +81,7 @@ import { empty, filterMultipleArrays, indentWrap as unboundIndentWrap, + stitchArrays, withEntries, } from '#sugar'; @@ -102,20 +108,22 @@ import { linkWikiDataArrays, loadYAMLDocumentsFromDataSteps, processThingsFromDataSteps, - saveThingsFromDataSteps, + connectThingsFromDataSteps, + makeWikiDataFromDataSteps, sortWikiDataArrays, } from '#yaml'; import FileSizePreloader from './file-size-preloader.js'; import {listingSpec, listingTargetSpec} from './listing-spec.js'; import * as buildModes from './write/build-modes/index.js'; +import * as tidyModes from './write/tidy-modes/index.js'; const __dirname = path.dirname(fileURLToPath(import.meta.url)); let COMMIT; try { COMMIT = execSync('git log --format="%h %B" -n 1 HEAD', {cwd: __dirname}).toString().trim(); -} catch (error) { +} catch { COMMIT = '(failed to detect)'; } @@ -126,6 +134,7 @@ const STATUS_NOT_APPLICABLE = `not applicable`; const STATUS_STARTED_NOT_DONE = `started but not yet done`; const STATUS_DONE_CLEAN = `done without warnings`; const STATUS_FATAL_ERROR = `fatal error`; +const STATUS_INVALID_SIGNAL = `invalid exit signal`; const STATUS_HAS_WARNINGS = `has warnings`; const defaultStepStatus = {status: STATUS_NOT_STARTED, annotation: null}; @@ -133,8 +142,8 @@ const defaultStepStatus = {status: STATUS_NOT_STARTED, annotation: null}; // Defined globally for quick access outside the main() function's contents. // This will be initialized and mutated over the course of main(). let stepStatusSummary; -let showStepStatusSummary = false; -let showStepMemoryInSummary = false; +let shouldShowStepStatusSummary = false; +let shouldShowStepMemoryInSummary = false; async function main() { Error.stackTraceLimit = Infinity; @@ -174,6 +183,10 @@ async function main() { {...defaultStepStatus, name: `report directory errors`, for: ['verify']}, + reportOrphanedArtworks: + {...defaultStepStatus, name: `report orphaned artworks`, + for: ['verify']}, + filterReferenceErrors: {...defaultStepStatus, name: `filter reference errors`, for: ['verify']}, @@ -190,6 +203,9 @@ async function main() { {...defaultStepStatus, name: `precache nearly all data`, for: ['build']}, + checkWikiDataSourceFileSorting: + {...defaultStepStatus, name: `check sorting rules against wiki data files`}, + loadURLFiles: {...defaultStepStatus, name: `load internal & custom url spec files`, for: ['build']}, @@ -235,6 +251,14 @@ async function main() { {...defaultStepStatus, name: `identify web routes`, for: ['build']}, + reformatCuratedURLs: + {...defaultStepStatus, name: `reformat curated URLs`, + for: ['build']}, + + sortWikiDataSourceFiles: + {...defaultStepStatus, name: `apply sorting rules to wiki data files`, + for: ['build']}, + performBuild: {...defaultStepStatus, name: `perform selected build mode`, for: ['build']}, @@ -256,32 +280,77 @@ async function main() { const defaultQueueSize = 500; - const buildModeFlagOptions = ( + const buildModeFlagOptions = withEntries(buildModes, entries => entries.map(([key, mode]) => [key, { help: mode.description, type: 'flag', - }]))); + }])); - const selectedBuildModeFlags = Object.keys( - await parseOptions(process.argv.slice(2), { - [parseOptions.handleUnknown]: () => {}, - ...buildModeFlagOptions, - })); + const selectedBuildModeFlags = + Object.keys( + await parseOptions(process.argv.slice(2), { + [parseOptions.handleUnknown]: () => {}, + ...buildModeFlagOptions, + })); - let selectedBuildModeFlag; + const tidyModeFlagOptions = + withEntries(tidyModes, entries => + entries.map(([key, mode]) => [key, { + help: mode.description, + type: 'flag', + }])); + + const selectedTidyModeFlags = + Object.keys( + await parseOptions(process.argv.slice(2), { + [parseOptions.handleUnknown]: () => {}, + ...tidyModeFlagOptions, + })); + + if (selectedTidyModeFlags.includes('format-urls')) { + Object.assign(stepStatusSummary.reformatCuratedURLs, { + status: STATUS_NOT_STARTED, + annotation: `--format-urls provided`, + }); + } else { + Object.assign(stepStatusSummary.reformatCuratedURLs, { + status: STATUS_NOT_APPLICABLE, + annotation: `--format-urls not provided`, + }); + } - if (empty(selectedBuildModeFlags)) { - // No build mode selected. This is not a valid state for building the wiki, - // but we want to let access to --help, so we'll show a message about what - // to do later. - selectedBuildModeFlag = null; - } else if (selectedBuildModeFlags.length > 1) { - logError`Building multiple modes (${selectedBuildModeFlags.join(', ')}) at once not supported.`; - logError`Please specify one build mode.`; - return false; + if (selectedTidyModeFlags.includes('sort')) { + Object.assign(stepStatusSummary.sortWikiDataSourceFiles, { + status: STATUS_NOT_STARTED, + annotation: `--sort provided`, + }); + + Object.assign(stepStatusSummary.checkWikiDataSourceFileSorting, { + status: STATUS_NOT_APPLICABLE, + annotation: `--sort provided, dry run not applicable`, + }); } else { - selectedBuildModeFlag = selectedBuildModeFlags[0]; + Object.assign(stepStatusSummary.sortWikiDataSourceFiles, { + status: STATUS_NOT_APPLICABLE, + annotation: `--sort not provided, dry run only`, + }); + + Object.assign(stepStatusSummary.checkWikiDataSourceFileSorting, { + status: STATUS_NOT_STARTED, + annotation: `--sort not provided, dry run applicable`, + }); + } + + let selectedBuildModeFlag; + switch (selectedBuildModeFlags.length) { + case 0: selectedBuildModeFlag = null; break; + case 1: selectedBuildModeFlag = selectedBuildModeFlags[0]; break; + default: { + logError`Building multiple modes (${selectedBuildModeFlags.join(', ')}) at once not supported.`; + logError`Please specify one build mode.`; + return false; + } } const selectedBuildMode = @@ -289,16 +358,25 @@ async function main() { ? buildModes[selectedBuildModeFlag] : null); - // This is about to get a whole lot more stuff put in it. - const wikiData = { - listingSpec, - listingTargetSpec, - }; + const selectedTidyModes = + selectedTidyModeFlags + .map(flag => tidyModes[flag]); - const buildOptions = - (selectedBuildMode - ? selectedBuildMode.getCLIOptions() - : {}); + const tidyingOnly = + !selectedBuildMode && + !empty(selectedTidyModes); + + const selectedBuildModeOptions = + selectedBuildMode?.getCLIOptions?.() ?? + {}; + + const selectedTidyModeOptions = + selectedTidyModes.map(tidyMode => + tidyMode.getCLIOptions?.() ?? + {}); + + const selectedTidyModeOptionsFlat = + Object.fromEntries(selectedTidyModeOptions.flat()); const commonOptions = { 'help': { @@ -362,6 +440,11 @@ async function main() { type: 'flag', }, + 'skip-orphaned-artwork-validation': { + help: `Skips checking for internally orphaned artworks, which is a bad idea, unless you're debugging those in particular`, + type: 'flag', + }, + 'skip-reference-validation': { help: `Skips checking and reporting reference errors, which speeds up the build but may silently allow erroneous data to pass through`, type: 'flag', @@ -402,6 +485,11 @@ async function main() { type: 'flag', }, + 'skip-curated-url-validation': { + help: `Skips checking if URLs match a set of standardizing rules; only intended for use with old data`, + type: 'flag', + }, + 'skip-file-sizes': { help: `Skips preloading file sizes for images and additional files, which will be left blank in the build`, type: 'flag', @@ -412,6 +500,11 @@ async function main() { type: 'flag', }, + 'skip-sorting-validation': { + help: `Skips checking the if custom sorting rules for this wiki are satisfied`, + type: 'flag', + }, + 'skip-media-validation': { help: `Skips checking and reporting missing and misplaced media files, which isn't necessary if you aren't adding or removing data or updating directories`, type: 'flag', @@ -466,6 +559,11 @@ async function main() { type: 'flag', }, + 'skip-self-diagnosis': { + help: `Disable some runtime validation for the wiki's own code, which speeds up long builds, but may allow unpredicted corner cases to fail strangely and silently`, + type: 'flag', + }, + 'queue-size': { help: `Process more or fewer disk files at once to optimize performance or avoid I/O errors, unlimited if set to 0 (between 500 and 700 is usually a safe range for building HSMusic on Windows machines)\nDefaults to ${defaultQueueSize}`, type: 'value', @@ -520,13 +618,15 @@ async function main() { // here, even though we won't be doing anything with them later. // (This is a bit of a hack.) ...buildModeFlagOptions, + ...tidyModeFlagOptions, ...commonOptions, - ...buildOptions, + ...selectedTidyModeOptionsFlat, + ...selectedBuildModeOptions, }); - showStepStatusSummary = cliOptions['show-step-summary'] ?? false; - showStepMemoryInSummary = cliOptions['show-step-memory'] ?? false; + shouldShowStepStatusSummary = cliOptions['show-step-summary'] ?? false; + shouldShowStepMemoryInSummary = cliOptions['show-step-memory'] ?? false; if (cliOptions['help']) { console.log( @@ -539,7 +639,7 @@ async function main() { `and website content/structure ` + `from provided data, media, and language directories.\n` + `\n` + - `CLI options are divided into three groups:\n`)); + `CLI options are divided into five groups:\n`)); console.log(` 1) ` + indentWrap( `Common options: ` + @@ -548,37 +648,63 @@ async function main() { {spaces: 4, bullet: true})); console.log(` 2) ` + indentWrap( + `Tidying mode selection: ` + + `One or more tidying mode may be selected, ` + + `and they adjust the contents of data files ` + + `to satisfy predefined or data-configured standardization rules`, + {spaces: 4, bullet: true})); + + console.log(` 3) ` + indentWrap( + `Tidying mode options: ` + + `Each tidy mode may `)) + + console.log(` 4) ` + indentWrap( `Build mode selection: ` + `One build mode should be selected, ` + `and it decides the main set of behavior to use ` + `for presenting or interacting with site content`, {spaces: 4, bullet: true})); - console.log(` 3) ` + indentWrap( - `Build options: ` + + console.log(` 5) ` + indentWrap( + `Build mode options: ` + `Each build mode has a set of unique options ` + `which customize behavior for that build mode`, {spaces: 4, bullet: true})); + console.log(`All options may be specified in any order.`); + console.log(``); showHelpForOptions({ heading: `Common options`, options: commonOptions, - wrap, }); showHelpForOptions({ + heading: `Tidying mode selection`, + options: tidyModeFlagOptions, + }); + + stitchArrays({ + flag: selectedTidyModeFlags, + options: selectedTidyModeOptions, + }).forEach(({flag, options}) => { + showHelpForOptions({ + heading: `Options for tidying mode --${flag}`, + options, + silentIfNoOptions: false, + }); + }); + + showHelpForOptions({ heading: `Build mode selection`, options: buildModeFlagOptions, - wrap, }); if (selectedBuildMode) { showHelpForOptions({ - heading: `Build options for --${selectedBuildModeFlag}`, - options: buildOptions, - wrap, + heading: `Options for build mode --${selectedBuildModeFlag}`, + options: selectedBuildModeOptions, }); } else { console.log( @@ -604,7 +730,8 @@ async function main() { const thumbsOnly = cliOptions['thumbs-only'] ?? false; const noInput = cliOptions['no-input'] ?? false; - const showAggregateTraces = cliOptions['show-traces'] ?? false; + const skipSelfDiagnosis = cliOptions['skip-self-diagnosis'] ?? false; + const showTraces = cliOptions['show-traces'] ?? false; const precacheMode = cliOptions['precache-mode'] ?? 'common'; @@ -643,6 +770,34 @@ async function main() { }); } + if (tidyingOnly) { + Object.assign(stepStatusSummary.performBuild, { + status: STATUS_NOT_APPLICABLE, + annotation: `tidying modes provided`, + }); + + for (const key of [ + 'preloadFileSizes', + 'watchLanguageFiles', + 'verifyImagePaths', + 'buildSearchIndex', + 'generateThumbnails', + 'identifyWebRoutes', + 'checkWikiDataSourceFileSorting', + ]) { + Object.assign(stepStatusSummary[key], { + status: STATUS_NOT_APPLICABLE, + annotation: `tidying modes provided without build mode`, + }); + } + } + + if (cliOptions['skip-curated-url-validation']) { + logWarn`Won't check if any URLs match the curated URL rules this run`; + logWarn `(--skip-curated-url-validation passed).`; + disableCuratedURLValidation(); + } + // Finish setting up defaults by combining information from all options. const _fallbackStep = (stepKey, { @@ -801,6 +956,16 @@ async function main() { }, }); + fallbackStep('reportOrphanedArtworks', { + default: 'perform', + cli: { + flag: 'skip-orphaned-artwork-validation', + negate: true, + warn: + `Skipping orphaned artwork validation. Hopefully you're debugging!`, + }, + }); + fallbackStep('filterReferenceErrors', { default: 'perform', cli: { @@ -889,6 +1054,10 @@ async function main() { break decideBuildSearchIndex; } + if (tidyingOnly) { + break decideBuildSearchIndex; + } + const indexFile = path.join(wikiCachePath, 'search', 'index.json') let stats; try { @@ -946,6 +1115,18 @@ async function main() { paragraph = false; } + fallbackStep('checkWikiDataSourceFileSorting', { + default: 'perform', + buildConfig: 'sort', + cli: { + flag: 'skip-sorting-validation', + negate: true, + warning: + `Skipping sorting validation. If any of this wiki's sorting rules are not\n` + + `satisfied, those errors will be silently passed along to the build.`, + }, + }); + fallbackStep('verifyImagePaths', { default: 'perform', buildConfig: 'mediaValidation', @@ -1082,6 +1263,20 @@ async function main() { return false; } + if (skipSelfDiagnosis) { + logWarn`${'Skipping code self-diagnosis.'} (--skip-self-diagnosis provided)`; + logWarn`This build should run substantially faster, but corner cases`; + logWarn`not previously predicted may fail strangely and silently.`; + + html.disableSlotValidation(); + } + + if (!showTraces) { + html.disableTagTracing(); + } + + await quickstat.track(mediaPath, {readdir: true, stat: true}); + Object.assign(stepStatusSummary.determineMediaCachePath, { status: STATUS_STARTED_NOT_DONE, timeStart: Date.now(), @@ -1260,7 +1455,7 @@ async function main() { const niceShowAggregate = (error, ...opts) => { showAggregate(error, { - showTraces: showAggregateTraces, + showTraces, pathToFileURL: (f) => path.relative(__dirname, fileURLToPath(f)), ...opts, }); @@ -1381,6 +1576,12 @@ async function main() { timeStart: Date.now(), }); + // This is about to get a whole lot more stuff put in it. + const wikiData = { + listingSpec, + listingTargetSpec, + }; + let yamlDataSteps; let yamlDocumentProcessingAggregate; @@ -1389,7 +1590,7 @@ async function main() { if (!paragraph) console.log(''); console.error(error); - niceShowAggregate(error); + niceShowAggregate(error, {showTraces: true}); logError`There was a JavaScript error ${stage}.`; fileIssue(); @@ -1406,7 +1607,8 @@ async function main() { let loadAggregate, loadResult; let processAggregate, processResult; - let saveAggregate, saveResult; + let connectAggregate; + let makeWikiDataResult; const dataSteps = getAllDataSteps(); @@ -1456,21 +1658,29 @@ async function main() { } try { - ({aggregate: saveAggregate, result: saveResult} = - saveThingsFromDataSteps( + ({aggregate: connectAggregate} = + connectThingsFromDataSteps( processResult, dataSteps)); - saveAggregate.close(); - saveAggregate = undefined; + connectAggregate.close(); } catch (error) { return whoops(error, `finalizing data files`); } + try { + makeWikiDataResult = + makeWikiDataFromDataSteps( + processResult, + dataSteps); + } catch (error) { + return whoops(error, 'preparing wikiData object'); + } + yamlDataSteps = dataSteps; yamlDocumentProcessingAggregate = processAggregate; - Object.assign(wikiData, saveResult); + Object.assign(wikiData, makeWikiDataResult); } { @@ -1480,6 +1690,10 @@ async function main() { ? prop : wikiData[prop]); + if (array && empty(array)) { + return; + } + logInfo` - ${array?.length ?? colors.red('(Missing!)')} ${colors.normal(colors.dim(label))}`; } @@ -1506,6 +1720,7 @@ async function main() { logThings('newsData', 'news entries'); } logThings('staticPageData', 'static pages'); + logThings('sortingRules', 'sorting rules'); if (wikiData.homepageLayout) { logInfo` - ${1} homepage layout (${ wikiData.homepageLayout.sections.length @@ -1654,7 +1869,7 @@ async function main() { } } catch (error) { if (!paragraph) console.log(''); - niceShowAggregate(error); + niceShowAggregate(error, {showTraces: true}); console.log(''); logError`There was an error precaching internal data objects.`; @@ -1677,8 +1892,8 @@ async function main() { }); } - // Filter out any things with duplicate directories throughout the data, - // warning about them too. + // Check for things with duplicate directories throughout the data, + // and halt if any are found. if (stepStatusSummary.reportDirectoryErrors.status === STATUS_NOT_STARTED) { Object.assign(stepStatusSummary.reportDirectoryErrors, { @@ -1700,10 +1915,17 @@ async function main() { if (!paragraph) console.log(''); niceShowAggregate(aggregate); - logWarn`The above duplicate directories were detected while reviewing data files.`; - logWarn`Since it's impossible to automatically determine which one's directory is`; - logWarn`correct, the build can't continue. Specify unique 'Directory' fields in`; - logWarn`some or all of these data entries to resolve the errors.`; + if (aggregate.errors?.find(err => err.message.toLowerCase().includes('duplicate'))) { + logWarn`The above duplicate directories were detected while reviewing data files.`; + logWarn`Since it's impossible to automatically determine which one's directory is`; + logWarn`correct, the build can't continue. Specify unique 'Directory' fields in`; + logWarn`some or all of these data entries to resolve the errors.`; + } else { + logWarn`The above directory errors were detected while reviewing data files.`; + logWarn`Since it's impossible to automatically fill in working directories,`; + logWarn`the build can't continue. Manually specify 'Directory' fields in`; + logWarn`some or all of these data entries to resolve the errors.`; + } console.log(''); paragraph = true; @@ -1719,8 +1941,42 @@ async function main() { } } - // Filter out any reference errors throughout the data, warning about them - // too. + // Check for artwork objects which have been orphaned from their things, + // and halt if any are found. + + if (stepStatusSummary.reportOrphanedArtworks.status === STATUS_NOT_STARTED) { + Object.assign(stepStatusSummary.reportOrphanedArtworks, { + status: STATUS_STARTED_NOT_DONE, + timeStart: Date.now(), + }); + + try { + reportOrphanedArtworks(wikiData, {getAllFindSpecs}); + + Object.assign(stepStatusSummary.reportOrphanedArtworks, { + status: STATUS_DONE_CLEAN, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + } catch (aggregate) { + if (!paragraph) console.log(''); + niceShowAggregate(aggregate, {showTraces: true}); + + logError`Failed to initialize artwork data connections properly.`; + fileIssue(); + + Object.assign(stepStatusSummary.reportOrphanedArtworks, { + status: STATUS_FATAL_ERROR, + annotation: `orphaned artworks found`, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + + return false; + } + } + + // Filter out any reference errors throughout the data, warning about these. if (stepStatusSummary.filterReferenceErrors.status === STATUS_NOT_STARTED) { Object.assign(stepStatusSummary.filterReferenceErrors, { @@ -1841,6 +2097,44 @@ async function main() { }); } + if (stepStatusSummary.checkWikiDataSourceFileSorting.status === STATUS_NOT_STARTED) { + Object.assign(stepStatusSummary.checkWikiDataSourceFileSorting, { + status: STATUS_STARTED_NOT_DONE, + timeStart: Date.now(), + }); + + const {SortingRule} = thingConstructors; + const results = + await Array.fromAsync(SortingRule.go({dataPath, wikiData, dry: true})); + + const needed = results.filter(result => result.changed); + + if (empty(needed)) { + logInfo`All sorting rules are satisfied. Nice!`; + paragraph = false; + + Object.assign(stepStatusSummary.checkWikiDataSourceFileSorting, { + status: STATUS_DONE_CLEAN, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + } else { + logWarn`Some of this wiki's sorting rules currently aren't satisfied:`; + for (const {rule} of needed) { + logWarn`- ${rule.message}`; + } + logWarn`Run ${'hsmusic --sort'} to automatically update data files.`; + paragraph = false; + + Object.assign(stepStatusSummary.checkWikiDataSourceFileSorting, { + status: STATUS_HAS_WARNINGS, + annotation: `not all rules satisfied`, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + } + } + if (stepStatusSummary.performBuild.status === STATUS_NOT_APPLICABLE) { displayCompositeCacheAnalysis(); @@ -1863,7 +2157,7 @@ async function main() { aggregate.close(); } catch (error) { - niceShowAggregate(error); + niceShowAggregate(error, {showTraces: true}); logError`Couldn't load internal default URL spec.`; logError`This is required to build the wiki, so stopping here.`; fileIssue(); @@ -1977,7 +2271,7 @@ async function main() { try { processURLSpecsAggregate.close(); } catch (error) { - niceShowAggregate(error); + niceShowAggregate(error, {showTraces: true}); logWarn`There were errors loading the optional URL specs you`; logWarn`selected using ${'--urls'}. Since they might misfunction,`; logWarn`debug the errors or remove the failing ones from ${'--urls'}.`; @@ -2235,7 +2529,7 @@ async function main() { }); internalDefaultLanguage = internalDefaultLanguageWatcher.language; - } catch (_error) { + } catch { // No need to display the error here - it's already printed by // watchLanguageFile. errorLoadingInternalDefaultLanguage = true; @@ -2246,7 +2540,7 @@ async function main() { try { internalDefaultLanguage = await processLanguageFile(internalDefaultStringsFile); } catch (error) { - niceShowAggregate(error); + niceShowAggregate(error, {showTraces: true}); errorLoadingInternalDefaultLanguage = true; } } @@ -2398,7 +2692,7 @@ async function main() { for (const {status, value: language, reason: error} of results) { if (status === 'rejected') { errorLoadingCustomLanguages = true; - niceShowAggregate(error); + niceShowAggregate(error, {showTraces: true}); } else { languages[language.code] = language; } @@ -2888,7 +3182,7 @@ async function main() { }); } catch (error) { if (!paragraph) console.log(''); - niceShowAggregate(error); + niceShowAggregate(error, {showTraces: true}); logError`There was an error preparing or writing search data.`; fileIssue(); @@ -2938,7 +3232,7 @@ async function main() { preparedWebRoutes = result; } catch (error) { if (!paragraph) console.log(''); - niceShowAggregate(error); + niceShowAggregate(error, {showTraces: true}); logError`There was an issue identifying web routes!`; fileIssue(); @@ -2972,10 +3266,81 @@ async function main() { .some(({to}) => to[0].startsWith('searchData')) : null); + quickstat.reset(); + + let restartBeforeBuild = false; + const updatedTidyModes = []; + + for (const [step, tidyMode] of [ + ['reformatCuratedURLs', 'format-urls'], + ['sortWikiDataSourceFiles', 'sort'], + ]) { + if (stepStatusSummary[step].status !== STATUS_NOT_STARTED) { + continue; + } + + Object.assign(stepStatusSummary[step], { + status: STATUS_STARTED_NOT_DONE, + timeStart: Date.now(), + }); + + const tidySignal = + await tidyModes[tidyMode].go({ + wikiData, + dataPath, + tidyingOnly, + }); + + switch (tidySignal) { + case 'clean': { + Object.assign(stepStatusSummary[step], { + status: STATUS_DONE_CLEAN, + annotation: `no changes needed`, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + + break; + } + + case 'updated': { + Object.assign(stepStatusSummary[step], { + status: STATUS_DONE_CLEAN, + annotation: `changes cueing restart`, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + + restartBeforeBuild = true; + updatedTidyModes.push(tidyMode); + + break; + } + + default: { + Object.assign(stepStatusSummary[step], { + status: STATUS_INVALID_SIGNAL, + annotation: `unknown: ${tidySignal}`, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + + logError`Invalid exit signal for ${'--' + tidyMode}: ${tidySignal}`; + fileIssue(); + + return false; + } + } + } + if (stepStatusSummary.performBuild.status === STATUS_NOT_APPLICABLE) { return true; } + if (restartBeforeBuild) { + return 'restart'; + } + const developersComment = `<!--\n` + [ wikiData.wikiInfo.canonicalBase @@ -3023,6 +3388,7 @@ async function main() { developersComment, languages, missingImagePaths, + niceShowAggregate, thumbsCache, urlSpec, urls, @@ -3085,145 +3451,71 @@ async function main() { } // TODO: isMain detection isn't consistent across platforms here -/* eslint-disable-next-line no-constant-condition */ +// eslint-disable-next-line no-constant-binary-expression if (true || isMain(import.meta.url) || path.basename(process.argv[1]) === 'hsmusic') { (async () => { let result; + let numRestarts = 0; const totalTimeStart = Date.now(); - try { - result = await main(); - } catch (error) { - if (error instanceof AggregateError) { - showAggregate(error); - } else if (error.cause) { - console.error(error); - showAggregate(error); - } else { - console.error(error); - } - } - - const totalTimeEnd = Date.now(); - - const formatDuration = timeDelta => { - const seconds = timeDelta / 1000; - - if (seconds > 90) { - const modSeconds = Math.floor(seconds % 60); - const minutes = Math.floor(seconds - seconds % 60) / 60; - return `${minutes}m${modSeconds}s`; - } - - if (seconds < 0.1) { - return 'instant'; + while (true) { + try { + result = await main(); + } catch (error) { + if (error instanceof AggregateError) { + showAggregate(error); + } else if (error.cause) { + console.error(error); + showAggregate(error); + } else { + console.error(error); + } } - const precision = (seconds > 1 ? 3 : 2); - return `${seconds.toPrecision(precision)}s`; - }; - - if (showStepStatusSummary) { - const totalDuration = formatDuration(totalTimeEnd - totalTimeStart); + if (result === 'restart') { + console.log(''); - console.error(colors.bright(`Step summary:`)); - - const longestNameLength = - Math.max(... - Object.values(stepStatusSummary) - .map(({name}) => name.length)); - - const stepsNotClean = - Object.values(stepStatusSummary) - .map(({status}) => - status === STATUS_HAS_WARNINGS || - status === STATUS_FATAL_ERROR || - status === STATUS_STARTED_NOT_DONE); - - const anyStepsNotClean = - stepsNotClean.includes(true); - - const stepDetails = Object.values(stepStatusSummary); - - const stepDurations = - stepDetails.map(({status, timeStart, timeEnd}) => { - if ( - status === STATUS_NOT_APPLICABLE || - status === STATUS_NOT_STARTED || - status === STATUS_STARTED_NOT_DONE - ) { - return '-'; + if (shouldShowStepStatusSummary) { + if (numRestarts >= 1) { + console.error(colors.bright(`Step summary since latest restart:`)); + } else { + console.error(colors.bright(`Step summary before restart:`)); } - if (typeof timeStart !== 'number' || typeof timeEnd !== 'number') { - return 'unknown'; - } - - return formatDuration(timeEnd - timeStart); - }); - - const longestDurationLength = - Math.max(...stepDurations.map(duration => duration.length)); - - const stepMemories = - stepDetails.map(({memory}) => - (memory - ? Math.round(memory["heapUsed"] / 1024 / 1024) + 'MB' - : '-')); - - const longestMemoryLength = - Math.max(...stepMemories.map(memory => memory.length)); - - for (let index = 0; index < stepDetails.length; index++) { - const {name, status, annotation} = stepDetails[index]; - const duration = stepDurations[index]; - const memory = stepMemories[index]; - - let message = - (stepsNotClean[index] - ? `!! ` - : ` - `); - - message += `(${duration} `.padStart(longestDurationLength + 2, ' '); - - if (showStepMemoryInSummary) { - message += ` ${memory})`.padStart(longestMemoryLength + 2, ' '); + showStepStatusSummary(); + console.log(''); } - message += ` `; - message += `${name}: `.padEnd(longestNameLength + 4, '.'); - message += ` `; - message += status; - - if (annotation) { - message += ` (${annotation})`; + if (numRestarts > 2) { + logError`A restart was cued, but we've restarted a bunch already.`; + logError`Exiting because this is probably a bug!`; + console.log(''); + break; + } else { + console.log(''); + console.log(`A restart was cued. This is probably normal, and required`); + console.log(`to load updated data files. Restarting automatically now!`); + console.log(''); + numRestarts++; } + } else { + break; + } + } - switch (status) { - case STATUS_DONE_CLEAN: - console.error(colors.green(message)); - break; - - case STATUS_NOT_STARTED: - case STATUS_NOT_APPLICABLE: - console.error(colors.dim(message)); - break; - - case STATUS_HAS_WARNINGS: - case STATUS_STARTED_NOT_DONE: - console.error(colors.yellow(message)); - break; + if (shouldShowStepStatusSummary) { + if (numRestarts >= 1) { + console.error(colors.bright(`Step summary after final restart:`)); + } else { + console.error(colors.bright(`Step summary:`)); + } - case STATUS_FATAL_ERROR: - console.error(colors.red(message)); - break; + const {anyStepsNotClean} = + showStepStatusSummary(); - default: - console.error(message); - break; - } - } + const totalTimeEnd = Date.now(); + const totalDuration = formatDuration(totalTimeEnd - totalTimeStart); console.error(colors.bright(`Done in ${totalDuration}.`)); @@ -3252,3 +3544,111 @@ if (true || isMain(import.meta.url) || path.basename(process.argv[1]) === 'hsmus process.exit(0); })(); } + +function showStepStatusSummary() { + const longestNameLength = + Math.max(... + Object.values(stepStatusSummary) + .map(({name}) => name.length)); + + const stepsNotClean = + Object.values(stepStatusSummary) + .map(({status}) => + status === STATUS_HAS_WARNINGS || + status === STATUS_FATAL_ERROR || + status === STATUS_INVALID_SIGNAL || + status === STATUS_NOT_STARTED || + status === STATUS_STARTED_NOT_DONE); + + const anyStepsNotClean = + stepsNotClean.includes(true); + + const stepDetails = Object.values(stepStatusSummary); + + const stepDurations = + stepDetails.map(({status, timeStart, timeEnd}) => { + if ( + status === STATUS_NOT_APPLICABLE || + status === STATUS_NOT_STARTED || + status === STATUS_STARTED_NOT_DONE + ) { + return '-'; + } + + if (typeof timeStart !== 'number' || typeof timeEnd !== 'number') { + return 'unknown'; + } + + return formatDuration(timeEnd - timeStart); + }); + + const longestDurationLength = + Math.max(...stepDurations.map(duration => duration.length)); + + const stepMemories = + stepDetails.map(({memory}) => + (memory + ? Math.round(memory["heapUsed"] / 1024 / 1024) + 'MB' + : '-')); + + const longestMemoryLength = + Math.max(...stepMemories.map(memory => memory.length)); + + for (let index = 0; index < stepDetails.length; index++) { + const {name, status, annotation} = stepDetails[index]; + const duration = stepDurations[index]; + const memory = stepMemories[index]; + + let message = + (stepsNotClean[index] + ? `!! ` + : ` - `); + + message += `(${duration} `.padStart(longestDurationLength + 2, ' '); + + if (shouldShowStepMemoryInSummary) { + message += ` ${memory})`.padStart(longestMemoryLength + 2, ' '); + } + + message += ` `; + message += `${name}: `.padEnd(longestNameLength + 4, '.'); + if (stepsNotClean[index]) { + message += `! `; + } else { + message += ` `; + } + + message += status; + + if (annotation) { + message += ` (${annotation})`; + } + + switch (status) { + case STATUS_DONE_CLEAN: + console.error(colors.green(message)); + break; + + case STATUS_NOT_APPLICABLE: + console.error(colors.dim(message)); + break; + + case STATUS_HAS_WARNINGS: + case STATUS_NOT_STARTED: + case STATUS_STARTED_NOT_DONE: + console.error(colors.yellow(message)); + break; + + case STATUS_FATAL_ERROR: + case STATUS_INVALID_SIGNAL: + console.error(colors.red(message)); + break; + + default: + console.error(message); + break; + } + } + + return {anyStepsNotClean}; +} |