diff options
Diffstat (limited to 'src/upd8.js')
| -rwxr-xr-x | src/upd8.js | 1021 |
1 files changed, 802 insertions, 219 deletions
diff --git a/src/upd8.js b/src/upd8.js index 6bd52da9..9e4ef4fb 100755 --- a/src/upd8.js +++ b/src/upd8.js @@ -31,38 +31,26 @@ // Oh yeah, like. Just run this through some relatively recent version of // node.js and you'll 8e fine. ...Within the project root. O8viously. +import '#import-heck'; + import {execSync} from 'node:child_process'; -import {readdir, readFile} from 'node:fs/promises'; +import {readdir, readFile, stat} from 'node:fs/promises'; import * as path from 'node:path'; import {fileURLToPath} from 'node:url'; import wrap from 'word-wrap'; -// Due to import time shenanigans, these imports have to come in the specified -// order. This obviously needs fixing up. - -/* precede #find */ -import { - filterReferenceErrors, - reportDuplicateDirectories, - reportContentTextErrors, -} from '#data-checks'; - -import {bindFind, getAllFindSpecs} from '#find'; - -// End of import time shenanigans (hopefully) - -import {showAggregate} from '#aggregate'; +import {mapAggregate, showAggregate} from '#aggregate'; import CacheableObject from '#cacheable-object'; import {displayCompositeCacheAnalysis} from '#composite'; +import {bindFind, getAllFindSpecs} from '#find'; import {processLanguageFile, watchLanguageFile, internalDefaultStringsFile} from '#language'; import {isMain, traverse} from '#node-utils'; +import {writeSearchData} from '#search'; import {sortByName} from '#sort'; -import {empty, withEntries} from '#sugar'; import {generateURLs, urlSpec} from '#urls'; -import {linkWikiDataArrays, loadAndProcessDataDocuments, sortWikiDataArrays} - from '#yaml'; +import {identifyAllWebRoutes} from '#web-routes'; import { colors, @@ -73,8 +61,22 @@ import { logError, parseOptions, progressCallAll, + showHelpForOptions as unboundShowHelpForOptions, } from '#cli'; +import { + filterReferenceErrors, + reportDirectoryErrors, + reportContentTextErrors, +} from '#data-checks'; + +import { + bindOpts, + empty, + indentWrap as unboundIndentWrap, + withEntries, +} from '#sugar'; + import genThumbs, { CACHE_FILE as thumbsCacheFile, defaultMagickThreads, @@ -84,14 +86,21 @@ import genThumbs, { verifyImagePaths, } from '#thumbs'; +import { + getAllDataSteps, + linkWikiDataArrays, + loadYAMLDocumentsFromDataSteps, + processThingsFromDataSteps, + saveThingsFromDataSteps, + sortWikiDataArrays, +} from '#yaml'; + import FileSizePreloader from './file-size-preloader.js'; import {listingSpec, listingTargetSpec} from './listing-spec.js'; import * as buildModes from './write/build-modes/index.js'; const __dirname = path.dirname(fileURLToPath(import.meta.url)); -const CACHEBUST = 23; - let COMMIT; try { COMMIT = execSync('git log --format="%h %B" -n 1 HEAD', {cwd: __dirname}).toString().trim(); @@ -118,66 +127,109 @@ let showStepStatusSummary = false; async function main() { Error.stackTraceLimit = Infinity; + let paragraph = true; + stepStatusSummary = { determineMediaCachePath: - {...defaultStepStatus, name: `determine media cache path`}, + {...defaultStepStatus, name: `determine media cache path`, + for: ['thumbs', 'build']}, migrateThumbnails: - {...defaultStepStatus, name: `migrate thumbnails`}, + {...defaultStepStatus, name: `migrate thumbnails`, + for: ['thumbs']}, loadThumbnailCache: - {...defaultStepStatus, name: `load thumbnail cache file`}, + {...defaultStepStatus, name: `load thumbnail cache file`, + for: ['thumbs', 'build']}, generateThumbnails: - {...defaultStepStatus, name: `generate thumbnails`}, + {...defaultStepStatus, name: `generate thumbnails`, + for: ['thumbs']}, loadDataFiles: - {...defaultStepStatus, name: `load and process data files`}, + {...defaultStepStatus, name: `load and process data files`, + for: ['build']}, linkWikiDataArrays: - {...defaultStepStatus, name: `link wiki data arrays`}, + {...defaultStepStatus, name: `link wiki data arrays`, + for: ['build']}, precacheCommonData: - {...defaultStepStatus, name: `precache common data`}, + {...defaultStepStatus, name: `precache common data`, + for: ['build']}, - reportDuplicateDirectories: - {...defaultStepStatus, name: `report duplicate directories`}, + reportDirectoryErrors: + {...defaultStepStatus, name: `report directory errors`, + for: ['verify']}, filterReferenceErrors: - {...defaultStepStatus, name: `filter reference errors`}, + {...defaultStepStatus, name: `filter reference errors`, + for: ['verify']}, reportContentTextErrors: - {...defaultStepStatus, name: `report content text errors`}, + {...defaultStepStatus, name: `report content text errors`, + for: ['verify']}, sortWikiDataArrays: - {...defaultStepStatus, name: `sort wiki data arrays`}, + {...defaultStepStatus, name: `sort wiki data arrays`, + for: ['build']}, precacheAllData: - {...defaultStepStatus, name: `precache nearly all data`}, + {...defaultStepStatus, name: `precache nearly all data`, + for: ['build']}, // TODO: This should be split into load/watch steps. loadInternalDefaultLanguage: - {...defaultStepStatus, name: `load internal default language`}, + {...defaultStepStatus, name: `load internal default language`, + for: ['build']}, loadLanguageFiles: - {...defaultStepStatus, name: `statically load custom language files`}, + {...defaultStepStatus, name: `statically load custom language files`, + for: ['build']}, watchLanguageFiles: - {...defaultStepStatus, name: `watch custom language files`}, + {...defaultStepStatus, name: `watch custom language files`, + for: ['build']}, initializeDefaultLanguage: - {...defaultStepStatus, name: `initialize default language`}, + {...defaultStepStatus, name: `initialize default language`, + for: ['build']}, verifyImagePaths: - {...defaultStepStatus, name: `verify missing/misplaced image paths`}, + {...defaultStepStatus, name: `verify missing/misplaced image paths`, + for: ['verify']}, preloadFileSizes: - {...defaultStepStatus, name: `preload file sizes`}, + {...defaultStepStatus, name: `preload file sizes`, + for: ['build']}, + + buildSearchIndex: + {...defaultStepStatus, name: `generate search index`, + for: ['build', 'search']}, + + identifyWebRoutes: + {...defaultStepStatus, name: `identify web routes`, + for: ['build']}, performBuild: - {...defaultStepStatus, name: `perform selected build mode`}, + {...defaultStepStatus, name: `perform selected build mode`, + for: ['build']}, }; + const stepsWhich = condition => + Object.entries(stepStatusSummary) + .filter(([_key, value]) => condition(value)) + .map(([key]) => key); + + /* eslint-disable-next-line no-unused-vars */ + const stepsFor = (...which) => + stepsWhich(step => + which.some(w => step.for?.includes(w))); + + const stepsNotFor = (...which) => + stepsWhich(step => + which.every(w => !step.for?.includes(w))); + const defaultQueueSize = 500; const buildModeFlagOptions = ( @@ -194,21 +246,24 @@ async function main() { })); let selectedBuildModeFlag; - let usingDefaultBuildMode; if (empty(selectedBuildModeFlags)) { - selectedBuildModeFlag = 'static-build'; - usingDefaultBuildMode = true; + // No build mode selected. This is not a valid state for building the wiki, + // but we want to let access to --help, so we'll show a message about what + // to do later. + selectedBuildModeFlag = null; } else if (selectedBuildModeFlags.length > 1) { logError`Building multiple modes (${selectedBuildModeFlags.join(', ')}) at once not supported.`; - logError`Please specify a maximum of one build mode.`; + logError`Please specify one build mode.`; return false; } else { selectedBuildModeFlag = selectedBuildModeFlags[0]; - usingDefaultBuildMode = false; } - const selectedBuildMode = buildModes[selectedBuildModeFlag]; + const selectedBuildMode = + (selectedBuildModeFlag + ? buildModes[selectedBuildModeFlag] + : null); // This is about to get a whole lot more stuff put in it. const wikiData = { @@ -216,7 +271,10 @@ async function main() { listingTargetSpec, }; - const buildOptions = selectedBuildMode.getCLIOptions(); + const buildOptions = + (selectedBuildMode + ? selectedBuildMode.getCLIOptions() + : {}); const commonOptions = { 'help': { @@ -228,7 +286,7 @@ async function main() { // and like a jillion other things too. Pretty much everything which // makes an individual wiki what it is goes here! 'data-path': { - help: `Specify path to data directory, including YAML files that cover all info about wiki content, layout, and structure\n\nAlways required for wiki building, but may be provided via the HSMUSIC_DATA environment variable instead`, + help: `Specify path to data directory, including YAML files that cover all info about wiki content, layout, and structure\n\nAlways required for wiki building; may be provided via the HSMUSIC_DATA environment variable`, type: 'value', }, @@ -236,12 +294,17 @@ async function main() { // categorized; check out MEDIA_ALBUM_ART_DIRECTORY and other constants // near the top of this file (upd8.js). 'media-path': { - help: `Specify path to media directory, including album artwork and additional files, as well as custom site layout media and other media files for reference or linking in wiki content\n\nAlways required for wiki building, but may be provided via the HSMUSIC_MEDIA environment variable instead`, + help: `Specify path to media directory, including album artwork and additional files, as well as custom site layout media and other media files for reference or linking in wiki content\n\nAlways required for wiki building; may be provided via the HSMUSIC_MEDIA environment variable`, type: 'value', }, 'media-cache-path': { - help: `Specify path to media cache directory, including automatically generated thumbnails\n\nThis usually doesn't need to be provided, and will be inferred by adding "-cache" to the end of the media directory`, + help: `Specify path to media cache directory, including automatically generated thumbnails\n\nThis usually doesn't need to be provided, and will be inferred either by loading "media-cache" from --cache-path, or by adding "-cache" to the end of the media directory\n\nMay be provided via the HSMUSIC_MEDIA_CACHE environment variable`, + type: 'value', + }, + + 'cache-path': { + help: `Specify path to general cache directory, usually containing generated thumbnails and assorted files reused between builds\n\nAlways required for wiki building; may be provided via the HSMUSIC_CACHE environment varaible`, type: 'value', }, @@ -285,6 +348,11 @@ async function main() { type: 'flag', }, + 'new-thumbs': { + help: `Repair a media cache that's completely missing its index file by starting clean and not reusing any existing thumbnails`, + type: 'flag', + }, + 'skip-file-sizes': { help: `Skips preloading file sizes for images and additional files, which will be left blank in the build`, type: 'flag', @@ -295,6 +363,16 @@ async function main() { type: 'flag', }, + 'refresh-search': { + help: `Generate the text search index this build, instead of waiting for the automatic delay`, + type: 'flag', + }, + + 'skip-search': { + help: `Skip creation of the text search index no matter what, even if it'd normally be scheduled for now`, + type: 'flag', + }, + // Just working on data entries and not interested in actually // generating site HTML yet? This flag will cut execution off right // 8efore any site 8uilding actually happens. @@ -374,6 +452,18 @@ async function main() { }, }; + const indentWrap = + bindOpts(unboundIndentWrap, { + wrap, + }); + + const showHelpForOptions = + bindOpts(unboundShowHelpForOptions, { + [bindOpts.bindIndex]: 0, + indentWrap, + sort: sortByName, + }); + const cliOptions = await parseOptions(process.argv.slice(2), { // We don't want to error when we receive these options, so specify them // here, even though we won't be doing anything with them later. @@ -384,89 +474,71 @@ async function main() { ...buildOptions, }); - if (cliOptions['help']) { - const indentWrap = (spaces, str) => wrap(str, {width: 60 - spaces, indent: ' '.repeat(spaces)}); - - const showOptions = (msg, options) => { - console.log(colors.bright(msg)); - - const entries = Object.entries(options); - const sortedOptions = sortByName(entries - .map(([name, descriptor]) => ({name, descriptor}))); - - if (!sortedOptions.length) { - console.log(`(No options available)`) - } - - let justInsertedPaddingLine = false; - - for (const {name, descriptor} of sortedOptions) { - if (descriptor.alias) { - continue; - } - - const aliases = entries - .filter(([_name, {alias}]) => alias === name) - .map(([name]) => name); - - let wrappedHelp, wrappedHelpLines = 0; - if (descriptor.help) { - wrappedHelp = indentWrap(4, descriptor.help); - wrappedHelpLines = wrappedHelp.split('\n').length; - } - - if (wrappedHelpLines > 0 && !justInsertedPaddingLine) { - console.log(''); - } - - console.log(colors.bright(` --` + name) + - (aliases.length - ? ` (or: ${aliases.map(alias => colors.bright(`--` + alias)).join(', ')})` - : '') + - (descriptor.help - ? '' - : colors.dim(' (no help provided)'))); - - if (wrappedHelp) { - console.log(wrappedHelp); - } - - if (wrappedHelpLines > 1) { - console.log(''); - justInsertedPaddingLine = true; - } else { - justInsertedPaddingLine = false; - } - } - - if (!justInsertedPaddingLine) { - console.log(``); - } - }; + showStepStatusSummary = cliOptions['show-step-summary'] ?? false; + if (cliOptions['help']) { console.log( - colors.bright(`hsmusic (aka. Homestuck Music Wiki)\n`) + + colors.bright(`hsmusic (aka. Homestuck Music Wiki, HSMusic Wiki)\n`) + `static wiki software cataloguing collaborative creation\n`); - console.log(indentWrap(0, - `The \`hsmusic\` command provides basic control over all parts of generating user-visible HTML pages and website content/structure from provided data, media, and language directories.\n` + + console.log(indentWrap( + `The \`hsmusic\` command provides basic control over ` + + `all parts of generating user-visible HTML pages ` + + `and website content/structure ` + + `from provided data, media, and language directories.\n` + `\n` + `CLI options are divided into three groups:\n`)); - console.log(` 1) ` + indentWrap(4, - `Common options: These are shared by all build modes and always have the same essential behavior`).trim()); - console.log(` 2) ` + indentWrap(4, - `Build mode selection: One build mode may be selected (or else the default, --static-build, is used), and it decides which entire set of behavior to use for providing site content to the user`).trim()); - console.log(` 3) ` + indentWrap(4, - `Build options: Each build mode has a set of unique options which customize behavior for that build mode`).trim()); + + console.log(` 1) ` + indentWrap( + `Common options: ` + + `These are shared by all build modes ` + + `and always have the same essential behavior`, + {spaces: 4, bullet: true})); + + console.log(` 2) ` + indentWrap( + `Build mode selection: ` + + `One build mode should be selected, ` + + `and it decides the main set of behavior to use ` + + `for presenting or interacting with site content`, + {spaces: 4, bullet: true})); + + console.log(` 3) ` + indentWrap( + `Build options: ` + + `Each build mode has a set of unique options ` + + `which customize behavior for that build mode`, + {spaces: 4, bullet: true})); + console.log(``); - showOptions(`Common options`, commonOptions); - showOptions(`Build mode selection`, buildModeFlagOptions); + showHelpForOptions({ + heading: `Common options`, + options: commonOptions, + wrap, + }); + + showHelpForOptions({ + heading: `Build mode selection`, + options: buildModeFlagOptions, + wrap, + }); + + if (selectedBuildMode) { + showHelpForOptions({ + heading: `Build options for --${selectedBuildModeFlag}`, + options: buildOptions, + wrap, + }); + } else { + console.log( + `Specify a build mode and run with ${colors.bright('--help')} again for info\n` + + `about the options for that build mode.`); + } - if (buildOptions) { - showOptions(`Build options for --${selectedBuildModeFlag} (${ - usingDefaultBuildMode ? 'default' : 'selected' - })`, buildOptions); + for (const step of Object.values(stepStatusSummary)) { + Object.assign(step, { + status: STATUS_NOT_APPLICABLE, + annotation: `--help provided`, + }); } return true; @@ -474,13 +546,12 @@ async function main() { const dataPath = cliOptions['data-path'] || process.env.HSMUSIC_DATA; const mediaPath = cliOptions['media-path'] || process.env.HSMUSIC_MEDIA; + const wikiCachePath = cliOptions['cache-path'] || process.env.HSMUSIC_CACHE; const langPath = cliOptions['lang-path'] || process.env.HSMUSIC_LANG; // Can 8e left unset! const thumbsOnly = cliOptions['thumbs-only'] ?? false; const noInput = cliOptions['no-input'] ?? false; - showStepStatusSummary = cliOptions['show-step-summary'] ?? false; - const showAggregateTraces = cliOptions['show-traces'] ?? false; const precacheMode = cliOptions['precache-mode'] ?? 'common'; @@ -501,7 +572,11 @@ async function main() { logError`${`Expected --media-path option or HSMUSIC_MEDIA to be set`}`; } - if (!dataPath || !mediaPath) { + if (!wikiCachePath) { + logError`${`Expected --cache-path option or HSMUSIC_CACHE to be set`}`; + } + + if (!dataPath || !mediaPath || !wikiCachePath) { return false; } @@ -512,60 +587,92 @@ async function main() { status: STATUS_NOT_APPLICABLE, annotation: `--no-build provided`, }); - } else { - if (usingDefaultBuildMode) { - logInfo`No build mode specified, will use default: ${selectedBuildModeFlag}`; - } else { - logInfo`Will use specified build mode: ${selectedBuildModeFlag}`; - } } // Finish setting up defaults by combining information from all options. const _fallbackStep = (stepKey, { default: defaultValue, + cli: cliArg, + buildConfig: buildConfigKey = null, + }) => { + const buildConfig = selectedBuildMode?.config?.[buildConfigKey]; + const {[stepKey]: step} = stepStatusSummary; + + const cliEntries = + (cliArg === null || cliArg === undefined + ? [] + : Array.isArray(cliArg) + ? cliArg + : [cliArg]); - cli: { + for (const { flag: cliFlag = null, negate: cliFlagNegates = false, warn: cliFlagWarning = null, - } = {}, - - buildConfig: buildConfigKey, - }) => { - const {[buildConfigKey]: buildConfig} = selectedBuildMode.config; - const {[stepKey]: step} = stepStatusSummary; + disable: cliFlagDisablesSteps = [], + } of cliEntries) { + if (!cliOptions[cliFlag]) { + continue; + } - if (cliFlag && cliOptions[cliFlag]) { const cliPart = `--` + cliFlag; const modePart = `--` + selectedBuildModeFlag; + if (buildConfig?.applicable === false) { if (cliFlagNegates) { logWarn`${cliPart} provided, but ${modePart} already skips this step`; logWarn`Redundant option ${cliPart}`; + continue; } else { logWarn`${cliPart} provided, but this step isn't applicable for ${modePart}`; logWarn`Ignoring option ${cliPart}`; + continue; } - } else if (buildConfig?.required === true) { + } + + if (buildConfig?.required === true) { if (cliFlagNegates) { logWarn`${cliPart} provided, but ${modePart} requires this step`; logWarn`Ignoring option ${cliPart}`; + continue; } else { logWarn`${cliPart} provided, but ${modePart} already requires this step`; logWarn`Redundant option ${cliPart}`; + continue; } - } else { - if (cliFlagNegates) { - step.status = STATUS_NOT_APPLICABLE; - step.annotation = `--${cliFlag} provided`; + } + + step.status = + (cliFlagNegates + ? STATUS_NOT_APPLICABLE + : STATUS_NOT_STARTED); + + step.annotation = `--${cliFlag} provided`; + + if (cliFlagWarning) { + for (const line of cliFlagWarning.split('\n')) { + logWarn(line); } - if (cliFlagWarning) { - for (const line of cliFlagWarning.split('\n')) { - logWarn(line); - } + } + + for (const step of cliFlagDisablesSteps) { + const summary = stepStatusSummary[step]; + if (summary.status === STATUS_NOT_APPLICABLE && summary.annotation) { + stepStatusSummary.performBuild.annotation += `; --${cliFlag} provided`; + } else { + summary.status = STATUS_NOT_APPLICABLE; + summary.annotation = `--${cliFlag} provided`; } } + + return; + } + + if (buildConfig?.required === true) { + step.status = STATUS_NOT_STARTED; + step.annotation = `required for --${selectedBuildModeFlag}`; + return; } if (buildConfig?.applicable === false) { @@ -580,13 +687,29 @@ async function main() { return; } + if (buildConfig?.default === 'perform') { + step.status = STATUS_NOT_STARTED; + step.annotation = `default for --${selectedBuildModeFlag}`; + return; + } + switch (defaultValue) { - case 'skip': + case 'skip': { step.status = STATUS_NOT_APPLICABLE; - if (cliFlag && !cliFlagNegates) { - step.annotation = `--${cliFlag} not provided`; + + const enablingFlags = + cliEntries + .filter(({negate}) => !negate) + .map(({flag}) => flag); + + if (!empty(enablingFlags)) { + step.annotation = + enablingFlags.map(flag => `--${flag}`).join(', ') + + ` not provided`; } + break; + } case 'perform': break; @@ -611,7 +734,6 @@ async function main() { fallbackStep('filterReferenceErrors', { default: 'perform', - buildConfig: null, cli: { flag: 'skip-reference-validation', negate: true, @@ -624,17 +746,20 @@ async function main() { fallbackStep('generateThumbnails', { default: 'perform', buildConfig: 'thumbs', - cli: { - flag: 'skip-thumbs', - negate: true, - }, + cli: [ + {flag: 'thumbs-only', disable: stepsNotFor('thumbs')}, + {flag: 'skip-thumbs', negate: true}, + ], }); fallbackStep('migrateThumbnails', { default: 'skip', - buildConfig: null, cli: { flag: 'migrate-thumbs', + disable: [ + ...stepsNotFor('thumbs'), + 'generateThumbnails', + ], }, }); @@ -647,6 +772,99 @@ async function main() { }, }); + fallbackStep('identifyWebRoutes', { + default: 'perform', + buildConfig: 'webRoutes', + }); + + decideBuildSearchIndex: { + fallbackStep('buildSearchIndex', { + default: 'skip', + buildConfig: 'search', + cli: [ + {flag: 'refresh-search'}, + {flag: 'skip-search', negate: true}, + ], + }); + + if (cliOptions['refresh-search'] || cliOptions['skip-search']) { + if (cliOptions['refresh-search']) { + logInfo`${'--refresh-search'} provided, will generate search fresh this build.`; + } + + break decideBuildSearchIndex; + } + + if (stepStatusSummary.buildSearchIndex.status !== STATUS_NOT_APPLICABLE) { + break decideBuildSearchIndex; + } + + if (selectedBuildMode?.config?.search?.default === 'skip') { + break decideBuildSearchIndex; + } + + // TODO: OK this is a little silly. + if (stepStatusSummary.buildSearchIndex.annotation?.startsWith('N/A')) { + break decideBuildSearchIndex; + } + + const indexFile = path.join(wikiCachePath, 'search', 'index.json') + let stats; + try { + stats = await stat(indexFile); + } catch (error) { + if (error.code === 'ENOENT') { + Object.assign(stepStatusSummary.buildSearchIndex, { + status: STATUS_NOT_STARTED, + annotation: `search/index.json not present, will create`, + }); + + logInfo`Looks like the search cache doesn't exist.`; + logInfo`It'll be generated fresh, this build!`; + } else { + Object.assign(stepStatusSummary.buildSearchIndex, { + status: STATUS_NOT_APPLICABLE, + annotation: `error getting search index stats`, + }); + + if (!paragraph) console.log(''); + console.error(error); + + logWarn`There was an error checking the search index file, located at:`; + logWarn`${indexFile}`; + logWarn`You may want to toss out the "search" folder; it'll be generated`; + logWarn`anew, if you do, and may fix this error.`; + } + + paragraph = false; + break decideBuildSearchIndex; + } + + const delta = Date.now() - stats.mtimeMs; + const minute = 60 * 1000; + const delay = 45 * minute; + + const whenst = duration => `~${Math.ceil(duration / minute)} min`; + + if (delta < delay) { + logInfo`Search index was generated recently, skipping for this build.`; + logInfo`Next scheduled is in ${whenst(delay - delta)}, or by using ${'--refresh-search'}.`; + Object.assign(stepStatusSummary.buildSearchIndex, { + status: STATUS_NOT_APPLICABLE, + annotation: `earlier than scheduled based on file mtime`, + }); + } else { + logInfo`Search index hasn't been generated for a little while.`; + logInfo`It'll be generated this build, then again in ${whenst(delay)}.`; + Object.assign(stepStatusSummary.buildSearchIndex, { + status: STATUS_NOT_STARTED, + annotation: `past when shceduled based on file mtime`, + }); + } + + paragraph = false; + } + fallbackStep('verifyImagePaths', { default: 'perform', buildConfig: 'mediaValidation', @@ -687,33 +905,43 @@ async function main() { }); } + // TODO: These should error if the option was actually provided but + // the relevant steps were already disabled for some other reason. switch (precacheMode) { case 'common': - Object.assign(stepStatusSummary.precacheAllData, { - status: STATUS_NOT_APPLICABLE, - annotation: `--precache-mode is common, not all`, - }); + if (stepStatusSummary.precacheAllData.status === STATUS_NOT_STARTED) { + Object.assign(stepStatusSummary.precacheAllData, { + status: STATUS_NOT_APPLICABLE, + annotation: `--precache-mode is common, not all`, + }); + } break; case 'all': - Object.assign(stepStatusSummary.precacheCommonData, { - status: STATUS_NOT_APPLICABLE, - annotation: `--precache-mode is all, not common`, - }); + if (stepStatusSummary.precacheCommonData.status === STATUS_NOT_STARTED) { + Object.assign(stepStatusSummary.precacheCommonData, { + status: STATUS_NOT_APPLICABLE, + annotation: `--precache-mode is all, not common`, + }); + } break; case 'none': - Object.assign(stepStatusSummary.precacheCommonData, { - status: STATUS_NOT_APPLICABLE, - annotation: `--precache-mode is none`, - }); + if (stepStatusSummary.precacheCommonData.status === STATUS_NOT_STARTED) { + Object.assign(stepStatusSummary.precacheCommonData, { + status: STATUS_NOT_APPLICABLE, + annotation: `--precache-mode is none`, + }); + } - Object.assign(stepStatusSummary.precacheAllData, { - status: STATUS_NOT_APPLICABLE, - annotation: `--precache-mode is none`, - }); + if (stepStatusSummary.precacheAllData.status === STATUS_NOT_STARTED) { + Object.assign(stepStatusSummary.precacheAllData, { + status: STATUS_NOT_APPLICABLE, + annotation: `--precache-mode is none`, + }); + } break; } @@ -735,39 +963,162 @@ async function main() { return false; } + // If we're going to require a build mode and none is specified, + // exit and show what to do. This must not precede anything that might + // disable the build (e.g. changing its status to STATUS_NOT_APPLICABLE). + + if (stepStatusSummary.performBuild.status === STATUS_NOT_STARTED) { + if (selectedBuildMode) { + logInfo`Will use specified build mode: ${selectedBuildModeFlag}`; + } else { + showHelpForOptions({ + heading: `Please specify a build mode:`, + options: buildModeFlagOptions, + }); + + console.log( + `(Use ${colors.bright('--help')} for general info and all options, or specify\n` + + ` a build mode alongside ${colors.bright('--help')} for that mode's options!`); + + for (const step of Object.values(stepStatusSummary)) { + Object.assign(step, { + status: STATUS_NOT_APPLICABLE, + annotation: `no build mode provided`, + }); + } + + return false; + } + } else if (selectedBuildMode) { + if (stepStatusSummary.performBuild.annotation) { + logError`You've specified a build mode, ${selectedBuildModeFlag}, but it won't be used,`; + logError`according to the message: ${`"${stepStatusSummary.performBuild.annotation}"`}`; + } else { + logError`You've specified a build mode, ${selectedBuildModeFlag}, but it won't be used,`; + logError`probably because of another option you've provided.`; + } + logError`Please remove ${'--' + selectedBuildModeFlag} or the conflicting option.`; + return false; + } + Object.assign(stepStatusSummary.determineMediaCachePath, { status: STATUS_STARTED_NOT_DONE, timeStart: Date.now(), }); + const regenerateMissingThumbnailCache = + cliOptions['new-thumbs'] ?? false; + const {mediaCachePath, annotation: mediaCachePathAnnotation} = await determineMediaCachePath({ mediaPath, + wikiCachePath, + providedMediaCachePath: cliOptions['media-cache-path'] || process.env.HSMUSIC_MEDIA_CACHE, + + regenerateMissingThumbnailCache, + disallowDoubling: stepStatusSummary.migrateThumbnails.status === STATUS_NOT_STARTED, }); + if (regenerateMissingThumbnailCache) { + if ( + mediaCachePathAnnotation !== `contained path will regenerate missing cache` && + mediaCachePathAnnotation !== `adjacent path will regenerate missing cache` + ) { + if (mediaCachePath) { + logError`Determined a media cache path. (${mediaCachePathAnnotation})`; + console.error(''); + logWarn`By using ${'--new-thumbs'}, you requested to generate completely`; + logWarn`new thumbnails, but there's already a ${'thumbnail-cache.json'}`; + logWarn`file where it's expected, within this media cache:`; + logWarn`${path.resolve(mediaCachePath)}`; + console.error(''); + logWarn`If you really do want to completely regenerate all thumbnails`; + logWarn`and not reuse any existing ones, move aside ${'thumbnail-cache.json'}`; + logWarn`and run with ${'--new-thumbs'} again.`; + + Object.assign(stepStatusSummary.determineMediaCachePath, { + status: STATUS_FATAL_ERROR, + annotation: `--new-thumbs provided but regeneration not needed`, + timeEnd: Date.now(), + }); + + return false; + } else { + logError`Couldn't determine a media cache path. (${mediaCachePathAnnotation})`; + console.error(''); + logWarn`You requested to generate completely new thumbnails, but`; + logWarn`the media cache wasn't readable or just couldn't be found.`; + logWarn`Run again without ${'--new-thumbs'} - you should investigate`; + logWarn`what's going on before continuing.`; + + Object.assign(stepStatusSummary.determineMediaCachePath, { + status: STATUS_FATAL_ERROR, + annotation: mediaCachePathAnnotation, + timeEnd: Date.now(), + }); + + return false; + } + } + } + if (!mediaCachePath) { logError`Couldn't determine a media cache path. (${mediaCachePathAnnotation})`; switch (mediaCachePathAnnotation) { - case 'inferred path does not have cache': - logError`If you're certain this is the right path, you can provide it via`; - logError`${'--media-cache-path'} or ${'HSMUSIC_MEDIA_CACHE'}, and it should work.`; + case `contained path does not have cache`: + console.error(''); + logError`You've provided a ${'--cache-path'} or ${'HSMUSIC_CACHE_PATH'},`; + logError`${path.resolve(wikiCachePath)}`; + console.error(''); + logError`It contains a ${'media-cache'} folder, but this folder is`; + logError`missing its ${'thumbnail-cache.json'} file. This means there's`; + logError`no information available to reuse. If you use this cache,`; + logError`hsmusic will generate any existing thumbnails over again.`; + console.error(''); + logError`* Try to see if you can recover or locate a copy of your`; + logError` ${'thumbnail-cache.json'} file and put it back in place;`; + logError`* Or, generate all-new thumbnails with ${'--new-thumbs'}.`; + break; + + case 'adjacent path does not have cache': + console.error(''); + logError`You have an existing ${'media-cache'} folder next to your media path,`; + logError`${path.resolve(mediaPath)}`; + console.error(''); + logError`The ${'media-cache'} folder is missing its ${'thumbnail-cache.json'}`; + logError`file. This means there's no information available to reuse,`; + logError`and if you use this cache, hsmusic will generate any existing`; + logError`thumbnails over again.`; + console.error(''); + logError`* Try to see if you can recover or locate a copy of your`; + logError` ${'thumbnail-cache.json'} file and put it back in place;`; + logError`* Or, generate all-new thumbnails with ${'--new-thumbs'}.`; break; - case 'inferred path not readable': + case `contained path not readable`: + case `adjacent path not readable`: + console.error(''); logError`The folder couldn't be read, which usually indicates`; logError`a permissions error. Try to resolve this, or provide`; logError`a new path with ${'--media-cache-path'} or ${'HSMUSIC_MEDIA_CACHE'}.`; break; - case 'media path not provided': /* unreachable */ + case `media path not provided`: /* unreachable */ + console.error(''); logError`It seems a ${'--media-path'} (or ${'HSMUSIC_MEDIA'}) wasn't provided.`; logError`Make sure one of these is actually pointing to a path that exists.`; break; + + case `cache path not provided`: /* unreachable */ + console.error(''); + logError`It seems a ${'--cache-path'} (or ${'HSMUSIC_CACHE'}) wasn't provided.`; + logError`Make sure one of these is actually pointing to a path that exists.`; + break; } Object.assign(stepStatusSummary.determineMediaCachePath, { @@ -941,27 +1292,95 @@ async function main() { timeStart: Date.now(), }); - let processDataAggregate, wikiDataResult; + let yamlDataSteps; + let yamlDocumentProcessingAggregate; - try { - ({aggregate: processDataAggregate, result: wikiDataResult} = - await loadAndProcessDataDocuments({dataPath})); - } catch (error) { - console.error(error); + { + const whoops = (error, stage) => { + if (!paragraph) console.log(''); - logError`There was a JavaScript error loading data files.`; - fileIssue(); + console.error(error); + niceShowAggregate(error); - Object.assign(stepStatusSummary.loadDataFiles, { - status: STATUS_FATAL_ERROR, - annotation: `javascript error - view log for details`, - timeEnd: Date.now(), - }); + logError`There was a JavaScript error ${stage}.`; + fileIssue(); - return false; - } + Object.assign(stepStatusSummary.loadDataFiles, { + status: STATUS_FATAL_ERROR, + annotation: `javascript error - view log for details`, + timeEnd: Date.now(), + }); + + return false; + }; - Object.assign(wikiData, wikiDataResult); + let loadAggregate, loadResult; + let processAggregate, processResult; + let saveAggregate, saveResult; + + const dataSteps = getAllDataSteps(); + + try { + ({aggregate: loadAggregate, result: loadResult} = + await loadYAMLDocumentsFromDataSteps( + dataSteps, + {dataPath})); + } catch (error) { + return whoops(error, `loading data files`); + } + + try { + loadAggregate.close(); + } catch (error) { + if (!paragraph) console.log(''); + niceShowAggregate(error); + + logError`The above errors were detected while loading data files.`; + logError`Since this indicates some files weren't able to load at all,`; + logError`there would probably be pretty bad reference errors if the`; + logError`build were to continue. Please resolve these errors and`; + logError`then give it another go.`; + + paragraph = true; + console.log(''); + + Object.assign(stepStatusSummary.loadDataFiles, { + status: STATUS_FATAL_ERROR, + annotation: `error loading data files`, + timeEnd: Date.now(), + }); + + return false; + } + + try { + ({aggregate: processAggregate, result: processResult} = + await processThingsFromDataSteps( + loadResult.documentLists, + loadResult.fileLists, + dataSteps, + {dataPath})); + } catch (error) { + return whoops(error, `processing data files`); + } + + try { + ({aggregate: saveAggregate, result: saveResult} = + saveThingsFromDataSteps( + processResult, + dataSteps)); + + saveAggregate.close(); + saveAggregate = undefined; + } catch (error) { + return whoops(error, `finalizing data files`); + } + + yamlDataSteps = dataSteps; + yamlDocumentProcessingAggregate = processAggregate; + + Object.assign(wikiData, saveResult); + } { const logThings = (prop, label) => { @@ -974,10 +1393,16 @@ async function main() { } try { + if (!paragraph) console.log(''); + logInfo`Loaded data and processed objects:`; logThings('albumData', 'albums'); logThings('trackData', 'tracks'); - logThings(wikiData.artistData.filter(artist => !artist.isAlias), 'artists'); + logThings( + (wikiData.artistData + ? wikiData.artistData.filter(artist => !artist.isAlias) + : null), + 'artists'); if (wikiData.flashData) { logThings('flashData', 'flashes'); logThings('flashActData', 'flash acts'); @@ -998,21 +1423,28 @@ async function main() { if (wikiData.wikiInfo) { logInfo` - ${1} wiki config file`; } + + console.log(''); + paragraph = true; } catch (error) { console.error(`Error showing data summary:`, error); + paragraph = false; } let errorless = true; try { - processDataAggregate.close(); + yamlDocumentProcessingAggregate.close(); } catch (error) { + if (!paragraph) console.log(''); niceShowAggregate(error); + logWarn`The above errors were detected while processing data files.`; + errorless = false; } if (!wikiData.wikiInfo) { - logError`Can't proceed without wiki info file successfully loading`; + logError`Can't proceed without wiki info file successfully loading.`; Object.assign(stepStatusSummary.loadDataFiles, { status: STATUS_FATAL_ERROR, @@ -1025,15 +1457,20 @@ async function main() { if (errorless) { logInfo`All data files processed without any errors - nice!`; + paragraph = false; Object.assign(stepStatusSummary.loadDataFiles, { status: STATUS_DONE_CLEAN, timeEnd: Date.now(), }); } else { - logWarn`If the remaining valid data is complete enough, the wiki will`; - logWarn`still build - but all errored data will be skipped.`; - logWarn`(Resolve errors for more complete output!)`; + logWarn`This might indicate some fields in the YAML data weren't formatted`; + logWarn`correctly, for example. The build should still work, but invalid`; + logWarn`fields will be skipped. Take a look at the report above to see`; + logWarn`what needs fixing up, for a more complete build!`; + + console.log(''); + paragraph = true; Object.assign(stepStatusSummary.loadDataFiles, { status: STATUS_HAS_WARNINGS, @@ -1106,14 +1543,31 @@ async function main() { ]), }; - for (const [wikiDataKey, properties] of Object.entries(commonDataMap)) { - const thingData = wikiData[wikiDataKey]; - const allProperties = new Set(['name', 'directory', ...properties]); - for (const thing of thingData) { - for (const property of allProperties) { - void thing[property]; + try { + for (const [wikiDataKey, properties] of Object.entries(commonDataMap)) { + const thingData = wikiData[wikiDataKey]; + const allProperties = new Set(['name', 'directory', ...properties]); + for (const thing of thingData) { + for (const property of allProperties) { + void thing[property]; + } } } + } catch (error) { + if (!paragraph) console.log(''); + niceShowAggregate(error); + console.log(''); + + logError`There was an error precaching internal data objects.`; + fileIssue(); + + Object.assign(stepStatusSummary.precacheCommonData, { + status: STATUS_FATAL_ERROR, + annotation: `see log for details`, + timeEnd: Date.now(), + }); + + return false; } Object.assign(stepStatusSummary.precacheCommonData, { @@ -1122,23 +1576,27 @@ async function main() { }); } + const urls = generateURLs(urlSpec); + // Filter out any things with duplicate directories throughout the data, // warning about them too. - Object.assign(stepStatusSummary.reportDuplicateDirectories, { + Object.assign(stepStatusSummary.reportDirectoryErrors, { status: STATUS_STARTED_NOT_DONE, timeStart: Date.now(), }); try { - reportDuplicateDirectories(wikiData, {getAllFindSpecs}); + reportDirectoryErrors(wikiData, {getAllFindSpecs}); logInfo`No duplicate directories found - nice!`; + paragraph = false; - Object.assign(stepStatusSummary.reportDuplicateDirectories, { + Object.assign(stepStatusSummary.reportDirectoryErrors, { status: STATUS_DONE_CLEAN, timeEnd: Date.now(), }); } catch (aggregate) { + if (!paragraph) console.log(''); niceShowAggregate(aggregate); logWarn`The above duplicate directories were detected while reviewing data files.`; @@ -1146,7 +1604,10 @@ async function main() { logWarn`correct, the build can't continue. Specify unique 'Directory' fields in`; logWarn`some or all of these data entries to resolve the errors.`; - Object.assign(stepStatusSummary.reportDuplicateDirectories, { + console.log(''); + paragraph = true; + + Object.assign(stepStatusSummary.reportDirectoryErrors, { status: STATUS_FATAL_ERROR, annotation: `duplicate directories found`, timeEnd: Date.now(), @@ -1171,17 +1632,23 @@ async function main() { filterReferenceErrorsAggregate.close(); logInfo`All references validated without any errors - nice!`; + paragraph = false; Object.assign(stepStatusSummary.filterReferenceErrors, { status: STATUS_DONE_CLEAN, timeEnd: Date.now(), }); } catch (error) { + if (!paragraph) console.log(''); niceShowAggregate(error); logWarn`The above errors were detected while validating references in data files.`; - logWarn`The wiki will still build, but these connections between data objects`; - logWarn`will be completely skipped. Resolve the errors for more complete output.`; + logWarn`The wiki should still build, but these connections between data objects`; + logWarn`will be skipped, which might have unexpected consequences. Take a look at`; + logWarn`the report above to see what needs fixing up, for a more complete build!`; + + console.log(''); + paragraph = true; Object.assign(stepStatusSummary.filterReferenceErrors, { status: STATUS_HAS_WARNINGS, @@ -1199,19 +1666,25 @@ async function main() { try { reportContentTextErrors(wikiData, {bindFind}); + logInfo`All content text validated without any errors - nice!`; + paragraph = false; Object.assign(stepStatusSummary.reportContentTextErrors, { status: STATUS_DONE_CLEAN, timeEnd: Date.now(), }); } catch (error) { + if (!paragraph) console.log(''); niceShowAggregate(error); logWarn`The above errors were detected while processing content text in data files.`; logWarn`The wiki will still build, but placeholders will be displayed in these spots.`; logWarn`Resolve the errors for more complete output.`; + console.log(''); + paragraph = true; + Object.assign(stepStatusSummary.reportContentTextErrors, { status: STATUS_HAS_WARNINGS, annotation: `view log for details`, @@ -1228,7 +1701,7 @@ async function main() { timeStart: Date.now(), }); - sortWikiDataArrays(wikiData); + sortWikiDataArrays(yamlDataSteps, wikiData); Object.assign(stepStatusSummary.sortWikiDataArrays, { status: STATUS_DONE_CLEAN, @@ -1523,6 +1996,7 @@ async function main() { } logInfo`Applying new default strings from custom ${customDefaultLanguage.code} language file.`; + paragraph = false; finalDefaultLanguage = customDefaultLanguage; finalDefaultLanguageAnnotation = `using wiki-specified custom default language`; @@ -1603,6 +2077,7 @@ async function main() { } logInfo`Loaded language strings: ${Object.keys(languages).join(', ')}`; + paragraph = false; Object.assign(stepStatusSummary.initializeDefaultLanguage, { status: STATUS_DONE_CLEAN, @@ -1610,8 +2085,6 @@ async function main() { timeEnd: Date.now(), }); - const urls = generateURLs(urlSpec); - let missingImagePaths; if (stepStatusSummary.verifyImagePaths.status === STATUS_NOT_APPLICABLE) { @@ -1735,6 +2208,7 @@ async function main() { await fileSizePreloader.waitUntilDoneLoading(); logInfo`Preloading filesizes for ${imageFilePaths.length} full-resolution images...`; + paragraph = false; fileSizePreloader.loadPaths(...imageFilePaths.map((path) => path.device)); await fileSizePreloader.waitUntilDoneLoading(); @@ -1751,6 +2225,7 @@ async function main() { }); } else { logInfo`Done preloading filesizes without any errors - nice!`; + paragraph = false; Object.assign(stepStatusSummary.preloadFileSizes, { status: STATUS_DONE_CLEAN, @@ -1759,6 +2234,110 @@ async function main() { } } + if (stepStatusSummary.buildSearchIndex.status === STATUS_NOT_STARTED) { + Object.assign(stepStatusSummary.buildSearchIndex, { + status: STATUS_STARTED_NOT_DONE, + timeStart: Date.now(), + }); + + try { + await writeSearchData({ + thumbsCache, + urls, + wikiCachePath, + wikiData, + }); + + logInfo`Search data successfully written - nice!`; + paragraph = false; + + Object.assign(stepStatusSummary.buildSearchIndex, { + status: STATUS_DONE_CLEAN, + timeEnd: Date.now(), + }); + } catch (error) { + if (!paragraph) console.log(''); + niceShowAggregate(error); + + logError`There was an error preparing or writing search data.`; + fileIssue(); + logWarn`Any existing search data will be reused, and search may be`; + logWarn`generally dysfunctional. The site should work otherwise, though!`; + + console.log(''); + paragraph = true; + + Object.assign(stepStatusSummary.buildSearchIndex, { + status: STATUS_HAS_WARNINGS, + annotation: `see log for details`, + timeEnd: Date.now(), + }); + } + } + + let webRouteSources = null; + let preparedWebRoutes = null; + + if (stepStatusSummary.identifyWebRoutes.status === STATUS_NOT_STARTED) { + Object.assign(stepStatusSummary.identifyWebRoutes, { + status: STATUS_STARTED_NOT_DONE, + timeStart: Date.now(), + }); + + const fromRoot = urls.from('shared.root'); + + try { + webRouteSources = await identifyAllWebRoutes({ + mediaCachePath, + mediaPath, + wikiCachePath, + }); + + const {aggregate, result} = + mapAggregate( + webRouteSources, + ({to, ...rest}) => ({ + ...rest, + to: fromRoot.to(...to), + }), + {message: `Errors computing effective web route paths`},); + + aggregate.close(); + preparedWebRoutes = result; + } catch (error) { + if (!paragraph) console.log(''); + niceShowAggregate(error); + + logError`There was an issue identifying web routes!`; + fileIssue(); + + console.log(''); + paragraph = true; + + Object.assign(stepStatusSummary.identifyWebRoutes, { + status: STATUS_FATAL_ERROR, + message: `JavaScript error - view log for details`, + timeEnd: Date.now(), + }); + + return false; + } + + logInfo`Successfully determined web routes - nice!`; + paragraph = false; + + Object.assign(stepStatusSummary.identifyWebRoutes, { + status: STATUS_DONE_CLEAN, + timeEnd: Date.now(), + }); + } + + wikiData.wikiInfo.searchDataAvailable = + (webRouteSources + ? webRouteSources + .some(({to}) => to[0].startsWith('searchData')) + : null); + if (stepStatusSummary.performBuild.status === STATUS_NOT_APPLICABLE) { return true; } @@ -1800,12 +2379,16 @@ async function main() { let buildModeResult; + logInfo`Passing control over to build mode: ${selectedBuildModeFlag}`; + console.log(''); + try { buildModeResult = await selectedBuildMode.go({ cliOptions, dataPath, mediaPath, mediaCachePath, + wikiCachePath, queueSize, srcRootPath: __dirname, @@ -1815,9 +2398,9 @@ async function main() { thumbsCache, urls, urlSpec, + webRoutes: preparedWebRoutes, wikiData, - cachebust: '?' + CACHEBUST, closeLanguageWatchers, developersComment, getSizeOfAdditionalFile, |