diff options
Diffstat (limited to 'src/upd8.js')
| -rwxr-xr-x | src/upd8.js | 5112 |
1 files changed, 3180 insertions, 1932 deletions
diff --git a/src/upd8.js b/src/upd8.js index ba59068f..86ecab69 100755 --- a/src/upd8.js +++ b/src/upd8.js @@ -31,2203 +31,3451 @@ // Oh yeah, like. Just run this through some relatively recent version of // node.js and you'll 8e fine. ...Within the project root. O8viously. -import * as path from 'path'; -import { promisify } from 'util'; -import { fileURLToPath } from 'url'; +import '#import-heck'; + +import {execSync} from 'node:child_process'; +import {readdir, readFile, stat, writeFile} from 'node:fs/promises'; +import * as path from 'node:path'; +import {fileURLToPath} from 'node:url'; + +import wrap from 'word-wrap'; + +import {mapAggregate, openAggregate, showAggregate} from '#aggregate'; +import CacheableObject from '#cacheable-object'; +import {stringifyCache} from '#cli'; +import {displayCompositeCacheAnalysis} from '#composite'; +import find, {bindFind, getAllFindSpecs} from '#find'; +import {processLanguageFile, watchLanguageFile, internalDefaultStringsFile} + from '#language'; +import {isMain, traverse} from '#node-utils'; +import {bindReverse} from '#reverse'; +import {writeSearchData} from '#search'; +import {sortByName} from '#sort'; +import thingConstructors from '#things'; +import {identifyAllWebRoutes} from '#web-routes'; -// I made this dependency myself! A long, long time ago. It is pro8a8ly my -// most useful li8rary ever. I'm not sure 8esides me actually uses it, though. -import fixWS from 'fix-whitespace'; -// Wait nevermind, I forgot a8out why-do-kids-love-the-taste-of-cinnamon-toast- -// crunch. THAT is my 8est li8rary. +import { + colors, + decorateTime, + fileIssue, + logWarn, + logInfo, + logError, + parseOptions, + progressCallAll, + showHelpForOptions as unboundShowHelpForOptions, +} from '#cli'; -// It stands for "HTML Entities", apparently. Cursed. -import he from 'he'; +import { + filterReferenceErrors, + reportContentTextErrors, + reportDirectoryErrors, + reportOrphanedArtworks, +} from '#data-checks'; import { - copyFile, - mkdir, - readFile, - stat, - symlink, - writeFile, - unlink, -} from 'fs/promises'; + bindOpts, + empty, + filterMultipleArrays, + indentWrap as unboundIndentWrap, + withEntries, +} from '#sugar'; + +import genThumbs, { + CACHE_FILE as thumbsCacheFile, + defaultMagickThreads, + determineMediaCachePath, + isThumb, + migrateThumbsIntoDedicatedCacheDirectory, + verifyImagePaths, +} from '#thumbs'; -import { inspect as nodeInspect } from 'util'; +import { + applyLocalizedWithBaseDirectory, + applyURLSpecOverriding, + generateURLs, + getOrigin, + internalDefaultURLSpecFile, + processURLSpecFromFile, +} from '#urls'; -import genThumbs from './gen-thumbs.js'; -import { listingSpec, listingTargetSpec } from './listing-spec.js'; -import urlSpec from './url-spec.js'; -import * as pageSpecs from './page/index.js'; +import { + getAllDataSteps, + linkWikiDataArrays, + loadYAMLDocumentsFromDataSteps, + processThingsFromDataSteps, + saveThingsFromDataSteps, + sortWikiDataArrays, +} from '#yaml'; -import find, { bindFind } from './util/find.js'; -import * as html from './util/html.js'; -import unbound_link, {getLinkThemeString} from './util/link.js'; -import { findFiles } from './util/io.js'; +import FileSizePreloader from './file-size-preloader.js'; +import {listingSpec, listingTargetSpec} from './listing-spec.js'; +import * as buildModes from './write/build-modes/index.js'; -import CacheableObject from './data/cacheable-object.js'; +const __dirname = path.dirname(fileURLToPath(import.meta.url)); -import { serializeThings } from './data/serialize.js'; +let COMMIT; +try { + COMMIT = execSync('git log --format="%h %B" -n 1 HEAD', {cwd: __dirname}).toString().trim(); +} catch (error) { + COMMIT = '(failed to detect)'; +} -import { - Language, -} from './data/things.js'; +const BUILD_TIME = new Date(); -import { - filterDuplicateDirectories, - filterReferenceErrors, - linkWikiDataArrays, - loadAndProcessDataDocuments, - sortWikiDataArrays, - WIKI_INFO_FILE, -} from './data/yaml.js'; +const STATUS_NOT_STARTED = `not started`; +const STATUS_NOT_APPLICABLE = `not applicable`; +const STATUS_STARTED_NOT_DONE = `started but not yet done`; +const STATUS_DONE_CLEAN = `done without warnings`; +const STATUS_FATAL_ERROR = `fatal error`; +const STATUS_HAS_WARNINGS = `has warnings`; -import { - fancifyFlashURL, - fancifyURL, - generateAdditionalFilesShortcut, - generateAdditionalFilesList, - generateChronologyLinks, - generateCoverLink, - generateInfoGalleryLinks, - generatePreviousNextLinks, - generateTrackListDividedByGroups, - getAlbumGridHTML, - getAlbumStylesheet, - getArtistString, - getFlashGridHTML, - getFooterLocalizationLinks, - getGridHTML, - getRevealStringFromTags, - getRevealStringFromWarnings, - getThemeString, - iconifyURL -} from './misc-templates.js'; +const defaultStepStatus = {status: STATUS_NOT_STARTED, annotation: null}; -import { - color, - decorateTime, - logWarn, - logInfo, - logError, - parseOptions, - progressPromiseAll, - ENABLE_COLOR -} from './util/cli.js'; +// Defined globally for quick access outside the main() function's contents. +// This will be initialized and mutated over the course of main(). +let stepStatusSummary; +let shouldShowStepStatusSummary = false; +let shouldShowStepMemoryInSummary = false; -import { - validateReplacerSpec, - transformInline -} from './util/replacer.js'; +async function main() { + Error.stackTraceLimit = Infinity; -import { - chunkByConditions, - chunkByProperties, - getAlbumCover, - getAlbumListTag, - getAllTracks, - getArtistAvatar, - getArtistCommentary, - getArtistNumContributions, - getFlashCover, - getKebabCase, - getTotalDuration, - getTrackCover, - sortByArtDate, - sortByDate, - sortByName -} from './util/wiki-data.js'; + let paragraph = true; -import { - serializeContribs, - serializeCover, - serializeGroupsForAlbum, - serializeGroupsForTrack, - serializeImagePaths, - serializeLink -} from './util/serialize.js'; + stepStatusSummary = { + determineMediaCachePath: + {...defaultStepStatus, name: `determine media cache path`, + for: ['thumbs', 'build']}, -import { - bindOpts, - decorateErrorWithIndex, - filterAggregateAsync, - filterEmptyLines, - mapAggregate, - mapAggregateAsync, - openAggregate, - queue, - showAggregate, - splitArray, - unique, - withAggregate, - withEntries -} from './util/sugar.js'; + migrateThumbnails: + {...defaultStepStatus, name: `migrate thumbnails`, + for: ['thumbs']}, -import { - generateURLs, - thumb -} from './util/urls.js'; + loadOfflineThumbnailCache: + {...defaultStepStatus, name: `load offline thumbnail cache file`, + for: ['thumbs', 'build']}, -// Pensive emoji! -import { - FANDOM_GROUP_DIRECTORY, - OFFICIAL_GROUP_DIRECTORY -} from './util/magic-constants.js'; + generateThumbnails: + {...defaultStepStatus, name: `generate thumbnails`, + for: ['thumbs']}, -import FileSizePreloader from './file-size-preloader.js'; + loadDataFiles: + {...defaultStepStatus, name: `load and process data files`, + for: ['build']}, -const __dirname = path.dirname(fileURLToPath(import.meta.url)); + linkWikiDataArrays: + {...defaultStepStatus, name: `link wiki data arrays`, + for: ['build']}, -const CACHEBUST = 8; + precacheCommonData: + {...defaultStepStatus, name: `precache common data`, + for: ['build']}, -const DEFAULT_STRINGS_FILE = 'strings-default.json'; + reportDirectoryErrors: + {...defaultStepStatus, name: `report directory errors`, + for: ['verify']}, -// Code that's common 8etween the 8uild code (i.e. upd8.js) and gener8ted -// site code should 8e put here. Which, uh, ~~only really means this one -// file~~ is now a variety of useful utilities! -// -// Rather than hard code it, anything in this directory can 8e shared across -// 8oth ends of the code8ase. -// (This gets symlinked into the --data-path directory.) -const UTILITY_DIRECTORY = 'util'; + reportOrphanedArtworks: + {...defaultStepStatus, name: `report orphaned artworks`, + for: ['verify']}, + + filterReferenceErrors: + {...defaultStepStatus, name: `filter reference errors`, + for: ['verify']}, + + reportContentTextErrors: + {...defaultStepStatus, name: `report content text errors`, + for: ['verify']}, + + sortWikiDataArrays: + {...defaultStepStatus, name: `sort wiki data arrays`, + for: ['build']}, + + precacheAllData: + {...defaultStepStatus, name: `precache nearly all data`, + for: ['build']}, + + sortWikiDataSourceFiles: + {...defaultStepStatus, name: `apply sorting rules to wiki data files`, + for: ['build']}, + + checkWikiDataSourceFileSorting: + {...defaultStepStatus, name: `check sorting rules against wiki data files`}, + + loadURLFiles: + {...defaultStepStatus, name: `load internal & custom url spec files`, + for: ['build']}, + + loadOnlineThumbnailCache: + {...defaultStepStatus, name: `load online thumbnail cache file`, + for: ['thumbs', 'build']}, + + // TODO: This should be split into load/watch steps. + loadInternalDefaultLanguage: + {...defaultStepStatus, name: `load internal default language`, + for: ['build']}, + + loadLanguageFiles: + {...defaultStepStatus, name: `statically load custom language files`, + for: ['build']}, + + watchLanguageFiles: + {...defaultStepStatus, name: `watch custom language files`, + for: ['build']}, + + initializeDefaultLanguage: + {...defaultStepStatus, name: `initialize default language`, + for: ['build']}, + + verifyImagePaths: + {...defaultStepStatus, name: `verify missing/misplaced image paths`, + for: ['verify']}, + + preloadFileSizes: + {...defaultStepStatus, name: `preload file sizes`, + for: ['build']}, + + loadOnlineFileSizeCache: + {...defaultStepStatus, name: `load online file size cache file`, + for: ['build']}, + + buildSearchIndex: + {...defaultStepStatus, name: `generate search index`, + for: ['build', 'search']}, + + identifyWebRoutes: + {...defaultStepStatus, name: `identify web routes`, + for: ['build']}, + + performBuild: + {...defaultStepStatus, name: `perform selected build mode`, + for: ['build']}, + }; + + const stepsWhich = condition => + Object.entries(stepStatusSummary) + .filter(([_key, value]) => condition(value)) + .map(([key]) => key); + + /* eslint-disable-next-line no-unused-vars */ + const stepsFor = (...which) => + stepsWhich(step => + which.some(w => step.for?.includes(w))); + + const stepsNotFor = (...which) => + stepsWhich(step => + which.every(w => !step.for?.includes(w))); + + const defaultQueueSize = 500; + + const buildModeFlagOptions = ( + withEntries(buildModes, entries => + entries.map(([key, mode]) => [key, { + help: mode.description, + type: 'flag', + }]))); + + const selectedBuildModeFlags = Object.keys( + await parseOptions(process.argv.slice(2), { + [parseOptions.handleUnknown]: () => {}, + ...buildModeFlagOptions, + })); + + let selectedBuildModeFlag; + let sortInAdditionToBuild = false; + + // As an exception, --sort can be combined with another build mode. + if (selectedBuildModeFlags.length >= 2 && selectedBuildModeFlags.includes('sort')) { + sortInAdditionToBuild = true; + selectedBuildModeFlags.splice(selectedBuildModeFlags.indexOf('sort'), 1); + } + + if (sortInAdditionToBuild) { + Object.assign(stepStatusSummary.sortWikiDataSourceFiles, { + status: STATUS_NOT_STARTED, + annotation: `--sort provided with another build mode`, + }); -// Code that's used only in the static site! CSS, cilent JS, etc. -// (This gets symlinked into the --data-path directory.) -const STATIC_DIRECTORY = 'static'; + Object.assign(stepStatusSummary.checkWikiDataSourceFileSorting, { + status: STATUS_NOT_APPLICABLE, + annotation: `--sort provided, dry run not applicable`, + }); + } else { + Object.assign(stepStatusSummary.sortWikiDataSourceFiles, { + status: STATUS_NOT_APPLICABLE, + annotation: `--sort not provided, dry run only`, + }); -// This exists adjacent to index.html for any page with oEmbed metadata. -const OEMBED_JSON_FILE = 'oembed.json'; + Object.assign(stepStatusSummary.checkWikiDataSourceFileSorting, { + status: STATUS_NOT_STARTED, + annotation: `--sort not provided, dry run applicable`, + }); + } + + if (empty(selectedBuildModeFlags)) { + // No build mode selected. This is not a valid state for building the wiki, + // but we want to let access to --help, so we'll show a message about what + // to do later. + selectedBuildModeFlag = null; + } else if (selectedBuildModeFlags.length > 1) { + logError`Building multiple modes (${selectedBuildModeFlags.join(', ')}) at once not supported.`; + logError`Please specify one build mode.`; + return false; + } else { + selectedBuildModeFlag = selectedBuildModeFlags[0]; + } + + const selectedBuildMode = + (selectedBuildModeFlag + ? buildModes[selectedBuildModeFlag] + : null); + + // This is about to get a whole lot more stuff put in it. + const wikiData = { + listingSpec, + listingTargetSpec, + }; + + const buildOptions = + (selectedBuildMode + ? selectedBuildMode.getCLIOptions() + : {}); + + const commonOptions = { + 'help': { + help: `Display usage info and basic information for the \`hsmusic\` command`, + type: 'flag', + }, -// Automatically copied (if present) from media directory to site root. -const FAVICON_FILE = 'favicon.ico'; + // Data files for the site, including flash, artist, and al8um data, + // and like a jillion other things too. Pretty much everything which + // makes an individual wiki what it is goes here! + 'data-path': { + help: `Specify path to data directory, including YAML files that cover all info about wiki content, layout, and structure\n\nAlways required for wiki building; may be provided via the HSMUSIC_DATA environment variable`, + type: 'value', + }, -function inspect(value) { - return nodeInspect(value, {colors: ENABLE_COLOR}); -} + // Static media will 8e referenced in the site here! The contents are + // categorized; check out MEDIA_ALBUM_ART_DIRECTORY and other constants + // near the top of this file (upd8.js). + 'media-path': { + help: `Specify path to media directory, including album artwork and additional files, as well as custom site layout media and other media files for reference or linking in wiki content\n\nAlways required for wiki building; may be provided via the HSMUSIC_MEDIA environment variable`, + type: 'value', + }, -// Shared varia8les! These are more efficient to access than a shared varia8le -// (or at least I h8pe so), and are easier to pass across functions than a -// 8unch of specific arguments. -// -// Upd8: Okay yeah these aren't actually any different. Still cleaner than -// passing around a data object containing all this, though. -let dataPath; -let mediaPath; -let langPath; -let outputPath; + 'media-cache-path': { + help: `Specify path to media cache directory, including automatically generated thumbnails\n\nThis usually doesn't need to be provided, and will be inferred either by loading "media-cache" from --cache-path, or by adding "-cache" to the end of the media directory\n\nMay be provided via the HSMUSIC_MEDIA_CACHE environment variable`, + type: 'value', + }, -// Glo8al data o8ject shared 8etween 8uild functions and all that. This keeps -// everything encapsul8ted in one place, so it's easy to pass and share across -// modules! -let wikiData = {}; + 'cache-path': { + help: `Specify path to general cache directory, usually containing generated thumbnails and assorted files reused between builds\n\nAlways required for wiki building; may be provided via the HSMUSIC_CACHE environment varaible`, + type: 'value', + }, -let queueSize; + // String files! For the most part, this is used for translating the + // site to different languages, though you can also customize strings + // for your own 8uild of the site if you'd like. Files here should all + // match the format in strings-default.json in this repository. (If a + // language file is missing any strings, the site code will fall 8ack + // to what's specified in strings-default.json.) + // + // Unlike the other options here, this one's optional - the site will + // 8uild with the default (English) strings if this path is left + // unspecified. + 'lang-path': { + help: `Specify path to language directory, including JSON files that mapping internal string keys to localized language content, and various language metadata\n\nOptional for wiki building, unless the wiki's default language is not English; may be provided via the HSMUSIC_LANG environment variable instead`, + type: 'value', + }, -const urls = generateURLs(urlSpec); + 'urls': { + help: `Specify which optional URL specs to use for this build, customizing where pages are generated or resources are accessed from`, + type: 'value', + }, -function splitLines(text) { - return text.split(/\r\n|\r|\n/); -} + 'show-url-spec': { + help: `Displays the entire computed URL spec, after the data folder's default override and optional specs are applied. This is mostly useful for progammer debugging!`, + type: 'flag', + }, -const replacerSpec = { - 'album': { - find: 'album', - link: 'album' + 'skip-directory-validation': { + help: `Skips checking for duplicated directories, which speeds up the build but may cause the wiki to catch on fire`, + type: 'flag', }, - 'album-commentary': { - find: 'album', - link: 'albumCommentary' + + 'skip-orphaned-artwork-validation': { + help: `Skips checking for internally orphaned artworks, which is a bad idea, unless you're debugging those in particular`, + type: 'flag', }, - 'artist': { - find: 'artist', - link: 'artist' + + 'skip-reference-validation': { + help: `Skips checking and reporting reference errors, which speeds up the build but may silently allow erroneous data to pass through`, + type: 'flag', }, - 'artist-gallery': { - find: 'artist', - link: 'artistGallery' + + 'skip-content-text-validation': { + help: `Skips checking and reporting content text errors, which speeds up the build but may silently allow misformatted or mislinked content to pass through`, + type: 'flag', }, - 'commentary-index': { - find: null, - link: 'commentaryIndex' + + // Thum8nail gener8tion is *usually* something you want, 8ut it can 8e + // kinda a pain to run every time, since it does necessit8te reading + // every media file at run time. Pass this to skip it. + 'skip-thumbs': { + help: `Skip processing and generating thumbnails in media directory (speeds up subsequent builds, but remove this option [or use --thumbs-only] and re-run once when you add or modify media files to ensure thumbnails stay up-to-date!)`, + type: 'flag', }, - 'date': { - find: null, - value: ref => new Date(ref), - html: (date, {language}) => `<time datetime="${date.toString()}">${language.formatDate(date)}</time>` + + // Or, if you *only* want to gener8te newly upd8ted thum8nails, you can + // pass this flag! It exits 8efore 8uilding the rest of the site. + 'thumbs-only': { + help: `Skip everything besides processing media directory and generating up-to-date thumbnails (useful when using --skip-thumbs for most runs)`, + type: 'flag', }, - 'flash': { - find: 'flash', - link: 'flash', - transformName(name, node, input) { - const nextCharacter = input[node.iEnd]; - const lastCharacter = name[name.length - 1]; - if ( - ![' ', '\n', '<'].includes(nextCharacter) && - lastCharacter === '.' - ) { - return name.slice(0, -1); - } else { - return name; - } - } + + 'migrate-thumbs': { + help: `Transfer automatically generated thumbnail files out of an existing media directory and into the easier-to-manage media-cache directory`, + type: 'flag', }, - 'group': { - find: 'group', - link: 'groupInfo' + + 'new-thumbs': { + help: `Repair a media cache that's completely missing its index file by starting clean and not reusing any existing thumbnails`, + type: 'flag', }, - 'group-gallery': { - find: 'group', - link: 'groupGallery' + + 'refresh-online-thumbs': { + help: `Downloads a fresh copy of the online file size cache, so changes there are immediately reflected`, + type: 'flag', }, - 'home': { - find: null, - link: 'home' + + 'skip-file-sizes': { + help: `Skips preloading file sizes for images and additional files, which will be left blank in the build`, + type: 'flag', }, - 'listing-index': { - find: null, - link: 'listingIndex' + + 'refresh-online-file-sizes': { + help: `Downloads a fresh copy of the online file size cache, so changes there are immediately reflected`, + type: 'flag', }, - 'listing': { - find: 'listing', - link: 'listing' + + 'skip-sorting-validation': { + help: `Skips checking the if custom sorting rules for this wiki are satisfied`, + type: 'flag', }, - 'media': { - find: null, - link: 'media' + + 'skip-media-validation': { + help: `Skips checking and reporting missing and misplaced media files, which isn't necessary if you aren't adding or removing data or updating directories`, + type: 'flag', }, - 'news-index': { - find: null, - link: 'newsIndex' + + 'refresh-search': { + help: `Generate the text search index this build, instead of waiting for the automatic delay`, + type: 'flag', }, - 'news-entry': { - find: 'newsEntry', - link: 'newsEntry' + + 'skip-search': { + help: `Skip creation of the text search index no matter what, even if it'd normally be scheduled for now`, + type: 'flag', }, - 'root': { - find: null, - link: 'root' + + // Just working on data entries and not interested in actually + // generating site HTML yet? This flag will cut execution off right + // 8efore any site 8uilding actually happens. + 'no-build': { + help: `Don't run a build of the site at all; only process data/media and report any errors detected`, + type: 'flag', + }, + + 'no-input': { + help: `Don't wait on input from stdin - assume the device is headless`, + type: 'flag', + }, + + 'no-language-reloading': { + help: `Don't reload language files while the build is running\n\nApplied by default for --static-build`, + type: 'flag', + }, + + 'no-language-reload': {alias: 'no-language-reloading'}, + + // Want sweet, sweet trace8ack info in aggreg8te error messages? This + // will print all the juicy details (or at least the first relevant + // line) right to your output, 8ut also pro8a8ly give you a headache + // 8ecause wow that is a lot of visual noise. + 'show-traces': { + help: `Show JavaScript source code paths for reported errors in "aggregate" error displays\n\n(Debugging use only, but please enable this if you're reporting bugs for our issue tracker!)`, + type: 'flag', }, - 'site': { - find: null, - link: 'site' + + 'show-step-summary': { + help: `Show a summary of all the top-level build steps once hsmusic exits. This is mostly useful for progammer debugging!`, + type: 'flag', + }, + + 'show-step-memory': { + help: `Include total process memory usage traces at the time each top-level build step ends. Use with --show-step-summary. This is mostly useful for programmer debugging!`, + type: 'flag', }, - 'static': { - find: 'staticPage', - link: 'staticPage' + + 'queue-size': { + help: `Process more or fewer disk files at once to optimize performance or avoid I/O errors, unlimited if set to 0 (between 500 and 700 is usually a safe range for building HSMusic on Windows machines)\nDefaults to ${defaultQueueSize}`, + type: 'value', + validate(size) { + if (parseInt(size) !== parseFloat(size)) return 'an integer'; + if (parseInt(size) < 0) return 'a counting number or zero'; + return true; + }, }, - 'string': { - find: null, - value: ref => ref, - html: (ref, {language, args}) => language.$(ref, args) + queue: {alias: 'queue-size'}, + + 'magick-threads': { + help: `Process more or fewer thumbnail files at once with ImageMagick when generating thumbnails. (Each ImageMagick thread may also make use of multi-core processing at its own utility.)`, + type: 'value', + validate(threads) { + if (parseInt(threads) !== parseFloat(threads)) return 'an integer'; + if (parseInt(threads) < 0) return 'a counting number or zero'; + return true; + } }, - 'tag': { - find: 'artTag', - link: 'tag' + magick: {alias: 'magick-threads'}, + + 'precache-mode': { + help: + `Change the way certain runtime-computed values are preemptively evaluated and cached\n\n` + + `common: Preemptively compute certain properties which are needed for basic data loading and site generation\n\n` + + `all: Compute every visible data property, optimizing rate of content generation, but causing a long stall before the build actually starts\n\n` + + `none: Don't preemptively compute any values - strictly the most efficient, but may result in unpredictably "lopsided" performance for individual steps of loading data and building the site\n\n` + + `Defaults to 'common'`, + type: 'value', + validate(value) { + if (['common', 'all', 'none'].includes(value)) return true; + return 'common, all, or none'; + }, }, - 'track': { - find: 'track', - link: 'track' + }; + + const indentWrap = + bindOpts(unboundIndentWrap, { + wrap, + }); + + const showHelpForOptions = + bindOpts(unboundShowHelpForOptions, { + [bindOpts.bindIndex]: 0, + indentWrap, + sort: sortByName, + }); + + const cliOptions = await parseOptions(process.argv.slice(2), { + // We don't want to error when we receive these options, so specify them + // here, even though we won't be doing anything with them later. + // (This is a bit of a hack.) + ...buildModeFlagOptions, + + ...commonOptions, + ...buildOptions, + }); + + shouldShowStepStatusSummary = cliOptions['show-step-summary'] ?? false; + shouldShowStepMemoryInSummary = cliOptions['show-step-memory'] ?? false; + + if (cliOptions['help']) { + console.log( + colors.bright(`hsmusic (aka. Homestuck Music Wiki, HSMusic Wiki)\n`) + + `static wiki software cataloguing collaborative creation\n`); + + console.log(indentWrap( + `The \`hsmusic\` command provides basic control over ` + + `all parts of generating user-visible HTML pages ` + + `and website content/structure ` + + `from provided data, media, and language directories.\n` + + `\n` + + `CLI options are divided into three groups:\n`)); + + console.log(` 1) ` + indentWrap( + `Common options: ` + + `These are shared by all build modes ` + + `and always have the same essential behavior`, + {spaces: 4, bullet: true})); + + console.log(` 2) ` + indentWrap( + `Build mode selection: ` + + `One build mode should be selected, ` + + `and it decides the main set of behavior to use ` + + `for presenting or interacting with site content`, + {spaces: 4, bullet: true})); + + console.log(` 3) ` + indentWrap( + `Build options: ` + + `Each build mode has a set of unique options ` + + `which customize behavior for that build mode`, + {spaces: 4, bullet: true})); + + console.log(``); + + showHelpForOptions({ + heading: `Common options`, + options: commonOptions, + wrap, + }); + + showHelpForOptions({ + heading: `Build mode selection`, + options: buildModeFlagOptions, + wrap, + }); + + if (selectedBuildMode) { + showHelpForOptions({ + heading: `Build options for --${selectedBuildModeFlag}`, + options: buildOptions, + wrap, + }); + } else { + console.log( + `Specify a build mode and run with ${colors.bright('--help')} again for info\n` + + `about the options for that build mode.`); } -}; -if (!validateReplacerSpec(replacerSpec, {find, link: unbound_link})) { - process.exit(); -} + for (const step of Object.values(stepStatusSummary)) { + Object.assign(step, { + status: STATUS_NOT_APPLICABLE, + annotation: `--help provided`, + }); + } + + return true; + } + + const dataPath = cliOptions['data-path'] || process.env.HSMUSIC_DATA; + const mediaPath = cliOptions['media-path'] || process.env.HSMUSIC_MEDIA; + const wikiCachePath = cliOptions['cache-path'] || process.env.HSMUSIC_CACHE; + const langPath = cliOptions['lang-path'] || process.env.HSMUSIC_LANG; // Can 8e left unset! + + const thumbsOnly = cliOptions['thumbs-only'] ?? false; + const noInput = cliOptions['no-input'] ?? false; + + const showAggregateTraces = cliOptions['show-traces'] ?? false; + + const precacheMode = cliOptions['precache-mode'] ?? 'common'; + + const wantedURLSpecKeys = cliOptions['urls'] ?? []; + const showURLSpec = cliOptions['show-url-spec'] ?? false; + + // Makes writing nicer on the CPU and file I/O parts of the OS, with a + // marginal performance deficit while waiting for file writes to finish + // before proceeding to more page processing. + const queueSize = +(cliOptions['queue-size'] ?? defaultQueueSize); + + const magickThreads = +(cliOptions['magick-threads'] ?? defaultMagickThreads); + + if (!dataPath) { + logError`${`Expected --data-path option or HSMUSIC_DATA to be set`}`; + } -function parseAttributes(string, {to}) { - const attributes = Object.create(null); - const skipWhitespace = i => { - const ws = /\s/; - if (ws.test(string[i])) { - const match = string.slice(i).match(/[^\s]/); - if (match) { - return i + match.index; - } else { - return string.length; - } + if (!mediaPath) { + logError`${`Expected --media-path option or HSMUSIC_MEDIA to be set`}`; + } + + if (!wikiCachePath) { + logError`${`Expected --cache-path option or HSMUSIC_CACHE to be set`}`; + } + + if (!dataPath || !mediaPath || !wikiCachePath) { + return false; + } + + if (cliOptions['no-build']) { + logInfo`Won't generate any site or page files this run (--no-build passed).`; + + Object.assign(stepStatusSummary.performBuild, { + status: STATUS_NOT_APPLICABLE, + annotation: `--no-build provided`, + }); + } + + // Finish setting up defaults by combining information from all options. + + const _fallbackStep = (stepKey, { + default: defaultValue, + cli: cliArg, + buildConfig: buildConfigKey = null, + }) => { + const buildConfig = selectedBuildMode?.config?.[buildConfigKey]; + const {[stepKey]: step} = stepStatusSummary; + + const cliEntries = + (cliArg === null || cliArg === undefined + ? [] + : Array.isArray(cliArg) + ? cliArg + : [cliArg]); + + for (const { + flag: cliFlag = null, + negate: cliFlagNegates = false, + warn: cliFlagWarning = null, + disable: cliFlagDisablesSteps = [], + } of cliEntries) { + if (!cliOptions[cliFlag]) { + continue; + } + + const cliPart = `--` + cliFlag; + const modePart = `--` + selectedBuildModeFlag; + + if (buildConfig?.applicable === false) { + if (cliFlagNegates) { + logWarn`${cliPart} provided, but ${modePart} already skips this step`; + logWarn`Redundant option ${cliPart}`; + continue; } else { - return i; + logWarn`${cliPart} provided, but this step isn't applicable for ${modePart}`; + logWarn`Ignoring option ${cliPart}`; + continue; } - }; + } - for (let i = 0; i < string.length;) { - i = skipWhitespace(i); - const aStart = i; - const aEnd = i + string.slice(i).match(/[\s=]|$/).index; - const attribute = string.slice(aStart, aEnd); - i = skipWhitespace(aEnd); - if (string[i] === '=') { - i = skipWhitespace(i + 1); - let end, endOffset; - if (string[i] === '"' || string[i] === "'") { - end = string[i]; - endOffset = 1; - i++; - } else { - end = '\\s'; - endOffset = 0; - } - const vStart = i; - const vEnd = i + string.slice(i).match(new RegExp(`${end}|$`)).index; - const value = string.slice(vStart, vEnd); - i = vEnd + endOffset; - if (attribute === 'src' && value.startsWith('media/')) { - attributes[attribute] = to('media.path', value.slice('media/'.length)); - } else { - attributes[attribute] = value; - } + if (buildConfig?.required === true) { + if (cliFlagNegates) { + logWarn`${cliPart} provided, but ${modePart} requires this step`; + logWarn`Ignoring option ${cliPart}`; + continue; } else { - attributes[attribute] = attribute; + logWarn`${cliPart} provided, but ${modePart} already requires this step`; + logWarn`Redundant option ${cliPart}`; + continue; } - } - return Object.fromEntries(Object.entries(attributes).map(([ key, val ]) => [ - key, - val === 'true' ? true : - val === 'false' ? false : - val === key ? true : - val - ])); -} + } -function joinLineBreaks(sourceLines) { - const outLines = []; + step.status = + (cliFlagNegates + ? STATUS_NOT_APPLICABLE + : STATUS_NOT_STARTED); - let lineSoFar = ''; - for (let i = 0; i < sourceLines.length; i++) { - const line = sourceLines[i]; - lineSoFar += line; - if (!line.endsWith('<br>')) { - outLines.push(lineSoFar); - lineSoFar = ''; + step.annotation = `--${cliFlag} provided`; + + if (cliFlagWarning) { + if (!paragraph) console.log(''); + + for (const line of cliFlagWarning.split('\n')) { + logWarn(line); + } + + paragraph = false; + } + + for (const step of cliFlagDisablesSteps) { + const summary = stepStatusSummary[step]; + if (summary.status === STATUS_NOT_APPLICABLE && summary.annotation) { + stepStatusSummary.performBuild.annotation += `; --${cliFlag} provided`; + } else { + summary.status = STATUS_NOT_APPLICABLE; + summary.annotation = `--${cliFlag} provided`; } + } + + return; } - if (lineSoFar) { - outLines.push(lineSoFar); + if (buildConfig?.required === true) { + step.status = STATUS_NOT_STARTED; + step.annotation = `required for --${selectedBuildModeFlag}`; + return; } - return outLines; -} + if (buildConfig?.applicable === false) { + step.status = STATUS_NOT_APPLICABLE; + step.annotation = `N/A for --${selectedBuildModeFlag}`; + return; + } -function transformMultiline(text, { - parseAttributes, - transformInline -}) { - // Heck yes, HTML magics. + if (buildConfig?.default === 'skip') { + step.status = STATUS_NOT_APPLICABLE; + step.annotation = `default for --${selectedBuildModeFlag}`; + return; + } - text = transformInline(text.trim()); + if (buildConfig?.default === 'perform') { + step.status = STATUS_NOT_STARTED; + step.annotation = `default for --${selectedBuildModeFlag}`; + return; + } - const outLines = []; + switch (defaultValue) { + case 'skip': { + step.status = STATUS_NOT_APPLICABLE; - const indentString = ' '.repeat(4); + const enablingFlags = + cliEntries + .filter(({negate}) => !negate) + .map(({flag}) => flag); - let levelIndents = []; - const openLevel = indent => { - // opening a sublist is a pain: to be semantically *and* visually - // correct, we have to append the <ul> at the end of the existing - // previous <li> - const previousLine = outLines[outLines.length - 1]; - if (previousLine?.endsWith('</li>')) { - // we will re-close the <li> later - outLines[outLines.length - 1] = previousLine.slice(0, -5) + ' <ul>'; - } else { - // if the previous line isn't a list item, this is the opening of - // the first list level, so no need for indent - outLines.push('<ul>'); - } - levelIndents.push(indent); - }; - const closeLevel = () => { - levelIndents.pop(); - if (levelIndents.length) { - // closing a sublist, so close the list item containing it too - outLines.push(indentString.repeat(levelIndents.length) + '</ul></li>'); - } else { - // closing the final list level! no need for indent here - outLines.push('</ul>'); + if (!empty(enablingFlags)) { + step.annotation = + enablingFlags.map(flag => `--${flag}`).join(', ') + + ` not provided`; } + + break; + } + + case 'perform': + break; + + default: + throw new Error(`Invalid default step status ${defaultValue}`); + } + }; + + { + let errored = false; + + const fallbackStep = (stepKey, options) => { + try { + _fallbackStep(stepKey, options); + } catch (error) { + logError`Error determining fallback for step ${stepKey}`; + showAggregate(error); + errored = true; + } }; - // okay yes we should support nested formatting, more than one blockquote - // layer, etc, but hear me out here: making all that work would basically - // be the same as implementing an entire markdown converter, which im not - // interested in doing lol. sorry!!! - let inBlockquote = false; - - let lines = splitLines(text); - lines = joinLineBreaks(lines); - for (let line of lines) { - const imageLine = line.startsWith('<img'); - line = line.replace(/<img (.*?)>/g, (match, attributes) => img({ - lazy: true, - link: true, - thumb: 'medium', - ...parseAttributes(attributes) - })); - - let indentThisLine = 0; - let lineContent = line; - let lineTag = 'p'; - - const listMatch = line.match(/^( *)- *(.*)$/); - if (listMatch) { - // is a list item! - if (!levelIndents.length) { - // first level is always indent = 0, regardless of actual line - // content (this is to avoid going to a lesser indent than the - // initial level) - openLevel(0); - } else { - // find level corresponding to indent - const indent = listMatch[1].length; - let i; - for (i = levelIndents.length - 1; i >= 0; i--) { - if (levelIndents[i] <= indent) break; - } - // note: i cannot equal -1 because the first indentation level - // is always 0, and the minimum indentation is also 0 - if (levelIndents[i] === indent) { - // same indent! return to that level - while (levelIndents.length - 1 > i) closeLevel(); - // (if this is already the current level, the above loop - // will do nothing) - } else if (levelIndents[i] < indent) { - // lesser indent! branch based on index - if (i === levelIndents.length - 1) { - // top level is lesser: add a new level - openLevel(indent); - } else { - // lower level is lesser: return to that level - while (levelIndents.length - 1 > i) closeLevel(); - } - } - } - // finally, set variables for appending content line - indentThisLine = levelIndents.length; - lineContent = listMatch[2]; - lineTag = 'li'; - } else { - // not a list item! close any existing list levels - while (levelIndents.length) closeLevel(); - - // like i said, no nested shenanigans - quotes only appear outside - // of lists. sorry! - const quoteMatch = line.match(/^> *(.*)$/); - if (quoteMatch) { - // is a quote! open a blockquote tag if it doesnt already exist - if (!inBlockquote) { - inBlockquote = true; - outLines.push('<blockquote>'); - } - indentThisLine = 1; - lineContent = quoteMatch[1]; - } else if (inBlockquote) { - // not a quote! close a blockquote tag if it exists - inBlockquote = false; - outLines.push('</blockquote>'); - } - - // let some escaped symbols display as the normal symbol, since the - // point of escaping them is just to avoid having them be treated as - // syntax markers! - if (lineContent.match(/( *)\\-/)) { - lineContent = lineContent.replace('\\-', '-'); - } else if (lineContent.match(/( *)\\>/)) { - lineContent = lineContent.replace('\\>', '>'); - } - } + fallbackStep('reportDirectoryErrors', { + default: 'perform', + cli: { + flag: 'skip-directory-validation', + negate: true, + warn: + `Skipping directory validation. If any directories are duplicated\n` + + `in data, the build will probably fail in unpredictable ways.`, + }, + }); - if (lineTag === 'p') { - // certain inline element tags should still be postioned within a - // paragraph; other elements (e.g. headings) should be added as-is - const elementMatch = line.match(/^<(.*?)[ >]/); - if (elementMatch && !imageLine && !['a', 'abbr', 'b', 'bdo', 'br', 'cite', 'code', 'data', 'datalist', 'del', 'dfn', 'em', 'i', 'img', 'ins', 'kbd', 'mark', 'output', 'picture', 'q', 'ruby', 'samp', 'small', 'span', 'strong', 'sub', 'sup', 'svg', 'time', 'var', 'wbr'].includes(elementMatch[1])) { - lineTag = ''; - } + fallbackStep('reportOrphanedArtworks', { + default: 'perform', + cli: { + flag: 'skip-orphaned-artwork-validation', + negate: true, + warn: + `Skipping orphaned artwork validation. Hopefully you're debugging!`, + }, + }); + + fallbackStep('filterReferenceErrors', { + default: 'perform', + cli: { + flag: 'skip-reference-validation', + negate: true, + warn: + `Skipping reference validation. If any reference errors are present\n` + + `in data, they will be silently passed along to the build.`, + }, + }); + + fallbackStep('reportContentTextErrors', { + default: 'perform', + cli: { + flag: 'skip-content-text-validation', + negate: true, + warn: + `Skipping content text validation. If any commentary or other content\n` + + `is misformatted or has bad links, it will be silently passed along\n` + + `to the build.`, + }, + }); + + fallbackStep('generateThumbnails', { + default: 'perform', + buildConfig: 'thumbs', + cli: [ + {flag: 'thumbs-only', disable: stepsNotFor('thumbs')}, + {flag: 'skip-thumbs', negate: true}, + ], + }); + + fallbackStep('migrateThumbnails', { + default: 'skip', + cli: { + flag: 'migrate-thumbs', + disable: [ + ...stepsNotFor('thumbs'), + 'generateThumbnails', + ], + }, + }); + + fallbackStep('preloadFileSizes', { + default: 'perform', + buildConfig: 'fileSizes', + cli: { + flag: 'skip-file-sizes', + negate: true, + }, + }); + + fallbackStep('identifyWebRoutes', { + default: 'perform', + buildConfig: 'webRoutes', + }); + + decideBuildSearchIndex: { + fallbackStep('buildSearchIndex', { + default: 'skip', + buildConfig: 'search', + cli: [ + {flag: 'refresh-search'}, + {flag: 'skip-search', negate: true}, + ], + }); + + if (cliOptions['refresh-search'] || cliOptions['skip-search']) { + if (cliOptions['refresh-search']) { + logInfo`${'--refresh-search'} provided, will generate search fresh this build.`; } - let pushString = indentString.repeat(indentThisLine); - if (lineTag) { - pushString += `<${lineTag}>${lineContent}</${lineTag}>`; + break decideBuildSearchIndex; + } + + if (stepStatusSummary.buildSearchIndex.status !== STATUS_NOT_APPLICABLE) { + break decideBuildSearchIndex; + } + + if (selectedBuildMode?.config?.search?.default === 'skip') { + break decideBuildSearchIndex; + } + + // TODO: OK this is a little silly. + if (stepStatusSummary.buildSearchIndex.annotation?.startsWith('N/A')) { + break decideBuildSearchIndex; + } + + const indexFile = path.join(wikiCachePath, 'search', 'index.json') + let stats; + try { + stats = await stat(indexFile); + } catch (error) { + if (error.code === 'ENOENT') { + Object.assign(stepStatusSummary.buildSearchIndex, { + status: STATUS_NOT_STARTED, + annotation: `search/index.json not present, will create`, + }); + + logInfo`Looks like the search cache doesn't exist.`; + logInfo`It'll be generated fresh, this build!`; } else { - pushString += lineContent; + Object.assign(stepStatusSummary.buildSearchIndex, { + status: STATUS_NOT_APPLICABLE, + annotation: `error getting search index stats`, + }); + + if (!paragraph) console.log(''); + console.error(error); + + logWarn`There was an error checking the search index file, located at:`; + logWarn`${indexFile}`; + logWarn`You may want to toss out the "search" folder; it'll be generated`; + logWarn`anew, if you do, and may fix this error.`; } - outLines.push(pushString); + + paragraph = false; + break decideBuildSearchIndex; + } + + const delta = Date.now() - stats.mtimeMs; + const minute = 60 * 1000; + const delay = 45 * minute; + + const whenst = duration => `~${Math.ceil(duration / minute)} min`; + + if (delta < delay) { + logInfo`Search index was generated recently, skipping for this build.`; + logInfo`Next scheduled is in ${whenst(delay - delta)}, or by using ${'--refresh-search'}.`; + Object.assign(stepStatusSummary.buildSearchIndex, { + status: STATUS_NOT_APPLICABLE, + annotation: `earlier than scheduled`, + }); + } else { + logInfo`Search index hasn't been generated for a little while.`; + logInfo`It'll be generated this build, then again in ${whenst(delay)}.`; + Object.assign(stepStatusSummary.buildSearchIndex, { + status: STATUS_NOT_STARTED, + annotation: `past when shceduled`, + }); + } + + paragraph = false; } - // after processing all lines... + fallbackStep('checkWikiDataSourceFileSorting', { + default: 'perform', + buildConfig: 'sort', + cli: { + flag: 'skip-sorting-validation', + negate: true, + warning: + `Skipping sorting validation. If any of this wiki's sorting rules are not\n` + + `satisfied, those errors will be silently passed along to the build.`, + }, + }); + + fallbackStep('verifyImagePaths', { + default: 'perform', + buildConfig: 'mediaValidation', + cli: { + flag: 'skip-media-validation', + negate: true, + warning: + `Skipping media validation. If any media files are missing or misplaced,\n` + + `those errors will be silently passed along to the build.`, + }, + }); - // if still in a list, close all levels - while (levelIndents.length) closeLevel(); + fallbackStep('watchLanguageFiles', { + default: 'perform', + buildConfig: 'languageReloading', + cli: { + flag: 'no-language-reloading', + negate: true, + }, + }); - // if still in a blockquote, close its tag - if (inBlockquote) { - inBlockquote = false; - outLines.push('</blockquote>'); + if (errored) { + return false; } + } - return outLines.join('\n'); -} + if (stepStatusSummary.generateThumbnails.status === STATUS_NOT_STARTED) { + Object.assign(stepStatusSummary.loadOfflineThumbnailCache, { + status: STATUS_NOT_APPLICABLE, + annotation: `using cache from thumbnail generation`, + }); + } -function transformLyrics(text, { - transformInline, - transformMultiline -}) { - // Different from transformMultiline 'cuz it joins multiple lines together - // with line 8reaks (<br>); transformMultiline treats each line as its own - // complete paragraph (or list, etc). - - // If it looks like old data, then like, oh god. - // Use the normal transformMultiline tool. - if (text.includes('<br')) { - return transformMultiline(text); - } - - text = transformInline(text.trim()); - - let buildLine = ''; - const addLine = () => outLines.push(`<p>${buildLine}</p>`); - const outLines = []; - for (const line of text.split('\n')) { - if (line.length) { - if (buildLine.length) { - buildLine += '<br>'; - } - buildLine += line; - } else if (buildLine.length) { - addLine(); - buildLine = ''; - } + if (stepStatusSummary.watchLanguageFiles.status === STATUS_NOT_STARTED) { + Object.assign(stepStatusSummary.loadLanguageFiles, { + status: STATUS_NOT_APPLICABLE, + annotation: `watching for changes instead`, + }); + } + + // TODO: These should error if the option was actually provided but + // the relevant steps were already disabled for some other reason. + switch (precacheMode) { + case 'common': + if (stepStatusSummary.precacheAllData.status === STATUS_NOT_STARTED) { + Object.assign(stepStatusSummary.precacheAllData, { + status: STATUS_NOT_APPLICABLE, + annotation: `--precache-mode is common, not all`, + }); + } + + break; + + case 'all': + if (stepStatusSummary.precacheCommonData.status === STATUS_NOT_STARTED) { + Object.assign(stepStatusSummary.precacheCommonData, { + status: STATUS_NOT_APPLICABLE, + annotation: `--precache-mode is all, not common`, + }); + } + + break; + + case 'none': + if (stepStatusSummary.precacheCommonData.status === STATUS_NOT_STARTED) { + Object.assign(stepStatusSummary.precacheCommonData, { + status: STATUS_NOT_APPLICABLE, + annotation: `--precache-mode is none`, + }); + } + + if (stepStatusSummary.precacheAllData.status === STATUS_NOT_STARTED) { + Object.assign(stepStatusSummary.precacheAllData, { + status: STATUS_NOT_APPLICABLE, + annotation: `--precache-mode is none`, + }); + } + + break; + } + + if (!langPath) { + Object.assign(stepStatusSummary.loadLanguageFiles, { + status: STATUS_NOT_APPLICABLE, + annotation: `neither --lang-path nor HSMUSIC_LANG provided`, + }); + + Object.assign(stepStatusSummary.watchLanguageFiles, { + status: STATUS_NOT_APPLICABLE, + annotation: `neither --lang-path nor HSMUSIC_LANG provided`, + }); + } + + if (stepStatusSummary.generateThumbnails.status === STATUS_NOT_APPLICABLE && thumbsOnly) { + logInfo`Well, you've put yourself rather between a roc and a hard place, hmmmm?`; + return false; + } + + // If we're going to require a build mode and none is specified, + // exit and show what to do. This must not precede anything that might + // disable the build (e.g. changing its status to STATUS_NOT_APPLICABLE). + + if (stepStatusSummary.performBuild.status === STATUS_NOT_STARTED) { + if (selectedBuildMode) { + logInfo`Will use specified build mode: ${selectedBuildModeFlag}`; + } else { + showHelpForOptions({ + heading: `Please specify a build mode:`, + options: buildModeFlagOptions, + }); + + console.log( + `(Use ${colors.bright('--help')} for general info and all options, or specify\n` + + ` a build mode alongside ${colors.bright('--help')} for that mode's options!`); + + for (const step of Object.values(stepStatusSummary)) { + Object.assign(step, { + status: STATUS_NOT_APPLICABLE, + annotation: `no build mode provided`, + }); + } + + return false; } - if (buildLine.length) { - addLine(); + } else if (selectedBuildMode) { + if (stepStatusSummary.performBuild.annotation) { + logError`You've specified a build mode, ${selectedBuildModeFlag}, but it won't be used,`; + logError`according to the message: ${`"${stepStatusSummary.performBuild.annotation}"`}`; + } else { + logError`You've specified a build mode, ${selectedBuildModeFlag}, but it won't be used,`; + logError`probably because of another option you've provided.`; } - return outLines.join('\n'); -} + logError`Please remove ${'--' + selectedBuildModeFlag} or the conflicting option.`; + return false; + } -function stringifyThings(thingData) { - return JSON.stringify(serializeThings(thingData)); -} + Object.assign(stepStatusSummary.determineMediaCachePath, { + status: STATUS_STARTED_NOT_DONE, + timeStart: Date.now(), + }); -function img({ - src, - alt, - noSrcText = '', - thumb: thumbKey, - reveal, - id, - class: className, - width, - height, - link = false, - lazy = false, - square = false -}) { - const willSquare = square; - const willLink = typeof link === 'string' || link; - - const originalSrc = src; - const thumbSrc = src && (thumbKey ? thumb[thumbKey](src) : src); - - const imgAttributes = html.attributes({ - id: link ? '' : id, - class: className, - alt, - width, - height - }); - - const noSrcHTML = !src && wrap(`<div class="image-text-area">${noSrcText}</div>`); - const nonlazyHTML = src && wrap(`<img src="${thumbSrc}" ${imgAttributes}>`); - const lazyHTML = src && lazy && wrap(`<img class="lazy" data-original="${thumbSrc}" ${imgAttributes}>`, true); - - if (!src) { - return noSrcHTML; - } else if (lazy) { - return fixWS` - <noscript>${nonlazyHTML}</noscript> - ${lazyHTML} - `; - } else { - return nonlazyHTML; + const regenerateMissingThumbnailCache = + cliOptions['new-thumbs'] ?? false; + + const {mediaCachePath, annotation: mediaCachePathAnnotation} = + await determineMediaCachePath({ + mediaPath, + wikiCachePath, + + providedMediaCachePath: + cliOptions['media-cache-path'] || process.env.HSMUSIC_MEDIA_CACHE, + + regenerateMissingThumbnailCache, + + disallowDoubling: + stepStatusSummary.migrateThumbnails.status === STATUS_NOT_STARTED, + }); + + if (regenerateMissingThumbnailCache) { + if ( + mediaCachePathAnnotation !== `contained path will regenerate missing cache` && + mediaCachePathAnnotation !== `adjacent path will regenerate missing cache` + ) { + if (mediaCachePath) { + logError`Determined a media cache path. (${mediaCachePathAnnotation})`; + console.error(''); + logWarn`By using ${'--new-thumbs'}, you requested to generate completely`; + logWarn`new thumbnails, but there's already a ${'thumbnail-cache.json'}`; + logWarn`file where it's expected, within this media cache:`; + logWarn`${path.resolve(mediaCachePath)}`; + console.error(''); + logWarn`If you really do want to completely regenerate all thumbnails`; + logWarn`and not reuse any existing ones, move aside ${'thumbnail-cache.json'}`; + logWarn`and run with ${'--new-thumbs'} again.`; + + Object.assign(stepStatusSummary.determineMediaCachePath, { + status: STATUS_FATAL_ERROR, + annotation: `--new-thumbs provided but regeneration not needed`, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + + return false; + } else { + logError`Couldn't determine a media cache path. (${mediaCachePathAnnotation})`; + console.error(''); + logWarn`You requested to generate completely new thumbnails, but`; + logWarn`the media cache wasn't readable or just couldn't be found.`; + logWarn`Run again without ${'--new-thumbs'} - you should investigate`; + logWarn`what's going on before continuing.`; + + Object.assign(stepStatusSummary.determineMediaCachePath, { + status: STATUS_FATAL_ERROR, + annotation: mediaCachePathAnnotation, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + + return false; + } + } + } + + if (!mediaCachePath) { + logError`Couldn't determine a media cache path. (${mediaCachePathAnnotation})`; + + switch (mediaCachePathAnnotation) { + case `contained path does not have cache`: + console.error(''); + logError`You've provided a ${'--cache-path'} or ${'HSMUSIC_CACHE_PATH'},`; + logError`${path.resolve(wikiCachePath)}`; + console.error(''); + logError`It contains a ${'media-cache'} folder, but this folder is`; + logError`missing its ${'thumbnail-cache.json'} file. This means there's`; + logError`no information available to reuse. If you use this cache,`; + logError`hsmusic will generate any existing thumbnails over again.`; + console.error(''); + logError`* Try to see if you can recover or locate a copy of your`; + logError` ${'thumbnail-cache.json'} file and put it back in place;`; + logError`* Or, generate all-new thumbnails with ${'--new-thumbs'}.`; + break; + + case 'adjacent path does not have cache': + console.error(''); + logError`You have an existing ${'media-cache'} folder next to your media path,`; + logError`${path.resolve(mediaPath)}`; + console.error(''); + logError`The ${'media-cache'} folder is missing its ${'thumbnail-cache.json'}`; + logError`file. This means there's no information available to reuse,`; + logError`and if you use this cache, hsmusic will generate any existing`; + logError`thumbnails over again.`; + console.error(''); + logError`* Try to see if you can recover or locate a copy of your`; + logError` ${'thumbnail-cache.json'} file and put it back in place;`; + logError`* Or, generate all-new thumbnails with ${'--new-thumbs'}.`; + break; + + case `contained path not readable`: + case `adjacent path not readable`: + console.error(''); + logError`The folder couldn't be read, which usually indicates`; + logError`a permissions error. Try to resolve this, or provide`; + logError`a new path with ${'--media-cache-path'} or ${'HSMUSIC_MEDIA_CACHE'}.`; + break; + + case `media path not provided`: /* unreachable */ + console.error(''); + logError`It seems a ${'--media-path'} (or ${'HSMUSIC_MEDIA'}) wasn't provided.`; + logError`Make sure one of these is actually pointing to a path that exists.`; + break; + + case `cache path not provided`: /* unreachable */ + console.error(''); + logError`It seems a ${'--cache-path'} (or ${'HSMUSIC_CACHE'}) wasn't provided.`; + logError`Make sure one of these is actually pointing to a path that exists.`; + break; } - function wrap(input, hide = false) { - let wrapped = input; + Object.assign(stepStatusSummary.determineMediaCachePath, { + status: STATUS_FATAL_ERROR, + annotation: mediaCachePathAnnotation, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); - wrapped = `<div class="image-inner-area">${wrapped}</div>`; - wrapped = `<div class="image-container">${wrapped}</div>`; + return false; + } - if (reveal) { - wrapped = fixWS` - <div class="reveal"> - ${wrapped} - <span class="reveal-text">${reveal}</span> - </div> - `; - } + logInfo`Using media cache at: ${mediaCachePath} (${mediaCachePathAnnotation})`; - if (willSquare) { - wrapped = html.tag('div', {class: 'square-content'}, wrapped); - wrapped = html.tag('div', {class: ['square', hide && !willLink && 'js-hide']}, wrapped); - } + Object.assign(stepStatusSummary.determineMediaCachePath, { + status: STATUS_DONE_CLEAN, + annotation: mediaCachePathAnnotation, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); - if (willLink) { - wrapped = html.tag('a', { - id, - class: ['box', hide && 'js-hide'], - href: typeof link === 'string' ? link : originalSrc - }, wrapped); - } + if (stepStatusSummary.migrateThumbnails.status === STATUS_NOT_STARTED) { + Object.assign(stepStatusSummary.migrateThumbnails, { + status: STATUS_STARTED_NOT_DONE, + timeStart: Date.now(), + }); + + const result = await migrateThumbsIntoDedicatedCacheDirectory({ + mediaPath, + mediaCachePath, + queueSize, + }); - return wrapped; + if (result.succses) { + Object.assign(stepStatusSummary.migrateThumbnails, { + status: STATUS_FATAL_ERROR, + annotation: `view log for details`, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + + return false; } -} -function validateWritePath(path, urlGroup) { - if (!Array.isArray(path)) { - return {error: `Expected array, got ${path}`}; + logInfo`Good to go! Run hsmusic again without ${'--migrate-thumbs'} to start`; + logInfo`using the migrated media cache.`; + + Object.assign(stepStatusSummary.migrateThumbnails, { + status: STATUS_DONE_CLEAN, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + + return true; + } + + const niceShowAggregate = (error, ...opts) => { + showAggregate(error, { + showTraces: showAggregateTraces, + pathToFileURL: (f) => path.relative(__dirname, fileURLToPath(f)), + ...opts, + }); + }; + + if ( + stepStatusSummary.loadOfflineThumbnailCache.status === STATUS_NOT_STARTED && + stepStatusSummary.generateThumbnails.status === STATUS_NOT_STARTED + ) { + throw new Error(`Unable to continue with both loadOfflineThumbnailCache and generateThumbnails`); + } + + let thumbsCache; + + // TODO: Skip this step if we're using online thumbs + if (stepStatusSummary.loadOfflineThumbnailCache.status === STATUS_NOT_STARTED) { + Object.assign(stepStatusSummary.loadOfflineThumbnailCache, { + status: STATUS_STARTED_NOT_DONE, + timeStart: Date.now(), + }); + + const thumbsCachePath = path.join(mediaCachePath, thumbsCacheFile); + + try { + thumbsCache = JSON.parse(await readFile(thumbsCachePath)); + } catch (error) { + if (error.code === 'ENOENT') { + logError`The thumbnail cache doesn't exist, and it's necessary to build` + logError`the website. Please run once without ${'--skip-thumbs'} - after` + logError`that you'll be good to go and don't need to process thumbnails` + logError`again!`; + + Object.assign(stepStatusSummary.loadOfflineThumbnailCache, { + status: STATUS_FATAL_ERROR, + annotation: `cache does not exist`, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + + return false; + } else { + logError`Malformed or unreadable thumbnail cache file: ${error}`; + logError`Path: ${thumbsCachePath}`; + logError`The thumbbnail cache is necessary to build the site, so you'll`; + logError`have to investigate this to get the build working. Try running`; + logError`again without ${'--skip-thumbs'}. If you can't get it working,`; + logError`you're welcome to message in the HSMusic Discord and we'll try`; + logError`to help you out with troubleshooting!`; + logError`${'https://hsmusic.wiki/discord/'}`; + + Object.assign(stepStatusSummary.loadOfflineThumbnailCache, { + status: STATUS_FATAL_ERROR, + annotation: `cache malformed or unreadable`, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + + return false; + } } - const { paths } = urlGroup; + logInfo`Thumbnail cache file successfully read.`; + + Object.assign(stepStatusSummary.loadOfflineThumbnailCache, { + status: STATUS_DONE_CLEAN, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + + logInfo`Skipping thumbnail generation.`; + } else if (stepStatusSummary.generateThumbnails.status === STATUS_NOT_STARTED) { + Object.assign(stepStatusSummary.generateThumbnails, { + status: STATUS_STARTED_NOT_DONE, + timeStart: Date.now(), + }); + + logInfo`Begin thumbnail generation... -----+`; - const definedKeys = Object.keys(paths); - const specifiedKey = path[0]; + const result = await genThumbs({ + mediaPath, + mediaCachePath, - if (!definedKeys.includes(specifiedKey)) { - return {error: `Specified key ${specifiedKey} isn't defined`}; + queueSize, + magickThreads, + quiet: !thumbsOnly, + }); + + logInfo`Done thumbnail generation! --------+`; + + if (!result.success) { + Object.assign(stepStatusSummary.generateThumbnails, { + status: STATUS_FATAL_ERROR, + annotation: `view log for details`, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + + return false; } - const expectedArgs = paths[specifiedKey].match(/<>/g)?.length ?? 0; - const specifiedArgs = path.length - 1; + Object.assign(stepStatusSummary.generateThumbnails, { + status: STATUS_DONE_CLEAN, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); - if (specifiedArgs !== expectedArgs) { - return {error: `Expected ${expectedArgs} arguments, got ${specifiedArgs}`}; + if (thumbsOnly) { + return true; } - return {success: true}; -} + thumbsCache = result.cache; + } else { + thumbsCache = {}; + } + + Object.assign(stepStatusSummary.loadDataFiles, { + status: STATUS_STARTED_NOT_DONE, + timeStart: Date.now(), + }); + + let yamlDataSteps; + let yamlDocumentProcessingAggregate; -function validateWriteObject(obj) { - if (typeof obj !== 'object') { - return {error: `Expected object, got ${typeof obj}`}; + { + const whoops = (error, stage) => { + if (!paragraph) console.log(''); + + console.error(error); + niceShowAggregate(error); + + logError`There was a JavaScript error ${stage}.`; + fileIssue(); + + Object.assign(stepStatusSummary.loadDataFiles, { + status: STATUS_FATAL_ERROR, + annotation: `javascript error - view log for details`, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + + return false; + }; + + let loadAggregate, loadResult; + let processAggregate, processResult; + let saveAggregate, saveResult; + + const dataSteps = getAllDataSteps(); + + try { + ({aggregate: loadAggregate, result: loadResult} = + await loadYAMLDocumentsFromDataSteps( + dataSteps, + {dataPath})); + } catch (error) { + return whoops(error, `loading data files`); } - if (typeof obj.type !== 'string') { - return {error: `Expected type to be string, got ${obj.type}`}; + try { + loadAggregate.close(); + } catch (error) { + if (!paragraph) console.log(''); + niceShowAggregate(error); + + logError`The above errors were detected while loading data files.`; + logError`Since this indicates some files weren't able to load at all,`; + logError`there would probably be pretty bad reference errors if the`; + logError`build were to continue. Please resolve these errors and`; + logError`then give it another go.`; + + paragraph = true; + console.log(''); + + Object.assign(stepStatusSummary.loadDataFiles, { + status: STATUS_FATAL_ERROR, + annotation: `error loading data files`, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + + return false; } - switch (obj.type) { - case 'legacy': { - if (typeof obj.write !== 'function') { - return {error: `Expected write to be string, got ${obj.write}`}; - } + try { + ({aggregate: processAggregate, result: processResult} = + await processThingsFromDataSteps( + loadResult.documentLists, + loadResult.fileLists, + dataSteps, + {dataPath})); + } catch (error) { + return whoops(error, `processing data files`); + } - break; - } + try { + ({aggregate: saveAggregate, result: saveResult} = + saveThingsFromDataSteps( + processResult, + dataSteps)); - case 'page': { - const path = validateWritePath(obj.path, urlSpec.localized); - if (path.error) { - return {error: `Path validation failed: ${path.error}`}; - } + saveAggregate.close(); + saveAggregate = undefined; + } catch (error) { + return whoops(error, `finalizing data files`); + } - if (typeof obj.page !== 'function') { - return {error: `Expected page to be function, got ${obj.content}`}; - } + yamlDataSteps = dataSteps; + yamlDocumentProcessingAggregate = processAggregate; - break; - } + Object.assign(wikiData, saveResult); + } - case 'data': { - const path = validateWritePath(obj.path, urlSpec.data); - if (path.error) { - return {error: `Path validation failed: ${path.error}`}; - } + { + const logThings = (prop, label) => { + const array = + (Array.isArray(prop) + ? prop + : wikiData[prop]); - if (typeof obj.data !== 'function') { - return {error: `Expected data to be function, got ${obj.data}`}; - } + if (array && empty(array)) { + return; + } - break; - } + logInfo` - ${array?.length ?? colors.red('(Missing!)')} ${colors.normal(colors.dim(label))}`; + } - case 'redirect': { - const fromPath = validateWritePath(obj.fromPath, urlSpec.localized); - if (fromPath.error) { - return {error: `Path (fromPath) validation failed: ${fromPath.error}`}; - } + try { + if (!paragraph) console.log(''); + + logInfo`Loaded data and processed objects:`; + logThings('albumData', 'albums'); + logThings('trackData', 'tracks'); + logThings( + (wikiData.artistData + ? wikiData.artistData.filter(artist => !artist.isAlias) + : null), + 'artists'); + if (wikiData.flashData) { + logThings('flashData', 'flashes'); + logThings('flashActData', 'flash acts'); + logThings('flashSideData', 'flash sides'); + } + logThings('groupData', 'groups'); + logThings('groupCategoryData', 'group categories'); + logThings('artTagData', 'art tags'); + if (wikiData.newsData) { + logThings('newsData', 'news entries'); + } + logThings('staticPageData', 'static pages'); + logThings('sortingRules', 'sorting rules'); + if (wikiData.homepageLayout) { + logInfo` - ${1} homepage layout (${ + wikiData.homepageLayout.sections.length + } sections, ${ + wikiData.homepageLayout.sections + .flatMap(section => section.rows) + .length + } rows)`; + } + if (wikiData.wikiInfo) { + logInfo` - ${1} wiki config file`; + } + + console.log(''); + paragraph = true; + } catch (error) { + console.error(`Error showing data summary:`, error); + paragraph = false; + } - const toPath = validateWritePath(obj.toPath, urlSpec.localized); - if (toPath.error) { - return {error: `Path (toPath) validation failed: ${toPath.error}`}; - } + let errorless = true; + try { + yamlDocumentProcessingAggregate.close(); + } catch (error) { + if (!paragraph) console.log(''); + niceShowAggregate(error); - if (typeof obj.title !== 'function') { - return {error: `Expected title to be function, got ${obj.title}`}; - } + logWarn`The above errors were detected while processing data files.`; - break; - } + errorless = false; + } - default: { - return {error: `Unknown type: ${obj.type}`}; - } + if (!wikiData.wikiInfo) { + logError`Can't proceed without wiki info file successfully loading.`; + + Object.assign(stepStatusSummary.loadDataFiles, { + status: STATUS_FATAL_ERROR, + annotation: `wiki info object not available`, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + + return false; } - return {success: true}; -} + if (errorless) { + logInfo`All data files processed without any errors - nice!`; + paragraph = false; -/* -async function writeData(subKey, directory, data) { - const paths = writePage.paths('', 'data.' + subKey, directory, {file: 'data.json'}); - await writePage.write(JSON.stringify(data), {paths}); -} -*/ - -// This used to 8e a function! It's long 8een divided into multiple helper -// functions, and nowadays we just directly access those, rather than ever -// touching the original one (which had contained everything). -const writePage = {}; - -writePage.to = ({ - baseDirectory, - pageSubKey, - paths -}) => (targetFullKey, ...args) => { - const [ groupKey, subKey ] = targetFullKey.split('.'); - let path = paths.subdirectoryPrefix; - - let from; - let to; - - // When linking to *outside* the localized area of the site, we need to - // make sure the result is correctly relative to the 8ase directory. - if (groupKey !== 'localized' && groupKey !== 'localizedDefaultLanguage' && baseDirectory) { - from = 'localizedWithBaseDirectory.' + pageSubKey; - to = targetFullKey; - } else if (groupKey === 'localizedDefaultLanguage' && baseDirectory) { - // Special case for specifically linking *from* a page with base - // directory *to* a page without! Used for the language switcher and - // hopefully nothing else oh god. - from = 'localizedWithBaseDirectory.' + pageSubKey; - to = 'localized.' + subKey; - } else if (groupKey === 'localizedDefaultLanguage') { - // Linking to the default, except surprise, we're already IN the default - // (no baseDirectory set). - from = 'localized.' + pageSubKey; - to = 'localized.' + subKey; + Object.assign(stepStatusSummary.loadDataFiles, { + status: STATUS_DONE_CLEAN, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); } else { - // If we're linking inside the localized area (or there just is no - // 8ase directory), the 8ase directory doesn't matter. - from = 'localized.' + pageSubKey; - to = targetFullKey; + logWarn`This might indicate some fields in the YAML data weren't formatted`; + logWarn`correctly, for example. The build should still work, but invalid`; + logWarn`fields will be skipped. Take a look at the report above to see`; + logWarn`what needs fixing up, for a more complete build!`; + + console.log(''); + paragraph = true; + + Object.assign(stepStatusSummary.loadDataFiles, { + status: STATUS_HAS_WARNINGS, + annotation: `view log for details`, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); } + } - path += urls.from(from).to(to, ...args); + // Link data arrays so that all essential references between objects are + // complete, so properties (like dates!) are inherited where that's + // appropriate. - return path; -}; + Object.assign(stepStatusSummary.linkWikiDataArrays, { + status: STATUS_STARTED_NOT_DONE, + timeStart: Date.now(), + }); -writePage.html = (pageInfo, { - defaultLanguage, - language, - languages, - localizedPaths, - paths, - oEmbedJSONHref, - to, - transformMultiline, - wikiData -}) => { - const { wikiInfo } = wikiData; - - let { - title = '', - meta = {}, - theme = '', - stylesheet = '', - - showWikiNameInTitle = true, - - // missing properties are auto-filled, see below! - body = {}, - banner = {}, - main = {}, - sidebarLeft = {}, - sidebarRight = {}, - nav = {}, - footer = {}, - socialEmbed = {}, - } = pageInfo; - - body.style ??= ''; - - theme = theme || getThemeString(wikiInfo.color); - - banner ||= {}; - banner.classes ??= []; - banner.src ??= ''; - banner.position ??= ''; - banner.dimensions ??= [0, 0]; - - main.classes ??= []; - main.content ??= ''; - - sidebarLeft ??= {}; - sidebarRight ??= {}; - - for (const sidebar of [sidebarLeft, sidebarRight]) { - sidebar.classes ??= []; - sidebar.content ??= ''; - sidebar.collapse ??= true; - } - - nav.classes ??= []; - nav.content ??= ''; - nav.links ??= []; - - footer.classes ??= []; - footer.content ??= (wikiInfo.footerContent ? transformMultiline(wikiInfo.footerContent) : ''); - - footer.content += '\n' + getFooterLocalizationLinks(paths.pathname, { - defaultLanguage, languages, paths, language, to - }); - - const canonical = (wikiInfo.canonicalBase - ? wikiInfo.canonicalBase + (paths.pathname === '/' ? '' : paths.pathname) - : ''); - - const localizedCanonical = (wikiInfo.canonicalBase - ? Object.entries(localizedPaths).map(([ code, { pathname } ]) => ({ - lang: code, - href: wikiInfo.canonicalBase + (pathname === '/' ? '' : pathname) - })) - : []); - - const collapseSidebars = (sidebarLeft.collapse !== false) && (sidebarRight.collapse !== false); - - const mainHTML = main.content && html.tag('main', { - id: 'content', - class: main.classes - }, main.content); - - const footerHTML = footer.content && html.tag('footer', { - id: 'footer', - class: footer.classes - }, footer.content); - - const generateSidebarHTML = (id, { - content, - multiple, - classes, - collapse = true, - wide = false - }) => (content - ? html.tag('div', - {id, class: [ - 'sidebar-column', - 'sidebar', - wide && 'wide', - !collapse && 'no-hide', - ...classes - ]}, - content) - : multiple ? html.tag('div', - {id, class: [ - 'sidebar-column', - 'sidebar-multiple', - wide && 'wide', - !collapse && 'no-hide' - ]}, - multiple.map(content => html.tag('div', - {class: ['sidebar', ...classes]}, - content))) - : ''); - - const sidebarLeftHTML = generateSidebarHTML('sidebar-left', sidebarLeft); - const sidebarRightHTML = generateSidebarHTML('sidebar-right', sidebarRight); - - if (nav.simple) { - nav.links = [ - {toHome: true}, - {toCurrentPage: true} - ]; - } - - const links = (nav.links || []).filter(Boolean); - - const navLinkParts = []; - for (let i = 0; i < links.length; i++) { - let cur = links[i]; - const prev = links[i - 1]; - const next = links[i + 1]; - - let { title: linkTitle } = cur; - - if (cur.toHome) { - linkTitle ??= wikiInfo.nameShort; - } else if (cur.toCurrentPage) { - linkTitle ??= title; - } + linkWikiDataArrays(wikiData, {bindFind, bindReverse}); - let part = prev && (cur.divider ?? true) ? '/ ' : ''; + Object.assign(stepStatusSummary.linkWikiDataArrays, { + status: STATUS_DONE_CLEAN, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); - if (typeof cur.html === 'string') { - if (!cur.html) { - logWarn`Empty HTML in nav link ${JSON.stringify(cur)}`; - } - part += `<span>${cur.html}</span>`; - } else { - const attributes = { - class: (cur.toCurrentPage || i === links.length - 1) && 'current', - href: ( - cur.toCurrentPage ? '' : - cur.toHome ? to('localized.home') : - cur.path ? to(...cur.path) : - cur.href ? (() => { - logWarn`Using legacy href format nav link in ${paths.pathname}`; - return cur.href; - })() : - null) - }; - if (attributes.href === null) { - throw new Error(`Expected some href specifier for link to ${linkTitle} (${JSON.stringify(cur)})`); - } - part += html.tag('a', attributes, linkTitle); - } - navLinkParts.push(part); - } - - const navHTML = html.tag('nav', { - [html.onlyIfContent]: true, - id: 'header', - class: nav.classes - }, [ - links.length && html.tag('h2', {class: 'highlight-last-link'}, navLinkParts), - nav.content - ]); - - const bannerSrc = ( - banner.src ? banner.src : - banner.path ? to(...banner.path) : - null); - - const bannerHTML = banner.position && bannerSrc && html.tag('div', - { - id: 'banner', - class: banner.classes - }, - html.tag('img', { - src: bannerSrc, - alt: banner.alt, - width: banner.dimensions[0] || 1100, - height: banner.dimensions[1] || 200 - }) - ); - - const layoutHTML = [ - navHTML, - banner.position === 'top' && bannerHTML, - html.tag('div', - {class: ['layout-columns', !collapseSidebars && 'vertical-when-thin']}, - [ - sidebarLeftHTML, - mainHTML, - sidebarRightHTML - ]), - banner.position === 'bottom' && bannerHTML, - footerHTML - ].filter(Boolean).join('\n'); - - const infoCardHTML = fixWS` - <div id="info-card-container"> - <div class="info-card-decor"> - <div class="info-card"> - <div class="info-card-art-container no-reveal"> - ${img({ - class: 'info-card-art', - src: '', - link: true, - square: true - })} - </div> - <div class="info-card-art-container reveal"> - ${img({ - class: 'info-card-art', - src: '', - link: true, - square: true, - reveal: getRevealStringFromWarnings('<span class="info-card-art-warnings"></span>', {language}) - })} - </div> - <h1 class="info-card-name"><a></a></h1> - <p class="info-card-album">${language.$('releaseInfo.from', {album: '<a></a>'})}</p> - <p class="info-card-artists">${language.$('releaseInfo.by', {artists: '<span></span>'})}</p> - <p class="info-card-cover-artists">${language.$('releaseInfo.coverArtBy', {artists: '<span></span>'})}</p> - </div> - </div> - </div> - `; - - const socialEmbedHTML = [ - socialEmbed.title && html.tag('meta', {property: 'og:title', content: socialEmbed.title}), - socialEmbed.description && html.tag('meta', {property: 'og:description', content: socialEmbed.description}), - socialEmbed.image && html.tag('meta', {property: 'og:image', content: socialEmbed.image}), - socialEmbed.color && html.tag('meta', {name: 'theme-color', content: socialEmbed.color}), - oEmbedJSONHref && html.tag('link', {type: 'application/json+oembed', href: oEmbedJSONHref}), - ].filter(Boolean).join('\n'); - - return filterEmptyLines(fixWS` - <!DOCTYPE html> - <html ${html.attributes({ - lang: language.intlCode, - 'data-language-code': language.code, - 'data-url-key': paths.toPath[0], - ...Object.fromEntries(paths.toPath.slice(1).map((v, i) => [['data-url-value' + i], v])), - 'data-rebase-localized': to('localized.root'), - 'data-rebase-shared': to('shared.root'), - 'data-rebase-media': to('media.root'), - 'data-rebase-data': to('data.root') - })}> - <head> - <title>${(showWikiNameInTitle - ? language.formatString('misc.pageTitle.withWikiName', { - title, - wikiName: wikiInfo.nameShort - }) - : language.formatString('misc.pageTitle', {title}))}</title> - <meta charset="utf-8"> - <meta name="viewport" content="width=device-width, initial-scale=1"> - ${Object.entries(meta).filter(([ key, value ]) => value).map(([ key, value ]) => `<meta ${key}="${html.escapeAttributeValue(value)}">`).join('\n')} - ${canonical && `<link rel="canonical" href="${canonical}">`} - ${localizedCanonical.map(({ lang, href }) => `<link rel="alternate" hreflang="${lang}" href="${href}">`).join('\n')} - ${socialEmbedHTML} - <link rel="stylesheet" href="${to('shared.staticFile', `site.css?${CACHEBUST}`)}"> - ${(theme || stylesheet) && fixWS` - <style> - ${theme} - ${stylesheet} - </style> - `} - <script src="${to('shared.staticFile', `lazy-loading.js?${CACHEBUST}`)}"></script> - </head> - <body ${html.attributes({style: body.style || ''})}> - <div id="page-container"> - ${mainHTML && fixWS` - <div id="skippers"> - ${[ - ['#content', language.$('misc.skippers.skipToContent')], - sidebarLeftHTML && ['#sidebar-left', (sidebarRightHTML - ? language.$('misc.skippers.skipToSidebar.left') - : language.$('misc.skippers.skipToSidebar'))], - sidebarRightHTML && ['#sidebar-right', (sidebarLeftHTML - ? language.$('misc.skippers.skipToSidebar.right') - : language.$('misc.skippers.skipToSidebar'))], - footerHTML && ['#footer', language.$('misc.skippers.skipToFooter')] - ].filter(Boolean).map(([ href, title ]) => fixWS` - <span class="skipper"><a href="${href}">${title}</a></span> - `).join('\n')} - </div> - `} - ${layoutHTML} - </div> - ${infoCardHTML} - <script type="module" src="${to('shared.staticFile', `client.js?${CACHEBUST}`)}"></script> - </body> - </html> - `); -}; - -writePage.oEmbedJSON = (pageInfo, { - language, - wikiData, -}) => { - const { socialEmbed } = pageInfo; - const { wikiInfo } = wikiData; - const { canonicalBase, nameShort } = wikiInfo; - - if (!socialEmbed) return ''; - - const entries = [ - socialEmbed.heading && ['author_name', - language.$('misc.socialEmbed.heading', { - wikiName: nameShort, - heading: socialEmbed.heading - })], - socialEmbed.headingLink && canonicalBase && ['author_url', - canonicalBase.replace(/\/$/, '') + '/' + - socialEmbed.headingLink.replace(/^\//, '')], - ].filter(Boolean); - - if (!entries.length) return ''; - - return JSON.stringify(Object.fromEntries(entries)); -}; - -writePage.write = async ({ - html, - oEmbedJSON = '', - paths, -}) => { - await mkdir(paths.outputDirectory, {recursive: true}); - await Promise.all([ - writeFile(paths.outputFile, html), - oEmbedJSON && writeFile(paths.oEmbedJSONFile, oEmbedJSON) - ].filter(Boolean)); -}; - -// TODO: This only supports one <>-style argument. -writePage.paths = (baseDirectory, fullKey, directory = '', { - file = 'index.html' -} = {}) => { - const [ groupKey, subKey ] = fullKey.split('.'); - - const pathname = (groupKey === 'localized' && baseDirectory - ? urls.from('shared.root').toDevice('localizedWithBaseDirectory.' + subKey, baseDirectory, directory) - : urls.from('shared.root').toDevice(fullKey, directory)); - - // Needed for the rare directory which itself contains a slash, e.g. for - // listings, with directories like 'albums/by-name'. - const subdirectoryPrefix = '../'.repeat(directory.split('/').length - 1); - - const outputDirectory = path.join(outputPath, pathname); - const outputFile = path.join(outputDirectory, file); - const oEmbedJSONFile = path.join(outputDirectory, OEMBED_JSON_FILE); - - return { - toPath: [fullKey, directory], - pathname, - subdirectoryPrefix, - outputDirectory, outputFile, - oEmbedJSONFile, + if (precacheMode === 'common') { + Object.assign(stepStatusSummary.precacheCommonData, { + status: STATUS_STARTED_NOT_DONE, + timeStart: Date.now(), + }); + + const commonDataMap = { + albumData: new Set([ + // Needed for sorting + 'date', 'tracks', + // Needed for computing page paths + 'aliasedArtist', 'commentary', 'coverArtistContribs', + ]), + + artTagData: new Set([ + // Needed for computing page paths + 'isContentWarning', + ]), + + flashData: new Set([ + // Needed for sorting + 'act', 'date', + ]), + + flashActData: new Set([ + // Needed for sorting + 'flashes', + ]), + + groupData: new Set([ + // Needed for computing page paths + 'albums', + ]), + + listingSpec: new Set([ + // Needed for computing page paths + 'contentFunction', 'featureFlag', + ]), + + trackData: new Set([ + // Needed for sorting + 'album', 'date', + // Needed for computing page paths + 'commentary', 'coverArtistContribs', + ]), }; -}; -async function writeFavicon() { try { - await stat(path.join(mediaPath, FAVICON_FILE)); + for (const [wikiDataKey, properties] of Object.entries(commonDataMap)) { + const thingData = wikiData[wikiDataKey]; + const allProperties = new Set(['name', 'directory', ...properties]); + for (const thing of thingData) { + for (const property of allProperties) { + void thing[property]; + } + } + } } catch (error) { - return; + if (!paragraph) console.log(''); + niceShowAggregate(error); + console.log(''); + + logError`There was an error precaching internal data objects.`; + fileIssue(); + + Object.assign(stepStatusSummary.precacheCommonData, { + status: STATUS_FATAL_ERROR, + annotation: `see log for details`, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + + return false; + } + + Object.assign(stepStatusSummary.precacheCommonData, { + status: STATUS_DONE_CLEAN, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + } + + // Check for things with duplicate directories throughout the data, + // and halt if any are found. + + if (stepStatusSummary.reportDirectoryErrors.status === STATUS_NOT_STARTED) { + Object.assign(stepStatusSummary.reportDirectoryErrors, { + status: STATUS_STARTED_NOT_DONE, + timeStart: Date.now(), + }); + + try { + reportDirectoryErrors(wikiData, {getAllFindSpecs}); + logInfo`No duplicate directories found - nice!`; + paragraph = false; + + Object.assign(stepStatusSummary.reportDirectoryErrors, { + status: STATUS_DONE_CLEAN, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + } catch (aggregate) { + if (!paragraph) console.log(''); + niceShowAggregate(aggregate); + + logWarn`The above duplicate directories were detected while reviewing data files.`; + logWarn`Since it's impossible to automatically determine which one's directory is`; + logWarn`correct, the build can't continue. Specify unique 'Directory' fields in`; + logWarn`some or all of these data entries to resolve the errors.`; + + console.log(''); + paragraph = true; + + Object.assign(stepStatusSummary.reportDirectoryErrors, { + status: STATUS_FATAL_ERROR, + annotation: `duplicate directories found`, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + + return false; } + } + + // Check for artwork objects which have been orphaned from their things, + // and halt if any are found. + + if (stepStatusSummary.reportOrphanedArtworks.status === STATUS_NOT_STARTED) { + Object.assign(stepStatusSummary.reportOrphanedArtworks, { + status: STATUS_STARTED_NOT_DONE, + timeStart: Date.now(), + }); try { - await copyFile( - path.join(mediaPath, FAVICON_FILE), - path.join(outputPath, FAVICON_FILE) - ); + reportOrphanedArtworks(wikiData, {getAllFindSpecs}); + + Object.assign(stepStatusSummary.reportOrphanedArtworks, { + status: STATUS_DONE_CLEAN, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + } catch (aggregate) { + if (!paragraph) console.log(''); + niceShowAggregate(aggregate); + + logError`Failed to initialize artwork data connections properly.`; + fileIssue(); + + Object.assign(stepStatusSummary.reportOrphanedArtworks, { + status: STATUS_FATAL_ERROR, + annotation: `orphaned artworks found`, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + + return false; + } + } + + // Filter out any reference errors throughout the data, warning about these. + + if (stepStatusSummary.filterReferenceErrors.status === STATUS_NOT_STARTED) { + Object.assign(stepStatusSummary.filterReferenceErrors, { + status: STATUS_STARTED_NOT_DONE, + timeStart: Date.now(), + }); + + const filterReferenceErrorsAggregate = + filterReferenceErrors(wikiData, {find, bindFind}); + + try { + filterReferenceErrorsAggregate.close(); + + logInfo`All references validated without any errors - nice!`; + paragraph = false; + + Object.assign(stepStatusSummary.filterReferenceErrors, { + status: STATUS_DONE_CLEAN, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); } catch (error) { - logWarn`Failed to copy favicon! ${error.message}`; - return; + if (!paragraph) console.log(''); + niceShowAggregate(error); + + logWarn`The above errors were detected while validating references in data files.`; + logWarn`The wiki should still build, but these connections between data objects`; + logWarn`will be skipped, which might have unexpected consequences. Take a look at`; + logWarn`the report above to see what needs fixing up, for a more complete build!`; + + console.log(''); + paragraph = true; + + Object.assign(stepStatusSummary.filterReferenceErrors, { + status: STATUS_HAS_WARNINGS, + annotation: `view log for details`, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); } + } - logInfo`Copied favicon to site root.`; -} + if (stepStatusSummary.reportContentTextErrors.status === STATUS_NOT_STARTED) { + Object.assign(stepStatusSummary.reportContentTextErrors, { + status: STATUS_STARTED_NOT_DONE, + timeStart: Date.now(), + }); -function writeSymlinks() { - return progressPromiseAll('Writing site symlinks.', [ - link(path.join(__dirname, UTILITY_DIRECTORY), 'shared.utilityRoot'), - link(path.join(__dirname, STATIC_DIRECTORY), 'shared.staticRoot'), - link(mediaPath, 'media.root') - ]); - - async function link(directory, urlKey) { - const pathname = urls.from('shared.root').toDevice(urlKey); - const file = path.join(outputPath, pathname); - try { - await unlink(file); - } catch (error) { - if (error.code !== 'ENOENT') { - throw error; - } - } - try { - await symlink(path.resolve(directory), file); - } catch (error) { - if (error.code === 'EPERM') { - await symlink(path.resolve(directory), file, 'junction'); - } - } + try { + reportContentTextErrors(wikiData, {bindFind}); + + logInfo`All content text validated without any errors - nice!`; + paragraph = false; + + Object.assign(stepStatusSummary.reportContentTextErrors, { + status: STATUS_DONE_CLEAN, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + } catch (error) { + if (!paragraph) console.log(''); + niceShowAggregate(error); + + logWarn`The above errors were detected while processing content text in data files.`; + logWarn`The wiki will still build, but placeholders will be displayed in these spots.`; + logWarn`Resolve the errors for more complete output.`; + + console.log(''); + paragraph = true; + + Object.assign(stepStatusSummary.reportContentTextErrors, { + status: STATUS_HAS_WARNINGS, + annotation: `view log for details`, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); } -} + } -function writeSharedFilesAndPages({language, wikiData}) { - const { groupData, wikiInfo } = wikiData; + // Sort data arrays so that they're all in order! This may use properties + // which are only available after the initial linking. - const redirect = async (title, from, urlKey, directory) => { - const target = path.relative(from, urls.from('shared.root').to(urlKey, directory)); - const content = generateRedirectPage(title, target, {language}); - await mkdir(path.join(outputPath, from), {recursive: true}); - await writeFile(path.join(outputPath, from, 'index.html'), content); - }; + Object.assign(stepStatusSummary.sortWikiDataArrays, { + status: STATUS_STARTED_NOT_DONE, + timeStart: Date.now(), + }); - return progressPromiseAll(`Writing files & pages shared across languages.`, [ - groupData?.some(group => group.directory === 'fandom') && - redirect('Fandom - Gallery', 'albums/fandom', 'localized.groupGallery', 'fandom'), + sortWikiDataArrays(yamlDataSteps, wikiData, {bindFind, bindReverse}); - groupData?.some(group => group.directory === 'official') && - redirect('Official - Gallery', 'albums/official', 'localized.groupGallery', 'official'), + Object.assign(stepStatusSummary.sortWikiDataArrays, { + status: STATUS_DONE_CLEAN, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); - wikiInfo.enableListings && - redirect('Album Commentary', 'list/all-commentary', 'localized.commentaryIndex', ''), + if (precacheMode === 'all') { + Object.assign(stepStatusSummary.precacheAllData, { + status: STATUS_STARTED_NOT_DONE, + timeStart: Date.now(), + }); - writeFile(path.join(outputPath, 'data.json'), fixWS` - { - "albumData": ${stringifyThings(wikiData.albumData)}, - ${wikiInfo.enableFlashesAndGames && `"flashData": ${stringifyThings(wikiData.flashData)},`} - "artistData": ${stringifyThings(wikiData.artistData)} - } - `) - ].filter(Boolean)); -} + // TODO: Aggregate errors here, instead of just throwing. + progressCallAll('Caching all data values', Object.entries(wikiData) + .filter(([key]) => + key !== 'listingSpec' && + key !== 'listingTargetSpec') + .map(([key, value]) => + key === 'wikiInfo' ? [key, [value]] : + key === 'homepageLayout' ? [key, [value]] : + [key, value]) + .flatMap(([_key, things]) => things) + .map(thing => () => CacheableObject.cacheAllExposedProperties(thing))); + + Object.assign(stepStatusSummary.precacheAllData, { + status: STATUS_DONE_CLEAN, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + } -function generateRedirectPage(title, target, {language}) { - return fixWS` - <!DOCTYPE html> - <html> - <head> - <title>${language.$('redirectPage.title', {title})}</title> - <meta charset="utf-8"> - <meta http-equiv="refresh" content="0;url=${target}"> - <link rel="canonical" href="${target}"> - <link rel="stylesheet" href="static/site-basic.css"> - </head> - <body> - <main> - <h1>${language.$('redirectPage.title', {title})}</h1> - <p>${language.$('redirectPage.infoLine', { - target: `<a href="${target}">${target}</a>` - })}</p> - </main> - </body> - </html> - `; -} + if (stepStatusSummary.sortWikiDataSourceFiles.status === STATUS_NOT_STARTED) { + Object.assign(stepStatusSummary.sortWikiDataSourceFiles, { + status: STATUS_STARTED_NOT_DONE, + timeStart: Date.now(), + }); -// RIP toAnythingMan (previously getHrefOfAnythingMan), 2020-05-25<>2021-05-14. -// ........Yet the function 8reathes life anew as linkAnythingMan! ::::) -function linkAnythingMan(anythingMan, {link, wikiData, ...opts}) { - return ( - wikiData.albumData.includes(anythingMan) ? link.album(anythingMan, opts) : - wikiData.trackData.includes(anythingMan) ? link.track(anythingMan, opts) : - wikiData.flashData?.includes(anythingMan) ? link.flash(anythingMan, opts) : - 'idk bud' - ) -} + const {SortingRule} = thingConstructors; + const results = + await Array.fromAsync(SortingRule.go({dataPath, wikiData})); + + if (results.some(result => result.changed)) { + logInfo`Updated data files to satisfy sorting.`; + logInfo`Restarting automatically, since that's now needed!`; -async function processLanguageFile(file) { - const contents = await readFile(file, 'utf-8'); - const json = JSON.parse(contents); + Object.assign(stepStatusSummary.sortWikiDataSourceFiles, { + status: STATUS_DONE_CLEAN, + annotation: `changes cueing restart`, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); - const code = json['meta.languageCode']; - if (!code) { - throw new Error(`Missing language code (file: ${file})`); + return 'restart'; + } else { + logInfo`All sorting rules are satisfied. Nice!`; + paragraph = false; + + Object.assign(stepStatusSummary.sortWikiDataSourceFiles, { + status: STATUS_DONE_CLEAN, + annotation: `no changes needed`, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); } - delete json['meta.languageCode']; + } else if (stepStatusSummary.checkWikiDataSourceFileSorting.status === STATUS_NOT_STARTED) { + Object.assign(stepStatusSummary.checkWikiDataSourceFileSorting, { + status: STATUS_STARTED_NOT_DONE, + timeStart: Date.now(), + }); - const intlCode = json['meta.languageIntlCode'] ?? null; - delete json['meta.languageIntlCode']; + const {SortingRule} = thingConstructors; + const results = + await Array.fromAsync(SortingRule.go({dataPath, wikiData, dry: true})); - const name = json['meta.languageName']; - if (!name) { - throw new Error(`Missing language name (${code})`); - } - delete json['meta.languageName']; + const needed = results.filter(result => result.changed); - const hidden = json['meta.hidden'] ?? false; - delete json['meta.hidden']; + if (empty(needed)) { + logInfo`All sorting rules are satisfied. Nice!`; + paragraph = false; - if (json['meta.baseDirectory']) { - logWarn`(${code}) Language JSON still has unused meta.baseDirectory`; - delete json['meta.baseDirectory']; + Object.assign(stepStatusSummary.checkWikiDataSourceFileSorting, { + status: STATUS_DONE_CLEAN, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + } else { + logWarn`Some of this wiki's sorting rules currently aren't satisfied:`; + for (const {rule} of needed) { + logWarn`- ${rule.message}`; + } + logWarn`Run ${'hsmusic --sort'} to automatically update data files.`; + paragraph = false; + + Object.assign(stepStatusSummary.checkWikiDataSourceFileSorting, { + status: STATUS_HAS_WARNINGS, + annotation: `not all rules satisfied`, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); } + } - const language = new Language(); - language.code = code; - language.intlCode = intlCode; - language.name = name; - language.hidden = hidden; - language.escapeHTML = string => he.encode(string, {useNamedReferences: true}); - language.strings = json; - return language; -} + if (stepStatusSummary.performBuild.status === STATUS_NOT_APPLICABLE) { + displayCompositeCacheAnalysis(); -// Wrapper function for running a function once for all languages. -async function wrapLanguages(fn, {languages, writeOneLanguage = null}) { - const k = writeOneLanguage; - const languagesToRun = (k - ? {[k]: languages[k]} - : languages); + if (precacheMode === 'all') { + return true; + } + } + + Object.assign(stepStatusSummary.loadURLFiles, { + status: STATUS_STARTED_NOT_DONE, + timeStart: Date.now(), + }); + + let internalURLSpec = {}; + + try { + let aggregate; + ({aggregate, result: internalURLSpec} = + await processURLSpecFromFile(internalDefaultURLSpecFile)); + + aggregate.close(); + } catch (error) { + niceShowAggregate(error); + logError`Couldn't load internal default URL spec.`; + logError`This is required to build the wiki, so stopping here.`; + fileIssue(); + + Object.assign(stepStatusSummary.loadURLFiles, { + status: STATUS_FATAL_ERROR, + annotation: `see log for details`, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); - const entries = Object.entries(languagesToRun) - .filter(([ key ]) => key !== 'default'); + return false; + } - for (let i = 0; i < entries.length; i++) { - const [ key, language ] = entries[i]; + // We'll mutate this as we load other url spec files. + const urlSpec = structuredClone(internalURLSpec); - await fn(language, i, entries); + const allURLSpecDataFiles = + (await readdir(dataPath)) + .filter(name => + name.startsWith('urls') && + ['.json', '.yaml'].includes(path.extname(name))) + .sort() /* Just in case... */ + .map(name => path.join(dataPath, name)); + + const getURLSpecKeyFromFile = file => { + const base = path.basename(file, path.extname(file)); + if (base === 'urls') { + return base; + } else { + return base.replace(/^urls-/, ''); } -} + }; -async function main() { - Error.stackTraceLimit = Infinity; - - const WD = wikiData; - - WD.listingSpec = listingSpec; - WD.listingTargetSpec = listingTargetSpec; - - const miscOptions = await parseOptions(process.argv.slice(2), { - // Data files for the site, including flash, artist, and al8um data, - // and like a jillion other things too. Pretty much everything which - // makes an individual wiki what it is goes here! - 'data-path': { - type: 'value' - }, - - // Static media will 8e referenced in the site here! The contents are - // categorized; check out MEDIA_ALBUM_ART_DIRECTORY and other constants - // near the top of this file (upd8.js). - 'media-path': { - type: 'value' - }, - - // String files! For the most part, this is used for translating the - // site to different languages, though you can also customize strings - // for your own 8uild of the site if you'd like. Files here should all - // match the format in strings-default.json in this repository. (If a - // language file is missing any strings, the site code will fall 8ack - // to what's specified in strings-default.json.) - // - // Unlike the other options here, this one's optional - the site will - // 8uild with the default (English) strings if this path is left - // unspecified. - 'lang-path': { - type: 'value' - }, - - // This is the output directory. It's the one you'll upload online with - // rsync or whatever when you're pushing an upd8, and also the one - // you'd archive if you wanted to make a 8ackup of the whole dang - // site. Just keep in mind that the gener8ted result will contain a - // couple symlinked directories, so if you're uploading, you're pro8a8ly - // gonna want to resolve those yourself. - 'out-path': { - type: 'value' - }, - - // Thum8nail gener8tion is *usually* something you want, 8ut it can 8e - // kinda a pain to run every time, since it does necessit8te reading - // every media file at run time. Pass this to skip it. - 'skip-thumbs': { - type: 'flag' - }, - - // Or, if you *only* want to gener8te newly upd8ted thum8nails, you can - // pass this flag! It exits 8efore 8uilding the rest of the site. - 'thumbs-only': { - type: 'flag' - }, - - // Just working on data entries and not interested in actually - // generating site HTML yet? This flag will cut execution off right - // 8efore any site 8uilding actually happens. - 'no-build': { - type: 'flag' - }, - - // Only want to 8uild one language during testing? This can chop down - // 8uild times a pretty 8ig chunk! Just pass a single language code. - 'lang': { - type: 'value' - }, - - // Working without a dev server and just using file:// URLs in your we8 - // 8rowser? This will automatically append index.html to links across - // the site. Not recommended for production, since it isn't guaranteed - // 100% error-free (and index.html-style links are less pretty anyway). - 'append-index-html': { - type: 'flag' - }, - - // Want sweet, sweet trace8ack info in aggreg8te error messages? This - // will print all the juicy details (or at least the first relevant - // line) right to your output, 8ut also pro8a8ly give you a headache - // 8ecause wow that is a lot of visual noise. - 'show-traces': { - type: 'flag' - }, - - 'queue-size': { - type: 'value', - validate(size) { - if (parseInt(size) !== parseFloat(size)) return 'an integer'; - if (parseInt(size) < 0) return 'a counting number or zero'; - return true; - } - }, - queue: {alias: 'queue-size'}, - - // This option is super slow and has the potential for bugs! It puts - // CacheableObject in a mode where every instance is a Proxy which will - // keep track of invalid property accesses. - 'show-invalid-property-accesses': { - type: 'flag' - }, - - [parseOptions.handleUnknown]: () => {} - }); - - dataPath = miscOptions['data-path'] || process.env.HSMUSIC_DATA; - mediaPath = miscOptions['media-path'] || process.env.HSMUSIC_MEDIA; - langPath = miscOptions['lang-path'] || process.env.HSMUSIC_LANG; // Can 8e left unset! - outputPath = miscOptions['out-path'] || process.env.HSMUSIC_OUT; - - const writeOneLanguage = miscOptions['lang']; - - { - let errored = false; - const error = (cond, msg) => { - if (cond) { - console.error(`\x1b[31;1m${msg}\x1b[0m`); - errored = true; - } - }; - error(!dataPath, `Expected --data-path option or HSMUSIC_DATA to be set`); - error(!mediaPath, `Expected --media-path option or HSMUSIC_MEDIA to be set`); - error(!outputPath, `Expected --out-path option or HSMUSIC_OUT to be set`); - if (errored) { - return; - } + const isDefaultURLSpecFile = file => + getURLSpecKeyFromFile(file) === 'urls'; + + const overrideDefaultURLSpecFile = + allURLSpecDataFiles.find(file => isDefaultURLSpecFile(file)); + + const optionalURLSpecDataFiles = + allURLSpecDataFiles.filter(file => !isDefaultURLSpecFile(file)); + + const optionalURLSpecDataKeys = + optionalURLSpecDataFiles.map(file => getURLSpecKeyFromFile(file)); + + const selectedURLSpecDataKeys = optionalURLSpecDataKeys.slice(); + const selectedURLSpecDataFiles = optionalURLSpecDataFiles.slice(); + + const {removed: [unusedURLSpecDataKeys]} = + filterMultipleArrays( + selectedURLSpecDataKeys, + selectedURLSpecDataFiles, + (key, _file) => wantedURLSpecKeys.includes(key)); + + if (!empty(selectedURLSpecDataKeys)) { + logInfo`Using these optional URL specs: ${selectedURLSpecDataKeys.join(', ')}`; + if (!empty(unusedURLSpecDataKeys)) { + logInfo`Other available optional URL specs: ${unusedURLSpecDataKeys.join(', ')}`; } + } else if (!empty(unusedURLSpecDataKeys)) { + logInfo`Not using any optional URL specs.`; + logInfo`These are available with --urls: ${unusedURLSpecDataKeys.join(', ')}`; + } + + if (overrideDefaultURLSpecFile) { + try { + let aggregate; + let overrideDefaultURLSpec; + + ({aggregate, result: overrideDefaultURLSpec} = + await processURLSpecFromFile(overrideDefaultURLSpecFile)); + + aggregate.close(); - const appendIndexHTML = miscOptions['append-index-html'] ?? false; - if (appendIndexHTML) { - logWarn`Appending index.html to link hrefs. (Note: not recommended for production release!)`; - unbound_link.globalOptions.appendIndexHTML = true; + ({aggregate} = + applyURLSpecOverriding(overrideDefaultURLSpec, urlSpec)); + + aggregate.close(); + } catch (error) { + niceShowAggregate(error); + logError`Errors loading this data repo's ${'urls.yaml'} file.`; + logError`This provides essential overrides for this wiki,`; + logError`so stopping here. Debug the errors to continue.`; + + Object.assign(stepStatusSummary.loadURLFiles, { + status: STATUS_FATAL_ERROR, + annotation: `see log for details`, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + + return false; } + } + + const processURLSpecsAggregate = + openAggregate({message: `Errors processing URL specs`}); + + const selectedURLSpecs = + processURLSpecsAggregate.receive( + await Promise.all( + selectedURLSpecDataFiles + .map(file => processURLSpecFromFile(file)))); + + for (const selectedURLSpec of selectedURLSpecs) { + processURLSpecsAggregate.receive( + applyURLSpecOverriding(selectedURLSpec, urlSpec)); + } + + try { + processURLSpecsAggregate.close(); + } catch (error) { + niceShowAggregate(error); + logWarn`There were errors loading the optional URL specs you`; + logWarn`selected using ${'--urls'}. Since they might misfunction,`; + logWarn`debug the errors or remove the failing ones from ${'--urls'}.`; + + Object.assign(stepStatusSummary.loadURLFiles, { + status: STATUS_FATAL_ERROR, + annotation: `see log for details`, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + + return false; + } - const skipThumbs = miscOptions['skip-thumbs'] ?? false; - const thumbsOnly = miscOptions['thumbs-only'] ?? false; - const noBuild = miscOptions['no-build'] ?? false; - const showAggregateTraces = miscOptions['show-traces'] ?? false; + if (showURLSpec) { + if (!paragraph) console.log(''); - const niceShowAggregate = (error, ...opts) => { - showAggregate(error, { - showTraces: showAggregateTraces, - pathToFile: f => path.relative(__dirname, f), - ...opts + logInfo`Here's the final URL spec, via ${'--show-url-spec'}:` + console.log(urlSpec); + console.log(''); + + paragraph = true; + } + + Object.assign(stepStatusSummary.loadURLFiles, { + status: STATUS_DONE_CLEAN, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + + if (!getOrigin(urlSpec.thumb.prefix)) { + Object.assign(stepStatusSummary.loadOnlineThumbnailCache, { + status: STATUS_NOT_APPLICABLE, + annotation: `using offline thumbs`, + }); + } + + if (getOrigin(urlSpec.media.prefix)) { + Object.assign(stepStatusSummary.preloadFileSizes, { + status: STATUS_NOT_APPLICABLE, + annotation: `using online media`, + }); + } else { + Object.assign(stepStatusSummary.loadOnlineFileSizeCache, { + status: STATUS_NOT_APPLICABLE, + annotation: `using offline media`, + }); + } + + applyLocalizedWithBaseDirectory(urlSpec); + + const urls = generateURLs(urlSpec); + + if (stepStatusSummary.loadOnlineThumbnailCache.status === STATUS_NOT_STARTED) loadOnlineThumbnailCache: { + Object.assign(stepStatusSummary.loadOnlineThumbnailCache, { + status: STATUS_STARTED_NOT_DONE, + timeStart: Date.now(), + }); + + let onlineThumbsCache = null; + + const cacheFile = path.join(wikiCachePath, 'online-thumbnail-cache.json'); + + let readError = null; + let writeError = null; + + if (!cliOptions['refresh-online-thumbs']) { + try { + onlineThumbsCache = JSON.parse(await readFile(cacheFile)); + } catch (caughtError) { + readError = caughtError; + } + } + + if (onlineThumbsCache) obliterateLocalCopy: { + if (!onlineThumbsCache._urlPrefix) { + // Well, it doesn't even count. + onlineThumbsCache = null; + break obliterateLocalCopy; + } + + if (onlineThumbsCache._urlPrefix !== urlSpec.thumb.prefix) { + logInfo`Local copy of online thumbs cache is for a different prefix.`; + logInfo`It'll be downloaded and replaced, for reuse next time.`; + paragraph = false; + + onlineThumbsCache = null; + break obliterateLocalCopy; + } + + let stats; + try { + stats = await stat(cacheFile); + } catch { + logInfo`Unable to get the stats of local copy of online thumbs cache...`; + logInfo`This is really weird, since we *were* able to read it...`; + logInfo`We're just going to try writing to it and download fresh!`; + paragraph = false; + + onlineThumbsCache = null; + break obliterateLocalCopy; + } + + const delta = Date.now() - stats.mtimeMs; + const minute = 60 * 1000; + const delay = 60 * minute; + + const whenst = duration => `~${Math.ceil(duration / minute)} min`; + + if (delta < delay) { + logInfo`Online thumbs cache was downloaded recently, skipping for this build.`; + logInfo`Next scheduled is in ${whenst(delay - delta)}, or by using ${'--refresh-online-thumbs'}.`; + paragraph = false; + + Object.assign(stepStatusSummary.loadOnlineThumbnailCache, { + status: STATUS_DONE_CLEAN, + annotation: `reusing local copy, earlier than scheduled`, + timeEnd: Date.now(), + memory: process.memoryUsage(), }); - }; - if (skipThumbs && thumbsOnly) { - logInfo`Well, you've put yourself rather between a roc and a hard place, hmmmm?`; - return; + thumbsCache = onlineThumbsCache; + + break loadOnlineThumbnailCache; + } else { + logInfo`Online thumbs cache hasn't been downloaded for a little while.`; + logInfo`It'll be downloaded this build, then again in ${whenst(delay)}.`; + onlineThumbsCache = null; + paragraph = false; + } } - if (skipThumbs) { - logInfo`Skipping thumbnail generation.`; - } else { - logInfo`Begin thumbnail generation... -----+`; - const result = await genThumbs(mediaPath, {queueSize, quiet: true}); - logInfo`Done thumbnail generation! --------+`; - if (!result) return; - if (thumbsOnly) return; - } - - const showInvalidPropertyAccesses = miscOptions['show-invalid-property-accesses'] ?? false; - - if (showInvalidPropertyAccesses) { - CacheableObject.DEBUG_SLOW_TRACK_INVALID_PROPERTIES = true; - } - - const { - aggregate: processDataAggregate, - result: wikiDataResult - } = await loadAndProcessDataDocuments({dataPath}); - - Object.assign(wikiData, wikiDataResult); - - { - const logThings = (thingDataProp, label) => logInfo` - ${wikiData[thingDataProp]?.length ?? color.red('(Missing!)')} ${color.normal(color.dim(label))}`; - try { - logInfo`Loaded data and processed objects:`; - logThings('albumData', 'albums'); - logThings('trackData', 'tracks'); - logThings('artistData', 'artists'); - if (wikiData.flashData) { - logThings('flashData', 'flashes'); - logThings('flashActData', 'flash acts'); - } - logThings('groupData', 'groups'); - logThings('groupCategoryData', 'group categories'); - logThings('artTagData', 'art tags'); - if (wikiData.newsData) { - logThings('newsData', 'news entries'); - } - logThings('staticPageData', 'static pages'); - if (wikiData.homepageLayout) { - logInfo` - ${1} homepage layout (${wikiData.homepageLayout.rows.length} rows)`; - } - if (wikiData.wikiInfo) { - logInfo` - ${1} wiki config file`; - } - } catch (error) { - console.error(`Error showing data summary:`, error); - } + try { + await writeFile(cacheFile, stringifyCache(onlineThumbsCache)); + } catch (caughtError) { + writeError = caughtError; + } - let errorless = true; - try { - processDataAggregate.close(); - } catch (error) { - niceShowAggregate(error); - logWarn`The above errors were detected while processing data files.`; - logWarn`If the remaining valid data is complete enough, the wiki will`; - logWarn`still build - but all errored data will be skipped.`; - logWarn`(Resolve errors for more complete output!)`; - errorless = false; - } + if (readError && writeError && readError.code !== 'ENOENT') { + console.error(readError); + logWarn`Wasn't able to read the local copy of the`; + logWarn`online thumbs cache file...`; + console.error(writeError); + logWarn`...or write to it, either.`; + logWarn`The online thumbs cache will be downloaded`; + logWarn`for every build until you investigate this path:`; + logWarn`${cacheFile}`; + paragraph = false; + } else if (readError && readError.code === 'ENOENT' && !writeError) { + logInfo`No local copy of online thumbs cache.`; + logInfo`It'll be downloaded this time and reused next time.`; + paragraph = false; + } else if (readError && readError.code === 'ENOENT' && writeError) { + console.error(writeError); + logWarn`Doesn't look like we can write a local copy of`; + logWarn`the offline thumbs cache, at this path:`; + logWarn`${cacheFile}`; + logWarn`The online thumbs cache will be downloaded`; + logWarn`for every build until you investigate that.`; + paragraph = false; + } - if (errorless) { - logInfo`All data processed without any errors - nice!`; - logInfo`(This means all source files will be fully accounted for during page generation.)`; - } + const url = new URL(urlSpec.thumb.prefix); + url.pathname = path.posix.join(url.pathname, 'thumbnail-cache.json'); + + try { + onlineThumbsCache = await fetch(url).then(res => res.json()); + } catch (error) { + console.error(error); + logWarn`There was an error downloading the online thumbnail cache.`; + logWarn`The wiki will act as though no thumbs are available at all.`; + paragraph = false; + + Object.assign(stepStatusSummary.loadOnlineThumbnailCache, { + status: STATUS_HAS_WARNINGS, + annotation: `failed to download`, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + + onlineThumbsCache = {}; + thumbsCache = {}; + + break loadOnlineThumbnailCache; } - if (!WD.wikiInfo) { - logError`Can't proceed without wiki info file (${WIKI_INFO_FILE}) successfully loading`; - return; + onlineThumbsCache._urlPrefix = urlSpec.thumb.prefix; + + thumbsCache = onlineThumbsCache; + + if (onlineThumbsCache && !writeError) { + try { + await writeFile(cacheFile, stringifyCache(onlineThumbsCache)); + } catch (error) { + console.error(error); + logWarn`There was an error saving a local copy of the`; + logWarn`online thumbnail cache. It'll be fetched again`; + logWarn`next time.`; + paragraph = false; + + Object.assign(stepStatusSummary.loadOnlineThumbnailCache, { + status: STATUS_HAS_WARNINGS, + annotation: `failed to download`, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + + break loadOnlineThumbnailCache; + } } - let duplicateDirectoriesErrored = false; - - function filterAndShowDuplicateDirectories() { - const aggregate = filterDuplicateDirectories(wikiData); - let errorless = true; - try { - aggregate.close(); - } catch (aggregate) { - niceShowAggregate(aggregate); - logWarn`The above duplicate directories were detected while reviewing data files.`; - logWarn`Each thing listed above will been totally excempt from this build of the site!`; - logWarn`Specify unique 'Directory' fields in data entries to resolve these.`; - logWarn`${`Note:`} This will probably result in reference errors below.`; - logWarn`${`. . .`} You should fix duplicate directories first!`; - logWarn`(Resolve errors for more complete output!)`; - duplicateDirectoriesErrored = true; - errorless = false; - } - if (errorless) { - logInfo`No duplicate directories found - nice!`; - } + Object.assign(stepStatusSummary.loadOnlineThumbnailCache, { + status: STATUS_DONE_CLEAN, + timeStart: Date.now(), + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + } + + const languageReloading = + stepStatusSummary.watchLanguageFiles.status === STATUS_NOT_STARTED; + + Object.assign(stepStatusSummary.loadInternalDefaultLanguage, { + status: STATUS_STARTED_NOT_DONE, + timeStart: Date.now(), + }); + + let internalDefaultLanguage; + let internalDefaultLanguageWatcher; + + let errorLoadingInternalDefaultLanguage = false; + + if (languageReloading) { + internalDefaultLanguageWatcher = watchLanguageFile(internalDefaultStringsFile); + + try { + await new Promise((resolve, reject) => { + const watcher = internalDefaultLanguageWatcher; + + const onReady = () => { + watcher.removeListener('ready', onReady); + watcher.removeListener('error', onError); + resolve(); + }; + + const onError = error => { + watcher.removeListener('ready', onReady); + watcher.removeListener('error', onError); + watcher.close(); + reject(error); + }; + + watcher.on('ready', onReady); + watcher.on('error', onError); + }); + + internalDefaultLanguage = internalDefaultLanguageWatcher.language; + } catch (_error) { + // No need to display the error here - it's already printed by + // watchLanguageFile. + errorLoadingInternalDefaultLanguage = true; } + } else { + internalDefaultLanguageWatcher = null; - function filterAndShowReferenceErrors() { - const aggregate = filterReferenceErrors(wikiData); - let errorless = true; - try { - aggregate.close(); - } catch (error) { - niceShowAggregate(error); - logWarn`The above errors were detected while validating references in data files.`; - logWarn`If the remaining valid data is complete enough, the wiki will still build -`; - logWarn`but all errored references will be skipped.`; - if (duplicateDirectoriesErrored) { - logWarn`${`Note:`} Duplicate directories were found as well. Review those first,`; - logWarn`${`. . .`} as they may have caused some of the errors detected above.`; - } - logWarn`(Resolve errors for more complete output!)`; - errorless = false; - } - if (errorless) { - logInfo`All references validated without any errors - nice!`; - logInfo`(This means all references between things, such as leitmotif references` - logInfo` and artist credits, will be fully accounted for during page generation.)`; - } + try { + internalDefaultLanguage = await processLanguageFile(internalDefaultStringsFile); + } catch (error) { + niceShowAggregate(error); + errorLoadingInternalDefaultLanguage = true; + } + } + + if (errorLoadingInternalDefaultLanguage) { + logError`There was an error reading the internal language file.`; + fileIssue(); + + Object.assign(stepStatusSummary.loadInternalDefaultLanguage, { + status: STATUS_FATAL_ERROR, + annotation: `see log for details`, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + + return false; + } + + if (languageReloading) { + // Bypass node.js special-case handling for uncaught error events + internalDefaultLanguageWatcher.on('error', () => {}); + } + + Object.assign(stepStatusSummary.loadInternalDefaultLanguage, { + status: STATUS_DONE_CLEAN, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + + let customLanguageWatchers; + let languages; + + if (langPath) { + if (languageReloading) { + Object.assign(stepStatusSummary.watchLanguageFiles, { + status: STATUS_STARTED_NOT_DONE, + timeStart: Date.now(), + }); + } else { + Object.assign(stepStatusSummary.loadLanguageFiles, { + status: STATUS_STARTED_NOT_DONE, + timeStart: Date.now(), + }); } - // Link data arrays so that all essential references between objects are - // complete, so properties (like dates!) are inherited where that's - // appropriate. - linkWikiDataArrays(wikiData); + const languageDataFiles = + (await readdir(langPath)) + .filter(name => ['.json', '.yaml'].includes(path.extname(name))) + .map(name => path.join(langPath, name)); - // Filter out any things with duplicate directories throughout the data, - // warning about them too. - filterAndShowDuplicateDirectories(); + let errorLoadingCustomLanguages = false; - // Filter out any reference errors throughout the data, warning about them - // too. - filterAndShowReferenceErrors(); + if (languageReloading) watchCustomLanguages: { + Object.assign(stepStatusSummary.watchLanguageFiles, { + status: STATUS_STARTED_NOT_DONE, + timeStart: Date.now(), + }); - // Sort data arrays so that they're all in order! This may use properties - // which are only available after the initial linking. - sortWikiDataArrays(wikiData); + customLanguageWatchers = + languageDataFiles.map(file => { + const watcher = watchLanguageFile(file); - const internalDefaultLanguage = await processLanguageFile(path.join(__dirname, DEFAULT_STRINGS_FILE)); + // Bypass node.js special-case handling for uncaught error events + watcher.on('error', () => {}); - let languages; - if (langPath) { - const languageDataFiles = await findFiles(langPath, { - filter: f => path.extname(f) === '.json' + return watcher; }); - const results = await progressPromiseAll(`Reading & processing language files.`, languageDataFiles - .map(file => processLanguageFile(file))); + const waitingOnWatchers = new Set(customLanguageWatchers); + + const initialResults = + await Promise.allSettled( + customLanguageWatchers + .map(watcher => new Promise((resolve, reject) => { + const onReady = () => { + watcher.removeListener('ready', onReady); + watcher.removeListener('error', onError); + waitingOnWatchers.delete(watcher); + resolve(); + }; + + const onError = error => { + watcher.removeListener('ready', onReady); + watcher.removeListener('error', onError); + reject(error); + }; + + watcher.on('ready', onReady); + watcher.on('error', onError); + }))); + + if (initialResults.some(({status}) => status === 'rejected')) { + logWarn`There were errors loading custom languages from the language path`; + logWarn`provided: ${langPath}`; + + if (noInput) { + internalDefaultLanguageWatcher.close(); + + for (const watcher of Object.values(customLanguageWatchers)) { + watcher.close(); + } + + Object.assign(stepStatusSummary.watchLanguageFiles, { + status: STATUS_FATAL_ERROR, + annotation: `see log for details`, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + + errorLoadingCustomLanguages = true; + break watchCustomLanguages; + } - languages = Object.fromEntries(results.map(language => [language.code, language])); + logWarn`The build should start automatically if you investigate these.`; + logWarn`Or, exit by pressing ^C here (control+C) and run again without`; + logWarn`providing ${'--lang-path'} (or ${'HSMUSIC_LANG'}) to build without custom`; + logWarn`languages.`; + + await new Promise(resolve => { + for (const watcher of waitingOnWatchers) { + watcher.once('ready', () => { + waitingOnWatchers.remove(watcher); + if (empty(waitingOnWatchers)) { + resolve(); + } + }); + } + }); + } + + languages = + Object.fromEntries( + customLanguageWatchers + .map(({language}) => [language.code, language])); + + Object.assign(stepStatusSummary.watchLanguageFiles, { + status: STATUS_DONE_CLEAN, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); } else { - languages = {}; - } + languages = {}; - const customDefaultLanguage = languages[WD.wikiInfo.defaultLanguage ?? internalDefaultLanguage.code]; - let finalDefaultLanguage; + const results = + await Promise.allSettled( + languageDataFiles + .map(file => processLanguageFile(file))); - if (customDefaultLanguage) { - logInfo`Applying new default strings from custom ${customDefaultLanguage.code} language file.`; - customDefaultLanguage.inheritedStrings = internalDefaultLanguage.strings; - finalDefaultLanguage = customDefaultLanguage; - } else if (WD.wikiInfo.defaultLanguage) { - logError`Wiki info file specified default language is ${WD.wikiInfo.defaultLanguage}, but no such language file exists!`; - if (langPath) { - logError`Check if an appropriate file exists in ${langPath}?`; + for (const {status, value: language, reason: error} of results) { + if (status === 'rejected') { + errorLoadingCustomLanguages = true; + niceShowAggregate(error); } else { - logError`Be sure to specify ${'--lang'} or ${'HSMUSIC_LANG'} with the path to language files.`; + languages[language.code] = language; } - return; - } else { - languages[internalDefaultLanguage.code] = internalDefaultLanguage; - finalDefaultLanguage = internalDefaultLanguage; + } + + if (errorLoadingCustomLanguages) { + Object.assign(stepStatusSummary.loadLanguageFiles, { + status: STATUS_FATAL_ERROR, + annotation: `see log for details`, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + } else { + Object.assign(stepStatusSummary.loadLanguageFiles, { + status: STATUS_DONE_CLEAN, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + } + } + + if (errorLoadingCustomLanguages) { + logError`Failed to load language files. Please investigate these, or don't provide`; + logError`--lang-path (or HSMUSIC_LANG) and build again.`; + return false; + } + } else { + languages = {}; + } + + Object.assign(stepStatusSummary.initializeDefaultLanguage, { + status: STATUS_STARTED_NOT_DONE, + timeStart: Date.now(), + }); + + let finalDefaultLanguage; + let finalDefaultLanguageWatcher; + let finalDefaultLanguageAnnotation; + + if (wikiData.wikiInfo.defaultLanguage) { + const customDefaultLanguage = languages[wikiData.wikiInfo.defaultLanguage]; + + if (!customDefaultLanguage) { + logError`Wiki info file specified default language is ${wikiData.wikiInfo.defaultLanguage}, but no such language file exists!`; + if (langPath) { + logError`Check if an appropriate file exists in ${langPath}?`; + } else { + logError`Be sure to specify ${'--lang-path'} or ${'HSMUSIC_LANG'} with the path to language files.`; + } + + Object.assign(stepStatusSummary.initializeDefaultLanguage, { + status: STATUS_FATAL_ERROR, + annotation: `wiki specifies default language whose file is not available`, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + + return false; + } + + logInfo`Applying new default strings from custom ${customDefaultLanguage.code} language file.`; + paragraph = false; + + finalDefaultLanguage = customDefaultLanguage; + finalDefaultLanguageAnnotation = `using wiki-specified custom default language`; + + if (languageReloading) { + finalDefaultLanguageWatcher = + customLanguageWatchers + .find(({language}) => language === customDefaultLanguage); } + } else if (languages[internalDefaultLanguage.code]) { + const customDefaultLanguage = languages[internalDefaultLanguage.code]; + finalDefaultLanguage = customDefaultLanguage; + finalDefaultLanguageAnnotation = `using inferred custom default language`; + + if (languageReloading) { + finalDefaultLanguageWatcher = + customLanguageWatchers + .find(({language}) => language === customDefaultLanguage); + } + } else { + languages[internalDefaultLanguage.code] = internalDefaultLanguage; + + finalDefaultLanguage = internalDefaultLanguage; + finalDefaultLanguageAnnotation = `no custom default language specified`; + + if (languageReloading) { + finalDefaultLanguageWatcher = internalDefaultLanguageWatcher; + } + } + + const closeLanguageWatchers = () => { + if (languageReloading) { + for (const watcher of [ + internalDefaultLanguageWatcher, + ...customLanguageWatchers, + ]) { + watcher.close(); + } + } + }; + + const inheritStringsFromInternalLanguage = () => { + // The custom default language, if set, will be the new one providing fallback + // strings for other languages. But on its own, it still might not be a complete + // list of strings - so it falls back to the internal default language, which + // won't otherwise be presented in the build. + if (finalDefaultLanguage === internalDefaultLanguage) return; + const {strings: inheritedStrings} = internalDefaultLanguage; + Object.assign(finalDefaultLanguage, {inheritedStrings}); + }; + + const inheritStringsFromDefaultLanguage = () => { + const {strings: inheritedStrings} = finalDefaultLanguage; for (const language of Object.values(languages)) { - if (language === finalDefaultLanguage) { - continue; - } + if (language === finalDefaultLanguage) continue; + Object.assign(language, {inheritedStrings}); + } + }; - language.inheritedStrings = finalDefaultLanguage.strings; + if (finalDefaultLanguage !== internalDefaultLanguage) { + inheritStringsFromInternalLanguage(); + } + + inheritStringsFromDefaultLanguage(); + + if (languageReloading) { + if (finalDefaultLanguage !== internalDefaultLanguage) { + internalDefaultLanguageWatcher.on('update', () => { + inheritStringsFromInternalLanguage(); + inheritStringsFromDefaultLanguage(); + }); } - logInfo`Loaded language strings: ${Object.keys(languages).join(', ')}`; + finalDefaultLanguageWatcher.on('update', () => { + inheritStringsFromDefaultLanguage(); + }); + } + + logInfo`Loaded language strings: ${Object.keys(languages).join(', ')}`; + paragraph = false; + + Object.assign(stepStatusSummary.initializeDefaultLanguage, { + status: STATUS_DONE_CLEAN, + annotation: finalDefaultLanguageAnnotation, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + + let missingImagePaths; + + if (stepStatusSummary.verifyImagePaths.status === STATUS_NOT_APPLICABLE) { + missingImagePaths = []; + } else if (stepStatusSummary.verifyImagePaths.status === STATUS_NOT_STARTED) { + Object.assign(stepStatusSummary.verifyImagePaths, { + status: STATUS_STARTED_NOT_DONE, + timeStart: Date.now(), + }); - if (noBuild) { - logInfo`Not generating any site or page files this run (--no-build passed).`; - } else if (writeOneLanguage && !(writeOneLanguage in languages)) { - logError`Specified to write only ${writeOneLanguage}, but there is no strings file with this language code!`; - return; - } else if (writeOneLanguage) { - logInfo`Writing only language ${writeOneLanguage} this run.`; + const results = + await verifyImagePaths(mediaPath, {urls, wikiData}); + + missingImagePaths = results.missing; + const misplacedImagePaths = results.misplaced; + + if (empty(missingImagePaths) && empty(misplacedImagePaths)) { + Object.assign(stepStatusSummary.verifyImagePaths, { + status: STATUS_DONE_CLEAN, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + } else if (empty(missingImagePaths)) { + Object.assign(stepStatusSummary.verifyImagePaths, { + status: STATUS_HAS_WARNINGS, + annotation: `misplaced images detected`, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + } else if (empty(misplacedImagePaths)) { + Object.assign(stepStatusSummary.verifyImagePaths, { + status: STATUS_HAS_WARNINGS, + annotation: `missing images detected`, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); } else { - logInfo`Writing all languages.`; + Object.assign(stepStatusSummary.verifyImagePaths, { + status: STATUS_HAS_WARNINGS, + annotation: `missing and misplaced images detected`, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); } + } - { - const tagRefs = new Set([...WD.trackData, ...WD.albumData].flatMap(thing => thing.artTagsByRef ?? [])); + let getSizeOfMediaFile = () => null; - for (const ref of tagRefs) { - if (find.artTag(ref, WD.artTagData)) { - tagRefs.delete(ref); - } - } + const fileSizePreloader = + new FileSizePreloader({ + prefix: mediaPath, + }); - if (tagRefs.size) { - for (const ref of Array.from(tagRefs).sort()) { - console.log(`\x1b[33;1m- Missing tag: "${ref}"\x1b[0m`); - } - return; - } + if (stepStatusSummary.loadOnlineFileSizeCache.status === STATUS_NOT_STARTED) loadOnlineFileSizeCache: { + Object.assign(stepStatusSummary.loadOnlineFileSizeCache, { + status: STATUS_STARTED_NOT_DONE, + timeStart: Date.now(), + }); + + let onlineFileSizeCache = null; + + const makeFileSizeCacheAvailable = () => { + fileSizePreloader.loadFromCache(onlineFileSizeCache); + + getSizeOfMediaFile = p => + fileSizePreloader.getSizeOfPath( + path.resolve( + mediaPath, + decodeURIComponent(p).split('/').join(path.sep))); + }; + + const cacheFile = path.join(wikiCachePath, 'online-file-size-cache.json'); + + let readError = null; + let writeError = null; + + if (!cliOptions['refresh-online-file-sizes']) { + try { + onlineFileSizeCache = JSON.parse(await readFile(cacheFile)); + } catch (caughtError) { + readError = caughtError; + } } - WD.justEverythingMan = sortByDate([...WD.albumData, ...WD.trackData, ...(WD.flashData || [])]); - WD.justEverythingSortedByArtDateMan = sortByArtDate(WD.justEverythingMan.slice()); - // console.log(JSON.stringify(justEverythingSortedByArtDateMan.map(toAnythingMan), null, 2)); - - WD.officialAlbumData = WD.albumData.filter(album => album.groups.some(group => group.directory === OFFICIAL_GROUP_DIRECTORY)); - WD.fandomAlbumData = WD.albumData.filter(album => album.groups.every(group => group.directory !== OFFICIAL_GROUP_DIRECTORY)); - - const fileSizePreloader = new FileSizePreloader(); - - // File sizes of additional files need to be precalculated before we can - // actually reference 'em in site building, so get those loading right - // away. We actually need to keep track of two things here - the on-device - // file paths we're actually reading, and the corresponding on-site media - // paths that will be exposed in site build code. We'll build a mapping - // function between them so that when site code requests a site path, - // it'll get the size of the file at the corresponding device path. - const additionalFilePaths = [ - ...WD.albumData.flatMap(album => ( - [ - ...album.additionalFiles ?? [], - ...album.tracks.flatMap(track => track.additionalFiles ?? []) - ] - .flatMap(fileGroup => fileGroup.files) - .map(file => ({ - device: (path.join(mediaPath, urls - .from('media.root') - .toDevice('media.albumAdditionalFile', album.directory, file))), - media: (urls - .from('media.root') - .to('media.albumAdditionalFile', album.directory, file)) - })))), - ]; - - const getSizeOfAdditionalFile = mediaPath => { - const { device = null } = additionalFilePaths.find(({ media }) => media === mediaPath) || {}; - if (!device) return null; - return fileSizePreloader.getSizeOfPath(device); + if (onlineFileSizeCache) obliterateLocalCopy: { + if (!onlineFileSizeCache._urlPrefix) { + // Well, it doesn't even count. + onlineFileSizeCache = null; + break obliterateLocalCopy; + } + + if (onlineFileSizeCache._urlPrefix !== urlSpec.media.prefix) { + logInfo`Local copy of online file size cache is for a different prefix.`; + logInfo`It'll be downloaded and replaced, for reuse next time.`; + paragraph = false; + + onlineFileSizeCache = null; + break obliterateLocalCopy; + } + + let stats; + try { + stats = await stat(cacheFile); + } catch { + logInfo`Unable to get the stats of local copy of online file size cache...`; + logInfo`This is really weird, since we *were* able to read it...`; + logInfo`We're just going to try writing to it and download fresh!`; + paragraph = false; + + onlineFileSizeCache = null; + break obliterateLocalCopy; + } + + const delta = Date.now() - stats.mtimeMs; + const minute = 60 * 1000; + const delay = 60 * minute; + + const whenst = duration => `~${Math.ceil(duration / minute)} min`; + + if (delta < delay) { + logInfo`Online file size cache was downloaded recently, skipping for this build.`; + logInfo`Next scheduled is in ${whenst(delay - delta)}, or by using ${'--refresh-online-file-sizes'}.`; + paragraph = false; + + Object.assign(stepStatusSummary.loadOnlineFileSizeCache, { + status: STATUS_DONE_CLEAN, + annotation: `reusing local copy, earlier than scheduled`, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + + delete onlineFileSizeCache._urlPrefix; + + makeFileSizeCacheAvailable(); + + break loadOnlineFileSizeCache; + } else { + logInfo`Online file size hasn't been downloaded for a little while.`; + logInfo`It'll be downloaded this build, then again in ${whenst(delay)}.`; + onlineFileSizeCache = null; + paragraph = false; + } + } + + try { + await writeFile(cacheFile, stringifyCache(onlineFileSizeCache)); + } catch (caughtError) { + writeError = caughtError; + } + + if (readError && writeError && readError.code !== 'ENOENT') { + console.error(readError); + logWarn`Wasn't able to read the local copy of the`; + logWarn`online file size cache file...`; + console.error(writeError); + logWarn`...or write to it, either.`; + logWarn`The online file size cache will be downloaded`; + logWarn`for every build until you investigate this path:`; + logWarn`${cacheFile}`; + paragraph = false; + } else if (readError && readError.code === 'ENOENT' && !writeError) { + logInfo`No local copy of online file size cache.`; + logInfo`It'll be downloaded this time and reused next time.`; + paragraph = false; + } else if (readError && readError.code === 'ENOENT' && writeError) { + console.error(writeError); + logWarn`Doesn't look like we can write a local copy of`; + logWarn`the offline file size cache, at this path:`; + logWarn`${cacheFile}`; + logWarn`The online file size cache will be downloaded`; + logWarn`for every build until you investigate that.`; + paragraph = false; + } + + const url = new URL(urlSpec.media.prefix); + url.pathname = path.posix.join(url.pathname, 'file-size-cache.json'); + + try { + onlineFileSizeCache = await fetch(url).then(res => res.json()); + } catch (error) { + console.error(error); + logWarn`There was an error downloading the online file size cache.`; + logWarn`The wiki will act as though no file sizes are available at all.`; + paragraph = false; + + Object.assign(stepStatusSummary.loadOnlineFileSizeCache, { + status: STATUS_HAS_WARNINGS, + annotation: `failed to download`, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + + break loadOnlineFileSizeCache; + } + + makeFileSizeCacheAvailable(); + + onlineFileSizeCache._urlPrefix = urlSpec.media.prefix; + + if (onlineFileSizeCache && !writeError) { + try { + await writeFile(cacheFile, stringifyCache(onlineFileSizeCache)); + } catch (error) { + console.error(error); + logWarn`There was an error saving a local copy of the`; + logWarn`online file size cache. It'll be fetched again`; + logWarn`next time.`; + paragraph = false; + + Object.assign(stepStatusSummary.loadOnlineFileSizeCache, { + status: STATUS_HAS_WARNINGS, + annotation: `failed to download`, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + + break loadOnlineFileSizeCache; + } + } + + Object.assign(stepStatusSummary.loadOnlineFileSizeCache, { + status: STATUS_DONE_CLEAN, + timeStart: Date.now(), + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + } + + if (stepStatusSummary.preloadFileSizes.status === STATUS_NOT_STARTED) { + Object.assign(stepStatusSummary.preloadFileSizes, { + status: STATUS_STARTED_NOT_DONE, + timeStart: Date.now(), + }); + + const mediaFilePaths = + await traverse(mediaPath, { + pathStyle: 'device', + filterDir: dir => dir !== '.git', + filterFile: file => !isThumb(file), + }).then(files => files + .map(file => ({ + device: file, + media: + urls + .from('media.root') + .to('media.path', path.relative(mediaPath, file).split(path.sep).join('/')), + }))); + + getSizeOfMediaFile = mediaPath => { + const pair = mediaFilePaths.find(({media}) => media === mediaPath); + if (!pair) return null; + return fileSizePreloader.getSizeOfPath(pair.device); }; - logInfo`Preloading filesizes for ${additionalFilePaths.length} additional files...`; + logInfo`Preloading file sizes for ${mediaFilePaths.length} media files...`; - fileSizePreloader.loadPaths(...additionalFilePaths.map(path => path.device)); + fileSizePreloader.loadPaths(...mediaFilePaths.map(path => path.device)); await fileSizePreloader.waitUntilDoneLoading(); - logInfo`Done preloading filesizes!`; + if (fileSizePreloader.hasErrored) { + logWarn`Some media files couldn't be read for preloading file sizes.`; + logWarn`This means the wiki won't display file sizes for these files.`; + logWarn`Investigate missing or unreadable files to get that fixed!`; + + Object.assign(stepStatusSummary.preloadFileSizes, { + status: STATUS_HAS_WARNINGS, + annotation: `see log for details`, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + } else { + logInfo`Done preloading file sizes without any errors - nice!`; + paragraph = false; + + Object.assign(stepStatusSummary.preloadFileSizes, { + status: STATUS_DONE_CLEAN, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + } - if (noBuild) return; + // TODO: kinda jank that this is out of band of any particular step, + // even though it's operationally a follow-up to preloadFileSizes - // Makes writing a little nicer on CPU theoretically, 8ut also costs in - // performance right now 'cuz it'll w8 for file writes to 8e completed - // 8efore moving on to more data processing. So, defaults to zero, which - // disa8les the queue feature altogether. - queueSize = +(miscOptions['queue-size'] ?? 0); + let oopsCache = false; + saveFileSizeCache: { + let cache; + try { + cache = fileSizePreloader.saveAsCache(); + } catch (error) { + console.error(error); + logWarn`Couldn't compute file size preloader's cache.`; + oopsCache = true; + break saveFileSizeCache; + } - const buildDictionary = pageSpecs; + const cacheFile = path.join(mediaPath, 'file-size-cache.json'); - // NOT for ena8ling or disa8ling specific features of the site! - // This is only in charge of what general groups of files to 8uild. - // They're here to make development quicker when you're only working - // on some particular area(s) of the site rather than making changes - // across all of them. - const writeFlags = await parseOptions(process.argv.slice(2), { - all: {type: 'flag'}, // Defaults to true if none 8elow specified. + try { + await writeFile(cacheFile, stringifyCache(cache)); + } catch (error) { + console.error(error); + logWarn`Couldn't save preloaded file sizes to a cache file:`; + logWarn`${cacheFile}`; + oopsCache = true; + } + } - // Kinda a hack t8h! - ...Object.fromEntries(Object.keys(buildDictionary) - .map(key => [key, {type: 'flag'}])), + if (oopsCache) { + logWarn`This won't affect the build, but this build should not be used`; + logWarn`as a model for another build accessing its media files online.`; + } + } - [parseOptions.handleUnknown]: () => {} + if (stepStatusSummary.buildSearchIndex.status === STATUS_NOT_STARTED) { + Object.assign(stepStatusSummary.buildSearchIndex, { + status: STATUS_STARTED_NOT_DONE, + timeStart: Date.now(), }); - const writeAll = !Object.keys(writeFlags).length || writeFlags.all; + try { + await writeSearchData({ + thumbsCache, + urls, + wikiCachePath, + wikiData, + }); + + logInfo`Search data successfully written - nice!`; + paragraph = false; + + Object.assign(stepStatusSummary.buildSearchIndex, { + status: STATUS_DONE_CLEAN, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + } catch (error) { + if (!paragraph) console.log(''); + niceShowAggregate(error); + + logError`There was an error preparing or writing search data.`; + fileIssue(); + logWarn`Any existing search data will be reused, and search may be`; + logWarn`generally dysfunctional. The site should work otherwise, though!`; + + console.log(''); + paragraph = true; + + Object.assign(stepStatusSummary.buildSearchIndex, { + status: STATUS_HAS_WARNINGS, + annotation: `see log for details`, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + } + } - logInfo`Writing site pages: ${writeAll ? 'all' : Object.keys(writeFlags).join(', ')}`; + let webRouteSources = null; + let preparedWebRoutes = null; - await writeFavicon(); - await writeSymlinks(); - await writeSharedFilesAndPages({language: finalDefaultLanguage, wikiData}); + if (stepStatusSummary.identifyWebRoutes.status === STATUS_NOT_STARTED) { + Object.assign(stepStatusSummary.identifyWebRoutes, { + status: STATUS_STARTED_NOT_DONE, + timeStart: Date.now(), + }); - const buildSteps = (writeAll - ? Object.entries(buildDictionary) - : (Object.entries(buildDictionary) - .filter(([ flag ]) => writeFlags[flag]))); + const fromRoot = urls.from('shared.root'); - let writes; - { - let error = false; + try { + webRouteSources = await identifyAllWebRoutes({ + mediaCachePath, + mediaPath, + wikiCachePath, + }); + + const {aggregate, result} = + mapAggregate( + webRouteSources, + ({to, ...rest}) => ({ + ...rest, + to: fromRoot.to(...to), + }), + {message: `Errors computing effective web route paths`},); + + aggregate.close(); + preparedWebRoutes = result; + } catch (error) { + if (!paragraph) console.log(''); + niceShowAggregate(error); - const buildStepsWithTargets = buildSteps.map(([ flag, pageSpec ]) => { - // Condition not met: skip this build step altogether. - if (pageSpec.condition && !pageSpec.condition({wikiData})) { - return null; - } + logError`There was an issue identifying web routes!`; + fileIssue(); - // May still call writeTargetless if present. - if (!pageSpec.targets) { - return {flag, pageSpec, targets: []}; - } + console.log(''); + paragraph = true; - if (!pageSpec.write) { - logError`${flag + '.targets'} is specified, but ${flag + '.write'} is missing!`; - error = true; - return null; - } + Object.assign(stepStatusSummary.identifyWebRoutes, { + status: STATUS_FATAL_ERROR, + message: `JavaScript error - view log for details`, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); - const targets = pageSpec.targets({wikiData}); - if (!Array.isArray(targets)) { - logError`${flag + '.targets'} was called, but it didn't return an array! (${typeof targets})`; - error = true; - return null; - } + return false; + } - return {flag, pageSpec, targets}; - }).filter(Boolean); + logInfo`Successfully determined web routes - nice!`; + paragraph = false; - if (error) { - return; - } + Object.assign(stepStatusSummary.identifyWebRoutes, { + status: STATUS_DONE_CLEAN, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + } + + wikiData.wikiInfo.searchDataAvailable = + (webRouteSources + ? webRouteSources + .some(({to}) => to[0].startsWith('searchData')) + : null); + + if (stepStatusSummary.performBuild.status === STATUS_NOT_APPLICABLE) { + return true; + } + + const developersComment = + `<!--\n` + [ + wikiData.wikiInfo.canonicalBase + ? `hsmusic.wiki - ${wikiData.wikiInfo.name}, ${wikiData.wikiInfo.canonicalBase}` + : `hsmusic.wiki - ${wikiData.wikiInfo.name}`, + 'Code copyright 2019-2023 Quasar Nebula et al (MIT License)', + ...wikiData.wikiInfo.canonicalBase === 'https://hsmusic.wiki/' ? [ + 'Data avidly compiled and localization brought to you', + 'by our awesome team and community of wiki contributors', + '***', + 'Want to contribute? Join our Discord or leave feedback!', + '- https://hsmusic.wiki/discord/', + '- https://hsmusic.wiki/feedback/', + '- https://github.com/hsmusic/', + ] : [ + 'https://github.com/hsmusic/', + ], + '***', + BUILD_TIME && + `Site built: ${BUILD_TIME.toLocaleString('en-US', { + dateStyle: 'long', + timeStyle: 'long', + })}`, + COMMIT && + `Latest code commit: ${COMMIT}`, + ] + .filter(Boolean) + .map(line => ` ` + line) + .join('\n') + `\n-->`; + + Object.assign(stepStatusSummary.performBuild, { + status: STATUS_STARTED_NOT_DONE, + timeStart: Date.now(), + }); + + let buildModeResult; + + logInfo`Passing control over to build mode: ${selectedBuildModeFlag}`; + console.log(''); + + const universalUtilities = { + getSizeOfMediaFile, + + defaultLanguage: finalDefaultLanguage, + developersComment, + languages, + missingImagePaths, + thumbsCache, + urlSpec, + urls, + wikiData, + }; - const validateWrites = (writes, fnName) => { - // Do a quick valid8tion! If one of the writeThingPages functions go - // wrong, this will stall out early and tell us which did. - - if (!Array.isArray(writes)) { - logError`${fnName} didn't return an array!`; - error = true; - return false; - } - - if (!( - writes.every(obj => typeof obj === 'object') && - writes.every(obj => { - const result = validateWriteObject(obj); - if (result.error) { - logError`Validating write object failed: ${result.error}`; - return false; - } else { - return true; - } - }) - )) { - logError`${fnName} returned invalid entries!`; - error = true; - return false; - } - - return true; - }; + try { + buildModeResult = await selectedBuildMode.go({ + cliOptions, + queueSize, + + universalUtilities, + ...universalUtilities, + + dataPath, + mediaPath, + mediaCachePath, + wikiCachePath, + srcRootPath: __dirname, - // return; + webRoutes: preparedWebRoutes, - writes = buildStepsWithTargets.flatMap(({ flag, pageSpec, targets }) => { - const writes = targets.flatMap(target => - pageSpec.write(target, {wikiData})?.slice() || []); + closeLanguageWatchers, + niceShowAggregate, + }); + } catch (error) { + console.error(error); - if (!validateWrites(writes, flag + '.write')) { - return []; - } + logError`There was a JavaScript error performing the build.`; + fileIssue(); - if (pageSpec.writeTargetless) { - const writes2 = pageSpec.writeTargetless({wikiData}); + Object.assign(stepStatusSummary.performBuild, { + status: STATUS_FATAL_ERROR, + message: `javascript error - view log for details`, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); - if (!validateWrites(writes2, flag + '.writeTargetless')) { - return []; - } + return false; + } - writes.push(...writes2); - } + if (buildModeResult !== true) { + Object.assign(stepStatusSummary.performBuild, { + status: STATUS_HAS_WARNINGS, + annotation: `may not have completed - view log for details`, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); - return writes; - }); + return false; + } - if (error) { - return; + Object.assign(stepStatusSummary.performBuild, { + status: STATUS_DONE_CLEAN, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + + return true; +} + +// TODO: isMain detection isn't consistent across platforms here +/* eslint-disable-next-line no-constant-condition */ +if (true || isMain(import.meta.url) || path.basename(process.argv[1]) === 'hsmusic') { + (async () => { + let result; + let numRestarts = 0; + + const totalTimeStart = Date.now(); + + while (true) { + try { + result = await main(); + } catch (error) { + if (error instanceof AggregateError) { + showAggregate(error); + } else if (error.cause) { + console.error(error); + showAggregate(error); + } else { + console.error(error); } - } + } - const pageWrites = writes.filter(({ type }) => type === 'page'); - const dataWrites = writes.filter(({ type }) => type === 'data'); - const redirectWrites = writes.filter(({ type }) => type === 'redirect'); + if (result === 'restart') { + console.log(''); - if (writes.length) { - logInfo`Total of ${writes.length} writes returned. (${pageWrites.length} page, ${dataWrites.length} data [currently skipped], ${redirectWrites.length} redirect)`; - } else { - logWarn`No writes returned at all, so exiting early. This is probably a bug!`; - return; + if (shouldShowStepStatusSummary) { + if (numRestarts >= 1) { + console.error(colors.bright(`Step summary since latest restart:`)); + } else { + console.error(colors.bright(`Step summary before restart:`)); + } + + showStepStatusSummary(); + console.log(''); + } + + if (numRestarts > 5) { + logError`A restart was cued, but we've restarted a bunch already.`; + logError`Exiting because this is probably a bug!`; + console.log(''); + break; + } else { + console.log(''); + logInfo`A restart was cued. This is probably normal, and required`; + logInfo`to load updated data files. Restarting automatically now!`; + console.log(''); + numRestarts++; + } + } else { + break; + } } - /* - await progressPromiseAll(`Writing data files shared across languages.`, queue( - dataWrites.map(({path, data}) => () => { - const bound = {}; + if (shouldShowStepStatusSummary) { + if (numRestarts >= 1) { + console.error(colors.bright(`Step summary after final restart:`)); + } else { + console.error(colors.bright(`Step summary:`)); + } - bound.serializeLink = bindOpts(serializeLink, {}); + const {anyStepsNotClean} = + showStepStatusSummary(); - bound.serializeContribs = bindOpts(serializeContribs, {}); + const totalTimeEnd = Date.now(); + const totalDuration = formatDuration(totalTimeEnd - totalTimeStart); - bound.serializeImagePaths = bindOpts(serializeImagePaths, { - thumb - }); + console.error(colors.bright(`Done in ${totalDuration}.`)); - bound.serializeCover = bindOpts(serializeCover, { - [bindOpts.bindIndex]: 2, - serializeImagePaths: bound.serializeImagePaths, - urls - }); + if (result === true) { + if (anyStepsNotClean) { + console.error(colors.bright(`Final output is true, but some steps aren't clean.`)); + process.exit(1); + return; + } else { + console.error(colors.bright(`Final output is true and all steps are clean.`)); + } + } else if (result === false) { + console.error(colors.bright(`Final output is false.`)); + } else { + console.error(colors.bright(`Final output is not true (${result}).`)); + } + } - bound.serializeGroupsForAlbum = bindOpts(serializeGroupsForAlbum, { - serializeLink - }); + if (result !== true) { + process.exit(1); + return; + } - bound.serializeGroupsForTrack = bindOpts(serializeGroupsForTrack, { - serializeLink - }); + decorateTime.displayTime(); - // TODO: This only supports one <>-style argument. - return writeData(path[0], path[1], data({ - ...bound - })); - }), - queueSize - )); - */ - - const perLanguageFn = async (language, i, entries) => { - const baseDirectory = (language === finalDefaultLanguage ? '' : language.code); - - console.log(`\x1b[34;1m${ - (`[${i + 1}/${entries.length}] ${language.code} (-> /${baseDirectory}) ` - .padEnd(60, '-')) - }\x1b[0m`); - - await progressPromiseAll(`Writing ${language.code}`, queue([ - ...pageWrites.map(({type, ...props}) => () => { - const { path, page } = props; - - // TODO: This only supports one <>-style argument. - const pageSubKey = path[0]; - const directory = path[1]; - - const localizedPaths = Object.fromEntries(Object.entries(languages) - .filter(([ key, language ]) => key !== 'default' && !language.hidden) - .map(([ key, language ]) => [language.code, writePage.paths( - (language === finalDefaultLanguage ? '' : language.code), - 'localized.' + pageSubKey, - directory - )])); - - const paths = writePage.paths( - baseDirectory, - 'localized.' + pageSubKey, - directory - ); - - const to = writePage.to({ - baseDirectory, - pageSubKey, - paths - }); - - const absoluteTo = (targetFullKey, ...args) => { - const [ groupKey, subKey ] = targetFullKey.split('.'); - const from = urls.from('shared.root'); - return '/' + (groupKey === 'localized' && baseDirectory - ? from.to('localizedWithBaseDirectory.' + subKey, baseDirectory, ...args) - : from.to(targetFullKey, ...args)); - }; - - // TODO: Is there some nicer way to define these, - // may8e without totally re-8inding everything for - // each page? - const bound = {}; - - bound.link = withEntries(unbound_link, entries => entries - .map(([ key, fn ]) => [key, bindOpts(fn, {to})])); - - bound.linkAnythingMan = bindOpts(linkAnythingMan, { - link: bound.link, - wikiData - }); - - bound.parseAttributes = bindOpts(parseAttributes, { - to - }); - - bound.find = bindFind(wikiData, {mode: 'warn'}); - - bound.transformInline = bindOpts(transformInline, { - find: bound.find, - link: bound.link, - replacerSpec, - language, - to, - wikiData - }); - - bound.transformMultiline = bindOpts(transformMultiline, { - transformInline: bound.transformInline, - parseAttributes: bound.parseAttributes - }); - - bound.transformLyrics = bindOpts(transformLyrics, { - transformInline: bound.transformInline, - transformMultiline: bound.transformMultiline - }); - - bound.iconifyURL = bindOpts(iconifyURL, { - language, - to - }); - - bound.fancifyURL = bindOpts(fancifyURL, { - language - }); - - bound.fancifyFlashURL = bindOpts(fancifyFlashURL, { - [bindOpts.bindIndex]: 2, - language - }); - - bound.getLinkThemeString = getLinkThemeString; - - bound.getThemeString = getThemeString; - - bound.getArtistString = bindOpts(getArtistString, { - iconifyURL: bound.iconifyURL, - link: bound.link, - language - }); - - bound.getAlbumCover = bindOpts(getAlbumCover, { - to - }); - - bound.getTrackCover = bindOpts(getTrackCover, { - to - }); - - bound.getFlashCover = bindOpts(getFlashCover, { - to - }); - - bound.getArtistAvatar = bindOpts(getArtistAvatar, { - to - }); - - bound.generateAdditionalFilesShortcut = bindOpts(generateAdditionalFilesShortcut, { - language - }); - - bound.generateAdditionalFilesList = bindOpts(generateAdditionalFilesList, { - language - }); - - bound.generateChronologyLinks = bindOpts(generateChronologyLinks, { - link: bound.link, - linkAnythingMan: bound.linkAnythingMan, - language, - wikiData - }); - - bound.generateCoverLink = bindOpts(generateCoverLink, { - [bindOpts.bindIndex]: 0, - img, - link: bound.link, - language, - to, - wikiData - }); - - bound.generateInfoGalleryLinks = bindOpts(generateInfoGalleryLinks, { - [bindOpts.bindIndex]: 2, - link: bound.link, - language - }); - - bound.generatePreviousNextLinks = bindOpts(generatePreviousNextLinks, { - link: bound.link, - language - }); - - bound.generateTrackListDividedByGroups = bindOpts(generateTrackListDividedByGroups, { - language, - wikiData, - }); - - bound.getGridHTML = bindOpts(getGridHTML, { - [bindOpts.bindIndex]: 0, - img, - language - }); - - bound.getAlbumGridHTML = bindOpts(getAlbumGridHTML, { - [bindOpts.bindIndex]: 0, - getAlbumCover: bound.getAlbumCover, - getGridHTML: bound.getGridHTML, - link: bound.link, - language - }); - - bound.getFlashGridHTML = bindOpts(getFlashGridHTML, { - [bindOpts.bindIndex]: 0, - getFlashCover: bound.getFlashCover, - getGridHTML: bound.getGridHTML, - link: bound.link - }); - - bound.getRevealStringFromTags = bindOpts(getRevealStringFromTags, { - language - }); - - bound.getRevealStringFromWarnings = bindOpts(getRevealStringFromWarnings, { - language - }); - - bound.getAlbumStylesheet = bindOpts(getAlbumStylesheet, { - to - }); - - const pageInfo = page({ - ...bound, - - language, - - absoluteTo, - relativeTo: to, - to, - urls, - - getSizeOfAdditionalFile, - }); - - const oEmbedJSON = writePage.oEmbedJSON(pageInfo, { - language, - wikiData, - }); - - const oEmbedJSONHref = (oEmbedJSON && wikiData.wikiInfo.canonicalBase) && ( - wikiData.wikiInfo.canonicalBase + urls.from('shared.root').to('shared.path', paths.pathname + OEMBED_JSON_FILE)); - - const html = writePage.html(pageInfo, { - defaultLanguage: finalDefaultLanguage, - language, - languages, - localizedPaths, - oEmbedJSONHref, - paths, - to, - transformMultiline: bound.transformMultiline, - wikiData - }); - - return writePage.write({ - html, - oEmbedJSON, - paths, - }); - }), - ...redirectWrites.map(({fromPath, toPath, title: titleFn}) => () => { - const title = titleFn({ - language - }); - - // TODO: This only supports one <>-style argument. - const fromPaths = writePage.paths(baseDirectory, 'localized.' + fromPath[0], fromPath[1]); - const to = writePage.to({baseDirectory, pageSubKey: fromPath[0], paths: fromPaths}); - - const target = to('localized.' + toPath[0], ...toPath.slice(1)); - const html = generateRedirectPage(title, target, {language}); - return writePage.write({html, paths: fromPaths}); - }) - ], queueSize)); - }; + process.exit(0); + })(); +} - await wrapLanguages(perLanguageFn, { - languages, - writeOneLanguage, - }); +function formatDuration(timeDelta) { + const seconds = timeDelta / 1000; + + if (seconds > 90) { + const modSeconds = Math.floor(seconds % 60); + const minutes = Math.floor(seconds - seconds % 60) / 60; + return `${minutes}m${modSeconds}s`; + } - // The single most important step. - logInfo`Written!`; + if (seconds < 0.1) { + return 'instant'; + } + + const precision = (seconds > 1 ? 3 : 2); + return `${seconds.toPrecision(precision)}s`; } -main().catch(error => { - if (error instanceof AggregateError) { - showAggregate(error); - } else { - console.error(error); +function showStepStatusSummary() { + const longestNameLength = + Math.max(... + Object.values(stepStatusSummary) + .map(({name}) => name.length)); + + const stepsNotClean = + Object.values(stepStatusSummary) + .map(({status}) => + status === STATUS_HAS_WARNINGS || + status === STATUS_FATAL_ERROR || + status === STATUS_STARTED_NOT_DONE); + + const anyStepsNotClean = + stepsNotClean.includes(true); + + const stepDetails = Object.values(stepStatusSummary); + + const stepDurations = + stepDetails.map(({status, timeStart, timeEnd}) => { + if ( + status === STATUS_NOT_APPLICABLE || + status === STATUS_NOT_STARTED || + status === STATUS_STARTED_NOT_DONE + ) { + return '-'; + } + + if (typeof timeStart !== 'number' || typeof timeEnd !== 'number') { + return 'unknown'; + } + + return formatDuration(timeEnd - timeStart); + }); + + const longestDurationLength = + Math.max(...stepDurations.map(duration => duration.length)); + + const stepMemories = + stepDetails.map(({memory}) => + (memory + ? Math.round(memory["heapUsed"] / 1024 / 1024) + 'MB' + : '-')); + + const longestMemoryLength = + Math.max(...stepMemories.map(memory => memory.length)); + + for (let index = 0; index < stepDetails.length; index++) { + const {name, status, annotation} = stepDetails[index]; + const duration = stepDurations[index]; + const memory = stepMemories[index]; + + let message = + (stepsNotClean[index] + ? `!! ` + : ` - `); + + message += `(${duration} `.padStart(longestDurationLength + 2, ' '); + + if (shouldShowStepMemoryInSummary) { + message += ` ${memory})`.padStart(longestMemoryLength + 2, ' '); } -}).then(() => { - decorateTime.displayTime(); - CacheableObject.showInvalidAccesses(); -}); + + message += ` `; + message += `${name}: `.padEnd(longestNameLength + 4, '.'); + message += ` `; + message += status; + + if (annotation) { + message += ` (${annotation})`; + } + + switch (status) { + case STATUS_DONE_CLEAN: + console.error(colors.green(message)); + break; + + case STATUS_NOT_STARTED: + case STATUS_NOT_APPLICABLE: + console.error(colors.dim(message)); + break; + + case STATUS_HAS_WARNINGS: + case STATUS_STARTED_NOT_DONE: + console.error(colors.yellow(message)); + break; + + case STATUS_FATAL_ERROR: + console.error(colors.red(message)); + break; + + default: + console.error(message); + break; + } + } + + return {anyStepsNotClean}; +} |