diff options
| -rwxr-xr-x | src/upd8.js | 136 | ||||
| -rw-r--r-- | src/url-spec.js | 66 | ||||
| -rw-r--r-- | src/urls-default.yaml (renamed from src/url-spec-default.yaml) | 0 | ||||
| -rw-r--r-- | src/urls.js | 7 |
4 files changed, 186 insertions, 23 deletions
diff --git a/src/upd8.js b/src/upd8.js index b83b5171..187be4cc 100755 --- a/src/upd8.js +++ b/src/upd8.js @@ -40,7 +40,7 @@ import {fileURLToPath} from 'node:url'; import wrap from 'word-wrap'; -import {mapAggregate, showAggregate} from '#aggregate'; +import {mapAggregate, openAggregate, showAggregate} from '#aggregate'; import CacheableObject from '#cacheable-object'; import {displayCompositeCacheAnalysis} from '#composite'; import find, {bindFind, getAllFindSpecs} from '#find'; @@ -50,8 +50,6 @@ import {isMain, traverse} from '#node-utils'; import {bindReverse} from '#reverse'; import {writeSearchData} from '#search'; import {sortByName} from '#sort'; -import {internalDefaultURLSpecFile, generateURLs, processURLSpecFromFile} - from '#urls'; import {identifyAllWebRoutes} from '#web-routes'; import { @@ -75,6 +73,7 @@ import { import { bindOpts, empty, + filterMultipleArrays, indentWrap as unboundIndentWrap, withEntries, } from '#sugar'; @@ -89,6 +88,14 @@ import genThumbs, { } from '#thumbs'; import { + applyLocalizedWithBaseDirectory, + applyURLSpecOverriding, + generateURLs, + internalDefaultURLSpecFile, + processURLSpecFromFile, +} from '#urls'; + +import { getAllDataSteps, linkWikiDataArrays, loadYAMLDocumentsFromDataSteps, @@ -330,6 +337,11 @@ async function main() { type: 'value', }, + 'urls': { + help: `Specify which optional URL specs to use for this build, customizing where pages are generated or resources are accessed from`, + type: 'value', + }, + 'skip-directory-validation': { help: `Skips checking for duplicated directories, which speeds up the build but may cause the wiki to catch on fire`, type: 'flag', @@ -571,6 +583,8 @@ async function main() { const precacheMode = cliOptions['precache-mode'] ?? 'common'; + const wantedURLSpecKeys = cliOptions['urls'] ?? []; + // Makes writing nicer on the CPU and file I/O parts of the OS, with a // marginal performance deficit while waiting for file writes to finish // before proceeding to more page processing. @@ -1833,7 +1847,119 @@ async function main() { return false; } - const urlSpec = internalURLSpec; + // We'll mutate this as we load other url spec files. + const urlSpec = structuredClone(internalURLSpec); + + const allURLSpecDataFiles = + (await readdir(dataPath)) + .filter(name => + name.startsWith('urls') && + ['.json', '.yaml'].includes(path.extname(name))) + .sort() /* Just in case... */ + .map(name => path.join(dataPath, name)); + + const getURLSpecKeyFromFile = file => { + const base = path.basename(file, path.extname(file)); + if (base === 'urls') { + return base; + } else { + return base.replace(/^urls-/, ''); + } + }; + + const isDefaultURLSpecFile = file => + getURLSpecKeyFromFile(file) === 'urls'; + + const overrideDefaultURLSpecFile = + allURLSpecDataFiles.find(file => isDefaultURLSpecFile(file)); + + const optionalURLSpecDataFiles = + allURLSpecDataFiles.filter(file => !isDefaultURLSpecFile(file)); + + const optionalURLSpecDataKeys = + optionalURLSpecDataFiles.map(file => getURLSpecKeyFromFile(file)); + + const selectedURLSpecDataKeys = optionalURLSpecDataKeys.slice(); + const selectedURLSpecDataFiles = optionalURLSpecDataFiles.slice(); + + const {removed: [unusedURLSpecDataKeys]} = + filterMultipleArrays( + selectedURLSpecDataKeys, + selectedURLSpecDataFiles, + (key, _file) => wantedURLSpecKeys.includes(key)); + + if (!empty(selectedURLSpecDataKeys)) { + logInfo`Using these optional URL specs: ${selectedURLSpecDataKeys.join(', ')}`; + if (!empty(unusedURLSpecDataKeys)) { + logInfo`Other available optional URL specs: ${unusedURLSpecDataKeys.join(', ')}`; + } + } else if (!empty(unusedURLSpecDataKeys)) { + logInfo`Not using any optional URL specs.`; + logInfo`These are available with --urls: ${unusedURLSpecDataKeys.join(', ')}`; + } + + if (overrideDefaultURLSpecFile) { + try { + let aggregate; + let overrideDefaultURLSpec; + + ({aggregate, result: overrideDefaultURLSpec} = + await processURLSpecFromFile(overrideDefaultURLSpecFile)); + + aggregate.close(); + + ({aggregate} = + applyURLSpecOverriding(overrideDefaultURLSpec, urlSpec)); + + aggregate.close(); + } catch (error) { + niceShowAggregate(error); + logError`Errors loading this data repo's ${'urls.yaml'} file.`; + logError`This provides essential overrides for this wiki,`; + logError`so stopping here. Debug the errors to continue.`; + + Object.assign(stepStatusSummary.loadURLFiles, { + status: STATUS_FATAL_ERROR, + annotation: `see log for details`, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + + return false; + } + } + + const processURLSpecsAggregate = + openAggregate({message: `Errors processing URL specs`}); + + const selectedURLSpecs = + processURLSpecsAggregate.receive( + await Promise.all( + selectedURLSpecDataFiles + .map(file => processURLSpecFromFile(file)))); + + for (const selectedURLSpec of selectedURLSpecs) { + processURLSpecsAggregate.receive( + applyURLSpecOverriding(selectedURLSpec, urlSpec)); + } + + try { + processURLSpecsAggregate.close(); + } catch (error) { + niceShowAggregate(error); + logWarn`There were errors loading the optional URL specs you`; + logWarn`selected using ${'--urls'}. Since they might misfunction,`; + logWarn`debug the errors or remove the failing ones from ${'--urls'}.`; + + Object.assign(stepStatusSummary.loadURLFiles, { + status: STATUS_FATAL_ERROR, + annotation: `see log for details`, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + + return false; + } Object.assign(stepStatusSummary.loadURLFiles, { status: STATUS_DONE_CLEAN, @@ -1841,6 +1967,8 @@ async function main() { memory: process.memoryUsage(), }); + applyLocalizedWithBaseDirectory(urlSpec); + const urls = generateURLs(urlSpec); const languageReloading = diff --git a/src/url-spec.js b/src/url-spec.js index 42e3e45c..f8ab6c69 100644 --- a/src/url-spec.js +++ b/src/url-spec.js @@ -10,7 +10,7 @@ import yaml from 'js-yaml'; import {annotateError, annotateErrorWithFile, openAggregate} from '#aggregate'; import {empty, typeAppearance, withEntries} from '#sugar'; -export const DEFAULT_URL_SPEC_FILE = 'url-spec-default.yaml'; +export const DEFAULT_URL_SPEC_FILE = 'urls-default.yaml'; export const internalDefaultURLSpecFile = path.resolve( @@ -96,6 +96,8 @@ export function processURLSpec(sourceSpec) { const aggregate = openAggregate({message: `Errors processing URL spec`}); + sourceSpec ??= {}; + const urlSpec = structuredClone(sourceSpec); delete urlSpec.yamlAliases; @@ -115,29 +117,14 @@ export function processURLSpec(sourceSpec) { case '<auto>': { if (!urlSpec.localized) { aggregate.push(new Error( - `Couldn't prepare 'localizedWithBaseDirectory' group, ` + + `Not ready for 'localizedWithBaseDirectory' group, ` + `'localized' not available`)); - - break; - } - - if (!urlSpec.localized.paths) { + } else if (!urlSpec.localized.paths) { aggregate.push(new Error( - `Couldn't prepare 'localizedWithBaseDirectory' group, ` + + `Not ready for 'localizedWithBaseDirectory' group, ` + `'localized' group's paths not available`)); - - break; } - const paths = - withEntries(urlSpec.localized.paths, entries => - entries.map(([key, path]) => [key, '<>/' + path])); - - urlSpec.localizedWithBaseDirectory = - Object.assign( - structuredClone(urlSpec.localized), - {paths}); - break; } @@ -155,6 +142,47 @@ export function processURLSpec(sourceSpec) { return {aggregate, result: urlSpec}; } +export function applyURLSpecOverriding(overrideSpec, baseSpec) { + const aggregate = openAggregate({message: `Errors applying URL spec`}); + + for (const [groupKey, overrideGroupSpec] of Object.entries(overrideSpec)) { + const baseGroupSpec = baseSpec[groupKey]; + + if (!baseGroupSpec) { + aggregate.push(new Error(`Group key "${groupKey}" not available on base spec`)); + continue; + } + + if (overrideGroupSpec.prefix) { + baseGroupSpec.prefix = overrideGroupSpec.prefix; + } + + if (overrideGroupSpec.paths) { + for (const [pathKey, overridePathValue] of Object.entries(overrideGroupSpec.paths)) { + if (!baseGroupSpec.paths[pathKey]) { + aggregate.push(new Error(`Path key "${groupKey}.${pathKey}" not available on base spec`)); + continue; + } + + baseGroupSpec.paths[pathKey] = overridePathValue; + } + } + } + + return {aggregate}; +} + +export function applyLocalizedWithBaseDirectory(urlSpec) { + const paths = + withEntries(urlSpec.localized.paths, entries => + entries.map(([key, path]) => [key, '<>/' + path])); + + urlSpec.localizedWithBaseDirectory = + Object.assign( + structuredClone(urlSpec.localized), + {paths}); +} + export async function processURLSpecFromFile(file) { let contents; diff --git a/src/url-spec-default.yaml b/src/urls-default.yaml index 10bc0d23..10bc0d23 100644 --- a/src/url-spec-default.yaml +++ b/src/urls-default.yaml diff --git a/src/urls.js b/src/urls.js index 1a471b30..71173f7f 100644 --- a/src/urls.js +++ b/src/urls.js @@ -11,6 +11,13 @@ import {withEntries} from '#sugar'; export * from './url-spec.js'; export function generateURLs(urlSpec) { + if ( + typeof urlSpec.localized === 'object' && + typeof urlSpec.localizedWithBaseDirectory !== 'object' + ) { + throw new Error(`Provided urlSpec missing localizedWithBaseDirectory`); + } + const getValueForFullKey = (obj, fullKey) => { const [groupKey, subKey] = fullKey.split('.'); if (!groupKey || !subKey) { |