urls, upd8: default-override & optional url specs

4 files changed, 186 insertions, 23 deletions

diff --git a/src/upd8.js b/src/upd8.js
index b83b5171..187be4cc 100755
--- a/src/upd8.js
+++ b/src/upd8.js
@@ -40,7 +40,7 @@ import {fileURLToPath} from 'node:url';
 
 import wrap from 'word-wrap';
 
-import {mapAggregate, showAggregate} from '#aggregate';
+import {mapAggregate, openAggregate, showAggregate} from '#aggregate';
 import CacheableObject from '#cacheable-object';
 import {displayCompositeCacheAnalysis} from '#composite';
 import find, {bindFind, getAllFindSpecs} from '#find';
@@ -50,8 +50,6 @@ import {isMain, traverse} from '#node-utils';
 import {bindReverse} from '#reverse';
 import {writeSearchData} from '#search';
 import {sortByName} from '#sort';
-import {internalDefaultURLSpecFile, generateURLs, processURLSpecFromFile}
-  from '#urls';
 import {identifyAllWebRoutes} from '#web-routes';
 
 import {
@@ -75,6 +73,7 @@ import {
 import {
   bindOpts,
   empty,
+  filterMultipleArrays,
   indentWrap as unboundIndentWrap,
   withEntries,
 } from '#sugar';
@@ -89,6 +88,14 @@ import genThumbs, {
 } from '#thumbs';
 
 import {
+  applyLocalizedWithBaseDirectory,
+  applyURLSpecOverriding,
+  generateURLs,
+  internalDefaultURLSpecFile,
+  processURLSpecFromFile,
+} from '#urls';
+
+import {
   getAllDataSteps,
   linkWikiDataArrays,
   loadYAMLDocumentsFromDataSteps,
@@ -330,6 +337,11 @@ async function main() {
       type: 'value',
     },
 
+    'urls': {
+      help: `Specify which optional URL specs to use for this build, customizing where pages are generated or resources are accessed from`,
+      type: 'value',
+    },
+
     'skip-directory-validation': {
       help: `Skips checking for duplicated directories, which speeds up the build but may cause the wiki to catch on fire`,
       type: 'flag',
@@ -571,6 +583,8 @@ async function main() {
 
   const precacheMode = cliOptions['precache-mode'] ?? 'common';
 
+  const wantedURLSpecKeys = cliOptions['urls'] ?? [];
+
   // Makes writing nicer on the CPU and file I/O parts of the OS, with a
   // marginal performance deficit while waiting for file writes to finish
   // before proceeding to more page processing.
@@ -1833,7 +1847,119 @@ async function main() {
     return false;
   }
 
-  const urlSpec = internalURLSpec;
+  // We'll mutate this as we load other url spec files.
+  const urlSpec = structuredClone(internalURLSpec);
+
+  const allURLSpecDataFiles =
+    (await readdir(dataPath))
+      .filter(name =>
+        name.startsWith('urls') &&
+        ['.json', '.yaml'].includes(path.extname(name)))
+      .sort() /* Just in case... */
+      .map(name => path.join(dataPath, name));
+
+  const getURLSpecKeyFromFile = file => {
+    const base = path.basename(file, path.extname(file));
+    if (base === 'urls') {
+      return base;
+    } else {
+      return base.replace(/^urls-/, '');
+    }
+  };
+
+  const isDefaultURLSpecFile = file =>
+    getURLSpecKeyFromFile(file) === 'urls';
+
+  const overrideDefaultURLSpecFile =
+    allURLSpecDataFiles.find(file => isDefaultURLSpecFile(file));
+
+  const optionalURLSpecDataFiles =
+    allURLSpecDataFiles.filter(file => !isDefaultURLSpecFile(file));
+
+  const optionalURLSpecDataKeys =
+    optionalURLSpecDataFiles.map(file => getURLSpecKeyFromFile(file));
+
+  const selectedURLSpecDataKeys = optionalURLSpecDataKeys.slice();
+  const selectedURLSpecDataFiles = optionalURLSpecDataFiles.slice();
+
+  const {removed: [unusedURLSpecDataKeys]} =
+    filterMultipleArrays(
+      selectedURLSpecDataKeys,
+      selectedURLSpecDataFiles,
+      (key, _file) => wantedURLSpecKeys.includes(key));
+
+  if (!empty(selectedURLSpecDataKeys)) {
+    logInfo`Using these optional URL specs: ${selectedURLSpecDataKeys.join(', ')}`;
+    if (!empty(unusedURLSpecDataKeys)) {
+      logInfo`Other available optional URL specs: ${unusedURLSpecDataKeys.join(', ')}`;
+    }
+  } else if (!empty(unusedURLSpecDataKeys)) {
+    logInfo`Not using any optional URL specs.`;
+    logInfo`These are available with --urls: ${unusedURLSpecDataKeys.join(', ')}`;
+  }
+
+  if (overrideDefaultURLSpecFile) {
+    try {
+      let aggregate;
+      let overrideDefaultURLSpec;
+
+      ({aggregate, result: overrideDefaultURLSpec} =
+          await processURLSpecFromFile(overrideDefaultURLSpecFile));
+
+      aggregate.close();
+
+      ({aggregate} =
+          applyURLSpecOverriding(overrideDefaultURLSpec, urlSpec));
+
+      aggregate.close();
+    } catch (error) {
+      niceShowAggregate(error);
+      logError`Errors loading this data repo's ${'urls.yaml'} file.`;
+      logError`This provides essential overrides for this wiki,`;
+      logError`so stopping here. Debug the errors to continue.`;
+
+      Object.assign(stepStatusSummary.loadURLFiles, {
+        status: STATUS_FATAL_ERROR,
+        annotation: `see log for details`,
+        timeEnd: Date.now(),
+        memory: process.memoryUsage(),
+      });
+
+      return false;
+    }
+  }
+
+  const processURLSpecsAggregate =
+    openAggregate({message: `Errors processing URL specs`});
+
+  const selectedURLSpecs =
+    processURLSpecsAggregate.receive(
+      await Promise.all(
+        selectedURLSpecDataFiles
+          .map(file => processURLSpecFromFile(file))));
+
+  for (const selectedURLSpec of selectedURLSpecs) {
+    processURLSpecsAggregate.receive(
+      applyURLSpecOverriding(selectedURLSpec, urlSpec));
+  }
+
+  try {
+    processURLSpecsAggregate.close();
+  } catch (error) {
+    niceShowAggregate(error);
+    logWarn`There were errors loading the optional URL specs you`;
+    logWarn`selected using ${'--urls'}. Since they might misfunction,`;
+    logWarn`debug the errors or remove the failing ones from ${'--urls'}.`;
+
+    Object.assign(stepStatusSummary.loadURLFiles, {
+      status: STATUS_FATAL_ERROR,
+      annotation: `see log for details`,
+      timeEnd: Date.now(),
+      memory: process.memoryUsage(),
+    });
+
+    return false;
+  }
 
   Object.assign(stepStatusSummary.loadURLFiles, {
     status: STATUS_DONE_CLEAN,
@@ -1841,6 +1967,8 @@ async function main() {
     memory: process.memoryUsage(),
   });
 
+  applyLocalizedWithBaseDirectory(urlSpec);
+
   const urls = generateURLs(urlSpec);
 
   const languageReloading =
diff --git a/src/url-spec.js b/src/url-spec.js
index 42e3e45c..f8ab6c69 100644
--- a/src/url-spec.js
+++ b/src/url-spec.js
@@ -10,7 +10,7 @@ import yaml from 'js-yaml';
 import {annotateError, annotateErrorWithFile, openAggregate} from '#aggregate';
 import {empty, typeAppearance, withEntries} from '#sugar';
 
-export const DEFAULT_URL_SPEC_FILE = 'url-spec-default.yaml';
+export const DEFAULT_URL_SPEC_FILE = 'urls-default.yaml';
 
 export const internalDefaultURLSpecFile =
   path.resolve(
@@ -96,6 +96,8 @@ export function processURLSpec(sourceSpec) {
   const aggregate =
     openAggregate({message: `Errors processing URL spec`});
 
+  sourceSpec ??= {};
+
   const urlSpec = structuredClone(sourceSpec);
 
   delete urlSpec.yamlAliases;
@@ -115,29 +117,14 @@ export function processURLSpec(sourceSpec) {
     case '<auto>': {
       if (!urlSpec.localized) {
         aggregate.push(new Error(
-          `Couldn't prepare 'localizedWithBaseDirectory' group, ` +
+          `Not ready for 'localizedWithBaseDirectory' group, ` +
           `'localized' not available`));
-
-        break;
-      }
-
-      if (!urlSpec.localized.paths) {
+      } else if (!urlSpec.localized.paths) {
         aggregate.push(new Error(
-          `Couldn't prepare 'localizedWithBaseDirectory' group, ` +
+          `Not ready for 'localizedWithBaseDirectory' group, ` +
           `'localized' group's paths not available`));
-
-        break;
       }
 
-      const paths =
-        withEntries(urlSpec.localized.paths, entries =>
-          entries.map(([key, path]) => [key, '<>/' + path]));
-
-      urlSpec.localizedWithBaseDirectory =
-        Object.assign(
-          structuredClone(urlSpec.localized),
-          {paths});
-
       break;
     }
 
@@ -155,6 +142,47 @@ export function processURLSpec(sourceSpec) {
   return {aggregate, result: urlSpec};
 }
 
+export function applyURLSpecOverriding(overrideSpec, baseSpec) {
+  const aggregate = openAggregate({message: `Errors applying URL spec`});
+
+  for (const [groupKey, overrideGroupSpec] of Object.entries(overrideSpec)) {
+    const baseGroupSpec = baseSpec[groupKey];
+
+    if (!baseGroupSpec) {
+      aggregate.push(new Error(`Group key "${groupKey}" not available on base spec`));
+      continue;
+    }
+
+    if (overrideGroupSpec.prefix) {
+      baseGroupSpec.prefix = overrideGroupSpec.prefix;
+    }
+
+    if (overrideGroupSpec.paths) {
+      for (const [pathKey, overridePathValue] of Object.entries(overrideGroupSpec.paths)) {
+        if (!baseGroupSpec.paths[pathKey]) {
+          aggregate.push(new Error(`Path key "${groupKey}.${pathKey}" not available on base spec`));
+          continue;
+        }
+
+        baseGroupSpec.paths[pathKey] = overridePathValue;
+      }
+    }
+  }
+
+  return {aggregate};
+}
+
+export function applyLocalizedWithBaseDirectory(urlSpec) {
+  const paths =
+    withEntries(urlSpec.localized.paths, entries =>
+      entries.map(([key, path]) => [key, '<>/' + path]));
+
+  urlSpec.localizedWithBaseDirectory =
+    Object.assign(
+      structuredClone(urlSpec.localized),
+      {paths});
+}
+
 export async function processURLSpecFromFile(file) {
   let contents;
 
diff --git a/src/url-spec-default.yaml b/src/urls-default.yaml
index 10bc0d23..10bc0d23 100644
--- a/src/url-spec-default.yaml
+++ b/src/urls-default.yaml
diff --git a/src/urls.js b/src/urls.js
index 1a471b30..71173f7f 100644
--- a/src/urls.js
+++ b/src/urls.js
@@ -11,6 +11,13 @@ import {withEntries} from '#sugar';
 export * from './url-spec.js';
 
 export function generateURLs(urlSpec) {
+  if (
+    typeof urlSpec.localized === 'object' &&
+    typeof urlSpec.localizedWithBaseDirectory !== 'object'
+  ) {
+    throw new Error(`Provided urlSpec missing localizedWithBaseDirectory`);
+  }
+
   const getValueForFullKey = (obj, fullKey) => {
     const [groupKey, subKey] = fullKey.split('.');
     if (!groupKey || !subKey) {