urls, upd8: load internal url spec from yaml file

5 files changed, 380 insertions, 158 deletions

diff --git a/src/upd8.js b/src/upd8.js
index 6097f21f..b83b5171 100755
--- a/src/upd8.js
+++ b/src/upd8.js
@@ -50,7 +50,8 @@ import {isMain, traverse} from '#node-utils';
 import {bindReverse} from '#reverse';
 import {writeSearchData} from '#search';
 import {sortByName} from '#sort';
-import {generateURLs, urlSpec} from '#urls';
+import {internalDefaultURLSpecFile, generateURLs, processURLSpecFromFile}
+  from '#urls';
 import {identifyAllWebRoutes} from '#web-routes';
 
 import {
@@ -180,6 +181,10 @@ async function main() {
       {...defaultStepStatus, name: `precache nearly all data`,
         for: ['build']},
 
+    loadURLFiles:
+      {...defaultStepStatus, name: `load internal & custom url spec files`,
+        for: ['build']},
+
     // TODO: This should be split into load/watch steps.
     loadInternalDefaultLanguage:
       {...defaultStepStatus, name: `load internal default language`,
@@ -1627,8 +1632,6 @@ async function main() {
     });
   }
 
-  const urls = generateURLs(urlSpec);
-
   // Filter out any things with duplicate directories throughout the data,
   // warning about them too.
 
@@ -1801,6 +1804,45 @@ async function main() {
     }
   }
 
+  Object.assign(stepStatusSummary.loadURLFiles, {
+    status: STATUS_STARTED_NOT_DONE,
+    timeStart: Date.now(),
+  });
+
+  let internalURLSpec = {};
+
+  try {
+    let aggregate;
+    ({aggregate, result: internalURLSpec} =
+      await processURLSpecFromFile(internalDefaultURLSpecFile));
+
+    aggregate.close();
+  } catch (error) {
+    niceShowAggregate(error);
+    logError`Couldn't load internal default URL spec.`;
+    logError`This is required to build the wiki, so stopping here.`;
+    fileIssue();
+
+    Object.assign(stepStatusSummary.loadURLFiles, {
+      status: STATUS_FATAL_ERROR,
+      annotation: `see log for details`,
+      timeEnd: Date.now(),
+      memory: process.memoryUsage(),
+    });
+
+    return false;
+  }
+
+  const urlSpec = internalURLSpec;
+
+  Object.assign(stepStatusSummary.loadURLFiles, {
+    status: STATUS_DONE_CLEAN,
+    timeEnd: Date.now(),
+    memory: process.memoryUsage(),
+  });
+
+  const urls = generateURLs(urlSpec);
+
   const languageReloading =
     stepStatusSummary.watchLanguageFiles.status === STATUS_NOT_STARTED;
 
diff --git a/src/url-spec-default.yaml b/src/url-spec-default.yaml
new file mode 100644
index 00000000..10bc0d23
--- /dev/null
+++ b/src/url-spec-default.yaml
@@ -0,0 +1,143 @@
+# These are variables which are used to make expressing this
+# YAML file more convenient. They are not exposed externally.
+# (Stuff which uses this YAML file can't even see the names
+# for each variable!)
+yamlAliases:
+  - &genericPaths
+      root: ''
+      path: '<>'
+
+  # Static files are all grouped under a `static-${STATIC_VERSION}` folder as
+  # part of a build. This is so that multiple builds of a wiki can coexist
+  # served from the same server / file system root: older builds' HTML files
+  # refer to earlier values of STATIC_VERSION, avoiding name collisions.
+  - &staticVersion 3p4
+
+data:
+  prefix: 'data/'
+
+  paths:
+  - *genericPaths
+
+  - album: 'album/<>'
+    artist: 'artist/<>'
+    track: 'track/<>'
+
+localized:
+  paths:
+  - *genericPaths
+  - page: '<>/'
+
+    home: ''
+
+    album: 'album/<>/'
+    albumCommentary: 'commentary/album/<>/'
+    albumGallery: 'album/<>/gallery/'
+    albumReferencedArtworks: 'album/<>/referenced-art/'
+    albumReferencingArtworks: 'album/<>/referencing-art/'
+
+    artist: 'artist/<>/'
+    artistGallery: 'artist/<>/gallery/'
+
+    commentaryIndex: 'commentary/'
+
+    flashIndex: 'flash/'
+
+    flash: 'flash/<>/'
+
+    flashActGallery: 'flash-act/<>/'
+
+    groupInfo: 'group/<>/'
+    groupGallery: 'group/<>/gallery/'
+
+    listingIndex: 'list/'
+
+    listing: 'list/<>/'
+
+    newsIndex: 'news/'
+
+    newsEntry: 'news/<>/'
+
+    staticPage: '<>/'
+
+    tag: 'tag/<>/'
+
+    track: 'track/<>/'
+    trackReferencedArtworks: 'track/<>/referenced-art/'
+    trackReferencingArtworks: 'track/<>/referencing-art/'
+
+# This gets automatically switched in place when working from
+# a baseDirectory, so it should never be referenced manually.
+# It's also filled in externally to this YAML spec.
+localizedWithBaseDirectory: '<auto>'
+
+shared:
+  paths: *genericPaths
+
+staticCSS:
+  prefix:
+  - 'static-'
+  - *staticVersion
+  - '/css/'
+
+  paths: *genericPaths
+
+staticJS:
+  prefix:
+  - 'static-'
+  - *staticVersion
+  - '/js/'
+
+  paths: *genericPaths
+
+staticLib:
+  prefix:
+  - 'static-'
+  - *staticVersion
+  - '/lib/'
+
+  paths: *genericPaths
+
+staticMisc:
+  prefix:
+  - 'static-'
+  - *staticVersion
+  - '/misc/'
+
+  paths:
+  - *genericPaths
+  - icon: 'icons.svg#icon-<>'
+
+staticSharedUtil:
+  prefix:
+  - 'static-'
+  - *staticVersion
+  - '/shared-util/'
+
+  paths: *genericPaths
+
+media:
+  prefix: 'media/'
+
+  paths:
+  - *genericPaths
+
+  - albumAdditionalFile: 'album-additional/<>/<>'
+    albumBanner: 'album-art/<>/banner.<>'
+    albumCover: 'album-art/<>/cover.<>'
+    albumWallpaper: 'album-art/<>/bg.<>'
+    albumWallpaperPart: 'album-art/<>/<>'
+
+    artistAvatar: 'artist-avatar/<>.<>'
+
+    flashArt: 'flash-art/<>.<>'
+
+    trackCover: 'album-art/<>/<>.<>'
+
+thumb:
+  prefix: 'thumb/'
+  paths: *genericPaths
+
+searchData:
+  prefix: 'search-data/'
+  paths: *genericPaths
diff --git a/src/url-spec.js b/src/url-spec.js
index 6ca75e7d..42e3e45c 100644
--- a/src/url-spec.js
+++ b/src/url-spec.js
@@ -1,145 +1,191 @@
-import {withEntries} from '#sugar';
-
-// Static files are all grouped under a `static-${STATIC_VERSION}` folder as
-// part of a build. This is so that multiple builds of a wiki can coexist
-// served from the same server / file system root: older builds' HTML files
-// refer to earlier values of STATIC_VERSION, avoiding name collisions.
-const STATIC_VERSION = '3p3';
-
-const genericPaths = {
-  root: '',
-  path: '<>',
-};
-
-const urlSpec = {
-  data: {
-    prefix: 'data/',
-
-    paths: {
-      ...genericPaths,
-
-      album: 'album/<>',
-      artist: 'artist/<>',
-      track: 'track/<>',
-    },
-  },
-
-  localized: {
-    // TODO: Implement this.
-    // prefix: '_languageCode',
-
-    paths: {
-      ...genericPaths,
-      page: '<>/',
-
-      home: '',
-
-      album: 'album/<>/',
-      albumCommentary: 'commentary/album/<>/',
-      albumGallery: 'album/<>/gallery/',
-      albumReferencedArtworks: 'album/<>/referenced-art/',
-      albumReferencingArtworks: 'album/<>/referencing-art/',
-
-      artist: 'artist/<>/',
-      artistGallery: 'artist/<>/gallery/',
-
-      commentaryIndex: 'commentary/',
-
-      flashIndex: 'flash/',
-
-      flash: 'flash/<>/',
-
-      flashActGallery: 'flash-act/<>/',
-
-      groupInfo: 'group/<>/',
-      groupGallery: 'group/<>/gallery/',
-
-      listingIndex: 'list/',
-
-      listing: 'list/<>/',
-
-      newsIndex: 'news/',
-
-      newsEntry: 'news/<>/',
-
-      staticPage: '<>/',
-
-      tag: 'tag/<>/',
-
-      track: 'track/<>/',
-      trackReferencedArtworks: 'track/<>/referenced-art/',
-      trackReferencingArtworks: 'track/<>/referencing-art/',
-    },
-  },
-
-  shared: {
-    paths: genericPaths,
-  },
-
-  staticCSS: {
-    prefix: `static-${STATIC_VERSION}/css/`,
-    paths: genericPaths,
-  },
-
-  staticJS: {
-    prefix: `static-${STATIC_VERSION}/js/`,
-    paths: genericPaths,
-  },
-
-  staticLib: {
-    prefix: `static-${STATIC_VERSION}/lib/`,
-    paths: genericPaths,
-  },
-
-  staticMisc: {
-    prefix: `static-${STATIC_VERSION}/misc/`,
-    paths: {
-      ...genericPaths,
-      icon: 'icons.svg#icon-<>',
-    },
-  },
-
-  staticSharedUtil: {
-    prefix: `static-${STATIC_VERSION}/shared-util/`,
-    paths: genericPaths,
-  },
-
-  media: {
-    prefix: 'media/',
-
-    paths: {
-      ...genericPaths,
-
-      albumAdditionalFile: 'album-additional/<>/<>',
-      albumBanner: 'album-art/<>/banner.<>',
-      albumCover: 'album-art/<>/cover.<>',
-      albumWallpaper: 'album-art/<>/bg.<>',
-      albumWallpaperPart: 'album-art/<>/<>',
-
-      artistAvatar: 'artist-avatar/<>.<>',
-
-      flashArt: 'flash-art/<>.<>',
-
-      trackCover: 'album-art/<>/<>.<>',
-    },
-  },
-
-  thumb: {
-    prefix: 'thumb/',
-    paths: genericPaths,
-  },
-
-  searchData: {
-    prefix: 'search-data/',
-    paths: genericPaths,
-  },
-};
-
-// This gets automatically switched in place when working from a baseDirectory,
-// so it should never be referenced manually.
-urlSpec.localizedWithBaseDirectory = {
-  paths: withEntries(urlSpec.localized.paths, (entries) =>
-    entries.map(([key, path]) => [key, '<>/' + path])),
-};
-
-export default urlSpec;
+// Exports defined here are re-exported through urls.js,
+// so they're generally imported from '#urls'.
+
+import {readFile} from 'node:fs/promises';
+import * as path from 'node:path';
+import {fileURLToPath} from 'node:url';
+
+import yaml from 'js-yaml';
+
+import {annotateError, annotateErrorWithFile, openAggregate} from '#aggregate';
+import {empty, typeAppearance, withEntries} from '#sugar';
+
+export const DEFAULT_URL_SPEC_FILE = 'url-spec-default.yaml';
+
+export const internalDefaultURLSpecFile =
+  path.resolve(
+    path.dirname(fileURLToPath(import.meta.url)),
+    DEFAULT_URL_SPEC_FILE);
+
+function processStringToken(key, token) {
+  const oops = appearance =>
+    new Error(
+      `Expected ${key} to be a string or an array of strings, ` +
+      `got ${appearance}`);
+
+  if (typeof token === 'string') {
+    return token;
+  } else if (Array.isArray(token)) {
+    if (empty(token)) {
+      throw oops(`empty array`);
+    } else if (token.every(item => typeof item !== 'string')) {
+      throw oops(`array of non-strings`);
+    } else if (token.some(item => typeof item !== 'string')) {
+      throw oops(`array of mixed strings and non-strings`);
+    } else {
+      return token.join('');
+    }
+  } else {
+    throw oops(typeAppearance(token));
+  }
+}
+
+// Mutates, so don't even think about reusing the original representation.
+function processObjectToken(key, token) {
+  const oops = appearance =>
+    new Error(
+      `Expected ${key} to be an object or an array of objects, ` +
+      `got ${appearance}`);
+
+  const looksLikeObject = value =>
+    typeof value === 'object' &&
+    value !== null &&
+    !Array.isArray(value);
+
+  if (looksLikeObject(token)) {
+    return token;
+  } else if (Array.isArray(token)) {
+    if (empty(token)) {
+      throw oops(`empty array`);
+    } else if (token.every(item => !looksLikeObject(item))) {
+      throw oops(`array of non-objects`);
+    } else if (token.some(item => !looksLikeObject(item))) {
+      throw oops(`array of mixed objects and non-objects`);
+    } else {
+      return Object.assign(...token);
+    }
+  }
+}
+
+function makeProcessToken(aggregate) {
+  return (object, key, processFn) => {
+    if (key in object) {
+      const value = aggregate.call(processFn, key, object[key]);
+      if (value === null) {
+        delete object[key];
+      } else {
+        object[key] = value;
+      }
+    }
+  };
+}
+
+export function processGroupSpec(groupKey, groupSpec) {
+  const aggregate =
+    openAggregate({message: `Errors procsesing group "${groupKey}"`});
+
+  const processToken = makeProcessToken(aggregate);
+
+  processToken(groupSpec, 'prefix', processStringToken);
+  processToken(groupSpec, 'paths', processObjectToken);
+
+  return {aggregate, result: groupSpec};
+}
+
+export function processURLSpec(sourceSpec) {
+  const aggregate =
+    openAggregate({message: `Errors processing URL spec`});
+
+  const urlSpec = structuredClone(sourceSpec);
+
+  delete urlSpec.yamlAliases;
+  delete urlSpec.localizedWithBaseDirectory;
+
+  aggregate.nest({message: `Errors processing groups`}, groupsAggregate => {
+    Object.assign(urlSpec,
+      withEntries(urlSpec, entries =>
+        entries.map(([groupKey, groupSpec]) => [
+          groupKey,
+          groupsAggregate.receive(
+            processGroupSpec(groupKey, groupSpec)),
+        ])));
+  });
+
+  switch (sourceSpec.localizedWithBaseDirectory) {
+    case '<auto>': {
+      if (!urlSpec.localized) {
+        aggregate.push(new Error(
+          `Couldn't prepare 'localizedWithBaseDirectory' group, ` +
+          `'localized' not available`));
+
+        break;
+      }
+
+      if (!urlSpec.localized.paths) {
+        aggregate.push(new Error(
+          `Couldn't prepare 'localizedWithBaseDirectory' group, ` +
+          `'localized' group's paths not available`));
+
+        break;
+      }
+
+      const paths =
+        withEntries(urlSpec.localized.paths, entries =>
+          entries.map(([key, path]) => [key, '<>/' + path]));
+
+      urlSpec.localizedWithBaseDirectory =
+        Object.assign(
+          structuredClone(urlSpec.localized),
+          {paths});
+
+      break;
+    }
+
+    case undefined:
+      break;
+
+    default:
+      aggregate.push(new Error(
+        `Expected 'localizedWithBaseDirectory' group to have value '<auto>' ` +
+        `or not be set`));
+
+      break;
+  }
+
+  return {aggregate, result: urlSpec};
+}
+
+export async function processURLSpecFromFile(file) {
+  let contents;
+
+  try {
+    contents = await readFile(file, 'utf-8');
+  } catch (caughtError) {
+    throw annotateError(
+      new Error(`Failed to read URL spec file`, {cause: caughtError}),
+      error => annotateErrorWithFile(error, file));
+  }
+
+  let sourceSpec;
+  let parseLanguage;
+
+  try {
+    if (path.extname(file) === '.yaml') {
+      parseLanguage = 'YAML';
+      sourceSpec = yaml.load(contents);
+    } else {
+      parseLanguage = 'JSON';
+      sourceSpec = JSON.parse(contents);
+    }
+  } catch (caughtError) {
+    throw annotateError(
+      new Error(`Failed to parse URL spec file as valid ${parseLanguage}`, {cause: caughtError}),
+      error => annotateErrorWithFile(error, file));
+  }
+
+  try {
+    return processURLSpec(sourceSpec);
+  } catch (caughtError) {
+    throw annotateErrorWithFile(caughtError, file);
+  }
+}
diff --git a/src/util/urls.js b/src/urls.js
index 11b9b8b0..1a471b30 100644
--- a/src/util/urls.js
+++ b/src/urls.js
@@ -8,15 +8,7 @@ import * as path from 'node:path';
 
 import {withEntries} from '#sugar';
 
-// This export is only provided for convenience, i.e. to enable the following:
-//
-//   import {urlSpec} from '#urls';
-//
-// It's not actually defined in this module's variable scope, and functions
-// exported here require a urlSpec (whether this default one or another) to be
-// passed directly.
-//
-export {default as urlSpec} from '../url-spec.js';
+export * from './url-spec.js';
 
 export function generateURLs(urlSpec) {
   const getValueForFullKey = (obj, fullKey) => {
diff --git a/src/util/aggregate.js b/src/util/aggregate.js
index e8f45f3b..c7648c4c 100644
--- a/src/util/aggregate.js
+++ b/src/util/aggregate.js
@@ -110,7 +110,6 @@ export function openAggregate({
 
     return results.map(({aggregate, result}) => {
       if (!aggregate) {
-        console.log('nope:', results);
         throw new Error(`Expected an array of {aggregate, result} objects`);
       }