extract static-build, new modular build modes

1 files changed, 77 insertions, 417 deletions

diff --git a/src/upd8.js b/src/upd8.js
index d5a5c2a7..c3934a4c 100755
--- a/src/upd8.js
+++ b/src/upd8.js
@@ -40,7 +40,6 @@ import {listingSpec, listingTargetSpec} from './listing-spec.js';
 import urlSpec from './url-spec.js';
 
 import {processLanguageFile} from './data/language.js';
-import {serializeThings} from './data/serialize.js';
 
 import CacheableObject from './data/things/cacheable-object.js';
 
@@ -53,14 +52,17 @@ import {
   WIKI_INFO_FILE,
 } from './data/yaml.js';
 
-import * as pageSpecs from './page/index.js';
-
 import find from './util/find.js';
 import {findFiles} from './util/io.js';
 import link from './util/link.js';
 import {isMain} from './util/node-utils.js';
 import {validateReplacerSpec} from './util/replacer.js';
+import {empty, showAggregate} from './util/sugar.js';
 import {replacerSpec} from './util/transform-content.js';
+import {generateURLs} from './util/urls.js';
+
+import {generateDevelopersCommentHTML} from './write/page-template.js';
+import * as buildModes from './write/build-modes/index.js';
 
 import {
   color,
@@ -73,34 +75,6 @@ import {
   progressPromiseAll,
 } from './util/cli.js';
 
-import {
-  queue,
-  showAggregate,
-  withEntries,
-} from './util/sugar.js';
-
-import {
-  generateURLs,
-  getPagePaths,
-  getURLsFrom,
-} from './util/urls.js';
-
-import {bindUtilities} from './write/bind-utilities.js';
-import {validateWrites} from './write/validate-writes.js';
-
-import {
-  generateDocumentHTML,
-  generateGlobalWikiDataJSON,
-  generateOEmbedJSON,
-  generateRedirectHTML,
-} from './write/page-template.js';
-
-import {
-  writePage,
-  writeSharedFilesAndPages,
-  writeSymlinks,
-} from './write/write-files.js';
-
 /*
 import {
   serializeContribs,
@@ -133,24 +107,32 @@ if (!validateReplacerSpec(replacerSpec, {find, link})) {
   process.exit();
 }
 
-// Wrapper function for running a function once for all languages.
-async function wrapLanguages(fn, {languages, writeOneLanguage = null}) {
-  const k = writeOneLanguage;
-  const languagesToRun = k ? {[k]: languages[k]} : languages;
+async function main() {
+  Error.stackTraceLimit = Infinity;
+
+  const selectedBuildModeFlags = Object.keys(
+    await parseOptions(process.argv.slice(2), {
+      [parseOptions.handleUnknown]: () => {},
 
-  const entries = Object.entries(languagesToRun).filter(
-    ([key]) => key !== 'default'
-  );
+      ...Object.fromEntries(Object.keys(buildModes)
+        .map((key) => [key, {type: 'flag'}])),
+    }));
 
-  for (let i = 0; i < entries.length; i++) {
-    const [_key, language] = entries[i];
+  let selectedBuildModeFlag;
 
-    await fn(language, i, entries);
+  if (empty(selectedBuildModeFlags)) {
+    selectedBuildModeFlag = 'static-build';
+    logInfo`No build mode specified, using default: ${selectedBuildModeFlag}`;
+  } else if (selectedBuildModeFlags.length > 1) {
+    logError`Building multiple modes (${selectedBuildModeFlags.join(', ')}) at once not supported.`;
+    logError`Please specify a maximum of one build mode.`;
+    return;
+  } else {
+    selectedBuildModeFlag = selectedBuildModeFlags[0];
+    logInfo`Using specified build mode: ${selectedBuildModeFlag}`;
   }
-}
 
-async function main() {
-  Error.stackTraceLimit = Infinity;
+  const selectedBuildMode = buildModes[selectedBuildModeFlag];
 
   // This is about to get a whole lot more stuff put in it.
   const wikiData = {
@@ -158,7 +140,9 @@ async function main() {
     listingTargetSpec,
   };
 
-  const miscOptions = await parseOptions(process.argv.slice(2), {
+  const cliOptions = await parseOptions(process.argv.slice(2), {
+    ...selectedBuildMode.getCLIOptions(),
+
     // Data files for the site, including flash, artist, and al8um data,
     // and like a jillion other things too. Pretty much everything which
     // makes an individual wiki what it is goes here!
@@ -187,16 +171,6 @@ async function main() {
       type: 'value',
     },
 
-    // This is the output directory. It's the one you'll upload online with
-    // rsync or whatever when you're pushing an upd8, and also the one
-    // you'd archive if you wanted to make a 8ackup of the whole dang
-    // site. Just keep in mind that the gener8ted result will contain a
-    // couple symlinked directories, so if you're uploading, you're pro8a8ly
-    // gonna want to resolve those yourself.
-    'out-path': {
-      type: 'value',
-    },
-
     // Thum8nail gener8tion is *usually* something you want, 8ut it can 8e
     // kinda a pain to run every time, since it does necessit8te reading
     // every media file at run time. Pass this to skip it.
@@ -217,20 +191,6 @@ async function main() {
       type: 'flag',
     },
 
-    // Only want to 8uild one language during testing? This can chop down
-    // 8uild times a pretty 8ig chunk! Just pass a single language code.
-    lang: {
-      type: 'value',
-    },
-
-    // Working without a dev server and just using file:// URLs in your we8
-    // 8rowser? This will automatically append index.html to links across
-    // the site. Not recommended for production, since it isn't guaranteed
-    // 100% error-free (and index.html-style links are less pretty anyway).
-    'append-index-html': {
-      type: 'flag',
-    },
-
     // Want sweet, sweet trace8ack info in aggreg8te error messages? This
     // will print all the juicy details (or at least the first relevant
     // line) right to your output, 8ut also pro8a8ly give you a headache
@@ -268,32 +228,26 @@ async function main() {
     'precache-data': {
       type: 'flag',
     },
-
-    [parseOptions.handleUnknown]: () => {},
   });
 
-  const dataPath = miscOptions['data-path'] || process.env.HSMUSIC_DATA;
-  const mediaPath = miscOptions['media-path'] || process.env.HSMUSIC_MEDIA;
-  const langPath = miscOptions['lang-path'] || process.env.HSMUSIC_LANG; // Can 8e left unset!
-  const outputPath = miscOptions['out-path'] || process.env.HSMUSIC_OUT;
-
-  const skipThumbs = miscOptions['skip-thumbs'] ?? false;
-  const thumbsOnly = miscOptions['thumbs-only'] ?? false;
-  const noBuild = miscOptions['no-build'] ?? false;
-  const writeOneLanguage = miscOptions['lang'];
+  const dataPath = cliOptions['data-path'] || process.env.HSMUSIC_DATA;
+  const mediaPath = cliOptions['media-path'] || process.env.HSMUSIC_MEDIA;
+  const langPath = cliOptions['lang-path'] || process.env.HSMUSIC_LANG; // Can 8e left unset!
 
-  const showAggregateTraces = miscOptions['show-traces'] ?? false;
+  const skipThumbs = cliOptions['skip-thumbs'] ?? false;
+  const thumbsOnly = cliOptions['thumbs-only'] ?? false;
+  const noBuild = cliOptions['no-build'] ?? false;
 
-  const appendIndexHTML = miscOptions['append-index-html'] ?? false;
+  const showAggregateTraces = cliOptions['show-traces'] ?? false;
 
-  const precacheData = miscOptions['precache-data'] ?? false;
-  const showInvalidPropertyAccesses = miscOptions['show-invalid-property-accesses'] ?? false;
+  const precacheData = cliOptions['precache-data'] ?? false;
+  const showInvalidPropertyAccesses = cliOptions['show-invalid-property-accesses'] ?? false;
 
   // Makes writing a little nicer on CPU theoretically, 8ut also costs in
   // performance right now 'cuz it'll w8 for file writes to 8e completed
   // 8efore moving on to more data processing. So, defaults to zero, which
   // disa8les the queue feature altogether.
-  const queueSize = +(miscOptions['queue-size'] ?? 0);
+  const queueSize = +(cliOptions['queue-size'] ?? 0);
 
   {
     let errored = false;
@@ -305,39 +259,11 @@ async function main() {
     };
     error(!dataPath, `Expected --data-path option or HSMUSIC_DATA to be set`);
     error(!mediaPath, `Expected --media-path option or HSMUSIC_MEDIA to be set`);
-    error(!outputPath, `Expected --out-path option or HSMUSIC_OUT to be set`);
     if (errored) {
       return;
     }
   }
 
-  if (appendIndexHTML) {
-    logWarn`Appending index.html to link hrefs. (Note: not recommended for production release!)`;
-    link.globalOptions.appendIndexHTML = true;
-  }
-
-  // NOT for ena8ling or disa8ling specific features of the site!
-  // This is only in charge of what general groups of files to 8uild.
-  // They're here to make development quicker when you're only working
-  // on some particular area(s) of the site rather than making changes
-  // across all of them.
-  const writeFlags = await parseOptions(process.argv.slice(2), {
-    all: {type: 'flag'}, // Defaults to true if none 8elow specified.
-
-    // Kinda a hack t8h!
-    ...Object.fromEntries(
-      Object.keys(pageSpecs).map((key) => [key, {type: 'flag'}])
-    ),
-
-    [parseOptions.handleUnknown]: () => {},
-  });
-
-  const writeAll = !Object.keys(writeFlags).length || writeFlags.all;
-
-  logInfo`Writing site pages: ${
-    writeAll ? 'all' : Object.keys(writeFlags).join(', ')
-  }`;
-
   const niceShowAggregate = (error, ...opts) => {
     showAggregate(error, {
       showTraces: showAggregateTraces,
@@ -502,8 +428,7 @@ async function main() {
   }
 
   const internalDefaultLanguage = await processLanguageFile(
-    path.join(__dirname, DEFAULT_STRINGS_FILE)
-  );
+    path.join(__dirname, DEFAULT_STRINGS_FILE));
 
   let languages;
   if (langPath) {
@@ -511,14 +436,11 @@ async function main() {
       filter: (f) => path.extname(f) === '.json',
     });
 
-    const results = await progressPromiseAll(
-      `Reading & processing language files.`,
-      languageDataFiles.map((file) => processLanguageFile(file))
-    );
+    const results = await progressPromiseAll(`Reading & processing language files.`,
+      languageDataFiles.map((file) => processLanguageFile(file)));
 
     languages = Object.fromEntries(
-      results.map((language) => [language.code, language])
-    );
+      results.map((language) => [language.code, language]));
   } else {
     languages = {};
   }
@@ -556,13 +478,6 @@ async function main() {
 
   if (noBuild) {
     logInfo`Not generating any site or page files this run (--no-build passed).`;
-  } else if (writeOneLanguage && !(writeOneLanguage in languages)) {
-    logError`Specified to write only ${writeOneLanguage}, but there is no strings file with this language code!`;
-    return;
-  } else if (writeOneLanguage) {
-    logInfo`Writing only language ${writeOneLanguage} this run.`;
-  } else {
-    logInfo`Writing all languages.`;
   }
 
   {
@@ -632,310 +547,55 @@ async function main() {
 
   if (noBuild) return;
 
-  const buildDictionary = pageSpecs;
-
-  await writeSymlinks({
-    srcRootDirname: __dirname,
-    mediaPath,
-    outputPath,
-    urls,
+  const developersComment = generateDevelopersCommentHTML({
+    buildTime: BUILD_TIME,
+    commit: COMMIT,
+    wikiData,
   });
 
-  await writeSharedFilesAndPages({
+  return selectedBuildMode.go({
+    cliOptions,
+    dataPath,
     mediaPath,
-    outputPath,
-    urls,
+    queueSize,
+    srcRootPath: __dirname,
 
-    language: finalDefaultLanguage,
+    defaultLanguage: finalDefaultLanguage,
+    languages,
     wikiData,
-    wikiDataJSON: generateGlobalWikiDataJSON({
-      serializeThings,
-      wikiData,
-    })
-  });
-
-  const buildSteps = writeAll
-    ? Object.entries(buildDictionary)
-    : Object.entries(buildDictionary).filter(([flag]) => writeFlags[flag]);
-
-  let writes;
-  {
-    let error = false;
-
-    const buildStepsWithTargets = buildSteps
-      .map(([flag, pageSpec]) => {
-        // Condition not met: skip this build step altogether.
-        if (pageSpec.condition && !pageSpec.condition({wikiData})) {
-          return null;
-        }
-
-        // May still call writeTargetless if present.
-        if (!pageSpec.targets) {
-          return {flag, pageSpec, targets: []};
-        }
-
-        if (!pageSpec.write) {
-          logError`${flag + '.targets'} is specified, but ${flag + '.write'} is missing!`;
-          error = true;
-          return null;
-        }
-
-        const targets = pageSpec.targets({wikiData});
-        if (!Array.isArray(targets)) {
-          logError`${flag + '.targets'} was called, but it didn't return an array! (${typeof targets})`;
-          error = true;
-          return null;
-        }
-
-        return {flag, pageSpec, targets};
-      })
-      .filter(Boolean);
-
-    if (error) {
-      return;
-    }
-
-    writes = progressCallAll('Computing page & data writes.', buildStepsWithTargets.flatMap(({flag, pageSpec, targets}) => {
-      const writesFns = targets.map(target => () => {
-        const writes = pageSpec.write(target, {wikiData})?.slice() || [];
-        const valid = validateWrites(writes, {
-          functionName: flag + '.write',
-          urlSpec,
-        });
-        error ||=! valid;
-        return valid ? writes : [];
-      });
-
-      if (pageSpec.writeTargetless) {
-        writesFns.push(() => {
-          const writes = pageSpec.writeTargetless({wikiData});
-          const valid = validateWrites(writes, {
-            functionName: flag + '.writeTargetless',
-            urlSpec,
-          });
-          error ||=! valid;
-          return valid ? writes : [];
-        });
-      }
-
-      return writesFns;
-    })).flat();
-
-    if (error) {
-      return;
-    }
-  }
-
-  const pageWrites = writes.filter(({type}) => type === 'page');
-  const dataWrites = writes.filter(({type}) => type === 'data');
-  const redirectWrites = writes.filter(({type}) => type === 'redirect');
-
-  if (writes.length) {
-    logInfo`Total of ${writes.length} writes returned. (${pageWrites.length} page, ${dataWrites.length} data [currently skipped], ${redirectWrites.length} redirect)`;
-  } else {
-    logWarn`No writes returned at all, so exiting early. This is probably a bug!`;
-    return;
-  }
-
-  /*
-  await progressPromiseAll(`Writing data files shared across languages.`, queue(
-    dataWrites.map(({path, data}) => () => {
-      const bound = {};
-
-      bound.serializeLink = bindOpts(serializeLink, {});
-
-      bound.serializeContribs = bindOpts(serializeContribs, {});
-
-      bound.serializeImagePaths = bindOpts(serializeImagePaths, {
-        thumb
-      });
-
-      bound.serializeCover = bindOpts(serializeCover, {
-        [bindOpts.bindIndex]: 2,
-        serializeImagePaths: bound.serializeImagePaths,
-        urls
-      });
-
-      bound.serializeGroupsForAlbum = bindOpts(serializeGroupsForAlbum, {
-        serializeLink
-      });
-
-      bound.serializeGroupsForTrack = bindOpts(serializeGroupsForTrack, {
-        serializeLink
-      });
-
-      // TODO: This only supports one <>-style argument.
-      return writeData(path[0], path[1], data({...bound}));
-    }),
-    queueSize
-  ));
-  */
-
-  const perLanguageFn = async (language, i, entries) => {
-    const baseDirectory =
-      language === finalDefaultLanguage ? '' : language.code;
-
-    console.log(`\x1b[34;1m${`[${i + 1}/${entries.length}] ${language.code} (-> /${baseDirectory}) `.padEnd(60, '-')}\x1b[0m`);
-
-    await progressPromiseAll(`Writing ${language.code}`, queue([
-      ...pageWrites.map((props) => () => {
-        const {path, page} = props;
-
-        const pageSubKey = path[0];
-        const urlArgs = path.slice(1);
-
-        const localizedPaths = withEntries(languages, entries => entries
-          .filter(([key, language]) => key !== 'default' && !language.hidden)
-          .map(([_key, language]) => [
-            language.code,
-            getPagePaths({
-              outputPath,
-              urls,
-
-              baseDirectory:
-                (language === finalDefaultLanguage
-                  ? ''
-                  : language.code),
-              fullKey: 'localized.' + pageSubKey,
-              urlArgs,
-            }),
-          ]));
-
-        const paths = getPagePaths({
-          outputPath,
-          urls,
-
-          baseDirectory,
-          fullKey: 'localized.' + pageSubKey,
-          urlArgs,
-        });
-
-        const to = getURLsFrom({
-          urls,
-          baseDirectory,
-          pageSubKey,
-          paths,
-        });
-
-        const absoluteTo = (targetFullKey, ...args) => {
-          const [groupKey, subKey] = targetFullKey.split('.');
-          const from = urls.from('shared.root');
-          return (
-            '/' +
-            (groupKey === 'localized' && baseDirectory
-              ? from.to(
-                  'localizedWithBaseDirectory.' + subKey,
-                  baseDirectory,
-                  ...args
-                )
-              : from.to(targetFullKey, ...args))
-          );
-        };
-
-        const bound = bindUtilities({
-          language,
-          to,
-          wikiData,
-        });
-
-        const pageInfo = page({
-          ...bound,
-
-          language,
-
-          absoluteTo,
-          relativeTo: to,
-          to,
-          urls,
-
-          getSizeOfAdditionalFile,
-        });
-
-        const oEmbedJSON = generateOEmbedJSON(pageInfo, {
-          language,
-          wikiData,
-        });
-
-        const oEmbedJSONHref =
-          oEmbedJSON &&
-          wikiData.wikiInfo.canonicalBase &&
-          wikiData.wikiInfo.canonicalBase +
-            urls
-              .from('shared.root')
-              .to('shared.path', paths.pathname + 'oembed.json');
-
-        const pageHTML = generateDocumentHTML(pageInfo, {
-          buildTime: BUILD_TIME,
-          cachebust: '?' + CACHEBUST,
-          commit: COMMIT,
-          defaultLanguage: finalDefaultLanguage,
-          getThemeString: bound.getThemeString,
-          language,
-          languages,
-          localizedPaths,
-          oEmbedJSONHref,
-          paths,
-          to,
-          transformMultiline: bound.transformMultiline,
-          wikiData,
-        });
-
-        return writePage({
-          html: pageHTML,
-          oEmbedJSON,
-          paths,
-        });
-      }),
-      ...redirectWrites.map(({fromPath, toPath, title: titleFn}) => () => {
-        const title = titleFn({
-          language,
-        });
-
-        const from = getPagePaths({
-          outputPath,
-          urls,
-
-          baseDirectory,
-          fullKey: 'localized.' + fromPath[0],
-          urlArgs: fromPath.slice(1),
-        });
-
-        const to = getURLsFrom({
-          urls,
-          baseDirectory,
-          pageSubKey: fromPath[0],
-          paths: from,
-        });
-
-        const target = to('localized.' + toPath[0], ...toPath.slice(1));
-        const html = generateRedirectHTML(title, target, {language});
-        return writePage({html, paths: from});
-      }),
-    ], queueSize));
-  };
+    urls,
+    urlSpec,
 
-  await wrapLanguages(perLanguageFn, {
-    languages,
-    writeOneLanguage,
+    cachebust: '?' + CACHEBUST,
+    developersComment,
+    getSizeOfAdditionalFile,
   });
-
-  // The single most important step.
-  logInfo`Written!`;
 }
 
 // TODO: isMain detection isn't consistent across platforms here
 /* eslint-disable-next-line no-constant-condition */
 if (true || isMain(import.meta.url) || path.basename(process.argv[1]) === 'hsmusic') {
-  main()
-    .catch((error) => {
+  (async () => {
+    let result;
+
+    try {
+      result = await main();
+    } catch (error) {
       if (error instanceof AggregateError) {
         showAggregate(error);
       } else {
         console.error(error);
       }
-    })
-    .then(() => {
-      decorateTime.displayTime();
-      CacheableObject.showInvalidAccesses();
-    });
+    }
+
+    if (result !== true) {
+      process.exit(1);
+      return;
+    }
+
+    decorateTime.displayTime();
+    CacheableObject.showInvalidAccesses();
+
+    process.exit(0);
+  })();
 }