upd8: online thumbs cache

This code is actually from January 17th.

2 files changed, 191 insertions, 22 deletions

diff --git a/src/upd8.js b/src/upd8.js
index 6d6f51c1..30f1b143 100755
--- a/src/upd8.js
+++ b/src/upd8.js
@@ -34,7 +34,7 @@
 import '#import-heck';
 
 import {execSync} from 'node:child_process';
-import {readdir, readFile, stat} from 'node:fs/promises';
+import {readdir, readFile, stat, writeFile} from 'node:fs/promises';
 import * as path from 'node:path';
 import {fileURLToPath} from 'node:url';
 
@@ -91,6 +91,7 @@ import {
   applyLocalizedWithBaseDirectory,
   applyURLSpecOverriding,
   generateURLs,
+  getOrigin,
   internalDefaultURLSpecFile,
   processURLSpecFromFile,
 } from '#urls';
@@ -148,8 +149,8 @@ async function main() {
       {...defaultStepStatus, name: `migrate thumbnails`,
         for: ['thumbs']},
 
-    loadThumbnailCache:
-      {...defaultStepStatus, name: `load thumbnail cache file`,
+    loadOfflineThumbnailCache:
+      {...defaultStepStatus, name: `load offline thumbnail cache file`,
         for: ['thumbs', 'build']},
 
     generateThumbnails:
@@ -192,6 +193,10 @@ async function main() {
       {...defaultStepStatus, name: `load internal & custom url spec files`,
         for: ['build']},
 
+    loadOnlineThumbnailCache:
+      {...defaultStepStatus, name: `load online thumbnail cache file`,
+        for: ['thumbs', 'build']},
+
     // TODO: This should be split into load/watch steps.
     loadInternalDefaultLanguage:
       {...defaultStepStatus, name: `load internal default language`,
@@ -953,7 +958,7 @@ async function main() {
   }
 
   if (stepStatusSummary.generateThumbnails.status === STATUS_NOT_STARTED) {
-    Object.assign(stepStatusSummary.loadThumbnailCache, {
+    Object.assign(stepStatusSummary.loadOfflineThumbnailCache, {
       status: STATUS_NOT_APPLICABLE,
       annotation: `using cache from thumbnail generation`,
     });
@@ -1247,16 +1252,17 @@ async function main() {
   };
 
   if (
-    stepStatusSummary.loadThumbnailCache.status === STATUS_NOT_STARTED &&
+    stepStatusSummary.loadOfflineThumbnailCache.status === STATUS_NOT_STARTED &&
     stepStatusSummary.generateThumbnails.status === STATUS_NOT_STARTED
   ) {
-    throw new Error(`Unable to continue with both loadThumbnailCache and generateThumbnails`);
+    throw new Error(`Unable to continue with both loadOfflineThumbnailCache and generateThumbnails`);
   }
 
   let thumbsCache;
 
-  if (stepStatusSummary.loadThumbnailCache.status === STATUS_NOT_STARTED) {
-    Object.assign(stepStatusSummary.loadThumbnailCache, {
+  // TODO: Skip this step if we're using online thumbs
+  if (stepStatusSummary.loadOfflineThumbnailCache.status === STATUS_NOT_STARTED) {
+    Object.assign(stepStatusSummary.loadOfflineThumbnailCache, {
       status: STATUS_STARTED_NOT_DONE,
       timeStart: Date.now(),
     });
@@ -1272,7 +1278,7 @@ async function main() {
         logError`that you'll be good to go and don't need to process thumbnails`
         logError`again!`;
 
-        Object.assign(stepStatusSummary.loadThumbnailCache, {
+        Object.assign(stepStatusSummary.loadOfflineThumbnailCache, {
           status: STATUS_FATAL_ERROR,
           annotation: `cache does not exist`,
           timeEnd: Date.now(),
@@ -1290,7 +1296,7 @@ async function main() {
         logError`to help you out with troubleshooting!`;
         logError`${'https://hsmusic.wiki/discord/'}`;
 
-        Object.assign(stepStatusSummary.loadThumbnailCache, {
+        Object.assign(stepStatusSummary.loadOfflineThumbnailCache, {
           status: STATUS_FATAL_ERROR,
           annotation: `cache malformed or unreadable`,
           timeEnd: Date.now(),
@@ -1303,7 +1309,7 @@ async function main() {
 
     logInfo`Thumbnail cache file successfully read.`;
 
-    Object.assign(stepStatusSummary.loadThumbnailCache, {
+    Object.assign(stepStatusSummary.loadOfflineThumbnailCache, {
       status: STATUS_DONE_CLEAN,
       timeEnd: Date.now(),
       memory: process.memoryUsage(),
@@ -1983,10 +1989,173 @@ async function main() {
     memory: process.memoryUsage(),
   });
 
+  if (!getOrigin(urlSpec.thumb.prefix)) {
+    Object.assign(stepStatusSummary.loadOnlineThumbnailCache, {
+      status: STATUS_NOT_APPLICABLE,
+      annotation: `using offline thumbs`,
+    });
+  }
+
   applyLocalizedWithBaseDirectory(urlSpec);
 
   const urls = generateURLs(urlSpec);
 
+  if (stepStatusSummary.loadOnlineThumbnailCache.status === STATUS_NOT_STARTED) loadOnlineThumbnailCache: {
+    Object.assign(stepStatusSummary.loadOnlineThumbnailCache, {
+      status: STATUS_STARTED_NOT_DONE,
+      timeStart: Date.now(),
+    });
+
+    let onlineThumbsCache = null;
+
+    const cacheFile = path.join(wikiCachePath, 'online-thumbnail-cache.json');
+
+    let readError = null;
+    let writeError = null;
+
+    try {
+      onlineThumbsCache = JSON.parse(await readFile(cacheFile));
+    } catch (caughtError) {
+      readError = caughtError;
+    }
+
+    if (onlineThumbsCache) obliterateThumbsCache: {
+      if (!onlineThumbsCache._urlPrefix) {
+        // Well, it doesn't even count.
+        onlineThumbsCache = null;
+        break obliterateThumbsCache;
+      }
+
+      if (onlineThumbsCache._urlPrefix !== urlSpec.thumb.cache) {
+        logInfo`Local copy of online thumbs cache is for a different prefix.`;
+        logInfo`It'll be downloaded and replaced, for reuse next time.`;
+        paragraph = false;
+
+        onlineThumbsCache = null;
+        break obliterateThumbsCache;
+      }
+
+      let stats;
+      try {
+        stats = await stat(cacheFile);
+      } catch {
+        logInfo`Unable to get the stats of local copy of online thumbs cache...`;
+        logInfo`This is really weird, since we *were* able to read it...`;
+        logInfo`We're just going to try writing to it and download fresh!`;
+        paragraph = false;
+
+        onlineThumbsCache = null;
+        break obliterateThumbsCache;
+      }
+
+      const delta = Date.now() - stats.mtimeMs;
+      const minute = 60 * 1000;
+      const delay = 60 * minute;
+
+      const whenst = duration => `~${Math.ceil(duration / minute)} min`;
+
+      if (delta < delay) {
+        logInfo`Online thumbs cache was downloaded recently, skipping for this build.`;
+        logInfo`Next scheduled is in ${whenst(delay - delta)}, or by using ${'--refresh-online-thumbs'}.`;
+        paragraph = false;
+
+        Object.assign(stepStatusSummary.loadOnlineThumbnailCache, {
+          status: STATUS_DONE_CLEAN,
+          annotation: `reusing local copy, earlier than scheduled based on file mtime`,
+        });
+
+        thumbsCache = onlineThumbsCache;
+
+        break loadOnlineThumbnailCache;
+      } else {
+        logInfo`Online thumbs cache hasn't been downloaded for a little while.`;
+        logInfo`It'll be downloaded this build, then again in ${whenst(delay)}.`;
+        onlineThumbsCache = null;
+        paragraph = false;
+      }
+    }
+
+    try {
+      await writeFile(cacheFile, JSON.stringify(onlineThumbsCache ?? {}));
+    } catch (caughtError) {
+      writeError = caughtError;
+    }
+
+    if (readError && writeError && readError.code !== 'ENOENT') {
+      console.error(readError);
+      logWarn`Wasn't able to read the local copy of the`;
+      logWarn`online thumbs cache file...`;
+      console.error(writeError);
+      logWarn`...or write to it, either.`;
+      logWarn`The online thumbs cache will be downloaded`;
+      logWarn`for every build until you investigate this path:`;
+      logWarn`${cacheFile}`;
+      paragraph = false;
+    } else if (readError && readError.code === 'ENOENT' && !writeError) {
+      logInfo`No local copy of online thumbs cache.`;
+      logInfo`It'll be downloaded this time and reused next time.`;
+      paragraph = false;
+    } else if (readError && readError.code === 'ENOENT' && writeError) {
+      console.error(writeError);
+      logWarn`Doesn't look like we can write a local copy of`;
+      logWarn`the offline thumbs cache, at this path:`;
+      logWarn`${cacheFile}`;
+      logWarn`The online thumbs cache will be downloaded`;
+      logWarn`for every build until you investigate that.`;
+      paragraph = false;
+    }
+
+    const url = new URL(urlSpec.thumb.prefix);
+    url.pathname = path.posix.join(url.pathname, 'thumbnail-cache.json');
+
+    try {
+      onlineThumbsCache = await fetch(url).then(res => res.json());
+    } catch (error) {
+      console.error(error);
+      logWarn`There was an error downloading the online thumbnail cache.`;
+      logWarn`The wiki will act as though no thumbs are available at all.`;
+      paragraph = false;
+
+      Object.assign(stepStatusSummary.loadOnlineThumbnailCache, {
+        status: STATUS_HAS_WARNINGS,
+        annotation: `failed to download`,
+      });
+
+      onlineThumbsCache = {};
+      thumbsCache = {};
+
+      break loadOnlineThumbnailCache;
+    }
+
+    onlineThumbsCache._prefix = urlSpec.thumb.prefix;
+
+    thumbsCache = onlineThumbsCache;
+
+    if (onlineThumbsCache && !writeError) {
+      try {
+        await writeFile(cacheFile, JSON.stringify(onlineThumbsCache));
+      } catch (error) {
+        console.error(error);
+        logWarn`There was an error saving a local copy of the`;
+        logWarn`online thumbnail cache. It'll be fetched again`;
+        logWarn`next time.`;
+        paragraph = false;
+
+        Object.assign(stepStatusSummary.loadOnlineThumbnailCache, {
+          status: STATUS_HAS_WARNINGS,
+          annotation: `failed to download`,
+        });
+
+        break loadOnlineThumbnailCache;
+      }
+    }
+
+    Object.assign(stepStatusSummary.loadOnlineThumbnailCache, {
+      status: STATUS_DONE_CLEAN,
+      timeStart: Date.now(),
+    });
+  }
+
   const languageReloading =
     stepStatusSummary.watchLanguageFiles.status === STATUS_NOT_STARTED;
 
diff --git a/src/urls.js b/src/urls.js
index faa58237..83a8b904 100644
--- a/src/urls.js
+++ b/src/urls.js
@@ -53,15 +53,7 @@ export function generateURLs(urlSpec) {
     const rebasePrefix =
       '../'.repeat(fromPrefix.split('/').filter(Boolean).length);
 
-    const originOfPrefix = prefix => {
-      try {
-        return new URL(prefix).origin;
-      } catch {
-        return null;
-      }
-    };
-
-    const fromOrigin = originOfPrefix(fromPrefix);
+    const fromOrigin = getOrigin(fromPrefix);
 
     const pathHelper = (toPath, toGroup) => {
       let B = trimLeadingSlash(toPath);
@@ -74,9 +66,9 @@ export function generateURLs(urlSpec) {
       const toPrefix = toGroup.prefix;
 
       if (toPrefix !== fromPrefix) {
-        // Compare origins. Note that originOfPrefix() can
+        // Compare origins. Note that getOrigin() can
         // be null for both prefixes.
-        const toOrigin = originOfPrefix(toPrefix);
+        const toOrigin = getOrigin(toPrefix);
         if (fromOrigin === toOrigin) {
           // Go to the root, add the to-group's prefix, then
           // continue with normal path.relative() behavior.
@@ -233,6 +225,14 @@ export function generateURLs(urlSpec) {
   return generateFrom();
 }
 
+export function getOrigin(prefix) {
+  try {
+    return new URL(prefix).origin;
+  } catch {
+    return null;
+  }
+}
+
 const thumbnailHelper = (name) => (file) =>
   file.replace(/\.(jpg|png)$/, name + '.jpg');