--clear-thumbs utility

3 files changed, 158 insertions, 24 deletions

diff --git a/src/gen-thumbs.js b/src/gen-thumbs.js
index 64f1f27a..26ac035e 100644
--- a/src/gen-thumbs.js
+++ b/src/gen-thumbs.js
@@ -89,11 +89,17 @@ import {spawn} from 'child_process';
 import {createHash} from 'crypto';
 import * as path from 'path';
 
-import {readFile, writeFile} from 'fs/promises'; // Whatcha know! Nice.
+import {
+  readFile,
+  stat,
+  unlink,
+  writeFile,
+} from 'fs/promises'; // Whatcha know! Nice.
 
 import {createReadStream} from 'fs'; // Still gotta import from 8oth tho, for createReadStream.
 
 import {
+  fileIssue,
   logError,
   logInfo,
   logWarn,
@@ -110,6 +116,8 @@ import {
 
 import {delay, queue} from './util/sugar.js';
 
+export const defaultMagickThreads = 8;
+
 function readFileMD5(filePath) {
   return new Promise((resolve, reject) => {
     const md5 = createHash('md5');
@@ -189,8 +197,95 @@ function generateImageThumbnails(filePath, {spawnConvert}) {
         promisifyProcess(convert('.' + ext, details), false)));
 }
 
+export async function clearThumbs(mediaPath, {
+  queueSize = 0,
+} = {}) {
+  if (!mediaPath) {
+    throw new Error('Expected mediaPath to be passed');
+  }
+
+  logInfo`Looking for thumbnails to clear out...`;
+
+  const thumbFiles = await traverse(mediaPath, {
+    filterFile: file => isThumb(file),
+    filterDir: name => name !== '.git',
+  });
+
+  if (thumbFiles.length) {
+    // Double-check files. Since we're unlinking (deleting) files,
+    // we're better off safe than sorry!
+    const thumbtacks = Object.keys(thumbnailSpec);
+    const unsafeFiles = thumbFiles.filter(file => {
+      if (path.extname(file) !== '.jpg') return true;
+      if (thumbtacks.every(tack => !file.includes(tack))) return true;
+      return false;
+    });
+
+    if (unsafeFiles.length > 0) {
+      logError`Detected files which we thought were safe, but don't actually seem to be thumbnails!`;
+      logError`List of files that were invalid: ${`(Please remove any personal files before reporting)`}`;
+      for (const file of unsafeFiles) {
+        console.error(file);
+      }
+      fileIssue();
+      return;
+    }
+
+    logInfo`Clearing out ${thumbFiles.length} thumbs.`;
+
+    const errored = [];
+
+    await progressPromiseAll(`Removing thumbnail files`, queue(
+      thumbFiles.map(file => async () => {
+        try {
+          await unlink(path.join(mediaPath, file));
+        } catch (error) {
+          if (error.code !== 'ENOENT') {
+            errored.push(file);
+          }
+        }
+      }),
+      queueSize));
+
+    if (errored.length) {
+      logError`Couldn't remove these paths (${errored.length}):`;
+      for (const file of errored) {
+        console.error(file);
+      }
+      logError`Check for permission errors?`;
+    } else {
+      logInfo`Successfully deleted all ${thumbFiles.length} thumbnail files!`;
+    }
+  } else {
+    logInfo`Didn't find any thumbs in media directory.`;
+    logInfo`${mediaPath}`;
+  }
+
+  let cacheExists = false;
+  try {
+    await stat(path.join(mediaPath, CACHE_FILE));
+    cacheExists = true;
+  } catch (error) {
+    if (error.code === 'ENOENT') {
+      logInfo`Cache file already missing, nothing to remove there.`;
+    } else {
+      logWarn`Failed to access cache file. Check its permissions?`;
+    }
+  }
+
+  if (cacheExists) {
+    try {
+      unlink(path.join(mediaPath, CACHE_FILE));
+      logInfo`Removed thumbnail cache file.`;
+    } catch (error) {
+      logWarn`Failed to remove cache file. Check its permissions?`;
+    }
+  }
+}
+
 export default async function genThumbs(mediaPath, {
   queueSize = 0,
+  magickThreads = defaultMagickThreads,
   quiet = false,
 } = {}) {
   if (!mediaPath) {
@@ -226,6 +321,8 @@ export default async function genThumbs(mediaPath, {
     logInfo`Found ImageMagick binary: ${convertInfo}`;
   }
 
+  quietInfo`Running up to ${magickThreads + ' magick threads'} simultaneously.`;
+
   let cache,
     firstRun = false;
   try {
@@ -306,32 +403,28 @@ export default async function genThumbs(mediaPath, {
     return true;
   }
 
+  logInfo`Generating thumbnails for ${entriesToGenerate.length} media files.`;
+  if (entriesToGenerate.length > 250) {
+    logInfo`Go get a latte - this could take a while!`;
+  }
+
   const failed = [];
   const succeeded = [];
   const writeMessageFn = () =>
     `Writing image thumbnails. [failed: ${failed.length}]`;
 
-  // This is actually sort of a lie, 8ecause we aren't doing synchronicity.
-  // (We pass queueSize = 1 to queue().) 8ut we still use progressPromiseAll,
-  // 'cuz the progress indic8tor is very cool and good.
-  await progressPromiseAll(
-    writeMessageFn,
+  await progressPromiseAll(writeMessageFn,
     queue(
-      entriesToGenerate.map(
-        ([filePath, md5]) =>
-          () =>
-            generateImageThumbnails(path.join(mediaPath, filePath), {spawnConvert}).then(
-              () => {
-                updatedCache[filePath] = md5;
-                succeeded.push(filePath);
-              },
-              (error) => {
-                failed.push([filePath, error]);
-              }
-            )
-      )
-    )
-  );
+      entriesToGenerate.map(([filePath, md5]) => () =>
+        generateImageThumbnails(path.join(mediaPath, filePath), {spawnConvert}).then(
+          () => {
+            updatedCache[filePath] = md5;
+            succeeded.push(filePath);
+          },
+          error => {
+            failed.push([filePath, error]);
+          })),
+      magickThreads));
 
   if (failed.length > 0) {
     for (const [path, error] of failed) {
@@ -359,7 +452,7 @@ export default async function genThumbs(mediaPath, {
 }
 
 export function isThumb(file) {
-  const thumbnailLabel = file.match(/\.([^.]+)\.[^.]+$/)?.[1];
+  const thumbnailLabel = file.match(/\.([^.]+)\.jpg$/)?.[1];
   return Object.keys(thumbnailSpec).includes(thumbnailLabel);
 }
 
@@ -369,6 +462,7 @@ if (isMain(import.meta.url)) {
       'media-path': {
         type: 'value',
       },
+
       'queue-size': {
         type: 'value',
         validate(size) {
@@ -377,6 +471,7 @@ if (isMain(import.meta.url)) {
           return true;
         },
       },
+
       queue: {alias: 'queue-size'},
     });
 
diff --git a/src/upd8.js b/src/upd8.js
index 317ccb03..2b4fb5f6 100755
--- a/src/upd8.js
+++ b/src/upd8.js
@@ -36,7 +36,11 @@ import * as path from 'path';
 import {fileURLToPath} from 'url';
 import wrap from 'word-wrap';
 
-import genThumbs, {isThumb} from './gen-thumbs.js';
+import genThumbs, {
+  clearThumbs,
+  defaultMagickThreads,
+  isThumb,
+} from './gen-thumbs.js';
 import {listingSpec, listingTargetSpec} from './listing-spec.js';
 import urlSpec from './url-spec.js';
 
@@ -195,6 +199,11 @@ async function main() {
       type: 'flag',
     },
 
+    'clear-thumbs': {
+      help: `Clear the thumbnail cache and remove generated thumbnail files from media directory\n\n(This skips building. Run again without --clear-thumbs to build the site.)`,
+      type: 'flag',
+    },
+
     // Just working on data entries and not interested in actually
     // generating site HTML yet? This flag will cut execution off right
     // 8efore any site 8uilding actually happens.
@@ -223,6 +232,11 @@ async function main() {
     },
     queue: {alias: 'queue-size'},
 
+    'magick-threads': {
+      help: `Process more or fewer thumbnail files at once with ImageMagick when generating thumbnails. (Each ImageMagick thread may also make use of multi-core processing at its own utility.)`,
+    },
+    magick: {alias: 'magick-threads'},
+
     // This option is super slow and has the potential for bugs! It puts
     // CacheableObject in a mode where every instance is a Proxy which will
     // keep track of invalid property accesses.
@@ -350,6 +364,7 @@ async function main() {
 
   const skipThumbs = cliOptions['skip-thumbs'] ?? false;
   const thumbsOnly = cliOptions['thumbs-only'] ?? false;
+  const clearThumbsFlag = cliOptions['clear-thumbs'] ?? false;
   const noBuild = cliOptions['no-build'] ?? false;
 
   const showAggregateTraces = cliOptions['show-traces'] ?? false;
@@ -362,6 +377,8 @@ async function main() {
   // before proceeding to more page processing.
   const queueSize = +(cliOptions['queue-size'] ?? defaultQueueSize);
 
+  const magickThreads = +(cliOptions['magick-threads'] ?? defaultMagickThreads);
+
   {
     let errored = false;
     const error = (cond, msg) => {
@@ -390,11 +407,25 @@ async function main() {
     return;
   }
 
+  if (clearThumbsFlag) {
+    await clearThumbs(mediaPath, {queueSize});
+
+    logInfo`All done! Remove ${'--clear-thumbs'} to run the next build.`;
+    if (skipThumbs) {
+      logInfo`And don't forget to remove ${'--skip-thumbs'} too, eh?`;
+    }
+    return;
+  }
+
   if (skipThumbs) {
     logInfo`Skipping thumbnail generation.`;
   } else {
     logInfo`Begin thumbnail generation... -----+`;
-    const result = await genThumbs(mediaPath, {queueSize, quiet: true});
+    const result = await genThumbs(mediaPath, {
+      queueSize,
+      magickThreads,
+      quiet: !thumbsOnly,
+    });
     logInfo`Done thumbnail generation! --------+`;
     if (!result) return;
     if (thumbsOnly) return;
diff --git a/src/util/cli.js b/src/util/cli.js
index 1ddc90e0..f83c8061 100644
--- a/src/util/cli.js
+++ b/src/util/cli.js
@@ -330,3 +330,11 @@ export function progressCallAll(msgOrMsgFn, array) {
 
   return vals;
 }
+
+export function fileIssue({
+  topMessage = `This shouldn't happen.`,
+} = {}) {
+  console.error(color.red(`${topMessage} Please let the HSMusic developers know:`));
+  console.error(color.red(`- https://hsmusic.wiki/feedback/`));
+  console.error(color.red(`- https://github.com/hsmusic/hsmusic-wiki/issues/`));
+}