util: remove findFiles util, adapt traverse

5 files changed, 66 insertions, 75 deletions

diff --git a/src/data/yaml.js b/src/data/yaml.js
index 9ebce64..c0058da 100644
--- a/src/data/yaml.js
+++ b/src/data/yaml.js
@@ -30,7 +30,7 @@ import {
 } from '../util/wiki-data.js';
 
 import find, {bindFind} from '../util/find.js';
-import {findFiles} from '../util/io.js';
+import {traverse} from '../util/node-utils.js';
 
 // --> General supporting stuff
 
@@ -739,13 +739,12 @@ export const dataSteps = [
 
   {
     title: `Process album files`,
-    files: async (dataPath) =>
-      (
-        await findFiles(path.join(dataPath, DATA_ALBUM_DIRECTORY), {
-          filter: (f) => path.extname(f) === '.yaml',
-          joinParentDirectory: false,
-        })
-      ).map(file => path.join(DATA_ALBUM_DIRECTORY, file)),
+
+    files: dataPath =>
+      traverse(path.join(dataPath, DATA_ALBUM_DIRECTORY), {
+        filterFile: name => path.extname(name) === '.yaml',
+        prefixPath: DATA_ALBUM_DIRECTORY,
+      }),
 
     documentMode: documentModes.headerAndEntries,
     processHeaderDocument: processAlbumDocument,
@@ -977,13 +976,12 @@ export const dataSteps = [
 
   {
     title: `Process static page files`,
-    files: async (dataPath) =>
-      (
-        await findFiles(path.join(dataPath, DATA_STATIC_PAGE_DIRECTORY), {
-          filter: f => path.extname(f) === '.yaml',
-          joinParentDirectory: false,
-        })
-      ).map(file => path.join(DATA_STATIC_PAGE_DIRECTORY, file)),
+
+    files: dataPath =>
+      traverse(path.join(dataPath, DATA_STATIC_PAGE_DIRECTORY), {
+        filterFile: name => path.extname(name) === '.yaml',
+        prefixPath: DATA_STATIC_PAGE_DIRECTORY,
+      }),
 
     documentMode: documentModes.onePerFile,
     processDocument: processStaticPageDocument,
diff --git a/src/gen-thumbs.js b/src/gen-thumbs.js
index 655ab6d..2d432e4 100644
--- a/src/gen-thumbs.js
+++ b/src/gen-thumbs.js
@@ -220,6 +220,7 @@ export async function clearThumbs(mediaPath, {
     const unsafeFiles = thumbFiles.filter(file => {
       if (path.extname(file) !== '.jpg') return true;
       if (thumbtacks.every(tack => !file.includes(tack))) return true;
+      if (path.relative(mediaPath, file).startsWith('../')) return true;
       return false;
     });
 
@@ -240,7 +241,7 @@ export async function clearThumbs(mediaPath, {
     await progressPromiseAll(`Removing thumbnail files`, queue(
       thumbFiles.map(file => async () => {
         try {
-          await unlink(path.join(mediaPath, file));
+          await unlink(file);
         } catch (error) {
           if (error.code !== 'ENOENT') {
             errored.push(file);
@@ -539,6 +540,7 @@ export async function traverseSourceImagePaths(mediaPath, {target}) {
 
   return await traverse(mediaPath, {
     pathStyle: (target === 'verify' ? 'posix' : 'device'),
+    prefixPath: '',
 
     filterFile(name) {
       const ext = path.extname(name);
diff --git a/src/upd8.js b/src/upd8.js
index 4bbdb95..1e52c55 100755
--- a/src/upd8.js
+++ b/src/upd8.js
@@ -60,7 +60,6 @@ import {
   WIKI_INFO_FILE,
 } from './data/yaml.js';
 
-import {findFiles} from './util/io.js';
 import link from './util/link.js';
 import {isMain, traverse} from './util/node-utils.js';
 import {empty, showAggregate, withEntries} from './util/sugar.js';
@@ -601,8 +600,9 @@ async function main() {
 
   let languages;
   if (langPath) {
-    const languageDataFiles = await findFiles(langPath, {
-      filter: (f) => path.extname(f) === '.json',
+    const languageDataFiles = await traverse(langPath, {
+      filterFile: name => path.extname(name) === '.json',
+      pathStyle: 'device',
     });
 
     const results = await progressPromiseAll(`Reading & processing language files.`,
@@ -687,20 +687,21 @@ async function main() {
   // we can't super easily know which ones are referenced at runtime, just
   // cheat and get file sizes for all images under media. (This includes
   // additional files which are images.)
-  const imageFilePaths = (await traverse(mediaPath, {
-    pathStyle: 'device',
-    filterDir: dir => dir !== '.git',
-    filterFile: file => (
-      ['.png', '.gif', '.jpg'].includes(path.extname(file)) &&
-        !isThumb(file)),
-  }))
-    .map(file => ({
-      device: path.join(mediaPath, file),
-      media:
-        urls
-          .from('media.root')
-          .to('media.path', file.split(path.sep).join('/')),
-    }));
+  const imageFilePaths =
+    await traverse(mediaPath, {
+      pathStyle: 'device',
+      filterDir: dir => dir !== '.git',
+      filterFile: file =>
+        ['.png', '.gif', '.jpg'].includes(path.extname(file)) &&
+        !isThumb(file),
+    }).then(files => files
+        .map(file => ({
+          device: file,
+          media:
+            urls
+              .from('media.root')
+              .to('media.path', path.relative(mediaPath, file).split(path.sep).join('/')),
+        })));
 
   const getSizeOfMediaFileHelper = paths => (mediaPath) => {
     const pair = paths.find(({media}) => media === mediaPath);
diff --git a/src/util/io.js b/src/util/io.js
deleted file mode 100644
index 12e87f4..0000000
--- a/src/util/io.js
+++ /dev/null
@@ -1,23 +0,0 @@
-// Utility functions for interacting with files and other external data
-// interfacey constructs.
-
-import {readdir} from 'fs/promises';
-import * as path from 'path';
-
-export async function findFiles(dataPath, {
-  filter = () => true,
-  joinParentDirectory = true,
-} = {}) {
-  let files;
-  try {
-    files = await readdir(dataPath);
-  } catch (error) {
-    throw Object.assign(
-      new AggregateError([error], `Failed to list files from ${dataPath}`),
-      {code: error.code});
-  }
-
-  return files
-    .filter((file) => filter(file))
-    .map((file) => (joinParentDirectory ? path.join(dataPath, file) : file));
-}
diff --git a/src/util/node-utils.js b/src/util/node-utils.js
index 3c0dd4c..2fb7e8d 100644
--- a/src/util/node-utils.js
+++ b/src/util/node-utils.js
@@ -1,6 +1,6 @@
 // Utility functions which are only relevant to particular Node.js constructs.
 
-import {readdir} from 'fs/promises';
+import {readdir, stat} from 'fs/promises';
 import {fileURLToPath} from 'url';
 import * as path from 'path';
 
@@ -56,34 +56,47 @@ export function isMain(importMetaURL) {
   ].includes(relative);
 }
 
-// Like readdir... but it's recursive!
-export function traverse(startDirPath, {
+// Like readdir... but it's recursive! This returns a flat list of file paths.
+// By default, the paths include the provided top/root path, but this can be
+// changed with prefixPath to prefix some other path, or to just return paths
+// relative to the root. Change pathStyle to specify posix or win32, or leave
+// it as the default device-correct style. Provide a filterDir function to
+// control which directory names are traversed at all, and filterFile to
+// select which filenames are included in the final list.
+export async function traverse(rootPath, {
   pathStyle = 'device',
   filterFile = () => true,
-  filterDir = () => true
+  filterDir = () => true,
+  prefixPath = rootPath,
 } = {}) {
-  const pathJoin = {
+  const pathJoinDevice = path.join;
+  const pathJoinStyle = {
     'device': path.join,
     'posix': path.posix.join,
     'win32': path.win32.join,
   }[pathStyle];
 
-  if (!pathJoin) {
+  if (!pathJoinStyle) {
     throw new Error(`Expected pathStyle to be device, posix, or win32`);
   }
 
-  const recursive = (names, subDirPath) =>
-    Promise.all(names.map(name =>
-      readdir(pathJoin(startDirPath, subDirPath, name)).then(
-        names =>
-          (filterDir(name)
-            ? recursive(names, pathJoin(subDirPath, name))
-            : []),
-        () =>
-          (filterFile(name)
-            ? [pathJoin(subDirPath, name)]
-            : []))))
-      .then(pathArrays => pathArrays.flat());
+  const recursive = (names, ...subdirectories) =>
+    Promise.all(names.map(async name => {
+      const devicePath = pathJoinDevice(rootPath, ...subdirectories, name);
+      const stats = await stat(devicePath);
 
-  return readdir(startDirPath).then(names => recursive(names, ''));
+      if (stats.isDirectory() && !filterDir(name)) return [];
+      else if (stats.isFile() && !filterFile(name)) return [];
+      else if (!stats.isDirectory() && !stats.isFile()) return [];
+
+      if (stats.isDirectory()) {
+        return recursive(await readdir(devicePath), ...subdirectories, name);
+      } else {
+        return pathJoinStyle(prefixPath, ...subdirectories, name);
+      }
+    }));
+
+  const names = await readdir(rootPath);
+  const results = await recursive(names);
+  return results.flat(Infinity);
 }