diff options
| -rw-r--r-- | src/data/yaml.js | 28 | ||||
| -rw-r--r-- | src/gen-thumbs.js | 4 | ||||
| -rwxr-xr-x | src/upd8.js | 35 | ||||
| -rw-r--r-- | src/util/io.js | 23 | ||||
| -rw-r--r-- | src/util/node-utils.js | 51 |
5 files changed, 66 insertions, 75 deletions
diff --git a/src/data/yaml.js b/src/data/yaml.js index 9ebce642..c0058da3 100644 --- a/src/data/yaml.js +++ b/src/data/yaml.js @@ -30,7 +30,7 @@ import { } from '../util/wiki-data.js'; import find, {bindFind} from '../util/find.js'; -import {findFiles} from '../util/io.js'; +import {traverse} from '../util/node-utils.js'; // --> General supporting stuff @@ -739,13 +739,12 @@ export const dataSteps = [ { title: `Process album files`, - files: async (dataPath) => - ( - await findFiles(path.join(dataPath, DATA_ALBUM_DIRECTORY), { - filter: (f) => path.extname(f) === '.yaml', - joinParentDirectory: false, - }) - ).map(file => path.join(DATA_ALBUM_DIRECTORY, file)), + + files: dataPath => + traverse(path.join(dataPath, DATA_ALBUM_DIRECTORY), { + filterFile: name => path.extname(name) === '.yaml', + prefixPath: DATA_ALBUM_DIRECTORY, + }), documentMode: documentModes.headerAndEntries, processHeaderDocument: processAlbumDocument, @@ -977,13 +976,12 @@ export const dataSteps = [ { title: `Process static page files`, - files: async (dataPath) => - ( - await findFiles(path.join(dataPath, DATA_STATIC_PAGE_DIRECTORY), { - filter: f => path.extname(f) === '.yaml', - joinParentDirectory: false, - }) - ).map(file => path.join(DATA_STATIC_PAGE_DIRECTORY, file)), + + files: dataPath => + traverse(path.join(dataPath, DATA_STATIC_PAGE_DIRECTORY), { + filterFile: name => path.extname(name) === '.yaml', + prefixPath: DATA_STATIC_PAGE_DIRECTORY, + }), documentMode: documentModes.onePerFile, processDocument: processStaticPageDocument, diff --git a/src/gen-thumbs.js b/src/gen-thumbs.js index 655ab6d0..2d432e4b 100644 --- a/src/gen-thumbs.js +++ b/src/gen-thumbs.js @@ -220,6 +220,7 @@ export async function clearThumbs(mediaPath, { const unsafeFiles = thumbFiles.filter(file => { if (path.extname(file) !== '.jpg') return true; if (thumbtacks.every(tack => !file.includes(tack))) return true; + if (path.relative(mediaPath, file).startsWith('../')) return true; return false; }); @@ -240,7 +241,7 @@ export async function clearThumbs(mediaPath, { await progressPromiseAll(`Removing thumbnail files`, queue( thumbFiles.map(file => async () => { try { - await unlink(path.join(mediaPath, file)); + await unlink(file); } catch (error) { if (error.code !== 'ENOENT') { errored.push(file); @@ -539,6 +540,7 @@ export async function traverseSourceImagePaths(mediaPath, {target}) { return await traverse(mediaPath, { pathStyle: (target === 'verify' ? 'posix' : 'device'), + prefixPath: '', filterFile(name) { const ext = path.extname(name); diff --git a/src/upd8.js b/src/upd8.js index 4bbdb95e..1e52c555 100755 --- a/src/upd8.js +++ b/src/upd8.js @@ -60,7 +60,6 @@ import { WIKI_INFO_FILE, } from './data/yaml.js'; -import {findFiles} from './util/io.js'; import link from './util/link.js'; import {isMain, traverse} from './util/node-utils.js'; import {empty, showAggregate, withEntries} from './util/sugar.js'; @@ -601,8 +600,9 @@ async function main() { let languages; if (langPath) { - const languageDataFiles = await findFiles(langPath, { - filter: (f) => path.extname(f) === '.json', + const languageDataFiles = await traverse(langPath, { + filterFile: name => path.extname(name) === '.json', + pathStyle: 'device', }); const results = await progressPromiseAll(`Reading & processing language files.`, @@ -687,20 +687,21 @@ async function main() { // we can't super easily know which ones are referenced at runtime, just // cheat and get file sizes for all images under media. (This includes // additional files which are images.) - const imageFilePaths = (await traverse(mediaPath, { - pathStyle: 'device', - filterDir: dir => dir !== '.git', - filterFile: file => ( - ['.png', '.gif', '.jpg'].includes(path.extname(file)) && - !isThumb(file)), - })) - .map(file => ({ - device: path.join(mediaPath, file), - media: - urls - .from('media.root') - .to('media.path', file.split(path.sep).join('/')), - })); + const imageFilePaths = + await traverse(mediaPath, { + pathStyle: 'device', + filterDir: dir => dir !== '.git', + filterFile: file => + ['.png', '.gif', '.jpg'].includes(path.extname(file)) && + !isThumb(file), + }).then(files => files + .map(file => ({ + device: file, + media: + urls + .from('media.root') + .to('media.path', path.relative(mediaPath, file).split(path.sep).join('/')), + }))); const getSizeOfMediaFileHelper = paths => (mediaPath) => { const pair = paths.find(({media}) => media === mediaPath); diff --git a/src/util/io.js b/src/util/io.js deleted file mode 100644 index 12e87f4d..00000000 --- a/src/util/io.js +++ /dev/null @@ -1,23 +0,0 @@ -// Utility functions for interacting with files and other external data -// interfacey constructs. - -import {readdir} from 'fs/promises'; -import * as path from 'path'; - -export async function findFiles(dataPath, { - filter = () => true, - joinParentDirectory = true, -} = {}) { - let files; - try { - files = await readdir(dataPath); - } catch (error) { - throw Object.assign( - new AggregateError([error], `Failed to list files from ${dataPath}`), - {code: error.code}); - } - - return files - .filter((file) => filter(file)) - .map((file) => (joinParentDirectory ? path.join(dataPath, file) : file)); -} diff --git a/src/util/node-utils.js b/src/util/node-utils.js index 3c0dd4cd..2fb7e8dd 100644 --- a/src/util/node-utils.js +++ b/src/util/node-utils.js @@ -1,6 +1,6 @@ // Utility functions which are only relevant to particular Node.js constructs. -import {readdir} from 'fs/promises'; +import {readdir, stat} from 'fs/promises'; import {fileURLToPath} from 'url'; import * as path from 'path'; @@ -56,34 +56,47 @@ export function isMain(importMetaURL) { ].includes(relative); } -// Like readdir... but it's recursive! -export function traverse(startDirPath, { +// Like readdir... but it's recursive! This returns a flat list of file paths. +// By default, the paths include the provided top/root path, but this can be +// changed with prefixPath to prefix some other path, or to just return paths +// relative to the root. Change pathStyle to specify posix or win32, or leave +// it as the default device-correct style. Provide a filterDir function to +// control which directory names are traversed at all, and filterFile to +// select which filenames are included in the final list. +export async function traverse(rootPath, { pathStyle = 'device', filterFile = () => true, - filterDir = () => true + filterDir = () => true, + prefixPath = rootPath, } = {}) { - const pathJoin = { + const pathJoinDevice = path.join; + const pathJoinStyle = { 'device': path.join, 'posix': path.posix.join, 'win32': path.win32.join, }[pathStyle]; - if (!pathJoin) { + if (!pathJoinStyle) { throw new Error(`Expected pathStyle to be device, posix, or win32`); } - const recursive = (names, subDirPath) => - Promise.all(names.map(name => - readdir(pathJoin(startDirPath, subDirPath, name)).then( - names => - (filterDir(name) - ? recursive(names, pathJoin(subDirPath, name)) - : []), - () => - (filterFile(name) - ? [pathJoin(subDirPath, name)] - : [])))) - .then(pathArrays => pathArrays.flat()); + const recursive = (names, ...subdirectories) => + Promise.all(names.map(async name => { + const devicePath = pathJoinDevice(rootPath, ...subdirectories, name); + const stats = await stat(devicePath); - return readdir(startDirPath).then(names => recursive(names, '')); + if (stats.isDirectory() && !filterDir(name)) return []; + else if (stats.isFile() && !filterFile(name)) return []; + else if (!stats.isDirectory() && !stats.isFile()) return []; + + if (stats.isDirectory()) { + return recursive(await readdir(devicePath), ...subdirectories, name); + } else { + return pathJoinStyle(prefixPath, ...subdirectories, name); + } + })); + + const names = await readdir(rootPath); + const results = await recursive(names); + return results.flat(Infinity); } |