From ab038ee9aa304c7127ffd9832b236fb2a7a7a787 Mon Sep 17 00:00:00 2001 From: "(quasar) nebula" Date: Sat, 20 Dec 2025 18:30:14 -0400 Subject: sugar: wrapQueue, use this in traverse doubt this is half the answer for performance but... --- src/common-util/sugar.js | 29 +++++++++++++++++++++++++++++ src/node-utils.js | 7 ++++++- 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/src/common-util/sugar.js b/src/common-util/sugar.js index 354cf5cc..fba06d76 100644 --- a/src/common-util/sugar.js +++ b/src/common-util/sugar.js @@ -395,6 +395,35 @@ export function queue(functionList, queueSize = 50) { return promiseList; } +export function wrapQueue(fn, queueSize = 50) { + if (queueSize === 0) return fn; + + let running = 0; + let resume = []; + + let proceed = (...args) => { + running++; + return Promise.try(fn, ...args).finally(() => { + running--; + if (resume.length) { + resume.shift()(); + } + }); + }; + + return (...args) => { + if (running === queueSize) { + return new Promise(resolve => { + resume.push(resolve); + }).then(() => { + return proceed(...args); + }); + } else { + return proceed(...args); + } + }; +} + export function delay(ms) { return new Promise((res) => setTimeout(res, ms)); } diff --git a/src/node-utils.js b/src/node-utils.js index 345d10aa..d2e29b2f 100644 --- a/src/node-utils.js +++ b/src/node-utils.js @@ -6,6 +6,8 @@ import {fileURLToPath} from 'node:url'; import _commandExists from 'command-exists'; +import {wrapQueue} from '#sugar'; + // This package throws an error instead of returning false when the command // doesn't exist, for some reason. Yay for making logic more difficult! // Here's a straightforward workaround. @@ -68,6 +70,7 @@ export async function traverse(rootPath, { filterFile = () => true, filterDir = () => true, prefixPath = rootPath, + queueSize = 8, } = {}) { const pathJoinDevice = path.join; const pathJoinStyle = { @@ -80,6 +83,8 @@ export async function traverse(rootPath, { throw new Error(`Expected pathStyle to be device, posix, or win32`); } + const q_readdir = wrapQueue(readdir, queueSize); + const recursive = (names, ...subdirectories) => Promise.all(names.map(async name => { const devicePath = pathJoinDevice(rootPath, ...subdirectories, name); @@ -90,7 +95,7 @@ export async function traverse(rootPath, { else if (!stats.isDirectory() && !stats.isFile()) return []; if (stats.isDirectory()) { - return recursive(await readdir(devicePath), ...subdirectories, name); + return recursive(await q_readdir(devicePath), ...subdirectories, name); } else { return pathJoinStyle(prefixPath, ...subdirectories, name); } -- cgit 1.3.0-6-gf8a5