From 79b233cab5853b50717ffb281247485e26101ef0 Mon Sep 17 00:00:00 2001 From: "(quasar) nebula" Date: Sat, 4 Mar 2023 10:13:48 -0400 Subject: --clear-thumbs utility --- src/gen-thumbs.js | 139 +++++++++++++++++++++++++++++++++++++++++++++--------- src/upd8.js | 35 +++++++++++++- src/util/cli.js | 8 ++++ 3 files changed, 158 insertions(+), 24 deletions(-) diff --git a/src/gen-thumbs.js b/src/gen-thumbs.js index 64f1f27a..26ac035e 100644 --- a/src/gen-thumbs.js +++ b/src/gen-thumbs.js @@ -89,11 +89,17 @@ import {spawn} from 'child_process'; import {createHash} from 'crypto'; import * as path from 'path'; -import {readFile, writeFile} from 'fs/promises'; // Whatcha know! Nice. +import { + readFile, + stat, + unlink, + writeFile, +} from 'fs/promises'; // Whatcha know! Nice. import {createReadStream} from 'fs'; // Still gotta import from 8oth tho, for createReadStream. import { + fileIssue, logError, logInfo, logWarn, @@ -110,6 +116,8 @@ import { import {delay, queue} from './util/sugar.js'; +export const defaultMagickThreads = 8; + function readFileMD5(filePath) { return new Promise((resolve, reject) => { const md5 = createHash('md5'); @@ -189,8 +197,95 @@ function generateImageThumbnails(filePath, {spawnConvert}) { promisifyProcess(convert('.' + ext, details), false))); } +export async function clearThumbs(mediaPath, { + queueSize = 0, +} = {}) { + if (!mediaPath) { + throw new Error('Expected mediaPath to be passed'); + } + + logInfo`Looking for thumbnails to clear out...`; + + const thumbFiles = await traverse(mediaPath, { + filterFile: file => isThumb(file), + filterDir: name => name !== '.git', + }); + + if (thumbFiles.length) { + // Double-check files. Since we're unlinking (deleting) files, + // we're better off safe than sorry! + const thumbtacks = Object.keys(thumbnailSpec); + const unsafeFiles = thumbFiles.filter(file => { + if (path.extname(file) !== '.jpg') return true; + if (thumbtacks.every(tack => !file.includes(tack))) return true; + return false; + }); + + if (unsafeFiles.length > 0) { + logError`Detected files which we thought were safe, but don't actually seem to be thumbnails!`; + logError`List of files that were invalid: ${`(Please remove any personal files before reporting)`}`; + for (const file of unsafeFiles) { + console.error(file); + } + fileIssue(); + return; + } + + logInfo`Clearing out ${thumbFiles.length} thumbs.`; + + const errored = []; + + await progressPromiseAll(`Removing thumbnail files`, queue( + thumbFiles.map(file => async () => { + try { + await unlink(path.join(mediaPath, file)); + } catch (error) { + if (error.code !== 'ENOENT') { + errored.push(file); + } + } + }), + queueSize)); + + if (errored.length) { + logError`Couldn't remove these paths (${errored.length}):`; + for (const file of errored) { + console.error(file); + } + logError`Check for permission errors?`; + } else { + logInfo`Successfully deleted all ${thumbFiles.length} thumbnail files!`; + } + } else { + logInfo`Didn't find any thumbs in media directory.`; + logInfo`${mediaPath}`; + } + + let cacheExists = false; + try { + await stat(path.join(mediaPath, CACHE_FILE)); + cacheExists = true; + } catch (error) { + if (error.code === 'ENOENT') { + logInfo`Cache file already missing, nothing to remove there.`; + } else { + logWarn`Failed to access cache file. Check its permissions?`; + } + } + + if (cacheExists) { + try { + unlink(path.join(mediaPath, CACHE_FILE)); + logInfo`Removed thumbnail cache file.`; + } catch (error) { + logWarn`Failed to remove cache file. Check its permissions?`; + } + } +} + export default async function genThumbs(mediaPath, { queueSize = 0, + magickThreads = defaultMagickThreads, quiet = false, } = {}) { if (!mediaPath) { @@ -226,6 +321,8 @@ export default async function genThumbs(mediaPath, { logInfo`Found ImageMagick binary: ${convertInfo}`; } + quietInfo`Running up to ${magickThreads + ' magick threads'} simultaneously.`; + let cache, firstRun = false; try { @@ -306,32 +403,28 @@ export default async function genThumbs(mediaPath, { return true; } + logInfo`Generating thumbnails for ${entriesToGenerate.length} media files.`; + if (entriesToGenerate.length > 250) { + logInfo`Go get a latte - this could take a while!`; + } + const failed = []; const succeeded = []; const writeMessageFn = () => `Writing image thumbnails. [failed: ${failed.length}]`; - // This is actually sort of a lie, 8ecause we aren't doing synchronicity. - // (We pass queueSize = 1 to queue().) 8ut we still use progressPromiseAll, - // 'cuz the progress indic8tor is very cool and good. - await progressPromiseAll( - writeMessageFn, + await progressPromiseAll(writeMessageFn, queue( - entriesToGenerate.map( - ([filePath, md5]) => - () => - generateImageThumbnails(path.join(mediaPath, filePath), {spawnConvert}).then( - () => { - updatedCache[filePath] = md5; - succeeded.push(filePath); - }, - (error) => { - failed.push([filePath, error]); - } - ) - ) - ) - ); + entriesToGenerate.map(([filePath, md5]) => () => + generateImageThumbnails(path.join(mediaPath, filePath), {spawnConvert}).then( + () => { + updatedCache[filePath] = md5; + succeeded.push(filePath); + }, + error => { + failed.push([filePath, error]); + })), + magickThreads)); if (failed.length > 0) { for (const [path, error] of failed) { @@ -359,7 +452,7 @@ export default async function genThumbs(mediaPath, { } export function isThumb(file) { - const thumbnailLabel = file.match(/\.([^.]+)\.[^.]+$/)?.[1]; + const thumbnailLabel = file.match(/\.([^.]+)\.jpg$/)?.[1]; return Object.keys(thumbnailSpec).includes(thumbnailLabel); } @@ -369,6 +462,7 @@ if (isMain(import.meta.url)) { 'media-path': { type: 'value', }, + 'queue-size': { type: 'value', validate(size) { @@ -377,6 +471,7 @@ if (isMain(import.meta.url)) { return true; }, }, + queue: {alias: 'queue-size'}, }); diff --git a/src/upd8.js b/src/upd8.js index 317ccb03..2b4fb5f6 100755 --- a/src/upd8.js +++ b/src/upd8.js @@ -36,7 +36,11 @@ import * as path from 'path'; import {fileURLToPath} from 'url'; import wrap from 'word-wrap'; -import genThumbs, {isThumb} from './gen-thumbs.js'; +import genThumbs, { + clearThumbs, + defaultMagickThreads, + isThumb, +} from './gen-thumbs.js'; import {listingSpec, listingTargetSpec} from './listing-spec.js'; import urlSpec from './url-spec.js'; @@ -195,6 +199,11 @@ async function main() { type: 'flag', }, + 'clear-thumbs': { + help: `Clear the thumbnail cache and remove generated thumbnail files from media directory\n\n(This skips building. Run again without --clear-thumbs to build the site.)`, + type: 'flag', + }, + // Just working on data entries and not interested in actually // generating site HTML yet? This flag will cut execution off right // 8efore any site 8uilding actually happens. @@ -223,6 +232,11 @@ async function main() { }, queue: {alias: 'queue-size'}, + 'magick-threads': { + help: `Process more or fewer thumbnail files at once with ImageMagick when generating thumbnails. (Each ImageMagick thread may also make use of multi-core processing at its own utility.)`, + }, + magick: {alias: 'magick-threads'}, + // This option is super slow and has the potential for bugs! It puts // CacheableObject in a mode where every instance is a Proxy which will // keep track of invalid property accesses. @@ -350,6 +364,7 @@ async function main() { const skipThumbs = cliOptions['skip-thumbs'] ?? false; const thumbsOnly = cliOptions['thumbs-only'] ?? false; + const clearThumbsFlag = cliOptions['clear-thumbs'] ?? false; const noBuild = cliOptions['no-build'] ?? false; const showAggregateTraces = cliOptions['show-traces'] ?? false; @@ -362,6 +377,8 @@ async function main() { // before proceeding to more page processing. const queueSize = +(cliOptions['queue-size'] ?? defaultQueueSize); + const magickThreads = +(cliOptions['magick-threads'] ?? defaultMagickThreads); + { let errored = false; const error = (cond, msg) => { @@ -390,11 +407,25 @@ async function main() { return; } + if (clearThumbsFlag) { + await clearThumbs(mediaPath, {queueSize}); + + logInfo`All done! Remove ${'--clear-thumbs'} to run the next build.`; + if (skipThumbs) { + logInfo`And don't forget to remove ${'--skip-thumbs'} too, eh?`; + } + return; + } + if (skipThumbs) { logInfo`Skipping thumbnail generation.`; } else { logInfo`Begin thumbnail generation... -----+`; - const result = await genThumbs(mediaPath, {queueSize, quiet: true}); + const result = await genThumbs(mediaPath, { + queueSize, + magickThreads, + quiet: !thumbsOnly, + }); logInfo`Done thumbnail generation! --------+`; if (!result) return; if (thumbsOnly) return; diff --git a/src/util/cli.js b/src/util/cli.js index 1ddc90e0..f83c8061 100644 --- a/src/util/cli.js +++ b/src/util/cli.js @@ -330,3 +330,11 @@ export function progressCallAll(msgOrMsgFn, array) { return vals; } + +export function fileIssue({ + topMessage = `This shouldn't happen.`, +} = {}) { + console.error(color.red(`${topMessage} Please let the HSMusic developers know:`)); + console.error(color.red(`- https://hsmusic.wiki/feedback/`)); + console.error(color.red(`- https://github.com/hsmusic/hsmusic-wiki/issues/`)); +} -- cgit 1.3.0-6-gf8a5