diff options
Diffstat (limited to 'src/gen-thumbs.js')
| -rw-r--r-- | src/gen-thumbs.js | 538 |
1 files changed, 272 insertions, 266 deletions
diff --git a/src/gen-thumbs.js b/src/gen-thumbs.js index 839c1d4..b5b918f 100644 --- a/src/gen-thumbs.js +++ b/src/gen-thumbs.js @@ -1,4 +1,5 @@ #!/usr/bin/env node +/** @format */ // Ok, so the d8te is 3 March 2021, and the music wiki was initially released // on 15 November 2019. That is 474 days or 11376 hours. In my opinion, and @@ -77,316 +78,321 @@ const CACHE_FILE = 'thumbnail-cache.json'; const WARNING_DELAY_TIME = 10000; -import { spawn } from 'child_process'; -import { createHash } from 'crypto'; +import {spawn} from 'child_process'; +import {createHash} from 'crypto'; import * as path from 'path'; -import { - readdir, - readFile, - writeFile -} from 'fs/promises'; // Whatcha know! Nice. +import {readdir, readFile, writeFile} from 'fs/promises'; // Whatcha know! Nice. -import { - createReadStream -} from 'fs'; // Still gotta import from 8oth tho, for createReadStream. +import {createReadStream} from 'fs'; // Still gotta import from 8oth tho, for createReadStream. import { - logError, - logInfo, - logWarn, - parseOptions, - progressPromiseAll + logError, + logInfo, + logWarn, + parseOptions, + progressPromiseAll, } from './util/cli.js'; -import { - commandExists, - isMain, - promisifyProcess, -} from './util/node-utils.js'; +import {commandExists, isMain, promisifyProcess} from './util/node-utils.js'; + +import {delay, queue} from './util/sugar.js'; + +function traverse( + startDirPath, + {filterFile = () => true, filterDir = () => true} = {} +) { + const recursive = (names, subDirPath) => + Promise.all( + names.map((name) => + readdir(path.join(startDirPath, subDirPath, name)).then( + (names) => + filterDir(name) + ? recursive(names, path.join(subDirPath, name)) + : [], + () => (filterFile(name) ? [path.join(subDirPath, name)] : []) + ) + ) + ).then((pathArrays) => pathArrays.flatMap((x) => x)); -import { - delay, - queue, -} from './util/sugar.js'; - -function traverse(startDirPath, { - filterFile = () => true, - filterDir = () => true -} = {}) { - const recursive = (names, subDirPath) => Promise - .all(names.map(name => readdir(path.join(startDirPath, subDirPath, name)).then( - names => filterDir(name) ? recursive(names, path.join(subDirPath, name)) : [], - err => filterFile(name) ? [path.join(subDirPath, name)] : []))) - .then(pathArrays => pathArrays.flatMap(x => x)); - - return readdir(startDirPath) - .then(names => recursive(names, '')); + return readdir(startDirPath).then((names) => recursive(names, '')); } function readFileMD5(filePath) { - return new Promise((resolve, reject) => { - const md5 = createHash('md5'); - const stream = createReadStream(filePath); - stream.on('data', data => md5.update(data)); - stream.on('end', data => resolve(md5.digest('hex'))); - stream.on('error', err => reject(err)); - }); + return new Promise((resolve, reject) => { + const md5 = createHash('md5'); + const stream = createReadStream(filePath); + stream.on('data', (data) => md5.update(data)); + stream.on('end', () => resolve(md5.digest('hex'))); + stream.on('error', (err) => reject(err)); + }); } async function getImageMagickVersion(spawnConvert) { - const proc = spawnConvert(['--version'], false); + const proc = spawnConvert(['--version'], false); - let allData = ''; - proc.stdout.on('data', data => { - allData += data.toString(); - }); + let allData = ''; + proc.stdout.on('data', (data) => { + allData += data.toString(); + }); - await promisifyProcess(proc, false); + await promisifyProcess(proc, false); - if (!allData.match(/ImageMagick/i)) { - return null; - } + if (!allData.match(/ImageMagick/i)) { + return null; + } - const match = allData.match(/Version: (.*)/i); - if (!match) { - return 'unknown version'; - } + const match = allData.match(/Version: (.*)/i); + if (!match) { + return 'unknown version'; + } - return match[1]; + return match[1]; } async function getSpawnConvert() { - let fn, description, version; - if (await commandExists('convert')) { - fn = args => spawn('convert', args); - description = 'convert'; - } else if (await commandExists('magick')) { - fn = (args, prefix = true) => spawn('magick', - (prefix ? ['convert', ...args] : args)); - description = 'magick convert'; - } else { - return [`no convert or magick binary`, null]; - } - - version = await getImageMagickVersion(fn); - - if (version === null) { - return [`binary --version output didn't indicate it's ImageMagick`]; - } - - return [`${description} (${version})`, fn]; + let fn, description, version; + if (await commandExists('convert')) { + fn = (args) => spawn('convert', args); + description = 'convert'; + } else if (await commandExists('magick')) { + fn = (args, prefix = true) => + spawn('magick', prefix ? ['convert', ...args] : args); + description = 'magick convert'; + } else { + return [`no convert or magick binary`, null]; + } + + version = await getImageMagickVersion(fn); + + if (version === null) { + return [`binary --version output didn't indicate it's ImageMagick`]; + } + + return [`${description} (${version})`, fn]; } function generateImageThumbnails(filePath, {spawnConvert}) { - const dirname = path.dirname(filePath); - const extname = path.extname(filePath); - const basename = path.basename(filePath, extname); - const output = name => path.join(dirname, basename + name + '.jpg'); - - const convert = (name, {size, quality}) => spawnConvert([ - filePath, - '-strip', - '-resize', `${size}x${size}>`, - '-interlace', 'Plane', - '-quality', `${quality}%`, - output(name) - ]); - - return Promise.all([ - promisifyProcess(convert('.medium', {size: 400, quality: 95}), false), - promisifyProcess(convert('.small', {size: 250, quality: 85}), false) + const dirname = path.dirname(filePath); + const extname = path.extname(filePath); + const basename = path.basename(filePath, extname); + const output = (name) => path.join(dirname, basename + name + '.jpg'); + + const convert = (name, {size, quality}) => + spawnConvert([ + filePath, + '-strip', + '-resize', + `${size}x${size}>`, + '-interlace', + 'Plane', + '-quality', + `${quality}%`, + output(name), ]); - return new Promise((resolve, reject) => { - if (Math.random() < 0.2) { - reject(new Error(`Them's the 8r8ks, kiddo!`)); - } else { - resolve(); - } - }); + return Promise.all([ + promisifyProcess(convert('.medium', {size: 400, quality: 95}), false), + promisifyProcess(convert('.small', {size: 250, quality: 85}), false), + ]); } -export default async function genThumbs(mediaPath, { - queueSize = 0, - quiet = false -} = {}) { - if (!mediaPath) { - throw new Error('Expected mediaPath to be passed'); - } +export default async function genThumbs( + mediaPath, + {queueSize = 0, quiet = false} = {} +) { + if (!mediaPath) { + throw new Error('Expected mediaPath to be passed'); + } - const quietInfo = (quiet - ? () => null - : logInfo); - - const filterFile = name => { - // TODO: Why is this not working???????? - // thumbnail-cache.json is 8eing passed through, for some reason. - - const ext = path.extname(name); - if (ext !== '.jpg' && ext !== '.png') return false; - - const rest = path.basename(name, ext); - if (rest.endsWith('.medium') || rest.endsWith('.small')) return false; - - return true; - }; - - const filterDir = name => { - if (name === '.git') return false; - return true; - }; - - const [convertInfo, spawnConvert] = await getSpawnConvert() ?? []; - if (!spawnConvert) { - logError`${`It looks like you don't have ImageMagick installed.`}`; - logError`ImageMagick is required to generate thumbnails for display on the wiki.`; - logError`(Error message: ${convertInfo})`; - logInfo`You can find info to help install ImageMagick on Linux, Windows, or macOS`; - logInfo`from its official website: ${`https://imagemagick.org/script/download.php`}`; - logInfo`If you have trouble working ImageMagick and would like some help, feel free`; - logInfo`to drop a message in the HSMusic Discord server! ${'https://hsmusic.wiki/discord/'}`; - return false; - } else { - logInfo`Found ImageMagick binary: ${convertInfo}`; - } + const quietInfo = quiet ? () => null : logInfo; - let cache, firstRun = false, failedReadingCache = false; - try { - cache = JSON.parse(await readFile(path.join(mediaPath, CACHE_FILE))); - quietInfo`Cache file successfully read.`; - } catch (error) { - cache = {}; - if (error.code === 'ENOENT') { - firstRun = true; - } else { - failedReadingCache = true; - logWarn`Malformed or unreadable cache file: ${error}`; - logWarn`You may want to cancel and investigate this!`; - logWarn`All-new thumbnails and cache will be generated for this run.`; - await delay(WARNING_DELAY_TIME); - } - } + const filterFile = (name) => { + // TODO: Why is this not working???????? + // thumbnail-cache.json is 8eing passed through, for some reason. - try { - await writeFile(path.join(mediaPath, CACHE_FILE), JSON.stringify(cache)); - quietInfo`Writing to cache file appears to be working.`; - } catch (error) { - logWarn`Test of cache file writing failed: ${error}`; - if (cache) { - logWarn`Cache read succeeded: Any newly written thumbs will be unnecessarily regenerated on the next run.`; - } else if (firstRun) { - logWarn`No cache found: All thumbs will be generated now, and will be unnecessarily regenerated next run.`; - } else { - logWarn`Cache read failed: All thumbs will be regenerated now, and will be unnecessarily regenerated again next run.`; - } - logWarn`You may want to cancel and investigate this!`; - await delay(WARNING_DELAY_TIME); - } - - const imagePaths = await traverse(mediaPath, {filterFile, filterDir}); - - const imageToMD5Entries = await progressPromiseAll(`Generating MD5s of image files`, queue( - imagePaths.map(imagePath => () => readFileMD5(path.join(mediaPath, imagePath)).then( - md5 => [imagePath, md5], - error => [imagePath, {error}] - )), - queueSize - )); - - { - let error = false; - for (const entry of imageToMD5Entries) { - if (entry[1].error) { - logError`Failed to read ${entry[0]}: ${entry[1].error}`; - error = true; - } - } - if (error) { - logError`Failed to read at least one image file!`; - logError`This implies a thumbnail probably won't be generatable.`; - logError`So, exiting early.`; - return false; - } else { - quietInfo`All image files successfully read.`; - } - } + const ext = path.extname(name); + if (ext !== '.jpg' && ext !== '.png') return false; - // Technically we could pro8a8ly mut8te the cache varia8le in-place? - // 8ut that seems kinda iffy. - const updatedCache = Object.assign({}, cache); + const rest = path.basename(name, ext); + if (rest.endsWith('.medium') || rest.endsWith('.small')) return false; - const entriesToGenerate = imageToMD5Entries - .filter(([filePath, md5]) => md5 !== cache[filePath]); + return true; + }; - if (entriesToGenerate.length === 0) { - logInfo`All image thumbnails are already up-to-date - nice!`; - return true; + const filterDir = (name) => { + if (name === '.git') return false; + return true; + }; + + const [convertInfo, spawnConvert] = (await getSpawnConvert()) ?? []; + if (!spawnConvert) { + logError`${`It looks like you don't have ImageMagick installed.`}`; + logError`ImageMagick is required to generate thumbnails for display on the wiki.`; + logError`(Error message: ${convertInfo})`; + logInfo`You can find info to help install ImageMagick on Linux, Windows, or macOS`; + logInfo`from its official website: ${`https://imagemagick.org/script/download.php`}`; + logInfo`If you have trouble working ImageMagick and would like some help, feel free`; + logInfo`to drop a message in the HSMusic Discord server! ${'https://hsmusic.wiki/discord/'}`; + return false; + } else { + logInfo`Found ImageMagick binary: ${convertInfo}`; + } + + let cache, + firstRun = false; + try { + cache = JSON.parse(await readFile(path.join(mediaPath, CACHE_FILE))); + quietInfo`Cache file successfully read.`; + } catch (error) { + cache = {}; + if (error.code === 'ENOENT') { + firstRun = true; + } else { + logWarn`Malformed or unreadable cache file: ${error}`; + logWarn`You may want to cancel and investigate this!`; + logWarn`All-new thumbnails and cache will be generated for this run.`; + await delay(WARNING_DELAY_TIME); + } + } + + try { + await writeFile(path.join(mediaPath, CACHE_FILE), JSON.stringify(cache)); + quietInfo`Writing to cache file appears to be working.`; + } catch (error) { + logWarn`Test of cache file writing failed: ${error}`; + if (cache) { + logWarn`Cache read succeeded: Any newly written thumbs will be unnecessarily regenerated on the next run.`; + } else if (firstRun) { + logWarn`No cache found: All thumbs will be generated now, and will be unnecessarily regenerated next run.`; + } else { + logWarn`Cache read failed: All thumbs will be regenerated now, and will be unnecessarily regenerated again next run.`; + } + logWarn`You may want to cancel and investigate this!`; + await delay(WARNING_DELAY_TIME); + } + + const imagePaths = await traverse(mediaPath, {filterFile, filterDir}); + + const imageToMD5Entries = await progressPromiseAll( + `Generating MD5s of image files`, + queue( + imagePaths.map( + (imagePath) => () => + readFileMD5(path.join(mediaPath, imagePath)).then( + (md5) => [imagePath, md5], + (error) => [imagePath, {error}] + ) + ), + queueSize + ) + ); + + { + let error = false; + for (const entry of imageToMD5Entries) { + if (entry[1].error) { + logError`Failed to read ${entry[0]}: ${entry[1].error}`; + error = true; + } + } + if (error) { + logError`Failed to read at least one image file!`; + logError`This implies a thumbnail probably won't be generatable.`; + logError`So, exiting early.`; + return false; + } else { + quietInfo`All image files successfully read.`; } + } - const failed = []; - const succeeded = []; - const writeMessageFn = () => `Writing image thumbnails. [failed: ${failed.length}]`; + // Technically we could pro8a8ly mut8te the cache varia8le in-place? + // 8ut that seems kinda iffy. + const updatedCache = Object.assign({}, cache); - // This is actually sort of a lie, 8ecause we aren't doing synchronicity. - // (We pass queueSize = 1 to queue().) 8ut we still use progressPromiseAll, - // 'cuz the progress indic8tor is very cool and good. - await progressPromiseAll(writeMessageFn, queue(entriesToGenerate.map(([filePath, md5]) => - () => generateImageThumbnails(path.join(mediaPath, filePath)).then( - () => { + const entriesToGenerate = imageToMD5Entries.filter( + ([filePath, md5]) => md5 !== cache[filePath] + ); + + if (entriesToGenerate.length === 0) { + logInfo`All image thumbnails are already up-to-date - nice!`; + return true; + } + + const failed = []; + const succeeded = []; + const writeMessageFn = () => + `Writing image thumbnails. [failed: ${failed.length}]`; + + // This is actually sort of a lie, 8ecause we aren't doing synchronicity. + // (We pass queueSize = 1 to queue().) 8ut we still use progressPromiseAll, + // 'cuz the progress indic8tor is very cool and good. + await progressPromiseAll( + writeMessageFn, + queue( + entriesToGenerate.map( + ([filePath, md5]) => + () => + generateImageThumbnails(path.join(mediaPath, filePath)).then( + () => { updatedCache[filePath] = md5; succeeded.push(filePath); - }, - error => { + }, + (error) => { failed.push([filePath, error]); - } - ) - ))); - - if (failed.length > 0) { - for (const [path, error] of failed) { - logError`Thumbnails failed to generate for ${path} - ${error}`; - } - logWarn`Result is incomplete - the above ${failed.length} thumbnails should be checked for errors.`; - logWarn`${succeeded.length} successfully generated images won't be regenerated next run, though!`; - } else { - logInfo`Generated all (updated) thumbnails successfully!`; - } - - try { - await writeFile(path.join(mediaPath, CACHE_FILE), JSON.stringify(updatedCache)); - quietInfo`Updated cache file successfully written!`; - } catch (error) { - logWarn`Failed to write updated cache file: ${error}`; - logWarn`Any newly (re)generated thumbnails will be regenerated next run.`; - logWarn`Sorry about that!`; + } + ) + ) + ) + ); + + if (failed.length > 0) { + for (const [path, error] of failed) { + logError`Thumbnails failed to generate for ${path} - ${error}`; } - - return true; + logWarn`Result is incomplete - the above ${failed.length} thumbnails should be checked for errors.`; + logWarn`${succeeded.length} successfully generated images won't be regenerated next run, though!`; + } else { + logInfo`Generated all (updated) thumbnails successfully!`; + } + + try { + await writeFile( + path.join(mediaPath, CACHE_FILE), + JSON.stringify(updatedCache) + ); + quietInfo`Updated cache file successfully written!`; + } catch (error) { + logWarn`Failed to write updated cache file: ${error}`; + logWarn`Any newly (re)generated thumbnails will be regenerated next run.`; + logWarn`Sorry about that!`; + } + + return true; } if (isMain(import.meta.url)) { - (async function() { - const miscOptions = await parseOptions(process.argv.slice(2), { - 'media-path': { - type: 'value' - }, - 'queue-size': { - type: 'value', - validate(size) { - if (parseInt(size) !== parseFloat(size)) return 'an integer'; - if (parseInt(size) < 0) return 'a counting number or zero'; - return true; - } - }, - queue: {alias: 'queue-size'}, - }); - - const mediaPath = miscOptions['media-path'] || process.env.HSMUSIC_MEDIA; - const queueSize = +(miscOptions['queue-size'] ?? 0); - - await genThumbs(mediaPath, {queueSize}); - })().catch(err => { - console.error(err); + (async function () { + const miscOptions = await parseOptions(process.argv.slice(2), { + 'media-path': { + type: 'value', + }, + 'queue-size': { + type: 'value', + validate(size) { + if (parseInt(size) !== parseFloat(size)) return 'an integer'; + if (parseInt(size) < 0) return 'a counting number or zero'; + return true; + }, + }, + queue: {alias: 'queue-size'}, }); + + const mediaPath = miscOptions['media-path'] || process.env.HSMUSIC_MEDIA; + const queueSize = +(miscOptions['queue-size'] ?? 0); + + await genThumbs(mediaPath, {queueSize}); + })().catch((err) => { + console.error(err); + }); } |