diff options
Diffstat (limited to 'upd8/gen-thumbs.js')
| -rw-r--r-- | upd8/gen-thumbs.js | 323 |
1 files changed, 0 insertions, 323 deletions
diff --git a/upd8/gen-thumbs.js b/upd8/gen-thumbs.js deleted file mode 100644 index bec01d1..0000000 --- a/upd8/gen-thumbs.js +++ /dev/null @@ -1,323 +0,0 @@ -#!/usr/bin/env node - -// Ok, so the d8te is 3 March 2021, and the music wiki was initially released -// on 15 November 2019. That is 474 days or 11376 hours. In my opinion, and -// pro8a8ly the opinions of at least one other person, that is WAY TOO LONG -// to go without media thum8nails!!!! So that's what this file is here to do. -// -// This program takes a path to the media folder (via --media or the environ. -// varia8le HSMUSIC_MEDIA), traverses su8directories to locate image files, -// and gener8tes lower-resolution/file-size versions of all that are new or -// have 8een modified since the last run. We use a JSON-format cache of MD5s -// for each file to perform this comparision; we gener8te files (using ffmpeg) -// in "medium" and "small" sizes adjacent to the existing PNG for easy and -// versatile access in site gener8tion code. -// -// So for example, on the very first run, you might have a media folder which -// looks something like this: -// -// media/ -// album-art/ -// one-year-older/ -// cover.jpg -// firefly-cloud.jpg -// october.jpg -// ... -// flash-art/ -// 413.jpg -// ... -// bg.jpg -// ... -// -// After running gen-thumbs.js with the path to that folder passed, you'd end -// up with something like this: -// -// media/ -// album-art/ -// one-year-older/ -// cover.jpg -// cover.medium.jpg -// cover.small.jpg -// firefly-cloud.jpg -// firefly-cloud.medium.jpg -// firefly-cloud.small.jpg -// october.jpg -// october.medium.jpg -// october.small.jpg -// ... -// flash-art/ -// 413.jpg -// 413.medium.jpg -// 413.small.jpg -// ... -// bg.jpg -// bg.medium.jpg -// bg.small.jpg -// thumbs-cache.json -// ... -// -// (Do note that while 8oth JPG and PNG are supported, gener8ted files will -// always 8e in JPG format and file extension. GIFs are skipped since there -// aren't any super gr8 ways to make those more efficient!) -// -// And then in gener8tion code, you'd reference the medium/small or original -// version of each file, as decided is appropriate. Here are some guidelines: -// -// - Small: Grid tiles on the homepage and in galleries. -// - Medium: Cover art on individual al8um and track pages, etc. -// - Original: Only linked to, not embedded. -// -// The traversal code is indiscrimin8te: there are no special cases to, say, -// not gener8te thum8nails for the bg.jpg file (since those would generally go -// unused). This is just to make the code more porta8le and sta8le, long-term, -// since it avoids a lot of otherwise implic8ted maintenance. - -'use strict'; - -const CACHE_FILE = 'thumbnail-cache.json'; -const WARNING_DELAY_TIME = 10000; - -const { spawn } = require('child_process'); -const crypto = require('crypto'); -const fsp = require('fs/promises'); // Whatcha know! Nice. -const fs = require('fs'); // Still gotta include 8oth tho, for createReadStream. -const path = require('path'); - -const { - delay, - logError, - logInfo, - logWarn, - parseOptions, - progressPromiseAll, - promisifyProcess, - queue, -} = require('./util'); - -function traverse(startDirPath, { - filterFile = () => true, - filterDir = () => true -} = {}) { - const recursive = (names, subDirPath) => Promise - .all(names.map(name => fsp.readdir(path.join(startDirPath, subDirPath, name)).then( - names => filterDir(name) ? recursive(names, path.join(subDirPath, name)) : [], - err => filterFile(name) ? [path.join(subDirPath, name)] : []))) - .then(pathArrays => pathArrays.flatMap(x => x)); - - return fsp.readdir(startDirPath) - .then(names => recursive(names, '')); -} - -function readFileMD5(filePath) { - return new Promise((resolve, reject) => { - const md5 = crypto.createHash('md5'); - const stream = fs.createReadStream(filePath); - stream.on('data', data => md5.update(data)); - stream.on('end', data => resolve(md5.digest('hex'))); - stream.on('error', err => reject(err)); - }); -} - -function generateImageThumbnails(filePath) { - const dirname = path.dirname(filePath); - const extname = path.extname(filePath); - const basename = path.basename(filePath, extname); - const output = name => path.join(dirname, basename + name + '.jpg'); - - const convert = (name, {size, quality}) => spawn('convert', [ - '-strip', - '-resize', `${size}x${size}>`, - '-interlace', 'Plane', - '-quality', `${quality}%`, - filePath, - output(name) - ]); - - return Promise.all([ - promisifyProcess(convert('.medium', {size: 400, quality: 95}), false), - promisifyProcess(convert('.small', {size: 250, quality: 85}), false) - ]); - - return new Promise((resolve, reject) => { - if (Math.random() < 0.2) { - reject(new Error(`Them's the 8r8ks, kiddo!`)); - } else { - resolve(); - } - }); -} - -async function genThumbs(mediaPath, { - queueSize = 0, - quiet = false -} = {}) { - if (!mediaPath) { - throw new Error('Expected mediaPath to be passed'); - } - - const quietInfo = (quiet - ? () => null - : logInfo); - - const filterFile = name => { - // TODO: Why is this not working???????? - // thumbnail-cache.json is 8eing passed through, for some reason. - - const ext = path.extname(name); - if (ext !== '.jpg' && ext !== '.png') return false; - - const rest = path.basename(name, ext); - if (rest.endsWith('.medium') || rest.endsWith('.small')) return false; - - return true; - }; - - const filterDir = name => { - if (name === '.git') return false; - return true; - }; - - let cache, firstRun = false, failedReadingCache = false; - try { - cache = JSON.parse(await fsp.readFile(path.join(mediaPath, CACHE_FILE))); - quietInfo`Cache file successfully read.`; - } catch (error) { - cache = {}; - if (error.code === 'ENOENT') { - firstRun = true; - } else { - failedReadingCache = true; - logWarn`Malformed or unreadable cache file: ${error}`; - logWarn`You may want to cancel and investigate this!`; - logWarn`All-new thumbnails and cache will be generated for this run.`; - await delay(WARNING_DELAY_TIME); - } - } - - try { - await fsp.writeFile(path.join(mediaPath, CACHE_FILE), JSON.stringify(cache)); - quietInfo`Writing to cache file appears to be working.`; - } catch (error) { - logWarn`Test of cache file writing failed: ${error}`; - if (cache) { - logWarn`Cache read succeeded: Any newly written thumbs will be unnecessarily regenerated on the next run.`; - } else if (firstRun) { - logWarn`No cache found: All thumbs will be generated now, and will be unnecessarily regenerated next run.`; - } else { - logWarn`Cache read failed: All thumbs will be regenerated now, and will be unnecessarily regenerated again next run.`; - } - logWarn`You may want to cancel and investigate this!`; - await delay(WARNING_DELAY_TIME); - } - - const imagePaths = await traverse(mediaPath, {filterFile, filterDir}); - - const imageToMD5Entries = await progressPromiseAll(`Generating MD5s of image files`, queue( - imagePaths.map(imagePath => () => readFileMD5(path.join(mediaPath, imagePath)).then( - md5 => [imagePath, md5], - error => [imagePath, {error}] - )), - queueSize - )); - - { - let error = false; - for (const entry of imageToMD5Entries) { - if (entry[1].error) { - logError`Failed to read ${entry[0]}: ${entry[1].error}`; - error = true; - } - } - if (error) { - logError`Failed to read at least one image file!`; - logError`This implies a thumbnail probably won't be generatable.`; - logError`So, exiting early.`; - return false; - } else { - quietInfo`All image files successfully read.`; - } - } - - // Technically we could pro8a8ly mut8te the cache varia8le in-place? - // 8ut that seems kinda iffy. - const updatedCache = Object.assign({}, cache); - - const entriesToGenerate = imageToMD5Entries - .filter(([filePath, md5]) => md5 !== cache[filePath]); - - if (entriesToGenerate.length === 0) { - logInfo`All image thumbnails are already up-to-date - nice!`; - return true; - } - - const failed = []; - const succeeded = []; - const writeMessageFn = () => `Writing image thumbnails. [failed: ${failed.length}]`; - - // This is actually sort of a lie, 8ecause we aren't doing synchronicity. - // (We pass queueSize = 1 to queue().) 8ut we still use progressPromiseAll, - // 'cuz the progress indic8tor is very cool and good. - await progressPromiseAll(writeMessageFn, queue(entriesToGenerate.map(([filePath, md5]) => - () => generateImageThumbnails(path.join(mediaPath, filePath)).then( - () => { - updatedCache[filePath] = md5; - succeeded.push(filePath); - }, - error => { - failed.push([filePath, error]); - } - ) - ))); - - if (failed.length > 0) { - for (const [path, error] of failed) { - logError`Thumbnails failed to generate for ${path} - ${error}`; - } - logWarn`Result is incomplete - the above ${failed.length} thumbnails should be checked for errors.`; - logWarn`${succeeded.length} successfully generated images won't be regenerated next run, though!`; - } else { - logInfo`Generated all (updated) thumbnails successfully!`; - } - - try { - await fsp.writeFile(path.join(mediaPath, CACHE_FILE), JSON.stringify(updatedCache)); - quietInfo`Updated cache file successfully written!`; - } catch (error) { - logWarn`Failed to write updated cache file: ${error}`; - logWarn`Any newly (re)generated thumbnails will be regenerated next run.`; - logWarn`Sorry about that!`; - } - - return true; -}; - -module.exports = genThumbs; - -if (require.main === module) { - (async () => { - const miscOptions = await parseOptions(process.argv.slice(2), { - 'media': { - type: 'value' - }, - - 'queue-size': { - type: 'value', - validate(size) { - if (parseInt(size) !== parseFloat(size)) return 'an integer'; - if (parseInt(size) < 0) return 'a counting number or zero'; - return true; - } - }, - queue: {alias: 'queue-size'} - }); - - const mediaPath = miscOptions.media || process.env.HSMUSIC_MEDIA; - if (!mediaPath) { - logError`Expected --media option or HSMUSIC_MEDIA to be set`; - } - - const queueSize = +(miscOptions['queue-size'] ?? 0); - - await genThumbs(mediaPath, {queueSize}); - })().catch(err => console.error(err)); -} |