diff options
| -rw-r--r-- | gen-thumbs.js | 323 | ||||
| -rw-r--r-- | upd8-util.js | 41 | ||||
| -rwxr-xr-x | upd8.js | 31 |
3 files changed, 388 insertions, 7 deletions
diff --git a/gen-thumbs.js b/gen-thumbs.js new file mode 100644 index 00000000..3887aaea --- /dev/null +++ b/gen-thumbs.js @@ -0,0 +1,323 @@ +#!/usr/bin/env node + +// Ok, so the d8te is 3 March 2021, and the music wiki was initially released +// on 15 November 2019. That is 474 days or 11376 hours. In my opinion, and +// pro8a8ly the opinions of at least one other person, that is WAY TOO LONG +// to go without media thum8nails!!!! So that's what this file is here to do. +// +// This program takes a path to the media folder (via --media or the environ. +// varia8le HSMUSIC_MEDIA), traverses su8directories to locate image files, +// and gener8tes lower-resolution/file-size versions of all that are new or +// have 8een modified since the last run. We use a JSON-format cache of MD5s +// for each file to perform this comparision; we gener8te files (using ffmpeg) +// in "medium" and "small" sizes adjacent to the existing PNG for easy and +// versatile access in site gener8tion code. +// +// So for example, on the very first run, you might have a media folder which +// looks something like this: +// +// media/ +// album-art/ +// one-year-older/ +// cover.jpg +// firefly-cloud.jpg +// october.jpg +// ... +// flash-art/ +// 413.jpg +// ... +// bg.jpg +// ... +// +// After running gen-thumbs.js with the path to that folder passed, you'd end +// up with something like this: +// +// media/ +// album-art/ +// one-year-older/ +// cover.jpg +// cover.medium.jpg +// cover.small.jpg +// firefly-cloud.jpg +// firefly-cloud.medium.jpg +// firefly-cloud.small.jpg +// october.jpg +// october.medium.jpg +// october.small.jpg +// ... +// flash-art/ +// 413.jpg +// 413.medium.jpg +// 413.small.jpg +// ... +// bg.jpg +// bg.medium.jpg +// bg.small.jpg +// thumbs-cache.json +// ... +// +// (Do note that while 8oth JPG and PNG are supported, gener8ted files will +// always 8e in JPG format and file extension. GIFs are skipped since there +// aren't any super gr8 ways to make those more efficient!) +// +// And then in gener8tion code, you'd reference the medium/small or original +// version of each file, as decided is appropriate. Here are some guidelines: +// +// - Small: Grid tiles on the homepage and in galleries. +// - Medium: Cover art on individual al8um and track pages, etc. +// - Original: Only linked to, not embedded. +// +// The traversal code is indiscrimin8te: there are no special cases to, say, +// not gener8te thum8nails for the bg.jpg file (since those would generally go +// unused). This is just to make the code more porta8le and sta8le, long-term, +// since it avoids a lot of otherwise implic8ted maintenance. + +'use strict'; + +const CACHE_FILE = 'thumbnail-cache.json'; +const WARNING_DELAY_TIME = 10000; + +const { spawn } = require('child_process'); +const crypto = require('crypto'); +const fsp = require('fs/promises'); // Whatcha know! Nice. +const fs = require('fs'); // Still gotta include 8oth tho, for createReadStream. +const path = require('path'); + +const { + delay, + logError, + logInfo, + logWarn, + parseOptions, + progressPromiseAll, + promisifyProcess, + queue, +} = require('./upd8-util'); + +function traverse(startDirPath, { + filterFile = () => true, + filterDir = () => true +} = {}) { + const recursive = (names, subDirPath) => Promise + .all(names.map(name => fsp.readdir(path.join(startDirPath, subDirPath, name)).then( + names => filterDir(name) ? recursive(names, path.join(subDirPath, name)) : [], + err => filterFile(name) ? [path.join(subDirPath, name)] : []))) + .then(pathArrays => pathArrays.flatMap(x => x)); + + return fsp.readdir(startDirPath) + .then(names => recursive(names, '')); +} + +function readFileMD5(filePath) { + return new Promise((resolve, reject) => { + const md5 = crypto.createHash('md5'); + const stream = fs.createReadStream(filePath); + stream.on('data', data => md5.update(data)); + stream.on('end', data => resolve(md5.digest('hex'))); + stream.on('error', err => reject(err)); + }); +} + +function generateImageThumbnails(filePath) { + const dirname = path.dirname(filePath); + const extname = path.extname(filePath); + const basename = path.basename(filePath, extname); + const output = name => path.join(dirname, basename + name + '.jpg'); + + const convert = (name, {size, quality}) => spawn('convert', [ + '-strip', + '-resize', `${size}x${size}>`, + '-interlace', 'Plane', + '-quality', `${quality}%`, + filePath, + output(name) + ]); + + return Promise.all([ + promisifyProcess(convert('.medium', {size: 400, quality: 95}), false), + promisifyProcess(convert('.small', {size: 250, quality: 85}), false) + ]); + + return new Promise((resolve, reject) => { + if (Math.random() < 0.2) { + reject(new Error(`Them's the 8r8ks, kiddo!`)); + } else { + resolve(); + } + }); +} + +async function genThumbs(mediaPath, { + queueSize = 0, + quiet = false +} = {}) { + if (!mediaPath) { + throw new Error('Expected mediaPath to be passed'); + } + + const quietInfo = (quiet + ? () => null + : logInfo); + + const filterFile = name => { + // TODO: Why is this not working???????? + // thumbnail-cache.json is 8eing passed through, for some reason. + + const ext = path.extname(name); + if (ext !== '.jpg' && ext !== '.png') return false; + + const rest = path.basename(name, ext); + if (rest.endsWith('.medium') || rest.endsWith('.small')) return false; + + return true; + }; + + const filterDir = name => { + if (name === '.git') return false; + return true; + }; + + let cache, firstRun = false, failedReadingCache = false; + try { + cache = JSON.parse(await fsp.readFile(path.join(mediaPath, CACHE_FILE))); + quietInfo`Cache file successfully read.`; + } catch (error) { + cache = {}; + if (error.code === 'ENOENT') { + firstRun = true; + } else { + failedReadingCache = true; + logWarn`Malformed or unreadable cache file: ${error}`; + logWarn`You may want to cancel and investigate this!`; + logWarn`All-new thumbnails and cache will be generated for this run.`; + await delay(WARNING_DELAY_TIME); + } + } + + try { + await fsp.writeFile(path.join(mediaPath, CACHE_FILE), JSON.stringify(cache)); + quietInfo`Writing to cache file appears to be working.`; + } catch (error) { + logWarn`Test of cache file writing failed: ${error}`; + if (cache) { + logWarn`Cache read succeeded: Any newly written thumbs will be unnecessarily regenerated on the next run.`; + } else if (firstRun) { + logWarn`No cache found: All thumbs will be generated now, and will be unnecessarily regenerated next run.`; + } else { + logWarn`Cache read failed: All thumbs will be regenerated now, and will be unnecessarily regenerated again next run.`; + } + logWarn`You may want to cancel and investigate this!`; + await delay(WARNING_DELAY_TIME); + } + + const imagePaths = await traverse(mediaPath, {filterFile, filterDir}); + + const imageToMD5Entries = await progressPromiseAll(`Generating MD5s of image files`, queue( + imagePaths.map(imagePath => () => readFileMD5(path.join(mediaPath, imagePath)).then( + md5 => [imagePath, md5], + error => [imagePath, {error}] + )), + queueSize + )); + + { + let error = false; + for (const entry of imageToMD5Entries) { + if (entry[1].error) { + logError`Failed to read ${entry[0]}: ${entry[1].error}`; + error = true; + } + } + if (error) { + logError`Failed to read at least one image file!`; + logError`This implies a thumbnail probably won't be generatable.`; + logError`So, exiting early.`; + return false; + } else { + quietInfo`All image files successfully read.`; + } + } + + // Technically we could pro8a8ly mut8te the cache varia8le in-place? + // 8ut that seems kinda iffy. + const updatedCache = Object.assign({}, cache); + + const entriesToGenerate = imageToMD5Entries + .filter(([filePath, md5]) => md5 !== cache[filePath]); + + if (entriesToGenerate.length === 0) { + logInfo`All image thumbnails are already up-to-date - nice!`; + return true; + } + + const failed = []; + const succeeded = []; + const writeMessageFn = () => `Writing image thumbnails. [failed: ${failed.length}]`; + + // This is actually sort of a lie, 8ecause we aren't doing synchronicity. + // (We pass queueSize = 1 to queue().) 8ut we still use progressPromiseAll, + // 'cuz the progress indic8tor is very cool and good. + await progressPromiseAll(writeMessageFn, queue(entriesToGenerate.map(([filePath, md5]) => + () => generateImageThumbnails(path.join(mediaPath, filePath)).then( + () => { + updatedCache[filePath] = md5; + succeeded.push(filePath); + }, + error => { + failed.push([filePath, error]); + } + ) + ))); + + if (failed.length > 0) { + for (const [path, error] of failed) { + logError`Thumbnails failed to generate for ${path} - ${error}`; + } + logWarn`Result is incomplete - the above ${failed.length} thumbnails should be checked for errors.`; + logWarn`${succeeded.length} successfully generated images won't be regenerated next run, though!`; + } else { + logInfo`Generated all (updated) thumbnails successfully!`; + } + + try { + await fsp.writeFile(path.join(mediaPath, CACHE_FILE), JSON.stringify(updatedCache)); + quietInfo`Updated cache file successfully written!`; + } catch (error) { + logWarn`Failed to write updated cache file: ${error}`; + logWarn`Any newly (re)generated thumbnails will be regenerated next run.`; + logWarn`Sorry about that!`; + } + + return true; +}; + +module.exports = genThumbs; + +if (require.main === module) { + (async () => { + const miscOptions = await parseOptions(process.argv.slice(2), { + 'media': { + type: 'value' + }, + + 'queue-size': { + type: 'value', + validate(size) { + if (parseInt(size) !== parseFloat(size)) return 'an integer'; + if (parseInt(size) < 0) return 'a counting number or zero'; + return true; + } + }, + queue: {alias: 'queue-size'} + }); + + const mediaPath = miscOptions.media || process.env.HSMUSIC_MEDIA; + if (!mediaPath) { + logError`Expected --media option or HSMUSIC_MEDIA to be set`; + } + + const queueSize = +(miscOptions['queue-size'] ?? 0); + + await genThumbs(mediaPath, {queueSize}); + })().catch(err => console.error(err)); +} diff --git a/upd8-util.js b/upd8-util.js index e188ed4c..64983313 100644 --- a/upd8-util.js +++ b/upd8-util.js @@ -46,13 +46,17 @@ module.exports.joinNoOxford = function(array, plural = 'and') { return `${array.slice(0, -1).join(', ')} ${plural} ${array[array.length - 1]}`; }; -module.exports.progressPromiseAll = function (msg, array) { +module.exports.progressPromiseAll = function (msgOrMsgFn, array) { if (!array.length) { return Promise.resolve([]); } + const msgFn = (typeof msgOrMsgFn === 'function' + ? msgOrMsgFn + : () => msgOrMsgFn); + let done = 0, total = array.length; - process.stdout.write(`\r${msg} [0/${total}]`); + process.stdout.write(`\r${msgFn()} [0/${total}]`); const start = Date.now(); return Promise.all(array.map(promise => promise.then(val => { done++; @@ -60,9 +64,9 @@ module.exports.progressPromiseAll = function (msg, array) { const pc = (Math.round(done / total * 1000) / 10 + '%').padEnd('99.9%'.length, ' '); if (done === total) { const time = Date.now() - start; - process.stdout.write(`\r\x1b[2m${msg} [${pc}] \x1b[0;32mDone! \x1b[0;2m(${time} ms) \x1b[0m\n`) + process.stdout.write(`\r\x1b[2m${msgFn()} [${pc}] \x1b[0;32mDone! \x1b[0;2m(${time} ms) \x1b[0m\n`) } else { - process.stdout.write(`\r${msg} [${pc}] `); + process.stdout.write(`\r${msgFn()} [${pc}] `); } return val; }))); @@ -95,6 +99,8 @@ module.exports.queue = function (array, max = 50) { return ret; }; +module.exports.delay = ms => new Promise(res => setTimeout(res, ms)); + module.exports.th = function (n) { if (n % 10 === 1 && n !== 11) { return n + 'st'; @@ -321,6 +327,7 @@ const logColor = color => (literals, ...values) => { w(`\x1b[0m\n`); }; +module.exports.logInfo = logColor(2); module.exports.logWarn = logColor(33); module.exports.logError = logColor(31); @@ -369,3 +376,29 @@ module.exports.chunkByProperties = function(array, properties) { chunk })); }; + +// Very cool function origin8ting in... http-music pro8a8ly! +// Sorry if we happen to 8e violating past-us's copyright, lmao. +module.exports.promisifyProcess = function(proc, showLogging = true) { + // Takes a process (from the child_process module) and returns a promise + // that resolves when the process exits (or rejects, if the exit code is + // non-zero). + // + // Ayy look, no alpha8etical second letter! Couldn't tell this was written + // like three years ago 8efore I was me. 8888) + + return new Promise((resolve, reject) => { + if (showLogging) { + proc.stdout.pipe(process.stdout); + proc.stderr.pipe(process.stderr); + } + + proc.on('exit', code => { + if (code === 0) { + resolve(); + } else { + reject(code); + } + }) + }) +}; diff --git a/upd8.js b/upd8.js index 07aaa139..21f16052 100755 --- a/upd8.js +++ b/upd8.js @@ -115,6 +115,7 @@ const { joinNoOxford, mapInPlace, logWarn, + logInfo, logError, parseOptions, progressPromiseAll, @@ -126,6 +127,8 @@ const { unique } = require('./upd8-util'); +const genThumbs = require('./gen-thumbs'); + const C = require('./common/common'); const CACHEBUST = 3; @@ -263,6 +266,14 @@ const link = { site: linkPathname('site', {color: false}) }; +const thumbnailHelper = name => file => + file.replace(/\.(jpg|png)$/, name + '.jpg'); + +const thumb = { + medium: thumbnailHelper('.medium'), + small: thumbnailHelper('.small') +}; + function generateURLs(fromPath) { const helper = toPath => { let argIndex = 0; @@ -958,6 +969,7 @@ function transformMultiline(text, {strings, to}) { line = line.replace(/<img (.*?)>/g, (match, attributes) => img({ lazy: true, link: true, + thumb: 'medium', ...parseAttributes(attributes, {to}) })); @@ -1878,6 +1890,7 @@ function attributes(attribs) { function img({ src = '', alt = '', + thumb: thumbKey = '', reveal = '', id = '', width = '', @@ -1889,6 +1902,9 @@ function img({ const willSquare = square; const willLink = typeof link === 'string' || link; + const originalSrc = src; + const thumbSrc = thumbKey ? thumb[thumbKey](src) : src; + const imgAttributes = attributes({ id: link ? '' : id, alt, @@ -1896,8 +1912,8 @@ function img({ height }); - const nonlazyHTML = wrap(`<img src="${src}" ${imgAttributes}>`); - const lazyHTML = lazy && wrap(`<img class="lazy" data-original="${src}" ${imgAttributes}>`, true); + const nonlazyHTML = wrap(`<img src="${thumbSrc}" ${imgAttributes}>`); + const lazyHTML = lazy && wrap(`<img class="lazy" data-original="${thumbSrc}" ${imgAttributes}>`, true); if (lazy) { return fixWS` @@ -1933,7 +1949,7 @@ function img({ if (willLink) { html = `<a ${classes('box', hide && 'js-hide')} ${attributes({ id, - href: typeof link === 'string' ? link : src + href: typeof link === 'string' ? link : originalSrc })}>${html}</a>`; } @@ -2202,6 +2218,7 @@ function getGridHTML({ ${img({ src: srcFn(item), alt: altFn(item), + thumb: 'small', lazy: (typeof lazy === 'number' ? i >= lazy : lazy), square: true, reveal: getRevealString(item.artTags, {strings}) @@ -2534,6 +2551,7 @@ function generateCoverLink({ ${img({ src, alt, + thumb: 'medium', id: 'cover-art', link: true, square: true, @@ -5174,6 +5192,13 @@ async function main() { } } + logInfo`Begin thumbnail generation... -----+`; + const result = await genThumbs(mediaPath, {queueSize, quiet: true}); + logInfo`Done thumbnail generation! --------+`; + if (!result) { + return; + } + const defaultStrings = await processLanguageFile(path.join(__dirname, DEFAULT_STRINGS_FILE)); if (defaultStrings.error) { logError`Error loading default strings: ${defaultStrings.error}`; |