From 0944e67a92b2ac7203af1e7152a33395b32923a2 Mon Sep 17 00:00:00 2001 From: "(quasar) nebula" Date: Sat, 27 May 2023 12:08:26 -0300 Subject: thumbs: imagemagick is fricking killing me --- src/upd8.js | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) (limited to 'src/upd8.js') diff --git a/src/upd8.js b/src/upd8.js index bfdd1c2a..2051517b 100755 --- a/src/upd8.js +++ b/src/upd8.js @@ -418,11 +418,12 @@ async function main() { } if (clearThumbsFlag) { - await clearThumbs(mediaPath, {queueSize}); - - logInfo`All done! Remove ${'--clear-thumbs'} to run the next build.`; - if (skipThumbs) { - logInfo`And don't forget to remove ${'--skip-thumbs'} too, eh?`; + const result = await clearThumbs(mediaPath, {queueSize}); + if (result.success) { + logInfo`All done! Remove ${'--clear-thumbs'} to run the next build.`; + if (skipThumbs) { + logInfo`And don't forget to remove ${'--skip-thumbs'} too, eh?`; + } } return; } @@ -437,7 +438,7 @@ async function main() { quiet: !thumbsOnly, }); logInfo`Done thumbnail generation! --------+`; - if (!result) return; + if (!result.success) return; if (thumbsOnly) return; } -- cgit 1.3.0-6-gf8a5 From 029210cc329a015a939472a688209d3f3423242b Mon Sep 17 00:00:00 2001 From: "(quasar) nebula" Date: Tue, 30 May 2023 09:51:26 -0300 Subject: thumbs, content: integrate cached thumb sizes into content --- src/upd8.js | 36 +++++++++++++++++++++++++++++++++--- 1 file changed, 33 insertions(+), 3 deletions(-) (limited to 'src/upd8.js') diff --git a/src/upd8.js b/src/upd8.js index 2051517b..2f08204a 100755 --- a/src/upd8.js +++ b/src/upd8.js @@ -32,6 +32,7 @@ // node.js and you'll 8e fine. ...Within the project root. O8viously. import {execSync} from 'node:child_process'; +import {readFile} from 'node:fs/promises'; import * as path from 'node:path'; import {fileURLToPath} from 'node:url'; @@ -57,6 +58,7 @@ import { } from '#cli'; import genThumbs, { + CACHE_FILE as thumbsCacheFile, clearThumbs, defaultMagickThreads, isThumb, @@ -428,7 +430,33 @@ async function main() { return; } + let thumbsCache; + if (skipThumbs) { + const thumbsCachePath = path.join(mediaPath, thumbsCacheFile); + try { + thumbsCache = JSON.parse(await readFile(thumbsCachePath)); + logInfo`Thumbnail cache file successfully read.`; + } catch (error) { + if (error.code === 'ENOENT') { + logError`The thumbnail cache doesn't exist, and it's necessary to build` + logError`the website. Please run once without ${'--skip-thumbs'} - after` + logError`that you'll be good to go and don't need to process thumbnails` + logError`again!`; + return; + } else { + logError`Malformed or unreadable thumbnail cache file: ${error}`; + logError`Path: ${thumbsCachePath}`; + logError`The thumbbnail cache is necessary to build the site, so you'll`; + logError`have to investigate this to get the build working. Try running`; + logError`again without ${'--skip-thumbs'}. If you can't get it working,`; + logError`you're welcome to message in the HSMusic Discord and we'll try`; + logError`to help you out with troubleshooting!`; + logError`${'https://hsmusic.wiki/discord/'}`; + return; + } + } + logInfo`Skipping thumbnail generation.`; } else { logInfo`Begin thumbnail generation... -----+`; @@ -440,6 +468,7 @@ async function main() { logInfo`Done thumbnail generation! --------+`; if (!result.success) return; if (thumbsOnly) return; + thumbsCache = result.cache; } if (noBuild) { @@ -705,7 +734,7 @@ async function main() { }; const getSizeOfAdditionalFile = getSizeOfMediaFileHelper(additionalFilePaths); - const getSizeOfImageFile = getSizeOfMediaFileHelper(imageFilePaths); + const getSizeOfImagePath = getSizeOfMediaFileHelper(imageFilePaths); logInfo`Preloading filesizes for ${additionalFilePaths.length} additional files...`; @@ -760,14 +789,15 @@ async function main() { defaultLanguage: finalDefaultLanguage, languages, - wikiData, + thumbsCache, urls, urlSpec, + wikiData, cachebust: '?' + CACHEBUST, developersComment, getSizeOfAdditionalFile, - getSizeOfImageFile, + getSizeOfImagePath, niceShowAggregate, }); } -- cgit 1.3.0-6-gf8a5 From 75a7b56d3616d384b31757c9537a6e27f4e9b350 Mon Sep 17 00:00:00 2001 From: "(quasar) nebula" Date: Mon, 28 Aug 2023 14:46:06 -0300 Subject: upd8: accept and pass --magick-threads through properly --- src/upd8.js | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'src/upd8.js') diff --git a/src/upd8.js b/src/upd8.js index 2f08204a..2ec231c9 100755 --- a/src/upd8.js +++ b/src/upd8.js @@ -233,6 +233,12 @@ async function main() { 'magick-threads': { help: `Process more or fewer thumbnail files at once with ImageMagick when generating thumbnails. (Each ImageMagick thread may also make use of multi-core processing at its own utility.)`, + type: 'value', + validate(threads) { + if (parseInt(threads) !== parseFloat(threads)) return 'an integer'; + if (parseInt(threads) < 0) return 'a counting number or zero'; + return true; + } }, magick: {alias: 'magick-threads'}, -- cgit 1.3.0-6-gf8a5 From eb00f2993a1aaaba171ad6c918656552f80bb748 Mon Sep 17 00:00:00 2001 From: "(quasar) nebula" Date: Thu, 7 Sep 2023 12:38:34 -0300 Subject: data: import Thing.common utilities directly Also rename 'color' (from #cli) to 'colors'. --- src/upd8.js | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'src/upd8.js') diff --git a/src/upd8.js b/src/upd8.js index 2ec231c9..f6091ca2 100755 --- a/src/upd8.js +++ b/src/upd8.js @@ -47,7 +47,7 @@ import {generateURLs, urlSpec} from '#urls'; import {sortByName} from '#wiki-data'; import { - color, + colors, decorateTime, logWarn, logInfo, @@ -279,7 +279,7 @@ async function main() { const indentWrap = (spaces, str) => wrap(str, {width: 60 - spaces, indent: ' '.repeat(spaces)}); const showOptions = (msg, options) => { - console.log(color.bright(msg)); + console.log(colors.bright(msg)); const entries = Object.entries(options); const sortedOptions = sortByName(entries @@ -310,13 +310,13 @@ async function main() { console.log(''); } - console.log(color.bright(` --` + name) + + console.log(colors.bright(` --` + name) + (aliases.length - ? ` (or: ${aliases.map(alias => color.bright(`--` + alias)).join(', ')})` + ? ` (or: ${aliases.map(alias => colors.bright(`--` + alias)).join(', ')})` : '') + (descriptor.help ? '' - : color.dim(' (no help provided)'))); + : colors.dim(' (no help provided)'))); if (wrappedHelp) { console.log(wrappedHelp); @@ -336,7 +336,7 @@ async function main() { }; console.log( - color.bright(`hsmusic (aka. Homestuck Music Wiki)\n`) + + colors.bright(`hsmusic (aka. Homestuck Music Wiki)\n`) + `static wiki software cataloguing collaborative creation\n`); console.log(indentWrap(0, @@ -496,7 +496,7 @@ async function main() { { const logThings = (thingDataProp, label) => - logInfo` - ${wikiData[thingDataProp]?.length ?? color.red('(Missing!)')} ${color.normal(color.dim(label))}`; + logInfo` - ${wikiData[thingDataProp]?.length ?? colors.red('(Missing!)')} ${colors.normal(colors.dim(label))}`; try { logInfo`Loaded data and processed objects:`; logThings('albumData', 'albums'); -- cgit 1.3.0-6-gf8a5 From c4f6c41a248ba9ef4f802cc03c20757d417540e4 Mon Sep 17 00:00:00 2001 From: "(quasar) nebula" Date: Sat, 9 Sep 2023 21:08:06 -0300 Subject: data: WIP cached composition nonsense --- src/upd8.js | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) (limited to 'src/upd8.js') diff --git a/src/upd8.js b/src/upd8.js index f6091ca2..7f423271 100755 --- a/src/upd8.js +++ b/src/upd8.js @@ -38,6 +38,7 @@ import {fileURLToPath} from 'node:url'; import wrap from 'word-wrap'; +import {displayCompositeCacheAnalysis} from '#composite'; import {processLanguageFile} from '#language'; import {isMain, traverse} from '#node-utils'; import bootRepl from '#repl'; @@ -612,6 +613,10 @@ async function main() { // which are only available after the initial linking. sortWikiDataArrays(wikiData); + console.log( + CacheableObject.getUpdateValue(wikiData.albumData[0], 'trackSections'), + wikiData.albumData[0].trackSections); + if (precacheData) { progressCallAll('Caching all data values', Object.entries(wikiData) .filter(([key]) => @@ -625,6 +630,11 @@ async function main() { .map(thing => () => CacheableObject.cacheAllExposedProperties(thing))); } + if (noBuild) { + displayCompositeCacheAnalysis(); + if (precacheData) return; + } + const internalDefaultLanguage = await processLanguageFile( path.join(__dirname, DEFAULT_STRINGS_FILE)); @@ -754,7 +764,9 @@ async function main() { logInfo`Done preloading filesizes!`; - if (noBuild) return; + if (noBuild) { + return; + } const developersComment = ``; - return selectedBuildMode.go({ - cliOptions, - dataPath, - mediaPath, - queueSize, - srcRootPath: __dirname, - - defaultLanguage: finalDefaultLanguage, - languages, - missingImagePaths, - thumbsCache, - urls, - urlSpec, - wikiData, - - cachebust: '?' + CACHEBUST, - developersComment, - getSizeOfAdditionalFile, - getSizeOfImagePath, - niceShowAggregate, - }); + stepStatusSummary.performBuild.status = STATUS_STARTED_NOT_DONE; + + let buildModeResult; + + try { + buildModeResult = await selectedBuildMode.go({ + cliOptions, + dataPath, + mediaPath, + queueSize, + srcRootPath: __dirname, + + defaultLanguage: finalDefaultLanguage, + languages, + missingImagePaths, + thumbsCache, + urls, + urlSpec, + wikiData, + + cachebust: '?' + CACHEBUST, + developersComment, + getSizeOfAdditionalFile, + getSizeOfImagePath, + niceShowAggregate, + }); + } catch (error) { + console.error(error); + + logError`There was a JavaScript error performing the build.`; + fileIssue(); + + Object.assign(stepStatusSummary.performBuild, { + status: STATUS_FATAL_ERROR, + message: `javascript error - view log for details`, + }); + + return false; + } + + if (buildModeResult !== true) { + Object.assign(stepStatusSummary.performBuild, { + status: STATUS_HAS_WARNINGS, + message: `may not have completed - view log for details`, + }); + + return false; + } + + stepStatusSummary.performBuild.status = STATUS_DONE_CLEAN; + + return true; } // TODO: isMain detection isn't consistent across platforms here @@ -834,6 +1103,65 @@ if (true || isMain(import.meta.url) || path.basename(process.argv[1]) === 'hsmus } } + if (showStepStatusSummary) { + console.error(colors.bright(`Step summary:`)); + + const longestNameLength = + Math.max(... + Object.values(stepStatusSummary) + .map(({name}) => name.length)); + + const anyStepsNotClean = + Object.values(stepStatusSummary) + .some(({status}) => + status === STATUS_HAS_WARNINGS || + status === STATUS_FATAL_ERROR || + status === STATUS_STARTED_NOT_DONE); + + for (const {name, status, annotation} of Object.values(stepStatusSummary)) { + let message = `${(name + ': ').padEnd(longestNameLength + 4, '.')} ${status}`; + if (annotation) { + message += ` (${annotation})`; + } + + switch (status) { + case STATUS_DONE_CLEAN: + console.error(colors.green(message)); + break; + + case STATUS_NOT_STARTED: + case STATUS_NOT_APPLICABLE: + console.error(colors.dim(message)); + break; + + case STATUS_HAS_WARNINGS: + case STATUS_STARTED_NOT_DONE: + console.error(colors.yellow(message)); + break; + + case STATUS_FATAL_ERROR: + console.error(colors.red(message)); + break; + + default: + console.error(message); + break; + } + } + + if (result === true) { + if (anyStepsNotClean) { + console.error(colors.bright(`Final output is true, but some steps aren't clean.`)); + process.exit(1); + return; + } else { + console.error(colors.bright(`Final output is true and all steps are clean.`)); + } + } else { + console.error(colors.bright(`Final output is not true (${result}).`)); + } + } + if (result !== true) { process.exit(1); return; -- cgit 1.3.0-6-gf8a5