diff options
Diffstat (limited to 'src/upd8.js')
| -rwxr-xr-x | src/upd8.js | 403 |
1 files changed, 350 insertions, 53 deletions
diff --git a/src/upd8.js b/src/upd8.js index 4c79007..600cc25 100755 --- a/src/upd8.js +++ b/src/upd8.js @@ -31,6 +31,8 @@ // Oh yeah, like. Just run this through some relatively recent version of // node.js and you'll 8e fine. ...Within the project root. O8viously. +import '#import-heck'; + import {execSync} from 'node:child_process'; import {readdir, readFile} from 'node:fs/promises'; import * as path from 'node:path'; @@ -38,31 +40,17 @@ import {fileURLToPath} from 'node:url'; import wrap from 'word-wrap'; -// Due to import time shenanigans, these imports have to come in the specified -// order. This obviously needs fixing up. - -/* precede #find */ -import { - filterReferenceErrors, - reportDuplicateDirectories, - reportContentTextErrors, -} from '#data-checks'; - -import {bindFind, getAllFindSpecs} from '#find'; - -// End of import time shenanigans (hopefully) - -import {showAggregate} from '#aggregate'; +import {mapAggregate, showAggregate} from '#aggregate'; import CacheableObject from '#cacheable-object'; import {displayCompositeCacheAnalysis} from '#composite'; +import {bindFind, getAllFindSpecs} from '#find'; import {processLanguageFile, watchLanguageFile, internalDefaultStringsFile} from '#language'; import {isMain, traverse} from '#node-utils'; import {sortByName} from '#sort'; import {empty, withEntries} from '#sugar'; import {generateURLs, urlSpec} from '#urls'; -import {linkWikiDataArrays, loadAndProcessDataDocuments, sortWikiDataArrays} - from '#yaml'; +import {identifyAllWebRoutes} from '#web-routes'; import { colors, @@ -75,6 +63,12 @@ import { progressCallAll, } from '#cli'; +import { + filterReferenceErrors, + reportDuplicateDirectories, + reportContentTextErrors, +} from '#data-checks'; + import genThumbs, { CACHE_FILE as thumbsCacheFile, defaultMagickThreads, @@ -84,6 +78,15 @@ import genThumbs, { verifyImagePaths, } from '#thumbs'; +import { + getAllDataSteps, + linkWikiDataArrays, + loadYAMLDocumentsFromDataSteps, + processThingsFromDataSteps, + saveThingsFromDataSteps, + sortWikiDataArrays, +} from '#yaml'; + import FileSizePreloader from './file-size-preloader.js'; import {listingSpec, listingTargetSpec} from './listing-spec.js'; import * as buildModes from './write/build-modes/index.js'; @@ -174,6 +177,9 @@ async function main() { preloadFileSizes: {...defaultStepStatus, name: `preload file sizes`}, + identifyWebRoutes: + {...defaultStepStatus, name: `identify web routes`}, + performBuild: {...defaultStepStatus, name: `perform selected build mode`}, }; @@ -241,7 +247,12 @@ async function main() { }, 'media-cache-path': { - help: `Specify path to media cache directory, including automatically generated thumbnails\n\nThis usually doesn't need to be provided, and will be inferred by adding "-cache" to the end of the media directory`, + help: `Specify path to media cache directory, including automatically generated thumbnails\n\nThis usually doesn't need to be provided, and will be inferred either by loading "media-cache" from --cache-path, or by adding "-cache" to the end of the media directory\n\nAlso may be provided via the HSMUSIC_MEDIA_CACHE environment variable`, + type: 'value', + }, + + 'cache-path': { + help: `Specify path to general cache directory, usually containing generated thumbnails and assorted files reused between builds\n\nRequired for some features and may always be required if you're starting a new workspace\n\nAlso may be provided via the HSMUSIC_CACHE environment varaible`, type: 'value', }, @@ -285,6 +296,11 @@ async function main() { type: 'flag', }, + 'new-thumbs': { + help: `Repair a media cache that's completely missing its index file by starting clean and not reusing any existing thumbnails`, + type: 'flag', + }, + 'skip-file-sizes': { help: `Skips preloading file sizes for images and additional files, which will be left blank in the build`, type: 'flag', @@ -474,6 +490,7 @@ async function main() { const dataPath = cliOptions['data-path'] || process.env.HSMUSIC_DATA; const mediaPath = cliOptions['media-path'] || process.env.HSMUSIC_MEDIA; + const wikiCachePath = cliOptions['cache-path'] || process.env.HSMUSIC_CACHE; const langPath = cliOptions['lang-path'] || process.env.HSMUSIC_LANG; // Can 8e left unset! const thumbsOnly = cliOptions['thumbs-only'] ?? false; @@ -501,6 +518,10 @@ async function main() { logError`${`Expected --media-path option or HSMUSIC_MEDIA to be set`}`; } + if (!wikiCachePath) { + logWarn`No --cache-path option nor HSMUSIC_CACHE set; provide for more features`; + } + if (!dataPath || !mediaPath) { return false; } @@ -556,18 +577,29 @@ async function main() { logWarn`Redundant option ${cliPart}`; } } else { - if (cliFlagNegates) { - step.status = STATUS_NOT_APPLICABLE; - step.annotation = `--${cliFlag} provided`; - } + step.status = + (cliFlagNegates + ? STATUS_NOT_APPLICABLE + : STATUS_NOT_STARTED); + + step.annotation = `--${cliFlag} provided`; + if (cliFlagWarning) { for (const line of cliFlagWarning.split('\n')) { logWarn(line); } } + + return; } } + if (buildConfig?.required === true) { + step.status = STATUS_NOT_STARTED; + step.annotation = `required for --${selectedBuildModeFlag}`; + return; + } + if (buildConfig?.applicable === false) { step.status = STATUS_NOT_APPLICABLE; step.annotation = `N/A for --${selectedBuildModeFlag}`; @@ -580,6 +612,12 @@ async function main() { return; } + if (buildConfig?.default === 'perform') { + step.status = STATUS_NOT_STARTED; + step.annotation = `default for --${selectedBuildModeFlag}`; + return; + } + switch (defaultValue) { case 'skip': step.status = STATUS_NOT_APPLICABLE; @@ -647,6 +685,11 @@ async function main() { }, }); + fallbackStep('identifyWebRoutes', { + default: 'skip', + buildConfig: 'webRoutes', + }); + fallbackStep('verifyImagePaths', { default: 'perform', buildConfig: 'mediaValidation', @@ -740,31 +783,118 @@ async function main() { timeStart: Date.now(), }); + const regenerateMissingThumbnailCache = + cliOptions['new-thumbs'] ?? false; + const {mediaCachePath, annotation: mediaCachePathAnnotation} = await determineMediaCachePath({ mediaPath, + wikiCachePath, + providedMediaCachePath: cliOptions['media-cache-path'] || process.env.HSMUSIC_MEDIA_CACHE, + + regenerateMissingThumbnailCache, + disallowDoubling: stepStatusSummary.migrateThumbnails.status === STATUS_NOT_STARTED, }); + if (regenerateMissingThumbnailCache) { + if ( + mediaCachePathAnnotation !== `contained path will regenerate missing cache` && + mediaCachePathAnnotation !== `adjacent path will regenerate missing cache` + ) { + if (mediaCachePath) { + logError`Determined a media cache path. (${mediaCachePathAnnotation})`; + console.error(''); + logWarn`By using ${'--new-thumbs'}, you requested to generate completely`; + logWarn`new thumbnails, but there's already a ${'thumbnail-cache.json'}`; + logWarn`file where it's expected, within this media cache:`; + logWarn`${path.resolve(mediaCachePath)}`; + console.error(''); + logWarn`If you really do want to completely regenerate all thumbnails`; + logWarn`and not reuse any existing ones, move aside ${'thumbnail-cache.json'}`; + logWarn`and run with ${'--new-thumbs'} again.`; + + Object.assign(stepStatusSummary.determineMediaCachePath, { + status: STATUS_FATAL_ERROR, + annotation: `--new-thumbs provided but regeneration not needed`, + timeEnd: Date.now(), + }); + + return false; + } else { + logError`Couldn't determine a media cache path. (${mediaCachePathAnnotation})`; + console.error(''); + logWarn`You requested to generate completely new thumbnails, but`; + logWarn`the media cache wasn't readable or just couldn't be found.`; + logWarn`Run again without ${'--new-thumbs'} - you should investigate`; + logWarn`what's going on before continuing.`; + + Object.assign(stepStatusSummary.determineMediaCachePath, { + status: STATUS_FATAL_ERROR, + annotation: mediaCachePathAnnotation, + timeEnd: Date.now(), + }); + + return false; + } + } + } + if (!mediaCachePath) { logError`Couldn't determine a media cache path. (${mediaCachePathAnnotation})`; switch (mediaCachePathAnnotation) { - case 'inferred path does not have cache': - logError`If you're certain this is the right path, you can provide it via`; - logError`${'--media-cache-path'} or ${'HSMUSIC_MEDIA_CACHE'}, and it should work.`; + case `contained path does not have cache`: + console.error(''); + logError`You've provided a ${'--cache-path'} or ${'HSMUSIC_CACHE_PATH'},`; + logError`${path.resolve(wikiCachePath)}`; + console.error(''); + logError`It contains a ${'media-cache'} folder, but this folder is`; + logError`missing its ${'thumbnail-cache.json'} file. This means there's`; + logError`no information available to reuse. If you use this cache,`; + logError`hsmusic will generate any existing thumbnails over again.`; + console.error(''); + logError`* Try to see if you can recover or locate a copy of your`; + logError` ${'thumbnail-cache.json'} file and put it back in place;`; + logError`* Or, generate all-new thumbnails with ${'--new-thumbs'}.`; break; - case 'inferred path not readable': + case 'adjacent path does not have cache': + console.error(''); + logError`You have an existing ${'media-cache'} folder next to your media path,`; + logError`${path.resolve(mediaPath)}`; + console.error(''); + logError`The ${'media-cache'} folder is missing its ${'thumbnail-cache.json'}`; + logError`file. This means there's no information available to reuse,`; + logError`and if you use this cache, hsmusic will generate any existing`; + logError`thumbnails over again.`; + console.error(''); + logError`* Try to see if you can recover or locate a copy of your`; + logError` ${'thumbnail-cache.json'} file and put it back in place;`; + logError`* Or, generate all-new thumbnails with ${'--new-thumbs'}.`; + break; + + case `contained path not readable`: + case `adjacent path not readable`: + console.error(''); logError`The folder couldn't be read, which usually indicates`; logError`a permissions error. Try to resolve this, or provide`; logError`a new path with ${'--media-cache-path'} or ${'HSMUSIC_MEDIA_CACHE'}.`; break; - case 'media path not provided': /* unreachable */ + case `missing wiki cache to create media cache inside`: + console.error(''); + logError`It looks like you're starting totally fresh, so please`; + logError`create a ${'cache'} folder and provide it with ${'--cache-path'}`; + logError`or ${'HSMUSIC_CACHE'}. The media cache will automatically be`; + logError`generated inside of this folder!`; + break; + + case `media path not provided`: /* unreachable */ + console.error(''); logError`It seems a ${'--media-path'} (or ${'HSMUSIC_MEDIA'}) wasn't provided.`; logError`Make sure one of these is actually pointing to a path that exists.`; break; @@ -936,32 +1066,102 @@ async function main() { CacheableObject.DEBUG_SLOW_TRACK_INVALID_PROPERTIES = true; } + let paragraph = false; + Object.assign(stepStatusSummary.loadDataFiles, { status: STATUS_STARTED_NOT_DONE, timeStart: Date.now(), }); - let processDataAggregate, wikiDataResult; + let yamlDataSteps; + let yamlDocumentProcessingAggregate; - try { - ({aggregate: processDataAggregate, result: wikiDataResult} = - await loadAndProcessDataDocuments({dataPath})); - } catch (error) { - console.error(error); + { + const whoops = (error, stage) => { + if (!paragraph) console.log(''); - logError`There was a JavaScript error loading data files.`; - fileIssue(); + console.error(error); + niceShowAggregate(error); - Object.assign(stepStatusSummary.loadDataFiles, { - status: STATUS_FATAL_ERROR, - annotation: `javascript error - view log for details`, - timeEnd: Date.now(), - }); + logError`There was a JavaScript error ${stage}.`; + fileIssue(); - return false; - } + Object.assign(stepStatusSummary.loadDataFiles, { + status: STATUS_FATAL_ERROR, + annotation: `javascript error - view log for details`, + timeEnd: Date.now(), + }); + + return false; + }; + + let loadAggregate, loadResult; + let processAggregate, processResult; + let saveAggregate, saveResult; + + const dataSteps = getAllDataSteps(); + + try { + ({aggregate: loadAggregate, result: loadResult} = + await loadYAMLDocumentsFromDataSteps( + dataSteps, + {dataPath})); + } catch (error) { + return whoops(error, `loading data files`); + } + + try { + loadAggregate.close(); + } catch (error) { + if (!paragraph) console.log(''); + niceShowAggregate(error); + + logError`The above errors were detected while loading data files.`; + logError`Since this indicates some files weren't able to load at all,`; + logError`there would probably be pretty bad reference errors if the`; + logError`build were to continue. Please resolve these errors and`; + logError`then give it another go.`; + + paragraph = true; + console.log(''); + + Object.assign(stepStatusSummary.loadDataFiles, { + status: STATUS_FATAL_ERROR, + annotation: `error loading data files`, + timeEnd: Date.now(), + }); + + return false; + } + + try { + ({aggregate: processAggregate, result: processResult} = + await processThingsFromDataSteps( + loadResult.documentLists, + loadResult.fileLists, + dataSteps, + {dataPath})); + } catch (error) { + return whoops(error, `processing data files`); + } + + try { + ({aggregate: saveAggregate, result: saveResult} = + saveThingsFromDataSteps( + processResult, + dataSteps)); + + saveAggregate.close(); + saveAggregate = undefined; + } catch (error) { + return whoops(error, `finalizing data files`); + } - Object.assign(wikiData, wikiDataResult); + yamlDataSteps = dataSteps; + yamlDocumentProcessingAggregate = processAggregate; + + Object.assign(wikiData, saveResult); + } { const logThings = (prop, label) => { @@ -974,13 +1174,20 @@ async function main() { } try { + if (!paragraph) console.log(''); + logInfo`Loaded data and processed objects:`; logThings('albumData', 'albums'); logThings('trackData', 'tracks'); - logThings(wikiData.artistData.filter(artist => !artist.isAlias), 'artists'); + logThings( + (wikiData.artistData + ? wikiData.artistData.filter(artist => !artist.isAlias) + : null), + 'artists'); if (wikiData.flashData) { logThings('flashData', 'flashes'); logThings('flashActData', 'flash acts'); + logThings('flashSideData', 'flash sides'); } logThings('groupData', 'groups'); logThings('groupCategoryData', 'group categories'); @@ -997,21 +1204,28 @@ async function main() { if (wikiData.wikiInfo) { logInfo` - ${1} wiki config file`; } + + console.log(''); + paragraph = true; } catch (error) { console.error(`Error showing data summary:`, error); + paragraph = false; } let errorless = true; try { - processDataAggregate.close(); + yamlDocumentProcessingAggregate.close(); } catch (error) { + if (!paragraph) console.log(''); niceShowAggregate(error); + logWarn`The above errors were detected while processing data files.`; + errorless = false; } if (!wikiData.wikiInfo) { - logError`Can't proceed without wiki info file successfully loading`; + logError`Can't proceed without wiki info file successfully loading.`; Object.assign(stepStatusSummary.loadDataFiles, { status: STATUS_FATAL_ERROR, @@ -1024,15 +1238,20 @@ async function main() { if (errorless) { logInfo`All data files processed without any errors - nice!`; + paragraph = false; Object.assign(stepStatusSummary.loadDataFiles, { status: STATUS_DONE_CLEAN, timeEnd: Date.now(), }); } else { - logWarn`If the remaining valid data is complete enough, the wiki will`; - logWarn`still build - but all errored data will be skipped.`; - logWarn`(Resolve errors for more complete output!)`; + logWarn`This might indicate some fields in the YAML data weren't formatted`; + logWarn`correctly, for example. The build should still work, but invalid`; + logWarn`fields will be skipped. Take a look at the report above to see`; + logWarn`what needs fixing up, for a more complete build!`; + + console.log(''); + paragraph = true; Object.assign(stepStatusSummary.loadDataFiles, { status: STATUS_HAS_WARNINGS, @@ -1132,12 +1351,14 @@ async function main() { try { reportDuplicateDirectories(wikiData, {getAllFindSpecs}); logInfo`No duplicate directories found - nice!`; + paragraph = false; Object.assign(stepStatusSummary.reportDuplicateDirectories, { status: STATUS_DONE_CLEAN, timeEnd: Date.now(), }); } catch (aggregate) { + if (!paragraph) console.log(''); niceShowAggregate(aggregate); logWarn`The above duplicate directories were detected while reviewing data files.`; @@ -1145,6 +1366,9 @@ async function main() { logWarn`correct, the build can't continue. Specify unique 'Directory' fields in`; logWarn`some or all of these data entries to resolve the errors.`; + console.log(''); + paragraph = true; + Object.assign(stepStatusSummary.reportDuplicateDirectories, { status: STATUS_FATAL_ERROR, annotation: `duplicate directories found`, @@ -1170,17 +1394,23 @@ async function main() { filterReferenceErrorsAggregate.close(); logInfo`All references validated without any errors - nice!`; + paragraph = false; Object.assign(stepStatusSummary.filterReferenceErrors, { status: STATUS_DONE_CLEAN, timeEnd: Date.now(), }); } catch (error) { + if (!paragraph) console.log(''); niceShowAggregate(error); logWarn`The above errors were detected while validating references in data files.`; - logWarn`The wiki will still build, but these connections between data objects`; - logWarn`will be completely skipped. Resolve the errors for more complete output.`; + logWarn`The wiki should still build, but these connections between data objects`; + logWarn`will be skipped, which might have unexpected consequences. Take a look at`; + logWarn`the report above to see what needs fixing up, for a more complete build!`; + + console.log(''); + paragraph = true; Object.assign(stepStatusSummary.filterReferenceErrors, { status: STATUS_HAS_WARNINGS, @@ -1198,19 +1428,25 @@ async function main() { try { reportContentTextErrors(wikiData, {bindFind}); + logInfo`All content text validated without any errors - nice!`; + paragraph = false; Object.assign(stepStatusSummary.reportContentTextErrors, { status: STATUS_DONE_CLEAN, timeEnd: Date.now(), }); } catch (error) { + if (!paragraph) console.log(''); niceShowAggregate(error); logWarn`The above errors were detected while processing content text in data files.`; logWarn`The wiki will still build, but placeholders will be displayed in these spots.`; logWarn`Resolve the errors for more complete output.`; + console.log(''); + paragraph = true; + Object.assign(stepStatusSummary.reportContentTextErrors, { status: STATUS_HAS_WARNINGS, annotation: `view log for details`, @@ -1227,7 +1463,7 @@ async function main() { timeStart: Date.now(), }); - sortWikiDataArrays(wikiData); + sortWikiDataArrays(yamlDataSteps, wikiData); Object.assign(stepStatusSummary.sortWikiDataArrays, { status: STATUS_DONE_CLEAN, @@ -1522,6 +1758,7 @@ async function main() { } logInfo`Applying new default strings from custom ${customDefaultLanguage.code} language file.`; + paragraph = false; finalDefaultLanguage = customDefaultLanguage; finalDefaultLanguageAnnotation = `using wiki-specified custom default language`; @@ -1602,6 +1839,7 @@ async function main() { } logInfo`Loaded language strings: ${Object.keys(languages).join(', ')}`; + paragraph = false; Object.assign(stepStatusSummary.initializeDefaultLanguage, { status: STATUS_DONE_CLEAN, @@ -1684,7 +1922,7 @@ async function main() { ...(track.midiProjectFiles ?? []), ]), ] - .flatMap((fileGroup) => fileGroup.files) + .flatMap((fileGroup) => fileGroup.files ?? []) .map((file) => ({ device: path.join( mediaPath, @@ -1734,6 +1972,7 @@ async function main() { await fileSizePreloader.waitUntilDoneLoading(); logInfo`Preloading filesizes for ${imageFilePaths.length} full-resolution images...`; + paragraph = false; fileSizePreloader.loadPaths(...imageFilePaths.map((path) => path.device)); await fileSizePreloader.waitUntilDoneLoading(); @@ -1750,6 +1989,7 @@ async function main() { }); } else { logInfo`Done preloading filesizes without any errors - nice!`; + paragraph = false; Object.assign(stepStatusSummary.preloadFileSizes, { status: STATUS_DONE_CLEAN, @@ -1758,6 +1998,62 @@ async function main() { } } + let webRoutes = null; + + if (stepStatusSummary.identifyWebRoutes.status === STATUS_NOT_STARTED) { + Object.assign(stepStatusSummary.identifyWebRoutes, { + status: STATUS_STARTED_NOT_DONE, + timeStart: Date.now(), + }); + + const fromRoot = urls.from('shared.root'); + + try { + const webRouteSources = await identifyAllWebRoutes({ + mediaCachePath, + mediaPath, + wikiCachePath, + }); + + const {aggregate, result} = + mapAggregate( + webRouteSources, + ({to, ...rest}) => ({ + ...rest, + to: fromRoot.to(...to), + }), + {message: `Errors computing effective web route paths`},); + + aggregate.close(); + webRoutes = result; + } catch (error) { + if (!paragraph) console.log(''); + niceShowAggregate(error); + + logError`There was an issue identifying web routes!`; + fileIssue(); + + console.log(''); + paragraph = true; + + Object.assign(stepStatusSummary.identifyWebRoutes, { + status: STATUS_FATAL_ERROR, + message: `JavaScript error - view log for details`, + timeEnd: Date.now(), + }); + + return false; + } + + logInfo`Successfully determined web routes.`; + paragraph = false; + + Object.assign(stepStatusSummary.identifyWebRoutes, { + status: STATUS_DONE_CLEAN, + timeEnd: Date.now(), + }); + } + if (stepStatusSummary.performBuild.status === STATUS_NOT_APPLICABLE) { return true; } @@ -1814,6 +2110,7 @@ async function main() { thumbsCache, urls, urlSpec, + webRoutes, wikiData, cachebust: '?' + CACHEBUST, |