From 90d466482e73e7a00023151e1c894f66d1c2ad79 Mon Sep 17 00:00:00 2001 From: "(quasar) nebula" Date: Fri, 17 Jan 2025 07:57:26 -0400 Subject: urls, upd8: default-override & optional url specs --- src/upd8.js | 136 +++++++++++++++++++++++++++++++++++++++++-- src/url-spec-default.yaml | 143 ---------------------------------------------- src/url-spec.js | 66 +++++++++++++++------ src/urls-default.yaml | 143 ++++++++++++++++++++++++++++++++++++++++++++++ src/urls.js | 7 +++ 5 files changed, 329 insertions(+), 166 deletions(-) delete mode 100644 src/url-spec-default.yaml create mode 100644 src/urls-default.yaml (limited to 'src') diff --git a/src/upd8.js b/src/upd8.js index b83b5171..187be4cc 100755 --- a/src/upd8.js +++ b/src/upd8.js @@ -40,7 +40,7 @@ import {fileURLToPath} from 'node:url'; import wrap from 'word-wrap'; -import {mapAggregate, showAggregate} from '#aggregate'; +import {mapAggregate, openAggregate, showAggregate} from '#aggregate'; import CacheableObject from '#cacheable-object'; import {displayCompositeCacheAnalysis} from '#composite'; import find, {bindFind, getAllFindSpecs} from '#find'; @@ -50,8 +50,6 @@ import {isMain, traverse} from '#node-utils'; import {bindReverse} from '#reverse'; import {writeSearchData} from '#search'; import {sortByName} from '#sort'; -import {internalDefaultURLSpecFile, generateURLs, processURLSpecFromFile} - from '#urls'; import {identifyAllWebRoutes} from '#web-routes'; import { @@ -75,6 +73,7 @@ import { import { bindOpts, empty, + filterMultipleArrays, indentWrap as unboundIndentWrap, withEntries, } from '#sugar'; @@ -88,6 +87,14 @@ import genThumbs, { verifyImagePaths, } from '#thumbs'; +import { + applyLocalizedWithBaseDirectory, + applyURLSpecOverriding, + generateURLs, + internalDefaultURLSpecFile, + processURLSpecFromFile, +} from '#urls'; + import { getAllDataSteps, linkWikiDataArrays, @@ -330,6 +337,11 @@ async function main() { type: 'value', }, + 'urls': { + help: `Specify which optional URL specs to use for this build, customizing where pages are generated or resources are accessed from`, + type: 'value', + }, + 'skip-directory-validation': { help: `Skips checking for duplicated directories, which speeds up the build but may cause the wiki to catch on fire`, type: 'flag', @@ -571,6 +583,8 @@ async function main() { const precacheMode = cliOptions['precache-mode'] ?? 'common'; + const wantedURLSpecKeys = cliOptions['urls'] ?? []; + // Makes writing nicer on the CPU and file I/O parts of the OS, with a // marginal performance deficit while waiting for file writes to finish // before proceeding to more page processing. @@ -1833,7 +1847,119 @@ async function main() { return false; } - const urlSpec = internalURLSpec; + // We'll mutate this as we load other url spec files. + const urlSpec = structuredClone(internalURLSpec); + + const allURLSpecDataFiles = + (await readdir(dataPath)) + .filter(name => + name.startsWith('urls') && + ['.json', '.yaml'].includes(path.extname(name))) + .sort() /* Just in case... */ + .map(name => path.join(dataPath, name)); + + const getURLSpecKeyFromFile = file => { + const base = path.basename(file, path.extname(file)); + if (base === 'urls') { + return base; + } else { + return base.replace(/^urls-/, ''); + } + }; + + const isDefaultURLSpecFile = file => + getURLSpecKeyFromFile(file) === 'urls'; + + const overrideDefaultURLSpecFile = + allURLSpecDataFiles.find(file => isDefaultURLSpecFile(file)); + + const optionalURLSpecDataFiles = + allURLSpecDataFiles.filter(file => !isDefaultURLSpecFile(file)); + + const optionalURLSpecDataKeys = + optionalURLSpecDataFiles.map(file => getURLSpecKeyFromFile(file)); + + const selectedURLSpecDataKeys = optionalURLSpecDataKeys.slice(); + const selectedURLSpecDataFiles = optionalURLSpecDataFiles.slice(); + + const {removed: [unusedURLSpecDataKeys]} = + filterMultipleArrays( + selectedURLSpecDataKeys, + selectedURLSpecDataFiles, + (key, _file) => wantedURLSpecKeys.includes(key)); + + if (!empty(selectedURLSpecDataKeys)) { + logInfo`Using these optional URL specs: ${selectedURLSpecDataKeys.join(', ')}`; + if (!empty(unusedURLSpecDataKeys)) { + logInfo`Other available optional URL specs: ${unusedURLSpecDataKeys.join(', ')}`; + } + } else if (!empty(unusedURLSpecDataKeys)) { + logInfo`Not using any optional URL specs.`; + logInfo`These are available with --urls: ${unusedURLSpecDataKeys.join(', ')}`; + } + + if (overrideDefaultURLSpecFile) { + try { + let aggregate; + let overrideDefaultURLSpec; + + ({aggregate, result: overrideDefaultURLSpec} = + await processURLSpecFromFile(overrideDefaultURLSpecFile)); + + aggregate.close(); + + ({aggregate} = + applyURLSpecOverriding(overrideDefaultURLSpec, urlSpec)); + + aggregate.close(); + } catch (error) { + niceShowAggregate(error); + logError`Errors loading this data repo's ${'urls.yaml'} file.`; + logError`This provides essential overrides for this wiki,`; + logError`so stopping here. Debug the errors to continue.`; + + Object.assign(stepStatusSummary.loadURLFiles, { + status: STATUS_FATAL_ERROR, + annotation: `see log for details`, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + + return false; + } + } + + const processURLSpecsAggregate = + openAggregate({message: `Errors processing URL specs`}); + + const selectedURLSpecs = + processURLSpecsAggregate.receive( + await Promise.all( + selectedURLSpecDataFiles + .map(file => processURLSpecFromFile(file)))); + + for (const selectedURLSpec of selectedURLSpecs) { + processURLSpecsAggregate.receive( + applyURLSpecOverriding(selectedURLSpec, urlSpec)); + } + + try { + processURLSpecsAggregate.close(); + } catch (error) { + niceShowAggregate(error); + logWarn`There were errors loading the optional URL specs you`; + logWarn`selected using ${'--urls'}. Since they might misfunction,`; + logWarn`debug the errors or remove the failing ones from ${'--urls'}.`; + + Object.assign(stepStatusSummary.loadURLFiles, { + status: STATUS_FATAL_ERROR, + annotation: `see log for details`, + timeEnd: Date.now(), + memory: process.memoryUsage(), + }); + + return false; + } Object.assign(stepStatusSummary.loadURLFiles, { status: STATUS_DONE_CLEAN, @@ -1841,6 +1967,8 @@ async function main() { memory: process.memoryUsage(), }); + applyLocalizedWithBaseDirectory(urlSpec); + const urls = generateURLs(urlSpec); const languageReloading = diff --git a/src/url-spec-default.yaml b/src/url-spec-default.yaml deleted file mode 100644 index 10bc0d23..00000000 --- a/src/url-spec-default.yaml +++ /dev/null @@ -1,143 +0,0 @@ -# These are variables which are used to make expressing this -# YAML file more convenient. They are not exposed externally. -# (Stuff which uses this YAML file can't even see the names -# for each variable!) -yamlAliases: - - &genericPaths - root: '' - path: '<>' - - # Static files are all grouped under a `static-${STATIC_VERSION}` folder as - # part of a build. This is so that multiple builds of a wiki can coexist - # served from the same server / file system root: older builds' HTML files - # refer to earlier values of STATIC_VERSION, avoiding name collisions. - - &staticVersion 3p4 - -data: - prefix: 'data/' - - paths: - - *genericPaths - - - album: 'album/<>' - artist: 'artist/<>' - track: 'track/<>' - -localized: - paths: - - *genericPaths - - page: '<>/' - - home: '' - - album: 'album/<>/' - albumCommentary: 'commentary/album/<>/' - albumGallery: 'album/<>/gallery/' - albumReferencedArtworks: 'album/<>/referenced-art/' - albumReferencingArtworks: 'album/<>/referencing-art/' - - artist: 'artist/<>/' - artistGallery: 'artist/<>/gallery/' - - commentaryIndex: 'commentary/' - - flashIndex: 'flash/' - - flash: 'flash/<>/' - - flashActGallery: 'flash-act/<>/' - - groupInfo: 'group/<>/' - groupGallery: 'group/<>/gallery/' - - listingIndex: 'list/' - - listing: 'list/<>/' - - newsIndex: 'news/' - - newsEntry: 'news/<>/' - - staticPage: '<>/' - - tag: 'tag/<>/' - - track: 'track/<>/' - trackReferencedArtworks: 'track/<>/referenced-art/' - trackReferencingArtworks: 'track/<>/referencing-art/' - -# This gets automatically switched in place when working from -# a baseDirectory, so it should never be referenced manually. -# It's also filled in externally to this YAML spec. -localizedWithBaseDirectory: '' - -shared: - paths: *genericPaths - -staticCSS: - prefix: - - 'static-' - - *staticVersion - - '/css/' - - paths: *genericPaths - -staticJS: - prefix: - - 'static-' - - *staticVersion - - '/js/' - - paths: *genericPaths - -staticLib: - prefix: - - 'static-' - - *staticVersion - - '/lib/' - - paths: *genericPaths - -staticMisc: - prefix: - - 'static-' - - *staticVersion - - '/misc/' - - paths: - - *genericPaths - - icon: 'icons.svg#icon-<>' - -staticSharedUtil: - prefix: - - 'static-' - - *staticVersion - - '/shared-util/' - - paths: *genericPaths - -media: - prefix: 'media/' - - paths: - - *genericPaths - - - albumAdditionalFile: 'album-additional/<>/<>' - albumBanner: 'album-art/<>/banner.<>' - albumCover: 'album-art/<>/cover.<>' - albumWallpaper: 'album-art/<>/bg.<>' - albumWallpaperPart: 'album-art/<>/<>' - - artistAvatar: 'artist-avatar/<>.<>' - - flashArt: 'flash-art/<>.<>' - - trackCover: 'album-art/<>/<>.<>' - -thumb: - prefix: 'thumb/' - paths: *genericPaths - -searchData: - prefix: 'search-data/' - paths: *genericPaths diff --git a/src/url-spec.js b/src/url-spec.js index 42e3e45c..f8ab6c69 100644 --- a/src/url-spec.js +++ b/src/url-spec.js @@ -10,7 +10,7 @@ import yaml from 'js-yaml'; import {annotateError, annotateErrorWithFile, openAggregate} from '#aggregate'; import {empty, typeAppearance, withEntries} from '#sugar'; -export const DEFAULT_URL_SPEC_FILE = 'url-spec-default.yaml'; +export const DEFAULT_URL_SPEC_FILE = 'urls-default.yaml'; export const internalDefaultURLSpecFile = path.resolve( @@ -96,6 +96,8 @@ export function processURLSpec(sourceSpec) { const aggregate = openAggregate({message: `Errors processing URL spec`}); + sourceSpec ??= {}; + const urlSpec = structuredClone(sourceSpec); delete urlSpec.yamlAliases; @@ -115,29 +117,14 @@ export function processURLSpec(sourceSpec) { case '': { if (!urlSpec.localized) { aggregate.push(new Error( - `Couldn't prepare 'localizedWithBaseDirectory' group, ` + + `Not ready for 'localizedWithBaseDirectory' group, ` + `'localized' not available`)); - - break; - } - - if (!urlSpec.localized.paths) { + } else if (!urlSpec.localized.paths) { aggregate.push(new Error( - `Couldn't prepare 'localizedWithBaseDirectory' group, ` + + `Not ready for 'localizedWithBaseDirectory' group, ` + `'localized' group's paths not available`)); - - break; } - const paths = - withEntries(urlSpec.localized.paths, entries => - entries.map(([key, path]) => [key, '<>/' + path])); - - urlSpec.localizedWithBaseDirectory = - Object.assign( - structuredClone(urlSpec.localized), - {paths}); - break; } @@ -155,6 +142,47 @@ export function processURLSpec(sourceSpec) { return {aggregate, result: urlSpec}; } +export function applyURLSpecOverriding(overrideSpec, baseSpec) { + const aggregate = openAggregate({message: `Errors applying URL spec`}); + + for (const [groupKey, overrideGroupSpec] of Object.entries(overrideSpec)) { + const baseGroupSpec = baseSpec[groupKey]; + + if (!baseGroupSpec) { + aggregate.push(new Error(`Group key "${groupKey}" not available on base spec`)); + continue; + } + + if (overrideGroupSpec.prefix) { + baseGroupSpec.prefix = overrideGroupSpec.prefix; + } + + if (overrideGroupSpec.paths) { + for (const [pathKey, overridePathValue] of Object.entries(overrideGroupSpec.paths)) { + if (!baseGroupSpec.paths[pathKey]) { + aggregate.push(new Error(`Path key "${groupKey}.${pathKey}" not available on base spec`)); + continue; + } + + baseGroupSpec.paths[pathKey] = overridePathValue; + } + } + } + + return {aggregate}; +} + +export function applyLocalizedWithBaseDirectory(urlSpec) { + const paths = + withEntries(urlSpec.localized.paths, entries => + entries.map(([key, path]) => [key, '<>/' + path])); + + urlSpec.localizedWithBaseDirectory = + Object.assign( + structuredClone(urlSpec.localized), + {paths}); +} + export async function processURLSpecFromFile(file) { let contents; diff --git a/src/urls-default.yaml b/src/urls-default.yaml new file mode 100644 index 00000000..10bc0d23 --- /dev/null +++ b/src/urls-default.yaml @@ -0,0 +1,143 @@ +# These are variables which are used to make expressing this +# YAML file more convenient. They are not exposed externally. +# (Stuff which uses this YAML file can't even see the names +# for each variable!) +yamlAliases: + - &genericPaths + root: '' + path: '<>' + + # Static files are all grouped under a `static-${STATIC_VERSION}` folder as + # part of a build. This is so that multiple builds of a wiki can coexist + # served from the same server / file system root: older builds' HTML files + # refer to earlier values of STATIC_VERSION, avoiding name collisions. + - &staticVersion 3p4 + +data: + prefix: 'data/' + + paths: + - *genericPaths + + - album: 'album/<>' + artist: 'artist/<>' + track: 'track/<>' + +localized: + paths: + - *genericPaths + - page: '<>/' + + home: '' + + album: 'album/<>/' + albumCommentary: 'commentary/album/<>/' + albumGallery: 'album/<>/gallery/' + albumReferencedArtworks: 'album/<>/referenced-art/' + albumReferencingArtworks: 'album/<>/referencing-art/' + + artist: 'artist/<>/' + artistGallery: 'artist/<>/gallery/' + + commentaryIndex: 'commentary/' + + flashIndex: 'flash/' + + flash: 'flash/<>/' + + flashActGallery: 'flash-act/<>/' + + groupInfo: 'group/<>/' + groupGallery: 'group/<>/gallery/' + + listingIndex: 'list/' + + listing: 'list/<>/' + + newsIndex: 'news/' + + newsEntry: 'news/<>/' + + staticPage: '<>/' + + tag: 'tag/<>/' + + track: 'track/<>/' + trackReferencedArtworks: 'track/<>/referenced-art/' + trackReferencingArtworks: 'track/<>/referencing-art/' + +# This gets automatically switched in place when working from +# a baseDirectory, so it should never be referenced manually. +# It's also filled in externally to this YAML spec. +localizedWithBaseDirectory: '' + +shared: + paths: *genericPaths + +staticCSS: + prefix: + - 'static-' + - *staticVersion + - '/css/' + + paths: *genericPaths + +staticJS: + prefix: + - 'static-' + - *staticVersion + - '/js/' + + paths: *genericPaths + +staticLib: + prefix: + - 'static-' + - *staticVersion + - '/lib/' + + paths: *genericPaths + +staticMisc: + prefix: + - 'static-' + - *staticVersion + - '/misc/' + + paths: + - *genericPaths + - icon: 'icons.svg#icon-<>' + +staticSharedUtil: + prefix: + - 'static-' + - *staticVersion + - '/shared-util/' + + paths: *genericPaths + +media: + prefix: 'media/' + + paths: + - *genericPaths + + - albumAdditionalFile: 'album-additional/<>/<>' + albumBanner: 'album-art/<>/banner.<>' + albumCover: 'album-art/<>/cover.<>' + albumWallpaper: 'album-art/<>/bg.<>' + albumWallpaperPart: 'album-art/<>/<>' + + artistAvatar: 'artist-avatar/<>.<>' + + flashArt: 'flash-art/<>.<>' + + trackCover: 'album-art/<>/<>.<>' + +thumb: + prefix: 'thumb/' + paths: *genericPaths + +searchData: + prefix: 'search-data/' + paths: *genericPaths diff --git a/src/urls.js b/src/urls.js index 1a471b30..71173f7f 100644 --- a/src/urls.js +++ b/src/urls.js @@ -11,6 +11,13 @@ import {withEntries} from '#sugar'; export * from './url-spec.js'; export function generateURLs(urlSpec) { + if ( + typeof urlSpec.localized === 'object' && + typeof urlSpec.localizedWithBaseDirectory !== 'object' + ) { + throw new Error(`Provided urlSpec missing localizedWithBaseDirectory`); + } + const getValueForFullKey = (obj, fullKey) => { const [groupKey, subKey] = fullKey.split('.'); if (!groupKey || !subKey) { -- cgit 1.3.0-6-gf8a5