From db816222a07a4a8dff9f0f8b66e7aa7cb7c15eb5 Mon Sep 17 00:00:00 2001 From: "(quasar) nebula" Date: Sun, 8 Jan 2023 10:52:12 -0400 Subject: extract static-build, new modular build modes --- src/write/build-modes/static-build.js | 423 ++++++++++++++++++++++++++++++++++ 1 file changed, 423 insertions(+) create mode 100644 src/write/build-modes/static-build.js (limited to 'src/write/build-modes/static-build.js') diff --git a/src/write/build-modes/static-build.js b/src/write/build-modes/static-build.js new file mode 100644 index 00000000..eafb53d6 --- /dev/null +++ b/src/write/build-modes/static-build.js @@ -0,0 +1,423 @@ +import {bindUtilities} from '../bind-utilities.js'; +import {validateWrites} from '../validate-writes.js'; + +import { + generateDocumentHTML, + generateGlobalWikiDataJSON, + generateOEmbedJSON, + generateRedirectHTML, +} from '../page-template.js'; + +import { + writePage, + writeSharedFilesAndPages, + writeSymlinks, +} from '../write-files.js'; + +import {serializeThings} from '../../data/serialize.js'; + +import * as pageSpecs from '../../page/index.js'; + +import link from '../../util/link.js'; +import {empty, queue, withEntries} from '../../util/sugar.js'; +import {getPagePaths, getURLsFrom} from '../../util/urls.js'; + +import { + logError, + logInfo, + logWarn, + progressCallAll, + progressPromiseAll, +} from '../../util/cli.js'; + +const pageFlags = Object.keys(pageSpecs); + +export function getCLIOptions() { + return { + // This is the output directory. It's the one you'll upload online with + // rsync or whatever when you're pushing an upd8, and also the one + // you'd archive if you wanted to make a 8ackup of the whole dang + // site. Just keep in mind that the gener8ted result will contain a + // couple symlinked directories, so if you're uploading, you're pro8a8ly + // gonna want to resolve those yourself. + 'out-path': { + type: 'value', + }, + + // Working without a dev server and just using file:// URLs in your we8 + // 8rowser? This will automatically append index.html to links across + // the site. Not recommended for production, since it isn't guaranteed + // 100% error-free (and index.html-style links are less pretty anyway). + 'append-index-html': { + type: 'flag', + }, + + // Only want to 8uild one language during testing? This can chop down + // 8uild times a pretty 8ig chunk! Just pass a single language code. + 'lang': { + type: 'value', + }, + + // NOT for neatly ena8ling or disa8ling specific features of the site! + // This is only in charge of what general groups of files to write. + // They're here to make development quicker when you're only working + // on some particular area(s) of the site rather than making changes + // across all of them. + ...Object.fromEntries(pageFlags.map((key) => [key, {type: 'flag'}])), + }; +} + +export async function go({ + cliOptions, + _dataPath, + mediaPath, + queueSize, + + defaultLanguage, + languages, + srcRootPath, + urls, + urlSpec, + wikiData, + + cachebust, + developersComment, + getSizeOfAdditionalFile, +}) { + const outputPath = cliOptions['out-path'] || process.env.HSMUSIC_OUT; + const appendIndexHTML = cliOptions['append-index-html'] ?? false; + const writeOneLanguage = cliOptions['lang'] ?? null; + + if (!outputPath) { + logError`Expected ${'--out-path'} option or ${'HSMUSIC_OUT'} to be set`; + return false; + } + + if (appendIndexHTML) { + logWarn`Appending index.html to link hrefs. (Note: not recommended for production release!)`; + link.globalOptions.appendIndexHTML = true; + } + + if (writeOneLanguage && !(writeOneLanguage in languages)) { + logError`Specified to write only ${writeOneLanguage}, but there is no strings file with this language code!`; + return false; + } else if (writeOneLanguage) { + logInfo`Writing only language ${writeOneLanguage} this run.`; + } else { + logInfo`Writing all languages.`; + } + + const selectedPageFlags = Object.keys(cliOptions) + .filter(key => pageFlags.includes(key)); + + const writeAll = empty(selectedPageFlags) || selectedPageFlags.includes('all'); + logInfo`Writing site pages: ${writeAll ? 'all' : selectedPageFlags.join(', ')}`; + + await writeSymlinks({ + srcRootPath, + mediaPath, + outputPath, + urls, + }); + + await writeSharedFilesAndPages({ + mediaPath, + outputPath, + urls, + + language: defaultLanguage, + wikiData, + wikiDataJSON: generateGlobalWikiDataJSON({ + serializeThings, + wikiData, + }) + }); + + const buildSteps = writeAll + ? Object.entries(pageSpecs) + : Object.entries(pageSpecs) + .filter(([flag]) => selectedPageFlags.includes(flag)); + + let writes; + { + let error = false; + + const buildStepsWithTargets = buildSteps + .map(([flag, pageSpec]) => { + // Condition not met: skip this build step altogether. + if (pageSpec.condition && !pageSpec.condition({wikiData})) { + return null; + } + + // May still call writeTargetless if present. + if (!pageSpec.targets) { + return {flag, pageSpec, targets: []}; + } + + if (!pageSpec.write) { + logError`${flag + '.targets'} is specified, but ${flag + '.write'} is missing!`; + error = true; + return null; + } + + const targets = pageSpec.targets({wikiData}); + if (!Array.isArray(targets)) { + logError`${flag + '.targets'} was called, but it didn't return an array! (${typeof targets})`; + error = true; + return null; + } + + return {flag, pageSpec, targets}; + }) + .filter(Boolean); + + if (error) { + return false; + } + + writes = progressCallAll('Computing page & data writes.', buildStepsWithTargets.flatMap(({flag, pageSpec, targets}) => { + const writesFns = targets.map(target => () => { + const writes = pageSpec.write(target, {wikiData})?.slice() || []; + const valid = validateWrites(writes, { + functionName: flag + '.write', + urlSpec, + }); + error ||=! valid; + return valid ? writes : []; + }); + + if (pageSpec.writeTargetless) { + writesFns.push(() => { + const writes = pageSpec.writeTargetless({wikiData}); + const valid = validateWrites(writes, { + functionName: flag + '.writeTargetless', + urlSpec, + }); + error ||=! valid; + return valid ? writes : []; + }); + } + + return writesFns; + })).flat(); + + if (error) { + return false; + } + } + + const pageWrites = writes.filter(({type}) => type === 'page'); + const dataWrites = writes.filter(({type}) => type === 'data'); + const redirectWrites = writes.filter(({type}) => type === 'redirect'); + + if (writes.length) { + logInfo`Total of ${writes.length} writes returned. (${pageWrites.length} page, ${dataWrites.length} data [currently skipped], ${redirectWrites.length} redirect)`; + } else { + logWarn`No writes returned at all, so exiting early. This is probably a bug!`; + return false; + } + + /* + await progressPromiseAll(`Writing data files shared across languages.`, queue( + dataWrites.map(({path, data}) => () => { + const bound = {}; + + bound.serializeLink = bindOpts(serializeLink, {}); + + bound.serializeContribs = bindOpts(serializeContribs, {}); + + bound.serializeImagePaths = bindOpts(serializeImagePaths, { + thumb + }); + + bound.serializeCover = bindOpts(serializeCover, { + [bindOpts.bindIndex]: 2, + serializeImagePaths: bound.serializeImagePaths, + urls + }); + + bound.serializeGroupsForAlbum = bindOpts(serializeGroupsForAlbum, { + serializeLink + }); + + bound.serializeGroupsForTrack = bindOpts(serializeGroupsForTrack, { + serializeLink + }); + + // TODO: This only supports one <>-style argument. + return writeData(path[0], path[1], data({...bound})); + }), + queueSize + )); + */ + + const perLanguageFn = async (language, i, entries) => { + const baseDirectory = + language === defaultLanguage ? '' : language.code; + + console.log(`\x1b[34;1m${`[${i + 1}/${entries.length}] ${language.code} (-> /${baseDirectory}) `.padEnd(60, '-')}\x1b[0m`); + + await progressPromiseAll(`Writing ${language.code}`, queue([ + ...pageWrites.map((props) => () => { + const {path, page} = props; + + const pageSubKey = path[0]; + const urlArgs = path.slice(1); + + const localizedPaths = withEntries(languages, entries => entries + .filter(([key, language]) => key !== 'default' && !language.hidden) + .map(([_key, language]) => [ + language.code, + getPagePaths({ + outputPath, + urls, + + baseDirectory: + (language === defaultLanguage + ? '' + : language.code), + fullKey: 'localized.' + pageSubKey, + urlArgs, + }), + ])); + + const paths = getPagePaths({ + outputPath, + urls, + + baseDirectory, + fullKey: 'localized.' + pageSubKey, + urlArgs, + }); + + const to = getURLsFrom({ + urls, + baseDirectory, + pageSubKey, + paths, + }); + + const absoluteTo = (targetFullKey, ...args) => { + const [groupKey, subKey] = targetFullKey.split('.'); + const from = urls.from('shared.root'); + return ( + '/' + + (groupKey === 'localized' && baseDirectory + ? from.to( + 'localizedWithBaseDirectory.' + subKey, + baseDirectory, + ...args + ) + : from.to(targetFullKey, ...args)) + ); + }; + + const bound = bindUtilities({ + language, + to, + wikiData, + }); + + const pageInfo = page({ + ...bound, + + language, + + absoluteTo, + relativeTo: to, + to, + urls, + + getSizeOfAdditionalFile, + }); + + const oEmbedJSON = generateOEmbedJSON(pageInfo, { + language, + wikiData, + }); + + const oEmbedJSONHref = + oEmbedJSON && + wikiData.wikiInfo.canonicalBase && + wikiData.wikiInfo.canonicalBase + + urls + .from('shared.root') + .to('shared.path', paths.pathname + 'oembed.json'); + + const pageHTML = generateDocumentHTML(pageInfo, { + cachebust, + defaultLanguage, + developersComment, + getThemeString: bound.getThemeString, + language, + languages, + localizedPaths, + oEmbedJSONHref, + paths, + to, + transformMultiline: bound.transformMultiline, + wikiData, + }); + + return writePage({ + html: pageHTML, + oEmbedJSON, + paths, + }); + }), + ...redirectWrites.map(({fromPath, toPath, title: titleFn}) => () => { + const title = titleFn({ + language, + }); + + const from = getPagePaths({ + outputPath, + urls, + + baseDirectory, + fullKey: 'localized.' + fromPath[0], + urlArgs: fromPath.slice(1), + }); + + const to = getURLsFrom({ + urls, + baseDirectory, + pageSubKey: fromPath[0], + paths: from, + }); + + const target = to('localized.' + toPath[0], ...toPath.slice(1)); + const html = generateRedirectHTML(title, target, {language}); + return writePage({html, paths: from}); + }), + ], queueSize)); + }; + + await wrapLanguages(perLanguageFn, { + languages, + writeOneLanguage, + }); + + // The single most important step. + logInfo`Written!`; + return true; +} + +// Wrapper function for running a function once for all languages. +async function wrapLanguages(fn, { + languages, + writeOneLanguage = null, +}) { + const k = writeOneLanguage; + const languagesToRun = k ? {[k]: languages[k]} : languages; + + const entries = Object.entries(languagesToRun).filter( + ([key]) => key !== 'default' + ); + + for (let i = 0; i < entries.length; i++) { + const [_key, language] = entries[i]; + + await fn(language, i, entries); + } +} -- cgit 1.3.0-6-gf8a5 From 2e90eaed378491142cb9d57ce705c58f4a598a10 Mon Sep 17 00:00:00 2001 From: "(quasar) nebula" Date: Sun, 8 Jan 2023 10:59:13 -0400 Subject: move static-specific write fns into build mode --- src/write/build-modes/static-build.js | 148 ++++++++++++++++++++++++++++++++-- 1 file changed, 140 insertions(+), 8 deletions(-) (limited to 'src/write/build-modes/static-build.js') diff --git a/src/write/build-modes/static-build.js b/src/write/build-modes/static-build.js index eafb53d6..b3700c43 100644 --- a/src/write/build-modes/static-build.js +++ b/src/write/build-modes/static-build.js @@ -1,3 +1,5 @@ +import * as path from 'path'; + import {bindUtilities} from '../bind-utilities.js'; import {validateWrites} from '../validate-writes.js'; @@ -8,12 +10,6 @@ import { generateRedirectHTML, } from '../page-template.js'; -import { - writePage, - writeSharedFilesAndPages, - writeSymlinks, -} from '../write-files.js'; - import {serializeThings} from '../../data/serialize.js'; import * as pageSpecs from '../../page/index.js'; @@ -120,12 +116,15 @@ export async function go({ urls, }); - await writeSharedFilesAndPages({ + await writeFavicon({ mediaPath, outputPath, - urls, + }); + await writeSharedFilesAndPages({ language: defaultLanguage, + outputPath, + urls, wikiData, wikiDataJSON: generateGlobalWikiDataJSON({ serializeThings, @@ -421,3 +420,136 @@ async function wrapLanguages(fn, { await fn(language, i, entries); } } + +import { + copyFile, + mkdir, + stat, + symlink, + writeFile, + unlink, +} from 'fs/promises'; + +async function writePage({ + html, + oEmbedJSON = '', + paths, +}) { + await mkdir(paths.output.directory, {recursive: true}); + + await Promise.all([ + writeFile(paths.output.documentHTML, html), + + oEmbedJSON && + writeFile(paths.output.oEmbedJSON, oEmbedJSON), + ].filter(Boolean)); +} + +function writeSymlinks({ + srcRootPath, + mediaPath, + outputPath, + urls, +}) { + return progressPromiseAll('Writing site symlinks.', [ + link(path.join(srcRootPath, 'util'), 'shared.utilityRoot'), + link(path.join(srcRootPath, 'static'), 'shared.staticRoot'), + link(mediaPath, 'media.root'), + ]); + + async function link(directory, urlKey) { + const pathname = urls.from('shared.root').toDevice(urlKey); + const file = path.join(outputPath, pathname); + + try { + await unlink(file); + } catch (error) { + if (error.code !== 'ENOENT') { + throw error; + } + } + + try { + await symlink(path.resolve(directory), file); + } catch (error) { + if (error.code === 'EPERM') { + await symlink(path.resolve(directory), file, 'junction'); + } + } + } +} + +async function writeFavicon({ + mediaPath, + outputPath, +}) { + const faviconFile = 'favicon.ico'; + + try { + await stat(path.join(mediaPath, faviconFile)); + } catch (error) { + return; + } + + try { + await copyFile( + path.join(mediaPath, faviconFile), + path.join(outputPath, faviconFile)); + } catch (error) { + logWarn`Failed to copy favicon! ${error.message}`; + return; + } + + logInfo`Copied favicon to site root.`; +} + +async function writeSharedFilesAndPages({ + language, + outputPath, + urls, + wikiData, + wikiDataJSON, +}) { + const {groupData, wikiInfo} = wikiData; + + return progressPromiseAll(`Writing files & pages shared across languages.`, [ + groupData?.some((group) => group.directory === 'fandom') && + redirect( + 'Fandom - Gallery', + 'albums/fandom', + 'localized.groupGallery', + 'fandom' + ), + + groupData?.some((group) => group.directory === 'official') && + redirect( + 'Official - Gallery', + 'albums/official', + 'localized.groupGallery', + 'official' + ), + + wikiInfo.enableListings && + redirect( + 'Album Commentary', + 'list/all-commentary', + 'localized.commentaryIndex', + '' + ), + + wikiDataJSON && + writeFile( + path.join(outputPath, 'data.json'), + wikiDataJSON), + ].filter(Boolean)); + + async function redirect(title, from, urlKey, directory) { + const target = path.relative( + from, + urls.from('shared.root').to(urlKey, directory) + ); + const content = generateRedirectHTML(title, target, {language}); + await mkdir(path.join(outputPath, from), {recursive: true}); + await writeFile(path.join(outputPath, from, 'index.html'), content); + } +} -- cgit 1.3.0-6-gf8a5 From 594e8dd46f9e6cc74c680536a1d820eef27133f0 Mon Sep 17 00:00:00 2001 From: "(quasar) nebula" Date: Mon, 9 Jan 2023 21:07:16 -0400 Subject: most essential behavior for live-dev-server --- src/write/build-modes/static-build.js | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) (limited to 'src/write/build-modes/static-build.js') diff --git a/src/write/build-modes/static-build.js b/src/write/build-modes/static-build.js index b3700c43..1544a122 100644 --- a/src/write/build-modes/static-build.js +++ b/src/write/build-modes/static-build.js @@ -16,7 +16,6 @@ import * as pageSpecs from '../../page/index.js'; import link from '../../util/link.js'; import {empty, queue, withEntries} from '../../util/sugar.js'; -import {getPagePaths, getURLsFrom} from '../../util/urls.js'; import { logError, @@ -26,6 +25,13 @@ import { progressPromiseAll, } from '../../util/cli.js'; +import { + getPagePathname, + getPagePaths, + getPageSubdirectoryPrefix, + getURLsFrom, +} from '../../util/urls.js'; + const pageFlags = Object.keys(pageSpecs); export function getCLIOptions() { @@ -263,20 +269,18 @@ export async function go({ const pageSubKey = path[0]; const urlArgs = path.slice(1); - const localizedPaths = withEntries(languages, entries => entries + const localizedPathnames = withEntries(languages, entries => entries .filter(([key, language]) => key !== 'default' && !language.hidden) .map(([_key, language]) => [ language.code, - getPagePaths({ - outputPath, - urls, - + getPagePathname({ baseDirectory: (language === defaultLanguage ? '' : language.code), fullKey: 'localized.' + pageSubKey, urlArgs, + urls, }), ])); @@ -293,7 +297,9 @@ export async function go({ urls, baseDirectory, pageSubKey, - paths, + subdirectoryPrefix: getPageSubdirectoryPrefix({ + urlArgs: page.path.slice(1), + }), }); const absoluteTo = (targetFullKey, ...args) => { @@ -320,8 +326,6 @@ export async function go({ const pageInfo = page({ ...bound, - language, - absoluteTo, relativeTo: to, to, @@ -350,7 +354,7 @@ export async function go({ getThemeString: bound.getThemeString, language, languages, - localizedPaths, + localizedPathnames, oEmbedJSONHref, paths, to, @@ -382,7 +386,9 @@ export async function go({ urls, baseDirectory, pageSubKey: fromPath[0], - paths: from, + subdirectoryPrefix: getPageSubdirectoryPrefix({ + urlArgs: fromPath.slice(1), + }), }); const target = to('localized.' + toPath[0], ...toPath.slice(1)); -- cgit 1.3.0-6-gf8a5 From d442546057f8280a141d4aa54f633a09c429e2d3 Mon Sep 17 00:00:00 2001 From: "(quasar) nebula" Date: Tue, 10 Jan 2023 16:55:04 -0400 Subject: extract absoluteTo --- src/write/build-modes/static-build.js | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-) (limited to 'src/write/build-modes/static-build.js') diff --git a/src/write/build-modes/static-build.js b/src/write/build-modes/static-build.js index 1544a122..c2bb02f2 100644 --- a/src/write/build-modes/static-build.js +++ b/src/write/build-modes/static-build.js @@ -30,6 +30,7 @@ import { getPagePaths, getPageSubdirectoryPrefix, getURLsFrom, + getURLsFromRoot, } from '../../util/urls.js'; const pageFlags = Object.keys(pageSpecs); @@ -302,20 +303,10 @@ export async function go({ }), }); - const absoluteTo = (targetFullKey, ...args) => { - const [groupKey, subKey] = targetFullKey.split('.'); - const from = urls.from('shared.root'); - return ( - '/' + - (groupKey === 'localized' && baseDirectory - ? from.to( - 'localizedWithBaseDirectory.' + subKey, - baseDirectory, - ...args - ) - : from.to(targetFullKey, ...args)) - ); - }; + const absoluteTo = getURLsFromRoot({ + baseDirectory, + urls, + }); const bound = bindUtilities({ language, @@ -327,7 +318,6 @@ export async function go({ ...bound, absoluteTo, - relativeTo: to, to, urls, -- cgit 1.3.0-6-gf8a5 From 4de56d21db5004e5b2c2a49caedee0908b9c6a9f Mon Sep 17 00:00:00 2001 From: "(quasar) nebula" Date: Tue, 10 Jan 2023 17:01:54 -0400 Subject: bind more utilities in bindUtilities --- src/write/build-modes/static-build.js | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) (limited to 'src/write/build-modes/static-build.js') diff --git a/src/write/build-modes/static-build.js b/src/write/build-modes/static-build.js index c2bb02f2..dab2598a 100644 --- a/src/write/build-modes/static-build.js +++ b/src/write/build-modes/static-build.js @@ -309,20 +309,15 @@ export async function go({ }); const bound = bindUtilities({ + absoluteTo, + getSizeOfAdditionalFile, language, to, + urls, wikiData, }); - const pageInfo = page({ - ...bound, - - absoluteTo, - to, - urls, - - getSizeOfAdditionalFile, - }); + const pageInfo = page(bound); const oEmbedJSON = generateOEmbedJSON(pageInfo, { language, -- cgit 1.3.0-6-gf8a5