diff options
Diffstat (limited to 'src/write')
| -rw-r--r-- | src/write/bind-utilities.js | 12 | ||||
| -rw-r--r-- | src/write/build-modes/index.js | 1 | ||||
| -rw-r--r-- | src/write/build-modes/live-dev-server.js | 213 | ||||
| -rw-r--r-- | src/write/build-modes/repl.js | 41 | ||||
| -rw-r--r-- | src/write/build-modes/sort.js | 76 | ||||
| -rw-r--r-- | src/write/build-modes/static-build.js | 186 |
6 files changed, 380 insertions, 149 deletions
diff --git a/src/write/bind-utilities.js b/src/write/bind-utilities.js index 3d4ecc7a..afbf8b2f 100644 --- a/src/write/bind-utilities.js +++ b/src/write/bind-utilities.js @@ -19,14 +19,14 @@ import { export function bindUtilities({ absoluteTo, - cachebust, defaultLanguage, - getSizeOfAdditionalFile, - getSizeOfImagePath, + getSizeOfMediaFile, language, languages, missingImagePaths, + niceShowAggregate, pagePath, + pagePathStringFromRoot, thumbsCache, to, urls, @@ -36,16 +36,16 @@ export function bindUtilities({ Object.assign(bound, { absoluteTo, - cachebust, defaultLanguage, - getSizeOfAdditionalFile, - getSizeOfImagePath, + getSizeOfMediaFile, getThumbnailsAvailableForDimensions, html, language, languages, missingImagePaths, + niceShowAggregate, pagePath, + pagePathStringFromRoot, thumb, to, urls, diff --git a/src/write/build-modes/index.js b/src/write/build-modes/index.js index 3ae2cfc6..4b61619d 100644 --- a/src/write/build-modes/index.js +++ b/src/write/build-modes/index.js @@ -1,3 +1,4 @@ export * as 'live-dev-server' from './live-dev-server.js'; export * as 'repl' from './repl.js'; +export * as 'sort' from './sort.js'; export * as 'static-build' from './static-build.js'; diff --git a/src/write/build-modes/live-dev-server.js b/src/write/build-modes/live-dev-server.js index 03ef6049..5dece8d0 100644 --- a/src/write/build-modes/live-dev-server.js +++ b/src/write/build-modes/live-dev-server.js @@ -1,11 +1,14 @@ import {spawn} from 'node:child_process'; import * as http from 'node:http'; import {open, stat} from 'node:fs/promises'; +import * as os from 'node:os'; import * as path from 'node:path'; import {pipeline} from 'node:stream/promises'; import {inspect as nodeInspect} from 'node:util'; -import {ENABLE_COLOR, colors, logInfo, logWarn, progressCallAll} from '#cli'; +import {openAggregate} from '#aggregate'; +import {ENABLE_COLOR, colors, fileIssue, logInfo, logWarn, progressCallAll} + from '#cli'; import {watchContentDependencies} from '#content-dependencies'; import {quickEvaluate} from '#content-function'; import * as html from '#html'; @@ -24,7 +27,7 @@ import {generateRandomLinkDataJSON, generateRedirectHTML} from '../common-templa const defaultHost = '0.0.0.0'; const defaultPort = 8002; -export const description = `Hosts a local HTTP server which generates page content as it is requested, instead of all at once; reacts to changes in data files, so new reloads will be up-to-date with on-disk YAML data (<- not implemented yet, check back soon!)\n\nIntended for local development ONLY; this custom HTTP server is NOT rigorously tested and almost certainly has security flaws`; +export const description = `Hosts a local HTTP server which generates page content as it is requested, instead of all at once\n\nIntended for local development ONLY; this custom HTTP server is NOT rigorously tested and almost certainly has security flaws`; export const config = { fileSizes: { @@ -91,22 +94,49 @@ export function getCLIOptions() { }; } +const getContentType = extname => ({ + // BRB covering all my bases + 'aac': 'audio/aac', + 'bmp': 'image/bmp', + 'css': 'text/css', + 'csv': 'text/csv', + 'gif': 'image/gif', + 'ico': 'image/vnd.microsoft.icon', + 'jpg': 'image/jpeg', + 'jpeg': 'image/jpeg', + 'js': 'text/javascript', + 'mjs': 'text/javascript', + 'mp3': 'audio/mpeg', + 'mp4': 'video/mp4', + 'oga': 'audio/ogg', + 'ogg': 'audio/ogg', + 'ogv': 'video/ogg', + 'opus': 'audio/opus', + 'png': 'image/png', + 'pdf': 'application/pdf', + 'svg': 'image/svg+xml', + 'ttf': 'font/ttf', + 'txt': 'text/plain', + 'wav': 'audio/wav', + 'weba': 'audio/webm', + 'webm': 'video/webm', + 'woff': 'font/woff', + 'woff2': 'font/woff2', + 'xml': 'application/xml', + 'zip': 'application/zip', +})[extname]; + export async function go({ cliOptions, + universalUtilities, + defaultLanguage, languages, - missingImagePaths, - srcRootPath, - thumbsCache, urls, webRoutes, wikiData, - cachebust, - developersComment: _developersComment, - getSizeOfAdditionalFile, - getSizeOfImagePath, niceShowAggregate, }) { const showError = (error) => { @@ -136,21 +166,49 @@ export async function go({ contentDependenciesWatcher.on('error', () => {}); await new Promise(resolve => contentDependenciesWatcher.once('ready', resolve)); + const commonUtilities = {...universalUtilities}; + + const pathAggregate = openAggregate({message: `Errors computing page paths`}); + let targetSpecPairs = getPageSpecsWithTargets({wikiData}); - const pages = progressCallAll(`Computing page data & paths for ${targetSpecPairs.length} targets.`, + const pages = progressCallAll(`Computing page paths for ${targetSpecPairs.length} targets.`, targetSpecPairs.flatMap(({ pageSpec, target, targetless, }) => () => { - if (targetless) { - const result = pageSpec.pathsTargetless({wikiData}); - return Array.isArray(result) ? result : [result]; - } else { - return pageSpec.pathsForTarget(target); + try { + if (targetless) { + const result = pageSpec.pathsTargetless({wikiData}); + return Array.isArray(result) ? result : [result]; + } else { + return pageSpec.pathsForTarget(target); + } + } catch (caughtError) { + if (targetless) { + pathAggregate.push(new Error( + `Failed to compute targetless paths for ` + + inspect(pageSpec, {compact: true}), + {cause: caughtError})); + } else { + pathAggregate.push(new Error( + `Failed to compute paths for ` + + inspect(target), + {cause: caughtError})); + } + return []; } })).flat(); + try { + pathAggregate.close(); + } catch (error) { + niceShowAggregate(error); + logWarn`Failed to compute page paths for some targets.`; + logWarn`This means some pages that normally exist will be 404s.`; + fileIssue(); + } + logInfo`Will be serving a total of ${pages.length} pages.`; const urlToPageMap = Object.fromEntries(pages @@ -195,7 +253,7 @@ export async function go({ let url; try { url = new URL(request.url, `http://${request.headers.host}`); - } catch (error) { + } catch { response.writeHead(500, contentTypePlain); response.end('Failed to parse request URL\n'); return; @@ -242,7 +300,7 @@ export async function go({ let filePath; try { filePath = path.resolve(localDirectory, decodeURI(safePath.split('/').join(path.sep))); - } catch (error) { + } catch { response.writeHead(404, contentTypePlain); response.end(`File not found for: ${safePath}`); console.log(`${requestHead} [404] ${pathname}`); @@ -267,38 +325,7 @@ export async function go({ } const extname = path.extname(safePath).slice(1).toLowerCase(); - - const contentType = { - // BRB covering all my bases - 'aac': 'audio/aac', - 'bmp': 'image/bmp', - 'css': 'text/css', - 'csv': 'text/csv', - 'gif': 'image/gif', - 'ico': 'image/vnd.microsoft.icon', - 'jpg': 'image/jpeg', - 'jpeg': 'image/jpeg', - 'js': 'text/javascript', - 'mjs': 'text/javascript', - 'mp3': 'audio/mpeg', - 'mp4': 'video/mp4', - 'oga': 'audio/ogg', - 'ogg': 'audio/ogg', - 'ogv': 'video/ogg', - 'opus': 'audio/opus', - 'png': 'image/png', - 'pdf': 'application/pdf', - 'svg': 'image/svg+xml', - 'ttf': 'font/ttf', - 'txt': 'text/plain', - 'wav': 'audio/wav', - 'weba': 'audio/webm', - 'webm': 'video/webm', - 'woff': 'font/woff', - 'woff2': 'font/woff2', - 'xml': 'application/xml', - 'zip': 'application/zip', - }[extname]; + const contentType = getContentType(extname); let fd, size; try { @@ -323,7 +350,16 @@ export async function go({ 'Content-Length': size, }); - await pipeline(fd.createReadStream(), response); + try { + await pipeline(fd.createReadStream(), response); + } catch (error) { + if (error.code === 'ERR_STREAM_PREMATURE_CLOSE') { + // Connection was dropped, this is OK. + return; + } else { + throw error; + } + } if (loudResponses) console.log(`${requestHead} [200] ${pathname} (${colors.magenta(`web route`)})`); @@ -332,13 +368,36 @@ export async function go({ // Other routes determined by page and URL specs + const startTiming = () => { + if (!showTimings) { + return () => ''; + } + + const timeStart = Date.now(); + + return () => { + const timeEnd = Date.now(); + const timeDelta = timeEnd - timeStart; + + if (timeDelta > 100) { + return `${(timeDelta / 1000).toFixed(2)}s`; + } else { + return `${timeDelta}ms`; + } + }; + }; + // URL to page map expects trailing slash but no leading slash. const pathnameKey = pathname.replace(/^\//, '') + (pathname.endsWith('/') ? '' : '/'); - if (!Object.hasOwn(urlToPageMap, pathnameKey)) { + const is404 = + !Object.hasOwn(urlToPageMap, pathnameKey) || + !(urlToPageMap[pathnameKey].page.condition?.() ?? true); + + if (is404) { response.writeHead(404, contentTypePlain); response.end(`No page found for: ${pathnameKey}\n`); - if (loudResponses) console.log(`${requestHead} [404] ${pathname}`); + if (loudResponses) console.log(`${requestHead} [404] ${pathname} (no page)`); return; } @@ -395,22 +454,16 @@ export async function go({ return; } - const timeStart = Date.now(); + const timing = startTiming(); const bound = bindUtilities({ + ...commonUtilities, + absoluteTo, - cachebust, - defaultLanguage, - getSizeOfAdditionalFile, - getSizeOfImagePath, language, - languages, - missingImagePaths, pagePath: servePath, - thumbsCache, - to, - urls, - wikiData, + pagePathStringFromRoot: pathname.replace(/^\//, ''), + to: page.absoluteLinks ? absoluteTo : to, }); const topLevelResult = @@ -424,18 +477,10 @@ export async function go({ const {pageHTML} = html.resolve(topLevelResult); - const timeEnd = Date.now(); - const timeDelta = timeEnd - timeStart; - - if (showTimings) { - const timeString = - (timeDelta > 100 - ? `${(timeDelta / 1000).toFixed(2)}s` - : `${timeDelta}ms`); - - console.log(`${requestHead} [200, ${timeString}] ${pathname} (${colors.blue(`page`)})`); - } else if (loudResponses) { - console.log(`${requestHead} [200] ${pathname} (${colors.blue(`page`)})`); + const timeString = timing(); + const status = (timeString ? `200 ${timeString}` : `200`); + if (showTimings || loudResponses) { + console.log(`${requestHead} [${status}] ${pathname} (${colors.blue(`page`)})`); } response.writeHead(200, contentTypeHTML); @@ -448,7 +493,13 @@ export async function go({ } }); - const address = `http://${host}:${port}/`; + const addresses = + (host === '0.0.0.0' + ? [`http://localhost:${port}/`, + `http://${os.hostname()}:${port}/`] + : host === '127.0.0.1' + ? [`http://localhost:${port}/`] + : [`http://${host}:${port}/`]); server.on('error', error => { if (error.code === 'EADDRINUSE') { @@ -465,7 +516,15 @@ export async function go({ }); server.on('listening', () => { - logInfo`${'All done!'} Listening at: ${address}`; + if (addresses.length === 1) { + logInfo`${'All done!'} Listening at: ${addresses[0]}`; + } else { + logInfo`${`All done!`} Listening at:`; + for (const address of addresses) { + logInfo`- ${address}`; + } + } + logInfo`Press ^C here (control+C) to stop the server and exit.`; if (showTimings && loudResponses) { logInfo`Printing all HTTP responses, plus page generation timings.`; diff --git a/src/write/build-modes/repl.js b/src/write/build-modes/repl.js index b300e8e8..920ad9f7 100644 --- a/src/write/build-modes/repl.js +++ b/src/write/build-modes/repl.js @@ -13,6 +13,10 @@ export const config = { default: 'skip', }, + search: { + default: 'skip', + }, + thumbs: { applicable: false, }, @@ -32,6 +36,7 @@ import * as path from 'node:path'; import * as repl from 'node:repl'; import _find, {bindFind} from '#find'; +import _reverse, {bindReverse} from '#reverse'; import CacheableObject from '#cacheable-object'; import {logWarn} from '#cli'; import {debugComposite} from '#composite'; @@ -46,16 +51,12 @@ export async function getContextAssignments({ mediaPath, mediaCachePath, + universalUtilities, + defaultLanguage, - languages, - missingImagePaths, - thumbsCache, - urls, wikiData, - getSizeOfAdditionalFile, - getSizeOfImagePath, - niceShowAggregate, + niceShowAggregate: showAggregate, }) { let find; try { @@ -66,19 +67,25 @@ export async function getContextAssignments({ logWarn`\`find\` variable will be missing`; } + let reverse; + try { + reverse = bindReverse(wikiData); + } catch (error) { + console.error(error); + logWarn`Failed to prepare wikiData-bound reverse() functions`; + logWarn`\`reverse\` variable will be missing`; + } + const replContext = { + universalUtilities, + ...universalUtilities, + dataPath, mediaPath, mediaCachePath, - languages, - defaultLanguage, language: defaultLanguage, - missingImagePaths, - thumbsCache, - urls, - wikiData, ...wikiData, WD: wikiData, @@ -98,9 +105,11 @@ export async function getContextAssignments({ find, bindFind, - getSizeOfAdditionalFile, - getSizeOfImagePath, - showAggregate: niceShowAggregate, + _reverse, + reverse, + bindReverse, + + showAggregate, }; replContext.replContext = replContext; diff --git a/src/write/build-modes/sort.js b/src/write/build-modes/sort.js new file mode 100644 index 00000000..1a738ac8 --- /dev/null +++ b/src/write/build-modes/sort.js @@ -0,0 +1,76 @@ +export const description = `Update data files in-place to satisfy custom sorting rules`; + +import {logInfo} from '#cli'; +import {empty} from '#sugar'; +import thingConstructors from '#things'; + +export const config = { + fileSizes: { + applicable: false, + }, + + languageReloading: { + applicable: false, + }, + + mediaValidation: { + applicable: false, + }, + + search: { + applicable: false, + }, + + thumbs: { + applicable: false, + }, + + webRoutes: { + applicable: false, + }, + + sort: { + applicable: false, + }, +}; + +export function getCLIOptions() { + return {}; +} + +export async function go({wikiData, dataPath}) { + if (empty(wikiData.sortingRules)) { + logInfo`There aren't any sorting rules in for this wiki.`; + return true; + } + + const {SortingRule} = thingConstructors; + + let numUpdated = 0; + let numActive = 0; + + for await (const result of SortingRule.go({wikiData, dataPath})) { + numActive++; + + const niceMessage = `"${result.rule.message}"`; + + if (result.changed) { + numUpdated++; + logInfo`Updating to satisfy ${niceMessage}.`; + } else { + logInfo`Already good: ${niceMessage}`; + } + } + + if (numUpdated > 1) { + logInfo`Updated data files to satisfy ${numUpdated} sorting rules.`; + } else if (numUpdated === 1) { + logInfo`Updated data files to satisfy ${1} sorting rule.` + } else if (numActive >= 1) { + logInfo`All sorting rules were already satisfied. Good to go!`; + } else { + logInfo`No sorting rules are currently active.`; + } + + return true; +} diff --git a/src/write/build-modes/static-build.js b/src/write/build-modes/static-build.js index 68cf0949..b5ded04c 100644 --- a/src/write/build-modes/static-build.js +++ b/src/write/build-modes/static-build.js @@ -1,13 +1,7 @@ +import {cp, mkdir, stat, symlink, writeFile, unlink} from 'node:fs/promises'; import * as path from 'node:path'; -import { - copyFile, - mkdir, - stat, - symlink, - writeFile, - unlink, -} from 'node:fs/promises'; +import {rimraf} from 'rimraf'; import {quickLoadContentDependencies} from '#content-dependencies'; import {quickEvaluate} from '#content-function'; @@ -24,6 +18,7 @@ import { } from '#cli'; import { + getOrigin, getPagePathname, getURLsFrom, getURLsFromRoot, @@ -49,6 +44,10 @@ export const config = { default: 'perform', }, + search: { + default: 'perform', + }, + thumbs: { default: 'perform', }, @@ -103,22 +102,16 @@ export function getCLIOptions() { export async function go({ cliOptions, - mediaPath, queueSize, + universalUtilities, + defaultLanguage, languages, - missingImagePaths, - srcRootPath, - thumbsCache, urls, webRoutes, wikiData, - cachebust, - developersComment: _developersComment, - getSizeOfAdditionalFile, - getSizeOfImagePath, niceShowAggregate, }) { const outputPath = cliOptions['out-path'] || process.env.HSMUSIC_OUT; @@ -156,12 +149,12 @@ export async function go({ webRoutes, }); - if (writeAll) { - await writeFavicon({ - mediaPath, - outputPath, - }); + await writeWebRouteCopies({ + outputPath, + webRoutes, + }); + if (writeAll) { await writeSharedFilesAndPages({ outputPath, randomLinkDataJSON: generateRandomLinkDataJSON({wikiData}), @@ -186,7 +179,7 @@ export async function go({ return null; } - const paths = []; + let paths = []; if (pageSpec.pathsTargetless) { const result = pageSpec.pathsTargetless({wikiData}); @@ -216,6 +209,9 @@ export async function go({ // TODO: Validate each pathsForTargets entry } + paths = + paths.filter(path => path.condition?.() ?? true); + return paths; }) .filter(Boolean) @@ -277,6 +273,8 @@ export async function go({ showAggregate: niceShowAggregate, }); + const commonUtilities = {...universalUtilities}; + const perLanguageFn = async (language, i, entries) => { const baseDirectory = language === defaultLanguage ? '' : language.code; @@ -305,19 +303,13 @@ export async function go({ }); const bound = bindUtilities({ + ...commonUtilities, + absoluteTo, - cachebust, - defaultLanguage, - getSizeOfAdditionalFile, - getSizeOfImagePath, language, - languages, - missingImagePaths, pagePath, - thumbsCache, - to, - urls, - wikiData, + pagePathStringFromRoot: pathname, + to: page.absoluteLinks ? absoluteTo : to, }); let pageHTML, oEmbedJSON; @@ -432,12 +424,21 @@ async function writePage({ ].filter(Boolean)); } +function filterNoOrigin(route) { + return !getOrigin(route.to); +} + function writeWebRouteSymlinks({ outputPath, webRoutes, }) { + const symlinkRoutes = + webRoutes + .filter(route => route.statically === 'symlink') + .filter(filterNoOrigin); + const promises = - webRoutes.map(async route => { + symlinkRoutes.map(async route => { const parts = route.to.split('/'); const parentDirectoryParts = parts.slice(0, -1); const symlinkNamePart = parts.at(-1); @@ -469,28 +470,113 @@ function writeWebRouteSymlinks({ return progressPromiseAll(`Writing web route symlinks.`, promises); } -async function writeFavicon({ - mediaPath, +async function writeWebRouteCopies({ outputPath, + webRoutes, }) { - const faviconFile = 'favicon.ico'; + const copyRoutes = + webRoutes + .filter(route => route.statically === 'copy') + .filter(filterNoOrigin); - try { - await stat(path.join(mediaPath, faviconFile)); - } catch (error) { - return; - } + const promises = + copyRoutes.map(async route => { + const permissionName = '__hsmusic-ok-for-deletion.txt'; - try { - await copyFile( - path.join(mediaPath, faviconFile), - path.join(outputPath, faviconFile)); - } catch (error) { - logWarn`Failed to copy favicon! ${error.message}`; - return; - } + const parts = route.to.split('/'); + const parentDirectoryParts = parts.slice(0, -1); + const copyNamePart = parts.at(-1); + + const parentDirectory = path.join(outputPath, ...parentDirectoryParts); + const copyPath = path.join(parentDirectory, copyNamePart); + + // We're going to do a rimraf call! This is freaking terrifying, + // so nope out on a couple important conditions. - logInfo`Copied favicon to site root.`; + let needsDelete; + try { + await stat(copyPath); + needsDelete = true; + } catch (error) { + if (error.code === 'ENOENT') { + needsDelete = false; + } else { + throw error; + } + } + + if (needsDelete) { + // First remove it directly, in case it's a symlink. + try { + await unlink(copyPath); + needsDelete = false; + } catch (error) { + // EPERM is POSIX, but libuv may or may not flat-out just raise + // the system error (which is ostensibly EISDIR on Linux). + // https://github.com/nodejs/node-v0.x-archive/issues/5791 + // https://man7.org/linux/man-pages/man2/unlink.2.html + // + // Both of these indidcate "a directory, probably" and we'll + // still check for the deletion permission file where we expect + // it before actually touching anything. + if (error.code !== 'EPERM' && error.code !== 'EISDIR') { + throw error; + } + } + } + + if (needsDelete) { + // Then check that the deletion permission file exists + // where we expect it. + try { + await stat(path.join(copyPath, permissionName)); + } catch (error) { + if (error.code === 'ENOENT') { + throw new Error(`Couldn't find ${permissionName} in ${copyPath} - please delete or move away this folder manually`); + } else { + throw error; + } + } + + // And *then* actually delete that directory. + await rimraf(copyPath); + } + + // Actually copy the source path where it's wanted. + await cp(route.from, copyPath, {recursive: true}); + + // And certify that it's OK to delete this path, next time around. + await writeFile(path.join(copyPath, permissionName), + `The presence of this file (by its name, not its contents)\n` + + `indicates hsmusic may delete everything contained in this\n` + + `directory (the one which directly contains this file, *not*\n` + + `any further-up parent directories).\n` + + `\n` + + `If you make edits, or add any files, they will be deleted or\n` + + `overwritten the next time you run the build.\n` + + `\n` + + `If you delete *this* file, hsmusic will error during the next\n` + + `build, and will ask that you delete the containing directory\n` + + `yourself.\n`); + }); + + const results = + await Promise.allSettled(promises); + + const errors = + results + .filter(({status}) => status === 'rejected') + .map(({reason}) => reason) + .map(err => + (err.message.startsWith(`Couldn't find`) + ? err.message + : err)); + + if (empty(errors)) { + logInfo`Wrote web route copies.`; + } else { + throw new AggregateError(errors, `Errors copying internal files ("web routes")`); + } } async function writeSharedFilesAndPages({ |