From db816222a07a4a8dff9f0f8b66e7aa7cb7c15eb5 Mon Sep 17 00:00:00 2001 From: "(quasar) nebula" Date: Sun, 8 Jan 2023 10:52:12 -0400 Subject: extract static-build, new modular build modes --- src/write/build-modes/live-dev-server.js | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 src/write/build-modes/live-dev-server.js (limited to 'src/write/build-modes/live-dev-server.js') diff --git a/src/write/build-modes/live-dev-server.js b/src/write/build-modes/live-dev-server.js new file mode 100644 index 00000000..76b6d47f --- /dev/null +++ b/src/write/build-modes/live-dev-server.js @@ -0,0 +1,31 @@ +import {logInfo} from '../../util/cli.js'; + +export function getCLIOptions() { + // Stub. + return {}; +} + +export async function go({ + _cliOptions, + _dataPath, + _mediaPath, + _queueSize, + + _defaultLanguage, + _languages, + _srcRootPath, + _urls, + _urlSpec, + _wikiData, + + _cachebust, + _developersComment, + _getSizeOfAdditionalFile, +}) { + // Stub. + logInfo`So we are back in the mine!`; + logInfo`We are swinging our pickaxe in-`; + logInfo`...multiple directions,`; + logInfo`...multiple directions.`; + return true; +} -- cgit 1.3.0-6-gf8a5 From 154c050db72ebf06d4514326c696ab43fb3f8dc8 Mon Sep 17 00:00:00 2001 From: "(quasar) nebula" Date: Sun, 8 Jan 2023 11:49:44 -0400 Subject: basic actual stub for live-dev-server --- src/write/build-modes/live-dev-server.js | 43 ++++++++++++++++++++++++++------ 1 file changed, 36 insertions(+), 7 deletions(-) (limited to 'src/write/build-modes/live-dev-server.js') diff --git a/src/write/build-modes/live-dev-server.js b/src/write/build-modes/live-dev-server.js index 76b6d47f..c3094712 100644 --- a/src/write/build-modes/live-dev-server.js +++ b/src/write/build-modes/live-dev-server.js @@ -1,4 +1,9 @@ -import {logInfo} from '../../util/cli.js'; +import * as pageSpecs from '../../page/index.js'; + +import { + logInfo, + progressCallAll, +} from '../../util/cli.js'; export function getCLIOptions() { // Stub. @@ -16,16 +21,40 @@ export async function go({ _srcRootPath, _urls, _urlSpec, - _wikiData, + wikiData, _cachebust, _developersComment, _getSizeOfAdditionalFile, }) { - // Stub. - logInfo`So we are back in the mine!`; - logInfo`We are swinging our pickaxe in-`; - logInfo`...multiple directions,`; - logInfo`...multiple directions.`; + let targetSpecPairs = getPageSpecsWithTargets({wikiData}); + const writes = progressCallAll(`Computing page data & paths for ${targetSpecPairs.length} targets.`, + targetSpecPairs.map(({ + pageSpec, + target, + targetless, + }) => () => + targetless + ? pageSpec.writeTargetless({wikiData}) + : pageSpec.write(target, {wikiData}))).flat(); + + logInfo`Will be serving a total of ${writes.length} pages.`; + return true; } + +function getPageSpecsWithTargets({ + wikiData, +}) { + return Object.values(pageSpecs) + .filter(pageSpec => pageSpec.condition?.({wikiData}) ?? true) + .flatMap(pageSpec => [ + ...pageSpec.targets + ? pageSpec.targets({wikiData}) + .map(target => ({pageSpec, target})) + : [], + Object.hasOwn(pageSpec, 'writeTargetless') && + {pageSpec, targetless: true}, + ]) + .filter(Boolean); +} -- cgit 1.3.0-6-gf8a5 From 594e8dd46f9e6cc74c680536a1d820eef27133f0 Mon Sep 17 00:00:00 2001 From: "(quasar) nebula" Date: Mon, 9 Jan 2023 21:07:16 -0400 Subject: most essential behavior for live-dev-server --- src/write/build-modes/live-dev-server.js | 287 +++++++++++++++++++++++++++++-- 1 file changed, 273 insertions(+), 14 deletions(-) (limited to 'src/write/build-modes/live-dev-server.js') diff --git a/src/write/build-modes/live-dev-server.js b/src/write/build-modes/live-dev-server.js index c3094712..80badeb9 100644 --- a/src/write/build-modes/live-dev-server.js +++ b/src/write/build-modes/live-dev-server.js @@ -1,34 +1,55 @@ +import * as http from 'http'; +import {createReadStream} from 'fs'; +import {stat} from 'fs/promises'; +import * as path from 'path'; +import {pipeline} from 'stream/promises' + +import {bindUtilities} from '../bind-utilities.js'; + +import {serializeThings} from '../../data/serialize.js'; + import * as pageSpecs from '../../page/index.js'; +import {logInfo, progressCallAll} from '../../util/cli.js'; +import {withEntries} from '../../util/sugar.js'; + +import { + getPagePathname, + getPageSubdirectoryPrefix, + getURLsFrom, +} from '../../util/urls.js'; + import { - logInfo, - progressCallAll, -} from '../../util/cli.js'; + generateDocumentHTML, + generateGlobalWikiDataJSON, + generateRedirectHTML, +} from '../page-template.js'; export function getCLIOptions() { - // Stub. return {}; } export async function go({ _cliOptions, _dataPath, - _mediaPath, + mediaPath, _queueSize, - _defaultLanguage, - _languages, - _srcRootPath, - _urls, + defaultLanguage, + languages, + srcRootPath, + urls, _urlSpec, wikiData, - _cachebust, - _developersComment, - _getSizeOfAdditionalFile, + cachebust, + developersComment, + getSizeOfAdditionalFile, }) { + const port = 8002; + let targetSpecPairs = getPageSpecsWithTargets({wikiData}); - const writes = progressCallAll(`Computing page data & paths for ${targetSpecPairs.length} targets.`, + const pages = progressCallAll(`Computing page data & paths for ${targetSpecPairs.length} targets.`, targetSpecPairs.map(({ pageSpec, target, @@ -38,7 +59,245 @@ export async function go({ ? pageSpec.writeTargetless({wikiData}) : pageSpec.write(target, {wikiData}))).flat(); - logInfo`Will be serving a total of ${writes.length} pages.`; + logInfo`Will be serving a total of ${pages.length} pages.`; + + const urlToPageMap = Object.fromEntries(pages + .filter(page => page.type === 'page' || page.type === 'redirect') + .flatMap(page => { + let servePath; + if (page.type === 'page') + servePath = page.path; + else if (page.type === 'redirect') + servePath = page.fromPath; + + const fullKey = 'localized.' + servePath[0]; + const urlArgs = servePath.slice(1); + + return Object.values(languages).map(language => { + const baseDirectory = + language === defaultLanguage ? '' : language.code; + + const pathname = getPagePathname({ + baseDirectory, + fullKey, + urlArgs, + urls, + }); + + return [pathname, { + baseDirectory, + language, + page, + servePath, + }]; + }); + })); + + const server = http.createServer(async (request, response) => { + const contentTypeHTML = {'Content-Type': 'text/html; charset=utf-8'}; + const contentTypeJSON = {'Content-Type': 'text/json; charset=utf-8'}; + const contentTypePlain = {'Content-Type': 'text/plain; charset=utf-8'}; + + const requestTime = new Date().toLocaleDateString('en-US', {hour: '2-digit', minute: '2-digit', second: '2-digit'}); + const requestHead = `${requestTime} - ${request.socket.remoteAddress}`; + + let url; + try { + url = new URL(request.url, `http://${request.headers.host}`); + } catch (error) { + response.writeHead(500, contentTypePlain); + response.end('Failed to parse request URL\n'); + return; + } + + const {pathname} = url; + + // Specialized routes + + if (pathname === '/data.json') { + response.writeHead(200, contentTypeJSON); + response.end(generateGlobalWikiDataJSON({ + serializeThings, + wikiData, + })); + return; + } + + const { + area: localFileArea, + path: localFilePath + } = pathname.match(/^\/(?static|media)\/(?.*)/)?.groups ?? {}; + + if (localFileArea) { + // Not security tested, man, this is a dev server!! + const safePath = path.resolve('/', localFilePath).replace(/^\//, ''); + + let localDirectory; + if (localFileArea === 'static') { + localDirectory = path.join(srcRootPath, 'static'); + } else if (localFileArea === 'media') { + localDirectory = mediaPath; + } + + const filePath = path.resolve(localDirectory, safePath); + + try { + await stat(filePath); + } catch (error) { + if (error.code === 'ENOENT') { + response.writeHead(404, contentTypePlain); + response.end(`No ${localFileArea} file found for: ${safePath}`); + console.log(`${requestHead} [404] ${pathname}`); + console.log(`ENOENT for stat: ${filePath}`); + } else { + response.writeHead(500, contentTypePlain); + response.end(`Internal error accessing ${localFileArea} file for: ${safePath}`); + console.error(`${requestHead} [500] ${pathname}`); + console.error(error); + } + return; + } + + try { + response.writeHead(200); // Sorry, no MIME type for now + await pipeline( + createReadStream(filePath), + response); + console.log(`${requestHead} [200] ${pathname}`); + } catch (error) { + response.writeHead(500, contentTypePlain); + response.end(`Failed during file-to-response pipeline`); + console.error(`${requestHead} [500] ${pathname}`); + console.error(error); + return; + } + } + + // Other routes determined by page and URL specs + + // URL to page map expects trailing slash but no leading slash. + const pathnameKey = pathname.replace(/^\//, '') + (pathname.endsWith('/') ? '' : '/'); + + if (!Object.hasOwn(urlToPageMap, pathnameKey)) { + response.writeHead(404, contentTypePlain); + response.end(`No page found for: ${pathnameKey}\n`); + console.log(`${requestHead} [404] ${pathname}`); + return; + } + + const { + baseDirectory, + language, + page, + servePath, + } = urlToPageMap[pathnameKey]; + + const to = getURLsFrom({ + urls, + baseDirectory, + pageSubKey: servePath[0], + subdirectoryPrefix: getPageSubdirectoryPrefix({ + urlArgs: servePath.slice(1), + }), + }); + + const absoluteTo = (targetFullKey, ...args) => { + const [groupKey, subKey] = targetFullKey.split('.'); + const from = urls.from('shared.root'); + return ( + '/' + + (groupKey === 'localized' && baseDirectory + ? from.to( + 'localizedWithBaseDirectory.' + subKey, + baseDirectory, + ...args + ) + : from.to(targetFullKey, ...args)) + ); + }; + + try { + const pageSubKey = servePath[0]; + const urlArgs = servePath.slice(1); + + if (page.type === 'redirect') { + response.writeHead(301, contentTypeHTML); + + const target = to('localized.' + page.toPath[0], ...page.toPath.slice(1)); + const redirectHTML = generateRedirectHTML(page.title, target, {language}); + + response.end(redirectHTML); + + console.log(`${requestHead} [301] (redirect) ${pathname}`); + return; + } + + response.writeHead(200, contentTypeHTML); + + const localizedPathnames = withEntries(languages, entries => entries + .filter(([key, language]) => key !== 'default' && !language.hidden) + .map(([_key, language]) => [ + language.code, + getPagePathname({ + baseDirectory: + (language === defaultLanguage + ? '' + : language.code), + fullKey: 'localized.' + pageSubKey, + urlArgs, + urls, + }), + ])); + + const bound = bindUtilities({ + language, + to, + wikiData, + }); + + const pageInfo = page.page({ + ...bound, + + absoluteTo, + relativeTo: to, + to, + urls, + + getSizeOfAdditionalFile, + }); + + const pageHTML = generateDocumentHTML(pageInfo, { + cachebust, + defaultLanguage, + developersComment, + getThemeString: bound.getThemeString, + language, + languages, + localizedPathnames, + oEmbedJSONHref: null, // No oEmbed support for live dev server + pageSubKey, + pathname, + urlArgs, + to, + transformMultiline: bound.transformMultiline, + wikiData, + }); + + console.log(`${requestHead} [200] ${pathname}`); + response.end(pageHTML); + } catch (error) { + response.writeHead(500, contentTypePlain); + response.end(`Error generating page, view server log for details\n`); + console.error(`${requestHead} [500] ${pathname}`); + console.error(error); + } + }); + + server.listen(port); + logInfo`${'All done!'} Listening at ${`http://0.0.0.0:${port}/`}`; + + // Just keep going... forever!!! + await new Promise(() => {}); return true; } -- cgit 1.3.0-6-gf8a5 From a532f066ffbe0387216c85323fdf81aff36c1d6c Mon Sep 17 00:00:00 2001 From: "(quasar) nebula" Date: Mon, 9 Jan 2023 21:07:37 -0400 Subject: finish up the rest necessary for live-dev-server --- src/write/build-modes/live-dev-server.js | 113 +++++++++++++++++++++++++------ 1 file changed, 94 insertions(+), 19 deletions(-) (limited to 'src/write/build-modes/live-dev-server.js') diff --git a/src/write/build-modes/live-dev-server.js b/src/write/build-modes/live-dev-server.js index 80badeb9..f322a79b 100644 --- a/src/write/build-modes/live-dev-server.js +++ b/src/write/build-modes/live-dev-server.js @@ -10,7 +10,7 @@ import {serializeThings} from '../../data/serialize.js'; import * as pageSpecs from '../../page/index.js'; -import {logInfo, progressCallAll} from '../../util/cli.js'; +import {logInfo, logWarn, progressCallAll} from '../../util/cli.js'; import {withEntries} from '../../util/sugar.js'; import { @@ -26,27 +26,39 @@ import { } from '../page-template.js'; export function getCLIOptions() { - return {}; + return { + host: { + type: 'value', + }, + + port: { + type: 'value', + validate(size) { + if (parseInt(size) !== parseFloat(size)) return 'an integer'; + if (parseInt(size) < 1024 || parseInt(size) > 49151) return 'a user/registered port (1024-49151)'; + return true; + }, + }, + }; } export async function go({ - _cliOptions, + cliOptions, _dataPath, mediaPath, - _queueSize, defaultLanguage, languages, srcRootPath, urls, - _urlSpec, wikiData, cachebust, developersComment, getSizeOfAdditionalFile, }) { - const port = 8002; + const host = cliOptions['host'] ?? '0.0.0.0'; + const port = parseInt(cliOptions['port'] ?? 8002); let targetSpecPairs = getPageSpecsWithTargets({wikiData}); const pages = progressCallAll(`Computing page data & paths for ${targetSpecPairs.length} targets.`, @@ -95,7 +107,7 @@ export async function go({ const server = http.createServer(async (request, response) => { const contentTypeHTML = {'Content-Type': 'text/html; charset=utf-8'}; - const contentTypeJSON = {'Content-Type': 'text/json; charset=utf-8'}; + const contentTypeJSON = {'Content-Type': 'application/json; charset=utf-8'}; const contentTypePlain = {'Content-Type': 'text/plain; charset=utf-8'}; const requestTime = new Date().toLocaleDateString('en-US', {hour: '2-digit', minute: '2-digit', second: '2-digit'}); @@ -115,26 +127,35 @@ export async function go({ // Specialized routes if (pathname === '/data.json') { - response.writeHead(200, contentTypeJSON); - response.end(generateGlobalWikiDataJSON({ - serializeThings, - wikiData, - })); + try { + const json = generateGlobalWikiDataJSON({ + serializeThings, + wikiData, + }); + response.writeHead(200, contentTypeJSON); + response.end(json); + console.log(`${requestHead} [200] /data.json`); + } catch (error) { + response.writeHead(500, contentTypeJSON); + response.end({error: `Internal error serializing wiki JSON`}); + console.error(`${requestHead} [500] /data.json`); + console.error(error); + } return; } const { area: localFileArea, path: localFilePath - } = pathname.match(/^\/(?static|media)\/(?.*)/)?.groups ?? {}; + } = pathname.match(/^\/(?static|util|media)\/(?.*)/)?.groups ?? {}; if (localFileArea) { // Not security tested, man, this is a dev server!! const safePath = path.resolve('/', localFilePath).replace(/^\//, ''); let localDirectory; - if (localFileArea === 'static') { - localDirectory = path.join(srcRootPath, 'static'); + if (localFileArea === 'static' || localFileArea === 'util') { + localDirectory = path.join(srcRootPath, localFileArea); } else if (localFileArea === 'media') { localDirectory = mediaPath; } @@ -158,8 +179,42 @@ export async function go({ return; } + const extname = path.extname(safePath).slice(1).toLowerCase(); + + const contentType = { + // BRB covering all my bases + 'aac': 'audio/aac', + 'bmp': 'image/bmp', + 'css': 'text/css', + 'csv': 'text/csv', + 'gif': 'image/gif', + 'ico': 'image/vnd.microsoft.icon', + 'jpg': 'image/jpeg', + 'jpeg:': 'image/jpeg', + 'js': 'text/javascript', + 'mjs': 'text/javascript', + 'mp3': 'audio/mpeg', + 'mp4': 'video/mp4', + 'oga': 'audio/ogg', + 'ogg': 'audio/ogg', + 'ogv': 'video/ogg', + 'opus': 'audio/opus', + 'png': 'image/png', + 'pdf': 'application/pdf', + 'svg': 'image/svg+xml', + 'ttf': 'font/ttf', + 'txt': 'text/plain', + 'wav': 'audio/wav', + 'weba': 'audio/webm', + 'webm': 'video/webm', + 'woff': 'font/woff', + 'woff2': 'font/woff2', + 'xml': 'application/xml', + 'zip': 'application/zip', + }[extname]; + try { - response.writeHead(200); // Sorry, no MIME type for now + response.writeHead(200, contentType ? {'Content-Type': contentType} : {}); await pipeline( createReadStream(filePath), response); @@ -169,8 +224,8 @@ export async function go({ response.end(`Failed during file-to-response pipeline`); console.error(`${requestHead} [500] ${pathname}`); console.error(error); - return; } + return; } // Other routes determined by page and URL specs @@ -293,8 +348,28 @@ export async function go({ } }); - server.listen(port); - logInfo`${'All done!'} Listening at ${`http://0.0.0.0:${port}/`}`; + const address = `http://${host}:${port}/`; + + server.on('error', error => { + if (error.code === 'EADDRINUSE') { + logWarn`Port ${port} is already in use - will (continually) retry after 10 seconds.`; + logWarn`Press ^C here (control+C) to exit and change ${'--port'} number, or stop the server currently running on port ${port}.`; + setTimeout(() => { + server.close(); + server.listen(port, host); + }, 10_000); + } else { + console.error(`Server error detected (code: ${error.code})`); + console.error(error); + } + }); + + server.on('listening', () => { + logInfo`${'All done!'} Listening at: ${address}`; + logInfo`Press ^C here (control+C) to stop the server and exit.`; + }); + + server.listen(port, host); // Just keep going... forever!!! await new Promise(() => {}); -- cgit 1.3.0-6-gf8a5 From 0480e9a6502695fb741fc300db03ef734f38b51c Mon Sep 17 00:00:00 2001 From: "(quasar) nebula" Date: Mon, 9 Jan 2023 21:42:25 -0400 Subject: attempt to fix path issues on windows --- src/write/build-modes/live-dev-server.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src/write/build-modes/live-dev-server.js') diff --git a/src/write/build-modes/live-dev-server.js b/src/write/build-modes/live-dev-server.js index f322a79b..a8e54247 100644 --- a/src/write/build-modes/live-dev-server.js +++ b/src/write/build-modes/live-dev-server.js @@ -151,7 +151,7 @@ export async function go({ if (localFileArea) { // Not security tested, man, this is a dev server!! - const safePath = path.resolve('/', localFilePath).replace(/^\//, ''); + const safePath = path.posix.resolve('/', localFilePath).replace(/^\//, ''); let localDirectory; if (localFileArea === 'static' || localFileArea === 'util') { @@ -160,7 +160,7 @@ export async function go({ localDirectory = mediaPath; } - const filePath = path.resolve(localDirectory, safePath); + const filePath = path.resolve(localDirectory, safePath.split('/').join(path.sep)); try { await stat(filePath); -- cgit 1.3.0-6-gf8a5 From d442546057f8280a141d4aa54f633a09c429e2d3 Mon Sep 17 00:00:00 2001 From: "(quasar) nebula" Date: Tue, 10 Jan 2023 16:55:04 -0400 Subject: extract absoluteTo --- src/write/build-modes/live-dev-server.js | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-) (limited to 'src/write/build-modes/live-dev-server.js') diff --git a/src/write/build-modes/live-dev-server.js b/src/write/build-modes/live-dev-server.js index a8e54247..d481b480 100644 --- a/src/write/build-modes/live-dev-server.js +++ b/src/write/build-modes/live-dev-server.js @@ -17,6 +17,7 @@ import { getPagePathname, getPageSubdirectoryPrefix, getURLsFrom, + getURLsFromRoot, } from '../../util/urls.js'; import { @@ -256,20 +257,10 @@ export async function go({ }), }); - const absoluteTo = (targetFullKey, ...args) => { - const [groupKey, subKey] = targetFullKey.split('.'); - const from = urls.from('shared.root'); - return ( - '/' + - (groupKey === 'localized' && baseDirectory - ? from.to( - 'localizedWithBaseDirectory.' + subKey, - baseDirectory, - ...args - ) - : from.to(targetFullKey, ...args)) - ); - }; + const absoluteTo = getURLsFromRoot({ + baseDirectory, + urls, + }); try { const pageSubKey = servePath[0]; @@ -314,7 +305,6 @@ export async function go({ ...bound, absoluteTo, - relativeTo: to, to, urls, -- cgit 1.3.0-6-gf8a5 From 4de56d21db5004e5b2c2a49caedee0908b9c6a9f Mon Sep 17 00:00:00 2001 From: "(quasar) nebula" Date: Tue, 10 Jan 2023 17:01:54 -0400 Subject: bind more utilities in bindUtilities --- src/write/build-modes/live-dev-server.js | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) (limited to 'src/write/build-modes/live-dev-server.js') diff --git a/src/write/build-modes/live-dev-server.js b/src/write/build-modes/live-dev-server.js index d481b480..b6bf662b 100644 --- a/src/write/build-modes/live-dev-server.js +++ b/src/write/build-modes/live-dev-server.js @@ -296,20 +296,15 @@ export async function go({ ])); const bound = bindUtilities({ + absoluteTo, + getSizeOfAdditionalFile, language, to, + urls, wikiData, }); - const pageInfo = page.page({ - ...bound, - - absoluteTo, - to, - urls, - - getSizeOfAdditionalFile, - }); + const pageInfo = page.page(bound); const pageHTML = generateDocumentHTML(pageInfo, { cachebust, -- cgit 1.3.0-6-gf8a5