From 2260541dc69c19e7444348ac3243f96e4321b781 Mon Sep 17 00:00:00 2001 From: "(quasar) nebula" Date: Wed, 5 May 2021 10:10:50 -0300 Subject: move upd8 code files into their own directory --- README.md | 4 +- gen-thumbs.js | 323 --- package.json | 2 +- strings-default.json | 305 --- upd8-util.js | 423 --- upd8.js | 6597 --------------------------------------------- upd8/gen-thumbs.js | 323 +++ upd8/main.js | 6597 +++++++++++++++++++++++++++++++++++++++++++++ upd8/strings-default.json | 305 +++ upd8/util.js | 423 +++ 10 files changed, 7651 insertions(+), 7651 deletions(-) delete mode 100644 gen-thumbs.js delete mode 100644 strings-default.json delete mode 100644 upd8-util.js delete mode 100755 upd8.js create mode 100644 upd8/gen-thumbs.js create mode 100755 upd8/main.js create mode 100644 upd8/strings-default.json create mode 100644 upd8/util.js diff --git a/README.md b/README.md index 9bc0d286..f0e2493b 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ HSMusic, short for the *Homestuck Music Wiki*, is a revitalization and reimagini **Disclaimer:** most of the code here *sucks*. It's been shambled together over the course of over a year, and while we're fairly confident it's all at minimum functional, we can't guarantee the same about its understandability! Still, for the official release of [hsmusic.wiki][hsmusic], we've done our best to put together a codebase which is *somewhat* navigable. The description below summarizes it: -* `upd8.js`: "Build" code for the site. Everything specific to generating the structure and HTML content of the website is conatined in this file. As expected, it's pretty massive. +* `upd8`: "Build" code for the site. Everything specific to generating the structure and HTML content of the website is conatined in this folder. As expected, it's pretty massive, and is currently undergoing some much-belated restructuring. * `static`: Static code and supporting files. Everything here is wholly client-side and referenced by the generated HTML files. * `common`: Code which is depended upon by both client- and server-side code. For the most part, this is constants such as directory paths, though there are a few handy algorithms here too. * In the not quite so far past, we used to have `data` and `media` folders too. Today, for portability and convenience in project structure, those are saved in separate repositories, and you can pass hsmusic paths to them through the `--data` and `--media` options, or the `HSMUSIC_DATA` and `HSMUSIC_MEDIA` environment variables. @@ -14,7 +14,7 @@ HSMusic, short for the *Homestuck Music Wiki*, is a revitalization and reimagini * Media directory: Images and other static files referenced by generated and static content across the site. Many of the files here are cover art, and their names match the automatically generated "kebab case" identifiers for tracks and albums (or a manually overridden one). * Same for the output root: previously it was in a `site` folder; today, use `--out` or `HSMUSIC_OUT`! -The code process for upd8.js was politely introduced by 2019!us back when we were beginning the site, and it's essentially the same structure followed today. In summary: +The upd8 code process was politely introduced by 2019!us back when we were beginning the site, and it's essentially the same structure followed today. In summary: 1. Locate and read data files, processing them into relatively usable JS object-style formats. (The formats themselves are hard-coded and somewhat arbitrary, and are often extended when more or different data is useful.) 2. Validate the data and show any errors that might've been caught during processing. (These aren't exhaustive test cases; they're designed to catch a majority of common errors and typos.) diff --git a/gen-thumbs.js b/gen-thumbs.js deleted file mode 100644 index 3887aaea..00000000 --- a/gen-thumbs.js +++ /dev/null @@ -1,323 +0,0 @@ -#!/usr/bin/env node - -// Ok, so the d8te is 3 March 2021, and the music wiki was initially released -// on 15 November 2019. That is 474 days or 11376 hours. In my opinion, and -// pro8a8ly the opinions of at least one other person, that is WAY TOO LONG -// to go without media thum8nails!!!! So that's what this file is here to do. -// -// This program takes a path to the media folder (via --media or the environ. -// varia8le HSMUSIC_MEDIA), traverses su8directories to locate image files, -// and gener8tes lower-resolution/file-size versions of all that are new or -// have 8een modified since the last run. We use a JSON-format cache of MD5s -// for each file to perform this comparision; we gener8te files (using ffmpeg) -// in "medium" and "small" sizes adjacent to the existing PNG for easy and -// versatile access in site gener8tion code. -// -// So for example, on the very first run, you might have a media folder which -// looks something like this: -// -// media/ -// album-art/ -// one-year-older/ -// cover.jpg -// firefly-cloud.jpg -// october.jpg -// ... -// flash-art/ -// 413.jpg -// ... -// bg.jpg -// ... -// -// After running gen-thumbs.js with the path to that folder passed, you'd end -// up with something like this: -// -// media/ -// album-art/ -// one-year-older/ -// cover.jpg -// cover.medium.jpg -// cover.small.jpg -// firefly-cloud.jpg -// firefly-cloud.medium.jpg -// firefly-cloud.small.jpg -// october.jpg -// october.medium.jpg -// october.small.jpg -// ... -// flash-art/ -// 413.jpg -// 413.medium.jpg -// 413.small.jpg -// ... -// bg.jpg -// bg.medium.jpg -// bg.small.jpg -// thumbs-cache.json -// ... -// -// (Do note that while 8oth JPG and PNG are supported, gener8ted files will -// always 8e in JPG format and file extension. GIFs are skipped since there -// aren't any super gr8 ways to make those more efficient!) -// -// And then in gener8tion code, you'd reference the medium/small or original -// version of each file, as decided is appropriate. Here are some guidelines: -// -// - Small: Grid tiles on the homepage and in galleries. -// - Medium: Cover art on individual al8um and track pages, etc. -// - Original: Only linked to, not embedded. -// -// The traversal code is indiscrimin8te: there are no special cases to, say, -// not gener8te thum8nails for the bg.jpg file (since those would generally go -// unused). This is just to make the code more porta8le and sta8le, long-term, -// since it avoids a lot of otherwise implic8ted maintenance. - -'use strict'; - -const CACHE_FILE = 'thumbnail-cache.json'; -const WARNING_DELAY_TIME = 10000; - -const { spawn } = require('child_process'); -const crypto = require('crypto'); -const fsp = require('fs/promises'); // Whatcha know! Nice. -const fs = require('fs'); // Still gotta include 8oth tho, for createReadStream. -const path = require('path'); - -const { - delay, - logError, - logInfo, - logWarn, - parseOptions, - progressPromiseAll, - promisifyProcess, - queue, -} = require('./upd8-util'); - -function traverse(startDirPath, { - filterFile = () => true, - filterDir = () => true -} = {}) { - const recursive = (names, subDirPath) => Promise - .all(names.map(name => fsp.readdir(path.join(startDirPath, subDirPath, name)).then( - names => filterDir(name) ? recursive(names, path.join(subDirPath, name)) : [], - err => filterFile(name) ? [path.join(subDirPath, name)] : []))) - .then(pathArrays => pathArrays.flatMap(x => x)); - - return fsp.readdir(startDirPath) - .then(names => recursive(names, '')); -} - -function readFileMD5(filePath) { - return new Promise((resolve, reject) => { - const md5 = crypto.createHash('md5'); - const stream = fs.createReadStream(filePath); - stream.on('data', data => md5.update(data)); - stream.on('end', data => resolve(md5.digest('hex'))); - stream.on('error', err => reject(err)); - }); -} - -function generateImageThumbnails(filePath) { - const dirname = path.dirname(filePath); - const extname = path.extname(filePath); - const basename = path.basename(filePath, extname); - const output = name => path.join(dirname, basename + name + '.jpg'); - - const convert = (name, {size, quality}) => spawn('convert', [ - '-strip', - '-resize', `${size}x${size}>`, - '-interlace', 'Plane', - '-quality', `${quality}%`, - filePath, - output(name) - ]); - - return Promise.all([ - promisifyProcess(convert('.medium', {size: 400, quality: 95}), false), - promisifyProcess(convert('.small', {size: 250, quality: 85}), false) - ]); - - return new Promise((resolve, reject) => { - if (Math.random() < 0.2) { - reject(new Error(`Them's the 8r8ks, kiddo!`)); - } else { - resolve(); - } - }); -} - -async function genThumbs(mediaPath, { - queueSize = 0, - quiet = false -} = {}) { - if (!mediaPath) { - throw new Error('Expected mediaPath to be passed'); - } - - const quietInfo = (quiet - ? () => null - : logInfo); - - const filterFile = name => { - // TODO: Why is this not working???????? - // thumbnail-cache.json is 8eing passed through, for some reason. - - const ext = path.extname(name); - if (ext !== '.jpg' && ext !== '.png') return false; - - const rest = path.basename(name, ext); - if (rest.endsWith('.medium') || rest.endsWith('.small')) return false; - - return true; - }; - - const filterDir = name => { - if (name === '.git') return false; - return true; - }; - - let cache, firstRun = false, failedReadingCache = false; - try { - cache = JSON.parse(await fsp.readFile(path.join(mediaPath, CACHE_FILE))); - quietInfo`Cache file successfully read.`; - } catch (error) { - cache = {}; - if (error.code === 'ENOENT') { - firstRun = true; - } else { - failedReadingCache = true; - logWarn`Malformed or unreadable cache file: ${error}`; - logWarn`You may want to cancel and investigate this!`; - logWarn`All-new thumbnails and cache will be generated for this run.`; - await delay(WARNING_DELAY_TIME); - } - } - - try { - await fsp.writeFile(path.join(mediaPath, CACHE_FILE), JSON.stringify(cache)); - quietInfo`Writing to cache file appears to be working.`; - } catch (error) { - logWarn`Test of cache file writing failed: ${error}`; - if (cache) { - logWarn`Cache read succeeded: Any newly written thumbs will be unnecessarily regenerated on the next run.`; - } else if (firstRun) { - logWarn`No cache found: All thumbs will be generated now, and will be unnecessarily regenerated next run.`; - } else { - logWarn`Cache read failed: All thumbs will be regenerated now, and will be unnecessarily regenerated again next run.`; - } - logWarn`You may want to cancel and investigate this!`; - await delay(WARNING_DELAY_TIME); - } - - const imagePaths = await traverse(mediaPath, {filterFile, filterDir}); - - const imageToMD5Entries = await progressPromiseAll(`Generating MD5s of image files`, queue( - imagePaths.map(imagePath => () => readFileMD5(path.join(mediaPath, imagePath)).then( - md5 => [imagePath, md5], - error => [imagePath, {error}] - )), - queueSize - )); - - { - let error = false; - for (const entry of imageToMD5Entries) { - if (entry[1].error) { - logError`Failed to read ${entry[0]}: ${entry[1].error}`; - error = true; - } - } - if (error) { - logError`Failed to read at least one image file!`; - logError`This implies a thumbnail probably won't be generatable.`; - logError`So, exiting early.`; - return false; - } else { - quietInfo`All image files successfully read.`; - } - } - - // Technically we could pro8a8ly mut8te the cache varia8le in-place? - // 8ut that seems kinda iffy. - const updatedCache = Object.assign({}, cache); - - const entriesToGenerate = imageToMD5Entries - .filter(([filePath, md5]) => md5 !== cache[filePath]); - - if (entriesToGenerate.length === 0) { - logInfo`All image thumbnails are already up-to-date - nice!`; - return true; - } - - const failed = []; - const succeeded = []; - const writeMessageFn = () => `Writing image thumbnails. [failed: ${failed.length}]`; - - // This is actually sort of a lie, 8ecause we aren't doing synchronicity. - // (We pass queueSize = 1 to queue().) 8ut we still use progressPromiseAll, - // 'cuz the progress indic8tor is very cool and good. - await progressPromiseAll(writeMessageFn, queue(entriesToGenerate.map(([filePath, md5]) => - () => generateImageThumbnails(path.join(mediaPath, filePath)).then( - () => { - updatedCache[filePath] = md5; - succeeded.push(filePath); - }, - error => { - failed.push([filePath, error]); - } - ) - ))); - - if (failed.length > 0) { - for (const [path, error] of failed) { - logError`Thumbnails failed to generate for ${path} - ${error}`; - } - logWarn`Result is incomplete - the above ${failed.length} thumbnails should be checked for errors.`; - logWarn`${succeeded.length} successfully generated images won't be regenerated next run, though!`; - } else { - logInfo`Generated all (updated) thumbnails successfully!`; - } - - try { - await fsp.writeFile(path.join(mediaPath, CACHE_FILE), JSON.stringify(updatedCache)); - quietInfo`Updated cache file successfully written!`; - } catch (error) { - logWarn`Failed to write updated cache file: ${error}`; - logWarn`Any newly (re)generated thumbnails will be regenerated next run.`; - logWarn`Sorry about that!`; - } - - return true; -}; - -module.exports = genThumbs; - -if (require.main === module) { - (async () => { - const miscOptions = await parseOptions(process.argv.slice(2), { - 'media': { - type: 'value' - }, - - 'queue-size': { - type: 'value', - validate(size) { - if (parseInt(size) !== parseFloat(size)) return 'an integer'; - if (parseInt(size) < 0) return 'a counting number or zero'; - return true; - } - }, - queue: {alias: 'queue-size'} - }); - - const mediaPath = miscOptions.media || process.env.HSMUSIC_MEDIA; - if (!mediaPath) { - logError`Expected --media option or HSMUSIC_MEDIA to be set`; - } - - const queueSize = +(miscOptions['queue-size'] ?? 0); - - await genThumbs(mediaPath, {queueSize}); - })().catch(err => console.error(err)); -} diff --git a/package.json b/package.json index aaaf8f68..80ee2c52 100644 --- a/package.json +++ b/package.json @@ -4,7 +4,7 @@ "description": "static wiki software cataloguing collaborative creation", "main": "upd8.js", "bin": { - "hsmusic": "./upd8.js" + "hsmusic": "./upd8/main.js" }, "dependencies": { "fix-whitespace": "^1.0.4", diff --git a/strings-default.json b/strings-default.json deleted file mode 100644 index 7a948d64..00000000 --- a/strings-default.json +++ /dev/null @@ -1,305 +0,0 @@ -{ - "meta.languageCode": "en", - "count.tracks": "{TRACKS}", - "count.tracks.withUnit.zero": "", - "count.tracks.withUnit.one": "{TRACKS} track", - "count.tracks.withUnit.two": "", - "count.tracks.withUnit.few": "", - "count.tracks.withUnit.many": "", - "count.tracks.withUnit.other": "{TRACKS} tracks", - "count.albums": "{ALBUMS}", - "count.albums.withUnit.zero": "", - "count.albums.withUnit.one": "{ALBUMS} album", - "count.albums.withUnit.two": "", - "count.albums.withUnit.two": "", - "count.albums.withUnit.few": "", - "count.albums.withUnit.many": "", - "count.albums.withUnit.other": "{ALBUMS} albums", - "count.commentaryEntries": "{ENTRIES}", - "count.commentaryEntries.withUnit.zero": "", - "count.commentaryEntries.withUnit.one": "{ENTRIES} entry", - "count.commentaryEntries.withUnit.two": "", - "count.commentaryEntries.withUnit.few": "", - "count.commentaryEntries.withUnit.many": "", - "count.commentaryEntries.withUnit.other": "{ENTRIES} entries", - "count.contributions": "{CONTRIBUTIONS}", - "count.contributions.withUnit.zero": "", - "count.contributions.withUnit.one": "{CONTRIBUTIONS} contribution", - "count.contributions.withUnit.two": "", - "count.contributions.withUnit.few": "", - "count.contributions.withUnit.many": "", - "count.contributions.withUnit.other": "{CONTRIBUTIONS} contributions", - "count.coverArts": "{COVER_ARTS}", - "count.coverArts.withUnit.zero": "", - "count.coverArts.withUnit.one": "{COVER_ARTS} cover art", - "count.coverArts.withUnit.two": "", - "count.coverArts.withUnit.few": "", - "count.coverArts.withUnit.many": "", - "count.coverArts.withUnit.other": "{COVER_ARTS} cover arts", - "count.timesReferenced": "{TIMES_REFERENCED}", - "count.timesReferenced.withUnit.zero": "", - "count.timesReferenced.withUnit.one": "{TIMES_REFERENCED} time referenced", - "count.timesReferenced.withUnit.two": "", - "count.timesReferenced.withUnit.few": "", - "count.timesReferenced.withUnit.many": "", - "count.timesReferenced.withUnit.other": "{TIMES_REFERENCED} times referenced", - "count.words": "{WORDS}", - "count.words.thousand": "{WORDS}k", - "count.words.withUnit.zero": "", - "count.words.withUnit.one": "{WORDS} word", - "count.words.withUnit.two": "", - "count.words.withUnit.few": "", - "count.words.withUnit.many": "", - "count.words.withUnit.other": "{WORDS} words", - "count.timesUsed": "{TIMES_USED}", - "count.timesUsed.withUnit.zero": "", - "count.timesUsed.withUnit.one": "used {TIMES_USED} time", - "count.timesUsed.withUnit.two": "", - "count.timesUsed.withUnit.few": "", - "count.timesUsed.withUnit.many": "", - "count.timesUsed.withUnit.other": "used {TIMES_USED} times", - "count.index.zero": "", - "count.index.one": "{INDEX}st", - "count.index.two": "{INDEX}nd", - "count.index.few": "{INDEX}rd", - "count.index.many": "", - "count.index.other": "{INDEX}th", - "count.duration.hours": "{HOURS}:{MINUTES}:{SECONDS}", - "count.duration.hours.withUnit": "{HOURS}:{MINUTES}:{SECONDS} hours", - "count.duration.minutes": "{MINUTES}:{SECONDS}", - "count.duration.minutes.withUnit": "{MINUTES}:{SECONDS} minutes", - "count.duration.approximate": "~{DURATION}", - "count.duration.missing": "_:__", - "releaseInfo.by": "By {ARTISTS}.", - "releaseInfo.from": "From {ALBUM}.", - "releaseInfo.coverArtBy": "Cover art by {ARTISTS}.", - "releaseInfo.wallpaperArtBy": "Wallpaper art by {ARTISTS}.", - "releaseInfo.bannerArtBy": "Banner art by {ARTISTS}.", - "releaseInfo.released": "Released {DATE}.", - "releaseInfo.artReleased": "Art released {DATE}.", - "releaseInfo.addedToWiki": "Added to wiki {DATE}.", - "releaseInfo.duration": "Duration: {DURATION}.", - "releaseInfo.viewCommentary": "View {LINK}!", - "releaseInfo.viewCommentary.link": "commentary page", - "releaseInfo.listenOn": "Listen on {LINKS}.", - "releaseInfo.listenOn.noLinks": "This track has no URLs at which it can be listened.", - "releaseInfo.visitOn": "Visit on {LINKS}.", - "releaseInfo.playOn": "Play on {LINKS}.", - "releaseInfo.alsoReleasedAs": "Also released as:", - "releaseInfo.alsoReleasedAs.item": "{TRACK} (on {ALBUM})", - "releaseInfo.contributors": "Contributors:", - "releaseInfo.tracksReferenced": "Tracks that {TRACK} references:", - "releaseInfo.tracksThatReference": "Tracks that reference {TRACK}:", - "releaseInfo.flashesThatFeature": "Flashes & games that feature {TRACK}:", - "releaseInfo.flashesThatFeature.item": "{FLASH}", - "releaseInfo.flashesThatFeature.item.asDifferentRelease": "{FLASH} (as {TRACK})", - "releaseInfo.lyrics": "Lyrics:", - "releaseInfo.artistCommentary": "Artist commentary:", - "releaseInfo.artistCommentary.seeOriginalRelease": "See {ORIGINAL}!", - "releaseInfo.artTags": "Tags:", - "releaseInfo.note": "Note:", - "trackList.group": "{GROUP} ({DURATION}):", - "trackList.item.withDuration": "({DURATION}) {TRACK}", - "trackList.item.withDuration.withArtists": "({DURATION}) {TRACK} {BY}", - "trackList.item.withArtists": "{TRACK} {BY}", - "trackList.item.withArtists.by": "by {ARTISTS}", - "trackList.item.rerelease": "{TRACK} (re-release)", - "misc.alt.albumCover": "album cover", - "misc.alt.albumBanner": "album banner", - "misc.alt.trackCover": "track cover", - "misc.alt.artistAvatar": "artist avatar", - "misc.alt.flashArt": "flash art", - "misc.chronology.seeArtistPages": "(See artist pages for chronology info!)", - "misc.chronology.heading.coverArt": "{INDEX} cover art by {ARTIST}", - "misc.chronology.heading.flash": "{INDEX} flash/game by {ARTIST}", - "misc.chronology.heading.track": "{INDEX} track by {ARTIST}", - "misc.external.domain": "External ({DOMAIN})", - "misc.external.bandcamp": "Bandcamp", - "misc.external.bandcamp.domain": "Bandcamp ({DOMAIN})", - "misc.external.deviantart": "DeviantArt", - "misc.external.instagram": "Instagram", - "misc.external.mastodon": "Mastodon", - "misc.external.mastodon.domain": "Mastodon ({DOMAIN})", - "misc.external.patreon": "Patreon", - "misc.external.poetryFoundation": "Poetry Foundation", - "misc.external.soundcloud": "SoundCloud", - "misc.external.tumblr": "Tumblr", - "misc.external.twitter": "Twitter", - "misc.external.wikipedia": "Wikipedia", - "misc.external.youtube": "YouTube", - "misc.external.youtube.playlist": "YouTube (playlist)", - "misc.external.youtube.fullAlbum": "YouTube (full album)", - "misc.external.flash.bgreco": "{LINK} (HQ Audio)", - "misc.external.flash.homestuck.page": "{LINK} (page {PAGE})", - "misc.external.flash.homestuck.secret": "{LINK} (secret page)", - "misc.external.flash.youtube": "{LINK} (on any device)", - "misc.nav.previous": "Previous", - "misc.nav.next": "Next", - "misc.nav.info": "Info", - "misc.nav.gallery": "Gallery", - "misc.skippers.skipToContent": "Skip to content", - "misc.skippers.skipToSidebar": "Skip to sidebar", - "misc.skippers.skipToSidebar.left": "Skip to sidebar (left)", - "misc.skippers.skipToSidebar.right": "Skip to sidebar (right)", - "misc.skippers.skipToFooter": "Skip to footer", - "misc.jumpTo": "Jump to:", - "misc.jumpTo.withLinks": "Jump to: {LINKS}.", - "misc.contentWarnings": "cw: {WARNINGS}", - "misc.contentWarnings.reveal": "click to show", - "misc.albumGridDetails": "({TRACKS}, {TIME})", - "homepage.title": "{TITLE}", - "homepage.news.title": "News", - "homepage.news.entry.viewRest": "(View rest of entry!)", - "albumSidebar.trackList.group": "{GROUP}", - "albumSidebar.trackList.group.withRange": "{GROUP} ({RANGE})", - "albumSidebar.trackList.item": "{TRACK}", - "albumSidebar.groupBox.title": "{GROUP}", - "albumSidebar.groupBox.next": "Next: {ALBUM}", - "albumSidebar.groupBox.previous": "Previous: {ALBUM}", - "albumPage.title": "{ALBUM}", - "albumPage.nav.album": "{ALBUM}", - "albumPage.nav.randomTrack": "Random Track", - "albumCommentaryPage.title": "{ALBUM} - Commentary", - "albumCommentaryPage.infoLine": "{WORDS} across {ENTRIES}.", - "albumCommentaryPage.nav.album": "Album: {ALBUM}", - "albumCommentaryPage.entry.title.albumCommentary": "Album commentary", - "albumCommentaryPage.entry.title.trackCommentary": "{TRACK}", - "artistPage.title": "{ARTIST}", - "artistPage.creditList.album": "{ALBUM}", - "artistPage.creditList.album.withDate": "{ALBUM} ({DATE})", - "artistPage.creditList.album.withDate.withDuration": "{ALBUM} ({DATE}; {DURATION})", - "artistPage.creditList.flashAct": "{ACT}", - "artistPage.creditList.flashAct.withDateRange": "{ACT} ({DATE_RANGE})", - "artistPage.creditList.entry.track": "{TRACK}", - "artistPage.creditList.entry.track.withDuration": "({DURATION}) {TRACK}", - "artistPage.creditList.entry.album.coverArt": "(cover art)", - "artistPage.creditList.entry.album.wallpaperArt": "(wallpaper art)", - "artistPage.creditList.entry.album.bannerArt": "(banner art)", - "artistPage.creditList.entry.album.commentary": "(album commentary)", - "artistPage.creditList.entry.flash": "{FLASH}", - "artistPage.creditList.entry.rerelease": "{ENTRY} (re-release)", - "artistPage.creditList.entry.withContribution": "{ENTRY} ({CONTRIBUTION})", - "artistPage.creditList.entry.withArtists": "{ENTRY} (with {ARTISTS})", - "artistPage.creditList.entry.withArtists.withContribution": "{ENTRY} ({CONTRIBUTION}; with {ARTISTS})", - "artistPage.contributedDurationLine": "{ARTIST} has contributed {DURATION} of music shared on this wiki.", - "artistPage.musicGroupsLine": "Contributed music to groups: {GROUPS}", - "artistPage.artGroupsLine": "Contributed art to groups: {GROUPS}", - "artistPage.groupsLine.item": "{GROUP} ({CONTRIBUTIONS})", - "artistPage.trackList.title": "Tracks", - "artistPage.unreleasedTrackList.title": "Unreleased Tracks", - "artistPage.artList.title": "Art", - "artistPage.flashList.title": "Flashes & Games", - "artistPage.commentaryList.title": "Commentary", - "artistPage.viewArtGallery": "View {LINK}!", - "artistPage.viewArtGallery.orBrowseList": "View {LINK}! Or browse the list:", - "artistPage.viewArtGallery.link": "art gallery", - "artistPage.nav.artist": "Artist: {ARTIST}", - "artistGalleryPage.title": "{ARTIST} - Gallery", - "artistGalleryPage.infoLine": "Contributed to {COVER_ARTS}.", - "commentaryIndex.title": "Commentary", - "commentaryIndex.infoLine": "{WORDS} across {ENTRIES}, in all.", - "commentaryIndex.albumList.title": "Choose an album:", - "commentaryIndex.albumList.item": "{ALBUM} ({WORDS} across {ENTRIES})", - "flashIndex.title": "Flashes & Games", - "flashPage.title": "{FLASH}", - "flashPage.nav.flash": "{FLASH}", - "groupSidebar.title": "Groups", - "groupSidebar.groupList.category": "{CATEGORY}", - "groupSidebar.groupList.item": "{GROUP}", - "groupPage.nav.group": "Group: {GROUP}", - "groupInfoPage.title": "{GROUP}", - "groupInfoPage.viewAlbumGallery": "View {LINK}! Or browse the list:", - "groupInfoPage.viewAlbumGallery.link": "album gallery", - "groupInfoPage.albumList.title": "Albums", - "groupInfoPage.albumList.item": "({YEAR}) {ALBUM}", - "groupGalleryPage.title": "{GROUP} - Gallery", - "groupGalleryPage.infoLine": "{TRACKS} across {ALBUMS}, totaling {TIME}.", - "listingIndex.title": "Listings", - "listingIndex.infoLine": "{WIKI}: {TRACKS} across {ALBUMS}, totaling {DURATION}.", - "listingIndex.exploreList": "Feel free to explore any of the listings linked below and in the sidebar!", - "listingPage.listAlbums.byName.title": "Albums - by Name", - "listingPage.listAlbums.byName.item": "{ALBUM} ({TRACKS})", - "listingPage.listAlbums.byTracks.title": "Albums - by Tracks", - "listingPage.listAlbums.byTracks.item": "{ALBUM} ({TRACKS})", - "listingPage.listAlbums.byDuration.title": "Albums - by Duration", - "listingPage.listAlbums.byDuration.item": "{ALBUM} ({DURATION})", - "listingPage.listAlbums.byDate.title": "Albums - by Date", - "listingPage.listAlbums.byDate.item": "{ALBUM} ({DATE})", - "listingPage.listAlbums.byDateAdded.title": "Albums - by Date Added to Wiki", - "listingPage.listAlbums.byDateAdded.date": "{DATE}", - "listingPage.listAlbums.byDateAdded.album": "{ALBUM}", - "listingPage.listArtists.byName.title": "Artists - by Name", - "listingPage.listArtists.byName.item": "{ARTIST} ({CONTRIBUTIONS})", - "listingPage.listArtists.byContribs.title": "Artists - by Contributions", - "listingPage.listArtists.byContribs.item": "{ARTIST} ({CONTRIBUTIONS})", - "listingPage.listArtists.byCommentary.title": "Artists - by Commentary Entries", - "listingPage.listArtists.byCommentary.item": "{ARTIST} ({ENTRIES})", - "listingPage.listArtists.byDuration.title": "Artists - by Duration", - "listingPage.listArtists.byDuration.item": "{ARTIST} ({DURATION})", - "listingPage.listArtists.byLatest.title": "Artists - by Latest Contribution", - "listingPage.listArtists.byLatest.item": "{ARTIST} ({DATE})", - "listingPage.listGroups.byName.title": "Groups - by Name", - "listingPage.listGroups.byName.item": "{GROUP} ({GALLERY})", - "listingPage.listGroups.byName.item.gallery": "Gallery", - "listingPage.listGroups.byCategory.title": "Groups - by Category", - "listingPage.listGroups.byCategory.category": "{CATEGORY}", - "listingPage.listGroups.byCategory.group": "{GROUP} ({GALLERY})", - "listingPage.listGroups.byCategory.group.gallery": "Gallery", - "listingPage.listGroups.byAlbums.title": "Groups - by Albums", - "listingPage.listGroups.byAlbums.item": "{GROUP} ({ALBUMS})", - "listingPage.listGroups.byTracks.title": "Groups - by Tracks", - "listingPage.listGroups.byTracks.item": "{GROUP} ({TRACKS})", - "listingPage.listGroups.byDuration.title": "Groups - by Duration", - "listingPage.listGroups.byDuration.item": "{GROUP} ({DURATION})", - "listingPage.listGroups.byLatest.title": "Groups - by Latest Album", - "listingPage.listGroups.byLatest.item": "{GROUP} ({DATE})", - "listingPage.listTracks.byName.title": "Tracks - by Name", - "listingPage.listTracks.byName.item": "{TRACK}", - "listingPage.listTracks.byAlbum.title": "Tracks - by Album", - "listingPage.listTracks.byAlbum.album": "{ALBUM}", - "listingPage.listTracks.byAlbum.track": "{TRACK}", - "listingPage.listTracks.byDate.title": "Tracks - by Date", - "listingPage.listTracks.byDate.album": "{ALBUM} ({DATE})", - "listingPage.listTracks.byDate.track": "{TRACK}", - "listingPage.listTracks.byDate.track.rerelease": "{TRACK} (re-release)", - "listingPage.listTracks.byDuration.title": "Tracks - by Duration", - "listingPage.listTracks.byDuration.item": "{TRACK} ({DURATION})", - "listingPage.listTracks.byDurationInAlbum.title": "Tracks - by Duration (in Album)", - "listingPage.listTracks.byDurationInAlbum.album": "{ALBUM}", - "listingPage.listTracks.byDurationInAlbum.track": "{TRACK} ({DURATION})", - "listingPage.listTracks.byTimesReferenced.title": "Tracks - by Times Referenced", - "listingPage.listTracks.byTimesReferenced.item": "{TRACK} ({TIMES_REFERENCED})", - "listingPage.listTracks.inFlashes.byAlbum.title": "Tracks - in Flashes & Games (by Album)", - "listingPage.listTracks.inFlashes.byAlbum.album": "{ALBUM} ({DATE})", - "listingPage.listTracks.inFlashes.byAlbum.track": "{TRACK} (in {FLASHES})", - "listingPage.listTracks.inFlashes.byFlash.title": "Tracks - in Flashes & Games (by Flash)", - "listingPage.listTracks.inFlashes.byFlash.flash": "{FLASH} ({DATE})", - "listingPage.listTracks.inFlashes.byFlash.track": "{TRACK} (from {ALBUM})", - "listingPage.listTracks.withLyrics.title": "Tracks - with Lyrics", - "listingPage.listTracks.withLyrics.album": "{ALBUM} ({DATE})", - "listingPage.listTracks.withLyrics.track": "{TRACK}", - "listingPage.listTags.byName.title": "Tags - by Name", - "listingPage.listTags.byName.item": "{TAG} ({TIMES_USED})", - "listingPage.listTags.byUses.title": "Tags - by Uses", - "listingPage.listTags.byUses.item": "{TAG} ({TIMES_USED})", - "listingPage.misc.trackContributors": "Track Contributors", - "listingPage.misc.artContributors": "Art Contributors", - "listingPage.misc.artAndFlashContributors": "Art & Flash Contributors", - "newsIndex.title": "News", - "newsIndex.entry.viewRest": "(View rest of entry!)", - "newsEntryPage.title": "{ENTRY}", - "newsEntryPage.published": "(Published {DATE}.)", - "newsEntryPage.nav.news": "News", - "newsEntryPage.nav.entry": "{DATE}: {ENTRY}", - "redirectPage.title": "Moved to {TITLE}", - "redirectPage.infoLine": "This page has been moved to {TARGET}.", - "tagPage.title": "{TAG}", - "tagPage.infoLine": "Appears in {COVER_ARTS}.", - "tagPage.nav.tag": "Tag: {TAG}", - "trackPage.title": "{TRACK}", - "trackPage.referenceList.fandom": "Fandom:", - "trackPage.referenceList.official": "Official:", - "trackPage.nav.track": "{TRACK}", - "trackPage.nav.track.withNumber": "{NUMBER}. {TRACK}", - "trackPage.nav.random": "Random" -} diff --git a/upd8-util.js b/upd8-util.js deleted file mode 100644 index 4c4186f7..00000000 --- a/upd8-util.js +++ /dev/null @@ -1,423 +0,0 @@ -// This is used by upd8.js! It's part of the 8ackend. Read the notes there if -// you're curious. -// -// Friendly(!) disclaimer: these utility functions haven't 8een tested all that -// much. Do not assume it will do exactly what you want it to do in all cases. -// It will likely only do exactly what I want it to, and only in the cases I -// decided were relevant enough to 8other handling. - -'use strict'; - -// Apparently JavaScript doesn't come with a function to split an array into -// chunks! Weird. Anyway, this is an awesome place to use a generator, even -// though we don't really make use of the 8enefits of generators any time we -// actually use this. 8ut it's still awesome, 8ecause I say so. -module.exports.splitArray = function*(array, fn) { - let lastIndex = 0; - while (lastIndex < array.length) { - let nextIndex = array.findIndex((item, index) => index >= lastIndex && fn(item)); - if (nextIndex === -1) { - nextIndex = array.length; - } - yield array.slice(lastIndex, nextIndex); - // Plus one because we don't want to include the dividing line in the - // next array we yield. - lastIndex = nextIndex + 1; - } -}; - -// This function's name is a joke. Jokes! Hahahahahahahaha. Funny. -module.exports.joinNoOxford = function(array, plural = 'and') { - array = array.filter(Boolean); - - if (array.length === 0) { - // ???????? - return ''; - } - - if (array.length === 1) { - return array[0]; - } - - if (array.length === 2) { - return `${array[0]} ${plural} ${array[1]}`; - } - - return `${array.slice(0, -1).join(', ')} ${plural} ${array[array.length - 1]}`; -}; - -module.exports.progressPromiseAll = function (msgOrMsgFn, array) { - if (!array.length) { - return Promise.resolve([]); - } - - const msgFn = (typeof msgOrMsgFn === 'function' - ? msgOrMsgFn - : () => msgOrMsgFn); - - let done = 0, total = array.length; - process.stdout.write(`\r${msgFn()} [0/${total}]`); - const start = Date.now(); - return Promise.all(array.map(promise => promise.then(val => { - done++; - // const pc = `${done}/${total}`; - const pc = (Math.round(done / total * 1000) / 10 + '%').padEnd('99.9%'.length, ' '); - if (done === total) { - const time = Date.now() - start; - process.stdout.write(`\r\x1b[2m${msgFn()} [${pc}] \x1b[0;32mDone! \x1b[0;2m(${time} ms) \x1b[0m\n`) - } else { - process.stdout.write(`\r${msgFn()} [${pc}] `); - } - return val; - }))); -}; - -module.exports.queue = function (array, max = 50) { - if (max === 0) { - return array.map(fn => fn()); - } - - const begin = []; - let current = 0; - const ret = array.map(fn => new Promise((resolve, reject) => { - begin.push(() => { - current++; - Promise.resolve(fn()).then(value => { - current--; - if (current < max && begin.length) { - begin.shift()(); - } - resolve(value); - }, reject); - }); - })); - - for (let i = 0; i < max && begin.length; i++) { - begin.shift()(); - } - - return ret; -}; - -module.exports.delay = ms => new Promise(res => setTimeout(res, ms)); - -module.exports.th = function (n) { - if (n % 10 === 1 && n !== 11) { - return n + 'st'; - } else if (n % 10 === 2 && n !== 12) { - return n + 'nd'; - } else if (n % 10 === 3 && n !== 13) { - return n + 'rd'; - } else { - return n + 'th'; - } -}; - -// My function names just keep getting 8etter. -module.exports.s = function (n, word) { - return `${n} ${word}` + (n === 1 ? '' : 's'); -}; - -// Hey, did you know I apparently put a space 8efore the parameters in function -// names? 8ut only in function expressions, not declar8tions? I mean, I guess -// you did. You're pro8a8ly more familiar with my code than I am 8y this -// point. I haven't messed with any of this code in ages. Yay!!!!!!!! -// -// This function only does anything on o8jects you're going to 8e reusing. -// Argua8ly I could use a WeakMap here, 8ut since the o8ject needs to 8e -// reused to 8e useful anyway, I just store the result with a symbol. -// Sorry if it's 8een frozen I guess?? -module.exports.cacheOneArg = function (fn) { - const symbol = Symbol('Cache'); - return arg => { - if (!arg[symbol]) { - arg[symbol] = fn(arg); - } - return arg[symbol]; - }; -}; - -const decorateTime = function (functionToBeWrapped) { - const fn = function(...args) { - const start = Date.now(); - const ret = functionToBeWrapped(...args); - const end = Date.now(); - fn.timeSpent += end - start; - fn.timesCalled++; - return ret; - }; - - fn.wrappedName = functionToBeWrapped.name; - fn.timeSpent = 0; - fn.timesCalled = 0; - fn.displayTime = function() { - const averageTime = fn.timeSpent / fn.timesCalled; - console.log(`\x1b[1m${fn.wrappedName}(...):\x1b[0m ${fn.timeSpent} ms / ${fn.timesCalled} calls \x1b[2m(avg: ${averageTime} ms)\x1b[0m`); - }; - - decorateTime.decoratedFunctions.push(fn); - - return fn; -}; - -decorateTime.decoratedFunctions = []; -decorateTime.displayTime = function() { - if (decorateTime.decoratedFunctions.length) { - console.log(`\x1b[1mdecorateTime results: ` + '-'.repeat(40) + '\x1b[0m'); - for (const fn of decorateTime.decoratedFunctions) { - fn.displayTime(); - } - } -}; - -module.exports.decorateTime = decorateTime; - -// Stolen as #@CK from mtui! -const parseOptions = async function(options, optionDescriptorMap) { - // This function is sorely lacking in comments, but the basic usage is - // as such: - // - // options is the array of options you want to process; - // optionDescriptorMap is a mapping of option names to objects that describe - // the expected value for their corresponding options. - // Returned is a mapping of any specified option names to their values, or - // a process.exit(1) and error message if there were any issues. - // - // Here are examples of optionDescriptorMap to cover all the things you can - // do with it: - // - // optionDescriptorMap: { - // 'telnet-server': {type: 'flag'}, - // 't': {alias: 'telnet-server'} - // } - // - // options: ['t'] -> result: {'telnet-server': true} - // - // optionDescriptorMap: { - // 'directory': { - // type: 'value', - // validate(name) { - // // const whitelistedDirectories = ['apple', 'banana'] - // if (whitelistedDirectories.includes(name)) { - // return true - // } else { - // return 'a whitelisted directory' - // } - // } - // }, - // 'files': {type: 'series'} - // } - // - // ['--directory', 'apple'] -> {'directory': 'apple'} - // ['--directory', 'artichoke'] -> (error) - // ['--files', 'a', 'b', 'c', ';'] -> {'files': ['a', 'b', 'c']} - // - // TODO: Be able to validate the values in a series option. - - const handleDashless = optionDescriptorMap[parseOptions.handleDashless]; - const handleUnknown = optionDescriptorMap[parseOptions.handleUnknown]; - const result = Object.create(null); - for (let i = 0; i < options.length; i++) { - const option = options[i]; - if (option.startsWith('--')) { - // --x can be a flag or expect a value or series of values - let name = option.slice(2).split('=')[0]; // '--x'.split('=') = ['--x'] - let descriptor = optionDescriptorMap[name]; - if (!descriptor) { - if (handleUnknown) { - handleUnknown(option); - } else { - console.error(`Unknown option name: ${name}`); - process.exit(1); - } - continue; - } - if (descriptor.alias) { - name = descriptor.alias; - descriptor = optionDescriptorMap[name]; - } - if (descriptor.type === 'flag') { - result[name] = true; - } else if (descriptor.type === 'value') { - let value = option.slice(2).split('=')[1]; - if (!value) { - value = options[++i]; - if (!value || value.startsWith('-')) { - value = null; - } - } - if (!value) { - console.error(`Expected a value for --${name}`); - process.exit(1); - } - result[name] = value; - } else if (descriptor.type === 'series') { - if (!options.slice(i).includes(';')) { - console.error(`Expected a series of values concluding with ; (\\;) for --${name}`); - process.exit(1); - } - const endIndex = i + options.slice(i).indexOf(';'); - result[name] = options.slice(i + 1, endIndex); - i = endIndex; - } - if (descriptor.validate) { - const validation = await descriptor.validate(result[name]); - if (validation !== true) { - console.error(`Expected ${validation} for --${name}`); - process.exit(1); - } - } - } else if (option.startsWith('-')) { - // mtui doesn't use any -x=y or -x y format optionuments - // -x will always just be a flag - let name = option.slice(1); - let descriptor = optionDescriptorMap[name]; - if (!descriptor) { - if (handleUnknown) { - handleUnknown(option); - } else { - console.error(`Unknown option name: ${name}`); - process.exit(1); - } - continue; - } - if (descriptor.alias) { - name = descriptor.alias; - descriptor = optionDescriptorMap[name]; - } - if (descriptor.type === 'flag') { - result[name] = true; - } else { - console.error(`Use --${name} (value) to specify ${name}`); - process.exit(1); - } - } else if (handleDashless) { - handleDashless(option); - } - } - return result; -} - -parseOptions.handleDashless = Symbol(); -parseOptions.handleUnknown = Symbol(); - -module.exports.parseOptions = parseOptions; - -// Cheap FP for a cheap dyke! -// I have no idea if this is what curry actually means. -module.exports.curry = f => x => (...args) => f(x, ...args); - -module.exports.mapInPlace = (array, fn) => array.splice(0, array.length, ...array.map(fn)); - -module.exports.filterEmptyLines = string => string.split('\n').filter(line => line.trim()).join('\n'); - -module.exports.unique = arr => Array.from(new Set(arr)); - -const logColor = color => (literals, ...values) => { - const w = s => process.stdout.write(s); - w(`\x1b[${color}m`); - for (let i = 0; i < literals.length; i++) { - w(literals[i]); - if (values[i] !== undefined) { - w(`\x1b[1m`); - w(String(values[i])); - w(`\x1b[0;${color}m`); - } - } - w(`\x1b[0m\n`); -}; - -module.exports.logInfo = logColor(2); -module.exports.logWarn = logColor(33); -module.exports.logError = logColor(31); - -module.exports.sortByName = (a, b) => { - let an = a.name.toLowerCase(); - let bn = b.name.toLowerCase(); - if (an.startsWith('the ')) an = an.slice(4); - if (bn.startsWith('the ')) bn = bn.slice(4); - return an < bn ? -1 : an > bn ? 1 : 0; -}; - -module.exports.chunkByConditions = function(array, conditions) { - if (array.length === 0) { - return []; - } else if (conditions.length === 0) { - return [array]; - } - - const out = []; - let cur = [array[0]]; - for (let i = 1; i < array.length; i++) { - const item = array[i]; - const prev = array[i - 1]; - let chunk = false; - for (const condition of conditions) { - if (condition(item, prev)) { - chunk = true; - break; - } - } - if (chunk) { - out.push(cur); - cur = [item]; - } else { - cur.push(item); - } - } - out.push(cur); - return out; -}; - -module.exports.chunkByProperties = function(array, properties) { - return module.exports.chunkByConditions(array, properties.map(p => (a, b) => { - if (a[p] instanceof Date && b[p] instanceof Date) - return +a[p] !== +b[p]; - - if (a[p] !== b[p]) return true; - - // Not sure if this line is still necessary with the specific check for - // d8tes a8ove, 8ut, uh, keeping it anyway, just in case....? - if (a[p] != b[p]) return true; - - return false; - })) - .map(chunk => ({ - ...Object.fromEntries(properties.map(p => [p, chunk[0][p]])), - chunk - })); -}; - -// Very cool function origin8ting in... http-music pro8a8ly! -// Sorry if we happen to 8e violating past-us's copyright, lmao. -module.exports.promisifyProcess = function(proc, showLogging = true) { - // Takes a process (from the child_process module) and returns a promise - // that resolves when the process exits (or rejects, if the exit code is - // non-zero). - // - // Ayy look, no alpha8etical second letter! Couldn't tell this was written - // like three years ago 8efore I was me. 8888) - - return new Promise((resolve, reject) => { - if (showLogging) { - proc.stdout.pipe(process.stdout); - proc.stderr.pipe(process.stderr); - } - - proc.on('exit', code => { - if (code === 0) { - resolve(); - } else { - reject(code); - } - }) - }) -}; - -// Stolen from jq! Which pro8a8ly stole the concept from other places. Nice. -module.exports.withEntries = (obj, fn) => Object.fromEntries(fn(Object.entries(obj))); - -// Nothin' more to it than what it says. Runs a function in-place. Provides an -// altern8tive syntax to the usual IIFEs (e.g. (() => {})()) when you want to -// open a scope and run some statements while inside an existing expression. -module.exports.call = fn => fn(); diff --git a/upd8.js b/upd8.js deleted file mode 100755 index e17005dd..00000000 --- a/upd8.js +++ /dev/null @@ -1,6597 +0,0 @@ -#!/usr/bin/env node - -// HEY N8RDS! -// -// This is one of the 8ACKEND FILES. It's not used anywhere on the actual site -// you are pro8a8ly using right now. -// -// Specifically, this one does all the actual work of the music wiki. The -// process looks something like this: -// -// 1. Crawl the music directories. Well, not so much "crawl" as "look inside -// the folders for each al8um, and read the metadata file descri8ing that -// al8um and the tracks within." -// -// 2. Read that metadata. I'm writing this 8efore actually doing any of the -// code, and I've gotta admit I have no idea what file format they're -// going to 8e in. May8e JSON, 8ut more likely some weird custom format -// which will 8e a lot easier to edit. -// -// 3. Generate the page files! They're just static index.html files, and are -// what gh-pages (or wherever this is hosted) will show to clients. -// Hopefully pretty minimalistic HTML, 8ut like, shrug. They'll reference -// CSS (and maaaaaaaay8e JS) files, hard-coded somewhere near the root. -// -// 4. Print an awesome message which says the process is done. This is the -// most important step. -// -// Oh yeah, like. Just run this through some relatively recent version of -// node.js and you'll 8e fine. ...Within the project root. O8viously. - -// HEY FUTURE ME!!!!!!!! Don't forget to implement artist pages! Those are, -// like, the coolest idea you've had yet, so DO NOT FORGET. (Remem8er, link -// from track listings, etc!) --- Thanks, past me. To futurerer me: an al8um -// listing page (a list of all the al8ums)! Make sure to sort these 8y date - -// we'll need a new field for al8ums. - -// ^^^^^^^^ DID THAT! 8ut also, artist images. Pro8a8ly stolen from the fandom -// wiki (I found half those images anywayz). - -// TRACK ART CREDITS. This is a must. - -// 2020-08-23 -// ATTENTION ALL 8*TCHES AND OTHER GENDER TRUCKERS: AS IT TURNS OUT, THIS CODE -// ****SUCKS****. I DON'T THINK ANYTHING WILL EVER REDEEM IT, 8UT THAT DOESN'T -// MEAN WE CAN'T TAKE SOME ACTION TO MAKE WRITING IT A LITTLE LESS TERRI8LE. -// We're gonna start defining STRUCTURES to make things suck less!!!!!!!! -// No classes 8ecause those are a huge pain and like, pro8a8ly 8ad performance -// or whatever -- just some standard structures that should 8e followed -// wherever reasona8le. Only one I need today is the contri8 one 8ut let's put -// any new general-purpose structures here too, ok? -// -// Contri8ution: {who, what, date, thing}. D8 and thing are the new fields. -// -// Use these wisely, which is to say all the time and instead of whatever -// terri8le new pseudo structure you're trying to invent!!!!!!!! -// -// Upd8 2021-01-03: Soooooooo we didn't actually really end up using these, -// lol? Well there's still only one anyway. Kinda ended up doing a 8ig refactor -// of all the o8ject structures today. It's not *especially* relevant 8ut feels -// worth mentioning? I'd get rid of this comment 8lock 8ut I like it too much! -// Even though I haven't actually reread it, lol. 8ut yeah, hopefully in the -// spirit of this "make things more consistent" attitude I 8rought up 8ack in -// August, stuff's lookin' 8etter than ever now. W00t! - -'use strict'; - -const fs = require('fs'); -const path = require('path'); -const util = require('util'); - -// I made this dependency myself! A long, long time ago. It is pro8a8ly my -// most useful li8rary ever. I'm not sure 8esides me actually uses it, though. -const fixWS = require('fix-whitespace'); -// Wait nevermind, I forgot a8out why-do-kids-love-the-taste-of-cinnamon-toast- -// crunch. THAT is my 8est li8rary. - -// The require function just returns whatever the module exports, so there's -// no reason you can't wrap it in some decorator right out of the 8ox. Which is -// exactly what we do here. -const mkdirp = util.promisify(require('mkdirp')); - -// It stands for "HTML Entities", apparently. Cursed. -const he = require('he'); - -// This is the dum8est name for a function possi8le. Like, SURE, fine, may8e -// the UNIX people had some valid reason to go with the weird truncated -// lowercased convention they did. 8ut Node didn't have to ALSO use that -// convention! Would it have 8een so hard to just name the function something -// like fs.readDirectory???????? No, it wouldn't have 8een. -const readdir = util.promisify(fs.readdir); -// 8ut okay, like, look at me. DOING THE SAME THING. See, *I* could have named -// my promisified function differently, and yet I did not. I literally cannot -// explain why. We are all used to following in the 8ad decisions of our -// ancestors, and never never never never never never never consider that hey, -// may8e we don't need to make the exact same decisions they did. Even when -// we're perfectly aware th8t's exactly what we're doing! Programmers, -// including me, are all pretty stupid. - -// 8ut I mean, come on. Look. Node decided to use readFile, instead of like, -// what, cat? Why couldn't they rename readdir too???????? As Johannes Kepler -// once so elegantly put it: "Shrug." -const readFile = util.promisify(fs.readFile); -const writeFile = util.promisify(fs.writeFile); -const access = util.promisify(fs.access); -const symlink = util.promisify(fs.symlink); -const unlink = util.promisify(fs.unlink); - -const { - cacheOneArg, - call, - chunkByConditions, - chunkByProperties, - curry, - decorateTime, - filterEmptyLines, - joinNoOxford, - mapInPlace, - logWarn, - logInfo, - logError, - parseOptions, - progressPromiseAll, - queue, - s, - sortByName, - splitArray, - th, - unique, - withEntries -} = require('./upd8-util'); - -const genThumbs = require('./gen-thumbs'); - -const C = require('./common/common'); - -const CACHEBUST = 5; - -const WIKI_INFO_FILE = 'wiki-info.txt'; -const HOMEPAGE_INFO_FILE = 'homepage.txt'; -const ARTIST_DATA_FILE = 'artists.txt'; -const FLASH_DATA_FILE = 'flashes.txt'; -const NEWS_DATA_FILE = 'news.txt'; -const TAG_DATA_FILE = 'tags.txt'; -const GROUP_DATA_FILE = 'groups.txt'; -const STATIC_PAGE_DATA_FILE = 'static-pages.txt'; -const DEFAULT_STRINGS_FILE = 'strings-default.json'; - -const CSS_FILE = 'site.css'; - -// Shared varia8les! These are more efficient to access than a shared varia8le -// (or at least I h8pe so), and are easier to pass across functions than a -// 8unch of specific arguments. -// -// Upd8: Okay yeah these aren't actually any different. Still cleaner than -// passing around a data object containing all this, though. -let dataPath; -let mediaPath; -let langPath; -let outputPath; - -let wikiInfo; -let homepageInfo; -let albumData; -let trackData; -let flashData; -let flashActData; -let newsData; -let tagData; -let groupData; -let groupCategoryData; -let staticPageData; - -let artistNames; -let artistData; -let artistAliasData; - -let officialAlbumData; -let fandomAlbumData; -let justEverythingMan; // tracks, albums, flashes -- don't forget to upd8 toAnythingMan! -let justEverythingSortedByArtDateMan; -let contributionData; - -let queueSize; - -let languages; - -const html = { - // Non-comprehensive. ::::P - selfClosingTags: ['br', 'img'], - - // Pass to tag() as an attri8utes key to make tag() return a 8lank string - // if the provided content is empty. Useful for when you'll only 8e showing - // an element according to the presence of content that would 8elong there. - onlyIfContent: Symbol(), - - tag(tagName, ...args) { - const selfClosing = html.selfClosingTags.includes(tagName); - - let openTag; - let content; - let attrs; - - if (typeof args[0] === 'object' && !Array.isArray(args[0])) { - attrs = args[0]; - content = args[1]; - } else { - content = args[0]; - } - - if (selfClosing && content) { - throw new Error(`Tag <${tagName}> is self-closing but got content!`); - } - - if (attrs?.[html.onlyIfContent] && !content) { - return ''; - } - - if (attrs) { - const attrString = html.attributes(args[0]); - if (attrString) { - openTag = `${tagName} ${attrString}`; - } - } - - if (!openTag) { - openTag = tagName; - } - - if (Array.isArray(content)) { - content = content.filter(Boolean).join('\n'); - } - - if (content) { - if (content.includes('\n')) { - return fixWS` - <${openTag}> - ${content} - - `; - } else { - return `<${openTag}>${content}`; - } - } else { - if (selfClosing) { - return `<${openTag}>`; - } else { - return `<${openTag}>`; - } - } - }, - - escapeAttributeValue(value) { - return value - .replaceAll('"', '"') - .replaceAll("'", '''); - }, - - attributes(attribs) { - return Object.entries(attribs) - .map(([ key, val ]) => { - if (!val) - return [key, val]; - else if (typeof val === 'string' || typeof val === 'boolean') - return [key, val]; - else if (typeof val === 'number') - return [key, val.toString()]; - else if (Array.isArray(val)) - return [key, val.join(' ')]; - else - throw new Error(`Attribute value for ${key} should be primitive or array, got ${typeof val}`); - }) - .filter(([ key, val ]) => val) - .map(([ key, val ]) => (typeof val === 'boolean' - ? `${key}` - : `${key}="${html.escapeAttributeValue(val)}"`)) - .join(' '); - } -}; - -const urlSpec = { - data: { - prefix: 'data/', - - paths: { - root: '', - path: '<>', - - album: 'album/<>', - artist: 'artist/<>', - track: 'track/<>' - } - }, - - localized: { - // TODO: Implement this. - // prefix: '_languageCode', - - paths: { - root: '', - path: '<>', - - home: '', - - album: 'album/<>/', - albumCommentary: 'commentary/album/<>/', - - artist: 'artist/<>/', - artistGallery: 'artist/<>/gallery/', - - commentaryIndex: 'commentary/', - - flashIndex: 'flash/', - flash: 'flash/<>/', - - groupInfo: 'group/<>/', - groupGallery: 'group/<>/gallery/', - - listingIndex: 'list/', - listing: 'list/<>/', - - newsIndex: 'news/', - newsEntry: 'news/<>/', - - staticPage: '<>/', - tag: 'tag/<>/', - track: 'track/<>/' - } - }, - - shared: { - paths: { - root: '', - path: '<>', - - commonFile: 'common/<>', - staticFile: 'static/<>' - } - }, - - media: { - prefix: 'media/', - - paths: { - root: '', - path: '<>', - - albumCover: 'album-art/<>/cover.jpg', - albumWallpaper: 'album-art/<>/bg.jpg', - albumBanner: 'album-art/<>/banner.jpg', - trackCover: 'album-art/<>/<>.jpg', - artistAvatar: 'artist-avatar/<>.jpg', - flashArt: 'flash-art/<>.jpg' - } - } -}; - -// This gets automatically switched in place when working from a baseDirectory, -// so it should never be referenced manually. -urlSpec.localizedWithBaseDirectory = { - paths: withEntries( - urlSpec.localized.paths, - entries => entries.map(([key, path]) => [key, '<>/' + path]) - ) -}; - -const linkHelper = (hrefFn, {color = true, attr = null} = {}) => - (thing, { - strings, to, - text = '', - class: className = '', - hash = '' - }) => ( - html.tag('a', { - ...attr ? attr(thing) : {}, - href: hrefFn(thing, {to}) + (hash ? (hash.startsWith('#') ? '' : '#') + hash : ''), - style: color ? getLinkThemeString(thing) : '', - class: className - }, text || thing.name) - ); - -const linkDirectory = (key, {expose = null, attr = null, ...conf} = {}) => - linkHelper((thing, {to}) => to('localized.' + key, thing.directory), { - attr: thing => ({ - ...attr ? attr(thing) : {}, - ...expose ? {[expose]: thing.directory} : {} - }), - ...conf - }); - -const linkPathname = (key, conf) => linkHelper(({directory: pathname}, {to}) => to(key, pathname), conf); -const linkIndex = (key, conf) => linkHelper((_, {to}) => to('localized.' + key), conf); - -const link = { - album: linkDirectory('album'), - albumCommentary: linkDirectory('albumCommentary'), - artist: linkDirectory('artist', {color: false}), - artistGallery: linkDirectory('artistGallery', {color: false}), - commentaryIndex: linkIndex('commentaryIndex', {color: false}), - flashIndex: linkIndex('flashIndex', {color: false}), - flash: linkDirectory('flash'), - groupInfo: linkDirectory('groupInfo'), - groupGallery: linkDirectory('groupGallery'), - home: linkIndex('home', {color: false}), - listingIndex: linkIndex('listingIndex'), - listing: linkDirectory('listing'), - newsIndex: linkIndex('newsIndex', {color: false}), - newsEntry: linkDirectory('newsEntry', {color: false}), - staticPage: linkDirectory('staticPage', {color: false}), - tag: linkDirectory('tag'), - track: linkDirectory('track', {expose: 'data-track'}), - - media: linkPathname('media.path', {color: false}), - root: linkPathname('shared.path', {color: false}), - data: linkPathname('data.path', {color: false}), - site: linkPathname('localized.path', {color: false}) -}; - -const thumbnailHelper = name => file => - file.replace(/\.(jpg|png)$/, name + '.jpg'); - -const thumb = { - medium: thumbnailHelper('.medium'), - small: thumbnailHelper('.small') -}; - -function generateURLs(fromPath) { - const getValueForFullKey = (obj, fullKey, prop = null) => { - const [ groupKey, subKey ] = fullKey.split('.'); - if (!groupKey || !subKey) { - throw new Error(`Expected group key and subkey (got ${fullKey})`); - } - - if (!obj.hasOwnProperty(groupKey)) { - throw new Error(`Expected valid group key (got ${groupKey})`); - } - - const group = obj[groupKey]; - - if (!group.hasOwnProperty(subKey)) { - throw new Error(`Expected valid subkey (got ${subKey} for group ${groupKey})`); - } - - return { - value: group[subKey], - group - }; - }; - - const generateTo = (fromPath, fromGroup) => { - const rebasePrefix = '../'.repeat((fromGroup.prefix || '').split('/').filter(Boolean).length); - - const pathHelper = (toPath, toGroup) => { - let target = toPath; - - let argIndex = 0; - target = target.replaceAll('<>', () => `<${argIndex++}>`); - - if (toGroup.prefix !== fromGroup.prefix) { - // TODO: Handle differing domains in prefixes. - target = rebasePrefix + (toGroup.prefix || '') + target; - } - - return (path.relative(fromPath, target) - + (toPath.endsWith('/') ? '/' : '')); - }; - - const groupSymbol = Symbol(); - - const groupHelper = urlGroup => ({ - [groupSymbol]: urlGroup, - ...withEntries(urlGroup.paths, entries => entries - .map(([key, path]) => [key, pathHelper(path, urlGroup)])) - }); - - const relative = withEntries(urlSpec, entries => entries - .map(([key, urlGroup]) => [key, groupHelper(urlGroup)])); - - const to = (key, ...args) => { - const { value: template, group: {[groupSymbol]: toGroup} } = getValueForFullKey(relative, key) - let result = template.replaceAll(/<([0-9]+)>/g, (match, n) => args[n]); - - // Kinda hacky lol, 8ut it works. - const missing = result.match(/<([0-9]+)>/g); - if (missing) { - throw new Error(`Expected ${missing[missing.length - 1]} arguments, got ${args.length}`); - } - - return result; - }; - - return {to, relative}; - }; - - const generateFrom = () => { - const map = withEntries(urlSpec, entries => entries - .map(([key, group]) => [key, withEntries(group.paths, entries => entries - .map(([key, path]) => [key, generateTo(path, group)]) - )])); - - const from = key => getValueForFullKey(map, key).value; - - return {from, map}; - }; - - return generateFrom(); -} - -const urls = generateURLs(); - -const searchHelper = (keys, dataFn, findFn) => ref => { - if (!ref) return null; - ref = ref.replace(new RegExp(`^(${keys.join('|')}):`), ''); - const found = findFn(ref, dataFn()); - if (!found) { - logWarn`Didn't match anything for ${ref}! (${keys.join(', ')})`; - } - return found; -}; - -const matchDirectory = (ref, data) => data.find(({ directory }) => directory === ref); - -const matchDirectoryOrName = (ref, data) => { - let thing; - - thing = matchDirectory(ref, data); - if (thing) return thing; - - thing = data.find(({ name }) => name === ref); - if (thing) return thing; - - thing = data.find(({ name }) => name.toLowerCase() === ref.toLowerCase()); - if (thing) { - logWarn`Bad capitalization: ${'\x1b[31m' + ref} -> ${'\x1b[32m' + thing.name}`; - return thing; - } - - return null; -}; - -const search = { - album: searchHelper(['album', 'album-commentary'], () => albumData, matchDirectoryOrName), - artist: searchHelper(['artist', 'artist-gallery'], () => artistData, matchDirectoryOrName), - flash: searchHelper(['flash'], () => flashData, matchDirectory), - group: searchHelper(['group', 'group-gallery'], () => groupData, matchDirectoryOrName), - listing: searchHelper(['listing'], () => listingSpec, matchDirectory), - newsEntry: searchHelper(['news-entry'], () => newsData, matchDirectory), - staticPage: searchHelper(['static'], () => staticPageData, matchDirectory), - tag: searchHelper(['tag'], () => tagData, (ref, data) => - matchDirectoryOrName(ref.startsWith('cw: ') ? ref.slice(4) : ref, data)), - track: searchHelper(['track'], () => trackData, matchDirectoryOrName) -}; - -// Localiz8tion time! Or l10n as the neeeeeeeerds call it. Which is a terri8le -// name and not one I intend on using, thank you very much. (Don't even get me -// started on """"a11y"""".) -// -// All the default strings are in strings-default.json, if you're curious what -// those actually look like. Pretty much it's "I like {ANIMAL}" for example. -// For each language, the o8ject gets turned into a single function of form -// f(key, {args}). It searches for a key in the o8ject and uses the string it -// finds (or the one in strings-default.json) as a templ8 evaluated with the -// arguments passed. (This function gets treated as an o8ject too; it gets -// the language code attached.) -// -// The function's also responsi8le for getting rid of dangerous characters -// (quotes and angle tags), though only within the templ8te (not the args), -// and it converts the keys of the arguments o8ject from camelCase to -// CONSTANT_CASE too. -function genStrings(stringsJSON, defaultJSON = null) { - // genStrings will only 8e called once for each language, and it happens - // right at the start of the program (or at least 8efore 8uilding pages). - // So, now's a good time to valid8te the strings and let any warnings be - // known. - - // May8e contrary to the argument name, the arguments should 8e o8jects, - // not actual JSON-formatted strings! - if (typeof stringsJSON !== 'object' || stringsJSON.constructor !== Object) { - return {error: `Expected an object (parsed JSON) for stringsJSON.`}; - } - if (typeof defaultJSON !== 'object') { // typeof null === object. I h8 JS. - return {error: `Expected an object (parsed JSON) or null for defaultJSON.`}; - } - - // All languages require a language code. - const code = stringsJSON['meta.languageCode']; - if (!code) { - return {error: `Missing language code.`}; - } - if (typeof code !== 'string') { - return {error: `Expected language code to be a string.`}; - } - - // Every value on the provided o8ject should be a string. - // (This is lazy, but we only 8other checking this on stringsJSON, on the - // assumption that defaultJSON was passed through this function too, and so - // has already been valid8ted.) - { - let err = false; - for (const [ key, value ] of Object.entries(stringsJSON)) { - if (typeof value !== 'string') { - logError`(${code}) The value for ${key} should be a string.`; - err = true; - } - } - if (err) { - return {error: `Expected all values to be a string.`}; - } - } - - // Checking is generally done against the default JSON, so we'll skip out - // if that isn't provided (which should only 8e the case when it itself is - // 8eing processed as the first loaded language). - if (defaultJSON) { - // Warn for keys that are missing or unexpected. - const expectedKeys = Object.keys(defaultJSON); - const presentKeys = Object.keys(stringsJSON); - for (const key of presentKeys) { - if (!expectedKeys.includes(key)) { - logWarn`(${code}) Unexpected translation key: ${key} - this won't be used!`; - } - } - for (const key of expectedKeys) { - if (!presentKeys.includes(key)) { - logWarn`(${code}) Missing translation key: ${key} - this won't be localized!`; - } - } - } - - // Valid8tion is complete, 8ut We can still do a little caching to make - // repeated actions faster. - - // We're gonna 8e mut8ting the strings dictionary o8ject from here on out. - // We make a copy so we don't mess with the one which was given to us. - stringsJSON = Object.assign({}, stringsJSON); - - // Preemptively pass everything through HTML encoding. This will prevent - // strings from embedding HTML tags or accidentally including characters - // that throw HTML parsers off. - for (const key of Object.keys(stringsJSON)) { - stringsJSON[key] = he.encode(stringsJSON[key], {useNamedReferences: true}); - } - - // It's time to cre8te the actual langauge function! - - // In the function, we don't actually distinguish 8etween the primary and - // default (fall8ack) strings - any relevant warnings have already 8een - // presented a8ove, at the time the language JSON is processed. Now we'll - // only 8e using them for indexing strings to use as templ8tes, and we can - // com8ine them for that. - const stringIndex = Object.assign({}, defaultJSON, stringsJSON); - - // We do still need the list of valid keys though. That's 8ased upon the - // default strings. (Or stringsJSON, 8ut only if the defaults aren't - // provided - which indic8tes that the single o8ject provided *is* the - // default.) - const validKeys = Object.keys(defaultJSON || stringsJSON); - - const invalidKeysFound = []; - - const strings = (key, args = {}) => { - // Ok, with the warning out of the way, it's time to get to work. - // First make sure we're even accessing a valid key. (If not, return - // an error string as su8stitute.) - if (!validKeys.includes(key)) { - // We only want to warn a8out a given key once. More than that is - // just redundant! - if (!invalidKeysFound.includes(key)) { - invalidKeysFound.push(key); - logError`(${code}) Accessing invalid key ${key}. Fix a typo or provide this in strings-default.json!`; - } - return `MISSING: ${key}`; - } - - const template = stringIndex[key]; - - // Convert the keys on the args dict from camelCase to CONSTANT_CASE. - // (This isn't an OUTRAGEOUSLY versatile algorithm for doing that, 8ut - // like, who cares, dude?) Also, this is an array, 8ecause it's handy - // for the iterating we're a8out to do. - const processedArgs = Object.entries(args) - .map(([ k, v ]) => [k.replace(/[A-Z]/g, '_$&').toUpperCase(), v]); - - // Replacement time! Woot. Reduce comes in handy here! - const output = processedArgs.reduce( - (x, [ k, v ]) => x.replaceAll(`{${k}}`, v), - template); - - // Post-processing: if any expected arguments *weren't* replaced, that - // is almost definitely an error. - if (output.match(/\{[A-Z_]+\}/)) { - logError`(${code}) Args in ${key} were missing - output: ${output}`; - } - - return output; - }; - - // And lastly, we add some utility stuff to the strings function. - - // Store the language code, for convenience of access. - strings.code = code; - - // Store the strings dictionary itself, also for convenience. - strings.json = stringsJSON; - - // Store Intl o8jects that can 8e reused for value formatting. - strings.intl = { - date: new Intl.DateTimeFormat(code, {full: true}), - number: new Intl.NumberFormat(code), - list: { - conjunction: new Intl.ListFormat(code, {type: 'conjunction'}), - disjunction: new Intl.ListFormat(code, {type: 'disjunction'}), - unit: new Intl.ListFormat(code, {type: 'unit'}) - }, - plural: { - cardinal: new Intl.PluralRules(code, {type: 'cardinal'}), - ordinal: new Intl.PluralRules(code, {type: 'ordinal'}) - } - }; - - const bindUtilities = (obj, bind) => Object.fromEntries(Object.entries(obj).map( - ([ key, fn ]) => [key, (value, opts = {}) => fn(value, {...bind, ...opts})] - )); - - // There are a 8unch of handy count functions which expect a strings value; - // for a more terse syntax, we'll stick 'em on the strings function itself, - // with automatic 8inding for the strings argument. - strings.count = bindUtilities(count, {strings}); - - // The link functions also expect the strings o8ject(*). May as well hand - // 'em over here too! Keep in mind they still expect {to} though, and that - // isn't something we have access to from this scope (so calls such as - // strings.link.album(...) still need to provide it themselves). - // - // (*) At time of writing, it isn't actually used for anything, 8ut future- - // proofing, ok???????? - strings.link = bindUtilities(link, {strings}); - - // List functions, too! - strings.list = bindUtilities(list, {strings}); - - return strings; -}; - -const countHelper = (stringKey, argName = stringKey) => (value, {strings, unit = false}) => strings( - (unit - ? `count.${stringKey}.withUnit.` + strings.intl.plural.cardinal.select(value) - : `count.${stringKey}`), - {[argName]: strings.intl.number.format(value)}); - -const count = { - date: (date, {strings}) => { - return strings.intl.date.format(date); - }, - - dateRange: ([startDate, endDate], {strings}) => { - return strings.intl.date.formatRange(startDate, endDate); - }, - - duration: (secTotal, {strings, approximate = false, unit = false}) => { - if (secTotal === 0) { - return strings('count.duration.missing'); - } - - const hour = Math.floor(secTotal / 3600); - const min = Math.floor((secTotal - hour * 3600) / 60); - const sec = Math.floor(secTotal - hour * 3600 - min * 60); - - const pad = val => val.toString().padStart(2, '0'); - - const stringSubkey = unit ? '.withUnit' : ''; - - const duration = (hour > 0 - ? strings('count.duration.hours' + stringSubkey, { - hours: hour, - minutes: pad(min), - seconds: pad(sec) - }) - : strings('count.duration.minutes' + stringSubkey, { - minutes: min, - seconds: pad(sec) - })); - - return (approximate - ? strings('count.duration.approximate', {duration}) - : duration); - }, - - index: (value, {strings}) => { - return strings('count.index.' + strings.intl.plural.ordinal.select(value), {index: value}); - }, - - number: value => strings.intl.number.format(value), - - words: (value, {strings, unit = false}) => { - const num = strings.intl.number.format(value > 1000 - ? Math.floor(value / 100) / 10 - : value); - - const words = (value > 1000 - ? strings('count.words.thousand', {words: num}) - : strings('count.words', {words: num})); - - return strings('count.words.withUnit.' + strings.intl.plural.cardinal.select(value), {words}); - }, - - albums: countHelper('albums'), - commentaryEntries: countHelper('commentaryEntries', 'entries'), - contributions: countHelper('contributions'), - coverArts: countHelper('coverArts'), - timesReferenced: countHelper('timesReferenced'), - timesUsed: countHelper('timesUsed'), - tracks: countHelper('tracks') -}; - -const listHelper = type => (list, {strings}) => strings.intl.list[type].format(list); - -const list = { - unit: listHelper('unit'), - or: listHelper('disjunction'), - and: listHelper('conjunction') -}; - -// Note there isn't a 'find track data files' function. I plan on including the -// data for all tracks within an al8um collected in the single metadata file -// for that al8um. Otherwise there'll just 8e way too many files, and I'd also -// have to worry a8out linking track files to al8um files (which would contain -// only the track listing, not track data itself), and dealing with errors of -// missing track files (or track files which are not linked to al8ums). All a -// 8unch of stuff that's a pain to deal with for no apparent 8enefit. -async function findFiles(dataPath, filter = f => true) { - return (await readdir(dataPath)) - .map(file => path.join(dataPath, file)) - .filter(file => filter(file)); -} - -function* getSections(lines) { - // ::::) - const isSeparatorLine = line => /^-{8,}$/.test(line); - yield* splitArray(lines, isSeparatorLine); -} - -function getBasicField(lines, name) { - const line = lines.find(line => line.startsWith(name + ':')); - return line && line.slice(name.length + 1).trim(); -} - -function getBooleanField(lines, name) { - // The ?? oper8tor (which is just, hilariously named, lol) can 8e used to - // specify a default! - const value = getBasicField(lines, name); - switch (value) { - case 'yes': - case 'true': - return true; - case 'no': - case 'false': - return false; - default: - return null; - } -} - -function getListField(lines, name) { - let startIndex = lines.findIndex(line => line.startsWith(name + ':')); - // If callers want to default to an empty array, they should stick - // "|| []" after the call. - if (startIndex === -1) { - return null; - } - // We increment startIndex 8ecause we don't want to include the - // "heading" line (e.g. "URLs:") in the actual data. - startIndex++; - let endIndex = lines.findIndex((line, index) => index >= startIndex && !line.startsWith('- ')); - if (endIndex === -1) { - endIndex = lines.length; - } - if (endIndex === startIndex) { - // If there is no list that comes after the heading line, treat the - // heading line itself as the comma-separ8ted array value, using - // the 8asic field function to do that. (It's l8 and my 8rain is - // sleepy. Please excuse any unhelpful comments I may write, or may - // have already written, in this st8. Thanks!) - const value = getBasicField(lines, name); - return value && value.split(',').map(val => val.trim()); - } - const listLines = lines.slice(startIndex, endIndex); - return listLines.map(line => line.slice(2)); -}; - -function getContributionField(section, name) { - let contributors = getListField(section, name); - - if (!contributors) { - return null; - } - - if (contributors.length === 1 && contributors[0].startsWith('')) { - const arr = []; - arr.textContent = contributors[0]; - return arr; - } - - contributors = contributors.map(contrib => { - // 8asically, the format is "Who (What)", or just "Who". 8e sure to - // keep in mind that "what" doesn't necessarily have a value! - const match = contrib.match(/^(.*?)( \((.*)\))?$/); - if (!match) { - return contrib; - } - const who = match[1]; - const what = match[3] || null; - return {who, what}; - }); - - const badContributor = contributors.find(val => typeof val === 'string'); - if (badContributor) { - return {error: `An entry has an incorrectly formatted contributor, "${badContributor}".`}; - } - - if (contributors.length === 1 && contributors[0].who === 'none') { - return null; - } - - return contributors; -}; - -function getMultilineField(lines, name) { - // All this code is 8asically the same as the getListText - just with a - // different line prefix (four spaces instead of a dash and a space). - let startIndex = lines.findIndex(line => line.startsWith(name + ':')); - if (startIndex === -1) { - return null; - } - startIndex++; - let endIndex = lines.findIndex((line, index) => index >= startIndex && line.length && !line.startsWith(' ')); - if (endIndex === -1) { - endIndex = lines.length; - } - // If there aren't any content lines, don't return anything! - if (endIndex === startIndex) { - return null; - } - // We also join the lines instead of returning an array. - const listLines = lines.slice(startIndex, endIndex); - return listLines.map(line => line.slice(4)).join('\n'); -}; - -const replacerSpec = { - 'album': { - search: 'album', - link: 'album' - }, - 'album-commentary': { - search: 'album', - link: 'albumCommentary' - }, - 'artist': { - search: 'artist', - link: 'artist' - }, - 'artist-gallery': { - search: 'artist', - link: 'artistGallery' - }, - 'commentary-index': { - search: null, - link: 'commentaryIndex' - }, - 'date': { - search: null, - value: ref => new Date(ref), - html: (date, {strings}) => `` - }, - 'flash': { - search: 'flash', - link: 'flash', - transformName(name, search, offset, text) { - const nextCharacter = text[offset + search.length]; - const lastCharacter = name[name.length - 1]; - if ( - ![' ', '\n', '<'].includes(nextCharacter) && - lastCharacter === '.' - ) { - return name.slice(0, -1); - } else { - return name; - } - } - }, - 'group': { - search: 'group', - link: 'groupInfo' - }, - 'group-gallery': { - search: 'group', - link: 'groupGallery' - }, - 'listing-index': { - search: null, - link: 'listingIndex' - }, - 'listing': { - search: 'listing', - link: 'listing' - }, - 'media': { - search: null, - link: 'media' - }, - 'news-index': { - search: null, - link: 'newsIndex' - }, - 'news-entry': { - search: 'newsEntry', - link: 'newsEntry' - }, - 'root': { - search: null, - link: 'root' - }, - 'site': { - search: null, - link: 'site' - }, - 'static': { - search: 'staticPage', - link: 'staticPage' - }, - 'tag': { - search: 'tag', - link: 'tag' - }, - 'track': { - search: 'track', - link: 'track' - } -}; - -{ - let error = false; - for (const [key, {link: linkKey, search: searchKey, value, html}] of Object.entries(replacerSpec)) { - if (!html && !link[linkKey]) { - logError`The replacer spec ${key} has invalid link key ${linkKey}! Specify it in link specs or fix typo.`; - error = true; - } - if (searchKey && !search[searchKey]) { - logError`The replacer spec ${key} has invalid search key ${searchKey}! Specify it in search specs or fix typo.`; - error = true; - } - } - if (error) process.exit(); - - const categoryPart = Object.keys(replacerSpec).join('|'); - transformInline.regexp = new RegExp(String.raw`(? { - if (!category) { - category = 'track'; - } - - const { - search: searchKey, - link: linkKey, - value: valueFn, - html: htmlFn, - transformName - } = replacerSpec[category]; - - const value = ( - valueFn ? valueFn(ref) : - searchKey ? search[searchKey](ref) : - { - directory: ref.replace(category + ':', ''), - name: null - }); - - if (!value) { - logWarn`The link ${match} does not match anything!`; - return match; - } - - const label = (enteredName - || transformName && transformName(value.name, match, offset, text) - || value.name); - - if (!valueFn && !label) { - logWarn`The link ${match} requires a label be entered!`; - return match; - } - - const fn = (htmlFn - ? htmlFn - : strings.link[linkKey]); - - try { - return fn(value, {text: label, hash, strings, to}); - } catch (error) { - logError`The link ${match} failed to be processed: ${error}`; - return match; - } - }).replaceAll(String.raw`\[[`, '[['); -} - -function parseAttributes(string, {to}) { - const attributes = Object.create(null); - const skipWhitespace = i => { - const ws = /\s/; - if (ws.test(string[i])) { - const match = string.slice(i).match(/[^\s]/); - if (match) { - return i + match.index; - } else { - return string.length; - } - } else { - return i; - } - }; - - for (let i = 0; i < string.length;) { - i = skipWhitespace(i); - const aStart = i; - const aEnd = i + string.slice(i).match(/[\s=]|$/).index; - const attribute = string.slice(aStart, aEnd); - i = skipWhitespace(aEnd); - if (string[i] === '=') { - i = skipWhitespace(i + 1); - let end, endOffset; - if (string[i] === '"' || string[i] === "'") { - end = string[i]; - endOffset = 1; - i++; - } else { - end = '\\s'; - endOffset = 0; - } - const vStart = i; - const vEnd = i + string.slice(i).match(new RegExp(`${end}|$`)).index; - const value = string.slice(vStart, vEnd); - i = vEnd + endOffset; - if (attribute === 'src' && value.startsWith('media/')) { - attributes[attribute] = to('media.path', value.slice('media/'.length)); - } else { - attributes[attribute] = value; - } - } else { - attributes[attribute] = attribute; - } - } - return Object.fromEntries(Object.entries(attributes).map(([ key, val ]) => [ - key, - val === 'true' ? true : - val === 'false' ? false : - val === key ? true : - val - ])); -} - -function transformMultiline(text, {strings, to}) { - // Heck yes, HTML magics. - - text = transformInline(text.trim(), {strings, to}); - - const outLines = []; - - const indentString = ' '.repeat(4); - - let levelIndents = []; - const openLevel = indent => { - // opening a sublist is a pain: to be semantically *and* visually - // correct, we have to append the - `).join('\n')} - - `; - } - }, - - { - directory: 'tracks/by-times-referenced', - title: ({strings}) => strings('listingPage.listTracks.byTimesReferenced.title'), - - data() { - return trackData - .map(track => ({track, timesReferenced: track.referencedBy.length})) - .filter(({ timesReferenced }) => timesReferenced > 0) - .sort((a, b) => b.timesReferenced - a.timesReferenced); - }, - - row({track, timesReferenced}, {strings, to}) { - return strings('listingPage.listTracks.byTimesReferenced.item', { - track: strings.link.track(track, {to}), - timesReferenced: strings.count.timesReferenced(timesReferenced, {unit: true}) - }); - } - }, - - { - directory: 'tracks/in-flashes/by-album', - title: ({strings}) => strings('listingPage.listTracks.inFlashes.byAlbum.title'), - condition: () => wikiInfo.features.flashesAndGames, - - data() { - return chunkByProperties(trackData.filter(t => t.flashes.length > 0), ['album']) - .filter(({ album }) => album.directory !== C.UNRELEASED_TRACKS_DIRECTORY); - }, - - html(chunks, {strings, to}) { - return fixWS` -
- ${chunks.map(({album, chunk: tracks}) => fixWS` -
${strings('listingPage.listTracks.inFlashes.byAlbum.album', { - album: strings.link.album(album, {to}), - date: strings.count.date(album.date) - })}
-
    - ${(tracks - .map(track => strings('listingPage.listTracks.inFlashes.byAlbum.track', { - track: strings.link.track(track, {to}), - flashes: strings.list.and(track.flashes.map(flash => strings.link.flash(flash, {to}))) - })) - .map(row => `
  • ${row}
    • `) - .join('\n'))} -
- `).join('\n')} -
- `; - } - }, - - { - directory: 'tracks/in-flashes/by-flash', - title: ({strings}) => strings('listingPage.listTracks.inFlashes.byFlash.title'), - condition: () => wikiInfo.features.flashesAndGames, - - html({strings, to}) { - return fixWS` -
- ${C.sortByDate(flashData.slice()).map(flash => fixWS` -
${strings('listingPage.listTracks.inFlashes.byFlash.flash', { - flash: strings.link.flash(flash, {to}), - date: strings.count.date(flash.date) - })}
-
    - ${(flash.tracks - .map(track => strings('listingPage.listTracks.inFlashes.byFlash.track', { - track: strings.link.track(track, {to}), - album: strings.link.album(track.album, {to}) - })) - .map(row => `
  • ${row}
    • `) - .join('\n'))} -
- `).join('\n')} -
- `; - } - }, - - { - directory: 'tracks/with-lyrics', - title: ({strings}) => strings('listingPage.listTracks.withLyrics.title'), - - data() { - return chunkByProperties(trackData.filter(t => t.lyrics), ['album']); - }, - - html(chunks, {strings, to}) { - return fixWS` -
- ${chunks.map(({album, chunk: tracks}) => fixWS` -
${strings('listingPage.listTracks.withLyrics.album', { - album: strings.link.album(album, {to}), - date: strings.count.date(album.date) - })}
-
    - ${(tracks - .map(track => strings('listingPage.listTracks.withLyrics.track', { - track: strings.link.track(track, {to}), - })) - .map(row => `
  • ${row}
    • `) - .join('\n'))} -
- `).join('\n')} -
- `; - } - }, - - { - directory: 'tags/by-name', - title: ({strings}) => strings('listingPage.listTags.byName.title'), - condition: () => wikiInfo.features.artTagUI, - - data() { - return tagData - .filter(tag => !tag.isCW) - .sort(sortByName) - .map(tag => ({tag, timesUsed: tag.things.length})); - }, - - row({tag, timesUsed}, {strings, to}) { - return strings('listingPage.listTags.byName.item', { - tag: strings.link.tag(tag, {to}), - timesUsed: strings.count.timesUsed(timesUsed, {unit: true}) - }); - } - }, - - { - directory: 'tags/by-uses', - title: ({strings}) => strings('listingPage.listTags.byUses.title'), - condition: () => wikiInfo.features.artTagUI, - - data() { - return tagData - .filter(tag => !tag.isCW) - .map(tag => ({tag, timesUsed: tag.things.length})) - .sort((a, b) => b.timesUsed - a.timesUsed); - }, - - row({tag, timesUsed}, {strings, to}) { - return strings('listingPage.listTags.byUses.item', { - tag: strings.link.tag(tag, {to}), - timesUsed: strings.count.timesUsed(timesUsed, {unit: true}) - }); - } - }, - - { - directory: 'random', - title: ({strings}) => `Random Pages`, - html: ({strings, to}) => fixWS` -

Choose a link to go to a random page in that category or album! If your browser doesn't support relatively modern JavaScript or you've disabled it, these links won't work - sorry.

-

(Data files are downloading in the background! Please wait for data to load.)

-

(Data files have finished being downloaded. The links should work!)

-
-
Miscellaneous:
-
- ${[ - {name: 'Official', albumData: officialAlbumData, code: 'official'}, - {name: 'Fandom', albumData: fandomAlbumData, code: 'fandom'} - ].map(category => fixWS` -
${category.name}: (Random Album, Random Track)
-
    ${category.albumData.map(album => fixWS` -
  • ${album.name}
    • - `).join('\n')}
- `).join('\n')} -
- ` - } -]; - -function writeListingPages() { - if (!wikiInfo.features.listings) { - return; - } - - return [ - writeListingIndex(), - ...listingSpec.map(writeListingPage).filter(Boolean) - ]; -} - -function writeListingIndex() { - const releasedTracks = trackData.filter(track => track.album.directory !== C.UNRELEASED_TRACKS_DIRECTORY); - const releasedAlbums = albumData.filter(album => album.directory !== C.UNRELEASED_TRACKS_DIRECTORY); - const duration = getTotalDuration(releasedTracks); - - return ({strings, writePage}) => writePage('listingIndex', '', ({to}) => ({ - title: strings('listingIndex.title'), - - main: { - content: fixWS` -

${strings('listingIndex.title')}

-

${strings('listingIndex.infoLine', { - wiki: wikiInfo.name, - tracks: `${strings.count.tracks(releasedTracks.length, {unit: true})}`, - albums: `${strings.count.albums(releasedAlbums.length, {unit: true})}`, - duration: `${strings.count.duration(duration, {approximate: true, unit: true})}` - })}

-
-

${strings('listingIndex.exploreList')}

- ${generateLinkIndexForListings(null, {strings, to})} - ` - }, - - sidebarLeft: { - content: generateSidebarForListings(null, {strings, to}) - }, - - nav: {simple: true} - })) -} - -function writeListingPage(listing) { - if (listing.condition && !listing.condition()) { - return null; - } - - const data = (listing.data - ? listing.data() - : null); - - return ({strings, writePage}) => writePage('listing', listing.directory, ({to}) => ({ - title: listing.title({strings}), - - main: { - content: fixWS` -

${listing.title({strings})}

- ${listing.html && (listing.data - ? listing.html(data, {strings, to}) - : listing.html({strings, to}))} - ${listing.row && fixWS` - - `} - ` - }, - - sidebarLeft: { - content: generateSidebarForListings(listing, {strings, to}) - }, - - nav: { - links: [ - { - href: to('localized.home'), - title: wikiInfo.shortName - }, - { - href: to('localized.listingIndex'), - title: strings('listingIndex.title') - }, - { - href: '', - title: listing.title({strings}) - } - ] - } - })); -} - -function generateSidebarForListings(currentListing, {strings, to}) { - return fixWS` -

${strings.link.listingIndex('', {text: strings('listingIndex.title'), to})}

- ${generateLinkIndexForListings(currentListing, {strings, to})} - `; -} - -function generateLinkIndexForListings(currentListing, {strings, to}) { - return fixWS` - - `; -} - -function filterAlbumsByCommentary() { - return albumData.filter(album => [album, ...album.tracks].some(x => x.commentary)); -} - -function writeCommentaryPages() { - if (!filterAlbumsByCommentary().length) { - return; - } - - return [ - writeCommentaryIndex(), - ...filterAlbumsByCommentary().map(writeAlbumCommentaryPage) - ]; -} - -function writeCommentaryIndex() { - const data = filterAlbumsByCommentary() - .map(album => ({ - album, - entries: [album, ...album.tracks].filter(x => x.commentary).map(x => x.commentary) - })) - .map(({ album, entries }) => ({ - album, entries, - words: entries.join(' ').split(' ').length - })); - - const totalEntries = data.reduce((acc, {entries}) => acc + entries.length, 0); - const totalWords = data.reduce((acc, {words}) => acc + words, 0); - - return ({strings, writePage}) => writePage('commentaryIndex', '', ({to}) => ({ - title: strings('commentaryIndex.title'), - - main: { - content: fixWS` -
-

${strings('commentaryIndex.title')}

-

${strings('commentaryIndex.infoLine', { - words: `${strings.count.words(totalWords, {unit: true})}`, - entries: `${strings.count.commentaryEntries(totalEntries, {unit: true})}` - })}

-

${strings('commentaryIndex.albumList.title')}

- -
- ` - }, - - nav: {simple: true} - })); -} - -function writeAlbumCommentaryPage(album) { - const entries = [album, ...album.tracks].filter(x => x.commentary).map(x => x.commentary); - const words = entries.join(' ').split(' ').length; - - return ({strings, writePage}) => writePage('albumCommentary', album.directory, ({to}) => ({ - title: strings('albumCommentaryPage.title', {album: album.name}), - stylesheet: getAlbumStylesheet(album, {to}), - theme: getThemeString(album), - - main: { - content: fixWS` -
-

${strings('albumCommentaryPage.title', { - album: strings.link.album(album, {to}) - })}

-

${strings('albumCommentaryPage.infoLine', { - words: `${strings.count.words(words, {unit: true})}`, - entries: `${strings.count.commentaryEntries(entries.length, {unit: true})}` - })}

- ${album.commentary && fixWS` -

${strings('albumCommentaryPage.entry.title.albumCommentary')}

-
- ${transformMultiline(album.commentary, {strings, to})} -
- `} - ${album.tracks.filter(t => t.commentary).map(track => fixWS` -

${strings('albumCommentaryPage.entry.title.trackCommentary', { - track: strings.link.track(track, {to}) - })}

-
- ${transformMultiline(track.commentary, {strings, to})} -
- `).join('\n')} -
- ` - }, - - nav: { - links: [ - { - href: to('localized.home'), - title: wikiInfo.shortName - }, - { - href: to('localized.commentaryIndex'), - title: strings('commentaryIndex.title') - }, - { - html: strings('albumCommentaryPage.nav.album', { - album: strings.link.albumCommentary(album, {class: 'current', to}) - }) - } - ] - } - })); -} - -function writeTagPages() { - if (!wikiInfo.features.artTagUI) { - return; - } - - return tagData.filter(tag => !tag.isCW).map(writeTagPage); -} - -function writeTagPage(tag) { - const { things } = tag; - - return ({strings, writePage}) => writePage('tag', tag.directory, ({to}) => ({ - title: strings('tagPage.title', {tag: tag.name}), - theme: getThemeString(tag), - - main: { - classes: ['top-index'], - content: fixWS` -

${strings('tagPage.title', {tag: tag.name})}

-

${strings('tagPage.infoLine', { - coverArts: strings.count.coverArts(things.length, {unit: true}) - })}

-
- ${getGridHTML({ - strings, to, - entries: things.map(item => ({item})), - srcFn: thing => (thing.album - ? getTrackCover(thing, {to}) - : getAlbumCover(thing, {to})), - hrefFn: thing => (thing.album - ? to('localized.track', thing.directory) - : to('localized.album', thing.directory)) - })} -
- ` - }, - - nav: { - links: [ - { - href: to('localized.home'), - title: wikiInfo.shortName - }, - wikiInfo.features.listings && - { - href: to('localized.listingIndex'), - title: strings('listingIndex.title') - }, - { - html: strings('tagPage.nav.tag', { - tag: strings.link.tag(tag, {class: 'current', to}) - }) - } - ] - } - })); -} - -function getArtistString(artists, {strings, to, showIcons = false, showContrib = false}) { - return strings.list.and(artists.map(({ who, what }) => { - const { urls, directory, name } = who; - return [ - strings.link.artist(who, {to}), - showContrib && what && `(${what})`, - showIcons && urls.length && `(${ - strings.list.unit(urls.map(url => iconifyURL(url, {strings, to}))) - })` - ].filter(Boolean).join(' '); - })); -} - -function getLinkThemeString(thing) { - const { primary, dim } = C.getColors(thing.color || wikiInfo.color); - return `--primary-color: ${primary}; --dim-color: ${dim}`; -} - -function getThemeString(thing, additionalVariables = []) { - const { primary, dim } = C.getColors(thing.color || wikiInfo.color); - - const variables = [ - `--primary-color: ${primary}`, - `--dim-color: ${dim}`, - ...additionalVariables - ].filter(Boolean); - - return fixWS` - ${variables.length && fixWS` - :root { - ${variables.map(line => line + ';').join('\n')} - } - `} - `; -} - -function getFlashDirectory(flash) { - // const kebab = getKebabCase(flash.name.replace('[S] ', '')); - // return flash.page + (kebab ? '-' + kebab : ''); - // return '' + flash.page; - return '' + flash.directory; -} - -function getTagDirectory({name}) { - return C.getKebabCase(name); -} - -function getAlbumListTag(album) { - if (album.directory === C.UNRELEASED_TRACKS_DIRECTORY) { - return 'ul'; - } else { - return 'ol'; - } -} - -function fancifyURL(url, {strings, album = false} = {}) { - const domain = new URL(url).hostname; - return fixWS`${ - domain.includes('bandcamp.com') ? strings('misc.external.bandcamp') : - [ - 'music.solatrux.com' - ].includes(domain) ? strings('misc.external.bandcamp.domain', {domain}) : - [ - 'types.pl' - ].includes(domain) ? strings('misc.external.mastodon.domain', {domain}) : - domain.includes('youtu') ? (album - ? (url.includes('list=') - ? strings('misc.external.youtube.playlist') - : strings('misc.external.youtube.fullAlbum')) - : strings('misc.external.youtube')) : - domain.includes('soundcloud') ? strings('misc.external.soundcloud') : - domain.includes('tumblr.com') ? strings('misc.external.tumblr') : - domain.includes('twitter.com') ? strings('misc.external.twitter') : - domain.includes('deviantart.com') ? strings('misc.external.deviantart') : - domain.includes('wikipedia.org') ? strings('misc.external.wikipedia') : - domain.includes('poetryfoundation.org') ? strings('misc.external.poetryFoundation') : - domain.includes('instagram.com') ? strings('misc.external.instagram') : - domain.includes('patreon.com') ? strings('misc.external.patreon') : - domain - }`; -} - -function fancifyFlashURL(url, flash, {strings}) { - const link = fancifyURL(url, {strings}); - return `${ - url.includes('homestuck.com') ? (isNaN(Number(flash.page)) - ? strings('misc.external.flash.homestuck.secret', {link}) - : strings('misc.external.flash.homestuck.page', {link, page: flash.page})) : - url.includes('bgreco.net') ? strings('misc.external.flash.bgreco', {link}) : - url.includes('youtu') ? strings('misc.external.flash.youtube', {link}) : - link - }`; -} - -function iconifyURL(url, {strings, to}) { - const domain = new URL(url).hostname; - const [ id, msg ] = ( - domain.includes('bandcamp.com') ? ['bandcamp', strings('misc.external.bandcamp')] : - ( - domain.includes('music.solatrus.com') - ) ? ['bandcamp', strings('misc.external.bandcamp.domain', {domain})] : - ( - domain.includes('types.pl') - ) ? ['mastodon', strings('misc.external.mastodon.domain', {domain})] : - domain.includes('youtu') ? ['youtube', strings('misc.external.youtube')] : - domain.includes('soundcloud') ? ['soundcloud', strings('misc.external.soundcloud')] : - domain.includes('tumblr.com') ? ['tumblr', strings('misc.external.tumblr')] : - domain.includes('twitter.com') ? ['twitter', strings('misc.external.twitter')] : - domain.includes('deviantart.com') ? ['deviantart', strings('misc.external.deviantart')] : - domain.includes('instagram.com') ? ['instagram', strings('misc.external.bandcamp')] : - ['globe', strings('misc.external.domain', {domain})] - ); - return fixWS`${msg}`; -} - -function chronologyLinks(currentThing, { - strings, to, - headingString, - contribKey, - getThings -}) { - const contributions = currentThing[contribKey]; - if (!contributions) { - return ''; - } - - if (contributions.length > 8) { - return `
${strings('misc.chronology.seeArtistPages')}
`; - } - - return contributions.map(({ who: artist }) => { - const things = C.sortByDate(unique(getThings(artist))); - const releasedThings = things.filter(thing => { - const album = albumData.includes(thing) ? thing : thing.album; - return !(album && album.directory === C.UNRELEASED_TRACKS_DIRECTORY); - }); - const index = releasedThings.indexOf(currentThing); - - if (index === -1) return ''; - - // TODO: This can pro8a8ly 8e made to use generatePreviousNextLinks? - // We'd need to make generatePreviousNextLinks use toAnythingMan tho. - const previous = releasedThings[index - 1]; - const next = releasedThings[index + 1]; - const parts = [ - previous && `Previous`, - next && `Next` - ].filter(Boolean); - - const stringOpts = { - index: strings.count.index(index + 1, {strings}), - artist: strings.link.artist(artist, {to}) - }; - - return fixWS` -
- ${strings(headingString, stringOpts)} - ${parts.length && `(${parts.join(', ')})`} -
- `; - }).filter(Boolean).join('\n'); -} - -function generateAlbumNavLinks(album, currentTrack, {strings, to}) { - if (album.tracks.length <= 1) { - return ''; - } - - const previousNextLinks = currentTrack && generatePreviousNextLinks('localized.track', currentTrack, album.tracks, {strings, to}) - const randomLink = `${ - (currentTrack - ? strings('trackPage.nav.random') - : strings('albumPage.nav.randomTrack')) - }`; - - return (previousNextLinks - ? `(${previousNextLinks}, ${randomLink})` - : `(${randomLink})`); -} - -function generateAlbumChronologyLinks(album, currentTrack, {strings, to}) { - return [ - currentTrack && chronologyLinks(currentTrack, { - strings, to, - headingString: 'misc.chronology.heading.track', - contribKey: 'artists', - getThings: artist => [...artist.tracks.asArtist, ...artist.tracks.asContributor] - }), - chronologyLinks(currentTrack || album, { - strings, to, - headingString: 'misc.chronology.heading.coverArt', - contribKey: 'coverArtists', - getThings: artist => [...artist.albums.asCoverArtist, ...artist.tracks.asCoverArtist] - }) - ].filter(Boolean).join('\n'); -} - -function generateSidebarForAlbum(album, currentTrack, {strings, to}) { - const listTag = getAlbumListTag(album); - - const trackToListItem = track => `
  • ${ - strings('albumSidebar.trackList.item', { - track: strings.link.track(track, {to}) - }) - }
    • `; - - const trackListPart = fixWS` -

    ${album.name}

    - ${album.trackGroups ? fixWS` -
    - ${album.trackGroups.map(({ name, color, startIndex, tracks }) => fixWS` -
    ${ - (listTag === 'ol' - ? strings('albumSidebar.trackList.group.withRange', { - group: strings.link.track(tracks[0], {to, text: name}), - range: `${startIndex + 1}–${startIndex + tracks.length}` - }) - : strings('albumSidebar.trackList.group', { - group: strings.link.track(tracks[0], {to, text: name}) - })) - }
    - ${(!currentTrack || tracks.includes(currentTrack)) && fixWS` -
    <${listTag === 'ol' ? `ol start="${startIndex + 1}"` : listTag}> - ${tracks.map(trackToListItem).join('\n')} -
    - `} - `).join('\n')} -
    - ` : fixWS` - <${listTag}> - ${album.tracks.map(trackToListItem).join('\n')} - - `} - `; - - const { groups } = album; - - const groupParts = groups.map(group => { - const index = group.albums.indexOf(album); - const next = group.albums[index + 1]; - const previous = group.albums[index - 1]; - return {group, next, previous}; - }).map(({group, next, previous}) => fixWS` -

    ${ - strings('albumSidebar.groupBox.title', { - group: `${group.name}` - }) - }

    - ${!currentTrack && transformMultiline(group.descriptionShort, {strings, to})} - ${group.urls.length && `

    ${ - strings('releaseInfo.visitOn', { - links: strings.list.or(group.urls.map(url => fancifyURL(url, {strings}))) - }) - }

    `} - ${!currentTrack && fixWS` - ${next && `

    ${ - strings('albumSidebar.groupBox.next', { - album: `${next.name}` - }) - }

    `} - ${previous && `

    ${ - strings('albumSidebar.groupBox.previous', { - album: `${previous.name}` - }) - }

    `} - `} - `); - - if (groupParts.length) { - if (currentTrack) { - const combinedGroupPart = groupParts.join('\n
    \n'); - return { - multiple: [ - trackListPart, - combinedGroupPart - ] - }; - } else { - return { - multiple: [ - ...groupParts, - trackListPart - ] - }; - } - } else { - return { - content: trackListPart - }; - } -} - -function generateSidebarForGroup(currentGroup, {strings, to, isGallery}) { - if (!wikiInfo.features.groupUI) { - return null; - } - - const urlKey = isGallery ? 'localized.groupGallery' : 'localized.groupInfo'; - - return { - content: fixWS` -

    ${strings('groupSidebar.title')}

    -
    - ${groupCategoryData.map(category => [ - fixWS` -
    ${ - strings('groupSidebar.groupList.category', { - category: `${category.name}` - }) - }
    -
      - ${category.groups.map(group => fixWS` -
    • ${ - strings('groupSidebar.groupList.item', { - group: `${group.name}` - }) - }
      • - `).join('\n')} -
    - ` - ]).join('\n')} -
    - ` - }; -} - -function generateInfoGalleryLinks(urlKeyInfo, urlKeyGallery, currentThing, isGallery, {strings, to}) { - return [ - strings.link[urlKeyInfo](currentThing, { - to, - class: isGallery ? '' : 'current', - text: strings('misc.nav.info') - }), - strings.link[urlKeyGallery](currentThing, { - to, - class: isGallery ? 'current' : '', - text: strings('misc.nav.gallery') - }) - ].join(', '); -} - -function generatePreviousNextLinks(urlKey, currentThing, thingData, {strings, to}) { - const index = thingData.indexOf(currentThing); - const previous = thingData[index - 1]; - const next = thingData[index + 1]; - - return [ - previous && `${strings('misc.nav.previous')}`, - next && `${strings('misc.nav.next')}` - ].filter(Boolean).join(', '); -} - -function generateNavForGroup(currentGroup, {strings, to, isGallery}) { - if (!wikiInfo.features.groupUI) { - return {simple: true}; - } - - const urlKey = isGallery ? 'localized.groupGallery' : 'localized.groupInfo'; - const linkKey = isGallery ? 'groupGallery' : 'groupInfo'; - - const infoGalleryLinks = generateInfoGalleryLinks('groupInfo', 'groupGallery', currentGroup, isGallery, {strings, to}); - const previousNextLinks = generatePreviousNextLinks(urlKey, currentGroup, groupData, {strings, to}) - - return { - links: [ - { - href: to('localized.home'), - title: wikiInfo.shortName - }, - wikiInfo.features.listings && - { - href: to('localized.listingIndex'), - title: strings('listingIndex.title') - }, - { - html: strings('groupPage.nav.group', { - group: strings.link[linkKey](currentGroup, {class: 'current', to}) - }) - }, - { - divider: false, - html: (previousNextLinks - ? `(${infoGalleryLinks}; ${previousNextLinks})` - : `(${previousNextLinks})`) - } - ] - }; -} - -function writeGroupPages() { - return groupData.map(writeGroupPage); -} - -function writeGroupPage(group) { - const releasedAlbums = group.albums.filter(album => album.directory !== C.UNRELEASED_TRACKS_DIRECTORY); - const releasedTracks = releasedAlbums.flatMap(album => album.tracks); - const totalDuration = getTotalDuration(releasedTracks); - - return async ({strings, writePage}) => { - await writePage('groupInfo', group.directory, ({to}) => ({ - title: strings('groupInfoPage.title', {group: group.name}), - theme: getThemeString(group), - - main: { - content: fixWS` -

    ${strings('groupInfoPage.title', {group: group.name})}

    - ${group.urls.length && `

    ${ - strings('releaseInfo.visitOn', { - links: strings.list.or(group.urls.map(url => fancifyURL(url, {strings}))) - }) - }

    `} -
    - ${transformMultiline(group.description, {strings, to})} -
    -

    ${strings('groupInfoPage.albumList.title')}

    -

    ${ - strings('groupInfoPage.viewAlbumGallery', { - link: `${ - strings('groupInfoPage.viewAlbumGallery.link') - }` - }) - }

    - - ` - }, - - sidebarLeft: generateSidebarForGroup(group, {strings, to, isGallery: false}), - nav: generateNavForGroup(group, {strings, to, isGallery: false}) - })); - - await writePage('groupGallery', group.directory, ({to}) => ({ - title: strings('groupGalleryPage.title', {group: group.name}), - theme: getThemeString(group), - - main: { - classes: ['top-index'], - content: fixWS` -

    ${strings('groupGalleryPage.title', {group: group.name})}

    -

    ${ - strings('groupGalleryPage.infoLine', { - tracks: `${strings.count.tracks(releasedTracks.length, {unit: true})}`, - albums: `${strings.count.albums(releasedAlbums.length, {unit: true})}`, - time: `${strings.count.duration(totalDuration, {unit: true})}` - }) - }

    - ${wikiInfo.features.groupUI && wikiInfo.features.listings && `

    (Choose another group to filter by!)

    `} -
    - ${getAlbumGridHTML({ - strings, to, - entries: C.sortByDate(group.albums.map(item => ({item}))).reverse(), - details: true - })} -
    - ` - }, - - sidebarLeft: generateSidebarForGroup(group, {strings, to, isGallery: true}), - nav: generateNavForGroup(group, {strings, to, isGallery: true}) - })); - }; -} - -function toAnythingMan(anythingMan, to) { - return ( - albumData.includes(anythingMan) ? to('localized.album', anythingMan.directory) : - trackData.includes(anythingMan) ? to('localized.track', anythingMan.directory) : - flashData?.includes(anythingMan) ? to('localized.flash', anythingMan.directory) : - 'idk-bud' - ) -} - -function getAlbumCover(album, {to}) { - return to('media.albumCover', album.directory); -} - -function getTrackCover(track, {to}) { - // Some al8ums don't have any track art at all, and in those, every track - // just inherits the al8um's own cover art. - if (track.coverArtists === null) { - return getAlbumCover(track.album, {to}); - } else { - return to('media.trackCover', track.album.directory, track.directory); - } -} - -function getFlashLink(flash) { - return `https://homestuck.com/story/${flash.page}`; -} - -function classes(...args) { - const values = args.filter(Boolean); - return `class="${values.join(' ')}"`; -} - -async function processLanguageFile(file, defaultStrings = null) { - let contents; - try { - contents = await readFile(file, 'utf-8'); - } catch (error) { - return {error: `Could not read ${file} (${error.code}).`}; - } - - let json; - try { - json = JSON.parse(contents); - } catch (error) { - return {error: `Could not parse JSON from ${file} (${error}).`}; - } - - return genStrings(json, defaultStrings); -} - -// Wrapper function for running a function once for all languages. It provides: -// * the language strings -// * a shadowing writePages function for outputing to the appropriate subdir -// * a shadowing urls object for linking to the appropriate relative paths -async function wrapLanguages(fn, writeOneLanguage = null) { - const k = writeOneLanguage - const languagesToRun = (k - ? {[k]: languages[k]} - : languages) - - const entries = Object.entries(languagesToRun) - .filter(([ key ]) => key !== 'default'); - - for (let i = 0; i < entries.length; i++) { - const [ key, strings ] = entries[i]; - - const baseDirectory = (strings === languages.default ? '' : strings.code); - - const shadow_writePage = (urlKey, directory, pageFn) => writePage(strings, baseDirectory, urlKey, directory, pageFn); - - // 8ring the utility functions over too! - Object.assign(shadow_writePage, writePage); - - await fn({ - baseDirectory, - strings, - writePage: shadow_writePage - }, i, entries); - } -} - -async function main() { - const miscOptions = await parseOptions(process.argv.slice(2), { - // Data files for the site, including flash, artist, and al8um data, - // and like a jillion other things too. Pretty much everything which - // makes an individual wiki what it is goes here! - 'data-path': { - type: 'value' - }, - - // Static media will 8e referenced in the site here! The contents are - // categorized; check out MEDIA_DIRECTORY and rel8ted constants in - // common/common.js. (This gets symlinked into the --data directory.) - 'media-path': { - type: 'value' - }, - - // String files! For the most part, this is used for translating the - // site to different languages, though you can also customize strings - // for your own 8uild of the site if you'd like. Files here should all - // match the format in strings-default.json in this repository. (If a - // language file is missing any strings, the site code will fall 8ack - // to what's specified in strings-default.json.) - // - // Unlike the other options here, this one's optional - the site will - // 8uild with the default (English) strings if this path is left - // unspecified. - 'lang-path': { - type: 'value' - }, - - // This is the output directory. It's the one you'll upload online with - // rsync or whatever when you're pushing an upd8, and also the one - // you'd archive if you wanted to make a 8ackup of the whole dang - // site. Just keep in mind that the gener8ted result will contain a - // couple symlinked directories, so if you're uploading, you're pro8a8ly - // gonna want to resolve those yourself. - 'out-path': { - type: 'value' - }, - - // Thum8nail gener8tion is *usually* something you want, 8ut it can 8e - // kinda a pain to run every time, since it does necessit8te reading - // every media file at run time. Pass this to skip it. - 'skip-thumbs': { - type: 'flag' - }, - - // Only want 8uild one language during testing? This can chop down - // 8uild times a pretty 8ig chunk! Just pass a single language code. - 'lang': { - type: 'value' - }, - - 'queue-size': { - type: 'value', - validate(size) { - if (parseInt(size) !== parseFloat(size)) return 'an integer'; - if (parseInt(size) < 0) return 'a counting number or zero'; - return true; - } - }, - queue: {alias: 'queue-size'}, - - [parseOptions.handleUnknown]: () => {} - }); - - dataPath = miscOptions['data-path'] || process.env.HSMUSIC_DATA; - mediaPath = miscOptions['media-path'] || process.env.HSMUSIC_MEDIA; - langPath = miscOptions['lang-path'] || process.env.HSMUSIC_LANG; // Can 8e left unset! - outputPath = miscOptions['out-path'] || process.env.HSMUSIC_OUT; - - const writeOneLanguage = miscOptions['lang']; - - { - let errored = false; - const error = (cond, msg) => { - if (cond) { - console.error(`\x1b[31;1m${msg}\x1b[0m`); - errored = true; - } - }; - error(!dataPath, `Expected --data option or HSMUSIC_DATA to be set`); - error(!mediaPath, `Expected --media option or HSMUSIC_MEDIA to be set`); - error(!outputPath, `Expected --out option or HSMUSIC_OUT to be set`); - if (errored) { - return; - } - } - - const skipThumbs = miscOptions['skip-thumbs'] ?? false; - - if (skipThumbs) { - logInfo`Skipping thumbnail generation.`; - } else { - logInfo`Begin thumbnail generation... -----+`; - const result = await genThumbs(mediaPath, {queueSize, quiet: true}); - logInfo`Done thumbnail generation! --------+`; - if (!result) { - return; - } - } - - const defaultStrings = await processLanguageFile(path.join(__dirname, DEFAULT_STRINGS_FILE)); - if (defaultStrings.error) { - logError`Error loading default strings: ${defaultStrings.error}`; - return; - } - - if (langPath) { - const languageDataFiles = await findFiles(langPath, f => path.extname(f) === '.json'); - const results = await progressPromiseAll(`Reading & processing language files.`, languageDataFiles - .map(file => processLanguageFile(file, defaultStrings.json))); - - let error = false; - for (const strings of results) { - if (strings.error) { - logError`Error loading provided strings: ${strings.error}`; - error = true; - } - } - if (error) return; - - languages = Object.fromEntries(results.map(strings => [strings.code, strings])); - } else { - languages = {}; - } - - if (!languages[defaultStrings.code]) { - languages[defaultStrings.code] = defaultStrings; - } - - logInfo`Loaded language strings: ${Object.keys(languages).join(', ')}`; - - if (writeOneLanguage && !(writeOneLanguage in languages)) { - logError`Specified to write only ${writeOneLanguage}, but there is no strings file with this language code!`; - return; - } else if (writeOneLanguage) { - logInfo`Writing only language ${writeOneLanguage} this run.`; - } else { - logInfo`Writing all languages.`; - } - - wikiInfo = await processWikiInfoFile(path.join(dataPath, WIKI_INFO_FILE)); - if (wikiInfo.error) { - console.log(`\x1b[31;1m${wikiInfo.error}\x1b[0m`); - return; - } - - // Update languages o8ject with the wiki-specified default language! - // This will make page files for that language 8e gener8ted at the root - // directory, instead of the language-specific su8directory. - if (wikiInfo.defaultLanguage) { - if (Object.keys(languages).includes(wikiInfo.defaultLanguage)) { - languages.default = languages[wikiInfo.defaultLanguage]; - } else { - logError`Wiki info file specified default language is ${wikiInfo.defaultLanguage}, but no such language file exists!`; - if (langPath) { - logError`Check if an appropriate file exists in ${langPath}?`; - } else { - logError`Be sure to specify ${'--lang'} or ${'HSMUSIC_LANG'} with the path to language files.`; - } - return; - } - } else { - languages.default = defaultStrings; - } - - homepageInfo = await processHomepageInfoFile(path.join(dataPath, HOMEPAGE_INFO_FILE)); - - if (homepageInfo.error) { - console.log(`\x1b[31;1m${homepageInfo.error}\x1b[0m`); - return; - } - - { - const errors = homepageInfo.rows.filter(obj => obj.error); - if (errors.length) { - for (const error of errors) { - console.log(`\x1b[31;1m${error.error}\x1b[0m`); - } - return; - } - } - - // 8ut wait, you might say, how do we know which al8um these data files - // correspond to???????? You wouldn't dare suggest we parse the actual - // paths returned 8y this function, which ought to 8e of effectively - // unknown format except for their purpose as reada8le data files!? - // To that, I would say, yeah, you're right. Thanks a 8unch, my projection - // of "you". We're going to read these files later, and contained within - // will 8e the actual directory names that the data correspond to. Yes, - // that's redundant in some ways - we COULD just return the directory name - // in addition to the data path, and duplicating that name within the file - // itself suggests we 8e careful to avoid mismatching it - 8ut doing it - // this way lets the data files themselves 8e more porta8le (meaning we - // could store them all in one folder, if we wanted, and this program would - // still output to the correct al8um directories), and also does make the - // function's signature simpler (an array of strings, rather than some kind - // of structure containing 8oth data file paths and output directories). - // This is o8jectively a good thing, 8ecause it means the function can stay - // truer to its name, and have a narrower purpose: it doesn't need to - // concern itself with where we *output* files, or whatever other reasons - // we might (hypothetically) have for knowing the containing directory. - // And, in the strange case where we DO really need to know that info, we - // callers CAN use path.dirname to find out that data. 8ut we'll 8e - // avoiding that in our code 8ecause, again, we want to avoid assuming the - // format of the returned paths here - they're only meant to 8e used for - // reading as-is. - const albumDataFiles = await findFiles(path.join(dataPath, C.DATA_ALBUM_DIRECTORY)); - - // Technically, we could do the data file reading and output writing at the - // same time, 8ut that kinda makes the code messy, so I'm not 8othering - // with it. - albumData = await progressPromiseAll(`Reading & processing album files.`, albumDataFiles.map(processAlbumDataFile)); - - { - const errors = albumData.filter(obj => obj.error); - if (errors.length) { - for (const error of errors) { - console.log(`\x1b[31;1m${error.error}\x1b[0m`); - } - return; - } - } - - C.sortByDate(albumData); - - artistData = await processArtistDataFile(path.join(dataPath, ARTIST_DATA_FILE)); - if (artistData.error) { - console.log(`\x1b[31;1m${artistData.error}\x1b[0m`); - return; - } - - { - const errors = artistData.filter(obj => obj.error); - if (errors.length) { - for (const error of errors) { - console.log(`\x1b[31;1m${error.error}\x1b[0m`); - } - return; - } - } - - artistAliasData = artistData.filter(x => x.alias); - artistData = artistData.filter(x => !x.alias); - - trackData = C.getAllTracks(albumData); - - if (wikiInfo.features.flashesAndGames) { - flashData = await processFlashDataFile(path.join(dataPath, FLASH_DATA_FILE)); - if (flashData.error) { - console.log(`\x1b[31;1m${flashData.error}\x1b[0m`); - return; - } - - const errors = flashData.filter(obj => obj.error); - if (errors.length) { - for (const error of errors) { - console.log(`\x1b[31;1m${error.error}\x1b[0m`); - } - return; - } - } - - flashActData = flashData?.filter(x => x.act8r8k); - flashData = flashData?.filter(x => !x.act8r8k); - - artistNames = Array.from(new Set([ - ...artistData.filter(artist => !artist.alias).map(artist => artist.name), - ...[ - ...albumData.flatMap(album => [ - ...album.artists || [], - ...album.coverArtists || [], - ...album.wallpaperArtists || [], - ...album.tracks.flatMap(track => [ - ...track.artists, - ...track.coverArtists || [], - ...track.contributors || [] - ]) - ]), - ...(flashData?.flatMap(flash => [ - ...flash.contributors || [] - ]) || []) - ].map(contribution => contribution.who) - ])); - - tagData = await processTagDataFile(path.join(dataPath, TAG_DATA_FILE)); - if (tagData.error) { - console.log(`\x1b[31;1m${tagData.error}\x1b[0m`); - return; - } - - { - const errors = tagData.filter(obj => obj.error); - if (errors.length) { - for (const error of errors) { - console.log(`\x1b[31;1m${error.error}\x1b[0m`); - } - return; - } - } - - groupData = await processGroupDataFile(path.join(dataPath, GROUP_DATA_FILE)); - if (groupData.error) { - console.log(`\x1b[31;1m${groupData.error}\x1b[0m`); - return; - } - - { - const errors = groupData.filter(obj => obj.error); - if (errors.length) { - for (const error of errors) { - console.log(`\x1b[31;1m${error.error}\x1b[0m`); - } - return; - } - } - - groupCategoryData = groupData.filter(x => x.isCategory); - groupData = groupData.filter(x => x.isGroup); - - staticPageData = await processStaticPageDataFile(path.join(dataPath, STATIC_PAGE_DATA_FILE)); - if (staticPageData.error) { - console.log(`\x1b[31;1m${staticPageData.error}\x1b[0m`); - return; - } - - { - const errors = staticPageData.filter(obj => obj.error); - if (errors.length) { - for (const error of errors) { - console.log(`\x1b[31;1m${error.error}\x1b[0m`); - } - return; - } - } - - if (wikiInfo.features.news) { - newsData = await processNewsDataFile(path.join(dataPath, NEWS_DATA_FILE)); - if (newsData.error) { - console.log(`\x1b[31;1m${newsData.error}\x1b[0m`); - return; - } - - const errors = newsData.filter(obj => obj.error); - if (errors.length) { - for (const error of errors) { - console.log(`\x1b[31;1m${error.error}\x1b[0m`); - } - return; - } - - C.sortByDate(newsData); - newsData.reverse(); - } - - { - const tagNames = new Set([...trackData, ...albumData].flatMap(thing => thing.artTags)); - - for (let { name, isCW } of tagData) { - if (isCW) { - name = 'cw: ' + name; - } - tagNames.delete(name); - } - - if (tagNames.size) { - for (const name of Array.from(tagNames).sort()) { - console.log(`\x1b[33;1m- Missing tag: "${name}"\x1b[0m`); - } - return; - } - } - - artistNames.sort((a, b) => a.toLowerCase() < b.toLowerCase() ? -1 : a.toLowerCase() > b.toLowerCase() ? 1 : 0); - - justEverythingMan = C.sortByDate([...albumData, ...trackData, ...(flashData || [])]); - justEverythingSortedByArtDateMan = C.sortByArtDate(justEverythingMan.slice()); - // console.log(JSON.stringify(justEverythingSortedByArtDateMan.map(toAnythingMan), null, 2)); - - { - let buffer = []; - const clearBuffer = function() { - if (buffer.length) { - for (const entry of buffer.slice(0, -1)) { - console.log(`\x1b[2m... ${entry.name} ...\x1b[0m`); - } - const lastEntry = buffer[buffer.length - 1]; - console.log(`\x1b[2m... \x1b[0m${lastEntry.name}\x1b[0;2m ...\x1b[0m`); - buffer = []; - } - }; - const showWhere = (name, color) => { - const where = justEverythingMan.filter(thing => [ - ...thing.coverArtists || [], - ...thing.contributors || [], - ...thing.artists || [] - ].some(({ who }) => who === name)); - for (const thing of where) { - console.log(`\x1b[${color}m- ` + (thing.album ? `(\x1b[1m${thing.album.name}\x1b[0;${color}m)` : '') + ` \x1b[1m${thing.name}\x1b[0m`); - } - }; - let CR4SH = false; - for (let name of artistNames) { - const entry = [...artistData, ...artistAliasData].find(entry => entry.name === name || entry.name.toLowerCase() === name.toLowerCase()); - if (!entry) { - clearBuffer(); - console.log(`\x1b[31mMissing entry for artist "\x1b[1m${name}\x1b[0;31m"\x1b[0m`); - showWhere(name, 31); - CR4SH = true; - } else if (entry.alias) { - console.log(`\x1b[33mArtist "\x1b[1m${name}\x1b[0;33m" should be named "\x1b[1m${entry.alias}\x1b[0;33m"\x1b[0m`); - showWhere(name, 33); - CR4SH = true; - } else if (entry.name !== name) { - console.log(`\x1b[33mArtist "\x1b[1m${name}\x1b[0;33m" should be named "\x1b[1m${entry.name}\x1b[0;33m"\x1b[0m`); - showWhere(name, 33); - CR4SH = true; - } else { - buffer.push(entry); - if (buffer.length > 3) { - buffer.shift(); - } - } - } - if (CR4SH) { - return; - } - } - - { - const directories = []; - for (const { directory, name } of albumData) { - if (directories.includes(directory)) { - console.log(`\x1b[31;1mDuplicate album directory "${directory}" (${name})\x1b[0m`); - return; - } - directories.push(directory); - } - } - - { - const directories = []; - const where = {}; - for (const { directory, album } of trackData) { - if (directories.includes(directory)) { - console.log(`\x1b[31;1mDuplicate track directory "${directory}"\x1b[0m`); - console.log(`Shows up in:`); - console.log(`- ${album.name}`); - console.log(`- ${where[directory].name}`); - return; - } - directories.push(directory); - where[directory] = album; - } - } - - { - const artists = []; - const artistsLC = []; - for (const name of artistNames) { - if (!artists.includes(name) && artistsLC.includes(name.toLowerCase())) { - const other = artists.find(oth => oth.toLowerCase() === name.toLowerCase()); - console.log(`\x1b[31;1mMiscapitalized artist name: ${name}, ${other}\x1b[0m`); - return; - } - artists.push(name); - artistsLC.push(name.toLowerCase()); - } - } - - { - for (const { references, name, album } of trackData) { - for (const ref of references) { - if (!search.track(ref)) { - logWarn`Track not found "${ref}" in ${name} (${album.name})`; - } - } - } - } - - contributionData = Array.from(new Set([ - ...trackData.flatMap(track => [...track.artists || [], ...track.contributors || [], ...track.coverArtists || []]), - ...albumData.flatMap(album => [...album.artists || [], ...album.coverArtists || [], ...album.wallpaperArtists || [], ...album.bannerArtists || []]), - ...(flashData?.flatMap(flash => [...flash.contributors || []]) || []) - ])); - - // Now that we have all the data, resolve references all 8efore actually - // gener8ting any of the pages, 8ecause page gener8tion is going to involve - // accessing these references a lot, and there's no reason to resolve them - // more than once. (We 8uild a few additional links that can't 8e cre8ted - // at initial data processing time here too.) - - const filterNullArray = (parent, key) => { - for (const obj of parent) { - const array = obj[key]; - for (let i = 0; i < array.length; i++) { - if (!array[i]) { - const prev = array[i - 1] && array[i - 1].name; - const next = array[i + 1] && array[i + 1].name; - logWarn`Unexpected null in ${obj.name} (${obj.what}) (array key ${key} - prev: ${prev}, next: ${next})`; - } - } - array.splice(0, array.length, ...array.filter(Boolean)); - } - }; - - const filterNullValue = (parent, key) => { - parent.splice(0, parent.length, ...parent.filter(obj => { - if (!obj[key]) { - logWarn`Unexpected null in ${obj.name} (value key ${key})`; - } - })); - }; - - trackData.forEach(track => mapInPlace(track.references, search.track)); - trackData.forEach(track => track.aka = search.track(track.aka)); - trackData.forEach(track => mapInPlace(track.artTags, search.tag)); - albumData.forEach(album => mapInPlace(album.groups, search.group)); - albumData.forEach(album => mapInPlace(album.artTags, search.tag)); - artistAliasData.forEach(artist => artist.alias = search.artist(artist.alias)); - contributionData.forEach(contrib => contrib.who = search.artist(contrib.who)); - - filterNullArray(trackData, 'references'); - filterNullArray(trackData, 'artTags'); - filterNullArray(albumData, 'groups'); - filterNullArray(albumData, 'artTags'); - filterNullValue(artistAliasData, 'alias'); - filterNullValue(contributionData, 'who'); - - trackData.forEach(track1 => track1.referencedBy = trackData.filter(track2 => track2.references.includes(track1))); - groupData.forEach(group => group.albums = albumData.filter(album => album.groups.includes(group))); - tagData.forEach(tag => tag.things = C.sortByArtDate([...albumData, ...trackData]).filter(thing => thing.artTags.includes(tag))); - - groupData.forEach(group => group.category = groupCategoryData.find(x => x.name === group.category)); - groupCategoryData.forEach(category => category.groups = groupData.filter(x => x.category === category)); - - trackData.forEach(track => track.otherReleases = [ - track.aka, - ...trackData.filter(({ aka }) => aka === track || (track.aka && aka === track.aka)), - ].filter(x => x && x !== track)); - - if (wikiInfo.features.flashesAndGames) { - flashData.forEach(flash => mapInPlace(flash.tracks, search.track)); - flashData.forEach(flash => flash.act = flashActData.find(act => act.name === flash.act)); - flashActData.forEach(act => act.flashes = flashData.filter(flash => flash.act === act)); - - filterNullArray(flashData, 'tracks'); - - trackData.forEach(track => track.flashes = flashData.filter(flash => flash.tracks.includes(track))); - } - - artistData.forEach(artist => { - const filterProp = (array, prop) => array.filter(thing => thing[prop]?.some(({ who }) => who === artist)); - const filterCommentary = array => array.filter(thing => thing.commentary && thing.commentary.replace(/<\/?b>/g, '').includes('' + artist.name + ':')); - artist.tracks = { - asArtist: filterProp(trackData, 'artists'), - asCommentator: filterCommentary(trackData), - asContributor: filterProp(trackData, 'contributors'), - asCoverArtist: filterProp(trackData, 'coverArtists'), - asAny: trackData.filter(track => ( - [...track.artists, ...track.contributors, ...track.coverArtists || []].some(({ who }) => who === artist) - )) - }; - artist.albums = { - asArtist: filterProp(albumData, 'artists'), - asCommentator: filterCommentary(albumData), - asCoverArtist: filterProp(albumData, 'coverArtists'), - asWallpaperArtist: filterProp(albumData, 'wallpaperArtists'), - asBannerArtist: filterProp(albumData, 'bannerArtists') - }; - if (wikiInfo.features.flashesAndGames) { - artist.flashes = { - asContributor: filterProp(flashData, 'contributors') - }; - } - }); - - officialAlbumData = albumData.filter(album => album.groups.some(group => group.directory === C.OFFICIAL_GROUP_DIRECTORY)); - fandomAlbumData = albumData.filter(album => album.groups.every(group => group.directory !== C.OFFICIAL_GROUP_DIRECTORY)); - - // Makes writing a little nicer on CPU theoretically, 8ut also costs in - // performance right now 'cuz it'll w8 for file writes to 8e completed - // 8efore moving on to more data processing. So, defaults to zero, which - // disa8les the queue feature altogether. - queueSize = +(miscOptions['queue-size'] ?? 0); - - // NOT for ena8ling or disa8ling specific features of the site! - // This is only in charge of what general groups of files to 8uild. - // They're here to make development quicker when you're only working - // on some particular area(s) of the site rather than making changes - // across all of them. - const writeFlags = await parseOptions(process.argv.slice(2), { - all: {type: 'flag'}, // Defaults to true if none 8elow specified. - - album: {type: 'flag'}, - artist: {type: 'flag'}, - commentary: {type: 'flag'}, - flash: {type: 'flag'}, - group: {type: 'flag'}, - list: {type: 'flag'}, - misc: {type: 'flag'}, - news: {type: 'flag'}, - static: {type: 'flag'}, - tag: {type: 'flag'}, - track: {type: 'flag'}, - - [parseOptions.handleUnknown]: () => {} - }); - - const writeAll = !Object.keys(writeFlags).length || writeFlags.all; - - logInfo`Writing site pages: ${writeAll ? 'all' : Object.keys(writeFlags).join(', ')}`; - - await writeSymlinks(); - await writeSharedFilesAndPages({strings: defaultStrings}); - - const buildDictionary = { - misc: writeMiscellaneousPages, - news: writeNewsPages, - list: writeListingPages, - tag: writeTagPages, - commentary: writeCommentaryPages, - static: writeStaticPages, - group: writeGroupPages, - album: writeAlbumPages, - track: writeTrackPages, - artist: writeArtistPages, - flash: writeFlashPages - }; - - const buildSteps = (writeAll - ? Object.values(buildDictionary) - : (Object.entries(buildDictionary) - .filter(([ flag ]) => writeFlags[flag]) - .map(([ flag, fn ]) => fn))); - - // *NB: While what's 8elow is 8asically still true in principle, the - // format is QUITE DIFFERENT than what's descri8ed here! There - // will 8e actual document8tion on like, what the return format - // looks like soon, once we implement a 8unch of other pages and - // are certain what they actually, uh, will look like, in the end.* - // - // The writeThingPages functions don't actually immediately do any file - // writing themselves; an initial call will only gather the relevant data - // which is *then* used for writing. So the return value is a function - // (or an array of functions) which expects {writePage, strings}, and - // *that's* what we call after -- multiple times, once for each language. - let writes; - { - let error = false; - - writes = buildSteps.flatMap(fn => { - const fns = fn() || []; - - // Do a quick valid8tion! If one of the writeThingPages functions go - // wrong, this will stall out early and tell us which did. - if (!Array.isArray(fns)) { - logError`${fn.name} didn't return an array!`; - error = true; - } else if (fns.every(entry => Array.isArray(entry))) { - if (!( - fns.every(entry => entry.every(obj => typeof obj === 'object')) && - fns.every(entry => entry.every(obj => { - const result = validateWriteObject(obj); - if (result.error) { - logError`Validating write object failed: ${result.error}`; - return false; - } else { - return true; - } - })) - )) { - logError`${fn.name} uses updated format, but entries are invalid!`; - error = true; - } - - return fns.flatMap(writes => writes); - } else if (fns.some(fn => typeof fn !== 'function')) { - logError`${fn.name} didn't return all functions or all arrays!`; - error = true; - } - - return fns; - }); - - if (error) { - return; - } - - // The modern(TM) return format for each writeThingPages function is an - // array of arrays, each of which's items are 8ig Complicated Objects - // that 8asically look like {type, path, content}. 8ut surprise, these - // aren't actually implemented in most places yet! So, we transform - // stuff in the old format here. 'Scept keep in mind, the OLD FORMAT - // doesn't really give us most of the info we want for Cool And Modern - // Reasons, so they're going into a fancy {type: 'legacy'} sort of - // o8ject, with a plain {write} property for, uh, the writing stuff, - // same as usual. - // - // I promise this document8tion will get 8etter when we make progress - // actually moving old pages over. Also it'll 8e hecks of less work - // than previous restructures, don't worry. - writes = writes.map(entry => - typeof entry === 'object' ? entry : - typeof entry === 'function' ? {type: 'legacy', write: entry} : - {type: 'wut', entry}); - - const wut = writes.filter(({ type }) => type === 'wut'); - if (wut.length) { - // Oh g*d oh h*ck. - logError`Uhhhhh writes contains something 8esides o8jects and functions?`; - logError`Definitely a 8ug!`; - console.log(wut); - return; - } - } - - const localizedWrites = writes.filter(({ type }) => type === 'page' || type === 'legacy'); - const dataWrites = writes.filter(({ type }) => type === 'data'); - - await progressPromiseAll(`Writing data files shared across languages.`, queue( - // TODO: This only supports one <>-style argument. - dataWrites.map(({path, data}) => () => writeData(path[0], path[1], data())), - queueSize - )); - - await wrapLanguages(async ({strings, ...opts}, i, entries) => { - console.log(`\x1b[34;1m${ - (`[${i + 1}/${entries.length}] ${strings.code} (-> /${opts.baseDirectory}) ` - .padEnd(60, '-')) - }\x1b[0m`); - await progressPromiseAll(`Writing ${strings.code}`, queue( - localizedWrites.map(({type, ...props}) => () => { - switch (type) { - case 'legacy': { - const { write } = props; - return write({strings, ...opts}); - } - case 'page': { - const { path, page } = props; - // TODO: This only supports one <>-style argument. - return opts.writePage(path[0], path[1], ({to}) => page({strings, to})); - } - } - }), - queueSize - )); - }, writeOneLanguage); - - decorateTime.displayTime(); - - // The single most important step. - logInfo`Written!`; -} - -main().catch(error => console.error(error)); diff --git a/upd8/gen-thumbs.js b/upd8/gen-thumbs.js new file mode 100644 index 00000000..bec01d1d --- /dev/null +++ b/upd8/gen-thumbs.js @@ -0,0 +1,323 @@ +#!/usr/bin/env node + +// Ok, so the d8te is 3 March 2021, and the music wiki was initially released +// on 15 November 2019. That is 474 days or 11376 hours. In my opinion, and +// pro8a8ly the opinions of at least one other person, that is WAY TOO LONG +// to go without media thum8nails!!!! So that's what this file is here to do. +// +// This program takes a path to the media folder (via --media or the environ. +// varia8le HSMUSIC_MEDIA), traverses su8directories to locate image files, +// and gener8tes lower-resolution/file-size versions of all that are new or +// have 8een modified since the last run. We use a JSON-format cache of MD5s +// for each file to perform this comparision; we gener8te files (using ffmpeg) +// in "medium" and "small" sizes adjacent to the existing PNG for easy and +// versatile access in site gener8tion code. +// +// So for example, on the very first run, you might have a media folder which +// looks something like this: +// +// media/ +// album-art/ +// one-year-older/ +// cover.jpg +// firefly-cloud.jpg +// october.jpg +// ... +// flash-art/ +// 413.jpg +// ... +// bg.jpg +// ... +// +// After running gen-thumbs.js with the path to that folder passed, you'd end +// up with something like this: +// +// media/ +// album-art/ +// one-year-older/ +// cover.jpg +// cover.medium.jpg +// cover.small.jpg +// firefly-cloud.jpg +// firefly-cloud.medium.jpg +// firefly-cloud.small.jpg +// october.jpg +// october.medium.jpg +// october.small.jpg +// ... +// flash-art/ +// 413.jpg +// 413.medium.jpg +// 413.small.jpg +// ... +// bg.jpg +// bg.medium.jpg +// bg.small.jpg +// thumbs-cache.json +// ... +// +// (Do note that while 8oth JPG and PNG are supported, gener8ted files will +// always 8e in JPG format and file extension. GIFs are skipped since there +// aren't any super gr8 ways to make those more efficient!) +// +// And then in gener8tion code, you'd reference the medium/small or original +// version of each file, as decided is appropriate. Here are some guidelines: +// +// - Small: Grid tiles on the homepage and in galleries. +// - Medium: Cover art on individual al8um and track pages, etc. +// - Original: Only linked to, not embedded. +// +// The traversal code is indiscrimin8te: there are no special cases to, say, +// not gener8te thum8nails for the bg.jpg file (since those would generally go +// unused). This is just to make the code more porta8le and sta8le, long-term, +// since it avoids a lot of otherwise implic8ted maintenance. + +'use strict'; + +const CACHE_FILE = 'thumbnail-cache.json'; +const WARNING_DELAY_TIME = 10000; + +const { spawn } = require('child_process'); +const crypto = require('crypto'); +const fsp = require('fs/promises'); // Whatcha know! Nice. +const fs = require('fs'); // Still gotta include 8oth tho, for createReadStream. +const path = require('path'); + +const { + delay, + logError, + logInfo, + logWarn, + parseOptions, + progressPromiseAll, + promisifyProcess, + queue, +} = require('./util'); + +function traverse(startDirPath, { + filterFile = () => true, + filterDir = () => true +} = {}) { + const recursive = (names, subDirPath) => Promise + .all(names.map(name => fsp.readdir(path.join(startDirPath, subDirPath, name)).then( + names => filterDir(name) ? recursive(names, path.join(subDirPath, name)) : [], + err => filterFile(name) ? [path.join(subDirPath, name)] : []))) + .then(pathArrays => pathArrays.flatMap(x => x)); + + return fsp.readdir(startDirPath) + .then(names => recursive(names, '')); +} + +function readFileMD5(filePath) { + return new Promise((resolve, reject) => { + const md5 = crypto.createHash('md5'); + const stream = fs.createReadStream(filePath); + stream.on('data', data => md5.update(data)); + stream.on('end', data => resolve(md5.digest('hex'))); + stream.on('error', err => reject(err)); + }); +} + +function generateImageThumbnails(filePath) { + const dirname = path.dirname(filePath); + const extname = path.extname(filePath); + const basename = path.basename(filePath, extname); + const output = name => path.join(dirname, basename + name + '.jpg'); + + const convert = (name, {size, quality}) => spawn('convert', [ + '-strip', + '-resize', `${size}x${size}>`, + '-interlace', 'Plane', + '-quality', `${quality}%`, + filePath, + output(name) + ]); + + return Promise.all([ + promisifyProcess(convert('.medium', {size: 400, quality: 95}), false), + promisifyProcess(convert('.small', {size: 250, quality: 85}), false) + ]); + + return new Promise((resolve, reject) => { + if (Math.random() < 0.2) { + reject(new Error(`Them's the 8r8ks, kiddo!`)); + } else { + resolve(); + } + }); +} + +async function genThumbs(mediaPath, { + queueSize = 0, + quiet = false +} = {}) { + if (!mediaPath) { + throw new Error('Expected mediaPath to be passed'); + } + + const quietInfo = (quiet + ? () => null + : logInfo); + + const filterFile = name => { + // TODO: Why is this not working???????? + // thumbnail-cache.json is 8eing passed through, for some reason. + + const ext = path.extname(name); + if (ext !== '.jpg' && ext !== '.png') return false; + + const rest = path.basename(name, ext); + if (rest.endsWith('.medium') || rest.endsWith('.small')) return false; + + return true; + }; + + const filterDir = name => { + if (name === '.git') return false; + return true; + }; + + let cache, firstRun = false, failedReadingCache = false; + try { + cache = JSON.parse(await fsp.readFile(path.join(mediaPath, CACHE_FILE))); + quietInfo`Cache file successfully read.`; + } catch (error) { + cache = {}; + if (error.code === 'ENOENT') { + firstRun = true; + } else { + failedReadingCache = true; + logWarn`Malformed or unreadable cache file: ${error}`; + logWarn`You may want to cancel and investigate this!`; + logWarn`All-new thumbnails and cache will be generated for this run.`; + await delay(WARNING_DELAY_TIME); + } + } + + try { + await fsp.writeFile(path.join(mediaPath, CACHE_FILE), JSON.stringify(cache)); + quietInfo`Writing to cache file appears to be working.`; + } catch (error) { + logWarn`Test of cache file writing failed: ${error}`; + if (cache) { + logWarn`Cache read succeeded: Any newly written thumbs will be unnecessarily regenerated on the next run.`; + } else if (firstRun) { + logWarn`No cache found: All thumbs will be generated now, and will be unnecessarily regenerated next run.`; + } else { + logWarn`Cache read failed: All thumbs will be regenerated now, and will be unnecessarily regenerated again next run.`; + } + logWarn`You may want to cancel and investigate this!`; + await delay(WARNING_DELAY_TIME); + } + + const imagePaths = await traverse(mediaPath, {filterFile, filterDir}); + + const imageToMD5Entries = await progressPromiseAll(`Generating MD5s of image files`, queue( + imagePaths.map(imagePath => () => readFileMD5(path.join(mediaPath, imagePath)).then( + md5 => [imagePath, md5], + error => [imagePath, {error}] + )), + queueSize + )); + + { + let error = false; + for (const entry of imageToMD5Entries) { + if (entry[1].error) { + logError`Failed to read ${entry[0]}: ${entry[1].error}`; + error = true; + } + } + if (error) { + logError`Failed to read at least one image file!`; + logError`This implies a thumbnail probably won't be generatable.`; + logError`So, exiting early.`; + return false; + } else { + quietInfo`All image files successfully read.`; + } + } + + // Technically we could pro8a8ly mut8te the cache varia8le in-place? + // 8ut that seems kinda iffy. + const updatedCache = Object.assign({}, cache); + + const entriesToGenerate = imageToMD5Entries + .filter(([filePath, md5]) => md5 !== cache[filePath]); + + if (entriesToGenerate.length === 0) { + logInfo`All image thumbnails are already up-to-date - nice!`; + return true; + } + + const failed = []; + const succeeded = []; + const writeMessageFn = () => `Writing image thumbnails. [failed: ${failed.length}]`; + + // This is actually sort of a lie, 8ecause we aren't doing synchronicity. + // (We pass queueSize = 1 to queue().) 8ut we still use progressPromiseAll, + // 'cuz the progress indic8tor is very cool and good. + await progressPromiseAll(writeMessageFn, queue(entriesToGenerate.map(([filePath, md5]) => + () => generateImageThumbnails(path.join(mediaPath, filePath)).then( + () => { + updatedCache[filePath] = md5; + succeeded.push(filePath); + }, + error => { + failed.push([filePath, error]); + } + ) + ))); + + if (failed.length > 0) { + for (const [path, error] of failed) { + logError`Thumbnails failed to generate for ${path} - ${error}`; + } + logWarn`Result is incomplete - the above ${failed.length} thumbnails should be checked for errors.`; + logWarn`${succeeded.length} successfully generated images won't be regenerated next run, though!`; + } else { + logInfo`Generated all (updated) thumbnails successfully!`; + } + + try { + await fsp.writeFile(path.join(mediaPath, CACHE_FILE), JSON.stringify(updatedCache)); + quietInfo`Updated cache file successfully written!`; + } catch (error) { + logWarn`Failed to write updated cache file: ${error}`; + logWarn`Any newly (re)generated thumbnails will be regenerated next run.`; + logWarn`Sorry about that!`; + } + + return true; +}; + +module.exports = genThumbs; + +if (require.main === module) { + (async () => { + const miscOptions = await parseOptions(process.argv.slice(2), { + 'media': { + type: 'value' + }, + + 'queue-size': { + type: 'value', + validate(size) { + if (parseInt(size) !== parseFloat(size)) return 'an integer'; + if (parseInt(size) < 0) return 'a counting number or zero'; + return true; + } + }, + queue: {alias: 'queue-size'} + }); + + const mediaPath = miscOptions.media || process.env.HSMUSIC_MEDIA; + if (!mediaPath) { + logError`Expected --media option or HSMUSIC_MEDIA to be set`; + } + + const queueSize = +(miscOptions['queue-size'] ?? 0); + + await genThumbs(mediaPath, {queueSize}); + })().catch(err => console.error(err)); +} diff --git a/upd8/main.js b/upd8/main.js new file mode 100755 index 00000000..84bab6c1 --- /dev/null +++ b/upd8/main.js @@ -0,0 +1,6597 @@ +#!/usr/bin/env node + +// HEY N8RDS! +// +// This is one of the 8ACKEND FILES. It's not used anywhere on the actual site +// you are pro8a8ly using right now. +// +// Specifically, this one does all the actual work of the music wiki. The +// process looks something like this: +// +// 1. Crawl the music directories. Well, not so much "crawl" as "look inside +// the folders for each al8um, and read the metadata file descri8ing that +// al8um and the tracks within." +// +// 2. Read that metadata. I'm writing this 8efore actually doing any of the +// code, and I've gotta admit I have no idea what file format they're +// going to 8e in. May8e JSON, 8ut more likely some weird custom format +// which will 8e a lot easier to edit. +// +// 3. Generate the page files! They're just static index.html files, and are +// what gh-pages (or wherever this is hosted) will show to clients. +// Hopefully pretty minimalistic HTML, 8ut like, shrug. They'll reference +// CSS (and maaaaaaaay8e JS) files, hard-coded somewhere near the root. +// +// 4. Print an awesome message which says the process is done. This is the +// most important step. +// +// Oh yeah, like. Just run this through some relatively recent version of +// node.js and you'll 8e fine. ...Within the project root. O8viously. + +// HEY FUTURE ME!!!!!!!! Don't forget to implement artist pages! Those are, +// like, the coolest idea you've had yet, so DO NOT FORGET. (Remem8er, link +// from track listings, etc!) --- Thanks, past me. To futurerer me: an al8um +// listing page (a list of all the al8ums)! Make sure to sort these 8y date - +// we'll need a new field for al8ums. + +// ^^^^^^^^ DID THAT! 8ut also, artist images. Pro8a8ly stolen from the fandom +// wiki (I found half those images anywayz). + +// TRACK ART CREDITS. This is a must. + +// 2020-08-23 +// ATTENTION ALL 8*TCHES AND OTHER GENDER TRUCKERS: AS IT TURNS OUT, THIS CODE +// ****SUCKS****. I DON'T THINK ANYTHING WILL EVER REDEEM IT, 8UT THAT DOESN'T +// MEAN WE CAN'T TAKE SOME ACTION TO MAKE WRITING IT A LITTLE LESS TERRI8LE. +// We're gonna start defining STRUCTURES to make things suck less!!!!!!!! +// No classes 8ecause those are a huge pain and like, pro8a8ly 8ad performance +// or whatever -- just some standard structures that should 8e followed +// wherever reasona8le. Only one I need today is the contri8 one 8ut let's put +// any new general-purpose structures here too, ok? +// +// Contri8ution: {who, what, date, thing}. D8 and thing are the new fields. +// +// Use these wisely, which is to say all the time and instead of whatever +// terri8le new pseudo structure you're trying to invent!!!!!!!! +// +// Upd8 2021-01-03: Soooooooo we didn't actually really end up using these, +// lol? Well there's still only one anyway. Kinda ended up doing a 8ig refactor +// of all the o8ject structures today. It's not *especially* relevant 8ut feels +// worth mentioning? I'd get rid of this comment 8lock 8ut I like it too much! +// Even though I haven't actually reread it, lol. 8ut yeah, hopefully in the +// spirit of this "make things more consistent" attitude I 8rought up 8ack in +// August, stuff's lookin' 8etter than ever now. W00t! + +'use strict'; + +const fs = require('fs'); +const path = require('path'); +const util = require('util'); + +// I made this dependency myself! A long, long time ago. It is pro8a8ly my +// most useful li8rary ever. I'm not sure 8esides me actually uses it, though. +const fixWS = require('fix-whitespace'); +// Wait nevermind, I forgot a8out why-do-kids-love-the-taste-of-cinnamon-toast- +// crunch. THAT is my 8est li8rary. + +// The require function just returns whatever the module exports, so there's +// no reason you can't wrap it in some decorator right out of the 8ox. Which is +// exactly what we do here. +const mkdirp = util.promisify(require('mkdirp')); + +// It stands for "HTML Entities", apparently. Cursed. +const he = require('he'); + +// This is the dum8est name for a function possi8le. Like, SURE, fine, may8e +// the UNIX people had some valid reason to go with the weird truncated +// lowercased convention they did. 8ut Node didn't have to ALSO use that +// convention! Would it have 8een so hard to just name the function something +// like fs.readDirectory???????? No, it wouldn't have 8een. +const readdir = util.promisify(fs.readdir); +// 8ut okay, like, look at me. DOING THE SAME THING. See, *I* could have named +// my promisified function differently, and yet I did not. I literally cannot +// explain why. We are all used to following in the 8ad decisions of our +// ancestors, and never never never never never never never consider that hey, +// may8e we don't need to make the exact same decisions they did. Even when +// we're perfectly aware th8t's exactly what we're doing! Programmers, +// including me, are all pretty stupid. + +// 8ut I mean, come on. Look. Node decided to use readFile, instead of like, +// what, cat? Why couldn't they rename readdir too???????? As Johannes Kepler +// once so elegantly put it: "Shrug." +const readFile = util.promisify(fs.readFile); +const writeFile = util.promisify(fs.writeFile); +const access = util.promisify(fs.access); +const symlink = util.promisify(fs.symlink); +const unlink = util.promisify(fs.unlink); + +const { + cacheOneArg, + call, + chunkByConditions, + chunkByProperties, + curry, + decorateTime, + filterEmptyLines, + joinNoOxford, + mapInPlace, + logWarn, + logInfo, + logError, + parseOptions, + progressPromiseAll, + queue, + s, + sortByName, + splitArray, + th, + unique, + withEntries +} = require('./util'); + +const genThumbs = require('./gen-thumbs'); + +const C = require('../common/common'); + +const CACHEBUST = 5; + +const WIKI_INFO_FILE = 'wiki-info.txt'; +const HOMEPAGE_INFO_FILE = 'homepage.txt'; +const ARTIST_DATA_FILE = 'artists.txt'; +const FLASH_DATA_FILE = 'flashes.txt'; +const NEWS_DATA_FILE = 'news.txt'; +const TAG_DATA_FILE = 'tags.txt'; +const GROUP_DATA_FILE = 'groups.txt'; +const STATIC_PAGE_DATA_FILE = 'static-pages.txt'; +const DEFAULT_STRINGS_FILE = 'strings-default.json'; + +const CSS_FILE = 'site.css'; + +// Shared varia8les! These are more efficient to access than a shared varia8le +// (or at least I h8pe so), and are easier to pass across functions than a +// 8unch of specific arguments. +// +// Upd8: Okay yeah these aren't actually any different. Still cleaner than +// passing around a data object containing all this, though. +let dataPath; +let mediaPath; +let langPath; +let outputPath; + +let wikiInfo; +let homepageInfo; +let albumData; +let trackData; +let flashData; +let flashActData; +let newsData; +let tagData; +let groupData; +let groupCategoryData; +let staticPageData; + +let artistNames; +let artistData; +let artistAliasData; + +let officialAlbumData; +let fandomAlbumData; +let justEverythingMan; // tracks, albums, flashes -- don't forget to upd8 toAnythingMan! +let justEverythingSortedByArtDateMan; +let contributionData; + +let queueSize; + +let languages; + +const html = { + // Non-comprehensive. ::::P + selfClosingTags: ['br', 'img'], + + // Pass to tag() as an attri8utes key to make tag() return a 8lank string + // if the provided content is empty. Useful for when you'll only 8e showing + // an element according to the presence of content that would 8elong there. + onlyIfContent: Symbol(), + + tag(tagName, ...args) { + const selfClosing = html.selfClosingTags.includes(tagName); + + let openTag; + let content; + let attrs; + + if (typeof args[0] === 'object' && !Array.isArray(args[0])) { + attrs = args[0]; + content = args[1]; + } else { + content = args[0]; + } + + if (selfClosing && content) { + throw new Error(`Tag <${tagName}> is self-closing but got content!`); + } + + if (attrs?.[html.onlyIfContent] && !content) { + return ''; + } + + if (attrs) { + const attrString = html.attributes(args[0]); + if (attrString) { + openTag = `${tagName} ${attrString}`; + } + } + + if (!openTag) { + openTag = tagName; + } + + if (Array.isArray(content)) { + content = content.filter(Boolean).join('\n'); + } + + if (content) { + if (content.includes('\n')) { + return fixWS` + <${openTag}> + ${content} + + `; + } else { + return `<${openTag}>${content}`; + } + } else { + if (selfClosing) { + return `<${openTag}>`; + } else { + return `<${openTag}>`; + } + } + }, + + escapeAttributeValue(value) { + return value + .replaceAll('"', '"') + .replaceAll("'", '''); + }, + + attributes(attribs) { + return Object.entries(attribs) + .map(([ key, val ]) => { + if (!val) + return [key, val]; + else if (typeof val === 'string' || typeof val === 'boolean') + return [key, val]; + else if (typeof val === 'number') + return [key, val.toString()]; + else if (Array.isArray(val)) + return [key, val.join(' ')]; + else + throw new Error(`Attribute value for ${key} should be primitive or array, got ${typeof val}`); + }) + .filter(([ key, val ]) => val) + .map(([ key, val ]) => (typeof val === 'boolean' + ? `${key}` + : `${key}="${html.escapeAttributeValue(val)}"`)) + .join(' '); + } +}; + +const urlSpec = { + data: { + prefix: 'data/', + + paths: { + root: '', + path: '<>', + + album: 'album/<>', + artist: 'artist/<>', + track: 'track/<>' + } + }, + + localized: { + // TODO: Implement this. + // prefix: '_languageCode', + + paths: { + root: '', + path: '<>', + + home: '', + + album: 'album/<>/', + albumCommentary: 'commentary/album/<>/', + + artist: 'artist/<>/', + artistGallery: 'artist/<>/gallery/', + + commentaryIndex: 'commentary/', + + flashIndex: 'flash/', + flash: 'flash/<>/', + + groupInfo: 'group/<>/', + groupGallery: 'group/<>/gallery/', + + listingIndex: 'list/', + listing: 'list/<>/', + + newsIndex: 'news/', + newsEntry: 'news/<>/', + + staticPage: '<>/', + tag: 'tag/<>/', + track: 'track/<>/' + } + }, + + shared: { + paths: { + root: '', + path: '<>', + + commonFile: 'common/<>', + staticFile: 'static/<>' + } + }, + + media: { + prefix: 'media/', + + paths: { + root: '', + path: '<>', + + albumCover: 'album-art/<>/cover.jpg', + albumWallpaper: 'album-art/<>/bg.jpg', + albumBanner: 'album-art/<>/banner.jpg', + trackCover: 'album-art/<>/<>.jpg', + artistAvatar: 'artist-avatar/<>.jpg', + flashArt: 'flash-art/<>.jpg' + } + } +}; + +// This gets automatically switched in place when working from a baseDirectory, +// so it should never be referenced manually. +urlSpec.localizedWithBaseDirectory = { + paths: withEntries( + urlSpec.localized.paths, + entries => entries.map(([key, path]) => [key, '<>/' + path]) + ) +}; + +const linkHelper = (hrefFn, {color = true, attr = null} = {}) => + (thing, { + strings, to, + text = '', + class: className = '', + hash = '' + }) => ( + html.tag('a', { + ...attr ? attr(thing) : {}, + href: hrefFn(thing, {to}) + (hash ? (hash.startsWith('#') ? '' : '#') + hash : ''), + style: color ? getLinkThemeString(thing) : '', + class: className + }, text || thing.name) + ); + +const linkDirectory = (key, {expose = null, attr = null, ...conf} = {}) => + linkHelper((thing, {to}) => to('localized.' + key, thing.directory), { + attr: thing => ({ + ...attr ? attr(thing) : {}, + ...expose ? {[expose]: thing.directory} : {} + }), + ...conf + }); + +const linkPathname = (key, conf) => linkHelper(({directory: pathname}, {to}) => to(key, pathname), conf); +const linkIndex = (key, conf) => linkHelper((_, {to}) => to('localized.' + key), conf); + +const link = { + album: linkDirectory('album'), + albumCommentary: linkDirectory('albumCommentary'), + artist: linkDirectory('artist', {color: false}), + artistGallery: linkDirectory('artistGallery', {color: false}), + commentaryIndex: linkIndex('commentaryIndex', {color: false}), + flashIndex: linkIndex('flashIndex', {color: false}), + flash: linkDirectory('flash'), + groupInfo: linkDirectory('groupInfo'), + groupGallery: linkDirectory('groupGallery'), + home: linkIndex('home', {color: false}), + listingIndex: linkIndex('listingIndex'), + listing: linkDirectory('listing'), + newsIndex: linkIndex('newsIndex', {color: false}), + newsEntry: linkDirectory('newsEntry', {color: false}), + staticPage: linkDirectory('staticPage', {color: false}), + tag: linkDirectory('tag'), + track: linkDirectory('track', {expose: 'data-track'}), + + media: linkPathname('media.path', {color: false}), + root: linkPathname('shared.path', {color: false}), + data: linkPathname('data.path', {color: false}), + site: linkPathname('localized.path', {color: false}) +}; + +const thumbnailHelper = name => file => + file.replace(/\.(jpg|png)$/, name + '.jpg'); + +const thumb = { + medium: thumbnailHelper('.medium'), + small: thumbnailHelper('.small') +}; + +function generateURLs(fromPath) { + const getValueForFullKey = (obj, fullKey, prop = null) => { + const [ groupKey, subKey ] = fullKey.split('.'); + if (!groupKey || !subKey) { + throw new Error(`Expected group key and subkey (got ${fullKey})`); + } + + if (!obj.hasOwnProperty(groupKey)) { + throw new Error(`Expected valid group key (got ${groupKey})`); + } + + const group = obj[groupKey]; + + if (!group.hasOwnProperty(subKey)) { + throw new Error(`Expected valid subkey (got ${subKey} for group ${groupKey})`); + } + + return { + value: group[subKey], + group + }; + }; + + const generateTo = (fromPath, fromGroup) => { + const rebasePrefix = '../'.repeat((fromGroup.prefix || '').split('/').filter(Boolean).length); + + const pathHelper = (toPath, toGroup) => { + let target = toPath; + + let argIndex = 0; + target = target.replaceAll('<>', () => `<${argIndex++}>`); + + if (toGroup.prefix !== fromGroup.prefix) { + // TODO: Handle differing domains in prefixes. + target = rebasePrefix + (toGroup.prefix || '') + target; + } + + return (path.relative(fromPath, target) + + (toPath.endsWith('/') ? '/' : '')); + }; + + const groupSymbol = Symbol(); + + const groupHelper = urlGroup => ({ + [groupSymbol]: urlGroup, + ...withEntries(urlGroup.paths, entries => entries + .map(([key, path]) => [key, pathHelper(path, urlGroup)])) + }); + + const relative = withEntries(urlSpec, entries => entries + .map(([key, urlGroup]) => [key, groupHelper(urlGroup)])); + + const to = (key, ...args) => { + const { value: template, group: {[groupSymbol]: toGroup} } = getValueForFullKey(relative, key) + let result = template.replaceAll(/<([0-9]+)>/g, (match, n) => args[n]); + + // Kinda hacky lol, 8ut it works. + const missing = result.match(/<([0-9]+)>/g); + if (missing) { + throw new Error(`Expected ${missing[missing.length - 1]} arguments, got ${args.length}`); + } + + return result; + }; + + return {to, relative}; + }; + + const generateFrom = () => { + const map = withEntries(urlSpec, entries => entries + .map(([key, group]) => [key, withEntries(group.paths, entries => entries + .map(([key, path]) => [key, generateTo(path, group)]) + )])); + + const from = key => getValueForFullKey(map, key).value; + + return {from, map}; + }; + + return generateFrom(); +} + +const urls = generateURLs(); + +const searchHelper = (keys, dataFn, findFn) => ref => { + if (!ref) return null; + ref = ref.replace(new RegExp(`^(${keys.join('|')}):`), ''); + const found = findFn(ref, dataFn()); + if (!found) { + logWarn`Didn't match anything for ${ref}! (${keys.join(', ')})`; + } + return found; +}; + +const matchDirectory = (ref, data) => data.find(({ directory }) => directory === ref); + +const matchDirectoryOrName = (ref, data) => { + let thing; + + thing = matchDirectory(ref, data); + if (thing) return thing; + + thing = data.find(({ name }) => name === ref); + if (thing) return thing; + + thing = data.find(({ name }) => name.toLowerCase() === ref.toLowerCase()); + if (thing) { + logWarn`Bad capitalization: ${'\x1b[31m' + ref} -> ${'\x1b[32m' + thing.name}`; + return thing; + } + + return null; +}; + +const search = { + album: searchHelper(['album', 'album-commentary'], () => albumData, matchDirectoryOrName), + artist: searchHelper(['artist', 'artist-gallery'], () => artistData, matchDirectoryOrName), + flash: searchHelper(['flash'], () => flashData, matchDirectory), + group: searchHelper(['group', 'group-gallery'], () => groupData, matchDirectoryOrName), + listing: searchHelper(['listing'], () => listingSpec, matchDirectory), + newsEntry: searchHelper(['news-entry'], () => newsData, matchDirectory), + staticPage: searchHelper(['static'], () => staticPageData, matchDirectory), + tag: searchHelper(['tag'], () => tagData, (ref, data) => + matchDirectoryOrName(ref.startsWith('cw: ') ? ref.slice(4) : ref, data)), + track: searchHelper(['track'], () => trackData, matchDirectoryOrName) +}; + +// Localiz8tion time! Or l10n as the neeeeeeeerds call it. Which is a terri8le +// name and not one I intend on using, thank you very much. (Don't even get me +// started on """"a11y"""".) +// +// All the default strings are in strings-default.json, if you're curious what +// those actually look like. Pretty much it's "I like {ANIMAL}" for example. +// For each language, the o8ject gets turned into a single function of form +// f(key, {args}). It searches for a key in the o8ject and uses the string it +// finds (or the one in strings-default.json) as a templ8 evaluated with the +// arguments passed. (This function gets treated as an o8ject too; it gets +// the language code attached.) +// +// The function's also responsi8le for getting rid of dangerous characters +// (quotes and angle tags), though only within the templ8te (not the args), +// and it converts the keys of the arguments o8ject from camelCase to +// CONSTANT_CASE too. +function genStrings(stringsJSON, defaultJSON = null) { + // genStrings will only 8e called once for each language, and it happens + // right at the start of the program (or at least 8efore 8uilding pages). + // So, now's a good time to valid8te the strings and let any warnings be + // known. + + // May8e contrary to the argument name, the arguments should 8e o8jects, + // not actual JSON-formatted strings! + if (typeof stringsJSON !== 'object' || stringsJSON.constructor !== Object) { + return {error: `Expected an object (parsed JSON) for stringsJSON.`}; + } + if (typeof defaultJSON !== 'object') { // typeof null === object. I h8 JS. + return {error: `Expected an object (parsed JSON) or null for defaultJSON.`}; + } + + // All languages require a language code. + const code = stringsJSON['meta.languageCode']; + if (!code) { + return {error: `Missing language code.`}; + } + if (typeof code !== 'string') { + return {error: `Expected language code to be a string.`}; + } + + // Every value on the provided o8ject should be a string. + // (This is lazy, but we only 8other checking this on stringsJSON, on the + // assumption that defaultJSON was passed through this function too, and so + // has already been valid8ted.) + { + let err = false; + for (const [ key, value ] of Object.entries(stringsJSON)) { + if (typeof value !== 'string') { + logError`(${code}) The value for ${key} should be a string.`; + err = true; + } + } + if (err) { + return {error: `Expected all values to be a string.`}; + } + } + + // Checking is generally done against the default JSON, so we'll skip out + // if that isn't provided (which should only 8e the case when it itself is + // 8eing processed as the first loaded language). + if (defaultJSON) { + // Warn for keys that are missing or unexpected. + const expectedKeys = Object.keys(defaultJSON); + const presentKeys = Object.keys(stringsJSON); + for (const key of presentKeys) { + if (!expectedKeys.includes(key)) { + logWarn`(${code}) Unexpected translation key: ${key} - this won't be used!`; + } + } + for (const key of expectedKeys) { + if (!presentKeys.includes(key)) { + logWarn`(${code}) Missing translation key: ${key} - this won't be localized!`; + } + } + } + + // Valid8tion is complete, 8ut We can still do a little caching to make + // repeated actions faster. + + // We're gonna 8e mut8ting the strings dictionary o8ject from here on out. + // We make a copy so we don't mess with the one which was given to us. + stringsJSON = Object.assign({}, stringsJSON); + + // Preemptively pass everything through HTML encoding. This will prevent + // strings from embedding HTML tags or accidentally including characters + // that throw HTML parsers off. + for (const key of Object.keys(stringsJSON)) { + stringsJSON[key] = he.encode(stringsJSON[key], {useNamedReferences: true}); + } + + // It's time to cre8te the actual langauge function! + + // In the function, we don't actually distinguish 8etween the primary and + // default (fall8ack) strings - any relevant warnings have already 8een + // presented a8ove, at the time the language JSON is processed. Now we'll + // only 8e using them for indexing strings to use as templ8tes, and we can + // com8ine them for that. + const stringIndex = Object.assign({}, defaultJSON, stringsJSON); + + // We do still need the list of valid keys though. That's 8ased upon the + // default strings. (Or stringsJSON, 8ut only if the defaults aren't + // provided - which indic8tes that the single o8ject provided *is* the + // default.) + const validKeys = Object.keys(defaultJSON || stringsJSON); + + const invalidKeysFound = []; + + const strings = (key, args = {}) => { + // Ok, with the warning out of the way, it's time to get to work. + // First make sure we're even accessing a valid key. (If not, return + // an error string as su8stitute.) + if (!validKeys.includes(key)) { + // We only want to warn a8out a given key once. More than that is + // just redundant! + if (!invalidKeysFound.includes(key)) { + invalidKeysFound.push(key); + logError`(${code}) Accessing invalid key ${key}. Fix a typo or provide this in strings-default.json!`; + } + return `MISSING: ${key}`; + } + + const template = stringIndex[key]; + + // Convert the keys on the args dict from camelCase to CONSTANT_CASE. + // (This isn't an OUTRAGEOUSLY versatile algorithm for doing that, 8ut + // like, who cares, dude?) Also, this is an array, 8ecause it's handy + // for the iterating we're a8out to do. + const processedArgs = Object.entries(args) + .map(([ k, v ]) => [k.replace(/[A-Z]/g, '_$&').toUpperCase(), v]); + + // Replacement time! Woot. Reduce comes in handy here! + const output = processedArgs.reduce( + (x, [ k, v ]) => x.replaceAll(`{${k}}`, v), + template); + + // Post-processing: if any expected arguments *weren't* replaced, that + // is almost definitely an error. + if (output.match(/\{[A-Z_]+\}/)) { + logError`(${code}) Args in ${key} were missing - output: ${output}`; + } + + return output; + }; + + // And lastly, we add some utility stuff to the strings function. + + // Store the language code, for convenience of access. + strings.code = code; + + // Store the strings dictionary itself, also for convenience. + strings.json = stringsJSON; + + // Store Intl o8jects that can 8e reused for value formatting. + strings.intl = { + date: new Intl.DateTimeFormat(code, {full: true}), + number: new Intl.NumberFormat(code), + list: { + conjunction: new Intl.ListFormat(code, {type: 'conjunction'}), + disjunction: new Intl.ListFormat(code, {type: 'disjunction'}), + unit: new Intl.ListFormat(code, {type: 'unit'}) + }, + plural: { + cardinal: new Intl.PluralRules(code, {type: 'cardinal'}), + ordinal: new Intl.PluralRules(code, {type: 'ordinal'}) + } + }; + + const bindUtilities = (obj, bind) => Object.fromEntries(Object.entries(obj).map( + ([ key, fn ]) => [key, (value, opts = {}) => fn(value, {...bind, ...opts})] + )); + + // There are a 8unch of handy count functions which expect a strings value; + // for a more terse syntax, we'll stick 'em on the strings function itself, + // with automatic 8inding for the strings argument. + strings.count = bindUtilities(count, {strings}); + + // The link functions also expect the strings o8ject(*). May as well hand + // 'em over here too! Keep in mind they still expect {to} though, and that + // isn't something we have access to from this scope (so calls such as + // strings.link.album(...) still need to provide it themselves). + // + // (*) At time of writing, it isn't actually used for anything, 8ut future- + // proofing, ok???????? + strings.link = bindUtilities(link, {strings}); + + // List functions, too! + strings.list = bindUtilities(list, {strings}); + + return strings; +}; + +const countHelper = (stringKey, argName = stringKey) => (value, {strings, unit = false}) => strings( + (unit + ? `count.${stringKey}.withUnit.` + strings.intl.plural.cardinal.select(value) + : `count.${stringKey}`), + {[argName]: strings.intl.number.format(value)}); + +const count = { + date: (date, {strings}) => { + return strings.intl.date.format(date); + }, + + dateRange: ([startDate, endDate], {strings}) => { + return strings.intl.date.formatRange(startDate, endDate); + }, + + duration: (secTotal, {strings, approximate = false, unit = false}) => { + if (secTotal === 0) { + return strings('count.duration.missing'); + } + + const hour = Math.floor(secTotal / 3600); + const min = Math.floor((secTotal - hour * 3600) / 60); + const sec = Math.floor(secTotal - hour * 3600 - min * 60); + + const pad = val => val.toString().padStart(2, '0'); + + const stringSubkey = unit ? '.withUnit' : ''; + + const duration = (hour > 0 + ? strings('count.duration.hours' + stringSubkey, { + hours: hour, + minutes: pad(min), + seconds: pad(sec) + }) + : strings('count.duration.minutes' + stringSubkey, { + minutes: min, + seconds: pad(sec) + })); + + return (approximate + ? strings('count.duration.approximate', {duration}) + : duration); + }, + + index: (value, {strings}) => { + return strings('count.index.' + strings.intl.plural.ordinal.select(value), {index: value}); + }, + + number: value => strings.intl.number.format(value), + + words: (value, {strings, unit = false}) => { + const num = strings.intl.number.format(value > 1000 + ? Math.floor(value / 100) / 10 + : value); + + const words = (value > 1000 + ? strings('count.words.thousand', {words: num}) + : strings('count.words', {words: num})); + + return strings('count.words.withUnit.' + strings.intl.plural.cardinal.select(value), {words}); + }, + + albums: countHelper('albums'), + commentaryEntries: countHelper('commentaryEntries', 'entries'), + contributions: countHelper('contributions'), + coverArts: countHelper('coverArts'), + timesReferenced: countHelper('timesReferenced'), + timesUsed: countHelper('timesUsed'), + tracks: countHelper('tracks') +}; + +const listHelper = type => (list, {strings}) => strings.intl.list[type].format(list); + +const list = { + unit: listHelper('unit'), + or: listHelper('disjunction'), + and: listHelper('conjunction') +}; + +// Note there isn't a 'find track data files' function. I plan on including the +// data for all tracks within an al8um collected in the single metadata file +// for that al8um. Otherwise there'll just 8e way too many files, and I'd also +// have to worry a8out linking track files to al8um files (which would contain +// only the track listing, not track data itself), and dealing with errors of +// missing track files (or track files which are not linked to al8ums). All a +// 8unch of stuff that's a pain to deal with for no apparent 8enefit. +async function findFiles(dataPath, filter = f => true) { + return (await readdir(dataPath)) + .map(file => path.join(dataPath, file)) + .filter(file => filter(file)); +} + +function* getSections(lines) { + // ::::) + const isSeparatorLine = line => /^-{8,}$/.test(line); + yield* splitArray(lines, isSeparatorLine); +} + +function getBasicField(lines, name) { + const line = lines.find(line => line.startsWith(name + ':')); + return line && line.slice(name.length + 1).trim(); +} + +function getBooleanField(lines, name) { + // The ?? oper8tor (which is just, hilariously named, lol) can 8e used to + // specify a default! + const value = getBasicField(lines, name); + switch (value) { + case 'yes': + case 'true': + return true; + case 'no': + case 'false': + return false; + default: + return null; + } +} + +function getListField(lines, name) { + let startIndex = lines.findIndex(line => line.startsWith(name + ':')); + // If callers want to default to an empty array, they should stick + // "|| []" after the call. + if (startIndex === -1) { + return null; + } + // We increment startIndex 8ecause we don't want to include the + // "heading" line (e.g. "URLs:") in the actual data. + startIndex++; + let endIndex = lines.findIndex((line, index) => index >= startIndex && !line.startsWith('- ')); + if (endIndex === -1) { + endIndex = lines.length; + } + if (endIndex === startIndex) { + // If there is no list that comes after the heading line, treat the + // heading line itself as the comma-separ8ted array value, using + // the 8asic field function to do that. (It's l8 and my 8rain is + // sleepy. Please excuse any unhelpful comments I may write, or may + // have already written, in this st8. Thanks!) + const value = getBasicField(lines, name); + return value && value.split(',').map(val => val.trim()); + } + const listLines = lines.slice(startIndex, endIndex); + return listLines.map(line => line.slice(2)); +}; + +function getContributionField(section, name) { + let contributors = getListField(section, name); + + if (!contributors) { + return null; + } + + if (contributors.length === 1 && contributors[0].startsWith('')) { + const arr = []; + arr.textContent = contributors[0]; + return arr; + } + + contributors = contributors.map(contrib => { + // 8asically, the format is "Who (What)", or just "Who". 8e sure to + // keep in mind that "what" doesn't necessarily have a value! + const match = contrib.match(/^(.*?)( \((.*)\))?$/); + if (!match) { + return contrib; + } + const who = match[1]; + const what = match[3] || null; + return {who, what}; + }); + + const badContributor = contributors.find(val => typeof val === 'string'); + if (badContributor) { + return {error: `An entry has an incorrectly formatted contributor, "${badContributor}".`}; + } + + if (contributors.length === 1 && contributors[0].who === 'none') { + return null; + } + + return contributors; +}; + +function getMultilineField(lines, name) { + // All this code is 8asically the same as the getListText - just with a + // different line prefix (four spaces instead of a dash and a space). + let startIndex = lines.findIndex(line => line.startsWith(name + ':')); + if (startIndex === -1) { + return null; + } + startIndex++; + let endIndex = lines.findIndex((line, index) => index >= startIndex && line.length && !line.startsWith(' ')); + if (endIndex === -1) { + endIndex = lines.length; + } + // If there aren't any content lines, don't return anything! + if (endIndex === startIndex) { + return null; + } + // We also join the lines instead of returning an array. + const listLines = lines.slice(startIndex, endIndex); + return listLines.map(line => line.slice(4)).join('\n'); +}; + +const replacerSpec = { + 'album': { + search: 'album', + link: 'album' + }, + 'album-commentary': { + search: 'album', + link: 'albumCommentary' + }, + 'artist': { + search: 'artist', + link: 'artist' + }, + 'artist-gallery': { + search: 'artist', + link: 'artistGallery' + }, + 'commentary-index': { + search: null, + link: 'commentaryIndex' + }, + 'date': { + search: null, + value: ref => new Date(ref), + html: (date, {strings}) => `` + }, + 'flash': { + search: 'flash', + link: 'flash', + transformName(name, search, offset, text) { + const nextCharacter = text[offset + search.length]; + const lastCharacter = name[name.length - 1]; + if ( + ![' ', '\n', '<'].includes(nextCharacter) && + lastCharacter === '.' + ) { + return name.slice(0, -1); + } else { + return name; + } + } + }, + 'group': { + search: 'group', + link: 'groupInfo' + }, + 'group-gallery': { + search: 'group', + link: 'groupGallery' + }, + 'listing-index': { + search: null, + link: 'listingIndex' + }, + 'listing': { + search: 'listing', + link: 'listing' + }, + 'media': { + search: null, + link: 'media' + }, + 'news-index': { + search: null, + link: 'newsIndex' + }, + 'news-entry': { + search: 'newsEntry', + link: 'newsEntry' + }, + 'root': { + search: null, + link: 'root' + }, + 'site': { + search: null, + link: 'site' + }, + 'static': { + search: 'staticPage', + link: 'staticPage' + }, + 'tag': { + search: 'tag', + link: 'tag' + }, + 'track': { + search: 'track', + link: 'track' + } +}; + +{ + let error = false; + for (const [key, {link: linkKey, search: searchKey, value, html}] of Object.entries(replacerSpec)) { + if (!html && !link[linkKey]) { + logError`The replacer spec ${key} has invalid link key ${linkKey}! Specify it in link specs or fix typo.`; + error = true; + } + if (searchKey && !search[searchKey]) { + logError`The replacer spec ${key} has invalid search key ${searchKey}! Specify it in search specs or fix typo.`; + error = true; + } + } + if (error) process.exit(); + + const categoryPart = Object.keys(replacerSpec).join('|'); + transformInline.regexp = new RegExp(String.raw`(? { + if (!category) { + category = 'track'; + } + + const { + search: searchKey, + link: linkKey, + value: valueFn, + html: htmlFn, + transformName + } = replacerSpec[category]; + + const value = ( + valueFn ? valueFn(ref) : + searchKey ? search[searchKey](ref) : + { + directory: ref.replace(category + ':', ''), + name: null + }); + + if (!value) { + logWarn`The link ${match} does not match anything!`; + return match; + } + + const label = (enteredName + || transformName && transformName(value.name, match, offset, text) + || value.name); + + if (!valueFn && !label) { + logWarn`The link ${match} requires a label be entered!`; + return match; + } + + const fn = (htmlFn + ? htmlFn + : strings.link[linkKey]); + + try { + return fn(value, {text: label, hash, strings, to}); + } catch (error) { + logError`The link ${match} failed to be processed: ${error}`; + return match; + } + }).replaceAll(String.raw`\[[`, '[['); +} + +function parseAttributes(string, {to}) { + const attributes = Object.create(null); + const skipWhitespace = i => { + const ws = /\s/; + if (ws.test(string[i])) { + const match = string.slice(i).match(/[^\s]/); + if (match) { + return i + match.index; + } else { + return string.length; + } + } else { + return i; + } + }; + + for (let i = 0; i < string.length;) { + i = skipWhitespace(i); + const aStart = i; + const aEnd = i + string.slice(i).match(/[\s=]|$/).index; + const attribute = string.slice(aStart, aEnd); + i = skipWhitespace(aEnd); + if (string[i] === '=') { + i = skipWhitespace(i + 1); + let end, endOffset; + if (string[i] === '"' || string[i] === "'") { + end = string[i]; + endOffset = 1; + i++; + } else { + end = '\\s'; + endOffset = 0; + } + const vStart = i; + const vEnd = i + string.slice(i).match(new RegExp(`${end}|$`)).index; + const value = string.slice(vStart, vEnd); + i = vEnd + endOffset; + if (attribute === 'src' && value.startsWith('media/')) { + attributes[attribute] = to('media.path', value.slice('media/'.length)); + } else { + attributes[attribute] = value; + } + } else { + attributes[attribute] = attribute; + } + } + return Object.fromEntries(Object.entries(attributes).map(([ key, val ]) => [ + key, + val === 'true' ? true : + val === 'false' ? false : + val === key ? true : + val + ])); +} + +function transformMultiline(text, {strings, to}) { + // Heck yes, HTML magics. + + text = transformInline(text.trim(), {strings, to}); + + const outLines = []; + + const indentString = ' '.repeat(4); + + let levelIndents = []; + const openLevel = indent => { + // opening a sublist is a pain: to be semantically *and* visually + // correct, we have to append the + `).join('\n')} + + `; + } + }, + + { + directory: 'tracks/by-times-referenced', + title: ({strings}) => strings('listingPage.listTracks.byTimesReferenced.title'), + + data() { + return trackData + .map(track => ({track, timesReferenced: track.referencedBy.length})) + .filter(({ timesReferenced }) => timesReferenced > 0) + .sort((a, b) => b.timesReferenced - a.timesReferenced); + }, + + row({track, timesReferenced}, {strings, to}) { + return strings('listingPage.listTracks.byTimesReferenced.item', { + track: strings.link.track(track, {to}), + timesReferenced: strings.count.timesReferenced(timesReferenced, {unit: true}) + }); + } + }, + + { + directory: 'tracks/in-flashes/by-album', + title: ({strings}) => strings('listingPage.listTracks.inFlashes.byAlbum.title'), + condition: () => wikiInfo.features.flashesAndGames, + + data() { + return chunkByProperties(trackData.filter(t => t.flashes.length > 0), ['album']) + .filter(({ album }) => album.directory !== C.UNRELEASED_TRACKS_DIRECTORY); + }, + + html(chunks, {strings, to}) { + return fixWS` +
    + ${chunks.map(({album, chunk: tracks}) => fixWS` +
    ${strings('listingPage.listTracks.inFlashes.byAlbum.album', { + album: strings.link.album(album, {to}), + date: strings.count.date(album.date) + })}
    +
      + ${(tracks + .map(track => strings('listingPage.listTracks.inFlashes.byAlbum.track', { + track: strings.link.track(track, {to}), + flashes: strings.list.and(track.flashes.map(flash => strings.link.flash(flash, {to}))) + })) + .map(row => `
    • ${row}
      • `) + .join('\n'))} +
    + `).join('\n')} +
    + `; + } + }, + + { + directory: 'tracks/in-flashes/by-flash', + title: ({strings}) => strings('listingPage.listTracks.inFlashes.byFlash.title'), + condition: () => wikiInfo.features.flashesAndGames, + + html({strings, to}) { + return fixWS` +
    + ${C.sortByDate(flashData.slice()).map(flash => fixWS` +
    ${strings('listingPage.listTracks.inFlashes.byFlash.flash', { + flash: strings.link.flash(flash, {to}), + date: strings.count.date(flash.date) + })}
    +
      + ${(flash.tracks + .map(track => strings('listingPage.listTracks.inFlashes.byFlash.track', { + track: strings.link.track(track, {to}), + album: strings.link.album(track.album, {to}) + })) + .map(row => `
    • ${row}
      • `) + .join('\n'))} +
    + `).join('\n')} +
    + `; + } + }, + + { + directory: 'tracks/with-lyrics', + title: ({strings}) => strings('listingPage.listTracks.withLyrics.title'), + + data() { + return chunkByProperties(trackData.filter(t => t.lyrics), ['album']); + }, + + html(chunks, {strings, to}) { + return fixWS` +
    + ${chunks.map(({album, chunk: tracks}) => fixWS` +
    ${strings('listingPage.listTracks.withLyrics.album', { + album: strings.link.album(album, {to}), + date: strings.count.date(album.date) + })}
    +
      + ${(tracks + .map(track => strings('listingPage.listTracks.withLyrics.track', { + track: strings.link.track(track, {to}), + })) + .map(row => `
    • ${row}
      • `) + .join('\n'))} +
    + `).join('\n')} +
    + `; + } + }, + + { + directory: 'tags/by-name', + title: ({strings}) => strings('listingPage.listTags.byName.title'), + condition: () => wikiInfo.features.artTagUI, + + data() { + return tagData + .filter(tag => !tag.isCW) + .sort(sortByName) + .map(tag => ({tag, timesUsed: tag.things.length})); + }, + + row({tag, timesUsed}, {strings, to}) { + return strings('listingPage.listTags.byName.item', { + tag: strings.link.tag(tag, {to}), + timesUsed: strings.count.timesUsed(timesUsed, {unit: true}) + }); + } + }, + + { + directory: 'tags/by-uses', + title: ({strings}) => strings('listingPage.listTags.byUses.title'), + condition: () => wikiInfo.features.artTagUI, + + data() { + return tagData + .filter(tag => !tag.isCW) + .map(tag => ({tag, timesUsed: tag.things.length})) + .sort((a, b) => b.timesUsed - a.timesUsed); + }, + + row({tag, timesUsed}, {strings, to}) { + return strings('listingPage.listTags.byUses.item', { + tag: strings.link.tag(tag, {to}), + timesUsed: strings.count.timesUsed(timesUsed, {unit: true}) + }); + } + }, + + { + directory: 'random', + title: ({strings}) => `Random Pages`, + html: ({strings, to}) => fixWS` +

    Choose a link to go to a random page in that category or album! If your browser doesn't support relatively modern JavaScript or you've disabled it, these links won't work - sorry.

    +

    (Data files are downloading in the background! Please wait for data to load.)

    +

    (Data files have finished being downloaded. The links should work!)

    +
    +
    Miscellaneous:
    +
    + ${[ + {name: 'Official', albumData: officialAlbumData, code: 'official'}, + {name: 'Fandom', albumData: fandomAlbumData, code: 'fandom'} + ].map(category => fixWS` +
    ${category.name}: (Random Album, Random Track)
    +
      ${category.albumData.map(album => fixWS` +
    • ${album.name}
      • + `).join('\n')}
    + `).join('\n')} +
    + ` + } +]; + +function writeListingPages() { + if (!wikiInfo.features.listings) { + return; + } + + return [ + writeListingIndex(), + ...listingSpec.map(writeListingPage).filter(Boolean) + ]; +} + +function writeListingIndex() { + const releasedTracks = trackData.filter(track => track.album.directory !== C.UNRELEASED_TRACKS_DIRECTORY); + const releasedAlbums = albumData.filter(album => album.directory !== C.UNRELEASED_TRACKS_DIRECTORY); + const duration = getTotalDuration(releasedTracks); + + return ({strings, writePage}) => writePage('listingIndex', '', ({to}) => ({ + title: strings('listingIndex.title'), + + main: { + content: fixWS` +

    ${strings('listingIndex.title')}

    +

    ${strings('listingIndex.infoLine', { + wiki: wikiInfo.name, + tracks: `${strings.count.tracks(releasedTracks.length, {unit: true})}`, + albums: `${strings.count.albums(releasedAlbums.length, {unit: true})}`, + duration: `${strings.count.duration(duration, {approximate: true, unit: true})}` + })}

    +
    +

    ${strings('listingIndex.exploreList')}

    + ${generateLinkIndexForListings(null, {strings, to})} + ` + }, + + sidebarLeft: { + content: generateSidebarForListings(null, {strings, to}) + }, + + nav: {simple: true} + })) +} + +function writeListingPage(listing) { + if (listing.condition && !listing.condition()) { + return null; + } + + const data = (listing.data + ? listing.data() + : null); + + return ({strings, writePage}) => writePage('listing', listing.directory, ({to}) => ({ + title: listing.title({strings}), + + main: { + content: fixWS` +

    ${listing.title({strings})}

    + ${listing.html && (listing.data + ? listing.html(data, {strings, to}) + : listing.html({strings, to}))} + ${listing.row && fixWS` + + `} + ` + }, + + sidebarLeft: { + content: generateSidebarForListings(listing, {strings, to}) + }, + + nav: { + links: [ + { + href: to('localized.home'), + title: wikiInfo.shortName + }, + { + href: to('localized.listingIndex'), + title: strings('listingIndex.title') + }, + { + href: '', + title: listing.title({strings}) + } + ] + } + })); +} + +function generateSidebarForListings(currentListing, {strings, to}) { + return fixWS` +

    ${strings.link.listingIndex('', {text: strings('listingIndex.title'), to})}

    + ${generateLinkIndexForListings(currentListing, {strings, to})} + `; +} + +function generateLinkIndexForListings(currentListing, {strings, to}) { + return fixWS` + + `; +} + +function filterAlbumsByCommentary() { + return albumData.filter(album => [album, ...album.tracks].some(x => x.commentary)); +} + +function writeCommentaryPages() { + if (!filterAlbumsByCommentary().length) { + return; + } + + return [ + writeCommentaryIndex(), + ...filterAlbumsByCommentary().map(writeAlbumCommentaryPage) + ]; +} + +function writeCommentaryIndex() { + const data = filterAlbumsByCommentary() + .map(album => ({ + album, + entries: [album, ...album.tracks].filter(x => x.commentary).map(x => x.commentary) + })) + .map(({ album, entries }) => ({ + album, entries, + words: entries.join(' ').split(' ').length + })); + + const totalEntries = data.reduce((acc, {entries}) => acc + entries.length, 0); + const totalWords = data.reduce((acc, {words}) => acc + words, 0); + + return ({strings, writePage}) => writePage('commentaryIndex', '', ({to}) => ({ + title: strings('commentaryIndex.title'), + + main: { + content: fixWS` +
    +

    ${strings('commentaryIndex.title')}

    +

    ${strings('commentaryIndex.infoLine', { + words: `${strings.count.words(totalWords, {unit: true})}`, + entries: `${strings.count.commentaryEntries(totalEntries, {unit: true})}` + })}

    +

    ${strings('commentaryIndex.albumList.title')}

    + +
    + ` + }, + + nav: {simple: true} + })); +} + +function writeAlbumCommentaryPage(album) { + const entries = [album, ...album.tracks].filter(x => x.commentary).map(x => x.commentary); + const words = entries.join(' ').split(' ').length; + + return ({strings, writePage}) => writePage('albumCommentary', album.directory, ({to}) => ({ + title: strings('albumCommentaryPage.title', {album: album.name}), + stylesheet: getAlbumStylesheet(album, {to}), + theme: getThemeString(album), + + main: { + content: fixWS` +
    +

    ${strings('albumCommentaryPage.title', { + album: strings.link.album(album, {to}) + })}

    +

    ${strings('albumCommentaryPage.infoLine', { + words: `${strings.count.words(words, {unit: true})}`, + entries: `${strings.count.commentaryEntries(entries.length, {unit: true})}` + })}

    + ${album.commentary && fixWS` +

    ${strings('albumCommentaryPage.entry.title.albumCommentary')}

    +
    + ${transformMultiline(album.commentary, {strings, to})} +
    + `} + ${album.tracks.filter(t => t.commentary).map(track => fixWS` +

    ${strings('albumCommentaryPage.entry.title.trackCommentary', { + track: strings.link.track(track, {to}) + })}

    +
    + ${transformMultiline(track.commentary, {strings, to})} +
    + `).join('\n')} +
    + ` + }, + + nav: { + links: [ + { + href: to('localized.home'), + title: wikiInfo.shortName + }, + { + href: to('localized.commentaryIndex'), + title: strings('commentaryIndex.title') + }, + { + html: strings('albumCommentaryPage.nav.album', { + album: strings.link.albumCommentary(album, {class: 'current', to}) + }) + } + ] + } + })); +} + +function writeTagPages() { + if (!wikiInfo.features.artTagUI) { + return; + } + + return tagData.filter(tag => !tag.isCW).map(writeTagPage); +} + +function writeTagPage(tag) { + const { things } = tag; + + return ({strings, writePage}) => writePage('tag', tag.directory, ({to}) => ({ + title: strings('tagPage.title', {tag: tag.name}), + theme: getThemeString(tag), + + main: { + classes: ['top-index'], + content: fixWS` +

    ${strings('tagPage.title', {tag: tag.name})}

    +

    ${strings('tagPage.infoLine', { + coverArts: strings.count.coverArts(things.length, {unit: true}) + })}

    +
    + ${getGridHTML({ + strings, to, + entries: things.map(item => ({item})), + srcFn: thing => (thing.album + ? getTrackCover(thing, {to}) + : getAlbumCover(thing, {to})), + hrefFn: thing => (thing.album + ? to('localized.track', thing.directory) + : to('localized.album', thing.directory)) + })} +
    + ` + }, + + nav: { + links: [ + { + href: to('localized.home'), + title: wikiInfo.shortName + }, + wikiInfo.features.listings && + { + href: to('localized.listingIndex'), + title: strings('listingIndex.title') + }, + { + html: strings('tagPage.nav.tag', { + tag: strings.link.tag(tag, {class: 'current', to}) + }) + } + ] + } + })); +} + +function getArtistString(artists, {strings, to, showIcons = false, showContrib = false}) { + return strings.list.and(artists.map(({ who, what }) => { + const { urls, directory, name } = who; + return [ + strings.link.artist(who, {to}), + showContrib && what && `(${what})`, + showIcons && urls.length && `(${ + strings.list.unit(urls.map(url => iconifyURL(url, {strings, to}))) + })` + ].filter(Boolean).join(' '); + })); +} + +function getLinkThemeString(thing) { + const { primary, dim } = C.getColors(thing.color || wikiInfo.color); + return `--primary-color: ${primary}; --dim-color: ${dim}`; +} + +function getThemeString(thing, additionalVariables = []) { + const { primary, dim } = C.getColors(thing.color || wikiInfo.color); + + const variables = [ + `--primary-color: ${primary}`, + `--dim-color: ${dim}`, + ...additionalVariables + ].filter(Boolean); + + return fixWS` + ${variables.length && fixWS` + :root { + ${variables.map(line => line + ';').join('\n')} + } + `} + `; +} + +function getFlashDirectory(flash) { + // const kebab = getKebabCase(flash.name.replace('[S] ', '')); + // return flash.page + (kebab ? '-' + kebab : ''); + // return '' + flash.page; + return '' + flash.directory; +} + +function getTagDirectory({name}) { + return C.getKebabCase(name); +} + +function getAlbumListTag(album) { + if (album.directory === C.UNRELEASED_TRACKS_DIRECTORY) { + return 'ul'; + } else { + return 'ol'; + } +} + +function fancifyURL(url, {strings, album = false} = {}) { + const domain = new URL(url).hostname; + return fixWS`${ + domain.includes('bandcamp.com') ? strings('misc.external.bandcamp') : + [ + 'music.solatrux.com' + ].includes(domain) ? strings('misc.external.bandcamp.domain', {domain}) : + [ + 'types.pl' + ].includes(domain) ? strings('misc.external.mastodon.domain', {domain}) : + domain.includes('youtu') ? (album + ? (url.includes('list=') + ? strings('misc.external.youtube.playlist') + : strings('misc.external.youtube.fullAlbum')) + : strings('misc.external.youtube')) : + domain.includes('soundcloud') ? strings('misc.external.soundcloud') : + domain.includes('tumblr.com') ? strings('misc.external.tumblr') : + domain.includes('twitter.com') ? strings('misc.external.twitter') : + domain.includes('deviantart.com') ? strings('misc.external.deviantart') : + domain.includes('wikipedia.org') ? strings('misc.external.wikipedia') : + domain.includes('poetryfoundation.org') ? strings('misc.external.poetryFoundation') : + domain.includes('instagram.com') ? strings('misc.external.instagram') : + domain.includes('patreon.com') ? strings('misc.external.patreon') : + domain + }`; +} + +function fancifyFlashURL(url, flash, {strings}) { + const link = fancifyURL(url, {strings}); + return `${ + url.includes('homestuck.com') ? (isNaN(Number(flash.page)) + ? strings('misc.external.flash.homestuck.secret', {link}) + : strings('misc.external.flash.homestuck.page', {link, page: flash.page})) : + url.includes('bgreco.net') ? strings('misc.external.flash.bgreco', {link}) : + url.includes('youtu') ? strings('misc.external.flash.youtube', {link}) : + link + }`; +} + +function iconifyURL(url, {strings, to}) { + const domain = new URL(url).hostname; + const [ id, msg ] = ( + domain.includes('bandcamp.com') ? ['bandcamp', strings('misc.external.bandcamp')] : + ( + domain.includes('music.solatrus.com') + ) ? ['bandcamp', strings('misc.external.bandcamp.domain', {domain})] : + ( + domain.includes('types.pl') + ) ? ['mastodon', strings('misc.external.mastodon.domain', {domain})] : + domain.includes('youtu') ? ['youtube', strings('misc.external.youtube')] : + domain.includes('soundcloud') ? ['soundcloud', strings('misc.external.soundcloud')] : + domain.includes('tumblr.com') ? ['tumblr', strings('misc.external.tumblr')] : + domain.includes('twitter.com') ? ['twitter', strings('misc.external.twitter')] : + domain.includes('deviantart.com') ? ['deviantart', strings('misc.external.deviantart')] : + domain.includes('instagram.com') ? ['instagram', strings('misc.external.bandcamp')] : + ['globe', strings('misc.external.domain', {domain})] + ); + return fixWS`${msg}`; +} + +function chronologyLinks(currentThing, { + strings, to, + headingString, + contribKey, + getThings +}) { + const contributions = currentThing[contribKey]; + if (!contributions) { + return ''; + } + + if (contributions.length > 8) { + return `
    ${strings('misc.chronology.seeArtistPages')}
    `; + } + + return contributions.map(({ who: artist }) => { + const things = C.sortByDate(unique(getThings(artist))); + const releasedThings = things.filter(thing => { + const album = albumData.includes(thing) ? thing : thing.album; + return !(album && album.directory === C.UNRELEASED_TRACKS_DIRECTORY); + }); + const index = releasedThings.indexOf(currentThing); + + if (index === -1) return ''; + + // TODO: This can pro8a8ly 8e made to use generatePreviousNextLinks? + // We'd need to make generatePreviousNextLinks use toAnythingMan tho. + const previous = releasedThings[index - 1]; + const next = releasedThings[index + 1]; + const parts = [ + previous && `Previous`, + next && `Next` + ].filter(Boolean); + + const stringOpts = { + index: strings.count.index(index + 1, {strings}), + artist: strings.link.artist(artist, {to}) + }; + + return fixWS` +
    + ${strings(headingString, stringOpts)} + ${parts.length && `(${parts.join(', ')})`} +
    + `; + }).filter(Boolean).join('\n'); +} + +function generateAlbumNavLinks(album, currentTrack, {strings, to}) { + if (album.tracks.length <= 1) { + return ''; + } + + const previousNextLinks = currentTrack && generatePreviousNextLinks('localized.track', currentTrack, album.tracks, {strings, to}) + const randomLink = `${ + (currentTrack + ? strings('trackPage.nav.random') + : strings('albumPage.nav.randomTrack')) + }`; + + return (previousNextLinks + ? `(${previousNextLinks}, ${randomLink})` + : `(${randomLink})`); +} + +function generateAlbumChronologyLinks(album, currentTrack, {strings, to}) { + return [ + currentTrack && chronologyLinks(currentTrack, { + strings, to, + headingString: 'misc.chronology.heading.track', + contribKey: 'artists', + getThings: artist => [...artist.tracks.asArtist, ...artist.tracks.asContributor] + }), + chronologyLinks(currentTrack || album, { + strings, to, + headingString: 'misc.chronology.heading.coverArt', + contribKey: 'coverArtists', + getThings: artist => [...artist.albums.asCoverArtist, ...artist.tracks.asCoverArtist] + }) + ].filter(Boolean).join('\n'); +} + +function generateSidebarForAlbum(album, currentTrack, {strings, to}) { + const listTag = getAlbumListTag(album); + + const trackToListItem = track => `
  • ${ + strings('albumSidebar.trackList.item', { + track: strings.link.track(track, {to}) + }) + }
    • `; + + const trackListPart = fixWS` +

    ${album.name}

    + ${album.trackGroups ? fixWS` +
    + ${album.trackGroups.map(({ name, color, startIndex, tracks }) => fixWS` +
    ${ + (listTag === 'ol' + ? strings('albumSidebar.trackList.group.withRange', { + group: strings.link.track(tracks[0], {to, text: name}), + range: `${startIndex + 1}–${startIndex + tracks.length}` + }) + : strings('albumSidebar.trackList.group', { + group: strings.link.track(tracks[0], {to, text: name}) + })) + }
    + ${(!currentTrack || tracks.includes(currentTrack)) && fixWS` +
    <${listTag === 'ol' ? `ol start="${startIndex + 1}"` : listTag}> + ${tracks.map(trackToListItem).join('\n')} +
    + `} + `).join('\n')} +
    + ` : fixWS` + <${listTag}> + ${album.tracks.map(trackToListItem).join('\n')} + + `} + `; + + const { groups } = album; + + const groupParts = groups.map(group => { + const index = group.albums.indexOf(album); + const next = group.albums[index + 1]; + const previous = group.albums[index - 1]; + return {group, next, previous}; + }).map(({group, next, previous}) => fixWS` +

    ${ + strings('albumSidebar.groupBox.title', { + group: `${group.name}` + }) + }

    + ${!currentTrack && transformMultiline(group.descriptionShort, {strings, to})} + ${group.urls.length && `

    ${ + strings('releaseInfo.visitOn', { + links: strings.list.or(group.urls.map(url => fancifyURL(url, {strings}))) + }) + }

    `} + ${!currentTrack && fixWS` + ${next && `

    ${ + strings('albumSidebar.groupBox.next', { + album: `${next.name}` + }) + }

    `} + ${previous && `

    ${ + strings('albumSidebar.groupBox.previous', { + album: `${previous.name}` + }) + }

    `} + `} + `); + + if (groupParts.length) { + if (currentTrack) { + const combinedGroupPart = groupParts.join('\n
    \n'); + return { + multiple: [ + trackListPart, + combinedGroupPart + ] + }; + } else { + return { + multiple: [ + ...groupParts, + trackListPart + ] + }; + } + } else { + return { + content: trackListPart + }; + } +} + +function generateSidebarForGroup(currentGroup, {strings, to, isGallery}) { + if (!wikiInfo.features.groupUI) { + return null; + } + + const urlKey = isGallery ? 'localized.groupGallery' : 'localized.groupInfo'; + + return { + content: fixWS` +

    ${strings('groupSidebar.title')}

    +
    + ${groupCategoryData.map(category => [ + fixWS` +
    ${ + strings('groupSidebar.groupList.category', { + category: `${category.name}` + }) + }
    +
      + ${category.groups.map(group => fixWS` +
    • ${ + strings('groupSidebar.groupList.item', { + group: `${group.name}` + }) + }
      • + `).join('\n')} +
    + ` + ]).join('\n')} +
    + ` + }; +} + +function generateInfoGalleryLinks(urlKeyInfo, urlKeyGallery, currentThing, isGallery, {strings, to}) { + return [ + strings.link[urlKeyInfo](currentThing, { + to, + class: isGallery ? '' : 'current', + text: strings('misc.nav.info') + }), + strings.link[urlKeyGallery](currentThing, { + to, + class: isGallery ? 'current' : '', + text: strings('misc.nav.gallery') + }) + ].join(', '); +} + +function generatePreviousNextLinks(urlKey, currentThing, thingData, {strings, to}) { + const index = thingData.indexOf(currentThing); + const previous = thingData[index - 1]; + const next = thingData[index + 1]; + + return [ + previous && `${strings('misc.nav.previous')}`, + next && `${strings('misc.nav.next')}` + ].filter(Boolean).join(', '); +} + +function generateNavForGroup(currentGroup, {strings, to, isGallery}) { + if (!wikiInfo.features.groupUI) { + return {simple: true}; + } + + const urlKey = isGallery ? 'localized.groupGallery' : 'localized.groupInfo'; + const linkKey = isGallery ? 'groupGallery' : 'groupInfo'; + + const infoGalleryLinks = generateInfoGalleryLinks('groupInfo', 'groupGallery', currentGroup, isGallery, {strings, to}); + const previousNextLinks = generatePreviousNextLinks(urlKey, currentGroup, groupData, {strings, to}) + + return { + links: [ + { + href: to('localized.home'), + title: wikiInfo.shortName + }, + wikiInfo.features.listings && + { + href: to('localized.listingIndex'), + title: strings('listingIndex.title') + }, + { + html: strings('groupPage.nav.group', { + group: strings.link[linkKey](currentGroup, {class: 'current', to}) + }) + }, + { + divider: false, + html: (previousNextLinks + ? `(${infoGalleryLinks}; ${previousNextLinks})` + : `(${previousNextLinks})`) + } + ] + }; +} + +function writeGroupPages() { + return groupData.map(writeGroupPage); +} + +function writeGroupPage(group) { + const releasedAlbums = group.albums.filter(album => album.directory !== C.UNRELEASED_TRACKS_DIRECTORY); + const releasedTracks = releasedAlbums.flatMap(album => album.tracks); + const totalDuration = getTotalDuration(releasedTracks); + + return async ({strings, writePage}) => { + await writePage('groupInfo', group.directory, ({to}) => ({ + title: strings('groupInfoPage.title', {group: group.name}), + theme: getThemeString(group), + + main: { + content: fixWS` +

    ${strings('groupInfoPage.title', {group: group.name})}

    + ${group.urls.length && `

    ${ + strings('releaseInfo.visitOn', { + links: strings.list.or(group.urls.map(url => fancifyURL(url, {strings}))) + }) + }

    `} +
    + ${transformMultiline(group.description, {strings, to})} +
    +

    ${strings('groupInfoPage.albumList.title')}

    +

    ${ + strings('groupInfoPage.viewAlbumGallery', { + link: `${ + strings('groupInfoPage.viewAlbumGallery.link') + }` + }) + }

    + + ` + }, + + sidebarLeft: generateSidebarForGroup(group, {strings, to, isGallery: false}), + nav: generateNavForGroup(group, {strings, to, isGallery: false}) + })); + + await writePage('groupGallery', group.directory, ({to}) => ({ + title: strings('groupGalleryPage.title', {group: group.name}), + theme: getThemeString(group), + + main: { + classes: ['top-index'], + content: fixWS` +

    ${strings('groupGalleryPage.title', {group: group.name})}

    +

    ${ + strings('groupGalleryPage.infoLine', { + tracks: `${strings.count.tracks(releasedTracks.length, {unit: true})}`, + albums: `${strings.count.albums(releasedAlbums.length, {unit: true})}`, + time: `${strings.count.duration(totalDuration, {unit: true})}` + }) + }

    + ${wikiInfo.features.groupUI && wikiInfo.features.listings && `

    (Choose another group to filter by!)

    `} +
    + ${getAlbumGridHTML({ + strings, to, + entries: C.sortByDate(group.albums.map(item => ({item}))).reverse(), + details: true + })} +
    + ` + }, + + sidebarLeft: generateSidebarForGroup(group, {strings, to, isGallery: true}), + nav: generateNavForGroup(group, {strings, to, isGallery: true}) + })); + }; +} + +function toAnythingMan(anythingMan, to) { + return ( + albumData.includes(anythingMan) ? to('localized.album', anythingMan.directory) : + trackData.includes(anythingMan) ? to('localized.track', anythingMan.directory) : + flashData?.includes(anythingMan) ? to('localized.flash', anythingMan.directory) : + 'idk-bud' + ) +} + +function getAlbumCover(album, {to}) { + return to('media.albumCover', album.directory); +} + +function getTrackCover(track, {to}) { + // Some al8ums don't have any track art at all, and in those, every track + // just inherits the al8um's own cover art. + if (track.coverArtists === null) { + return getAlbumCover(track.album, {to}); + } else { + return to('media.trackCover', track.album.directory, track.directory); + } +} + +function getFlashLink(flash) { + return `https://homestuck.com/story/${flash.page}`; +} + +function classes(...args) { + const values = args.filter(Boolean); + return `class="${values.join(' ')}"`; +} + +async function processLanguageFile(file, defaultStrings = null) { + let contents; + try { + contents = await readFile(file, 'utf-8'); + } catch (error) { + return {error: `Could not read ${file} (${error.code}).`}; + } + + let json; + try { + json = JSON.parse(contents); + } catch (error) { + return {error: `Could not parse JSON from ${file} (${error}).`}; + } + + return genStrings(json, defaultStrings); +} + +// Wrapper function for running a function once for all languages. It provides: +// * the language strings +// * a shadowing writePages function for outputing to the appropriate subdir +// * a shadowing urls object for linking to the appropriate relative paths +async function wrapLanguages(fn, writeOneLanguage = null) { + const k = writeOneLanguage + const languagesToRun = (k + ? {[k]: languages[k]} + : languages) + + const entries = Object.entries(languagesToRun) + .filter(([ key ]) => key !== 'default'); + + for (let i = 0; i < entries.length; i++) { + const [ key, strings ] = entries[i]; + + const baseDirectory = (strings === languages.default ? '' : strings.code); + + const shadow_writePage = (urlKey, directory, pageFn) => writePage(strings, baseDirectory, urlKey, directory, pageFn); + + // 8ring the utility functions over too! + Object.assign(shadow_writePage, writePage); + + await fn({ + baseDirectory, + strings, + writePage: shadow_writePage + }, i, entries); + } +} + +async function main() { + const miscOptions = await parseOptions(process.argv.slice(2), { + // Data files for the site, including flash, artist, and al8um data, + // and like a jillion other things too. Pretty much everything which + // makes an individual wiki what it is goes here! + 'data-path': { + type: 'value' + }, + + // Static media will 8e referenced in the site here! The contents are + // categorized; check out MEDIA_DIRECTORY and rel8ted constants in + // common/common.js. (This gets symlinked into the --data directory.) + 'media-path': { + type: 'value' + }, + + // String files! For the most part, this is used for translating the + // site to different languages, though you can also customize strings + // for your own 8uild of the site if you'd like. Files here should all + // match the format in strings-default.json in this repository. (If a + // language file is missing any strings, the site code will fall 8ack + // to what's specified in strings-default.json.) + // + // Unlike the other options here, this one's optional - the site will + // 8uild with the default (English) strings if this path is left + // unspecified. + 'lang-path': { + type: 'value' + }, + + // This is the output directory. It's the one you'll upload online with + // rsync or whatever when you're pushing an upd8, and also the one + // you'd archive if you wanted to make a 8ackup of the whole dang + // site. Just keep in mind that the gener8ted result will contain a + // couple symlinked directories, so if you're uploading, you're pro8a8ly + // gonna want to resolve those yourself. + 'out-path': { + type: 'value' + }, + + // Thum8nail gener8tion is *usually* something you want, 8ut it can 8e + // kinda a pain to run every time, since it does necessit8te reading + // every media file at run time. Pass this to skip it. + 'skip-thumbs': { + type: 'flag' + }, + + // Only want 8uild one language during testing? This can chop down + // 8uild times a pretty 8ig chunk! Just pass a single language code. + 'lang': { + type: 'value' + }, + + 'queue-size': { + type: 'value', + validate(size) { + if (parseInt(size) !== parseFloat(size)) return 'an integer'; + if (parseInt(size) < 0) return 'a counting number or zero'; + return true; + } + }, + queue: {alias: 'queue-size'}, + + [parseOptions.handleUnknown]: () => {} + }); + + dataPath = miscOptions['data-path'] || process.env.HSMUSIC_DATA; + mediaPath = miscOptions['media-path'] || process.env.HSMUSIC_MEDIA; + langPath = miscOptions['lang-path'] || process.env.HSMUSIC_LANG; // Can 8e left unset! + outputPath = miscOptions['out-path'] || process.env.HSMUSIC_OUT; + + const writeOneLanguage = miscOptions['lang']; + + { + let errored = false; + const error = (cond, msg) => { + if (cond) { + console.error(`\x1b[31;1m${msg}\x1b[0m`); + errored = true; + } + }; + error(!dataPath, `Expected --data option or HSMUSIC_DATA to be set`); + error(!mediaPath, `Expected --media option or HSMUSIC_MEDIA to be set`); + error(!outputPath, `Expected --out option or HSMUSIC_OUT to be set`); + if (errored) { + return; + } + } + + const skipThumbs = miscOptions['skip-thumbs'] ?? false; + + if (skipThumbs) { + logInfo`Skipping thumbnail generation.`; + } else { + logInfo`Begin thumbnail generation... -----+`; + const result = await genThumbs(mediaPath, {queueSize, quiet: true}); + logInfo`Done thumbnail generation! --------+`; + if (!result) { + return; + } + } + + const defaultStrings = await processLanguageFile(path.join(__dirname, DEFAULT_STRINGS_FILE)); + if (defaultStrings.error) { + logError`Error loading default strings: ${defaultStrings.error}`; + return; + } + + if (langPath) { + const languageDataFiles = await findFiles(langPath, f => path.extname(f) === '.json'); + const results = await progressPromiseAll(`Reading & processing language files.`, languageDataFiles + .map(file => processLanguageFile(file, defaultStrings.json))); + + let error = false; + for (const strings of results) { + if (strings.error) { + logError`Error loading provided strings: ${strings.error}`; + error = true; + } + } + if (error) return; + + languages = Object.fromEntries(results.map(strings => [strings.code, strings])); + } else { + languages = {}; + } + + if (!languages[defaultStrings.code]) { + languages[defaultStrings.code] = defaultStrings; + } + + logInfo`Loaded language strings: ${Object.keys(languages).join(', ')}`; + + if (writeOneLanguage && !(writeOneLanguage in languages)) { + logError`Specified to write only ${writeOneLanguage}, but there is no strings file with this language code!`; + return; + } else if (writeOneLanguage) { + logInfo`Writing only language ${writeOneLanguage} this run.`; + } else { + logInfo`Writing all languages.`; + } + + wikiInfo = await processWikiInfoFile(path.join(dataPath, WIKI_INFO_FILE)); + if (wikiInfo.error) { + console.log(`\x1b[31;1m${wikiInfo.error}\x1b[0m`); + return; + } + + // Update languages o8ject with the wiki-specified default language! + // This will make page files for that language 8e gener8ted at the root + // directory, instead of the language-specific su8directory. + if (wikiInfo.defaultLanguage) { + if (Object.keys(languages).includes(wikiInfo.defaultLanguage)) { + languages.default = languages[wikiInfo.defaultLanguage]; + } else { + logError`Wiki info file specified default language is ${wikiInfo.defaultLanguage}, but no such language file exists!`; + if (langPath) { + logError`Check if an appropriate file exists in ${langPath}?`; + } else { + logError`Be sure to specify ${'--lang'} or ${'HSMUSIC_LANG'} with the path to language files.`; + } + return; + } + } else { + languages.default = defaultStrings; + } + + homepageInfo = await processHomepageInfoFile(path.join(dataPath, HOMEPAGE_INFO_FILE)); + + if (homepageInfo.error) { + console.log(`\x1b[31;1m${homepageInfo.error}\x1b[0m`); + return; + } + + { + const errors = homepageInfo.rows.filter(obj => obj.error); + if (errors.length) { + for (const error of errors) { + console.log(`\x1b[31;1m${error.error}\x1b[0m`); + } + return; + } + } + + // 8ut wait, you might say, how do we know which al8um these data files + // correspond to???????? You wouldn't dare suggest we parse the actual + // paths returned 8y this function, which ought to 8e of effectively + // unknown format except for their purpose as reada8le data files!? + // To that, I would say, yeah, you're right. Thanks a 8unch, my projection + // of "you". We're going to read these files later, and contained within + // will 8e the actual directory names that the data correspond to. Yes, + // that's redundant in some ways - we COULD just return the directory name + // in addition to the data path, and duplicating that name within the file + // itself suggests we 8e careful to avoid mismatching it - 8ut doing it + // this way lets the data files themselves 8e more porta8le (meaning we + // could store them all in one folder, if we wanted, and this program would + // still output to the correct al8um directories), and also does make the + // function's signature simpler (an array of strings, rather than some kind + // of structure containing 8oth data file paths and output directories). + // This is o8jectively a good thing, 8ecause it means the function can stay + // truer to its name, and have a narrower purpose: it doesn't need to + // concern itself with where we *output* files, or whatever other reasons + // we might (hypothetically) have for knowing the containing directory. + // And, in the strange case where we DO really need to know that info, we + // callers CAN use path.dirname to find out that data. 8ut we'll 8e + // avoiding that in our code 8ecause, again, we want to avoid assuming the + // format of the returned paths here - they're only meant to 8e used for + // reading as-is. + const albumDataFiles = await findFiles(path.join(dataPath, C.DATA_ALBUM_DIRECTORY)); + + // Technically, we could do the data file reading and output writing at the + // same time, 8ut that kinda makes the code messy, so I'm not 8othering + // with it. + albumData = await progressPromiseAll(`Reading & processing album files.`, albumDataFiles.map(processAlbumDataFile)); + + { + const errors = albumData.filter(obj => obj.error); + if (errors.length) { + for (const error of errors) { + console.log(`\x1b[31;1m${error.error}\x1b[0m`); + } + return; + } + } + + C.sortByDate(albumData); + + artistData = await processArtistDataFile(path.join(dataPath, ARTIST_DATA_FILE)); + if (artistData.error) { + console.log(`\x1b[31;1m${artistData.error}\x1b[0m`); + return; + } + + { + const errors = artistData.filter(obj => obj.error); + if (errors.length) { + for (const error of errors) { + console.log(`\x1b[31;1m${error.error}\x1b[0m`); + } + return; + } + } + + artistAliasData = artistData.filter(x => x.alias); + artistData = artistData.filter(x => !x.alias); + + trackData = C.getAllTracks(albumData); + + if (wikiInfo.features.flashesAndGames) { + flashData = await processFlashDataFile(path.join(dataPath, FLASH_DATA_FILE)); + if (flashData.error) { + console.log(`\x1b[31;1m${flashData.error}\x1b[0m`); + return; + } + + const errors = flashData.filter(obj => obj.error); + if (errors.length) { + for (const error of errors) { + console.log(`\x1b[31;1m${error.error}\x1b[0m`); + } + return; + } + } + + flashActData = flashData?.filter(x => x.act8r8k); + flashData = flashData?.filter(x => !x.act8r8k); + + artistNames = Array.from(new Set([ + ...artistData.filter(artist => !artist.alias).map(artist => artist.name), + ...[ + ...albumData.flatMap(album => [ + ...album.artists || [], + ...album.coverArtists || [], + ...album.wallpaperArtists || [], + ...album.tracks.flatMap(track => [ + ...track.artists, + ...track.coverArtists || [], + ...track.contributors || [] + ]) + ]), + ...(flashData?.flatMap(flash => [ + ...flash.contributors || [] + ]) || []) + ].map(contribution => contribution.who) + ])); + + tagData = await processTagDataFile(path.join(dataPath, TAG_DATA_FILE)); + if (tagData.error) { + console.log(`\x1b[31;1m${tagData.error}\x1b[0m`); + return; + } + + { + const errors = tagData.filter(obj => obj.error); + if (errors.length) { + for (const error of errors) { + console.log(`\x1b[31;1m${error.error}\x1b[0m`); + } + return; + } + } + + groupData = await processGroupDataFile(path.join(dataPath, GROUP_DATA_FILE)); + if (groupData.error) { + console.log(`\x1b[31;1m${groupData.error}\x1b[0m`); + return; + } + + { + const errors = groupData.filter(obj => obj.error); + if (errors.length) { + for (const error of errors) { + console.log(`\x1b[31;1m${error.error}\x1b[0m`); + } + return; + } + } + + groupCategoryData = groupData.filter(x => x.isCategory); + groupData = groupData.filter(x => x.isGroup); + + staticPageData = await processStaticPageDataFile(path.join(dataPath, STATIC_PAGE_DATA_FILE)); + if (staticPageData.error) { + console.log(`\x1b[31;1m${staticPageData.error}\x1b[0m`); + return; + } + + { + const errors = staticPageData.filter(obj => obj.error); + if (errors.length) { + for (const error of errors) { + console.log(`\x1b[31;1m${error.error}\x1b[0m`); + } + return; + } + } + + if (wikiInfo.features.news) { + newsData = await processNewsDataFile(path.join(dataPath, NEWS_DATA_FILE)); + if (newsData.error) { + console.log(`\x1b[31;1m${newsData.error}\x1b[0m`); + return; + } + + const errors = newsData.filter(obj => obj.error); + if (errors.length) { + for (const error of errors) { + console.log(`\x1b[31;1m${error.error}\x1b[0m`); + } + return; + } + + C.sortByDate(newsData); + newsData.reverse(); + } + + { + const tagNames = new Set([...trackData, ...albumData].flatMap(thing => thing.artTags)); + + for (let { name, isCW } of tagData) { + if (isCW) { + name = 'cw: ' + name; + } + tagNames.delete(name); + } + + if (tagNames.size) { + for (const name of Array.from(tagNames).sort()) { + console.log(`\x1b[33;1m- Missing tag: "${name}"\x1b[0m`); + } + return; + } + } + + artistNames.sort((a, b) => a.toLowerCase() < b.toLowerCase() ? -1 : a.toLowerCase() > b.toLowerCase() ? 1 : 0); + + justEverythingMan = C.sortByDate([...albumData, ...trackData, ...(flashData || [])]); + justEverythingSortedByArtDateMan = C.sortByArtDate(justEverythingMan.slice()); + // console.log(JSON.stringify(justEverythingSortedByArtDateMan.map(toAnythingMan), null, 2)); + + { + let buffer = []; + const clearBuffer = function() { + if (buffer.length) { + for (const entry of buffer.slice(0, -1)) { + console.log(`\x1b[2m... ${entry.name} ...\x1b[0m`); + } + const lastEntry = buffer[buffer.length - 1]; + console.log(`\x1b[2m... \x1b[0m${lastEntry.name}\x1b[0;2m ...\x1b[0m`); + buffer = []; + } + }; + const showWhere = (name, color) => { + const where = justEverythingMan.filter(thing => [ + ...thing.coverArtists || [], + ...thing.contributors || [], + ...thing.artists || [] + ].some(({ who }) => who === name)); + for (const thing of where) { + console.log(`\x1b[${color}m- ` + (thing.album ? `(\x1b[1m${thing.album.name}\x1b[0;${color}m)` : '') + ` \x1b[1m${thing.name}\x1b[0m`); + } + }; + let CR4SH = false; + for (let name of artistNames) { + const entry = [...artistData, ...artistAliasData].find(entry => entry.name === name || entry.name.toLowerCase() === name.toLowerCase()); + if (!entry) { + clearBuffer(); + console.log(`\x1b[31mMissing entry for artist "\x1b[1m${name}\x1b[0;31m"\x1b[0m`); + showWhere(name, 31); + CR4SH = true; + } else if (entry.alias) { + console.log(`\x1b[33mArtist "\x1b[1m${name}\x1b[0;33m" should be named "\x1b[1m${entry.alias}\x1b[0;33m"\x1b[0m`); + showWhere(name, 33); + CR4SH = true; + } else if (entry.name !== name) { + console.log(`\x1b[33mArtist "\x1b[1m${name}\x1b[0;33m" should be named "\x1b[1m${entry.name}\x1b[0;33m"\x1b[0m`); + showWhere(name, 33); + CR4SH = true; + } else { + buffer.push(entry); + if (buffer.length > 3) { + buffer.shift(); + } + } + } + if (CR4SH) { + return; + } + } + + { + const directories = []; + for (const { directory, name } of albumData) { + if (directories.includes(directory)) { + console.log(`\x1b[31;1mDuplicate album directory "${directory}" (${name})\x1b[0m`); + return; + } + directories.push(directory); + } + } + + { + const directories = []; + const where = {}; + for (const { directory, album } of trackData) { + if (directories.includes(directory)) { + console.log(`\x1b[31;1mDuplicate track directory "${directory}"\x1b[0m`); + console.log(`Shows up in:`); + console.log(`- ${album.name}`); + console.log(`- ${where[directory].name}`); + return; + } + directories.push(directory); + where[directory] = album; + } + } + + { + const artists = []; + const artistsLC = []; + for (const name of artistNames) { + if (!artists.includes(name) && artistsLC.includes(name.toLowerCase())) { + const other = artists.find(oth => oth.toLowerCase() === name.toLowerCase()); + console.log(`\x1b[31;1mMiscapitalized artist name: ${name}, ${other}\x1b[0m`); + return; + } + artists.push(name); + artistsLC.push(name.toLowerCase()); + } + } + + { + for (const { references, name, album } of trackData) { + for (const ref of references) { + if (!search.track(ref)) { + logWarn`Track not found "${ref}" in ${name} (${album.name})`; + } + } + } + } + + contributionData = Array.from(new Set([ + ...trackData.flatMap(track => [...track.artists || [], ...track.contributors || [], ...track.coverArtists || []]), + ...albumData.flatMap(album => [...album.artists || [], ...album.coverArtists || [], ...album.wallpaperArtists || [], ...album.bannerArtists || []]), + ...(flashData?.flatMap(flash => [...flash.contributors || []]) || []) + ])); + + // Now that we have all the data, resolve references all 8efore actually + // gener8ting any of the pages, 8ecause page gener8tion is going to involve + // accessing these references a lot, and there's no reason to resolve them + // more than once. (We 8uild a few additional links that can't 8e cre8ted + // at initial data processing time here too.) + + const filterNullArray = (parent, key) => { + for (const obj of parent) { + const array = obj[key]; + for (let i = 0; i < array.length; i++) { + if (!array[i]) { + const prev = array[i - 1] && array[i - 1].name; + const next = array[i + 1] && array[i + 1].name; + logWarn`Unexpected null in ${obj.name} (${obj.what}) (array key ${key} - prev: ${prev}, next: ${next})`; + } + } + array.splice(0, array.length, ...array.filter(Boolean)); + } + }; + + const filterNullValue = (parent, key) => { + parent.splice(0, parent.length, ...parent.filter(obj => { + if (!obj[key]) { + logWarn`Unexpected null in ${obj.name} (value key ${key})`; + } + })); + }; + + trackData.forEach(track => mapInPlace(track.references, search.track)); + trackData.forEach(track => track.aka = search.track(track.aka)); + trackData.forEach(track => mapInPlace(track.artTags, search.tag)); + albumData.forEach(album => mapInPlace(album.groups, search.group)); + albumData.forEach(album => mapInPlace(album.artTags, search.tag)); + artistAliasData.forEach(artist => artist.alias = search.artist(artist.alias)); + contributionData.forEach(contrib => contrib.who = search.artist(contrib.who)); + + filterNullArray(trackData, 'references'); + filterNullArray(trackData, 'artTags'); + filterNullArray(albumData, 'groups'); + filterNullArray(albumData, 'artTags'); + filterNullValue(artistAliasData, 'alias'); + filterNullValue(contributionData, 'who'); + + trackData.forEach(track1 => track1.referencedBy = trackData.filter(track2 => track2.references.includes(track1))); + groupData.forEach(group => group.albums = albumData.filter(album => album.groups.includes(group))); + tagData.forEach(tag => tag.things = C.sortByArtDate([...albumData, ...trackData]).filter(thing => thing.artTags.includes(tag))); + + groupData.forEach(group => group.category = groupCategoryData.find(x => x.name === group.category)); + groupCategoryData.forEach(category => category.groups = groupData.filter(x => x.category === category)); + + trackData.forEach(track => track.otherReleases = [ + track.aka, + ...trackData.filter(({ aka }) => aka === track || (track.aka && aka === track.aka)), + ].filter(x => x && x !== track)); + + if (wikiInfo.features.flashesAndGames) { + flashData.forEach(flash => mapInPlace(flash.tracks, search.track)); + flashData.forEach(flash => flash.act = flashActData.find(act => act.name === flash.act)); + flashActData.forEach(act => act.flashes = flashData.filter(flash => flash.act === act)); + + filterNullArray(flashData, 'tracks'); + + trackData.forEach(track => track.flashes = flashData.filter(flash => flash.tracks.includes(track))); + } + + artistData.forEach(artist => { + const filterProp = (array, prop) => array.filter(thing => thing[prop]?.some(({ who }) => who === artist)); + const filterCommentary = array => array.filter(thing => thing.commentary && thing.commentary.replace(/<\/?b>/g, '').includes('' + artist.name + ':')); + artist.tracks = { + asArtist: filterProp(trackData, 'artists'), + asCommentator: filterCommentary(trackData), + asContributor: filterProp(trackData, 'contributors'), + asCoverArtist: filterProp(trackData, 'coverArtists'), + asAny: trackData.filter(track => ( + [...track.artists, ...track.contributors, ...track.coverArtists || []].some(({ who }) => who === artist) + )) + }; + artist.albums = { + asArtist: filterProp(albumData, 'artists'), + asCommentator: filterCommentary(albumData), + asCoverArtist: filterProp(albumData, 'coverArtists'), + asWallpaperArtist: filterProp(albumData, 'wallpaperArtists'), + asBannerArtist: filterProp(albumData, 'bannerArtists') + }; + if (wikiInfo.features.flashesAndGames) { + artist.flashes = { + asContributor: filterProp(flashData, 'contributors') + }; + } + }); + + officialAlbumData = albumData.filter(album => album.groups.some(group => group.directory === C.OFFICIAL_GROUP_DIRECTORY)); + fandomAlbumData = albumData.filter(album => album.groups.every(group => group.directory !== C.OFFICIAL_GROUP_DIRECTORY)); + + // Makes writing a little nicer on CPU theoretically, 8ut also costs in + // performance right now 'cuz it'll w8 for file writes to 8e completed + // 8efore moving on to more data processing. So, defaults to zero, which + // disa8les the queue feature altogether. + queueSize = +(miscOptions['queue-size'] ?? 0); + + // NOT for ena8ling or disa8ling specific features of the site! + // This is only in charge of what general groups of files to 8uild. + // They're here to make development quicker when you're only working + // on some particular area(s) of the site rather than making changes + // across all of them. + const writeFlags = await parseOptions(process.argv.slice(2), { + all: {type: 'flag'}, // Defaults to true if none 8elow specified. + + album: {type: 'flag'}, + artist: {type: 'flag'}, + commentary: {type: 'flag'}, + flash: {type: 'flag'}, + group: {type: 'flag'}, + list: {type: 'flag'}, + misc: {type: 'flag'}, + news: {type: 'flag'}, + static: {type: 'flag'}, + tag: {type: 'flag'}, + track: {type: 'flag'}, + + [parseOptions.handleUnknown]: () => {} + }); + + const writeAll = !Object.keys(writeFlags).length || writeFlags.all; + + logInfo`Writing site pages: ${writeAll ? 'all' : Object.keys(writeFlags).join(', ')}`; + + await writeSymlinks(); + await writeSharedFilesAndPages({strings: defaultStrings}); + + const buildDictionary = { + misc: writeMiscellaneousPages, + news: writeNewsPages, + list: writeListingPages, + tag: writeTagPages, + commentary: writeCommentaryPages, + static: writeStaticPages, + group: writeGroupPages, + album: writeAlbumPages, + track: writeTrackPages, + artist: writeArtistPages, + flash: writeFlashPages + }; + + const buildSteps = (writeAll + ? Object.values(buildDictionary) + : (Object.entries(buildDictionary) + .filter(([ flag ]) => writeFlags[flag]) + .map(([ flag, fn ]) => fn))); + + // *NB: While what's 8elow is 8asically still true in principle, the + // format is QUITE DIFFERENT than what's descri8ed here! There + // will 8e actual document8tion on like, what the return format + // looks like soon, once we implement a 8unch of other pages and + // are certain what they actually, uh, will look like, in the end.* + // + // The writeThingPages functions don't actually immediately do any file + // writing themselves; an initial call will only gather the relevant data + // which is *then* used for writing. So the return value is a function + // (or an array of functions) which expects {writePage, strings}, and + // *that's* what we call after -- multiple times, once for each language. + let writes; + { + let error = false; + + writes = buildSteps.flatMap(fn => { + const fns = fn() || []; + + // Do a quick valid8tion! If one of the writeThingPages functions go + // wrong, this will stall out early and tell us which did. + if (!Array.isArray(fns)) { + logError`${fn.name} didn't return an array!`; + error = true; + } else if (fns.every(entry => Array.isArray(entry))) { + if (!( + fns.every(entry => entry.every(obj => typeof obj === 'object')) && + fns.every(entry => entry.every(obj => { + const result = validateWriteObject(obj); + if (result.error) { + logError`Validating write object failed: ${result.error}`; + return false; + } else { + return true; + } + })) + )) { + logError`${fn.name} uses updated format, but entries are invalid!`; + error = true; + } + + return fns.flatMap(writes => writes); + } else if (fns.some(fn => typeof fn !== 'function')) { + logError`${fn.name} didn't return all functions or all arrays!`; + error = true; + } + + return fns; + }); + + if (error) { + return; + } + + // The modern(TM) return format for each writeThingPages function is an + // array of arrays, each of which's items are 8ig Complicated Objects + // that 8asically look like {type, path, content}. 8ut surprise, these + // aren't actually implemented in most places yet! So, we transform + // stuff in the old format here. 'Scept keep in mind, the OLD FORMAT + // doesn't really give us most of the info we want for Cool And Modern + // Reasons, so they're going into a fancy {type: 'legacy'} sort of + // o8ject, with a plain {write} property for, uh, the writing stuff, + // same as usual. + // + // I promise this document8tion will get 8etter when we make progress + // actually moving old pages over. Also it'll 8e hecks of less work + // than previous restructures, don't worry. + writes = writes.map(entry => + typeof entry === 'object' ? entry : + typeof entry === 'function' ? {type: 'legacy', write: entry} : + {type: 'wut', entry}); + + const wut = writes.filter(({ type }) => type === 'wut'); + if (wut.length) { + // Oh g*d oh h*ck. + logError`Uhhhhh writes contains something 8esides o8jects and functions?`; + logError`Definitely a 8ug!`; + console.log(wut); + return; + } + } + + const localizedWrites = writes.filter(({ type }) => type === 'page' || type === 'legacy'); + const dataWrites = writes.filter(({ type }) => type === 'data'); + + await progressPromiseAll(`Writing data files shared across languages.`, queue( + // TODO: This only supports one <>-style argument. + dataWrites.map(({path, data}) => () => writeData(path[0], path[1], data())), + queueSize + )); + + await wrapLanguages(async ({strings, ...opts}, i, entries) => { + console.log(`\x1b[34;1m${ + (`[${i + 1}/${entries.length}] ${strings.code} (-> /${opts.baseDirectory}) ` + .padEnd(60, '-')) + }\x1b[0m`); + await progressPromiseAll(`Writing ${strings.code}`, queue( + localizedWrites.map(({type, ...props}) => () => { + switch (type) { + case 'legacy': { + const { write } = props; + return write({strings, ...opts}); + } + case 'page': { + const { path, page } = props; + // TODO: This only supports one <>-style argument. + return opts.writePage(path[0], path[1], ({to}) => page({strings, to})); + } + } + }), + queueSize + )); + }, writeOneLanguage); + + decorateTime.displayTime(); + + // The single most important step. + logInfo`Written!`; +} + +main().catch(error => console.error(error)); diff --git a/upd8/strings-default.json b/upd8/strings-default.json new file mode 100644 index 00000000..7a948d64 --- /dev/null +++ b/upd8/strings-default.json @@ -0,0 +1,305 @@ +{ + "meta.languageCode": "en", + "count.tracks": "{TRACKS}", + "count.tracks.withUnit.zero": "", + "count.tracks.withUnit.one": "{TRACKS} track", + "count.tracks.withUnit.two": "", + "count.tracks.withUnit.few": "", + "count.tracks.withUnit.many": "", + "count.tracks.withUnit.other": "{TRACKS} tracks", + "count.albums": "{ALBUMS}", + "count.albums.withUnit.zero": "", + "count.albums.withUnit.one": "{ALBUMS} album", + "count.albums.withUnit.two": "", + "count.albums.withUnit.two": "", + "count.albums.withUnit.few": "", + "count.albums.withUnit.many": "", + "count.albums.withUnit.other": "{ALBUMS} albums", + "count.commentaryEntries": "{ENTRIES}", + "count.commentaryEntries.withUnit.zero": "", + "count.commentaryEntries.withUnit.one": "{ENTRIES} entry", + "count.commentaryEntries.withUnit.two": "", + "count.commentaryEntries.withUnit.few": "", + "count.commentaryEntries.withUnit.many": "", + "count.commentaryEntries.withUnit.other": "{ENTRIES} entries", + "count.contributions": "{CONTRIBUTIONS}", + "count.contributions.withUnit.zero": "", + "count.contributions.withUnit.one": "{CONTRIBUTIONS} contribution", + "count.contributions.withUnit.two": "", + "count.contributions.withUnit.few": "", + "count.contributions.withUnit.many": "", + "count.contributions.withUnit.other": "{CONTRIBUTIONS} contributions", + "count.coverArts": "{COVER_ARTS}", + "count.coverArts.withUnit.zero": "", + "count.coverArts.withUnit.one": "{COVER_ARTS} cover art", + "count.coverArts.withUnit.two": "", + "count.coverArts.withUnit.few": "", + "count.coverArts.withUnit.many": "", + "count.coverArts.withUnit.other": "{COVER_ARTS} cover arts", + "count.timesReferenced": "{TIMES_REFERENCED}", + "count.timesReferenced.withUnit.zero": "", + "count.timesReferenced.withUnit.one": "{TIMES_REFERENCED} time referenced", + "count.timesReferenced.withUnit.two": "", + "count.timesReferenced.withUnit.few": "", + "count.timesReferenced.withUnit.many": "", + "count.timesReferenced.withUnit.other": "{TIMES_REFERENCED} times referenced", + "count.words": "{WORDS}", + "count.words.thousand": "{WORDS}k", + "count.words.withUnit.zero": "", + "count.words.withUnit.one": "{WORDS} word", + "count.words.withUnit.two": "", + "count.words.withUnit.few": "", + "count.words.withUnit.many": "", + "count.words.withUnit.other": "{WORDS} words", + "count.timesUsed": "{TIMES_USED}", + "count.timesUsed.withUnit.zero": "", + "count.timesUsed.withUnit.one": "used {TIMES_USED} time", + "count.timesUsed.withUnit.two": "", + "count.timesUsed.withUnit.few": "", + "count.timesUsed.withUnit.many": "", + "count.timesUsed.withUnit.other": "used {TIMES_USED} times", + "count.index.zero": "", + "count.index.one": "{INDEX}st", + "count.index.two": "{INDEX}nd", + "count.index.few": "{INDEX}rd", + "count.index.many": "", + "count.index.other": "{INDEX}th", + "count.duration.hours": "{HOURS}:{MINUTES}:{SECONDS}", + "count.duration.hours.withUnit": "{HOURS}:{MINUTES}:{SECONDS} hours", + "count.duration.minutes": "{MINUTES}:{SECONDS}", + "count.duration.minutes.withUnit": "{MINUTES}:{SECONDS} minutes", + "count.duration.approximate": "~{DURATION}", + "count.duration.missing": "_:__", + "releaseInfo.by": "By {ARTISTS}.", + "releaseInfo.from": "From {ALBUM}.", + "releaseInfo.coverArtBy": "Cover art by {ARTISTS}.", + "releaseInfo.wallpaperArtBy": "Wallpaper art by {ARTISTS}.", + "releaseInfo.bannerArtBy": "Banner art by {ARTISTS}.", + "releaseInfo.released": "Released {DATE}.", + "releaseInfo.artReleased": "Art released {DATE}.", + "releaseInfo.addedToWiki": "Added to wiki {DATE}.", + "releaseInfo.duration": "Duration: {DURATION}.", + "releaseInfo.viewCommentary": "View {LINK}!", + "releaseInfo.viewCommentary.link": "commentary page", + "releaseInfo.listenOn": "Listen on {LINKS}.", + "releaseInfo.listenOn.noLinks": "This track has no URLs at which it can be listened.", + "releaseInfo.visitOn": "Visit on {LINKS}.", + "releaseInfo.playOn": "Play on {LINKS}.", + "releaseInfo.alsoReleasedAs": "Also released as:", + "releaseInfo.alsoReleasedAs.item": "{TRACK} (on {ALBUM})", + "releaseInfo.contributors": "Contributors:", + "releaseInfo.tracksReferenced": "Tracks that {TRACK} references:", + "releaseInfo.tracksThatReference": "Tracks that reference {TRACK}:", + "releaseInfo.flashesThatFeature": "Flashes & games that feature {TRACK}:", + "releaseInfo.flashesThatFeature.item": "{FLASH}", + "releaseInfo.flashesThatFeature.item.asDifferentRelease": "{FLASH} (as {TRACK})", + "releaseInfo.lyrics": "Lyrics:", + "releaseInfo.artistCommentary": "Artist commentary:", + "releaseInfo.artistCommentary.seeOriginalRelease": "See {ORIGINAL}!", + "releaseInfo.artTags": "Tags:", + "releaseInfo.note": "Note:", + "trackList.group": "{GROUP} ({DURATION}):", + "trackList.item.withDuration": "({DURATION}) {TRACK}", + "trackList.item.withDuration.withArtists": "({DURATION}) {TRACK} {BY}", + "trackList.item.withArtists": "{TRACK} {BY}", + "trackList.item.withArtists.by": "by {ARTISTS}", + "trackList.item.rerelease": "{TRACK} (re-release)", + "misc.alt.albumCover": "album cover", + "misc.alt.albumBanner": "album banner", + "misc.alt.trackCover": "track cover", + "misc.alt.artistAvatar": "artist avatar", + "misc.alt.flashArt": "flash art", + "misc.chronology.seeArtistPages": "(See artist pages for chronology info!)", + "misc.chronology.heading.coverArt": "{INDEX} cover art by {ARTIST}", + "misc.chronology.heading.flash": "{INDEX} flash/game by {ARTIST}", + "misc.chronology.heading.track": "{INDEX} track by {ARTIST}", + "misc.external.domain": "External ({DOMAIN})", + "misc.external.bandcamp": "Bandcamp", + "misc.external.bandcamp.domain": "Bandcamp ({DOMAIN})", + "misc.external.deviantart": "DeviantArt", + "misc.external.instagram": "Instagram", + "misc.external.mastodon": "Mastodon", + "misc.external.mastodon.domain": "Mastodon ({DOMAIN})", + "misc.external.patreon": "Patreon", + "misc.external.poetryFoundation": "Poetry Foundation", + "misc.external.soundcloud": "SoundCloud", + "misc.external.tumblr": "Tumblr", + "misc.external.twitter": "Twitter", + "misc.external.wikipedia": "Wikipedia", + "misc.external.youtube": "YouTube", + "misc.external.youtube.playlist": "YouTube (playlist)", + "misc.external.youtube.fullAlbum": "YouTube (full album)", + "misc.external.flash.bgreco": "{LINK} (HQ Audio)", + "misc.external.flash.homestuck.page": "{LINK} (page {PAGE})", + "misc.external.flash.homestuck.secret": "{LINK} (secret page)", + "misc.external.flash.youtube": "{LINK} (on any device)", + "misc.nav.previous": "Previous", + "misc.nav.next": "Next", + "misc.nav.info": "Info", + "misc.nav.gallery": "Gallery", + "misc.skippers.skipToContent": "Skip to content", + "misc.skippers.skipToSidebar": "Skip to sidebar", + "misc.skippers.skipToSidebar.left": "Skip to sidebar (left)", + "misc.skippers.skipToSidebar.right": "Skip to sidebar (right)", + "misc.skippers.skipToFooter": "Skip to footer", + "misc.jumpTo": "Jump to:", + "misc.jumpTo.withLinks": "Jump to: {LINKS}.", + "misc.contentWarnings": "cw: {WARNINGS}", + "misc.contentWarnings.reveal": "click to show", + "misc.albumGridDetails": "({TRACKS}, {TIME})", + "homepage.title": "{TITLE}", + "homepage.news.title": "News", + "homepage.news.entry.viewRest": "(View rest of entry!)", + "albumSidebar.trackList.group": "{GROUP}", + "albumSidebar.trackList.group.withRange": "{GROUP} ({RANGE})", + "albumSidebar.trackList.item": "{TRACK}", + "albumSidebar.groupBox.title": "{GROUP}", + "albumSidebar.groupBox.next": "Next: {ALBUM}", + "albumSidebar.groupBox.previous": "Previous: {ALBUM}", + "albumPage.title": "{ALBUM}", + "albumPage.nav.album": "{ALBUM}", + "albumPage.nav.randomTrack": "Random Track", + "albumCommentaryPage.title": "{ALBUM} - Commentary", + "albumCommentaryPage.infoLine": "{WORDS} across {ENTRIES}.", + "albumCommentaryPage.nav.album": "Album: {ALBUM}", + "albumCommentaryPage.entry.title.albumCommentary": "Album commentary", + "albumCommentaryPage.entry.title.trackCommentary": "{TRACK}", + "artistPage.title": "{ARTIST}", + "artistPage.creditList.album": "{ALBUM}", + "artistPage.creditList.album.withDate": "{ALBUM} ({DATE})", + "artistPage.creditList.album.withDate.withDuration": "{ALBUM} ({DATE}; {DURATION})", + "artistPage.creditList.flashAct": "{ACT}", + "artistPage.creditList.flashAct.withDateRange": "{ACT} ({DATE_RANGE})", + "artistPage.creditList.entry.track": "{TRACK}", + "artistPage.creditList.entry.track.withDuration": "({DURATION}) {TRACK}", + "artistPage.creditList.entry.album.coverArt": "(cover art)", + "artistPage.creditList.entry.album.wallpaperArt": "(wallpaper art)", + "artistPage.creditList.entry.album.bannerArt": "(banner art)", + "artistPage.creditList.entry.album.commentary": "(album commentary)", + "artistPage.creditList.entry.flash": "{FLASH}", + "artistPage.creditList.entry.rerelease": "{ENTRY} (re-release)", + "artistPage.creditList.entry.withContribution": "{ENTRY} ({CONTRIBUTION})", + "artistPage.creditList.entry.withArtists": "{ENTRY} (with {ARTISTS})", + "artistPage.creditList.entry.withArtists.withContribution": "{ENTRY} ({CONTRIBUTION}; with {ARTISTS})", + "artistPage.contributedDurationLine": "{ARTIST} has contributed {DURATION} of music shared on this wiki.", + "artistPage.musicGroupsLine": "Contributed music to groups: {GROUPS}", + "artistPage.artGroupsLine": "Contributed art to groups: {GROUPS}", + "artistPage.groupsLine.item": "{GROUP} ({CONTRIBUTIONS})", + "artistPage.trackList.title": "Tracks", + "artistPage.unreleasedTrackList.title": "Unreleased Tracks", + "artistPage.artList.title": "Art", + "artistPage.flashList.title": "Flashes & Games", + "artistPage.commentaryList.title": "Commentary", + "artistPage.viewArtGallery": "View {LINK}!", + "artistPage.viewArtGallery.orBrowseList": "View {LINK}! Or browse the list:", + "artistPage.viewArtGallery.link": "art gallery", + "artistPage.nav.artist": "Artist: {ARTIST}", + "artistGalleryPage.title": "{ARTIST} - Gallery", + "artistGalleryPage.infoLine": "Contributed to {COVER_ARTS}.", + "commentaryIndex.title": "Commentary", + "commentaryIndex.infoLine": "{WORDS} across {ENTRIES}, in all.", + "commentaryIndex.albumList.title": "Choose an album:", + "commentaryIndex.albumList.item": "{ALBUM} ({WORDS} across {ENTRIES})", + "flashIndex.title": "Flashes & Games", + "flashPage.title": "{FLASH}", + "flashPage.nav.flash": "{FLASH}", + "groupSidebar.title": "Groups", + "groupSidebar.groupList.category": "{CATEGORY}", + "groupSidebar.groupList.item": "{GROUP}", + "groupPage.nav.group": "Group: {GROUP}", + "groupInfoPage.title": "{GROUP}", + "groupInfoPage.viewAlbumGallery": "View {LINK}! Or browse the list:", + "groupInfoPage.viewAlbumGallery.link": "album gallery", + "groupInfoPage.albumList.title": "Albums", + "groupInfoPage.albumList.item": "({YEAR}) {ALBUM}", + "groupGalleryPage.title": "{GROUP} - Gallery", + "groupGalleryPage.infoLine": "{TRACKS} across {ALBUMS}, totaling {TIME}.", + "listingIndex.title": "Listings", + "listingIndex.infoLine": "{WIKI}: {TRACKS} across {ALBUMS}, totaling {DURATION}.", + "listingIndex.exploreList": "Feel free to explore any of the listings linked below and in the sidebar!", + "listingPage.listAlbums.byName.title": "Albums - by Name", + "listingPage.listAlbums.byName.item": "{ALBUM} ({TRACKS})", + "listingPage.listAlbums.byTracks.title": "Albums - by Tracks", + "listingPage.listAlbums.byTracks.item": "{ALBUM} ({TRACKS})", + "listingPage.listAlbums.byDuration.title": "Albums - by Duration", + "listingPage.listAlbums.byDuration.item": "{ALBUM} ({DURATION})", + "listingPage.listAlbums.byDate.title": "Albums - by Date", + "listingPage.listAlbums.byDate.item": "{ALBUM} ({DATE})", + "listingPage.listAlbums.byDateAdded.title": "Albums - by Date Added to Wiki", + "listingPage.listAlbums.byDateAdded.date": "{DATE}", + "listingPage.listAlbums.byDateAdded.album": "{ALBUM}", + "listingPage.listArtists.byName.title": "Artists - by Name", + "listingPage.listArtists.byName.item": "{ARTIST} ({CONTRIBUTIONS})", + "listingPage.listArtists.byContribs.title": "Artists - by Contributions", + "listingPage.listArtists.byContribs.item": "{ARTIST} ({CONTRIBUTIONS})", + "listingPage.listArtists.byCommentary.title": "Artists - by Commentary Entries", + "listingPage.listArtists.byCommentary.item": "{ARTIST} ({ENTRIES})", + "listingPage.listArtists.byDuration.title": "Artists - by Duration", + "listingPage.listArtists.byDuration.item": "{ARTIST} ({DURATION})", + "listingPage.listArtists.byLatest.title": "Artists - by Latest Contribution", + "listingPage.listArtists.byLatest.item": "{ARTIST} ({DATE})", + "listingPage.listGroups.byName.title": "Groups - by Name", + "listingPage.listGroups.byName.item": "{GROUP} ({GALLERY})", + "listingPage.listGroups.byName.item.gallery": "Gallery", + "listingPage.listGroups.byCategory.title": "Groups - by Category", + "listingPage.listGroups.byCategory.category": "{CATEGORY}", + "listingPage.listGroups.byCategory.group": "{GROUP} ({GALLERY})", + "listingPage.listGroups.byCategory.group.gallery": "Gallery", + "listingPage.listGroups.byAlbums.title": "Groups - by Albums", + "listingPage.listGroups.byAlbums.item": "{GROUP} ({ALBUMS})", + "listingPage.listGroups.byTracks.title": "Groups - by Tracks", + "listingPage.listGroups.byTracks.item": "{GROUP} ({TRACKS})", + "listingPage.listGroups.byDuration.title": "Groups - by Duration", + "listingPage.listGroups.byDuration.item": "{GROUP} ({DURATION})", + "listingPage.listGroups.byLatest.title": "Groups - by Latest Album", + "listingPage.listGroups.byLatest.item": "{GROUP} ({DATE})", + "listingPage.listTracks.byName.title": "Tracks - by Name", + "listingPage.listTracks.byName.item": "{TRACK}", + "listingPage.listTracks.byAlbum.title": "Tracks - by Album", + "listingPage.listTracks.byAlbum.album": "{ALBUM}", + "listingPage.listTracks.byAlbum.track": "{TRACK}", + "listingPage.listTracks.byDate.title": "Tracks - by Date", + "listingPage.listTracks.byDate.album": "{ALBUM} ({DATE})", + "listingPage.listTracks.byDate.track": "{TRACK}", + "listingPage.listTracks.byDate.track.rerelease": "{TRACK} (re-release)", + "listingPage.listTracks.byDuration.title": "Tracks - by Duration", + "listingPage.listTracks.byDuration.item": "{TRACK} ({DURATION})", + "listingPage.listTracks.byDurationInAlbum.title": "Tracks - by Duration (in Album)", + "listingPage.listTracks.byDurationInAlbum.album": "{ALBUM}", + "listingPage.listTracks.byDurationInAlbum.track": "{TRACK} ({DURATION})", + "listingPage.listTracks.byTimesReferenced.title": "Tracks - by Times Referenced", + "listingPage.listTracks.byTimesReferenced.item": "{TRACK} ({TIMES_REFERENCED})", + "listingPage.listTracks.inFlashes.byAlbum.title": "Tracks - in Flashes & Games (by Album)", + "listingPage.listTracks.inFlashes.byAlbum.album": "{ALBUM} ({DATE})", + "listingPage.listTracks.inFlashes.byAlbum.track": "{TRACK} (in {FLASHES})", + "listingPage.listTracks.inFlashes.byFlash.title": "Tracks - in Flashes & Games (by Flash)", + "listingPage.listTracks.inFlashes.byFlash.flash": "{FLASH} ({DATE})", + "listingPage.listTracks.inFlashes.byFlash.track": "{TRACK} (from {ALBUM})", + "listingPage.listTracks.withLyrics.title": "Tracks - with Lyrics", + "listingPage.listTracks.withLyrics.album": "{ALBUM} ({DATE})", + "listingPage.listTracks.withLyrics.track": "{TRACK}", + "listingPage.listTags.byName.title": "Tags - by Name", + "listingPage.listTags.byName.item": "{TAG} ({TIMES_USED})", + "listingPage.listTags.byUses.title": "Tags - by Uses", + "listingPage.listTags.byUses.item": "{TAG} ({TIMES_USED})", + "listingPage.misc.trackContributors": "Track Contributors", + "listingPage.misc.artContributors": "Art Contributors", + "listingPage.misc.artAndFlashContributors": "Art & Flash Contributors", + "newsIndex.title": "News", + "newsIndex.entry.viewRest": "(View rest of entry!)", + "newsEntryPage.title": "{ENTRY}", + "newsEntryPage.published": "(Published {DATE}.)", + "newsEntryPage.nav.news": "News", + "newsEntryPage.nav.entry": "{DATE}: {ENTRY}", + "redirectPage.title": "Moved to {TITLE}", + "redirectPage.infoLine": "This page has been moved to {TARGET}.", + "tagPage.title": "{TAG}", + "tagPage.infoLine": "Appears in {COVER_ARTS}.", + "tagPage.nav.tag": "Tag: {TAG}", + "trackPage.title": "{TRACK}", + "trackPage.referenceList.fandom": "Fandom:", + "trackPage.referenceList.official": "Official:", + "trackPage.nav.track": "{TRACK}", + "trackPage.nav.track.withNumber": "{NUMBER}. {TRACK}", + "trackPage.nav.random": "Random" +} diff --git a/upd8/util.js b/upd8/util.js new file mode 100644 index 00000000..4c4186f7 --- /dev/null +++ b/upd8/util.js @@ -0,0 +1,423 @@ +// This is used by upd8.js! It's part of the 8ackend. Read the notes there if +// you're curious. +// +// Friendly(!) disclaimer: these utility functions haven't 8een tested all that +// much. Do not assume it will do exactly what you want it to do in all cases. +// It will likely only do exactly what I want it to, and only in the cases I +// decided were relevant enough to 8other handling. + +'use strict'; + +// Apparently JavaScript doesn't come with a function to split an array into +// chunks! Weird. Anyway, this is an awesome place to use a generator, even +// though we don't really make use of the 8enefits of generators any time we +// actually use this. 8ut it's still awesome, 8ecause I say so. +module.exports.splitArray = function*(array, fn) { + let lastIndex = 0; + while (lastIndex < array.length) { + let nextIndex = array.findIndex((item, index) => index >= lastIndex && fn(item)); + if (nextIndex === -1) { + nextIndex = array.length; + } + yield array.slice(lastIndex, nextIndex); + // Plus one because we don't want to include the dividing line in the + // next array we yield. + lastIndex = nextIndex + 1; + } +}; + +// This function's name is a joke. Jokes! Hahahahahahahaha. Funny. +module.exports.joinNoOxford = function(array, plural = 'and') { + array = array.filter(Boolean); + + if (array.length === 0) { + // ???????? + return ''; + } + + if (array.length === 1) { + return array[0]; + } + + if (array.length === 2) { + return `${array[0]} ${plural} ${array[1]}`; + } + + return `${array.slice(0, -1).join(', ')} ${plural} ${array[array.length - 1]}`; +}; + +module.exports.progressPromiseAll = function (msgOrMsgFn, array) { + if (!array.length) { + return Promise.resolve([]); + } + + const msgFn = (typeof msgOrMsgFn === 'function' + ? msgOrMsgFn + : () => msgOrMsgFn); + + let done = 0, total = array.length; + process.stdout.write(`\r${msgFn()} [0/${total}]`); + const start = Date.now(); + return Promise.all(array.map(promise => promise.then(val => { + done++; + // const pc = `${done}/${total}`; + const pc = (Math.round(done / total * 1000) / 10 + '%').padEnd('99.9%'.length, ' '); + if (done === total) { + const time = Date.now() - start; + process.stdout.write(`\r\x1b[2m${msgFn()} [${pc}] \x1b[0;32mDone! \x1b[0;2m(${time} ms) \x1b[0m\n`) + } else { + process.stdout.write(`\r${msgFn()} [${pc}] `); + } + return val; + }))); +}; + +module.exports.queue = function (array, max = 50) { + if (max === 0) { + return array.map(fn => fn()); + } + + const begin = []; + let current = 0; + const ret = array.map(fn => new Promise((resolve, reject) => { + begin.push(() => { + current++; + Promise.resolve(fn()).then(value => { + current--; + if (current < max && begin.length) { + begin.shift()(); + } + resolve(value); + }, reject); + }); + })); + + for (let i = 0; i < max && begin.length; i++) { + begin.shift()(); + } + + return ret; +}; + +module.exports.delay = ms => new Promise(res => setTimeout(res, ms)); + +module.exports.th = function (n) { + if (n % 10 === 1 && n !== 11) { + return n + 'st'; + } else if (n % 10 === 2 && n !== 12) { + return n + 'nd'; + } else if (n % 10 === 3 && n !== 13) { + return n + 'rd'; + } else { + return n + 'th'; + } +}; + +// My function names just keep getting 8etter. +module.exports.s = function (n, word) { + return `${n} ${word}` + (n === 1 ? '' : 's'); +}; + +// Hey, did you know I apparently put a space 8efore the parameters in function +// names? 8ut only in function expressions, not declar8tions? I mean, I guess +// you did. You're pro8a8ly more familiar with my code than I am 8y this +// point. I haven't messed with any of this code in ages. Yay!!!!!!!! +// +// This function only does anything on o8jects you're going to 8e reusing. +// Argua8ly I could use a WeakMap here, 8ut since the o8ject needs to 8e +// reused to 8e useful anyway, I just store the result with a symbol. +// Sorry if it's 8een frozen I guess?? +module.exports.cacheOneArg = function (fn) { + const symbol = Symbol('Cache'); + return arg => { + if (!arg[symbol]) { + arg[symbol] = fn(arg); + } + return arg[symbol]; + }; +}; + +const decorateTime = function (functionToBeWrapped) { + const fn = function(...args) { + const start = Date.now(); + const ret = functionToBeWrapped(...args); + const end = Date.now(); + fn.timeSpent += end - start; + fn.timesCalled++; + return ret; + }; + + fn.wrappedName = functionToBeWrapped.name; + fn.timeSpent = 0; + fn.timesCalled = 0; + fn.displayTime = function() { + const averageTime = fn.timeSpent / fn.timesCalled; + console.log(`\x1b[1m${fn.wrappedName}(...):\x1b[0m ${fn.timeSpent} ms / ${fn.timesCalled} calls \x1b[2m(avg: ${averageTime} ms)\x1b[0m`); + }; + + decorateTime.decoratedFunctions.push(fn); + + return fn; +}; + +decorateTime.decoratedFunctions = []; +decorateTime.displayTime = function() { + if (decorateTime.decoratedFunctions.length) { + console.log(`\x1b[1mdecorateTime results: ` + '-'.repeat(40) + '\x1b[0m'); + for (const fn of decorateTime.decoratedFunctions) { + fn.displayTime(); + } + } +}; + +module.exports.decorateTime = decorateTime; + +// Stolen as #@CK from mtui! +const parseOptions = async function(options, optionDescriptorMap) { + // This function is sorely lacking in comments, but the basic usage is + // as such: + // + // options is the array of options you want to process; + // optionDescriptorMap is a mapping of option names to objects that describe + // the expected value for their corresponding options. + // Returned is a mapping of any specified option names to their values, or + // a process.exit(1) and error message if there were any issues. + // + // Here are examples of optionDescriptorMap to cover all the things you can + // do with it: + // + // optionDescriptorMap: { + // 'telnet-server': {type: 'flag'}, + // 't': {alias: 'telnet-server'} + // } + // + // options: ['t'] -> result: {'telnet-server': true} + // + // optionDescriptorMap: { + // 'directory': { + // type: 'value', + // validate(name) { + // // const whitelistedDirectories = ['apple', 'banana'] + // if (whitelistedDirectories.includes(name)) { + // return true + // } else { + // return 'a whitelisted directory' + // } + // } + // }, + // 'files': {type: 'series'} + // } + // + // ['--directory', 'apple'] -> {'directory': 'apple'} + // ['--directory', 'artichoke'] -> (error) + // ['--files', 'a', 'b', 'c', ';'] -> {'files': ['a', 'b', 'c']} + // + // TODO: Be able to validate the values in a series option. + + const handleDashless = optionDescriptorMap[parseOptions.handleDashless]; + const handleUnknown = optionDescriptorMap[parseOptions.handleUnknown]; + const result = Object.create(null); + for (let i = 0; i < options.length; i++) { + const option = options[i]; + if (option.startsWith('--')) { + // --x can be a flag or expect a value or series of values + let name = option.slice(2).split('=')[0]; // '--x'.split('=') = ['--x'] + let descriptor = optionDescriptorMap[name]; + if (!descriptor) { + if (handleUnknown) { + handleUnknown(option); + } else { + console.error(`Unknown option name: ${name}`); + process.exit(1); + } + continue; + } + if (descriptor.alias) { + name = descriptor.alias; + descriptor = optionDescriptorMap[name]; + } + if (descriptor.type === 'flag') { + result[name] = true; + } else if (descriptor.type === 'value') { + let value = option.slice(2).split('=')[1]; + if (!value) { + value = options[++i]; + if (!value || value.startsWith('-')) { + value = null; + } + } + if (!value) { + console.error(`Expected a value for --${name}`); + process.exit(1); + } + result[name] = value; + } else if (descriptor.type === 'series') { + if (!options.slice(i).includes(';')) { + console.error(`Expected a series of values concluding with ; (\\;) for --${name}`); + process.exit(1); + } + const endIndex = i + options.slice(i).indexOf(';'); + result[name] = options.slice(i + 1, endIndex); + i = endIndex; + } + if (descriptor.validate) { + const validation = await descriptor.validate(result[name]); + if (validation !== true) { + console.error(`Expected ${validation} for --${name}`); + process.exit(1); + } + } + } else if (option.startsWith('-')) { + // mtui doesn't use any -x=y or -x y format optionuments + // -x will always just be a flag + let name = option.slice(1); + let descriptor = optionDescriptorMap[name]; + if (!descriptor) { + if (handleUnknown) { + handleUnknown(option); + } else { + console.error(`Unknown option name: ${name}`); + process.exit(1); + } + continue; + } + if (descriptor.alias) { + name = descriptor.alias; + descriptor = optionDescriptorMap[name]; + } + if (descriptor.type === 'flag') { + result[name] = true; + } else { + console.error(`Use --${name} (value) to specify ${name}`); + process.exit(1); + } + } else if (handleDashless) { + handleDashless(option); + } + } + return result; +} + +parseOptions.handleDashless = Symbol(); +parseOptions.handleUnknown = Symbol(); + +module.exports.parseOptions = parseOptions; + +// Cheap FP for a cheap dyke! +// I have no idea if this is what curry actually means. +module.exports.curry = f => x => (...args) => f(x, ...args); + +module.exports.mapInPlace = (array, fn) => array.splice(0, array.length, ...array.map(fn)); + +module.exports.filterEmptyLines = string => string.split('\n').filter(line => line.trim()).join('\n'); + +module.exports.unique = arr => Array.from(new Set(arr)); + +const logColor = color => (literals, ...values) => { + const w = s => process.stdout.write(s); + w(`\x1b[${color}m`); + for (let i = 0; i < literals.length; i++) { + w(literals[i]); + if (values[i] !== undefined) { + w(`\x1b[1m`); + w(String(values[i])); + w(`\x1b[0;${color}m`); + } + } + w(`\x1b[0m\n`); +}; + +module.exports.logInfo = logColor(2); +module.exports.logWarn = logColor(33); +module.exports.logError = logColor(31); + +module.exports.sortByName = (a, b) => { + let an = a.name.toLowerCase(); + let bn = b.name.toLowerCase(); + if (an.startsWith('the ')) an = an.slice(4); + if (bn.startsWith('the ')) bn = bn.slice(4); + return an < bn ? -1 : an > bn ? 1 : 0; +}; + +module.exports.chunkByConditions = function(array, conditions) { + if (array.length === 0) { + return []; + } else if (conditions.length === 0) { + return [array]; + } + + const out = []; + let cur = [array[0]]; + for (let i = 1; i < array.length; i++) { + const item = array[i]; + const prev = array[i - 1]; + let chunk = false; + for (const condition of conditions) { + if (condition(item, prev)) { + chunk = true; + break; + } + } + if (chunk) { + out.push(cur); + cur = [item]; + } else { + cur.push(item); + } + } + out.push(cur); + return out; +}; + +module.exports.chunkByProperties = function(array, properties) { + return module.exports.chunkByConditions(array, properties.map(p => (a, b) => { + if (a[p] instanceof Date && b[p] instanceof Date) + return +a[p] !== +b[p]; + + if (a[p] !== b[p]) return true; + + // Not sure if this line is still necessary with the specific check for + // d8tes a8ove, 8ut, uh, keeping it anyway, just in case....? + if (a[p] != b[p]) return true; + + return false; + })) + .map(chunk => ({ + ...Object.fromEntries(properties.map(p => [p, chunk[0][p]])), + chunk + })); +}; + +// Very cool function origin8ting in... http-music pro8a8ly! +// Sorry if we happen to 8e violating past-us's copyright, lmao. +module.exports.promisifyProcess = function(proc, showLogging = true) { + // Takes a process (from the child_process module) and returns a promise + // that resolves when the process exits (or rejects, if the exit code is + // non-zero). + // + // Ayy look, no alpha8etical second letter! Couldn't tell this was written + // like three years ago 8efore I was me. 8888) + + return new Promise((resolve, reject) => { + if (showLogging) { + proc.stdout.pipe(process.stdout); + proc.stderr.pipe(process.stderr); + } + + proc.on('exit', code => { + if (code === 0) { + resolve(); + } else { + reject(code); + } + }) + }) +}; + +// Stolen from jq! Which pro8a8ly stole the concept from other places. Nice. +module.exports.withEntries = (obj, fn) => Object.fromEntries(fn(Object.entries(obj))); + +// Nothin' more to it than what it says. Runs a function in-place. Provides an +// altern8tive syntax to the usual IIFEs (e.g. (() => {})()) when you want to +// open a scope and run some statements while inside an existing expression. +module.exports.call = fn => fn(); -- cgit 1.3.0-6-gf8a5