From 2260541dc69c19e7444348ac3243f96e4321b781 Mon Sep 17 00:00:00 2001 From: "(quasar) nebula" Date: Wed, 5 May 2021 10:10:50 -0300 Subject: move upd8 code files into their own directory --- upd8/main.js | 6597 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 6597 insertions(+) create mode 100755 upd8/main.js (limited to 'upd8/main.js') diff --git a/upd8/main.js b/upd8/main.js new file mode 100755 index 0000000..84bab6c --- /dev/null +++ b/upd8/main.js @@ -0,0 +1,6597 @@ +#!/usr/bin/env node + +// HEY N8RDS! +// +// This is one of the 8ACKEND FILES. It's not used anywhere on the actual site +// you are pro8a8ly using right now. +// +// Specifically, this one does all the actual work of the music wiki. The +// process looks something like this: +// +// 1. Crawl the music directories. Well, not so much "crawl" as "look inside +// the folders for each al8um, and read the metadata file descri8ing that +// al8um and the tracks within." +// +// 2. Read that metadata. I'm writing this 8efore actually doing any of the +// code, and I've gotta admit I have no idea what file format they're +// going to 8e in. May8e JSON, 8ut more likely some weird custom format +// which will 8e a lot easier to edit. +// +// 3. Generate the page files! They're just static index.html files, and are +// what gh-pages (or wherever this is hosted) will show to clients. +// Hopefully pretty minimalistic HTML, 8ut like, shrug. They'll reference +// CSS (and maaaaaaaay8e JS) files, hard-coded somewhere near the root. +// +// 4. Print an awesome message which says the process is done. This is the +// most important step. +// +// Oh yeah, like. Just run this through some relatively recent version of +// node.js and you'll 8e fine. ...Within the project root. O8viously. + +// HEY FUTURE ME!!!!!!!! Don't forget to implement artist pages! Those are, +// like, the coolest idea you've had yet, so DO NOT FORGET. (Remem8er, link +// from track listings, etc!) --- Thanks, past me. To futurerer me: an al8um +// listing page (a list of all the al8ums)! Make sure to sort these 8y date - +// we'll need a new field for al8ums. + +// ^^^^^^^^ DID THAT! 8ut also, artist images. Pro8a8ly stolen from the fandom +// wiki (I found half those images anywayz). + +// TRACK ART CREDITS. This is a must. + +// 2020-08-23 +// ATTENTION ALL 8*TCHES AND OTHER GENDER TRUCKERS: AS IT TURNS OUT, THIS CODE +// ****SUCKS****. I DON'T THINK ANYTHING WILL EVER REDEEM IT, 8UT THAT DOESN'T +// MEAN WE CAN'T TAKE SOME ACTION TO MAKE WRITING IT A LITTLE LESS TERRI8LE. +// We're gonna start defining STRUCTURES to make things suck less!!!!!!!! +// No classes 8ecause those are a huge pain and like, pro8a8ly 8ad performance +// or whatever -- just some standard structures that should 8e followed +// wherever reasona8le. Only one I need today is the contri8 one 8ut let's put +// any new general-purpose structures here too, ok? +// +// Contri8ution: {who, what, date, thing}. D8 and thing are the new fields. +// +// Use these wisely, which is to say all the time and instead of whatever +// terri8le new pseudo structure you're trying to invent!!!!!!!! +// +// Upd8 2021-01-03: Soooooooo we didn't actually really end up using these, +// lol? Well there's still only one anyway. Kinda ended up doing a 8ig refactor +// of all the o8ject structures today. It's not *especially* relevant 8ut feels +// worth mentioning? I'd get rid of this comment 8lock 8ut I like it too much! +// Even though I haven't actually reread it, lol. 8ut yeah, hopefully in the +// spirit of this "make things more consistent" attitude I 8rought up 8ack in +// August, stuff's lookin' 8etter than ever now. W00t! + +'use strict'; + +const fs = require('fs'); +const path = require('path'); +const util = require('util'); + +// I made this dependency myself! A long, long time ago. It is pro8a8ly my +// most useful li8rary ever. I'm not sure 8esides me actually uses it, though. +const fixWS = require('fix-whitespace'); +// Wait nevermind, I forgot a8out why-do-kids-love-the-taste-of-cinnamon-toast- +// crunch. THAT is my 8est li8rary. + +// The require function just returns whatever the module exports, so there's +// no reason you can't wrap it in some decorator right out of the 8ox. Which is +// exactly what we do here. +const mkdirp = util.promisify(require('mkdirp')); + +// It stands for "HTML Entities", apparently. Cursed. +const he = require('he'); + +// This is the dum8est name for a function possi8le. Like, SURE, fine, may8e +// the UNIX people had some valid reason to go with the weird truncated +// lowercased convention they did. 8ut Node didn't have to ALSO use that +// convention! Would it have 8een so hard to just name the function something +// like fs.readDirectory???????? No, it wouldn't have 8een. +const readdir = util.promisify(fs.readdir); +// 8ut okay, like, look at me. DOING THE SAME THING. See, *I* could have named +// my promisified function differently, and yet I did not. I literally cannot +// explain why. We are all used to following in the 8ad decisions of our +// ancestors, and never never never never never never never consider that hey, +// may8e we don't need to make the exact same decisions they did. Even when +// we're perfectly aware th8t's exactly what we're doing! Programmers, +// including me, are all pretty stupid. + +// 8ut I mean, come on. Look. Node decided to use readFile, instead of like, +// what, cat? Why couldn't they rename readdir too???????? As Johannes Kepler +// once so elegantly put it: "Shrug." +const readFile = util.promisify(fs.readFile); +const writeFile = util.promisify(fs.writeFile); +const access = util.promisify(fs.access); +const symlink = util.promisify(fs.symlink); +const unlink = util.promisify(fs.unlink); + +const { + cacheOneArg, + call, + chunkByConditions, + chunkByProperties, + curry, + decorateTime, + filterEmptyLines, + joinNoOxford, + mapInPlace, + logWarn, + logInfo, + logError, + parseOptions, + progressPromiseAll, + queue, + s, + sortByName, + splitArray, + th, + unique, + withEntries +} = require('./util'); + +const genThumbs = require('./gen-thumbs'); + +const C = require('../common/common'); + +const CACHEBUST = 5; + +const WIKI_INFO_FILE = 'wiki-info.txt'; +const HOMEPAGE_INFO_FILE = 'homepage.txt'; +const ARTIST_DATA_FILE = 'artists.txt'; +const FLASH_DATA_FILE = 'flashes.txt'; +const NEWS_DATA_FILE = 'news.txt'; +const TAG_DATA_FILE = 'tags.txt'; +const GROUP_DATA_FILE = 'groups.txt'; +const STATIC_PAGE_DATA_FILE = 'static-pages.txt'; +const DEFAULT_STRINGS_FILE = 'strings-default.json'; + +const CSS_FILE = 'site.css'; + +// Shared varia8les! These are more efficient to access than a shared varia8le +// (or at least I h8pe so), and are easier to pass across functions than a +// 8unch of specific arguments. +// +// Upd8: Okay yeah these aren't actually any different. Still cleaner than +// passing around a data object containing all this, though. +let dataPath; +let mediaPath; +let langPath; +let outputPath; + +let wikiInfo; +let homepageInfo; +let albumData; +let trackData; +let flashData; +let flashActData; +let newsData; +let tagData; +let groupData; +let groupCategoryData; +let staticPageData; + +let artistNames; +let artistData; +let artistAliasData; + +let officialAlbumData; +let fandomAlbumData; +let justEverythingMan; // tracks, albums, flashes -- don't forget to upd8 toAnythingMan! +let justEverythingSortedByArtDateMan; +let contributionData; + +let queueSize; + +let languages; + +const html = { + // Non-comprehensive. ::::P + selfClosingTags: ['br', 'img'], + + // Pass to tag() as an attri8utes key to make tag() return a 8lank string + // if the provided content is empty. Useful for when you'll only 8e showing + // an element according to the presence of content that would 8elong there. + onlyIfContent: Symbol(), + + tag(tagName, ...args) { + const selfClosing = html.selfClosingTags.includes(tagName); + + let openTag; + let content; + let attrs; + + if (typeof args[0] === 'object' && !Array.isArray(args[0])) { + attrs = args[0]; + content = args[1]; + } else { + content = args[0]; + } + + if (selfClosing && content) { + throw new Error(`Tag <${tagName}> is self-closing but got content!`); + } + + if (attrs?.[html.onlyIfContent] && !content) { + return ''; + } + + if (attrs) { + const attrString = html.attributes(args[0]); + if (attrString) { + openTag = `${tagName} ${attrString}`; + } + } + + if (!openTag) { + openTag = tagName; + } + + if (Array.isArray(content)) { + content = content.filter(Boolean).join('\n'); + } + + if (content) { + if (content.includes('\n')) { + return fixWS` + <${openTag}> + ${content} + + `; + } else { + return `<${openTag}>${content}`; + } + } else { + if (selfClosing) { + return `<${openTag}>`; + } else { + return `<${openTag}>`; + } + } + }, + + escapeAttributeValue(value) { + return value + .replaceAll('"', '"') + .replaceAll("'", '''); + }, + + attributes(attribs) { + return Object.entries(attribs) + .map(([ key, val ]) => { + if (!val) + return [key, val]; + else if (typeof val === 'string' || typeof val === 'boolean') + return [key, val]; + else if (typeof val === 'number') + return [key, val.toString()]; + else if (Array.isArray(val)) + return [key, val.join(' ')]; + else + throw new Error(`Attribute value for ${key} should be primitive or array, got ${typeof val}`); + }) + .filter(([ key, val ]) => val) + .map(([ key, val ]) => (typeof val === 'boolean' + ? `${key}` + : `${key}="${html.escapeAttributeValue(val)}"`)) + .join(' '); + } +}; + +const urlSpec = { + data: { + prefix: 'data/', + + paths: { + root: '', + path: '<>', + + album: 'album/<>', + artist: 'artist/<>', + track: 'track/<>' + } + }, + + localized: { + // TODO: Implement this. + // prefix: '_languageCode', + + paths: { + root: '', + path: '<>', + + home: '', + + album: 'album/<>/', + albumCommentary: 'commentary/album/<>/', + + artist: 'artist/<>/', + artistGallery: 'artist/<>/gallery/', + + commentaryIndex: 'commentary/', + + flashIndex: 'flash/', + flash: 'flash/<>/', + + groupInfo: 'group/<>/', + groupGallery: 'group/<>/gallery/', + + listingIndex: 'list/', + listing: 'list/<>/', + + newsIndex: 'news/', + newsEntry: 'news/<>/', + + staticPage: '<>/', + tag: 'tag/<>/', + track: 'track/<>/' + } + }, + + shared: { + paths: { + root: '', + path: '<>', + + commonFile: 'common/<>', + staticFile: 'static/<>' + } + }, + + media: { + prefix: 'media/', + + paths: { + root: '', + path: '<>', + + albumCover: 'album-art/<>/cover.jpg', + albumWallpaper: 'album-art/<>/bg.jpg', + albumBanner: 'album-art/<>/banner.jpg', + trackCover: 'album-art/<>/<>.jpg', + artistAvatar: 'artist-avatar/<>.jpg', + flashArt: 'flash-art/<>.jpg' + } + } +}; + +// This gets automatically switched in place when working from a baseDirectory, +// so it should never be referenced manually. +urlSpec.localizedWithBaseDirectory = { + paths: withEntries( + urlSpec.localized.paths, + entries => entries.map(([key, path]) => [key, '<>/' + path]) + ) +}; + +const linkHelper = (hrefFn, {color = true, attr = null} = {}) => + (thing, { + strings, to, + text = '', + class: className = '', + hash = '' + }) => ( + html.tag('a', { + ...attr ? attr(thing) : {}, + href: hrefFn(thing, {to}) + (hash ? (hash.startsWith('#') ? '' : '#') + hash : ''), + style: color ? getLinkThemeString(thing) : '', + class: className + }, text || thing.name) + ); + +const linkDirectory = (key, {expose = null, attr = null, ...conf} = {}) => + linkHelper((thing, {to}) => to('localized.' + key, thing.directory), { + attr: thing => ({ + ...attr ? attr(thing) : {}, + ...expose ? {[expose]: thing.directory} : {} + }), + ...conf + }); + +const linkPathname = (key, conf) => linkHelper(({directory: pathname}, {to}) => to(key, pathname), conf); +const linkIndex = (key, conf) => linkHelper((_, {to}) => to('localized.' + key), conf); + +const link = { + album: linkDirectory('album'), + albumCommentary: linkDirectory('albumCommentary'), + artist: linkDirectory('artist', {color: false}), + artistGallery: linkDirectory('artistGallery', {color: false}), + commentaryIndex: linkIndex('commentaryIndex', {color: false}), + flashIndex: linkIndex('flashIndex', {color: false}), + flash: linkDirectory('flash'), + groupInfo: linkDirectory('groupInfo'), + groupGallery: linkDirectory('groupGallery'), + home: linkIndex('home', {color: false}), + listingIndex: linkIndex('listingIndex'), + listing: linkDirectory('listing'), + newsIndex: linkIndex('newsIndex', {color: false}), + newsEntry: linkDirectory('newsEntry', {color: false}), + staticPage: linkDirectory('staticPage', {color: false}), + tag: linkDirectory('tag'), + track: linkDirectory('track', {expose: 'data-track'}), + + media: linkPathname('media.path', {color: false}), + root: linkPathname('shared.path', {color: false}), + data: linkPathname('data.path', {color: false}), + site: linkPathname('localized.path', {color: false}) +}; + +const thumbnailHelper = name => file => + file.replace(/\.(jpg|png)$/, name + '.jpg'); + +const thumb = { + medium: thumbnailHelper('.medium'), + small: thumbnailHelper('.small') +}; + +function generateURLs(fromPath) { + const getValueForFullKey = (obj, fullKey, prop = null) => { + const [ groupKey, subKey ] = fullKey.split('.'); + if (!groupKey || !subKey) { + throw new Error(`Expected group key and subkey (got ${fullKey})`); + } + + if (!obj.hasOwnProperty(groupKey)) { + throw new Error(`Expected valid group key (got ${groupKey})`); + } + + const group = obj[groupKey]; + + if (!group.hasOwnProperty(subKey)) { + throw new Error(`Expected valid subkey (got ${subKey} for group ${groupKey})`); + } + + return { + value: group[subKey], + group + }; + }; + + const generateTo = (fromPath, fromGroup) => { + const rebasePrefix = '../'.repeat((fromGroup.prefix || '').split('/').filter(Boolean).length); + + const pathHelper = (toPath, toGroup) => { + let target = toPath; + + let argIndex = 0; + target = target.replaceAll('<>', () => `<${argIndex++}>`); + + if (toGroup.prefix !== fromGroup.prefix) { + // TODO: Handle differing domains in prefixes. + target = rebasePrefix + (toGroup.prefix || '') + target; + } + + return (path.relative(fromPath, target) + + (toPath.endsWith('/') ? '/' : '')); + }; + + const groupSymbol = Symbol(); + + const groupHelper = urlGroup => ({ + [groupSymbol]: urlGroup, + ...withEntries(urlGroup.paths, entries => entries + .map(([key, path]) => [key, pathHelper(path, urlGroup)])) + }); + + const relative = withEntries(urlSpec, entries => entries + .map(([key, urlGroup]) => [key, groupHelper(urlGroup)])); + + const to = (key, ...args) => { + const { value: template, group: {[groupSymbol]: toGroup} } = getValueForFullKey(relative, key) + let result = template.replaceAll(/<([0-9]+)>/g, (match, n) => args[n]); + + // Kinda hacky lol, 8ut it works. + const missing = result.match(/<([0-9]+)>/g); + if (missing) { + throw new Error(`Expected ${missing[missing.length - 1]} arguments, got ${args.length}`); + } + + return result; + }; + + return {to, relative}; + }; + + const generateFrom = () => { + const map = withEntries(urlSpec, entries => entries + .map(([key, group]) => [key, withEntries(group.paths, entries => entries + .map(([key, path]) => [key, generateTo(path, group)]) + )])); + + const from = key => getValueForFullKey(map, key).value; + + return {from, map}; + }; + + return generateFrom(); +} + +const urls = generateURLs(); + +const searchHelper = (keys, dataFn, findFn) => ref => { + if (!ref) return null; + ref = ref.replace(new RegExp(`^(${keys.join('|')}):`), ''); + const found = findFn(ref, dataFn()); + if (!found) { + logWarn`Didn't match anything for ${ref}! (${keys.join(', ')})`; + } + return found; +}; + +const matchDirectory = (ref, data) => data.find(({ directory }) => directory === ref); + +const matchDirectoryOrName = (ref, data) => { + let thing; + + thing = matchDirectory(ref, data); + if (thing) return thing; + + thing = data.find(({ name }) => name === ref); + if (thing) return thing; + + thing = data.find(({ name }) => name.toLowerCase() === ref.toLowerCase()); + if (thing) { + logWarn`Bad capitalization: ${'\x1b[31m' + ref} -> ${'\x1b[32m' + thing.name}`; + return thing; + } + + return null; +}; + +const search = { + album: searchHelper(['album', 'album-commentary'], () => albumData, matchDirectoryOrName), + artist: searchHelper(['artist', 'artist-gallery'], () => artistData, matchDirectoryOrName), + flash: searchHelper(['flash'], () => flashData, matchDirectory), + group: searchHelper(['group', 'group-gallery'], () => groupData, matchDirectoryOrName), + listing: searchHelper(['listing'], () => listingSpec, matchDirectory), + newsEntry: searchHelper(['news-entry'], () => newsData, matchDirectory), + staticPage: searchHelper(['static'], () => staticPageData, matchDirectory), + tag: searchHelper(['tag'], () => tagData, (ref, data) => + matchDirectoryOrName(ref.startsWith('cw: ') ? ref.slice(4) : ref, data)), + track: searchHelper(['track'], () => trackData, matchDirectoryOrName) +}; + +// Localiz8tion time! Or l10n as the neeeeeeeerds call it. Which is a terri8le +// name and not one I intend on using, thank you very much. (Don't even get me +// started on """"a11y"""".) +// +// All the default strings are in strings-default.json, if you're curious what +// those actually look like. Pretty much it's "I like {ANIMAL}" for example. +// For each language, the o8ject gets turned into a single function of form +// f(key, {args}). It searches for a key in the o8ject and uses the string it +// finds (or the one in strings-default.json) as a templ8 evaluated with the +// arguments passed. (This function gets treated as an o8ject too; it gets +// the language code attached.) +// +// The function's also responsi8le for getting rid of dangerous characters +// (quotes and angle tags), though only within the templ8te (not the args), +// and it converts the keys of the arguments o8ject from camelCase to +// CONSTANT_CASE too. +function genStrings(stringsJSON, defaultJSON = null) { + // genStrings will only 8e called once for each language, and it happens + // right at the start of the program (or at least 8efore 8uilding pages). + // So, now's a good time to valid8te the strings and let any warnings be + // known. + + // May8e contrary to the argument name, the arguments should 8e o8jects, + // not actual JSON-formatted strings! + if (typeof stringsJSON !== 'object' || stringsJSON.constructor !== Object) { + return {error: `Expected an object (parsed JSON) for stringsJSON.`}; + } + if (typeof defaultJSON !== 'object') { // typeof null === object. I h8 JS. + return {error: `Expected an object (parsed JSON) or null for defaultJSON.`}; + } + + // All languages require a language code. + const code = stringsJSON['meta.languageCode']; + if (!code) { + return {error: `Missing language code.`}; + } + if (typeof code !== 'string') { + return {error: `Expected language code to be a string.`}; + } + + // Every value on the provided o8ject should be a string. + // (This is lazy, but we only 8other checking this on stringsJSON, on the + // assumption that defaultJSON was passed through this function too, and so + // has already been valid8ted.) + { + let err = false; + for (const [ key, value ] of Object.entries(stringsJSON)) { + if (typeof value !== 'string') { + logError`(${code}) The value for ${key} should be a string.`; + err = true; + } + } + if (err) { + return {error: `Expected all values to be a string.`}; + } + } + + // Checking is generally done against the default JSON, so we'll skip out + // if that isn't provided (which should only 8e the case when it itself is + // 8eing processed as the first loaded language). + if (defaultJSON) { + // Warn for keys that are missing or unexpected. + const expectedKeys = Object.keys(defaultJSON); + const presentKeys = Object.keys(stringsJSON); + for (const key of presentKeys) { + if (!expectedKeys.includes(key)) { + logWarn`(${code}) Unexpected translation key: ${key} - this won't be used!`; + } + } + for (const key of expectedKeys) { + if (!presentKeys.includes(key)) { + logWarn`(${code}) Missing translation key: ${key} - this won't be localized!`; + } + } + } + + // Valid8tion is complete, 8ut We can still do a little caching to make + // repeated actions faster. + + // We're gonna 8e mut8ting the strings dictionary o8ject from here on out. + // We make a copy so we don't mess with the one which was given to us. + stringsJSON = Object.assign({}, stringsJSON); + + // Preemptively pass everything through HTML encoding. This will prevent + // strings from embedding HTML tags or accidentally including characters + // that throw HTML parsers off. + for (const key of Object.keys(stringsJSON)) { + stringsJSON[key] = he.encode(stringsJSON[key], {useNamedReferences: true}); + } + + // It's time to cre8te the actual langauge function! + + // In the function, we don't actually distinguish 8etween the primary and + // default (fall8ack) strings - any relevant warnings have already 8een + // presented a8ove, at the time the language JSON is processed. Now we'll + // only 8e using them for indexing strings to use as templ8tes, and we can + // com8ine them for that. + const stringIndex = Object.assign({}, defaultJSON, stringsJSON); + + // We do still need the list of valid keys though. That's 8ased upon the + // default strings. (Or stringsJSON, 8ut only if the defaults aren't + // provided - which indic8tes that the single o8ject provided *is* the + // default.) + const validKeys = Object.keys(defaultJSON || stringsJSON); + + const invalidKeysFound = []; + + const strings = (key, args = {}) => { + // Ok, with the warning out of the way, it's time to get to work. + // First make sure we're even accessing a valid key. (If not, return + // an error string as su8stitute.) + if (!validKeys.includes(key)) { + // We only want to warn a8out a given key once. More than that is + // just redundant! + if (!invalidKeysFound.includes(key)) { + invalidKeysFound.push(key); + logError`(${code}) Accessing invalid key ${key}. Fix a typo or provide this in strings-default.json!`; + } + return `MISSING: ${key}`; + } + + const template = stringIndex[key]; + + // Convert the keys on the args dict from camelCase to CONSTANT_CASE. + // (This isn't an OUTRAGEOUSLY versatile algorithm for doing that, 8ut + // like, who cares, dude?) Also, this is an array, 8ecause it's handy + // for the iterating we're a8out to do. + const processedArgs = Object.entries(args) + .map(([ k, v ]) => [k.replace(/[A-Z]/g, '_$&').toUpperCase(), v]); + + // Replacement time! Woot. Reduce comes in handy here! + const output = processedArgs.reduce( + (x, [ k, v ]) => x.replaceAll(`{${k}}`, v), + template); + + // Post-processing: if any expected arguments *weren't* replaced, that + // is almost definitely an error. + if (output.match(/\{[A-Z_]+\}/)) { + logError`(${code}) Args in ${key} were missing - output: ${output}`; + } + + return output; + }; + + // And lastly, we add some utility stuff to the strings function. + + // Store the language code, for convenience of access. + strings.code = code; + + // Store the strings dictionary itself, also for convenience. + strings.json = stringsJSON; + + // Store Intl o8jects that can 8e reused for value formatting. + strings.intl = { + date: new Intl.DateTimeFormat(code, {full: true}), + number: new Intl.NumberFormat(code), + list: { + conjunction: new Intl.ListFormat(code, {type: 'conjunction'}), + disjunction: new Intl.ListFormat(code, {type: 'disjunction'}), + unit: new Intl.ListFormat(code, {type: 'unit'}) + }, + plural: { + cardinal: new Intl.PluralRules(code, {type: 'cardinal'}), + ordinal: new Intl.PluralRules(code, {type: 'ordinal'}) + } + }; + + const bindUtilities = (obj, bind) => Object.fromEntries(Object.entries(obj).map( + ([ key, fn ]) => [key, (value, opts = {}) => fn(value, {...bind, ...opts})] + )); + + // There are a 8unch of handy count functions which expect a strings value; + // for a more terse syntax, we'll stick 'em on the strings function itself, + // with automatic 8inding for the strings argument. + strings.count = bindUtilities(count, {strings}); + + // The link functions also expect the strings o8ject(*). May as well hand + // 'em over here too! Keep in mind they still expect {to} though, and that + // isn't something we have access to from this scope (so calls such as + // strings.link.album(...) still need to provide it themselves). + // + // (*) At time of writing, it isn't actually used for anything, 8ut future- + // proofing, ok???????? + strings.link = bindUtilities(link, {strings}); + + // List functions, too! + strings.list = bindUtilities(list, {strings}); + + return strings; +}; + +const countHelper = (stringKey, argName = stringKey) => (value, {strings, unit = false}) => strings( + (unit + ? `count.${stringKey}.withUnit.` + strings.intl.plural.cardinal.select(value) + : `count.${stringKey}`), + {[argName]: strings.intl.number.format(value)}); + +const count = { + date: (date, {strings}) => { + return strings.intl.date.format(date); + }, + + dateRange: ([startDate, endDate], {strings}) => { + return strings.intl.date.formatRange(startDate, endDate); + }, + + duration: (secTotal, {strings, approximate = false, unit = false}) => { + if (secTotal === 0) { + return strings('count.duration.missing'); + } + + const hour = Math.floor(secTotal / 3600); + const min = Math.floor((secTotal - hour * 3600) / 60); + const sec = Math.floor(secTotal - hour * 3600 - min * 60); + + const pad = val => val.toString().padStart(2, '0'); + + const stringSubkey = unit ? '.withUnit' : ''; + + const duration = (hour > 0 + ? strings('count.duration.hours' + stringSubkey, { + hours: hour, + minutes: pad(min), + seconds: pad(sec) + }) + : strings('count.duration.minutes' + stringSubkey, { + minutes: min, + seconds: pad(sec) + })); + + return (approximate + ? strings('count.duration.approximate', {duration}) + : duration); + }, + + index: (value, {strings}) => { + return strings('count.index.' + strings.intl.plural.ordinal.select(value), {index: value}); + }, + + number: value => strings.intl.number.format(value), + + words: (value, {strings, unit = false}) => { + const num = strings.intl.number.format(value > 1000 + ? Math.floor(value / 100) / 10 + : value); + + const words = (value > 1000 + ? strings('count.words.thousand', {words: num}) + : strings('count.words', {words: num})); + + return strings('count.words.withUnit.' + strings.intl.plural.cardinal.select(value), {words}); + }, + + albums: countHelper('albums'), + commentaryEntries: countHelper('commentaryEntries', 'entries'), + contributions: countHelper('contributions'), + coverArts: countHelper('coverArts'), + timesReferenced: countHelper('timesReferenced'), + timesUsed: countHelper('timesUsed'), + tracks: countHelper('tracks') +}; + +const listHelper = type => (list, {strings}) => strings.intl.list[type].format(list); + +const list = { + unit: listHelper('unit'), + or: listHelper('disjunction'), + and: listHelper('conjunction') +}; + +// Note there isn't a 'find track data files' function. I plan on including the +// data for all tracks within an al8um collected in the single metadata file +// for that al8um. Otherwise there'll just 8e way too many files, and I'd also +// have to worry a8out linking track files to al8um files (which would contain +// only the track listing, not track data itself), and dealing with errors of +// missing track files (or track files which are not linked to al8ums). All a +// 8unch of stuff that's a pain to deal with for no apparent 8enefit. +async function findFiles(dataPath, filter = f => true) { + return (await readdir(dataPath)) + .map(file => path.join(dataPath, file)) + .filter(file => filter(file)); +} + +function* getSections(lines) { + // ::::) + const isSeparatorLine = line => /^-{8,}$/.test(line); + yield* splitArray(lines, isSeparatorLine); +} + +function getBasicField(lines, name) { + const line = lines.find(line => line.startsWith(name + ':')); + return line && line.slice(name.length + 1).trim(); +} + +function getBooleanField(lines, name) { + // The ?? oper8tor (which is just, hilariously named, lol) can 8e used to + // specify a default! + const value = getBasicField(lines, name); + switch (value) { + case 'yes': + case 'true': + return true; + case 'no': + case 'false': + return false; + default: + return null; + } +} + +function getListField(lines, name) { + let startIndex = lines.findIndex(line => line.startsWith(name + ':')); + // If callers want to default to an empty array, they should stick + // "|| []" after the call. + if (startIndex === -1) { + return null; + } + // We increment startIndex 8ecause we don't want to include the + // "heading" line (e.g. "URLs:") in the actual data. + startIndex++; + let endIndex = lines.findIndex((line, index) => index >= startIndex && !line.startsWith('- ')); + if (endIndex === -1) { + endIndex = lines.length; + } + if (endIndex === startIndex) { + // If there is no list that comes after the heading line, treat the + // heading line itself as the comma-separ8ted array value, using + // the 8asic field function to do that. (It's l8 and my 8rain is + // sleepy. Please excuse any unhelpful comments I may write, or may + // have already written, in this st8. Thanks!) + const value = getBasicField(lines, name); + return value && value.split(',').map(val => val.trim()); + } + const listLines = lines.slice(startIndex, endIndex); + return listLines.map(line => line.slice(2)); +}; + +function getContributionField(section, name) { + let contributors = getListField(section, name); + + if (!contributors) { + return null; + } + + if (contributors.length === 1 && contributors[0].startsWith('')) { + const arr = []; + arr.textContent = contributors[0]; + return arr; + } + + contributors = contributors.map(contrib => { + // 8asically, the format is "Who (What)", or just "Who". 8e sure to + // keep in mind that "what" doesn't necessarily have a value! + const match = contrib.match(/^(.*?)( \((.*)\))?$/); + if (!match) { + return contrib; + } + const who = match[1]; + const what = match[3] || null; + return {who, what}; + }); + + const badContributor = contributors.find(val => typeof val === 'string'); + if (badContributor) { + return {error: `An entry has an incorrectly formatted contributor, "${badContributor}".`}; + } + + if (contributors.length === 1 && contributors[0].who === 'none') { + return null; + } + + return contributors; +}; + +function getMultilineField(lines, name) { + // All this code is 8asically the same as the getListText - just with a + // different line prefix (four spaces instead of a dash and a space). + let startIndex = lines.findIndex(line => line.startsWith(name + ':')); + if (startIndex === -1) { + return null; + } + startIndex++; + let endIndex = lines.findIndex((line, index) => index >= startIndex && line.length && !line.startsWith(' ')); + if (endIndex === -1) { + endIndex = lines.length; + } + // If there aren't any content lines, don't return anything! + if (endIndex === startIndex) { + return null; + } + // We also join the lines instead of returning an array. + const listLines = lines.slice(startIndex, endIndex); + return listLines.map(line => line.slice(4)).join('\n'); +}; + +const replacerSpec = { + 'album': { + search: 'album', + link: 'album' + }, + 'album-commentary': { + search: 'album', + link: 'albumCommentary' + }, + 'artist': { + search: 'artist', + link: 'artist' + }, + 'artist-gallery': { + search: 'artist', + link: 'artistGallery' + }, + 'commentary-index': { + search: null, + link: 'commentaryIndex' + }, + 'date': { + search: null, + value: ref => new Date(ref), + html: (date, {strings}) => `` + }, + 'flash': { + search: 'flash', + link: 'flash', + transformName(name, search, offset, text) { + const nextCharacter = text[offset + search.length]; + const lastCharacter = name[name.length - 1]; + if ( + ![' ', '\n', '<'].includes(nextCharacter) && + lastCharacter === '.' + ) { + return name.slice(0, -1); + } else { + return name; + } + } + }, + 'group': { + search: 'group', + link: 'groupInfo' + }, + 'group-gallery': { + search: 'group', + link: 'groupGallery' + }, + 'listing-index': { + search: null, + link: 'listingIndex' + }, + 'listing': { + search: 'listing', + link: 'listing' + }, + 'media': { + search: null, + link: 'media' + }, + 'news-index': { + search: null, + link: 'newsIndex' + }, + 'news-entry': { + search: 'newsEntry', + link: 'newsEntry' + }, + 'root': { + search: null, + link: 'root' + }, + 'site': { + search: null, + link: 'site' + }, + 'static': { + search: 'staticPage', + link: 'staticPage' + }, + 'tag': { + search: 'tag', + link: 'tag' + }, + 'track': { + search: 'track', + link: 'track' + } +}; + +{ + let error = false; + for (const [key, {link: linkKey, search: searchKey, value, html}] of Object.entries(replacerSpec)) { + if (!html && !link[linkKey]) { + logError`The replacer spec ${key} has invalid link key ${linkKey}! Specify it in link specs or fix typo.`; + error = true; + } + if (searchKey && !search[searchKey]) { + logError`The replacer spec ${key} has invalid search key ${searchKey}! Specify it in search specs or fix typo.`; + error = true; + } + } + if (error) process.exit(); + + const categoryPart = Object.keys(replacerSpec).join('|'); + transformInline.regexp = new RegExp(String.raw`(? { + if (!category) { + category = 'track'; + } + + const { + search: searchKey, + link: linkKey, + value: valueFn, + html: htmlFn, + transformName + } = replacerSpec[category]; + + const value = ( + valueFn ? valueFn(ref) : + searchKey ? search[searchKey](ref) : + { + directory: ref.replace(category + ':', ''), + name: null + }); + + if (!value) { + logWarn`The link ${match} does not match anything!`; + return match; + } + + const label = (enteredName + || transformName && transformName(value.name, match, offset, text) + || value.name); + + if (!valueFn && !label) { + logWarn`The link ${match} requires a label be entered!`; + return match; + } + + const fn = (htmlFn + ? htmlFn + : strings.link[linkKey]); + + try { + return fn(value, {text: label, hash, strings, to}); + } catch (error) { + logError`The link ${match} failed to be processed: ${error}`; + return match; + } + }).replaceAll(String.raw`\[[`, '[['); +} + +function parseAttributes(string, {to}) { + const attributes = Object.create(null); + const skipWhitespace = i => { + const ws = /\s/; + if (ws.test(string[i])) { + const match = string.slice(i).match(/[^\s]/); + if (match) { + return i + match.index; + } else { + return string.length; + } + } else { + return i; + } + }; + + for (let i = 0; i < string.length;) { + i = skipWhitespace(i); + const aStart = i; + const aEnd = i + string.slice(i).match(/[\s=]|$/).index; + const attribute = string.slice(aStart, aEnd); + i = skipWhitespace(aEnd); + if (string[i] === '=') { + i = skipWhitespace(i + 1); + let end, endOffset; + if (string[i] === '"' || string[i] === "'") { + end = string[i]; + endOffset = 1; + i++; + } else { + end = '\\s'; + endOffset = 0; + } + const vStart = i; + const vEnd = i + string.slice(i).match(new RegExp(`${end}|$`)).index; + const value = string.slice(vStart, vEnd); + i = vEnd + endOffset; + if (attribute === 'src' && value.startsWith('media/')) { + attributes[attribute] = to('media.path', value.slice('media/'.length)); + } else { + attributes[attribute] = value; + } + } else { + attributes[attribute] = attribute; + } + } + return Object.fromEntries(Object.entries(attributes).map(([ key, val ]) => [ + key, + val === 'true' ? true : + val === 'false' ? false : + val === key ? true : + val + ])); +} + +function transformMultiline(text, {strings, to}) { + // Heck yes, HTML magics. + + text = transformInline(text.trim(), {strings, to}); + + const outLines = []; + + const indentString = ' '.repeat(4); + + let levelIndents = []; + const openLevel = indent => { + // opening a sublist is a pain: to be semantically *and* visually + // correct, we have to append the + `).join('\n')} + + `; + } + }, + + { + directory: 'tracks/by-times-referenced', + title: ({strings}) => strings('listingPage.listTracks.byTimesReferenced.title'), + + data() { + return trackData + .map(track => ({track, timesReferenced: track.referencedBy.length})) + .filter(({ timesReferenced }) => timesReferenced > 0) + .sort((a, b) => b.timesReferenced - a.timesReferenced); + }, + + row({track, timesReferenced}, {strings, to}) { + return strings('listingPage.listTracks.byTimesReferenced.item', { + track: strings.link.track(track, {to}), + timesReferenced: strings.count.timesReferenced(timesReferenced, {unit: true}) + }); + } + }, + + { + directory: 'tracks/in-flashes/by-album', + title: ({strings}) => strings('listingPage.listTracks.inFlashes.byAlbum.title'), + condition: () => wikiInfo.features.flashesAndGames, + + data() { + return chunkByProperties(trackData.filter(t => t.flashes.length > 0), ['album']) + .filter(({ album }) => album.directory !== C.UNRELEASED_TRACKS_DIRECTORY); + }, + + html(chunks, {strings, to}) { + return fixWS` +
+ ${chunks.map(({album, chunk: tracks}) => fixWS` +
${strings('listingPage.listTracks.inFlashes.byAlbum.album', { + album: strings.link.album(album, {to}), + date: strings.count.date(album.date) + })}
+
    + ${(tracks + .map(track => strings('listingPage.listTracks.inFlashes.byAlbum.track', { + track: strings.link.track(track, {to}), + flashes: strings.list.and(track.flashes.map(flash => strings.link.flash(flash, {to}))) + })) + .map(row => `
  • ${row}
    • `) + .join('\n'))} +
+ `).join('\n')} +
+ `; + } + }, + + { + directory: 'tracks/in-flashes/by-flash', + title: ({strings}) => strings('listingPage.listTracks.inFlashes.byFlash.title'), + condition: () => wikiInfo.features.flashesAndGames, + + html({strings, to}) { + return fixWS` +
+ ${C.sortByDate(flashData.slice()).map(flash => fixWS` +
${strings('listingPage.listTracks.inFlashes.byFlash.flash', { + flash: strings.link.flash(flash, {to}), + date: strings.count.date(flash.date) + })}
+
    + ${(flash.tracks + .map(track => strings('listingPage.listTracks.inFlashes.byFlash.track', { + track: strings.link.track(track, {to}), + album: strings.link.album(track.album, {to}) + })) + .map(row => `
  • ${row}
    • `) + .join('\n'))} +
+ `).join('\n')} +
+ `; + } + }, + + { + directory: 'tracks/with-lyrics', + title: ({strings}) => strings('listingPage.listTracks.withLyrics.title'), + + data() { + return chunkByProperties(trackData.filter(t => t.lyrics), ['album']); + }, + + html(chunks, {strings, to}) { + return fixWS` +
+ ${chunks.map(({album, chunk: tracks}) => fixWS` +
${strings('listingPage.listTracks.withLyrics.album', { + album: strings.link.album(album, {to}), + date: strings.count.date(album.date) + })}
+
    + ${(tracks + .map(track => strings('listingPage.listTracks.withLyrics.track', { + track: strings.link.track(track, {to}), + })) + .map(row => `
  • ${row}
    • `) + .join('\n'))} +
+ `).join('\n')} +
+ `; + } + }, + + { + directory: 'tags/by-name', + title: ({strings}) => strings('listingPage.listTags.byName.title'), + condition: () => wikiInfo.features.artTagUI, + + data() { + return tagData + .filter(tag => !tag.isCW) + .sort(sortByName) + .map(tag => ({tag, timesUsed: tag.things.length})); + }, + + row({tag, timesUsed}, {strings, to}) { + return strings('listingPage.listTags.byName.item', { + tag: strings.link.tag(tag, {to}), + timesUsed: strings.count.timesUsed(timesUsed, {unit: true}) + }); + } + }, + + { + directory: 'tags/by-uses', + title: ({strings}) => strings('listingPage.listTags.byUses.title'), + condition: () => wikiInfo.features.artTagUI, + + data() { + return tagData + .filter(tag => !tag.isCW) + .map(tag => ({tag, timesUsed: tag.things.length})) + .sort((a, b) => b.timesUsed - a.timesUsed); + }, + + row({tag, timesUsed}, {strings, to}) { + return strings('listingPage.listTags.byUses.item', { + tag: strings.link.tag(tag, {to}), + timesUsed: strings.count.timesUsed(timesUsed, {unit: true}) + }); + } + }, + + { + directory: 'random', + title: ({strings}) => `Random Pages`, + html: ({strings, to}) => fixWS` +

Choose a link to go to a random page in that category or album! If your browser doesn't support relatively modern JavaScript or you've disabled it, these links won't work - sorry.

+

(Data files are downloading in the background! Please wait for data to load.)

+

(Data files have finished being downloaded. The links should work!)

+
+
Miscellaneous:
+
+ ${[ + {name: 'Official', albumData: officialAlbumData, code: 'official'}, + {name: 'Fandom', albumData: fandomAlbumData, code: 'fandom'} + ].map(category => fixWS` +
${category.name}: (Random Album, Random Track)
+
    ${category.albumData.map(album => fixWS` +
  • ${album.name}
    • + `).join('\n')}
+ `).join('\n')} +
+ ` + } +]; + +function writeListingPages() { + if (!wikiInfo.features.listings) { + return; + } + + return [ + writeListingIndex(), + ...listingSpec.map(writeListingPage).filter(Boolean) + ]; +} + +function writeListingIndex() { + const releasedTracks = trackData.filter(track => track.album.directory !== C.UNRELEASED_TRACKS_DIRECTORY); + const releasedAlbums = albumData.filter(album => album.directory !== C.UNRELEASED_TRACKS_DIRECTORY); + const duration = getTotalDuration(releasedTracks); + + return ({strings, writePage}) => writePage('listingIndex', '', ({to}) => ({ + title: strings('listingIndex.title'), + + main: { + content: fixWS` +

${strings('listingIndex.title')}

+

${strings('listingIndex.infoLine', { + wiki: wikiInfo.name, + tracks: `${strings.count.tracks(releasedTracks.length, {unit: true})}`, + albums: `${strings.count.albums(releasedAlbums.length, {unit: true})}`, + duration: `${strings.count.duration(duration, {approximate: true, unit: true})}` + })}

+
+

${strings('listingIndex.exploreList')}

+ ${generateLinkIndexForListings(null, {strings, to})} + ` + }, + + sidebarLeft: { + content: generateSidebarForListings(null, {strings, to}) + }, + + nav: {simple: true} + })) +} + +function writeListingPage(listing) { + if (listing.condition && !listing.condition()) { + return null; + } + + const data = (listing.data + ? listing.data() + : null); + + return ({strings, writePage}) => writePage('listing', listing.directory, ({to}) => ({ + title: listing.title({strings}), + + main: { + content: fixWS` +

${listing.title({strings})}

+ ${listing.html && (listing.data + ? listing.html(data, {strings, to}) + : listing.html({strings, to}))} + ${listing.row && fixWS` + + `} + ` + }, + + sidebarLeft: { + content: generateSidebarForListings(listing, {strings, to}) + }, + + nav: { + links: [ + { + href: to('localized.home'), + title: wikiInfo.shortName + }, + { + href: to('localized.listingIndex'), + title: strings('listingIndex.title') + }, + { + href: '', + title: listing.title({strings}) + } + ] + } + })); +} + +function generateSidebarForListings(currentListing, {strings, to}) { + return fixWS` +

${strings.link.listingIndex('', {text: strings('listingIndex.title'), to})}

+ ${generateLinkIndexForListings(currentListing, {strings, to})} + `; +} + +function generateLinkIndexForListings(currentListing, {strings, to}) { + return fixWS` + + `; +} + +function filterAlbumsByCommentary() { + return albumData.filter(album => [album, ...album.tracks].some(x => x.commentary)); +} + +function writeCommentaryPages() { + if (!filterAlbumsByCommentary().length) { + return; + } + + return [ + writeCommentaryIndex(), + ...filterAlbumsByCommentary().map(writeAlbumCommentaryPage) + ]; +} + +function writeCommentaryIndex() { + const data = filterAlbumsByCommentary() + .map(album => ({ + album, + entries: [album, ...album.tracks].filter(x => x.commentary).map(x => x.commentary) + })) + .map(({ album, entries }) => ({ + album, entries, + words: entries.join(' ').split(' ').length + })); + + const totalEntries = data.reduce((acc, {entries}) => acc + entries.length, 0); + const totalWords = data.reduce((acc, {words}) => acc + words, 0); + + return ({strings, writePage}) => writePage('commentaryIndex', '', ({to}) => ({ + title: strings('commentaryIndex.title'), + + main: { + content: fixWS` +
+

${strings('commentaryIndex.title')}

+

${strings('commentaryIndex.infoLine', { + words: `${strings.count.words(totalWords, {unit: true})}`, + entries: `${strings.count.commentaryEntries(totalEntries, {unit: true})}` + })}

+

${strings('commentaryIndex.albumList.title')}

+ +
+ ` + }, + + nav: {simple: true} + })); +} + +function writeAlbumCommentaryPage(album) { + const entries = [album, ...album.tracks].filter(x => x.commentary).map(x => x.commentary); + const words = entries.join(' ').split(' ').length; + + return ({strings, writePage}) => writePage('albumCommentary', album.directory, ({to}) => ({ + title: strings('albumCommentaryPage.title', {album: album.name}), + stylesheet: getAlbumStylesheet(album, {to}), + theme: getThemeString(album), + + main: { + content: fixWS` +
+

${strings('albumCommentaryPage.title', { + album: strings.link.album(album, {to}) + })}

+

${strings('albumCommentaryPage.infoLine', { + words: `${strings.count.words(words, {unit: true})}`, + entries: `${strings.count.commentaryEntries(entries.length, {unit: true})}` + })}

+ ${album.commentary && fixWS` +

${strings('albumCommentaryPage.entry.title.albumCommentary')}

+
+ ${transformMultiline(album.commentary, {strings, to})} +
+ `} + ${album.tracks.filter(t => t.commentary).map(track => fixWS` +

${strings('albumCommentaryPage.entry.title.trackCommentary', { + track: strings.link.track(track, {to}) + })}

+
+ ${transformMultiline(track.commentary, {strings, to})} +
+ `).join('\n')} +
+ ` + }, + + nav: { + links: [ + { + href: to('localized.home'), + title: wikiInfo.shortName + }, + { + href: to('localized.commentaryIndex'), + title: strings('commentaryIndex.title') + }, + { + html: strings('albumCommentaryPage.nav.album', { + album: strings.link.albumCommentary(album, {class: 'current', to}) + }) + } + ] + } + })); +} + +function writeTagPages() { + if (!wikiInfo.features.artTagUI) { + return; + } + + return tagData.filter(tag => !tag.isCW).map(writeTagPage); +} + +function writeTagPage(tag) { + const { things } = tag; + + return ({strings, writePage}) => writePage('tag', tag.directory, ({to}) => ({ + title: strings('tagPage.title', {tag: tag.name}), + theme: getThemeString(tag), + + main: { + classes: ['top-index'], + content: fixWS` +

${strings('tagPage.title', {tag: tag.name})}

+

${strings('tagPage.infoLine', { + coverArts: strings.count.coverArts(things.length, {unit: true}) + })}

+
+ ${getGridHTML({ + strings, to, + entries: things.map(item => ({item})), + srcFn: thing => (thing.album + ? getTrackCover(thing, {to}) + : getAlbumCover(thing, {to})), + hrefFn: thing => (thing.album + ? to('localized.track', thing.directory) + : to('localized.album', thing.directory)) + })} +
+ ` + }, + + nav: { + links: [ + { + href: to('localized.home'), + title: wikiInfo.shortName + }, + wikiInfo.features.listings && + { + href: to('localized.listingIndex'), + title: strings('listingIndex.title') + }, + { + html: strings('tagPage.nav.tag', { + tag: strings.link.tag(tag, {class: 'current', to}) + }) + } + ] + } + })); +} + +function getArtistString(artists, {strings, to, showIcons = false, showContrib = false}) { + return strings.list.and(artists.map(({ who, what }) => { + const { urls, directory, name } = who; + return [ + strings.link.artist(who, {to}), + showContrib && what && `(${what})`, + showIcons && urls.length && `(${ + strings.list.unit(urls.map(url => iconifyURL(url, {strings, to}))) + })` + ].filter(Boolean).join(' '); + })); +} + +function getLinkThemeString(thing) { + const { primary, dim } = C.getColors(thing.color || wikiInfo.color); + return `--primary-color: ${primary}; --dim-color: ${dim}`; +} + +function getThemeString(thing, additionalVariables = []) { + const { primary, dim } = C.getColors(thing.color || wikiInfo.color); + + const variables = [ + `--primary-color: ${primary}`, + `--dim-color: ${dim}`, + ...additionalVariables + ].filter(Boolean); + + return fixWS` + ${variables.length && fixWS` + :root { + ${variables.map(line => line + ';').join('\n')} + } + `} + `; +} + +function getFlashDirectory(flash) { + // const kebab = getKebabCase(flash.name.replace('[S] ', '')); + // return flash.page + (kebab ? '-' + kebab : ''); + // return '' + flash.page; + return '' + flash.directory; +} + +function getTagDirectory({name}) { + return C.getKebabCase(name); +} + +function getAlbumListTag(album) { + if (album.directory === C.UNRELEASED_TRACKS_DIRECTORY) { + return 'ul'; + } else { + return 'ol'; + } +} + +function fancifyURL(url, {strings, album = false} = {}) { + const domain = new URL(url).hostname; + return fixWS`${ + domain.includes('bandcamp.com') ? strings('misc.external.bandcamp') : + [ + 'music.solatrux.com' + ].includes(domain) ? strings('misc.external.bandcamp.domain', {domain}) : + [ + 'types.pl' + ].includes(domain) ? strings('misc.external.mastodon.domain', {domain}) : + domain.includes('youtu') ? (album + ? (url.includes('list=') + ? strings('misc.external.youtube.playlist') + : strings('misc.external.youtube.fullAlbum')) + : strings('misc.external.youtube')) : + domain.includes('soundcloud') ? strings('misc.external.soundcloud') : + domain.includes('tumblr.com') ? strings('misc.external.tumblr') : + domain.includes('twitter.com') ? strings('misc.external.twitter') : + domain.includes('deviantart.com') ? strings('misc.external.deviantart') : + domain.includes('wikipedia.org') ? strings('misc.external.wikipedia') : + domain.includes('poetryfoundation.org') ? strings('misc.external.poetryFoundation') : + domain.includes('instagram.com') ? strings('misc.external.instagram') : + domain.includes('patreon.com') ? strings('misc.external.patreon') : + domain + }`; +} + +function fancifyFlashURL(url, flash, {strings}) { + const link = fancifyURL(url, {strings}); + return `${ + url.includes('homestuck.com') ? (isNaN(Number(flash.page)) + ? strings('misc.external.flash.homestuck.secret', {link}) + : strings('misc.external.flash.homestuck.page', {link, page: flash.page})) : + url.includes('bgreco.net') ? strings('misc.external.flash.bgreco', {link}) : + url.includes('youtu') ? strings('misc.external.flash.youtube', {link}) : + link + }`; +} + +function iconifyURL(url, {strings, to}) { + const domain = new URL(url).hostname; + const [ id, msg ] = ( + domain.includes('bandcamp.com') ? ['bandcamp', strings('misc.external.bandcamp')] : + ( + domain.includes('music.solatrus.com') + ) ? ['bandcamp', strings('misc.external.bandcamp.domain', {domain})] : + ( + domain.includes('types.pl') + ) ? ['mastodon', strings('misc.external.mastodon.domain', {domain})] : + domain.includes('youtu') ? ['youtube', strings('misc.external.youtube')] : + domain.includes('soundcloud') ? ['soundcloud', strings('misc.external.soundcloud')] : + domain.includes('tumblr.com') ? ['tumblr', strings('misc.external.tumblr')] : + domain.includes('twitter.com') ? ['twitter', strings('misc.external.twitter')] : + domain.includes('deviantart.com') ? ['deviantart', strings('misc.external.deviantart')] : + domain.includes('instagram.com') ? ['instagram', strings('misc.external.bandcamp')] : + ['globe', strings('misc.external.domain', {domain})] + ); + return fixWS`${msg}`; +} + +function chronologyLinks(currentThing, { + strings, to, + headingString, + contribKey, + getThings +}) { + const contributions = currentThing[contribKey]; + if (!contributions) { + return ''; + } + + if (contributions.length > 8) { + return `
${strings('misc.chronology.seeArtistPages')}
`; + } + + return contributions.map(({ who: artist }) => { + const things = C.sortByDate(unique(getThings(artist))); + const releasedThings = things.filter(thing => { + const album = albumData.includes(thing) ? thing : thing.album; + return !(album && album.directory === C.UNRELEASED_TRACKS_DIRECTORY); + }); + const index = releasedThings.indexOf(currentThing); + + if (index === -1) return ''; + + // TODO: This can pro8a8ly 8e made to use generatePreviousNextLinks? + // We'd need to make generatePreviousNextLinks use toAnythingMan tho. + const previous = releasedThings[index - 1]; + const next = releasedThings[index + 1]; + const parts = [ + previous && `Previous`, + next && `Next` + ].filter(Boolean); + + const stringOpts = { + index: strings.count.index(index + 1, {strings}), + artist: strings.link.artist(artist, {to}) + }; + + return fixWS` +
+ ${strings(headingString, stringOpts)} + ${parts.length && `(${parts.join(', ')})`} +
+ `; + }).filter(Boolean).join('\n'); +} + +function generateAlbumNavLinks(album, currentTrack, {strings, to}) { + if (album.tracks.length <= 1) { + return ''; + } + + const previousNextLinks = currentTrack && generatePreviousNextLinks('localized.track', currentTrack, album.tracks, {strings, to}) + const randomLink = `${ + (currentTrack + ? strings('trackPage.nav.random') + : strings('albumPage.nav.randomTrack')) + }`; + + return (previousNextLinks + ? `(${previousNextLinks}, ${randomLink})` + : `(${randomLink})`); +} + +function generateAlbumChronologyLinks(album, currentTrack, {strings, to}) { + return [ + currentTrack && chronologyLinks(currentTrack, { + strings, to, + headingString: 'misc.chronology.heading.track', + contribKey: 'artists', + getThings: artist => [...artist.tracks.asArtist, ...artist.tracks.asContributor] + }), + chronologyLinks(currentTrack || album, { + strings, to, + headingString: 'misc.chronology.heading.coverArt', + contribKey: 'coverArtists', + getThings: artist => [...artist.albums.asCoverArtist, ...artist.tracks.asCoverArtist] + }) + ].filter(Boolean).join('\n'); +} + +function generateSidebarForAlbum(album, currentTrack, {strings, to}) { + const listTag = getAlbumListTag(album); + + const trackToListItem = track => `
  • ${ + strings('albumSidebar.trackList.item', { + track: strings.link.track(track, {to}) + }) + }
    • `; + + const trackListPart = fixWS` +

    ${album.name}

    + ${album.trackGroups ? fixWS` +
    + ${album.trackGroups.map(({ name, color, startIndex, tracks }) => fixWS` +
    ${ + (listTag === 'ol' + ? strings('albumSidebar.trackList.group.withRange', { + group: strings.link.track(tracks[0], {to, text: name}), + range: `${startIndex + 1}–${startIndex + tracks.length}` + }) + : strings('albumSidebar.trackList.group', { + group: strings.link.track(tracks[0], {to, text: name}) + })) + }
    + ${(!currentTrack || tracks.includes(currentTrack)) && fixWS` +
    <${listTag === 'ol' ? `ol start="${startIndex + 1}"` : listTag}> + ${tracks.map(trackToListItem).join('\n')} +
    + `} + `).join('\n')} +
    + ` : fixWS` + <${listTag}> + ${album.tracks.map(trackToListItem).join('\n')} + + `} + `; + + const { groups } = album; + + const groupParts = groups.map(group => { + const index = group.albums.indexOf(album); + const next = group.albums[index + 1]; + const previous = group.albums[index - 1]; + return {group, next, previous}; + }).map(({group, next, previous}) => fixWS` +

    ${ + strings('albumSidebar.groupBox.title', { + group: `${group.name}` + }) + }

    + ${!currentTrack && transformMultiline(group.descriptionShort, {strings, to})} + ${group.urls.length && `

    ${ + strings('releaseInfo.visitOn', { + links: strings.list.or(group.urls.map(url => fancifyURL(url, {strings}))) + }) + }

    `} + ${!currentTrack && fixWS` + ${next && `

    ${ + strings('albumSidebar.groupBox.next', { + album: `${next.name}` + }) + }

    `} + ${previous && `

    ${ + strings('albumSidebar.groupBox.previous', { + album: `${previous.name}` + }) + }

    `} + `} + `); + + if (groupParts.length) { + if (currentTrack) { + const combinedGroupPart = groupParts.join('\n
    \n'); + return { + multiple: [ + trackListPart, + combinedGroupPart + ] + }; + } else { + return { + multiple: [ + ...groupParts, + trackListPart + ] + }; + } + } else { + return { + content: trackListPart + }; + } +} + +function generateSidebarForGroup(currentGroup, {strings, to, isGallery}) { + if (!wikiInfo.features.groupUI) { + return null; + } + + const urlKey = isGallery ? 'localized.groupGallery' : 'localized.groupInfo'; + + return { + content: fixWS` +

    ${strings('groupSidebar.title')}

    +
    + ${groupCategoryData.map(category => [ + fixWS` +
    ${ + strings('groupSidebar.groupList.category', { + category: `${category.name}` + }) + }
    +
      + ${category.groups.map(group => fixWS` +
    • ${ + strings('groupSidebar.groupList.item', { + group: `${group.name}` + }) + }
      • + `).join('\n')} +
    + ` + ]).join('\n')} +
    + ` + }; +} + +function generateInfoGalleryLinks(urlKeyInfo, urlKeyGallery, currentThing, isGallery, {strings, to}) { + return [ + strings.link[urlKeyInfo](currentThing, { + to, + class: isGallery ? '' : 'current', + text: strings('misc.nav.info') + }), + strings.link[urlKeyGallery](currentThing, { + to, + class: isGallery ? 'current' : '', + text: strings('misc.nav.gallery') + }) + ].join(', '); +} + +function generatePreviousNextLinks(urlKey, currentThing, thingData, {strings, to}) { + const index = thingData.indexOf(currentThing); + const previous = thingData[index - 1]; + const next = thingData[index + 1]; + + return [ + previous && `${strings('misc.nav.previous')}`, + next && `${strings('misc.nav.next')}` + ].filter(Boolean).join(', '); +} + +function generateNavForGroup(currentGroup, {strings, to, isGallery}) { + if (!wikiInfo.features.groupUI) { + return {simple: true}; + } + + const urlKey = isGallery ? 'localized.groupGallery' : 'localized.groupInfo'; + const linkKey = isGallery ? 'groupGallery' : 'groupInfo'; + + const infoGalleryLinks = generateInfoGalleryLinks('groupInfo', 'groupGallery', currentGroup, isGallery, {strings, to}); + const previousNextLinks = generatePreviousNextLinks(urlKey, currentGroup, groupData, {strings, to}) + + return { + links: [ + { + href: to('localized.home'), + title: wikiInfo.shortName + }, + wikiInfo.features.listings && + { + href: to('localized.listingIndex'), + title: strings('listingIndex.title') + }, + { + html: strings('groupPage.nav.group', { + group: strings.link[linkKey](currentGroup, {class: 'current', to}) + }) + }, + { + divider: false, + html: (previousNextLinks + ? `(${infoGalleryLinks}; ${previousNextLinks})` + : `(${previousNextLinks})`) + } + ] + }; +} + +function writeGroupPages() { + return groupData.map(writeGroupPage); +} + +function writeGroupPage(group) { + const releasedAlbums = group.albums.filter(album => album.directory !== C.UNRELEASED_TRACKS_DIRECTORY); + const releasedTracks = releasedAlbums.flatMap(album => album.tracks); + const totalDuration = getTotalDuration(releasedTracks); + + return async ({strings, writePage}) => { + await writePage('groupInfo', group.directory, ({to}) => ({ + title: strings('groupInfoPage.title', {group: group.name}), + theme: getThemeString(group), + + main: { + content: fixWS` +

    ${strings('groupInfoPage.title', {group: group.name})}

    + ${group.urls.length && `

    ${ + strings('releaseInfo.visitOn', { + links: strings.list.or(group.urls.map(url => fancifyURL(url, {strings}))) + }) + }

    `} +
    + ${transformMultiline(group.description, {strings, to})} +
    +

    ${strings('groupInfoPage.albumList.title')}

    +

    ${ + strings('groupInfoPage.viewAlbumGallery', { + link: `${ + strings('groupInfoPage.viewAlbumGallery.link') + }` + }) + }

    + + ` + }, + + sidebarLeft: generateSidebarForGroup(group, {strings, to, isGallery: false}), + nav: generateNavForGroup(group, {strings, to, isGallery: false}) + })); + + await writePage('groupGallery', group.directory, ({to}) => ({ + title: strings('groupGalleryPage.title', {group: group.name}), + theme: getThemeString(group), + + main: { + classes: ['top-index'], + content: fixWS` +

    ${strings('groupGalleryPage.title', {group: group.name})}

    +

    ${ + strings('groupGalleryPage.infoLine', { + tracks: `${strings.count.tracks(releasedTracks.length, {unit: true})}`, + albums: `${strings.count.albums(releasedAlbums.length, {unit: true})}`, + time: `${strings.count.duration(totalDuration, {unit: true})}` + }) + }

    + ${wikiInfo.features.groupUI && wikiInfo.features.listings && `

    (Choose another group to filter by!)

    `} +
    + ${getAlbumGridHTML({ + strings, to, + entries: C.sortByDate(group.albums.map(item => ({item}))).reverse(), + details: true + })} +
    + ` + }, + + sidebarLeft: generateSidebarForGroup(group, {strings, to, isGallery: true}), + nav: generateNavForGroup(group, {strings, to, isGallery: true}) + })); + }; +} + +function toAnythingMan(anythingMan, to) { + return ( + albumData.includes(anythingMan) ? to('localized.album', anythingMan.directory) : + trackData.includes(anythingMan) ? to('localized.track', anythingMan.directory) : + flashData?.includes(anythingMan) ? to('localized.flash', anythingMan.directory) : + 'idk-bud' + ) +} + +function getAlbumCover(album, {to}) { + return to('media.albumCover', album.directory); +} + +function getTrackCover(track, {to}) { + // Some al8ums don't have any track art at all, and in those, every track + // just inherits the al8um's own cover art. + if (track.coverArtists === null) { + return getAlbumCover(track.album, {to}); + } else { + return to('media.trackCover', track.album.directory, track.directory); + } +} + +function getFlashLink(flash) { + return `https://homestuck.com/story/${flash.page}`; +} + +function classes(...args) { + const values = args.filter(Boolean); + return `class="${values.join(' ')}"`; +} + +async function processLanguageFile(file, defaultStrings = null) { + let contents; + try { + contents = await readFile(file, 'utf-8'); + } catch (error) { + return {error: `Could not read ${file} (${error.code}).`}; + } + + let json; + try { + json = JSON.parse(contents); + } catch (error) { + return {error: `Could not parse JSON from ${file} (${error}).`}; + } + + return genStrings(json, defaultStrings); +} + +// Wrapper function for running a function once for all languages. It provides: +// * the language strings +// * a shadowing writePages function for outputing to the appropriate subdir +// * a shadowing urls object for linking to the appropriate relative paths +async function wrapLanguages(fn, writeOneLanguage = null) { + const k = writeOneLanguage + const languagesToRun = (k + ? {[k]: languages[k]} + : languages) + + const entries = Object.entries(languagesToRun) + .filter(([ key ]) => key !== 'default'); + + for (let i = 0; i < entries.length; i++) { + const [ key, strings ] = entries[i]; + + const baseDirectory = (strings === languages.default ? '' : strings.code); + + const shadow_writePage = (urlKey, directory, pageFn) => writePage(strings, baseDirectory, urlKey, directory, pageFn); + + // 8ring the utility functions over too! + Object.assign(shadow_writePage, writePage); + + await fn({ + baseDirectory, + strings, + writePage: shadow_writePage + }, i, entries); + } +} + +async function main() { + const miscOptions = await parseOptions(process.argv.slice(2), { + // Data files for the site, including flash, artist, and al8um data, + // and like a jillion other things too. Pretty much everything which + // makes an individual wiki what it is goes here! + 'data-path': { + type: 'value' + }, + + // Static media will 8e referenced in the site here! The contents are + // categorized; check out MEDIA_DIRECTORY and rel8ted constants in + // common/common.js. (This gets symlinked into the --data directory.) + 'media-path': { + type: 'value' + }, + + // String files! For the most part, this is used for translating the + // site to different languages, though you can also customize strings + // for your own 8uild of the site if you'd like. Files here should all + // match the format in strings-default.json in this repository. (If a + // language file is missing any strings, the site code will fall 8ack + // to what's specified in strings-default.json.) + // + // Unlike the other options here, this one's optional - the site will + // 8uild with the default (English) strings if this path is left + // unspecified. + 'lang-path': { + type: 'value' + }, + + // This is the output directory. It's the one you'll upload online with + // rsync or whatever when you're pushing an upd8, and also the one + // you'd archive if you wanted to make a 8ackup of the whole dang + // site. Just keep in mind that the gener8ted result will contain a + // couple symlinked directories, so if you're uploading, you're pro8a8ly + // gonna want to resolve those yourself. + 'out-path': { + type: 'value' + }, + + // Thum8nail gener8tion is *usually* something you want, 8ut it can 8e + // kinda a pain to run every time, since it does necessit8te reading + // every media file at run time. Pass this to skip it. + 'skip-thumbs': { + type: 'flag' + }, + + // Only want 8uild one language during testing? This can chop down + // 8uild times a pretty 8ig chunk! Just pass a single language code. + 'lang': { + type: 'value' + }, + + 'queue-size': { + type: 'value', + validate(size) { + if (parseInt(size) !== parseFloat(size)) return 'an integer'; + if (parseInt(size) < 0) return 'a counting number or zero'; + return true; + } + }, + queue: {alias: 'queue-size'}, + + [parseOptions.handleUnknown]: () => {} + }); + + dataPath = miscOptions['data-path'] || process.env.HSMUSIC_DATA; + mediaPath = miscOptions['media-path'] || process.env.HSMUSIC_MEDIA; + langPath = miscOptions['lang-path'] || process.env.HSMUSIC_LANG; // Can 8e left unset! + outputPath = miscOptions['out-path'] || process.env.HSMUSIC_OUT; + + const writeOneLanguage = miscOptions['lang']; + + { + let errored = false; + const error = (cond, msg) => { + if (cond) { + console.error(`\x1b[31;1m${msg}\x1b[0m`); + errored = true; + } + }; + error(!dataPath, `Expected --data option or HSMUSIC_DATA to be set`); + error(!mediaPath, `Expected --media option or HSMUSIC_MEDIA to be set`); + error(!outputPath, `Expected --out option or HSMUSIC_OUT to be set`); + if (errored) { + return; + } + } + + const skipThumbs = miscOptions['skip-thumbs'] ?? false; + + if (skipThumbs) { + logInfo`Skipping thumbnail generation.`; + } else { + logInfo`Begin thumbnail generation... -----+`; + const result = await genThumbs(mediaPath, {queueSize, quiet: true}); + logInfo`Done thumbnail generation! --------+`; + if (!result) { + return; + } + } + + const defaultStrings = await processLanguageFile(path.join(__dirname, DEFAULT_STRINGS_FILE)); + if (defaultStrings.error) { + logError`Error loading default strings: ${defaultStrings.error}`; + return; + } + + if (langPath) { + const languageDataFiles = await findFiles(langPath, f => path.extname(f) === '.json'); + const results = await progressPromiseAll(`Reading & processing language files.`, languageDataFiles + .map(file => processLanguageFile(file, defaultStrings.json))); + + let error = false; + for (const strings of results) { + if (strings.error) { + logError`Error loading provided strings: ${strings.error}`; + error = true; + } + } + if (error) return; + + languages = Object.fromEntries(results.map(strings => [strings.code, strings])); + } else { + languages = {}; + } + + if (!languages[defaultStrings.code]) { + languages[defaultStrings.code] = defaultStrings; + } + + logInfo`Loaded language strings: ${Object.keys(languages).join(', ')}`; + + if (writeOneLanguage && !(writeOneLanguage in languages)) { + logError`Specified to write only ${writeOneLanguage}, but there is no strings file with this language code!`; + return; + } else if (writeOneLanguage) { + logInfo`Writing only language ${writeOneLanguage} this run.`; + } else { + logInfo`Writing all languages.`; + } + + wikiInfo = await processWikiInfoFile(path.join(dataPath, WIKI_INFO_FILE)); + if (wikiInfo.error) { + console.log(`\x1b[31;1m${wikiInfo.error}\x1b[0m`); + return; + } + + // Update languages o8ject with the wiki-specified default language! + // This will make page files for that language 8e gener8ted at the root + // directory, instead of the language-specific su8directory. + if (wikiInfo.defaultLanguage) { + if (Object.keys(languages).includes(wikiInfo.defaultLanguage)) { + languages.default = languages[wikiInfo.defaultLanguage]; + } else { + logError`Wiki info file specified default language is ${wikiInfo.defaultLanguage}, but no such language file exists!`; + if (langPath) { + logError`Check if an appropriate file exists in ${langPath}?`; + } else { + logError`Be sure to specify ${'--lang'} or ${'HSMUSIC_LANG'} with the path to language files.`; + } + return; + } + } else { + languages.default = defaultStrings; + } + + homepageInfo = await processHomepageInfoFile(path.join(dataPath, HOMEPAGE_INFO_FILE)); + + if (homepageInfo.error) { + console.log(`\x1b[31;1m${homepageInfo.error}\x1b[0m`); + return; + } + + { + const errors = homepageInfo.rows.filter(obj => obj.error); + if (errors.length) { + for (const error of errors) { + console.log(`\x1b[31;1m${error.error}\x1b[0m`); + } + return; + } + } + + // 8ut wait, you might say, how do we know which al8um these data files + // correspond to???????? You wouldn't dare suggest we parse the actual + // paths returned 8y this function, which ought to 8e of effectively + // unknown format except for their purpose as reada8le data files!? + // To that, I would say, yeah, you're right. Thanks a 8unch, my projection + // of "you". We're going to read these files later, and contained within + // will 8e the actual directory names that the data correspond to. Yes, + // that's redundant in some ways - we COULD just return the directory name + // in addition to the data path, and duplicating that name within the file + // itself suggests we 8e careful to avoid mismatching it - 8ut doing it + // this way lets the data files themselves 8e more porta8le (meaning we + // could store them all in one folder, if we wanted, and this program would + // still output to the correct al8um directories), and also does make the + // function's signature simpler (an array of strings, rather than some kind + // of structure containing 8oth data file paths and output directories). + // This is o8jectively a good thing, 8ecause it means the function can stay + // truer to its name, and have a narrower purpose: it doesn't need to + // concern itself with where we *output* files, or whatever other reasons + // we might (hypothetically) have for knowing the containing directory. + // And, in the strange case where we DO really need to know that info, we + // callers CAN use path.dirname to find out that data. 8ut we'll 8e + // avoiding that in our code 8ecause, again, we want to avoid assuming the + // format of the returned paths here - they're only meant to 8e used for + // reading as-is. + const albumDataFiles = await findFiles(path.join(dataPath, C.DATA_ALBUM_DIRECTORY)); + + // Technically, we could do the data file reading and output writing at the + // same time, 8ut that kinda makes the code messy, so I'm not 8othering + // with it. + albumData = await progressPromiseAll(`Reading & processing album files.`, albumDataFiles.map(processAlbumDataFile)); + + { + const errors = albumData.filter(obj => obj.error); + if (errors.length) { + for (const error of errors) { + console.log(`\x1b[31;1m${error.error}\x1b[0m`); + } + return; + } + } + + C.sortByDate(albumData); + + artistData = await processArtistDataFile(path.join(dataPath, ARTIST_DATA_FILE)); + if (artistData.error) { + console.log(`\x1b[31;1m${artistData.error}\x1b[0m`); + return; + } + + { + const errors = artistData.filter(obj => obj.error); + if (errors.length) { + for (const error of errors) { + console.log(`\x1b[31;1m${error.error}\x1b[0m`); + } + return; + } + } + + artistAliasData = artistData.filter(x => x.alias); + artistData = artistData.filter(x => !x.alias); + + trackData = C.getAllTracks(albumData); + + if (wikiInfo.features.flashesAndGames) { + flashData = await processFlashDataFile(path.join(dataPath, FLASH_DATA_FILE)); + if (flashData.error) { + console.log(`\x1b[31;1m${flashData.error}\x1b[0m`); + return; + } + + const errors = flashData.filter(obj => obj.error); + if (errors.length) { + for (const error of errors) { + console.log(`\x1b[31;1m${error.error}\x1b[0m`); + } + return; + } + } + + flashActData = flashData?.filter(x => x.act8r8k); + flashData = flashData?.filter(x => !x.act8r8k); + + artistNames = Array.from(new Set([ + ...artistData.filter(artist => !artist.alias).map(artist => artist.name), + ...[ + ...albumData.flatMap(album => [ + ...album.artists || [], + ...album.coverArtists || [], + ...album.wallpaperArtists || [], + ...album.tracks.flatMap(track => [ + ...track.artists, + ...track.coverArtists || [], + ...track.contributors || [] + ]) + ]), + ...(flashData?.flatMap(flash => [ + ...flash.contributors || [] + ]) || []) + ].map(contribution => contribution.who) + ])); + + tagData = await processTagDataFile(path.join(dataPath, TAG_DATA_FILE)); + if (tagData.error) { + console.log(`\x1b[31;1m${tagData.error}\x1b[0m`); + return; + } + + { + const errors = tagData.filter(obj => obj.error); + if (errors.length) { + for (const error of errors) { + console.log(`\x1b[31;1m${error.error}\x1b[0m`); + } + return; + } + } + + groupData = await processGroupDataFile(path.join(dataPath, GROUP_DATA_FILE)); + if (groupData.error) { + console.log(`\x1b[31;1m${groupData.error}\x1b[0m`); + return; + } + + { + const errors = groupData.filter(obj => obj.error); + if (errors.length) { + for (const error of errors) { + console.log(`\x1b[31;1m${error.error}\x1b[0m`); + } + return; + } + } + + groupCategoryData = groupData.filter(x => x.isCategory); + groupData = groupData.filter(x => x.isGroup); + + staticPageData = await processStaticPageDataFile(path.join(dataPath, STATIC_PAGE_DATA_FILE)); + if (staticPageData.error) { + console.log(`\x1b[31;1m${staticPageData.error}\x1b[0m`); + return; + } + + { + const errors = staticPageData.filter(obj => obj.error); + if (errors.length) { + for (const error of errors) { + console.log(`\x1b[31;1m${error.error}\x1b[0m`); + } + return; + } + } + + if (wikiInfo.features.news) { + newsData = await processNewsDataFile(path.join(dataPath, NEWS_DATA_FILE)); + if (newsData.error) { + console.log(`\x1b[31;1m${newsData.error}\x1b[0m`); + return; + } + + const errors = newsData.filter(obj => obj.error); + if (errors.length) { + for (const error of errors) { + console.log(`\x1b[31;1m${error.error}\x1b[0m`); + } + return; + } + + C.sortByDate(newsData); + newsData.reverse(); + } + + { + const tagNames = new Set([...trackData, ...albumData].flatMap(thing => thing.artTags)); + + for (let { name, isCW } of tagData) { + if (isCW) { + name = 'cw: ' + name; + } + tagNames.delete(name); + } + + if (tagNames.size) { + for (const name of Array.from(tagNames).sort()) { + console.log(`\x1b[33;1m- Missing tag: "${name}"\x1b[0m`); + } + return; + } + } + + artistNames.sort((a, b) => a.toLowerCase() < b.toLowerCase() ? -1 : a.toLowerCase() > b.toLowerCase() ? 1 : 0); + + justEverythingMan = C.sortByDate([...albumData, ...trackData, ...(flashData || [])]); + justEverythingSortedByArtDateMan = C.sortByArtDate(justEverythingMan.slice()); + // console.log(JSON.stringify(justEverythingSortedByArtDateMan.map(toAnythingMan), null, 2)); + + { + let buffer = []; + const clearBuffer = function() { + if (buffer.length) { + for (const entry of buffer.slice(0, -1)) { + console.log(`\x1b[2m... ${entry.name} ...\x1b[0m`); + } + const lastEntry = buffer[buffer.length - 1]; + console.log(`\x1b[2m... \x1b[0m${lastEntry.name}\x1b[0;2m ...\x1b[0m`); + buffer = []; + } + }; + const showWhere = (name, color) => { + const where = justEverythingMan.filter(thing => [ + ...thing.coverArtists || [], + ...thing.contributors || [], + ...thing.artists || [] + ].some(({ who }) => who === name)); + for (const thing of where) { + console.log(`\x1b[${color}m- ` + (thing.album ? `(\x1b[1m${thing.album.name}\x1b[0;${color}m)` : '') + ` \x1b[1m${thing.name}\x1b[0m`); + } + }; + let CR4SH = false; + for (let name of artistNames) { + const entry = [...artistData, ...artistAliasData].find(entry => entry.name === name || entry.name.toLowerCase() === name.toLowerCase()); + if (!entry) { + clearBuffer(); + console.log(`\x1b[31mMissing entry for artist "\x1b[1m${name}\x1b[0;31m"\x1b[0m`); + showWhere(name, 31); + CR4SH = true; + } else if (entry.alias) { + console.log(`\x1b[33mArtist "\x1b[1m${name}\x1b[0;33m" should be named "\x1b[1m${entry.alias}\x1b[0;33m"\x1b[0m`); + showWhere(name, 33); + CR4SH = true; + } else if (entry.name !== name) { + console.log(`\x1b[33mArtist "\x1b[1m${name}\x1b[0;33m" should be named "\x1b[1m${entry.name}\x1b[0;33m"\x1b[0m`); + showWhere(name, 33); + CR4SH = true; + } else { + buffer.push(entry); + if (buffer.length > 3) { + buffer.shift(); + } + } + } + if (CR4SH) { + return; + } + } + + { + const directories = []; + for (const { directory, name } of albumData) { + if (directories.includes(directory)) { + console.log(`\x1b[31;1mDuplicate album directory "${directory}" (${name})\x1b[0m`); + return; + } + directories.push(directory); + } + } + + { + const directories = []; + const where = {}; + for (const { directory, album } of trackData) { + if (directories.includes(directory)) { + console.log(`\x1b[31;1mDuplicate track directory "${directory}"\x1b[0m`); + console.log(`Shows up in:`); + console.log(`- ${album.name}`); + console.log(`- ${where[directory].name}`); + return; + } + directories.push(directory); + where[directory] = album; + } + } + + { + const artists = []; + const artistsLC = []; + for (const name of artistNames) { + if (!artists.includes(name) && artistsLC.includes(name.toLowerCase())) { + const other = artists.find(oth => oth.toLowerCase() === name.toLowerCase()); + console.log(`\x1b[31;1mMiscapitalized artist name: ${name}, ${other}\x1b[0m`); + return; + } + artists.push(name); + artistsLC.push(name.toLowerCase()); + } + } + + { + for (const { references, name, album } of trackData) { + for (const ref of references) { + if (!search.track(ref)) { + logWarn`Track not found "${ref}" in ${name} (${album.name})`; + } + } + } + } + + contributionData = Array.from(new Set([ + ...trackData.flatMap(track => [...track.artists || [], ...track.contributors || [], ...track.coverArtists || []]), + ...albumData.flatMap(album => [...album.artists || [], ...album.coverArtists || [], ...album.wallpaperArtists || [], ...album.bannerArtists || []]), + ...(flashData?.flatMap(flash => [...flash.contributors || []]) || []) + ])); + + // Now that we have all the data, resolve references all 8efore actually + // gener8ting any of the pages, 8ecause page gener8tion is going to involve + // accessing these references a lot, and there's no reason to resolve them + // more than once. (We 8uild a few additional links that can't 8e cre8ted + // at initial data processing time here too.) + + const filterNullArray = (parent, key) => { + for (const obj of parent) { + const array = obj[key]; + for (let i = 0; i < array.length; i++) { + if (!array[i]) { + const prev = array[i - 1] && array[i - 1].name; + const next = array[i + 1] && array[i + 1].name; + logWarn`Unexpected null in ${obj.name} (${obj.what}) (array key ${key} - prev: ${prev}, next: ${next})`; + } + } + array.splice(0, array.length, ...array.filter(Boolean)); + } + }; + + const filterNullValue = (parent, key) => { + parent.splice(0, parent.length, ...parent.filter(obj => { + if (!obj[key]) { + logWarn`Unexpected null in ${obj.name} (value key ${key})`; + } + })); + }; + + trackData.forEach(track => mapInPlace(track.references, search.track)); + trackData.forEach(track => track.aka = search.track(track.aka)); + trackData.forEach(track => mapInPlace(track.artTags, search.tag)); + albumData.forEach(album => mapInPlace(album.groups, search.group)); + albumData.forEach(album => mapInPlace(album.artTags, search.tag)); + artistAliasData.forEach(artist => artist.alias = search.artist(artist.alias)); + contributionData.forEach(contrib => contrib.who = search.artist(contrib.who)); + + filterNullArray(trackData, 'references'); + filterNullArray(trackData, 'artTags'); + filterNullArray(albumData, 'groups'); + filterNullArray(albumData, 'artTags'); + filterNullValue(artistAliasData, 'alias'); + filterNullValue(contributionData, 'who'); + + trackData.forEach(track1 => track1.referencedBy = trackData.filter(track2 => track2.references.includes(track1))); + groupData.forEach(group => group.albums = albumData.filter(album => album.groups.includes(group))); + tagData.forEach(tag => tag.things = C.sortByArtDate([...albumData, ...trackData]).filter(thing => thing.artTags.includes(tag))); + + groupData.forEach(group => group.category = groupCategoryData.find(x => x.name === group.category)); + groupCategoryData.forEach(category => category.groups = groupData.filter(x => x.category === category)); + + trackData.forEach(track => track.otherReleases = [ + track.aka, + ...trackData.filter(({ aka }) => aka === track || (track.aka && aka === track.aka)), + ].filter(x => x && x !== track)); + + if (wikiInfo.features.flashesAndGames) { + flashData.forEach(flash => mapInPlace(flash.tracks, search.track)); + flashData.forEach(flash => flash.act = flashActData.find(act => act.name === flash.act)); + flashActData.forEach(act => act.flashes = flashData.filter(flash => flash.act === act)); + + filterNullArray(flashData, 'tracks'); + + trackData.forEach(track => track.flashes = flashData.filter(flash => flash.tracks.includes(track))); + } + + artistData.forEach(artist => { + const filterProp = (array, prop) => array.filter(thing => thing[prop]?.some(({ who }) => who === artist)); + const filterCommentary = array => array.filter(thing => thing.commentary && thing.commentary.replace(/<\/?b>/g, '').includes('' + artist.name + ':')); + artist.tracks = { + asArtist: filterProp(trackData, 'artists'), + asCommentator: filterCommentary(trackData), + asContributor: filterProp(trackData, 'contributors'), + asCoverArtist: filterProp(trackData, 'coverArtists'), + asAny: trackData.filter(track => ( + [...track.artists, ...track.contributors, ...track.coverArtists || []].some(({ who }) => who === artist) + )) + }; + artist.albums = { + asArtist: filterProp(albumData, 'artists'), + asCommentator: filterCommentary(albumData), + asCoverArtist: filterProp(albumData, 'coverArtists'), + asWallpaperArtist: filterProp(albumData, 'wallpaperArtists'), + asBannerArtist: filterProp(albumData, 'bannerArtists') + }; + if (wikiInfo.features.flashesAndGames) { + artist.flashes = { + asContributor: filterProp(flashData, 'contributors') + }; + } + }); + + officialAlbumData = albumData.filter(album => album.groups.some(group => group.directory === C.OFFICIAL_GROUP_DIRECTORY)); + fandomAlbumData = albumData.filter(album => album.groups.every(group => group.directory !== C.OFFICIAL_GROUP_DIRECTORY)); + + // Makes writing a little nicer on CPU theoretically, 8ut also costs in + // performance right now 'cuz it'll w8 for file writes to 8e completed + // 8efore moving on to more data processing. So, defaults to zero, which + // disa8les the queue feature altogether. + queueSize = +(miscOptions['queue-size'] ?? 0); + + // NOT for ena8ling or disa8ling specific features of the site! + // This is only in charge of what general groups of files to 8uild. + // They're here to make development quicker when you're only working + // on some particular area(s) of the site rather than making changes + // across all of them. + const writeFlags = await parseOptions(process.argv.slice(2), { + all: {type: 'flag'}, // Defaults to true if none 8elow specified. + + album: {type: 'flag'}, + artist: {type: 'flag'}, + commentary: {type: 'flag'}, + flash: {type: 'flag'}, + group: {type: 'flag'}, + list: {type: 'flag'}, + misc: {type: 'flag'}, + news: {type: 'flag'}, + static: {type: 'flag'}, + tag: {type: 'flag'}, + track: {type: 'flag'}, + + [parseOptions.handleUnknown]: () => {} + }); + + const writeAll = !Object.keys(writeFlags).length || writeFlags.all; + + logInfo`Writing site pages: ${writeAll ? 'all' : Object.keys(writeFlags).join(', ')}`; + + await writeSymlinks(); + await writeSharedFilesAndPages({strings: defaultStrings}); + + const buildDictionary = { + misc: writeMiscellaneousPages, + news: writeNewsPages, + list: writeListingPages, + tag: writeTagPages, + commentary: writeCommentaryPages, + static: writeStaticPages, + group: writeGroupPages, + album: writeAlbumPages, + track: writeTrackPages, + artist: writeArtistPages, + flash: writeFlashPages + }; + + const buildSteps = (writeAll + ? Object.values(buildDictionary) + : (Object.entries(buildDictionary) + .filter(([ flag ]) => writeFlags[flag]) + .map(([ flag, fn ]) => fn))); + + // *NB: While what's 8elow is 8asically still true in principle, the + // format is QUITE DIFFERENT than what's descri8ed here! There + // will 8e actual document8tion on like, what the return format + // looks like soon, once we implement a 8unch of other pages and + // are certain what they actually, uh, will look like, in the end.* + // + // The writeThingPages functions don't actually immediately do any file + // writing themselves; an initial call will only gather the relevant data + // which is *then* used for writing. So the return value is a function + // (or an array of functions) which expects {writePage, strings}, and + // *that's* what we call after -- multiple times, once for each language. + let writes; + { + let error = false; + + writes = buildSteps.flatMap(fn => { + const fns = fn() || []; + + // Do a quick valid8tion! If one of the writeThingPages functions go + // wrong, this will stall out early and tell us which did. + if (!Array.isArray(fns)) { + logError`${fn.name} didn't return an array!`; + error = true; + } else if (fns.every(entry => Array.isArray(entry))) { + if (!( + fns.every(entry => entry.every(obj => typeof obj === 'object')) && + fns.every(entry => entry.every(obj => { + const result = validateWriteObject(obj); + if (result.error) { + logError`Validating write object failed: ${result.error}`; + return false; + } else { + return true; + } + })) + )) { + logError`${fn.name} uses updated format, but entries are invalid!`; + error = true; + } + + return fns.flatMap(writes => writes); + } else if (fns.some(fn => typeof fn !== 'function')) { + logError`${fn.name} didn't return all functions or all arrays!`; + error = true; + } + + return fns; + }); + + if (error) { + return; + } + + // The modern(TM) return format for each writeThingPages function is an + // array of arrays, each of which's items are 8ig Complicated Objects + // that 8asically look like {type, path, content}. 8ut surprise, these + // aren't actually implemented in most places yet! So, we transform + // stuff in the old format here. 'Scept keep in mind, the OLD FORMAT + // doesn't really give us most of the info we want for Cool And Modern + // Reasons, so they're going into a fancy {type: 'legacy'} sort of + // o8ject, with a plain {write} property for, uh, the writing stuff, + // same as usual. + // + // I promise this document8tion will get 8etter when we make progress + // actually moving old pages over. Also it'll 8e hecks of less work + // than previous restructures, don't worry. + writes = writes.map(entry => + typeof entry === 'object' ? entry : + typeof entry === 'function' ? {type: 'legacy', write: entry} : + {type: 'wut', entry}); + + const wut = writes.filter(({ type }) => type === 'wut'); + if (wut.length) { + // Oh g*d oh h*ck. + logError`Uhhhhh writes contains something 8esides o8jects and functions?`; + logError`Definitely a 8ug!`; + console.log(wut); + return; + } + } + + const localizedWrites = writes.filter(({ type }) => type === 'page' || type === 'legacy'); + const dataWrites = writes.filter(({ type }) => type === 'data'); + + await progressPromiseAll(`Writing data files shared across languages.`, queue( + // TODO: This only supports one <>-style argument. + dataWrites.map(({path, data}) => () => writeData(path[0], path[1], data())), + queueSize + )); + + await wrapLanguages(async ({strings, ...opts}, i, entries) => { + console.log(`\x1b[34;1m${ + (`[${i + 1}/${entries.length}] ${strings.code} (-> /${opts.baseDirectory}) ` + .padEnd(60, '-')) + }\x1b[0m`); + await progressPromiseAll(`Writing ${strings.code}`, queue( + localizedWrites.map(({type, ...props}) => () => { + switch (type) { + case 'legacy': { + const { write } = props; + return write({strings, ...opts}); + } + case 'page': { + const { path, page } = props; + // TODO: This only supports one <>-style argument. + return opts.writePage(path[0], path[1], ({to}) => page({strings, to})); + } + } + }), + queueSize + )); + }, writeOneLanguage); + + decorateTime.displayTime(); + + // The single most important step. + logInfo`Written!`; +} + +main().catch(error => console.error(error)); -- cgit 1.3.0-6-gf8a5