#!/usr/bin/env node
// HEY N8RDS!
//
// This is one of the 8ACKEND FILES. It's not used anywhere on the actual site
// you are pro8a8ly using right now.
//
// Specifically, this one does all the actual work of the music wiki. The
// process looks something like this:
//
// 1. Crawl the music directories. Well, not so much "crawl" as "look inside
// the folders for each al8um, and read the metadata file descri8ing that
// al8um and the tracks within."
//
// 2. Read that metadata. I'm writing this 8efore actually doing any of the
// code, and I've gotta admit I have no idea what file format they're
// going to 8e in. May8e JSON, 8ut more likely some weird custom format
// which will 8e a lot easier to edit.
//
// 3. Generate the page files! They're just static index.html files, and are
// what gh-pages (or wherever this is hosted) will show to clients.
// Hopefully pretty minimalistic HTML, 8ut like, shrug. They'll reference
// CSS (and maaaaaaaay8e JS) files, hard-coded somewhere near the root.
//
// 4. Print an awesome message which says the process is done. This is the
// most important step.
//
// Oh yeah, like. Just run this through some relatively recent version of
// node.js and you'll 8e fine. ...Within the project root. O8viously.
// HEY FUTURE ME!!!!!!!! Don't forget to implement artist pages! Those are,
// like, the coolest idea you've had yet, so DO NOT FORGET. (Remem8er, link
// from track listings, etc!) --- Thanks, past me. To futurerer me: an al8um
// listing page (a list of all the al8ums)! Make sure to sort these 8y date -
// we'll need a new field for al8ums.
// ^^^^^^^^ DID THAT! 8ut also, artist images. Pro8a8ly stolen from the fandom
// wiki (I found half those images anywayz).
// TRACK ART CREDITS. This is a must.
// 2020-08-23
// ATTENTION ALL 8*TCHES AND OTHER GENDER TRUCKERS: AS IT TURNS OUT, THIS CODE
// ****SUCKS****. I DON'T THINK ANYTHING WILL EVER REDEEM IT, 8UT THAT DOESN'T
// MEAN WE CAN'T TAKE SOME ACTION TO MAKE WRITING IT A LITTLE LESS TERRI8LE.
// We're gonna start defining STRUCTURES to make things suck less!!!!!!!!
// No classes 8ecause those are a huge pain and like, pro8a8ly 8ad performance
// or whatever -- just some standard structures that should 8e followed
// wherever reasona8le. Only one I need today is the contri8 one 8ut let's put
// any new general-purpose structures here too, ok?
//
// Contri8ution: {who, what, date, thing}. D8 and thing are the new fields.
//
// Use these wisely, which is to say all the time and instead of whatever
// terri8le new pseudo structure you're trying to invent!!!!!!!!
//
// Upd8 2021-01-03: Soooooooo we didn't actually really end up using these,
// lol? Well there's still only one anyway. Kinda ended up doing a 8ig refactor
// of all the o8ject structures today. It's not *especially* relevant 8ut feels
// worth mentioning? I'd get rid of this comment 8lock 8ut I like it too much!
// Even though I haven't actually reread it, lol. 8ut yeah, hopefully in the
// spirit of this "make things more consistent" attitude I 8rought up 8ack in
// August, stuff's lookin' 8etter than ever now. W00t!
import * as path from 'path';
import { promisify } from 'util';
import { fileURLToPath } from 'url';
// I made this dependency myself! A long, long time ago. It is pro8a8ly my
// most useful li8rary ever. I'm not sure 8esides me actually uses it, though.
import fixWS from 'fix-whitespace';
// Wait nevermind, I forgot a8out why-do-kids-love-the-taste-of-cinnamon-toast-
// crunch. THAT is my 8est li8rary.
// It stands for "HTML Entities", apparently. Cursed.
import he from 'he';
import {
// This is the dum8est name for a function possi8le. Like, SURE, fine, may8e
// the UNIX people had some valid reason to go with the weird truncated
// lowercased convention they did. 8ut Node didn't have to ALSO use that
// convention! Would it have 8een so hard to just name the function
// something like fs.readDirectory???????? No, it wouldn't have 8een.
readdir,
// ~~ 8ut okay, like, look at me. DOING THE SAME THING. See, *I* could have
// named my promisified function differently, and yet I did not. I literally
// cannot explain why. We are all used to following in the 8ad decisions of
// our ancestors, and never never never never never never never consider
// that hey, may8e we don't need to make the exact same decisions they did.
// Even when we're perfectly aware th8t's exactly what we're doing! ~~
//
// 2021 ADDENDUM: Ok, a year and a half later the a8ove is still true,
// except for the part a8out promisifying, since fs/promises
// already does that for us. 8ut I could STILL import it
// using my own name (`readdir as readDirectory`), and yet
// here I am, defin8tely not doing that.
// SOME THINGS NEVER CHANGE.
//
// Programmers, including me, are all pretty stupid.
// 8ut I mean, come on. Look. Node decided to use readFile, instead of like,
// what, cat? Why couldn't they rename readdir too???????? As Johannes
// Kepler once so elegantly put it: "Shrug."
readFile,
writeFile,
access,
mkdir,
symlink,
unlink
} from 'fs/promises';
import genThumbs from './gen-thumbs.js';
import * as html from './util/html.js';
import link from './util/link.js';
import {
decorateTime,
logWarn,
logInfo,
logError,
parseOptions,
progressPromiseAll
} from './util/cli.js';
import {
getLinkThemeString,
getThemeString
} from './util/colors.js';
import {
chunkByConditions,
chunkByProperties,
getAllTracks,
getArtistCommentary,
getArtistNumContributions,
getKebabCase,
sortByArtDate,
sortByDate,
sortByName
} from './util/wiki-data.js';
import {
call,
escapeRegex,
filterEmptyLines,
mapInPlace,
queue,
splitArray,
unique,
withEntries
} from './util/sugar.js';
import {
generateURLs,
thumb
} from './util/urls.js';
const __dirname = path.dirname(fileURLToPath(import.meta.url));
const CACHEBUST = 5;
const WIKI_INFO_FILE = 'wiki-info.txt';
const HOMEPAGE_INFO_FILE = 'homepage.txt';
const ARTIST_DATA_FILE = 'artists.txt';
const FLASH_DATA_FILE = 'flashes.txt';
const NEWS_DATA_FILE = 'news.txt';
const TAG_DATA_FILE = 'tags.txt';
const GROUP_DATA_FILE = 'groups.txt';
const STATIC_PAGE_DATA_FILE = 'static-pages.txt';
const DEFAULT_STRINGS_FILE = 'strings-default.json';
const UNRELEASED_TRACKS_DIRECTORY = 'unreleased-tracks';
const OFFICIAL_GROUP_DIRECTORY = 'official';
const FANDOM_GROUP_DIRECTORY = 'fandom';
// Code that's common 8etween the 8uild code (i.e. upd8.js) and gener8ted
// site code should 8e put here. Which, uh, ~~only really means this one
// file~~ is now a variety of useful utilities!
//
// Rather than hard code it, anything in this directory can 8e shared across
// 8oth ends of the code8ase.
// (This gets symlinked into the --data directory.)
const UTILITY_DIRECTORY = 'util';
// Code that's used only in the static site! CSS, cilent JS, etc.
// (This gets symlinked into the --data directory.)
const STATIC_DIRECTORY = 'static';
// Su8directory under provided --data directory for al8um files, which are
// read from and processed to compose the majority of album and track data.
const DATA_ALBUM_DIRECTORY = 'album';
// Shared varia8les! These are more efficient to access than a shared varia8le
// (or at least I h8pe so), and are easier to pass across functions than a
// 8unch of specific arguments.
//
// Upd8: Okay yeah these aren't actually any different. Still cleaner than
// passing around a data object containing all this, though.
let dataPath;
let mediaPath;
let langPath;
let outputPath;
// Glo8al data o8ject shared 8etween 8uild functions and all that. This keeps
// everything encapsul8ted in one place, so it's easy to pass and share across
// modules!
let wikiData = {};
let queueSize;
let languages;
const urlSpec = {
data: {
prefix: 'data/',
paths: {
root: '',
path: '<>',
album: 'album/<>',
artist: 'artist/<>',
track: 'track/<>'
}
},
localized: {
// TODO: Implement this.
// prefix: '_languageCode',
paths: {
root: '',
path: '<>',
home: '',
album: 'album/<>/',
albumCommentary: 'commentary/album/<>/',
artist: 'artist/<>/',
artistGallery: 'artist/<>/gallery/',
commentaryIndex: 'commentary/',
flashIndex: 'flash/',
flash: 'flash/<>/',
groupInfo: 'group/<>/',
groupGallery: 'group/<>/gallery/',
listingIndex: 'list/',
listing: 'list/<>/',
newsIndex: 'news/',
newsEntry: 'news/<>/',
staticPage: '<>/',
tag: 'tag/<>/',
track: 'track/<>/'
}
},
shared: {
paths: {
root: '',
path: '<>',
utilityRoot: 'util',
staticRoot: 'static',
utilityFile: 'util/<>',
staticFile: 'static/<>'
}
},
media: {
prefix: 'media/',
paths: {
root: '',
path: '<>',
albumCover: 'album-art/<>/cover.jpg',
albumWallpaper: 'album-art/<>/bg.jpg',
albumBanner: 'album-art/<>/banner.jpg',
trackCover: 'album-art/<>/<>.jpg',
artistAvatar: 'artist-avatar/<>.jpg',
flashArt: 'flash-art/<>.jpg'
}
}
};
// This gets automatically switched in place when working from a baseDirectory,
// so it should never be referenced manually.
urlSpec.localizedWithBaseDirectory = {
paths: withEntries(
urlSpec.localized.paths,
entries => entries.map(([key, path]) => [key, '<>/' + path])
)
};
const urls = generateURLs(urlSpec);
const searchHelper = (keys, dataProp, findFn) => (ref, {wikiData}) => {
if (!ref) return null;
ref = ref.replace(new RegExp(`^(${keys.join('|')}):`), '');
if (!wikiData[dataProp]) {
console.log('find', dataProp, Object.keys(wikiData));
}
const found = findFn(ref, wikiData[dataProp]);
if (!found) {
logWarn`Didn't match anything for ${ref}! (${keys.join(', ')})`;
}
return found;
};
const matchDirectory = (ref, data) => data.find(({ directory }) => directory === ref);
const matchDirectoryOrName = (ref, data) => {
let thing;
thing = matchDirectory(ref, data);
if (thing) return thing;
thing = data.find(({ name }) => name === ref);
if (thing) return thing;
thing = data.find(({ name }) => name.toLowerCase() === ref.toLowerCase());
if (thing) {
logWarn`Bad capitalization: ${'\x1b[31m' + ref} -> ${'\x1b[32m' + thing.name}`;
return thing;
}
return null;
};
const search = {
album: searchHelper(['album', 'album-commentary'], 'albumData', matchDirectoryOrName),
artist: searchHelper(['artist', 'artist-gallery'], 'artistData', matchDirectoryOrName),
flash: searchHelper(['flash'], 'flashData', matchDirectory),
group: searchHelper(['group', 'group-gallery'], 'groupData', matchDirectoryOrName),
listing: searchHelper(['listing'], 'listingSpec', matchDirectory),
newsEntry: searchHelper(['news-entry'], 'newsData', matchDirectory),
staticPage: searchHelper(['static'], 'staticPageData', matchDirectory),
tag: searchHelper(['tag'], 'tagData', (ref, data) =>
matchDirectoryOrName(ref.startsWith('cw: ') ? ref.slice(4) : ref, data)),
track: searchHelper(['track'], 'trackData', matchDirectoryOrName)
};
// Localiz8tion time! Or l10n as the neeeeeeeerds call it. Which is a terri8le
// name and not one I intend on using, thank you very much. (Don't even get me
// started on """"a11y"""".)
//
// All the default strings are in strings-default.json, if you're curious what
// those actually look like. Pretty much it's "I like {ANIMAL}" for example.
// For each language, the o8ject gets turned into a single function of form
// f(key, {args}). It searches for a key in the o8ject and uses the string it
// finds (or the one in strings-default.json) as a templ8 evaluated with the
// arguments passed. (This function gets treated as an o8ject too; it gets
// the language code attached.)
//
// The function's also responsi8le for getting rid of dangerous characters
// (quotes and angle tags), though only within the templ8te (not the args),
// and it converts the keys of the arguments o8ject from camelCase to
// CONSTANT_CASE too.
function genStrings(stringsJSON, defaultJSON = null) {
// genStrings will only 8e called once for each language, and it happens
// right at the start of the program (or at least 8efore 8uilding pages).
// So, now's a good time to valid8te the strings and let any warnings be
// known.
// May8e contrary to the argument name, the arguments should 8e o8jects,
// not actual JSON-formatted strings!
if (typeof stringsJSON !== 'object' || stringsJSON.constructor !== Object) {
return {error: `Expected an object (parsed JSON) for stringsJSON.`};
}
if (typeof defaultJSON !== 'object') { // typeof null === object. I h8 JS.
return {error: `Expected an object (parsed JSON) or null for defaultJSON.`};
}
// All languages require a language code.
const code = stringsJSON['meta.languageCode'];
if (!code) {
return {error: `Missing language code.`};
}
if (typeof code !== 'string') {
return {error: `Expected language code to be a string.`};
}
// Every value on the provided o8ject should be a string.
// (This is lazy, but we only 8other checking this on stringsJSON, on the
// assumption that defaultJSON was passed through this function too, and so
// has already been valid8ted.)
{
let err = false;
for (const [ key, value ] of Object.entries(stringsJSON)) {
if (typeof value !== 'string') {
logError`(${code}) The value for ${key} should be a string.`;
err = true;
}
}
if (err) {
return {error: `Expected all values to be a string.`};
}
}
// Checking is generally done against the default JSON, so we'll skip out
// if that isn't provided (which should only 8e the case when it itself is
// 8eing processed as the first loaded language).
if (defaultJSON) {
// Warn for keys that are missing or unexpected.
const expectedKeys = Object.keys(defaultJSON);
const presentKeys = Object.keys(stringsJSON);
for (const key of presentKeys) {
if (!expectedKeys.includes(key)) {
logWarn`(${code}) Unexpected translation key: ${key} - this won't be used!`;
}
}
for (const key of expectedKeys) {
if (!presentKeys.includes(key)) {
logWarn`(${code}) Missing translation key: ${key} - this won't be localized!`;
}
}
}
// Valid8tion is complete, 8ut We can still do a little caching to make
// repeated actions faster.
// We're gonna 8e mut8ting the strings dictionary o8ject from here on out.
// We make a copy so we don't mess with the one which was given to us.
stringsJSON = Object.assign({}, stringsJSON);
// Preemptively pass everything through HTML encoding. This will prevent
// strings from embedding HTML tags or accidentally including characters
// that throw HTML parsers off.
for (const key of Object.keys(stringsJSON)) {
stringsJSON[key] = he.encode(stringsJSON[key], {useNamedReferences: true});
}
// It's time to cre8te the actual langauge function!
// In the function, we don't actually distinguish 8etween the primary and
// default (fall8ack) strings - any relevant warnings have already 8een
// presented a8ove, at the time the language JSON is processed. Now we'll
// only 8e using them for indexing strings to use as templ8tes, and we can
// com8ine them for that.
const stringIndex = Object.assign({}, defaultJSON, stringsJSON);
// We do still need the list of valid keys though. That's 8ased upon the
// default strings. (Or stringsJSON, 8ut only if the defaults aren't
// provided - which indic8tes that the single o8ject provided *is* the
// default.)
const validKeys = Object.keys(defaultJSON || stringsJSON);
const invalidKeysFound = [];
const strings = (key, args = {}) => {
// Ok, with the warning out of the way, it's time to get to work.
// First make sure we're even accessing a valid key. (If not, return
// an error string as su8stitute.)
if (!validKeys.includes(key)) {
// We only want to warn a8out a given key once. More than that is
// just redundant!
if (!invalidKeysFound.includes(key)) {
invalidKeysFound.push(key);
logError`(${code}) Accessing invalid key ${key}. Fix a typo or provide this in strings-default.json!`;
}
return `MISSING: ${key}`;
}
const template = stringIndex[key];
// Convert the keys on the args dict from camelCase to CONSTANT_CASE.
// (This isn't an OUTRAGEOUSLY versatile algorithm for doing that, 8ut
// like, who cares, dude?) Also, this is an array, 8ecause it's handy
// for the iterating we're a8out to do.
const processedArgs = Object.entries(args)
.map(([ k, v ]) => [k.replace(/[A-Z]/g, '_$&').toUpperCase(), v]);
// Replacement time! Woot. Reduce comes in handy here!
const output = processedArgs.reduce(
(x, [ k, v ]) => x.replaceAll(`{${k}}`, v),
template);
// Post-processing: if any expected arguments *weren't* replaced, that
// is almost definitely an error.
if (output.match(/\{[A-Z_]+\}/)) {
logError`(${code}) Args in ${key} were missing - output: ${output}`;
}
return output;
};
// And lastly, we add some utility stuff to the strings function.
// Store the language code, for convenience of access.
strings.code = code;
// Store the strings dictionary itself, also for convenience.
strings.json = stringsJSON;
// Store Intl o8jects that can 8e reused for value formatting.
strings.intl = {
date: new Intl.DateTimeFormat(code, {full: true}),
number: new Intl.NumberFormat(code),
list: {
conjunction: new Intl.ListFormat(code, {type: 'conjunction'}),
disjunction: new Intl.ListFormat(code, {type: 'disjunction'}),
unit: new Intl.ListFormat(code, {type: 'unit'})
},
plural: {
cardinal: new Intl.PluralRules(code, {type: 'cardinal'}),
ordinal: new Intl.PluralRules(code, {type: 'ordinal'})
}
};
const bindUtilities = (obj, bind) => Object.fromEntries(Object.entries(obj).map(
([ key, fn ]) => [key, (value, opts = {}) => fn(value, {...bind, ...opts})]
));
// There are a 8unch of handy count functions which expect a strings value;
// for a more terse syntax, we'll stick 'em on the strings function itself,
// with automatic 8inding for the strings argument.
strings.count = bindUtilities(count, {strings});
// The link functions also expect the strings o8ject(*). May as well hand
// 'em over here too! Keep in mind they still expect {to} though, and that
// isn't something we have access to from this scope (so calls such as
// strings.link.album(...) still need to provide it themselves).
//
// (*) At time of writing, it isn't actually used for anything, 8ut future-
// proofing, ok????????
strings.link = bindUtilities(link, {strings});
// List functions, too!
strings.list = bindUtilities(list, {strings});
return strings;
};
const countHelper = (stringKey, argName = stringKey) => (value, {strings, unit = false}) => strings(
(unit
? `count.${stringKey}.withUnit.` + strings.intl.plural.cardinal.select(value)
: `count.${stringKey}`),
{[argName]: strings.intl.number.format(value)});
const count = {
date: (date, {strings}) => {
return strings.intl.date.format(date);
},
dateRange: ([startDate, endDate], {strings}) => {
return strings.intl.date.formatRange(startDate, endDate);
},
duration: (secTotal, {strings, approximate = false, unit = false}) => {
if (secTotal === 0) {
return strings('count.duration.missing');
}
const hour = Math.floor(secTotal / 3600);
const min = Math.floor((secTotal - hour * 3600) / 60);
const sec = Math.floor(secTotal - hour * 3600 - min * 60);
const pad = val => val.toString().padStart(2, '0');
const stringSubkey = unit ? '.withUnit' : '';
const duration = (hour > 0
? strings('count.duration.hours' + stringSubkey, {
hours: hour,
minutes: pad(min),
seconds: pad(sec)
})
: strings('count.duration.minutes' + stringSubkey, {
minutes: min,
seconds: pad(sec)
}));
return (approximate
? strings('count.duration.approximate', {duration})
: duration);
},
index: (value, {strings}) => {
return strings('count.index.' + strings.intl.plural.ordinal.select(value), {index: value});
},
number: value => strings.intl.number.format(value),
words: (value, {strings, unit = false}) => {
const num = strings.intl.number.format(value > 1000
? Math.floor(value / 100) / 10
: value);
const words = (value > 1000
? strings('count.words.thousand', {words: num})
: strings('count.words', {words: num}));
return strings('count.words.withUnit.' + strings.intl.plural.cardinal.select(value), {words});
},
albums: countHelper('albums'),
commentaryEntries: countHelper('commentaryEntries', 'entries'),
contributions: countHelper('contributions'),
coverArts: countHelper('coverArts'),
timesReferenced: countHelper('timesReferenced'),
timesUsed: countHelper('timesUsed'),
tracks: countHelper('tracks')
};
const listHelper = type => (list, {strings}) => strings.intl.list[type].format(list);
const list = {
unit: listHelper('unit'),
or: listHelper('disjunction'),
and: listHelper('conjunction')
};
// Note there isn't a 'find track data files' function. I plan on including the
// data for all tracks within an al8um collected in the single metadata file
// for that al8um. Otherwise there'll just 8e way too many files, and I'd also
// have to worry a8out linking track files to al8um files (which would contain
// only the track listing, not track data itself), and dealing with errors of
// missing track files (or track files which are not linked to al8ums). All a
// 8unch of stuff that's a pain to deal with for no apparent 8enefit.
async function findFiles(dataPath, filter = f => true) {
return (await readdir(dataPath))
.map(file => path.join(dataPath, file))
.filter(file => filter(file));
}
function* getSections(lines) {
// ::::)
const isSeparatorLine = line => /^-{8,}$/.test(line);
yield* splitArray(lines, isSeparatorLine);
}
function getBasicField(lines, name) {
const line = lines.find(line => line.startsWith(name + ':'));
return line && line.slice(name.length + 1).trim();
}
function getDimensionsField(lines, name) {
const string = getBasicField(lines, name);
if (!string) return string;
const parts = string.split(/[x,* ]+/g);
if (parts.length !== 2) throw new Error(`Invalid dimensions: ${string} (expected width & height)`);
const nums = parts.map(part => Number(part.trim()));
if (nums.includes(NaN)) throw new Error(`Invalid dimensions: ${string} (couldn't parse as numbers)`);
return nums;
}
function getBooleanField(lines, name) {
// The ?? oper8tor (which is just, hilariously named, lol) can 8e used to
// specify a default!
const value = getBasicField(lines, name);
switch (value) {
case 'yes':
case 'true':
return true;
case 'no':
case 'false':
return false;
default:
return null;
}
}
function getListField(lines, name) {
let startIndex = lines.findIndex(line => line.startsWith(name + ':'));
// If callers want to default to an empty array, they should stick
// "|| []" after the call.
if (startIndex === -1) {
return null;
}
// We increment startIndex 8ecause we don't want to include the
// "heading" line (e.g. "URLs:") in the actual data.
startIndex++;
let endIndex = lines.findIndex((line, index) => index >= startIndex && !line.startsWith('- '));
if (endIndex === -1) {
endIndex = lines.length;
}
if (endIndex === startIndex) {
// If there is no list that comes after the heading line, treat the
// heading line itself as the comma-separ8ted array value, using
// the 8asic field function to do that. (It's l8 and my 8rain is
// sleepy. Please excuse any unhelpful comments I may write, or may
// have already written, in this st8. Thanks!)
const value = getBasicField(lines, name);
return value && value.split(',').map(val => val.trim());
}
const listLines = lines.slice(startIndex, endIndex);
return listLines.map(line => line.slice(2));
};
function getContributionField(section, name) {
let contributors = getListField(section, name);
if (!contributors) {
return null;
}
if (contributors.length === 1 && contributors[0].startsWith('')) {
const arr = [];
arr.textContent = contributors[0];
return arr;
}
contributors = contributors.map(contrib => {
// 8asically, the format is "Who (What)", or just "Who". 8e sure to
// keep in mind that "what" doesn't necessarily have a value!
const match = contrib.match(/^(.*?)( \((.*)\))?$/);
if (!match) {
return contrib;
}
const who = match[1];
const what = match[3] || null;
return {who, what};
});
const badContributor = contributors.find(val => typeof val === 'string');
if (badContributor) {
return {error: `An entry has an incorrectly formatted contributor, "${badContributor}".`};
}
if (contributors.length === 1 && contributors[0].who === 'none') {
return null;
}
return contributors;
};
function getMultilineField(lines, name) {
// All this code is 8asically the same as the getListText - just with a
// different line prefix (four spaces instead of a dash and a space).
let startIndex = lines.findIndex(line => line.startsWith(name + ':'));
if (startIndex === -1) {
return null;
}
startIndex++;
let endIndex = lines.findIndex((line, index) => index >= startIndex && line.length && !line.startsWith(' '));
if (endIndex === -1) {
endIndex = lines.length;
}
// If there aren't any content lines, don't return anything!
if (endIndex === startIndex) {
return null;
}
// We also join the lines instead of returning an array.
const listLines = lines.slice(startIndex, endIndex);
return listLines.map(line => line.slice(4)).join('\n');
};
const replacerSpec = {
'album': {
search: 'album',
link: 'album'
},
'album-commentary': {
search: 'album',
link: 'albumCommentary'
},
'artist': {
search: 'artist',
link: 'artist'
},
'artist-gallery': {
search: 'artist',
link: 'artistGallery'
},
'commentary-index': {
search: null,
link: 'commentaryIndex'
},
'date': {
search: null,
value: ref => new Date(ref),
html: (date, {strings}) => ``
},
'flash': {
search: 'flash',
link: 'flash',
transformName(name, node, input) {
const nextCharacter = input[node.iEnd];
const lastCharacter = name[name.length - 1];
if (
![' ', '\n', '<'].includes(nextCharacter) &&
lastCharacter === '.'
) {
return name.slice(0, -1);
} else {
return name;
}
}
},
'group': {
search: 'group',
link: 'groupInfo'
},
'group-gallery': {
search: 'group',
link: 'groupGallery'
},
'listing-index': {
search: null,
link: 'listingIndex'
},
'listing': {
search: 'listing',
link: 'listing'
},
'media': {
search: null,
link: 'media'
},
'news-index': {
search: null,
link: 'newsIndex'
},
'news-entry': {
search: 'newsEntry',
link: 'newsEntry'
},
'root': {
search: null,
link: 'root'
},
'site': {
search: null,
link: 'site'
},
'static': {
search: 'staticPage',
link: 'staticPage'
},
'string': {
search: null,
value: ref => ref,
html: (ref, {strings, args}) => strings(ref, args)
},
'tag': {
search: 'tag',
link: 'tag'
},
'track': {
search: 'track',
link: 'track'
}
};
{
let error = false;
for (const [key, {link: linkKey, search: searchKey, value, html}] of Object.entries(replacerSpec)) {
if (!html && !link[linkKey]) {
logError`The replacer spec ${key} has invalid link key ${linkKey}! Specify it in link specs or fix typo.`;
error = true;
}
if (searchKey && !search[searchKey]) {
logError`The replacer spec ${key} has invalid search key ${searchKey}! Specify it in search specs or fix typo.`;
error = true;
}
}
if (error) process.exit();
// Syntax literals.
const tagBeginning = '[[';
const tagEnding = ']]';
const tagReplacerValue = ':';
const tagHash = '#';
const tagArgument = '*';
const tagArgumentValue = '=';
const tagLabel = '|';
const noPrecedingWhitespace = '(? ({i, type: 'error', data: {message}});
const endOfInput = (i, comment) => makeError(i, `Unexpected end of input (${comment}).`);
// These are 8asically stored on the glo8al scope, which might seem odd
// for a recursive function, 8ut the values are only ever used immediately
// after they're set.
let stopped,
stop_iMatch,
stop_iParse,
stop_literal;
const parseOneTextNode = function(input, i, stopAt) {
return parseNodes(input, i, stopAt, true)[0];
};
const parseNodes = function(input, i, stopAt, textOnly) {
let nodes = [];
let escapeNext = false;
let string = '';
let iString = 0;
stopped = false;
const pushTextNode = (isLast) => {
string = input.slice(iString, i);
// If this is the last text node 8efore stopping (at a stopAt match
// or the end of the input), trim off whitespace at the end.
if (isLast) {
string = string.trimEnd();
}
if (string.length) {
nodes.push({i: iString, iEnd: i, type: 'text', data: string});
string = '';
}
};
const literalsToMatch = stopAt ? stopAt.concat([R_tagBeginning]) : [R_tagBeginning];
// The 8ackslash stuff here is to only match an even (or zero) num8er
// of sequential 'slashes. Even amounts always cancel out! Odd amounts
// don't, which would mean the following literal is 8eing escaped and
// should 8e counted only as part of the current string/text.
//
// Inspired 8y this: https://stackoverflow.com/a/41470813
const regexpSource = `(?= 0) {
lineStart += 1;
} else {
lineStart = 0;
}
let lineEnd = input.slice(i).indexOf('\n');
if (lineEnd >= 0) {
lineEnd += i;
} else {
lineEnd = input.length;
}
const line = input.slice(lineStart, lineEnd);
const cursor = i - lineStart;
throw new SyntaxError(fixWS`
Parse error (at pos ${i}): ${message}
${line}
${'-'.repeat(cursor) + '^'}
`);
}
};
}
/*
{
const show = input => process.stdout.write(`-- ${input}\n` + util.inspect(
transformInline.parse(input),
{
depth: null,
colors: true
}
) + '\n\n');
show(`[[album:are-you-lost|Cristata's new album]]`);
show(`[[string:content.donate.patreonLine*link=[[external:https://www.patreon.com/qznebula|Patreon]]]]`);
}
{
const test = input => {
let n = 0;
const s = 5;
const start = Date.now();
const end = start + s * 1000;
while (Date.now() < end) {
transformInline.parse(input);
n++;
}
console.log(`Ran ${Math.round(n / s)} times/sec.`);
}
test(fixWS`
[[string:content.donate.patreonLine*link=[[external:https://www.patreon.com/qznebula|Patreon]]]]
Hello, world! Wow [[album:the-beans-zone]] is some cool stuff.
`);
process.exit();
}
*/
{
const evaluateTag = function(node, opts) {
const { input, strings, to, wikiData } = opts;
const source = input.slice(node.i, node.iEnd);
const replacerKey = node.data.replacerKey?.data || 'track';
if (!replacerSpec[replacerKey]) {
logWarn`The link ${source} has an invalid replacer key!`;
return source;
}
const {
search: searchKey,
link: linkKey,
value: valueFn,
html: htmlFn,
transformName
} = replacerSpec[replacerKey];
const replacerValue = transformNodes(node.data.replacerValue, opts);
const value = (
valueFn ? valueFn(replacerValue) :
searchKey ? search[searchKey](replacerValue, {wikiData}) :
{
directory: replacerValue,
name: null
});
if (!value) {
logWarn`The link ${source} does not match anything!`;
return source;
}
const enteredLabel = node.data.label && transformNode(node.data.label, opts);
const label = (enteredLabel
|| transformName && transformName(value.name, node, input)
|| value.name);
if (!valueFn && !label) {
logWarn`The link ${source} requires a label be entered!`;
return source;
}
const hash = node.data.hash && transformNodes(node.data.hash, opts);
const args = node.data.args && Object.fromEntries(node.data.args.map(
({ key, value }) => [
transformNode(key, opts),
transformNodes(value, opts)
]));
const fn = (htmlFn
? htmlFn
: strings.link[linkKey]);
try {
return fn(value, {text: label, hash, args, strings, to});
} catch (error) {
logError`The link ${source} failed to be processed: ${error}`;
return source;
}
};
const transformNode = function(node, opts) {
if (!node) {
throw new Error('Expected a node!');
}
if (Array.isArray(node)) {
throw new Error('Got an array - use transformNodes here!');
}
switch (node.type) {
case 'text':
return node.data;
case 'tag':
return evaluateTag(node, opts);
default:
throw new Error(`Unknown node type ${node.type}`);
}
};
const transformNodes = function(nodes, opts) {
if (!nodes || !Array.isArray(nodes)) {
throw new Error(`Expected an array of nodes! Got: ${nodes}`);
}
return nodes.map(node => transformNode(node, opts)).join('');
};
Object.assign(transformInline, {
evaluateTag,
transformNode,
transformNodes
});
}
function transformInline(input, {strings, to, wikiData}) {
if (!strings) throw new Error('Expected strings');
if (!to) throw new Error('Expected to');
if (!wikiData) throw new Error('Expected wikiData');
const nodes = transformInline.parse(input);
return transformInline.transformNodes(nodes, {input, strings, to, wikiData});
}
function parseAttributes(string, {to}) {
const attributes = Object.create(null);
const skipWhitespace = i => {
const ws = /\s/;
if (ws.test(string[i])) {
const match = string.slice(i).match(/[^\s]/);
if (match) {
return i + match.index;
} else {
return string.length;
}
} else {
return i;
}
};
for (let i = 0; i < string.length;) {
i = skipWhitespace(i);
const aStart = i;
const aEnd = i + string.slice(i).match(/[\s=]|$/).index;
const attribute = string.slice(aStart, aEnd);
i = skipWhitespace(aEnd);
if (string[i] === '=') {
i = skipWhitespace(i + 1);
let end, endOffset;
if (string[i] === '"' || string[i] === "'") {
end = string[i];
endOffset = 1;
i++;
} else {
end = '\\s';
endOffset = 0;
}
const vStart = i;
const vEnd = i + string.slice(i).match(new RegExp(`${end}|$`)).index;
const value = string.slice(vStart, vEnd);
i = vEnd + endOffset;
if (attribute === 'src' && value.startsWith('media/')) {
attributes[attribute] = to('media.path', value.slice('media/'.length));
} else {
attributes[attribute] = value;
}
} else {
attributes[attribute] = attribute;
}
}
return Object.fromEntries(Object.entries(attributes).map(([ key, val ]) => [
key,
val === 'true' ? true :
val === 'false' ? false :
val === key ? true :
val
]));
}
function transformMultiline(text, {strings, to, wikiData}) {
// Heck yes, HTML magics.
if (!strings) throw new Error('Expected strings');
if (!to) throw new Error('Expected to');
if (!wikiData) throw new Error('Expected wikiData');
text = transformInline(text.trim(), {strings, to, wikiData});
const outLines = [];
const indentString = ' '.repeat(4);
let levelIndents = [];
const openLevel = indent => {
// opening a sublist is a pain: to be semantically *and* visually
// correct, we have to append the at the end of the existing
// previous
');
}
};
// okay yes we should support nested formatting, more than one blockquote
// layer, etc, but hear me out here: making all that work would basically
// be the same as implementing an entire markdown converter, which im not
// interested in doing lol. sorry!!!
let inBlockquote = false;
for (let line of text.split(/\r|\n|\r\n/)) {
const imageLine = line.startsWith('/g, (match, attributes) => img({
lazy: true,
link: true,
thumb: 'medium',
...parseAttributes(attributes, {to})
}));
let indentThisLine = 0;
let lineContent = line;
let lineTag = 'p';
const listMatch = line.match(/^( *)- *(.*)$/);
if (listMatch) {
// is a list item!
if (!levelIndents.length) {
// first level is always indent = 0, regardless of actual line
// content (this is to avoid going to a lesser indent than the
// initial level)
openLevel(0);
} else {
// find level corresponding to indent
const indent = listMatch[1].length;
let i;
for (i = levelIndents.length - 1; i >= 0; i--) {
if (levelIndents[i] <= indent) break;
}
// note: i cannot equal -1 because the first indentation level
// is always 0, and the minimum indentation is also 0
if (levelIndents[i] === indent) {
// same indent! return to that level
while (levelIndents.length - 1 > i) closeLevel();
// (if this is already the current level, the above loop
// will do nothing)
} else if (levelIndents[i] < indent) {
// lesser indent! branch based on index
if (i === levelIndents.length - 1) {
// top level is lesser: add a new level
openLevel(indent);
} else {
// lower level is lesser: return to that level
while (levelIndents.length - 1 > i) closeLevel();
}
}
}
// finally, set variables for appending content line
indentThisLine = levelIndents.length;
lineContent = listMatch[2];
lineTag = 'li';
} else {
// not a list item! close any existing list levels
while (levelIndents.length) closeLevel();
// like i said, no nested shenanigans - quotes only appear outside
// of lists. sorry!
const quoteMatch = line.match(/^> *(.*)$/);
if (quoteMatch) {
// is a quote! open a blockquote tag if it doesnt already exist
if (!inBlockquote) {
inBlockquote = true;
outLines.push('';
} else {
// if the previous line isn't a list item, this is the opening of
// the first list level, so no need for indent
outLines.push('
');
}
levelIndents.push(indent);
};
const closeLevel = () => {
levelIndents.pop();
if (levelIndents.length) {
// closing a sublist, so close the list item containing it too
outLines.push(indentString.repeat(levelIndents.length) + '
');
}
indentThisLine = 1;
lineContent = quoteMatch[1];
} else if (inBlockquote) {
// not a quote! close a blockquote tag if it exists
inBlockquote = false;
outLines.push('
');
}
}
if (lineTag === 'p') {
// certain inline element tags should still be postioned within a
// paragraph; other elements (e.g. headings) should be added as-is
const elementMatch = line.match(/^<(.*?)[ >]/);
if (elementMatch && !imageLine && !['a', 'abbr', 'b', 'bdo', 'br', 'cite', 'code', 'data', 'datalist', 'del', 'dfn', 'em', 'i', 'img', 'ins', 'kbd', 'mark', 'output', 'picture', 'q', 'ruby', 'samp', 'small', 'span', 'strong', 'sub', 'sup', 'svg', 'time', 'var', 'wbr'].includes(elementMatch[1])) {
lineTag = '';
}
}
let pushString = indentString.repeat(indentThisLine);
if (lineTag) {
pushString += `<${lineTag}>${lineContent}${lineTag}>`;
} else {
pushString += lineContent;
}
outLines.push(pushString);
}
// after processing all lines...
// if still in a list, close all levels
while (levelIndents.length) closeLevel();
// if still in a blockquote, close its tag
if (inBlockquote) {
inBlockquote = false;
outLines.push('');
}
return outLines.join('\n');
}
function transformLyrics(text, {strings, to, wikiData}) {
// Different from transformMultiline 'cuz it joins multiple lines together
// with line 8reaks (
); transformMultiline treats each line as its own
// complete paragraph (or list, etc).
// If it looks like old data, then like, oh god.
// Use the normal transformMultiline tool.
if (text.includes('
outLines.push(`
${buildLine}
`); const outLines = []; for (const line of text.split('\n')) { if (line.length) { if (buildLine.length) { buildLine += '${strings('releaseInfo.from', {album: ''})}
${strings('releaseInfo.by', {artists: ''})}
${strings('releaseInfo.coverArtBy', {artists: ''})}
__GENERATE_NEWS__
', wikiInfo.features.news ? fixWS`News requested in content description but this feature isn't enabled
`)) }, nav: { content: fixWS`${strings.link.newsEntry(entry, { to, text: strings('newsIndex.entry.viewRest') })}
`}${strings('newsEntryPage.published', {date: strings.count.date(entry.date)})}
${transformMultiline(entry.body, {strings, to, wikiData})}
${[
album.artists && strings('releaseInfo.by', {
artists: getArtistString(album.artists, {
strings, to,
showContrib: true,
showIcons: true
})
}),
album.coverArtists && strings('releaseInfo.coverArtBy', {
artists: getArtistString(album.coverArtists, {
strings, to,
showContrib: true,
showIcons: true
})
}),
album.wallpaperArtists && strings('releaseInfo.wallpaperArtBy', {
artists: getArtistString(album.wallpaperArtists, {
strings, to,
showContrib: true,
showIcons: true
})
}),
album.bannerArtists && strings('releaseInfo.bannerArtBy', {
artists: getArtistString(album.bannerArtists, {
strings, to,
showContrib: true,
showIcons: true
})
}),
strings('releaseInfo.released', {
date: strings.count.date(album.date)
}),
+album.coverArtDate !== +album.date && strings('releaseInfo.artReleased', {
date: strings.count.date(album.coverArtDate)
}),
strings('releaseInfo.duration', {
duration: strings.count.duration(albumDuration, {approximate: album.tracks.length > 1})
})
].filter(Boolean).join('
\n')}
${ strings('releaseInfo.viewCommentary', { link: `${ strings('releaseInfo.viewCommentary.link') }` }) }
`} ${album.urls.length && `${ strings('releaseInfo.listenOn', { links: strings.list.or(album.urls.map(url => fancifyURL(url, {album: true, strings}))) }) }
`} ${album.trackGroups ? fixWS`
${[
strings('releaseInfo.addedToWiki', {
date: strings.count.date(album.dateAdded)
})
].filter(Boolean).join('
\n')}
${strings('releaseInfo.artistCommentary')}
${transformMultiline(album.commentary, {strings, to, wikiData})}`} ` }, sidebarLeft: generateSidebarForAlbum(album, null, {strings, to, wikiData}), nav: { links: [ { href: to('localized.home'), title: wikiInfo.shortName }, { html: strings('albumPage.nav.album', { album: strings.link.album(album, {class: 'current', to}) }) }, { divider: false, html: generateAlbumNavLinks(album, null, {strings, to}) } ], content: fixWS`
${[
strings('releaseInfo.by', {
artists: getArtistString(track.artists, {
strings, to,
showContrib: true,
showIcons: true
})
}),
track.coverArtists && strings('releaseInfo.coverArtBy', {
artists: getArtistString(track.coverArtists, {
strings, to,
showContrib: true,
showIcons: true
})
}),
album.directory !== UNRELEASED_TRACKS_DIRECTORY && strings('releaseInfo.released', {
date: strings.count.date(track.date)
}),
+track.coverArtDate !== +track.date && strings('releaseInfo.artReleased', {
date: strings.count.date(track.coverArtDate)
}),
track.duration && strings('releaseInfo.duration', {
duration: strings.count.duration(track.duration)
})
].filter(Boolean).join('
\n')}
${ (track.urls.length ? strings('releaseInfo.listenOn', { links: strings.list.or(track.urls.map(url => fancifyURL(url, {strings}))) }) : strings('releaseInfo.listenOn.noLinks')) }
${otherReleases.length && fixWS`${strings('releaseInfo.alsoReleasedAs')}
${strings('releaseInfo.contributors')}
${transformInline(track.contributors.textContent, {strings, to, wikiData})}
${strings('releaseInfo.contributors')}
${strings('releaseInfo.tracksReferenced', {track: `${track.name}`})}
${generateTrackList(tracksReferenced, {strings, to})} `} ${tracksThatReference.length && fixWS`${strings('releaseInfo.tracksThatReference', {track: `${track.name}`})}
${useDividedReferences && fixWS`${strings('releaseInfo.flashesThatFeature', {track: `${track.name}`})}
${strings('releaseInfo.lyrics')}
${transformLyrics(track.lyrics, {strings, to, wikiData})}`} ${hasCommentary && fixWS`
${strings('releaseInfo.artistCommentary')}
${generateCommentary({strings, to})}`} ` }, sidebarLeft: generateSidebarForAlbum(album, track, {strings, to, wikiData}), nav: { links: [ { href: to('localized.home'), title: wikiInfo.shortName }, { href: to('localized.album', album.directory), title: album.name }, listTag === 'ol' ? { html: strings('trackPage.nav.track.withNumber', { number: album.tracks.indexOf(track) + 1, track: strings.link.track(track, {class: 'current', to}) }) } : { html: strings('trackPage.nav.track', { track: strings.link.track(track, {class: 'current', to}) }) }, { divider: false, html: generateAlbumNavLinks(album, track, {strings, to}) } ].filter(Boolean), content: fixWS`
${strings('releaseInfo.note')}
${transformMultiline(note, {strings, to, wikiData})}
${strings('releaseInfo.visitOn', { links: strings.list.or(urls.map(url => fancifyURL(url, {strings}))) })}
`} ${hasGallery && `${strings('artistPage.viewArtGallery', { link: strings.link.artistGallery(artist, { to, text: strings('artistPage.viewArtGallery.link') }) })}
`}${strings('misc.jumpTo.withLinks', { links: strings.list.unit([ [ [...releasedTracks, ...unreleasedTracks].length && `${strings('artistPage.trackList.title')}`, unreleasedTracks.length && `(${strings('artistPage.unreleasedTrackList.title')})` ].filter(Boolean).join(' '), artThingsAll.length && `${strings('artistPage.artList.title')}`, wikiInfo.features.flashesAndGames && flashes.length && `${strings('artistPage.flashList.title')}`, commentaryThings.length && `${strings('artistPage.commentaryList.title')}` ].filter(Boolean)) })}
${(releasedTracks.length || unreleasedTracks.length) && fixWS`${strings('artistPage.contributedDurationLine', { artist: artist.name, duration: strings.count.duration(totalReleasedDuration, {approximate: true, unit: true}) })}
${strings('artistPage.musicGroupsLine', { groups: strings.list.unit(musicGroups .map(({ group, contributions }) => strings('artistPage.groupsLine.item', { group: strings.link.groupInfo(group, {to}), contributions: strings.count.contributions(contributions) }))) })}
${generateTrackList(releasedTrackListChunks, {strings, to})} `} ${unreleasedTracks.length && fixWS`${strings('artistPage.viewArtGallery.orBrowseList', { link: strings.link.artistGallery(artist, { to, text: strings('artistPage.viewArtGallery.link') }) })}
`}${strings('artistPage.artGroupsLine', { groups: strings.list.unit(artGroups .map(({ group, contributions }) => strings('artistPage.groupsLine.item', { group: strings.link.groupInfo(group, {to}), contributions: strings.count.contributions(contributions) }))) })}
${strings('artistGalleryPage.infoLine', { coverArts: strings.count.coverArts(artThingsGallery.length, {unit: true}) })}
${strings('redirectPage.infoLine', { target: `${target}` })}
${strings('misc.jumpTo')}
${strings('releaseInfo.released', {date: strings.count.date(flash.date)})}
${(flash.page || flash.urls.length) && `${strings('releaseInfo.playOn', { links: strings.list.or([ flash.page && getFlashLink(flash), ...flash.urls ].map(url => fancifyFlashURL(url, flash, {strings}))) })}
`} ${flash.tracks.length && fixWS`Tracks featured in ${flash.name.replace(/\.$/, '')}:
${strings('releaseInfo.contributors')}
${transformInline(flash.contributors.textContent, {strings, to, wikiData})}
${strings('releaseInfo.contributors')}
Choose a link to go to a random page in that category or album! If your browser doesn't support relatively modern JavaScript or you've disabled it, these links won't work - sorry.
(Data files have finished being downloaded. The links should work!)
${strings('listingIndex.infoLine', { wiki: wikiInfo.name, tracks: `${strings.count.tracks(releasedTracks.length, {unit: true})}`, albums: `${strings.count.albums(releasedAlbums.length, {unit: true})}`, duration: `${strings.count.duration(duration, {approximate: true, unit: true})}` })}
${strings('listingIndex.exploreList')}
${generateLinkIndexForListings(null, {strings, to, wikiData})} ` }, sidebarLeft: { content: generateSidebarForListings(null, {strings, to, wikiData}) }, nav: {simple: true} })) } function writeListingPage(listing, {wikiData}) { if (listing.condition && !listing.condition({wikiData})) { return null; } const { wikiInfo } = wikiData; const data = (listing.data ? listing.data({wikiData}) : null); return ({strings, writePage}) => writePage('listing', listing.directory, ({to}) => ({ title: listing.title({strings}), main: { content: fixWS`${strings('commentaryIndex.infoLine', { words: `${strings.count.words(totalWords, {unit: true})}`, entries: `${strings.count.commentaryEntries(totalEntries, {unit: true})}` })}
${strings('commentaryIndex.albumList.title')}
${strings('albumCommentaryPage.infoLine', { words: `${strings.count.words(words, {unit: true})}`, entries: `${strings.count.commentaryEntries(entries.length, {unit: true})}` })}
${album.commentary && fixWS`${transformMultiline(album.commentary, {strings, to, wikiData})}`} ${album.tracks.filter(t => t.commentary).map(track => fixWS`
${transformMultiline(track.commentary, {strings, to, wikiData})}`).join('\n')}
${strings('tagPage.infoLine', { coverArts: strings.count.coverArts(things.length, {unit: true}) })}
${ strings('releaseInfo.visitOn', { links: strings.list.or(group.urls.map(url => fancifyURL(url, {strings}))) }) }
`} ${!currentTrack && fixWS` ${next && `${ strings('albumSidebar.groupBox.next', { album: `${next.name}` }) }
`} ${previous && `${ strings('albumSidebar.groupBox.previous', { album: `${previous.name}` }) }
`} `} `); if (groupParts.length) { if (currentTrack) { const combinedGroupPart = groupParts.join('\n${ strings('releaseInfo.visitOn', { links: strings.list.or(group.urls.map(url => fancifyURL(url, {strings}))) }) }
`}${transformMultiline(group.description, {strings, to, wikiData})}
${ strings('groupInfoPage.viewAlbumGallery', { link: `${ strings('groupInfoPage.viewAlbumGallery.link') }` }) }
${ strings('groupGalleryPage.infoLine', { tracks: `${strings.count.tracks(releasedTracks.length, {unit: true})}`, albums: `${strings.count.albums(releasedAlbums.length, {unit: true})}`, time: `${strings.count.duration(totalDuration, {unit: true})}` }) }
${wikiInfo.features.groupUI && wikiInfo.features.listings && `(Choose another group to filter by!)
`}