diff options
| -rw-r--r-- | src/data/things.js | 8 | ||||
| -rw-r--r-- | src/data/yaml.js | 13 | ||||
| -rw-r--r-- | src/listing-spec.js | 47 | ||||
| -rw-r--r-- | src/misc-templates.js | 18 | ||||
| -rw-r--r-- | src/page/artist.js | 25 | ||||
| -rw-r--r-- | src/page/group.js | 9 | ||||
| -rw-r--r-- | src/page/track.js | 13 | ||||
| -rwxr-xr-x | src/upd8.js | 8 | ||||
| -rw-r--r-- | src/util/wiki-data.js | 242 |
9 files changed, 296 insertions, 87 deletions
diff --git a/src/data/things.js b/src/data/things.js index daec610..035879f 100644 --- a/src/data/things.js +++ b/src/data/things.js @@ -33,7 +33,7 @@ import * as S from './serialize.js'; import { getKebabCase, - sortByArtDate, + sortAlbumsTracksChronologically, } from '../util/wiki-data.js'; import find from '../util/find.js'; @@ -1128,8 +1128,10 @@ ArtTag.propertyDescriptors = { expose: { dependencies: ['albumData', 'trackData'], compute: ({ albumData, trackData, [ArtTag.instance]: artTag }) => ( - sortByArtDate([...albumData, ...trackData] - .filter(thing => thing.artTags?.includes(artTag)))) + sortAlbumsTracksChronologically( + ([...albumData, ...trackData] + .filter(thing => thing.artTags?.includes(artTag))), + {getDate: o => o.coverArtDate})) } } }; diff --git a/src/data/yaml.js b/src/data/yaml.js index 32cf729..763dfd2 100644 --- a/src/data/yaml.js +++ b/src/data/yaml.js @@ -42,8 +42,9 @@ import { } from '../util/sugar.js'; import { - sortByDate, - sortByName, + sortAlbumsTracksChronologically, + sortAlphabetically, + sortChronologically, } from '../util/wiki-data.js'; import find, { bindFind } from '../util/find.js'; @@ -861,7 +862,7 @@ export const dataSteps = [ processDocument: processNewsEntryDocument, save(newsData) { - sortByDate(newsData); + sortChronologically(newsData); newsData.reverse(); return {newsData}; @@ -876,7 +877,7 @@ export const dataSteps = [ processDocument: processArtTagDocument, save(artTagData) { - artTagData.sort(sortByName); + sortAlphabetically(artTagData); return {artTagData}; } @@ -1108,8 +1109,8 @@ export function linkWikiDataArrays(wikiData) { export function sortWikiDataArrays(wikiData) { Object.assign(wikiData, { - albumData: sortByDate(wikiData.albumData.slice()), - trackData: sortByDate(wikiData.trackData.slice()) + albumData: sortChronologically(wikiData.albumData.slice()), + trackData: sortAlbumsTracksChronologically(wikiData.trackData.slice()), }); // Re-link data arrays, so that every object has the new, sorted versions. diff --git a/src/listing-spec.js b/src/listing-spec.js index bb0c0a5..1c1dbd5 100644 --- a/src/listing-spec.js +++ b/src/listing-spec.js @@ -4,8 +4,8 @@ import { chunkByProperties, getArtistNumContributions, getTotalDuration, - sortByDate, - sortByName + sortAlphabetically, + sortChronologically, } from './util/wiki-data.js'; const listingSpec = [ @@ -14,8 +14,7 @@ const listingSpec = [ stringsKey: 'listAlbums.byName', data({wikiData}) { - return wikiData.albumData.slice() - .sort(sortByName); + return sortAlphabetically(wikiData.albumData.slice()); }, row(album, {link, language}) { @@ -66,7 +65,7 @@ const listingSpec = [ stringsKey: 'listAlbums.byDate', data({wikiData}) { - return sortByDate(wikiData.albumData.filter(album => album.date)); + return sortChronologically(wikiData.albumData.filter(album => album.date)); }, row(album, {link, language}) { @@ -114,8 +113,7 @@ const listingSpec = [ stringsKey: 'listArtists.byName', data({wikiData}) { - return wikiData.artistData.slice() - .sort(sortByName) + return sortAlphabetically(wikiData.artistData.slice()) .map(artist => ({artist, contributions: getArtistNumContributions(artist)})); }, @@ -254,22 +252,23 @@ const listingSpec = [ stringsKey: 'listArtists.byLatest', data({wikiData}) { - const reversedTracks = wikiData.trackData.filter(t => t.date).reverse(); - const reversedArtThings = wikiData.justEverythingSortedByArtDateMan.filter(t => t.date).reverse(); + const reversedTracks = sortChronologically(wikiData.trackData.filter(t => t.date)).reverse(); + const reversedArtThings = sortChronologically([...wikiData.trackData, ...wikiData.albumData].filter(t => t.coverArtDate)).reverse(); return { - toTracks: sortByDate(wikiData.artistData + toTracks: sortChronologically(wikiData.artistData .map(artist => ({ artist, + directory: artist.directory, + name: artist.name, date: reversedTracks.find(track => ([ ...track.artistContribs ?? [], ...track.contributorContribs ?? [] ].some(({ who }) => who === artist)))?.date })) - .filter(({ date }) => date) - .sort((a, b) => a.name < b.name ? 1 : a.name > b.name ? -1 : 0)).reverse(), + .filter(({ date }) => date)).reverse(), - toArtAndFlashes: sortByDate(wikiData.artistData + toArtAndFlashes: sortChronologically(wikiData.artistData .map(artist => { const thing = reversedArtThings.find(thing => ([ ...thing.coverArtistContribs ?? [], @@ -277,6 +276,8 @@ const listingSpec = [ ].some(({ who }) => who === artist))); return thing && { artist, + directory: artist.directory, + name: artist.name, date: (thing.coverArtistContribs?.some(({ who }) => who === artist) ? thing.coverArtDate : thing.date) @@ -332,7 +333,7 @@ const listingSpec = [ directory: 'groups/by-name', stringsKey: 'listGroups.byName', condition: ({wikiData}) => wikiData.wikiInfo.enableGroupUI, - data: ({wikiData}) => wikiData.groupData.slice().sort(sortByName), + data: ({wikiData}) => sortAlphabetically(wikiData.groupData.slice()), row(group, {link, language}) { return language.$('listingPage.listGroups.byCategory.group', { @@ -437,11 +438,13 @@ const listingSpec = [ condition: ({wikiData}) => wikiData.wikiInfo.enableGroupUI, data({wikiData}) { - return sortByDate(wikiData.groupData + return sortChronologically(wikiData.groupData .map(group => { const albums = group.albums.filter(a => a.date); return albums.length && { group, + directory: group.directory, + name: group.name, date: albums[albums.length - 1].date }; }) @@ -456,8 +459,8 @@ const listingSpec = [ // l8ter, that flips them, and UMSPAF ends up displaying 8efore // Fandom. So we do an extra reverse here, which will fix that // and only affect groups that share the same d8te (8ecause - // groups that don't will 8e moved 8y the sortByDate call - // surrounding this). + // groups that don't will 8e moved 8y the sortChronologically + // call surrounding this). .reverse()).reverse() }, @@ -474,7 +477,7 @@ const listingSpec = [ stringsKey: 'listTracks.byName', data({wikiData}) { - return wikiData.trackData.slice().sort(sortByName); + return sortAlphabetically(wikiData.trackData.slice()); }, row(track, {link, language}) { @@ -516,7 +519,7 @@ const listingSpec = [ data({wikiData}) { return chunkByProperties( - sortByDate(wikiData.trackData.filter(t => t.date)), + sortChronologically(wikiData.trackData.filter(t => t.date)), ['album', 'date'] ); }, @@ -659,7 +662,7 @@ const listingSpec = [ html(flashData, {link, language}) { return fixWS` <dl> - ${sortByDate(flashData.slice()).map(flash => fixWS` + ${sortChronologically(flashData.slice()).map(flash => fixWS` <dt>${language.$('listingPage.listTracks.inFlashes.byFlash.flash', { flash: link.flash(flash), date: language.formatDate(flash.date) @@ -715,9 +718,7 @@ const listingSpec = [ condition: ({wikiData}) => wikiData.wikiInfo.enableArtTagUI, data({wikiData}) { - return wikiData.artTagData - .filter(tag => !tag.isContentWarning) - .sort(sortByName) + return sortAlphabetically(wikiData.artTagData.filter(tag => !tag.isContentWarning)) .map(tag => ({tag, timesUsed: tag.taggedInThings?.length})); }, diff --git a/src/misc-templates.js b/src/misc-templates.js index f40229d..c337f6e 100644 --- a/src/misc-templates.js +++ b/src/misc-templates.js @@ -7,6 +7,11 @@ import fixWS from 'fix-whitespace'; import * as html from './util/html.js'; import { + Track, + Album, +} from './data/things.js'; + +import { getColors } from './util/colors.js'; @@ -16,7 +21,8 @@ import { import { getTotalDuration, - sortByDate + sortAlbumsTracksChronologically, + sortChronologically, } from './util/wiki-data.js'; const BANDCAMP_DOMAINS = [ @@ -112,7 +118,15 @@ export function generateChronologyLinks(currentThing, { } return contributions.map(({ who: artist }) => { - const things = sortByDate(unique(getThings(artist)).filter(t => t[dateKey]), dateKey); + const thingsUnsorted = unique(getThings(artist)).filter(t => t[dateKey]); + + // Kinda a hack, but we automatically detect which is (probably) the + // right function to use here. + const args = [thingsUnsorted, {getDate: t => t[dateKey]}]; + const things = (thingsUnsorted.every(t => t instanceof Album || t instanceof Track) + ? sortAlbumsTracksChronologically(...args) + : sortChronologically(...args)); + const index = things.indexOf(currentThing); if (index === -1) return ''; diff --git a/src/page/artist.js b/src/page/artist.js index c15e034..17ff5b6 100644 --- a/src/page/artist.js +++ b/src/page/artist.js @@ -16,7 +16,10 @@ import { import { chunkByProperties, getTotalDuration, - sortByDate + sortAlbumsTracksChronologically, + sortByDate, + sortByDirectory, + sortChronologically, } from '../util/wiki-data.js'; // Page exports @@ -30,19 +33,19 @@ export function write(artist, {wikiData}) { const { name, urls, contextNotes } = artist; - const artThingsAll = sortByDate(unique([ + const artThingsAll = sortAlbumsTracksChronologically(unique([ ...artist.albumsAsCoverArtist ?? [], ...artist.albumsAsWallpaperArtist ?? [], ...artist.albumsAsBannerArtist ?? [], ...artist.tracksAsCoverArtist ?? [] - ])); + ]), {getDate: o => o.coverArtDate}); - const artThingsGallery = sortByDate([ + const artThingsGallery = sortAlbumsTracksChronologically([ ...artist.albumsAsCoverArtist ?? [], ...artist.tracksAsCoverArtist ?? [] - ]); + ], {getDate: o => o.coverArtDate}); - const commentaryThings = sortByDate([ + const commentaryThings = sortAlbumsTracksChronologically([ ...artist.albumsAsCommentator ?? [], ...artist.tracksAsCommentator ?? [] ]); @@ -56,24 +59,24 @@ export function write(artist, {wikiData}) { key }); - const artListChunks = chunkByProperties(sortByDate(artThingsAll.flatMap(thing => + const artListChunks = chunkByProperties(artThingsAll.flatMap(thing => (['coverArtistContribs', 'wallpaperArtistContribs', 'bannerArtistContribs'] .map(key => getArtistsAndContrib(thing, key)) .filter(({ contrib }) => contrib) .map(props => ({ album: thing.album || thing, track: thing.album ? thing : null, - date: +(thing.coverArtDate || thing.date), + date: thing.date, ...props }))) - )), ['date', 'album']); + ), ['date', 'album']); const commentaryListChunks = chunkByProperties(commentaryThings.map(thing => ({ album: thing.album || thing, track: thing.album ? thing : null })), ['album']); - const allTracks = sortByDate(unique([ + const allTracks = sortAlbumsTracksChronologically(unique([ ...artist.tracksAsArtist ?? [], ...artist.tracksAsContributor ?? [] ])); @@ -119,7 +122,7 @@ export function write(artist, {wikiData}) { let flashes, flashListChunks; if (wikiInfo.enableFlashesAndGames) { - flashes = sortByDate(artist.flashesAsContributor?.slice() ?? []); + flashes = sortChronologically(artist.flashesAsContributor?.slice() ?? []); flashListChunks = ( chunkByProperties(flashes.map(flash => ({ act: flash.act, diff --git a/src/page/group.js b/src/page/group.js index eb401dd..4eb8fb3 100644 --- a/src/page/group.js +++ b/src/page/group.js @@ -8,7 +8,7 @@ import * as html from '../util/html.js'; import { getTotalDuration, - sortByDate + sortChronologically, } from '../util/wiki-data.js'; // Page exports @@ -142,7 +142,12 @@ export function write(group, {wikiData}) { )} <div class="grid-listing"> ${getAlbumGridHTML({ - entries: sortByDate(group.albums.map(item => ({item}))).reverse(), + entries: sortChronologically(group.albums.map(album => ({ + item: album, + directory: album.directory, + name: album.name, + date: album.date, + }))).reverse(), details: true })} </div> diff --git a/src/page/track.js b/src/page/track.js index d51cee2..04e00ee 100644 --- a/src/page/track.js +++ b/src/page/track.js @@ -19,7 +19,7 @@ import { import { getTrackCover, getAlbumListTag, - sortByDate + sortChronologically, } from '../util/wiki-data.js'; // Page exports @@ -36,8 +36,15 @@ export function write(track, {wikiData}) { let flashesThatFeature; if (wikiInfo.enableFlashesAndGames) { - flashesThatFeature = sortByDate([track, ...otherReleases] - .flatMap(track => track.featuredInFlashes.map(flash => ({flash, as: track})))); + flashesThatFeature = sortChronologically([track, ...otherReleases] + .flatMap(track => track.featuredInFlashes + .map(flash => ({ + flash, + as: track, + directory: flash.directory, + name: flash.name, + date: flash.date + })))); } const unbound_getTrackItem = (track, {getArtistString, link, language}) => ( diff --git a/src/upd8.js b/src/upd8.js index ba59068..1631c7b 100755 --- a/src/upd8.js +++ b/src/upd8.js @@ -128,15 +128,11 @@ import { getAlbumListTag, getAllTracks, getArtistAvatar, - getArtistCommentary, getArtistNumContributions, getFlashCover, getKebabCase, getTotalDuration, getTrackCover, - sortByArtDate, - sortByDate, - sortByName } from './util/wiki-data.js'; import { @@ -1738,10 +1734,6 @@ async function main() { } } - WD.justEverythingMan = sortByDate([...WD.albumData, ...WD.trackData, ...(WD.flashData || [])]); - WD.justEverythingSortedByArtDateMan = sortByArtDate(WD.justEverythingMan.slice()); - // console.log(JSON.stringify(justEverythingSortedByArtDateMan.map(toAnythingMan), null, 2)); - WD.officialAlbumData = WD.albumData.filter(album => album.groups.some(group => group.directory === OFFICIAL_GROUP_DIRECTORY)); WD.fandomAlbumData = WD.albumData.filter(album => album.groups.every(group => group.directory !== OFFICIAL_GROUP_DIRECTORY)); diff --git a/src/util/wiki-data.js b/src/util/wiki-data.js index b4f7f21..aba508c 100644 --- a/src/util/wiki-data.js +++ b/src/util/wiki-data.js @@ -1,5 +1,7 @@ // Utility functions for interacting with wiki data. +import { Album, Track } from '../data/things.js'; + // Generic value operations export function getKebabCase(name) { @@ -62,32 +64,115 @@ export function chunkByProperties(array, properties) { })); } -// Sorting functions +// Sorting functions - all utils here are mutating, so make sure to initially +// slice/filter/somehow generate a new array from input data if retaining the +// initial sort matters! (Spoilers: If what you're doing involves any kind of +// parallelization, it definitely matters.) + +// General sorting utilities! These don't do any sorting on their own but are +// handy in the sorting functions below (or if you're making your own sort). + +export function compareCaseLessSensitive(a, b) { + // Compare two strings without considering capitalization... unless they + // happen to be the same that way. + + const al = a.toLowerCase(); + const bl = b.toLowerCase(); + + return (al === bl + ? a.localeCompare(b, undefined, {numeric: true}) + : al.localeCompare(bl, undefined, {numeric: true})); +} + +// Subtract common prefixes and other characters which some people don't like +// to have considered while sorting. The words part of this is English-only for +// now, which is totally evil. +export function normalizeName(s) { + // Turn (some) ligatures into expanded variant for cleaner sorting, e.g. + // "ff" into "ff", in decompose mode, so that "ü" is represented as two + // bytes ("u" + \u0308 combining diaeresis). + s = s.normalize('NFKD'); -export function sortByName(a, b) { - let an = a.name.toLowerCase(); - let bn = b.name.toLowerCase(); - if (an.startsWith('the ')) an = an.slice(4); - if (bn.startsWith('the ')) bn = bn.slice(4); - return an < bn ? -1 : an > bn ? 1 : 0; + // Replace one or more whitespace of any kind in a row, as well as certain + // punctuation, with a single typical space, then trim the ends. + s = s.replace(/[\p{Separator}\p{Dash_Punctuation}\p{Connector_Punctuation}]+/gu, ' ').trim(); + + // Discard anything that isn't a letter, number, or space. + s = s.replace(/[^\p{Letter}\p{Number} ]/gu, ''); + + // Remove common English (only, for now) prefixes. + s = s.replace(/^(?:an?|the) /i, ''); + + return s; +} + +// Component sort functions - these sort by one particular property, applying +// unique particulars where appropriate. Usually you don't want to use these +// directly, but if you're making a custom sort they can come in handy. + +// Universal method for sorting things into a predictable order, as directory +// is taken to be unique. There are two exceptions where this function (and +// thus any of the composite functions that start with it) *can't* be taken as +// deterministic: +// +// 1) Mixed data of two different Things, as directories are only taken as +// unique within one given class of Things. For example, this function +// won't be deterministic if its array contains both <album:ithaca> and +// <track:ithaca>. +// +// 2) Duplicate directories, or multiple instances of the "same" Thing. +// This function doesn't differentiate between two objects of the same +// directory, regardless of any other properties or the overall "identity" +// of the object. +// +// These exceptions are unavoidable except for not providing that kind of data +// in the first place, but you can still ensure the overall program output is +// deterministic by ensuring the input is arbitrarily sorted according to some +// other criteria - ex, although sortByDirectory itself isn't determinstic when +// given mixed track and album data, the final output (what goes on the site) +// will always be the same if you're doing sortByDirectory([...albumData, +// ...trackData]), because the initial sort places albums before tracks - and +// sortByDirectory will handle the rest, given all directories are unique +// except when album and track directories overlap with each other. +export function sortByDirectory(data, { + getDirectory = o => o.directory +} = {}) { + return data.sort((a, b) => { + const ad = getDirectory(a); + const bd = getDirectory(b); + return compareCaseLessSensitive(ad, bd) + }); } -// This function was originally made to sort just al8um data, 8ut its exact -// code works fine for sorting tracks too, so I made the varia8les and names -// more general. -export function sortByDate(data, dateKey = 'date') { - // Just to 8e clear: sort is a mutating function! I only return the array - // 8ecause then you don't have to define it as a separate varia8le 8efore - // passing it into this function. - return data.sort(({ [dateKey]: a }, { [dateKey]: b }) => { +export function sortByName(data, { + getName = o => o.name +} = {}) { + return data.sort((a, b) => { + const an = getName(a); + const bn = getName(b); + const ann = normalizeName(an); + const bnn = normalizeName(bn); + return ( + compareCaseLessSensitive(ann, bnn) || + compareCaseLessSensitive(an, bn)); + }); +} + +export function sortByDate(data, { + getDate = o => o.date +} = {}) { + return data.sort((a, b) => { + const ad = getDate(a); + const bd = getDate(b); + // It's possible for objects with and without dates to be mixed // together in the same array. If that's the case, we put all items // without dates at the end. - if (a && b) { - return a - b; - } else if (a) { + if (ad && bd) { + return ad - bd; + } else if (ad) { return -1; - } else if (b) { + } else if (bd) { return 1; } else { // If neither of the items being compared have a date, don't move @@ -99,9 +184,115 @@ export function sortByDate(data, dateKey = 'date') { }); } -// Same details as the sortByDate, 8ut for covers~ -export function sortByArtDate(data) { - return data.sort((a, b) => (a.coverArtDate || a.date) - (b.coverArtDate || b.date)); +export function sortByPositionInAlbum(data) { + return data.sort((a, b) => { + const aa = a.album; + const ba = b.album; + + // Don't change the sort when the two tracks are from separate albums. + // This function doesn't change the order of albums or try to "merge" + // two separated chunks of tracks from the same album together. + if (aa !== ba) { + return 0; + } + + // Don't change the sort when only one (or neither) item is actually + // a track (i.e. has an album). + if (!aa || !ba) { + return 0; + } + + const ai = aa.tracks.indexOf(a); + const bi = ba.tracks.indexOf(b); + + // There's no reason this two-way reference (a track's album and the + // album's track list) should be broken, but if for any reason it is, + // don't change the sort. + if (ai === -1 || bi === -1) { + return 0; + } + + return ai - bi; + }); +} + +// Note that this function only checks constructor equality, not inheritence! +// So it won't group subclasses together (as though they were the same type). +export function sortByThingType(data, thingConstructors) { + data.sort((a, b) => { + const ai = thingConstructors.indexOf(a.constructor); + const bi = thingConstructors.indexOf(b.constructor); + + if (ai >= 0 && bi >= 0) { + return ai - bi; + } else if (ai >= 0) { + return -1; + } else if (bi >= 0) { + return 1; + } else { + return 0; + } + }); +} + +// Composite sorting functions - these consider multiple properties, generally +// always returning the same output regardless of how the input was originally +// sorted (or left unsorted). If you're working with arbitrarily sorted inputs +// (typically wiki data, either in full or unsorted filter), these make sure +// what gets put on the actual website (or wherever) is deterministic. Also +// they're just handy sorting utilities. +// +// Note that because these are each comprised of multiple component sorting +// functions, they expect more than just one property to be present for full +// sorting (listed above each function). If you're mapping thing objects to +// another representation, try to include all of these listed properties. + +// Expects thing properties: +// * directory (or override getDirectory) +// * name (or override getName) +export function sortAlphabetically(data, {getDirectory, getName} = {}) { + sortByDirectory(data, {getDirectory}); + sortByName(data, {getName}); + return data; +} + +// Expects thing properties: +// * directory (or override getDirectory) +// * name (or override getName) +// * date (or override getDate) +export function sortChronologically(data, {getDirectory, getName, getDate} = {}) { + sortAlphabetically(data, {getDirectory, getName}); + sortByDate(data, {getDate}); + return data; +} + +// Highly contextual sort functions - these are only for very specific types +// of Things, and have appropriately hard-coded behavior. + +// Sorts so that tracks from the same album are generally grouped together in +// their original (album track list) order, while prioritizing date (by default +// release date but can be overridden) above all else. +// +// This function also works for data lists which contain only tracks. +export function sortAlbumsTracksChronologically(data, {getDate} = {}) { + // Sort albums before tracks... + sortByThingType(data, [Album, Track]); + + // Group tracks by album... + sortByDirectory(data, { + getDirectory: t => (t.album ? t.album.directory : t.directory) + }); + + // Sort tracks by position in album... + sortByPositionInAlbum(data); + + // ...and finally sort by date. If tracks from more than one album were + // released on the same date, they'll still be grouped together by album, + // and tracks within an album will retain their relative positioning (i.e. + // stay in the same order as part of the album's track listing). + sortByDate(data, {getDate}); + + return data; } // Specific data utilities @@ -152,13 +343,6 @@ export function getArtistNumContributions(artist) { ); } -export function getArtistCommentary(artist, {justEverythingMan}) { - return justEverythingMan.filter(thing => - (thing?.commentary - .replace(/<\/?b>/g, '') - .includes('<i>' + artist.name + ':</i>'))); -} - export function getFlashCover(flash, {to}) { return to('media.flashArt', flash.directory, flash.coverArtFileExtension); } |