preload, map and format sizes of additional files

6 files changed, 195 insertions, 9 deletions

diff --git a/src/data/things.js b/src/data/things.js
index 80e22e3..1865ee4 100644
--- a/src/data/things.js
+++ b/src/data/things.js
@@ -1649,6 +1649,28 @@ Object.assign(Language.prototype, {
         return this.intl_listUnit.format(array);
     },
 
+    // File sizes: 42.5 kB, 127.2 MB, 4.13 GB, 998.82 TB
+    formatFileSize(bytes) {
+        if (!bytes) return '';
+
+        bytes = parseInt(bytes);
+        if (isNaN(bytes)) return '';
+
+        const round = exp => Math.round(bytes / 10 ** (exp - 1)) / 10;
+
+        if (bytes >= 10 ** 12) {
+            return this.formatString('count.fileSize.terabytes', {terabytes: round(12)});
+        } else if (bytes >= 10 ** 9) {
+            return this.formatString('count.fileSize.gigabytes', {gigabytes: round(9)});
+        } else if (bytes >= 10 ** 6) {
+            return this.formatString('count.fileSize.megabytes', {megabytes: round(6)});
+        } else if (bytes >= 10 ** 3) {
+            return this.formatString('count.fileSize.kilobytes', {kilobytes: round(3)});
+        } else {
+            return this.formatString('count.fileSize.bytes', {bytes});
+        }
+    },
+
     // TODO: These are hard-coded. Is there a better way?
     countAlbums: countHelper('albums'),
     countCommentaryEntries: countHelper('commentaryEntries', 'entries'),
diff --git a/src/file-size-preloader.js b/src/file-size-preloader.js
new file mode 100644
index 0000000..d0807cc
--- /dev/null
+++ b/src/file-size-preloader.js
@@ -0,0 +1,100 @@
+// Very simple, bare-bones file size loader which takes a bunch of file
+// paths, gets their filesizes, and resolves a promise when it's done.
+//
+// Once the size of a path has been loaded, it's available synchronously -
+// so this may be provided to code areas which don't support async code!
+//
+// This class also supports loading more paths after the initial batch is
+// done (it uses a queue system) - but make sure you pause any sync code
+// depending on the results until it's finished. waitUntilDoneLoading will
+// always hold until the queue is completely emptied, including waiting for
+// any entries to finish which were added after the wait function itself was
+// called. (Same if you decide to await loadPaths. Sorry that function won't
+// resolve as soon as just the paths it provided are finished - that's not
+// really a worthwhile feature to support for its complexity here, since
+// basically all this should process almost instantaneously anyway!)
+//
+// This only processes files one at a time because I'm lazy and stat calls
+// are very, very fast.
+
+import { stat } from 'fs/promises';
+import { logWarn } from './util/cli.js';
+
+export default class FileSizePreloader {
+    #paths = [];
+    #sizes = [];
+    #loadedPathIndex = -1;
+
+    #loadingPromise = null;
+    #resolveLoadingPromise = null;
+
+    loadPaths(...paths) {
+        this.#paths.push(...paths.filter(p => !this.#paths.includes(p)));
+        return this.#startLoadingPaths();
+    }
+
+    waitUntilDoneLoading() {
+        return this.#loadingPromise ?? Promise.resolve();
+    }
+
+    #startLoadingPaths() {
+        if (this.#loadingPromise) {
+            return this.#loadingPromise;
+        }
+
+        this.#loadingPromise = new Promise((resolve => {
+            this.#resolveLoadingPromise = resolve;
+        }));
+
+        this.#loadNextPath();
+
+        return this.#loadingPromise;
+    }
+
+    async #loadNextPath() {
+        if (this.#loadedPathIndex === this.#paths.length - 1) {
+            return this.#doneLoadingPaths();
+        }
+
+        let size;
+
+        const path = this.#paths[this.#loadedPathIndex + 1];
+
+        try {
+            size = await this.readFileSize(path);
+        } catch (error) {
+            // Oops! Discard that path, and don't increment the index before
+            // moving on, since the next path will now be in its place.
+            this.#paths.splice(this.#loadedPathIndex + 1, 1);
+            logWarn`Failed to process file size for ${path}: ${error.message}`;
+            return this.#loadNextPath();
+        }
+
+        this.#sizes.push(size);
+        this.#loadedPathIndex++;
+        return this.#loadNextPath();
+    }
+
+    #doneLoadingPaths() {
+        this.#resolveLoadingPromise();
+        this.#loadingPromise = null;
+        this.#resolveLoadingPromise = null;
+    }
+
+    // Override me if you want?
+    // The rest of the code here is literally just a queue system, so you could
+    // pretty much repurpose it for anything... but there are probably cleaner
+    // ways than making an instance or subclass of this and overriding this one
+    // method!
+    async readFileSize(path) {
+        const stats = await stat(path);
+        return stats.size;
+    }
+
+    getSizeOfPath(path) {
+        const index = this.#paths.indexOf(path);
+        if (index === -1) return null;
+        if (index > this.#loadedPathIndex) return null;
+        return this.#sizes[index];
+    }
+}
diff --git a/src/misc-templates.js b/src/misc-templates.js
index ec58398..58c45f5 100644
--- a/src/misc-templates.js
+++ b/src/misc-templates.js
@@ -39,7 +39,7 @@ export function generateAdditionalFilesShortcut(additionalFiles, {language}) {
     });
 }
 
-export function generateAdditionalFilesList(additionalFiles, {language, linkFile}) {
+export function generateAdditionalFilesList(additionalFiles, {language, getFileSize, linkFile}) {
     if (!additionalFiles?.length) return '';
 
     const fileCount = additionalFiles.flatMap(g => g.files).length;
@@ -52,10 +52,17 @@ export function generateAdditionalFilesList(additionalFiles, {language, linkFile
                     ? language.$('releaseInfo.additionalFiles.entry.withDescription', {title, description})
                     : language.$('releaseInfo.additionalFiles.entry', {title}))}</dt>
                 <dd><ul>
-                    ${files.map(file => `<li>${language.$('releaseInfo.additionalFiles.file', {
-                        file: linkFile(file),
-                        size: '<i>pre-computed size</i>'
-                    })}</li>`).join('\n')}
+                    ${files.map(file => {
+                        const size = getFileSize(file);
+                        return (size
+                            ? `<li>${language.$('releaseInfo.additionalFiles.file.withSize', {
+                                file: linkFile(file),
+                                size: language.formatFileSize(getFileSize(file))
+                            })}</li>`
+                            : `<li>${language.$('releaseInfo.additionalFiles.file', {
+                                file: linkFile(file)
+                            })}</li>`);
+                    }).join('\n')}
                 </ul></dd>
             `).join('\n')}
         </dl>
diff --git a/src/page/album.js b/src/page/album.js
index 8df8a67..76c9c5f 100644
--- a/src/page/album.js
+++ b/src/page/album.js
@@ -109,10 +109,12 @@ export function write(album, {wikiData}) {
             getAlbumStylesheet,
             getArtistString,
             getLinkThemeString,
+            getSizeOfAdditionalFile,
             getThemeString,
             link,
             language,
-            transformMultiline
+            transformMultiline,
+            urls,
         }) => {
             const trackToListItem = bindOpts(unbound_trackToListItem, {
                 getArtistString,
@@ -219,7 +221,11 @@ export function write(album, {wikiData}) {
                             </${listTag}>
                         `}
                         ${hasAdditionalFiles && generateAdditionalFilesList(album.additionalFiles, {
-                            linkFile: file => link.albumAdditionalFile({album, file})
+                            // TODO: Kinda near the metal here...
+                            getFileSize: file => getSizeOfAdditionalFile(urls
+                                .from('media.root')
+                                .to('media.albumAdditionalFile', album.directory, file)),
+                            linkFile: file => link.albumAdditionalFile({album, file}),
                         })}
                         ${album.dateAddedToWiki && fixWS`
                             <p>
diff --git a/src/strings-default.json b/src/strings-default.json
index 78de7a8..cc4ec1d 100644
--- a/src/strings-default.json
+++ b/src/strings-default.json
@@ -71,6 +71,11 @@
     "count.duration.minutes.withUnit": "{MINUTES}:{SECONDS} minutes",
     "count.duration.approximate": "~{DURATION}",
     "count.duration.missing": "_:__",
+    "count.fileSize.terabytes": "{TERABYTES} TB",
+    "count.fileSize.gigabytes": "{GIGABYTES} GB",
+    "count.fileSize.megabytes": "{MEGABYTES} MB",
+    "count.fileSize.kilobytes": "{KILOBYTES} kB",
+    "count.fileSize.bytes": "{BYTES} bytes",
     "releaseInfo.by": "By {ARTISTS}.",
     "releaseInfo.from": "From {ALBUM}.",
     "releaseInfo.coverArtBy": "Cover art by {ARTISTS}.",
@@ -103,7 +108,8 @@
     "releaseInfo.additionalFiles.heading": "Has {FILE_COUNT} additional files:",
     "releaseInfo.additionalFiles.entry": "{TITLE}",
     "releaseInfo.additionalFiles.entry.withDescription": "{TITLE}: {DESCRIPTION}",
-    "releaseInfo.additionalFiles.file": "{FILE} ({SIZE})",
+    "releaseInfo.additionalFiles.file": "{FILE}",
+    "releaseInfo.additionalFiles.file.withSize": "{FILE} ({SIZE})",
     "releaseInfo.note": "Note:",
     "trackList.group": "{GROUP} ({DURATION}):",
     "trackList.item.withDuration": "({DURATION}) {TRACK}",
diff --git a/src/upd8.js b/src/upd8.js
index b55ddda..12f1af3 100755
--- a/src/upd8.js
+++ b/src/upd8.js
@@ -173,6 +173,8 @@ import {
     OFFICIAL_GROUP_DIRECTORY
 } from './util/magic-constants.js';
 
+import FileSizePreloader from './file-size-preloader.js';
+
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
 
 const CACHEBUST = 8;
@@ -1655,6 +1657,45 @@ async function main() {
     WD.officialAlbumData = WD.albumData.filter(album => album.groups.some(group => group.directory === OFFICIAL_GROUP_DIRECTORY));
     WD.fandomAlbumData = WD.albumData.filter(album => album.groups.every(group => group.directory !== OFFICIAL_GROUP_DIRECTORY));
 
+    const fileSizePreloader = new FileSizePreloader();
+
+    // File sizes of additional files need to be precalculated before we can
+    // actually reference 'em in site building, so get those loading right
+    // away. We actually need to keep track of two things here - the on-device
+    // file paths we're actually reading, and the corresponding on-site media
+    // paths that will be exposed in site build code. We'll build a mapping
+    // function between them so that when site code requests a site path,
+    // it'll get the size of the file at the corresponding device path.
+    const additionalFilePaths = [
+        ...WD.albumData.flatMap(album => (
+            [
+                ...album.additionalFiles ?? [],
+                ...album.tracks.flatMap(track => track.additionalFiles ?? [])
+            ]
+            .flatMap(fileGroup => fileGroup.files)
+            .map(file => ({
+                device: (path.join(mediaPath, urls
+                    .from('media.root')
+                    .toDevice('media.albumAdditionalFile', album.directory, file))),
+                media: (urls
+                    .from('media.root')
+                    .to('media.albumAdditionalFile', album.directory, file))
+            })))),
+    ];
+
+    const getSizeOfAdditionalFile = mediaPath => {
+        const { device = null } = additionalFilePaths.find(({ media }) => media === mediaPath) || {};
+        if (!device) return null;
+        return fileSizePreloader.getSizeOfPath(device);
+    };
+
+    logInfo`Preloading filesizes for ${additionalFilePaths.length} additional files...`;
+
+    fileSizePreloader.loadPaths(...additionalFilePaths.map(path => path.device));
+    await fileSizePreloader.waitUntilDoneLoading();
+
+    logInfo`Done preloading filesizes!`;
+
     if (noBuild) return;
 
     // Makes writing a little nicer on CPU theoretically, 8ut also costs in
@@ -2016,8 +2057,12 @@ async function main() {
 
                 const pageFn = () => page({
                     ...bound,
+
                     language,
-                    to
+                    to,
+                    urls,
+
+                    getSizeOfAdditionalFile,
                 });
 
                 const content = writePage.html(pageFn, {