From 0daee54085810797eea235b513b8357636cda49f Mon Sep 17 00:00:00 2001 From: Liam Date: Sat, 15 Jul 2017 11:45:55 -0400 Subject: Begin updating the playlist format --- src/crawl-itunes.js | 3 + src/download-playlist.js | 183 +++++++++++++++++++---------------------------- src/http-music.js | 65 +++++------------ src/loop-play.js | 14 ++-- src/pickers.js | 16 ++--- src/playlist-utils.js | 158 ++++++++++++++++++++++++++++++++++------ 6 files changed, 246 insertions(+), 193 deletions(-) mode change 100644 => 100755 src/crawl-itunes.js mode change 100644 => 100755 src/download-playlist.js (limited to 'src') diff --git a/src/crawl-itunes.js b/src/crawl-itunes.js old mode 100644 new mode 100755 index ec6c3ec..1c678dd --- a/src/crawl-itunes.js +++ b/src/crawl-itunes.js @@ -1,3 +1,6 @@ +#!/usr/bin/env node + +'use strict' const fs = require('fs') const path = require('path') diff --git a/src/download-playlist.js b/src/download-playlist.js old mode 100644 new mode 100755 index c8476e4..18e1a7f --- a/src/download-playlist.js +++ b/src/download-playlist.js @@ -1,20 +1,18 @@ -// TODO: This almost definitely doesn't work, ever since downloaders were -// removed! Maybe it's possible to make mpv only download (and not play) a -// file? +#!/usr/bin/env node 'use strict' const fs = require('fs') -const downloaders = require('./downloaders') const path = require('path') -const processArgv = require('./process-argv') const sanitize = require('sanitize-filename') +const promisifyProcess = require('./promisify-process') const { - isGroup, isTrack + isGroup, isTrack, flattenPlaylist, updatePlaylistFormat } = require('./playlist-utils') const { promisify } = require('util') +const { spawn } = require('child_process') const access = promisify(fs.access) const mkdir = promisify(fs.mkdir) @@ -22,107 +20,86 @@ const readFile = promisify(fs.readFile) const readdir = promisify(fs.readdir) const stat = promisify(fs.stat) const writeFile = promisify(fs.writeFile) -const ncp = promisify(require('ncp').ncp) -// It's typically bad to attempt to download or copy a million files at once, -// so we create a "promise delayer" that forces only several promises to run at -// at one time. -let delayPromise -{ - const INTERVAL = 50 - const MAX = 5 +async function downloadCrawl(topPlaylist, initialOutPath = './out/') { + let doneCount = 0 + let total = flattenPlaylist(topPlaylist).length - let active = 0 - - let queue = [] - - delayPromise = function(promiseMaker) { - return new Promise((resolve, reject) => { - queue.push([promiseMaker, resolve, reject]) - }) + const status = function() { + const percent = Math.trunc(doneCount / total * 10000) / 100 + console.log( + `\x1b[1mDownload crawler - ${percent}% completed ` + + `(${doneCount}/${total} tracks)\x1b[0m`) } - setInterval(async () => { - if (active >= MAX) { - return + const recursive = async function(groupContents, outPath) { + // If the output folder doesn't exist, we should create it. + let doesExist = true + try { + doesExist = (await stat(outPath)).isDirectory() + } catch(err) { + doesExist = false } - const top = queue.pop() - - if (top) { - const [ promiseMaker, resolve, reject ] = top - - active++ - - console.log('Going - queue: ' + queue.length) + if (!doesExist) { + await mkdir(outPath) + } - try { - resolve(await promiseMaker()) - } catch(err) { - reject(err) + let outPlaylist = [] + + for (let item of groupContents) { + if (isGroup(item)) { + // TODO: Not sure if this is the best way to pick the next out dir. + const out = outPath + sanitize(item[0]) + '/' + + outPlaylist.push([item[0], await recursive(item[1], out)]) + } else if (isTrack(item)) { + const base = sanitize(path.basename(item[0], path.extname(item[0]))) + const out = outPath + sanitize(base) + '.mp3' + + // If we've already downloaded a file at some point in previous time, + // there's no need to download it again! + // + // Since we can't guarantee the extension name of the file, we only + // compare bases. + // + // TODO: This probably doesn't work well with things like the YouTube + // downloader. + const items = await readdir(outPath) + const match = items.find(item => { + const itemBase = sanitize(path.basename(item, path.extname(item))) + return itemBase === base + }) + + if (match) { + console.log(`\x1b[32;2mAlready downloaded: ${out}\x1b[0m`) + outPlaylist.push([item[0], outPath + match]) + doneCount++ + status() + continue + } + + console.log(`\x1b[2mDownloading: ${item[0]} - ${item[1]}\x1b[0m`) + + console.log(out) + + await promisifyProcess(spawn('mpv', [ + '--no-audio-display', + item[1], '-o', out, + '-oac', 'libmp3lame' + ])) + + outPlaylist.push([item[0], out]) + doneCount++ + + status() } - - active-- } - }, INTERVAL) -} - -async function downloadCrawl(playlist, downloader, outPath = './out/') { - // If the output folder doesn't exist, we should create it. - let doesExist = true - try { - doesExist = (await stat(outPath)).isDirectory() - } catch(err) { - doesExist = false - } - if (!doesExist) { - await mkdir(outPath) + return outPlaylist } - return Promise.all(playlist.map(async (item) => { - if (isGroup(item)) { - // TODO: Not sure if this is the best way to pick the next out dir. - const out = outPath + sanitize(item[0]) + '/' - - return [item[0], await downloadCrawl(item[1], downloader, out)] - } else if (isTrack(item)) { - // TODO: How should we deal with songs that don't have an extension? - const ext = path.extname(item[1]) - const base = path.basename(item[1], ext) - const out = outPath + base + ext - - // If we've already downloaded a file at some point in previous time, - // there's no need to download it again! - // - // Since we can't guarantee the extension name of the file, we only - // compare bases. - // - // TODO: This probably doesn't work well with things like the YouTube - // downloader. - const items = await readdir(outPath) - const match = items.find(x => path.basename(x, path.extname(x)) === base) - if (match) { - console.log(`\x1b[32;2mAlready downloaded: ${out}\x1b[0m`) - return [item[0], outPath + match] - } - - console.log(`\x1b[2mDownloading: ${item[0]} - ${item[1]}\x1b[0m`) - - const downloadFile = await delayPromise(() => downloader(item[1])) - // console.log(downloadFile, path.resolve(out)) - - try { - await delayPromise(() => ncp(downloadFile, path.resolve(out))) - console.log(`\x1b[32;1mDownloaded: ${out}\x1b[0m`) - return [item[0], out] - } catch(err) { - console.error(`\x1b[31mFailed: ${out}\x1b[0m`) - console.error(err) - return false - } - } - })).then(p => p.filter(Boolean)) + return recursive(topPlaylist.items, initialOutPath) } async function main() { @@ -133,21 +110,11 @@ async function main() { return } - const playlist = JSON.parse(await readFile(process.argv[2])) - - let downloaderType = 'http' - - processArgv(process.argv.slice(3), { - '-downloader': util => { - downloaderType = util.nextArg() - } - }) - - const dl = downloaders.makePowerfulDownloader( - downloaders.getDownloader(downloaderType) + const playlist = updatePlaylistFormat( + JSON.parse(await readFile(process.argv[2])) ) - const outPlaylist = await downloadCrawl(playlist, dl) + const outPlaylist = await downloadCrawl(playlist) await writeFile('out/playlist.json', JSON.stringify(outPlaylist, null, 2)) diff --git a/src/http-music.js b/src/http-music.js index a6c9479..26a9675 100755 --- a/src/http-music.js +++ b/src/http-music.js @@ -10,7 +10,8 @@ const processArgv = require('./process-argv') const fetch = require('node-fetch') const { - filterPlaylistByPathString, removeGroupByPathString, getPlaylistTreeString + filterPlaylistByPathString, removeGroupByPathString, getPlaylistTreeString, + updatePlaylistFormat } = require('./playlist-utils') const readFile = promisify(fs.readFile) @@ -35,7 +36,7 @@ function downloadPlaylistFromOptionValue(arg) { Promise.resolve() .then(async () => { let sourcePlaylist = null - let activePlaylist = null + let activePlaylistGroup = null let pickerType = 'shuffle' let playOpts = [] @@ -64,46 +65,16 @@ Promise.resolve() return false } - const openedPlaylist = JSON.parse(playlistText) + const openedPlaylist = updatePlaylistFormat(JSON.parse(playlistText)) - // Playlists can be in two formats... - if (Array.isArray(openedPlaylist)) { - // ..the first, a simple array of tracks and groups; + sourcePlaylist = openedPlaylist + activePlaylistGroup = {items: openedPlaylist.items} - sourcePlaylist = openedPlaylist - activePlaylist = openedPlaylist - } else if (typeof openedPlaylist === 'object') { - // ..or an object including metadata and configuration as well as the - // array described in the first. - - if (!('tracks' in openedPlaylist)) { - throw new Error( - "Trackless object-type playlist (requires 'tracks' property)" - ) - } - - sourcePlaylist = openedPlaylist.tracks - activePlaylist = openedPlaylist.tracks - - // What's handy about the object-type playlist is that you can pass - // options that will be run every time the playlist is opened: - if ('options' in openedPlaylist) { - if (Array.isArray(openedPlaylist.options)) { - processArgv(openedPlaylist.options, optionFunctions) - } else { - throw new Error( - "Invalid 'options' property (expected array): " + file - ) - } - } - } else { - // Otherwise something's gone horribly wrong..! - throw new Error("Invalid playlist file contents: " + file) - } + processArgv(openedPlaylist.options, optionFunctions) } function requiresOpenPlaylist() { - if (activePlaylist === null) { + if (activePlaylistGroup === null) { throw new Error( "This action requires an open playlist - try --open (file)" ) @@ -142,7 +113,7 @@ Promise.resolve() requiresOpenPlaylist() - activePlaylist = [] + activePlaylistGroup = [] }, 'c': util => util.alias('-clear'), @@ -158,7 +129,7 @@ Promise.resolve() const pathString = util.nextArg() const group = filterPlaylistByPathString(sourcePlaylist, pathString) - activePlaylist.push(group) + activePlaylistGroup.push(group) }, 'k': util => util.alias('-keep'), @@ -171,7 +142,7 @@ Promise.resolve() const pathString = util.nextArg() console.log("Ignoring path: " + pathString) - removeGroupByPathString(activePlaylist, pathString) + removeGroupByPathString(activePlaylistGroup, pathString) }, 'r': util => util.alias('-remove'), @@ -183,7 +154,7 @@ Promise.resolve() requiresOpenPlaylist() - console.log(getPlaylistTreeString(activePlaylist)) + console.log(getPlaylistTreeString(activePlaylistGroup)) // If this is the last item in the argument list, the user probably // only wants to get the list, so we'll mark the 'should run' flag @@ -202,7 +173,7 @@ Promise.resolve() requiresOpenPlaylist() - console.log(getPlaylistTreeString(activePlaylist, true)) + console.log(getPlaylistTreeString(activePlaylistGroup, true)) // As with -l, if this is the last item in the argument list, we // won't actually be playing the playlist. @@ -255,7 +226,7 @@ Promise.resolve() requiresOpenPlaylist() - console.log(JSON.stringify(activePlaylist, null, 2)) + console.log(JSON.stringify(activePlaylistGroup, null, 2)) } } @@ -263,7 +234,7 @@ Promise.resolve() await processArgv(process.argv, optionFunctions) - if (activePlaylist === null) { + if (activePlaylistGroup === null) { throw new Error( "Cannot play - no open playlist. Try --open ?" ) @@ -273,10 +244,10 @@ Promise.resolve() let picker if (pickerType === 'shuffle') { console.log("Using shuffle picker.") - picker = pickers.makeShufflePlaylistPicker(activePlaylist) + picker = pickers.makeShufflePlaylistPicker(activePlaylistGroup) } else if (pickerType === 'ordered') { console.log("Using ordered picker.") - picker = pickers.makeOrderedPlaylistPicker(activePlaylist) + picker = pickers.makeOrderedPlaylistPicker(activePlaylistGroup) } else { console.error("Invalid picker type: " + pickerType) return @@ -373,7 +344,7 @@ Promise.resolve() return playPromise } else { - return activePlaylist + return activePlaylistGroup } }) .catch(err => console.error(err)) diff --git a/src/loop-play.js b/src/loop-play.js index 8ca4ae4..701e590 100644 --- a/src/loop-play.js +++ b/src/loop-play.js @@ -93,10 +93,8 @@ class PlayController { if (picked === null) { return null } else { - // TODO: Is there a function for this? - const arg = picked[1] - const downloader = getDownloaderFor(arg) - this.downloadController.download(downloader, arg) + const downloader = getDownloaderFor(picked.downloaderArg) + this.downloadController.download(downloader, picked.downloaderArg) return picked } } @@ -223,15 +221,15 @@ class PlayController { logTrackInfo() { if (this.currentTrack) { - const [ title, arg ] = this.currentTrack - console.log(`Playing: \x1b[1m${title} \x1b[2m${arg}\x1b[0m`) + const t = this.currentTrack + console.log(`Playing: \x1b[1m${t.name} \x1b[2m${t.downloaderArg}\x1b[0m`) } else { console.log("No song currently playing.") } if (this.nextTrack) { - const [ title, arg ] = this.nextTrack - console.log(`Up next: \x1b[1m${title} \x1b[2m${arg}\x1b[0m`) + const t = this.nextTrack + console.log(`Up next: \x1b[1m${t.name} \x1b[2m${t.downloaderArg}\x1b[0m`) } else { console.log("No song up next.") } diff --git a/src/pickers.js b/src/pickers.js index 92a9641..ee886bc 100644 --- a/src/pickers.js +++ b/src/pickers.js @@ -1,12 +1,12 @@ 'use strict' -const { flattenPlaylist } = require('./playlist-utils') +const { flattenGrouplike } = require('./playlist-utils') -function makeOrderedPlaylistPicker(playlist) { - // Ordered playlist picker - this plays all the tracks in a playlist in +function makeOrderedPlaylistPicker(grouplike) { + // Ordered playlist picker - this plays all the tracks in a group in // order, after flattening it. - const allSongs = flattenPlaylist(playlist) + const allSongs = flattenGrouplike(groupContents) let index = 0 return function() { @@ -20,15 +20,15 @@ function makeOrderedPlaylistPicker(playlist) { } } -function makeShufflePlaylistPicker(playlist) { +function makeShufflePlaylistPicker(grouplike) { // Shuffle playlist picker - this selects a random track at any index in // the playlist, after flattening it. - const allSongs = flattenPlaylist(playlist) + const flatGroup = flattenGrouplike(grouplike) return function() { - const index = Math.floor(Math.random() * allSongs.length) - const picked = allSongs[index] + const index = Math.floor(Math.random() * flatGroup.items.length) + const picked = flatGroup.items[index] return picked } } diff --git a/src/playlist-utils.js b/src/playlist-utils.js index 13c6003..ae28659 100644 --- a/src/playlist-utils.js +++ b/src/playlist-utils.js @@ -1,25 +1,134 @@ 'use strict' -function flattenPlaylist(playlist) { - // Flattens a playlist, taking all of the non-group items (tracks) at all - // levels in the playlist tree and returns them as a single-level array of - // tracks. +// TODO: Use this when loading playlists. Also grab things from http-music.js. +function updatePlaylistFormat(playlist) { + const defaultPlaylist = { + items: [], + options: [] + } + + let playlistObj = {} + + // Playlists can be in two formats... + if (Array.isArray(playlist)) { + // ..the first, a simple array of tracks and groups; + + playlistObj = {items: playlist} + } else { + // ..or an object including metadata and configuration as well as the + // array described in the first. + + playlistObj = playlist + + // The 'tracks' property was used for a while, but it doesn't really make + // sense, since we also store groups in the 'tracks' property. So it was + // renamed to 'items'. + if ('tracks' in playlistObj) { + playlistObj.items = playlistObj.tracks + delete playlistObj.tracks + } + } + + const fullPlaylistObj = Object.assign(defaultPlaylist, playlistObj) + + const handleGroupContents = groupContents => { + return groupContents.map(item => { + if (Array.isArray(item[1])) { + return {name: item[0], items: handleGroupContents(item[1])} + } else { + return updateTrackFormat(item) + } + }) + } - const groups = playlist.filter(x => isGroup(x)) - const nonGroups = playlist.filter(x => !isGroup(x)) + fullPlaylistObj.items = handleGroupContents(fullPlaylistObj.items) - return groups.map(g => flattenPlaylist(getGroupContents(g))) - .reduce((a, b) => a.concat(b), nonGroups) + return fullPlaylistObj +} + +function updateTrackFormat(track) { + const defaultTrack = { + name: '', + downloaderArg: '' + } + + let trackObj = {} + + if (Array.isArray(track)) { + if (track.length === 2) { + trackObj = {name: track[0], downloaderArg: track[1]} + } else { + throw new Error("Unexpected non-length 2 array-format track") + } + } else { + trackObj = track + } + + return Object.assign(defaultTrack, trackObj) +} + +function updateGroupFormat(group) { + const defaultGroup = { + name: '', + items: [] + } + + let groupObj + + if (Array.isArray(group)) { + if (group.length === 2) { + groupObj = {name: group[0], items: group[1]} + } else { + throw new Error("Unexpected non-length 2 array-format group") + } + } else { + groupObj = group + } + + return Object.assign(defaultGroup, groupObj) +} + +function mapGrouplikeItems(grouplike, handleTrack) { + if (typeof handleTrack === 'undefined') { + throw new Error("Missing track handler function") + } + + return { + items: grouplike.items.map(item => { + if (isTrack(item)) { + return handleTrack(item) + } else if (isGroup(item)) { + return mapGrouplikeItems(item, handleTrack, handleGroup) + } else { + throw new Error('Non-track/group item') + } + }) + } +} + +function flattenGrouplike(grouplike) { + // Flattens a group-like, taking all of the non-group items (tracks) at all + // levels in the group tree and returns them as a new group containing those + // tracks. + + return { + items: grouplike.items.map(item => { + if (isGroup(item)) { + return flattenGrouplike(item).items + } else { + return [item] + } + }).reduce((a, b) => a.concat(b), []) + } } function filterPlaylistByPathString(playlist, pathString) { - // Calls filterPlaylistByPath, taking a path string, rather than a parsed - // path. + // Calls filterGroupContentsByPath, taking an unparsed path string. - return filterPlaylistByPath(playlist, parsePathString(pathString)) + return filterGroupContentsByPath(playlist, parsePathString(pathString)) } -function filterPlaylistByPath(playlist, pathParts) { +function filterGroupContentsByPath(groupContents, pathParts) { // Finds a group by following the given group path and returns it. If the // function encounters an item in the group path that is not found, it logs // a warning message and returns the group found up to that point. @@ -38,22 +147,22 @@ function filterPlaylistByPath(playlist, pathParts) { const cur = pathParts[0] - let match = playlist.find(g => titleMatch(g, false)) + let match = groupContents.find(g => titleMatch(g, false)) if (!match) { - match = playlist.find(g => titleMatch(g, true)) + match = groupContents.find(g => titleMatch(g, true)) } if (match) { if (pathParts.length > 1) { const rest = pathParts.slice(1) - return filterPlaylistByPath(getGroupContents(match), rest) + return filterGroupContentsByPath(getGroupContents(match), rest) } else { return match } } else { console.warn(`Not found: "${cur}"`) - return playlist + return groupContents } } @@ -139,17 +248,22 @@ function getGroupContents(group) { return group[1] } -function isGroup(array) { - return Array.isArray(array[1]) +function isGroup(obj) { + return obj && obj.items + + // return Array.isArray(array[1]) } -function isTrack(array) { - return typeof array[1] === 'string' +function isTrack(obj) { + return obj && obj.downloaderArg + + // return typeof array[1] === 'string' } module.exports = { - flattenPlaylist, - filterPlaylistByPathString, filterPlaylistByPath, + updatePlaylistFormat, updateTrackFormat, + flattenGrouplike, + filterPlaylistByPathString, filterGroupContentsByPath, removeGroupByPathString, removeGroupByPath, getPlaylistTreeString, parsePathString, -- cgit 1.3.0-6-gf8a5