diff options
| -rw-r--r-- | package.json | 1 | ||||
| -rwxr-xr-x | src/download-playlist.js | 186 | ||||
| -rw-r--r-- | src/playlist-utils.js | 10 | ||||
| -rw-r--r-- | yarn.lock | 10 |
4 files changed, 100 insertions, 107 deletions
diff --git a/package.json b/package.json index 688267a..8432825 100644 --- a/package.json +++ b/package.json @@ -25,6 +25,7 @@ "command-exists": "^1.2.2", "fifo-js": "^2.1.0", "fs-extra": "^3.0.1", + "mkdirp": "^0.5.1", "ncp": "^2.0.0", "node-fetch": "^1.7.0", "node-natural-sort": "^0.8.6", diff --git a/src/download-playlist.js b/src/download-playlist.js index 390d241..dcaa7b7 100755 --- a/src/download-playlist.js +++ b/src/download-playlist.js @@ -8,137 +8,112 @@ const sanitize = require('sanitize-filename') const promisifyProcess = require('./promisify-process') const { - isGroup, isTrack, flattenGrouplike, updatePlaylistFormat + flattenGrouplike, updatePlaylistFormat, getItemPath } = require('./playlist-utils') const { getDownloaderFor, makePowerfulDownloader } = require('./downloaders') const { promisify } = require('util') const { spawn } = require('child_process') -const access = promisify(fs.access) -const mkdir = promisify(fs.mkdir) +const mkdirp = promisify(require('mkdirp')) + const readFile = promisify(fs.readFile) const readdir = promisify(fs.readdir) -const stat = promisify(fs.stat) -const writeFile = promisify(fs.writeFile) -async function downloadCrawl(topPlaylist, initialOutPath = './out/') { +async function downloadCrawl(playlist, topOut = './out/') { + const flat = flattenGrouplike(playlist) let doneCount = 0 - let total = flattenGrouplike(topPlaylist).items.length const status = function() { + const total = flat.items.length const percent = Math.trunc(doneCount / total * 10000) / 100 console.log( `\x1b[1mDownload crawler - ${percent}% completed ` + `(${doneCount}/${total} tracks)\x1b[0m`) } - const recursive = async function(groupContents, outPath) { - // If the output folder doesn't exist, we should create it. - let doesExist = true - try { - doesExist = (await stat(outPath)).isDirectory() - } catch(err) { - doesExist = false - } - - if (!doesExist) { - await mkdir(outPath) + for (let item of flat.items) { + const parentGroups = getItemPath(item).slice(0, -1) + + const dir = parentGroups.reduce((a, b) => { + return a + '/' + sanitize(b.name) + }, topOut) + '/' + + await mkdirp(dir) + + const base = path.basename(item.name, path.extname(item.name)) + const targetFile = dir + sanitize(base) + '.mp3' + + // If we've already downloaded a file at some point in previous time, + // there's no need to download it again! + // + // Since we can't guarantee the extension name of the file, we only + // compare bases. + // + // TODO: This probably doesn't work well with things like the YouTube + // downloader. + const items = await readdir(dir) + const match = items.find(item => { + const itemBase = sanitize(path.basename(item, path.extname(item))) + return itemBase === base + }) + + if (match) { + console.log(`\x1b[32;2mAlready downloaded: ${targetFile}\x1b[0m`) + doneCount++ + status() + continue } - let outPlaylist = [] - - for (let item of groupContents) { - if (isGroup(item)) { - // TODO: Not sure if this is the best way to pick the next out dir. - const out = outPath + sanitize(item.name) + '/' - - outPlaylist.push({ - name: item.name, - items: await recursive(item.items, out) - }) - } else if (isTrack(item)) { - const base = path.basename(item.name, path.extname(item.name)) - const targetFile = outPath + sanitize(base) + '.mp3' - - // If we've already downloaded a file at some point in previous time, - // there's no need to download it again! - // - // Since we can't guarantee the extension name of the file, we only - // compare bases. - // - // TODO: This probably doesn't work well with things like the YouTube - // downloader. - const items = await readdir(outPath) - const match = items.find(item => { - const itemBase = sanitize(path.basename(item, path.extname(item))) - return itemBase === base - }) - - if (match) { - console.log(`\x1b[32;2mAlready downloaded: ${targetFile}\x1b[0m`) - outPlaylist.push({name: item.name, downloaderArg: outPath + match}) - doneCount++ - status() - continue - } - - console.log( - `\x1b[2mDownloading: ${item.name} - ${item.downloaderArg}` + - ` => ${targetFile}\x1b[0m` + console.log( + `\x1b[2mDownloading: ${item.name} - ${item.downloaderArg}` + + ` => ${targetFile}\x1b[0m` + ) + + // Woo-hoo, using block labels for their intended purpose! (Maybe?) + downloadProcess: { + const downloader = makePowerfulDownloader( + getDownloaderFor(item.downloaderArg) + ) + + const outputtedFile = await downloader(item.downloaderArg) + + // If the return of the downloader is false, then the download + // failed. + if (outputtedFile === false) { + console.error( + `\x1b[33;1mDownload failed (item skipped): ${item.name}\x1b[0m` ) - // Woo-hoo, using block labels for their intended purpose! (Maybe?) - downloadProcess: { - const downloader = makePowerfulDownloader( - getDownloaderFor(item.downloaderArg) - ) - - const outputtedFile = await downloader(item.downloaderArg) - - // If the return of the downloader is false, then the download - // failed. - if (outputtedFile === false) { - console.error( - `\x1b[33;1mDownload failed (item skipped): ${item.name}\x1b[0m` - ) - - break downloadProcess - } - - try { - await promisifyProcess(spawn('ffmpeg', [ - '-i', outputtedFile, - - // A bug (in ffmpeg or macOS; not this) makes it necessary to have - // these options on macOS, otherwise the outputted file length is - // wrong. - '-write_xing', '0', - - targetFile - ]), false) - } catch(err) { - console.error( - `\x1b[33;1mFFmpeg failed (item skipped): ${item.name}\x1b[0m` - ) + break downloadProcess + } - break downloadProcess - } + try { + await promisifyProcess(spawn('ffmpeg', [ + '-i', outputtedFile, - console.log('Added:', item.name) - outPlaylist.push({name: item.name, downloaderArg: targetFile}) - } + // A bug (in ffmpeg or macOS; not this) makes it necessary to have + // these options on macOS, otherwise the outputted file length is + // wrong. + '-write_xing', '0', - doneCount++ + targetFile + ]), false) + } catch(err) { + console.error( + `\x1b[33;1mFFmpeg failed (item skipped): ${item.name}\x1b[0m` + ) - status() + break downloadProcess } + + console.log('Added:', item.name) } - return outPlaylist - } + doneCount++ - return {items: await recursive(topPlaylist.items, initialOutPath)} + status() + } } async function main(args) { @@ -151,12 +126,11 @@ async function main(args) { const playlist = updatePlaylistFormat(JSON.parse(await readFile(args[0]))) - const outPlaylist = await downloadCrawl(playlist) - - await writeFile('out/playlist.json', JSON.stringify(outPlaylist, null, 2)) + await downloadCrawl(playlist) - console.log('Done - saved playlist to out/playlist.json.') - process.exit(0) + console.log( + 'Done - downloaded to out/. (Use crawl-local out/ to create a playlist.)' + ) } module.exports = main diff --git a/src/playlist-utils.js b/src/playlist-utils.js index 5efd478..aed4964 100644 --- a/src/playlist-utils.js +++ b/src/playlist-utils.js @@ -245,6 +245,14 @@ function getPlaylistTreeString(playlist, showTracks = false) { return recursive(playlist) } +function getItemPath(item) { + if (item[parentSymbol]) { + return [...getItemPath(item[parentSymbol]), item] + } else { + return [item] + } +} + function getItemPathString(item) { // Gets the playlist path of an item by following its parent chain. // Returns a string in format Foo/Bar/Baz, where Foo and Bar are group @@ -286,7 +294,7 @@ module.exports = { filterPlaylistByPathString, filterGrouplikeByPath, removeGroupByPathString, removeGroupByPath, getPlaylistTreeString, - getItemPathString, + getItemPath, getItemPathString, parsePathString, isGroup, isTrack } diff --git a/yarn.lock b/yarn.lock index d3d249f..bbb0f25 100644 --- a/yarn.lock +++ b/yarn.lock @@ -149,6 +149,16 @@ lodash@^4.15.0: version "4.17.4" resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.4.tgz#78203a4d1c328ae1d86dca6460e369b57f4055ae" +minimist@0.0.8: + version "0.0.8" + resolved "https://registry.yarnpkg.com/minimist/-/minimist-0.0.8.tgz#857fcabfc3397d2625b8228262e86aa7a011b05d" + +mkdirp: + version "0.5.1" + resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-0.5.1.tgz#30057438eac6cf7f8c4767f38648d6697d75c903" + dependencies: + minimist "0.0.8" + ncp: version "2.0.0" resolved "https://registry.yarnpkg.com/ncp/-/ncp-2.0.0.tgz#195a21d6c46e361d2fb1281ba38b91e9df7bdbb3" |