From 3dd9d4c40e3b81beebbe7c0a66df93e226ac0bd1 Mon Sep 17 00:00:00 2001 From: Florrie Date: Fri, 4 Aug 2017 18:12:57 -0300 Subject: Change around some things in download-playlist --- src/download-playlist.js | 186 ++++++++++++++++++++--------------------------- 1 file changed, 80 insertions(+), 106 deletions(-) (limited to 'src/download-playlist.js') diff --git a/src/download-playlist.js b/src/download-playlist.js index 390d241..dcaa7b7 100755 --- a/src/download-playlist.js +++ b/src/download-playlist.js @@ -8,137 +8,112 @@ const sanitize = require('sanitize-filename') const promisifyProcess = require('./promisify-process') const { - isGroup, isTrack, flattenGrouplike, updatePlaylistFormat + flattenGrouplike, updatePlaylistFormat, getItemPath } = require('./playlist-utils') const { getDownloaderFor, makePowerfulDownloader } = require('./downloaders') const { promisify } = require('util') const { spawn } = require('child_process') -const access = promisify(fs.access) -const mkdir = promisify(fs.mkdir) +const mkdirp = promisify(require('mkdirp')) + const readFile = promisify(fs.readFile) const readdir = promisify(fs.readdir) -const stat = promisify(fs.stat) -const writeFile = promisify(fs.writeFile) -async function downloadCrawl(topPlaylist, initialOutPath = './out/') { +async function downloadCrawl(playlist, topOut = './out/') { + const flat = flattenGrouplike(playlist) let doneCount = 0 - let total = flattenGrouplike(topPlaylist).items.length const status = function() { + const total = flat.items.length const percent = Math.trunc(doneCount / total * 10000) / 100 console.log( `\x1b[1mDownload crawler - ${percent}% completed ` + `(${doneCount}/${total} tracks)\x1b[0m`) } - const recursive = async function(groupContents, outPath) { - // If the output folder doesn't exist, we should create it. - let doesExist = true - try { - doesExist = (await stat(outPath)).isDirectory() - } catch(err) { - doesExist = false - } - - if (!doesExist) { - await mkdir(outPath) + for (let item of flat.items) { + const parentGroups = getItemPath(item).slice(0, -1) + + const dir = parentGroups.reduce((a, b) => { + return a + '/' + sanitize(b.name) + }, topOut) + '/' + + await mkdirp(dir) + + const base = path.basename(item.name, path.extname(item.name)) + const targetFile = dir + sanitize(base) + '.mp3' + + // If we've already downloaded a file at some point in previous time, + // there's no need to download it again! + // + // Since we can't guarantee the extension name of the file, we only + // compare bases. + // + // TODO: This probably doesn't work well with things like the YouTube + // downloader. + const items = await readdir(dir) + const match = items.find(item => { + const itemBase = sanitize(path.basename(item, path.extname(item))) + return itemBase === base + }) + + if (match) { + console.log(`\x1b[32;2mAlready downloaded: ${targetFile}\x1b[0m`) + doneCount++ + status() + continue } - let outPlaylist = [] - - for (let item of groupContents) { - if (isGroup(item)) { - // TODO: Not sure if this is the best way to pick the next out dir. - const out = outPath + sanitize(item.name) + '/' - - outPlaylist.push({ - name: item.name, - items: await recursive(item.items, out) - }) - } else if (isTrack(item)) { - const base = path.basename(item.name, path.extname(item.name)) - const targetFile = outPath + sanitize(base) + '.mp3' - - // If we've already downloaded a file at some point in previous time, - // there's no need to download it again! - // - // Since we can't guarantee the extension name of the file, we only - // compare bases. - // - // TODO: This probably doesn't work well with things like the YouTube - // downloader. - const items = await readdir(outPath) - const match = items.find(item => { - const itemBase = sanitize(path.basename(item, path.extname(item))) - return itemBase === base - }) - - if (match) { - console.log(`\x1b[32;2mAlready downloaded: ${targetFile}\x1b[0m`) - outPlaylist.push({name: item.name, downloaderArg: outPath + match}) - doneCount++ - status() - continue - } - - console.log( - `\x1b[2mDownloading: ${item.name} - ${item.downloaderArg}` + - ` => ${targetFile}\x1b[0m` + console.log( + `\x1b[2mDownloading: ${item.name} - ${item.downloaderArg}` + + ` => ${targetFile}\x1b[0m` + ) + + // Woo-hoo, using block labels for their intended purpose! (Maybe?) + downloadProcess: { + const downloader = makePowerfulDownloader( + getDownloaderFor(item.downloaderArg) + ) + + const outputtedFile = await downloader(item.downloaderArg) + + // If the return of the downloader is false, then the download + // failed. + if (outputtedFile === false) { + console.error( + `\x1b[33;1mDownload failed (item skipped): ${item.name}\x1b[0m` ) - // Woo-hoo, using block labels for their intended purpose! (Maybe?) - downloadProcess: { - const downloader = makePowerfulDownloader( - getDownloaderFor(item.downloaderArg) - ) - - const outputtedFile = await downloader(item.downloaderArg) - - // If the return of the downloader is false, then the download - // failed. - if (outputtedFile === false) { - console.error( - `\x1b[33;1mDownload failed (item skipped): ${item.name}\x1b[0m` - ) - - break downloadProcess - } - - try { - await promisifyProcess(spawn('ffmpeg', [ - '-i', outputtedFile, - - // A bug (in ffmpeg or macOS; not this) makes it necessary to have - // these options on macOS, otherwise the outputted file length is - // wrong. - '-write_xing', '0', - - targetFile - ]), false) - } catch(err) { - console.error( - `\x1b[33;1mFFmpeg failed (item skipped): ${item.name}\x1b[0m` - ) + break downloadProcess + } - break downloadProcess - } + try { + await promisifyProcess(spawn('ffmpeg', [ + '-i', outputtedFile, - console.log('Added:', item.name) - outPlaylist.push({name: item.name, downloaderArg: targetFile}) - } + // A bug (in ffmpeg or macOS; not this) makes it necessary to have + // these options on macOS, otherwise the outputted file length is + // wrong. + '-write_xing', '0', - doneCount++ + targetFile + ]), false) + } catch(err) { + console.error( + `\x1b[33;1mFFmpeg failed (item skipped): ${item.name}\x1b[0m` + ) - status() + break downloadProcess } + + console.log('Added:', item.name) } - return outPlaylist - } + doneCount++ - return {items: await recursive(topPlaylist.items, initialOutPath)} + status() + } } async function main(args) { @@ -151,12 +126,11 @@ async function main(args) { const playlist = updatePlaylistFormat(JSON.parse(await readFile(args[0]))) - const outPlaylist = await downloadCrawl(playlist) - - await writeFile('out/playlist.json', JSON.stringify(outPlaylist, null, 2)) + await downloadCrawl(playlist) - console.log('Done - saved playlist to out/playlist.json.') - process.exit(0) + console.log( + 'Done - downloaded to out/. (Use crawl-local out/ to create a playlist.)' + ) } module.exports = main -- cgit 1.3.0-6-gf8a5