From 84a48215f19d271cfce88439a1300817f0666e85 Mon Sep 17 00:00:00 2001 From: liam4 Date: Wed, 21 Jun 2017 17:10:26 -0300 Subject: Various updates.. maybe finished?? --- src/crawl-http.js | 4 +-- src/download-playlist.js | 90 ++++++++++++++++++++++++++++++++++++++++-------- src/downloaders.js | 14 +++++++- src/http-music.js | 13 ++----- 4 files changed, 92 insertions(+), 29 deletions(-) (limited to 'src') diff --git a/src/crawl-http.js b/src/crawl-http.js index 7f51c87..020506b 100755 --- a/src/crawl-http.js +++ b/src/crawl-http.js @@ -97,9 +97,9 @@ function crawl(absURL, opts = {}, internals = {}) { err => { console.warn("Failed to download: " + absURL) - if (attempts < maxAttempts) { + if (internals.attempts < maxAttempts) { console.warn( - `Trying again. Attempt ${attempts + 1}/${maxAttempts}...` + `Trying again. Attempt ${internals.attempts + 1}/${maxAttempts}...` ) return crawl(absURL, opts, Object.assign({}, internals, { diff --git a/src/download-playlist.js b/src/download-playlist.js index eb6375a..bb6b86c 100644 --- a/src/download-playlist.js +++ b/src/download-playlist.js @@ -3,6 +3,7 @@ const fs = require('fs') const downloaders = require('./downloaders') const path = require('path') +const processArgv = require('./process-argv') const sanitize = require('sanitize-filename') const { @@ -15,11 +16,55 @@ const access = promisify(fs.access) const mkdir = promisify(fs.mkdir) const readFile = promisify(fs.readFile) const readdir = promisify(fs.readdir) -const rename = promisify(fs.rename) const stat = promisify(fs.stat) const writeFile = promisify(fs.writeFile) +const ncp = promisify(require('ncp').ncp) + +// It's typically bad to attempt to download or copy a million files at once, +// so we create a "promise delayer" that forces only several promises to run at +// at one time. +let delayPromise +{ + const INTERVAL = 50 + const MAX = 5 + + let active = 0 + + let queue = [] + + delayPromise = function(promiseMaker) { + return new Promise((resolve, reject) => { + queue.push([promiseMaker, resolve, reject]) + }) + } + + setInterval(async () => { + if (active >= MAX) { + return + } + + const top = queue.pop() + + if (top) { + const [ promiseMaker, resolve, reject ] = top + + active++ + + console.log('Going - queue: ' + queue.length) + + try { + resolve(await promiseMaker()) + } catch(err) { + reject(err) + } + + active-- + } + }, INTERVAL) +} async function downloadCrawl(playlist, downloader, outPath = './out/') { + // If the output folder doesn't exist, we should create it. let doesExist = true try { doesExist = (await stat(outPath)).isDirectory() @@ -38,30 +83,37 @@ async function downloadCrawl(playlist, downloader, outPath = './out/') { return [item[0], await downloadCrawl(item[1], downloader, out)] } else if (isTrack(item)) { - console.log(`\x1b[2m${item[0]} - ${item[1]}\x1b[0m`) - - // TODO: How to deal with songs that don't have an extension? + // TODO: How should we deal with songs that don't have an extension? const ext = path.extname(item[1]) const base = path.basename(item[1], ext) - + const out = outPath + base + ext + + // If we've already downloaded a file at some point in previous time, + // there's no need to download it again! + // + // Since we can't guarantee the extension name of the file, we only + // compare bases. + // + // TODO: This probably doesn't work well with things like the YouTube + // downloader. const items = await readdir(outPath) const match = items.find(x => path.basename(x, path.extname(x)) === base) if (match) { + console.log(`\x1b[32;2mAlready downloaded: ${out}\x1b[0m`) return [item[0], outPath + match] } - const downloadFile = await downloader(item[1]) - // const base = path.basename(downloadFile) - // const out = outPath + base + console.log(`\x1b[2mDownloading: ${item[0]} - ${item[1]}\x1b[0m`) - // console.log(`\x1b[1m${downloadFile}\x1b[0m`) + const downloadFile = await delayPromise(() => downloader(item[1])) + // console.log(downloadFile, path.resolve(out)) try { - await rename(downloadFile, path.resolve(out)) - console.log(`\x1b[1m${out}\x1b[0m`) + await delayPromise(() => ncp(downloadFile, path.resolve(out))) + console.log(`\x1b[32;1mDownloaded: ${out}\x1b[0m`) return [item[0], out] } catch(err) { - console.error(`\x1b[31mFAILED: ${out}\x1b[0m`) + console.error(`\x1b[31mFailed: ${out}\x1b[0m`) console.error(err) return false } @@ -74,21 +126,29 @@ async function main() { if (process.argv.length === 2) { console.error('Usage: download-playlist [opts]') - process.exit(1) return } const playlist = JSON.parse(await readFile(process.argv[2])) + let downloaderType = 'http' + + processArgv(process.argv.slice(3), { + '-downloader': util => { + downloaderType = util.nextArg() + } + }) + const dl = downloaders.makePowerfulDownloader( - downloaders.makeHTTPDownloader() + downloaders.getDownloader(downloaderType) ) const outPlaylist = await downloadCrawl(playlist, dl) - writeFile('out/playlist.json', JSON.stringify(outPlaylist, null, 2)) + await writeFile('out/playlist.json', JSON.stringify(outPlaylist, null, 2)) console.log('Done - saved playlist to out/playlist.json.') + process.exit(0) } main() diff --git a/src/downloaders.js b/src/downloaders.js index 2b193eb..8fa830c 100644 --- a/src/downloaders.js +++ b/src/downloaders.js @@ -70,5 +70,17 @@ module.exports = { makeHTTPDownloader, makeYouTubeDownloader, makeLocalDownloader, - makePowerfulDownloader + makePowerfulDownloader, + + getDownloader: downloaderType => { + if (downloaderType === 'http') { + return makeHTTPDownloader() + } else if (downloaderType === 'youtube') { + return makeYouTubeDownloader() + } else if (downloaderType === 'local') { + return makeLocalDownloader() + } else { + return null + } + } } diff --git a/src/http-music.js b/src/http-music.js index ed79878..68bfa77 100755 --- a/src/http-music.js +++ b/src/http-music.js @@ -224,17 +224,8 @@ setupDefaultPlaylist('./playlist.json') return } - let downloader - if (downloaderType === 'http') { - console.log("Using HTTP downloader.") - downloader = downloaders.makeHTTPDownloader() - } else if (downloaderType === 'youtube') { - console.log("Using YouTube downloader.") - downloader = downloaders.makeYouTubeDownloader() - } else if (downloaderType === 'local') { - console.log("Using local file downloader.") - downloader = downloaders.makeLocalDownloader() - } else { + let downloader = downloaders.getDownloader(downloaderType) + if (!downloader) { console.error("Invalid downloader type: " + downloaderType) return } -- cgit 1.3.0-6-gf8a5