Merge branch 'workspace'

5 files changed, 199 insertions, 15 deletions

diff --git a/src/crawl-http.js b/src/crawl-http.js
index 7f51c87..020506b 100755
--- a/src/crawl-http.js
+++ b/src/crawl-http.js
@@ -97,9 +97,9 @@ function crawl(absURL, opts = {}, internals = {}) {
       err => {
         console.warn("Failed to download: " + absURL)
 
-        if (attempts < maxAttempts) {
+        if (internals.attempts < maxAttempts) {
           console.warn(
-            `Trying again. Attempt ${attempts + 1}/${maxAttempts}...`
+            `Trying again. Attempt ${internals.attempts + 1}/${maxAttempts}...`
           )
 
           return crawl(absURL, opts, Object.assign({}, internals, {
diff --git a/src/download-playlist.js b/src/download-playlist.js
new file mode 100644
index 0000000..bb6b86c
--- /dev/null
+++ b/src/download-playlist.js
@@ -0,0 +1,155 @@
+'use strict'
+
+const fs = require('fs')
+const downloaders = require('./downloaders')
+const path = require('path')
+const processArgv = require('./process-argv')
+const sanitize = require('sanitize-filename')
+
+const {
+  isGroup, isTrack
+} = require('./playlist-utils')
+
+const { promisify } = require('util')
+
+const access = promisify(fs.access)
+const mkdir = promisify(fs.mkdir)
+const readFile = promisify(fs.readFile)
+const readdir = promisify(fs.readdir)
+const stat = promisify(fs.stat)
+const writeFile = promisify(fs.writeFile)
+const ncp = promisify(require('ncp').ncp)
+
+// It's typically bad to attempt to download or copy a million files at once,
+// so we create a "promise delayer" that forces only several promises to run at
+// at one time.
+let delayPromise
+{
+  const INTERVAL = 50
+  const MAX = 5
+
+  let active = 0
+
+  let queue = []
+
+  delayPromise = function(promiseMaker) {
+    return new Promise((resolve, reject) => {
+      queue.push([promiseMaker, resolve, reject])
+    })
+  }
+
+  setInterval(async () => {
+    if (active >= MAX) {
+      return
+    }
+
+    const top = queue.pop()
+
+    if (top) {
+      const [ promiseMaker, resolve, reject ] = top
+
+      active++
+
+      console.log('Going - queue: ' + queue.length)
+
+      try {
+        resolve(await promiseMaker())
+      } catch(err) {
+        reject(err)
+      }
+
+      active--
+    }
+  }, INTERVAL)
+}
+
+async function downloadCrawl(playlist, downloader, outPath = './out/') {
+  // If the output folder doesn't exist, we should create it.
+  let doesExist = true
+  try {
+    doesExist = (await stat(outPath)).isDirectory()
+  } catch(err) {
+    doesExist = false
+  }
+
+  if (!doesExist) {
+    await mkdir(outPath)
+  }
+
+  return Promise.all(playlist.map(async (item) => {
+    if (isGroup(item)) {
+      // TODO: Not sure if this is the best way to pick the next out dir.
+      const out = outPath + sanitize(item[0]) + '/'
+
+      return [item[0], await downloadCrawl(item[1], downloader, out)]
+    } else if (isTrack(item)) {
+      // TODO: How should we deal with songs that don't have an extension?
+      const ext = path.extname(item[1])
+      const base = path.basename(item[1], ext)
+      const out = outPath + base + ext
+
+      // If we've already downloaded a file at some point in previous time,
+      // there's no need to download it again!
+      //
+      // Since we can't guarantee the extension name of the file, we only
+      // compare bases.
+      //
+      // TODO: This probably doesn't work well with things like the YouTube
+      // downloader.
+      const items = await readdir(outPath)
+      const match = items.find(x => path.basename(x, path.extname(x)) === base)
+      if (match) {
+        console.log(`\x1b[32;2mAlready downloaded: ${out}\x1b[0m`)
+        return [item[0], outPath + match]
+      }
+
+      console.log(`\x1b[2mDownloading: ${item[0]} - ${item[1]}\x1b[0m`)
+
+      const downloadFile = await delayPromise(() => downloader(item[1]))
+      // console.log(downloadFile, path.resolve(out))
+
+      try {
+        await delayPromise(() => ncp(downloadFile, path.resolve(out)))
+        console.log(`\x1b[32;1mDownloaded: ${out}\x1b[0m`)
+        return [item[0], out]
+      } catch(err) {
+        console.error(`\x1b[31mFailed: ${out}\x1b[0m`)
+        console.error(err)
+        return false
+      }
+    }
+  })).then(p => p.filter(Boolean))
+}
+
+async function main() {
+  // TODO: Implement command line stuff here
+
+  if (process.argv.length === 2) {
+    console.error('Usage: download-playlist <playlistFile> [opts]')
+    return
+  }
+
+  const playlist = JSON.parse(await readFile(process.argv[2]))
+
+  let downloaderType = 'http'
+
+  processArgv(process.argv.slice(3), {
+    '-downloader': util => {
+      downloaderType = util.nextArg()
+    }
+  })
+
+  const dl = downloaders.makePowerfulDownloader(
+    downloaders.getDownloader(downloaderType)
+  )
+
+  const outPlaylist = await downloadCrawl(playlist, dl)
+
+  await writeFile('out/playlist.json', JSON.stringify(outPlaylist, null, 2))
+
+  console.log('Done - saved playlist to out/playlist.json.')
+  process.exit(0)
+}
+
+main()
+  .catch(err => console.error(err))
diff --git a/src/downloaders.js b/src/downloaders.js
index fa1f337..8fa830c 100644
--- a/src/downloaders.js
+++ b/src/downloaders.js
@@ -1,3 +1,5 @@
+'use strict'
+
 const fs = require('fs')
 const fetch = require('node-fetch')
 const promisifyProcess = require('./promisify-process')
@@ -47,8 +49,38 @@ function makeLocalDownloader() {
   }
 }
 
+function makePowerfulDownloader(downloader, maxAttempts = 5) {
+  // This should totally be named better..
+
+  return async function recursive(arg, attempts = 0) {
+    try {
+      return await downloader(arg)
+    } catch(err) {
+      if (attempts < maxAttempts) {
+        console.warn('Failed - attempting again:', arg)
+        return await recursive(arg, attempts + 1)
+      } else {
+        throw err
+      }
+    }
+  }
+}
+
 module.exports = {
   makeHTTPDownloader,
   makeYouTubeDownloader,
-  makeLocalDownloader
+  makeLocalDownloader,
+  makePowerfulDownloader,
+
+  getDownloader: downloaderType => {
+    if (downloaderType === 'http') {
+      return makeHTTPDownloader()
+    } else if (downloaderType === 'youtube') {
+      return makeYouTubeDownloader()
+    } else if (downloaderType === 'local') {
+      return makeLocalDownloader()
+    } else {
+      return null
+    }
+  }
 }
diff --git a/src/http-music.js b/src/http-music.js
index ed79878..68bfa77 100755
--- a/src/http-music.js
+++ b/src/http-music.js
@@ -224,17 +224,8 @@ setupDefaultPlaylist('./playlist.json')
         return
       }
 
-      let downloader
-      if (downloaderType === 'http') {
-        console.log("Using HTTP downloader.")
-        downloader = downloaders.makeHTTPDownloader()
-      } else if (downloaderType === 'youtube') {
-        console.log("Using YouTube downloader.")
-        downloader = downloaders.makeYouTubeDownloader()
-      } else if (downloaderType === 'local') {
-        console.log("Using local file downloader.")
-        downloader = downloaders.makeLocalDownloader()
-      } else {
+      let downloader = downloaders.getDownloader(downloaderType)
+      if (!downloader) {
         console.error("Invalid downloader type: " + downloaderType)
         return
       }
diff --git a/src/playlist-utils.js b/src/playlist-utils.js
index ff19ea9..13c6003 100644
--- a/src/playlist-utils.js
+++ b/src/playlist-utils.js
@@ -130,6 +130,7 @@ function parsePathString(pathString) {
   return pathParts
 }
 
+// TODO: Are these two functions actually useful??
 function getGroupTitle(group) {
   return group[0]
 }
@@ -142,11 +143,16 @@ function isGroup(array) {
   return Array.isArray(array[1])
 }
 
+function isTrack(array) {
+  return typeof array[1] === 'string'
+}
+
 module.exports = {
   flattenPlaylist,
   filterPlaylistByPathString, filterPlaylistByPath,
   removeGroupByPathString, removeGroupByPath,
   getPlaylistTreeString,
   parsePathString,
-  getGroupTitle, getGroupContents
+  getGroupTitle, getGroupContents,
+  isGroup, isTrack
 }