diff options
Diffstat (limited to 'downloaders.js')
| -rw-r--r-- | downloaders.js | 153 |
1 files changed, 83 insertions, 70 deletions
diff --git a/downloaders.js b/downloaders.js index 4b4750c..4c1ce38 100644 --- a/downloaders.js +++ b/downloaders.js @@ -1,7 +1,7 @@ const { promisifyProcess } = require('./general-util') const { promisify } = require('util') const { spawn } = require('child_process') -const { Base64 } = require('js-base64') +const { URL } = require('url') const mkdirp = promisify(require('mkdirp')) const fs = require('fs') const fetch = require('node-fetch') @@ -31,7 +31,9 @@ const copyFile = (source, target) => { }) } -const cachify = (identifier, baseFunction) => { +const disableBackResolving = arg => arg.split('/').map(str => str.replace(/^\../, '_..')).join('/') + +const cachify = (identifier, keyFunction, baseFunction) => { return async arg => { // If there was no argument passed (or it aws empty), nothing will work.. if (!arg) { @@ -41,7 +43,7 @@ const cachify = (identifier, baseFunction) => { // Determine where the final file will end up. This is just a directory - // the file's own name is determined by the downloader. const cacheDir = downloaders.rootCacheDir + '/' + identifier - const finalDirectory = cacheDir + '/' + Base64.encode(arg) + const finalDirectory = cacheDir + '/' + disableBackResolving(keyFunction(arg)) // Check if that directory only exists. If it does, return the file in it, // because it being there means we've already downloaded it at some point @@ -105,73 +107,84 @@ const downloaders = { // TODO: Cross-platform stuff rootCacheDir: process.env.HOME + '/.mtui/downloads', - http: cachify('http', arg => { - const out = ( - tempy.directory() + '/' + - sanitize(decodeURIComponent(path.basename(arg)))) - - return fetch(arg) - .then(response => response.buffer()) - .then(buffer => writeFile(out, buffer)) - .then(() => out) - }), - - youtubedl: cachify('youtubedl', arg => { - const outDir = tempy.directory() - const outFile = outDir + '/%(id)s-%(uploader)s-%(title)s.%(ext)s' - - const opts = [ - '--quiet', - '--no-warnings', - '--extract-audio', - '--audio-format', downloaders.extension, - '--output', outFile, - arg - ] - - return promisifyProcess(spawn('youtube-dl', opts)) - .then(() => readdir(outDir)) - .then(files => outDir + '/' + files[0]) - }), - - local: cachify('local', arg => { - // Usually we'd just return the given argument in a local - // downloader, which is efficient, since there's no need to - // copy a file from one place on the hard drive to another. - // But reading from a separate drive (e.g. a USB stick or a - // CD) can take a lot longer than reading directly from the - // computer's own drive, so this downloader copies the file - // to a temporary file on the computer's drive. - // Ideally, we'd be able to check whether a file is on the - // computer's main drive mount or not before going through - // the steps to copy, but I'm not sure if there's a way to - // do that (and it's even less likely there'd be a cross- - // platform way). - - // It's possible the downloader argument starts with the "file://" - // protocol string; in that case we'll want to snip it off and URL- - // decode the string. - arg = removeFileProtocol(arg) - - // TODO: Is it necessary to sanitize here? - // Haha, the answer to "should I sanitize" is probably always YES.. - const base = path.basename(arg, path.extname(arg)) - const out = tempy.directory() + '/' + sanitize(base) + path.extname(arg) - - return copyFile(arg, out) - .then(() => out) - }), - - locallink: cachify('locallink', arg => { - // Like the local downloader, but creates a symbolic link to the argument. - - arg = removeFileProtocol(arg) - const base = path.basename(arg, path.extname(arg)) - const out = tempy.directory() + '/' + sanitize(base) + path.extname(arg) - - return symlink(path.resolve(arg), out) - .then(() => out) - }), + http: cachify('http', + arg => { + const url = new URL(arg) + return url.hostname + url.pathname + }, + arg => { + const out = ( + tempy.directory() + '/' + + sanitize(decodeURIComponent(path.basename(arg)))) + + return fetch(arg) + .then(response => response.buffer()) + .then(buffer => writeFile(out, buffer)) + .then(() => out) + }), + + youtubedl: cachify('youtubedl', + arg => (arg.match(/watch\?v=(.*)/) || ['', arg])[1], + arg => { + const outDir = tempy.directory() + const outFile = outDir + '/%(id)s-%(uploader)s-%(title)s.%(ext)s' + + const opts = [ + '--quiet', + '--no-warnings', + '--extract-audio', + '--audio-format', downloaders.extension, + '--output', outFile, + arg + ] + + return promisifyProcess(spawn('youtube-dl', opts)) + .then(() => readdir(outDir)) + .then(files => outDir + '/' + files[0]) + }), + + local: cachify('local', + arg => arg, + arg => { + // Usually we'd just return the given argument in a local + // downloader, which is efficient, since there's no need to + // copy a file from one place on the hard drive to another. + // But reading from a separate drive (e.g. a USB stick or a + // CD) can take a lot longer than reading directly from the + // computer's own drive, so this downloader copies the file + // to a temporary file on the computer's drive. + // Ideally, we'd be able to check whether a file is on the + // computer's main drive mount or not before going through + // the steps to copy, but I'm not sure if there's a way to + // do that (and it's even less likely there'd be a cross- + // platform way). + + // It's possible the downloader argument starts with the "file://" + // protocol string; in that case we'll want to snip it off and URL- + // decode the string. + arg = removeFileProtocol(arg) + + // TODO: Is it necessary to sanitize here? + // Haha, the answer to "should I sanitize" is probably always YES.. + const base = path.basename(arg, path.extname(arg)) + const out = tempy.directory() + '/' + sanitize(base) + path.extname(arg) + + return copyFile(arg, out) + .then(() => out) + }), + + locallink: cachify('locallink', + arg => arg, + arg => { + // Like the local downloader, but creates a symbolic link to the argument. + + arg = removeFileProtocol(arg) + const base = path.basename(arg, path.extname(arg)) + const out = tempy.directory() + '/' + sanitize(base) + path.extname(arg) + + return symlink(path.resolve(arg), out) + .then(() => out) + }), echo: arg => arg, |