1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
|
const { promisifyProcess } = require('./general-util')
const { promisify } = require('util')
const { spawn } = require('child_process')
const { Base64 } = require('js-base64')
const mkdirp = promisify(require('mkdirp'))
const fs = require('fs')
const fetch = require('node-fetch')
const tempy = require('tempy')
const path = require('path')
const sanitize = require('sanitize-filename')
const writeFile = promisify(fs.writeFile)
const rename = promisify(fs.rename)
const stat = promisify(fs.stat)
const readdir = promisify(fs.readdir)
const symlink = promisify(fs.symlink)
const copyFile = (source, target) => {
// Stolen from https://stackoverflow.com/a/30405105/4633828
const rd = fs.createReadStream(source)
const wr = fs.createWriteStream(target)
return new Promise((resolve, reject) => {
rd.on('error', reject)
wr.on('error', reject)
wr.on('finish', resolve)
rd.pipe(wr)
}).catch(function(error) {
rd.destroy()
wr.end()
throw error
})
}
const cachify = (identifier, baseFunction) => {
return async arg => {
// If there was no argument passed (or it aws empty), nothing will work..
if (!arg) {
throw new TypeError('Expected a downloader argument')
}
// Determine where the final file will end up. This is just a directory -
// the file's own name is determined by the downloader.
const cacheDir = downloaders.rootCacheDir + '/' + identifier
const finalDirectory = cacheDir + '/' + Base64.encode(arg)
// Check if that directory only exists. If it does, return the file in it,
// because it being there means we've already downloaded it at some point
// in the past.
let exists
try {
await stat(finalDirectory)
exists = true
} catch (error) {
// ENOENT means the folder doesn't exist, which is one of the potential
// expected outputs, so do nothing and let the download continue.
if (error.code === 'ENOENT') {
exists = false
}
// Otherwise, there was some unexpected error, so throw it:
else {
throw error
}
}
// If the directory exists, return the file in it. Downloaders always
// return only one file, so it's expected that the directory will only
// contain a single file. We ignore any other files. Note we also allow
// the download to continue if there aren't any files in the directory -
// that would mean that the file (but not the directory) was unexpectedly
// deleted.
if (exists) {
const files = await readdir(finalDirectory)
if (files.length >= 1) {
return finalDirectory + '/' + files[0]
}
}
// The "temporary" output, aka the download location. Generally in a
// temporary location as returned by tempy.
const tempFile = await baseFunction(arg)
// Then move the download to the final location. First we need to make the
// folder exist, then we move the file.
const finalFile = finalDirectory + '/' + path.basename(tempFile)
await mkdirp(finalDirectory)
await rename(tempFile, finalFile)
// And return.
return finalFile
}
}
const removeFileProtocol = arg => {
const fileProto = 'file://'
if (arg.startsWith(fileProto)) {
return decodeURIComponent(arg.slice(fileProto.length))
} else {
return arg
}
}
const downloaders = {
extension: 'mp3', // Generally target file extension, used by youtube-dl
// TODO: Cross-platform stuff
rootCacheDir: process.env.HOME + '/.mtui/downloads',
http: cachify('http', arg => {
const out = (
tempy.directory() + '/' +
sanitize(decodeURIComponent(path.basename(arg))))
return fetch(arg)
.then(response => response.buffer())
.then(buffer => writeFile(out, buffer))
.then(() => out)
}),
youtubedl: cachify('youtubedl', arg => {
const outDir = tempy.directory()
const outFile = outDir + '/%(id)s-%(uploader)s-%(title)s.%(ext)s'
const opts = [
'--quiet',
'--no-warnings',
'--extract-audio',
'--audio-format', downloaders.extension,
'--output', outFile,
arg
]
return promisifyProcess(spawn('youtube-dl', opts))
.then(() => readdir(outDir))
.then(files => outDir + '/' + files[0])
}),
local: cachify('local', arg => {
// Usually we'd just return the given argument in a local
// downloader, which is efficient, since there's no need to
// copy a file from one place on the hard drive to another.
// But reading from a separate drive (e.g. a USB stick or a
// CD) can take a lot longer than reading directly from the
// computer's own drive, so this downloader copies the file
// to a temporary file on the computer's drive.
// Ideally, we'd be able to check whether a file is on the
// computer's main drive mount or not before going through
// the steps to copy, but I'm not sure if there's a way to
// do that (and it's even less likely there'd be a cross-
// platform way).
// It's possible the downloader argument starts with the "file://"
// protocol string; in that case we'll want to snip it off and URL-
// decode the string.
arg = removeFileProtocol(arg)
// TODO: Is it necessary to sanitize here?
// Haha, the answer to "should I sanitize" is probably always YES..
const base = path.basename(arg, path.extname(arg))
const out = tempy.directory() + '/' + sanitize(base) + path.extname(arg)
return copyFile(arg, out)
.then(() => out)
}),
locallink: cachify('locallink', arg => {
// Like the local downloader, but creates a symbolic link to the argument.
arg = removeFileProtocol(arg)
const base = path.basename(arg, path.extname(arg))
const out = tempy.directory() + '/' + sanitize(base) + path.extname(arg)
return symlink(path.resolve(arg), out)
.then(() => out)
}),
echo: arg => arg,
getDownloaderFor: arg => {
if (arg.startsWith('http://') || arg.startsWith('https://')) {
if (arg.includes('youtube.com')) {
return downloaders.youtubedl
} else {
return downloaders.http
}
} else {
// return downloaders.local
return downloaders.locallink
}
}
}
module.exports = downloaders
|
