From 69cf7222241ee9ed1f2aae3fba48061f05dbd56f Mon Sep 17 00:00:00 2001 From: liam4 Date: Wed, 31 May 2017 19:59:16 -0300 Subject: Generally improve how scripts and running works --- README.md | 32 +++++++-- crawl-recursive.js | 88 ----------------------- package-lock.json | 189 +++++++++++++++++++++++++++++++++++++++++++++++++ package.json | 8 +++ src/crawl-recursive.js | 91 ++++++++++++++++++++++++ src/play.js | 2 + 6 files changed, 318 insertions(+), 92 deletions(-) delete mode 100644 crawl-recursive.js create mode 100644 package-lock.json create mode 100755 src/crawl-recursive.js mode change 100644 => 100755 src/play.js diff --git a/README.md b/README.md index 7e7478b..1a1b75a 100644 --- a/README.md +++ b/README.md @@ -13,9 +13,33 @@ $ python3 -m http.server # On the client; that is, the device with http-music: $ cd http-music $ yarn # to install Node.js dependencies; you'll also need `avconv` and `play` (sox). -$ node crawl-recursive.js > playlist.json -$ node play.js # Go! +$ npm run crawl-recursive -- > playlist.json +$ node . # Go! ``` -**Zomg command line arguments????** — Yes; read the end of the `play.js` file. -There's a bunch of JS-comment-based documentation there. +**Zomg command line arguments documentation????** — Yes; read the end of the +`play.js` file. There's a bunch of JS-comment-based documentation there. + +There's actually three proper ways to run `http-music`: + +* **Run `$ npm link` and then use `$ http-music`.** This gives you the + advantage of having a proper command you can use anywhere; however it does + mean installing to /usr/bin (or wherever your `npm-link` command puts + things). + +* **Run `$ node .` while `cd`'d into `http-music`.** This is essentially the + same as using `npm-link`, but it requires you to be in the repository folder. + That's alright if you're developing, or just directly downloaded the entire + repository, but probably isn't otherwise useful. + +* **Run `$ npm run play`.** (You might need to do `$ npm run http-music play`.) + This way *works*, but it's not suggested; command line arguments need to be + passed after `--`, e.g. `npm run play -- -c -k CoolArtist123` instead of + `node . -c -k CoolArtist123` or `http-music -c -k CoolArtist123`. Use + whatever you prefer, I guess. + +**If you're running with `npm run`,** you need to use `--` before any of your +own options, e.g. `npm run play -- -c -k CoolArtist123`. I know, it looks +stupid; but it's really just the way `npm run` works. You're probably better +off with `node .` while `cd`'d into the `http-music` directory, or maybe you'd +rather `npm link` it so you can use it anywhere. diff --git a/crawl-recursive.js b/crawl-recursive.js deleted file mode 100644 index d53f7d1..0000000 --- a/crawl-recursive.js +++ /dev/null @@ -1,88 +0,0 @@ -'use strict' - -const MAX_DOWNLOAD_ATTEMPTS = 5 - -const fetch = require('node-fetch') -const $ = require('cheerio') - -function crawl(absURL, attempts = 0) { - // Recursively crawls a given URL, following every link to a deeper path and - // recording all links in a tree (in the same format playlists use). Makes - // multiple attempts to download failed paths. - - return fetch(absURL) - .then( - res => res.text().then(text => { - const links = getHTMLLinks(text) - const verbose = process.argv.includes('--verbose') - - return Promise.all(links.map(link => { - const [ title, href ] = link - - if (href.endsWith('/')) { - // It's a directory! - - if (verbose) console.log("[Dir] " + absURL + href) - return crawl(absURL + href) - .then(res => [title, res]) - } else { - // It's a file! - - if (verbose) console.log("[File] " + absURL + href) - return Promise.resolve([title, absURL + href]) - } - })) - }), - - err => { - console.warn("Failed to download: " + absURL) - - if (attempts < MAX_DOWNLOAD_ATTEMPTS) { - console.warn( - "Trying again. Attempt " + (attempts + 1) + - "/" + MAX_DOWNLOAD_ATTEMPTS + "..." - ) - - return crawl(absURL, attempts + 1) - } else { - console.error( - "We've hit the download attempt limit (" + - MAX_DOWNLOAD_ATTEMPTS + "). Giving up on this path." - ) - - throw 'FAILED_DOWNLOAD' - } - } - ) - .catch(error => { - if (error === 'FAILED_DOWNLOAD') { - // Debug logging for this is already handled above. - return [] - } else { - throw error - } - }) -} - -function getHTMLLinks(text) { - // Never parse HTML with a regex! - - return $(text).find('a').get().map(a => { - const $a = $(a) - return [$a.text(), $a.attr('href')] - }) -} - -if (process.argv.length === 2) { - console.log("Usage: crawl-recursive http://example.com/example/path") -} else { - let url = process.argv[2] - - if (!(url.endsWith('/'))) { - url = url + '/' - } - - crawl(url) - .then(res => console.log(JSON.stringify(res, null, 2))) - .catch(err => console.error(err)) -} diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000..4bc7556 --- /dev/null +++ b/package-lock.json @@ -0,0 +1,189 @@ +{ + "name": "http-music", + "version": "0.0.1", + "lockfileVersion": 1, + "dependencies": { + "@types/node": { + "version": "6.0.73", + "resolved": "https://registry.npmjs.org/@types/node/-/node-6.0.73.tgz", + "integrity": "sha1-hdxLtvElN3x13dJRmh7rY/Ck7XA=" + }, + "boolbase": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz", + "integrity": "sha1-aN/1++YMUes3cl6p4+0xDcwed24=" + }, + "buffer-shims": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/buffer-shims/-/buffer-shims-1.0.0.tgz", + "integrity": "sha1-mXjOMXOIxkmth5MCjDR37wRKi1E=" + }, + "cheerio": { + "version": "1.0.0-rc.1", + "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.0.0-rc.1.tgz", + "integrity": "sha1-KvNzOeq3E+9rcs3pjO+mcrh2Qf4=" + }, + "core-util-is": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", + "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=" + }, + "crypto-random-string": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/crypto-random-string/-/crypto-random-string-1.0.0.tgz", + "integrity": "sha1-ojD2T1aDEOFJgAmUB5DsmVRbyn4=" + }, + "css-select": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/css-select/-/css-select-1.2.0.tgz", + "integrity": "sha1-KzoRBTnFNV8c2NMUYj6HCxIeyFg=" + }, + "css-what": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/css-what/-/css-what-2.1.0.tgz", + "integrity": "sha1-lGfQMsOM+u+58teVASUwYvh/ob0=" + }, + "dom-serializer": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-0.1.0.tgz", + "integrity": "sha1-BzxpdUbOB4DOI75KKOKT5AvDDII=", + "dependencies": { + "domelementtype": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-1.1.3.tgz", + "integrity": "sha1-vSh3PiZCiBrsUVRJJCmcXNgiGFs=" + } + } + }, + "domelementtype": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-1.3.0.tgz", + "integrity": "sha1-sXrtguirWeUt2cGbF1bg/BhyBMI=" + }, + "domhandler": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-2.4.1.tgz", + "integrity": "sha1-iS5HAAqZvlW783dP/qBWHYh5wlk=" + }, + "domutils": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/domutils/-/domutils-1.5.1.tgz", + "integrity": "sha1-3NhIiib1Y9YQeeSMn3t+Mjc2gs8=" + }, + "encoding": { + "version": "0.1.12", + "resolved": "https://registry.npmjs.org/encoding/-/encoding-0.1.12.tgz", + "integrity": "sha1-U4tm8+5izRq1HsMjgp0flIDHS+s=" + }, + "entities": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/entities/-/entities-1.1.1.tgz", + "integrity": "sha1-blwtClYhtdra7O+AuQ7ftc13cvA=" + }, + "htmlparser2": { + "version": "3.9.2", + "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-3.9.2.tgz", + "integrity": "sha1-G9+HrMoPP55T+k/M6w9LTLsAszg=" + }, + "iconv-lite": { + "version": "0.4.17", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.17.tgz", + "integrity": "sha1-T9qjs4rLwsAxsEXQ7c3+HsqxjI0=" + }, + "inherits": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", + "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=" + }, + "is-stream": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-1.1.0.tgz", + "integrity": "sha1-EtSj3U5o4Lec6428hBc66A2RykQ=" + }, + "isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" + }, + "lodash": { + "version": "4.17.4", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.4.tgz", + "integrity": "sha1-eCA6TRwyiuHYbcpkYONptX9AVa4=" + }, + "node-fetch": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-1.7.0.tgz", + "integrity": "sha1-P/bFZUT5t/sAaCM4u1Xub1SooO8=" + }, + "nth-check": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-1.0.1.tgz", + "integrity": "sha1-mSms32KPwsQQmN6rgqxYDPFJquQ=" + }, + "parse5": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/parse5/-/parse5-3.0.2.tgz", + "integrity": "sha1-Be/1fw70V3+xRKefi5qWemzERRA=" + }, + "pify": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", + "integrity": "sha1-7RQaasBDqEnqWISY59yosVMw6Qw=" + }, + "process-nextick-args": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-1.0.7.tgz", + "integrity": "sha1-FQ4gt1ZZCtP5EJPyWk8q2L/zC6M=" + }, + "readable-stream": { + "version": "2.2.9", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.2.9.tgz", + "integrity": "sha1-z3jsb0ptHrQ9JkiMrJfwQudLf8g=" + }, + "safe-buffer": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.0.1.tgz", + "integrity": "sha1-0mPKVGls2KMGtcplUekt5XkY++c=" + }, + "sanitize-filename": { + "version": "1.6.1", + "resolved": "https://registry.npmjs.org/sanitize-filename/-/sanitize-filename-1.6.1.tgz", + "integrity": "sha1-YS2hyWRz+gLczaktzVtKsWSmdyo=" + }, + "string_decoder": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.0.1.tgz", + "integrity": "sha1-YuIA8DmVWmgQ2N8KM//A8BNmLZg=" + }, + "temp-dir": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/temp-dir/-/temp-dir-1.0.0.tgz", + "integrity": "sha1-CnwOom06Oa+n4OvqnB/AvE2qAR0=" + }, + "tempy": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/tempy/-/tempy-0.1.0.tgz", + "integrity": "sha1-hSdBPNBxAINPzJy7gkK+lboOH+4=" + }, + "truncate-utf8-bytes": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/truncate-utf8-bytes/-/truncate-utf8-bytes-1.0.2.tgz", + "integrity": "sha1-QFkjkJWS1W94pYGENLC3hInKXys=" + }, + "unique-string": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/unique-string/-/unique-string-1.0.0.tgz", + "integrity": "sha1-nhBXzKhRq7kzmPizOuGHuZyuwRo=" + }, + "utf8-byte-length": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/utf8-byte-length/-/utf8-byte-length-1.0.4.tgz", + "integrity": "sha1-9F8VDExm7uloGGUFq5P8u4rWv2E=" + }, + "util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" + } + } +} diff --git a/package.json b/package.json index 804df46..86856d9 100644 --- a/package.json +++ b/package.json @@ -2,6 +2,14 @@ "name": "http-music", "version": "0.0.1", "main": "src/play.js", + "scripts": { + "play": "node src/play.js", + "crawl-recursive": "node src/crawl-recursive" + }, + "bin": { + "http-music": "./src/play.js", + "http-music-crawl-recursive": "./src/crawl-recursive.js" + }, "dependencies": { "cheerio": "^1.0.0-rc.1", "node-fetch": "^1.7.0", diff --git a/src/crawl-recursive.js b/src/crawl-recursive.js new file mode 100755 index 0000000..2656279 --- /dev/null +++ b/src/crawl-recursive.js @@ -0,0 +1,91 @@ +#!/usr/bin/env node + +'use strict' + +const MAX_DOWNLOAD_ATTEMPTS = 5 + +const fetch = require('node-fetch') +const $ = require('cheerio') + +function crawl(absURL, attempts = 0) { + // Recursively crawls a given URL, following every link to a deeper path and + // recording all links in a tree (in the same format playlists use). Makes + // multiple attempts to download failed paths. + + return fetch(absURL) + .then( + res => res.text().then(text => { + const links = getHTMLLinks(text) + const verbose = process.argv.includes('--verbose') + + return Promise.all(links.map(link => { + const [ title, href ] = link + + if (href.endsWith('/')) { + // It's a directory! + + if (verbose) console.log("[Dir] " + absURL + href) + return crawl(absURL + href) + .then(res => [title, res]) + } else { + // It's a file! + + if (verbose) console.log("[File] " + absURL + href) + return Promise.resolve([title, absURL + href]) + } + })) + }), + + err => { + console.warn("Failed to download: " + absURL) + + if (attempts < MAX_DOWNLOAD_ATTEMPTS) { + console.warn( + "Trying again. Attempt " + (attempts + 1) + + "/" + MAX_DOWNLOAD_ATTEMPTS + "..." + ) + + return crawl(absURL, attempts + 1) + } else { + console.error( + "We've hit the download attempt limit (" + + MAX_DOWNLOAD_ATTEMPTS + "). Giving up on this path." + ) + + throw 'FAILED_DOWNLOAD' + } + } + ) + .catch(error => { + if (error === 'FAILED_DOWNLOAD') { + // Debug logging for this is already handled above. + return [] + } else { + throw error + } + }) +} + +function getHTMLLinks(text) { + // Never parse HTML with a regex! + + return $(text).find('a').get().map(a => { + const $a = $(a) + return [$a.text(), $a.attr('href')] + }) +} + +if (process.argv.length === 2) { + console.log("Usage: http-music-crawl-recursive http://.../example/path/") + console.log("..or, npm run crawl-recursive -- http://...") +} else { + let url = process.argv[2] + + if (!(url.endsWith('/'))) { + url = url + '/' + } + + crawl(url) + .then(res => console.log(JSON.stringify(res, null, 2))) + .catch(err => console.error(err)) +} diff --git a/src/play.js b/src/play.js old mode 100644 new mode 100755 index 5e3e04b..2e47fba --- a/src/play.js +++ b/src/play.js @@ -1,3 +1,5 @@ +#!/usr/bin/env node + 'use strict' const fs = require('fs') -- cgit 1.3.0-6-gf8a5