diff options
Diffstat (limited to 'crawlers.js')
-rw-r--r-- | crawlers.js | 65 |
1 files changed, 36 insertions, 29 deletions
diff --git a/crawlers.js b/crawlers.js index 3f6e391..8197095 100644 --- a/crawlers.js +++ b/crawlers.js @@ -1,15 +1,25 @@ -const fs = require('fs') -const path = require('path') -const expandHomeDir = require('expand-home-dir') -const fetch = require('node-fetch') -const url = require('url') -const { downloadPlaylistFromOptionValue, promisifyProcess } = require('./general-util') -const { spawn } = require('child_process') -const { orderBy } = require('natural-orderby') - -const { promisify } = require('util') -const readDir = promisify(fs.readdir) -const stat = promisify(fs.stat) +import {spawn} from 'node:child_process' +import {readdir, stat} from 'node:fs/promises' +import url from 'node:url' +import path from 'node:path' + +import {orderBy} from 'natural-orderby' +import expandHomeDir from 'expand-home-dir' +// import fetch from 'node-fetch' + +import {downloadPlaylistFromOptionValue, promisifyProcess} from './general-util.js' + +export const musicExtensions = [ + 'ogg', 'oga', + 'wav', 'mp3', 'm4a', 'aac', 'flac', 'opus', + 'mp4', 'mov', 'mkv', + 'mod' +] + +export const skipNames = [ + '.DS_Store', + '.git', +] // Each value is a function with these additional properties: // * crawlerName: The name of the crawler, such as "crawl-http". Used by @@ -21,7 +31,7 @@ const stat = promisify(fs.stat) const allCrawlers = {} /* TODO: Removed cheerio, so crawl-http no longer works. -function crawlHTTP(absURL, opts = {}, internals = {}) { +export function crawlHTTP(absURL, opts = {}, internals = {}) { // Recursively crawls a given URL, following every link to a deeper path and // recording all links in a tree (in the same format playlists use). Makes // multiple attempts to download failed paths. @@ -229,12 +239,7 @@ function getHTMLLinks(text) { } */ -function crawlLocal(dirPath, extensions = [ - 'ogg', 'oga', - 'wav', 'mp3', 'm4a', 'aac', 'flac', 'opus', - 'mp4', 'mov', 'mkv', - 'mod' -], isTop = true) { +function crawlLocal(dirPath, extensions = musicExtensions, isTop = true) { // If the passed path is a file:// URL, try to decode it: try { const url = new URL(dirPath) @@ -247,10 +252,16 @@ function crawlLocal(dirPath, extensions = [ dirPath = expandHomeDir(dirPath) } - return readDir(dirPath).then(items => { + return readdir(dirPath).then(items => { items = orderBy(items) return Promise.all(items.map(item => { + // There are a few files which are just never what we're looking for. + // We skip including or searching under these altogether. + if (skipNames.includes(item)) { + return null + } + const itemPath = path.join(dirPath, item) const itemURL = url.pathToFileURL(itemPath).href @@ -274,7 +285,7 @@ function crawlLocal(dirPath, extensions = [ return {name: item, url: itemURL} } } - }, statErr => null) + }, _statErr => null) })) }, err => { if (err.code === 'ENOENT') { @@ -321,7 +332,7 @@ crawlLocal.isAppropriateForArg = function(arg) { allCrawlers.crawlLocal = crawlLocal -async function crawlYouTube(url) { +export async function crawlYouTube(url) { const ytdl = spawn('youtube-dl', [ '-j', // Output as JSON '--flat-playlist', @@ -381,7 +392,7 @@ crawlYouTube.isAppropriateForArg = function(arg) { allCrawlers.crawlYouTube = crawlYouTube -async function openFile(input) { +export async function openFile(input) { return JSON.parse(await downloadPlaylistFromOptionValue(input)) } @@ -394,14 +405,10 @@ openFile.isAppropriateForArg = function(arg) { allCrawlers.openFile = openFile -// Actual module.exports stuff: - -Object.assign(module.exports, allCrawlers) - -module.exports.getCrawlerByName = function(name) { +export function getCrawlerByName(name) { return Object.values(allCrawlers).find(fn => fn.crawlerName === name) } -module.exports.getAllCrawlersForArg = function(arg) { +export function getAllCrawlersForArg(arg) { return Object.values(allCrawlers).filter(fn => fn.isAppropriateForArg(arg)) } |