path: root/crawlers.js

const path = require('path')
const { downloadPlaylistFromOptionValue, promisifyProcess } = require('./general-util')

// Each value is a function with these additional properties:
// * crawlerName: The name of the crawler, such as "crawl-http". Used by
//   getCrawlerByName.
// * isAppropriateForArg: A function returning whether an argument is valid for
//   the crawler. For example, crawlHTTP.isAppropriateForArg returns whether or
//   not the passed argument is a valid URL of the HTTP/HTTPS protocol. Used by
//   getAllCrawlersForArg.
const allCrawlers = {}

function sortIgnoreCase(sortFunction) {
  return function(a, b) {
    return sortFunction(a.toLowerCase(), b.toLowerCase())
  }
}

function crawlHTTP(absURL, opts = {}, internals = {}) {
  // Recursively crawls a given URL, following every link to a deeper path and
  // recording all links in a tree (in the same format playlists use). Makes
  // multiple attempts to download failed paths.

  const {
    verbose = false,

    maxAttempts = 5,

    allowedExternalHostRegex = null,
    stayInSameDirectory = true,

    keepAnyFileType = false,
    fileTypes = ['wav', 'ogg', 'oga', 'mp3', 'mp4', 'm4a', 'mov', 'mpga', 'mod'],

    forceGroupRegex = null,
    filterRegex = null
  } = opts

  if (!internals.attempts) internals.attempts = 0

  // TODO: Should absURL initially be added into this array? I'd like to
  // re-program this entire crawl function to make more sense - "internal"
  // dictionaries aren't quite easy to reason about!
  if (!internals.allURLs) internals.allURLs = []

  const verboseLog = text => {
    if (verbose) {
      console.error(text)
    }
  }

  const absURLObj = new URL(absURL)

  return fetch(absURL)
    .then(
      res => res.text().then(async text => {
        const links = getHTMLLinks(text)

        const items = []

        for (const link of links) {
          let [ name, href ] = link

          if (!href) {
            continue
          }

          // If the name (that's the content inside of <a>..</a>) ends with a
          // slash, that's probably just an artifact of a directory lister;
          // not actually part of the intended content. So we remove it!
          if (name.endsWith('/')) {
            name = name.slice(0, -1)
          }

          name = name.trim()

          let base
          if (path.extname(absURL)) {
            base = path.dirname(absURL) + '/'
          } else {
            base = absURL
          }

          const urlObj = new URL(href, base)
          const linkURL = urlObj.toString()

          if (internals.allURLs.includes(linkURL)) {
            verboseLog("[Ignored] Already done this URL: " + linkURL)
            continue
          }

          internals.allURLs.push(linkURL)

          if (filterRegex && !(filterRegex.test(linkURL))) {
            verboseLog("[Ignored] Failed regex: " + linkURL)
            continue
          }

          if (urlObj.host !== absURLObj.host && !(
            allowedExternalHostRegex && new RegExp(allowedExternalHostRegex)
              .test(urlObj.host))) {
            verboseLog("[Ignored] Inconsistent host: " + linkURL)
            continue
          }

          if (stayInSameDirectory) sameDir: {
            // Don't bother with staying in the same directory if it's on a
            // different host.
            if (urlObj.host !== absURLObj.host) {
              break sameDir
            }

            const relative = path.relative((new URL(base)).pathname, urlObj.pathname)
            if (relative.startsWith('..') || path.isAbsolute(relative)) {
              verboseLog("[Ignored] Outside of parent directory: " + linkURL + "\n-- relative: " + relative + "\n-- to base: " + base)
              continue
            }
          }

          if (href.endsWith('/') || (forceGroupRegex && new RegExp(forceGroupRegex).test(href))) {
            // It's a directory!

            verboseLog("[Dir] " + linkURL)

            items.push(await (
              crawlHTTP(linkURL, opts, Object.assign({}, internals))
                .then(({ items }) => ({name, items}))
            ))
          } else {
            // It's a file!

            const extensions = fileTypes.map(t => '.' + t)

            if (
              !keepAnyFileType &&
              !(extensions.includes(path.extname(href)))
            ) {
              verboseLog("[Ignored] Bad extension: " + linkURL)
              continue
            }

            verboseLog("[File] " + linkURL)
            items.push({name, downloaderArg: linkURL})
          }
        }

        return {items}
      }),

      err => {
        console.warn("Failed to download: " + absURL)

        if (internals.attempts < maxAttempts) {
          console.warn(
            `Trying again. Attempt ${internals.attempts + 1}/${maxAttempts}...`
          )

          return crawlHTTP(absURL, opts, Object.assign({}, internals, {
            attempts: internals.attempts + 1
          }))
        } else {
          console.error(
            "We've hit the download attempt limit (" + maxAttempts + "). " +
            "Giving up on this path."
          )

          throw 'FAILED_DOWNLOAD'
        }
      }
    )
    .catch(error => {
      if (error === 'FAILED_DOWNLOAD') {
        // Debug logging for this is already handled above.
        return []
      } else {
        throw error
      }
    })
}

function getHTMLLinks(text) {
  // Never parse HTML with a regex!
  const doc = (new DOMParser()).parseFromString(text, 'text/html')

  return Array.from(doc.getElementsByTagName('a')).map(el => {
    return [el.innerText, el.getAttribute('href')]
  })
}

crawlHTTP.crawlerName = 'crawl-http'

crawlHTTP.isAppropriateForArg = function(arg) {
  // It is only used for HTTP(S) servers:
  if (!(arg.startsWith('http://') || arg.startsWith('https://'))) {
    return false
  }

  // It will definitely only work for valid URLs:
  let url
  try {
    url = new URL(arg)
  } catch (error) {
    return false
  }

  // If the URL ends with a .json, it is probably meant to be used for a direct
  // playlist download, not to be crawled.
  if (path.extname(url.pathname) === '.json') {
    return false
  }

  return true
}

allCrawlers.crawlHTTP = crawlHTTP

async function openFile(input) {
  return JSON.parse(await downloadPlaylistFromOptionValue(input))
}

openFile.crawlerName = 'open-file'

openFile.isAppropriateForArg = function(arg) {
  // It is only valid for arguments that end with .json:
  return path.extname(arg) === '.json'
}

allCrawlers.openFile = openFile

// Actual module.exports stuff:

Object.assign(module.exports, allCrawlers)

module.exports.getCrawlerByName = function(name) {
  return Object.values(allCrawlers).find(fn => fn.crawlerName === name)
}

module.exports.getAllCrawlersForArg = function(arg) {
  return Object.values(allCrawlers).filter(fn => fn.isAppropriateForArg(arg))
}