HTTP crawler stuff

Basically, support for cors-anywhere.

(Cherry-picked from web-mtui branch)

1 files changed, 11 insertions, 3 deletions

diff --git a/crawlers.js b/crawlers.js
index feeedf2..c12948f 100644
--- a/crawlers.js
+++ b/crawlers.js
@@ -86,7 +86,15 @@ function crawlHTTP(absURL, opts = {}, internals = {}) {
 
           name = name.trim()
 
-          const urlObj = new url.URL(href, absURL + '/')
+          let base
+          if (path.extname(absURL)) {
+            base = path.dirname(absURL) + '/'
+            console.log('extname:', path.extname(absURL), 'so base:', base)
+          } else {
+            base = absURL
+          }
+
+          const urlObj = new url.URL(href, base)
           const linkURL = url.format(urlObj)
 
           if (internals.allURLs.includes(linkURL)) {
@@ -115,9 +123,9 @@ function crawlHTTP(absURL, opts = {}, internals = {}) {
               break sameDir
             }
 
-            const relative = path.relative(absURLObj.pathname, urlObj.pathname)
+            const relative = path.relative((new url.URL(base)).pathname, urlObj.pathname)
             if (relative.startsWith('..') || path.isAbsolute(relative)) {
-              verboseLog("[Ignored] Outside of parent directory: " + linkURL)
+              verboseLog("[Ignored] Outside of parent directory: " + linkURL + "\n-- relative: " + relative + "\n-- to base: " + base)
               continue
             }
           }