blob: 7a98be93303f1b360cca915b8e1c6531b58b11cd (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
|
const fetch = require('node-fetch')
function parseDirectoryListing(text) {
// Matches all links in a directory listing.
// Returns an array where each item is in the format [href, label].
if (!(text.includes('Directory listing for'))) {
console.warn("Not a directory listing! Crawl returning empty array.")
return []
}
const regex = /<a href="([^"]*)">([^>]*)<\/a>/g
let matches, output = []
while (matches = regex.exec(text)) {
output.push([matches[1], matches[2]])
}
return output
}
function crawl(absURL) {
return fetch(absURL)
.then(res => res.text(), err => {
console.warn('FAILED: ' + absURL)
return 'Oops'
})
.then(text => parseDirectoryListing(text))
.then(links => Promise.all(links.map(link => {
const [ href, title ] = link
if (href.endsWith('/')) {
// It's a directory!
console.log('[Dir] ' + absURL + href)
return crawl(absURL + href).then(res => [title, res])
} else {
// It's a file!
console.log('[File] ' + absURL + href)
return Promise.resolve([title, absURL + href])
}
})))
}
crawl('http://192.168.2.19:1233/')
.then(res => console.log(JSON.stringify(res, null, 2)))
.catch(err => console.error(err))
|
