-
Notifications
You must be signed in to change notification settings - Fork 2
/
clone.js
87 lines (76 loc) · 2.11 KB
/
clone.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
const puppeteer = require('puppeteer');
const { URL } = require('url');
const fs = require('fs-extra');
const path = require('path');
var args = [];
process.argv.slice(2).forEach(function (val, index, array) {
let a = val.split('=');
args[a[0]] = a[1];
});
if (! args.hasOwnProperty('url') || args.url.length === 0) {
console.error('ERROR: Please specify the `url`');
return;
}
const target = args['url'];
let disable = args.hasOwnProperty('disable') ? args['disable'] : null;
const disabled = disable !== null ? disable.split(',') : [];
const hostname = new URL(target);
const host = hostname.hostname;
(async () => {
const browser = await puppeteer.launch({
headless: false,
args: [
'--window-size=1024,600'
],
defaultViewport: {
width: 1024,
height: 600
}
});
const [page] = await browser.pages();
await page.setRequestInterception(true);
var dis = [];
let pairs = {
'image': 'image',
'css': 'stylesheet',
'js': 'script',
'font': 'font'
};
Object.keys(pairs).forEach(function(v, i) {
if (disabled.includes(v)) {
dis.push(pairs[v]);
}
});
// never load image
page.on('request', (req) => {
if (dis.includes(req.resourceType())) {
req.abort();
} else {
req.continue();
}
});
// intercept response and save content
page.on('response', async (res) => {
const url = new URL(res.url());
const regex = /http(s|):\/\//is;
if (! regex.test(url.pathname)) {
let filePath = path.resolve(`.${path.sep}output${path.sep}${host}${url.pathname}`);
if (path.extname(url.pathname).trim() === '') {
filePath = `${filePath}${path.sep}index.html`;
}
if (! args.hasOwnProperty('--no-progress')) {
console.info('Saving: ' + res.url() + ' => ' + filePath);
}
await fs.outputFile(filePath, await res.buffer());
}
});
// go to target url
await page.goto(target, {
waitUntil: ['domcontentloaded', 'networkidle0'],
timeout: 120000 // 120 seconds
});
// wait 2 seconds and close browser
setTimeout(async () => {
await browser.close();
}, 2 * 1000);
})();