drpy-node/spider/js/_lib.request.js at c6910af6824e77e164041246846f57114239be24 · hjdhnx/drpy-node · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
const iconv = require('iconv-lite');

function sanitizeUserAgent(headers) {
    if (!headers) {
        return headers;
    }
    const keys = Object.keys(headers);
    for (const key of keys) {
        if (key.toLowerCase() === 'user-agent' && headers[key] === 'RemoveUserAgent') {
            delete headers[key];
            break;
        }
    }
    return headers;
}

async function requestHtml(url, options) {
    try {
        let html = (await req(url, options)).content;
        // log(html);
        return html
    } catch (e) {
        log(`requestHtml error:${e.message}`);
        return ''
    }

}

async function requestJson(url, options) {
    try {
        let html = (await req(url, options)).content;
        return JSON.parse(html)
    } catch (e) {
        log(`requestJson error:${e.message}`);
        return {}
    }
}

async function getPublicIp() {
    let ip_obj = await requestJson('http://httpbin.org/ip');
    // log('ip_obj:',ip_obj);
    return ip_obj.origin
}

async function getHtml(config) {
    try {
        if (typeof config === "string") {
            return await axios.request(config)
        }
        const cfg = {
            url: config.url,
            method: config.method || 'GET',
            headers: config.headers || {
                'User-Agent': PC_UA
            },
            data: config.data || '',
            responseType: config.responseType || ''
        };
        cfg.headers = sanitizeUserAgent(cfg.headers);
        return await axios.request(cfg)
    } catch (e) {
        return e.response
    }

}

async function req_(reqUrl, mt, headers, data) {
    let config = {
        method: mt || 'Get',
        url: reqUrl,
        headers: headers || {
            'User-Agent': 'Mozilla/5.0 (Linux; Android 9; RMX1931 Build/PQ3A.190605.05081124; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/91.0.4472.114 Mobile Safari/537.36 QSTAPP/1.6.9 Html5Plus/1.0',
        },
        data: data || '',
    };
    config.headers = sanitizeUserAgent(config.headers);
    let res = await axios.request(config);
    return res.data;
}

async function req_encoding(reqUrl, mt, headers, encoding, data) {
    let config = {
        method: mt || 'Get',
        url: reqUrl,
        headers: headers || {
            'User-Agent': 'Mozilla/5.0 (Linux; Android 9; RMX1931 Build/PQ3A.190605.05081124; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/91.0.4472.114 Mobile Safari/537.36 QSTAPP/1.6.9 Html5Plus/1.0',
        },
        data: data || '',
        responseType: 'arraybuffer'
    };
    config.headers = sanitizeUserAgent(config.headers);
    let res = await axios.request(config);
    if (encoding) {
        res.data = iconv.decode(res.data, encoding);
    }
    return res.data;
}

async function req_proxy(reqUrl, mt, headers, data) {
    let config = {
        method: mt || 'Get',
        url: reqUrl,
        headers: headers || {
            'User-Agent': 'Mozilla/5.0 (Linux; Android 9; RMX1931 Build/PQ3A.190605.05081124; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/91.0.4472.114 Mobile Safari/537.36 QSTAPP/1.6.9 Html5Plus/1.0',
        },
        proxy: {
            protocol: 'http',
            host: '127.0.0.1',
            port: "7890"
        }
    };
    config.headers = sanitizeUserAgent(config.headers);
    if (data) {
        config.data = data;
    }
    let res = await axios.request(config);
    return res.data;
}

$.exports = {
    requestHtml,
    requestJson,
    getPublicIp,
    getHtml,
    req_,
    req_encoding,
    req_proxy,
    // axios // 没法import系统库
}