feat: 尝试增加底层拦截请求头的ua手段

Author: Taois

libs_drpy/fetchAxios.js

@@ -4,6 +4,50 @@
  */
 import FormData from 'form-data';
 import https from "https";
+import diagnosticsChannel from 'diagnostics_channel';
+
+let undiciStripUASubscribed = false;
+
+function ensureUndiciStripUASubscription() {
+    if (undiciStripUASubscribed) {
+        return;
+    }
+    undiciStripUASubscribed = true;
+
+    diagnosticsChannel.channel('undici:request:create').subscribe(({request}) => {
+        if (!request || !Array.isArray(request.headers)) {
+            return;
+        }
+        const headers = request.headers;
+
+        let shouldStrip = false;
+        for (let i = 0; i < headers.length; i += 2) {
+            const k = headers[i];
+            if (typeof k === 'string' && k.toLowerCase() === 'x-remove-user-agent') {
+                shouldStrip = true;
+                break;
+            }
+        }
+        if (!shouldStrip) {
+            return;
+        }
+
+        for (let i = 0; i < headers.length; i += 2) {
+            const k = headers[i];
+            if (typeof k === 'string' && k.toLowerCase() === 'x-remove-user-agent') {
+                headers.splice(i, 2);
+                i -= 2;
+            }
+        }
+        for (let i = 0; i < headers.length; i += 2) {
+            const k = headers[i];
+            if (typeof k === 'string' && k.toLowerCase() === 'user-agent') {
+                headers.splice(i, 2);
+                i -= 2;
+            }
+        }
+    });
+}
 
 /**
  * FetchAxios类 - HTTP客户端实现
@@ -71,6 +115,18 @@ class FetchAxios {
             finalConfig = await interceptor(finalConfig) || finalConfig;
         }
 
+        if (finalConfig.headers) {
+            const headerKeys = Object.keys(finalConfig.headers);
+            for (const key of headerKeys) {
+                if (key.toLowerCase() === 'user-agent' && finalConfig.headers[key] === 'RemoveUserAgent') {
+                    delete finalConfig.headers[key];
+                    finalConfig.headers['x-remove-user-agent'] = '1';
+                    ensureUndiciStripUASubscription();
+                    break;
+                }
+            }
+        }
+
         // 拼接查询参数
         if (finalConfig.params) {
             const query = new URLSearchParams(finalConfig.params).toString();
@@ -300,4 +356,4 @@ export function createHttpsInstance() {
         responseType: 'arraybuffer',
         httpsAgent: httpsAgent
     });
-}
+}

spider/js/_debug.js

@@ -0,0 +1,85 @@
+// _debug.js
+// 测试方法: http://localhost:5757/api/_debug?pwd=dzyyds
+var rule = {
+    title: '_debug',
+    description: '这是描述',
+    类型: '测试',
+    searchUrl: '',
+    class_parse: async () => {
+        log(`[${rule.title}] --class_parse--`);
+        return [
+            {type_id: '1', type_name: '电影'},
+            {type_id: '2', type_name: '电视剧'},
+            {type_id: '3', type_name: '综艺'},
+            {type_id: '4', type_name: '动漫'},
+        ]
+    },
+    预处理: async () => {
+        log(`[${rule.title}] --预处理--`);
+        rule.title = '_debug';
+    },
+    推荐: async () => {
+        // return '这是推荐:' + rule.title;
+        let d = [];
+        let html = '{}';
+        html = await request('https://httpbin.org/headers', {
+            headers: {
+                'Accept': '*/*',
+                'User-Agent': ''
+            }
+        });
+        // log(html);
+        d.push({
+            title: 'request结果1-传空UA',
+            content: html.parseX.headers,
+        });
+
+        html = await request('https://httpbin.org/headers', {
+            headers: {
+                'Accept': '*/*',
+                'User-Agent': 'RemoveUserAgent',
+            }
+        });
+        // log(html);
+        d.push({
+            title: 'request结果2-不传UA',
+            content: html.parseX.headers,
+        });
+
+        html = (await req('https://httpbin.org/headers', {
+            headers: {
+                'Accept': '*/*',
+                'User-Agent': 'RemoveUserAgent',
+            }
+        })).content;
+        d.push({
+            title: 'req结果-不传UA',
+            content: html.parseX.headers,
+        });
+
+        html = (await req('https://conn.origjoy.com/auth/init?appid=d4eeacc6cec3434fbc8c41608a3056a0&mac=0afa691314fd_a12d4a7c9n12&sn=a12d4a7c9n12&time=1768728113&ver=2.0&vn=4.1.3.03281430&sign=6a1ee16242b93a3ae6492bc55992b691',
+            {
+                headers: {
+                    'Accept': '*/*',
+                    'User-Agent': 'RemoveUserAgent',
+                }
+            })).content;
+        d.push({
+            title: 'req结果-60wmv',
+            content: html,
+        });
+        return d;
+    },
+    一级: async () => {
+        return '这是一级:' + rule.title
+    },
+    二级: async () => {
+        return '这是二级:' + rule.title
+    },
+    搜索: async () => {
+        return ['这是搜索:' + rule.title]
+    },
+    lazy: async () => {
+        return '这是播放:' + rule.title
+    },
+};

spider/js/_lib.request.js

@@ -1,5 +1,19 @@
 const iconv = require('iconv-lite');
 
+function sanitizeUserAgent(headers) {
+    if (!headers) {
+        return headers;
+    }
+    const keys = Object.keys(headers);
+    for (const key of keys) {
+        if (key.toLowerCase() === 'user-agent' && headers[key] === 'RemoveUserAgent') {
+            delete headers[key];
+            break;
+        }
+    }
+    return headers;
+}
+
 async function requestHtml(url, options) {
     try {
         let html = (await req(url, options)).content;
@@ -30,15 +44,20 @@ async function getPublicIp() {
 
 async function getHtml(config) {
     try {
-        return await axios.request(typeof config === "string" ? config : {
+        if (typeof config === "string") {
+            return await axios.request(config)
+        }
+        const cfg = {
             url: config.url,
             method: config.method || 'GET',
             headers: config.headers || {
                 'User-Agent': PC_UA
             },
             data: config.data || '',
-            responseType: config.responseType || '',//'arraybuffer'
-        })
+            responseType: config.responseType || ''
+        };
+        cfg.headers = sanitizeUserAgent(cfg.headers);
+        return await axios.request(cfg)
     } catch (e) {
         return e.response
     }
@@ -54,6 +73,7 @@ async function req_(reqUrl, mt, headers, data) {
         },
         data: data || '',
     };
+    config.headers = sanitizeUserAgent(config.headers);
     let res = await axios.request(config);
     return res.data;
 }
@@ -68,6 +88,7 @@ async function req_encoding(reqUrl, mt, headers, encoding, data) {
         data: data || '',
         responseType: 'arraybuffer'
     };
+    config.headers = sanitizeUserAgent(config.headers);
     let res = await axios.request(config);
     if (encoding) {
         res.data = iconv.decode(res.data, encoding);
@@ -88,6 +109,7 @@ async function req_proxy(reqUrl, mt, headers, data) {
             port: "7890"
         }
     };
+    config.headers = sanitizeUserAgent(config.headers);
     if (data) {
         config.data = data;
     }

utils/createAxiosAgent.js

@@ -39,14 +39,27 @@ export function createAxiosInstance(options = {}) {
 
     const httpsAgent = new https.Agent(httpsAgentOptions);
 
-    // 配置 axios 使用代理
     const _axios = axios.create({
-        httpAgent,  // 用于 HTTP 请求的代理
-        httpsAgent, // 用于 HTTPS 请求的代理
+        httpAgent,
+        httpsAgent,
+    });
+
+    _axios.interceptors.request.use(config => {
+        if (config && config.headers) {
+            const headers = config.headers;
+            const keys = Object.keys(headers);
+            for (const key of keys) {
+                if (key.toLowerCase() === 'user-agent' && headers[key] === 'RemoveUserAgent') {
+                    delete headers[key];
+                    break;
+                }
+            }
+        }
+        return config;
     });
 
     return _axios;
 }
 
 // 默认导出
-export default createAxiosInstance;
+export default createAxiosInstance;