Skip to content

Commit d9a2927

Browse files
hjdhnxhjdhnx
committed
update:支持解析与嗅探
1 parent 1bc9198 commit d9a2927

File tree

7 files changed

+236
-57
lines changed

7 files changed

+236
-57
lines changed

‎README.md‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,9 +43,11 @@ yarn build
4343

4444
# 更新记录
4545

46-
### 20250118
46+
### 20250119
4747

4848
1. 支持通用嗅探
49+
2. 支持hipy-t4源
50+
3. 支持解析
4951

5052
### hipy嗅探器说明
5153

‎nodejs/package.json‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,8 +38,10 @@
3838
"crypto-js": "^4.2.0",
3939
"dayjs": "^1.11.10",
4040
"fastify": "^4.26.0",
41+
"fastq": "^1.18.0",
4142
"hls-parser": "^0.10.8",
4243
"iconv-lite": "^0.6.3",
44+
"jsonpath-plus": "^10.2.0",
4345
"node-json-db": "^2.3.0",
4446
"node-rsa": "^1.1.1",
4547
"pako": "^2.1.0",

‎nodejs/src/index.config.js‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,15 @@
11
export default {
22
drpyS: {
3-
config_url: 'http://localhost:5757/config/1?sub=all&pwd=',
3+
config_url: 'http://localhost:5757/config/1?sub=all&pwd=', // 本地ds 支持
4+
// config_url: 'http://localhost:5707/config/0?sub=dzyyds', // 本地hipy-t4 支持
45
home_site: 'http://localhost:5757/api/设置中心',
56
enable_dspush: 1,
67
enable_home_site: 0,
78
sniffer_rule: 'http((?!http).){12,}?\\.(m3u8|mp4|flv|avi|mkv|rm|wmv|mpg|m4a|mp3)\\?.*|http((?!http).){12,}\\.(m3u8|mp4|flv|avi|mkv|rm|wmv|mpg|m4a|mp3)|http((?!http).)*?video/tos*|http((?!http).)*?obj/tos*',
89
enable_hipy_sniffer: 0,
910
hipy_sniffer_url: 'http://127.0.0.1:5708/sniffer?url=',
11+
parse_count: 6, // 最多显示多少条解析
12+
parse_timeout: 5000, // 解析超时毫秒数
1013
},
1114
ffm3u8: {
1215
url: 'https://cj.ffzyapi.com/api.php/provide/vod/from/ffm3u8',

‎nodejs/src/router.js‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ export default async function router(fastify) {
4747
try {
4848

4949
drpyS_data = await request(drpyS_config_url);
50-
if (drpyS_data.sites_count && drpyS_data.homepage === 'https://github.com/hjdhnx/drpy-node') {
50+
if (drpyS_data.homepage && drpyS_data.homepage.startsWith('https://github.com/hjdhnx')) {
5151
let drpyS_sites = drpyS_data.sites.filter(site => site.type === 4);
5252

5353
// console.log(drpyS_sites);

‎nodejs/src/spider/video/drpyS.js‎

Lines changed: 119 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import {request, post, mergeQuery} from '../../util/request.js';
22
import {base64Encode, md5} from '../../util/crypto-util.js';
3-
import {extractTags} from "../../util/tool.js";
3+
import {extractTags, shuffleArray, jsonpath} from "../../util/tool.js";
4+
import batchExecute from '../../util/batchExecute.js';
45
import * as cfg from '../../index.config.js';
56

67
const meta = {
@@ -246,67 +247,131 @@ async function play(_inReq, _outResp) {
246247
}
247248
result = images
248249
} else { //影视类执行解析、免嗅、嗅探逻辑
249-
if (result && result.jx) {
250-
console.log(DsCache.parses);
250+
if (result && result.jx && result.url) {
251+
const input = result.url;
252+
// console.log(DsCache.parses);
251253
// 筛选出json解析
252254
let parses = DsCache.parses.filter(it => it.type === 1);
253-
} else if (result && result.parse) {
254-
const sniffer_rule = cfg.default.drpyS.sniffer_rule || 'http((?!http).){12,}?\\.m3u8(?!\\?)';
255-
const regex = new RegExp(sniffer_rule);
256-
const realUrl = result.url;
257-
if (realUrl && regex.test(realUrl)) {
258-
result.parse = 0;
259-
} else if (realUrl && !regex.test(realUrl)) {
260-
if (cfg.default.drpyS.enable_hipy_sniffer && cfg.default.drpyS.hipy_sniffer_url) {
261-
const _js = result.js;
262-
const _parse_extra = result.parse_extra;
263-
const _query = {
264-
url: realUrl,
265-
script: _js ? base64Encode(_js) : undefined,
266-
}
267-
let _url = mergeQuery(cfg.default.drpyS.hipy_sniffer_url, _query);
268-
if (_parse_extra) {
269-
_url += _parse_extra;
270-
}
271-
try {
272-
let _result = await request(_url);
273-
console.log(`hipy嗅探器任务执行${_url} 完毕: ${_result.url}`);
274-
return {
275-
parse: 0,
276-
url: _result.url,
277-
header: _result.headers
255+
parses = shuffleArray(parses); // 随机打乱顺序
256+
let successCount = Number(cfg.default.drpyS.parse_count) || 6;
257+
let parse_timeout = Number(cfg.default.drpyS.parse_timeout) || 5000;
258+
console.log(`待并发的json解析数量: ${parses.length}`);
259+
let realUrls = [];
260+
const tasks = parses.map((jxObj, index) => {
261+
let task_id = jxObj.url + input;
262+
return {
263+
func: async function parseTask({jxObj, task_id}) {
264+
let json = await request(task_id, {timeout: parse_timeout}); // 解析5秒超时
265+
let _url = jsonpath.query(json, '$.url');
266+
if (Array.isArray(_url)) {
267+
_url = _url[0];
268+
}
269+
console.log('_url:', _url);
270+
if (!json.code || json.code === 200 || ![-1, 404, 403].includes(json.code)) {
271+
if (_url) {
272+
let lastIndex = _url.lastIndexOf('/');
273+
let lastLength = _url.slice(lastIndex + 1).length;
274+
// console.log('lastLength:', lastLength);
275+
if (lastLength > 10) {
276+
// console.log(`code:${json.code} , url:${json.url}`);
277+
return {...json, ...{name: jxObj.name}}
278+
}
279+
}
280+
throw new Error(`${jxObj.name} 解析 ${input} 失败: ${JSON.stringify(json)}`);
281+
} else {
282+
throw new Error(`${jxObj.name} 解析 ${input} 失败`);
278283
}
279-
} catch (e) {
280-
console.log(`hipy嗅探器嗅探错误: ${e.message}`);
284+
},
285+
param: {jxObj, task_id},
286+
id: task_id
287+
}
288+
});
289+
const listener = {
290+
func: (param, id, error, result) => {
291+
if (error) {
292+
console.error(`Task ${id} failed with error: ${error.message}`);
293+
} else if (result) {
294+
// log(`Task ${id} succeeded with result: `, result);
295+
realUrls.push({original: id, ...result});
281296
}
297+
// 中断逻辑示例
298+
if (param.stopOnFirst && result && result.url) {
299+
return 'break';
300+
}
301+
},
302+
param: {stopOnFirst: false},
303+
}
304+
await batchExecute(tasks, listener, successCount, 16);
305+
// console.log(realUrls);
306+
const playUrls = [];
307+
realUrls.forEach((item) => {
308+
playUrls.push(item.name, item.url);
309+
});
310+
return {
311+
parse: 0,
312+
url: playUrls,
313+
// header: headers
314+
}
282315

283-
} else {
284-
const sniffer = await _inReq.server.messageToDart({
285-
action: 'sniff',
286-
opt: {
287-
url: realUrl,
288-
timeout: 10000,
289-
rule: sniffer_rule,
290-
},
291-
});
292-
if (sniffer && sniffer.url) {
293-
const hds = {};
294-
if (sniffer.headers) {
295-
if (sniffer.headers['user-agent']) {
296-
hds['User-Agent'] = sniffer.headers['user-agent'];
297-
}
298-
if (sniffer.headers['referer']) {
299-
hds['Referer'] = sniffer.headers['referer'];
316+
} else if (result && result.parse && result.url) {
317+
const input = result.url;
318+
if (input && input.startsWith('http')) { // lazy返回结果是url http开头才走嗅探和免嗅逻辑
319+
const sniffer_rule = cfg.default.drpyS.sniffer_rule || 'http((?!http).){12,}?\\.m3u8(?!\\?)';
320+
const regex = new RegExp(sniffer_rule);
321+
if (regex.test(input)) {
322+
result.parse = 0;
323+
} else if (!regex.test(input)) {
324+
if (cfg.default.drpyS.enable_hipy_sniffer && cfg.default.drpyS.hipy_sniffer_url) {
325+
const _js = result.js;
326+
const _parse_extra = result.parse_extra;
327+
const _query = {
328+
url: input,
329+
script: _js ? base64Encode(_js) : undefined,
330+
}
331+
let _url = mergeQuery(cfg.default.drpyS.hipy_sniffer_url, _query);
332+
if (_parse_extra) {
333+
_url += _parse_extra;
334+
}
335+
try {
336+
let _result = await request(_url);
337+
console.log(`hipy嗅探器任务执行${_url} 完毕: ${_result.url}`);
338+
return {
339+
parse: 0,
340+
url: _result.url,
341+
header: _result.headers
300342
}
301-
if (sniffer.headers['cookie']) {
302-
hds['Cookie'] = sniffer.headers['cookie'];
343+
} catch (e) {
344+
console.log(`hipy嗅探器嗅探错误: ${e.message}`);
345+
}
346+
347+
} else {
348+
const sniffer = await _inReq.server.messageToDart({
349+
action: 'sniff',
350+
opt: {
351+
url: input,
352+
timeout: 10000,
353+
rule: sniffer_rule,
354+
},
355+
});
356+
if (sniffer && sniffer.url) {
357+
const hds = {};
358+
if (sniffer.headers) {
359+
if (sniffer.headers['user-agent']) {
360+
hds['User-Agent'] = sniffer.headers['user-agent'];
361+
}
362+
if (sniffer.headers['referer']) {
363+
hds['Referer'] = sniffer.headers['referer'];
364+
}
365+
if (sniffer.headers['cookie']) {
366+
hds['Cookie'] = sniffer.headers['cookie'];
367+
}
303368
}
369+
return {
370+
parse: 0,
371+
url: sniffer.url,
372+
header: hds,
373+
};
304374
}
305-
return {
306-
parse: 0,
307-
url: sniffer.url,
308-
header: hds,
309-
};
310375
}
311376
}
312377
}

‎nodejs/src/util/batchExecute.js‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
import fastq from 'fastq';
2+
3+
/**
4+
* Batch execution function with concurrency control and early termination.
5+
* @param {Array} tasks - Array of task objects, each containing func, param, and id.
6+
* @param {Object} listener - Progress listener object containing func and param.
7+
* @param {number} [successCount] - Number of successful tasks to wait for before stopping.
8+
* @param {number} max_task - Maximum number of concurrent tasks.
9+
* @returns {Promise<Array>} - Resolves with the successful results when the required tasks are complete.
10+
*/
11+
async function batchExecute(tasks, listener, successCount, max_task = 0) {
12+
const maxConcurrency = Number(max_task) || Number(process.env.MAX_TASK) || 16; // Default concurrency
13+
// console.log(`batchExecute with max_task: ${maxConcurrency}`);
14+
15+
let completedSuccess = 0;
16+
let stopExecution = false;
17+
const successfulResults = []; // To store successful results
18+
19+
const queue = fastq.promise(async (task) => {
20+
if (stopExecution) return; // Skip processing if execution has stopped
21+
22+
const {func, param, id} = task;
23+
try {
24+
// Check for stop condition at the start of each task
25+
if (stopExecution) return;
26+
27+
const result = await func({...param, stopExecution: () => stopExecution});
28+
if (stopExecution) return; // Check again after task execution
29+
30+
// if (result && result.url) { // Success condition
31+
successfulResults.push(result);
32+
completedSuccess++;
33+
// }
34+
35+
if (listener && typeof listener.func === 'function') {
36+
const listenerResult = listener.func(listener.param, id, null, result);
37+
if (listenerResult === 'break') {
38+
stopExecution = true;
39+
}
40+
}
41+
42+
if (successCount && completedSuccess >= successCount) {
43+
stopExecution = true;
44+
}
45+
} catch (error) {
46+
if (listener && typeof listener.func === 'function') {
47+
listener.func(listener.param, id, error, null);
48+
}
49+
}
50+
}, maxConcurrency);
51+
52+
// Enqueue tasks with a stop check
53+
tasks.forEach((task) => {
54+
queue.push(task).catch((err) => {
55+
console.error(`Task queue error for task ${task.id}:`, err);
56+
});
57+
});
58+
59+
// Monitor the queue and clear it on stopExecution
60+
const stopMonitor = new Promise((resolve) => {
61+
const interval = setInterval(() => {
62+
if (stopExecution) {
63+
queue.kill(); // Clear all pending tasks
64+
clearInterval(interval);
65+
resolve();
66+
}
67+
}, 50); // Check every 50ms
68+
});
69+
70+
// Wait for either stopExecution or all tasks to finish
71+
await Promise.race([queue.drained(), stopMonitor]);
72+
73+
console.log(`batchExecute completed with max_task: ${maxConcurrency} and ${completedSuccess} successful tasks.`);
74+
return successfulResults;
75+
}
76+
77+
export default batchExecute;

‎nodejs/src/util/tool.js‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,11 @@
11
import net from 'net';
2+
import {JSONPath} from 'jsonpath-plus';
3+
4+
export const jsonpath = {
5+
query(jsonObject, path) {
6+
return JSONPath({path: path, json: jsonObject})
7+
}
8+
};
29

310
// 检查端口是否被占用
411
function checkPort(port) {
@@ -34,6 +41,29 @@ export function extractTags(filename) {
3441
return [...new Set(tags.flat())];
3542
}
3643

44+
export function getRandomFromList(list) {
45+
// 将列表转换为数组
46+
const array = Array.isArray(list) ? list : Array.from(list);
47+
// 获取随机索引
48+
const randomIndex = Math.floor(Math.random() * array.length);
49+
// 返回随机选取的元素
50+
return array[randomIndex];
51+
}
52+
53+
/**
54+
* 对数组进行随机乱序（Fisher-Yates 洗牌算法）
55+
* @param {Array} array - 需要乱序的数组
56+
* @returns {Array} - 返回乱序后的新数组
57+
*/
58+
export function shuffleArray(array) {
59+
const result = [...array]; // 创建数组副本，避免修改原数组
60+
for (let i = result.length - 1; i > 0; i--) {
61+
const randomIndex = Math.floor(Math.random() * (i + 1)); // 随机索引
62+
[result[i], result[randomIndex]] = [result[randomIndex], result[i]]; // 交换元素
63+
}
64+
return result;
65+
}
66+
3767
// 示例
3868
// console.log(extractTags('1[画]')); // 输出 ['画']
3969
// console.log(extractTags('xxxx[书密]')); // 输出 ['书', '密']

0 commit comments

Comments
 (0)