From 26d4e22cf30ec8d01838064a2ef9f3257d6274ed Mon Sep 17 00:00:00 2001
From: zhiyu <542716863@qq.com>
Date: Wed, 31 Jan 2024 14:39:33 +0800
Subject: [PATCH] =?UTF-8?q?=E2=9C=A8=20feat:=20=E6=9B=B4=E6=8D=A2=E6=B0=B4?=
=?UTF-8?q?=E5=8D=B0=E7=9A=84xhs=E8=A7=A3=E6=9E=90=E4=B8=BA=E6=97=A0?=
=?UTF-8?q?=E6=B0=B4=E5=8D=B0=E7=9A=84xhs=E8=A7=A3=E6=9E=90?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
1. 删除有水印的xhs解析中的CK冗余请求
2. 新增更换有水印的xhs解析为无水印的xhs解析
---
apps/tools.js | 67 +++++++++++++++++++++++--------------------
config/version.yaml | 3 +-
constants/constant.js | 21 +++++++++++++-
constants/xhs.js | 5 ++++
4 files changed, 63 insertions(+), 33 deletions(-)
create mode 100644 constants/xhs.js
diff --git a/apps/tools.js b/apps/tools.js
index 28f6386..fb67227 100644
--- a/apps/tools.js
+++ b/apps/tools.js
@@ -5,15 +5,14 @@ import fs from "node:fs";
import axios from "axios";
import _ from "lodash";
import tunnel from "tunnel";
-import HttpProxyAgent, { HttpsProxyAgent} from "https-proxy-agent";
+import HttpProxyAgent, { HttpsProxyAgent } from "https-proxy-agent";
import { mkdirIfNotExists, checkAndRemoveFile, deleteFolderRecursive } from "../utils/file.js";
import { downloadBFile, getAudioUrl, getDownloadUrl, mergeFileToMp4 } from "../utils/bilibili.js";
import { parseUrl, parseM3u8, downloadM3u8Videos, mergeAcFileToMp4 } from "../utils/acfun.js";
import {
transMap,
douyinTypeMap,
- XHS_CK,
- RESTRICTION_DESCRIPTION,
+ RESTRICTION_DESCRIPTION, XHS_NO_WATERMARK_HEADER,
} from "../constants/constant.js";
import { formatBiliInfo, getIdVideo, secondsToTime } from "../utils/common.js";
import config from "../model/index.js";
@@ -26,6 +25,7 @@ import querystring from "querystring";
import TokenBucket from "../utils/token-bucket.js";
import { getWbi } from "../utils/biliWbi.js";
import { BILI_SUMMARY } from "../constants/bili.js";
+import { XHS_VIDEO } from "../constants/xhs.js";
export class tools extends plugin {
constructor() {
@@ -65,7 +65,7 @@ export class tools extends plugin {
},
{
reg: "(xhslink.com|xiaohongshu.com)",
- fnc: "redbook",
+ fnc: "xhs",
},
{
reg: "(instagram.com)",
@@ -246,17 +246,17 @@ export class tools extends plugin {
// API链接
const API_URL = `https://api16-normal-c-useast1a.tiktokv.com/aweme/v1/feed/?aweme_id=${ tiktokVideoId }`;
await fetch(API_URL, {
- headers: {
- "User-Agent":
- "Mozilla/5.0 (Linux; Android 5.0; SM-G900P Build/LRX21T) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.25 Mobile Safari/537.36",
- "Content-Type": "application/json",
- "Accept-Encoding": "gzip,deflate,compress",
- },
- // redirect: "follow",
- follow: 10,
- timeout: 10000,
- agent: new HttpsProxyAgent(this.myProxy),
- })
+ headers: {
+ "User-Agent":
+ "Mozilla/5.0 (Linux; Android 5.0; SM-G900P Build/LRX21T) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.25 Mobile Safari/537.36",
+ "Content-Type": "application/json",
+ "Accept-Encoding": "gzip,deflate,compress",
+ },
+ // redirect: "follow",
+ follow: 10,
+ timeout: 10000,
+ agent: new HttpsProxyAgent(this.myProxy),
+ })
.then(async resp => {
const respJson = await resp.json();
const data = respJson.aweme_list[0];
@@ -593,7 +593,7 @@ export class tools extends plugin {
}
// 小红书解析
- async redbook(e) {
+ async xhs(e) {
// 正则说明：匹配手机链接、匹配小程序、匹配PC链接
let msgUrl =
/(http:|https:)\/\/(xhslink|xiaohongshu).com\/[A-Za-z\d._?%&+\-=\/#@]*/.exec(
@@ -622,30 +622,35 @@ export class tools extends plugin {
}
const downloadPath = `${ this.defaultPath }${ this.e.group_id || this.e.user_id }`;
// 获取信息
- fetch(`https://www.xiaohongshu.com/discovery/item/${ id }`, {
- headers: {
- "user-agent":
- "Mozilla/5.0 (iPhone; CPU iPhone OS 13_2_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.3 Mobile/15E148 Safari/604.1 Edg/110.0.0.0",
- cookie: Buffer.from(XHS_CK, "base64").toString("utf-8"),
- },
+ fetch(`https://www.xiaohongshu.com/explore/${ id }`, {
+ headers: XHS_NO_WATERMARK_HEADER,
}).then(async resp => {
const xhsHtml = await resp.text();
- const reg = /window.__INITIAL_STATE__=(.*?)<\/script>/;
- const resJson = xhsHtml.match(reg)[0];
- const res = JSON.parse(resJson.match(reg)[1]);
- const noteData = res.noteData.data.noteData;
+ const reg = /window\.__INITIAL_STATE__=(.*?)<\/script>/;
+ const res = xhsHtml.match(reg)[1].replace(/undefined/g, "null");
+ const resJson = JSON.parse(res);
+ const noteData = resJson.note.noteDetailMap[id].note
const { title, desc, type } = noteData;
- e.reply(`识别：小红书, ${ title }\n${ desc }`);
let imgPromise = [];
if (type === "video") {
- const url = noteData.video.url;
- this.downloadVideo(url).then(path => {
+ // 封面
+ const cover = noteData.imageList?.[0].urlDefault;
+ e.reply([segment.image(cover), `识别：小红书, ${ title }\n${ desc }`]);
+ // 构造xhs视频链接
+ const xhsVideoUrl = `${ XHS_VIDEO }${ noteData.video.consumer.originVideoKey.replace('pre_post\/', '') }`;
+ // 下载视频
+ this.downloadVideo(xhsVideoUrl).then(path => {
+ if (path === undefined) {
+ // 创建文件，如果不存在
+ path = `${ this.defaultPath }${ this.e.group_id || this.e.user_id }/`;
+ }
e.reply(segment.video(path + "/temp.mp4"));
});
return true;
} else if (type === "normal") {
+ e.reply(`识别：小红书, ${ title }\n${ desc }`);
noteData.imageList.map(async (item, index) => {
- imgPromise.push(this.downloadImg(item.url, downloadPath, index.toString()));
+ imgPromise.push(this.downloadImg(item.urlDefault, downloadPath, index.toString()));
});
}
const paths = await Promise.all(imgPromise);
@@ -1045,7 +1050,7 @@ export class tools extends plugin {
writer.on("error", reject);
});
} catch (err) {
- logger.error("下载视频发生错误！");
+ logger.error(`下载视频发生错误！\ninfo:${err}`);
}
}
diff --git a/config/version.yaml b/config/version.yaml
index a1a693d..ad77098 100644
--- a/config/version.yaml
+++ b/config/version.yaml
@@ -1,7 +1,8 @@
- {
- version: 1.2.2,
+ version: 1.2.3,
data:
[
+ 新增小红书无水印下载功能,
新增哔哩哔哩官方AI总结功能,
新增哔哩哔哩音乐提取功能,
新增快手解析功能,
diff --git a/constants/constant.js b/constants/constant.js
index 88bde22..94b85b3 100644
--- a/constants/constant.js
+++ b/constants/constant.js
@@ -38,6 +38,25 @@ export const TEN_THOUSAND = 10000;
export const CAT_LIMIT = 10;
-export const XHS_CK = 'eGhzVHJhY2tlcklkPTczODhhYmY2LTI0MDgtNGU5YS04MTUyLTE0MGVhOGY1MTQ5ZjsgeGhzVHJhY2tlcklkLnNpZz1UcGUxTkNaX3B3UkFYdG01SVJmVEs0SWUxM0xBaGZuNmNZU2N4Vi1JYWxFOyBhMT0xODY2ZDkwMDM0NmI2NmppcjMzcGpxZ2MwM3JvcG1mczAydXMxdWNoeDEwMDAwMTM1MDUzOyB3ZWJJZD1mMTNkOGJkYjhiZGM3ZGE0MzY0NjA4NWJjYzQ1MDQ1YTsgZ2lkPXlZS0tmajg4SzA4MnlZS0tmajg4cUo3UzRLREtLVjNGcXFVVjd4Q0FrUzhxRk15OGxVNmlNeTg4OHlxMjgycThmMlk0UzAySjsgZ2lkLnNpZ249YlpzcFFzSUxEUmN5akZLQmN2L1FMWVhkU3lvPTsgd2ViX3Nlc3Npb249MDMwMDM3YTRjMDQyYjE1ZTVjMTg4OTUwOGIyNDRhZDExM2UwNTM7IHhoc1RyYWNrZXI9dXJsPW5vdGVEZXRhaWwmeGhzc2hhcmU9V2VpeGluU2Vzc2lvbjsgeGhzVHJhY2tlci5zaWc9YzdmcDVRclk2SGNvVERhUzluX2N3Z2RCRHh2MFZmWnpSU1NTcnlzbG5lQTsgZXh0cmFfZXhwX2lkcz1oNV8yMzAyMDExX29yaWdpbixoNV8xMjA4X2NsdCxoNV8xMTMwX2NsdCxpb3Nfd3hfbGF1bmNoX29wZW5fYXBwX2V4cCxoNV92aWRlb191aV9leHAzLHd4X2xhdW5jaF9vcGVuX2FwcF9kdXJhdGlvbl9vcmlnaW4scXVlc19jbHQyOyBleHRyYV9leHBfaWRzLnNpZz1DVUdrR3NYT3lBZmpVSXkyVGo3SjN4YmRNakFfSnpoR1JkYWd6cVlkbmJnOyB3ZWJCdWlsZD0xLjEuMjE7IHhzZWNhcHBpZD14aHMtcGMtd2ViOyB3ZWJzZWN0aWdhPTU5ZDNlZjFlNjBjNGFhMzdhN2RmM2MyMzQ2N2JkNDZkN2YxZGEwYjE5MThjZjMzNWVlN2YyZTllNTJhYzA0Y2Y7IHNlY19wb2lzb25faWQ9MTI0OTE1NWQtOWU5ZS00MzkyLTg2NTgtNTA1Yzc0YTUzMTM1'
+/**
+ * 有水印的头请求
+ * @type {{cookie: string, "User-Agent": string, accept: string}}
+ */
+export const XHS_WATERMARK_HEADER = {
+ 'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
+ 'cookie': '',
+ 'User-Agent':
+ "Mozilla/5.0 (iPhone; CPU iPhone OS 13_2_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.3 Mobile/15E148 Safari/604.1 Edg/110.0.0.0",
+}
+
+/**
+ * 无水印的头请求
+ * @type {{cookie: string, "User-Agent": string, accept: string}}
+ */
+export const XHS_NO_WATERMARK_HEADER = {
+ 'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9 ',
+ 'cookie': '',
+ 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 UBrowser/6.2.4098.3 Safari/537.36',
+}
export const RESTRICTION_DESCRIPTION = "\n-----------------------限制说明-----------------------"
\ No newline at end of file
diff --git a/constants/xhs.js b/constants/xhs.js
new file mode 100644
index 0000000..5a305c0
--- /dev/null
+++ b/constants/xhs.js
@@ -0,0 +1,5 @@
+/**
+ * 视频请求链接CDN
+ * @type {string}
+ */
+export const XHS_VIDEO = "http://sns-video-bd.xhscdn.com/"
\ No newline at end of file