🎈 pref:图片下载加入Aria2多线程下载 & 重构部分代码

This commit is contained in:
zhiyu1998 2024-08-15 19:49:05 +08:00
parent fdfc4a0a75
commit 8af1e3e345
3 changed files with 191 additions and 77 deletions

View File

@ -376,7 +376,7 @@ export class tools extends plugin {
// e.reply(segment.image(i.url_list[0])); // e.reply(segment.image(i.url_list[0]));
} }
// console.log(no_watermark_image_list) // console.log(no_watermark_image_list)
await this.reply(await Bot.makeForwardMsg(no_watermark_image_list)); await e.reply(await Bot.makeForwardMsg(no_watermark_image_list));
} }
// 如果开启评论的就调用 // 如果开启评论的就调用
await this.douyinComment(e, douId, headers); await this.douyinComment(e, douId, headers);
@ -740,7 +740,7 @@ export class tools extends plugin {
user_id: e.user_id, user_id: e.user_id,
}); });
}); });
await this.reply(await Bot.makeForwardMsg(dynamicSrcMsg)); await e.reply(await Bot.makeForwardMsg(dynamicSrcMsg));
} else { } else {
e.reply(`识别:哔哩哔哩动态, 但是失败!`); e.reply(`识别:哔哩哔哩动态, 但是失败!`);
} }
@ -929,12 +929,17 @@ export class tools extends plugin {
} else { } else {
// 非海外使用🪜下载 // 非海外使用🪜下载
const localPath = this.getCurDownloadPath(e); const localPath = this.getCurDownloadPath(e);
downloadImg(url, localPath, "", !isOversea, {}, { const xImgPath = await downloadImg({
proxyAddr: this.proxyAddr, img: url,
proxyPort: this.proxyPort dir: localPath,
}).then(async _ => { isProxy: !isOversea,
e.reply(segment.image(fs.readFileSync(localPath + "/" + url.split("/").pop()))); proxyInfo: {
}); proxyAddr: this.proxyAddr,
proxyPort: this.proxyPort
},
numThread: this.videoDownloadConcurrency,
})
e.reply(segment.image(xImgPath));
} }
} else { } else {
this.downloadVideo(url, !isOversea).then(path => { this.downloadVideo(url, !isOversea).then(path => {
@ -1015,7 +1020,6 @@ export class tools extends plugin {
const resJson = JSON.parse(res); const resJson = JSON.parse(res);
const noteData = resJson.note.noteDetailMap[id].note; const noteData = resJson.note.noteDetailMap[id].note;
const { title, desc, type } = noteData; const { title, desc, type } = noteData;
let imgPromise = [];
if (type === "video") { if (type === "video") {
// 封面 // 封面
const cover = noteData.imageList?.[0].urlDefault; const cover = noteData.imageList?.[0].urlDefault;
@ -1036,27 +1040,34 @@ export class tools extends plugin {
return true; return true;
} else if (type === "normal") { } else if (type === "normal") {
e.reply(`识别:小红书, ${ title }\n${ desc }`); e.reply(`识别:小红书, ${ title }\n${ desc }`);
noteData.imageList.map(async (item, index) => { const imagePromises = [];
imgPromise.push(downloadImg(item.urlDefault, downloadPath, index.toString())); // 使用 for..of 循环处理异步下载操作
}); for (let [index, item] of noteData.imageList.entries()) {
} imagePromises.push(downloadImg({
const paths = await Promise.all(imgPromise); img: item.urlDefault,
const imagesData = await Promise.all( dir: downloadPath,
paths.map(async item => { fileName: `${index}.png`,
const fileContent = await fs.promises.readFile(item); numThread: this.videoDownloadConcurrency,
}));
}
// 等待所有图片下载完成
const paths = await Promise.all(imagePromises);
// 直接构造 imagesData 数组
const imagesData = await Promise.all(paths.map(async (item) => {
return { return {
message: segment.image(fileContent), message: segment.image(await fs.promises.readFile(item)),
nickname: e.sender.card || e.user_id, nickname: e.sender.card || e.user_id,
user_id: e.user_id, user_id: e.user_id,
}; };
}), }));
);
// Reply with forward message // 回复带有转发消息的图片数据
e.reply(await Bot.makeForwardMsg(imagesData)); e.reply(await Bot.makeForwardMsg(imagesData));
// Clean up files // 批量删除下载的文件
await Promise.all(paths.map(item => fs.promises.unlink(item))); await Promise.all(paths.map(item => fs.promises.rm(item, { force: true })));
}
}); });
return true; return true;
} }
@ -1244,33 +1255,37 @@ export class tools extends plugin {
.then(async resp => { .then(async resp => {
const wbData = resp.data.data; const wbData = resp.data.data;
const { text, status_title, source, region_name, pics, page_info } = wbData; const { text, status_title, source, region_name, pics, page_info } = wbData;
e.reply(`识别:微博,${ text.replace(/<[^>]+>/g, '') }\n${ status_title }\n${ source }\t${ region_name }`); e.reply(`识别:微博,${ text.replace(/<[^>]+>/g, '') }\n${ status_title }\n${ source }\t${ region_name ?? '' }`);
if (pics) { if (pics) {
const removePath = []; // 下载图片并格式化消息
// 图片
const imagesPromise = pics.map(item => { const imagesPromise = pics.map(item => {
// 下载 return downloadImg({
return downloadImg(item?.large.url || item.url, this.getCurDownloadPath(e), "", false, { img: item?.large.url || item.url,
"Referer": "http://blog.sina.com.cn/", dir: this.getCurDownloadPath(e),
}); headersExt: {
}) "Referer": "http://blog.sina.com.cn/",
const images = await Promise.all(imagesPromise).then(paths => { },
return paths.map(item => { numThread: this.videoDownloadConcurrency,
// 记录删除的路径 }).then(async (filePath) => {
removePath.push(item); // 格式化为消息对象
// 格式化发送图片
return { return {
message: segment.image(fs.readFileSync(item)), message: segment.image(await fs.promises.readFile(filePath)),
nickname: e.sender.card || e.user_id, nickname: e.sender.card || e.user_id,
user_id: e.user_id, user_id: e.user_id,
} // 返回路径以便后续删除
}) filePath
}) };
});
});
// 等待所有图片处理完
const images = await Promise.all(imagesPromise);
// 回复合并的消息
await e.reply(await Bot.makeForwardMsg(images)); await e.reply(await Bot.makeForwardMsg(images));
// 发送完就删除
removePath.forEach(async item => { // 并行删除文件
checkAndRemoveFile(item); await Promise.all(images.map(({ filePath }) => checkAndRemoveFile(filePath)));
})
} }
if (page_info) { if (page_info) {
// 视频 // 视频

View File

@ -6,7 +6,8 @@ import schedule from "node-schedule";
import fs from "node:fs"; import fs from "node:fs";
import os from "os"; import os from "os";
import common from "../../../lib/common/common.js"; import common from "../../../lib/common/common.js";
import { TEN_THOUSAND } from "../constants/constant.js"; import path from 'path';
import { COMMON_USER_AGENT, TEN_THOUSAND } from "../constants/constant.js";
import { mkdirIfNotExists } from "./file.js"; import { mkdirIfNotExists } from "./file.js";
/** /**
@ -127,21 +128,21 @@ export function generateRandomStr(randomlength = 16) {
/** /**
* 下载mp3 * 下载mp3
* @param mp3Url MP3地址 * @param mp3Url MP3地址
* @param path 下载目录 * @param filePath 下载目录
* @param title 音乐名 * @param title 音乐名
* @param redirect 是否要重定向 * @param redirect 是否要重定向
* @param audioType 建议填写 mp3 / m4a / flac 类型 * @param audioType 建议填写 mp3 / m4a / flac 类型
* @returns {Promise<unknown>} * @returns {Promise<unknown>}
*/ */
export async function downloadAudio(mp3Url, path, title = "temp", redirect = "manual", audioType = "mp3") { export async function downloadAudio(mp3Url, filePath, title = "temp", redirect = "manual", audioType = "mp3") {
// 如果没有目录就创建一个 // 如果没有目录就创建一个
await mkdirIfNotExists(path) await mkdirIfNotExists(filePath)
// 补充保存文件名 // 补充保存文件名
path += `/${ title }.${audioType}`; filePath += `/${ title }.${ audioType }`;
if (fs.existsSync(path)) { if (fs.existsSync(filePath)) {
console.log(`音频已存在`); console.log(`音频已存在`);
fs.unlinkSync(path); fs.unlinkSync(filePath);
} }
// 发起请求 // 发起请求
@ -155,7 +156,7 @@ export async function downloadAudio(mp3Url, path, title = "temp", redirect = "ma
}); });
if (!response.ok) { if (!response.ok) {
throw new Error(`Failed to fetch ${response.statusText}`); throw new Error(`Failed to fetch ${ response.statusText }`);
} }
try { try {
@ -169,32 +170,78 @@ export async function downloadAudio(mp3Url, path, title = "temp", redirect = "ma
}); });
// 开始下载 // 开始下载
const writer = fs.createWriteStream(path); const writer = fs.createWriteStream(filePath);
response.data.pipe(writer); response.data.pipe(writer);
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
writer.on('finish', () => resolve(path)); writer.on('finish', () => resolve(filePath));
writer.on('error', reject); writer.on('error', reject);
}); });
} catch (error) { } catch (error) {
console.error(`下载音乐失败,错误信息为: ${error.message}`); console.error(`下载音乐失败,错误信息为: ${ error.message }`);
throw error; throw error;
} }
} }
/** /**
* 下载一张网络图片(自动以url的最后一个为名字) * 下载图片网关
* @param {string} img * @param {Object} options 参数对象
* @param {string} dir * @param {string} options.img 图片的URL
* @param {string} fileName * @param {string} options.dir 保存图片的目录
* @param {boolean} isProxy * @param {string} [options.fileName] 自定义文件名 (可选)
* @param {Object} headersExt * @param {boolean} [options.isProxy] 是否使用代理 (可选)
* @param {Object} proxyInfo 参数proxyAddr=地址proxyPort=端口 * @param {Object} [options.headersExt] 自定义请求头 (可选)
* @returns {Promise<unknown>} * @param {Object} [options.proxyInfo] 代理信息 (可选)
* @returns {Promise<string>}
*/ */
export async function downloadImg(img, dir, fileName = "", isProxy = false, headersExt = {}, proxyInfo = {}) { export async function downloadImg({
img,
dir,
fileName = "",
isProxy = false,
headersExt = {},
proxyInfo = {},
numThread = 1,
}) {
const downloadImgParams = {
img,
dir,
fileName,
isProxy,
headersExt,
proxyInfo,
numThread,
}
logger.info(logger.yellow(`[R插件][图片下载] 当前使用线程数:${ numThread }`));
if (numThread === 1) {
return normalDownloadImg(downloadImgParams);
} else if (numThread > 1) {
return downloadImgWithAria2(downloadImgParams);
}
}
/**
* 正常下载图片
* @param {Object} options 参数对象
* @param {string} options.img 图片的URL
* @param {string} options.dir 保存图片的目录
* @param {string} [options.fileName] 自定义文件名 (可选)
* @param {boolean} [options.isProxy] 是否使用代理 (可选)
* @param {Object} [options.headersExt] 自定义请求头 (可选)
* @param {Object} [options.proxyInfo] 代理信息 (可选)
* @returns {Promise<string>}
*/
async function normalDownloadImg({
img,
dir,
fileName = "",
isProxy = false,
headersExt = {},
proxyInfo = {}
}) {
if (fileName === "") { if (fileName === "") {
fileName = img.split("/").pop(); fileName = img.split("/").pop();
} }
@ -203,8 +250,7 @@ export async function downloadImg(img, dir, fileName = "", isProxy = false, head
const writer = fs.createWriteStream(filepath); const writer = fs.createWriteStream(filepath);
const axiosConfig = { const axiosConfig = {
headers: { headers: {
"User-Agent": "User-Agent": COMMON_USER_AGENT,
"Mozilla/5.0 (Linux; Android 5.0; SM-G900P Build/LRX21T) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.25 Mobile Safari/537.36",
...headersExt ...headersExt
}, },
responseType: "stream", responseType: "stream",
@ -233,10 +279,64 @@ export async function downloadImg(img, dir, fileName = "", isProxy = false, head
}); });
}); });
} catch (err) { } catch (err) {
logger.error(`图片下载失败, 原因:${err}`); logger.error(`图片下载失败, 原因:${ err }`);
} }
} }
/**
* 下载一张网络图片(使用aria2加速下载)
* @param {Object} options 参数对象
* @param {string} options.img 图片的URL
* @param {string} options.dir 保存图片的目录
* @param {string} [options.fileName] 自定义文件名 (可选)
* @param {boolean} [options.isProxy] 是否使用代理 (可选)
* @param {Object} [options.headersExt] 自定义请求头 (可选)
* @param {Object} [options.proxyInfo] 代理信息 (可选)
* @returns {Promise<unknown>}
*/
async function downloadImgWithAria2({
img,
dir,
fileName = "",
isProxy = false,
headersExt = {},
proxyInfo = {},
numThread = 1,
}) {
if (fileName === "") {
fileName = img.split("/").pop();
}
const filepath = path.resolve(dir, fileName);
await mkdirIfNotExists(dir);
// 构建 aria2c 命令
let aria2cCmd = `aria2c "${ img }" --dir="${ dir }" --out="${ fileName }" --max-connection-per-server=${numThread} --split=${numThread} --min-split-size=1M --continue`;
// 如果需要代理
if (isProxy) {
aria2cCmd += ` --all-proxy="http://${ proxyInfo.proxyAddr }:${ proxyInfo.proxyPort }"`;
}
// 添加自定义headers
if (headersExt && Object.keys(headersExt).length > 0) {
for (const [headerName, headerValue] of Object.entries(headersExt)) {
aria2cCmd += ` --header="${ headerName }: ${ headerValue }"`;
}
}
return new Promise((resolve, reject) => {
exec(aria2cCmd, (error, stdout, stderr) => {
if (error) {
logger.error(`图片下载失败, 原因:${ error.message }`);
reject(error);
return;
}
resolve(filepath);
});
});
}
/** /**
* 千位数的数据处理 * 千位数的数据处理
* @param data * @param data
@ -325,9 +425,9 @@ export function truncateString(inputString, maxLength = 50) {
* 测试当前是否存在🪜 * 测试当前是否存在🪜
* @returns {Promise<Boolean>} * @returns {Promise<Boolean>}
*/ */
export async function testProxy(host='127.0.0.1', port=7890) { export async function testProxy(host = '127.0.0.1', port = 7890) {
// 创建一个代理隧道 // 创建一个代理隧道
const httpsAgent = new HttpsProxyAgent(`http://${host}:${port}`); const httpsAgent = new HttpsProxyAgent(`http://${ host }:${ port }`);
try { try {
// 通过代理服务器发起请求 // 通过代理服务器发起请求
@ -343,7 +443,7 @@ export async function testProxy(host='127.0.0.1', port=7890) {
export function formatSeconds(seconds) { export function formatSeconds(seconds) {
const minutes = Math.floor(seconds / 60); const minutes = Math.floor(seconds / 60);
const remainingSeconds = seconds % 60; const remainingSeconds = seconds % 60;
return `${minutes}${remainingSeconds}`; return `${ minutes }${ remainingSeconds }`;
} }
/** /**
@ -362,7 +462,7 @@ export async function retryAxiosReq(requestFunction, retries = 3, delay = 1000)
return response.data; return response.data;
} catch (error) { } catch (error) {
if (retries > 0) { if (retries > 0) {
logger.mark(`[R插件][重试模块]重试中... (${3 - retries + 1}/3) 次`); logger.mark(`[R插件][重试模块]重试中... (${ 3 - retries + 1 }/3) 次`);
await new Promise(resolve => setTimeout(resolve, delay)); await new Promise(resolve => setTimeout(resolve, delay));
return retryAxiosReq(requestFunction, retries - 1, delay); return retryAxiosReq(requestFunction, retries - 1, delay);
} else { } else {
@ -406,7 +506,7 @@ export function estimateReadingTime(text, wpm = 200) {
*/ */
export function checkCommandExists(command) { export function checkCommandExists(command) {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
exec(`which ${command}`, (error, stdout, stderr) => { exec(`which ${ command }`, (error, stdout, stderr) => {
if (error) { if (error) {
// Command not found // Command not found
resolve(false); resolve(false);
@ -465,15 +565,15 @@ export function cleanFilename(filename) {
export function checkToolInCurEnv(someCommand) { export function checkToolInCurEnv(someCommand) {
// 根据操作系统选择命令 // 根据操作系统选择命令
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
const command = os.platform() === 'win32' ? `where ${someCommand}` : `which ${someCommand}`; const command = os.platform() === 'win32' ? `where ${ someCommand }` : `which ${ someCommand }`;
exec(command, (error, stdout, stderr) => { exec(command, (error, stdout, stderr) => {
if (error) { if (error) {
logger.error(`[R插件][checkTool]未找到${someCommand}: ${stderr || error.message}`); logger.error(`[R插件][checkTool]未找到${ someCommand }: ${ stderr || error.message }`);
resolve(false); resolve(false);
return; return;
} }
logger.info(`[R插件][checkTool]找到${someCommand}: ${stdout.trim()}`); logger.info(`[R插件][checkTool]找到${ someCommand }: ${ stdout.trim() }`);
resolve(true); resolve(true);
}); });
}); });

View File

@ -10,7 +10,6 @@ export async function checkAndRemoveFile(file) {
try { try {
await fs.promises.access(file); await fs.promises.access(file);
await fs.promises.unlink(file); await fs.promises.unlink(file);
logger.mark('文件已存在');
} catch (err) { } catch (err) {
if (err.code !== 'ENOENT') { if (err.code !== 'ENOENT') {
throw err; throw err;