🎈 pref:图片下载加入Aria2多线程下载 & 重构部分代码

This commit is contained in:
zhiyu1998 2024-08-15 19:49:05 +08:00
parent fdfc4a0a75
commit 8af1e3e345
3 changed files with 191 additions and 77 deletions

View File

@ -376,7 +376,7 @@ export class tools extends plugin {
// e.reply(segment.image(i.url_list[0])); // e.reply(segment.image(i.url_list[0]));
} }
// console.log(no_watermark_image_list) // console.log(no_watermark_image_list)
await this.reply(await Bot.makeForwardMsg(no_watermark_image_list)); await e.reply(await Bot.makeForwardMsg(no_watermark_image_list));
} }
// 如果开启评论的就调用 // 如果开启评论的就调用
await this.douyinComment(e, douId, headers); await this.douyinComment(e, douId, headers);
@ -740,7 +740,7 @@ export class tools extends plugin {
user_id: e.user_id, user_id: e.user_id,
}); });
}); });
await this.reply(await Bot.makeForwardMsg(dynamicSrcMsg)); await e.reply(await Bot.makeForwardMsg(dynamicSrcMsg));
} else { } else {
e.reply(`识别:哔哩哔哩动态, 但是失败!`); e.reply(`识别:哔哩哔哩动态, 但是失败!`);
} }
@ -929,12 +929,17 @@ export class tools extends plugin {
} else { } else {
// 非海外使用🪜下载 // 非海外使用🪜下载
const localPath = this.getCurDownloadPath(e); const localPath = this.getCurDownloadPath(e);
downloadImg(url, localPath, "", !isOversea, {}, { const xImgPath = await downloadImg({
img: url,
dir: localPath,
isProxy: !isOversea,
proxyInfo: {
proxyAddr: this.proxyAddr, proxyAddr: this.proxyAddr,
proxyPort: this.proxyPort proxyPort: this.proxyPort
}).then(async _ => { },
e.reply(segment.image(fs.readFileSync(localPath + "/" + url.split("/").pop()))); numThread: this.videoDownloadConcurrency,
}); })
e.reply(segment.image(xImgPath));
} }
} else { } else {
this.downloadVideo(url, !isOversea).then(path => { this.downloadVideo(url, !isOversea).then(path => {
@ -1015,7 +1020,6 @@ export class tools extends plugin {
const resJson = JSON.parse(res); const resJson = JSON.parse(res);
const noteData = resJson.note.noteDetailMap[id].note; const noteData = resJson.note.noteDetailMap[id].note;
const { title, desc, type } = noteData; const { title, desc, type } = noteData;
let imgPromise = [];
if (type === "video") { if (type === "video") {
// 封面 // 封面
const cover = noteData.imageList?.[0].urlDefault; const cover = noteData.imageList?.[0].urlDefault;
@ -1036,27 +1040,34 @@ export class tools extends plugin {
return true; return true;
} else if (type === "normal") { } else if (type === "normal") {
e.reply(`识别:小红书, ${ title }\n${ desc }`); e.reply(`识别:小红书, ${ title }\n${ desc }`);
noteData.imageList.map(async (item, index) => { const imagePromises = [];
imgPromise.push(downloadImg(item.urlDefault, downloadPath, index.toString())); // 使用 for..of 循环处理异步下载操作
}); for (let [index, item] of noteData.imageList.entries()) {
imagePromises.push(downloadImg({
img: item.urlDefault,
dir: downloadPath,
fileName: `${index}.png`,
numThread: this.videoDownloadConcurrency,
}));
} }
const paths = await Promise.all(imgPromise); // 等待所有图片下载完成
const imagesData = await Promise.all( const paths = await Promise.all(imagePromises);
paths.map(async item => {
const fileContent = await fs.promises.readFile(item); // 直接构造 imagesData 数组
const imagesData = await Promise.all(paths.map(async (item) => {
return { return {
message: segment.image(fileContent), message: segment.image(await fs.promises.readFile(item)),
nickname: e.sender.card || e.user_id, nickname: e.sender.card || e.user_id,
user_id: e.user_id, user_id: e.user_id,
}; };
}), }));
);
// Reply with forward message // 回复带有转发消息的图片数据
e.reply(await Bot.makeForwardMsg(imagesData)); e.reply(await Bot.makeForwardMsg(imagesData));
// Clean up files // 批量删除下载的文件
await Promise.all(paths.map(item => fs.promises.unlink(item))); await Promise.all(paths.map(item => fs.promises.rm(item, { force: true })));
}
}); });
return true; return true;
} }
@ -1244,33 +1255,37 @@ export class tools extends plugin {
.then(async resp => { .then(async resp => {
const wbData = resp.data.data; const wbData = resp.data.data;
const { text, status_title, source, region_name, pics, page_info } = wbData; const { text, status_title, source, region_name, pics, page_info } = wbData;
e.reply(`识别:微博,${ text.replace(/<[^>]+>/g, '') }\n${ status_title }\n${ source }\t${ region_name }`); e.reply(`识别:微博,${ text.replace(/<[^>]+>/g, '') }\n${ status_title }\n${ source }\t${ region_name ?? '' }`);
if (pics) { if (pics) {
const removePath = []; // 下载图片并格式化消息
// 图片
const imagesPromise = pics.map(item => { const imagesPromise = pics.map(item => {
// 下载 return downloadImg({
return downloadImg(item?.large.url || item.url, this.getCurDownloadPath(e), "", false, { img: item?.large.url || item.url,
dir: this.getCurDownloadPath(e),
headersExt: {
"Referer": "http://blog.sina.com.cn/", "Referer": "http://blog.sina.com.cn/",
}); },
}) numThread: this.videoDownloadConcurrency,
const images = await Promise.all(imagesPromise).then(paths => { }).then(async (filePath) => {
return paths.map(item => { // 格式化为消息对象
// 记录删除的路径
removePath.push(item);
// 格式化发送图片
return { return {
message: segment.image(fs.readFileSync(item)), message: segment.image(await fs.promises.readFile(filePath)),
nickname: e.sender.card || e.user_id, nickname: e.sender.card || e.user_id,
user_id: e.user_id, user_id: e.user_id,
} // 返回路径以便后续删除
}) filePath
}) };
});
});
// 等待所有图片处理完
const images = await Promise.all(imagesPromise);
// 回复合并的消息
await e.reply(await Bot.makeForwardMsg(images)); await e.reply(await Bot.makeForwardMsg(images));
// 发送完就删除
removePath.forEach(async item => { // 并行删除文件
checkAndRemoveFile(item); await Promise.all(images.map(({ filePath }) => checkAndRemoveFile(filePath)));
})
} }
if (page_info) { if (page_info) {
// 视频 // 视频

View File

@ -6,7 +6,8 @@ import schedule from "node-schedule";
import fs from "node:fs"; import fs from "node:fs";
import os from "os"; import os from "os";
import common from "../../../lib/common/common.js"; import common from "../../../lib/common/common.js";
import { TEN_THOUSAND } from "../constants/constant.js"; import path from 'path';
import { COMMON_USER_AGENT, TEN_THOUSAND } from "../constants/constant.js";
import { mkdirIfNotExists } from "./file.js"; import { mkdirIfNotExists } from "./file.js";
/** /**
@ -127,21 +128,21 @@ export function generateRandomStr(randomlength = 16) {
/** /**
* 下载mp3 * 下载mp3
* @param mp3Url MP3地址 * @param mp3Url MP3地址
* @param path 下载目录 * @param filePath 下载目录
* @param title 音乐名 * @param title 音乐名
* @param redirect 是否要重定向 * @param redirect 是否要重定向
* @param audioType 建议填写 mp3 / m4a / flac 类型 * @param audioType 建议填写 mp3 / m4a / flac 类型
* @returns {Promise<unknown>} * @returns {Promise<unknown>}
*/ */
export async function downloadAudio(mp3Url, path, title = "temp", redirect = "manual", audioType = "mp3") { export async function downloadAudio(mp3Url, filePath, title = "temp", redirect = "manual", audioType = "mp3") {
// 如果没有目录就创建一个 // 如果没有目录就创建一个
await mkdirIfNotExists(path) await mkdirIfNotExists(filePath)
// 补充保存文件名 // 补充保存文件名
path += `/${ title }.${audioType}`; filePath += `/${ title }.${ audioType }`;
if (fs.existsSync(path)) { if (fs.existsSync(filePath)) {
console.log(`音频已存在`); console.log(`音频已存在`);
fs.unlinkSync(path); fs.unlinkSync(filePath);
} }
// 发起请求 // 发起请求
@ -169,12 +170,12 @@ export async function downloadAudio(mp3Url, path, title = "temp", redirect = "ma
}); });
// 开始下载 // 开始下载
const writer = fs.createWriteStream(path); const writer = fs.createWriteStream(filePath);
response.data.pipe(writer); response.data.pipe(writer);
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
writer.on('finish', () => resolve(path)); writer.on('finish', () => resolve(filePath));
writer.on('error', reject); writer.on('error', reject);
}); });
@ -184,17 +185,63 @@ export async function downloadAudio(mp3Url, path, title = "temp", redirect = "ma
} }
} }
/** /**
* 下载一张网络图片(自动以url的最后一个为名字) * 下载图片网关
* @param {string} img * @param {Object} options 参数对象
* @param {string} dir * @param {string} options.img 图片的URL
* @param {string} fileName * @param {string} options.dir 保存图片的目录
* @param {boolean} isProxy * @param {string} [options.fileName] 自定义文件名 (可选)
* @param {Object} headersExt * @param {boolean} [options.isProxy] 是否使用代理 (可选)
* @param {Object} proxyInfo 参数proxyAddr=地址proxyPort=端口 * @param {Object} [options.headersExt] 自定义请求头 (可选)
* @returns {Promise<unknown>} * @param {Object} [options.proxyInfo] 代理信息 (可选)
* @returns {Promise<string>}
*/ */
export async function downloadImg(img, dir, fileName = "", isProxy = false, headersExt = {}, proxyInfo = {}) { export async function downloadImg({
img,
dir,
fileName = "",
isProxy = false,
headersExt = {},
proxyInfo = {},
numThread = 1,
}) {
const downloadImgParams = {
img,
dir,
fileName,
isProxy,
headersExt,
proxyInfo,
numThread,
}
logger.info(logger.yellow(`[R插件][图片下载] 当前使用线程数:${ numThread }`));
if (numThread === 1) {
return normalDownloadImg(downloadImgParams);
} else if (numThread > 1) {
return downloadImgWithAria2(downloadImgParams);
}
}
/**
* 正常下载图片
* @param {Object} options 参数对象
* @param {string} options.img 图片的URL
* @param {string} options.dir 保存图片的目录
* @param {string} [options.fileName] 自定义文件名 (可选)
* @param {boolean} [options.isProxy] 是否使用代理 (可选)
* @param {Object} [options.headersExt] 自定义请求头 (可选)
* @param {Object} [options.proxyInfo] 代理信息 (可选)
* @returns {Promise<string>}
*/
async function normalDownloadImg({
img,
dir,
fileName = "",
isProxy = false,
headersExt = {},
proxyInfo = {}
}) {
if (fileName === "") { if (fileName === "") {
fileName = img.split("/").pop(); fileName = img.split("/").pop();
} }
@ -203,8 +250,7 @@ export async function downloadImg(img, dir, fileName = "", isProxy = false, head
const writer = fs.createWriteStream(filepath); const writer = fs.createWriteStream(filepath);
const axiosConfig = { const axiosConfig = {
headers: { headers: {
"User-Agent": "User-Agent": COMMON_USER_AGENT,
"Mozilla/5.0 (Linux; Android 5.0; SM-G900P Build/LRX21T) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.25 Mobile Safari/537.36",
...headersExt ...headersExt
}, },
responseType: "stream", responseType: "stream",
@ -237,6 +283,60 @@ export async function downloadImg(img, dir, fileName = "", isProxy = false, head
} }
} }
/**
* 下载一张网络图片(使用aria2加速下载)
* @param {Object} options 参数对象
* @param {string} options.img 图片的URL
* @param {string} options.dir 保存图片的目录
* @param {string} [options.fileName] 自定义文件名 (可选)
* @param {boolean} [options.isProxy] 是否使用代理 (可选)
* @param {Object} [options.headersExt] 自定义请求头 (可选)
* @param {Object} [options.proxyInfo] 代理信息 (可选)
* @returns {Promise<unknown>}
*/
async function downloadImgWithAria2({
img,
dir,
fileName = "",
isProxy = false,
headersExt = {},
proxyInfo = {},
numThread = 1,
}) {
if (fileName === "") {
fileName = img.split("/").pop();
}
const filepath = path.resolve(dir, fileName);
await mkdirIfNotExists(dir);
// 构建 aria2c 命令
let aria2cCmd = `aria2c "${ img }" --dir="${ dir }" --out="${ fileName }" --max-connection-per-server=${numThread} --split=${numThread} --min-split-size=1M --continue`;
// 如果需要代理
if (isProxy) {
aria2cCmd += ` --all-proxy="http://${ proxyInfo.proxyAddr }:${ proxyInfo.proxyPort }"`;
}
// 添加自定义headers
if (headersExt && Object.keys(headersExt).length > 0) {
for (const [headerName, headerValue] of Object.entries(headersExt)) {
aria2cCmd += ` --header="${ headerName }: ${ headerValue }"`;
}
}
return new Promise((resolve, reject) => {
exec(aria2cCmd, (error, stdout, stderr) => {
if (error) {
logger.error(`图片下载失败, 原因:${ error.message }`);
reject(error);
return;
}
resolve(filepath);
});
});
}
/** /**
* 千位数的数据处理 * 千位数的数据处理
* @param data * @param data

View File

@ -10,7 +10,6 @@ export async function checkAndRemoveFile(file) {
try { try {
await fs.promises.access(file); await fs.promises.access(file);
await fs.promises.unlink(file); await fs.promises.unlink(file);
logger.mark('文件已存在');
} catch (err) { } catch (err) {
if (err.code !== 'ENOENT') { if (err.code !== 'ENOENT') {
throw err; throw err;