mirror of
https://github.com/Jerryplusy/rc-plugin.git
synced 2025-10-14 08:09:19 +00:00
✨ feat: 新增识图功能
1. 新增OpenAI转换器，可以自由转换Kimi和OpenAI 2. 新增识图功能，暂时作为测试功能，因为prompt没有写好
This commit is contained in:
zhiyu1998
parent
d300635a72
commit
d604620ba8
17
README.md
17
README.md
@ -48,7 +48,7 @@ git clone https://gitee.com/kyrzy0416/rconsole-plugin.git ./plugins/rconsole-plu
|
||||
git clone https://github.com/zhiyu1998/rconsole-plugin.git ./plugins/rconsole-plugin/
|
||||
```
|
||||
|
||||
2.【必要】在`Yunzai-Bot / Miao-Yunzai`目录下安装axios(0.27.2)、魔法工具（tunnel）、二维码处理工具（qrcode）、高性能下载队列（p-queue）
|
||||
2.【必要】在`Yunzai-Bot / Miao-Yunzai`目录下安装axios(0.27.2)、魔法工具（tunnel）、二维码处理工具（qrcode）、高性能下载队列（p-queue）、用于拉格朗日（ws）、用于识图（openai）
|
||||
|
||||
|
||||
```shell
|
||||
@ -198,6 +198,21 @@ git clone -b 1.6.7-lts https://gitee.com/kyrzy0416/rconsole-plugin.git
|
||||
|
||||
<img src="./img/lagrange.webp" width="30%" height="30%">
|
||||
|
||||
### 🤖 关于识图 [beta功能]
|
||||
|
||||
R 插件集成了我的新作品`gpt2txt`：https://github.com/zhiyu1998/gpt2txt
|
||||
|
||||
使用需要在锅巴 or tools.yaml修改以下内容：
|
||||
```yaml
|
||||
aiBaseURL: '' # 用于识图的接口，kimi默认接口为：https://api.moonshot.cn，其他服务商自己填写
|
||||
aiApiKey: '' # 用于识图的api key，kimi接口申请：https://platform.moonshot.cn/console/api-keys
|
||||
aiModel: 'claude-3-haiku-20240307' # 模型，使用kimi不用填写，其他要填写
|
||||
```
|
||||
|
||||
`Kimi`用户只需填写`aiBaseURL` 和 `aiApiKey`，其他用户都需要填写！效果展示如下：
|
||||
|
||||
![]()
|
||||
|
||||
## 🤺 R插件交流群
|
||||
扫码不行就：575663150
|
||||
|
||||
|
||||
@ -4,12 +4,19 @@ import fetch from "node-fetch";
|
||||
import puppeteer from "../../../lib/puppeteer/puppeteer.js";
|
||||
// http库
|
||||
import axios from "axios";
|
||||
// url库
|
||||
import url from 'url';
|
||||
// 常量
|
||||
import { CAT_LIMIT } from "../constants/constant.js";
|
||||
// 配置文件
|
||||
import config from "../model/index.js";
|
||||
// 书库
|
||||
import { getYiBook, getZBook, getZHelper } from "../utils/books.js";
|
||||
// 工具类
|
||||
import TokenBucket from '../utils/token-bucket.js'
|
||||
import { downloadImg } from "../utils/common.js";
|
||||
import { checkAndRemoveFile, toBase64 } from "../utils/file.js";
|
||||
import { OpenaiBuilder } from "../utils/openai-builder.js";
|
||||
|
||||
export class query extends plugin {
|
||||
/**
|
||||
@ -56,9 +63,23 @@ export class query extends plugin {
|
||||
{
|
||||
reg: "^#(wiki|百科)(.*)$",
|
||||
fnc: "wiki",
|
||||
},
|
||||
{
|
||||
reg: "^识图",
|
||||
fnc: "openAiOCR"
|
||||
}
|
||||
],
|
||||
});
|
||||
// 配置文件
|
||||
this.toolsConfig = config.getConfig("tools");
|
||||
// 视频保存路径
|
||||
this.defaultPath = this.toolsConfig.defaultPath;
|
||||
// ai接口
|
||||
this.aiBaseURL = this.toolsConfig.aiBaseURL;
|
||||
// ai api key
|
||||
this.aiApiKey = this.toolsConfig.aiApiKey;
|
||||
// ai模型
|
||||
this.aiModel = this.toolsConfig.aiModel;
|
||||
}
|
||||
|
||||
async doctor(e) {
|
||||
@ -317,6 +338,66 @@ export class query extends plugin {
|
||||
return true;
|
||||
}
|
||||
|
||||
// 识图
|
||||
async openAiOCR(e) {
|
||||
if (e.source) {
|
||||
let reply;
|
||||
if (e.isGroup) {
|
||||
reply = (await e.group.getChatHistory(this.e.source.seq, 1)).pop()?.message;
|
||||
} else {
|
||||
reply = (await e.friend.getChatHistory(this.e.source.time, 1)).pop()?.message;
|
||||
}
|
||||
if (reply) {
|
||||
for (let val of reply) {
|
||||
if (val.type == "image") {
|
||||
e.img = [val.url];
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!e.img) {
|
||||
this.setContext('openAiProcess');
|
||||
await e.reply("「R插件 x 月之暗面 Kimi」联合识别提醒你：请发送图片！", false, { at: true });
|
||||
} else {
|
||||
this.openAiProcess();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* AI引擎提供图像识别能力
|
||||
* @return {Promise<boolean>}
|
||||
*/
|
||||
async openAiProcess() {
|
||||
if (!this.e.img) {
|
||||
e.reply("「R插件 x 月之暗面 Kimi」联合识别提醒你：无法找到图片！")
|
||||
return true;
|
||||
}
|
||||
const img = this.e.img.find(item => item.startsWith("http"));
|
||||
const parsedUrl = url.parse(img);
|
||||
const pathArray = parsedUrl.pathname.split('/');
|
||||
const filenameWithExtension = pathArray[pathArray.length - 1];
|
||||
const path = `${ this.defaultPath }${ this.e.group_id || this.e.user_id }`
|
||||
// 下载图片
|
||||
const imgPath = await downloadImg(img, path, filenameWithExtension);
|
||||
// 构造OpenAI引擎
|
||||
try {
|
||||
const { model, ans } = await new OpenaiBuilder()
|
||||
.setBaseURL(this.aiBaseURL)
|
||||
.setApiKey(this.aiApiKey)
|
||||
.setModel(this.aiModel)
|
||||
.setPath(imgPath)
|
||||
.build();
|
||||
this.e.reply(`「R插件 x ${ model }」联合识别：\n${ ans }`);
|
||||
await checkAndRemoveFile(filenameWithExtension);
|
||||
} catch (err) {
|
||||
e.reply("「R插件 x 月之暗面 Kimi」联合识别提醒你：无法找到图片路径！")
|
||||
logger.error(err);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* 限制用户调用（默认1分钟1次）
|
||||
* @param e
|
||||
|
||||
@ -18,3 +18,7 @@ videoDownloadConcurrency: 1 # 下载视频是否使用多线程，如果不使
|
||||
lagrangeForwardWebSocket: 'ws://127.0.0.1:9091/' # 格式：ws://地址:端口/，拉格朗日正向连接地址，用于适配拉格朗日上传群文件，解决部分用户无法查看视频问题
|
||||
|
||||
autoclearTrashtime: '0 0 8 * * ?' #每天早上8点自动清理视频缓存，cron可自定义时间
|
||||
|
||||
aiBaseURL: '' # 用于识图的接口，kimi默认接口为：https://api.moonshot.cn，其他服务商自己填写
|
||||
aiApiKey: '' # 用于识图的api key，kimi接口申请：https://platform.moonshot.cn/console/api-keys
|
||||
aiModel: 'claude-3-haiku-20240307' # 模型，使用kimi不用填写，其他要填写
|
||||
@ -151,6 +151,39 @@ export function supportGuoba() {
|
||||
componentProps: {
|
||||
placeholder: "请输入拉格朗日正向WebSocket连接地址",
|
||||
},
|
||||
},
|
||||
{
|
||||
field: "tools.aiBaseURL",
|
||||
label: "AI接口地址",
|
||||
bottomHelpMessage:
|
||||
"支持Kimi、OpenAI、Claude等",
|
||||
component: "Input",
|
||||
required: false,
|
||||
componentProps: {
|
||||
placeholder: "请输入AI接口地址",
|
||||
},
|
||||
},
|
||||
{
|
||||
field: "tools.aiApiKey",
|
||||
label: "AI的key",
|
||||
bottomHelpMessage:
|
||||
"服务商提供的api key",
|
||||
component: "Input",
|
||||
required: false,
|
||||
componentProps: {
|
||||
placeholder: "请输入AI的key",
|
||||
},
|
||||
},
|
||||
{
|
||||
field: "tools.aiModel",
|
||||
label: "AI的模型",
|
||||
bottomHelpMessage:
|
||||
"默认使用的是Claude，也可以自定义模型",
|
||||
component: "Input",
|
||||
required: false,
|
||||
componentProps: {
|
||||
placeholder: "请输入AI的模型，例如：claude-3-haiku-20240307，使用kimi则不用填写",
|
||||
},
|
||||
}
|
||||
],
|
||||
getConfigData() {
|
||||
|
||||
BIN
img/imageRecognition.webp
Normal file
BIN
img/imageRecognition.webp
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 551 KiB |
@ -7,6 +7,7 @@
|
||||
"tunnel": "^0.0.6",
|
||||
"qrcode": "^1.5.3",
|
||||
"p-queue": "^8.0.1",
|
||||
"ws": "^8.17.0"
|
||||
"ws": "^8.17.0",
|
||||
"openai": "^4.47.1"
|
||||
}
|
||||
}
|
||||
|
||||
@ -101,3 +101,41 @@ export async function copyFiles(srcDir, destDir) {
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* 转换路径图片为base64格式
|
||||
* @param filePath 图片路径
|
||||
* @return {Promise<string>}
|
||||
*/
|
||||
export async function toBase64(filePath) {
|
||||
try {
|
||||
// 读取文件数据
|
||||
const fileData = await fs.readFileSync(filePath);
|
||||
// 将文件数据转换为Base64字符串
|
||||
const base64Data = fileData.toString('base64');
|
||||
// 返回Base64字符串
|
||||
return `data:${getMimeType(filePath)};base64,${base64Data}`;
|
||||
} catch (error) {
|
||||
throw new Error(`读取文件时出错: ${error.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 辅助函数：根据文件扩展名获取MIME类型
|
||||
* @param filePath
|
||||
* @return {*|string}
|
||||
*/
|
||||
function getMimeType(filePath) {
|
||||
const mimeTypes = {
|
||||
'.jpg': 'image/jpeg',
|
||||
'.jpeg': 'image/jpeg',
|
||||
'.png': 'image/png',
|
||||
'.gif': 'image/gif',
|
||||
'.pdf': 'application/pdf',
|
||||
'.txt': 'text/plain',
|
||||
// 添加其他文件类型和MIME类型的映射
|
||||
};
|
||||
|
||||
const ext = filePath.substring(filePath.lastIndexOf('.')).toLowerCase();
|
||||
return mimeTypes[ext] || 'application/octet-stream';
|
||||
}
|
||||
|
||||
117
utils/openai-builder.js
Normal file
117
utils/openai-builder.js
Normal file
@ -0,0 +1,117 @@
|
||||
import { toBase64 } from "./file.js";
|
||||
// openai库
|
||||
import OpenAI from 'openai';
|
||||
// fs
|
||||
import fs from "node:fs";
|
||||
|
||||
export class OpenaiBuilder {
|
||||
constructor() {
|
||||
this.baseURL = "https://api.moonshot.cn"; // 默认模型
|
||||
this.apiKey = ""; // 默认API密钥
|
||||
this.prompt = "描述一下这个图片"; // 默认提示
|
||||
this.model = 'claude-3-haiku-20240307'
|
||||
this.path = ''; // 上传文件的路径
|
||||
}
|
||||
|
||||
setBaseURL(baseURL) {
|
||||
this.baseURL = baseURL + "/v1";
|
||||
return this;
|
||||
}
|
||||
|
||||
setApiKey(apiKey) {
|
||||
this.apiKey = apiKey;
|
||||
return this;
|
||||
}
|
||||
|
||||
setPrompt(prompt) {
|
||||
this.prompt = prompt;
|
||||
return this;
|
||||
}
|
||||
|
||||
setModel(model) {
|
||||
this.model = model;
|
||||
return this;
|
||||
}
|
||||
|
||||
setPath(path) {
|
||||
this.path = path;
|
||||
return this;
|
||||
}
|
||||
|
||||
async build() {
|
||||
if (this.path === '') {
|
||||
throw Error("无法获取到文件路径");
|
||||
return null;
|
||||
}
|
||||
// logger.info(this.baseURL, this.apiKey)
|
||||
// 创建客户端
|
||||
this.client = new OpenAI({
|
||||
baseURL: this.baseURL,
|
||||
apiKey: this.apiKey
|
||||
});
|
||||
// 构建
|
||||
if (this.baseURL.includes("api.moonshot.cn")) {
|
||||
return await this.kimi(this.path);
|
||||
} else {
|
||||
return await this.openai(this.path);
|
||||
}
|
||||
}
|
||||
|
||||
async kimi(path) {
|
||||
let file_object = await this.client.files.create({
|
||||
file: fs.createReadStream(path),
|
||||
purpose: "file-extract"
|
||||
})
|
||||
let file_content = await (await this.client.files.content(file_object.id)).text()
|
||||
// 请求OpenAI
|
||||
const completion = await this.client.chat.completions.create({
|
||||
model: "moonshot-v1-8k",
|
||||
messages: [
|
||||
{
|
||||
"role": "system",
|
||||
"content": file_content,
|
||||
},
|
||||
{
|
||||
role: "user",
|
||||
content: this.prompt
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
return {
|
||||
"model": "月之暗面 Kimi",
|
||||
"ans": completion.choices[0].message.content
|
||||
}
|
||||
}
|
||||
|
||||
async openai(path) {
|
||||
// 转换base64
|
||||
const pic = await toBase64(path);
|
||||
const completion = await this.client.chat.completions.create({
|
||||
model: this.model,
|
||||
messages: [
|
||||
{
|
||||
role: "user",
|
||||
content: [
|
||||
{
|
||||
type: "image_url",
|
||||
image_url: {
|
||||
url: pic,
|
||||
},
|
||||
},
|
||||
{
|
||||
type: "text",
|
||||
text: this.prompt,
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
use_search: false,
|
||||
});
|
||||
|
||||
return {
|
||||
"model": "OpenAI",
|
||||
"ans": completion.choices[0].message.content
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
x
Reference in New Issue
Block a user