diff --git a/apps/ai.js b/apps/ai.js index 2f4e02a..d24560e 100644 --- a/apps/ai.js +++ b/apps/ai.js @@ -130,7 +130,7 @@ async function extractUserMessage(msg, nickname, e) { originalMessages.push({ type: 'image_url', image_url: { - url: message.image + url: message.url } }); } @@ -142,7 +142,12 @@ async function extractUserMessage(msg, nickname, e) { text.forEach((message) => { if(message === '') { } else { - returnMessage += `[${e.sender?.nickname},id:${e.user_id},seq:${e.message_id}]说:${message}\n`; + const tempMessage = `[${e.sender?.nickname},id:${e.user_id},seq:${e.message_id}]说:${message}\n` + returnMessage += tempMessage; + originalMessages.push({ + type: 'text', + content: tempMessage + }); } }); } @@ -187,7 +192,7 @@ async function extractUserMessage(msg, nickname, e) { originalMessages.push({ type: 'image_url', image_url: { - url: msg.image + url: msg.url } }); } diff --git a/lib/ai/aiCaller.js b/lib/ai/aiCaller.js index 2243598..add22fa 100644 --- a/lib/ai/aiCaller.js +++ b/lib/ai/aiCaller.js @@ -107,7 +107,7 @@ class AiCaller { */ async callMultimodalAi(originalMessages, chatHistory = [], memories = [], e) { try { - const messages = this.formatMultimodalMessages(originalMessages, chatHistory, memories, e); + const messages = await this.formatMultimodalMessages(originalMessages, chatHistory, memories, e); const apiCaller = this.openaiChat; const result = await apiCaller.callAi({ messages: messages, @@ -145,31 +145,45 @@ class AiCaller { const systemPrompt = await this.getSystemPrompt(e, memories); messages.push({ role: 'system', - content: [{ type: 'text', text: systemPrompt }] + content: [ + { type: 'text', text: systemPrompt } + ] }); for (const history of chatHistory) { const role = history.role === 'user' ? 'user' : 'assistant'; messages.push({ - role: role, - content: [{ type: 'text', text: history.content }] + role, + content: [ + { type: 'text', text: history.content } + ] }); } + const mergedUserContent = []; for (const msg of originalMessages) { if (msg.type === 'text' && msg.content) { - messages.push({ - role: 'user', - content: [{ type: 'text', text: msg.content }] + mergedUserContent.push({ + type: 'text', + text: msg.content }); - } else if (msg.type === 'image_url' && msg.image_url) { - messages.push({ - role: 'user', - content: [{ type: 'image_url', image_url: { url: msg.image_url.url } }] + } + + if (msg.type === 'image_url' && msg.image_url?.url) { + mergedUserContent.push({ + type: 'image_url', + image_url: { url: msg.image_url.url } }); } } + if (mergedUserContent.length > 0) { + messages.push({ + role: 'user', + content: mergedUserContent + }); + } return messages; } + /** * 构造完整的prompt * @param prompt diff --git a/modules/openai/openaiChat.js b/modules/openai/openaiChat.js index 5977d04..e59f8f6 100644 --- a/modules/openai/openaiChat.js +++ b/modules/openai/openaiChat.js @@ -25,13 +25,13 @@ class OpenaiChat { * @param messages 多模态消息数组 * @returns {Promise<{success: boolean, aiResponse: string}|{}>} */ - async callAi({ prompt, chatHistory = [], model, temperature, customPrompt, messages }) { + async callAi({ prompt, chatHistory = [], model, temperature, customPrompt, messages = [] }) { if (!this.openai) { logger.error('[crystelf-ai] ai未初始化..'); return { success: false }; } let finalMessages; - if (messages && messages.length > 0) { + if (messages.length > 0) { finalMessages = messages; } else { let systemMessage = {