From dbf33728e607f3efaa317703320eeafa08c474fb Mon Sep 17 00:00:00 2001 From: jonmatthis Date: Wed, 15 May 2024 17:31:44 -0400 Subject: [PATCH 1/5] add gpt-4o --- .gitignore | 4 ++++ src/core/ai/openai/openai-chat.service.ts | 1 + 2 files changed, 5 insertions(+) diff --git a/.gitignore b/.gitignore index a33a866..682114f 100644 --- a/.gitignore +++ b/.gitignore @@ -36,7 +36,11 @@ lerna-debug.log* .env.slack .env.discord +.env.analysis /.env.openai /src/interfaces/discord/commands/server-config-command/server-config-examples/capstone/2024-capstone-category-per-student-config.yaml /src/interfaces/discord/commands/server-config-command/server-config-examples/capstone/2024-capstone-category-per-student-config-CHANGE_NICKNAMES.yaml /src/interfaces/discord/commands/server-config-command/server-config-examples/capstone/student-identifiers.ts + +venv/ +.venv/ diff --git a/src/core/ai/openai/openai-chat.service.ts b/src/core/ai/openai/openai-chat.service.ts index f5ab821..738bca4 100644 --- a/src/core/ai/openai/openai-chat.service.ts +++ b/src/core/ai/openai/openai-chat.service.ts @@ -7,6 +7,7 @@ export interface OpenAiChatConfig { //https://platform.openai.com/docs/api-reference/chat messages: any[]; model: + | 'gpt-4o' | 'gpt-4-1106-preview' | 'gpt-4' | 'gpt-4-vision-preview' From 855c6d5d376591e5d66440b67e4b5f1f25443959 Mon Sep 17 00:00:00 2001 From: jonmatthis Date: Wed, 15 May 2024 17:59:29 -0400 Subject: [PATCH 2/5] don't respond in top levels (even if mentioned) --- .../discord/services/discord-on-message.service.ts | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/interfaces/discord/services/discord-on-message.service.ts b/src/interfaces/discord/services/discord-on-message.service.ts index 6ec7e4f..637e820 100644 --- a/src/interfaces/discord/services/discord-on-message.service.ts +++ b/src/interfaces/discord/services/discord-on-message.service.ts @@ -47,11 +47,12 @@ export class DiscordOnMessageService { const chatConfig = { messages: [], - model: 'gpt-4-vision-preview', + model: 'gpt-4o', temperature: 0.7, stream: true, max_tokens: 4096, } as OpenAiChatConfig; + this._openaiChatService.createChat(aiChatId, contextPrompt, chatConfig); const aiChatDocument = await this._aiChatsService.createAiChat({ aiChatId, @@ -78,11 +79,6 @@ export class DiscordOnMessageService { } const botId = message.client.user.id; - // Respond to messages that mention the bot - if (message.mentions.has(botId)) { - return true; - } - // Respond to messages in threads the bot created/owns if ( message.channel instanceof ThreadChannel && From 4879352dbd286fc38a3e2a0106d1658180b5ca18 Mon Sep 17 00:00:00 2001 From: jonmatthis Date: Wed, 15 May 2024 18:19:31 -0400 Subject: [PATCH 3/5] include attachements on pinned message --- .../discord/services/discord-context-prompt.service.ts | 10 +++++++++- .../discord/services/discord-on-message.service.ts | 2 +- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/interfaces/discord/services/discord-context-prompt.service.ts b/src/interfaces/discord/services/discord-context-prompt.service.ts index 16d9d30..0863b10 100644 --- a/src/interfaces/discord/services/discord-context-prompt.service.ts +++ b/src/interfaces/discord/services/discord-context-prompt.service.ts @@ -11,6 +11,7 @@ import { TextChannel, ThreadChannel, } from 'discord.js'; +import { DiscordMessageService } from './discord-message.service'; @Injectable() export class DiscordContextPromptService { @@ -19,6 +20,8 @@ export class DiscordContextPromptService { instructionsChannelPattern = new RegExp('.*?prompt-?settings.*', 'i'); botPromptEmoji = '🤖'; + constructor(private readonly _messageService: DiscordMessageService) {} + async getContextPromptFromMessage(message: Message) { try { if (message.channel instanceof DMChannel) { @@ -209,7 +212,12 @@ export class DiscordContextPromptService { let pinnedMessageCount = 0; for (const message of pinnedMessages.values()) { pinnedMessagesContent += `Pinned message ${pinnedMessageCount++}:\n`; - pinnedMessagesContent += message.content + '\n'; + const { humanInputText, attachmentText, imageURLs } = + await this._messageService.extractMessageContent(message); + + pinnedMessagesContent += humanInputText; + pinnedMessagesContent += attachmentText; + pinnedMessagesContent += imageURLs?.join('\n') || ''; } pinnedMessagesContent += '\nEND PINNED MESSAGES'; return pinnedMessagesContent; diff --git a/src/interfaces/discord/services/discord-on-message.service.ts b/src/interfaces/discord/services/discord-on-message.service.ts index 637e820..571e440 100644 --- a/src/interfaces/discord/services/discord-on-message.service.ts +++ b/src/interfaces/discord/services/discord-on-message.service.ts @@ -60,7 +60,7 @@ export class DiscordOnMessageService { contextRoute, contextInstructions: contextPrompt, couplets: [], - modelName: 'gpt-4-vision-preview', + modelName: 'gpt-4o', }); this.logger.debug(`Adding threadId ${aiChatId} to active listeners`); From 069df07ca9efe4484f6d2ba51845ad238836a0bc Mon Sep 17 00:00:00 2001 From: jonmatthis Date: Wed, 15 May 2024 20:54:10 -0400 Subject: [PATCH 4/5] we can implement the 'chat-to-markdown' stuff in a different way --- src/core/ai/openai/openai-chat.service.ts | 1 + .../discord/services/discord-persistence.service.ts | 9 +++++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/core/ai/openai/openai-chat.service.ts b/src/core/ai/openai/openai-chat.service.ts index 738bca4..7bdf17c 100644 --- a/src/core/ai/openai/openai-chat.service.ts +++ b/src/core/ai/openai/openai-chat.service.ts @@ -137,6 +137,7 @@ export class OpenaiChatService implements OnModuleInit { max_tokens: 4096, } as OpenAiChatConfig; } + private _reloadMessageHistoryFromAiChatDocument(aiChat: AiChatDocument) { const chatConfig = this._getConfigOrThrow(aiChat.aiChatId); diff --git a/src/interfaces/discord/services/discord-persistence.service.ts b/src/interfaces/discord/services/discord-persistence.service.ts index 861f703..26f6060 100644 --- a/src/interfaces/discord/services/discord-persistence.service.ts +++ b/src/interfaces/discord/services/discord-persistence.service.ts @@ -10,6 +10,7 @@ import { TextChannel, } from 'discord.js'; import { AiChatDocument } from '../../../core/database/collections/ai-chats/ai-chat.schema'; +import { CoupletDocument } from '../../../core/database/collections/couplets/couplet.schema'; @Injectable() export class DiscordPersistenceService { @@ -73,20 +74,22 @@ export class DiscordPersistenceService { }); await this._aiChatsService.addCouplets(aiChatId, [couplet]); - await this.attachAiChatToOldestMessage( + await this._updateInChatPersistence( aiChatId, await this._aiChatsService.findOne(aiChatId), discordMessage.channel as TextChannel, + couplet, ); } catch (error) { this.logger.error(`Error persisting interaction: ${error}`); } } - public async attachAiChatToOldestMessage( + private async _updateInChatPersistence( aiChatId: string, aiChatDocument: AiChatDocument, channel: TextChannel, + couplet: CoupletDocument, ) { this.logger.debug( `Attaching chat document to oldest message in thread ${aiChatId}`, @@ -96,6 +99,8 @@ export class DiscordPersistenceService { delete aiChatAsJson._id; delete aiChatAsJson.__v; + console.log(JSON.stringify(couplet, null, 2)); + const oldestMessage = await this._findOldestMessage(channel); const aiChatAttachmentName = `chat-${aiChatId}.json`; const aiChatAttachment = new AttachmentBuilder( From 1ed52a60076d2e920bc95490ca3c5ad6162dcc0f Mon Sep 17 00:00:00 2001 From: jonmatthis Date: Wed, 15 May 2024 22:16:35 -0400 Subject: [PATCH 5/5] clean up --- .../discord-context-prompt.service.ts | 21 ++++++++++++------- .../services/discord-message.service.ts | 17 +++++++++++++++ .../services/discord-persistence.service.ts | 2 -- 3 files changed, 30 insertions(+), 10 deletions(-) diff --git a/src/interfaces/discord/services/discord-context-prompt.service.ts b/src/interfaces/discord/services/discord-context-prompt.service.ts index 0863b10..4b7d16d 100644 --- a/src/interfaces/discord/services/discord-context-prompt.service.ts +++ b/src/interfaces/discord/services/discord-context-prompt.service.ts @@ -36,13 +36,17 @@ export class DiscordContextPromptService { const server = await channel.client.guilds.fetch(channel.guildId); const channelTopic = `CHANNEL ${channel.name} TOPIC:\n\n${channel.topic}` || ''; + const channelPinnedInstructions = await this._getPinnedInstructions(channel); + const categoryInstructions = await this.getCategoryInstructions( server, channel.parent as CategoryChannel, ); + const serverInstructions = await this.getServerInstructions(server); + return [ serverInstructions, categoryInstructions, @@ -183,7 +187,9 @@ export class DiscordContextPromptService { ); return instructionMessages - .map((message: Message) => message.content) + .map((message: Message) => + this._messageService.extractMessageContentAsString(message), + ) .join('\n'); } @@ -211,15 +217,14 @@ export class DiscordContextPromptService { let pinnedMessageCount = 0; for (const message of pinnedMessages.values()) { - pinnedMessagesContent += `Pinned message ${pinnedMessageCount++}:\n`; - const { humanInputText, attachmentText, imageURLs } = - await this._messageService.extractMessageContent(message); + pinnedMessagesContent += `BEGIN PINNED MESSAGE ${pinnedMessageCount++}:\n\n`; + const content = + await this._messageService.extractMessageContentAsString(message); + pinnedMessagesContent += content; + pinnedMessagesContent += `END PINNED MESSAGE ${pinnedMessageCount++}:\n\n`; - pinnedMessagesContent += humanInputText; - pinnedMessagesContent += attachmentText; - pinnedMessagesContent += imageURLs?.join('\n') || ''; + pinnedMessagesContent += '\nEND PINNED MESSAGES'; } - pinnedMessagesContent += '\nEND PINNED MESSAGES'; return pinnedMessagesContent; } } diff --git a/src/interfaces/discord/services/discord-message.service.ts b/src/interfaces/discord/services/discord-message.service.ts index 04abfab..8f4ccbc 100644 --- a/src/interfaces/discord/services/discord-message.service.ts +++ b/src/interfaces/discord/services/discord-message.service.ts @@ -9,6 +9,7 @@ import { OpenaiChatService } from '../../../core/ai/openai/openai-chat.service'; export class DiscordMessageService { private readonly maxMessageLength = 2000 * 0.85; // discord max message length is 2000 characters (and * 0.85 to be safe) private readonly logger = new Logger(DiscordMessageService.name); + constructor( private readonly _persistenceService: DiscordPersistenceService, private readonly _contextService: DiscordContextRouteService, @@ -191,6 +192,21 @@ export class DiscordMessageService { } } + public async extractMessageContentAsString(discordMessage: Message) { + const { humanInputText, attachmentText, imageURLs } = + await this.extractMessageContent(discordMessage); + let fullText = `BEGIN MESSAGE CONTENT:\n\n ${humanInputText}\n\n END MESSAGE CONTENT\n\n`; + if (attachmentText) { + fullText += attachmentText; + } + if (imageURLs.length > 0) { + fullText += '\n\nBEGIN IMAGE URLS:\n\n'; + fullText += imageURLs.join('\n'); + fullText += '\n\nEND IMAGE URLS\n\n'; + } + return fullText; + } + public async extractMessageContent( discordMessage: Message, respondToChannelOrMessage?: Message | TextBasedChannel, @@ -246,6 +262,7 @@ export class DiscordMessageService { attachmentText += 'END TEXT FROM ATTACHMENTS'; } } + return { humanInputText, attachmentText, imageURLs }; } diff --git a/src/interfaces/discord/services/discord-persistence.service.ts b/src/interfaces/discord/services/discord-persistence.service.ts index 26f6060..caf1d37 100644 --- a/src/interfaces/discord/services/discord-persistence.service.ts +++ b/src/interfaces/discord/services/discord-persistence.service.ts @@ -99,8 +99,6 @@ export class DiscordPersistenceService { delete aiChatAsJson._id; delete aiChatAsJson.__v; - console.log(JSON.stringify(couplet, null, 2)); - const oldestMessage = await this._findOldestMessage(channel); const aiChatAttachmentName = `chat-${aiChatId}.json`; const aiChatAttachment = new AttachmentBuilder(