From 9d6048f06f12958e9e161c93342325dd7a60b9b6 Mon Sep 17 00:00:00 2001 From: Victor Ilyushchenko Date: Thu, 6 Mar 2025 17:50:14 +0300 Subject: [PATCH] Added summarization action for meeting minutes (#8143) * Added summarization action for meeting minutes Signed-off-by: Victor Ilyushchenko * tweaks Signed-off-by: Victor Ilyushchenko * fmt Signed-off-by: Victor Ilyushchenko * ff Signed-off-by: Victor Ilyushchenko --------- Signed-off-by: Victor Ilyushchenko --- models/chunter/src/actions.ts | 15 ++ models/chunter/src/plugin.ts | 3 +- .../src/components/markup/NodeContent.svelte | 3 + plugins/ai-bot-resources/src/requests.ts | 45 +++++- plugins/ai-bot/src/rest.ts | 22 ++- plugins/chunter-assets/lang/cs.json | 3 +- plugins/chunter-assets/lang/de.json | 3 +- plugins/chunter-assets/lang/en.json | 3 +- plugins/chunter-assets/lang/es.json | 3 +- plugins/chunter-assets/lang/fr.json | 3 +- plugins/chunter-assets/lang/it.json | 3 +- plugins/chunter-assets/lang/pt.json | 3 +- plugins/chunter-assets/lang/ru.json | 3 +- plugins/chunter-assets/lang/zh.json | 3 +- plugins/chunter-resources/package.json | 3 +- plugins/chunter-resources/src/index.ts | 6 +- plugins/chunter-resources/src/utils.ts | 37 +++-- plugins/chunter/src/index.ts | 5 +- services/ai-bot/pod-ai-bot/src/config.ts | 2 + services/ai-bot/pod-ai-bot/src/controller.ts | 129 +++++++++++++++++- .../ai-bot/pod-ai-bot/src/server/server.ts | 20 ++- .../ai-bot/pod-ai-bot/src/utils/openai.ts | 72 +++++++++- 22 files changed, 355 insertions(+), 34 deletions(-) diff --git a/models/chunter/src/actions.ts b/models/chunter/src/actions.ts index c1940c5d58..17562c3e02 100644 --- a/models/chunter/src/actions.ts +++ b/models/chunter/src/actions.ts @@ -118,6 +118,21 @@ function defineMessageActions (builder: Builder): void { chunter.action.DeleteChatMessage ) + createAction( + builder, + { + action: chunter.actionImpl.SummarizeMessages, + label: chunter.string.SummarizeMessages, + icon: view.icon.Feather, + input: 'focus', + category: chunter.category.Chunter, + target: core.class.Doc, + context: { mode: ['context', 'browser'], group: 'tools' }, + visibilityTester: chunter.function.CanSummarizeMessages + }, + chunter.action.SummarizeMessages + ) + createAction( builder, { diff --git a/models/chunter/src/plugin.ts b/models/chunter/src/plugin.ts index cb431c5753..8dd737ffef 100644 --- a/models/chunter/src/plugin.ts +++ b/models/chunter/src/plugin.ts @@ -56,7 +56,8 @@ export default mergeIds(chunterId, chunter, { OpenInSidebar: '' as ViewAction, TranslateMessage: '' as ViewAction, ShowOriginalMessage: '' as ViewAction, - StartConversation: '' as ViewAction + StartConversation: '' as ViewAction, + SummarizeMessages: '' as ViewAction }, category: { Chunter: '' as Ref diff --git a/packages/presentation/src/components/markup/NodeContent.svelte b/packages/presentation/src/components/markup/NodeContent.svelte index 33a59cbe25..7aede5b461 100644 --- a/packages/presentation/src/components/markup/NodeContent.svelte +++ b/packages/presentation/src/components/markup/NodeContent.svelte @@ -48,6 +48,9 @@ const checkEmoji = (nodes: MarkupNode[]): boolean => { const matches: boolean[] = [] + if (nodes.some((node) => node.type !== 'text')) { + return false + } nodes.forEach((node) => { const reg = node.text?.match(/\P{Emoji}/gu) matches.push(reg != null && reg.length > 0 && [65039, 65038, 8205].every((code) => code !== reg[0].charCodeAt(0))) diff --git a/plugins/ai-bot-resources/src/requests.ts b/plugins/ai-bot-resources/src/requests.ts index 3bc8387761..800437aa69 100644 --- a/plugins/ai-bot-resources/src/requests.ts +++ b/plugins/ai-bot-resources/src/requests.ts @@ -12,16 +12,18 @@ // See the License for the specific language governing permissions and // limitations under the License. // -import { concatLink, type Markup, type Ref } from '@hcengineering/core' -import { getMetadata } from '@hcengineering/platform' -import presentation from '@hcengineering/presentation' import { type ConnectMeetingRequest, type DisconnectMeetingRequest, + type SummarizeMessagesRequest, + type SummarizeMessagesResponse, type TranslateRequest, type TranslateResponse } from '@hcengineering/ai-bot' +import { type Class, concatLink, type Doc, type Markup, type Ref } from '@hcengineering/core' import { type Room, type RoomLanguage } from '@hcengineering/love' +import { getMetadata } from '@hcengineering/platform' +import presentation from '@hcengineering/presentation' import aiBot from './plugin' @@ -54,6 +56,43 @@ export async function translate (text: Markup, lang: string): Promise, + targetClass: Ref> +): Promise { + const url = getMetadata(aiBot.metadata.EndpointURL) ?? '' + const token = getMetadata(presentation.metadata.Token) ?? '' + + if (url === '' || token === '') { + return undefined + } + + try { + const req: SummarizeMessagesRequest = { + target, + targetClass, + lang + } + const resp = await fetch(concatLink(url, '/summarize'), { + method: 'POST', + headers: { + Authorization: 'Bearer ' + token, + 'Content-Type': 'application/json' + }, + body: JSON.stringify(req) + }) + if (!resp.ok) { + return undefined + } + + return (await resp.json()) as SummarizeMessagesResponse + } catch (error) { + console.error(error) + return undefined + } +} + export async function connectMeeting ( roomId: Ref, language: RoomLanguage, diff --git a/plugins/ai-bot/src/rest.ts b/plugins/ai-bot/src/rest.ts index 36fc24b41d..e7c6aba37d 100644 --- a/plugins/ai-bot/src/rest.ts +++ b/plugins/ai-bot/src/rest.ts @@ -15,7 +15,7 @@ import { Class, Doc, Markup, PersonId, Ref, Space, Timestamp } from '@hcengineering/core' import { Room, RoomLanguage } from '@hcengineering/love' -import { Person } from '@hcengineering/contact' +import { Contact, Person } from '@hcengineering/contact' import { ChatMessage } from '@hcengineering/chunter' export interface AIEventRequest { @@ -35,6 +35,26 @@ export interface TranslateRequest { lang: string } +export interface PersonMessage { + personRef: Ref + personName: string + + time: Timestamp + text: string +} + +export interface SummarizeMessagesRequest { + lang: string + + target: Ref + targetClass: Ref> +} + +export interface SummarizeMessagesResponse { + text: Markup + lang: string +} + export interface TranslateResponse { text: Markup lang: string diff --git a/plugins/chunter-assets/lang/cs.json b/plugins/chunter-assets/lang/cs.json index 990b31668f..6746cab96e 100644 --- a/plugins/chunter-assets/lang/cs.json +++ b/plugins/chunter-assets/lang/cs.json @@ -131,6 +131,7 @@ "ViewingThreadFromArchivedChannel": "Prohlížíte vlákno z archivovaného kanálu", "ViewingArchivedChannel": "Prohlížíte archivovaný kanál", "OpenChatInSidebar": "Otevřít chat v postranním panelu", - "NoThreadsYet": "Zatím nejsou žádné vlákna." + "NoThreadsYet": "Zatím nejsou žádné vlákna.", + "SummarizeMessages": "Shrnout diskuzi" } } \ No newline at end of file diff --git a/plugins/chunter-assets/lang/de.json b/plugins/chunter-assets/lang/de.json index d05bf5fb57..08c9b94837 100644 --- a/plugins/chunter-assets/lang/de.json +++ b/plugins/chunter-assets/lang/de.json @@ -132,6 +132,7 @@ "ViewingArchivedChannel": "Sie sehen einen archivierten Kanal", "OpenChatInSidebar": "Chat in Seitenleiste öffnen", "ResolveThread": "Thread abschließen", - "NoThreadsYet": "Es gibt noch keine Threads." + "NoThreadsYet": "Es gibt noch keine Threads.", + "SummarizeMessages": "Diskussion zusammenfassen" } } \ No newline at end of file diff --git a/plugins/chunter-assets/lang/en.json b/plugins/chunter-assets/lang/en.json index e0e8c4161e..f37cefafcc 100644 --- a/plugins/chunter-assets/lang/en.json +++ b/plugins/chunter-assets/lang/en.json @@ -132,6 +132,7 @@ "ViewingArchivedChannel": "You are viewing an archived channel", "OpenChatInSidebar": "Open chat in sidebar", "ResolveThread": "Resolve", - "NoThreadsYet": "There are no threads yet." + "NoThreadsYet": "There are no threads yet.", + "SummarizeMessages": "Summarize discussion" } } \ No newline at end of file diff --git a/plugins/chunter-assets/lang/es.json b/plugins/chunter-assets/lang/es.json index 35a29796f1..cbee7b9338 100644 --- a/plugins/chunter-assets/lang/es.json +++ b/plugins/chunter-assets/lang/es.json @@ -131,6 +131,7 @@ "ViewingThreadFromArchivedChannel": "Estás viendo un hilo de un canal archivado", "ViewingArchivedChannel": "Estás viendo un canal archivado", "OpenChatInSidebar": "Abrir chat en la barra lateral", - "NoThreadsYet": "No hay hilos todavía." + "NoThreadsYet": "No hay hilos todavía.", + "SummarizeMessages": "Resumir la discusión" } } \ No newline at end of file diff --git a/plugins/chunter-assets/lang/fr.json b/plugins/chunter-assets/lang/fr.json index 415d555247..5653ff1569 100644 --- a/plugins/chunter-assets/lang/fr.json +++ b/plugins/chunter-assets/lang/fr.json @@ -131,6 +131,7 @@ "ViewingThreadFromArchivedChannel": "Vous consultez un fil de discussion d'un canal archivé", "ViewingArchivedChannel": "Vous consultez un canal archivé", "OpenChatInSidebar": "Ouvrir le chat dans la barre latérale", - "NoThreadsYet": "Il n'y a pas encore de fils de discussion." + "NoThreadsYet": "Il n'y a pas encore de fils de discussion.", + "SummarizeMessages": "Résumer la discussion" } } \ No newline at end of file diff --git a/plugins/chunter-assets/lang/it.json b/plugins/chunter-assets/lang/it.json index 633a2fac1f..e44f2a7e7c 100644 --- a/plugins/chunter-assets/lang/it.json +++ b/plugins/chunter-assets/lang/it.json @@ -131,6 +131,7 @@ "ViewingThreadFromArchivedChannel": "Stai visualizzando una discussione da un canale archiviato", "ViewingArchivedChannel": "Stai visualizzando un canale archiviato", "OpenChatInSidebar": "Apri chat nella barra laterale", - "NoThreadsYet": "Non ci sono ancora discussioni." + "NoThreadsYet": "Non ci sono ancora discussioni.", + "SummarizeMessages": "Riassumere la discussione" } } diff --git a/plugins/chunter-assets/lang/pt.json b/plugins/chunter-assets/lang/pt.json index 0abbb3c33c..9ae35077a0 100644 --- a/plugins/chunter-assets/lang/pt.json +++ b/plugins/chunter-assets/lang/pt.json @@ -131,6 +131,7 @@ "ViewingThreadFromArchivedChannel": "Está a visualizar uma conversa em cadeia de um canal arquivado", "ViewingArchivedChannel": "Está a visualizar um canal arquivado", "OpenChatInSidebar": "Abrir chat na barra lateral", - "NoThreadsYet": "Não há conversas ainda." + "NoThreadsYet": "Não há conversas ainda.", + "SummarizeMessages": "Resumir discussão" } } \ No newline at end of file diff --git a/plugins/chunter-assets/lang/ru.json b/plugins/chunter-assets/lang/ru.json index c7849904d6..e808e58bff 100644 --- a/plugins/chunter-assets/lang/ru.json +++ b/plugins/chunter-assets/lang/ru.json @@ -132,6 +132,7 @@ "ViewingArchivedChannel": "Вы просматриваете архивированный канал", "OpenChatInSidebar": "Открыть чат в боковой панели", "ResolveThread": "Пометить завершенным", - "NoThreadsYet": "Пока нет обсуждений." + "NoThreadsYet": "Пока нет обсуждений.", + "SummarizeMessages": "Подвести итоги обсуждения" } } \ No newline at end of file diff --git a/plugins/chunter-assets/lang/zh.json b/plugins/chunter-assets/lang/zh.json index 596ee9773d..4164d6a8e1 100644 --- a/plugins/chunter-assets/lang/zh.json +++ b/plugins/chunter-assets/lang/zh.json @@ -131,6 +131,7 @@ "ViewingThreadFromArchivedChannel": "你正在查看已归档频道的线程", "ViewingArchivedChannel": "你正在查看已归档频道", "OpenChatInSidebar": "在侧边栏中打开聊天", - "NoThreadsYet": "还没有线程。" + "NoThreadsYet": "还没有线程。", + "SummarizeMessages": "总结讨论" } } diff --git a/plugins/chunter-resources/package.json b/plugins/chunter-resources/package.json index 15fe246d85..5d9d8603ea 100644 --- a/plugins/chunter-resources/package.json +++ b/plugins/chunter-resources/package.json @@ -66,6 +66,7 @@ "fast-equals": "^5.2.2", "svelte": "^4.2.19", "@hcengineering/text-editor-resources": "^0.6.0", - "@hcengineering/text-editor": "^0.6.0" + "@hcengineering/text-editor": "^0.6.0", + "@hcengineering/love": "^0.6.0" } } diff --git a/plugins/chunter-resources/src/index.ts b/plugins/chunter-resources/src/index.ts index f298a9a76c..39f0c3845e 100644 --- a/plugins/chunter-resources/src/index.ts +++ b/plugins/chunter-resources/src/index.ts @@ -84,7 +84,9 @@ import { translateMessage, showOriginalMessage, canTranslateMessage, - startConversationAction + startConversationAction, + summarizeMessages, + canSummarizeMessages } from './utils' export { default as ChannelEmbeddedContent } from './components/ChannelEmbeddedContent.svelte' @@ -213,6 +215,7 @@ export default async (): Promise => ({ CloseChatWidgetTab: closeChatWidgetTab, OpenChannelInSidebar: openChannelInSidebar, CanTranslateMessage: canTranslateMessage, + CanSummarizeMessages: canSummarizeMessages, OpenThreadInSidebar: openThreadInSidebar, LocationDataResolver: locationDataResolver }, @@ -226,6 +229,7 @@ export default async (): Promise => ({ ReplyToThread: replyToThread, OpenInSidebar: openChannelInSidebarAction, TranslateMessage: translateMessage, + SummarizeMessages: summarizeMessages, ShowOriginalMessage: showOriginalMessage, StartConversation: startConversationAction } diff --git a/plugins/chunter-resources/src/utils.ts b/plugins/chunter-resources/src/utils.ts index bde8d5ba5b..e2c615ff1e 100644 --- a/plugins/chunter-resources/src/utils.ts +++ b/plugins/chunter-resources/src/utils.ts @@ -20,24 +20,26 @@ import activity, { type DocUpdateMessage } from '@hcengineering/activity' import { isReactionMessage } from '@hcengineering/activity-resources' +import aiBot from '@hcengineering/ai-bot' +import { summarizeMessages as aiSummarizeMessages, translate as aiTranslate } from '@hcengineering/ai-bot-resources' import { type Channel, type ChatMessage, type DirectMessage, type ThreadMessage } from '@hcengineering/chunter' -import contact, { getName, getCurrentEmployee, type Employee, type Person } from '@hcengineering/contact' +import contact, { getCurrentEmployee, getName, type Employee, type Person } from '@hcengineering/contact' import { - PersonIcon, employeeByAccountStore, employeeByIdStore, + PersonIcon, personRefByAccountUuidStore } from '@hcengineering/contact-resources' import core, { getCurrentAccount, + notEmpty, + type AccountUuid, type Class, type Client, type Doc, type Ref, type Space, - type Timestamp, - type AccountUuid, - notEmpty + type Timestamp } from '@hcengineering/core' import notification, { type DocNotifyContext, type InboxNotification } from '@hcengineering/notification' import { @@ -45,19 +47,18 @@ import { isActivityNotification, isMentionNotification } from '@hcengineering/notification-resources' -import { translate, type Asset, getMetadata } from '@hcengineering/platform' +import { getMetadata, translate, type Asset } from '@hcengineering/platform' import { getClient } from '@hcengineering/presentation' -import { type AnySvelteComponent, languageStore } from '@hcengineering/ui' +import { languageStore, type AnySvelteComponent } from '@hcengineering/ui' import { classIcon, getDocLinkTitle, getDocTitle } from '@hcengineering/view-resources' import { get, writable, type Unsubscriber } from 'svelte/store' -import aiBot from '@hcengineering/ai-bot' -import { translate as aiTranslate } from '@hcengineering/ai-bot-resources' import ChannelIcon from './components/ChannelIcon.svelte' import DirectIcon from './components/DirectIcon.svelte' import { openChannelInSidebar, resetChunterLocIfEqual } from './navigation' import chunter from './plugin' import { shownTranslatedMessagesStore, translatedMessagesStore, translatingMessagesStore } from './stores' +import love, { type MeetingMinutes } from '@hcengineering/love' export async function getDmName (client: Client, space?: Space): Promise { if (space === undefined) { @@ -531,6 +532,24 @@ export async function canTranslateMessage (): Promise { return url !== '' } +export async function summarizeMessages (doc: Doc): Promise { + await aiSummarizeMessages(get(languageStore), doc._id, doc._class) +} + +export async function canSummarizeMessages (doc: Doc): Promise { + if (doc?._id === undefined) return false + + const url = getMetadata(aiBot.metadata.EndpointURL) ?? '' + if (url === '') return false + + const client = getClient() + const hierarchy = client.getHierarchy() + + if (!hierarchy.isDerived(doc._class, love.class.MeetingMinutes)) return false + + return ((doc as MeetingMinutes).transcription ?? 0) > 0 +} + export async function startConversationAction (docs?: Employee | Employee[]): Promise { if (docs === undefined) return const employees = Array.isArray(docs) ? docs : [docs] diff --git a/plugins/chunter/src/index.ts b/plugins/chunter/src/index.ts index 3fddb81e47..45e555ed65 100644 --- a/plugins/chunter/src/index.ts +++ b/plugins/chunter/src/index.ts @@ -203,7 +203,8 @@ export default plugin(chunterId, { StartConversation: '' as IntlString, ViewingThreadFromArchivedChannel: '' as IntlString, ViewingArchivedChannel: '' as IntlString, - OpenChatInSidebar: '' as IntlString + OpenChatInSidebar: '' as IntlString, + SummarizeMessages: '' as IntlString }, ids: { DMNotification: '' as Ref, @@ -221,11 +222,13 @@ export default plugin(chunterId, { LeaveChannel: '' as Ref, RemoveChannel: '' as Ref, TranslateMessage: '' as Ref, + SummarizeMessages: '' as Ref, ShowOriginalMessage: '' as Ref, CloseConversation: '' as Ref }, function: { CanTranslateMessage: '' as Resource<(doc?: Doc | Doc[]) => Promise>, + CanSummarizeMessages: '' as Resource<(doc?: Doc | Doc[]) => Promise>, OpenThreadInSidebar: '' as Resource< ( _id: Ref, diff --git a/services/ai-bot/pod-ai-bot/src/config.ts b/services/ai-bot/pod-ai-bot/src/config.ts index ea699a49ba..2757691439 100644 --- a/services/ai-bot/pod-ai-bot/src/config.ts +++ b/services/ai-bot/pod-ai-bot/src/config.ts @@ -31,6 +31,7 @@ interface Config { OpenAIModel: OpenAI.ChatModel OpenAIBaseUrl: string OpenAITranslateModel: OpenAI.ChatModel + OpenAISummaryModel: OpenAI.ChatModel MaxContentTokens: number MaxHistoryRecords: number Port: number @@ -56,6 +57,7 @@ const config: Config = (() => { OpenAIKey: process.env.OPENAI_API_KEY ?? '', OpenAIModel: (process.env.OPENAI_MODEL ?? 'gpt-4o-mini') as OpenAI.ChatModel, OpenAITranslateModel: (process.env.OPENAI_TRANSLATE_MODEL ?? 'gpt-4o-mini') as OpenAI.ChatModel, + OpenAISummaryModel: (process.env.OPENAI_SUMMARY_MODEL ?? 'gpt-4o-mini') as OpenAI.ChatModel, OpenAIBaseUrl: process.env.OPENAI_BASE_URL ?? '', MaxContentTokens: parseNumber(process.env.MAX_CONTENT_TOKENS) ?? 128 * 100, MaxHistoryRecords: parseNumber(process.env.MAX_HISTORY_RECORDS) ?? 500, diff --git a/services/ai-bot/pod-ai-bot/src/controller.ts b/services/ai-bot/pod-ai-bot/src/controller.ts index 6ce317b1b0..61c4a9d1f0 100644 --- a/services/ai-bot/pod-ai-bot/src/controller.ts +++ b/services/ai-bot/pod-ai-bot/src/controller.ts @@ -13,32 +13,48 @@ // limitations under the License. // +import { isWorkspaceLoginInfo } from '@hcengineering/account-client' import { AIEventRequest, ConnectMeetingRequest, DisconnectMeetingRequest, IdentityResponse, + PersonMessage, PostTranscriptRequest, + SummarizeMessagesRequest, + SummarizeMessagesResponse, TranslateRequest, TranslateResponse } from '@hcengineering/ai-bot' -import { AccountUuid, MeasureContext, Ref, SocialId, type WorkspaceIds, type WorkspaceUuid } from '@hcengineering/core' +import core, { + AccountUuid, + MeasureContext, + PersonId, + Ref, + SocialId, + SortingOrder, + toIdMap, + type WorkspaceIds, + type WorkspaceUuid +} from '@hcengineering/core' import { Room } from '@hcengineering/love' import { WorkspaceInfoRecord } from '@hcengineering/server-ai-bot' import { getAccountClient } from '@hcengineering/server-client' import { generateToken } from '@hcengineering/server-token' -import { htmlToMarkup, markupToJSON, jsonToHTML } from '@hcengineering/text' -import { isWorkspaceLoginInfo } from '@hcengineering/account-client' +import { htmlToMarkup, jsonToHTML, jsonToMarkup, markupToJSON } from '@hcengineering/text' import { encodingForModel } from 'js-tiktoken' import OpenAI from 'openai' +import chunter from '@hcengineering/chunter' import { StorageAdapter } from '@hcengineering/server-core' import { buildStorageFromConfig, storageConfigFromEnv } from '@hcengineering/server-storage' +import { markdownToMarkup, markupToMarkdown } from '@hcengineering/text-markdown' import config from './config' import { DbStorage } from './storage' -import { WorkspaceClient } from './workspace/workspaceClient' -import { translateHtml } from './utils/openai' import { tryAssignToWorkspace } from './utils/account' +import { summarizeMessages, translateHtml } from './utils/openai' +import { WorkspaceClient } from './workspace/workspaceClient' +import contact, { Contact, getName } from '@hcengineering/contact' const CLOSE_INTERVAL_MS = 10 * 60 * 1000 // 10 minutes @@ -204,6 +220,109 @@ export class AIControl { } } + async summarizeMessages ( + workspace: WorkspaceUuid, + req: SummarizeMessagesRequest + ): Promise { + if (this.openai === undefined) return + if (req.target === undefined || req.targetClass === undefined) return + + const wsClient = await this.getWorkspaceClient(workspace) + if (wsClient === undefined) return + + const opClient = await wsClient.opClient + if (opClient === undefined) return + + const client = wsClient.client + if (client === undefined) return + + const target = await client.findOne(req.targetClass, { _id: req.target }) + if (target === undefined) return + + const messages = await client.findAll( + chunter.class.ChatMessage, + { + attachedTo: target._id, + collection: { $in: ['messages', 'transcription'] } + }, + { + sort: { createdOn: SortingOrder.Ascending }, + limit: 5000 + } + ) + + const personIds = new Set() + for (const m of messages) { + if (m.createdBy !== undefined) personIds.add(m.createdBy) + } + const identities = await client.findAll(contact.class.SocialIdentity, { key: { $in: Array.from(personIds) } }) + const contacts = await client.findAll(contact.class.Contact, { _id: { $in: identities.map((i) => i.attachedTo) } }) + const contactById = toIdMap(contacts) + const contactByPersonId = new Map() + for (const identity of identities) { + const contact = contactById.get(identity.attachedTo) + if (contact !== undefined) contactByPersonId.set(identity.key, contact) + } + + const messagesToSummarize: PersonMessage[] = [] + + for (const m of messages) { + const author = m.createdBy + if (author === undefined) continue + + const contact = contactByPersonId.get(author) + if (contact === undefined) continue + + const personName = getName(client.getHierarchy(), contact) + const text = markupToMarkdown(markupToJSON(m.message)) + + const lastPiece = messagesToSummarize[messagesToSummarize.length - 1] + if (lastPiece?.personRef === contact._id) { + lastPiece.text += (m.collection === 'transcription' ? ' ' : '\n') + text + } else { + messagesToSummarize.push({ + personRef: contact._id, + personName, + time: m.createdOn ?? 0, + text + }) + } + } + + const summary = await summarizeMessages(this.openai, messagesToSummarize, req.lang) + if (summary === undefined) return + + const summaryMarkup = jsonToMarkup(markdownToMarkup(summary)) + + const lastMessage = await client.findOne( + chunter.class.ChatMessage, + { + attachedTo: target._id, + collection: { $in: ['messages', 'transcription', 'summary'] } + }, + { + sort: { createdOn: SortingOrder.Descending }, + limit: 1 + } + ) + + const op = opClient.apply(undefined, 'AISummarizeMessagesRequestEvent') + + if (lastMessage?.collection === 'summary' && lastMessage.createdBy === opClient.user) { + await op.update(lastMessage, { message: summaryMarkup, editedOn: Date.now() }) + } else { + await op.addCollection(chunter.class.ChatMessage, core.space.Workspace, target._id, target._class, 'summary', { + message: summaryMarkup + }) + } + await op.commit() + + return { + text: summaryMarkup, + lang: req.lang + } + } + async processEvent (workspace: WorkspaceUuid, events: AIEventRequest[]): Promise { if (this.openai === undefined) return diff --git a/services/ai-bot/pod-ai-bot/src/server/server.ts b/services/ai-bot/pod-ai-bot/src/server/server.ts index 162725970d..ed7566cc25 100644 --- a/services/ai-bot/pod-ai-bot/src/server/server.ts +++ b/services/ai-bot/pod-ai-bot/src/server/server.ts @@ -21,7 +21,8 @@ import { ConnectMeetingRequest, DisconnectMeetingRequest, AIEventRequest, - PostTranscriptRequest + PostTranscriptRequest, + SummarizeMessagesRequest } from '@hcengineering/ai-bot' import { extractToken } from '@hcengineering/server-client' import { MeasureContext } from '@hcengineering/core' @@ -73,6 +74,23 @@ export function createServer (controller: AIControl, ctx: MeasureContext): Expre }) ) + app.post( + '/summarize', + wrapRequest(async (req, res, token) => { + if (req.body == null || Array.isArray(req.body) || typeof req.body !== 'object') { + throw new ApiError(400) + } + + const response = await controller.summarizeMessages(token.workspace, req.body as SummarizeMessagesRequest) + if (response === undefined) { + throw new ApiError(500) + } + + res.status(200) + res.json(response) + }) + ) + app.post( '/connect', wrapRequest(async (_, res, token) => { diff --git a/services/ai-bot/pod-ai-bot/src/utils/openai.ts b/services/ai-bot/pod-ai-bot/src/utils/openai.ts index e38a669ff7..1d8f106ed3 100644 --- a/services/ai-bot/pod-ai-bot/src/utils/openai.ts +++ b/services/ai-bot/pod-ai-bot/src/utils/openai.ts @@ -13,11 +13,13 @@ // limitations under the License. // +import { AccountUuid, Ref } from '@hcengineering/core' import { countTokens } from '@hcengineering/openai' import { Tiktoken } from 'js-tiktoken' import OpenAI from 'openai' -import { AccountUuid } from '@hcengineering/core' +import { PersonMessage } from '@hcengineering/ai-bot' +import contact, { Contact } from '@hcengineering/contact' import config from '../config' import { HistoryRecord } from '../types' import { WorkspaceClient } from '../workspace/workspaceClient' @@ -25,7 +27,7 @@ import { getTools } from './tools' export async function translateHtml (client: OpenAI, html: string, lang: string): Promise { const response = await client.chat.completions.create({ - model: config.OpenAITranslateModel, + model: config.OpenAISummaryModel, messages: [ { role: 'system', @@ -41,6 +43,72 @@ export async function translateHtml (client: OpenAI, html: string, lang: string) return response.choices[0].message.content ?? undefined } +export async function summarizeMessages ( + client: OpenAI, + messages: PersonMessage[], + lang: string +): Promise { + const personToName = new Map, string>() + for (const m of messages) { + if (!personToName.has(m.personRef)) { + personToName.set(m.personRef, m.personName) + } + } + + const nameUsage = new Map() + for (const [personRef, name] of personToName) { + const idx = nameUsage.get(name) ?? 0 + if (idx > 0) { + personToName.set(personRef, name + ` no.${idx}`) + } + nameUsage.set(name, idx + 1) + } + + const text = messages.map((p) => '---\n\n@' + p.personName + '\n' + p.text).join('\n\n') + + const prompt = `Generate a summary from the provided sequence of messages by creating separate bullet lists for each participant, ensuring that each bullet point includes only the key points, problems and further work plans without any chit-chat, and clearly label each participant so that their individual contributions are distinctly summarized. + Use following structure for output: + **@Participant Name** + - Key point 1 + - Key point 2 + - ... + **@Participant Name** + - Key point 1 + - ... + Don't introduce any other elements of the structure. + If a bullet point implies a reference to another participant include a reference according to this format: **@Participant Name** + The response should be translated into ${lang} regardless of the original language. Don't translate the names of the participants and leave them exactly as they appear in the text.` + + const response = await client.chat.completions.create({ + model: config.OpenAIModel, + messages: [ + { + role: 'system', + content: prompt + }, + // We could also pack them into separate messages, + // but for now it seems that this option is more preferable + { + role: 'user', + content: text + } + ] + }) + + let responseText = response.choices[0].message.content ?? undefined + if (responseText === undefined) return + + const classURI = encodeURIComponent(contact.class.Contact) + for (const [personRef, name] of personToName) { + const idURI = encodeURIComponent(personRef) + const nameURI = encodeURIComponent(name) + const refString = `[](ref://?_class=${classURI}&_id=${idURI}&label=${nameURI})` + responseText = responseText.replaceAll(`**@${name}**`, refString) + } + + return responseText +} + export async function createChatCompletion ( client: OpenAI, message: OpenAI.ChatCompletionMessageParam,