Fix meetings transcription (#7240)

Signed-off-by: Kristina Fefelova <kristin.fefelova@gmail.com>
This commit is contained in:
Kristina 2024-11-29 19:07:46 +04:00 committed by GitHub
parent a6fb3713f6
commit f9dff9573d
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
17 changed files with 98 additions and 105 deletions

View File

@ -48,7 +48,6 @@ import {
type Builder,
Collection,
Collection as PropCollection,
Hidden,
Index,
Mixin,
Model,
@ -203,9 +202,6 @@ export class TMeetingMinutes extends TAttachedDoc implements MeetingMinutes, Tod
@ReadOnly()
declare attachedTo: Ref<Doc>
@Hidden()
sid!: string
@Prop(TypeString(), view.string.Title)
@Index(IndexKind.FullText)
title!: string

View File

@ -56,7 +56,6 @@ export async function translate (text: Markup, lang: string): Promise<TranslateR
export async function connectMeeting (
roomId: Ref<Room>,
sid: string,
language: RoomLanguage,
options: Partial<ConnectMeetingRequest>
): Promise<void> {
@ -68,7 +67,7 @@ export async function connectMeeting (
}
try {
const req: ConnectMeetingRequest = { roomId, roomSid: sid, transcription: options.transcription ?? false, language }
const req: ConnectMeetingRequest = { roomId, transcription: options.transcription ?? false, language }
await fetch(concatLink(url, 'love/connect'), {
method: 'POST',
headers: {

View File

@ -61,7 +61,6 @@ export interface TranslateResponse {
export interface ConnectMeetingRequest {
roomId: Ref<Room>
roomSid: string
language: RoomLanguage
transcription: boolean
}
@ -74,7 +73,6 @@ export interface PostTranscriptRequest {
transcript: string
participant: Ref<Person>
roomName: string
final: boolean
}
export interface IdentityResponse {

View File

@ -67,11 +67,9 @@
const client = getClient()
const hierarchy = client.getHierarchy()
if ($isConnected && $currentRoom?._id === room._id) {
const sid = await lk.getSid()
let meeting = $currentMeetingMinutes
if (meeting?.sid !== sid || meeting?.attachedTo !== room._id || meeting?.status !== MeetingStatus.Active) {
if (meeting?.attachedTo !== room._id || meeting?.status !== MeetingStatus.Active) {
meeting = await client.findOne(love.class.MeetingMinutes, {
sid,
attachedTo: room._id,
status: MeetingStatus.Active
})

View File

@ -20,7 +20,7 @@
import love from '../../plugin'
import VideoTab from './VideoTab.svelte'
import { isCurrentInstanceConnected, lk } from '../../utils'
import { isCurrentInstanceConnected } from '../../utils'
import { currentMeetingMinutes, currentRoom } from '../../stores'
import ChatTab from './ChatTab.svelte'
import TranscriptionTab from './TranscriptionTab.svelte'
@ -35,14 +35,9 @@
let isMeetingMinutesLoaded = false
let room: Room | undefined = undefined
let sid: string | undefined = undefined
$: room = $currentRoom
void lk.getSid().then((res) => {
sid = res
})
$: if (
!$isCurrentInstanceConnected ||
widgetState?.data?.room === undefined ||
@ -52,15 +47,10 @@
closeWidget(love.ids.MeetingWidget)
}
$: if (meetingMinutes?.sid !== sid) {
meetingMinutes = undefined
isMeetingMinutesLoaded = false
}
$: if (sid != null && room !== undefined) {
$: if (room !== undefined) {
meetingQuery.query(
love.class.MeetingMinutes,
{ sid, attachedTo: room._id, status: MeetingStatus.Active },
{ attachedTo: room._id, status: MeetingStatus.Active },
async (res) => {
meetingMinutes = res[0]
if (meetingMinutes) {
@ -72,7 +62,7 @@
} else {
meetingQuery.unsubscribe()
meetingMinutes = undefined
isMeetingMinutesLoaded = sid !== undefined
isMeetingMinutesLoaded = false
}
function handleClose (): void {

View File

@ -414,6 +414,13 @@ async function initRoomMetadata (metadata: string | undefined): Promise<void> {
isTranscription.set(data.transcription === TranscriptionStatus.InProgress)
const room = get(currentRoom)
const meetingMinutes = get(currentMeetingMinutes)
const isValidMeeting =
meetingMinutes != null && meetingMinutes.attachedTo === room?._id && meetingMinutes.status === MeetingStatus.Active
if (room != null && !isValidMeeting) {
await initMeetingMinutes(room)
}
if (
(data.transcription == null || data.transcription === TranscriptionStatus.Idle) &&
@ -422,7 +429,7 @@ async function initRoomMetadata (metadata: string | undefined): Promise<void> {
await startTranscription(room)
}
if (data.recording == null && room?.startWithRecording === true) {
if (get(isRecordingAvailable) && data.recording == null && room?.startWithRecording === true) {
await record(room)
}
}
@ -657,11 +664,9 @@ async function navigateToOfficeDoc (hierarchy: Hierarchy, object: Doc): Promise<
navigate(loc)
}
async function openMeetingMinutes (room: Room): Promise<void> {
async function initMeetingMinutes (room: Room): Promise<void> {
const client = getClient()
const sid = await lk.getSid()
const doc = await client.findOne(love.class.MeetingMinutes, {
sid,
attachedTo: room._id,
status: MeetingStatus.Active
})
@ -682,7 +687,6 @@ async function openMeetingMinutes (room: Room): Promise<void> {
const newDoc: MeetingMinutes = {
_id,
_class: love.class.MeetingMinutes,
sid,
attachedTo: room._id,
attachedToClass: room._class,
collection: 'meetings',
@ -699,7 +703,7 @@ async function openMeetingMinutes (room: Room): Promise<void> {
room._id,
room._class,
'meetings',
{ sid, title: newDoc.title, description: newDoc.description, status: newDoc.status },
{ title: newDoc.title, description: newDoc.description, status: newDoc.status },
_id
)
currentMeetingMinutes.set(newDoc)
@ -734,7 +738,7 @@ export async function connectRoom (
3,
1000
)
await openMeetingMinutes(room)
await initMeetingMinutes(room)
} catch (err) {
console.error(err)
await leaveRoom(currentInfo, get(myOffice))
@ -993,8 +997,7 @@ export async function startTranscription (room: Room): Promise<void> {
const current = get(currentRoom)
if (current === undefined || room._id !== current._id) return
const sid = await lk.getSid()
await connectMeeting(room._id, sid, room.language, { transcription: true })
await connectMeeting(room._id, room.language, { transcription: true })
}
export async function stopTranscription (room: Room): Promise<void> {

View File

@ -166,8 +166,6 @@ export enum MeetingStatus {
}
export interface MeetingMinutes extends AttachedDoc {
sid: string
title: string
description: MarkupBlobRef | null

View File

@ -1,7 +1,7 @@
import esbuild from 'esbuild';
await esbuild.build({
entryPoints: ['src/index.ts', 'src/start.ts', 'src/config.ts', 'src/agent.ts', 'src/stt.ts', 'src/type.ts'],
entryPoints: ['src/**/*.ts'],
platform: 'node',
bundle: false,
minify: false,

View File

@ -35,8 +35,8 @@
},
"dependencies": {
"@deepgram/sdk": "^3.9.0",
"@livekit/agents": "^0.4.1",
"@livekit/rtc-node": "^0.11.1",
"@livekit/agents": "^0.4.6",
"@livekit/rtc-node": "^0.12.1",
"dotenv": "^16.4.5"
}
}

View File

@ -12,11 +12,11 @@ importers:
specifier: ^3.9.0
version: 3.9.0
'@livekit/agents':
specifier: ^0.4.1
version: 0.4.1
specifier: ^0.4.6
version: 0.4.6(@livekit/rtc-node@0.12.1)
'@livekit/rtc-node':
specifier: ^0.11.1
version: 0.11.1
specifier: ^0.12.1
version: 0.12.1
dotenv:
specifier: ^16.4.5
version: 16.4.5
@ -243,8 +243,10 @@ packages:
resolution: {integrity: sha512-93zYdMES/c1D69yZiKDBj0V24vqNzB/koF26KPaagAfd3P/4gUlh3Dys5ogAK+Exi9QyzlD8x/08Zt7wIKcDcA==}
deprecated: Use @eslint/object-schema instead
'@livekit/agents@0.4.1':
resolution: {integrity: sha512-zZnd19CWvm1i6PKzAgUw6gLdZOJ/QKzbIVSLNAZPQphCEEg3cPoe3fEf9XE7o9+n6e5sZ6FfmSUh/c7jxWmujw==}
'@livekit/agents@0.4.6':
resolution: {integrity: sha512-SVA6tW3o1ZVdQ0fGWwie+mMB4jkcNZzI0iYrOb0L9lDs9vW4C/4YlVtp8bOQSCxisG8HsCy+beVKbMTd0Mx9Fg==}
peerDependencies:
'@livekit/rtc-node': ^0.11.1
'@livekit/mutex@1.1.0':
resolution: {integrity: sha512-XRLG+z/0uoyDioupjUiskjI06Y51U/IXVPJn7qJ+R3J75XX01irYVBM9MpxeJahpVoe9QhU4moIEolX+HO9U9g==}
@ -252,38 +254,38 @@ packages:
'@livekit/protocol@1.27.1':
resolution: {integrity: sha512-ISEp7uWdV82mtCR1eyHFTzdRZTVbe2+ZztjmjiMPzR/KPrI1Ma/u5kLh87NNuY3Rn8wv1VlEvGHHsFjQ+dKVUw==}
'@livekit/rtc-node-darwin-arm64@0.11.1':
resolution: {integrity: sha512-M+Ui87H06ae19GGI7r937dS6hI84MBBTQAkkNlL7qd+pvdCAk25u0FYa8r4SOElKJ0VR3AbzeDoXTihLgpvjMg==}
'@livekit/rtc-node-darwin-arm64@0.12.1':
resolution: {integrity: sha512-GF+QnRp7yBK4AyeG3eZiuAzbQOH4+bkLSyTgb5A1CzGEca2XD5CSgKUeGtIgIglq0XmYzN6bvl260xUWPl0WCQ==}
engines: {node: '>= 10'}
cpu: [arm64]
os: [darwin]
'@livekit/rtc-node-darwin-x64@0.11.1':
resolution: {integrity: sha512-7G92fyuK2p+jdTH2cUJTNeAmtknTGsXuy0xbI727V7VzQvHFDXULCExRlgwn4t9TxvNlIjUpiltiQ6RCSai6zw==}
'@livekit/rtc-node-darwin-x64@0.12.1':
resolution: {integrity: sha512-uORO+/W5jvCfuFcmzQ4q5WCgJX5EW7eUxgfUHSaIuzcwgxBxG4yagWDl9WkK6MY9V1j6fhAiNlJ3hPl8cYGUjA==}
engines: {node: '>= 10'}
cpu: [x64]
os: [darwin]
'@livekit/rtc-node-linux-arm64-gnu@0.11.1':
resolution: {integrity: sha512-vqZN9+87Pvxit7auYVW69M+GvUPnf+EwipIJ92GgCJA3Ir1Tcceu5ud5/Ic+0FzSoV0cotVVlQNm74F0tQvyCg==}
'@livekit/rtc-node-linux-arm64-gnu@0.12.1':
resolution: {integrity: sha512-8LIHq0nSwCjUMVPNfrohupH2ynhEFcYmidYa1KmHLkVlW+gyXBfy+QE7voAbXhTSbLZLDX58N5KGCQa61LSVAA==}
engines: {node: '>= 10'}
cpu: [arm64]
os: [linux]
'@livekit/rtc-node-linux-x64-gnu@0.11.1':
resolution: {integrity: sha512-smHZUMfgILQh6/eoauYNe/VlKwQCp4/4jWxiIADHY+mtDtVSvQ9zB6y4GP8FrpohRwFWesKCUpvPBypU0Icrng==}
'@livekit/rtc-node-linux-x64-gnu@0.12.1':
resolution: {integrity: sha512-8x5IKzcTmwGyy0I0CU5vzealCl7c0Fp4KMBakq/uUOavbh/Rs1nAsjT8flrXC09F4njXD1E9uYN6CMTN9uZeKw==}
engines: {node: '>= 10'}
cpu: [x64]
os: [linux]
'@livekit/rtc-node-win32-x64-msvc@0.11.1':
resolution: {integrity: sha512-bTWVtb+UiRPFjiuhrqq40gt5vs5mMPTa1e+kd2jGQPTOlKZPLArQ0WgFcep2TAy1zmcpOgfeM1XRPVFhZl7G1A==}
'@livekit/rtc-node-win32-x64-msvc@0.12.1':
resolution: {integrity: sha512-zPbvUnpHdDlkEW3AN8wIqa3Wwae26MV/gQmqmP+3AjZRog8hMZy9Fe48lJgZR6VobjFjw99wSgIytYZCdIOUqQ==}
engines: {node: '>= 10'}
cpu: [x64]
os: [win32]
'@livekit/rtc-node@0.11.1':
resolution: {integrity: sha512-EFw+giPll12fcXATZpN2zKkE3umYJAdHvfjW+Yu0aBjwfxbUBXu8rz6le2CzDNvGmRwR888DSZXFZfYikwZgiw==}
'@livekit/rtc-node@0.12.1':
resolution: {integrity: sha512-VNRobdI/CmZ8mVUV105uqFOH7BMDmK8VsNq4GrpWAdXKnQMVKoAjLmvFU2LVQMKP4qE9BG3aTi59BGYt/PCLiA==}
engines: {node: '>= 18'}
'@livekit/typed-emitter@3.0.0':
@ -1556,11 +1558,11 @@ snapshots:
'@humanwhocodes/object-schema@2.0.3': {}
'@livekit/agents@0.4.1':
'@livekit/agents@0.4.6(@livekit/rtc-node@0.12.1)':
dependencies:
'@livekit/mutex': 1.1.0
'@livekit/protocol': 1.27.1
'@livekit/rtc-node': 0.11.1
'@livekit/rtc-node': 0.12.1
'@livekit/typed-emitter': 3.0.0
commander: 12.1.0
livekit-server-sdk: 2.8.1
@ -1578,32 +1580,32 @@ snapshots:
dependencies:
'@bufbuild/protobuf': 1.10.0
'@livekit/rtc-node-darwin-arm64@0.11.1':
'@livekit/rtc-node-darwin-arm64@0.12.1':
optional: true
'@livekit/rtc-node-darwin-x64@0.11.1':
'@livekit/rtc-node-darwin-x64@0.12.1':
optional: true
'@livekit/rtc-node-linux-arm64-gnu@0.11.1':
'@livekit/rtc-node-linux-arm64-gnu@0.12.1':
optional: true
'@livekit/rtc-node-linux-x64-gnu@0.11.1':
'@livekit/rtc-node-linux-x64-gnu@0.12.1':
optional: true
'@livekit/rtc-node-win32-x64-msvc@0.11.1':
'@livekit/rtc-node-win32-x64-msvc@0.12.1':
optional: true
'@livekit/rtc-node@0.11.1':
'@livekit/rtc-node@0.12.1':
dependencies:
'@bufbuild/protobuf': 2.2.2
'@livekit/mutex': 1.1.0
'@livekit/typed-emitter': 3.0.0
optionalDependencies:
'@livekit/rtc-node-darwin-arm64': 0.11.1
'@livekit/rtc-node-darwin-x64': 0.11.1
'@livekit/rtc-node-linux-arm64-gnu': 0.11.1
'@livekit/rtc-node-linux-x64-gnu': 0.11.1
'@livekit/rtc-node-win32-x64-msvc': 0.11.1
'@livekit/rtc-node-darwin-arm64': 0.12.1
'@livekit/rtc-node-darwin-x64': 0.12.1
'@livekit/rtc-node-linux-arm64-gnu': 0.12.1
'@livekit/rtc-node-linux-x64-gnu': 0.12.1
'@livekit/rtc-node-win32-x64-msvc': 0.12.1
'@livekit/typed-emitter@3.0.0': {}

View File

@ -17,7 +17,7 @@ import { cli, defineAgent, type JobContext, JobRequest, WorkerOptions } from '@l
import { fileURLToPath } from 'node:url'
import { RemoteParticipant, RemoteTrack, RemoteTrackPublication, RoomEvent, TrackKind } from '@livekit/rtc-node'
import { STT } from './stt.js'
import { STT } from './deepgram/stt.js'
import { Metadata, TranscriptionStatus } from './type.js'
import config from './config.js'

View File

@ -24,7 +24,7 @@ import {
SOCKET_STATES
} from '@deepgram/sdk'
import config from './config.js'
import config from '../config.js'
const KEEP_ALIVE_INTERVAL = 10 * 1000
@ -165,9 +165,9 @@ export class STT {
}
if (data.speech_final === true) {
void this.sendToPlatform(transcript, sid, true)
void this.sendToPlatform(transcript, sid)
} else if (data.is_final === true) {
void this.sendToPlatform(transcript, sid, false)
void this.sendToPlatform(transcript, sid)
}
})
@ -199,12 +199,11 @@ export class STT {
}
}
async sendToPlatform (transcript: string, sid: string, isFinal = false): Promise<void> {
async sendToPlatform (transcript: string, sid: string): Promise<void> {
const request = {
transcript,
participant: this.participantBySid.get(sid)?.identity,
roomName: this.name,
final: isFinal
roomName: this.name
}
try {

View File

@ -308,6 +308,6 @@ export class AIControl {
const wsClient = await this.getWorkspaceClient(workspace)
if (wsClient === undefined) return
await wsClient.processLoveTranscript(request.transcript, request.participant, roomId as Ref<Room>, request.final)
await wsClient.processLoveTranscript(request.transcript, request.participant, roomId as Ref<Room>)
}
}

View File

@ -1,3 +1,18 @@
//
// Copyright © 2024 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
import { ConnectMeetingRequest } from '@hcengineering/ai-bot'
import chunter from '@hcengineering/chunter'
import { Person } from '@hcengineering/contact'
@ -27,7 +42,6 @@ import { jsonToMarkup, MarkupNodeType } from '@hcengineering/text'
import config from '../config'
export class LoveController {
private readonly roomSidById = new Map<Ref<Room>, string>()
private readonly connectedRooms = new Set<Ref<Room>>()
private participantsInfo: ParticipantInfo[] = []
@ -44,7 +58,7 @@ export class LoveController {
void this.initData()
setInterval(() => {
void this.checkConnection()
}, 5000)
}, 10 * 1000)
}
getIdentity (): { identity: Ref<Person>, name: string } {
@ -119,12 +133,10 @@ export class LoveController {
if (room === undefined) {
this.ctx.error('Room not found', request)
this.roomSidById.delete(request.roomId)
this.connectedRooms.delete(request.roomId)
return
}
this.roomSidById.set(request.roomId, request.roomSid)
this.connectedRooms.add(request.roomId)
this.ctx.info('Connecting', { room: room.name, roomId: room._id })
@ -155,11 +167,10 @@ export class LoveController {
await stopTranscription(this.token, getTokenRoomName(this.workspace, room.name, room._id), room.name)
}
this.roomSidById.delete(roomId)
this.connectedRooms.delete(roomId)
}
async processTranscript (text: string, person: Ref<Person>, roomId: Ref<Room>, final: boolean): Promise<void> {
async processTranscript (text: string, person: Ref<Person>, roomId: Ref<Room>): Promise<void> {
const room = await this.getRoom(roomId)
const participant = await this.getRoomParticipant(roomId, person)
@ -167,14 +178,8 @@ export class LoveController {
return
}
const sid = this.roomSidById.get(roomId)
if (sid === undefined) {
return
}
const personAccount = this.client.getModel().getAccountByPersonId(participant.person)[0]
const doc = await this.getMeetingMinutes(room, sid)
const doc = await this.getMeetingMinutes(room)
if (doc === undefined) return
const op = this.client.apply(undefined, undefined, true)
@ -210,13 +215,10 @@ export class LoveController {
)
}
async getMeetingMinutes (room: Room, sid: string): Promise<MeetingMinutes | undefined> {
if (sid === '') return undefined
async getMeetingMinutes (room: Room): Promise<MeetingMinutes | undefined> {
const doc =
this.meetingMinutes.find(
(m) => m.sid === sid && m.attachedTo === room._id && m.status === MeetingStatus.Active
) ?? (await this.client.findOne(love.class.MeetingMinutes, { sid, room: room._id, status: MeetingStatus.Active }))
this.meetingMinutes.find((m) => m.attachedTo === room._id && m.status === MeetingStatus.Active) ??
(await this.client.findOne(love.class.MeetingMinutes, { attachedTo: room._id, status: MeetingStatus.Active }))
if (doc === undefined) {
return undefined

View File

@ -709,7 +709,7 @@ export class WorkspaceClient {
await this.love.disconnect(request.roomId)
}
async processLoveTranscript (text: string, participant: Ref<Person>, room: Ref<Room>, final: boolean): Promise<void> {
async processLoveTranscript (text: string, participant: Ref<Person>, room: Ref<Room>): Promise<void> {
// Just wait initialization
await this.opClient
@ -718,7 +718,7 @@ export class WorkspaceClient {
return
}
await this.love.processTranscript(text, participant, room, final)
await this.love.processTranscript(text, participant, room)
}
async getLoveIdentity (): Promise<IdentityResponse | undefined> {

View File

@ -86,7 +86,7 @@ export const main = async (): Promise<void> => {
const filename = stripPrefix(prefix, res.filename)
const storedBlob = await storageAdapter.stat(ctx, data.workspaceId, filename)
if (storedBlob !== undefined) {
const client = await WorkspaceClient.create(data.workspace)
const client = await WorkspaceClient.create(data.workspace, ctx)
await client.saveFile(filename, data.name, storedBlob, data.meetingMinutes)
await client.close()
}
@ -137,6 +137,7 @@ export const main = async (): Promise<void> => {
const name = `${room}_${dateStr}.mp4`
const id = await startRecord(storageConfig, egressClient, roomClient, roomName, workspace)
dataByUUID.set(id, { name, workspace: workspace.name, workspaceId: workspace, meetingMinutes })
ctx.info('Start recording', { workspace: workspace.name, roomName, meetingMinutes })
res.send()
} catch (e) {
console.error(e)

View File

@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
import core, { Client, Ref, TxOperations, type Blob, Data } from '@hcengineering/core'
import core, { Client, Ref, TxOperations, type Blob, Data, MeasureContext } from '@hcengineering/core'
import drive, { createFile } from '@hcengineering/drive'
import love, { MeetingMinutes } from '@hcengineering/love'
import { generateToken } from '@hcengineering/server-token'
@ -23,10 +23,13 @@ import config from './config'
export class WorkspaceClient {
private client!: TxOperations
private constructor (private readonly workspace: string) {}
private constructor (
private readonly workspace: string,
private readonly ctx: MeasureContext
) {}
static async create (workspace: string): Promise<WorkspaceClient> {
const instance = new WorkspaceClient(workspace)
static async create (workspace: string, ctx: MeasureContext): Promise<WorkspaceClient> {
const instance = new WorkspaceClient(workspace, ctx)
await instance.initClient(workspace)
return instance
}
@ -43,6 +46,7 @@ export class WorkspaceClient {
}
async saveFile (uuid: string, name: string, blob: Blob, meetingMinutes?: Ref<MeetingMinutes>): Promise<void> {
this.ctx.info('Save recording', { workspace: this.workspace, meetingMinutes })
const current = await this.client.findOne(drive.class.Drive, { _id: love.space.Drive })
if (current === undefined) {
await this.client.createDoc(
@ -83,7 +87,10 @@ export class WorkspaceClient {
if (ref === undefined) return
const meeting = await this.client.findOne(love.class.MeetingMinutes, { _id: ref })
if (meeting === undefined) return
if (meeting === undefined) {
this.ctx.error('Meeting not found', { _id: ref })
return
}
await this.client.addCollection(
attachment.class.Attachment,