Define love agent (#7098)

services/ai-bot/love-agent/.eslintrc.cjs

@@ -0,0 +1,15 @@
+module.exports = {
+  "extends": [
+    "standard-with-typescript"
+  ],
+  "ignorePatterns": ["*.json", "node_modules/*", ".eslintrc.cjs", "esbuild.config.js"],
+  "rules": {
+    "@typescript-eslint/array-type": "off",
+    "@typescript-eslint/promise-function-async": "off",
+    "@typescript-eslint/consistent-type-imports": "off"
+  },
+  parserOptions: {
+    tsconfigRootDir: __dirname,
+    project: './tsconfig.json',
+  }
+}

services/ai-bot/love-agent/.prettierrc

@@ -0,0 +1,12 @@
+{
+    "$schema": "http://json-schema.org/draft-04/schema#",
+    "trailingComma": "none",
+    "tabWidth": 2,
+    "semi": false,
+    "singleQuote": true,
+    "printWidth": 120,
+    "useTabs": false,
+    "bracketSpacing": true,
+    "proseWrap": "preserve",
+    "plugins": []
+}

services/ai-bot/love-agent/Dockerfile

@@ -0,0 +1,25 @@
+FROM node:20
+
+WORKDIR /usr/src/app
+RUN npm install --ignore-scripts=false --verbose bufferutil utf-8-validate @mongodb-js/zstd snappy --unsafe-perm
+
+RUN apt-get update
+RUN apt-get install libjemalloc2
+RUN apt-get clean
+
+ENV LD_PRELOAD=libjemalloc.so.2
+ENV MALLOC_CONF=dirty_decay_ms:1000,narenas:2,background_thread:true
+
+RUN npm install -g pnpm
+
+ENV PNPM_HOME="/pnpm"
+ENV PATH="$PNPM_HOME:$PATH"
+
+COPY package.json package.json
+COPY pnpm-lock.yaml pnpm-lock.yaml
+
+RUN pnpm install
+COPY ./lib .
+
+EXPOSE 4012
+CMD [ "node", "index.js" , "start"]

services/ai-bot/love-agent/esbuild.config.js

@@ -0,0 +1,15 @@
+import esbuild from 'esbuild';
+
+await esbuild.build({
+  entryPoints: ['src/index.ts', 'src/start.ts', 'src/config.ts', 'src/agent.ts', 'src/stt.ts'],
+  platform: 'node',
+  bundle: false,
+  minify: false,
+  outdir: 'lib',
+  keepNames: true,
+  sourcemap: 'inline',
+  allowOverwrite: true,
+  loader: {
+    '.ts': 'ts',
+  },
+}).catch(() => process.exit(1))

services/ai-bot/love-agent/package.json

@@ -0,0 +1,40 @@
+{
+  "name": "@hcengineering/love-agent",
+  "version": "0.6.0",
+  "main": "lib/index.js",
+  "types": "types/index.d.ts",
+  "author": "Anticrm Platform Contributors",
+  "license": "EPL-2.0",
+  "files": [
+    "lib/**/*",
+    "types/**/*",
+    "tsconfig.json"
+  ],
+  "type": "module",
+  "scripts": {
+    "build": "node esbuild.config.js",
+    "lint": "eslint src/**/*.ts",
+    "lint:fix": "eslint --fix src/**/*.ts",
+    "format": "pnpm lint:fix && prettier --write src/**/*.ts"
+  },
+  "devDependencies": {
+    "@types/node": "~20.11.16",
+    "@typescript-eslint/eslint-plugin": "^6.11.0",
+    "@typescript-eslint/parser": "^6.11.0",
+    "esbuild": "^0.20.2",
+    "eslint": "^8.54.0",
+    "eslint-config-prettier": "^8.8.0",
+    "eslint-config-standard-with-typescript": "^40.0.0",
+    "eslint-plugin-import": "^2.26.0",
+    "eslint-plugin-n": "^15.4.0",
+    "eslint-plugin-promise": "^6.1.1",
+    "prettier": "^3.1.0",
+    "typescript": "^5.6.3"
+  },
+  "dependencies": {
+    "@deepgram/sdk": "^3.8.1",
+    "@livekit/agents": "^0.3.3",
+    "@livekit/rtc-node": "^0.9.2",
+    "dotenv": "^16.4.5"
+  }
+}

services/ai-bot/love-agent/src/agent.ts

@@ -0,0 +1,105 @@
+//
+// Copyright © 2024 Hardcore Engineering Inc.
+//
+// Licensed under the Eclipse Public License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License. You may
+// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+import { cli, defineAgent, type JobContext, WorkerOptions, WorkerPermissions } from '@livekit/agents'
+import { fileURLToPath } from 'node:url'
+import { RemoteParticipant, RemoteTrack, RemoteTrackPublication, RoomEvent, TrackKind } from '@livekit/rtc-node'
+
+import { STT } from './stt.js'
+import { Metadata } from './type.js'
+
+function parseMetadata (metadata: string): Metadata {
+  try {
+    return JSON.parse(metadata) as Metadata
+  } catch (e) {
+    console.error('Error parsing metadata', e)
+  }
+
+  return {}
+}
+
+function applyMetadata (data: string, stt: STT): void {
+  if (data === '') return
+  const metadata = parseMetadata(data)
+
+  if (metadata.language != null) {
+    stt.updateLanguage(metadata.language)
+  }
+
+  if (metadata.transcription === true) {
+    stt.start()
+  } else if (metadata.transcription === false) {
+    stt.stop()
+  }
+}
+
+export default defineAgent({
+  entry: async (ctx: JobContext) => {
+    await ctx.connect()
+    await ctx.waitForParticipant()
+
+    const stt = new STT(ctx.room.name)
+
+    applyMetadata(ctx.room.metadata, stt)
+
+    ctx.room.on(RoomEvent.RoomMetadataChanged, (data) => {
+      applyMetadata(data, stt)
+    })
+
+    ctx.room.on(
+      RoomEvent.TrackSubscribed,
+      (track: RemoteTrack, publication: RemoteTrackPublication, participant: RemoteParticipant) => {
+        if (publication.kind === TrackKind.KIND_AUDIO) {
+          stt.subscribe(track, publication, participant)
+        }
+      }
+    )
+
+    ctx.room.on(
+      RoomEvent.TrackUnsubscribed,
+      (track: RemoteTrack, publication: RemoteTrackPublication, participant: RemoteParticipant) => {
+        if (publication.kind === TrackKind.KIND_AUDIO) {
+          stt.unsubscribe(track, publication, participant)
+        }
+      }
+    )
+
+    ctx.room.on(
+      RoomEvent.TrackMuted,
+      (publication) => {
+        if (publication.kind === TrackKind.KIND_AUDIO) {
+          stt.mute(publication.sid)
+        }
+      }
+    )
+
+    ctx.room.on(
+      RoomEvent.TrackUnmuted,
+      (publication) => {
+        if (publication.kind === TrackKind.KIND_AUDIO) {
+          stt.unmute(publication.sid)
+        }
+      }
+    )
+
+    ctx.addShutdownCallback(async () => {
+      stt.close()
+    })
+  }
+})
+
+export function runAgent (): void {
+  cli.runApp(new WorkerOptions({ agent: fileURLToPath(import.meta.url), permissions: new WorkerPermissions(true, true, true, true, [], true) }))
+}

services/ai-bot/love-agent/src/config.ts

@@ -0,0 +1,44 @@
+//
+// Copyright © 2024 Hardcore Engineering Inc.
+//
+// Licensed under the Eclipse Public License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License. You may
+// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+interface Config {
+  Port: number
+  TranscriptDelay: number
+  DeepgramApiKey: string
+  PlatformUrl: string
+  PlatformToken: string
+}
+
+const parseNumber = (str: string | undefined): number | undefined => (str !== undefined ? Number(str) : undefined)
+
+const config: Config = (() => {
+  const params: Partial<Config> = {
+    Port: parseNumber(process.env.PORT) ?? 4020,
+    DeepgramApiKey: process.env.DEEPGRAM_API_KEY,
+    TranscriptDelay: parseNumber(process.env.TRANSCRIPT_DELAY) ?? 3000,
+    PlatformUrl: process.env.PLATFORM_URL,
+    PlatformToken: process.env.PLATFORM_TOKEN
+  }
+
+  const missingEnv = (Object.keys(params) as Array<keyof Config>).filter((key) => params[key] === undefined)
+
+  if (missingEnv.length > 0) {
+    throw Error(`Missing env variables: ${missingEnv.join(', ')}`)
+  }
+
+  return params as Config
+})()
+
+export default config

services/ai-bot/love-agent/src/index.ts

@@ -0,0 +1,21 @@
+//
+// Copyright © 2024 Hardcore Engineering Inc.
+//
+// Licensed under the Eclipse Public License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License. You may
+// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+import { config } from 'dotenv'
+
+import { start } from './start.js'
+
+config()
+void start()

services/ai-bot/love-agent/src/start.ts

@@ -0,0 +1,33 @@
+//
+// Copyright © 2024 Hardcore Engineering Inc.
+//
+// Licensed under the Eclipse Public License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License. You may
+// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+import config from './config.js'
+import { runAgent } from './agent.js'
+
+export const start = async (): Promise<void> => {
+  console.log('Starting love worker', config)
+
+  runAgent()
+  const onClose = (): void => {}
+
+  process.on('SIGINT', onClose)
+  process.on('SIGTERM', onClose)
+  process.on('uncaughtException', (e: Error) => {
+    console.error(e)
+  })
+  process.on('unhandledRejection', (e: Error) => {
+    console.error(e)
+  })
+}

services/ai-bot/love-agent/src/stt.ts

@@ -0,0 +1,252 @@
+//
+// Copyright © 2024 Hardcore Engineering Inc.
+//
+// Licensed under the Eclipse Public License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License. You may
+// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+import { AudioStream, RemoteParticipant, RemoteTrack, RemoteTrackPublication } from '@livekit/rtc-node'
+import {
+  createClient,
+  DeepgramClient,
+  ListenLiveClient,
+  LiveTranscriptionEvents,
+  LiveTranscriptionEvent,
+  LiveSchema
+} from '@deepgram/sdk'
+
+import config from './config.js'
+
+const KEEP_ALIVE_INTERVAL = 10 * 1000
+
+const dgSchema: LiveSchema = {
+  model: 'nova-2-general',
+  encoding: 'linear16',
+  smart_format: true,
+  endpointing: 500,
+  interim_results: true,
+  vad_events: true,
+  utterance_end_ms: 1000,
+
+  punctuate: true,
+  language: 'en'
+}
+
+export class STT {
+  private readonly deepgram: DeepgramClient
+
+  private isInProgress = false
+  private language: string = 'en'
+
+  private readonly trackBySid = new Map<string, RemoteTrack>()
+  private readonly streamBySid = new Map<string, AudioStream>()
+  private readonly mutedTracks = new Set<string>()
+  private readonly participantBySid = new Map<string, RemoteParticipant>()
+
+  private readonly dgConnectionBySid = new Map<string, ListenLiveClient>()
+  private readonly intervalBySid = new Map<string, NodeJS.Timeout>()
+
+  private readonly transcriptsBySid = new Map<string, { value: string, startedOn: number }>()
+
+  private readonly interval: NodeJS.Timeout
+
+  constructor (private readonly name: string) {
+    this.deepgram = createClient(config.DeepgramApiKey)
+    this.interval = this.interval = setInterval(() => {
+      this.sendTranscriptToPlatform()
+    }, config.TranscriptDelay)
+  }
+
+  sendTranscriptToPlatform (): void {
+    const now = Date.now()
+    for (const [sid, transcript] of this.transcriptsBySid.entries()) {
+      if (now - transcript.startedOn > config.TranscriptDelay) {
+        void this.sendToPlatform(transcript.value, sid)
+        this.transcriptsBySid.delete(sid)
+      }
+    }
+  }
+
+  updateLanguage (language: string): void {
+    const shouldRestart = (this.language ?? 'en') !== language
+    this.language = language
+    if (shouldRestart) {
+      this.stop()
+      this.start()
+    }
+  }
+
+  start (): void {
+    if (this.isInProgress) return
+    this.isInProgress = true
+
+    for (const sid of this.trackBySid.keys()) {
+      this.processTrack(sid)
+    }
+  }
+
+  stop (): void {
+    if (!this.isInProgress) return
+    this.isInProgress = false
+    for (const sid of this.trackBySid.keys()) {
+      this.stopDeepgram(sid)
+    }
+  }
+
+  mute (sid: string): void {
+    this.mutedTracks.add(sid)
+  }
+
+  unmute (sid: string): void {
+    this.mutedTracks.delete(sid)
+  }
+
+  subscribe (track: RemoteTrack, publication: RemoteTrackPublication, participant: RemoteParticipant): void {
+    if (this.trackBySid.has(publication.sid)) return
+    this.trackBySid.set(publication.sid, track)
+    this.participantBySid.set(publication.sid, participant)
+    if (track.muted) {
+      this.mutedTracks.add(publication.sid)
+    }
+    if (this.isInProgress) {
+      this.processTrack(publication.sid)
+    }
+  }
+
+  unsubscribe (
+    _: RemoteTrack | undefined,
+    publication: RemoteTrackPublication,
+    participant: RemoteParticipant
+  ): void {
+    this.trackBySid.delete(publication.sid)
+    this.participantBySid.delete(participant.sid)
+    this.mutedTracks.delete(publication.sid)
+    this.stopDeepgram(publication.sid)
+  }
+
+  stopDeepgram (sid: string): void {
+    const stream = this.streamBySid.get(sid)
+    if (stream !== undefined) {
+      stream.close()
+    }
+
+    const dgConnection = this.dgConnectionBySid.get(sid)
+    if (dgConnection !== undefined) {
+      dgConnection.disconnect()
+    }
+
+    const interval = this.intervalBySid.get(sid)
+    if (interval !== undefined) {
+      clearInterval(interval)
+    }
+
+    this.intervalBySid.delete(sid)
+    this.dgConnectionBySid.delete(sid)
+    this.streamBySid.delete(sid)
+  }
+
+  processTrack (sid: string): void {
+    const track = this.trackBySid.get(sid)
+    if (track === undefined) return
+    if (this.dgConnectionBySid.has(sid)) return
+
+    const stream = new AudioStream(track)
+    const dgConnection = this.deepgram.listen.live({
+      ...dgSchema,
+      channels: stream.numChannels,
+      sample_rate: stream.sampleRate,
+      language: this.language ?? 'en'
+    })
+
+    const interval = setInterval(() => {
+      dgConnection.keepAlive()
+    }, KEEP_ALIVE_INTERVAL)
+
+    this.streamBySid.set(sid, stream)
+    this.dgConnectionBySid.set(track.sid, dgConnection)
+    this.intervalBySid.set(track.sid, interval)
+
+    dgConnection.on(LiveTranscriptionEvents.Open, () => {
+      dgConnection.on(LiveTranscriptionEvents.Transcript, (data: LiveTranscriptionEvent) => {
+        const transcript = data?.channel?.alternatives[0].transcript
+        if (transcript != null && transcript !== '') {
+          const prevData = this.transcriptsBySid.get(sid)
+          const prevValue = prevData?.value ?? ''
+          if (data.is_final === true) {
+            // TODO: how to join the final transcript ?
+            this.transcriptsBySid.set(sid, { value: prevValue + ' ' + transcript, startedOn: prevData?.startedOn ?? Date.now() })
+          }
+          if (data.speech_final === true) {
+            const result = this.transcriptsBySid.get(sid)?.value
+            if (result != null) {
+              void this.sendToPlatform(result, sid)
+            }
+            this.transcriptsBySid.delete(sid)
+          }
+        }
+      })
+
+      dgConnection.on(LiveTranscriptionEvents.Close, (d) => {
+        console.log('Connection closed.', d, track.sid)
+      })
+
+      dgConnection.on(LiveTranscriptionEvents.Error, (err) => {
+        console.error(err)
+      })
+    })
+
+    void this.streamToDeepgram(sid, stream)
+  }
+
+  async streamToDeepgram (sid: string, stream: AudioStream): Promise<void> {
+    for await (const frame of stream) {
+      if (!this.isInProgress) continue
+      if (this.mutedTracks.has(sid)) continue
+      const dgConnection = this.dgConnectionBySid.get(sid)
+      if (dgConnection === undefined) {
+        stream.close()
+        return
+      }
+      const buf = Buffer.from(frame.data.buffer)
+      dgConnection.send(buf)
+    }
+  }
+
+  async sendToPlatform (transcript: string, sid: string): Promise<void> {
+    const request = {
+      transcript,
+      participant: this.participantBySid.get(sid)?.identity,
+      roomName: this.name
+    }
+
+    try {
+      await fetch(`${config.PlatformUrl}/transcript`, {
+        method: 'POST',
+        headers: {
+          'Content-Type': 'application/json',
+          Authorization: 'Bearer ' + config.PlatformToken
+        },
+        body: JSON.stringify(request)
+      })
+    } catch (e) {
+      console.error('Error sending to platform', e)
+    }
+  }
+
+  close (): void {
+    clearInterval(this.interval)
+    for (const sid of this.transcriptsBySid.keys()) {
+      this.trackBySid.delete(sid)
+      this.participantBySid.delete(sid)
+      this.stopDeepgram(sid)
+    }
+  }
+}

services/ai-bot/love-agent/src/type.ts

@@ -0,0 +1,19 @@
+//
+// Copyright © 2024 Hardcore Engineering Inc.
+//
+// Licensed under the Eclipse Public License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License. You may
+// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+export interface Metadata {
+  transcription?: boolean
+  language?: string
+}

services/ai-bot/love-agent/tsconfig.json

@@ -0,0 +1,28 @@
+{
+  "compilerOptions": {
+    "module": "ESNext",
+    "esModuleInterop": true,
+    "target": "ESNext",
+    "moduleResolution": "node",
+    "sourceMap": true,
+
+    "declaration": true,
+    "strict": true,
+    "resolveJsonModule": true,
+    "skipLibCheck": true,
+    "skipDefaultLibCheck": true,
+    "declarationMap": true,
+    "disableReferencedProjectLoad": true,
+    "emitDecoratorMetadata": true,
+    "experimentalDecorators": true,
+    "incremental": true,
+    "isolatedModules": true,
+
+    "rootDir": "./src",
+    "outDir": "./lib",
+    "declarationDir": "./types",
+    "tsBuildInfoFile": ".build/build.tsbuildinfo"
+  },
+  "include": ["src/**/*"],
+  "exclude": ["node_modules"]
+}