wieerwill/platform
1
0
Fork 0
mirror of https://github.com/hcengineering/platform.git synced 2025-05-11 01:40:32 +00:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Try nova-3 multi-lang (#8791)
Some checks are pending
CI / build (push) Waiting to run
Details
CI / svelte-check (push) Blocked by required conditions
Details
CI / formatting (push) Blocked by required conditions
Details
CI / test (push) Blocked by required conditions
Details
CI / uitest (push) Waiting to run
Details
CI / uitest-pg (push) Waiting to run
Details
CI / uitest-qms (push) Waiting to run
Details
CI / uitest-workspaces (push) Waiting to run
Details
CI / docker-build (push) Blocked by required conditions
Details
CI / dist-build (push) Blocked by required conditions
Details

Browse Source 
Signed-off-by: Kristina Fefelova <kristin.fefelova@gmail.com>
This commit is contained in:
Kristina 2025-04-30 20:24:44 +04:00 committed by GitHub
parent 230e1ec418
commit c2defd9638
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 51 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
services/ai-bot/love-agent/package.json
View File

@ -34,9 +34,9 @@
"typescript": "^5.6.3" "typescript": "^5.6.3"
}, },
"dependencies": { "dependencies": {
"@deepgram/sdk": "^3.11.3", "@deepgram/sdk": "^3.12.1",
"@livekit/agents": "^0.7.2", "@livekit/agents": "^0.7.4",
"@livekit/rtc-node": "^0.13.10", "@livekit/rtc-node": "^0.13.11",
"dotenv": "^16.4.5", "dotenv": "^16.4.5",
"ws": "^8.18.0" "ws": "^8.18.0"
} }

74
services/ai-bot/love-agent/pnpm-lock.yaml
View File

@ -9,14 +9,14 @@ importers:
.: .:
dependencies: dependencies:
'@deepgram/sdk': '@deepgram/sdk':
specifier: ^3.11.3 specifier: ^3.12.1
version: 3.11.3 version: 3.12.1
'@livekit/agents': '@livekit/agents':
specifier: ^0.7.2 specifier: ^0.7.4
version: 0.7.2(@livekit/rtc-node@0.13.10) version: 0.7.4(@livekit/rtc-node@0.13.11)
'@livekit/rtc-node': '@livekit/rtc-node':
specifier: ^0.13.10 specifier: ^0.13.11
version: 0.13.10 version: 0.13.11
dotenv: dotenv:
specifier: ^16.4.5 specifier: ^16.4.5
version: 16.4.7 version: 16.4.7
@ -70,8 +70,8 @@ packages:
resolution: {integrity: sha512-8B1C/oTxTxyHlSFubAhNRgCbQ2SQ5wwvtlByn8sDYZvdDtdn/VE2yEPZ4BvUnrKWmsbTQY6/ooLV+9Ka2qmDSQ==} resolution: {integrity: sha512-8B1C/oTxTxyHlSFubAhNRgCbQ2SQ5wwvtlByn8sDYZvdDtdn/VE2yEPZ4BvUnrKWmsbTQY6/ooLV+9Ka2qmDSQ==}
engines: {node: '>=18.0.0'} engines: {node: '>=18.0.0'}
'@deepgram/sdk@3.11.3': '@deepgram/sdk@3.12.1':
resolution: {integrity: sha512-erOxfbci5beg2wiEUwmloXsg+JcSW+PGk2BVTMFaQvFxSOdJHN+tw8ymNktwxijaKrrL0osqkEpZ1YWbCxMJbQ==} resolution: {integrity: sha512-MNnCnlyxdf0IY4Dkt+YcgCb8sy14zAzJ5BCNMwzv20tSxweoj8mk6eM063zQTuHWc3x2giKvS9x6/IBtV9zJLg==}
engines: {node: '>=18.0.0'} engines: {node: '>=18.0.0'}
'@esbuild/aix-ppc64@0.20.2': '@esbuild/aix-ppc64@0.20.2':
@ -243,10 +243,10 @@ packages:
resolution: {integrity: sha512-93zYdMES/c1D69yZiKDBj0V24vqNzB/koF26KPaagAfd3P/4gUlh3Dys5ogAK+Exi9QyzlD8x/08Zt7wIKcDcA==} resolution: {integrity: sha512-93zYdMES/c1D69yZiKDBj0V24vqNzB/koF26KPaagAfd3P/4gUlh3Dys5ogAK+Exi9QyzlD8x/08Zt7wIKcDcA==}
deprecated: Use @eslint/object-schema instead deprecated: Use @eslint/object-schema instead
'@livekit/agents@0.7.2': '@livekit/agents@0.7.4':
resolution: {integrity: sha512-FBORsCwwRb0WK9Tik8HecbTBSOG0Ibr4pUZM4eDBhmlJHtDPdcXTSrir6R/gAoQMGS/q1I3qyUNnG72jx6qKDw==} resolution: {integrity: sha512-ocpmr1IzWj1oi8W0paAA7Fhu6KtnCcuwbKU1P6fPa2aTsyD4vciRRec4matzP6MlS/LoCGQYnTG0ACLGwPqDow==}
peerDependencies: peerDependencies:
'@livekit/rtc-node': ^0.13.4 '@livekit/rtc-node': ^0.13.11
'@livekit/mutex@1.1.1': '@livekit/mutex@1.1.1':
resolution: {integrity: sha512-EsshAucklmpuUAfkABPxJNhzj9v2sG7JuzFDL4ML1oJQSV14sqrpTYnsaOudMAw9yOaW53NU3QQTlUQoRs4czw==} resolution: {integrity: sha512-EsshAucklmpuUAfkABPxJNhzj9v2sG7JuzFDL4ML1oJQSV14sqrpTYnsaOudMAw9yOaW53NU3QQTlUQoRs4czw==}
@ -254,38 +254,38 @@ packages:
'@livekit/protocol@1.32.1': '@livekit/protocol@1.32.1':
resolution: {integrity: sha512-nzVSZ1Sp+6smF/nrcZU5xg6ieVanC1OeZcRqnRNnH9XFGY8txevboXdeBlj5j6E62YdP7X7NvotR6Tep5gSHiw==} resolution: {integrity: sha512-nzVSZ1Sp+6smF/nrcZU5xg6ieVanC1OeZcRqnRNnH9XFGY8txevboXdeBlj5j6E62YdP7X7NvotR6Tep5gSHiw==}
'@livekit/rtc-node-darwin-arm64@0.13.10': '@livekit/rtc-node-darwin-arm64@0.13.11':
resolution: {integrity: sha512-GNSHjpoHqIK9l9kgBpWtk+x5MFhfLv3BdLATsfiu9cE34mgmIr25H7NuO1z+SasFze3RmP6vGpPcHHjw23x/WA==} resolution: {integrity: sha512-XqbVUW5rVrRdVzxUI3+f8K6A1bnzAXytbCmPx7YiGOXVNRCV1kC84R7fap7OgrgN/rAtObhyYK882xdJVG/BYA==}
engines: {node: '>= 10'} engines: {node: '>= 10'}
cpu: [arm64] cpu: [arm64]
os: [darwin] os: [darwin]
'@livekit/rtc-node-darwin-x64@0.13.10': '@livekit/rtc-node-darwin-x64@0.13.11':
resolution: {integrity: sha512-YoKqFUQrqw3aukk0QNWdKDA0D0vannXYaiJbKQqt5SJpUbqxhe24hJM4zXcwPvEfdl1uX6Q6oUo5owDoq6cM1Q==} resolution: {integrity: sha512-UFe9Lp+7Z8UZcJq2oOH8+6nCKWlX0PVorB4jeCRZuVa4QL2PL1CcGvo9/kNNw5aA25AkPUgDjMXj2WbfEPNMKA==}
engines: {node: '>= 10'} engines: {node: '>= 10'}
cpu: [x64] cpu: [x64]
os: [darwin] os: [darwin]
'@livekit/rtc-node-linux-arm64-gnu@0.13.10': '@livekit/rtc-node-linux-arm64-gnu@0.13.11':
resolution: {integrity: sha512-NkruUR4rY0sYdR8wZCZDJbNtljl922mJbWJm3cGv+mNjBSHEYMjLiLliryM8jdclQeVSo4SFwcTEpvSSgZdpYw==} resolution: {integrity: sha512-GuJtl1nJhJnFEMI9plJqlIJ0BJCWuynJzbhhD7Yd/Zuw/NYzzzIf+wQ2mIZ0Zk9/EUV4oMYJqacJiZXvOvUQ3A==}
engines: {node: '>= 10'} engines: {node: '>= 10'}
cpu: [arm64] cpu: [arm64]
os: [linux] os: [linux]
'@livekit/rtc-node-linux-x64-gnu@0.13.10': '@livekit/rtc-node-linux-x64-gnu@0.13.11':
resolution: {integrity: sha512-CrTkAQqjnV5YoL1wawm5k4aQqBnn7hzNow4NAF73ef+n8GY1gw7M2KofItR3uv7b9yEeVr5Kkf9bnjFAZoT71g==} resolution: {integrity: sha512-Zi7Elg29JSmDzikxL2Q9YAAka2Khi7GwYHYBv69W6XXHqz3MN4wtnUGShclmqC7aITkHF0tVNLHdexFmMc3trA==}
engines: {node: '>= 10'} engines: {node: '>= 10'}
cpu: [x64] cpu: [x64]
os: [linux] os: [linux]
'@livekit/rtc-node-win32-x64-msvc@0.13.10': '@livekit/rtc-node-win32-x64-msvc@0.13.11':
resolution: {integrity: sha512-C1U+hEitE7Bv2MAuxtBOzJYWwTZGLoVK4A0RK9b3wwOCU7+vSHHIYBeQizb2wsHYLruEQrLVuli0HjqG98HyoA==} resolution: {integrity: sha512-NYemYGbc271SFv+ttYaNvEKLWmwkJqn988xOasq+lWd31kuhj0krR2cThM07HKhgoQwOfdyIcSV87b0mjPzb/A==}
engines: {node: '>= 10'} engines: {node: '>= 10'}
cpu: [x64] cpu: [x64]
os: [win32] os: [win32]
'@livekit/rtc-node@0.13.10': '@livekit/rtc-node@0.13.11':
resolution: {integrity: sha512-3tU1MTpAWblMxVKLBKXe6v4IfOcN1Wq8mRcGNcZF65MD2NNhpWvKQUmiCwR5FWEYJc/1VnZonLkIZsTXdIy++w==} resolution: {integrity: sha512-yq9uNRK+cdee0W6w0HPMSjTHovUteY4t4ZFrTdmpNt7fg/VxaJkdpXaG7cg8t+RX0pBT/NHskSQ4WFrqumezZg==}
engines: {node: '>= 18'} engines: {node: '>= 18'}
'@livekit/typed-emitter@3.0.0': '@livekit/typed-emitter@3.0.0':
@ -1528,7 +1528,7 @@ snapshots:
dependencies: dependencies:
dayjs: 1.11.13 dayjs: 1.11.13
'@deepgram/sdk@3.11.3': '@deepgram/sdk@3.12.1':
dependencies: dependencies:
'@deepgram/captions': 1.2.0 '@deepgram/captions': 1.2.0
'@types/node': 18.19.68 '@types/node': 18.19.68
@ -1645,11 +1645,11 @@ snapshots:
'@humanwhocodes/object-schema@2.0.3': {} '@humanwhocodes/object-schema@2.0.3': {}
'@livekit/agents@0.7.2(@livekit/rtc-node@0.13.10)': '@livekit/agents@0.7.4(@livekit/rtc-node@0.13.11)':
dependencies: dependencies:
'@livekit/mutex': 1.1.1 '@livekit/mutex': 1.1.1
'@livekit/protocol': 1.32.1 '@livekit/protocol': 1.32.1
'@livekit/rtc-node': 0.13.10 '@livekit/rtc-node': 0.13.11
'@livekit/typed-emitter': 3.0.0 '@livekit/typed-emitter': 3.0.0
commander: 12.1.0 commander: 12.1.0
livekit-server-sdk: 2.9.7 livekit-server-sdk: 2.9.7
@ -1667,22 +1667,22 @@ snapshots:
dependencies: dependencies:
'@bufbuild/protobuf': 1.10.0 '@bufbuild/protobuf': 1.10.0
'@livekit/rtc-node-darwin-arm64@0.13.10': '@livekit/rtc-node-darwin-arm64@0.13.11':
optional: true optional: true
'@livekit/rtc-node-darwin-x64@0.13.10': '@livekit/rtc-node-darwin-x64@0.13.11':
optional: true optional: true
'@livekit/rtc-node-linux-arm64-gnu@0.13.10': '@livekit/rtc-node-linux-arm64-gnu@0.13.11':
optional: true optional: true
'@livekit/rtc-node-linux-x64-gnu@0.13.10': '@livekit/rtc-node-linux-x64-gnu@0.13.11':
optional: true optional: true
'@livekit/rtc-node-win32-x64-msvc@0.13.10': '@livekit/rtc-node-win32-x64-msvc@0.13.11':
optional: true optional: true
'@livekit/rtc-node@0.13.10': '@livekit/rtc-node@0.13.11':
dependencies: dependencies:
'@bufbuild/protobuf': 1.10.0 '@bufbuild/protobuf': 1.10.0
'@livekit/mutex': 1.1.1 '@livekit/mutex': 1.1.1
@ -1690,11 +1690,11 @@ snapshots:
pino: 9.6.0 pino: 9.6.0
pino-pretty: 13.0.0 pino-pretty: 13.0.0
optionalDependencies: optionalDependencies:
'@livekit/rtc-node-darwin-arm64': 0.13.10 '@livekit/rtc-node-darwin-arm64': 0.13.11
'@livekit/rtc-node-darwin-x64': 0.13.10 '@livekit/rtc-node-darwin-x64': 0.13.11
'@livekit/rtc-node-linux-arm64-gnu': 0.13.10 '@livekit/rtc-node-linux-arm64-gnu': 0.13.11
'@livekit/rtc-node-linux-x64-gnu': 0.13.10 '@livekit/rtc-node-linux-x64-gnu': 0.13.11
'@livekit/rtc-node-win32-x64-msvc': 0.13.10 '@livekit/rtc-node-win32-x64-msvc': 0.13.11
'@livekit/typed-emitter@3.0.0': {} '@livekit/typed-emitter@3.0.0': {}

4
services/ai-bot/love-agent/src/config.ts
View File

@ -18,7 +18,6 @@ import { SttProvider } from './type.js'
interface Config { interface Config {
DeepgramApiKey: string DeepgramApiKey: string
DeepgramModel: string DeepgramModel: string
DeepgramEnModel: string
OpenAiTranscriptModel: string OpenAiTranscriptModel: string
OpenaiApiKey: string OpenaiApiKey: string
OpenaiBaseUrl: string OpenaiBaseUrl: string
@ -34,8 +33,7 @@ interface Config {
const config: Config = (() => { const config: Config = (() => {
const params: Partial<Config> = { const params: Partial<Config> = {
DeepgramApiKey: process.env.DEEPGRAM_API_KEY ?? '', DeepgramApiKey: process.env.DEEPGRAM_API_KEY ?? '',
DeepgramModel: process.env.DEEPGRAM_MODEL ?? 'nova-2-general', DeepgramModel: process.env.DEEPGRAM_MODEL ?? 'nova-3',
DeepgramEnModel: process.env.DEEPGRAM_EN_MODEL ?? 'nova-3-general',
OpenAiTranscriptModel: process.env.OPENAI_TRANSCRIPT_MODEL ?? 'gpt-4o-transcribe', OpenAiTranscriptModel: process.env.OPENAI_TRANSCRIPT_MODEL ?? 'gpt-4o-transcribe',
OpenaiApiKey: process.env.OPENAI_API_KEY ?? '', OpenaiApiKey: process.env.OPENAI_API_KEY ?? '',
OpenaiBaseUrl: process.env.OPENAI_BASE_URL ?? '', OpenaiBaseUrl: process.env.OPENAI_BASE_URL ?? '',

20
services/ai-bot/love-agent/src/deepgram/stt.ts
View File

@ -46,7 +46,7 @@ export class STT implements Stt {
private readonly deepgram: DeepgramClient private readonly deepgram: DeepgramClient
private isInProgress = false private isInProgress = false
private language: string = 'en' // private language: string = 'en'
private readonly trackBySid = new Map<string, RemoteTrack>() private readonly trackBySid = new Map<string, RemoteTrack>()
private readonly streamBySid = new Map<string, AudioStream>() private readonly streamBySid = new Map<string, AudioStream>()
@ -62,12 +62,12 @@ export class STT implements Stt {
} }
updateLanguage (language: string): void { updateLanguage (language: string): void {
const shouldRestart = (this.language ?? 'en') !== language // const shouldRestart = (this.language ?? 'en') !== language
this.language = language // this.language = language
if (shouldRestart) { // if (shouldRestart) {
this.stop() // this.stop()
this.start() // this.start()
} // }
} }
start (): void { start (): void {
@ -140,13 +140,13 @@ export class STT implements Stt {
if (this.dgConnectionBySid.has(sid)) return if (this.dgConnectionBySid.has(sid)) return
const stream = new AudioStream(track) const stream = new AudioStream(track)
const language = this.language ?? 'en' // const language = this.language ?? 'en'
const dgConnection = this.deepgram.listen.live({ const dgConnection = this.deepgram.listen.live({
...dgSchema, ...dgSchema,
channels: stream.numChannels, channels: stream.numChannels,
sample_rate: stream.sampleRate, sample_rate: stream.sampleRate,
language, language: 'multi',
model: language === 'en' || language === 'en-US' ? config.DeepgramEnModel : config.DeepgramModel model: config.DeepgramModel
}) })
console.log('Starting deepgram for track', this.room.name, sid) console.log('Starting deepgram for track', this.room.name, sid)