From 0ecbbe81263e86c1aeeedb5dde5b457599122250 Mon Sep 17 00:00:00 2001 From: Artyom Savchenko Date: Tue, 6 May 2025 11:27:58 +0700 Subject: [PATCH] UBERF-10491: Fix gmail client duplicates (#8837) * UBERF-10491: Fix gmail client duplicated Signed-off-by: Artem Savchenko * UBERF-10491: Clean up Signed-off-by: Artem Savchenko * UBERF-10491: Remove redundant check Signed-off-by: Artem Savchenko * UBERF-10491: Fix integration token Signed-off-by: Artem Savchenko * UBERF-10491: Mock test env Signed-off-by: Artem Savchenko --------- Signed-off-by: Artem Savchenko --- .../src/__tests__/gmailController.test.ts | 165 ++++++++++++++++++ services/gmail/pod-gmail/src/gmail.ts | 8 +- .../gmail/pod-gmail/src/gmailController.ts | 71 +++++--- services/gmail/pod-gmail/src/utils.ts | 6 +- 4 files changed, 218 insertions(+), 32 deletions(-) create mode 100644 services/gmail/pod-gmail/src/__tests__/gmailController.test.ts diff --git a/services/gmail/pod-gmail/src/__tests__/gmailController.test.ts b/services/gmail/pod-gmail/src/__tests__/gmailController.test.ts new file mode 100644 index 0000000000..49355dad89 --- /dev/null +++ b/services/gmail/pod-gmail/src/__tests__/gmailController.test.ts @@ -0,0 +1,165 @@ +// +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import { MeasureContext, WorkspaceUuid } from '@hcengineering/core' +import { StorageAdapter } from '@hcengineering/server-core' +import * as serverClient from '@hcengineering/server-client' + +import { GmailController } from '../gmailController' +import { GmailClient } from '../gmail' +import { WorkspaceClient } from '../workspaceClient' +import { Token } from '../types' +import * as integrations from '../integrations' +import * as tokens from '../tokens' + +jest.mock('../workspaceClient') +jest.mock('../gmail') +jest.mock('../integrations') +jest.mock('../tokens') +jest.mock('@hcengineering/server-client') +jest.mock('../utils') +jest.mock('../config') + +/* eslint-disable @typescript-eslint/unbound-method */ + +describe('GmailController', () => { + let gmailController: GmailController + let mockCtx: MeasureContext + let mockStorageAdapter: StorageAdapter + let mockWorkspaceClient: jest.Mocked + let mockGmailClients: Map> + + const workspaceAId: WorkspaceUuid = 'workspace-a' as any + const workspaceBId: WorkspaceUuid = 'workspace-b' as any + + const workspaceATokens: Token[] = [ + { userId: 'user1', workspace: workspaceAId, token: 'token1' } as any, + { userId: 'user2', workspace: workspaceAId, token: 'token2' } as any + ] + + const workspaceBTokens: Token[] = [ + { userId: 'user3', workspace: workspaceBId, token: 'token3' } as any, + { userId: 'user4', workspace: workspaceBId, token: 'token4' } as any, + { userId: 'user5', workspace: workspaceBId, token: 'token5' } as any + ] + + beforeEach(() => { + jest.clearAllMocks() + + mockCtx = { + info: (message: string, details: any) => { + console.log(message, details) + }, + error: (message: string, details: any) => { + console.error(message, details) + }, + warn: (message: string, details: any) => { + console.warn(message, details) + } + } as unknown as MeasureContext + + mockStorageAdapter = {} as unknown as StorageAdapter + + mockWorkspaceClient = { + createGmailClient: jest.fn(), + checkUsers: jest.fn().mockResolvedValue(undefined), + getNewMessages: jest.fn().mockResolvedValue(undefined), + close: jest.fn() + } as unknown as jest.Mocked + + // Create mock clients with unique properties + mockGmailClients = new Map() + + const allUsers = [...workspaceATokens, ...workspaceBTokens].map((token) => token.userId) + allUsers.forEach((userId) => { + mockGmailClients.set(userId, { + startSync: jest.fn().mockResolvedValue(undefined), + sync: jest.fn().mockResolvedValue(undefined), + close: jest.fn().mockResolvedValue(undefined), + getUserEmail: jest.fn().mockReturnValue(`${userId}@example.com`) + } as unknown as jest.Mocked) + }) + + // Mock WorkspaceClient.create + jest.spyOn(WorkspaceClient, 'create').mockResolvedValue(mockWorkspaceClient) + + // Mock getIntegrations + jest + .spyOn(integrations, 'getIntegrations') + .mockResolvedValue([ + { workspaceUuid: workspaceAId, socialId: 'user1' } as any, + { workspaceUuid: workspaceAId, socialId: 'user2' } as any, + { workspaceUuid: workspaceBId, socialId: 'user3' } as any, + { workspaceUuid: workspaceBId, socialId: 'user4' } as any, + { workspaceUuid: workspaceBId, socialId: 'user5' } as any + ]) + + // Mock getWorkspaceTokens + jest.spyOn(tokens, 'getWorkspaceTokens').mockImplementation(async (_, workspaceId) => { + if (workspaceId === workspaceAId) return workspaceATokens + if (workspaceId === workspaceBId) return workspaceBTokens + return [] + }) + + // Mock getAccountClient + jest.spyOn(serverClient, 'getAccountClient').mockReturnValue({ + getWorkspaceInfo: jest.fn().mockResolvedValue({ mode: 'active' }) + } as any) + + // Mock serviceToken + // eslint-disable-next-line @typescript-eslint/no-var-requires + jest.spyOn(require('../utils'), 'serviceToken').mockReturnValue('test-token') + + // Mock JSON.parse + jest.spyOn(JSON, 'parse').mockReturnValue({ + web: { client_id: 'id', client_secret: 'secret', redirect_uris: ['uri'] } + }) + + // Create GmailController + gmailController = GmailController.create(mockCtx, mockStorageAdapter) + + // Mock createGmailClient to return appropriate mock client + mockWorkspaceClient.createGmailClient.mockImplementation(async (token: Token) => { + const client = mockGmailClients.get(token.userId) + if (client == null) { + throw new Error(`No mock client for userId: ${token.userId}`) + } + return client + }) + }) + + it('should create clients for all tokens without duplicates', async () => { + // Execute startAll + await gmailController.startAll() + + // Verify workspaces were created + expect(WorkspaceClient.create).toHaveBeenCalledTimes(2) + expect(WorkspaceClient.create).toHaveBeenCalledWith( + expect.anything(), + expect.anything(), + mockStorageAdapter, + workspaceAId + ) + expect(WorkspaceClient.create).toHaveBeenCalledWith( + expect.anything(), + expect.anything(), + mockStorageAdapter, + workspaceBId + ) + + // Verify createGmailClient called 5 times + expect(mockWorkspaceClient.createGmailClient).toHaveBeenCalledTimes(5) + }) +}) diff --git a/services/gmail/pod-gmail/src/gmail.ts b/services/gmail/pod-gmail/src/gmail.ts index 0db65ef703..e013f547ce 100644 --- a/services/gmail/pod-gmail/src/gmail.ts +++ b/services/gmail/pod-gmail/src/gmail.ts @@ -99,7 +99,7 @@ export class GmailClient { private socialId: SocialId ) { this.email = email - this.integrationToken = serviceToken() + this.integrationToken = serviceToken(workspaceId) this.tokenStorage = new TokenStorage(this.ctx, workspaceId, this.integrationToken) this.client = new TxOperations(client, this.socialId._id) this.account = this.user.userId @@ -363,15 +363,11 @@ export class GmailClient { this.socialId = await getOrCreateSocialId(this.account, this.email) } - private async getCurrentToken (): Promise { - return await this.tokenStorage.getToken(this.socialId._id) - } - private async addClient (): Promise { try { this.ctx.info('Register client', { socialId: this.socialId._id, email: this.email }) const controller = GmailController.getGmailController() - controller.addClient(this.socialId._id, this) + controller.addClient(this.socialId._id, this.user.workspace, this) } catch (err) { this.ctx.error('Add client error', { workspaceUuid: this.user.workspace, diff --git a/services/gmail/pod-gmail/src/gmailController.ts b/services/gmail/pod-gmail/src/gmailController.ts index 6b39b3cf68..e6716b02c8 100644 --- a/services/gmail/pod-gmail/src/gmailController.ts +++ b/services/gmail/pod-gmail/src/gmailController.ts @@ -39,7 +39,11 @@ export class GmailController { private readonly workspaces: Map = new Map() private readonly credentials: ProjectCredentials - private readonly clients: Map = new Map() + private readonly clients: Map> = new Map< + PersonId, + Map + >() + private readonly initLimitter = new RateLimiter(config.InitLimit) private readonly authProvider @@ -75,24 +79,37 @@ export class GmailController { this.ctx.info('Start integrations', { count: integrations.length }) const limiter = new RateLimiter(config.InitLimit) - for (const integration of integrations) { - if (integration.workspaceUuid === null) continue + const workspaceIds = new Set( + integrations + .map((integration) => { + if (integration.workspaceUuid == null) { + this.ctx.info('No workspace found', { integration }) + return undefined + } + return integration.workspaceUuid + }) + .filter((id): id is WorkspaceUuid => id != null) + ) + this.ctx.info('Workspaces with integrations', { count: workspaceIds.size }) + + for (const workspace of workspaceIds) { + const wsToken = serviceToken(workspace) + const accountClient = getAccountClient(wsToken) + + const tokens = await getWorkspaceTokens(accountClient, workspace) await limiter.add(async () => { - if (integration.workspaceUuid === null) return - const accountClient = getAccountClient(token) const info = await accountClient.getWorkspaceInfo() if (info === undefined) { - this.ctx.info('workspace not found', { workspaceUuid: integration.workspaceUuid }) + this.ctx.info('workspace not found', { workspaceUuid: workspace }) return } if (!isActiveMode(info.mode)) { - this.ctx.info('workspace is not active', { workspaceUuid: integration.workspaceUuid }) + this.ctx.info('workspace is not active', { workspaceUuid: workspace }) return } - const tokens = await getWorkspaceTokens(accountClient, integration.workspaceUuid) this.ctx.info('Use stored tokens', { count: tokens.length }) - const startPromise = this.startWorkspace(integration.workspaceUuid, tokens) + const startPromise = this.startWorkspace(workspace, tokens) const timeoutPromise = new Promise((resolve) => { setTimeout(() => { resolve() @@ -142,27 +159,35 @@ export class GmailController { const data = JSON.parse(decode64(message)) const email = data.emailAddress const clients = this.clients.get(email) - for (const client of clients ?? []) { + if (clients === undefined) { + this.ctx.info('No clients found', { email }) + return + } + for (const client of clients.values()) { void client.sync() } } - addClient (socialId: PersonId, client: GmailClient): void { - const clients = this.clients.get(socialId) - if (clients === undefined) { - this.clients.set(socialId, [client]) - } else { - clients.push(client) - this.clients.set(socialId, clients) + addClient (socialId: PersonId, workspace: WorkspaceUuid, client: GmailClient): void { + let userClients = this.clients.get(socialId) + if (userClients === undefined) { + userClients = new Map() + this.clients.set(socialId, userClients) } - } - /* - async getGmailClient (userId: AccountUuid, workspace: WorkspaceUuid, token: string): Promise { - const workspaceClient = await this.getWorkspaceClient(workspace) - return await workspaceClient.createGmailClient({ userId, workspace, token }) + const existingClient = userClients.get(workspace) + if (existingClient != null) { + void existingClient.close().catch((err) => { + this.ctx.error('Error closing existing client', { + socialId, + workspace, + error: err.message + }) + }) + } + + userClients.set(workspace, client) } - */ getAuthProvider (): AuthProvider { return this.authProvider diff --git a/services/gmail/pod-gmail/src/utils.ts b/services/gmail/pod-gmail/src/utils.ts index d5de1d1d1b..ce5e1ea581 100644 --- a/services/gmail/pod-gmail/src/utils.ts +++ b/services/gmail/pod-gmail/src/utils.ts @@ -14,7 +14,7 @@ // limitations under the License. // -import { type Data, type Doc, type DocumentUpdate, systemAccountUuid } from '@hcengineering/core' +import { type Data, type Doc, type DocumentUpdate, systemAccountUuid, WorkspaceUuid } from '@hcengineering/core' import { generateToken } from '@hcengineering/server-token' import { deepEqual } from 'fast-equals' import { type KeyValueClient, getClient as getKeyValueClient } from '@hcengineering/kvs-client' @@ -72,8 +72,8 @@ export function addFooter (message: string): string { return message + config.FooterMessage.trim() } -export function serviceToken (): string { - return generateToken(systemAccountUuid, undefined, { service: 'gmail' }) +export function serviceToken (workspaceId?: WorkspaceUuid): string { + return generateToken(systemAccountUuid, workspaceId, { service: 'gmail' }) } export async function wait (sec: number): Promise {