diff --git a/dev/tool/src/calendar.ts b/dev/tool/src/calendar.ts new file mode 100644 index 0000000000..d6ef9ebd02 --- /dev/null +++ b/dev/tool/src/calendar.ts @@ -0,0 +1,251 @@ +import calendar from '@hcengineering/calendar' +import { type PersonId, type WorkspaceInfoWithStatus, type WorkspaceUuid, systemAccountUuid } from '@hcengineering/core' +import { getClient as getKvsClient } from '@hcengineering/kvs-client' +import { createClient, getAccountClient } from '@hcengineering/server-client' +import { generateToken } from '@hcengineering/server-token' +import setting from '@hcengineering/setting' +import type { Db } from 'mongodb' +import { getWorkspaceTransactorEndpoint } from './utils' + +interface Credentials { + refresh_token?: string | null + + expiry_date?: number | null + + access_token?: string | null + + token_type?: string | null + + id_token?: string | null + + scope?: string +} + +interface User extends Credentials { + userId: string + workspace: string + email: string +} + +// Updated token and history types +interface UserNew extends Credentials { + userId: PersonId + workspace: WorkspaceUuid + email: string +} + +interface OldHistory { + email: string + userId: string + workspace: string + historyId: string + calendarId?: string +} + +interface SyncHistory { + workspace: string + timestamp: number +} + +interface WorkspaceInfoProvider { + getWorkspaceInfo: (workspaceUuid: WorkspaceUuid) => Promise +} + +const CALENDAR_INTEGRATION = 'google-calendar' + +export async function performCalendarAccountMigrations (db: Db, region: string | null, kvsUrl: string): Promise { + console.log('Start calendar migrations') + const token = generateToken(systemAccountUuid, '' as WorkspaceUuid, { service: 'tool', admin: 'true' }) + const accountClient = getAccountClient(token) + + const allWorkpaces = await accountClient.listWorkspaces(region) + const byId = new Map(allWorkpaces.map((it) => [it.uuid, it])) + const oldNewIds = new Map(allWorkpaces.map((it) => [it.dataId ?? it.uuid, it])) + const workspaceProvider: WorkspaceInfoProvider = { + getWorkspaceInfo: async (workspaceUuid: WorkspaceUuid) => { + const ws = oldNewIds.get(workspaceUuid as any) ?? byId.get(workspaceUuid as any) + if (ws == null) { + console.error('No workspace found for token', workspaceUuid) + return undefined + } + return ws + } + } + + await migrateCalendarIntegrations(db, token, workspaceProvider) + + await migrateCalendarHistory(db, token, kvsUrl, workspaceProvider) + + console.log('Finished Calendar migrations') +} + +const workspacePersonsMap = new Map>() + +async function getPersonIdByEmail ( + workspace: WorkspaceUuid, + email: string, + oldId: string +): Promise { + const map = workspacePersonsMap.get(workspace) + if (map != null) { + return map[email] + } else { + const transactorUrl = await getWorkspaceTransactorEndpoint(workspace) + const token = generateToken(systemAccountUuid, workspace) + const client = await createClient(transactorUrl, token) + try { + const res: Record = {} + const integrations = await client.findAll(setting.class.Integration, { + type: calendar.integrationType.Calendar + }) + for (const integration of integrations) { + const val = integration.value.trim() + if (val === '') continue + res[val] = integration.createdBy ?? integration.modifiedBy + } + workspacePersonsMap.set(workspace, res) + return res[email] + } finally { + await client.close() + } + } +} + +async function migrateCalendarIntegrations ( + db: Db, + token: string, + workspaceProvider: WorkspaceInfoProvider +): Promise { + try { + const accountClient = getAccountClient(token) + + const tokens = db.collection('tokens') + + const allTokens = await tokens.find({}).toArray() + + for (const token of allTokens) { + try { + const ws = await workspaceProvider.getWorkspaceInfo(token.workspace as any) + if (ws == null) { + continue + } + token.workspace = ws.uuid + + const personId = await getPersonIdByEmail(ws.uuid, token.email, token.userId) + if (personId == null) { + console.error('No socialId found for token', token) + continue + } + // Check/create integration in account + const existing = await accountClient.getIntegration({ + kind: CALENDAR_INTEGRATION, + workspaceUuid: ws.uuid, + socialId: personId + }) + + if (existing == null) { + await accountClient.createIntegration({ + kind: CALENDAR_INTEGRATION, + workspaceUuid: ws.uuid, + socialId: personId + }) + } + + const existingToken = await accountClient.getIntegrationSecret({ + key: token.email, + kind: CALENDAR_INTEGRATION, + socialId: personId, + workspaceUuid: ws.uuid + }) + const newToken: UserNew = { + ...token, + workspace: ws?.uuid, + email: token.email, + userId: personId + } + if (existingToken == null) { + await accountClient.addIntegrationSecret({ + key: newToken.email, + kind: CALENDAR_INTEGRATION, + socialId: personId, + secret: JSON.stringify(newToken), + workspaceUuid: newToken.workspace + }) + } else { + const updatedToken = { + ...existingToken, + ...newToken + } + await accountClient.updateIntegrationSecret({ + key: newToken.email, + kind: CALENDAR_INTEGRATION, + socialId: personId, + secret: JSON.stringify(updatedToken), + workspaceUuid: newToken.workspace + }) + } + } catch (e) { + console.error('Error migrating token', token, e) + } + } + console.log('Calendar integrations migrations done, integration count:', allTokens.length) + } catch (e) { + console.error('Error migrating tokens', e) + } +} + +async function migrateCalendarHistory ( + db: Db, + token: string, + kvsUrl: string, + workspaceProvider: WorkspaceInfoProvider +): Promise { + try { + console.log('Start Calendar history migrations') + const history = db.collection('histories') + const allHistories = await history.find({}).toArray() + const calendarHistory = db.collection('calendarHistories') + const calendarHistories = await calendarHistory.find({}).toArray() + + const kvsClient = getKvsClient(CALENDAR_INTEGRATION, kvsUrl, token) + + for (const history of [...calendarHistories, ...allHistories]) { + try { + const ws = await workspaceProvider.getWorkspaceInfo(history.workspace as any) + if (ws == null) { + continue + } + + const personId = await getPersonIdByEmail(ws.uuid, history.email, history.userId) + if (personId == null) { + console.error('No socialId found for token', token) + continue + } + + const key = + history.calendarId != null + ? `${CALENDAR_INTEGRATION}:eventHistory:${ws.uuid}:${personId}:${history.email}:${history.calendarId}` + : `${CALENDAR_INTEGRATION}:calendarsHistory:${ws.uuid}:${personId}:${history.email}` + + await kvsClient.setValue(key, history.historyId) + } catch (e) { + console.error('Error migrating history', history, e) + } + } + + const syncHistory = db.collection('syncHistories') + const syncHistories = await syncHistory.find({}).toArray() + for (const history of syncHistories) { + const ws = await workspaceProvider.getWorkspaceInfo(history.workspace as any) + if (ws == null) { + continue + } + + const key = `${CALENDAR_INTEGRATION}:calendarSync:${ws.uuid}` + await kvsClient.setValue(key, history.timestamp) + } + console.log('Finished migrating gmail history, count:', allHistories.length) + } catch (e) { + console.error('Error migrating gmail history', e) + } +} diff --git a/dev/tool/src/index.ts b/dev/tool/src/index.ts index 765e7389b7..1816b833ec 100644 --- a/dev/tool/src/index.ts +++ b/dev/tool/src/index.ts @@ -97,6 +97,7 @@ import { performGithubAccountMigrations } from './github' import { migrateCreatedModifiedBy, ensureGlobalPersonsForLocalAccounts, moveAccountDbFromMongoToPG } from './db' import { getToolToken, getWorkspace, getWorkspaceTransactorEndpoint } from './utils' import { performGmailAccountMigrations } from './gmail' +import { performCalendarAccountMigrations } from './calendar' const colorConstants = { colorRed: '\u001b[31m', @@ -2447,6 +2448,21 @@ export function devTool ( client.close() }) + program + .command('migrate-calendar-integrations-data') + .option('--db ', 'DB name', 'calendar-service') + .option('--region ', 'DB region') + .action(async (cmd: { db: string, region?: string }) => { + const mongodbUri = getMongoDBUrl() + const client = getMongoClient(mongodbUri) + const _client = await client.getClient() + + const kvsUrl = getKvsUrl() + await performCalendarAccountMigrations(_client.db(cmd.db), cmd.region ?? null, kvsUrl) + await _client.close() + client.close() + }) + extendProgram?.(program) program.parse(process.argv) diff --git a/services/calendar/pod-calendar/src/kvsUtils.ts b/services/calendar/pod-calendar/src/kvsUtils.ts index 1944118dad..b0b34157ba 100644 --- a/services/calendar/pod-calendar/src/kvsUtils.ts +++ b/services/calendar/pod-calendar/src/kvsUtils.ts @@ -12,10 +12,11 @@ export function getKvsClient (): KeyValueClient { return keyValueClient } -export async function getSyncHistory (workspace: WorkspaceUuid): Promise { +export async function getSyncHistory (workspace: WorkspaceUuid): Promise { const client = getKvsClient() const key = `${CALENDAR_INTEGRATION}:calendarSync:${workspace}` - return await client.getValue(key) + const res = await client.getValue(key) + return res ?? undefined } export async function setSyncHistory (workspace: WorkspaceUuid): Promise { @@ -24,32 +25,37 @@ export async function setSyncHistory (workspace: WorkspaceUuid): Promise { await client.setValue(key, Date.now()) } -function calendarsHistoryKey (user: User): string { - return `${CALENDAR_INTEGRATION}:calendarsHistory:${user.workspace}:${user.userId}` +function calendarsHistoryKey (user: User, email: GoogleEmail): string { + return `${CALENDAR_INTEGRATION}:calendarsHistory:${user.workspace}:${user.userId}:${email}` } -export async function getCalendarsSyncHistory (user: User): Promise { +export async function getCalendarsSyncHistory (user: User, email: GoogleEmail): Promise { const client = getKvsClient() - return (await client.getValue(calendarsHistoryKey(user))) ?? undefined + return (await client.getValue(calendarsHistoryKey(user, email))) ?? undefined } -export async function setCalendarsSyncHistory (user: User, historyId: string): Promise { +export async function setCalendarsSyncHistory (user: User, email: GoogleEmail, historyId: string): Promise { const client = getKvsClient() - await client.setValue(calendarsHistoryKey(user), historyId) + await client.setValue(calendarsHistoryKey(user, email), historyId) } -function eventHistoryKey (user: User, calendarId: string): string { - return `${CALENDAR_INTEGRATION}:eventHistory:${user.workspace}:${user.userId}:${calendarId}` +function eventHistoryKey (user: User, email: GoogleEmail, calendarId: string): string { + return `${CALENDAR_INTEGRATION}:eventHistory:${user.workspace}:${user.userId}:${email}:${calendarId}` } -export async function getEventHistory (user: User, calendarId: string): Promise { +export async function getEventHistory (user: User, email: GoogleEmail, calendarId: string): Promise { const client = getKvsClient() - return (await client.getValue(eventHistoryKey(user, calendarId))) ?? undefined + return (await client.getValue(eventHistoryKey(user, email, calendarId))) ?? undefined } -export async function setEventHistory (user: User, calendarId: string, historyId: string): Promise { +export async function setEventHistory ( + user: User, + email: GoogleEmail, + calendarId: string, + historyId: string +): Promise { const client = getKvsClient() - await client.setValue(eventHistoryKey(user, calendarId), historyId) + await client.setValue(eventHistoryKey(user, email, calendarId), historyId) } export async function getUserByEmail (email: GoogleEmail): Promise { diff --git a/services/calendar/pod-calendar/src/sync.ts b/services/calendar/pod-calendar/src/sync.ts index c3e7b2e76f..dbee3ed539 100644 --- a/services/calendar/pod-calendar/src/sync.ts +++ b/services/calendar/pod-calendar/src/sync.ts @@ -191,7 +191,7 @@ export class IncomingSyncManager { } private async syncEvents (calendarId: string): Promise { - const history = await getEventHistory(this.user, calendarId) + const history = await getEventHistory(this.user, this.email, calendarId) await this.eventsSync(calendarId, history) } @@ -220,7 +220,7 @@ export class IncomingSyncManager { await this.eventsSync(calendarId, syncToken, nextPageToken) } if (res.data.nextSyncToken != null) { - await setEventHistory(this.user, calendarId, res.data.nextSyncToken) + await setEventHistory(this.user, this.email, calendarId, res.data.nextSyncToken) } // if resync } catch (err: any) { @@ -559,7 +559,7 @@ export class IncomingSyncManager { } async syncCalendars (): Promise { - const history = await getCalendarsSyncHistory(this.user) + const history = await getCalendarsSyncHistory(this.user, this.email) await this.calendarSync(history) const watchController = WatchController.get(this.accountClient) await this.rateLimiter.take(1) @@ -594,7 +594,7 @@ export class IncomingSyncManager { continue } if (res.data.nextSyncToken != null) { - await setCalendarsSyncHistory(this.user, res.data.nextSyncToken) + await setCalendarsSyncHistory(this.user, this.email, res.data.nextSyncToken) } return } catch (err: any) { diff --git a/services/calendar/pod-calendar/src/types.ts b/services/calendar/pod-calendar/src/types.ts index 786dcd61e4..3582f131e0 100644 --- a/services/calendar/pod-calendar/src/types.ts +++ b/services/calendar/pod-calendar/src/types.ts @@ -42,26 +42,8 @@ export interface EventWatch extends WatchBase { export type Watch = CalendarsWatch | EventWatch -export interface DummyWatch { - timer: NodeJS.Timeout - calendarId: string -} - export type Token = User & Credentials & { email: GoogleEmail } -export interface CalendarHistory { - userId: PersonId - workspace: string - historyId: string -} - -export interface EventHistory { - calendarId: string - userId: PersonId - workspace: string - historyId: string -} - export interface SyncHistory { workspace: string timestamp: number diff --git a/services/calendar/pod-calendar/src/watch.ts b/services/calendar/pod-calendar/src/watch.ts index 9ed8b75096..8afe977e15 100644 --- a/services/calendar/pod-calendar/src/watch.ts +++ b/services/calendar/pod-calendar/src/watch.ts @@ -31,7 +31,7 @@ export class WatchClient { private async getWatches (): Promise> { const client = getKvsClient() - const key = `${CALENDAR_INTEGRATION}:watch:${this.user.workspace}:${this.user.userId}` + const key = `${CALENDAR_INTEGRATION}:watch:${this.user.workspace}:${this.user.userId}:${this.user.email}` const watches = await client.listKeys(key) return watches ?? {} } @@ -104,7 +104,7 @@ async function watchCalendars (user: User, email: GoogleEmail, googleClient: cal const res = await googleClient.calendarList.watch({ requestBody: body }) if (res.data.expiration != null && res.data.resourceId !== null) { const client = getKvsClient() - const key = `${CALENDAR_INTEGRATION}:watch:${user.workspace}:${user.userId}:null` + const key = `${CALENDAR_INTEGRATION}:watch:${user.workspace}:${user.userId}:${email}:null` await client.setValue(key, { userId: user.userId, workspace: user.workspace, @@ -133,7 +133,7 @@ async function watchCalendar ( const res = await googleClient.events.watch({ calendarId, requestBody: body }) if (res.data.expiration != null && res.data.resourceId != null) { const client = getKvsClient() - const key = `${CALENDAR_INTEGRATION}:watch:${user.workspace}:${user.userId}:${calendarId}` + const key = `${CALENDAR_INTEGRATION}:watch:${user.workspace}:${user.userId}:${email}:${calendarId}` await client.setValue(key, { userId: user.userId, workspace: user.workspace, @@ -267,7 +267,7 @@ export class WatchController { ): Promise { if (!force) { const client = getKvsClient() - const key = `${CALENDAR_INTEGRATION}:watch:${user.workspace}:${user.userId}:${calendarId ?? 'null'}` + const key = `${CALENDAR_INTEGRATION}:watch:${user.workspace}:${user.userId}:${email}:${calendarId ?? 'null'}` const exists = await client.getValue(key) if (exists != null) { return