Calendar migration tool (#8873)

dev/tool/src/calendar.ts

@@ -0,0 +1,251 @@
+import calendar from '@hcengineering/calendar'
+import { type PersonId, type WorkspaceInfoWithStatus, type WorkspaceUuid, systemAccountUuid } from '@hcengineering/core'
+import { getClient as getKvsClient } from '@hcengineering/kvs-client'
+import { createClient, getAccountClient } from '@hcengineering/server-client'
+import { generateToken } from '@hcengineering/server-token'
+import setting from '@hcengineering/setting'
+import type { Db } from 'mongodb'
+import { getWorkspaceTransactorEndpoint } from './utils'
+
+interface Credentials {
+  refresh_token?: string | null
+
+  expiry_date?: number | null
+
+  access_token?: string | null
+
+  token_type?: string | null
+
+  id_token?: string | null
+
+  scope?: string
+}
+
+interface User extends Credentials {
+  userId: string
+  workspace: string
+  email: string
+}
+
+// Updated token and history types
+interface UserNew extends Credentials {
+  userId: PersonId
+  workspace: WorkspaceUuid
+  email: string
+}
+
+interface OldHistory {
+  email: string
+  userId: string
+  workspace: string
+  historyId: string
+  calendarId?: string
+}
+
+interface SyncHistory {
+  workspace: string
+  timestamp: number
+}
+
+interface WorkspaceInfoProvider {
+  getWorkspaceInfo: (workspaceUuid: WorkspaceUuid) => Promise<WorkspaceInfoWithStatus | undefined>
+}
+
+const CALENDAR_INTEGRATION = 'google-calendar'
+
+export async function performCalendarAccountMigrations (db: Db, region: string | null, kvsUrl: string): Promise<void> {
+  console.log('Start calendar migrations')
+  const token = generateToken(systemAccountUuid, '' as WorkspaceUuid, { service: 'tool', admin: 'true' })
+  const accountClient = getAccountClient(token)
+
+  const allWorkpaces = await accountClient.listWorkspaces(region)
+  const byId = new Map(allWorkpaces.map((it) => [it.uuid, it]))
+  const oldNewIds = new Map(allWorkpaces.map((it) => [it.dataId ?? it.uuid, it]))
+  const workspaceProvider: WorkspaceInfoProvider = {
+    getWorkspaceInfo: async (workspaceUuid: WorkspaceUuid) => {
+      const ws = oldNewIds.get(workspaceUuid as any) ?? byId.get(workspaceUuid as any)
+      if (ws == null) {
+        console.error('No workspace found for token', workspaceUuid)
+        return undefined
+      }
+      return ws
+    }
+  }
+
+  await migrateCalendarIntegrations(db, token, workspaceProvider)
+
+  await migrateCalendarHistory(db, token, kvsUrl, workspaceProvider)
+
+  console.log('Finished Calendar migrations')
+}
+
+const workspacePersonsMap = new Map<WorkspaceUuid, Record<string, PersonId>>()
+
+async function getPersonIdByEmail (
+  workspace: WorkspaceUuid,
+  email: string,
+  oldId: string
+): Promise<PersonId | undefined> {
+  const map = workspacePersonsMap.get(workspace)
+  if (map != null) {
+    return map[email]
+  } else {
+    const transactorUrl = await getWorkspaceTransactorEndpoint(workspace)
+    const token = generateToken(systemAccountUuid, workspace)
+    const client = await createClient(transactorUrl, token)
+    try {
+      const res: Record<string, PersonId> = {}
+      const integrations = await client.findAll(setting.class.Integration, {
+        type: calendar.integrationType.Calendar
+      })
+      for (const integration of integrations) {
+        const val = integration.value.trim()
+        if (val === '') continue
+        res[val] = integration.createdBy ?? integration.modifiedBy
+      }
+      workspacePersonsMap.set(workspace, res)
+      return res[email]
+    } finally {
+      await client.close()
+    }
+  }
+}
+
+async function migrateCalendarIntegrations (
+  db: Db,
+  token: string,
+  workspaceProvider: WorkspaceInfoProvider
+): Promise<void> {
+  try {
+    const accountClient = getAccountClient(token)
+
+    const tokens = db.collection<User>('tokens')
+
+    const allTokens = await tokens.find({}).toArray()
+
+    for (const token of allTokens) {
+      try {
+        const ws = await workspaceProvider.getWorkspaceInfo(token.workspace as any)
+        if (ws == null) {
+          continue
+        }
+        token.workspace = ws.uuid
+
+        const personId = await getPersonIdByEmail(ws.uuid, token.email, token.userId)
+        if (personId == null) {
+          console.error('No socialId found for token', token)
+          continue
+        }
+        // Check/create integration in account
+        const existing = await accountClient.getIntegration({
+          kind: CALENDAR_INTEGRATION,
+          workspaceUuid: ws.uuid,
+          socialId: personId
+        })
+
+        if (existing == null) {
+          await accountClient.createIntegration({
+            kind: CALENDAR_INTEGRATION,
+            workspaceUuid: ws.uuid,
+            socialId: personId
+          })
+        }
+
+        const existingToken = await accountClient.getIntegrationSecret({
+          key: token.email,
+          kind: CALENDAR_INTEGRATION,
+          socialId: personId,
+          workspaceUuid: ws.uuid
+        })
+        const newToken: UserNew = {
+          ...token,
+          workspace: ws?.uuid,
+          email: token.email,
+          userId: personId
+        }
+        if (existingToken == null) {
+          await accountClient.addIntegrationSecret({
+            key: newToken.email,
+            kind: CALENDAR_INTEGRATION,
+            socialId: personId,
+            secret: JSON.stringify(newToken),
+            workspaceUuid: newToken.workspace
+          })
+        } else {
+          const updatedToken = {
+            ...existingToken,
+            ...newToken
+          }
+          await accountClient.updateIntegrationSecret({
+            key: newToken.email,
+            kind: CALENDAR_INTEGRATION,
+            socialId: personId,
+            secret: JSON.stringify(updatedToken),
+            workspaceUuid: newToken.workspace
+          })
+        }
+      } catch (e) {
+        console.error('Error migrating token', token, e)
+      }
+    }
+    console.log('Calendar integrations migrations done, integration count:', allTokens.length)
+  } catch (e) {
+    console.error('Error migrating tokens', e)
+  }
+}
+
+async function migrateCalendarHistory (
+  db: Db,
+  token: string,
+  kvsUrl: string,
+  workspaceProvider: WorkspaceInfoProvider
+): Promise<void> {
+  try {
+    console.log('Start Calendar history migrations')
+    const history = db.collection<OldHistory>('histories')
+    const allHistories = await history.find({}).toArray()
+    const calendarHistory = db.collection<OldHistory>('calendarHistories')
+    const calendarHistories = await calendarHistory.find({}).toArray()
+
+    const kvsClient = getKvsClient(CALENDAR_INTEGRATION, kvsUrl, token)
+
+    for (const history of [...calendarHistories, ...allHistories]) {
+      try {
+        const ws = await workspaceProvider.getWorkspaceInfo(history.workspace as any)
+        if (ws == null) {
+          continue
+        }
+
+        const personId = await getPersonIdByEmail(ws.uuid, history.email, history.userId)
+        if (personId == null) {
+          console.error('No socialId found for token', token)
+          continue
+        }
+
+        const key =
+          history.calendarId != null
+            ? `${CALENDAR_INTEGRATION}:eventHistory:${ws.uuid}:${personId}:${history.email}:${history.calendarId}`
+            : `${CALENDAR_INTEGRATION}:calendarsHistory:${ws.uuid}:${personId}:${history.email}`
+
+        await kvsClient.setValue(key, history.historyId)
+      } catch (e) {
+        console.error('Error migrating history', history, e)
+      }
+    }
+
+    const syncHistory = db.collection<SyncHistory>('syncHistories')
+    const syncHistories = await syncHistory.find({}).toArray()
+    for (const history of syncHistories) {
+      const ws = await workspaceProvider.getWorkspaceInfo(history.workspace as any)
+      if (ws == null) {
+        continue
+      }
+
+      const key = `${CALENDAR_INTEGRATION}:calendarSync:${ws.uuid}`
+      await kvsClient.setValue(key, history.timestamp)
+    }
+    console.log('Finished migrating gmail history, count:', allHistories.length)
+  } catch (e) {
+    console.error('Error migrating gmail history', e)
+  }
+}

dev/tool/src/index.ts

@@ -97,6 +97,7 @@ import { performGithubAccountMigrations } from './github'
 import { migrateCreatedModifiedBy, ensureGlobalPersonsForLocalAccounts, moveAccountDbFromMongoToPG } from './db'
 import { getToolToken, getWorkspace, getWorkspaceTransactorEndpoint } from './utils'
 import { performGmailAccountMigrations } from './gmail'
+import { performCalendarAccountMigrations } from './calendar'
 
 const colorConstants = {
   colorRed: '\u001b[31m',
@@ -2447,6 +2448,21 @@ export function devTool (
       client.close()
     })
 
+  program
+    .command('migrate-calendar-integrations-data')
+    .option('--db <db>', 'DB name', 'calendar-service')
+    .option('--region <region>', 'DB region')
+    .action(async (cmd: { db: string, region?: string }) => {
+      const mongodbUri = getMongoDBUrl()
+      const client = getMongoClient(mongodbUri)
+      const _client = await client.getClient()
+
+      const kvsUrl = getKvsUrl()
+      await performCalendarAccountMigrations(_client.db(cmd.db), cmd.region ?? null, kvsUrl)
+      await _client.close()
+      client.close()
+    })
+
   extendProgram?.(program)
 
   program.parse(process.argv)

services/calendar/pod-calendar/src/kvsUtils.ts

@@ -12,10 +12,11 @@ export function getKvsClient (): KeyValueClient {
   return keyValueClient
 }
 
-export async function getSyncHistory (workspace: WorkspaceUuid): Promise<number | undefined | null> {
+export async function getSyncHistory (workspace: WorkspaceUuid): Promise<number | undefined> {
   const client = getKvsClient()
   const key = `${CALENDAR_INTEGRATION}:calendarSync:${workspace}`
-  return await client.getValue(key)
+  const res = await client.getValue<number>(key)
+  return res ?? undefined
 }
 
 export async function setSyncHistory (workspace: WorkspaceUuid): Promise<void> {
@@ -24,32 +25,37 @@ export async function setSyncHistory (workspace: WorkspaceUuid): Promise<void> {
   await client.setValue(key, Date.now())
 }
 
-function calendarsHistoryKey (user: User): string {
-  return `${CALENDAR_INTEGRATION}:calendarsHistory:${user.workspace}:${user.userId}`
+function calendarsHistoryKey (user: User, email: GoogleEmail): string {
+  return `${CALENDAR_INTEGRATION}:calendarsHistory:${user.workspace}:${user.userId}:${email}`
 }
 
-export async function getCalendarsSyncHistory (user: User): Promise<string | undefined> {
+export async function getCalendarsSyncHistory (user: User, email: GoogleEmail): Promise<string | undefined> {
   const client = getKvsClient()
-  return (await client.getValue(calendarsHistoryKey(user))) ?? undefined
+  return (await client.getValue(calendarsHistoryKey(user, email))) ?? undefined
 }
 
-export async function setCalendarsSyncHistory (user: User, historyId: string): Promise<void> {
+export async function setCalendarsSyncHistory (user: User, email: GoogleEmail, historyId: string): Promise<void> {
   const client = getKvsClient()
-  await client.setValue(calendarsHistoryKey(user), historyId)
+  await client.setValue(calendarsHistoryKey(user, email), historyId)
 }
 
-function eventHistoryKey (user: User, calendarId: string): string {
-  return `${CALENDAR_INTEGRATION}:eventHistory:${user.workspace}:${user.userId}:${calendarId}`
+function eventHistoryKey (user: User, email: GoogleEmail, calendarId: string): string {
+  return `${CALENDAR_INTEGRATION}:eventHistory:${user.workspace}:${user.userId}:${email}:${calendarId}`
 }
 
-export async function getEventHistory (user: User, calendarId: string): Promise<string | undefined> {
+export async function getEventHistory (user: User, email: GoogleEmail, calendarId: string): Promise<string | undefined> {
   const client = getKvsClient()
-  return (await client.getValue(eventHistoryKey(user, calendarId))) ?? undefined
+  return (await client.getValue(eventHistoryKey(user, email, calendarId))) ?? undefined
 }
 
-export async function setEventHistory (user: User, calendarId: string, historyId: string): Promise<void> {
+export async function setEventHistory (
+  user: User,
+  email: GoogleEmail,
+  calendarId: string,
+  historyId: string
+): Promise<void> {
   const client = getKvsClient()
-  await client.setValue(eventHistoryKey(user, calendarId), historyId)
+  await client.setValue(eventHistoryKey(user, email, calendarId), historyId)
 }
 
 export async function getUserByEmail (email: GoogleEmail): Promise<Token[]> {

services/calendar/pod-calendar/src/sync.ts

@@ -191,7 +191,7 @@ export class IncomingSyncManager {
   }
 
   private async syncEvents (calendarId: string): Promise<void> {
-    const history = await getEventHistory(this.user, calendarId)
+    const history = await getEventHistory(this.user, this.email, calendarId)
     await this.eventsSync(calendarId, history)
   }
 
@@ -220,7 +220,7 @@ export class IncomingSyncManager {
         await this.eventsSync(calendarId, syncToken, nextPageToken)
       }
       if (res.data.nextSyncToken != null) {
-        await setEventHistory(this.user, calendarId, res.data.nextSyncToken)
+        await setEventHistory(this.user, this.email, calendarId, res.data.nextSyncToken)
       }
       // if resync
     } catch (err: any) {
@@ -559,7 +559,7 @@ export class IncomingSyncManager {
   }
 
   async syncCalendars (): Promise<void> {
-    const history = await getCalendarsSyncHistory(this.user)
+    const history = await getCalendarsSyncHistory(this.user, this.email)
     await this.calendarSync(history)
     const watchController = WatchController.get(this.accountClient)
     await this.rateLimiter.take(1)
@@ -594,7 +594,7 @@ export class IncomingSyncManager {
           continue
         }
         if (res.data.nextSyncToken != null) {
-          await setCalendarsSyncHistory(this.user, res.data.nextSyncToken)
+          await setCalendarsSyncHistory(this.user, this.email, res.data.nextSyncToken)
         }
         return
       } catch (err: any) {

services/calendar/pod-calendar/src/types.ts

@@ -42,26 +42,8 @@ export interface EventWatch extends WatchBase {
 
 export type Watch = CalendarsWatch | EventWatch
 
-export interface DummyWatch {
-  timer: NodeJS.Timeout
-  calendarId: string
-}
-
 export type Token = User & Credentials & { email: GoogleEmail }
 
-export interface CalendarHistory {
-  userId: PersonId
-  workspace: string
-  historyId: string
-}
-
-export interface EventHistory {
-  calendarId: string
-  userId: PersonId
-  workspace: string
-  historyId: string
-}
-
 export interface SyncHistory {
   workspace: string
   timestamp: number

services/calendar/pod-calendar/src/watch.ts

@@ -31,7 +31,7 @@ export class WatchClient {
 
   private async getWatches (): Promise<Record<string, Watch>> {
     const client = getKvsClient()
-    const key = `${CALENDAR_INTEGRATION}:watch:${this.user.workspace}:${this.user.userId}`
+    const key = `${CALENDAR_INTEGRATION}:watch:${this.user.workspace}:${this.user.userId}:${this.user.email}`
     const watches = await client.listKeys<Watch>(key)
     return watches ?? {}
   }
@@ -104,7 +104,7 @@ async function watchCalendars (user: User, email: GoogleEmail, googleClient: cal
   const res = await googleClient.calendarList.watch({ requestBody: body })
   if (res.data.expiration != null && res.data.resourceId !== null) {
     const client = getKvsClient()
-    const key = `${CALENDAR_INTEGRATION}:watch:${user.workspace}:${user.userId}:null`
+    const key = `${CALENDAR_INTEGRATION}:watch:${user.workspace}:${user.userId}:${email}:null`
     await client.setValue<Watch>(key, {
       userId: user.userId,
       workspace: user.workspace,
@@ -133,7 +133,7 @@ async function watchCalendar (
   const res = await googleClient.events.watch({ calendarId, requestBody: body })
   if (res.data.expiration != null && res.data.resourceId != null) {
     const client = getKvsClient()
-    const key = `${CALENDAR_INTEGRATION}:watch:${user.workspace}:${user.userId}:${calendarId}`
+    const key = `${CALENDAR_INTEGRATION}:watch:${user.workspace}:${user.userId}:${email}:${calendarId}`
     await client.setValue<Watch>(key, {
       userId: user.userId,
       workspace: user.workspace,
@@ -267,7 +267,7 @@ export class WatchController {
   ): Promise<void> {
     if (!force) {
       const client = getKvsClient()
-      const key = `${CALENDAR_INTEGRATION}:watch:${user.workspace}:${user.userId}:${calendarId ?? 'null'}`
+      const key = `${CALENDAR_INTEGRATION}:watch:${user.workspace}:${user.userId}:${email}:${calendarId ?? 'null'}`
       const exists = await client.getValue<Watch>(key)
       if (exists != null) {
         return