UBERF-8169: Move account DB tool (#6772)

Signed-off-by: Alexey Zinoviev <alexey.zinoviev@xored.com>

dev/tool/src/db.ts

@@ -1,5 +1,22 @@
-import { type AccountDB, updateWorkspace, type Workspace } from '@hcengineering/account'
-import { type BackupClient, type Client, getWorkspaceId, systemAccountEmail, type Doc } from '@hcengineering/core'
+import {
+  type AccountDB,
+  listAccounts,
+  listWorkspacesPure,
+  listInvites,
+  updateWorkspace,
+  type Workspace,
+  type ObjectId,
+  getAccount,
+  getWorkspaceById
+} from '@hcengineering/account'
+import {
+  type BackupClient,
+  type Client,
+  getWorkspaceId,
+  systemAccountEmail,
+  type Doc,
+  type MeasureMetricsContext
+} from '@hcengineering/core'
 import { getMongoClient, getWorkspaceMongoDB } from '@hcengineering/mongo'
 import { convertDoc, createTable, getDBClient, retryTxn, translateDomain } from '@hcengineering/postgres'
 import { getTransactorEndpoint } from '@hcengineering/server-client'
@@ -117,3 +134,70 @@ export async function moveWorkspaceFromMongoToPG (
   pg.close()
   client.close()
 }
+
+export async function moveAccountDbFromMongoToPG (
+  ctx: MeasureMetricsContext,
+  mongoDb: AccountDB,
+  pgDb: AccountDB
+): Promise<void> {
+  const workspaceAssignments: [ObjectId, ObjectId][] = []
+  const accounts = await listAccounts(mongoDb)
+  const workspaces = await listWorkspacesPure(mongoDb)
+  const invites = await listInvites(mongoDb)
+
+  for (const mongoAccount of accounts) {
+    const pgAccount = {
+      ...mongoAccount,
+      _id: mongoAccount._id.toString()
+    }
+
+    delete (pgAccount as any).workspaces
+
+    const exists = await getAccount(pgDb, pgAccount.email)
+    if (exists === null) {
+      await pgDb.account.insertOne(pgAccount)
+      ctx.info('Moved account', { email: pgAccount.email })
+
+      for (const workspace of mongoAccount.workspaces) {
+        workspaceAssignments.push([pgAccount._id, workspace.toString()])
+      }
+    }
+  }
+
+  for (const mongoWorkspace of workspaces) {
+    const pgWorkspace = {
+      ...mongoWorkspace,
+      _id: mongoWorkspace._id.toString()
+    }
+
+    delete (pgWorkspace as any).accounts
+
+    const exists = await getWorkspaceById(pgDb, pgWorkspace.workspace)
+    if (exists === null) {
+      await pgDb.workspace.insertOne(pgWorkspace)
+      ctx.info('Moved workspace', {
+        workspace: pgWorkspace.workspace,
+        workspaceName: pgWorkspace.workspaceName,
+        workspaceUrl: pgWorkspace.workspaceUrl
+      })
+    }
+  }
+
+  for (const mongoInvite of invites) {
+    const pgInvite = {
+      ...mongoInvite,
+      _id: mongoInvite._id.toString()
+    }
+
+    const exists = await pgDb.invite.findOne({ _id: pgInvite._id })
+    if (exists === null) {
+      await pgDb.invite.insertOne(pgInvite)
+    }
+  }
+
+  if (workspaceAssignments.length > 0) {
+    for (const [accountId, workspaceId] of workspaceAssignments) {
+      await pgDb.assignWorkspace(accountId, workspaceId)
+    }
+  }
+}

dev/tool/src/index.ts

@@ -104,7 +104,7 @@ import {
   restoreRecruitingTaskTypes
 } from './clean'
 import { changeConfiguration } from './configuration'
-import { moveFromMongoToPG, moveWorkspaceFromMongoToPG } from './db'
+import { moveFromMongoToPG, moveWorkspaceFromMongoToPG, moveAccountDbFromMongoToPG } from './db'
 import { fixJsonMarkup, migrateMarkup, restoreLostMarkup } from './markup'
 import { fixMixinForeignAttributes, showMixinForeignAttributes } from './mixin'
 import { fixAccountEmails, renameAccount } from './renameAccount'
@@ -1631,6 +1631,24 @@ export function devTool (
     })
   })
 
+  program.command('move-account-db-to-pg').action(async () => {
+    const { dbUrl, mongodbUri } = prepareTools()
+
+    if (mongodbUri === undefined) {
+      throw new Error('MONGO_URL is not set')
+    }
+
+    if (mongodbUri === dbUrl) {
+      throw new Error('MONGO_URL and DB_URL are the same')
+    }
+
+    await withDatabase(dbUrl, async (pgDb) => {
+      await withDatabase(mongodbUri, async (mongoDb) => {
+        await moveAccountDbFromMongoToPG(toolCtx, mongoDb, pgDb)
+      })
+    })
+  })
+
   program
     .command('perfomance')
     .option('-p, --parallel', '', false)

pods/account/package.json

@@ -20,7 +20,7 @@
     "docker:abuild": "docker build -t hardcoreeng/account . --platform=linux/arm64 && ../../common/scripts/docker_tag_push.sh hardcoreeng/account",
     "docker:staging": "../../common/scripts/docker_tag.sh hardcoreeng/account staging",
     "docker:push": "../../common/scripts/docker_tag.sh hardcoreeng/account",
-    "run-local": "cross-env MONGO_URL=mongodb://localhost:27017 MINIO_ACCESS_KEY=minioadmi MINIO_SECRET_KEY=minioadmin MINIO_ENDPOINT=localhost SERVER_SECRET='secret' TRANSACTOR_URL=ws://localhost:3333 ts-node src/__start.ts",
+    "run-local": "cross-env DB_URL=mongodb://localhost:27017 MINIO_ACCESS_KEY=minioadmi MINIO_SECRET_KEY=minioadmin MINIO_ENDPOINT=localhost SERVER_SECRET='secret' TRANSACTOR_URL=ws://localhost:3333 ts-node src/__start.ts",
     "format": "format src",
     "test": "jest --passWithNoTests --silent --forceExit",
     "_phase:build": "compile transpile src",

pods/workspace/package.json

@@ -20,7 +20,7 @@
     "docker:abuild": "docker build -t hardcoreeng/workspace . --platform=linux/arm64 && ../../common/scripts/docker_tag_push.sh hardcoreeng/workspace",
     "docker:staging": "../../common/scripts/docker_tag.sh hardcoreeng/workspace staging",
     "docker:push": "../../common/scripts/docker_tag.sh hardcoreeng/workspace",
-    "run-local": "cross-env MONGO_URL=mongodb://localhost:27017 MINIO_ACCESS_KEY=minioadmi MINIO_SECRET_KEY=minioadmin MINIO_ENDPOINT=localhost SERVER_SECRET='secret' TRANSACTOR_URL=ws://localhost:3333 ts-node src/__start.ts",
+    "run-local": "cross-env DB_URL=mongodb://localhost:27017 MONGO_URL=mongodb://localhost:27017 MINIO_ACCESS_KEY=minioadmi MINIO_SECRET_KEY=minioadmin MINIO_ENDPOINT=localhost SERVER_SECRET='secret' TRANSACTOR_URL=ws://localhost:3333 ts-node src/__start.ts",
     "format": "format src",
     "test": "jest --passWithNoTests --silent --forceExit",
     "_phase:build": "compile transpile src",

server/account/src/collections/postgres.ts

@@ -274,7 +274,7 @@ export class AccountPostgresDbCollection extends PostgresDbCollection<Account> i
   async insertOne<K extends keyof Account>(data: Partial<Account>, idKey?: K): Promise<any> {
     if (data.workspaces !== undefined) {
       if (data.workspaces.length > 0) {
-        throw new Error('Cannot assign workspaces directly')
+        console.warn('Cannot assign workspaces directly')
       }
 
       delete data.workspaces
@@ -361,7 +361,7 @@ export class WorkspacePostgresDbCollection extends PostgresDbCollection<Workspac
 
     if (data.accounts !== undefined) {
       if (data.accounts.length > 0) {
-        throw new Error('Cannot assign workspaces directly')
+        console.warn('Cannot assign workspaces directly')
       }
 
       delete dbData.accounts

server/account/src/operations.ts

@@ -852,6 +852,13 @@ export async function listWorkspacesPure (db: AccountDB): Promise<Workspace[]> {
   return await db.workspace.find({})
 }
 
+/**
+ * @public
+ */
+export async function listInvites (db: AccountDB): Promise<Invite[]> {
+  return await db.invite.find({})
+}
+
 /**
  * @public
  */