UBER-1263 Fix blobs migration on new workspaces (#6586)

Signed-off-by: Alexander Onnikov <Alexander.Onnikov@xored.com>

models/core/src/migration.ts

@@ -15,7 +15,6 @@
 
 import { saveCollaborativeDoc } from '@hcengineering/collaboration'
 import core, {
-  DOMAIN_BLOB,
   DOMAIN_DOC_INDEX_STATE,
   DOMAIN_SPACE,
   DOMAIN_STATUS,
@@ -28,11 +27,9 @@ import core, {
   isClassIndexable,
   makeCollaborativeDoc,
   type AnyAttribute,
-  type Blob,
   type Doc,
   type Domain,
   type MeasureContext,
-  type Ref,
   type Space,
   type Status,
   type TxCreateDoc
@@ -48,7 +45,7 @@ import {
   type MigrationIterator,
   type MigrationUpgradeClient
 } from '@hcengineering/model'
-import { type StorageAdapter, type StorageAdapterEx } from '@hcengineering/storage'
+import { type StorageAdapter } from '@hcengineering/storage'
 import { markupToYDoc } from '@hcengineering/text'
 
 async function migrateStatusesToModel (client: MigrationClient): Promise<void> {
@@ -278,7 +275,6 @@ export const coreOperation: MigrateOperation = {
         }
       }
     )
-    const exAdapter: StorageAdapterEx = client.storageAdapter as StorageAdapterEx
     await tryMigrate(client, coreId, [
       {
         state: 'statuses-to-model',
@@ -292,12 +288,6 @@ export const coreOperation: MigrateOperation = {
         state: 'add-spaces-owner',
         func: migrateSpacesOwner
       },
-      {
-        state: 'storage_blobs_v1',
-        func: async (client: MigrationClient) => {
-          await migrateBlobData(exAdapter, client)
-        }
-      },
       {
         state: 'old-statuses-transactions',
         func: migrateStatusTransactions
@@ -340,43 +330,3 @@ export const coreOperation: MigrateOperation = {
     ])
   }
 }
-async function migrateBlobData (exAdapter: StorageAdapterEx, client: MigrationClient): Promise<void> {
-  const ctx = new MeasureMetricsContext('storage_upgrade', {})
-
-  for (const [provider, adapter] of exAdapter.adapters?.entries() ?? []) {
-    if (!(await adapter.exists(ctx, client.workspaceId))) {
-      continue
-    }
-    const blobs = await adapter.listStream(ctx, client.workspaceId)
-    const bulk = new Map<Ref<Blob>, Blob>()
-    try {
-      const push = async (force: boolean): Promise<void> => {
-        if (bulk.size > 1000 || force) {
-          await client.deleteMany(DOMAIN_BLOB, { _id: { $in: Array.from(bulk.keys()) } })
-          await client.create(DOMAIN_BLOB, Array.from(bulk.values()))
-          bulk.clear()
-        }
-      }
-      while (true) {
-        const blob = await blobs.next()
-        if (blob === undefined) {
-          break
-        }
-        // We need to state details for blob.
-        const blobData = await adapter.stat(ctx, client.workspaceId, blob._id)
-        if (blobData !== undefined) {
-          bulk.set(blobData._id, {
-            ...blobData,
-            provider
-          })
-        }
-        await push(false)
-      }
-      await push(true)
-    } catch (err: any) {
-      ctx.error('Error during blob migration', { error: err.message })
-    } finally {
-      await blobs.close()
-    }
-  }
-}

server/workspace-service/src/ws-operations.ts

@@ -200,9 +200,24 @@ export async function createWorkspace (
       ctx.info('Starting init script if any')
       await initializeWorkspace(ctx, branding, wsUrl, storageAdapter, client, ctxModellogger, async (value) => {
         ctx.info('Init script progress', { value })
-        await handleWsEvent?.('progress', version, 30 + Math.round((Math.min(value, 100) / 100) * 70))
+        await handleWsEvent?.('progress', version, 30 + Math.round((Math.min(value, 100) / 100) * 60))
       })
 
+      await upgradeWorkspace(
+        ctx,
+        version,
+        txes,
+        migrationOperation,
+        workspaceInfo,
+        ctxModellogger,
+        async (event, version, value) => {
+          ctx.info('Init script progress', { event, value })
+          await handleWsEvent?.('progress', version, 90 + Math.round((Math.min(value, 100) / 100) * 10))
+        },
+        false,
+        false
+      )
+
       await pipeline.close()
 
       await handleWsEvent?.('create-done', version, 100, '')