From 419a3ddfdbe4737ebdad43185f06864066e38720 Mon Sep 17 00:00:00 2001 From: Andrey Sobolev Date: Mon, 17 Mar 2025 17:44:24 +0700 Subject: [PATCH] UBERF-9633: Reduce migration calls during workspace creation (#8242) (#8244) Signed-off-by: Andrey Sobolev --- models/activity/src/migration.ts | 30 +++- models/chunter/src/migration.ts | 8 ++ models/contact/src/migration.ts | 3 +- models/core/src/migration.ts | 14 +- models/document/src/migration.ts | 11 ++ models/guest/src/migration.ts | 5 +- models/request/src/migration.ts | 5 +- models/server-activity/src/migration.ts | 1 + models/survey/src/migration.ts | 3 +- packages/model/src/migration.ts | 59 +++++--- pods/workspace/download-init-scripts.sh | 13 ++ server/backup/src/service.ts | 136 +++++++++--------- server/front/src/index.ts | 6 +- server/postgres/src/index.ts | 30 ++-- server/tool/src/index.ts | 16 ++- server/workspace-service/src/service.ts | 17 +-- server/workspace-service/src/ws-operations.ts | 35 +++-- server/ws/src/server_http.ts | 6 +- ws-tests/docker-compose.yaml | 1 + ws-tests/prepare_data.sh | 5 + 20 files changed, 260 insertions(+), 144 deletions(-) create mode 100755 pods/workspace/download-init-scripts.sh create mode 100644 ws-tests/prepare_data.sh diff --git a/models/activity/src/migration.ts b/models/activity/src/migration.ts index 434b8b3138..092cad49cf 100644 --- a/models/activity/src/migration.ts +++ b/models/activity/src/migration.ts @@ -38,6 +38,7 @@ import { type MigrationDocumentQuery, type MigrationIterator, type MigrationUpgradeClient, + type MigrateMode, tryMigrate } from '@hcengineering/model' import { htmlToMarkup } from '@hcengineering/text' @@ -456,18 +457,21 @@ async function migrateSocialIdsInDocUpdates (client: MigrationClient): Promise { + async migrate (client: MigrationClient, mode: MigrateMode): Promise { await tryMigrate(client, activityId, [ { state: 'reactions', + mode: 'upgrade', func: migrateReactions }, { state: 'markup', + mode: 'upgrade', func: migrateMarkup }, { state: 'migrate-doc-update-messages-space', + mode: 'upgrade', func: async (client) => { await migrateMessagesSpace( client, @@ -477,6 +481,30 @@ export const activityOperation: MigrateOperation = { ) } }, + { + state: 'migrate-employee-space-v1', + mode: 'upgrade', + func: async () => { + await client.update( + DOMAIN_ACTIVITY, + { space: 'contact:space:Employee' as Ref }, + { space: contact.space.Contacts } + ) + } + }, + { + state: 'migrate-activity-markup', + mode: 'upgrade', + func: migrateActivityMarkup + }, + { + state: 'move-reactions', + mode: 'upgrade', + func: async (client: MigrationClient): Promise => { + await client.move(DOMAIN_ACTIVITY, { _class: activity.class.Reaction }, DOMAIN_REACTION) + await client.move(DOMAIN_ACTIVITY, { _class: activity.class.UserMentionInfo }, DOMAIN_USER_MENTION) + } + }, { state: 'migrate-employee-space-v1', func: async () => { diff --git a/models/chunter/src/migration.ts b/models/chunter/src/migration.ts index c127aa3975..047891e2cf 100644 --- a/models/chunter/src/migration.ts +++ b/models/chunter/src/migration.ts @@ -236,14 +236,17 @@ export const chunterOperation: MigrateOperation = { await tryMigrate(client, chunterId, [ { state: 'create-chat-messages', + mode: 'upgrade', func: convertCommentsToChatMessages }, { state: 'remove-backlinks', + mode: 'upgrade', func: removeBacklinks }, { state: 'migrate-chat-messages-space', + mode: 'upgrade', func: async (client) => { await migrateMessagesSpace( client, @@ -255,6 +258,7 @@ export const chunterOperation: MigrateOperation = { }, { state: 'migrate-thread-messages-space', + mode: 'upgrade', func: async (client) => { await migrateMessagesSpace( client, @@ -266,18 +270,21 @@ export const chunterOperation: MigrateOperation = { }, { state: 'remove-old-classes-v1', + mode: 'upgrade', func: async (client) => { await removeOldClasses(client) } }, { state: 'remove-wrong-activity-v1', + mode: 'upgrade', func: async (client) => { await removeWrongActivity(client) } }, { state: 'remove-chat-info-v1', + mode: 'upgrade', func: async (client) => { await client.deleteMany(DOMAIN_CHUNTER, { _class: 'chunter:class:ChatInfo' as Ref> }) await client.deleteMany(DOMAIN_TX, { objectClass: 'chunter:class:ChatInfo' }) @@ -291,6 +298,7 @@ export const chunterOperation: MigrateOperation = { }, { state: 'remove-direct-doc-update-messages', + mode: 'upgrade', func: async (client) => { await client.deleteMany(DOMAIN_ACTIVITY, { _class: activity.class.DocUpdateMessage, diff --git a/models/contact/src/migration.ts b/models/contact/src/migration.ts index 306de295b5..6e16eed822 100644 --- a/models/contact/src/migration.ts +++ b/models/contact/src/migration.ts @@ -22,7 +22,6 @@ import { type MigrationClient, type MigrationDocumentQuery, type MigrationUpgradeClient, - type ModelLogger, tryMigrate, tryUpgrade } from '@hcengineering/model' @@ -264,7 +263,7 @@ async function createSocialIdentities (client: MigrationClient): Promise { } export const contactOperation: MigrateOperation = { - async migrate (client: MigrationClient, logger: ModelLogger): Promise { + async migrate (client: MigrationClient): Promise { await tryMigrate(client, contactId, [ { state: 'employees', diff --git a/models/core/src/migration.ts b/models/core/src/migration.ts index 419bfade16..bd99a9aca3 100644 --- a/models/core/src/migration.ts +++ b/models/core/src/migration.ts @@ -55,6 +55,7 @@ import { createDefaultSpace, tryMigrate, tryUpgrade, + type MigrateMode, type MigrateOperation, type MigrateUpdate, type MigrationClient, @@ -64,7 +65,7 @@ import { } from '@hcengineering/model' import { type StorageAdapter } from '@hcengineering/storage' -async function migrateStatusesToModel (client: MigrationClient): Promise { +async function migrateStatusesToModel (client: MigrationClient, mode: MigrateMode): Promise { // Move statuses to model: // Migrate the default ones with well-known ids as system's model // And the rest as user's model @@ -887,26 +888,32 @@ export const coreOperation: MigrateOperation = { await tryMigrate(client, coreId, [ { state: 'statuses-to-model', + mode: 'upgrade', func: migrateStatusesToModel }, { state: 'all-space-to-typed', + mode: 'upgrade', func: migrateAllSpaceToTyped }, { state: 'add-spaces-owner-v1', + mode: 'upgrade', func: migrateSpacesOwner }, { state: 'old-statuses-transactions', + mode: 'upgrade', func: migrateStatusTransactions }, { state: 'collaborative-content-to-storage', + mode: 'upgrade', func: migrateCollaborativeContentToStorage }, { state: 'fix-backups-hash-timestamp-v2', + mode: 'upgrade', func: async (client: MigrationClient): Promise => { const now = Date.now().toString(16) for (const d of client.hierarchy.domains()) { @@ -916,6 +923,7 @@ export const coreOperation: MigrateOperation = { }, { state: 'remove-collection-txes', + mode: 'upgrade', func: async (client) => { let processed = 0 let last = 0 @@ -959,6 +967,7 @@ export const coreOperation: MigrateOperation = { }, { state: 'move-model-txes', + mode: 'upgrade', func: async (client) => { await client.move( DOMAIN_TX, @@ -971,15 +980,18 @@ export const coreOperation: MigrateOperation = { }, { state: 'collaborative-docs-to-json', + mode: 'upgrade', func: migrateCollaborativeDocsToJson }, { state: 'accounts-to-social-ids', + mode: 'upgrade', func: migrateAccounts }, // ONLY FOR STAGING. REMOVE IT BEFORE MERGING TO PRODUCTION { state: 'space-members-to-account-uuids', + mode: 'upgrade', func: migrateSpaceMembersToAccountUuids } ]) diff --git a/models/document/src/migration.ts b/models/document/src/migration.ts index d0b5749a1d..b8d2bfeba2 100644 --- a/models/document/src/migration.ts +++ b/models/document/src/migration.ts @@ -357,42 +357,52 @@ export const documentOperation: MigrateOperation = { await tryMigrate(client, documentId, [ { state: 'updateDocumentIcons', + mode: 'upgrade', func: migrateDocumentIcons }, { state: 'migrate-timespaces', + mode: 'upgrade', func: migrateTeamspaces }, { state: 'migrate-teamspaces-mixins', + mode: 'upgrade', func: migrateTeamspacesMixins }, { state: 'migrateRank', + mode: 'upgrade', func: migrateRank }, { state: 'renameFields', + mode: 'upgrade', func: renameFields }, { state: 'renameFieldsRevert', + mode: 'upgrade', func: renameFieldsRevert }, { state: 'restoreContentField', + mode: 'upgrade', func: restoreContentField }, { state: 'migrateRanks', + mode: 'upgrade', func: migrateRanks }, { state: 'removeOldClasses', + mode: 'upgrade', func: removeOldClasses }, { state: 'migrateEmbeddings', + mode: 'upgrade', func: migrateEmbeddings }, { @@ -401,6 +411,7 @@ export const documentOperation: MigrateOperation = { }, { state: 'migrateEmbeddingsRefs', + mode: 'upgrade', func: migrateEmbeddingsRefs } ]) diff --git a/models/guest/src/migration.ts b/models/guest/src/migration.ts index 1a4251c858..beaf98af8d 100644 --- a/models/guest/src/migration.ts +++ b/models/guest/src/migration.ts @@ -3,12 +3,11 @@ import { tryMigrate, type MigrateOperation, type MigrationClient, - type MigrationUpgradeClient, - type ModelLogger + type MigrationUpgradeClient } from '@hcengineering/model' export const guestOperation: MigrateOperation = { - async migrate (client: MigrationClient, logger: ModelLogger): Promise { + async migrate (client: MigrationClient): Promise { await tryMigrate(client, guestId, []) }, async upgrade (state: Map>, client: () => Promise): Promise {} diff --git a/models/request/src/migration.ts b/models/request/src/migration.ts index 1b9e7248bc..5d92ea9bec 100644 --- a/models/request/src/migration.ts +++ b/models/request/src/migration.ts @@ -17,12 +17,11 @@ import { tryMigrate, type MigrateOperation, type MigrationClient, - type MigrationUpgradeClient, - type ModelLogger + type MigrationUpgradeClient } from '@hcengineering/model' export const requestOperation: MigrateOperation = { - async migrate (client: MigrationClient, logger: ModelLogger): Promise { + async migrate (client: MigrationClient): Promise { await tryMigrate(client, requestId, []) }, async upgrade (state: Map>, client: () => Promise): Promise {} diff --git a/models/server-activity/src/migration.ts b/models/server-activity/src/migration.ts index d4e626aa26..1d8e809c46 100644 --- a/models/server-activity/src/migration.ts +++ b/models/server-activity/src/migration.ts @@ -267,6 +267,7 @@ export const activityServerOperation: MigrateOperation = { await tryMigrate(client, serverActivityId, [ { state: 'doc-update-messages', + mode: 'upgrade', func: async (client) => { // Recreate activity to avoid duplicates await client.deleteMany(DOMAIN_ACTIVITY, { diff --git a/models/survey/src/migration.ts b/models/survey/src/migration.ts index 3248ce2754..0f88ca17a3 100644 --- a/models/survey/src/migration.ts +++ b/models/survey/src/migration.ts @@ -18,7 +18,6 @@ import { type MigrateOperation, type MigrationClient, type MigrationUpgradeClient, - type ModelLogger, tryMigrate, tryUpgrade } from '@hcengineering/model' @@ -26,7 +25,7 @@ import { import survey, { surveyId } from './index' export const surveyOperation: MigrateOperation = { - async migrate (client: MigrationClient, logger: ModelLogger): Promise { + async migrate (client: MigrationClient): Promise { await tryMigrate(client, surveyId, []) }, async upgrade (state: Map>, client: () => Promise): Promise { diff --git a/packages/model/src/migration.ts b/packages/model/src/migration.ts index b514ce1020..248ba1974f 100644 --- a/packages/model/src/migration.ts +++ b/packages/model/src/migration.ts @@ -122,20 +122,21 @@ export interface MigrationClient { * @public */ export type MigrationUpgradeClient = Client +export type MigrateMode = 'create' | 'upgrade' /** * @public */ export interface MigrateOperation { // Perform low level migration prior to the model update - preMigrate?: (client: MigrationClient, logger: ModelLogger) => Promise + preMigrate?: (client: MigrationClient, logger: ModelLogger, mode: MigrateMode) => Promise // Perform low level migration - migrate: (client: MigrationClient, logger: ModelLogger) => Promise + migrate: (client: MigrationClient, mode: MigrateMode) => Promise // Perform high level upgrade operations. upgrade: ( state: Map>, client: () => Promise, - logger: ModelLogger + mode: MigrateMode ) => Promise } @@ -144,7 +145,8 @@ export interface MigrateOperation { */ export interface Migrations { state: string - func: (client: MigrationClient) => Promise + mode?: MigrateMode // If set only applied to specified mode + func: (client: MigrationClient, mode: MigrateMode) => Promise } /** @@ -152,23 +154,31 @@ export interface Migrations { */ export interface UpgradeOperations { state: string - func: (client: MigrationUpgradeClient) => Promise + mode?: MigrateMode // If set only applied to specified mode + func: (client: MigrationUpgradeClient, mode: MigrateMode) => Promise } /** * @public */ -export async function tryMigrate (client: MigrationClient, plugin: string, migrations: Migrations[]): Promise { +export async function tryMigrate ( + client: MigrationClient, + plugin: string, + migrations: Migrations[], + mode: MigrateMode = 'upgrade' +): Promise { const states = client.migrateState.get(plugin) ?? new Set() for (const migration of migrations) { if (states.has(migration.state)) continue - try { - console.log('running migration', plugin, migration.state) - await migration.func(client) - } catch (err: any) { - console.error(err) - Analytics.handleError(err) - continue + if (migration.mode == null || migration.mode === mode) { + try { + console.log('running migration', plugin, migration.state) + await migration.func(client, mode) + } catch (err: any) { + console.error(err) + Analytics.handleError(err) + continue + } } const st: MigrationState = { plugin, @@ -190,22 +200,25 @@ export async function tryUpgrade ( state: Map>, client: () => Promise, plugin: string, - migrations: UpgradeOperations[] + migrations: UpgradeOperations[], + mode: MigrateMode = 'upgrade' ): Promise { const states = state.get(plugin) ?? new Set() - for (const migration of migrations) { - if (states.has(migration.state)) continue + for (const upgrades of migrations) { + if (states.has(upgrades.state)) continue const _client = await client() - try { - await migration.func(_client) - } catch (err: any) { - console.error(err) - Analytics.handleError(err) - continue + if (upgrades.mode == null || upgrades.mode === mode) { + try { + await upgrades.func(_client, mode) + } catch (err: any) { + console.error(err) + Analytics.handleError(err) + continue + } } const st: Data = { plugin, - state: migration.state + state: upgrades.state } const tx = new TxOperations(_client, core.account.System) await tx.createDoc(core.class.MigrationState, core.space.Configuration, st) diff --git a/pods/workspace/download-init-scripts.sh b/pods/workspace/download-init-scripts.sh new file mode 100755 index 0000000000..21aa5bfc74 --- /dev/null +++ b/pods/workspace/download-init-scripts.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +export INIT_SCRIPTS_BRANCH=${INIT_SCRIPTS_BRANCH:-unified-init-scripts} +# Download init repository +# Check if the file already exists +if [ -e "${INIT_SCRIPTS_BRANCH}.zip" ]; then + echo "File ${INIT_SCRIPTS_BRANCH}.zip already exists, skipping download" +else + wget https://github.com/hcengineering/init/archive/refs/heads/${INIT_SCRIPTS_BRANCH}.zip +fi + +unzip ${INIT_SCRIPTS_BRANCH}.zip -d ./temp +mv temp/init-${INIT_SCRIPTS_BRANCH} ./init +rm -rf ./temp diff --git a/server/backup/src/service.ts b/server/backup/src/service.ts index 7c3e826371..5d1c54a4df 100644 --- a/server/backup/src/service.ts +++ b/server/backup/src/service.ts @@ -263,7 +263,7 @@ class BackupWorker { workspace: ws.uuid, url: ws.url }) - const ctx = rootCtx.newChild(ws.uuid, { workspace: ws.uuid, url: ws.url }) + const ctx = rootCtx.newChild('doBackup', {}) const dataId = ws.dataId ?? (ws.uuid as unknown as WorkspaceDataId) let pipeline: Pipeline | undefined const backupIds = { @@ -278,54 +278,58 @@ class BackupWorker { dataId: ws.dataId, url: ws.url } - const result = await ctx.with('backup', { workspace: ws.uuid, url: ws.url }, (ctx) => - backup(ctx, '', wsIds, storage, { - skipDomains: this.skipDomains, - force: true, - timeout: this.config.Timeout * 1000, - connectTimeout: 5 * 60 * 1000, // 5 minutes to, - blobDownloadLimit: this.downloadLimit, - skipBlobContentTypes: [], - fullVerify: this.fullCheck, - storageAdapter: this.workspaceStorageAdapter, - getLastTx: async (): Promise => { - const config = this.getConfig(ctx, wsIds, null, this.workspaceStorageAdapter) - const adapterConf = config.adapters[config.domains[DOMAIN_TX]] - const hierarchy = new Hierarchy() - const modelDb = new ModelDb(hierarchy) - const txAdapter = await adapterConf.factory( - ctx, - this.contextVars, - hierarchy, - adapterConf.url, - wsIds, - modelDb, - this.workspaceStorageAdapter - ) - try { - await txAdapter.init?.(ctx, this.contextVars) + const result = await ctx.with( + 'backup', + {}, + (ctx) => + backup(ctx, '', wsIds, storage, { + skipDomains: this.skipDomains, + force: true, + timeout: this.config.Timeout * 1000, + connectTimeout: 5 * 60 * 1000, // 5 minutes to, + blobDownloadLimit: this.downloadLimit, + skipBlobContentTypes: [], + fullVerify: this.fullCheck, + storageAdapter: this.workspaceStorageAdapter, + getLastTx: async (): Promise => { + const config = this.getConfig(ctx, wsIds, null, this.workspaceStorageAdapter) + const adapterConf = config.adapters[config.domains[DOMAIN_TX]] + const hierarchy = new Hierarchy() + const modelDb = new ModelDb(hierarchy) + const txAdapter = await adapterConf.factory( + ctx, + this.contextVars, + hierarchy, + adapterConf.url, + wsIds, + modelDb, + this.workspaceStorageAdapter + ) + try { + await txAdapter.init?.(ctx, this.contextVars) - return ( - await txAdapter.rawFindAll( - DOMAIN_TX, - { objectSpace: { $ne: core.space.Model } }, - { limit: 1, sort: { modifiedOn: SortingOrder.Descending } } - ) - ).shift() - } finally { - await txAdapter.close() + return ( + await txAdapter.rawFindAll( + DOMAIN_TX, + { objectSpace: { $ne: core.space.Model } }, + { limit: 1, sort: { modifiedOn: SortingOrder.Descending } } + ) + ).shift() + } finally { + await txAdapter.close() + } + }, + getConnection: async () => { + if (pipeline === undefined) { + pipeline = await this.pipelineFactory(ctx, wsIds, true, () => {}, null) + } + return wrapPipeline(ctx, pipeline, wsIds) + }, + progress: (progress) => { + return notify?.(progress) ?? Promise.resolve() } - }, - getConnection: async () => { - if (pipeline === undefined) { - pipeline = await this.pipelineFactory(ctx, wsIds, true, () => {}, null) - } - return wrapPipeline(ctx, pipeline, wsIds) - }, - progress: (progress) => { - return notify?.(progress) ?? Promise.resolve() - } - }) + }), + { workspace: ws.uuid, url: ws.url } ) if (result.result) { @@ -444,28 +448,32 @@ export async function doRestoreWorkspace ( rootCtx.warn('\nRESTORE WORKSPACE ', { workspace: wsIds.uuid }) - const ctx = rootCtx.newChild(wsIds.uuid, { workspace: wsIds.uuid }) + const ctx = rootCtx.newChild('doRestore', {}) let pipeline: Pipeline | undefined try { const restoreIds = { uuid: bucketName as WorkspaceUuid, dataId: bucketName as WorkspaceDataId, url: '' } const storage = await createStorageBackupStorage(ctx, backupAdapter, restoreIds, wsIds.uuid) - const result: boolean = await ctx.with('restore', { workspace: wsIds.uuid }, (ctx) => - restore(ctx, '', wsIds, storage, { - date: -1, - skip: new Set(skipDomains), - recheck: false, // Do not need to recheck - storageAdapter: workspaceStorageAdapter, - cleanIndexState, - getConnection: async () => { - if (pipeline === undefined) { - pipeline = await pipelineFactory(ctx, wsIds, true, () => {}, null) + const result: boolean = await ctx.with( + 'restore', + {}, + (ctx) => + restore(ctx, '', wsIds, storage, { + date: -1, + skip: new Set(skipDomains), + recheck: false, // Do not need to recheck + storageAdapter: workspaceStorageAdapter, + cleanIndexState, + getConnection: async () => { + if (pipeline === undefined) { + pipeline = await pipelineFactory(ctx, wsIds, true, () => {}, null) + } + return wrapPipeline(ctx, pipeline, wsIds) + }, + progress: (progress) => { + return notify?.(progress) ?? Promise.resolve() } - return wrapPipeline(ctx, pipeline, wsIds) - }, - progress: (progress) => { - return notify?.(progress) ?? Promise.resolve() - } - }) + }), + { workspace: wsIds.uuid } ) return result } catch (err: any) { diff --git a/server/front/src/index.ts b/server/front/src/index.ts index 82af991264..c1089e264d 100644 --- a/server/front/src/index.ts +++ b/server/front/src/index.ts @@ -472,9 +472,9 @@ export function start ( return } - let blobInfo = await ctx.with('stat', { workspace: wsIds.uuid }, (ctx) => - config.storageAdapter.stat(ctx, wsIds, uuid) - ) + let blobInfo = await ctx.with('stat', {}, (ctx) => config.storageAdapter.stat(ctx, wsIds, uuid), { + workspace: wsIds.uuid + }) if (blobInfo === undefined) { ctx.error('No such key', { file: uuid, workspace: wsIds.uuid }) diff --git a/server/postgres/src/index.ts b/server/postgres/src/index.ts index 02893016f0..21b1b3c114 100644 --- a/server/postgres/src/index.ts +++ b/server/postgres/src/index.ts @@ -32,16 +32,26 @@ export function createPostgreeDestroyAdapter (url: string): WorkspaceDestroyAdap } const connection = await client.getClient() - await ctx.with('delete-workspace', {}, async () => { - // We need to clear information about workspace from all collections in schema - for (const [domain] of Object.entries(domainSchemas)) { - await ctx.with('delete-workspace-domain', {}, async () => { - await retryTxn(connection, async (client) => { - await client.unsafe(`delete from ${domain} where "workspaceId" = $1::uuid`, [workspaceUuid]) - }) - }) - } - }) + await ctx.with( + 'delete-workspace', + {}, + async (ctx) => { + // We need to clear information about workspace from all collections in schema + for (const [domain] of Object.entries(domainSchemas)) { + await ctx.with( + 'delete-workspace-domain', + {}, + async (ctx) => { + await retryTxn(connection, async (client) => { + await client.unsafe(`delete from ${domain} where "workspaceId" = $1::uuid`, [workspaceUuid]) + }) + }, + { domain } + ) + } + }, + { url: workspaceUuid } + ) } catch (err: any) { ctx.error('failed to clean workspace data', { err }) throw err diff --git a/server/tool/src/index.ts b/server/tool/src/index.ts index 242e7541da..db8a00a67f 100644 --- a/server/tool/src/index.ts +++ b/server/tool/src/index.ts @@ -39,7 +39,7 @@ import core, { type Ref, type WithLookup } from '@hcengineering/core' -import { consoleModelLogger, MigrateOperation, ModelLogger, tryMigrate } from '@hcengineering/model' +import { consoleModelLogger, MigrateOperation, ModelLogger, tryMigrate, type MigrateMode } from '@hcengineering/model' import { DomainIndexHelperImpl, Pipeline, StorageAdapter, type DbAdapter } from '@hcengineering/server-core' import { InitScript, WorkspaceInitializer } from './initializer' import toolPlugin from './plugin' @@ -150,7 +150,8 @@ export async function updateModel ( connection: TxOperations, pipeline: Pipeline, logger: ModelLogger = consoleModelLogger, - progress: (value: number) => Promise + progress: (value: number) => Promise, + mode: MigrateMode ): Promise { logger.log('connecting to transactor', { workspaceId }) @@ -167,7 +168,7 @@ export async function updateModel ( let i = 0 for (const op of migrateOperations) { const st = platformNow() - await op[1].upgrade(migrateState, async () => connection as any, logger) + await op[1].upgrade(migrateState, async () => connection as any, 'upgrade') const tdelta = platformNowDiff(st) if (tdelta > 0.5) { logger.log('Create', { name: op[0], time: tdelta }) @@ -244,7 +245,8 @@ export async function upgradeModel ( migrateOperations: [string, MigrateOperation][], logger: ModelLogger = consoleModelLogger, progress: (value: number) => Promise, - updateIndexes: 'perform' | 'skip' | 'disable' = 'skip' + updateIndexes: 'perform' | 'skip' | 'disable' = 'skip', + mode: MigrateMode = 'create' ): Promise { if (txes.some((tx) => tx.objectSpace !== core.space.Model)) { throw Error('Model txes must target only core.space.Model') @@ -276,7 +278,7 @@ export async function upgradeModel ( const t = platformNow() try { - await ctx.with(op[0], {}, (ctx) => preMigrate(preMigrateClient, logger)) + await ctx.with(op[0], {}, (ctx) => preMigrate(preMigrateClient, logger, mode)) } catch (err: any) { logger.error(`error during pre-migrate: ${op[0]} ${err.message}`, err) throw err @@ -320,7 +322,7 @@ export async function upgradeModel ( for (const op of migrateOperations) { try { const t = platformNow() - await ctx.with(op[0], {}, () => op[1].migrate(migrateClient, logger)) + await ctx.with(op[0], {}, () => op[1].migrate(migrateClient, mode)) const tdelta = platformNowDiff(t) if (tdelta > 0) { logger.log('migrate:', { workspaceId: wsIds, operation: op[0], time: tdelta }) @@ -349,7 +351,7 @@ export async function upgradeModel ( let i = 0 for (const op of migrateOperations) { const t = Date.now() - await ctx.with(op[0], {}, () => op[1].upgrade(migrateState, async () => connection, logger)) + await ctx.with(op[0], {}, () => op[1].upgrade(migrateState, async () => connection, mode)) const tdelta = Date.now() - t if (tdelta > 0) { logger.log('upgrade:', { operation: op[0], time: tdelta, workspaceId: wsIds }) diff --git a/server/workspace-service/src/service.ts b/server/workspace-service/src/service.ts index 61fe5cc519..43d4014550 100644 --- a/server/workspace-service/src/service.ts +++ b/server/workspace-service/src/service.ts @@ -181,17 +181,12 @@ export class WorkspaceWorker { await this.doSleep(ctx, opt) } else { void this.exec(async () => { - await this.doWorkspaceOperation( - ctx.newChild('workspaceOperation', { - workspace: workspace.uuid, - workspaceName: workspace.name - }), - workspace, - opt - ).catch((err) => { - Analytics.handleError(err) - ctx.error('error', { err }) - }) + await ctx + .with('workspaceOperation', {}, (ctx) => this.doWorkspaceOperation(ctx, workspace, opt)) + .catch((err) => { + Analytics.handleError(err) + ctx.error('error', { err }) + }) }) } } diff --git a/server/workspace-service/src/ws-operations.ts b/server/workspace-service/src/ws-operations.ts index 222bf5097c..1f152544fe 100644 --- a/server/workspace-service/src/ws-operations.ts +++ b/server/workspace-service/src/ws-operations.ts @@ -14,7 +14,7 @@ import core, { type WorkspaceIds, type WorkspaceInfoWithStatus } from '@hcengineering/core' -import { consoleModelLogger, type MigrateOperation, type ModelLogger } from '@hcengineering/model' +import { consoleModelLogger, type MigrateMode, type MigrateOperation, type ModelLogger } from '@hcengineering/model' import { getTransactorEndpoint } from '@hcengineering/server-client' import { SessionDataImpl, wrapPipeline, type Pipeline, type StorageAdapter } from '@hcengineering/server-core' import { getServerPipeline, getTxAdapterFactory, sharedPipelineContextVars } from '@hcengineering/server-pipeline' @@ -42,7 +42,7 @@ export async function createWorkspace ( ) => Promise, external: boolean = false ): Promise { - const childLogger = ctx.newChild('createWorkspace', {}, { workspace: workspaceInfo.uuid }) + const childLogger = ctx.newChild('createWorkspace', {}, {}) const ctxModellogger: ModelLogger = { log: (msg, data) => { childLogger.info(msg, data) @@ -96,9 +96,18 @@ export async function createWorkspace ( const client = new TxOperations(wrapPipeline(ctx, pipeline, wsIds), core.account.ConfigUser) - await updateModel(ctx, wsId, migrationOperation, client, pipeline, ctxModellogger, async (value) => { - await handleWsEvent?.('progress', version, 10 + Math.round((Math.min(value, 100) / 100) * 10)) - }) + await updateModel( + childLogger, + wsId, + migrationOperation, + client, + pipeline, + ctxModellogger, + async (value) => { + await handleWsEvent?.('progress', version, 10 + Math.round((Math.min(value, 100) / 100) * 10)) + }, + 'create' + ) ctx.info('Starting init script if any') const creatorUuid = workspaceInfo.createdBy @@ -108,7 +117,7 @@ export async function createWorkspace ( if (personInfo?.socialIds.length > 0) { await initializeWorkspace( - ctx, + childLogger, branding, wsIds, personInfo, @@ -128,7 +137,7 @@ export async function createWorkspace ( } await upgradeWorkspaceWith( - ctx, + childLogger, version, txes, migrationOperation, @@ -144,7 +153,8 @@ export async function createWorkspace ( }, false, 'disable', - external + external, + 'create' ) await handleWsEvent?.('create-done', version, 100, '') @@ -222,7 +232,8 @@ export async function upgradeWorkspace ( handleWsEvent, forceUpdate, forceIndexes ? 'perform' : 'skip', - external + external, + 'upgrade' ) } finally { await pipeline?.close() @@ -252,7 +263,8 @@ export async function upgradeWorkspaceWith ( ) => Promise, forceUpdate: boolean = true, updateIndexes: 'perform' | 'skip' | 'disable' = 'skip', - external: boolean = false + external: boolean = false, + mode: MigrateMode = 'create' ): Promise { const versionStr = versionToString(version) const workspaceVersion = { @@ -315,7 +327,8 @@ export async function upgradeWorkspaceWith ( async (value) => { progress = value }, - updateIndexes + updateIndexes, + mode ) await handleWsEvent?.('upgrade-done', version, 100, '') diff --git a/server/ws/src/server_http.ts b/server/ws/src/server_http.ts index 4b7d881619..f292cce060 100644 --- a/server/ws/src/server_http.ts +++ b/server/ws/src/server_http.ts @@ -379,9 +379,9 @@ export function startHttpServer ( const range = req.headers.range if (range !== undefined) { ctx - .with('file-range', { workspace: wsIds.uuid }, (ctx) => - getFileRange(ctx, range, externalStorage, wsIds, name, wrapRes(res)) - ) + .with('file-range', {}, (ctx) => getFileRange(ctx, range, externalStorage, wsIds, name, wrapRes(res)), { + workspace: wsIds.uuid + }) .catch((err) => { Analytics.handleError(err) ctx.error('/api/v1/blob get error', { err }) diff --git a/ws-tests/docker-compose.yaml b/ws-tests/docker-compose.yaml index 38c212a579..7d34079cc2 100644 --- a/ws-tests/docker-compose.yaml +++ b/ws-tests/docker-compose.yaml @@ -93,6 +93,7 @@ services: - STATS_URL=http://huly.local:4901 - BACKUP_STORAGE=${BACKUP_STORAGE_CONFIG} - BACKUP_BUCKET=${BACKUP_BUCKET_NAME} + # - INIT_WORKSPACE=huly restart: unless-stopped workspace_europe: image: hardcoreeng/workspace diff --git a/ws-tests/prepare_data.sh b/ws-tests/prepare_data.sh new file mode 100644 index 0000000000..a836283ebe --- /dev/null +++ b/ws-tests/prepare_data.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +./tool.sh create-workspace asa -w asa +./tool.sh backup-restore ${DUMP_ROOT}/asa asa --skip blob +./tool.sh assign-workspace user1 asa \ No newline at end of file