wieerwill/platform
1
0
Fork 0
mirror of https://github.com/hcengineering/platform.git synced 2025-04-07 16:41:31 +00:00
Code Issues Actions 5 Packages Projects Releases Wiki Activity

UBERF-9633: Reduce migration calls during workspace creation (#8242) (#8244)

Browse Source 
Signed-off-by: Andrey Sobolev <haiodo@gmail.com>
This commit is contained in:
Andrey Sobolev 2025-03-17 17:44:24 +07:00 committed by GitHub
parent 32550b6f9d
commit 419a3ddfdb
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
20 changed files with 260 additions and 144 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
models/activity/src/migration.ts
View File

@ -38,6 +38,7 @@ import {
type MigrationDocumentQuery,
type MigrationIterator,
type MigrationUpgradeClient,
type MigrateMode,
tryMigrate
} from '@hcengineering/model'
import { htmlToMarkup } from '@hcengineering/text'
@ -456,18 +457,21 @@ async function migrateSocialIdsInDocUpdates (client: MigrationClient): Promise<v
}
export const activityOperation: MigrateOperation = {
async migrate (client: MigrationClient): Promise<void> {
async migrate (client: MigrationClient, mode: MigrateMode): Promise<void> {
await tryMigrate(client, activityId, [
{
state: 'reactions',
mode: 'upgrade',
func: migrateReactions
},
{
state: 'markup',
mode: 'upgrade',
func: migrateMarkup
},
{
state: 'migrate-doc-update-messages-space',
mode: 'upgrade',
func: async (client) => {
await migrateMessagesSpace(
client,
@ -477,6 +481,30 @@ export const activityOperation: MigrateOperation = {
)
}
},
{
state: 'migrate-employee-space-v1',
mode: 'upgrade',
func: async () => {
await client.update<ActivityMessage>(
DOMAIN_ACTIVITY,
{ space: 'contact:space:Employee' as Ref<Space> },
{ space: contact.space.Contacts }
)
}
},
{
state: 'migrate-activity-markup',
mode: 'upgrade',
func: migrateActivityMarkup
},
{
state: 'move-reactions',
mode: 'upgrade',
func: async (client: MigrationClient): Promise<void> => {
await client.move(DOMAIN_ACTIVITY, { _class: activity.class.Reaction }, DOMAIN_REACTION)
await client.move(DOMAIN_ACTIVITY, { _class: activity.class.UserMentionInfo }, DOMAIN_USER_MENTION)
}
},
{
state: 'migrate-employee-space-v1',
func: async () => {

8
models/chunter/src/migration.ts
View File

@ -236,14 +236,17 @@ export const chunterOperation: MigrateOperation = {
await tryMigrate(client, chunterId, [
{
state: 'create-chat-messages',
mode: 'upgrade',
func: convertCommentsToChatMessages
},
{
state: 'remove-backlinks',
mode: 'upgrade',
func: removeBacklinks
},
{
state: 'migrate-chat-messages-space',
mode: 'upgrade',
func: async (client) => {
await migrateMessagesSpace(
client,
@ -255,6 +258,7 @@ export const chunterOperation: MigrateOperation = {
},
{
state: 'migrate-thread-messages-space',
mode: 'upgrade',
func: async (client) => {
await migrateMessagesSpace(
client,
@ -266,18 +270,21 @@ export const chunterOperation: MigrateOperation = {
},
{
state: 'remove-old-classes-v1',
mode: 'upgrade',
func: async (client) => {
await removeOldClasses(client)
}
},
{
state: 'remove-wrong-activity-v1',
mode: 'upgrade',
func: async (client) => {
await removeWrongActivity(client)
}
},
{
state: 'remove-chat-info-v1',
mode: 'upgrade',
func: async (client) => {
await client.deleteMany(DOMAIN_CHUNTER, { _class: 'chunter:class:ChatInfo' as Ref<Class<Doc>> })
await client.deleteMany(DOMAIN_TX, { objectClass: 'chunter:class:ChatInfo' })
@ -291,6 +298,7 @@ export const chunterOperation: MigrateOperation = {
},
{
state: 'remove-direct-doc-update-messages',
mode: 'upgrade',
func: async (client) => {
await client.deleteMany<DocUpdateMessage>(DOMAIN_ACTIVITY, {
_class: activity.class.DocUpdateMessage,

3
models/contact/src/migration.ts
View File

@ -22,7 +22,6 @@ import {
type MigrationClient,
type MigrationDocumentQuery,
type MigrationUpgradeClient,
type ModelLogger,
tryMigrate,
tryUpgrade
} from '@hcengineering/model'
@ -264,7 +263,7 @@ async function createSocialIdentities (client: MigrationClient): Promise<void> {
}
export const contactOperation: MigrateOperation = {
async migrate (client: MigrationClient, logger: ModelLogger): Promise<void> {
async migrate (client: MigrationClient): Promise<void> {
await tryMigrate(client, contactId, [
{
state: 'employees',

14
models/core/src/migration.ts
View File

@ -55,6 +55,7 @@ import {
createDefaultSpace,
tryMigrate,
tryUpgrade,
type MigrateMode,
type MigrateOperation,
type MigrateUpdate,
type MigrationClient,
@ -64,7 +65,7 @@ import {
} from '@hcengineering/model'
import { type StorageAdapter } from '@hcengineering/storage'
async function migrateStatusesToModel (client: MigrationClient): Promise<void> {
async function migrateStatusesToModel (client: MigrationClient, mode: MigrateMode): Promise<void> {
// Move statuses to model:
// Migrate the default ones with well-known ids as system's model
// And the rest as user's model
@ -887,26 +888,32 @@ export const coreOperation: MigrateOperation = {
await tryMigrate(client, coreId, [
{
state: 'statuses-to-model',
mode: 'upgrade',
func: migrateStatusesToModel
},
{
state: 'all-space-to-typed',
mode: 'upgrade',
func: migrateAllSpaceToTyped
},
{
state: 'add-spaces-owner-v1',
mode: 'upgrade',
func: migrateSpacesOwner
},
{
state: 'old-statuses-transactions',
mode: 'upgrade',
func: migrateStatusTransactions
},
{
state: 'collaborative-content-to-storage',
mode: 'upgrade',
func: migrateCollaborativeContentToStorage
},
{
state: 'fix-backups-hash-timestamp-v2',
mode: 'upgrade',
func: async (client: MigrationClient): Promise<void> => {
const now = Date.now().toString(16)
for (const d of client.hierarchy.domains()) {
@ -916,6 +923,7 @@ export const coreOperation: MigrateOperation = {
},
{
state: 'remove-collection-txes',
mode: 'upgrade',
func: async (client) => {
let processed = 0
let last = 0
@ -959,6 +967,7 @@ export const coreOperation: MigrateOperation = {
},
{
state: 'move-model-txes',
mode: 'upgrade',
func: async (client) => {
await client.move(
DOMAIN_TX,
@ -971,15 +980,18 @@ export const coreOperation: MigrateOperation = {
},
{
state: 'collaborative-docs-to-json',
mode: 'upgrade',
func: migrateCollaborativeDocsToJson
},
{
state: 'accounts-to-social-ids',
mode: 'upgrade',
func: migrateAccounts
},
// ONLY FOR STAGING. REMOVE IT BEFORE MERGING TO PRODUCTION
{
state: 'space-members-to-account-uuids',
mode: 'upgrade',
func: migrateSpaceMembersToAccountUuids
}
])

11
models/document/src/migration.ts
View File

@ -357,42 +357,52 @@ export const documentOperation: MigrateOperation = {
await tryMigrate(client, documentId, [
{
state: 'updateDocumentIcons',
mode: 'upgrade',
func: migrateDocumentIcons
},
{
state: 'migrate-timespaces',
mode: 'upgrade',
func: migrateTeamspaces
},
{
state: 'migrate-teamspaces-mixins',
mode: 'upgrade',
func: migrateTeamspacesMixins
},
{
state: 'migrateRank',
mode: 'upgrade',
func: migrateRank
},
{
state: 'renameFields',
mode: 'upgrade',
func: renameFields
},
{
state: 'renameFieldsRevert',
mode: 'upgrade',
func: renameFieldsRevert
},
{
state: 'restoreContentField',
mode: 'upgrade',
func: restoreContentField
},
{
state: 'migrateRanks',
mode: 'upgrade',
func: migrateRanks
},
{
state: 'removeOldClasses',
mode: 'upgrade',
func: removeOldClasses
},
{
state: 'migrateEmbeddings',
mode: 'upgrade',
func: migrateEmbeddings
},
{
@ -401,6 +411,7 @@ export const documentOperation: MigrateOperation = {
},
{
state: 'migrateEmbeddingsRefs',
mode: 'upgrade',
func: migrateEmbeddingsRefs
}
])

5
models/guest/src/migration.ts
View File

@ -3,12 +3,11 @@ import {
tryMigrate,
type MigrateOperation,
type MigrationClient,
type MigrationUpgradeClient,
type ModelLogger
type MigrationUpgradeClient
} from '@hcengineering/model'
export const guestOperation: MigrateOperation = {
async migrate (client: MigrationClient, logger: ModelLogger): Promise<void> {
async migrate (client: MigrationClient): Promise<void> {
await tryMigrate(client, guestId, [])
},
async upgrade (state: Map<string, Set<string>>, client: () => Promise<MigrationUpgradeClient>): Promise<void> {}

5
models/request/src/migration.ts
View File

@ -17,12 +17,11 @@ import {
tryMigrate,
type MigrateOperation,
type MigrationClient,
type MigrationUpgradeClient,
type ModelLogger
type MigrationUpgradeClient
} from '@hcengineering/model'
export const requestOperation: MigrateOperation = {
async migrate (client: MigrationClient, logger: ModelLogger): Promise<void> {
async migrate (client: MigrationClient): Promise<void> {
await tryMigrate(client, requestId, [])
},
async upgrade (state: Map<string, Set<string>>, client: () => Promise<MigrationUpgradeClient>): Promise<void> {}

1
models/server-activity/src/migration.ts
View File

@ -267,6 +267,7 @@ export const activityServerOperation: MigrateOperation = {
await tryMigrate(client, serverActivityId, [
{
state: 'doc-update-messages',
mode: 'upgrade',
func: async (client) => {
// Recreate activity to avoid duplicates
await client.deleteMany(DOMAIN_ACTIVITY, {

3
models/survey/src/migration.ts
View File

@ -18,7 +18,6 @@ import {
type MigrateOperation,
type MigrationClient,
type MigrationUpgradeClient,
type ModelLogger,
tryMigrate,
tryUpgrade
} from '@hcengineering/model'
@ -26,7 +25,7 @@ import {
import survey, { surveyId } from './index'
export const surveyOperation: MigrateOperation = {
async migrate (client: MigrationClient, logger: ModelLogger): Promise<void> {
async migrate (client: MigrationClient): Promise<void> {
await tryMigrate(client, surveyId, [])
},
async upgrade (state: Map<string, Set<string>>, client: () => Promise<MigrationUpgradeClient>): Promise<void> {

59
packages/model/src/migration.ts
View File

@ -122,20 +122,21 @@ export interface MigrationClient {
* @public
*/
export type MigrationUpgradeClient = Client
export type MigrateMode = 'create' | 'upgrade'
/**
* @public
*/
export interface MigrateOperation {
// Perform low level migration prior to the model update
preMigrate?: (client: MigrationClient, logger: ModelLogger) => Promise<void>
preMigrate?: (client: MigrationClient, logger: ModelLogger, mode: MigrateMode) => Promise<void>
// Perform low level migration
migrate: (client: MigrationClient, logger: ModelLogger) => Promise<void>
migrate: (client: MigrationClient, mode: MigrateMode) => Promise<void>
// Perform high level upgrade operations.
upgrade: (
state: Map<string, Set<string>>,
client: () => Promise<MigrationUpgradeClient>,
logger: ModelLogger
mode: MigrateMode
) => Promise<void>
}
@ -144,7 +145,8 @@ export interface MigrateOperation {
*/
export interface Migrations {
state: string
func: (client: MigrationClient) => Promise<void>
mode?: MigrateMode // If set only applied to specified mode
func: (client: MigrationClient, mode: MigrateMode) => Promise<void>
}
/**
@ -152,23 +154,31 @@ export interface Migrations {
*/
export interface UpgradeOperations {
state: string
func: (client: MigrationUpgradeClient) => Promise<void>
mode?: MigrateMode // If set only applied to specified mode
func: (client: MigrationUpgradeClient, mode: MigrateMode) => Promise<void>
}
/**
* @public
*/
export async function tryMigrate (client: MigrationClient, plugin: string, migrations: Migrations[]): Promise<void> {
export async function tryMigrate (
client: MigrationClient,
plugin: string,
migrations: Migrations[],
mode: MigrateMode = 'upgrade'
): Promise<void> {
const states = client.migrateState.get(plugin) ?? new Set()
for (const migration of migrations) {
if (states.has(migration.state)) continue
try {
console.log('running migration', plugin, migration.state)
await migration.func(client)
} catch (err: any) {
console.error(err)
Analytics.handleError(err)
continue
if (migration.mode == null || migration.mode === mode) {
try {
console.log('running migration', plugin, migration.state)
await migration.func(client, mode)
} catch (err: any) {
console.error(err)
Analytics.handleError(err)
continue
}
}
const st: MigrationState = {
plugin,
@ -190,22 +200,25 @@ export async function tryUpgrade (
state: Map<string, Set<string>>,
client: () => Promise<MigrationUpgradeClient>,
plugin: string,
migrations: UpgradeOperations[]
migrations: UpgradeOperations[],
mode: MigrateMode = 'upgrade'
): Promise<void> {
const states = state.get(plugin) ?? new Set()
for (const migration of migrations) {
if (states.has(migration.state)) continue
for (const upgrades of migrations) {
if (states.has(upgrades.state)) continue
const _client = await client()
try {
await migration.func(_client)
} catch (err: any) {
console.error(err)
Analytics.handleError(err)
continue
if (upgrades.mode == null || upgrades.mode === mode) {
try {
await upgrades.func(_client, mode)
} catch (err: any) {
console.error(err)
Analytics.handleError(err)
continue
}
}
const st: Data<MigrationState> = {
plugin,
state: migration.state
state: upgrades.state
}
const tx = new TxOperations(_client, core.account.System)
await tx.createDoc(core.class.MigrationState, core.space.Configuration, st)

13
pods/workspace/download-init-scripts.sh Executable file
View File

@ -0,0 +1,13 @@
#!/usr/bin/env bash
export INIT_SCRIPTS_BRANCH=${INIT_SCRIPTS_BRANCH:-unified-init-scripts}
# Download init repository
# Check if the file already exists
if [ -e "${INIT_SCRIPTS_BRANCH}.zip" ]; then
echo "File ${INIT_SCRIPTS_BRANCH}.zip already exists, skipping download"
else
wget https://github.com/hcengineering/init/archive/refs/heads/${INIT_SCRIPTS_BRANCH}.zip
fi
unzip ${INIT_SCRIPTS_BRANCH}.zip -d ./temp
mv temp/init-${INIT_SCRIPTS_BRANCH} ./init
rm -rf ./temp

136
server/backup/src/service.ts
View File

@ -263,7 +263,7 @@ class BackupWorker {
workspace: ws.uuid,
url: ws.url
})
const ctx = rootCtx.newChild(ws.uuid, { workspace: ws.uuid, url: ws.url })
const ctx = rootCtx.newChild('doBackup', {})
const dataId = ws.dataId ?? (ws.uuid as unknown as WorkspaceDataId)
let pipeline: Pipeline | undefined
const backupIds = {
@ -278,54 +278,58 @@ class BackupWorker {
dataId: ws.dataId,
url: ws.url
}
const result = await ctx.with('backup', { workspace: ws.uuid, url: ws.url }, (ctx) =>
backup(ctx, '', wsIds, storage, {
skipDomains: this.skipDomains,
force: true,
timeout: this.config.Timeout * 1000,
connectTimeout: 5 * 60 * 1000, // 5 minutes to,
blobDownloadLimit: this.downloadLimit,
skipBlobContentTypes: [],
fullVerify: this.fullCheck,
storageAdapter: this.workspaceStorageAdapter,
getLastTx: async (): Promise<Tx | undefined> => {
const config = this.getConfig(ctx, wsIds, null, this.workspaceStorageAdapter)
const adapterConf = config.adapters[config.domains[DOMAIN_TX]]
const hierarchy = new Hierarchy()
const modelDb = new ModelDb(hierarchy)
const txAdapter = await adapterConf.factory(
ctx,
this.contextVars,
hierarchy,
adapterConf.url,
wsIds,
modelDb,
this.workspaceStorageAdapter
)
try {
await txAdapter.init?.(ctx, this.contextVars)
const result = await ctx.with(
'backup',
{},
(ctx) =>
backup(ctx, '', wsIds, storage, {
skipDomains: this.skipDomains,
force: true,
timeout: this.config.Timeout * 1000,
connectTimeout: 5 * 60 * 1000, // 5 minutes to,
blobDownloadLimit: this.downloadLimit,
skipBlobContentTypes: [],
fullVerify: this.fullCheck,
storageAdapter: this.workspaceStorageAdapter,
getLastTx: async (): Promise<Tx | undefined> => {
const config = this.getConfig(ctx, wsIds, null, this.workspaceStorageAdapter)
const adapterConf = config.adapters[config.domains[DOMAIN_TX]]
const hierarchy = new Hierarchy()
const modelDb = new ModelDb(hierarchy)
const txAdapter = await adapterConf.factory(
ctx,
this.contextVars,
hierarchy,
adapterConf.url,
wsIds,
modelDb,
this.workspaceStorageAdapter
)
try {
await txAdapter.init?.(ctx, this.contextVars)
return (
await txAdapter.rawFindAll<Tx>(
DOMAIN_TX,
{ objectSpace: { $ne: core.space.Model } },
{ limit: 1, sort: { modifiedOn: SortingOrder.Descending } }
)
).shift()
} finally {
await txAdapter.close()
return (
await txAdapter.rawFindAll<Tx>(
DOMAIN_TX,
{ objectSpace: { $ne: core.space.Model } },
{ limit: 1, sort: { modifiedOn: SortingOrder.Descending } }
)
).shift()
} finally {
await txAdapter.close()
}
},
getConnection: async () => {
if (pipeline === undefined) {
pipeline = await this.pipelineFactory(ctx, wsIds, true, () => {}, null)
}
return wrapPipeline(ctx, pipeline, wsIds)
},
progress: (progress) => {
return notify?.(progress) ?? Promise.resolve()
}
},
getConnection: async () => {
if (pipeline === undefined) {
pipeline = await this.pipelineFactory(ctx, wsIds, true, () => {}, null)
}
return wrapPipeline(ctx, pipeline, wsIds)
},
progress: (progress) => {
return notify?.(progress) ?? Promise.resolve()
}
})
}),
{ workspace: ws.uuid, url: ws.url }
)
if (result.result) {
@ -444,28 +448,32 @@ export async function doRestoreWorkspace (
rootCtx.warn('\nRESTORE WORKSPACE ', {
workspace: wsIds.uuid
})
const ctx = rootCtx.newChild(wsIds.uuid, { workspace: wsIds.uuid })
const ctx = rootCtx.newChild('doRestore', {})
let pipeline: Pipeline | undefined
try {
const restoreIds = { uuid: bucketName as WorkspaceUuid, dataId: bucketName as WorkspaceDataId, url: '' }
const storage = await createStorageBackupStorage(ctx, backupAdapter, restoreIds, wsIds.uuid)
const result: boolean = await ctx.with('restore', { workspace: wsIds.uuid }, (ctx) =>
restore(ctx, '', wsIds, storage, {
date: -1,
skip: new Set(skipDomains),
recheck: false, // Do not need to recheck
storageAdapter: workspaceStorageAdapter,
cleanIndexState,
getConnection: async () => {
if (pipeline === undefined) {
pipeline = await pipelineFactory(ctx, wsIds, true, () => {}, null)
const result: boolean = await ctx.with(
'restore',
{},
(ctx) =>
restore(ctx, '', wsIds, storage, {
date: -1,
skip: new Set(skipDomains),
recheck: false, // Do not need to recheck
storageAdapter: workspaceStorageAdapter,
cleanIndexState,
getConnection: async () => {
if (pipeline === undefined) {
pipeline = await pipelineFactory(ctx, wsIds, true, () => {}, null)
}
return wrapPipeline(ctx, pipeline, wsIds)
},
progress: (progress) => {
return notify?.(progress) ?? Promise.resolve()
}
return wrapPipeline(ctx, pipeline, wsIds)
},
progress: (progress) => {
return notify?.(progress) ?? Promise.resolve()
}
})
}),
{ workspace: wsIds.uuid }
)
return result
} catch (err: any) {

6
server/front/src/index.ts
View File

@ -472,9 +472,9 @@ export function start (
return
}
let blobInfo = await ctx.with('stat', { workspace: wsIds.uuid }, (ctx) =>
config.storageAdapter.stat(ctx, wsIds, uuid)
)
let blobInfo = await ctx.with('stat', {}, (ctx) => config.storageAdapter.stat(ctx, wsIds, uuid), {
workspace: wsIds.uuid
})
if (blobInfo === undefined) {
ctx.error('No such key', { file: uuid, workspace: wsIds.uuid })

30
server/postgres/src/index.ts
View File

@ -32,16 +32,26 @@ export function createPostgreeDestroyAdapter (url: string): WorkspaceDestroyAdap
}
const connection = await client.getClient()
await ctx.with('delete-workspace', {}, async () => {
// We need to clear information about workspace from all collections in schema
for (const [domain] of Object.entries(domainSchemas)) {
await ctx.with('delete-workspace-domain', {}, async () => {
await retryTxn(connection, async (client) => {
await client.unsafe(`delete from ${domain} where "workspaceId" = $1::uuid`, [workspaceUuid])
})
})
}
})
await ctx.with(
'delete-workspace',
{},
async (ctx) => {
// We need to clear information about workspace from all collections in schema
for (const [domain] of Object.entries(domainSchemas)) {
await ctx.with(
'delete-workspace-domain',
{},
async (ctx) => {
await retryTxn(connection, async (client) => {
await client.unsafe(`delete from ${domain} where "workspaceId" = $1::uuid`, [workspaceUuid])
})
},
{ domain }
)
}
},
{ url: workspaceUuid }
)
} catch (err: any) {
ctx.error('failed to clean workspace data', { err })
throw err

16
server/tool/src/index.ts
View File

@ -39,7 +39,7 @@ import core, {
type Ref,
type WithLookup
} from '@hcengineering/core'
import { consoleModelLogger, MigrateOperation, ModelLogger, tryMigrate } from '@hcengineering/model'
import { consoleModelLogger, MigrateOperation, ModelLogger, tryMigrate, type MigrateMode } from '@hcengineering/model'
import { DomainIndexHelperImpl, Pipeline, StorageAdapter, type DbAdapter } from '@hcengineering/server-core'
import { InitScript, WorkspaceInitializer } from './initializer'
import toolPlugin from './plugin'
@ -150,7 +150,8 @@ export async function updateModel (
connection: TxOperations,
pipeline: Pipeline,
logger: ModelLogger = consoleModelLogger,
progress: (value: number) => Promise<void>
progress: (value: number) => Promise<void>,
mode: MigrateMode
): Promise<void> {
logger.log('connecting to transactor', { workspaceId })
@ -167,7 +168,7 @@ export async function updateModel (
let i = 0
for (const op of migrateOperations) {
const st = platformNow()
await op[1].upgrade(migrateState, async () => connection as any, logger)
await op[1].upgrade(migrateState, async () => connection as any, 'upgrade')
const tdelta = platformNowDiff(st)
if (tdelta > 0.5) {
logger.log('Create', { name: op[0], time: tdelta })
@ -244,7 +245,8 @@ export async function upgradeModel (
migrateOperations: [string, MigrateOperation][],
logger: ModelLogger = consoleModelLogger,
progress: (value: number) => Promise<void>,
updateIndexes: 'perform' | 'skip' | 'disable' = 'skip'
updateIndexes: 'perform' | 'skip' | 'disable' = 'skip',
mode: MigrateMode = 'create'
): Promise<Tx[]> {
if (txes.some((tx) => tx.objectSpace !== core.space.Model)) {
throw Error('Model txes must target only core.space.Model')
@ -276,7 +278,7 @@ export async function upgradeModel (
const t = platformNow()
try {
await ctx.with(op[0], {}, (ctx) => preMigrate(preMigrateClient, logger))
await ctx.with(op[0], {}, (ctx) => preMigrate(preMigrateClient, logger, mode))
} catch (err: any) {
logger.error(`error during pre-migrate: ${op[0]} ${err.message}`, err)
throw err
@ -320,7 +322,7 @@ export async function upgradeModel (
for (const op of migrateOperations) {
try {
const t = platformNow()
await ctx.with(op[0], {}, () => op[1].migrate(migrateClient, logger))
await ctx.with(op[0], {}, () => op[1].migrate(migrateClient, mode))
const tdelta = platformNowDiff(t)
if (tdelta > 0) {
logger.log('migrate:', { workspaceId: wsIds, operation: op[0], time: tdelta })
@ -349,7 +351,7 @@ export async function upgradeModel (
let i = 0
for (const op of migrateOperations) {
const t = Date.now()
await ctx.with(op[0], {}, () => op[1].upgrade(migrateState, async () => connection, logger))
await ctx.with(op[0], {}, () => op[1].upgrade(migrateState, async () => connection, mode))
const tdelta = Date.now() - t
if (tdelta > 0) {
logger.log('upgrade:', { operation: op[0], time: tdelta, workspaceId: wsIds })

17
server/workspace-service/src/service.ts
View File

@ -181,17 +181,12 @@ export class WorkspaceWorker {
await this.doSleep(ctx, opt)
} else {
void this.exec(async () => {
await this.doWorkspaceOperation(
ctx.newChild('workspaceOperation', {
workspace: workspace.uuid,
workspaceName: workspace.name
}),
workspace,
opt
).catch((err) => {
Analytics.handleError(err)
ctx.error('error', { err })
})
await ctx
.with('workspaceOperation', {}, (ctx) => this.doWorkspaceOperation(ctx, workspace, opt))
.catch((err) => {
Analytics.handleError(err)
ctx.error('error', { err })
})
})
}
}

35
server/workspace-service/src/ws-operations.ts
View File

@ -14,7 +14,7 @@ import core, {
type WorkspaceIds,
type WorkspaceInfoWithStatus
} from '@hcengineering/core'
import { consoleModelLogger, type MigrateOperation, type ModelLogger } from '@hcengineering/model'
import { consoleModelLogger, type MigrateMode, type MigrateOperation, type ModelLogger } from '@hcengineering/model'
import { getTransactorEndpoint } from '@hcengineering/server-client'
import { SessionDataImpl, wrapPipeline, type Pipeline, type StorageAdapter } from '@hcengineering/server-core'
import { getServerPipeline, getTxAdapterFactory, sharedPipelineContextVars } from '@hcengineering/server-pipeline'
@ -42,7 +42,7 @@ export async function createWorkspace (
) => Promise<void>,
external: boolean = false
): Promise<void> {
const childLogger = ctx.newChild('createWorkspace', {}, { workspace: workspaceInfo.uuid })
const childLogger = ctx.newChild('createWorkspace', {}, {})
const ctxModellogger: ModelLogger = {
log: (msg, data) => {
childLogger.info(msg, data)
@ -96,9 +96,18 @@ export async function createWorkspace (
const client = new TxOperations(wrapPipeline(ctx, pipeline, wsIds), core.account.ConfigUser)
await updateModel(ctx, wsId, migrationOperation, client, pipeline, ctxModellogger, async (value) => {
await handleWsEvent?.('progress', version, 10 + Math.round((Math.min(value, 100) / 100) * 10))
})
await updateModel(
childLogger,
wsId,
migrationOperation,
client,
pipeline,
ctxModellogger,
async (value) => {
await handleWsEvent?.('progress', version, 10 + Math.round((Math.min(value, 100) / 100) * 10))
},
'create'
)
ctx.info('Starting init script if any')
const creatorUuid = workspaceInfo.createdBy
@ -108,7 +117,7 @@ export async function createWorkspace (
if (personInfo?.socialIds.length > 0) {
await initializeWorkspace(
ctx,
childLogger,
branding,
wsIds,
personInfo,
@ -128,7 +137,7 @@ export async function createWorkspace (
}
await upgradeWorkspaceWith(
ctx,
childLogger,
version,
txes,
migrationOperation,
@ -144,7 +153,8 @@ export async function createWorkspace (
},
false,
'disable',
external
external,
'create'
)
await handleWsEvent?.('create-done', version, 100, '')
@ -222,7 +232,8 @@ export async function upgradeWorkspace (
handleWsEvent,
forceUpdate,
forceIndexes ? 'perform' : 'skip',
external
external,
'upgrade'
)
} finally {
await pipeline?.close()
@ -252,7 +263,8 @@ export async function upgradeWorkspaceWith (
) => Promise<void>,
forceUpdate: boolean = true,
updateIndexes: 'perform' | 'skip' | 'disable' = 'skip',
external: boolean = false
external: boolean = false,
mode: MigrateMode = 'create'
): Promise<void> {
const versionStr = versionToString(version)
const workspaceVersion = {
@ -315,7 +327,8 @@ export async function upgradeWorkspaceWith (
async (value) => {
progress = value
},
updateIndexes
updateIndexes,
mode
)
await handleWsEvent?.('upgrade-done', version, 100, '')

6
server/ws/src/server_http.ts
View File

@ -379,9 +379,9 @@ export function startHttpServer (
const range = req.headers.range
if (range !== undefined) {
ctx
.with('file-range', { workspace: wsIds.uuid }, (ctx) =>
getFileRange(ctx, range, externalStorage, wsIds, name, wrapRes(res))
)
.with('file-range', {}, (ctx) => getFileRange(ctx, range, externalStorage, wsIds, name, wrapRes(res)), {
workspace: wsIds.uuid
})
.catch((err) => {
Analytics.handleError(err)
ctx.error('/api/v1/blob get error', { err })

1
ws-tests/docker-compose.yaml
View File

@ -93,6 +93,7 @@ services:
- STATS_URL=http://huly.local:4901
- BACKUP_STORAGE=${BACKUP_STORAGE_CONFIG}
- BACKUP_BUCKET=${BACKUP_BUCKET_NAME}
# - INIT_WORKSPACE=huly
restart: unless-stopped
workspace_europe:
image: hardcoreeng/workspace

5
ws-tests/prepare_data.sh Normal file
View File

@ -0,0 +1,5 @@
#!/usr/bin/env bash
./tool.sh create-workspace asa -w asa
./tool.sh backup-restore ${DUMP_ROOT}/asa asa --skip blob
./tool.sh assign-workspace user1 asa