UBERF-9299: Fix backup service backup order (#7826)
Some checks are pending
CI / build (push) Waiting to run
CI / svelte-check (push) Blocked by required conditions
CI / formatting (push) Blocked by required conditions
CI / test (push) Blocked by required conditions
CI / uitest (push) Waiting to run
CI / uitest-pg (push) Waiting to run
CI / uitest-qms (push) Waiting to run
CI / docker-build (push) Blocked by required conditions
CI / dist-build (push) Blocked by required conditions

This commit is contained in:
Andrey Sobolev 2025-01-30 10:11:31 +07:00 committed by GitHub
parent 5c9e71be09
commit 9c3a1cc641
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 125 additions and 29 deletions

2
.vscode/launch.json vendored
View File

@ -394,7 +394,7 @@
"DB_URL": "mongodb://localhost:27017",
"MODEL_JSON": "${workspaceRoot}/models/all/bundle/model.json",
"SECRET": "secret",
"REGION": "pg",
"REGION": "cockroach",
"BUCKET_NAME":"backups",
"INTERVAL":"30"
},

View File

@ -51,6 +51,11 @@
<span class="fs-title overflow-label" class:content-color={contentColor}>
{#if label}<Label {label} />{/if}<slot name="title" />
</span>
{#if $$slots['title-tools']}
<div class="buttons-group small-gap">
<slot name="title-tools" />
</div>
{/if}
</div>
{#if $$slots.tools}
<div class="buttons-group small-gap">

View File

@ -1,5 +1,14 @@
<script lang="ts">
import { groupByArray, isActiveMode, type BaseWorkspaceInfo } from '@hcengineering/core'
import {
groupByArray,
isActiveMode,
isArchivingMode,
isDeletingMode,
isMigrationMode,
isRestoringMode,
reduceCalls,
type BaseWorkspaceInfo
} from '@hcengineering/core'
import { getEmbeddedLabel } from '@hcengineering/platform'
import { isAdminUser } from '@hcengineering/presentation'
import {
@ -14,7 +23,8 @@
Popup,
Scroller,
SearchEdit,
ticker
ticker,
CheckBox
} from '@hcengineering/ui'
import { workbenchId } from '@hcengineering/workbench'
import { getAllWorkspaces, getRegionInfo, performWorkspaceOperation, type RegionInfo } from '../utils'
@ -32,13 +42,14 @@
let workspaces: WorkspaceInfo[] = []
$: if ($ticker > 0) {
void getAllWorkspaces().then((res) => {
workspaces = res.sort((a, b) =>
(b.workspaceUrl ?? b.workspace).localeCompare(a.workspaceUrl ?? a.workspace)
) as WorkspaceInfo[]
})
}
const updateWorkspaces = reduceCalls(async (_: number) => {
const res = await getAllWorkspaces()
workspaces = res.sort((a, b) =>
(b.workspaceUrl ?? b.workspace).localeCompare(a.workspaceUrl ?? a.workspace)
) as WorkspaceInfo[]
})
$: void updateWorkspaces($ticker)
const now = Date.now()
@ -55,13 +66,26 @@
FewOrMoreYears: 10000000
}
let limit = 50
// Individual filters
let showActive: boolean = true
let showArchived: boolean = false
let showDeleted: boolean = true
let showOther: boolean = true
$: groupped = groupByArray(
workspaces.filter(
(it) =>
(it.workspaceName?.includes(search) ?? false) ||
(it.workspaceUrl?.includes(search) ?? false) ||
it.workspace?.includes(search) ||
it.createdBy?.includes(search)
((it.workspaceName?.includes(search) ?? false) ||
(it.workspaceUrl?.includes(search) ?? false) ||
it.workspace?.includes(search) ||
it.createdBy?.includes(search)) &&
((showActive && isActiveMode(it.mode)) ||
(showArchived && isArchivingMode(it.mode)) ||
(showDeleted && isDeletingMode(it.mode)) ||
(showOther && (isMigrationMode(it.mode) || isRestoringMode(it.mode))))
),
(it) => {
const lastUsageDays = Math.round((now - it.lastVisit) / (1000 * 3600 * 24))
@ -92,6 +116,26 @@
<SearchEdit bind:value={search} width={'100%'} />
</div>
<div class="p-3 flex-col">
<span class="fs-title mr-2">Filters: </span>
<div class="flex-row-center">
Show active workspaces:
<CheckBox bind:checked={showActive} />
</div>
<div class="flex-row-center">
<span class="mr-2">Show archived workspaces:</span>
<CheckBox bind:checked={showArchived} />
</div>
<div class="flex-row-center">
<span class="mr-2">Show deleted workspaces:</span>
<CheckBox bind:checked={showDeleted} />
</div>
<div class="flex-row-center">
<span class="mr-2">Show other workspaces:</span>
<CheckBox bind:checked={showOther} />
</div>
</div>
<div class="fs-title p-3 flex-row-center">
<span class="mr-2"> Migration region selector: </span>
<ButtonMenu
@ -109,6 +153,7 @@
<div class="mr-4">
{#each Object.keys(dayRanges) as k}
{@const v = groupped.get(k) ?? []}
{@const hasMore = (groupped.get(k) ?? []).length > limit}
{@const activeV = v.filter((it) => it.mode === 'active' && (it.region ?? '') !== selectedRegionId)}
{@const archiveV = v.filter((it) => it.mode === 'active')}
{@const archivedD = v.filter((it) => it.mode === 'archived')}
@ -116,13 +161,31 @@
{#if v.length > 0}
<Expandable expandable={true} bordered={true}>
<svelte:fragment slot="title">
<span class="fs-title focused-button">
{k} - {v.length}
<span class="fs-title focused-button flex-row-center">
{k} -
{#if hasMore}
{limit} of {v.length}
{:else}
{v.length}
{/if}
{#if av > 0}
- maitenance: {av}
{/if}
</span>
</svelte:fragment>
<svelte:fragment slot="title-tools">
{#if hasMore}
<div class="ml-4">
<Button
label={getEmbeddedLabel(`More ${k}`)}
kind={'link'}
on:click={() => {
limit += 50
}}
/>
</div>
{/if}
</svelte:fragment>
<svelte:fragment slot="tools">
{#if archiveV.length > 0}
<Button
@ -153,7 +216,7 @@
/>
{/if}
</svelte:fragment>
{#each v as workspace}
{#each v.slice(0, limit) as workspace}
{@const wsName = workspace.workspaceName ?? workspace.workspace}
{@const lastUsageDays = Math.round((Date.now() - workspace.lastVisit) / (1000 * 3600 * 24))}
<!-- svelte-ignore a11y-click-events-have-key-events -->

View File

@ -63,6 +63,7 @@ import { connect } from '@hcengineering/server-tool'
import { randomBytes } from 'crypto'
import otpGenerator from 'otp-generator'
import { getWorkspaceDestroyAdapter, sharedPipelineContextVars } from '@hcengineering/server-pipeline'
import { accountPlugin } from './plugin'
import type {
Account,
@ -92,7 +93,6 @@ import {
toAccountInfo,
verifyPassword
} from './utils'
import { getWorkspaceDestroyAdapter, sharedPipelineContextVars } from '@hcengineering/server-pipeline'
import MD5 from 'crypto-js/md5'
function buildGravatarId (email: string): string {
@ -866,7 +866,8 @@ export async function listWorkspaces (
db: AccountDB,
branding: Branding | null,
token: string,
region?: string | null
region?: string | null,
mode?: WorkspaceMode | null
): Promise<WorkspaceInfo[]> {
decodeToken(ctx, token) // Just verify token is valid
@ -874,9 +875,17 @@ export async function listWorkspaces (
region = null
}
return (await db.workspace.find(region != null ? { region } : {}))
.filter((it) => it.disabled !== true)
.map(trimWorkspaceInfo)
const q: Query<Workspace> = {
disabled: { $ne: true }
}
if (region != null) {
q.region = region
}
if (mode != null) {
q.mode = mode
}
return (await db.workspace.find(q)).map(trimWorkspaceInfo)
}
/**
@ -1701,7 +1710,8 @@ export async function getAllWorkspaces (
ctx: MeasureContext,
db: AccountDB,
branding: Branding | null,
token: string
token: string,
mode?: WorkspaceMode
): Promise<BaseWorkspaceInfo[]> {
const { email } = decodeToken(ctx, token)
const account = await getAccount(db, email)

View File

@ -96,16 +96,18 @@ class BackupWorker {
console.log('schedule backup with interval', this.config.Interval, 'seconds')
while (!this.canceled) {
try {
const res = await this.backup(ctx, this.config.CoolDown * 1000)
const res = await this.backup(ctx, (this.config.Interval / 4) * 1000)
this.printStats(ctx, res)
if (res.skipped === 0) {
console.log('cool down', this.config.CoolDown, 'seconds')
await new Promise<void>((resolve) => setTimeout(resolve, this.config.CoolDown * 1000))
}
} catch (err: any) {
Analytics.handleError(err)
ctx.error('error retry in cool down/5', { cooldown: this.config.CoolDown, error: err })
await new Promise<void>((resolve) => setTimeout(resolve, (this.config.CoolDown / 5) * 1000))
continue
}
console.log('cool down', this.config.CoolDown, 'seconds')
await new Promise<void>((resolve) => setTimeout(resolve, this.config.CoolDown * 1000))
}
}
@ -145,15 +147,31 @@ class BackupWorker {
return !workspacesIgnore.has(it.workspace)
})
workspaces.sort((a, b) => {
return (b.backupInfo?.backupSize ?? 0) - (a.backupInfo?.backupSize ?? 0)
const lastBackupMin = Math.round(((a.backupInfo?.lastBackup ?? 0) - (b.backupInfo?.lastBackup ?? 0)) / 60)
if (lastBackupMin === 0) {
// Same minute, sort by backup size
return (a.backupInfo?.backupSize ?? 0) - (b.backupInfo?.backupSize ?? 0)
}
return lastBackupMin
})
ctx.info('Preparing for BACKUP', {
ctx.warn('Preparing for BACKUP', {
total: workspaces.length,
skipped,
workspaces: workspaces.map((it) => it.workspace)
})
const part = workspaces.slice(0, 500)
let idx = 0
for (const ws of part) {
ctx.warn('prepare workspace', {
idx: ++idx,
workspace: ws.workspaceUrl ?? ws.workspace,
backupSize: ws.backupInfo?.backupSize ?? 0,
lastBackupSec: (Date.now() - (ws.backupInfo?.lastBackup ?? 0)) / 1000
})
}
return await this.doBackup(ctx, workspaces, recheckTimeout)
}

View File

@ -50,7 +50,7 @@ export async function listAccountWorkspaces (token: string, region: string | nul
},
body: JSON.stringify({
method: 'listWorkspaces',
params: [token, region]
params: [token, region, 'active']
})
})
).json()