UBERF-9299: Fix backup service backup order (#7826)
Some checks are pending
CI / build (push) Waiting to run
CI / svelte-check (push) Blocked by required conditions
CI / formatting (push) Blocked by required conditions
CI / test (push) Blocked by required conditions
CI / uitest (push) Waiting to run
CI / uitest-pg (push) Waiting to run
CI / uitest-qms (push) Waiting to run
CI / docker-build (push) Blocked by required conditions
CI / dist-build (push) Blocked by required conditions

This commit is contained in:
Andrey Sobolev 2025-01-30 10:11:31 +07:00 committed by GitHub
parent 5c9e71be09
commit 9c3a1cc641
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 125 additions and 29 deletions

2
.vscode/launch.json vendored
View File

@ -394,7 +394,7 @@
"DB_URL": "mongodb://localhost:27017", "DB_URL": "mongodb://localhost:27017",
"MODEL_JSON": "${workspaceRoot}/models/all/bundle/model.json", "MODEL_JSON": "${workspaceRoot}/models/all/bundle/model.json",
"SECRET": "secret", "SECRET": "secret",
"REGION": "pg", "REGION": "cockroach",
"BUCKET_NAME":"backups", "BUCKET_NAME":"backups",
"INTERVAL":"30" "INTERVAL":"30"
}, },

View File

@ -51,6 +51,11 @@
<span class="fs-title overflow-label" class:content-color={contentColor}> <span class="fs-title overflow-label" class:content-color={contentColor}>
{#if label}<Label {label} />{/if}<slot name="title" /> {#if label}<Label {label} />{/if}<slot name="title" />
</span> </span>
{#if $$slots['title-tools']}
<div class="buttons-group small-gap">
<slot name="title-tools" />
</div>
{/if}
</div> </div>
{#if $$slots.tools} {#if $$slots.tools}
<div class="buttons-group small-gap"> <div class="buttons-group small-gap">

View File

@ -1,5 +1,14 @@
<script lang="ts"> <script lang="ts">
import { groupByArray, isActiveMode, type BaseWorkspaceInfo } from '@hcengineering/core' import {
groupByArray,
isActiveMode,
isArchivingMode,
isDeletingMode,
isMigrationMode,
isRestoringMode,
reduceCalls,
type BaseWorkspaceInfo
} from '@hcengineering/core'
import { getEmbeddedLabel } from '@hcengineering/platform' import { getEmbeddedLabel } from '@hcengineering/platform'
import { isAdminUser } from '@hcengineering/presentation' import { isAdminUser } from '@hcengineering/presentation'
import { import {
@ -14,7 +23,8 @@
Popup, Popup,
Scroller, Scroller,
SearchEdit, SearchEdit,
ticker ticker,
CheckBox
} from '@hcengineering/ui' } from '@hcengineering/ui'
import { workbenchId } from '@hcengineering/workbench' import { workbenchId } from '@hcengineering/workbench'
import { getAllWorkspaces, getRegionInfo, performWorkspaceOperation, type RegionInfo } from '../utils' import { getAllWorkspaces, getRegionInfo, performWorkspaceOperation, type RegionInfo } from '../utils'
@ -32,13 +42,14 @@
let workspaces: WorkspaceInfo[] = [] let workspaces: WorkspaceInfo[] = []
$: if ($ticker > 0) { const updateWorkspaces = reduceCalls(async (_: number) => {
void getAllWorkspaces().then((res) => { const res = await getAllWorkspaces()
workspaces = res.sort((a, b) => workspaces = res.sort((a, b) =>
(b.workspaceUrl ?? b.workspace).localeCompare(a.workspaceUrl ?? a.workspace) (b.workspaceUrl ?? b.workspace).localeCompare(a.workspaceUrl ?? a.workspace)
) as WorkspaceInfo[] ) as WorkspaceInfo[]
}) })
}
$: void updateWorkspaces($ticker)
const now = Date.now() const now = Date.now()
@ -55,13 +66,26 @@
FewOrMoreYears: 10000000 FewOrMoreYears: 10000000
} }
let limit = 50
// Individual filters
let showActive: boolean = true
let showArchived: boolean = false
let showDeleted: boolean = true
let showOther: boolean = true
$: groupped = groupByArray( $: groupped = groupByArray(
workspaces.filter( workspaces.filter(
(it) => (it) =>
(it.workspaceName?.includes(search) ?? false) || ((it.workspaceName?.includes(search) ?? false) ||
(it.workspaceUrl?.includes(search) ?? false) || (it.workspaceUrl?.includes(search) ?? false) ||
it.workspace?.includes(search) || it.workspace?.includes(search) ||
it.createdBy?.includes(search) it.createdBy?.includes(search)) &&
((showActive && isActiveMode(it.mode)) ||
(showArchived && isArchivingMode(it.mode)) ||
(showDeleted && isDeletingMode(it.mode)) ||
(showOther && (isMigrationMode(it.mode) || isRestoringMode(it.mode))))
), ),
(it) => { (it) => {
const lastUsageDays = Math.round((now - it.lastVisit) / (1000 * 3600 * 24)) const lastUsageDays = Math.round((now - it.lastVisit) / (1000 * 3600 * 24))
@ -92,6 +116,26 @@
<SearchEdit bind:value={search} width={'100%'} /> <SearchEdit bind:value={search} width={'100%'} />
</div> </div>
<div class="p-3 flex-col">
<span class="fs-title mr-2">Filters: </span>
<div class="flex-row-center">
Show active workspaces:
<CheckBox bind:checked={showActive} />
</div>
<div class="flex-row-center">
<span class="mr-2">Show archived workspaces:</span>
<CheckBox bind:checked={showArchived} />
</div>
<div class="flex-row-center">
<span class="mr-2">Show deleted workspaces:</span>
<CheckBox bind:checked={showDeleted} />
</div>
<div class="flex-row-center">
<span class="mr-2">Show other workspaces:</span>
<CheckBox bind:checked={showOther} />
</div>
</div>
<div class="fs-title p-3 flex-row-center"> <div class="fs-title p-3 flex-row-center">
<span class="mr-2"> Migration region selector: </span> <span class="mr-2"> Migration region selector: </span>
<ButtonMenu <ButtonMenu
@ -109,6 +153,7 @@
<div class="mr-4"> <div class="mr-4">
{#each Object.keys(dayRanges) as k} {#each Object.keys(dayRanges) as k}
{@const v = groupped.get(k) ?? []} {@const v = groupped.get(k) ?? []}
{@const hasMore = (groupped.get(k) ?? []).length > limit}
{@const activeV = v.filter((it) => it.mode === 'active' && (it.region ?? '') !== selectedRegionId)} {@const activeV = v.filter((it) => it.mode === 'active' && (it.region ?? '') !== selectedRegionId)}
{@const archiveV = v.filter((it) => it.mode === 'active')} {@const archiveV = v.filter((it) => it.mode === 'active')}
{@const archivedD = v.filter((it) => it.mode === 'archived')} {@const archivedD = v.filter((it) => it.mode === 'archived')}
@ -116,13 +161,31 @@
{#if v.length > 0} {#if v.length > 0}
<Expandable expandable={true} bordered={true}> <Expandable expandable={true} bordered={true}>
<svelte:fragment slot="title"> <svelte:fragment slot="title">
<span class="fs-title focused-button"> <span class="fs-title focused-button flex-row-center">
{k} - {v.length} {k} -
{#if hasMore}
{limit} of {v.length}
{:else}
{v.length}
{/if}
{#if av > 0} {#if av > 0}
- maitenance: {av} - maitenance: {av}
{/if} {/if}
</span> </span>
</svelte:fragment> </svelte:fragment>
<svelte:fragment slot="title-tools">
{#if hasMore}
<div class="ml-4">
<Button
label={getEmbeddedLabel(`More ${k}`)}
kind={'link'}
on:click={() => {
limit += 50
}}
/>
</div>
{/if}
</svelte:fragment>
<svelte:fragment slot="tools"> <svelte:fragment slot="tools">
{#if archiveV.length > 0} {#if archiveV.length > 0}
<Button <Button
@ -153,7 +216,7 @@
/> />
{/if} {/if}
</svelte:fragment> </svelte:fragment>
{#each v as workspace} {#each v.slice(0, limit) as workspace}
{@const wsName = workspace.workspaceName ?? workspace.workspace} {@const wsName = workspace.workspaceName ?? workspace.workspace}
{@const lastUsageDays = Math.round((Date.now() - workspace.lastVisit) / (1000 * 3600 * 24))} {@const lastUsageDays = Math.round((Date.now() - workspace.lastVisit) / (1000 * 3600 * 24))}
<!-- svelte-ignore a11y-click-events-have-key-events --> <!-- svelte-ignore a11y-click-events-have-key-events -->

View File

@ -63,6 +63,7 @@ import { connect } from '@hcengineering/server-tool'
import { randomBytes } from 'crypto' import { randomBytes } from 'crypto'
import otpGenerator from 'otp-generator' import otpGenerator from 'otp-generator'
import { getWorkspaceDestroyAdapter, sharedPipelineContextVars } from '@hcengineering/server-pipeline'
import { accountPlugin } from './plugin' import { accountPlugin } from './plugin'
import type { import type {
Account, Account,
@ -92,7 +93,6 @@ import {
toAccountInfo, toAccountInfo,
verifyPassword verifyPassword
} from './utils' } from './utils'
import { getWorkspaceDestroyAdapter, sharedPipelineContextVars } from '@hcengineering/server-pipeline'
import MD5 from 'crypto-js/md5' import MD5 from 'crypto-js/md5'
function buildGravatarId (email: string): string { function buildGravatarId (email: string): string {
@ -866,7 +866,8 @@ export async function listWorkspaces (
db: AccountDB, db: AccountDB,
branding: Branding | null, branding: Branding | null,
token: string, token: string,
region?: string | null region?: string | null,
mode?: WorkspaceMode | null
): Promise<WorkspaceInfo[]> { ): Promise<WorkspaceInfo[]> {
decodeToken(ctx, token) // Just verify token is valid decodeToken(ctx, token) // Just verify token is valid
@ -874,9 +875,17 @@ export async function listWorkspaces (
region = null region = null
} }
return (await db.workspace.find(region != null ? { region } : {})) const q: Query<Workspace> = {
.filter((it) => it.disabled !== true) disabled: { $ne: true }
.map(trimWorkspaceInfo) }
if (region != null) {
q.region = region
}
if (mode != null) {
q.mode = mode
}
return (await db.workspace.find(q)).map(trimWorkspaceInfo)
} }
/** /**
@ -1701,7 +1710,8 @@ export async function getAllWorkspaces (
ctx: MeasureContext, ctx: MeasureContext,
db: AccountDB, db: AccountDB,
branding: Branding | null, branding: Branding | null,
token: string token: string,
mode?: WorkspaceMode
): Promise<BaseWorkspaceInfo[]> { ): Promise<BaseWorkspaceInfo[]> {
const { email } = decodeToken(ctx, token) const { email } = decodeToken(ctx, token)
const account = await getAccount(db, email) const account = await getAccount(db, email)

View File

@ -96,16 +96,18 @@ class BackupWorker {
console.log('schedule backup with interval', this.config.Interval, 'seconds') console.log('schedule backup with interval', this.config.Interval, 'seconds')
while (!this.canceled) { while (!this.canceled) {
try { try {
const res = await this.backup(ctx, this.config.CoolDown * 1000) const res = await this.backup(ctx, (this.config.Interval / 4) * 1000)
this.printStats(ctx, res) this.printStats(ctx, res)
if (res.skipped === 0) {
console.log('cool down', this.config.CoolDown, 'seconds')
await new Promise<void>((resolve) => setTimeout(resolve, this.config.CoolDown * 1000))
}
} catch (err: any) { } catch (err: any) {
Analytics.handleError(err) Analytics.handleError(err)
ctx.error('error retry in cool down/5', { cooldown: this.config.CoolDown, error: err }) ctx.error('error retry in cool down/5', { cooldown: this.config.CoolDown, error: err })
await new Promise<void>((resolve) => setTimeout(resolve, (this.config.CoolDown / 5) * 1000)) await new Promise<void>((resolve) => setTimeout(resolve, (this.config.CoolDown / 5) * 1000))
continue continue
} }
console.log('cool down', this.config.CoolDown, 'seconds')
await new Promise<void>((resolve) => setTimeout(resolve, this.config.CoolDown * 1000))
} }
} }
@ -145,15 +147,31 @@ class BackupWorker {
return !workspacesIgnore.has(it.workspace) return !workspacesIgnore.has(it.workspace)
}) })
workspaces.sort((a, b) => { workspaces.sort((a, b) => {
return (b.backupInfo?.backupSize ?? 0) - (a.backupInfo?.backupSize ?? 0) const lastBackupMin = Math.round(((a.backupInfo?.lastBackup ?? 0) - (b.backupInfo?.lastBackup ?? 0)) / 60)
if (lastBackupMin === 0) {
// Same minute, sort by backup size
return (a.backupInfo?.backupSize ?? 0) - (b.backupInfo?.backupSize ?? 0)
}
return lastBackupMin
}) })
ctx.info('Preparing for BACKUP', { ctx.warn('Preparing for BACKUP', {
total: workspaces.length, total: workspaces.length,
skipped, skipped,
workspaces: workspaces.map((it) => it.workspace) workspaces: workspaces.map((it) => it.workspace)
}) })
const part = workspaces.slice(0, 500)
let idx = 0
for (const ws of part) {
ctx.warn('prepare workspace', {
idx: ++idx,
workspace: ws.workspaceUrl ?? ws.workspace,
backupSize: ws.backupInfo?.backupSize ?? 0,
lastBackupSec: (Date.now() - (ws.backupInfo?.lastBackup ?? 0)) / 1000
})
}
return await this.doBackup(ctx, workspaces, recheckTimeout) return await this.doBackup(ctx, workspaces, recheckTimeout)
} }

View File

@ -50,7 +50,7 @@ export async function listAccountWorkspaces (token: string, region: string | nul
}, },
body: JSON.stringify({ body: JSON.stringify({
method: 'listWorkspaces', method: 'listWorkspaces',
params: [token, region] params: [token, region, 'active']
}) })
}) })
).json() ).json()