mirror of
https://github.com/hcengineering/platform.git
synced 2025-03-15 10:37:52 +00:00
UBERF-9299: Fix backup service backup order (#7826)
Some checks are pending
CI / build (push) Waiting to run
CI / svelte-check (push) Blocked by required conditions
CI / formatting (push) Blocked by required conditions
CI / test (push) Blocked by required conditions
CI / uitest (push) Waiting to run
CI / uitest-pg (push) Waiting to run
CI / uitest-qms (push) Waiting to run
CI / docker-build (push) Blocked by required conditions
CI / dist-build (push) Blocked by required conditions
Some checks are pending
CI / build (push) Waiting to run
CI / svelte-check (push) Blocked by required conditions
CI / formatting (push) Blocked by required conditions
CI / test (push) Blocked by required conditions
CI / uitest (push) Waiting to run
CI / uitest-pg (push) Waiting to run
CI / uitest-qms (push) Waiting to run
CI / docker-build (push) Blocked by required conditions
CI / dist-build (push) Blocked by required conditions
This commit is contained in:
parent
5c9e71be09
commit
9c3a1cc641
2
.vscode/launch.json
vendored
2
.vscode/launch.json
vendored
@ -394,7 +394,7 @@
|
|||||||
"DB_URL": "mongodb://localhost:27017",
|
"DB_URL": "mongodb://localhost:27017",
|
||||||
"MODEL_JSON": "${workspaceRoot}/models/all/bundle/model.json",
|
"MODEL_JSON": "${workspaceRoot}/models/all/bundle/model.json",
|
||||||
"SECRET": "secret",
|
"SECRET": "secret",
|
||||||
"REGION": "pg",
|
"REGION": "cockroach",
|
||||||
"BUCKET_NAME":"backups",
|
"BUCKET_NAME":"backups",
|
||||||
"INTERVAL":"30"
|
"INTERVAL":"30"
|
||||||
},
|
},
|
||||||
|
@ -51,6 +51,11 @@
|
|||||||
<span class="fs-title overflow-label" class:content-color={contentColor}>
|
<span class="fs-title overflow-label" class:content-color={contentColor}>
|
||||||
{#if label}<Label {label} />{/if}<slot name="title" />
|
{#if label}<Label {label} />{/if}<slot name="title" />
|
||||||
</span>
|
</span>
|
||||||
|
{#if $$slots['title-tools']}
|
||||||
|
<div class="buttons-group small-gap">
|
||||||
|
<slot name="title-tools" />
|
||||||
|
</div>
|
||||||
|
{/if}
|
||||||
</div>
|
</div>
|
||||||
{#if $$slots.tools}
|
{#if $$slots.tools}
|
||||||
<div class="buttons-group small-gap">
|
<div class="buttons-group small-gap">
|
||||||
|
@ -1,5 +1,14 @@
|
|||||||
<script lang="ts">
|
<script lang="ts">
|
||||||
import { groupByArray, isActiveMode, type BaseWorkspaceInfo } from '@hcengineering/core'
|
import {
|
||||||
|
groupByArray,
|
||||||
|
isActiveMode,
|
||||||
|
isArchivingMode,
|
||||||
|
isDeletingMode,
|
||||||
|
isMigrationMode,
|
||||||
|
isRestoringMode,
|
||||||
|
reduceCalls,
|
||||||
|
type BaseWorkspaceInfo
|
||||||
|
} from '@hcengineering/core'
|
||||||
import { getEmbeddedLabel } from '@hcengineering/platform'
|
import { getEmbeddedLabel } from '@hcengineering/platform'
|
||||||
import { isAdminUser } from '@hcengineering/presentation'
|
import { isAdminUser } from '@hcengineering/presentation'
|
||||||
import {
|
import {
|
||||||
@ -14,7 +23,8 @@
|
|||||||
Popup,
|
Popup,
|
||||||
Scroller,
|
Scroller,
|
||||||
SearchEdit,
|
SearchEdit,
|
||||||
ticker
|
ticker,
|
||||||
|
CheckBox
|
||||||
} from '@hcengineering/ui'
|
} from '@hcengineering/ui'
|
||||||
import { workbenchId } from '@hcengineering/workbench'
|
import { workbenchId } from '@hcengineering/workbench'
|
||||||
import { getAllWorkspaces, getRegionInfo, performWorkspaceOperation, type RegionInfo } from '../utils'
|
import { getAllWorkspaces, getRegionInfo, performWorkspaceOperation, type RegionInfo } from '../utils'
|
||||||
@ -32,13 +42,14 @@
|
|||||||
|
|
||||||
let workspaces: WorkspaceInfo[] = []
|
let workspaces: WorkspaceInfo[] = []
|
||||||
|
|
||||||
$: if ($ticker > 0) {
|
const updateWorkspaces = reduceCalls(async (_: number) => {
|
||||||
void getAllWorkspaces().then((res) => {
|
const res = await getAllWorkspaces()
|
||||||
workspaces = res.sort((a, b) =>
|
workspaces = res.sort((a, b) =>
|
||||||
(b.workspaceUrl ?? b.workspace).localeCompare(a.workspaceUrl ?? a.workspace)
|
(b.workspaceUrl ?? b.workspace).localeCompare(a.workspaceUrl ?? a.workspace)
|
||||||
) as WorkspaceInfo[]
|
) as WorkspaceInfo[]
|
||||||
})
|
})
|
||||||
}
|
|
||||||
|
$: void updateWorkspaces($ticker)
|
||||||
|
|
||||||
const now = Date.now()
|
const now = Date.now()
|
||||||
|
|
||||||
@ -55,13 +66,26 @@
|
|||||||
FewOrMoreYears: 10000000
|
FewOrMoreYears: 10000000
|
||||||
}
|
}
|
||||||
|
|
||||||
|
let limit = 50
|
||||||
|
|
||||||
|
// Individual filters
|
||||||
|
|
||||||
|
let showActive: boolean = true
|
||||||
|
let showArchived: boolean = false
|
||||||
|
let showDeleted: boolean = true
|
||||||
|
let showOther: boolean = true
|
||||||
|
|
||||||
$: groupped = groupByArray(
|
$: groupped = groupByArray(
|
||||||
workspaces.filter(
|
workspaces.filter(
|
||||||
(it) =>
|
(it) =>
|
||||||
(it.workspaceName?.includes(search) ?? false) ||
|
((it.workspaceName?.includes(search) ?? false) ||
|
||||||
(it.workspaceUrl?.includes(search) ?? false) ||
|
(it.workspaceUrl?.includes(search) ?? false) ||
|
||||||
it.workspace?.includes(search) ||
|
it.workspace?.includes(search) ||
|
||||||
it.createdBy?.includes(search)
|
it.createdBy?.includes(search)) &&
|
||||||
|
((showActive && isActiveMode(it.mode)) ||
|
||||||
|
(showArchived && isArchivingMode(it.mode)) ||
|
||||||
|
(showDeleted && isDeletingMode(it.mode)) ||
|
||||||
|
(showOther && (isMigrationMode(it.mode) || isRestoringMode(it.mode))))
|
||||||
),
|
),
|
||||||
(it) => {
|
(it) => {
|
||||||
const lastUsageDays = Math.round((now - it.lastVisit) / (1000 * 3600 * 24))
|
const lastUsageDays = Math.round((now - it.lastVisit) / (1000 * 3600 * 24))
|
||||||
@ -92,6 +116,26 @@
|
|||||||
<SearchEdit bind:value={search} width={'100%'} />
|
<SearchEdit bind:value={search} width={'100%'} />
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<div class="p-3 flex-col">
|
||||||
|
<span class="fs-title mr-2">Filters: </span>
|
||||||
|
<div class="flex-row-center">
|
||||||
|
Show active workspaces:
|
||||||
|
<CheckBox bind:checked={showActive} />
|
||||||
|
</div>
|
||||||
|
<div class="flex-row-center">
|
||||||
|
<span class="mr-2">Show archived workspaces:</span>
|
||||||
|
<CheckBox bind:checked={showArchived} />
|
||||||
|
</div>
|
||||||
|
<div class="flex-row-center">
|
||||||
|
<span class="mr-2">Show deleted workspaces:</span>
|
||||||
|
<CheckBox bind:checked={showDeleted} />
|
||||||
|
</div>
|
||||||
|
<div class="flex-row-center">
|
||||||
|
<span class="mr-2">Show other workspaces:</span>
|
||||||
|
<CheckBox bind:checked={showOther} />
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
<div class="fs-title p-3 flex-row-center">
|
<div class="fs-title p-3 flex-row-center">
|
||||||
<span class="mr-2"> Migration region selector: </span>
|
<span class="mr-2"> Migration region selector: </span>
|
||||||
<ButtonMenu
|
<ButtonMenu
|
||||||
@ -109,6 +153,7 @@
|
|||||||
<div class="mr-4">
|
<div class="mr-4">
|
||||||
{#each Object.keys(dayRanges) as k}
|
{#each Object.keys(dayRanges) as k}
|
||||||
{@const v = groupped.get(k) ?? []}
|
{@const v = groupped.get(k) ?? []}
|
||||||
|
{@const hasMore = (groupped.get(k) ?? []).length > limit}
|
||||||
{@const activeV = v.filter((it) => it.mode === 'active' && (it.region ?? '') !== selectedRegionId)}
|
{@const activeV = v.filter((it) => it.mode === 'active' && (it.region ?? '') !== selectedRegionId)}
|
||||||
{@const archiveV = v.filter((it) => it.mode === 'active')}
|
{@const archiveV = v.filter((it) => it.mode === 'active')}
|
||||||
{@const archivedD = v.filter((it) => it.mode === 'archived')}
|
{@const archivedD = v.filter((it) => it.mode === 'archived')}
|
||||||
@ -116,13 +161,31 @@
|
|||||||
{#if v.length > 0}
|
{#if v.length > 0}
|
||||||
<Expandable expandable={true} bordered={true}>
|
<Expandable expandable={true} bordered={true}>
|
||||||
<svelte:fragment slot="title">
|
<svelte:fragment slot="title">
|
||||||
<span class="fs-title focused-button">
|
<span class="fs-title focused-button flex-row-center">
|
||||||
{k} - {v.length}
|
{k} -
|
||||||
|
{#if hasMore}
|
||||||
|
{limit} of {v.length}
|
||||||
|
{:else}
|
||||||
|
{v.length}
|
||||||
|
{/if}
|
||||||
{#if av > 0}
|
{#if av > 0}
|
||||||
- maitenance: {av}
|
- maitenance: {av}
|
||||||
{/if}
|
{/if}
|
||||||
</span>
|
</span>
|
||||||
</svelte:fragment>
|
</svelte:fragment>
|
||||||
|
<svelte:fragment slot="title-tools">
|
||||||
|
{#if hasMore}
|
||||||
|
<div class="ml-4">
|
||||||
|
<Button
|
||||||
|
label={getEmbeddedLabel(`More ${k}`)}
|
||||||
|
kind={'link'}
|
||||||
|
on:click={() => {
|
||||||
|
limit += 50
|
||||||
|
}}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
{/if}
|
||||||
|
</svelte:fragment>
|
||||||
<svelte:fragment slot="tools">
|
<svelte:fragment slot="tools">
|
||||||
{#if archiveV.length > 0}
|
{#if archiveV.length > 0}
|
||||||
<Button
|
<Button
|
||||||
@ -153,7 +216,7 @@
|
|||||||
/>
|
/>
|
||||||
{/if}
|
{/if}
|
||||||
</svelte:fragment>
|
</svelte:fragment>
|
||||||
{#each v as workspace}
|
{#each v.slice(0, limit) as workspace}
|
||||||
{@const wsName = workspace.workspaceName ?? workspace.workspace}
|
{@const wsName = workspace.workspaceName ?? workspace.workspace}
|
||||||
{@const lastUsageDays = Math.round((Date.now() - workspace.lastVisit) / (1000 * 3600 * 24))}
|
{@const lastUsageDays = Math.round((Date.now() - workspace.lastVisit) / (1000 * 3600 * 24))}
|
||||||
<!-- svelte-ignore a11y-click-events-have-key-events -->
|
<!-- svelte-ignore a11y-click-events-have-key-events -->
|
||||||
|
@ -63,6 +63,7 @@ import { connect } from '@hcengineering/server-tool'
|
|||||||
import { randomBytes } from 'crypto'
|
import { randomBytes } from 'crypto'
|
||||||
import otpGenerator from 'otp-generator'
|
import otpGenerator from 'otp-generator'
|
||||||
|
|
||||||
|
import { getWorkspaceDestroyAdapter, sharedPipelineContextVars } from '@hcengineering/server-pipeline'
|
||||||
import { accountPlugin } from './plugin'
|
import { accountPlugin } from './plugin'
|
||||||
import type {
|
import type {
|
||||||
Account,
|
Account,
|
||||||
@ -92,7 +93,6 @@ import {
|
|||||||
toAccountInfo,
|
toAccountInfo,
|
||||||
verifyPassword
|
verifyPassword
|
||||||
} from './utils'
|
} from './utils'
|
||||||
import { getWorkspaceDestroyAdapter, sharedPipelineContextVars } from '@hcengineering/server-pipeline'
|
|
||||||
|
|
||||||
import MD5 from 'crypto-js/md5'
|
import MD5 from 'crypto-js/md5'
|
||||||
function buildGravatarId (email: string): string {
|
function buildGravatarId (email: string): string {
|
||||||
@ -866,7 +866,8 @@ export async function listWorkspaces (
|
|||||||
db: AccountDB,
|
db: AccountDB,
|
||||||
branding: Branding | null,
|
branding: Branding | null,
|
||||||
token: string,
|
token: string,
|
||||||
region?: string | null
|
region?: string | null,
|
||||||
|
mode?: WorkspaceMode | null
|
||||||
): Promise<WorkspaceInfo[]> {
|
): Promise<WorkspaceInfo[]> {
|
||||||
decodeToken(ctx, token) // Just verify token is valid
|
decodeToken(ctx, token) // Just verify token is valid
|
||||||
|
|
||||||
@ -874,9 +875,17 @@ export async function listWorkspaces (
|
|||||||
region = null
|
region = null
|
||||||
}
|
}
|
||||||
|
|
||||||
return (await db.workspace.find(region != null ? { region } : {}))
|
const q: Query<Workspace> = {
|
||||||
.filter((it) => it.disabled !== true)
|
disabled: { $ne: true }
|
||||||
.map(trimWorkspaceInfo)
|
}
|
||||||
|
if (region != null) {
|
||||||
|
q.region = region
|
||||||
|
}
|
||||||
|
if (mode != null) {
|
||||||
|
q.mode = mode
|
||||||
|
}
|
||||||
|
|
||||||
|
return (await db.workspace.find(q)).map(trimWorkspaceInfo)
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@ -1701,7 +1710,8 @@ export async function getAllWorkspaces (
|
|||||||
ctx: MeasureContext,
|
ctx: MeasureContext,
|
||||||
db: AccountDB,
|
db: AccountDB,
|
||||||
branding: Branding | null,
|
branding: Branding | null,
|
||||||
token: string
|
token: string,
|
||||||
|
mode?: WorkspaceMode
|
||||||
): Promise<BaseWorkspaceInfo[]> {
|
): Promise<BaseWorkspaceInfo[]> {
|
||||||
const { email } = decodeToken(ctx, token)
|
const { email } = decodeToken(ctx, token)
|
||||||
const account = await getAccount(db, email)
|
const account = await getAccount(db, email)
|
||||||
|
@ -96,16 +96,18 @@ class BackupWorker {
|
|||||||
console.log('schedule backup with interval', this.config.Interval, 'seconds')
|
console.log('schedule backup with interval', this.config.Interval, 'seconds')
|
||||||
while (!this.canceled) {
|
while (!this.canceled) {
|
||||||
try {
|
try {
|
||||||
const res = await this.backup(ctx, this.config.CoolDown * 1000)
|
const res = await this.backup(ctx, (this.config.Interval / 4) * 1000)
|
||||||
this.printStats(ctx, res)
|
this.printStats(ctx, res)
|
||||||
|
if (res.skipped === 0) {
|
||||||
|
console.log('cool down', this.config.CoolDown, 'seconds')
|
||||||
|
await new Promise<void>((resolve) => setTimeout(resolve, this.config.CoolDown * 1000))
|
||||||
|
}
|
||||||
} catch (err: any) {
|
} catch (err: any) {
|
||||||
Analytics.handleError(err)
|
Analytics.handleError(err)
|
||||||
ctx.error('error retry in cool down/5', { cooldown: this.config.CoolDown, error: err })
|
ctx.error('error retry in cool down/5', { cooldown: this.config.CoolDown, error: err })
|
||||||
await new Promise<void>((resolve) => setTimeout(resolve, (this.config.CoolDown / 5) * 1000))
|
await new Promise<void>((resolve) => setTimeout(resolve, (this.config.CoolDown / 5) * 1000))
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
console.log('cool down', this.config.CoolDown, 'seconds')
|
|
||||||
await new Promise<void>((resolve) => setTimeout(resolve, this.config.CoolDown * 1000))
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -145,15 +147,31 @@ class BackupWorker {
|
|||||||
return !workspacesIgnore.has(it.workspace)
|
return !workspacesIgnore.has(it.workspace)
|
||||||
})
|
})
|
||||||
workspaces.sort((a, b) => {
|
workspaces.sort((a, b) => {
|
||||||
return (b.backupInfo?.backupSize ?? 0) - (a.backupInfo?.backupSize ?? 0)
|
const lastBackupMin = Math.round(((a.backupInfo?.lastBackup ?? 0) - (b.backupInfo?.lastBackup ?? 0)) / 60)
|
||||||
|
if (lastBackupMin === 0) {
|
||||||
|
// Same minute, sort by backup size
|
||||||
|
return (a.backupInfo?.backupSize ?? 0) - (b.backupInfo?.backupSize ?? 0)
|
||||||
|
}
|
||||||
|
return lastBackupMin
|
||||||
})
|
})
|
||||||
|
|
||||||
ctx.info('Preparing for BACKUP', {
|
ctx.warn('Preparing for BACKUP', {
|
||||||
total: workspaces.length,
|
total: workspaces.length,
|
||||||
skipped,
|
skipped,
|
||||||
workspaces: workspaces.map((it) => it.workspace)
|
workspaces: workspaces.map((it) => it.workspace)
|
||||||
})
|
})
|
||||||
|
|
||||||
|
const part = workspaces.slice(0, 500)
|
||||||
|
let idx = 0
|
||||||
|
for (const ws of part) {
|
||||||
|
ctx.warn('prepare workspace', {
|
||||||
|
idx: ++idx,
|
||||||
|
workspace: ws.workspaceUrl ?? ws.workspace,
|
||||||
|
backupSize: ws.backupInfo?.backupSize ?? 0,
|
||||||
|
lastBackupSec: (Date.now() - (ws.backupInfo?.lastBackup ?? 0)) / 1000
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
return await this.doBackup(ctx, workspaces, recheckTimeout)
|
return await this.doBackup(ctx, workspaces, recheckTimeout)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -50,7 +50,7 @@ export async function listAccountWorkspaces (token: string, region: string | nul
|
|||||||
},
|
},
|
||||||
body: JSON.stringify({
|
body: JSON.stringify({
|
||||||
method: 'listWorkspaces',
|
method: 'listWorkspaces',
|
||||||
params: [token, region]
|
params: [token, region, 'active']
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
).json()
|
).json()
|
||||||
|
Loading…
Reference in New Issue
Block a user