UBERF-9299: Fix backup service backup order (#7826)

.vscode/launch.json

@@ -394,7 +394,7 @@
         "DB_URL": "mongodb://localhost:27017",
         "MODEL_JSON": "${workspaceRoot}/models/all/bundle/model.json",
         "SECRET": "secret",
-        "REGION": "pg",
+        "REGION": "cockroach",
         "BUCKET_NAME":"backups",
         "INTERVAL":"30"
       },

packages/ui/src/components/Expandable.svelte

@@ -51,6 +51,11 @@
       <span class="fs-title overflow-label" class:content-color={contentColor}>
         {#if label}<Label {label} />{/if}<slot name="title" />
       </span>
+      {#if $$slots['title-tools']}
+        <div class="buttons-group small-gap">
+          <slot name="title-tools" />
+        </div>
+      {/if}
     </div>
     {#if $$slots.tools}
       <div class="buttons-group small-gap">

plugins/login-resources/src/components/AdminWorkspaces.svelte

@@ -1,5 +1,14 @@
 <script lang="ts">
-  import { groupByArray, isActiveMode, type BaseWorkspaceInfo } from '@hcengineering/core'
+  import {
+    groupByArray,
+    isActiveMode,
+    isArchivingMode,
+    isDeletingMode,
+    isMigrationMode,
+    isRestoringMode,
+    reduceCalls,
+    type BaseWorkspaceInfo
+  } from '@hcengineering/core'
   import { getEmbeddedLabel } from '@hcengineering/platform'
   import { isAdminUser } from '@hcengineering/presentation'
   import {
@@ -14,7 +23,8 @@
     Popup,
     Scroller,
     SearchEdit,
-    ticker
+    ticker,
+    CheckBox
   } from '@hcengineering/ui'
   import { workbenchId } from '@hcengineering/workbench'
   import { getAllWorkspaces, getRegionInfo, performWorkspaceOperation, type RegionInfo } from '../utils'
@@ -32,13 +42,14 @@
 
   let workspaces: WorkspaceInfo[] = []
 
-  $: if ($ticker > 0) {
-    void getAllWorkspaces().then((res) => {
-      workspaces = res.sort((a, b) =>
-        (b.workspaceUrl ?? b.workspace).localeCompare(a.workspaceUrl ?? a.workspace)
-      ) as WorkspaceInfo[]
-    })
-  }
+  const updateWorkspaces = reduceCalls(async (_: number) => {
+    const res = await getAllWorkspaces()
+    workspaces = res.sort((a, b) =>
+      (b.workspaceUrl ?? b.workspace).localeCompare(a.workspaceUrl ?? a.workspace)
+    ) as WorkspaceInfo[]
+  })
+
+  $: void updateWorkspaces($ticker)
 
   const now = Date.now()
 
@@ -55,13 +66,26 @@
     FewOrMoreYears: 10000000
   }
 
+  let limit = 50
+
+  // Individual filters
+
+  let showActive: boolean = true
+  let showArchived: boolean = false
+  let showDeleted: boolean = true
+  let showOther: boolean = true
+
   $: groupped = groupByArray(
     workspaces.filter(
       (it) =>
-        (it.workspaceName?.includes(search) ?? false) ||
-        (it.workspaceUrl?.includes(search) ?? false) ||
-        it.workspace?.includes(search) ||
-        it.createdBy?.includes(search)
+        ((it.workspaceName?.includes(search) ?? false) ||
+          (it.workspaceUrl?.includes(search) ?? false) ||
+          it.workspace?.includes(search) ||
+          it.createdBy?.includes(search)) &&
+        ((showActive && isActiveMode(it.mode)) ||
+          (showArchived && isArchivingMode(it.mode)) ||
+          (showDeleted && isDeletingMode(it.mode)) ||
+          (showOther && (isMigrationMode(it.mode) || isRestoringMode(it.mode))))
     ),
     (it) => {
       const lastUsageDays = Math.round((now - it.lastVisit) / (1000 * 3600 * 24))
@@ -92,6 +116,26 @@
       <SearchEdit bind:value={search} width={'100%'} />
     </div>
 
+    <div class="p-3 flex-col">
+      <span class="fs-title mr-2">Filters: </span>
+      <div class="flex-row-center">
+        Show active workspaces:
+        <CheckBox bind:checked={showActive} />
+      </div>
+      <div class="flex-row-center">
+        <span class="mr-2">Show archived workspaces:</span>
+        <CheckBox bind:checked={showArchived} />
+      </div>
+      <div class="flex-row-center">
+        <span class="mr-2">Show deleted workspaces:</span>
+        <CheckBox bind:checked={showDeleted} />
+      </div>
+      <div class="flex-row-center">
+        <span class="mr-2">Show other workspaces:</span>
+        <CheckBox bind:checked={showOther} />
+      </div>
+    </div>
+
     <div class="fs-title p-3 flex-row-center">
       <span class="mr-2"> Migration region selector: </span>
       <ButtonMenu
@@ -109,6 +153,7 @@
         <div class="mr-4">
           {#each Object.keys(dayRanges) as k}
             {@const v = groupped.get(k) ?? []}
+            {@const hasMore = (groupped.get(k) ?? []).length > limit}
             {@const activeV = v.filter((it) => it.mode === 'active' && (it.region ?? '') !== selectedRegionId)}
             {@const archiveV = v.filter((it) => it.mode === 'active')}
             {@const archivedD = v.filter((it) => it.mode === 'archived')}
@@ -116,13 +161,31 @@
             {#if v.length > 0}
               <Expandable expandable={true} bordered={true}>
                 <svelte:fragment slot="title">
-                  <span class="fs-title focused-button">
-                    {k} - {v.length}
+                  <span class="fs-title focused-button flex-row-center">
+                    {k} -
+                    {#if hasMore}
+                      {limit} of {v.length}
+                    {:else}
+                      {v.length}
+                    {/if}
                     {#if av > 0}
                       - maitenance: {av}
                     {/if}
                   </span>
                 </svelte:fragment>
+                <svelte:fragment slot="title-tools">
+                  {#if hasMore}
+                    <div class="ml-4">
+                      <Button
+                        label={getEmbeddedLabel(`More ${k}`)}
+                        kind={'link'}
+                        on:click={() => {
+                          limit += 50
+                        }}
+                      />
+                    </div>
+                  {/if}
+                </svelte:fragment>
                 <svelte:fragment slot="tools">
                   {#if archiveV.length > 0}
                     <Button
@@ -153,7 +216,7 @@
                     />
                   {/if}
                 </svelte:fragment>
-                {#each v as workspace}
+                {#each v.slice(0, limit) as workspace}
                   {@const wsName = workspace.workspaceName ?? workspace.workspace}
                   {@const lastUsageDays = Math.round((Date.now() - workspace.lastVisit) / (1000 * 3600 * 24))}
                   <!-- svelte-ignore a11y-click-events-have-key-events -->

server/account/src/operations.ts

@@ -63,6 +63,7 @@ import { connect } from '@hcengineering/server-tool'
 import { randomBytes } from 'crypto'
 import otpGenerator from 'otp-generator'
 
+import { getWorkspaceDestroyAdapter, sharedPipelineContextVars } from '@hcengineering/server-pipeline'
 import { accountPlugin } from './plugin'
 import type {
   Account,
@@ -92,7 +93,6 @@ import {
   toAccountInfo,
   verifyPassword
 } from './utils'
-import { getWorkspaceDestroyAdapter, sharedPipelineContextVars } from '@hcengineering/server-pipeline'
 
 import MD5 from 'crypto-js/md5'
 function buildGravatarId (email: string): string {
@@ -866,7 +866,8 @@ export async function listWorkspaces (
   db: AccountDB,
   branding: Branding | null,
   token: string,
-  region?: string | null
+  region?: string | null,
+  mode?: WorkspaceMode | null
 ): Promise<WorkspaceInfo[]> {
   decodeToken(ctx, token) // Just verify token is valid
 
@@ -874,9 +875,17 @@ export async function listWorkspaces (
     region = null
   }
 
-  return (await db.workspace.find(region != null ? { region } : {}))
-    .filter((it) => it.disabled !== true)
-    .map(trimWorkspaceInfo)
+  const q: Query<Workspace> = {
+    disabled: { $ne: true }
+  }
+  if (region != null) {
+    q.region = region
+  }
+  if (mode != null) {
+    q.mode = mode
+  }
+
+  return (await db.workspace.find(q)).map(trimWorkspaceInfo)
 }
 
 /**
@@ -1701,7 +1710,8 @@ export async function getAllWorkspaces (
   ctx: MeasureContext,
   db: AccountDB,
   branding: Branding | null,
-  token: string
+  token: string,
+  mode?: WorkspaceMode
 ): Promise<BaseWorkspaceInfo[]> {
   const { email } = decodeToken(ctx, token)
   const account = await getAccount(db, email)

server/backup/src/service.ts

@@ -96,16 +96,18 @@ class BackupWorker {
     console.log('schedule backup with interval', this.config.Interval, 'seconds')
     while (!this.canceled) {
       try {
-        const res = await this.backup(ctx, this.config.CoolDown * 1000)
+        const res = await this.backup(ctx, (this.config.Interval / 4) * 1000)
         this.printStats(ctx, res)
+        if (res.skipped === 0) {
+          console.log('cool down', this.config.CoolDown, 'seconds')
+          await new Promise<void>((resolve) => setTimeout(resolve, this.config.CoolDown * 1000))
+        }
       } catch (err: any) {
         Analytics.handleError(err)
         ctx.error('error retry in cool down/5', { cooldown: this.config.CoolDown, error: err })
         await new Promise<void>((resolve) => setTimeout(resolve, (this.config.CoolDown / 5) * 1000))
         continue
       }
-      console.log('cool down', this.config.CoolDown, 'seconds')
-      await new Promise<void>((resolve) => setTimeout(resolve, this.config.CoolDown * 1000))
     }
   }
 
@@ -145,15 +147,31 @@ class BackupWorker {
       return !workspacesIgnore.has(it.workspace)
     })
     workspaces.sort((a, b) => {
-      return (b.backupInfo?.backupSize ?? 0) - (a.backupInfo?.backupSize ?? 0)
+      const lastBackupMin = Math.round(((a.backupInfo?.lastBackup ?? 0) - (b.backupInfo?.lastBackup ?? 0)) / 60)
+      if (lastBackupMin === 0) {
+        // Same minute, sort by backup size
+        return (a.backupInfo?.backupSize ?? 0) - (b.backupInfo?.backupSize ?? 0)
+      }
+      return lastBackupMin
     })
 
-    ctx.info('Preparing for BACKUP', {
+    ctx.warn('Preparing for BACKUP', {
       total: workspaces.length,
       skipped,
       workspaces: workspaces.map((it) => it.workspace)
     })
 
+    const part = workspaces.slice(0, 500)
+    let idx = 0
+    for (const ws of part) {
+      ctx.warn('prepare workspace', {
+        idx: ++idx,
+        workspace: ws.workspaceUrl ?? ws.workspace,
+        backupSize: ws.backupInfo?.backupSize ?? 0,
+        lastBackupSec: (Date.now() - (ws.backupInfo?.lastBackup ?? 0)) / 1000
+      })
+    }
+
     return await this.doBackup(ctx, workspaces, recheckTimeout)
   }
 

server/client/src/account.ts

@@ -50,7 +50,7 @@ export async function listAccountWorkspaces (token: string, region: string | nul
       },
       body: JSON.stringify({
         method: 'listWorkspaces',
-        params: [token, region]
+        params: [token, region, 'active']
       })
     })
   ).json()