Merge remote-tracking branch 'origin/develop' into staging

Signed-off-by: Andrey Sobolev <haiodo@gmail.com>

dev/tool/src/index.ts

@@ -40,6 +40,7 @@ import {
   backup,
   backupFind,
   backupList,
+  backupSize,
   compactBackup,
   createFileBackupStorage,
   createStorageBackupStorage,
@@ -59,7 +60,7 @@ import toolPlugin, { FileModelLogger } from '@hcengineering/server-tool'
 import { createWorkspace, upgradeWorkspace } from '@hcengineering/workspace-service'
 import path from 'path'
 
-import { buildStorageFromConfig, storageConfigFromEnv } from '@hcengineering/server-storage'
+import { buildStorageFromConfig, createStorageFromConfig, storageConfigFromEnv } from '@hcengineering/server-storage'
 import { program, type Command } from 'commander'
 import { type Db, type MongoClient } from 'mongodb'
 import { clearTelegramHistory } from './telegram'
@@ -86,6 +87,7 @@ import core, {
 import { consoleModelLogger, type MigrateOperation } from '@hcengineering/model'
 import contact from '@hcengineering/model-contact'
 import { getMongoClient, getWorkspaceDB, shutdown } from '@hcengineering/mongo'
+import { backupDownload } from '@hcengineering/server-backup/src/backup'
 import type { StorageAdapter, StorageAdapterEx } from '@hcengineering/server-core'
 import { deepEqual } from 'fast-equals'
 import { createWriteStream, readFileSync } from 'fs'
@@ -924,56 +926,79 @@ export function devTool (
     .command('backup-compact-s3 <bucketName> <dirName>')
     .description('Compact a given backup to just one snapshot')
     .option('-f, --force', 'Force compact.', false)
-    .action(async (bucketName: string, dirName: string, cmd: { force: boolean }) => {
-      const { mongodbUri } = prepareTools()
-      await withStorage(mongodbUri, async (adapter) => {
-        const storage = await createStorageBackupStorage(toolCtx, adapter, getWorkspaceId(bucketName), dirName)
+    .action(async (bucketName: string, dirName: string, cmd: { force: boolean, print: boolean }) => {
+      const backupStorageConfig = storageConfigFromEnv(process.env.STORAGE)
+      const storageAdapter = createStorageFromConfig(backupStorageConfig.storages[0])
+      try {
+        const storage = await createStorageBackupStorage(toolCtx, storageAdapter, getWorkspaceId(bucketName), dirName)
         await compactBackup(toolCtx, storage, cmd.force)
-      })
+      } catch (err: any) {
+        toolCtx.error('failed to size backup', { err })
+      }
+      await storageAdapter.close()
     })
 
-  program
-    .command('backup-compact-s3-all <bucketName>')
-    .description('Compact a given backup to just one snapshot')
-    .option('-f, --force', 'Force compact.', false)
-    .action(async (bucketName: string, dirName: string, cmd: { force: boolean }) => {
-      const { mongodbUri } = prepareTools()
-      await withDatabase(mongodbUri, async (db) => {
-        const { mongodbUri } = prepareTools()
-        await withStorage(mongodbUri, async (adapter) => {
-          const storage = await createStorageBackupStorage(toolCtx, adapter, getWorkspaceId(bucketName), dirName)
-          const workspaces = await listWorkspacesPure(db)
-
-          for (const w of workspaces) {
-            console.log(`clearing ${w.workspace} history:`)
-            await compactBackup(toolCtx, storage, cmd.force)
-          }
-        })
-      })
-    })
   program
     .command('backup-s3-restore <bucketName> <dirName> <workspace> [date]')
     .description('dump workspace transactions and minio resources')
     .action(async (bucketName: string, dirName: string, workspace: string, date, cmd) => {
-      const { mongodbUri } = prepareTools()
-      await withStorage(mongodbUri, async (adapter) => {
-        const storage = await createStorageBackupStorage(toolCtx, adapter, getWorkspaceId(bucketName), dirName)
+      const backupStorageConfig = storageConfigFromEnv(process.env.STORAGE)
+      const storageAdapter = createStorageFromConfig(backupStorageConfig.storages[0])
+      try {
+        const storage = await createStorageBackupStorage(toolCtx, storageAdapter, getWorkspaceId(bucketName), dirName)
         const wsid = getWorkspaceId(workspace)
         const endpoint = await getTransactorEndpoint(generateToken(systemAccountEmail, wsid), 'external')
         await restore(toolCtx, endpoint, wsid, storage, {
           date: parseInt(date ?? '-1')
         })
-      })
+      } catch (err: any) {
+        toolCtx.error('failed to size backup', { err })
+      }
+      await storageAdapter.close()
     })
   program
     .command('backup-s3-list <bucketName> <dirName>')
     .description('list snaphost ids for backup')
     .action(async (bucketName: string, dirName: string, cmd) => {
-      const { mongodbUri } = prepareTools()
-      await withStorage(mongodbUri, async (adapter) => {
-        const storage = await createStorageBackupStorage(toolCtx, adapter, getWorkspaceId(bucketName), dirName)
+      const backupStorageConfig = storageConfigFromEnv(process.env.STORAGE)
+      const storageAdapter = createStorageFromConfig(backupStorageConfig.storages[0])
+      try {
+        const storage = await createStorageBackupStorage(toolCtx, storageAdapter, getWorkspaceId(bucketName), dirName)
         await backupList(storage)
-      })
+      } catch (err: any) {
+        toolCtx.error('failed to size backup', { err })
+      }
+      await storageAdapter.close()
+    })
+
+  program
+    .command('backup-s3-size <bucketName> <dirName>')
+    .description('list snaphost ids for backup')
+    .action(async (bucketName: string, dirName: string, cmd) => {
+      const backupStorageConfig = storageConfigFromEnv(process.env.STORAGE)
+      const storageAdapter = createStorageFromConfig(backupStorageConfig.storages[0])
+      try {
+        const storage = await createStorageBackupStorage(toolCtx, storageAdapter, getWorkspaceId(bucketName), dirName)
+        await backupSize(storage)
+      } catch (err: any) {
+        toolCtx.error('failed to size backup', { err })
+      }
+      await storageAdapter.close()
+    })
+
+  program
+    .command('backup-s3-download <bucketName> <dirName> <storeIn>')
+    .description('Download a full backup from s3 to local dir')
+    .action(async (bucketName: string, dirName: string, storeIn: string, cmd) => {
+      const backupStorageConfig = storageConfigFromEnv(process.env.STORAGE)
+      const storageAdapter = createStorageFromConfig(backupStorageConfig.storages[0])
+      try {
+        const storage = await createStorageBackupStorage(toolCtx, storageAdapter, getWorkspaceId(bucketName), dirName)
+        await backupDownload(storage, storeIn)
+      } catch (err: any) {
+        toolCtx.error('failed to size backup', { err })
+      }
+      await storageAdapter.close()
     })
 
   program

packages/text/src/nodes/todo.ts

@@ -8,7 +8,7 @@ export const TodoItemNode = TaskItem.extend({
 
   addOptions () {
     return {
-      nested: false,
+      nested: true,
       HTMLAttributes: {},
       taskListTypeName: 'todoList'
     }

plugins/contact-resources/src/components/AvatarInstance.svelte

@@ -42,18 +42,25 @@
   }
 
   let fontSize: number = 16
+  let imgError = false
+
+  function handleImgError (): void {
+    imgError = true
+  }
+
+  $: hasImg = url != null && !imgError
 </script>
 
 {#if size === 'full' && !url && displayName && displayName !== ''}
   <div
     bind:this={element}
     class="hulyAvatar-container hulyAvatarSize-{size} {variant}"
-    class:no-img={!url && color}
-    class:bordered={!url && color === undefined}
+    class:no-img={!hasImg && color}
+    class:bordered={!hasImg && color === undefined}
     class:border={bColor !== undefined}
     class:withStatus
     style:--border-color={bColor ?? 'var(--primary-button-default)'}
-    style:background-color={color && !url ? color.icon : 'var(--theme-button-default)'}
+    style:background-color={color && !hasImg ? color.icon : 'var(--theme-button-default)'}
     use:resizeObserver={(element) => {
       fontSize = element.clientWidth * 0.6
     }}
@@ -69,15 +76,15 @@
   <div
     bind:this={element}
     class="hulyAvatar-container hulyAvatarSize-{size} stat {variant}"
-    class:no-img={!url && color}
-    class:bordered={!url && color === undefined}
+    class:no-img={!hasImg && color}
+    class:bordered={!hasImg && color === undefined}
     class:border={bColor !== undefined}
     class:withStatus
     style:--border-color={bColor ?? 'var(--primary-button-default)'}
-    style:background-color={color && !url ? color.icon : 'var(--theme-button-default)'}
+    style:background-color={color && !hasImg ? color.icon : 'var(--theme-button-default)'}
   >
-    {#if url}
-      <img class="hulyAvatarSize-{size} ava-image" src={url} {srcset} alt={''} />
+    {#if url && !imgError}
+      <img class="hulyAvatarSize-{size} ava-image" src={url} {srcset} alt={''} on:error={handleImgError} />
     {:else if displayName && displayName !== ''}
       <div
         class="ava-text"

plugins/time-resources/src/components/text-editor/node-view/ToDoItemNodeView.svelte

@@ -7,12 +7,14 @@
   import { NodeViewContent, NodeViewProps, NodeViewWrapper } from '@hcengineering/text-editor-resources'
   import time, { ToDo, ToDoPriority } from '@hcengineering/time'
   import { CheckBox, getEventPositionElement, showPopup } from '@hcengineering/ui'
+  import { onDestroy, onMount } from 'svelte'
 
   import timeRes from '../../../plugin'
 
   export let node: NodeViewProps['node']
   export let editor: NodeViewProps['editor']
   export let updateAttributes: NodeViewProps['updateAttributes']
+  export let getPos: NodeViewProps['getPos']
 
   export let objectId: Ref<Doc> | undefined = undefined
   export let objectClass: Ref<Class<Doc>> | undefined = undefined
@@ -21,6 +23,24 @@
   const client = getClient()
   const query = createQuery()
 
+  let focused = false
+
+  function handleSelectionUpdate (): void {
+    const selection = editor.state.selection
+    const pos = selection.$anchor.pos
+    const start = getPos()
+    const end = node.firstChild != null ? start + node.firstChild.nodeSize + 1 : start + node.nodeSize
+    focused = pos >= start && pos < end
+  }
+
+  onMount(() => {
+    editor.on('selectionUpdate', handleSelectionUpdate)
+  })
+
+  onDestroy(() => {
+    editor.off('selectionUpdate', handleSelectionUpdate)
+  })
+
   $: todoId = node.attrs.todoid as Ref<ToDo>
   $: userId = node.attrs.userid as Ref<Person>
   $: checked = node.attrs.checked ?? false
@@ -186,10 +206,11 @@
 
 <NodeViewWrapper data-drag-handle="" data-type="todoItem">
   <div
-    class="todo-item flex-row-top flex-gap-3"
+    class="todo-item flex-row-top flex-gap-2"
     class:empty={node.textContent.length === 0}
     class:unassigned={userId == null}
     class:hovered
+    class:focused
   >
     <div class="flex-center assignee" contenteditable="false">
       <EmployeePresenter
@@ -222,21 +243,21 @@
     }
 
     &.unassigned {
-      .assignee {
+      & > .assignee {
         opacity: 0;
       }
     }
 
     &.empty {
-      .assignee {
+      & > .assignee {
         visibility: hidden;
       }
     }
 
     &.hovered,
-    &:hover,
-    &:focus-within {
-      .assignee {
+    &.focused,
+    &:hover {
+      & > .assignee {
         opacity: 1;
       }
     }

plugins/time-resources/src/text-editor-extensions.ts

@@ -8,7 +8,7 @@ export const TodoItemExtension = TaskItem.extend({
 
   addOptions () {
     return {
-      nested: false,
+      nested: true,
       HTMLAttributes: {},
       taskListTypeName: 'todoList'
     }

server/backup/src/backup.ts

@@ -34,6 +34,7 @@ import core, {
   systemAccountEmail,
   TxCollectionCUD,
   WorkspaceId,
+  type BackupStatus,
   type Blob,
   type DocIndexState,
   type Tx
@@ -42,6 +43,8 @@ import { BlobClient, createClient } from '@hcengineering/server-client'
 import { fullTextPushStagePrefix, type StorageAdapter } from '@hcengineering/server-core'
 import { generateToken } from '@hcengineering/server-token'
 import { connect } from '@hcengineering/server-tool'
+import { createWriteStream, existsSync, mkdirSync, statSync } from 'node:fs'
+import { dirname } from 'node:path'
 import { PassThrough } from 'node:stream'
 import { createGzip } from 'node:zlib'
 import { join } from 'path'
@@ -49,7 +52,6 @@ import { Writable } from 'stream'
 import { extract, Pack, pack } from 'tar-stream'
 import { createGunzip, gunzipSync, gzipSync } from 'zlib'
 import { BackupStorage } from './storage'
-import type { BackupStatus } from '@hcengineering/core/src/classes'
 export * from './storage'
 
 const dataBlobSize = 50 * 1024 * 1024
@@ -1113,6 +1115,100 @@ export async function backupList (storage: BackupStorage): Promise<void> {
   }
 }
 
+/**
+ * @public
+ */
+export async function backupSize (storage: BackupStorage): Promise<void> {
+  const infoFile = 'backup.json.gz'
+
+  if (!(await storage.exists(infoFile))) {
+    throw new Error(`${infoFile} should present to restore`)
+  }
+  let size = 0
+
+  const backupInfo: BackupInfo = JSON.parse(gunzipSync(await storage.loadFile(infoFile)).toString())
+  console.log('workspace:', backupInfo.workspace ?? '', backupInfo.version)
+  const addFileSize = async (file: string | undefined | null): Promise<void> => {
+    if (file != null && (await storage.exists(file))) {
+      const fileSize = await storage.stat(file)
+      console.log(file, fileSize)
+      size += fileSize
+    }
+  }
+
+  // Let's calculate data size for backup
+  for (const sn of backupInfo.snapshots) {
+    for (const [, d] of Object.entries(sn.domains)) {
+      await addFileSize(d.snapshot)
+      for (const snp of d.snapshots ?? []) {
+        await addFileSize(snp)
+      }
+      for (const snp of d.storage ?? []) {
+        await addFileSize(snp)
+      }
+    }
+  }
+  await addFileSize(infoFile)
+
+  console.log('Backup size', size / (1024 * 1024), 'Mb')
+}
+
+/**
+ * @public
+ */
+export async function backupDownload (storage: BackupStorage, storeIn: string): Promise<void> {
+  const infoFile = 'backup.json.gz'
+
+  if (!(await storage.exists(infoFile))) {
+    throw new Error(`${infoFile} should present to restore`)
+  }
+  let size = 0
+
+  const backupInfo: BackupInfo = JSON.parse(gunzipSync(await storage.loadFile(infoFile)).toString())
+  console.log('workspace:', backupInfo.workspace ?? '', backupInfo.version)
+  const addFileSize = async (file: string | undefined | null): Promise<void> => {
+    if (file != null && (await storage.exists(file))) {
+      const fileSize = await storage.stat(file)
+      const target = join(storeIn, file)
+      const dir = dirname(target)
+      if (!existsSync(dir)) {
+        mkdirSync(dir, { recursive: true })
+      }
+      if (!existsSync(target) || fileSize !== statSync(target).size) {
+        console.log('downloading', file, fileSize)
+        const readStream = await storage.load(file)
+        const outp = createWriteStream(target)
+
+        readStream.pipe(outp)
+        await new Promise<void>((resolve) => {
+          readStream.on('end', () => {
+            readStream.destroy()
+            outp.close()
+            resolve()
+          })
+        })
+      }
+      size += fileSize
+    }
+  }
+
+  // Let's calculate data size for backup
+  for (const sn of backupInfo.snapshots) {
+    for (const [, d] of Object.entries(sn.domains)) {
+      await addFileSize(d.snapshot)
+      for (const snp of d.snapshots ?? []) {
+        await addFileSize(snp)
+      }
+      for (const snp of d.storage ?? []) {
+        await addFileSize(snp)
+      }
+    }
+  }
+  await addFileSize(infoFile)
+
+  console.log('Backup size', size / (1024 * 1024), 'Mb')
+}
+
 /**
  * @public
  */

server/core/src/server/aggregator.ts

@@ -124,21 +124,23 @@ export class AggregatorStorageAdapter implements StorageAdapter, StorageAdapterE
     let iterator: BlobStorageIterator | undefined
     return {
       next: async () => {
-        if (iterator === undefined && adapters.length > 0) {
-          iterator = await (adapters.shift() as StorageAdapter).listStream(ctx, workspaceId)
-        }
-        if (iterator === undefined) {
-          return []
-        }
-        const docInfos = await iterator.next()
-        if (docInfos.length > 0) {
-          // We need to check if our stored version is fine
-          return docInfos
-        } else {
-          // We need to take next adapter
-          await iterator.close()
-          iterator = undefined
-          return []
+        while (true) {
+          if (iterator === undefined && adapters.length > 0) {
+            iterator = await (adapters.shift() as StorageAdapter).listStream(ctx, workspaceId)
+          }
+          if (iterator === undefined) {
+            return []
+          }
+          const docInfos = await iterator.next()
+          if (docInfos.length > 0) {
+            // We need to check if our stored version is fine
+            return docInfos
+          } else {
+            // We need to take next adapter
+            await iterator.close()
+            iterator = undefined
+            continue
+          }
         }
       },
       close: async () => {