Merge remote-tracking branch 'origin/develop' into staging

Signed-off-by: Andrey Sobolev <haiodo@gmail.com>

common/config/rush/pnpm-lock.yaml

@@ -1223,6 +1223,9 @@ dependencies:
   '@types/domhandler':
     specifier: ^2.4.5
     version: 2.4.5
+  '@types/dompurify':
+    specifier: ^3.0.5
+    version: 3.0.5
   '@types/email-addresses':
     specifier: ^3.0.0
     version: 3.0.0
@@ -1451,6 +1454,9 @@ dependencies:
   domhandler:
     specifier: ^5.0.3
     version: 5.0.3
+  dompurify:
+    specifier: ^3.1.6
+    version: 3.1.6
   domutils:
     specifier: ^3.1.0
     version: 3.1.0
@@ -1586,6 +1592,9 @@ dependencies:
   highlight.js:
     specifier: ~11.8.0
     version: 11.8.0
+  hls.js:
+    specifier: ^1.5.15
+    version: 1.5.15
   html-to-text:
     specifier: ^9.0.3
     version: 9.0.5
@@ -8951,6 +8960,12 @@ packages:
     resolution: {integrity: sha512-lANhC2grmFG1gBac/8sDAKdIXx+TzAdkJIAjEOSMA+qW3297ybACEbacJnG15aNYfrzDO6fdcoouokqAKsy6aQ==}
     dev: false
 
+  /@types/dompurify@3.0.5:
+    resolution: {integrity: sha512-1Wg0g3BtQF7sSb27fJQAKck1HECM6zV1EB66j8JH9i3LCjYabJa0FSdiSgsD5K/RbrsR0SiraKacLB+T8ZVYAg==}
+    dependencies:
+      '@types/trusted-types': 2.0.7
+    dev: false
+
   /@types/domutils@1.7.8:
     resolution: {integrity: sha512-iZGboDV79ibrO3D625p9yD+VgmMDnyJocdIRJvu9Xz66R8SHfOY/XNgdjY5SFoFiLgILceVfSLt7IUhlk1Vhhg==}
     dependencies:
@@ -9531,6 +9546,10 @@ packages:
     resolution: {integrity: sha512-6WaYesThRMCl19iryMYP7/x2OVgCtbIVflDGFpWnb9irXI3UjYE4AzmYuiUKY1AJstGijoY+MgUszMgRxIYTYw==}
     dev: false
 
+  /@types/trusted-types@2.0.7:
+    resolution: {integrity: sha512-ScaPdn1dQczgbl0QFTeTOmVHFULt394XJgOQNoyVhZ6r2vLnMLJfBPd53SB52T/3G36VI1/g2MZaX0cwDuXsfw==}
+    dev: false
+
   /@types/unist@2.0.10:
     resolution: {integrity: sha512-IfYcSBWE3hLpBg8+X2SEa8LVkJdJEkT2Ese2aaLs3ptGdVtABxndrMaxuFlQ1qdFf9Q5rDvDpxI3WwgvKFAsQA==}
     dev: false
@@ -12955,6 +12974,10 @@ packages:
       domelementtype: 2.3.0
     dev: false
 
+  /dompurify@3.1.6:
+    resolution: {integrity: sha512-cTOAhc36AalkjtBpfG6O8JimdTMWNXjiePT2xQH/ppBGi/4uIpmj8eKyIkMJErXWARyINV/sB38yf8JCLF5pbQ==}
+    dev: false
+
   /domutils@1.5.1:
     resolution: {integrity: sha512-gSu5Oi/I+3wDENBsOWBiRK1eoGxcywYSqg3rR960/+EfY0CF4EX1VPkgHOZ3WiS/Jg2DtliF6BhWcHlfpYUcGw==}
     dependencies:
@@ -15598,6 +15621,10 @@ packages:
     requiresBuild: true
     dev: false
 
+  /hls.js@1.5.15:
+    resolution: {integrity: sha512-6cD7xN6bycBHaXz2WyPIaHn/iXFizE5au2yvY5q9aC4wfihxAr16C9fUy4nxh2a3wOw0fEgLRa9dN6wsYjlpNg==}
+    dev: false
+
   /hogan.js@3.0.2:
     resolution: {integrity: sha512-RqGs4wavGYJWE07t35JQccByczmNUXQT0E12ZYV1VKYu5UiAU9lsos/yBAcf840+zrUQQxgVduCR5/B8nNtibg==}
     hasBin: true
@@ -27291,7 +27318,7 @@ packages:
     dev: false
 
   file:projects/lead-resources.tgz(@types/node@20.11.19)(esbuild@0.20.1)(postcss-load-config@4.0.2)(postcss@8.4.35)(ts-node@10.9.2):
-    resolution: {integrity: sha512-TJVh5S1o+GvRWeeNWwXveGOpMtsQTR0n5RMGjK3kXsuAUYCQRPzgMh3noJxe2n4vvijK8IUNjAR9+AjeSPo5kw==, tarball: file:projects/lead-resources.tgz}
+    resolution: {integrity: sha512-xg8Fq55+BYSO+pwIkFTJFDJGPu1CWGB8CiZ64+J2jqzbAHkRaiOCP0u3R4lOw/z6k1tnqhL0m2bvV9pCUCYTHA==, tarball: file:projects/lead-resources.tgz}
     id: file:projects/lead-resources.tgz
     name: '@rush-temp/lead-resources'
     version: 0.0.0
@@ -30465,7 +30492,7 @@ packages:
     dev: false
 
   file:projects/presentation.tgz(@types/node@20.11.19)(esbuild@0.20.1)(postcss-load-config@4.0.2)(postcss@8.4.35)(ts-node@10.9.2):
-    resolution: {integrity: sha512-r+NP0EMgEeKbfaa4v8P1Iho0cfYqe9PhOBfV6SPd/9xnNPt42nK9Gu4r5so1LTolhEUzbFiKh7zSX1ADL5e/3g==, tarball: file:projects/presentation.tgz}
+    resolution: {integrity: sha512-ryBht4b1zE/Ik6KZqDL/joAzt3968bkRbGZOt3x+pE929i7yCtHmlMC7W65Nlr1eglhC2JTSy2NiKTNv9yjcuw==, tarball: file:projects/presentation.tgz}
     id: file:projects/presentation.tgz
     name: '@rush-temp/presentation'
     version: 0.0.0
@@ -30482,6 +30509,7 @@ packages:
       eslint-plugin-promise: 6.1.1(eslint@8.56.0)
       eslint-plugin-svelte: 2.35.1(eslint@8.56.0)(svelte@4.2.12)(ts-node@10.9.2)
       fast-equals: 5.0.1
+      hls.js: 1.5.15
       jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2)
       png-chunks-extract: 1.0.0
       prettier: 3.2.5
@@ -34795,7 +34823,7 @@ packages:
     dev: false
 
   file:projects/tool.tgz(bufferutil@4.0.8)(utf-8-validate@6.0.4):
-    resolution: {integrity: sha512-LwQbmBaSOZ5IKwCHz2mULcIuEr9rZ2b/7tqUGICHCawUzexUlQVxv2Yt0oFf2aZu83Sittt7dZwnN3sXHX9t9g==, tarball: file:projects/tool.tgz}
+    resolution: {integrity: sha512-sZH5yB7zg/kTpuIhLSqPYh0wFgw4aOpsriMq4wad8ZHRzlHASseyJAbEylIP8ltfPbFFN4Yy1nXaUOXS49anHg==, tarball: file:projects/tool.tgz}
     id: file:projects/tool.tgz
     name: '@rush-temp/tool'
     version: 0.0.0
@@ -35099,17 +35127,19 @@ packages:
     dev: false
 
   file:projects/ui.tgz(@types/node@20.11.19)(esbuild@0.20.1)(postcss-load-config@4.0.2)(postcss@8.4.35)(ts-node@10.9.2):
-    resolution: {integrity: sha512-WtSFJW84fNe+3lwzv2a8CRmyYIsY8B6HHJwg3YKLd7jWHF4T8hYIf892hAEv7kvh/vrZ7elq8E8b1znmCNd7Sw==, tarball: file:projects/ui.tgz}
+    resolution: {integrity: sha512-umESBjjPj7ES3uF9YcS31H5dwqZtMATByltYeDc+XG+7ovD1SOM11UAjBpHCqj026RvvqcSjE8lAQP1zRXxCoA==, tarball: file:projects/ui.tgz}
     id: file:projects/ui.tgz
     name: '@rush-temp/ui'
     version: 0.0.0
     dependencies:
+      '@types/dompurify': 3.0.5
       '@types/jest': 29.5.12
       '@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0)(eslint@8.56.0)(typescript@5.3.3)
       '@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.3.3)
       autolinker: 4.0.0
       date-fns: 2.30.0
       date-fns-tz: 2.0.0(date-fns@2.30.0)
+      dompurify: 3.1.6
       emoji-regex: 10.3.0
       eslint: 8.56.0
       eslint-config-standard-with-typescript: 40.0.0(@typescript-eslint/eslint-plugin@6.21.0)(eslint-plugin-import@2.29.1)(eslint-plugin-n@15.7.0)(eslint-plugin-promise@6.1.1)(eslint@8.56.0)(typescript@5.3.3)
@@ -35311,7 +35341,7 @@ packages:
     dev: false
 
   file:projects/view-resources.tgz(@types/node@20.11.19)(esbuild@0.20.1)(postcss-load-config@4.0.2)(postcss@8.4.35)(ts-node@10.9.2):
-    resolution: {integrity: sha512-l/K7osn3HZ3KIFeCyBe+rxQGUxvLvM+35if2HgylqgbWtD10Gk/rR+vuW1L54o8hT4ADMkbhvBW7VHE19isd+w==, tarball: file:projects/view-resources.tgz}
+    resolution: {integrity: sha512-g6op8hiY1zLsms7Sab4cAs29Ucbk6r20mx9hkZrhxn70uPW/VCLS+JW67cfWf85SyMwMloWuvY6ujfQfwNuScw==, tarball: file:projects/view-resources.tgz}
     id: file:projects/view-resources.tgz
     name: '@rush-temp/view-resources'
     version: 0.0.0
@@ -35326,6 +35356,7 @@ packages:
       eslint-plugin-promise: 6.1.1(eslint@8.56.0)
       eslint-plugin-svelte: 2.35.1(eslint@8.56.0)(svelte@4.2.12)(ts-node@10.9.2)
       fast-equals: 5.0.1
+      hls.js: 1.5.15
       jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2)
       prettier: 3.2.5
       prettier-plugin-svelte: 3.2.2(prettier@3.2.5)(svelte@4.2.12)

models/all/src/index.ts

@@ -178,7 +178,7 @@ export default function buildModel (enabled: string[] = ['*'], disabled: string[
       {
         label: calendar.string.ConfigLabel,
         description: calendar.string.ConfigDescription,
-        enabled: false,
+        enabled: true,
         beta: true,
         icon: calendar.icon.Calendar,
         classFilter: defaultFilter

models/calendar/src/index.ts

@@ -272,22 +272,22 @@ export function createModel (builder: Builder): void {
     calendar.category.Calendar
   )
 
-  createAction(
+  // createAction(
-    builder,
+  //   builder,
-    {
+  //   {
-      action: calendar.actionImpl.SaveEventReminder,
+  //     action: calendar.actionImpl.SaveEventReminder,
-      label: calendar.string.RemindMeAt,
+  //     label: calendar.string.RemindMeAt,
-      icon: calendar.icon.Reminder,
+  //     icon: calendar.icon.Reminder,
-      input: 'focus',
+  //     input: 'focus',
-      category: calendar.category.Calendar,
+  //     category: calendar.category.Calendar,
-      target: calendar.class.Event,
+  //     target: calendar.class.Event,
-      context: {
+  //     context: {
-        mode: 'context',
+  //       mode: 'context',
-        group: 'create'
+  //       group: 'create'
-      }
+  //     }
-    },
+  //   },
-    calendar.action.SaveEventReminder
+  //   calendar.action.SaveEventReminder
-  )
+  // )
 
   createAction(
     builder,

packages/core/src/memdb.ts

@@ -263,22 +263,25 @@ export abstract class MemDb extends TxProcessor implements Storage {
   }
 
   updateDoc (_id: Ref<Doc>, doc: Doc, update: TxUpdateDoc<Doc> | TxMixin<Doc, Doc>): void {
-    if (
+    if (this.hierarchy.isDerived(doc._class, core.class.Account) && update._class === core.class.TxUpdateDoc) {
-      this.hierarchy.isDerived(doc._class, core.class.Account) &&
+      const newEmail = (update as TxUpdateDoc<Account>).operations.email
-      update._class === core.class.TxUpdateDoc &&
+      if ((update as TxUpdateDoc<Account>).operations.person !== undefined) {
-      (update as TxUpdateDoc<Account>).operations.person !== undefined
+        const account = doc as Account
-    ) {
+        if (account.person !== undefined) {
-      const account = doc as Account
+          const acc = this.accountByPersonId.get(account.person) ?? []
-      if (account.person !== undefined) {
+          this.accountByPersonId.set(
-        const acc = this.accountByPersonId.get(account.person) ?? []
+            account.person,
-        this.accountByPersonId.set(
+            acc.filter((it) => it._id !== _id)
-          account.person,
+          )
-          acc.filter((it) => it._id !== _id)
+        }
-        )
+        const newPerson = (update as TxUpdateDoc<Account>).operations.person
-      }
+        if (newPerson !== undefined) {
-      const newPerson = (update as TxUpdateDoc<Account>).operations.person
+          this.accountByPersonId.set(newPerson, [...(this.accountByPersonId.get(newPerson) ?? []), account])
-      if (newPerson !== undefined) {
+        }
-        this.accountByPersonId.set(newPerson, [...(this.accountByPersonId.get(newPerson) ?? []), account])
+      } else if (newEmail !== undefined) {
+        const account = doc as Account
+        this.accountByEmail.delete(account.email)
+        this.accountByEmail.set(newEmail, account)
       }
     }
   }

packages/presentation/package.json

@@ -48,6 +48,7 @@
     "@hcengineering/ui": "^0.6.15",
     "@hcengineering/view": "^0.6.13",
     "@hcengineering/text": "^0.6.5",
+    "@hcengineering/diffview": "^0.6.0",
     "@hcengineering/uploader": "^0.6.0",
     "svelte": "^4.2.12",
     "@hcengineering/client": "^0.6.18",

packages/presentation/src/components/markup/CodeBlockNode.svelte

@@ -0,0 +1,32 @@
+<!--
+// Copyright © 2024 Hardcore Engineering Inc.
+//
+// Licensed under the Eclipse Public License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License. You may
+// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//
+// See the License for the specific language governing permissions and
+// limitations under the License.
+-->
+<script lang="ts">
+  import diffview from '@hcengineering/diffview'
+  import { MarkupNode } from '@hcengineering/text'
+  import { Component } from '@hcengineering/ui'
+
+  export let node: MarkupNode
+  export let preview = false
+
+  $: language = node.attrs?.language
+  $: content = node.content ?? []
+  $: value = content.map((node) => node.text).join('/n')
+</script>
+
+{#if node}
+  <pre class="proseCodeBlock" style:margin={preview ? '0' : null}><code
+      ><Component is={diffview.component.Highlight} props={{ value, language }} /></code
+    ></pre>
+{/if}

packages/presentation/src/components/markup/NodeContent.svelte

@@ -17,6 +17,7 @@
   import { AttrValue, MarkupNode, MarkupNodeType } from '@hcengineering/text'
 
   import MarkupNodes from './Nodes.svelte'
+  import CodeBlockNode from './CodeBlockNode.svelte'
   import ObjectNode from './ObjectNode.svelte'
 
   export let node: MarkupNode
@@ -71,7 +72,7 @@
       <MarkupNodes {nodes} {preview} />
     </svelte:element>
   {:else if node.type === MarkupNodeType.code_block}
-    <pre class="proseCodeBlock" style:margin={preview ? '0' : null}><code><MarkupNodes {nodes} {preview} /></code></pre>
+    <CodeBlockNode {node} {preview} />
   {:else if node.type === MarkupNodeType.image}
     {@const src = toString(attrs.src)}
     {@const alt = toString(attrs.alt)}

packages/presentation/src/preview.ts

@@ -6,27 +6,56 @@ import { getFileUrl, getCurrentWorkspaceId } from './file'
 import presentation from './plugin'
 
 export interface PreviewConfig {
-  previewUrl: string
+  image: string
+  video: string
 }
 
-const defaultPreview = (): string => `/files/${getCurrentWorkspaceId()}?file=:blobId&size=:size`
+export interface VideoMeta {
+  status: 'ready' | 'error' | 'inprogress' | 'queued' | 'downloading' | 'pendingupload'
+  thumbnail: string
+  hls: string
+}
+
+const defaultImagePreview = (): string => `/files/${getCurrentWorkspaceId()}?file=:blobId&size=:size`
 
 /**
  *
  * PREVIEW_CONFIG env variable format.
- * previewUrl - an Url with :workspace, :blobId, :downloadFile, :size placeholders, they will be replaced in UI with an appropriate blob values.
+ * - image - an Url with :workspace, :blobId, :downloadFile, :size placeholders.
+ * - video - an Url with :workspace, :blobId placeholders.
  */
 export function parsePreviewConfig (config?: string): PreviewConfig | undefined {
   if (config === undefined) {
     return
   }
-  return { previewUrl: config }
+
+  const previewConfig = { image: defaultImagePreview(), video: '' }
+
+  const configs = config.split(';')
+  for (const c of configs) {
+    if (c.includes('|')) {
+      const [key, value] = c.split('|')
+      if (key === 'image') {
+        previewConfig.image = value
+      } else if (key === 'video') {
+        previewConfig.video = value
+      } else {
+        throw new Error(`Unknown preview config key: ${key}`)
+      }
+    } else {
+      // fallback to image-only config for compatibility
+      previewConfig.image = c
+    }
+  }
+
+  return Object.freeze(previewConfig)
 }
 
 export function getPreviewConfig (): PreviewConfig {
   return (
     (getMetadata(presentation.metadata.PreviewConfig) as PreviewConfig) ?? {
-      previewUrl: defaultPreview()
+      image: defaultImagePreview(),
+      video: ''
     }
   )
 }
@@ -58,7 +87,7 @@ function blobToSrcSet (cfg: PreviewConfig, blob: Ref<Blob>, width: number | unde
     return ''
   }
 
-  let url = cfg.previewUrl.replaceAll(':workspace', encodeURIComponent(getCurrentWorkspaceId()))
+  let url = cfg.image.replaceAll(':workspace', encodeURIComponent(getCurrentWorkspaceId()))
   const downloadUrl = getFileUrl(blob)
 
   const frontUrl = getMetadata(presentation.metadata.FrontUrl) ?? window.location.origin
@@ -89,3 +118,25 @@ function blobToSrcSet (cfg: PreviewConfig, blob: Ref<Blob>, width: number | unde
 export function getFileSrcSet (_blob: Ref<Blob>, width?: number): string {
   return blobToSrcSet(getPreviewConfig(), _blob, width)
 }
+
+/**
+ * @public
+ */
+export async function getVideoMeta (file: string, filename?: string): Promise<VideoMeta | undefined> {
+  const cfg = getPreviewConfig()
+
+  const url = cfg.video
+    .replaceAll(':workspace', encodeURIComponent(getCurrentWorkspaceId()))
+    .replaceAll(':blobId', encodeURIComponent(file))
+
+  if (url === '') {
+    return undefined
+  }
+
+  try {
+    const response = await fetch(url)
+    if (response.ok) {
+      return (await response.json()) as VideoMeta
+    }
+  } catch {}
+}

packages/theme/styles/_colors.scss

@@ -77,7 +77,6 @@
   --text-editor-highlighted-node-delete-background-color: #F6DCDA;
   --text-editor-highlighted-node-delete-font-color: #54201C;
   
-  --text-editor-inline-code-color: #B02B46;
   --text-editor-table-marker-color: #bebebf;
 
   --theme-clockface-sec-arrow: conic-gradient(at 50% -10px, rgba(255, 0, 0, 0), rgba(255, 0, 0, 0) 49%, #F47758 50%, rgba(255, 0, 0, 0) 51%, rgba(255, 0, 0, 0) 100%);

packages/theme/styles/prose.scss

@@ -345,7 +345,6 @@ table.proseTable {
   margin: 0 1px;
   padding: 0 .25rem;
   font-family: var(--mono-font);
-  color: var(--text-editor-inline-code-color);
   background-color: var(--theme-button-default);
   border: 1px solid var(--theme-button-border);
   border-radius: .25rem;

packages/ui/package.json

@@ -33,6 +33,7 @@
     "prettier": "^3.1.0",
     "typescript": "^5.3.3",
     "@types/jest": "^29.5.5",
+    "@types/dompurify": "^3.0.5",
     "jest": "^29.7.0",
     "ts-jest": "^29.1.1",
     "svelte-eslint-parser": "^0.33.1"
@@ -47,6 +48,7 @@
     "emoji-regex": "^10.1.0",
     "date-fns": "^2.30.0",
     "date-fns-tz": "^2.0.0",
+    "dompurify": "^3.1.6",
     "@hcengineering/analytics": "^0.6.0"
   },
   "repository": "https://github.com/hcenginneing/anticrm",

packages/ui/src/components/Html.svelte

@@ -0,0 +1,23 @@
+<!--
+// Copyright © 2024 Hardcore Engineering Inc.
+//
+// Licensed under the Eclipse Public License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License. You may
+// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//
+// See the License for the specific language governing permissions and
+// limitations under the License.
+-->
+<script lang="ts">
+  import dompurify from 'dompurify'
+
+  export let value: string
+
+  $: sanitized = dompurify.sanitize(value)
+</script>
+
+{@html sanitized}

packages/ui/src/components/TooltipInstance.svelte

@@ -328,6 +328,18 @@
       />
     {/if}
   </div>
+  <svg class="svg-mask">
+    <clipPath id="nub-bg">
+      <path
+        d="M7.3.6 4.2 4.3C2.9 5.4 1.5 6 0 6v1h18V6c-1.5 0-2.9-.6-4.2-1.7L10.7.6C9.9-.1 8.5-.2 7.5.4c0 .1-.1.1-.2.2z"
+      />
+    </clipPath>
+    <clipPath id="nub-border">
+      <path
+        d="M4.8 5.1 8 1.3s.1 0 .1-.1c.5-.3 1.4-.3 1.9.1L13.1 5l.1.1 1.2.9H18c-1.5 0-2.9-.6-4.2-1.7L10.7.6C9.9-.1 8.5-.2 7.5.4c0 .1-.1.1-.2.2L4.2 4.3C2.9 5.4 1.5 6 0 6h3.6l1.2-.9z"
+      />
+    </clipPath>
+  </svg>
   <div
     bind:this={nubHTML}
     style:z-index={($modals.findIndex((t) => t.type === 'tooltip') ?? 1) + 10000}

packages/ui/src/index.ts

@@ -96,6 +96,7 @@ export { default as DatePresenter } from './components/calendar/DatePresenter.sv
 export { default as DueDatePresenter } from './components/calendar/DueDatePresenter.svelte'
 export { default as DateTimePresenter } from './components/calendar/DateTimePresenter.svelte'
 export { default as TimeInputBox } from './components/calendar/TimeInputBox.svelte'
+export { default as Html } from './components/Html.svelte'
 export { default as StylishEdit } from './components/StylishEdit.svelte'
 export { default as Grid } from './components/Grid.svelte'
 export { default as Row } from './components/Row.svelte'

plugins/diffview-resources/src/components/Highlight.svelte

@@ -0,0 +1,26 @@
+<!--
+// Copyright © 2024 Hardcore Engineering Inc.
+//
+// Licensed under the Eclipse Public License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License. You may
+// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//
+// See the License for the specific language governing permissions and
+// limitations under the License.
+-->
+<script lang="ts">
+  import { Html } from '@hcengineering/ui'
+
+  import { highlightText } from '../highlight'
+
+  export let value: string
+  export let language: string | undefined = undefined
+
+  $: highlighted = highlightText(value, { language })
+</script>
+
+<Html value={highlighted} />

plugins/diffview-resources/src/highlight/highlight.ts

@@ -19,18 +19,18 @@ import { hljsDefineSvelte } from './languages/svelte-hljs'
 hljs.registerLanguage('svelte', hljsDefineSvelte)
 
 export interface HighlightOptions {
-  language: string
+  language: string | undefined
 }
 
 export function highlightText (text: string, options: HighlightOptions): string {
   // We should always use highlighter because it sanitizes the input
   // We have to always use highlighter to ensure that the input is sanitized
-  const validLanguage = options.language !== '' && hljs.getLanguage(options.language) !== undefined
+  const { language } = options
-  const language = validLanguage ? options.language : 'text'
+  const validLanguage = language !== undefined && hljs.getLanguage(language) !== undefined
 
-  const { value: highlighted } = hljs.highlight(text, { language })
+  const { value: highlighted } = validLanguage ? hljs.highlight(text, { language }) : hljs.highlightAuto(text)
-  const normalized = normalizeHighlightTags(highlighted)
+
-  return normalized
+  return normalizeHighlightTags(highlighted)
 }
 
 export function highlightLines (lines: string[], options: HighlightOptions): string[] {

plugins/diffview-resources/src/index.ts

@@ -15,10 +15,12 @@
 
 import { type Resources } from '@hcengineering/platform'
 import DiffView from './components/DiffView.svelte'
+import Highlight from './components/Highlight.svelte'
 import InlineDiffView from './components/InlineDiffView.svelte'
 export default async (): Promise<Resources> => ({
   component: {
     DiffView,
-    InlineDiffView
+    InlineDiffView,
+    Highlight
   }
 })

plugins/diffview/src/index.ts

@@ -48,7 +48,8 @@ export interface DiffFileId {
 export default plugin(diffviewId, {
   component: {
     DiffView: '' as AnyComponent,
-    InlineDiffView: '' as AnyComponent
+    InlineDiffView: '' as AnyComponent,
+    Highlight: '' as AnyComponent
   },
   string: {
     ViewMode: '' as IntlString,

plugins/text-editor-resources/src/components/extension/codeblock.ts

@@ -13,12 +13,13 @@
 // limitations under the License.
 //
 
+import { codeBlockOptions } from '@hcengineering/text'
 import { DropdownLabelsPopup, getEventPositionElement, showPopup } from '@hcengineering/ui'
 import { type CodeBlockLowlightOptions, CodeBlockLowlight } from '@tiptap/extension-code-block-lowlight'
 import { type Node as ProseMirrorNode } from '@tiptap/pm/model'
 import { Plugin, PluginKey } from '@tiptap/pm/state'
 import { Decoration, DecorationSet, type EditorView } from '@tiptap/pm/view'
-import { type createLowlight } from 'lowlight'
+import { common, createLowlight } from 'lowlight'
 
 type Lowlight = ReturnType<typeof createLowlight>
 
@@ -26,14 +27,19 @@ const chevronSvg = `<svg width="16" height="16" viewBox="0 0 32 32" fill="curren
   <path d="M16 22L6 12L7.4 10.6L16 19.2L24.6 10.6L26 12L16 22Z" />
 </svg>`
 
-export const CodeBlockExtension = CodeBlockLowlight.extend<CodeBlockLowlightOptions>({
+export const codeBlockHighlightOptions: CodeBlockLowlightOptions = {
+  ...codeBlockOptions,
+  lowlight: createLowlight(common)
+}
+
+export const CodeBlockHighlighExtension = CodeBlockLowlight.extend<CodeBlockLowlightOptions>({
   addProseMirrorPlugins () {
     return [...(this.parent?.() ?? []), LanguageSelector(this.options)]
   }
 })
 
 export function LanguageSelector (options: CodeBlockLowlightOptions): Plugin {
-  return new Plugin({
+  return new Plugin<DecorationSet>({
     key: new PluginKey('codeblock-language-selector'),
     props: {
       decorations (state) {
@@ -41,13 +47,14 @@ export function LanguageSelector (options: CodeBlockLowlightOptions): Plugin {
       }
     },
     state: {
-      init () {
+      init (config, state) {
-        return DecorationSet.empty
+        return createDecorations(state.doc, options)
       },
       apply (tr, prev) {
         if (tr.docChanged) {
           return createDecorations(tr.doc, options)
         }
+
         return prev
       }
     }
@@ -84,7 +91,7 @@ function createDecorations (doc: ProseMirrorNode, options: CodeBlockLowlightOpti
 
 function createLangButton (language: string | null): HTMLButtonElement {
   const button = document.createElement('button')
-  button.className = 'antiButton ghost small sh-no-shape bs-none gap-medium iconR'
+  button.className = 'antiButton link-bordered small sh-no-shape bs-none gap-medium iconR'
   button.style.position = 'absolute'
   button.style.top = '0.375rem'
   button.style.right = '0.375rem'

plugins/text-editor-resources/src/components/extension/inlineCommands.ts

@@ -14,8 +14,8 @@
 //
 
 import { Extension } from '@tiptap/core'
-import Suggestion, { type SuggestionOptions } from './suggestion'
 import { PluginKey } from '@tiptap/pm/state'
+import Suggestion, { type SuggestionOptions } from './suggestion'
 
 export interface InlineCommandsOptions {
   suggestion: Omit<SuggestionOptions, 'editor'>
@@ -31,7 +31,11 @@ export const InlineCommandsExtension = Extension.create<InlineCommandsOptions>({
     return {
       suggestion: {
         char: '/',
-        startOfLine: true
+        allow: ({ state }) => {
+          const { $anchor } = state.selection
+          const parent = $anchor.parent
+          return parent.type.name === 'paragraph'
+        }
       }
     }
   },

plugins/text-editor-resources/src/kits/default-kit.ts

@@ -13,7 +13,7 @@
 // limitations under the License.
 //
 
-import { codeBlockOptions, codeOptions } from '@hcengineering/text'
+import { codeOptions } from '@hcengineering/text'
 import { showPopup } from '@hcengineering/ui'
 import { type Editor, Extension } from '@tiptap/core'
 import type { CodeOptions } from '@tiptap/extension-code'
@@ -25,10 +25,9 @@ import Link from '@tiptap/extension-link'
 import Typography from '@tiptap/extension-typography'
 import Underline from '@tiptap/extension-underline'
 import StarterKit from '@tiptap/starter-kit'
-import { common, createLowlight } from 'lowlight'
 
 import LinkPopup from '../components/LinkPopup.svelte'
-import { CodeBlockExtension } from '../components/extension/codeblock'
+import { CodeBlockHighlighExtension, codeBlockHighlightOptions } from '../components/extension/codeblock'
 
 export interface DefaultKitOptions {
   codeBlock?: Partial<CodeBlockOptions> | false
@@ -66,10 +65,7 @@ export const DefaultKit = Extension.create<DefaultKitOptions>({
         openOnClick: true,
         HTMLAttributes: { class: 'cursor-pointer', rel: 'noopener noreferrer', target: '_blank' }
       }),
-      CodeBlockExtension.configure({
+      CodeBlockHighlighExtension.configure(codeBlockHighlightOptions)
-        ...codeBlockOptions,
-        lowlight: createLowlight(common)
-      })
     ]
   }
 })

plugins/text-editor-resources/src/kits/editor-kit.ts

@@ -15,7 +15,7 @@
 import { type Class, type Doc, type Ref, type Space } from '@hcengineering/core'
 import { getResource } from '@hcengineering/platform'
 import { getBlobRef, getClient } from '@hcengineering/presentation'
-import { CodeBlockExtension, codeBlockOptions, CodeExtension, codeOptions } from '@hcengineering/text'
+import { CodeExtension, codeOptions } from '@hcengineering/text'
 import textEditor, { type ActionContext, type ExtensionCreator, type TextEditorMode } from '@hcengineering/text-editor'
 import { type AnyExtension, type Editor, Extension } from '@tiptap/core'
 import { type Level } from '@tiptap/extension-heading'
@@ -23,6 +23,7 @@ import ListKeymap from '@tiptap/extension-list-keymap'
 import TableHeader from '@tiptap/extension-table-header'
 import 'prosemirror-codemark/dist/codemark.css'
 
+import { CodeBlockHighlighExtension, codeBlockHighlightOptions } from '../components/extension/codeblock'
 import { NoteExtension, type NoteOptions } from '../components/extension/note'
 import { FileExtension, type FileOptions } from '../components/extension/fileExt'
 import { HardBreakExtension } from '../components/extension/hardBreak'
@@ -171,7 +172,7 @@ async function buildEditorKit (): Promise<Extension<EditorKitOptions, any>> {
                     }
                   })
                 ],
-                [200, CodeBlockExtension.configure(codeBlockOptions)],
+                [200, CodeBlockHighlighExtension.configure(codeBlockHighlightOptions)],
                 [210, CodeExtension.configure(codeOptions)],
                 [220, HardBreakExtension.configure({ shortcuts: mode })]
               ]

plugins/view-resources/package.json

@@ -57,6 +57,7 @@
     "@hcengineering/text-editor-resources": "^0.6.0",
     "@hcengineering/analytics": "^0.6.0",
     "@hcengineering/query": "^0.6.12",
-    "fast-equals": "^5.0.1"
+    "fast-equals": "^5.0.1",
+    "hls.js": "^1.5.15"
   }
 }

plugins/view-resources/src/components/viewer/VideoViewer.svelte

@@ -14,20 +14,46 @@
 -->
 <script lang="ts">
   import { type Blob, type Ref } from '@hcengineering/core'
-  import { getFileUrl, type BlobMetadata } from '@hcengineering/presentation'
+  import { getFileUrl, getVideoMeta, type BlobMetadata } from '@hcengineering/presentation'
+  import HLS from 'hls.js'
 
   export let value: Ref<Blob>
   export let name: string
   export let metadata: BlobMetadata | undefined
   export let fit: boolean = false
 
+  let video: HTMLVideoElement
+
+  async function fetchVideoMeta (value: Ref<Blob>, name: string): Promise<void> {
+    const src = getFileUrl(value, name)
+    const meta = await getVideoMeta(value, name)
+    if (meta != null && meta.status === 'ready' && HLS.isSupported()) {
+      const hls = new HLS()
+      hls.loadSource(meta.hls)
+      hls.attachMedia(video)
+      video.poster = meta.thumbnail
+    } else {
+      video.src = src
+    }
+  }
+
+  $: aspectRatio =
+    metadata?.originalWidth && metadata?.originalHeight
+      ? `${metadata.originalWidth} / ${metadata.originalHeight}`
+      : '16 / 9'
   $: maxWidth = metadata?.originalWidth ? `min(${metadata.originalWidth}px, 100%)` : undefined
   $: maxHeight = metadata?.originalHeight ? `min(${metadata.originalHeight}px, 80vh)` : undefined
-
+  $: void fetchVideoMeta(value, name)
-  $: src = getFileUrl(value, name)
 </script>
 
-<video style:max-width={fit ? '100%' : maxWidth} style:max-height={fit ? '100%' : maxHeight} controls preload={'auto'}>
+<video
-  <source {src} />
+  bind:this={video}
+  width="100%"
+  style:aspect-ratio={aspectRatio}
+  style:max-width={fit ? '100%' : maxWidth}
+  style:max-height={fit ? '100%' : maxHeight}
+  controls
+  preload={'auto'}
+>
   <track kind="captions" label={name} />
 </video>

plugins/workbench-resources/src/components/Workbench.svelte

@@ -641,16 +641,6 @@
       <path d="M10.5,12.2c0-2.9,2.4-5.2,5.2-5.2c0.6,0,1.2,0.1,1.8,0.3V0H0v17.5h15.8C12.9,17.5,10.5,15.1,10.5,12.2z" />
       <path d="M15.8,17.5h1.8v-0.4C17,17.4,16.4,17.5,15.8,17.5z" />
     </clipPath>
-    <clipPath id="nub-bg">
-      <path
-        d="M7.3.6 4.2 4.3C2.9 5.4 1.5 6 0 6v1h18V6c-1.5 0-2.9-.6-4.2-1.7L10.7.6C9.9-.1 8.5-.2 7.5.4c0 .1-.1.1-.2.2z"
-      />
-    </clipPath>
-    <clipPath id="nub-border">
-      <path
-        d="M4.8 5.1 8 1.3s.1 0 .1-.1c.5-.3 1.4-.3 1.9.1L13.1 5l.1.1 1.2.9H18c-1.5 0-2.9-.6-4.2-1.7L10.7.6C9.9-.1 8.5-.2 7.5.4c0 .1-.1.1-.2.2L4.2 4.3C2.9 5.4 1.5 6 0 6h3.6l1.2-.9z"
-      />
-    </clipPath>
   </svg>
   <div
     class="workbench-container"

pods/server/Dockerfile

@@ -16,4 +16,4 @@ COPY bundle/bundle.js.map ./
 
 EXPOSE 8080
 ENV UWS_HTTP_MAX_HEADERS_SIZE 32768
-CMD node --enable-source-maps --inspect=0.0.0.0:9229 bundle.js
+CMD node bundle.js

pods/server/package.json

@@ -8,14 +8,14 @@
   "template": "@hcengineering/node-package",
   "license": "EPL-2.0",
   "scripts": {
-    "start": "rush bundle --to @hcengineering/pod-server && cross-env NODE_ENV=production ELASTIC_INDEX_NAME=local_storage_index MODEL_VERSION=$(node ../../common/scripts/show_version.js) ACCOUNTS_URL=http://localhost:3000 REKONI_URL=http://localhost:4004 MONGO_URL=mongodb://localhost:27017 ELASTIC_URL=http://localhost:9200 FRONT_URL=http://localhost:8087 UPLOAD_URL=/upload MINIO_ENDPOINT=localhost MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin METRICS_CONSOLE=true SERVER_SECRET=secret OPERATION_PROFILING=false node --inspect --enable-source-maps bundle/bundle.js",
+    "start": "rush bundle --to @hcengineering/pod-server && cross-env NODE_ENV=production ELASTIC_INDEX_NAME=local_storage_index MODEL_VERSION=$(node ../../common/scripts/show_version.js) ACCOUNTS_URL=http://localhost:3000 REKONI_URL=http://localhost:4004 MONGO_URL=mongodb://localhost:27017 ELASTIC_URL=http://localhost:9200 FRONT_URL=http://localhost:8087 UPLOAD_URL=/upload MINIO_ENDPOINT=localhost MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin METRICS_CONSOLE=true SERVER_SECRET=secret OPERATION_PROFILING=false node bundle/bundle.js",
-    "start-u": "rush bundle --to @hcengineering/pod-server && cp ./node_modules/@hcengineering/uws/lib/*.node ./bundle/ && cross-env NODE_ENV=production SERVER_PROVIDER=uweb ELASTIC_INDEX_NAME=local_storage_index MODEL_VERSION=$(node ../../common/scripts/show_version.js) ACCOUNTS_URL=http://localhost:3000 REKONI_URL=http://localhost:4004 MONGO_URL=mongodb://localhost:27017 ELASTIC_URL=http://localhost:9200 FRONT_URL=http://localhost:8087 UPLOAD_URL=/upload MINIO_ENDPOINT=localhost MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin METRICS_CONSOLE=true SERVER_SECRET=secret node --inspect bundle/bundle.js",
+    "start-u": "rush bundle --to @hcengineering/pod-server && cp ./node_modules/@hcengineering/uws/lib/*.node ./bundle/ && cross-env NODE_ENV=production SERVER_PROVIDER=uweb ELASTIC_INDEX_NAME=local_storage_index MODEL_VERSION=$(node ../../common/scripts/show_version.js) ACCOUNTS_URL=http://localhost:3000 REKONI_URL=http://localhost:4004 MONGO_URL=mongodb://localhost:27017 ELASTIC_URL=http://localhost:9200 FRONT_URL=http://localhost:8087 UPLOAD_URL=/upload MINIO_ENDPOINT=localhost MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin METRICS_CONSOLE=true SERVER_SECRET=secret node bundle/bundle.js",
     "start-flame": "rush bundle --to @hcengineering/pod-server && cross-env NODE_ENV=production ELASTIC_INDEX_NAME=local_storage_index MODEL_VERSION=$(node ../../common/scripts/show_version.js) ACCOUNTS_URL=http://localhost:3000 REKONI_URL=http://localhost:4004 MONGO_URL=mongodb://localhost:27017 ELASTIC_URL=http://localhost:9200 FRONT_URL=http://localhost:8087 UPLOAD_URL=/upload MINIO_ENDPOINT=localhost MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin METRICS_CONSOLE=true SERVER_SECRET=secret clinic flame --dest ./out -- node --nolazy -r ts-node/register --enable-source-maps src/__start.ts",
     "build": "compile",
     "_phase:bundle": "rushx bundle",
     "_phase:docker-build": "rushx docker:build",
     "_phase:docker-staging": "rushx docker:staging",
-    "bundle": "mkdir -p bundle && esbuild src/__start.ts --sourcemap=inline --bundle --keep-names --platform=node --external:*.node --external:bufferutil --external:snappy --external:utf-8-validate --external:msgpackr-extract --define:process.env.GIT_REVISION=$(../../common/scripts/git_version.sh) --outfile=bundle/bundle.js --log-level=error --sourcemap=external",
+    "bundle": "mkdir -p bundle && esbuild src/__start.ts --sourcemap=inline --bundle --keep-names --platform=node --external:*.node --external:bufferutil --external:snappy --external:utf-8-validate --external:msgpackr-extract --define:process.env.MODEL_VERSION=$(node ../../common/scripts/show_version.js) --define:process.env.GIT_REVISION=$(../../common/scripts/git_version.sh) --outfile=bundle/bundle.js --log-level=error --sourcemap=external",
     "docker:build": "../../common/scripts/docker_build.sh hardcoreeng/transactor",
     "docker:tbuild": "docker build -t hardcoreeng/transactor . --platform=linux/amd64 && ../../common/scripts/docker_tag_push.sh hardcoreeng/transactor",
     "docker:abuild": "docker build -t hardcoreeng/transactor . --platform=linux/arm64 && ../../common/scripts/docker_tag_push.sh hardcoreeng/transactor",

server-plugins/time-resources/src/index.ts

@@ -582,7 +582,7 @@ async function changeIssueStatusHandler (
   return []
 }
 
-async function changeIssueNumberHandler (control: TriggerControl, issueId: Ref<Issue>): Promise<Tx[]> {
+async function changeIssueDataHandler (control: TriggerControl, issueId: Ref<Issue>): Promise<Tx[]> {
   const res: Tx[] = []
   const issue = (await control.findAll(control.ctx, tracker.class.Issue, { _id: issueId }))[0]
   if (issue !== undefined) {
@@ -632,9 +632,10 @@ async function updateIssueHandler (tx: TxUpdateDoc<Issue>, control: TriggerContr
   if (newStatus !== undefined) {
     res.push(...(await changeIssueStatusHandler(control, newStatus, tx.objectId)))
   }
+  const name = tx.operations.title
   const number = tx.operations.number
-  if (number !== undefined) {
+  if (number !== undefined || name !== undefined) {
-    res.push(...(await changeIssueNumberHandler(control, tx.objectId)))
+    res.push(...(await changeIssueDataHandler(control, tx.objectId)))
   }
   return res
 }

server/backup/src/backup.ts

@@ -636,11 +636,7 @@ export async function backup (
       // Load all digest from collection.
       while (true) {
         try {
-          const currentChunk = await ctx.with(
+          const currentChunk = await ctx.with('loadChunk', {}, () => connection.loadChunk(domain, idx, options.recheck))
-            'loadChunk',
-            {},
-            async () => await connection.loadChunk(domain, idx, options.recheck)
-          )
           idx = currentChunk.idx
 
           let needRetrieve: Ref<Doc>[] = []
@@ -1201,10 +1197,22 @@ export async function restore (
       workspace: workspaceId.name
     })
 
+    const doTrim = (s: string | undefined): string | undefined => {
+      if (s == null) {
+        return s
+      }
+      if (s.startsWith('"') && s.endsWith('"')) {
+        return s.slice(1, s.length - 1)
+      }
+      return s
+    }
+
     // Let's find difference
     const docsToAdd = new Map(
       Array.from(changeset.entries()).filter(
-        ([it]) => !serverChangeset.has(it) || (serverChangeset.has(it) && serverChangeset.get(it) !== changeset.get(it))
+        ([it]) =>
+          !serverChangeset.has(it) ||
+          (serverChangeset.has(it) && doTrim(serverChangeset.get(it)) !== doTrim(changeset.get(it)))
       )
     )
     const docsToRemove = Array.from(serverChangeset.keys()).filter((it) => !changeset.has(it))

server/core/src/storage.ts

@@ -9,7 +9,6 @@ import {
   type StorageIterator,
   type WorkspaceId
 } from '@hcengineering/core'
-import { estimateDocSize } from './utils'
 
 export * from '@hcengineering/storage'
 
@@ -77,7 +76,7 @@ export class BackupClientOps {
           break
         }
 
-        size += estimateDocSize(doc)
+        size += doc.size
         docs.push(doc)
       }
 
@@ -99,15 +98,15 @@ export class BackupClientOps {
     })
   }
 
-  async loadDocs (ctx: MeasureContext, domain: Domain, docs: Ref<Doc>[]): Promise<Doc[]> {
+  loadDocs (ctx: MeasureContext, domain: Domain, docs: Ref<Doc>[]): Promise<Doc[]> {
-    return await this.storage.load(ctx, domain, docs)
+    return this.storage.load(ctx, domain, docs)
   }
 
-  async upload (ctx: MeasureContext, domain: Domain, docs: Doc[]): Promise<void> {
+  upload (ctx: MeasureContext, domain: Domain, docs: Doc[]): Promise<void> {
-    await this.storage.upload(ctx, domain, docs)
+    return this.storage.upload(ctx, domain, docs)
   }
 
-  async clean (ctx: MeasureContext, domain: Domain, docs: Ref<Doc>[]): Promise<void> {
+  clean (ctx: MeasureContext, domain: Domain, docs: Ref<Doc>[]): Promise<void> {
-    await this.storage.clean(ctx, domain, docs)
+    return this.storage.clean(ctx, domain, docs)
   }
 }

server/front/readme.md

@@ -23,15 +23,12 @@ Front service is suited to deliver application bundles and resource assets, it a
 
 PREVIEW_CONFIG env variable format.
 
-A `;` separated list of triples, providerName|previewUrl|supportedFormats.
+A `;` separated list of pairs, mediaType|previewUrl.
 
-- providerName - a provider name should be same as in Storage configuration.
+* mediaType - a type of media, image or video.
-  It coult be empty and it will match by content types.
+* previewUrl - an Url with :workspace, :blobId, :downloadFile, :size placeholders, they will be replaced in UI with an appropriate blob values.
-- previewUrl - an Url with :workspace, :blobId, :downloadFile, :size placeholders, they will be replaced in UI with an appropriate blob values.
-- supportedFormats - a `,` separated list of file extensions.
-- contentTypes - a ',' separated list of content type patterns.
 
-PREVIEW_CONFIG=https://front.hc.engineering/files/:workspace/api/preview/?width=:size&image=:downloadFile
+PREVIEW_CONFIG=image|https://front.hc.engineering/files/:workspace/api/preview/?width=:size&image=:downloadFile
 
 ## Variables
 

server/mongo/src/storage.ts

@@ -1000,7 +1000,7 @@ abstract class MongoAdapterBase implements DbAdapter {
               await coll.bulkWrite(
                 Array.from(bulkUpdate.entries()).map((it) => ({
                   updateOne: {
-                    filter: { _id: it[0] },
+                    filter: { _id: it[0], '%hash%': null },
                     update: { $set: { '%hash%': it[1] } }
                   }
                 }))

server/postgres/src/storage.ts

@@ -753,12 +753,14 @@ abstract class PostgresAdapterBase implements DbAdapter {
     if (!isDataField(key)) return `"${key}"`
     const arr = key.split('.').filter((p) => p)
     let tKey = ''
+    let isNestedField = false
 
     for (let i = 0; i < arr.length; i++) {
       const element = arr[i]
       if (element === '$lookup') {
         tKey += arr[++i] + '_lookup'
       } else if (this.hierarchy.isMixin(element as Ref<Class<Doc>>)) {
+        isNestedField = true
         tKey += `${element}`
         if (i !== arr.length - 1) {
           tKey += "'->'"
@@ -773,7 +775,7 @@ abstract class PostgresAdapterBase implements DbAdapter {
       tKey = this.checkMixinKey<T>(tKey, _class, isDataArray)
     }
 
-    return isDataArray ? `data->'${tKey}'` : `data#>>'{${tKey}}'`
+    return isDataArray || isNestedField ? `data->'${tKey}'` : `data#>>'{${tKey}}'`
   }
 
   private checkMixinKey<T extends Doc>(key: string, _class: Ref<Class<T>>, isDataArray: boolean): string {
@@ -1046,7 +1048,9 @@ abstract class PostgresAdapterBase implements DbAdapter {
         const vals = part
           .map((doc) => {
             const d = convertDoc(doc, this.workspaceId.name)
-            return `('${d._id}', '${d.workspaceId}', '${d._class}', '${d.createdBy ?? d.modifiedBy}', '${d.modifiedBy}', ${d.modifiedOn}, ${d.createdOn ?? d.modifiedOn}, '${d.space}', '${d.attachedTo ?? '[NULL]'}', '${escapeBackticks(JSON.stringify(d.data))}')`
+            return `('${d._id}', '${d.workspaceId}', '${d._class}', '${d.createdBy ?? d.modifiedBy}', '${d.modifiedBy}', ${d.modifiedOn}, ${d.createdOn ?? d.modifiedOn}, '${d.space}', ${
+              d.attachedTo != null ? `'${d.attachedTo}'` : 'NULL'
+            }, '${escapeBackticks(JSON.stringify(d.data))}')`
           })
           .join(', ')
         await client.query(
@@ -1133,7 +1137,9 @@ abstract class PostgresAdapterBase implements DbAdapter {
         const vals = part
           .map((doc) => {
             const d = convertDoc(doc, this.workspaceId.name)
-            return `('${d._id}', '${d.workspaceId}', '${d._class}', '${d.createdBy ?? d.modifiedBy}', '${d.modifiedBy}', ${d.modifiedOn}, ${d.createdOn ?? d.modifiedOn}, '${d.space}', '${d.attachedTo ?? '[NULL]'}', '${escapeBackticks(JSON.stringify(d.data))}')`
+            return `('${d._id}', '${d.workspaceId}', '${d._class}', '${d.createdBy ?? d.modifiedBy}', '${d.modifiedBy}', ${d.modifiedOn}, ${d.createdOn ?? d.modifiedOn}, '${d.space}', ${
+              d.attachedTo != null ? `'${d.attachedTo}'` : 'NULL'
+            }, '${escapeBackticks(JSON.stringify(d.data))}')`
           })
           .join(', ')
         await client.query(

server/postgres/src/utils.ts

@@ -290,7 +290,7 @@ export function translateDomain (domain: string): string {
 export function parseDocWithProjection<T extends Doc> (doc: DBDoc, projection: Projection<T> | undefined): T {
   const { workspaceId, data, ...rest } = doc
   for (const key in rest) {
-    if ((rest as any)[key] === 'NULL') {
+    if ((rest as any)[key] === 'NULL' || (rest as any)[key] === null) {
       if (key === 'attachedTo') {
         // eslint-disable-next-line @typescript-eslint/no-dynamic-delete
         delete rest[key]
@@ -321,7 +321,7 @@ export function parseDocWithProjection<T extends Doc> (doc: DBDoc, projection: P
 export function parseDoc<T extends Doc> (doc: DBDoc): T {
   const { workspaceId, data, ...rest } = doc
   for (const key in rest) {
-    if ((rest as any)[key] === 'NULL') {
+    if ((rest as any)[key] === 'NULL' || (rest as any)[key] === null) {
       if (key === 'attachedTo') {
         // eslint-disable-next-line @typescript-eslint/no-dynamic-delete
         delete rest[key]

server/ws/src/client.ts

@@ -17,7 +17,6 @@ import core, {
   AccountRole,
   TxFactory,
   TxProcessor,
-  reduceCalls,
   type Account,
   type Branding,
   type Class,
@@ -55,8 +54,6 @@ export class ClientSession implements Session {
   sessionId = ''
   lastRequest = Date.now()
 
-  broadcastTx: Tx[] = []
-
   total: StatisticsElement = { find: 0, tx: 0 }
   current: StatisticsElement = { find: 0, tx: 0 }
   mins5: StatisticsElement = { find: 0, tx: 0 }
@@ -92,22 +89,7 @@ export class ClientSession implements Session {
   }
 
   async loadModel (ctx: ClientSessionCtx, lastModelTx: Timestamp, hash?: string): Promise<void> {
-    const contextData = new SessionDataImpl(
+    this.includeSessionContext(ctx.ctx)
-      this.token.email,
-      this.sessionId,
-      this.token.extra?.admin === 'true',
-      {
-        txes: [],
-        targets: {}
-      },
-      this.workspaceId,
-      this.branding,
-      false,
-      new Map(),
-      new Map(),
-      this._pipeline.context.modelDb
-    )
-    ctx.ctx.contextData = contextData
     const result = await ctx.ctx.with('load-model', {}, () => this._pipeline.loadModel(ctx.ctx, lastModelTx, hash))
     await ctx.sendResponse(result)
   }
@@ -129,22 +111,7 @@ export class ClientSession implements Session {
           },
           this.token.email as Ref<Account>
         )
-        const contextData = new SessionDataImpl(
+        this.includeSessionContext(ctx.ctx)
-          this.token.email,
-          this.sessionId,
-          this.token.extra?.admin === 'true',
-          {
-            txes: [],
-            targets: {}
-          },
-          this.workspaceId,
-          this.branding,
-          false,
-          new Map(),
-          new Map(),
-          this._pipeline.context.modelDb
-        )
-        ctx.ctx.contextData = contextData
         await this._pipeline.tx(ctx.ctx, [createTx])
         const acc = TxProcessor.createDoc2Doc(createTx)
         await ctx.sendResponse(acc)
@@ -157,15 +124,7 @@ export class ClientSession implements Session {
     await ctx.sendResponse(account)
   }
 
-  findAllRaw<T extends Doc>(
+  includeSessionContext (ctx: MeasureContext): void {
-    ctx: MeasureContext,
-    _class: Ref<Class<T>>,
-    query: DocumentQuery<T>,
-    options?: FindOptions<T>
-  ): Promise<FindResult<T>> {
-    this.lastRequest = Date.now()
-    this.total.find++
-    this.current.find++
     const contextData = new SessionDataImpl(
       this.token.email,
       this.sessionId,
@@ -182,6 +141,18 @@ export class ClientSession implements Session {
       this._pipeline.context.modelDb
     )
     ctx.contextData = contextData
+  }
+
+  findAllRaw<T extends Doc>(
+    ctx: MeasureContext,
+    _class: Ref<Class<T>>,
+    query: DocumentQuery<T>,
+    options?: FindOptions<T>
+  ): Promise<FindResult<T>> {
+    this.lastRequest = Date.now()
+    this.total.find++
+    this.current.find++
+    this.includeSessionContext(ctx)
     return this._pipeline.findAll(ctx, _class, query, options)
   }
 
@@ -196,22 +167,7 @@ export class ClientSession implements Session {
 
   async searchFulltext (ctx: ClientSessionCtx, query: SearchQuery, options: SearchOptions): Promise<void> {
     this.lastRequest = Date.now()
-    const contextData = new SessionDataImpl(
+    this.includeSessionContext(ctx.ctx)
-      this.token.email,
-      this.sessionId,
-      this.token.extra?.admin === 'true',
-      {
-        txes: [],
-        targets: {}
-      },
-      this.workspaceId,
-      this.branding,
-      false,
-      new Map(),
-      new Map(),
-      this._pipeline.context.modelDb
-    )
-    ctx.ctx.contextData = contextData
     await ctx.sendResponse(await this._pipeline.searchFulltext(ctx.ctx, query, options))
   }
 
@@ -219,22 +175,7 @@ export class ClientSession implements Session {
     this.lastRequest = Date.now()
     this.total.tx++
     this.current.tx++
-    const contextData = new SessionDataImpl(
+    this.includeSessionContext(ctx.ctx)
-      this.token.email,
-      this.sessionId,
-      this.token.extra?.admin === 'true',
-      {
-        txes: [],
-        targets: {}
-      },
-      this.workspaceId,
-      this.branding,
-      false,
-      new Map(),
-      new Map(),
-      this._pipeline.context.modelDb
-    )
-    ctx.ctx.contextData = contextData
 
     const result = await this._pipeline.tx(ctx.ctx, [tx])
 
@@ -245,10 +186,10 @@ export class ClientSession implements Session {
     await this._pipeline.handleBroadcast(ctx.ctx)
   }
 
-  doBroadcast = reduceCalls(async (ctx: MeasureContext, socket: ConnectionSocket) => {
+  broadcast (ctx: MeasureContext, socket: ConnectionSocket, tx: Tx[]): void {
-    if (this.broadcastTx.length > 10000) {
+    if (this.tx.length > 10000) {
       const classes = new Set<Ref<Class<Doc>>>()
-      for (const dtx of this.broadcastTx) {
+      for (const dtx of tx) {
         if (TxProcessor.isExtendsCUD(dtx._class)) {
           classes.add((dtx as TxCUD<Doc>).objectClass)
         }
@@ -258,7 +199,6 @@ export class ClientSession implements Session {
         }
       }
       const bevent = createBroadcastEvent(Array.from(classes))
-      this.broadcastTx = []
       socket.send(
         ctx,
         {
@@ -268,21 +208,7 @@ export class ClientSession implements Session {
         this.useCompression
       )
     } else {
-      const txes = [...this.broadcastTx]
+      void handleSend(ctx, socket, { result: tx }, 1024 * 1024, this.binaryMode, this.useCompression)
-      this.broadcastTx = []
-      await handleSend(ctx, socket, { result: txes }, 32 * 1024, this.binaryMode, this.useCompression)
     }
-  })
-
-  timeout: any
-
-  broadcast (ctx: MeasureContext, socket: ConnectionSocket, tx: Tx[]): void {
-    this.broadcastTx.push(...tx)
-    // We need to put into client broadcast queue, to send user requests first
-    // Collapse events in 1 second interval
-    clearTimeout(this.timeout)
-    this.timeout = setTimeout(() => {
-      void this.doBroadcast(ctx, socket)
-    }, 1)
   }
 }

server/ws/src/server_http.ts

@@ -206,7 +206,11 @@ export function startHttpServer (
       const contentType = req.query.contentType as string
       const size = parseInt((req.query.size as string) ?? '-1')
       if (Number.isNaN(size)) {
-        ctx.error('/api/v1/blob put error', { message: 'invalid NaN file size' })
+        ctx.error('/api/v1/blob put error', {
+          message: 'invalid NaN file size',
+          name,
+          workspace: payload.workspace.name
+        })
         res.writeHead(404, {})
         res.end()
         return

server/ws/src/types.ts

@@ -67,8 +67,6 @@ export interface Session {
 
   requests: Map<string, SessionRequest>
 
-  broadcastTx: Tx[]
-
   binaryMode: boolean
   useCompression: boolean
   total: StatisticsElement

services/github/pod-github/Dockerfile

@@ -15,4 +15,4 @@ COPY bundle/bundle.js ./
 COPY bundle/bundle.js.map ./
 
 EXPOSE 3078
-CMD [ "node", "--inspect", "--async-stack-traces", "--enable-source-maps", "bundle.js" ]
+CMD [ "node", "bundle.js" ]

services/github/pod-github/src/sync/comments.ts

@@ -12,8 +12,8 @@ import core, {
   Ref,
   TxOperations
 } from '@hcengineering/core'
-import { LiveQuery } from '@hcengineering/query'
 import github, { DocSyncInfo, GithubIntegrationRepository, GithubProject } from '@hcengineering/github'
+import { LiveQuery } from '@hcengineering/query'
 import { deepEqual } from 'fast-equals'
 import {
   ContainerFocus,
@@ -252,7 +252,7 @@ export class CommentSyncManager implements DocSyncManager {
     }
     if (info.external === undefined) {
       // TODO: Use selected repository
-      const repo = container.repository.find((it) => it._id === parent?.repository)
+      const repo = await this.provider.getRepositoryById(parent?.repository)
       if (repo?.nodeId === undefined) {
         // No need to sync if parent repository is not defined.
         return { needSync: githubSyncVersion }
@@ -303,7 +303,7 @@ export class CommentSyncManager implements DocSyncManager {
     comment: CommentExternalData,
     account: Ref<Account>
   ): Promise<void> {
-    const repository = container.repository.find((it) => it._id === info.repository)
+    const repository = await this.provider.getRepositoryById(info.repository)
     if (repository === undefined) {
       return
     }
@@ -384,7 +384,7 @@ export class CommentSyncManager implements DocSyncManager {
     derivedClient: TxOperations
   ): Promise<DocumentUpdate<DocSyncInfo>> {
     // TODO: Use selected repository
-    const repo = container.repository.find((it) => it._id === parent?.repository)
+    const repo = await this.provider.getRepositoryById(parent?.repository)
     if (repo?.nodeId === undefined) {
       // No need to sync if parent repository is not defined.
       return { needSync: githubSyncVersion }

services/github/pod-github/src/sync/issueBase.ts

@@ -302,8 +302,7 @@ export abstract class IssueSyncManagerBase {
                 const ff = await this.toPlatformField(
                   {
                     container: integration,
-                    project: prj,
+                    project: prj
-                    repository: repositories.filter((it) => it.githubProject === prj._id)
                   },
                   f,
                   target,
@@ -949,17 +948,19 @@ export abstract class IssueSyncManagerBase {
     existing: WithMarkup<Issue>,
     issueExternal: IssueExternalData
   ): Promise<void> {
-    const repo = container.repository.find((it) => it._id === info.repository) as GithubIntegrationRepository
+    const repo = await this.provider.getRepositoryById(info.repository)
-    await this.addConnectToMessage(
+    if (repo != null) {
-      existing._class === github.class.GithubPullRequest
+      await this.addConnectToMessage(
-        ? github.string.PullRequestConnectedActivityInfo
+        existing._class === github.class.GithubPullRequest
-        : github.string.IssueConnectedActivityInfo,
+          ? github.string.PullRequestConnectedActivityInfo
-      existing.space,
+          : github.string.IssueConnectedActivityInfo,
-      existing._id,
+        existing.space,
-      existing._class,
+        existing._id,
-      issueExternal,
+        existing._class,
-      repo
+        issueExternal,
-    )
+        repo
+      )
+    }
   }
 
   async collectIssueUpdate (
@@ -1125,7 +1126,7 @@ export abstract class IssueSyncManagerBase {
     container: IntegrationContainer,
     existingIssue: Issue | undefined,
     external: IssueExternalData
-  ): Promise<IssueSyncTarget | undefined | null> {
+  ): Promise<IssueSyncTarget | undefined> {
     if (existingIssue !== undefined) {
       // Select a milestone project
       if (existingIssue.milestone != null) {
@@ -1133,14 +1134,7 @@ export abstract class IssueSyncManagerBase {
           await this.provider.liveQuery.queryFind<GithubMilestone>(github.mixin.GithubMilestone, {})
         ).find((it) => it._id === existingIssue.milestone)
         if (milestone === undefined) {
-          // Let's search for milestone, and if it doesn't have mixin, return undefined.
+          return
-          const mstone = await this.client.findOne(github.mixin.GithubMilestone, {
-            _id: existingIssue.milestone as Ref<GithubMilestone>
-          })
-          if (mstone === undefined) {
-            return undefined
-          }
-          return null
         }
         return {
           project,

services/github/pod-github/src/sync/issues.ts

@@ -345,7 +345,9 @@ export class IssueSyncManager extends IssueSyncManagerBase implements DocSyncMan
         }
         if (info.repository == null) {
           // No need to sync if component it not yet set
-          const repos = container.repository.map((it) => it.name).join(', ')
+          const repos = (await this.provider.getProjectRepositories(container.project._id))
+            .map((it) => it.name)
+            .join(', ')
           this.ctx.error('Not syncing repository === null', {
             url: info.url,
             identifier: (existing as Issue).identifier,
@@ -360,9 +362,11 @@ export class IssueSyncManager extends IssueSyncManagerBase implements DocSyncMan
 
     let issueExternal = info.external as IssueExternalData
     if (info.external === undefined && existing !== undefined) {
-      const repository = container.repository.find((it) => it._id === info.repository)
+      const repository = await this.provider.getRepositoryById(info.repository)
       if (repository === undefined) {
-        const repos = container.repository.map((it) => it.name).join(', ')
+        const repos = (await this.provider.getProjectRepositories(container.project._id))
+          .map((it) => it.name)
+          .join(', ')
         this.ctx.error('Not syncing repository === undefined', {
           url: info.url,
           identifier: (existing as Issue).identifier,
@@ -427,11 +431,6 @@ export class IssueSyncManager extends IssueSyncManagerBase implements DocSyncMan
       existing as Issue,
       issueExternal
     )
-    if (target === null) {
-      // We need to wait, no milestone data yet.
-      this.ctx.error('target === null, no milestone data yet', { url: info.url })
-      return { needSync: githubSyncVersion }
-    }
     if (target === undefined) {
       target = this.getProjectIssueTarget(container.project, issueExternal)
     }
@@ -573,6 +572,11 @@ export class IssueSyncManager extends IssueSyncManagerBase implements DocSyncMan
           container.container,
           issueExternal.body
         )
+        const repo = await this.provider.getRepositoryById(info.repository)
+        if (repo == null) {
+          // No repository, it probable deleted
+          return { needSync: githubSyncVersion }
+        }
         await this.ctx.withLog(
           'create platform issue',
           {},
@@ -590,7 +594,7 @@ export class IssueSyncManager extends IssueSyncManagerBase implements DocSyncMan
               info.repository as Ref<GithubIntegrationRepository>,
               container.project,
               taskTypes[0]._id,
-              container.repository.find((it) => it._id === info.repository) as GithubIntegrationRepository & {
+              repo as GithubIntegrationRepository & {
                 repository: IntegrationRepositoryData
               },
               !markdownCompatible

services/github/pod-github/src/sync/projects.ts

@@ -12,11 +12,6 @@ import core, {
   TxOperations,
   generateId
 } from '@hcengineering/core'
-import { getEmbeddedLabel, translate } from '@hcengineering/platform'
-import { LiveQuery } from '@hcengineering/query'
-import task from '@hcengineering/task'
-import tracker, { Milestone } from '@hcengineering/tracker'
-import { RepositoryEvent } from '@octokit/webhooks-types'
 import github, {
   DocSyncInfo,
   GithubFieldMapping,
@@ -25,6 +20,11 @@ import github, {
   GithubProject,
   GithubProjectSyncData
 } from '@hcengineering/github'
+import { getEmbeddedLabel, translate } from '@hcengineering/platform'
+import { LiveQuery } from '@hcengineering/query'
+import task from '@hcengineering/task'
+import tracker, { Milestone } from '@hcengineering/tracker'
+import { RepositoryEvent } from '@octokit/webhooks-types'
 import { deepEqual } from 'fast-equals'
 import { Octokit } from 'octokit'
 import {
@@ -238,6 +238,28 @@ export class ProjectsSyncManager implements DocSyncManager {
           external: data,
           needSync: ''
         })
+
+        // We also need to notify all issues with milestone set to this milestone.
+        const milestonedIds = await this.client.findAll(
+          tracker.class.Issue,
+          { milestone: milestone._id },
+          { projection: { _id: 1 } }
+        )
+        while (milestonedIds.length > 0) {
+          const part = milestonedIds.splice(0, 100)
+          const docInfos = await this.client.findAll(
+            github.class.DocSyncInfo,
+            { _id: { $in: part.map((it) => it._id as unknown as Ref<DocSyncInfo>) } },
+            { projection: { _id: 1 } }
+          )
+          if (docInfos.length > 0) {
+            const ops = derivedClient.apply()
+            for (const d of docInfos) {
+              await ops.update(d, { needSync: '' })
+            }
+            await ops.commit()
+          }
+        }
       }
     }
   }

services/github/pod-github/src/sync/pullrequests.ts

@@ -32,7 +32,6 @@ import github, {
 import task, { TaskType, calcRank, makeRank } from '@hcengineering/task'
 import time, { ToDo, ToDoPriority } from '@hcengineering/time'
 import tracker, { Issue, IssuePriority, IssueStatus, Project } from '@hcengineering/tracker'
-import { OctokitResponse } from '@octokit/types'
 import { ProjectsV2ItemEvent, PullRequestEvent } from '@octokit/webhooks-types'
 import { Octokit } from 'octokit'
 import config from '../config'
@@ -59,7 +58,16 @@ import {
 } from './githubTypes'
 import { GithubIssueData, IssueSyncManagerBase, IssueSyncTarget, WithMarkup } from './issueBase'
 import { syncConfig } from './syncConfig'
-import { errorToObj, getSinceRaw, gqlp, guessStatus, isGHWriteAllowed, syncDerivedDocuments, syncRunner } from './utils'
+import {
+  errorToObj,
+  getSinceRaw,
+  gqlp,
+  guessStatus,
+  isGHWriteAllowed,
+  syncChilds,
+  syncDerivedDocuments,
+  syncRunner
+} from './utils'
 
 type GithubPullRequestData = GithubIssueData &
 Omit<GithubPullRequest, keyof Issue | 'commits' | 'reviews' | 'reviewComments'>
@@ -487,8 +495,8 @@ export class PullRequestSyncManager extends IssueSyncManagerBase implements DocS
         await this.ctx.withLog(
           'retrieve pull request patch',
           {},
-          async () =>
+          () =>
-            await this.handlePatch(
+            this.handlePatch(
               info,
               container,
               pullRequestExternal,
@@ -526,7 +534,7 @@ export class PullRequestSyncManager extends IssueSyncManagerBase implements DocS
               info.repository as Ref<GithubIntegrationRepository>,
               container.project,
               taskTypes[0]._id,
-              container.repository.find((it) => it._id === info.repository) as GithubIntegrationRepository,
+              (await this.provider.getRepositoryById(info.repository)) as GithubIntegrationRepository,
               !markdownCompatible
             )
           },
@@ -544,6 +552,9 @@ export class PullRequestSyncManager extends IssueSyncManagerBase implements DocS
 
         await op.commit()
 
+        // To sync reviews/review threads in case they are created before us.
+        await syncChilds(info, this.client, derivedClient)
+
         return {
           needSync: '',
           external: pullRequestExternal,
@@ -563,7 +574,7 @@ export class PullRequestSyncManager extends IssueSyncManagerBase implements DocS
           await this.ctx.withLog(
             'update pull request patch',
             {},
-            async () =>
+            async () => {
               await this.handlePatch(
                 info,
                 container,
@@ -575,7 +586,8 @@ export class PullRequestSyncManager extends IssueSyncManagerBase implements DocS
                 },
                 lastModified,
                 accountGH
-              ),
+              )
+            },
             { url: pullRequestExternal.url }
           )
         }
@@ -966,10 +978,6 @@ export class PullRequestSyncManager extends IssueSyncManagerBase implements DocS
       existing as Issue,
       pullRequestExternal
     )
-    if (target === null) {
-      // We need to wait, no milestone data yet.
-      return { needSync: '' }
-    }
     if (target === undefined) {
       target = this.getProjectIssueTarget(container.project, pullRequestExternal)
     }
@@ -1085,18 +1093,17 @@ export class PullRequestSyncManager extends IssueSyncManagerBase implements DocS
     existingPR: Pick<GithubPullRequest, '_id' | 'space' | '_class'>,
     lastModified: number,
     account: Ref<Account>
-  ): Promise<string | null> {
+  ): Promise<void> {
-    let patch: string | null = null
+    const repo = await this.provider.getRepositoryById(info.repository)
-    const repo = container.repository.find((it) => it._id === info.repository)
     if (repo?.nodeId === undefined) {
-      return null
+      return
     }
     if (info.external?.patch !== true) {
-      patch = await this.fetchPatch(pullRequestExternal, container.container.octokit, repo)
+      const { patch, contentType } = await this.fetchPatch(pullRequestExternal, container.container.octokit, repo)
 
       // Update attached patch data.
       const patchAttachment = await this.client.findOne(github.class.GithubPatch, { attachedTo: existingPR._id })
-      const blob = await this.provider.uploadFile(patch, patchAttachment?.file)
+      const blob = await this.provider.uploadFile(patch, patchAttachment?.file, contentType)
       if (blob !== undefined) {
         if (patchAttachment === undefined) {
           await this.client.addCollection(
@@ -1131,7 +1138,6 @@ export class PullRequestSyncManager extends IssueSyncManagerBase implements DocS
         }
       }
     }
-    return patch
   }
 
   private async createPullRequest (
@@ -1544,8 +1550,9 @@ export class PullRequestSyncManager extends IssueSyncManagerBase implements DocS
     pullRequest: PullRequestExternalData,
     octokit: Octokit,
     repository: GithubIntegrationRepository
-  ): Promise<string> {
+  ): Promise<{ patch: string, contentType: string }> {
     let patch = ''
+    let contentType = 'application/vnd.github.VERSION.diff'
     try {
       const patchContent = await octokit.rest.pulls.get({
         owner: repository.owner?.login as string,
@@ -1556,12 +1563,13 @@ export class PullRequestSyncManager extends IssueSyncManagerBase implements DocS
           'X-GitHub-Api-Version': '2022-11-28'
         }
       })
-      patch = ((patchContent as unknown as OctokitResponse<string>).data ?? '').slice(0, 2 * 1024 * 1024)
+      patch = (patchContent.data as unknown as string) ?? ''
+      contentType = patchContent.headers['content-type'] ?? 'application/vnd.github.VERSION.diff'
     } catch (err: any) {
       this.ctx.error('Error', { err })
       Analytics.handleError(err)
     }
-    return patch
+    return { patch, contentType }
   }
 
   async deleteGithubDocument (container: ContainerFocus, account: Ref<Account>, id: string): Promise<void> {

services/github/pod-github/src/sync/reviewComments.ts

@@ -310,11 +310,11 @@ export class ReviewCommentSyncManager implements DocSyncManager {
       return {}
     }
     if (parent === undefined) {
-      return { needSync: '' }
+      return { needSync: githubSyncVersion }
     }
     if (info.external === undefined) {
       // TODO: Use selected repository
-      const repo = container.repository.find((it) => it._id === parent?.repository)
+      const repo = await this.provider.getRepositoryById(parent?.repository)
       if (repo?.nodeId === undefined) {
         // No need to sync if parent repository is not defined.
         return { needSync: githubSyncVersion }
@@ -381,7 +381,7 @@ export class ReviewCommentSyncManager implements DocSyncManager {
     account: Ref<Account>,
     derivedClient: TxOperations
   ): Promise<void> {
-    const repository = container.repository.find((it) => it._id === info.repository)
+    const repository = await this.provider.getRepositoryById(info.repository)
     if (repository === undefined) {
       return
     }
@@ -473,7 +473,7 @@ export class ReviewCommentSyncManager implements DocSyncManager {
     derivedClient: TxOperations
   ): Promise<DocumentUpdate<DocSyncInfo>> {
     // TODO: Use selected repository
-    const repo = container.repository.find((it) => it._id === parent?.repository)
+    const repo = await this.provider.getRepositoryById(parent?.repository)
     if (repo?.nodeId === undefined) {
       // No need to sync if parent repository is not defined.
       return { needSync: githubSyncVersion }

services/github/pod-github/src/sync/reviewThreads.ts

@@ -11,14 +11,14 @@ import core, {
   Ref,
   TxOperations
 } from '@hcengineering/core'
-import { EmptyMarkup } from '@hcengineering/text'
-import { LiveQuery } from '@hcengineering/query'
 import github, {
   DocSyncInfo,
   GithubIntegrationRepository,
   GithubProject,
   GithubReviewThread
 } from '@hcengineering/github'
+import { LiveQuery } from '@hcengineering/query'
+import { EmptyMarkup } from '@hcengineering/text'
 import {
   ContainerFocus,
   DocSyncManager,
@@ -35,7 +35,7 @@ import {
   getUpdatedAtReviewThread,
   reviewThreadDetails
 } from './githubTypes'
-import { collectUpdate, deleteObjects, errorToObj, isGHWriteAllowed, syncDerivedDocuments } from './utils'
+import { collectUpdate, deleteObjects, errorToObj, isGHWriteAllowed, syncChilds, syncDerivedDocuments } from './utils'
 
 import { Analytics } from '@hcengineering/analytics'
 import { PullRequestReviewThreadEvent } from '@octokit/webhooks-types'
@@ -266,11 +266,11 @@ export class ReviewThreadSyncManager implements DocSyncManager {
       return {}
     }
     if (parent === undefined) {
-      return { needSync: '' }
+      return { needSync: githubSyncVersion }
     }
     if (info.external === undefined) {
       // TODO: Use selected repository
-      const repo = container.repository.find((it) => it._id === parent?.repository)
+      const repo = await this.provider.getRepositoryById(parent?.repository)
       if (repo?.nodeId === undefined) {
         // No need to sync if parent repository is not defined.
         return { needSync: githubSyncVersion }
@@ -305,6 +305,9 @@ export class ReviewThreadSyncManager implements DocSyncManager {
     if (existing === undefined) {
       try {
         await this.createReviewThread(info, messageData, parent, review, account)
+
+        // We need trigger comments, if their sync data created before
+        await syncChilds(info, this.client, derivedClient)
         return { needSync: githubSyncVersion, current: messageData }
       } catch (err: any) {
         this.ctx.error('Error', { err })
@@ -327,7 +330,7 @@ export class ReviewThreadSyncManager implements DocSyncManager {
     account: Ref<Account>,
     derivedClient: TxOperations
   ): Promise<void> {
-    const repository = container.repository.find((it) => it._id === info.repository)
+    const repository = await this.provider.getRepositoryById(info.repository)
     if (repository === undefined) {
       return
     }
@@ -422,7 +425,7 @@ export class ReviewThreadSyncManager implements DocSyncManager {
     derivedClient: TxOperations
   ): Promise<DocumentUpdate<DocSyncInfo>> {
     // TODO: Use selected repository
-    const repo = container.repository.find((it) => it._id === parent?.repository)
+    const repo = await this.provider.getRepositoryById(parent?.repository)
     if (repo?.nodeId === undefined) {
       // No need to sync if parent repository is not defined.
       return { needSync: githubSyncVersion }

services/github/pod-github/src/sync/reviews.ts

@@ -11,7 +11,6 @@ import core, {
   Ref,
   TxOperations
 } from '@hcengineering/core'
-import { LiveQuery } from '@hcengineering/query'
 import github, {
   DocSyncInfo,
   GithubIntegrationRepository,
@@ -19,6 +18,7 @@ import github, {
   GithubPullRequestReviewState,
   GithubReview
 } from '@hcengineering/github'
+import { LiveQuery } from '@hcengineering/query'
 import {
   ContainerFocus,
   DocSyncManager,
@@ -29,7 +29,7 @@ import {
   githubSyncVersion
 } from '../types'
 import { PullRequestExternalData, Review as ReviewExternalData, reviewDetails, toReviewState } from './githubTypes'
-import { collectUpdate, deleteObjects, errorToObj, isGHWriteAllowed } from './utils'
+import { collectUpdate, deleteObjects, errorToObj, isGHWriteAllowed, syncChilds } from './utils'
 
 import { Analytics } from '@hcengineering/analytics'
 import { PullRequestReviewEvent, PullRequestReviewSubmittedEvent } from '@octokit/webhooks-types'
@@ -284,11 +284,11 @@ export class ReviewSyncManager implements DocSyncManager {
       return {}
     }
     if (parent === undefined) {
-      return { needSync: '' }
+      return { needSync: githubSyncVersion }
     }
     if (info.external === undefined) {
       // TODO: Use selected repository
-      const repo = container.repository.find((it) => it._id === parent?.repository)
+      const repo = await this.provider.getRepositoryById(parent?.repository)
       if (repo?.nodeId === undefined) {
         // No need to sync if parent repository is not defined.
         return { needSync: githubSyncVersion }
@@ -310,6 +310,8 @@ export class ReviewSyncManager implements DocSyncManager {
     if (existing === undefined) {
       try {
         await this.createReview(info, messageData, parent, review, account)
+
+        await syncChilds(info, this.client, derivedClient)
         return { needSync: githubSyncVersion, current: messageData }
       } catch (err: any) {
         this.ctx.error('Error', { err })
@@ -331,7 +333,7 @@ export class ReviewSyncManager implements DocSyncManager {
     review: ReviewExternalData,
     account: Ref<Account>
   ): Promise<void> {
-    const repository = container.repository.find((it) => it._id === info.repository)
+    const repository = await this.provider.getRepositoryById(info.repository)
     if (repository === undefined) {
       return
     }
@@ -400,7 +402,7 @@ export class ReviewSyncManager implements DocSyncManager {
     derivedClient: TxOperations
   ): Promise<DocumentUpdate<DocSyncInfo>> {
     // TODO: Use selected repository
-    const repo = container.repository.find((it) => it._id === parent?.repository)
+    const repo = await this.provider.getRepositoryById(parent?.repository)
     if (repo?.nodeId === undefined) {
       // No need to sync if parent repository is not defined.
       return { needSync: githubSyncVersion }

services/github/pod-github/src/sync/utils.ts

@@ -16,15 +16,15 @@ import core, {
   Type,
   toIdMap
 } from '@hcengineering/core'
-import { PlatformError, unknownStatus } from '@hcengineering/platform'
-import task, { TaskType, calculateStatuses, createState, findStatusAttr } from '@hcengineering/task'
-import tracker, { IssueStatus } from '@hcengineering/tracker'
 import github, {
   DocSyncInfo,
   GithubIntegrationRepository,
   GithubIssueStateReason,
   GithubProject
 } from '@hcengineering/github'
+import { PlatformError, unknownStatus } from '@hcengineering/platform'
+import task, { TaskType, calculateStatuses, createState, findStatusAttr } from '@hcengineering/task'
+import tracker, { IssueStatus } from '@hcengineering/tracker'
 import { deepEqual } from 'fast-equals'
 import { IntegrationManager, githubExternalSyncVersion } from '../types'
 import { GithubDataType } from './githubTypes'
@@ -411,3 +411,14 @@ export function compareMarkdown (a: string, b: string): boolean {
 
   return na === nb
 }
+
+export async function syncChilds (info: DocSyncInfo, client: TxOperations, derivedClient: TxOperations): Promise<void> {
+  const childInfos = await client.findAll(github.class.DocSyncInfo, { parent: info.url.toLowerCase() })
+  if (childInfos.length > 0) {
+    const ops = derivedClient.apply()
+    for (const child of childInfos) {
+      await ops?.update(child, { needSync: '' })
+    }
+    await ops.commit()
+  }
+}

services/github/pod-github/src/types.ts

@@ -77,7 +77,6 @@ export type UserInfo = Data<GithubUserInfo>
 
 export interface ContainerFocus {
   container: IntegrationContainer
-  repository: GithubIntegrationRepository[]
   project: GithubProject
 }
 
@@ -96,7 +95,7 @@ export interface IntegrationManager {
   sync: () => void
   getGithubLogin: (container: IntegrationContainer, account: Ref<Person>) => Promise<UserInfo | undefined>
 
-  uploadFile: (patch: string, file?: string) => Promise<Blob | undefined>
+  uploadFile: (patch: string, file?: string, contentType?: string) => Promise<Blob | undefined>
 
   getStatuses: (type: Ref<TaskType> | undefined) => Promise<Status[]>
   getProjectStatuses: (type: Ref<ProjectType> | undefined) => Promise<Status[]>
@@ -126,6 +125,10 @@ export interface IntegrationManager {
   ) => Promise<{ markdownCompatible: boolean, markdown: string }>
 
   isPlatformUser: (account: Ref<PersonAccount>) => Promise<boolean>
+
+  getProjectRepositories: (space: Ref<Space>) => Promise<GithubIntegrationRepository[]>
+
+  getRepositoryById: (ref?: Ref<GithubIntegrationRepository> | null) => Promise<GithubIntegrationRepository | undefined>
 }
 
 export type ExternalSyncField = 'externalVersion' | 'derivedVersion'

services/github/pod-github/src/worker.ts

@@ -186,29 +186,45 @@ export class GithubWorker implements IntegrationManager {
   }
 
   async getContainer (space: Ref<Space>): Promise<ContainerFocus | undefined> {
-    for (const v of this.integrations.values()) {
+    const project = (
-      if (v.octokit === undefined) {
+      await this.liveQuery.queryFind<GithubProject>(github.mixin.GithubProject, {
-        continue
+        _id: space as Ref<GithubProject>
-      }
+      })
-      const project = (
+    ).shift()
-        await this.liveQuery.queryFind<GithubProject>(github.mixin.GithubProject, {
+    if (project !== undefined) {
-          _id: space as Ref<GithubProject>
+      for (const v of this.integrations.values()) {
-        })
+        if (v.octokit === undefined) {
-      ).shift()
+          continue
-      if (project !== undefined) {
+        }
-        const repositories = await this.liveQuery.queryFind<GithubIntegrationRepository>(
+        if (project.integration !== v.integration._id) {
-          github.class.GithubIntegrationRepository,
+          continue
-          {}
+        }
-        )
         return {
           container: v,
-          repository: repositories.filter((it) => it.githubProject === space),
           project
         }
       }
     }
   }
 
+  async getProjectRepositories (space: Ref<Space>): Promise<GithubIntegrationRepository[]> {
+    const repositories = await this.liveQuery.queryFind<GithubIntegrationRepository>(
+      github.class.GithubIntegrationRepository,
+      {}
+    )
+    return repositories.filter((it) => it.githubProject === space)
+  }
+
+  async getRepositoryById (
+    _id?: Ref<GithubIntegrationRepository> | null
+  ): Promise<GithubIntegrationRepository | undefined> {
+    if (_id != null) {
+      return (
+        await this.liveQuery.queryFind<GithubIntegrationRepository>(github.class.GithubIntegrationRepository, { _id })
+      ).shift()
+    }
+  }
+
   async getAccountU (user: User): Promise<PersonAccount | undefined> {
     return await this.getAccount({
       id: user.node_id,
@@ -584,9 +600,9 @@ export class GithubWorker implements IntegrationManager {
     return record !== undefined && accountRef !== undefined
   }
 
-  async uploadFile (patch: string, file?: string): Promise<Blob | undefined> {
+  async uploadFile (patch: string, file?: string, contentType?: string): Promise<Blob | undefined> {
     const id: string = file ?? generateId()
-    await this.storageAdapter.put(this.ctx, this.workspace, id, patch, 'text/x-patch', patch.length)
+    await this.storageAdapter.put(this.ctx, this.workspace, id, patch, contentType ?? 'text/x-patch')
     return await this.storageAdapter.stat(this.ctx, this.workspace, id)
   }
 
@@ -1101,17 +1117,6 @@ export class GithubWorker implements IntegrationManager {
     const _projects = projects.map((it) => it._id)
     const _repositories = repositories.map((it) => it._id)
 
-    const h = this.client.getHierarchy()
-    const sortCases = this.mappers
-      .map((it) => it._class)
-      .flat()
-      .map((it) => h.getDescendants(it))
-      .flat()
-      .map((it, idx) => ({
-        query: it,
-        index: idx
-      }))
-
     const docs = await this.ctx.with(
       'find-doc-sync-info',
       {},
@@ -1125,13 +1130,7 @@ export class GithubWorker implements IntegrationManager {
             repository: { $in: [null, ..._repositories] }
           },
           {
-            limit: 50,
+            limit: 50
-            sort: {
-              objectClass: {
-                order: SortingOrder.Ascending,
-                cases: sortCases
-              }
-            }
           }
         ),
       { _projects, _repositories }
@@ -1261,8 +1260,7 @@ export class GithubWorker implements IntegrationManager {
           })
           continue
         }
-        const container = await this.getContainer(info.space)
+        const repo = await this.getRepositoryById(info.repository)
-        const repo = container?.repository.find((it) => it._id === info.repository)
         if (repo !== undefined && !repo.enabled) {
           continue
         }