UBERF-4604 Load ydoc from minio (#4338)

Signed-off-by: Alexander Onnikov <alexander.onnikov@xored.com>

common/config/rush/pnpm-lock.yaml

@@ -998,6 +998,9 @@ dependencies:
   lexorank:
     specifier: ~1.0.4
     version: 1.0.5
+  lib0:
+    specifier: ^0.2.88
+    version: 0.2.88
   libphonenumber-js:
     specifier: ^1.9.46
     version: 1.10.47
@@ -12802,14 +12805,6 @@ packages:
     resolution: {integrity: sha512-K1B/Yr/gIU0wm68hk/yB0p/mv6xM3ShD5aci42vOwcjof8slG8Kpo3Q7+1WTv7DaRHKWRgLPqrFDt+4GtuFAtA==}
     dev: false
 
-  /lib0@0.2.86:
-    resolution: {integrity: sha512-kxigQTM4Q7NwJkEgdqQvU21qiR37twcqqLmh+/SbiGbRLfPlLVbHyY9sWp7PwXh0Xus9ELDSjsUOwcrdt5yZ4w==}
-    engines: {node: '>=16'}
-    hasBin: true
-    dependencies:
-      isomorphic.js: 0.2.5
-    dev: false
-
   /lib0@0.2.88:
     resolution: {integrity: sha512-KyroiEvCeZcZEMx5Ys+b4u4eEBbA1ch7XUaBhYpwa/nPMrzTjUhI4RfcytmQfYoTBPcdyx+FX6WFNIoNuJzJfQ==}
     engines: {node: '>=16'}
@@ -17241,7 +17236,7 @@ packages:
       y-protocols: ^1.0.1
       yjs: ^13.5.38
     dependencies:
-      lib0: 0.2.86
+      lib0: 0.2.88
       prosemirror-model: 1.19.3
       yjs: 13.6.8
     dev: false
@@ -17262,7 +17257,7 @@ packages:
     peerDependencies:
       yjs: ^13.5.6
     dependencies:
-      lib0: 0.2.86
+      lib0: 0.2.88
       lodash.debounce: 4.0.8
       y-protocols: 1.0.6(yjs@13.6.8)
       yjs: 13.6.8
@@ -17327,7 +17322,7 @@ packages:
     resolution: {integrity: sha512-ZPq0hpJQb6f59B++Ngg4cKexDJTvfOgeiv0sBc4sUm8CaBWH7OQC4kcCgrqbjJ/B2+6vO49exvTmYfdlPtcjbg==}
     engines: {node: '>=16.0.0', npm: '>=8.0.0'}
     dependencies:
-      lib0: 0.2.86
+      lib0: 0.2.88
     dev: false
 
   /ylru@1.3.2:
@@ -23653,7 +23648,7 @@ packages:
     dev: false
 
   file:projects/text-editor.tgz(@types/node@16.11.68)(bufferutil@4.0.7)(esbuild@0.16.17)(postcss-load-config@4.0.1)(postcss@8.4.31)(prosemirror-model@1.19.3)(ts-node@10.9.1):
-    resolution: {integrity: sha512-QgUDUW3v5zIVHETcfC/UsoZ8EPeauNcSyGO+I/J6rXLG2ndtNKIWRdl7Herj480/Q6JjPjWmm5RTIRAnz8lZXQ==, tarball: file:projects/text-editor.tgz}
+    resolution: {integrity: sha512-q4IthAiGXw8l3HkEitwxvXppLyB/mszif5wHdy0jcaz7Yc0moJOzS6ElJ3AgA/0IQHxKB7ladDAkU/wklo7icw==, tarball: file:projects/text-editor.tgz}
     id: file:projects/text-editor.tgz
     name: '@rush-temp/text-editor'
     version: 0.0.0
@@ -23696,6 +23691,7 @@ packages:
       eslint-plugin-promise: 6.1.1(eslint@8.54.0)
       eslint-plugin-svelte: 2.35.0(eslint@8.54.0)(svelte@4.2.5)(ts-node@10.9.1)
       jest: 29.7.0(@types/node@16.11.68)(ts-node@10.9.1)
+      lib0: 0.2.88
       prettier: 3.1.0
       prettier-plugin-svelte: 3.1.0(prettier@3.1.0)(svelte@4.2.5)
       prosemirror-codemark: 0.4.2(prosemirror-model@1.19.3)

dev/docker-compose.yaml

@@ -102,7 +102,7 @@ services:
     environment:
       - COLLABORATOR_PORT=3078
       - SECRET=secret
-      - TRANSACTOR_URL=ws://localhost:3333
+      - TRANSACTOR_URL=ws://transactor:3333
       - UPLOAD_URL=/files
       - MONGO_URL=mongodb://mongodb:27017
       - MINIO_ENDPOINT=minio

packages/text-editor/package.json

@@ -76,6 +76,7 @@
     "y-prosemirror": "^1.2.1",
     "rfc6902": "^5.0.1",
     "diff": "^5.1.0",
-    "slugify": "^1.6.6"
+    "slugify": "^1.6.6",
+    "lib0": "^0.2.88"
   }
 }

packages/text-editor/src/components/Collaboration.svelte

@@ -20,7 +20,7 @@
   import presentation from '@hcengineering/presentation'
 
   import textEditorPlugin from '../plugin'
-  import { DocumentId, TiptapCollabProvider, createTiptapCollaborationData } from '../provider'
+  import { DocumentId, TiptapCollabProvider, createTiptapCollaborationData } from '../provider/tiptap'
   import { CollaborationIds } from '../types'
 
   export let documentId: DocumentId

packages/text-editor/src/components/CollaborativeAttributeBox.svelte

@@ -21,7 +21,7 @@
   import { FocusExtension } from './extension/focus'
   import { FileAttachFunction } from './extension/imageExt'
   import textEditorPlugin from '../plugin'
-  import { minioDocumentId, mongodbDocumentId, platformDocumentId } from '../provider'
+  import { minioDocumentId, mongodbDocumentId, platformDocumentId } from '../provider/utils'
   import { RefAction, TextNodeAction } from '../types'
 
   export let object: Doc

packages/text-editor/src/components/CollaborativeTextEditor.svelte

@@ -29,7 +29,8 @@
   import { textEditorCommandHandler } from '../commands'
   import { EditorKit } from '../kits/editor-kit'
   import textEditorPlugin from '../plugin'
-  import { DocumentId, TiptapCollabProvider } from '../provider'
+  import { MinioProvider } from '../provider/minio'
+  import { DocumentId, TiptapCollabProvider } from '../provider/tiptap'
   import {
     CollaborationIds,
     RefAction,
@@ -90,7 +91,9 @@
   const ydoc = getContext<YDoc>(CollaborationIds.Doc) ?? new YDoc()
   const contextProvider = getContext<TiptapCollabProvider>(CollaborationIds.Provider)
 
-  const provider: TiptapCollabProvider =
+  const localProvider = contextProvider === undefined ? new MinioProvider(documentId, ydoc) : undefined
+
+  const remoteProvider: TiptapCollabProvider =
     contextProvider ??
     new TiptapCollabProvider({
       url: collaboratorURL,
@@ -103,8 +106,14 @@
       }
     })
 
-  let loading = true
-  void provider.loaded.then(() => (loading = false))
+  let localSynced = false
+  let remoteSynced = false
+
+  $: loading = !localSynced && !remoteSynced
+  $: editable = !readonly && remoteSynced
+
+  void localProvider?.loaded.then(() => (localSynced = true))
+  void remoteProvider.loaded.then(() => (remoteSynced = true))
 
   let editor: Editor
   let element: HTMLElement
@@ -142,11 +151,11 @@
   }
 
   export function takeSnapshot (snapshotId: string): void {
-    copyDocumentContent(documentId, snapshotId, { provider }, initialContentId)
+    copyDocumentContent(documentId, snapshotId, { provider: remoteProvider }, initialContentId)
   }
 
   export function copyField (srcFieldId: string, dstFieldId: string): void {
-    copyDocumentField(documentId, srcFieldId, dstFieldId, { provider }, initialContentId)
+    copyDocumentField(documentId, srcFieldId, dstFieldId, { provider: remoteProvider }, initialContentId)
   }
 
   export function isEditable (): boolean {
@@ -179,11 +188,11 @@
   }
 
   $: if (editor !== undefined) {
-    editor.setEditable(!readonly)
+    editor.setEditable(editable, true)
   }
 
   $: showTextStyleToolbar =
-    (!readonly || textFormatCategories.length > 0 || textNodeActions.length > 0) && canShowPopups
+    ((editable && textFormatCategories.length > 0) || textNodeActions.length > 0) && canShowPopups
 
   $: tippyOptions = {
     zIndex: 100000,
@@ -239,7 +248,7 @@
             appendTo: () => boundary ?? element
           },
           shouldShow: ({ editor }) => {
-            if (readonly || !canShowPopups) {
+            if (!editable || !canShowPopups) {
               return false
             }
             return editor.isActive('image')
@@ -250,7 +259,7 @@
           field
         }),
         CollaborationCursor.configure({
-          provider,
+          provider: remoteProvider,
           user,
           render: renderCursor,
           selectionRender: noSelectionRender
@@ -295,10 +304,11 @@
       try {
         editor.destroy()
       } catch (err: any) {}
-      if (contextProvider === undefined) {
-        provider.destroy()
-      }
     }
+    if (contextProvider === undefined) {
+      remoteProvider.destroy()
+    }
+    localProvider?.destroy()
   })
 </script>
 

packages/text-editor/src/components/CollaboratorEditor.svelte

@@ -21,7 +21,7 @@
   import { TextSelection } from '@tiptap/pm/state'
 
   import textEditorPlugin from '../plugin'
-  import { DocumentId } from '../provider'
+  import { DocumentId } from '../provider/tiptap'
   import { TextEditorCommandHandler, TextFormatCategory, TextNodeAction } from '../types'
 
   import CollaborativeTextEditor from './CollaborativeTextEditor.svelte'

packages/text-editor/src/components/node-view/svelte-node-view-renderer.ts

@@ -47,6 +47,8 @@ class SvelteNodeView extends NodeView<SvelteNodeViewComponent, Editor, SvelteNod
 
   contentDOMElement!: HTMLElement | null
 
+  isEditable!: boolean
+
   override mount (): void {
     const props: SvelteNodeViewProps = {
       editor: this.editor,
@@ -64,6 +66,8 @@ class SvelteNodeView extends NodeView<SvelteNodeViewComponent, Editor, SvelteNod
       ...(this.options.componentProps ?? {})
     }
 
+    this.isEditable = this.editor.isEditable
+
     const contentAs = this.options.contentAs ?? (this.node.isInline ? 'span' : 'div')
     const contentClass = this.options.contentClass ?? `node-${this.node.type.name}`
 
@@ -78,6 +82,8 @@ class SvelteNodeView extends NodeView<SvelteNodeViewComponent, Editor, SvelteNod
       }
     })
 
+    this.editor.on('update', this.handleEditorUpdate.bind(this))
+
     this.renderer = new SvelteRenderer(this.component, { element: target, props, context })
   }
 
@@ -126,9 +132,17 @@ class SvelteNodeView extends NodeView<SvelteNodeViewComponent, Editor, SvelteNod
     this.renderer.updateProps({ selected: false })
   }
 
+  handleEditorUpdate (): void {
+    if (this.isEditable !== this.editor.isEditable) {
+      this.isEditable = this.editor.isEditable
+      this.renderer.updateProps({ editor: this.editor })
+    }
+  }
+
   destroy (): void {
     this.renderer.destroy()
     this.contentDOMElement = null
+    this.editor.off('update', this.handleEditorUpdate.bind(this))
   }
 }
 

packages/text-editor/src/index.ts

@@ -71,11 +71,9 @@ export { TodoItemExtension, TodoListExtension } from './components/extension/tod
 export {
   TiptapCollabProvider,
   type TiptapCollabProviderConfiguration,
-  createTiptapCollaborationData,
-  minioDocumentId,
-  mongodbDocumentId,
-  platformDocumentId
-} from './provider'
+  createTiptapCollaborationData
+} from './provider/tiptap'
+export { minioDocumentId, mongodbDocumentId, platformDocumentId } from './provider/utils'
 export { CollaborationIds } from './types'
 
 export { textEditorId }

packages/text-editor/src/provider/minio.ts

@@ -0,0 +1,55 @@
+//
+// Copyright © 2023 Hardcore Engineering Inc.
+//
+// Licensed under the Eclipse Public License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License. You may
+// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+import { getMetadata } from '@hcengineering/platform'
+import presentation from '@hcengineering/presentation'
+import { concatLink } from '@hcengineering/core'
+import { ObservableV2 as Observable } from 'lib0/observable'
+import { type Doc as YDoc, applyUpdate } from 'yjs'
+
+interface EVENTS {
+  synced: (...args: any[]) => void
+}
+
+async function fetchContent (doc: YDoc, name: string): Promise<void> {
+  const frontUrl = getMetadata(presentation.metadata.FrontUrl) ?? window.location.origin
+
+  const res = await fetch(concatLink(frontUrl, `/files?file=${name}`))
+
+  if (res.ok) {
+    const blob = await res.blob()
+    const buffer = await blob.arrayBuffer()
+    applyUpdate(doc, new Uint8Array(buffer))
+  }
+}
+
+export class MinioProvider extends Observable<EVENTS> {
+  loaded: Promise<void>
+
+  constructor (name: string, doc: YDoc) {
+    super()
+
+    if (name.startsWith('minio://')) {
+      name = name.split('://', 2)[1]
+    }
+
+    void fetchContent(doc, name).then(() => {
+      this.emit('synced', [this])
+    })
+
+    this.loaded = new Promise((resolve) => {
+      this.on('synced', resolve)
+    })
+  }
+}

packages/text-editor/src/provider/tiptap.ts

@@ -13,10 +13,10 @@
 // limitations under the License.
 //
 import { Doc as Ydoc } from 'yjs'
-import type { Doc, Ref } from '@hcengineering/core'
-import { type KeyedAttribute, getClient } from '@hcengineering/presentation'
 import { HocuspocusProvider, type HocuspocusProviderConfiguration } from '@hocuspocus/provider'
 
+export type DocumentId = string
+
 export type TiptapCollabProviderConfiguration = HocuspocusProviderConfiguration &
 Required<Pick<HocuspocusProviderConfiguration, 'token'>> &
 Omit<HocuspocusProviderConfiguration, 'parameters'> & {
@@ -28,21 +28,6 @@ export interface TiptapCollabProviderURLParameters {
   targetContentId?: DocumentId
 }
 
-export type DocumentId = string
-
-export function minioDocumentId (docId: Ref<Doc>, attr?: KeyedAttribute): DocumentId {
-  return attr !== undefined ? `minio://${docId}%${attr.key}` : `minio://${docId}`
-}
-
-export function platformDocumentId (docId: Ref<Doc>, attr: KeyedAttribute): DocumentId {
-  return `platform://${attr.attr.attributeOf}/${docId}/${attr.key}`
-}
-
-export function mongodbDocumentId (docId: Ref<Doc>, attr: KeyedAttribute): DocumentId {
-  const domain = getClient().getHierarchy().getDomain(attr.attr.attributeOf)
-  return `mongodb://${domain}/${docId}/${attr.key}`
-}
-
 export class TiptapCollabProvider extends HocuspocusProvider {
   loaded: Promise<void>
 

packages/text-editor/src/provider/utils.ts

@@ -0,0 +1,32 @@
+//
+// Copyright © 2023 Hardcore Engineering Inc.
+//
+// Licensed under the Eclipse Public License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License. You may
+// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+import type { Doc, Ref } from '@hcengineering/core'
+import { type KeyedAttribute, getClient } from '@hcengineering/presentation'
+
+import { type DocumentId } from './tiptap'
+
+export function minioDocumentId (docId: Ref<Doc>, attr?: KeyedAttribute): DocumentId {
+  return attr !== undefined ? `minio://${docId}%${attr.key}` : `minio://${docId}`
+}
+
+export function platformDocumentId (docId: Ref<Doc>, attr: KeyedAttribute): DocumentId {
+  return `platform://${attr.attr.attributeOf}/${docId}/${attr.key}`
+}
+
+export function mongodbDocumentId (docId: Ref<Doc>, attr: KeyedAttribute): DocumentId {
+  const domain = getClient().getHierarchy().getDomain(attr.attr.attributeOf)
+  return `mongodb://${domain}/${docId}/${attr.key}`
+}

packages/text-editor/src/utils.ts

@@ -28,7 +28,7 @@ import {
   themeStore
 } from '@hcengineering/ui'
 
-import { type DocumentId, TiptapCollabProvider } from './provider'
+import { type DocumentId, TiptapCollabProvider } from './provider/tiptap'
 import { type CollaborationUser } from './types'
 
 type ProviderData = (