Merge remote-tracking branch 'origin/develop' into staging

Signed-off-by: Andrey Sobolev <haiodo@gmail.com>

.vscode/launch.json

@@ -85,7 +85,6 @@
         "SERVER_SECRET": "secret",
         "REKONI_URL": "http://localhost:4004",
         "MODEL_JSON": "${workspaceRoot}/models/all/bundle/model.json",
-        "REGION":"pg",
         "ELASTIC_INDEX_NAME": "local_storage_index",
         "STATS_URL":"http://host.docker.internal:4900",
         "ACCOUNTS_URL": "http://localhost:3000",

dev/tool/src/clean.ts

@@ -1152,7 +1152,9 @@ function isPersonAccount (tx: TxCUD<Doc>): boolean {
 }
 
 async function update<T extends Doc> (h: Hierarchy, db: Db, doc: T, update: DocumentUpdate<T>): Promise<void> {
-  await db.collection(h.getDomain(doc._class)).updateOne({ _id: doc._id }, { $set: { ...update, '%hash%': null } })
+  await db
+    .collection(h.getDomain(doc._class))
+    .updateOne({ _id: doc._id }, { $set: { ...update, '%hash%': Date.now().toString(16) } })
 }
 
 async function updateId (
@@ -1173,12 +1175,14 @@ async function updateId (
     const newId = generateId()
 
     // update txes
-    await db.collection(DOMAIN_TX).updateMany({ objectId: doc._id }, { $set: { objectId: newId, '%hash%': null } })
+    await db
+      .collection(DOMAIN_TX)
+      .updateMany({ objectId: doc._id }, { $set: { objectId: newId, '%hash%': Date.now().toString(16) } })
 
     // update nested txes
     await db
       .collection(DOMAIN_TX)
-      .updateMany({ 'tx.objectId': doc._id }, { $set: { 'tx.objectId': newId, '%hash%': null } })
+      .updateMany({ 'tx.objectId': doc._id }, { $set: { 'tx.objectId': newId, '%hash%': Date.now().toString(16) } })
 
     // we have generated ids for calendar, let's update in
     if (h.isDerived(doc._class, core.class.Account)) {
@@ -1232,7 +1236,7 @@ async function updateId (
       await db.collection(domain).insertOne({
         ...raw,
         _id: newId as any,
-        '%hash%': null
+        '%hash%': Date.now().toString(16)
       })
       await db.collection(domain).deleteOne({ _id: doc._id })
     }

models/activity/src/migration.ts

@@ -217,12 +217,6 @@ export const activityOperation: MigrateOperation = {
         state: 'migrate-activity-markup',
         func: migrateActivityMarkup
       },
-      {
-        state: 'fix-rename-backups',
-        func: async (client: MigrationClient): Promise<void> => {
-          await client.update(DOMAIN_ACTIVITY, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } })
-        }
-      },
       {
         state: 'move-reactions',
         func: async (client: MigrationClient): Promise<void> => {

models/attachment/src/migration.ts

@@ -24,12 +24,6 @@ import attachment, { attachmentId, DOMAIN_ATTACHMENT } from '.'
 export const attachmentOperation: MigrateOperation = {
   async migrate (client: MigrationClient): Promise<void> {
     await tryMigrate(client, attachmentId, [
-      {
-        state: 'fix-rename-backups',
-        func: async (client: MigrationClient): Promise<void> => {
-          await client.update(DOMAIN_ATTACHMENT, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } })
-        }
-      },
       {
         state: 'fix-attachedTo',
         func: async (client: MigrationClient): Promise<void> => {

models/chunter/src/migration.ts

@@ -362,12 +362,6 @@ export const chunterOperation: MigrateOperation = {
             'attributeUpdates.attrKey': 'members'
           })
         }
-      },
-      {
-        state: 'fix-rename-backups',
-        func: async (client: MigrationClient): Promise<void> => {
-          await client.update(DOMAIN_CHUNTER, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } })
-        }
       }
     ])
   },

models/contact/src/migration.ts

@@ -26,7 +26,7 @@ import activity, { DOMAIN_ACTIVITY } from '@hcengineering/model-activity'
 import core, { DOMAIN_SPACE } from '@hcengineering/model-core'
 import { DOMAIN_VIEW } from '@hcengineering/model-view'
 
-import contact, { contactId, DOMAIN_CHANNEL, DOMAIN_CONTACT } from './index'
+import contact, { contactId, DOMAIN_CONTACT } from './index'
 
 async function createEmployeeEmail (client: TxOperations): Promise<void> {
   const employees = await client.findAll(contact.mixin.Employee, {})
@@ -300,13 +300,6 @@ export const contactOperation: MigrateOperation = {
       {
         state: 'create-person-spaces-v1',
         func: createPersonSpaces
-      },
-      {
-        state: 'fix-rename-backups',
-        func: async (client: MigrationClient): Promise<void> => {
-          await client.update(DOMAIN_CONTACT, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } })
-          await client.update(DOMAIN_CHANNEL, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } })
-        }
       }
     ])
   },

models/core/src/migration.ts

@@ -424,10 +424,12 @@ export const coreOperation: MigrateOperation = {
         func: migrateCollaborativeContentToStorage
       },
       {
-        state: 'fix-rename-backups',
+        state: 'fix-backups-hash-timestamp',
         func: async (client: MigrationClient): Promise<void> => {
-          await client.update(DOMAIN_TX, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } })
+          const now = Date.now().toString(16)
-          await client.update(DOMAIN_SPACE, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } })
+          for (const d of client.hierarchy.domains()) {
+            await client.update(d, { '%hash%': { $in: [null, ''] } }, { $set: { '%hash%': now } })
+          }
         }
       },
       {

models/document/src/migration.ts

@@ -335,12 +335,6 @@ export const documentOperation: MigrateOperation = {
         state: 'renameFields',
         func: renameFields
       },
-      {
-        state: 'fix-rename-backups',
-        func: async (client: MigrationClient): Promise<void> => {
-          await client.update(DOMAIN_DOCUMENT, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } })
-        }
-      },
       {
         state: 'renameFieldsRevert',
         func: renameFieldsRevert

models/drive/src/migration.ts

@@ -132,12 +132,6 @@ export const driveOperation: MigrateOperation = {
       {
         state: 'renameFields',
         func: renameFields
-      },
-      {
-        state: 'fix-rename-backups',
-        func: async (client: MigrationClient): Promise<void> => {
-          await client.update(DOMAIN_DRIVE, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } })
-        }
       }
     ])
   },

models/notification/src/migration.ts

@@ -393,12 +393,6 @@ export const notificationOperation: MigrateOperation = {
           )
         }
       },
-      {
-        state: 'fix-rename-backups',
-        func: async (client: MigrationClient): Promise<void> => {
-          await client.update(DOMAIN_DOC_NOTIFY, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } })
-        }
-      },
       {
         state: 'remove-update-txes-docnotify-ctx-v2',
         func: async (client) => {

models/task/src/migration.ts

@@ -569,12 +569,6 @@ export const taskOperation: MigrateOperation = {
           await migrateSpace(client, task.space.Sequence, core.space.Workspace, [DOMAIN_KANBAN])
         }
       },
-      {
-        state: 'fix-rename-backups',
-        func: async (client: MigrationClient): Promise<void> => {
-          await client.update(DOMAIN_TASK, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } })
-        }
-      },
       {
         state: 'migrateRanks',
         func: migrateRanks

models/time/src/migration.ts

@@ -172,12 +172,6 @@ export const timeOperation: MigrateOperation = {
         func: async (client) => {
           await fillProps(client)
         }
-      },
-      {
-        state: 'fix-rename-backups',
-        func: async (client: MigrationClient): Promise<void> => {
-          await client.update(DOMAIN_TIME, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } })
-        }
       }
     ])
   },

models/view/src/migration.ts

@@ -86,12 +86,6 @@ export const viewOperation: MigrateOperation = {
       {
         state: 'remove-done-state-filter',
         func: removeDoneStateFilter
-      },
-      {
-        state: 'fix-rename-backups',
-        func: async (client: MigrationClient): Promise<void> => {
-          await client.update(DOMAIN_VIEW, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } })
-        }
       }
     ])
   },

packages/api-client/src/index.ts

@@ -14,5 +14,6 @@
 //
 
 export * from './client'
+export * from './markup/types'
 export * from './socket'
 export * from './types'

packages/core/src/client.ts

@@ -236,7 +236,7 @@ export async function createClient (
   let hierarchy = new Hierarchy()
   let model = new ModelDb(hierarchy)
 
-  let lastTx: number
+  let lastTx: number = 0
 
   function txHandler (...tx: Tx[]): void {
     if (tx == null || tx.length === 0) {
@@ -295,6 +295,10 @@ export async function createClient (
     }
 
     // We need to look for last {transactionThreshold} transactions and if it is more since lastTx one we receive, we need to perform full refresh.
+    if (lastTx === 0) {
+      await oldOnConnect?.(ClientConnectEvent.Refresh, data)
+      return
+    }
     const atxes = await ctx.with('find-atx', {}, () =>
       conn.findAll(
         core.class.Tx,

packages/presentation/src/components/DrawingBoard.svelte

@@ -16,7 +16,14 @@
   import { Analytics } from '@hcengineering/analytics'
   import { resizeObserver } from '@hcengineering/ui'
   import { onDestroy } from 'svelte'
-  import { drawing, type DrawingCmd, type DrawingData, type DrawingTool, type DrawTextCmd } from '../drawing'
+  import {
+    drawing,
+    makeCommandId,
+    type DrawingCmd,
+    type DrawingData,
+    type DrawingTool,
+    type DrawTextCmd
+  } from '../drawing'
   import DrawingBoardToolbar from './DrawingBoardToolbar.svelte'
 
   export let active = false
@@ -38,7 +45,8 @@
   let oldReadonly: boolean
   let oldDrawings: DrawingData[]
   let modified = false
-  let changingCmdIndex: number | undefined
+  let changingCmdId: string | undefined
+  let cmdEditor: HTMLDivElement | undefined
 
   $: updateToolbarPosition(readonly, board, toolbar)
   $: updateEditableState(drawings, readonly)
@@ -63,14 +71,15 @@
             commands = []
           } else {
             // Edit current content as a new drawing
-            commands = [...commands]
+            commands = commands.map((cmd) => ({ ...cmd, id: cmd.id ?? makeCommandId() }))
           }
           modified = false
         }
       } else {
         commands = undefined
       }
-      changingCmdIndex = undefined
+      changingCmdId = undefined
+      cmdEditor = undefined
       oldDrawings = drawings
       oldReadonly = readonly
     }
@@ -105,33 +114,40 @@
   function addCommand (cmd: DrawingCmd): void {
     if (commands !== undefined) {
       commands = [...commands, cmd]
-      changingCmdIndex = undefined
+      changingCmdId = undefined
+      cmdEditor = undefined
       modified = true
     }
   }
 
-  function showCommandProps (index: number): void {
+  function showCommandProps (id: string): void {
-    changingCmdIndex = index
+    changingCmdId = id
-    const anyCmd = commands?.[index]
+    for (const cmd of commands ?? []) {
-    if (anyCmd?.type === 'text') {
+      if (cmd.id === id) {
-      const cmd = anyCmd as DrawTextCmd
+        if (cmd.type === 'text') {
-      penColor = cmd.color
+          const textCmd = cmd as DrawTextCmd
-      fontSize = cmd.fontSize
+          penColor = textCmd.color
+          fontSize = textCmd.fontSize
+        }
+        break
+      }
     }
   }
 
-  function changeCommand (index: number, cmd: DrawingCmd): void {
+  function changeCommand (cmd: DrawingCmd): void {
     if (commands !== undefined) {
-      commands = commands.map((c, i) => (i === index ? cmd : c))
+      commands = commands.map((c) => (c.id === cmd.id ? cmd : c))
-      changingCmdIndex = undefined
+      changingCmdId = undefined
+      cmdEditor = undefined
       modified = true
     }
   }
 
-  function deleteCommand (index: number): void {
+  function deleteCommand (id: string): void {
     if (commands !== undefined) {
-      commands = commands.filter((_, i) => i !== index)
+      commands = commands.filter((c) => c.id !== id)
-      changingCmdIndex = undefined
+      changingCmdId = undefined
+      cmdEditor = undefined
       modified = true
     }
   }
@@ -159,19 +175,23 @@
       penWidth,
       eraserWidth,
       fontSize,
-      changingCmdIndex,
+      changingCmdId,
       cmdAdded: addCommand,
       cmdChanging: showCommandProps,
       cmdChanged: changeCommand,
       cmdUnchanged: () => {
-        changingCmdIndex = undefined
+        changingCmdId = undefined
       },
-      cmdDeleted: deleteCommand
+      cmdDeleted: deleteCommand,
+      editorCreated: (editor) => {
+        cmdEditor = editor
+      }
     }}
   >
     {#if !readonly}
       <DrawingBoardToolbar
         placeInside={toolbarInside}
+        {cmdEditor}
         bind:toolbar
         bind:tool
         bind:penColor

packages/presentation/src/components/DrawingBoardToolbar.svelte

@@ -53,6 +53,7 @@
   export let placeInside = false
   export let showPanTool = false
   export let toolbar: HTMLDivElement | undefined
+  export let cmdEditor: HTMLDivElement | undefined
 
   let colorSelector: HTMLInputElement
   let penColors: string[] = defaultColors
@@ -91,12 +92,14 @@
           penColors = penColors.filter((c: string) => c !== penColor)
           localStorage.setItem(storageKey.colors, JSON.stringify(penColors))
           selectColor(penColors[0])
+          focusEditor()
           break
         }
         case 'reset-colors': {
           penColors = defaultColors
           localStorage.removeItem(storageKey.colors)
           selectColor(penColors[0])
+          focusEditor()
           break
         }
         case undefined: {
@@ -115,6 +118,7 @@
       penColors = [...penColors, penColor]
       localStorage.setItem(storageKey.colors, JSON.stringify(penColors))
     }
+    focusEditor()
   }
 
   function selectColor (color: string): void {
@@ -148,6 +152,15 @@
 
   function updateFontSize (): void {
     localStorage.setItem(storageKey.fontSize, fontSize.toString())
+    focusEditor()
+  }
+
+  function focusEditor (): void {
+    setTimeout(() => {
+      if (cmdEditor !== undefined) {
+        cmdEditor.focus()
+      }
+    }, 100)
   }
 </script>
 
@@ -243,6 +256,7 @@
           tool = 'pen'
         }
         selectColor(color)
+        focusEditor()
       }}
     >
       <div slot="content" class="colorIcon" style:background={color} />

packages/presentation/src/drawing.ts

@@ -30,16 +30,18 @@ export interface DrawingProps {
   eraserWidth?: number
   fontSize?: number
   defaultCursor?: string
-  changingCmdIndex?: number
+  changingCmdId?: string
   cmdAdded?: (cmd: DrawingCmd) => void
-  cmdChanging?: (index: number) => void
+  cmdChanging?: (id: string) => void
-  cmdUnchanged?: (index: number) => void
+  cmdUnchanged?: (id: string) => void
-  cmdChanged?: (index: number, cmd: DrawingCmd) => void
+  cmdChanged?: (cmd: DrawingCmd) => void
-  cmdDeleted?: (index: number) => void
+  cmdDeleted?: (id: string) => void
+  editorCreated?: (editor: HTMLDivElement) => void
   panned?: (offset: Point) => void
 }
 
 export interface DrawingCmd {
+  id: string
   type: 'line' | 'text'
 }
 
@@ -71,6 +73,10 @@ function avgPoint (p1: Point, p2: Point): Point {
 
 const maxTextLength = 500
 
+export const makeCommandId = (): string => {
+  return crypto.randomUUID().toString()
+}
+
 const crossSvg = `<svg height="8" width="8" viewBox="0 0 16 16" fill="currentColor" xmlns="http://www.w3.org/2000/svg">
   <path d="m1.29 2.71 5.3 5.29-5.3 5.29c-.92.92.49 2.34 1.41 1.41l5.3-5.29 5.29 5.3c.92.92 2.34-.49 1.41-1.41l-5.29-5.3 5.3-5.29c.92-.93-.49-2.34-1.42-1.42l-5.29 5.3-5.29-5.3c-.93-.92-2.34.49-1.42 1.42z"/>
 </svg>`
@@ -294,13 +300,15 @@ export function drawing (
   draw.eraserWidth = props.eraserWidth ?? draw.eraserWidth
   draw.fontSize = props.fontSize ?? draw.fontSize
   draw.offset = props.offset ?? draw.offset
+
   updateCanvasCursor()
+  updateCanvasTouchAction()
 
   interface LiveTextBox {
     pos: Point
     box: HTMLDivElement
     editor: HTMLDivElement
-    cmdIndex: number
+    cmdId: string
   }
   let liveTextBox: LiveTextBox | undefined
 
@@ -328,6 +336,61 @@ export function drawing (
   })
   resizeObserver.observe(canvas)
 
+  let touchId: number | undefined
+
+  function findTouch (touches: TouchList, id: number | undefined = touchId): Touch | undefined {
+    for (let i = 0; i < touches.length; i++) {
+      const touch = touches[i]
+      if (touch.identifier === id) {
+        return touch
+      }
+    }
+  }
+
+  function touchToNodePoint (touch: Touch, node: HTMLElement): Point {
+    const rect = node.getBoundingClientRect()
+    return {
+      x: touch.clientX - rect.left,
+      y: touch.clientY - rect.top
+    }
+  }
+
+  function pointerToNodePoint (e: PointerEvent): Point {
+    return { x: e.offsetX, y: e.offsetY }
+  }
+
+  canvas.ontouchstart = (e) => {
+    if (readonly) {
+      return
+    }
+    const touch = e.changedTouches[0]
+    touchId = touch.identifier
+    drawStart(touchToNodePoint(touch, canvas))
+  }
+
+  canvas.ontouchmove = (e) => {
+    if (readonly) {
+      return
+    }
+    const touch = findTouch(e.changedTouches)
+    if (touch !== undefined) {
+      drawContinue(touchToNodePoint(touch, canvas))
+    }
+  }
+
+  canvas.ontouchend = (e) => {
+    if (readonly) {
+      return
+    }
+    const touch = findTouch(e.changedTouches)
+    if (touch !== undefined) {
+      drawEnd(touchToNodePoint(touch, canvas))
+    }
+    touchId = undefined
+  }
+
+  canvas.ontouchcancel = canvas.ontouchend
+
   canvas.onpointerdown = (e) => {
     if (readonly) {
       return
@@ -337,16 +400,7 @@ export function drawing (
     }
     e.preventDefault()
     canvas.setPointerCapture(e.pointerId)
-
+    drawStart(pointerToNodePoint(e))
-    const x = e.offsetX
-    const y = e.offsetY
-
-    draw.on = true
-    draw.points = []
-    prevPos = { x, y }
-    if (draw.isDrawingTool()) {
-      draw.addPoint(x, y)
-    }
   }
 
   canvas.onpointermove = (e) => {
@@ -354,35 +408,7 @@ export function drawing (
       return
     }
     e.preventDefault()
-
+    drawContinue(pointerToNodePoint(e))
-    const x = e.offsetX
-    const y = e.offsetY
-
-    if (draw.isDrawingTool()) {
-      const w = draw.cursorWidth()
-      canvasCursor.style.left = `${x - w / 2}px`
-      canvasCursor.style.top = `${y - w / 2}px`
-      if (draw.on) {
-        if (Math.hypot(prevPos.x - x, prevPos.y - y) < draw.minLineLength) {
-          return
-        }
-        draw.drawLive(x, y)
-        prevPos = { x, y }
-      }
-    }
-
-    if (draw.on && draw.tool === 'pan') {
-      requestAnimationFrame(() => {
-        draw.offset.x += x - prevPos.x
-        draw.offset.y += y - prevPos.y
-        replayCommands()
-        prevPos = { x, y }
-      })
-    }
-
-    if (draw.on && draw.tool === 'text') {
-      prevPos = { x, y }
-    }
   }
 
   canvas.onpointerup = (e) => {
@@ -391,24 +417,11 @@ export function drawing (
     }
     e.preventDefault()
     canvas.releasePointerCapture(e.pointerId)
-    if (draw.on) {
+    drawEnd(pointerToNodePoint(e))
-      if (draw.isDrawingTool()) {
-        draw.drawLive(e.offsetX, e.offsetY, true)
-        storeLineCommand()
-      } else if (draw.tool === 'pan') {
-        props.panned?.(draw.offset)
-      } else if (draw.tool === 'text') {
-        if (liveTextBox !== undefined) {
-          storeTextCommand()
-        } else {
-          const cmdIndex = findTextCommand(prevPos)
-          props.cmdChanging?.(cmdIndex)
-        }
-      }
-      draw.on = false
-    }
   }
 
+  canvas.onpointercancel = canvas.onpointerup
+
   canvas.onpointerenter = () => {
     if (!readonly && draw.isDrawingTool()) {
       canvasCursor.style.visibility = 'visible'
@@ -421,26 +434,86 @@ export function drawing (
     }
   }
 
-  function findTextCommand (mousePos: Point): number {
+  function drawStart (p: Point): void {
+    draw.on = true
+    draw.points = []
+    prevPos = p
+    if (draw.isDrawingTool()) {
+      draw.addPoint(p.x, p.y)
+    }
+  }
+
+  function drawContinue (p: Point): void {
+    if (draw.isDrawingTool()) {
+      const w = draw.cursorWidth()
+      canvasCursor.style.left = `${p.x - w / 2}px`
+      canvasCursor.style.top = `${p.y - w / 2}px`
+      if (draw.on) {
+        if (Math.hypot(prevPos.x - p.x, prevPos.y - p.y) < draw.minLineLength) {
+          return
+        }
+        draw.drawLive(p.x, p.y)
+        prevPos = p
+      }
+    }
+
+    if (draw.on && draw.tool === 'pan') {
+      requestAnimationFrame(() => {
+        draw.offset.x += p.x - prevPos.x
+        draw.offset.y += p.y - prevPos.y
+        replayCommands()
+        prevPos = p
+      })
+    }
+
+    if (draw.on && draw.tool === 'text') {
+      prevPos = p
+    }
+  }
+
+  function drawEnd (p: Point): void {
+    if (draw.on) {
+      if (draw.isDrawingTool()) {
+        draw.drawLive(p.x, p.y, true)
+        storeLineCommand()
+      } else if (draw.tool === 'pan') {
+        props.panned?.(draw.offset)
+      } else if (draw.tool === 'text') {
+        if (liveTextBox !== undefined) {
+          storeTextCommand()
+          closeLiveTextBox()
+        } else {
+          const cmd = findTextCommand(prevPos)
+          props.cmdChanging?.(cmd?.id ?? '')
+        }
+      }
+      draw.on = false
+    }
+  }
+
+  function findTextCommand (mousePos: Point): DrawTextCmd | undefined {
     const pos = draw.mouseToCanvasPoint(mousePos)
     for (let i = commands.length - 1; i >= 0; i--) {
       const anyCmd = commands[i]
       if (anyCmd.type === 'text') {
         const cmd = anyCmd as DrawTextCmd
         if (draw.isPointInText(pos, cmd)) {
-          return i
+          return cmd
         }
       }
     }
-    return -1
+    return undefined
   }
 
-  function makeLiveTextBox (cmdIndex: number): void {
+  function makeLiveTextBox (cmdId: string): void {
     let pos = prevPos
     let existingCmd: DrawTextCmd | undefined
-    if (cmdIndex >= 0 && commands[cmdIndex]?.type === 'text') {
+    for (const cmd of commands) {
-      existingCmd = commands[cmdIndex] as DrawTextCmd
+      if (cmd.id === cmdId && cmd.type === 'text') {
+        existingCmd = cmd as DrawTextCmd
         pos = draw.canvasToMousePoint(existingCmd.pos)
+        break
+      }
     }
 
     const padding = 6
@@ -455,6 +528,7 @@ export function drawing (
     box.style.borderRadius = 'var(--small-BorderRadius)'
     box.style.padding = `${padding}px`
     box.style.background = 'var(--theme-popup-header)'
+    box.style.touchAction = 'none'
     box.addEventListener('mousedown', (e) => {
       e.stopPropagation()
     })
@@ -513,19 +587,18 @@ export function drawing (
       if (e.key === 'Escape') {
         e.preventDefault()
         if (liveTextBox !== undefined) {
-          const cmdIndex = liveTextBox.cmdIndex
+          const cmdId = liveTextBox.cmdId
-          if (cmdIndex >= 0) {
+          // reset changingCmdId in clients
-            // reset changingCmdIndex in clients
           setTimeout(() => {
-              props.cmdUnchanged?.(cmdIndex)
+            props.cmdUnchanged?.(cmdId)
           }, 0)
         }
-        }
         closeLiveTextBox()
         replayCommands()
       } else if (e.key === 'Enter' && e.ctrlKey) {
         e.preventDefault()
         storeTextCommand()
+        closeLiveTextBox()
       }
     })
     box.appendChild(editor)
@@ -562,24 +635,7 @@ export function drawing (
       return handle
     }
 
-    const dragHandle = makeHandle()
+    const moveTextBox = (dx: number, dy: number): void => {
-    dragHandle.style.left = `-${handleSize / 2}px`
-    dragHandle.style.cursor = 'grab'
-    dragHandle.addEventListener('pointerdown', (e) => {
-      e.preventDefault()
-      dragHandle.style.cursor = 'grabbing'
-      dragHandle.setPointerCapture(e.pointerId)
-      const x = e.clientX
-      const y = e.clientY
-      const dragStart = { x, y }
-      const pointerMove = (e: PointerEvent): void => {
-        e.preventDefault()
-        const x = e.clientX
-        const y = e.clientY
-        const dx = x - dragStart.x
-        const dy = y - dragStart.y
-        dragStart.x = x
-        dragStart.y = y
       let newX = box.offsetLeft + dx
       let newY = box.offsetTop + dy
       // For screenshots the canvas always has the same size as the underlying image
@@ -601,6 +657,22 @@ export function drawing (
         liveTextBox.pos.y = newY + padding
       }
     }
+
+    const dragHandle = makeHandle()
+    dragHandle.style.left = `-${handleSize / 2}px`
+    dragHandle.style.cursor = 'grab'
+    dragHandle.style.touchAction = 'none'
+    dragHandle.addEventListener('pointerdown', (e) => {
+      e.preventDefault()
+      dragHandle.style.cursor = 'grabbing'
+      dragHandle.setPointerCapture(e.pointerId)
+      let prevPos = { x: e.clientX, y: e.clientY }
+      const pointerMove = (e: PointerEvent): void => {
+        e.preventDefault()
+        const p = { x: e.clientX, y: e.clientY }
+        moveTextBox(p.x - prevPos.x, p.y - prevPos.y)
+        prevPos = p
+      }
       const pointerUp = (e: PointerEvent): void => {
         setTimeout(() => {
           editor.focus()
@@ -610,9 +682,37 @@ export function drawing (
         dragHandle.releasePointerCapture(e.pointerId)
         dragHandle.removeEventListener('pointermove', pointerMove)
         dragHandle.removeEventListener('pointerup', pointerUp)
+        dragHandle.removeEventListener('pointercancel', pointerUp)
       }
       dragHandle.addEventListener('pointermove', pointerMove)
       dragHandle.addEventListener('pointerup', pointerUp)
+      dragHandle.addEventListener('pointercancel', pointerUp)
+    })
+    dragHandle.addEventListener('touchstart', (e) => {
+      dragHandle.style.cursor = 'grabbing'
+      const touch = e.changedTouches[0]
+      const touchId = touch.identifier
+      let prevPos = touchToNodePoint(touch, dragHandle)
+      const touchMove = (e: TouchEvent): void => {
+        const touch = findTouch(e.changedTouches, touchId)
+        if (touch !== undefined) {
+          const p = touchToNodePoint(touch, dragHandle)
+          moveTextBox(p.x - prevPos.x, p.y - prevPos.y)
+          prevPos = p
+        }
+      }
+      const touchEnd = (e: TouchEvent): void => {
+        setTimeout(() => {
+          editor.focus()
+        }, 100)
+        dragHandle.style.cursor = 'grab'
+        dragHandle.removeEventListener('touchmove', touchMove)
+        dragHandle.removeEventListener('touchend', touchEnd)
+        dragHandle.removeEventListener('touchcancel', touchEnd)
+      }
+      dragHandle.addEventListener('touchmove', touchMove)
+      dragHandle.addEventListener('touchend', touchEnd)
+      dragHandle.addEventListener('touchcancel', touchEnd)
     })
     box.appendChild(dragHandle)
 
@@ -622,20 +722,21 @@ export function drawing (
     deleteButton.innerHTML = crossSvg
     deleteButton.addEventListener('click', () => {
       node.removeChild(box)
-      if (liveTextBox?.cmdIndex !== undefined) {
+      if (liveTextBox?.cmdId !== undefined) {
-        props.cmdDeleted?.(liveTextBox.cmdIndex)
+        props.cmdDeleted?.(liveTextBox.cmdId)
       }
       liveTextBox = undefined
     })
     box.appendChild(deleteButton)
 
     node.appendChild(box)
-    liveTextBox = { box, editor, pos, cmdIndex }
+    liveTextBox = { box, editor, pos, cmdId }
     updateLiveTextBox()
     setTimeout(() => {
       editor.focus()
     }, 100)
     selectAll()
+    props.editorCreated?.(editor)
   }
 
   function updateLiveTextBox (): void {
@@ -658,7 +759,9 @@ export function drawing (
     if (liveTextBox !== undefined) {
       const text = (liveTextBox.editor.innerText ?? '').trim()
       if (text !== '') {
+        const cmdId = liveTextBox.cmdId
         const cmd: DrawTextCmd = {
+          id: cmdId === '' ? makeCommandId() : cmdId,
           type: 'text',
           text,
           pos: draw.mouseToCanvasPoint(liveTextBox.pos),
@@ -666,10 +769,9 @@ export function drawing (
           fontFace: draw.fontFace,
           color: draw.penColor
         }
-        const cmdIndex = liveTextBox.cmdIndex
         const notify = (): void => {
-          if (cmdIndex >= 0) {
+          if (cmdId !== '') {
-            props.cmdChanged?.(cmdIndex, cmd)
+            props.cmdChanged?.(cmd)
           } else {
             props.cmdAdded?.(cmd)
           }
@@ -680,7 +782,7 @@ export function drawing (
           notify()
         }
       } else {
-        props.cmdUnchanged?.(liveTextBox.cmdIndex)
+        props.cmdUnchanged?.(liveTextBox.cmdId)
       }
     }
   }
@@ -689,6 +791,7 @@ export function drawing (
     if (draw.points.length > 0) {
       const erasing = draw.tool === 'erase'
       const cmd: DrawLineCmd = {
+        id: makeCommandId(),
         type: 'line',
         lineWidth: erasing ? draw.eraserWidth : draw.penWidth,
         erasing,
@@ -726,13 +829,17 @@ export function drawing (
     }
   }
 
+  function updateCanvasTouchAction (): void {
+    canvas.style.touchAction = readonly ? 'unset' : 'none'
+  }
+
   function replayCommands (): void {
     draw.ctx.reset()
-    for (let i = 0; i < commands.length; i++) {
+    for (const cmd of commands) {
-      if (liveTextBox?.cmdIndex === i) {
+      if (cmd.id !== undefined && liveTextBox?.cmdId === cmd.id) {
         continue
       }
-      draw.drawCommand(commands[i])
+      draw.drawCommand(cmd)
     }
   }
 
@@ -774,9 +881,10 @@ export function drawing (
       }
       if (props.readonly !== readonly) {
         readonly = props.readonly ?? false
+        updateCanvasTouchAction()
         updateCursor = true
       }
-      if (props.changingCmdIndex === undefined) {
+      if (props.changingCmdId === undefined) {
         if (liveTextBox !== undefined) {
           storeTextCommand(true)
           closeLiveTextBox()
@@ -784,9 +892,9 @@ export function drawing (
         }
       } else {
         if (liveTextBox === undefined) {
-          makeLiveTextBox(props.changingCmdIndex)
+          makeLiveTextBox(props.changingCmdId)
           replay = true
-        } else if (liveTextBox.cmdIndex !== props.changingCmdIndex) {
+        } else if (liveTextBox.cmdId !== props.changingCmdId) {
           storeTextCommand(true)
           closeLiveTextBox()
           replay = true

packages/query/src/index.ts

@@ -364,7 +364,8 @@ export class LiveQuery implements WithTx, Client {
       total: 0,
       options: options as FindOptions<Doc>,
       callbacks: new Map(),
-      refresh: reduceCalls(() => this.doRefresh(q))
+      refresh: reduceCalls(() => this.doRefresh(q)),
+      refreshId: 0
     }
     if (callback !== undefined) {
       q.callbacks.set(callback.callbackId, callback.callback as unknown as Callback)
@@ -787,8 +788,9 @@ export class LiveQuery implements WithTx, Client {
   }
 
   private async doRefresh (q: Query): Promise<void> {
+    const qid = ++q.refreshId
     const res = await this.client.findAll(q._class, q.query, q.options)
-    if (!deepEqual(res, q.result) || (res.total !== q.total && q.options?.total === true)) {
+    if (q.refreshId === qid && (!deepEqual(res, q.result) || (res.total !== q.total && q.options?.total === true))) {
       q.result = new ResultArray(res, this.getHierarchy())
       q.total = res.total
       await this.callback(q)

packages/query/src/types.ts

@@ -12,6 +12,6 @@ export interface Query {
   options?: FindOptions<Doc>
   total: number
   callbacks: Map<string, Callback>
-
   refresh: () => Promise<void>
+  refreshId: number
 }

plugins/recruit-resources/src/components/CreateVacancy.svelte

@@ -103,7 +103,7 @@
     type: typeId as Ref<ProjectType>
   }
   export function canClose (): boolean {
-    return name === '' && typeId !== undefined
+    return name.trim() === '' && typeId !== undefined
   }
 
   const client = getClient()
@@ -185,7 +185,7 @@
     const resId: Ref<Issue> = generateId()
     const identifier = `${project?.identifier}-${number}`
     const data: AttachedData<Issue> = {
-      title: template.title + ` (${name})`,
+      title: template.title + ` (${name.trim()})`,
       description: null,
       assignee: template.assignee,
       component: template.component,
@@ -240,7 +240,7 @@
     const incResult = await client.update(sequence, { $inc: { sequence: 1 } }, true)
     const data: Data<Vacancy> = {
       ...vacancyData,
-      name,
+      name: name.trim(),
       description: template?.shortDescription ?? '',
       fullDescription: null,
       private: false,
@@ -336,7 +336,7 @@
 <Card
   label={recruit.string.CreateVacancy}
   okAction={createVacancy}
-  canSave={!!name}
+  canSave={name.trim() !== ''}
   gap={'gapV-4'}
   on:close={() => {
     dispatch('close')

plugins/recruit-resources/src/components/EditVacancy.svelte

@@ -101,9 +101,13 @@
 
     const updates: Partial<Data<Vacancy>> = {}
     const trimmedName = rawName.trim()
+    const trimmedNameOld = object.name?.trim()
 
-    if (trimmedName.length > 0 && trimmedName !== object.name?.trim()) {
+    if (trimmedName.length > 0 && (trimmedName !== trimmedNameOld || trimmedNameOld !== object.name)) {
       updates.name = trimmedName
+      rawName = trimmedName
+    } else {
+      rawName = object.name
     }
 
     if (rawDesc !== object.description) {

plugins/text-editor-resources/src/components/DrawingBoardEditor.svelte

@@ -13,7 +13,14 @@
 // limitations under the License.
 -->
 <script lang="ts">
-  import { DrawingBoardToolbar, DrawingCmd, DrawingTool, DrawTextCmd, drawing } from '@hcengineering/presentation'
+  import {
+    DrawingBoardToolbar,
+    DrawingCmd,
+    DrawingTool,
+    DrawTextCmd,
+    drawing,
+    makeCommandId
+  } from '@hcengineering/presentation'
   import { Loading } from '@hcengineering/ui'
   import { onMount, onDestroy } from 'svelte'
   import { Array as YArray, Map as YMap } from 'yjs'
@@ -34,11 +41,14 @@
   let fontSize: number
   let commands: DrawingCmd[] = []
   let offset: { x: number, y: number } = { x: 0, y: 0 }
-  let changingCmdIndex: number | undefined
+  let changingCmdId: string | undefined
+  let cmdEditor: HTMLDivElement | undefined
   let toolbar: HTMLDivElement
   let oldSelected = false
+  let oldReadonly = false
 
   $: onSelectedChanged(selected)
+  $: onReadonlyChanged(readonly)
 
   function listenSavedCommands (): void {
     commands = savedCmds.toArray()
@@ -50,25 +60,79 @@
     // offset = savedProps.get('offset')
   }
 
-  function showCommandProps (index: number): void {
+  function showCommandProps (id: string): void {
-    changingCmdIndex = index
+    changingCmdId = id
-    const anyCmd = commands[index]
+    for (const cmd of commands) {
-    if (anyCmd?.type === 'text') {
+      if (cmd.id === id) {
-      const cmd = anyCmd as DrawTextCmd
+        if (cmd.type === 'text') {
-      penColor = cmd.color
+          const textCmd = cmd as DrawTextCmd
-      fontSize = cmd.fontSize
+          penColor = textCmd.color
+          fontSize = textCmd.fontSize
         }
+        break
+      }
+    }
+  }
+
+  function changeCommand (cmd: DrawingCmd): void {
+    let index = -1
+    for (let i = 0; i < savedCmds.length; i++) {
+      if (savedCmds.get(i).id === cmd.id) {
+        savedCmds.delete(i)
+        index = i
+        break
+      }
+    }
+    if (index >= 0) {
+      savedCmds.insert(index, [cmd])
+    } else {
+      savedCmds.push([cmd])
+    }
+    changingCmdId = undefined
+    cmdEditor = undefined
+  }
+
+  function deleteCommand (id: string): void {
+    for (let i = 0; i < savedCmds.length; i++) {
+      if (savedCmds.get(i).id === id) {
+        savedCmds.delete(i)
+        break
+      }
+    }
+    changingCmdId = undefined
+    cmdEditor = undefined
   }
 
   function onSelectedChanged (selected: boolean): void {
     if (oldSelected !== selected) {
-      if (oldSelected && !selected && changingCmdIndex !== undefined) {
+      if (oldSelected && !selected && changingCmdId !== undefined) {
-        changingCmdIndex = undefined
+        changingCmdId = undefined
+        cmdEditor = undefined
       }
       oldSelected = selected
     }
   }
 
+  function onReadonlyChanged (readonly: boolean): void {
+    if (oldReadonly !== readonly) {
+      if (!readonly) {
+        let allHaveIds = true
+        for (let i = 0; i < savedCmds.length; i++) {
+          if (savedCmds.get(i).id === undefined) {
+            allHaveIds = false
+            break
+          }
+        }
+        if (!allHaveIds) {
+          const cmds = savedCmds.toArray()
+          savedCmds.delete(0, savedCmds.length)
+          savedCmds.push(cmds.map((cmd) => ({ ...cmd, id: cmd.id ?? makeCommandId() })))
+        }
+      }
+      oldReadonly = readonly
+    }
+  }
+
   onMount(() => {
     commands = savedCmds.toArray()
     // offset = savedProps.get('offset')
@@ -109,27 +173,23 @@
         penWidth,
         eraserWidth,
         fontSize,
-        changingCmdIndex,
+        changingCmdId,
         cmdAdded: (cmd) => {
           savedCmds.push([cmd])
-          changingCmdIndex = undefined
+          changingCmdId = undefined
         },
         cmdChanging: showCommandProps,
-        cmdChanged: (index, cmd) => {
+        cmdChanged: changeCommand,
-          savedCmds.delete(index)
-          savedCmds.insert(index, [cmd])
-          changingCmdIndex = undefined
-        },
         cmdUnchanged: () => {
-          changingCmdIndex = undefined
+          changingCmdId = undefined
-        },
-        cmdDeleted: (index) => {
-          savedCmds.delete(index)
-          changingCmdIndex = undefined
         },
+        cmdDeleted: deleteCommand,
         panned: (newOffset) => {
           offset = newOffset
           // savedProps.set('offset', offset)
+        },
+        editorCreated: (editor) => {
+          cmdEditor = editor
         }
       }}
     >
@@ -142,6 +202,7 @@
         <DrawingBoardToolbar
           placeInside={true}
           showPanTool={true}
+          {cmdEditor}
           bind:toolbar
           bind:tool
           bind:penColor

plugins/text-editor-resources/src/components/DrawingBoardNodeView.svelte

@@ -35,23 +35,43 @@
   let resizer: HTMLElement
   let startY: number
   let resizedHeight: number | undefined
+  let resizerTouchId: number | undefined
   let loading = true
   let loadingTimer: any
 
+  function resizeStart (y: number): void {
+    const height = node.attrs.height ?? defaultHeight
+    startY = y - height
+    resizedHeight = height
+  }
+
+  function resizeContinue (y: number): void {
+    resizedHeight = Math.max(minHeight, y - startY)
+    resizedHeight = Math.min(maxHeight, resizedHeight)
+  }
+
+  function resizeFinish (): void {
+    if (resizedHeight !== undefined) {
+      if (typeof getPos === 'function') {
+        const tr = editor.state.tr.setNodeMarkup(getPos(), undefined, { ...node.attrs, height: resizedHeight })
+        editor.view.dispatch(tr)
+      }
+      resizedHeight = undefined
+    }
+  }
+
   function onResizerPointerDown (e: PointerEvent): void {
     e.preventDefault()
-    const height = node.attrs.height ?? defaultHeight
-    startY = e.clientY - height
-    resizedHeight = height
     resizer.setPointerCapture(e.pointerId)
     resizer.addEventListener('pointermove', onResizerPointerMove)
     resizer.addEventListener('pointerup', onResizerPointerUp)
+    resizer.addEventListener('pointercancel', onResizerPointerUp)
+    resizeStart(e.clientY)
   }
 
   function onResizerPointerMove (e: PointerEvent): void {
     e.preventDefault()
-    resizedHeight = Math.max(minHeight, e.clientY - startY)
+    resizeContinue(e.clientY)
-    resizedHeight = Math.min(maxHeight, resizedHeight)
   }
 
   function onResizerPointerUp (e: PointerEvent): void {
@@ -59,11 +79,35 @@
     resizer.releasePointerCapture(e.pointerId)
     resizer.removeEventListener('pointermove', onResizerPointerMove)
     resizer.removeEventListener('pointerup', onResizerPointerUp)
-    if (typeof getPos === 'function') {
+    resizer.removeEventListener('pointercancel', onResizerPointerUp)
-      const tr = editor.state.tr.setNodeMarkup(getPos(), undefined, { ...node.attrs, height: resizedHeight })
+    resizeFinish()
-      editor.view.dispatch(tr)
   }
-    resizedHeight = undefined
+
+  function onResizerTouchStart (e: TouchEvent): void {
+    const touch = e.changedTouches[0]
+    resizerTouchId = touch.identifier
+    resizer.addEventListener('touchmove', onResizerTouchMove)
+    resizer.addEventListener('touchend', onResizerTouchEnd)
+    resizer.addEventListener('touchcancel', onResizerTouchEnd)
+    resizeStart(touch.clientY)
+  }
+
+  function onResizerTouchMove (e: TouchEvent): void {
+    for (let i = 0; i < e.changedTouches.length; i++) {
+      const touch = e.changedTouches[i]
+      if (touch.identifier === resizerTouchId) {
+        resizeContinue(touch.clientY)
+        return
+      }
+    }
+  }
+
+  function onResizerTouchEnd (): void {
+    resizer.removeEventListener('touchmove', onResizerTouchMove)
+    resizer.removeEventListener('touchend', onResizerTouchEnd)
+    resizer.removeEventListener('touchcancel', onResizerTouchEnd)
+    resizerTouchId = undefined
+    resizeFinish()
   }
 
   onMount(() => {
@@ -110,7 +154,12 @@
         />
       </div>
       {#if selected}
-        <div class="handle resizer" bind:this={resizer} on:pointerdown={onResizerPointerDown}>
+        <div
+          class="handle resizer"
+          bind:this={resizer}
+          on:pointerdown={onResizerPointerDown}
+          on:touchstart={onResizerTouchStart}
+        >
           <svg viewBox="0 0 60 4" height="4" width="60" fill="currentColor" xmlns="http://www.w3.org/2000/svg">
             <path
               d="m60 2a2 2 0 0 1 -2 2 2 2 0 0 1 -2-2 2 2 0 0 1 2-2 2 2 0 0 1 2 2zm-8 0a2 2 0 0 1 -2 2 2 2 0 0 1 -2-2 2 2 0 0 1 2-2 2 2 0 0 1 2 2zm-8 0a2 2 0 0 1 -2 2 2 2 0 0 1 -2-2 2 2 0 0 1 2-2 2 2 0 0 1 2 2zm-8 0a2 2 0 0 1 -2 2 2 2 0 0 1 -2-2 2 2 0 0 1 2-2 2 2 0 0 1 2 2zm-8 0a2 2 0 0 1 -2 2 2 2 0 0 1 -2-2 2 2 0 0 1 2-2 2 2 0 0 1 2 2zm-8 0a2 2 0 0 1 -2 2 2 2 0 0 1 -2-2 2 2 0 0 1 2-2 2 2 0 0 1 2 2zm-8 0a2 2 0 0 1 -2 2 2 2 0 0 1 -2-2 2 2 0 0 1 2-2 2 2 0 0 1 2 2zm-8 0a2 2 0 0 1 -2 2 2 2 0 0 1 -2-2 2 2 0 0 1 2-2 2 2 0 0 1 2 2z"
@@ -147,6 +196,7 @@
     align-items: center;
     justify-content: center;
     opacity: 0.5;
+    touch-action: none;
 
     &:hover {
       opacity: 1;

pods/backup/package.json

@@ -18,6 +18,7 @@
     "bundle": "rushx get-model && node ../../common/scripts/esbuild.js --entry=src/index.ts --keep-names=true --bundle=true --sourcemap=external --external=*.node",
     "docker:build": "../../common/scripts/docker_build.sh hardcoreeng/backup",
     "docker:tbuild": "docker build -t hardcoreeng/backup . --platform=linux/amd64 && ../../common/scripts/docker_tag_push.sh hardcoreeng/backup",
+    "docker:abuild": "docker build -t hardcoreeng/backup . --platform=linux/arm64 && ../../common/scripts/docker_tag_push.sh hardcoreeng/backup",
     "docker:staging": "../../common/scripts/docker_tag.sh hardcoreeng/backup staging",
     "docker:push": "../../common/scripts/docker_tag.sh hardcoreeng/backup",
     "run-local": "cross-env ACCOUNTS_URL=http://localhost:3000/ SECRET=secret MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin MINIO_ENDPOINT=localhost BUCKET_NAME=backups INTERVAL=30 ts-node src/index.ts",

server/indexer/src/indexer/indexer.ts

@@ -402,7 +402,7 @@ export class FullTextIndexPipeline implements FullTextPipeline {
   ): Promise<{ classUpdate: Ref<Class<Doc>>[], processed: number }> {
     const _classUpdate = new Set<Ref<Class<Doc>>>()
     let processed = 0
-    await rateLimiter.add(async () => {
+    await rateLimiter.exec(async () => {
       let st = Date.now()
 
       let groupBy = await this.storage.groupBy(ctx, DOMAIN_DOC_INDEX_STATE, 'objectClass', { needIndex: true })

server/middleware/src/spaceSecurity.ts

@@ -40,6 +40,7 @@ import core, {
   TxUpdateDoc,
   TxWorkspaceEvent,
   WorkspaceEvent,
+  clone,
   generateId,
   systemAccountEmail,
   toFindResult,
@@ -69,14 +70,14 @@ export class SpaceSecurityMiddleware extends BaseMiddleware implements Middlewar
 
   wasInit: Promise<void> | boolean = false
 
-  private readonly mainSpaces = [
+  private readonly mainSpaces = new Set([
     core.space.Configuration,
     core.space.DerivedTx,
     core.space.Model,
     core.space.Space,
     core.space.Workspace,
     core.space.Tx
-  ]
+  ])
 
   private constructor (
     private readonly skipFindCheck: boolean,
@@ -424,7 +425,7 @@ export class SpaceSecurityMiddleware extends BaseMiddleware implements Middlewar
     ctx.contextData.broadcast.targets.spaceSec = (tx) => {
       const space = this.spacesMap.get(tx.objectSpace)
       if (space === undefined) return undefined
-      if (this.systemSpaces.has(space._id) || this.mainSpaces.includes(space._id)) return undefined
+      if (this.systemSpaces.has(space._id) || this.mainSpaces.has(space._id)) return undefined
 
       return space.members.length === 0 ? undefined : this.getTargets(space?.members)
     }
@@ -455,12 +456,12 @@ export class SpaceSecurityMiddleware extends BaseMiddleware implements Middlewar
     ctx: MeasureContext,
     domain: Domain,
     spaces: Ref<Space>[]
-  ): Promise<{ result: Ref<Space>[], allDomainSpaces: boolean, domainSpaces: Set<Ref<Space>> }> {
+  ): Promise<{ result: Set<Ref<Space>>, allDomainSpaces: boolean, domainSpaces: Set<Ref<Space>> }> {
     const domainSpaces = await this.getDomainSpaces(ctx, domain)
-    const result = spaces.filter((p) => domainSpaces.has(p))
+    const result = new Set(spaces.filter((p) => domainSpaces.has(p)))
     return {
-      result: spaces.filter((p) => domainSpaces.has(p)),
+      result,
-      allDomainSpaces: result.length === domainSpaces.size,
+      allDomainSpaces: result.size === domainSpaces.size,
       domainSpaces
     }
   }
@@ -477,14 +478,14 @@ export class SpaceSecurityMiddleware extends BaseMiddleware implements Middlewar
       if (spaces.allDomainSpaces) {
         return undefined
       }
-      return { $in: spaces.result }
+      return { $in: Array.from(spaces.result) }
     }
     if (typeof query === 'string') {
-      if (!spaces.result.includes(query)) {
+      if (!spaces.result.has(query)) {
         return { $in: [] }
       }
     } else if (query.$in != null) {
-      query.$in = query.$in.filter((p) => spaces.result.includes(p))
+      query.$in = query.$in.filter((p) => spaces.result.has(p))
       if (query.$in.length === spaces.domainSpaces.size) {
         // all domain spaces
         delete query.$in
@@ -493,7 +494,7 @@ export class SpaceSecurityMiddleware extends BaseMiddleware implements Middlewar
       if (spaces.allDomainSpaces) {
         delete query.$in
       } else {
-        query.$in = spaces.result
+        query.$in = Array.from(spaces.result)
       }
     }
     if (Object.keys(query).length === 0) {
@@ -515,7 +516,7 @@ export class SpaceSecurityMiddleware extends BaseMiddleware implements Middlewar
     await this.init(ctx)
 
     const domain = this.context.hierarchy.getDomain(_class)
-    const newQuery = { ...query }
+    const newQuery = clone(query)
     const account = ctx.contextData.account
     const isSpace = this.context.hierarchy.isDerived(_class, core.class.Space)
     const field = this.getKey(domain)
@@ -528,12 +529,12 @@ export class SpaceSecurityMiddleware extends BaseMiddleware implements Middlewar
           const res = await this.mergeQuery(ctx, account, query[field], domain, isSpace)
           if (res === undefined) {
             // eslint-disable-next-line @typescript-eslint/no-dynamic-delete
-            delete (newQuery as any)[field]
+            delete newQuery[field]
           } else {
-            ;(newQuery as any)[field] = res
+            newQuery[field] = res
             if (typeof res === 'object') {
               if (Array.isArray(res.$in) && res.$in.length === 1 && Object.keys(res).length === 1) {
-                ;(newQuery as any)[field] = res.$in[0]
+                newQuery[field] = res.$in[0]
               }
             }
           }
@@ -541,25 +542,25 @@ export class SpaceSecurityMiddleware extends BaseMiddleware implements Middlewar
           const spaces = await this.filterByDomain(ctx, domain, this.getAllAllowedSpaces(account, !isSpace))
           if (spaces.allDomainSpaces) {
             // eslint-disable-next-line @typescript-eslint/no-dynamic-delete
-            delete (newQuery as any)[field]
+            delete newQuery[field]
-          } else if (spaces.result.length === 1) {
+          } else if (spaces.result.size === 1) {
-            ;(newQuery as any)[field] = spaces.result[0]
+            newQuery[field] = Array.from(spaces.result)[0]
             if (options !== undefined) {
-              options.allowedSpaces = spaces.result
+              options.allowedSpaces = Array.from(spaces.result)
             } else {
-              options = { allowedSpaces: spaces.result }
+              options = { allowedSpaces: Array.from(spaces.result) }
             }
           } else {
             // Check if spaces > 85% of all domain spaces, in this case return all and filter on client.
-            if (spaces.result.length / spaces.domainSpaces.size > 0.85 && options?.limit === undefined) {
+            if (spaces.result.size / spaces.domainSpaces.size > 0.85 && options?.limit === undefined) {
-              clientFilterSpaces = new Set(spaces.result)
+              clientFilterSpaces = spaces.result
               delete newQuery.space
             } else {
-              ;(newQuery as any)[field] = { $in: spaces.result }
+              newQuery[field] = { $in: Array.from(spaces.result) }
               if (options !== undefined) {
-                options.allowedSpaces = spaces.result
+                options.allowedSpaces = Array.from(spaces.result)
               } else {
-                options = { allowedSpaces: spaces.result }
+                options = { allowedSpaces: Array.from(spaces.result) }
               }
             }
           }
@@ -625,19 +626,19 @@ export class SpaceSecurityMiddleware extends BaseMiddleware implements Middlewar
     if (Object.keys(lookup).length === 0) return
     const account = ctx.contextData.account
     if (isSystem(account, ctx)) return
-    const allowedSpaces = this.getAllAllowedSpaces(account, true)
+    const allowedSpaces = new Set(this.getAllAllowedSpaces(account, true))
     for (const key in lookup) {
       const val = lookup[key]
       if (Array.isArray(val)) {
         const arr: AttachedDoc[] = []
         for (const value of val) {
-          if (allowedSpaces.includes(value.space)) {
+          if (allowedSpaces.has(value.space)) {
             arr.push(value)
           }
         }
         lookup[key] = arr as any
       } else if (val !== undefined) {
-        if (!allowedSpaces.includes(val.space)) {
+        if (!allowedSpaces.has(val.space)) {
           lookup[key] = undefined
         }
       }

server/mongo/src/storage.ts

@@ -1025,7 +1025,10 @@ abstract class MongoAdapterBase implements DbAdapter {
     return Date.now().toString(16) // Current hash value
   }
 
-  strimSize (str: string): string {
+  strimSize (str?: string): string {
+    if (str == null) {
+      return ''
+    }
     const pos = str.indexOf('|')
     if (pos > 0) {
       return str.substring(0, pos)
@@ -1041,8 +1044,6 @@ abstract class MongoAdapterBase implements DbAdapter {
     return {
       next: async () => {
         if (iterator === undefined) {
-          await coll.updateMany({ '%hash%': { $in: [null, ''] } }, { $set: { '%hash%': this.curHash() } })
-
           iterator = coll.find(
             {},
             {

server/postgres/src/schemas.ts

@@ -152,6 +152,11 @@ const docIndexStateSchema: Schema = {
     type: 'bool',
     notNull: true,
     index: true
+  },
+  objectClass: {
+    type: 'text',
+    notNull: true,
+    index: false
   }
 }
 
@@ -212,6 +217,40 @@ const eventSchema: Schema = {
   }
 }
 
+const docSyncInfo: Schema = {
+  ...baseSchema,
+  needSync: {
+    type: 'text',
+    notNull: false,
+    index: false
+  },
+  externalVersion: {
+    type: 'text',
+    notNull: false,
+    index: false
+  },
+  repository: {
+    type: 'text',
+    notNull: false,
+    index: false
+  },
+  url: {
+    type: 'text',
+    notNull: false,
+    index: false
+  },
+  objectClass: {
+    type: 'text',
+    notNull: false,
+    index: false
+  },
+  deleted: {
+    type: 'bool',
+    notNull: false,
+    index: false
+  }
+}
+
 export function addSchema (domain: string, schema: Schema): void {
   domainSchemas[translateDomain(domain)] = schema
   domainSchemaFields.set(domain, createSchemaFields(schema))
@@ -231,7 +270,8 @@ export const domainSchemas: Record<string, Schema> = {
   [translateDomain(DOMAIN_DOC_INDEX_STATE)]: docIndexStateSchema,
   notification: notificationSchema,
   [translateDomain('notification-dnc')]: dncSchema,
-  [translateDomain('notification-user')]: userNotificationSchema
+  [translateDomain('notification-user')]: userNotificationSchema,
+  github: docSyncInfo
 }
 
 export function getSchema (domain: string): Schema {

server/postgres/src/storage.ts

@@ -1276,7 +1276,10 @@ abstract class PostgresAdapterBase implements DbAdapter {
     return []
   }
 
-  strimSize (str: string): string {
+  strimSize (str?: string): string {
+    if (str == null) {
+      return ''
+    }
     const pos = str.indexOf('|')
     if (pos > 0) {
       return str.substring(0, pos)
@@ -1308,12 +1311,6 @@ abstract class PostgresAdapterBase implements DbAdapter {
           if (client === undefined) {
             client = await this.client.reserve()
           }
-
-          // We need update hash to be set properly
-          await client.unsafe(
-            `UPDATE ${tdomain} SET "%hash%" = '${this.curHash()}' WHERE "workspaceId" = '${this.workspaceId.name}' AND "%hash%" IS NULL OR "%hash%" = ''`
-          )
-
           initialized = true
           bulk = createBulk('_id, "%hash%"')
         }

server/postgres/src/utils.ts

@@ -283,7 +283,7 @@ export function convertDoc<T extends Doc> (
     modifiedOn: doc.modifiedOn,
     createdOn: doc.createdOn ?? doc.modifiedOn,
     _class: doc._class,
-    '%hash%': (doc as any)['%hash%'] ?? null
+    '%hash%': (doc as any)['%hash%'] ?? Date.now().toString(16)
   }
   const remainingData: Partial<T> = {}
 

server/rpc/src/rpc.ts

@@ -106,7 +106,13 @@ export class RPCHandler {
         const decoder = new TextDecoder()
         _data = decoder.decode(_data)
       }
+      try {
         return JSON.parse(_data.toString(), receiver)
+      } catch (err: any) {
+        if (((err.message as string) ?? '').includes('Unexpected token')) {
+          return this.packr.unpack(new Uint8Array(data))
+        }
+      }
     }
     return this.packr.unpack(new Uint8Array(data))
   }

workers/datalake/src/blob.ts

@@ -14,15 +14,14 @@
 //
 
 import { error, json } from 'itty-router'
-import { type Sql } from 'postgres'
+import { type BlobDB, withPostgres } from './db'
-import db, { withPostgres } from './db'
 import { cacheControl, hashLimit } from './const'
 import { toUUID } from './encodings'
 import { getSha256 } from './hash'
 import { selectStorage } from './storage'
 import { type BlobRequest, type WorkspaceRequest, type UUID } from './types'
 import { copyVideo, deleteVideo } from './video'
-import { measure, LoggedCache } from './measure'
+import { type MetricsContext, LoggedCache } from './metrics'
 
 interface BlobMetadata {
   lastModified: number
@@ -36,20 +35,24 @@ export function getBlobURL (request: Request, workspace: string, name: string):
   return new URL(path, request.url).toString()
 }
 
-export async function handleBlobGet (request: BlobRequest, env: Env, ctx: ExecutionContext): Promise<Response> {
+export async function handleBlobGet (
+  request: BlobRequest,
+  env: Env,
+  ctx: ExecutionContext,
+  metrics: MetricsContext
+): Promise<Response> {
   const { workspace, name } = request
 
-  const cache = new LoggedCache(caches.default)
+  const cache = new LoggedCache(caches.default, metrics)
   const cached = await cache.match(request)
   if (cached !== undefined) {
-    console.log({ message: 'cache hit' })
     return cached
   }
 
   const { bucket } = selectStorage(env, workspace)
 
-  const blob = await withPostgres(env, ctx, (sql) => {
+  const blob = await withPostgres(env, ctx, metrics, (db) => {
-    return db.getBlob(sql, { workspace, name })
+    return db.getBlob({ workspace, name })
   })
   if (blob === null || blob.deleted) {
     return error(404)
@@ -72,19 +75,25 @@ export async function handleBlobGet (request: BlobRequest, env: Env, ctx: Execut
   const response = new Response(object?.body, { headers, status })
 
   if (response.status === 200) {
-    ctx.waitUntil(cache.put(request, response.clone()))
+    const clone = metrics.withSync('response.clone', () => response.clone())
+    ctx.waitUntil(cache.put(request, clone))
   }
 
   return response
 }
 
-export async function handleBlobHead (request: BlobRequest, env: Env, ctx: ExecutionContext): Promise<Response> {
+export async function handleBlobHead (
+  request: BlobRequest,
+  env: Env,
+  ctx: ExecutionContext,
+  metrics: MetricsContext
+): Promise<Response> {
   const { workspace, name } = request
 
   const { bucket } = selectStorage(env, workspace)
 
-  const blob = await withPostgres(env, ctx, (sql) => {
+  const blob = await withPostgres(env, ctx, metrics, (db) => {
-    return db.getBlob(sql, { workspace, name })
+    return db.getBlob({ workspace, name })
   })
   if (blob === null || blob.deleted) {
     return error(404)
@@ -99,12 +108,17 @@ export async function handleBlobHead (request: BlobRequest, env: Env, ctx: Execu
   return new Response(null, { headers, status: 200 })
 }
 
-export async function handleBlobDelete (request: BlobRequest, env: Env, ctx: ExecutionContext): Promise<Response> {
+export async function handleBlobDelete (
+  request: BlobRequest,
+  env: Env,
+  ctx: ExecutionContext,
+  metrics: MetricsContext
+): Promise<Response> {
   const { workspace, name } = request
 
   try {
-    await withPostgres(env, ctx, (sql) => {
+    await withPostgres(env, ctx, metrics, (db) => {
-      return Promise.all([db.deleteBlob(sql, { workspace, name }), deleteVideo(env, workspace, name)])
+      return Promise.all([db.deleteBlob({ workspace, name }), deleteVideo(env, workspace, name)])
     })
 
     return new Response(null, { status: 204 })
@@ -118,7 +132,8 @@ export async function handleBlobDelete (request: BlobRequest, env: Env, ctx: Exe
 export async function handleUploadFormData (
   request: WorkspaceRequest,
   env: Env,
-  ctx: ExecutionContext
+  ctx: ExecutionContext,
+  metrics: MetricsContext
 ): Promise<Response> {
   const contentType = request.headers.get('Content-Type')
   if (contentType === null || !contentType.includes('multipart/form-data')) {
@@ -130,7 +145,7 @@ export async function handleUploadFormData (
 
   let formData: FormData
   try {
-    formData = await measure('fetch formdata', () => request.formData())
+    formData = await metrics.with('request.formData', () => request.formData())
   } catch (err: any) {
     const message = err instanceof Error ? err.message : String(err)
     console.error({ error: 'failed to parse form data', message })
@@ -146,8 +161,8 @@ export async function handleUploadFormData (
     files.map(async ([file, key]) => {
       const { name, type, lastModified } = file
       try {
-        const metadata = await withPostgres(env, ctx, (sql) => {
+        const metadata = await withPostgres(env, ctx, metrics, (db) => {
-          return saveBlob(env, sql, file.stream(), file.size, type, workspace, name, lastModified)
+          return saveBlob(env, db, file.stream(), file.size, type, workspace, name, lastModified)
         })
 
         // TODO this probably should happen via queue, let it be here for now
@@ -170,7 +185,7 @@ export async function handleUploadFormData (
 
 export async function saveBlob (
   env: Env,
-  sql: Sql,
+  db: BlobDB,
   stream: ReadableStream,
   size: number,
   type: string,
@@ -187,17 +202,15 @@ export async function saveBlob (
     const [hashStream, uploadStream] = stream.tee()
 
     const hash = await getSha256(hashStream)
-    const data = await db.getData(sql, { hash, location })
+    const data = await db.getData({ hash, location })
 
     if (data !== null) {
       // Lucky boy, nothing to upload, use existing blob
-      await db.createBlob(sql, { workspace, name, hash, location })
+      await db.createBlob({ workspace, name, hash, location })
     } else {
       await bucket.put(filename, uploadStream, { httpMetadata })
-      await sql.begin((sql) => [
+      await db.createData({ hash, location, filename, type, size })
-        db.createData(sql, { hash, location, filename, type, size }),
+      await db.createBlob({ workspace, name, hash, location })
-        db.createBlob(sql, { workspace, name, hash, location })
-      ])
     }
 
     return { type, size, lastModified, name }
@@ -205,17 +218,15 @@ export async function saveBlob (
     // For large files we cannot calculate checksum beforehead
     // upload file with unique filename and then obtain checksum
     const { hash } = await uploadLargeFile(bucket, stream, filename, { httpMetadata })
-    const data = await db.getData(sql, { hash, location })
+    const data = await db.getData({ hash, location })
     if (data !== null) {
       // We found an existing blob with the same hash
       // we can safely remove the existing blob from storage
-      await Promise.all([bucket.delete(filename), db.createBlob(sql, { workspace, name, hash, location })])
+      await Promise.all([bucket.delete(filename), db.createBlob({ workspace, name, hash, location })])
     } else {
       // Otherwise register a new hash and blob
-      await sql.begin((sql) => [
+      await db.createData({ hash, location, filename, type, size })
-        db.createData(sql, { hash, location, filename, type, size }),
+      await db.createBlob({ workspace, name, hash, location })
-        db.createBlob(sql, { workspace, name, hash, location })
-      ])
     }
 
     return { type, size, lastModified, name }
@@ -225,6 +236,7 @@ export async function saveBlob (
 export async function handleBlobUploaded (
   env: Env,
   ctx: ExecutionContext,
+  metrics: MetricsContext,
   workspace: string,
   name: string,
   filename: UUID
@@ -238,18 +250,16 @@ export async function handleBlobUploaded (
 
   const hash = object.checksums.md5 !== undefined ? digestToUUID(object.checksums.md5) : (crypto.randomUUID() as UUID)
 
-  await withPostgres(env, ctx, async (sql) => {
+  await withPostgres(env, ctx, metrics, async (db) => {
-    const data = await db.getData(sql, { hash, location })
+    const data = await db.getData({ hash, location })
     if (data !== null) {
-      await Promise.all([bucket.delete(filename), db.createBlob(sql, { workspace, name, hash, location })])
+      await Promise.all([bucket.delete(filename), db.createBlob({ workspace, name, hash, location })])
     } else {
       const size = object.size
       const type = object.httpMetadata?.contentType ?? 'application/octet-stream'
 
-      await sql.begin((sql) => [
+      await db.createData({ hash, location, filename, type, size })
-        db.createData(sql, { hash, location, filename, type, size }),
+      await db.createBlob({ workspace, name, hash, location })
-        db.createBlob(sql, { workspace, name, hash, location })
-      ])
     }
   })
 }

workers/datalake/src/db.ts

@@ -14,7 +14,7 @@
 //
 
 import postgres from 'postgres'
-import { measure, measureSync } from './measure'
+import { type MetricsContext } from './metrics'
 import { type Location, type UUID } from './types'
 
 export interface BlobDataId {
@@ -46,52 +46,62 @@ export interface BlobRecordWithFilename extends BlobRecord {
 export async function withPostgres<T> (
   env: Env,
   ctx: ExecutionContext,
-  fn: (sql: postgres.Sql) => Promise<T>
+  metrics: MetricsContext,
+  fn: (db: BlobDB) => Promise<T>
 ): Promise<T> {
-  const sql = measureSync('db.connect', () => {
+  const sql = metrics.withSync('db.connect', () => {
-    return postgres(env.HYPERDRIVE.connectionString)
+    return postgres(env.HYPERDRIVE.connectionString, {
+      connection: {
+        application_name: 'datalake'
+      }
     })
+  })
+  const db = new LoggedDB(new PostgresDB(sql), metrics)
+
   try {
-    return await fn(sql)
+    return await fn(db)
   } finally {
-    measureSync('db.close', () => {
+    metrics.withSync('db.disconnect', () => {
       ctx.waitUntil(sql.end({ timeout: 0 }))
     })
   }
 }
 
 export interface BlobDB {
-  getData: (sql: postgres.Sql, dataId: BlobDataId) => Promise<BlobDataRecord | null>
+  getData: (dataId: BlobDataId) => Promise<BlobDataRecord | null>
-  createData: (sql: postgres.Sql, data: BlobDataRecord) => Promise<void>
+  createData: (data: BlobDataRecord) => Promise<void>
-  getBlob: (sql: postgres.Sql, blobId: BlobId) => Promise<BlobRecordWithFilename | null>
+  getBlob: (blobId: BlobId) => Promise<BlobRecordWithFilename | null>
-  createBlob: (sql: postgres.Sql, blob: Omit<BlobRecord, 'filename' | 'deleted'>) => Promise<void>
+  createBlob: (blob: Omit<BlobRecord, 'filename' | 'deleted'>) => Promise<void>
-  deleteBlob: (sql: postgres.Sql, blob: BlobId) => Promise<void>
+  deleteBlob: (blob: BlobId) => Promise<void>
 }
 
-const db: BlobDB = {
+export class PostgresDB implements BlobDB {
-  async getData (sql: postgres.Sql, dataId: BlobDataId): Promise<BlobDataRecord | null> {
+  constructor (private readonly sql: postgres.Sql) {}
+
+  async getData (dataId: BlobDataId): Promise<BlobDataRecord | null> {
     const { hash, location } = dataId
-    const rows = await sql<BlobDataRecord[]>`
+    const rows = await this.sql<BlobDataRecord[]>`
       SELECT hash, location, filename, size, type
       FROM blob.data
       WHERE hash = ${hash} AND location = ${location}
     `
     return rows.length > 0 ? rows[0] : null
-  },
+  }
 
-  async createData (sql: postgres.Sql, data: BlobDataRecord): Promise<void> {
+  async createData (data: BlobDataRecord): Promise<void> {
     const { hash, location, filename, size, type } = data
 
-    await sql`
+    await this.sql`
       UPSERT INTO blob.data (hash, location, filename, size, type)
       VALUES (${hash}, ${location}, ${filename}, ${size}, ${type})
     `
-  },
+  }
 
-  async getBlob (sql: postgres.Sql, blobId: BlobId): Promise<BlobRecordWithFilename | null> {
+  async getBlob (blobId: BlobId): Promise<BlobRecordWithFilename | null> {
     const { workspace, name } = blobId
 
-    const rows = await sql<BlobRecordWithFilename[]>`
+    try {
+      const rows = await this.sql<BlobRecordWithFilename[]>`
         SELECT b.workspace, b.name, b.hash, b.location, b.deleted, d.filename
         FROM blob.blob AS b
         JOIN blob.data AS d ON b.hash = d.hash AND b.location = d.location
@@ -101,23 +111,26 @@ const db: BlobDB = {
       if (rows.length > 0) {
         return rows[0]
       }
+    } catch (err) {
+      console.error(err)
+    }
 
     return null
-  },
+  }
 
-  async createBlob (sql: postgres.Sql, blob: Omit<BlobRecord, 'filename' | 'deleted'>): Promise<void> {
+  async createBlob (blob: Omit<BlobRecord, 'filename' | 'deleted'>): Promise<void> {
     const { workspace, name, hash, location } = blob
 
-    await sql`
+    await this.sql`
       UPSERT INTO blob.blob (workspace, name, hash, location, deleted)
       VALUES (${workspace}, ${name}, ${hash}, ${location}, false)
     `
-  },
+  }
 
-  async deleteBlob (sql: postgres.Sql, blob: BlobId): Promise<void> {
+  async deleteBlob (blob: BlobId): Promise<void> {
     const { workspace, name } = blob
 
-    await sql`
+    await this.sql`
       UPDATE blob.blob
       SET deleted = true
       WHERE workspace = ${workspace} AND name = ${name}
@@ -125,12 +138,29 @@ const db: BlobDB = {
   }
 }
 
-export const measuredDb: BlobDB = {
+export class LoggedDB implements BlobDB {
-  getData: (sql, dataId) => measure('db.getData', () => db.getData(sql, dataId)),
+  constructor (
-  createData: (sql, data) => measure('db.createData', () => db.createData(sql, data)),
+    private readonly db: BlobDB,
-  getBlob: (sql, blobId) => measure('db.getBlob', () => db.getBlob(sql, blobId)),
+    private readonly ctx: MetricsContext
-  createBlob: (sql, blob) => measure('db.createBlob', () => db.createBlob(sql, blob)),
+  ) {}
-  deleteBlob: (sql, blob) => measure('db.deleteBlob', () => db.deleteBlob(sql, blob))
-}
 
-export default measuredDb
+  async getData (dataId: BlobDataId): Promise<BlobDataRecord | null> {
+    return await this.ctx.with('db.getData', () => this.db.getData(dataId))
+  }
+
+  async createData (data: BlobDataRecord): Promise<void> {
+    await this.ctx.with('db.createData', () => this.db.createData(data))
+  }
+
+  async getBlob (blobId: BlobId): Promise<BlobRecordWithFilename | null> {
+    return await this.ctx.with('db.getBlob', () => this.db.getBlob(blobId))
+  }
+
+  async createBlob (blob: Omit<BlobRecord, 'filename' | 'deleted'>): Promise<void> {
+    await this.ctx.with('db.createBlob', () => this.db.createBlob(blob))
+  }
+
+  async deleteBlob (blob: BlobId): Promise<void> {
+    await this.ctx.with('db.deleteBlob', () => this.db.deleteBlob(blob))
+  }
+}

workers/datalake/src/image.ts

@@ -14,11 +14,17 @@
 //
 
 import { getBlobURL } from './blob'
+import { type MetricsContext } from './metrics'
 import { type BlobRequest } from './types'
 
 const prefferedImageFormats = ['webp', 'avif', 'jpeg', 'png']
 
-export async function handleImageGet (request: BlobRequest): Promise<Response> {
+export async function handleImageGet (
+  request: BlobRequest,
+  env: Env,
+  ctx: ExecutionContext,
+  metrics: MetricsContext
+): Promise<Response> {
   const {
     workspace,
     name,
@@ -48,5 +54,5 @@ export async function handleImageGet (request: BlobRequest): Promise<Response> {
 
   const blobURL = getBlobURL(request, workspace, name)
   const imageRequest = new Request(blobURL, { headers: { Accept } })
-  return await fetch(imageRequest, { cf: { image, cacheTtl: 3600 } })
+  return await metrics.with('image.transform', () => fetch(imageRequest, { cf: { image, cacheTtl: 3600 } }))
 }

workers/datalake/src/index.ts

@@ -14,11 +14,11 @@
 //
 
 import { WorkerEntrypoint } from 'cloudflare:workers'
-import { type IRequestStrict, type RequestHandler, Router, error, html } from 'itty-router'
+import { type IRequest, type IRequestStrict, type RequestHandler, Router, error, html } from 'itty-router'
 
 import { handleBlobDelete, handleBlobGet, handleBlobHead, handleUploadFormData } from './blob'
 import { cors } from './cors'
-import { LoggedKVNamespace, LoggedR2Bucket, requestTimeAfter, requestTimeBefore } from './measure'
+import { LoggedKVNamespace, LoggedR2Bucket, MetricsContext } from './metrics'
 import { handleImageGet } from './image'
 import { handleS3Blob } from './s3'
 import { handleVideoMetaGet } from './video'
@@ -36,8 +36,8 @@ const { preflight, corsify } = cors({
 })
 
 const router = Router<IRequestStrict, [Env, ExecutionContext], Response>({
-  before: [preflight, requestTimeBefore],
+  before: [preflight],
-  finally: [corsify, requestTimeAfter]
+  finally: [corsify]
 })
 
 const withWorkspace: RequestHandler<WorkspaceRequest> = (request: WorkspaceRequest) => {
@@ -88,21 +88,29 @@ router
   .all('*', () => error(404))
 
 export default class DatalakeWorker extends WorkerEntrypoint<Env> {
-  constructor (ctx: ExecutionContext, env: Env) {
+  async fetch (request: IRequest): Promise<Response> {
-    env = {
+    const start = performance.now()
-      ...env,
+    const context = new MetricsContext()
-      datalake_blobs: new LoggedKVNamespace(env.datalake_blobs),
+
-      DATALAKE_APAC: new LoggedR2Bucket(env.DATALAKE_APAC),
+    const env = {
-      DATALAKE_EEUR: new LoggedR2Bucket(env.DATALAKE_EEUR),
+      ...this.env,
-      DATALAKE_WEUR: new LoggedR2Bucket(env.DATALAKE_WEUR),
+      datalake_blobs: new LoggedKVNamespace(this.env.datalake_blobs, context),
-      DATALAKE_ENAM: new LoggedR2Bucket(env.DATALAKE_ENAM),
+      DATALAKE_APAC: new LoggedR2Bucket(this.env.DATALAKE_APAC, context),
-      DATALAKE_WNAM: new LoggedR2Bucket(env.DATALAKE_WNAM)
+      DATALAKE_EEUR: new LoggedR2Bucket(this.env.DATALAKE_EEUR, context),
-    }
+      DATALAKE_WEUR: new LoggedR2Bucket(this.env.DATALAKE_WEUR, context),
-    super(ctx, env)
+      DATALAKE_ENAM: new LoggedR2Bucket(this.env.DATALAKE_ENAM, context),
+      DATALAKE_WNAM: new LoggedR2Bucket(this.env.DATALAKE_WNAM, context)
     }
 
-  async fetch (request: Request): Promise<Response> {
+    try {
-    return await router.fetch(request, this.env, this.ctx).catch(error)
+      return await router.fetch(request, env, this.ctx, context).catch(error)
+    } finally {
+      const total = performance.now() - start
+      const ops = context.metrics
+      const url = `${request.method} ${request.url}`
+      const message = `total=${total} ` + context.toString()
+      console.log({ message, total, ops, url })
+    }
   }
 
   async getBlob (workspace: string, name: string): Promise<ArrayBuffer> {

workers/datalake/src/metrics.ts

@@ -13,42 +13,47 @@
 // limitations under the License.
 //
 
-import { type IRequest, type ResponseHandler, type RequestHandler } from 'itty-router'
+export interface MetricsData {
+  name: string
+  time: number
+}
 
-export async function measure<T> (label: string, fn: () => Promise<T>): Promise<T> {
+export class MetricsContext {
+  metrics: Array<MetricsData> = []
+
+  async with<T>(name: string, fn: () => Promise<T>): Promise<T> {
     const start = performance.now()
     try {
       return await fn()
     } finally {
-    const duration = performance.now() - start
+      const time = performance.now() - start
-    console.log({ stage: label, duration })
+      this.metrics.push({ name, time })
+    }
   }
-}
 
-export function measureSync<T> (label: string, fn: () => T): T {
+  withSync<T>(name: string, fn: () => T): T {
     const start = performance.now()
     try {
       return fn()
     } finally {
-    const duration = performance.now() - start
+      const time = performance.now() - start
-    console.log({ stage: label, duration })
+      this.metrics.push({ name, time })
+    }
+  }
+
+  toString (): string {
+    return this.metrics.map((p) => `${p.name}=${p.time}`).join(' ')
   }
 }
 
-export const requestTimeBefore: RequestHandler<IRequest> = async (request: IRequest) => {
-  request.startTime = performance.now()
-}
-
-export const requestTimeAfter: ResponseHandler<Response> = async (response: Response, request: IRequest) => {
-  const duration = performance.now() - request.startTime
-  console.log({ stage: 'total', duration })
-}
-
 export class LoggedR2Bucket implements R2Bucket {
-  constructor (private readonly bucket: R2Bucket) {}
+  constructor (
+    private readonly bucket: R2Bucket,
+    private readonly ctx: MetricsContext
+  ) {}
 
   async head (key: string): Promise<R2Object | null> {
-    return await measure('r2.head', () => this.bucket.head(key))
+    return await this.ctx.with('r2.head', () => this.bucket.head(key))
   }
 
   async get (
@@ -57,7 +62,7 @@ export class LoggedR2Bucket implements R2Bucket {
       onlyIf?: R2Conditional | Headers
     }
   ): Promise<R2ObjectBody | null> {
-    return await measure('r2.get', () => this.bucket.get(key, options))
+    return await this.ctx.with('r2.get', () => this.bucket.get(key, options))
   }
 
   async put (
@@ -67,28 +72,31 @@ export class LoggedR2Bucket implements R2Bucket {
       onlyIf?: R2Conditional | Headers
     }
   ): Promise<R2Object> {
-    return await measure('r2.put', () => this.bucket.put(key, value, options))
+    return await this.ctx.with('r2.put', () => this.bucket.put(key, value, options))
   }
 
   async createMultipartUpload (key: string, options?: R2MultipartOptions): Promise<R2MultipartUpload> {
-    return await measure('r2.createMultipartUpload', () => this.bucket.createMultipartUpload(key, options))
+    return await this.ctx.with('r2.createMultipartUpload', () => this.bucket.createMultipartUpload(key, options))
   }
 
   resumeMultipartUpload (key: string, uploadId: string): R2MultipartUpload {
-    return measureSync('r2.resumeMultipartUpload', () => this.bucket.resumeMultipartUpload(key, uploadId))
+    return this.ctx.withSync('r2.resumeMultipartUpload', () => this.bucket.resumeMultipartUpload(key, uploadId))
   }
 
   async delete (keys: string | string[]): Promise<void> {
-    await measure('r2.delete', () => this.bucket.delete(keys))
+    await this.ctx.with('r2.delete', () => this.bucket.delete(keys))
   }
 
   async list (options?: R2ListOptions): Promise<R2Objects> {
-    return await measure('r2.list', () => this.bucket.list(options))
+    return await this.ctx.with('r2.list', () => this.bucket.list(options))
   }
 }
 
 export class LoggedKVNamespace implements KVNamespace {
-  constructor (private readonly kv: KVNamespace) {}
+  constructor (
+    private readonly kv: KVNamespace,
+    private readonly ctx: MetricsContext
+  ) {}
 
   get (key: string, options?: Partial<KVNamespaceGetOptions<undefined>>): Promise<string | null>
   get (key: string, type: 'text'): Promise<string | null>
@@ -100,7 +108,7 @@ export class LoggedKVNamespace implements KVNamespace {
   get (key: string, options?: KVNamespaceGetOptions<'arrayBuffer'>): Promise<ArrayBuffer | null>
   get (key: string, options?: KVNamespaceGetOptions<'stream'>): Promise<ReadableStream | null>
   async get (key: string, options?: any): Promise<any> {
-    return await measure('kv.get', () => this.kv.get(key, options))
+    return await this.ctx.with('kv.get', () => this.kv.get(key, options))
   }
 
   getWithMetadata<Metadata = unknown>(
@@ -140,11 +148,11 @@ export class LoggedKVNamespace implements KVNamespace {
     options?: KVNamespaceGetOptions<'stream'>
   ): Promise<KVNamespaceGetWithMetadataResult<ReadableStream, Metadata>>
   async getWithMetadata (key: string, options?: any): Promise<any> {
-    return await measure('kv.getWithMetadata', () => this.kv.getWithMetadata(key, options))
+    return await this.ctx.with('kv.getWithMetadata', () => this.kv.getWithMetadata(key, options))
   }
 
   async list<Metadata = unknown>(options?: KVNamespaceListOptions): Promise<KVNamespaceListResult<Metadata, string>> {
-    return await measure('kv.list', () => this.kv.list(options))
+    return await this.ctx.with('kv.list', () => this.kv.list(options))
   }
 
   async put (
@@ -152,26 +160,29 @@ export class LoggedKVNamespace implements KVNamespace {
     value: string | ArrayBuffer | ArrayBufferView | ReadableStream,
     options?: KVNamespacePutOptions
   ): Promise<void> {
-    await measure('kv.put', () => this.kv.put(key, value))
+    await this.ctx.with('kv.put', () => this.kv.put(key, value))
   }
 
   async delete (key: string): Promise<void> {
-    await measure('kv.delete', () => this.kv.delete(key))
+    await this.ctx.with('kv.delete', () => this.kv.delete(key))
   }
 }
 
 export class LoggedCache implements Cache {
-  constructor (private readonly cache: Cache) {}
+  constructor (
+    private readonly cache: Cache,
+    private readonly ctx: MetricsContext
+  ) {}
 
   async match (request: RequestInfo, options?: CacheQueryOptions): Promise<Response | undefined> {
-    return await measure('cache.match', () => this.cache.match(request, options))
+    return await this.ctx.with('cache.match', () => this.cache.match(request, options))
   }
 
   async delete (request: RequestInfo, options?: CacheQueryOptions): Promise<boolean> {
-    return await measure('cache.delete', () => this.cache.delete(request, options))
+    return await this.ctx.with('cache.delete', () => this.cache.delete(request, options))
   }
 
   async put (request: RequestInfo, response: Response): Promise<void> {
-    await measure('cache.put', () => this.cache.put(request, response))
+    await this.ctx.with('cache.put', () => this.cache.put(request, response))
   }
 }

workers/datalake/src/multipart.ts

@@ -14,9 +14,10 @@
 //
 
 import { error, json } from 'itty-router'
-import db, { withPostgres } from './db'
+import { withPostgres } from './db'
 import { cacheControl } from './const'
 import { toUUID } from './encodings'
+import { type MetricsContext } from './metrics'
 import { selectStorage } from './storage'
 import { type BlobRequest, type UUID } from './types'
 
@@ -82,7 +83,8 @@ export async function handleMultipartUploadPart (
 export async function handleMultipartUploadComplete (
   request: BlobRequest,
   env: Env,
-  ctx: ExecutionContext
+  ctx: ExecutionContext,
+  metrics: MetricsContext
 ): Promise<Response> {
   const { workspace, name } = request
 
@@ -105,17 +107,15 @@ export async function handleMultipartUploadComplete (
   const size = object.size ?? 0
   const filename = multipartKey as UUID
 
-  await withPostgres(env, ctx, async (sql) => {
+  await withPostgres(env, ctx, metrics, async (db) => {
-    const data = await db.getData(sql, { hash, location })
+    const data = await db.getData({ hash, location })
     if (data !== null) {
       // blob already exists
-      await Promise.all([bucket.delete(filename), db.createBlob(sql, { workspace, name, hash, location })])
+      await Promise.all([bucket.delete(filename), db.createBlob({ workspace, name, hash, location })])
     } else {
       // Otherwise register a new hash and blob
-      await sql.begin((sql) => [
+      await db.createData({ hash, location, filename, type, size })
-        db.createData(sql, { hash, location, filename, type, size }),
+      await db.createBlob({ workspace, name, hash, location })
-        db.createBlob(sql, { workspace, name, hash, location })
-      ])
     }
   })
 

workers/datalake/src/s3.ts

@@ -15,8 +15,9 @@
 
 import { AwsClient } from 'aws4fetch'
 import { error, json } from 'itty-router'
-import db, { withPostgres } from './db'
+import { withPostgres } from './db'
 import { saveBlob } from './blob'
+import { type MetricsContext } from './metrics'
 import { type BlobRequest } from './types'
 
 export interface S3UploadPayload {
@@ -35,16 +36,21 @@ function getS3Client (payload: S3UploadPayload): AwsClient {
   })
 }
 
-export async function handleS3Blob (request: BlobRequest, env: Env, ctx: ExecutionContext): Promise<Response> {
+export async function handleS3Blob (
+  request: BlobRequest,
+  env: Env,
+  ctx: ExecutionContext,
+  metrics: MetricsContext
+): Promise<Response> {
   const { workspace, name } = request
 
   const payload = await request.json<S3UploadPayload>()
 
   const client = getS3Client(payload)
 
-  return await withPostgres(env, ctx, async (sql) => {
+  return await withPostgres(env, ctx, metrics, async (db) => {
     // Ensure the blob does not exist
-    const blob = await db.getBlob(sql, { workspace, name })
+    const blob = await db.getBlob({ workspace, name })
     if (blob !== null) {
       return new Response(null, { status: 200 })
     }
@@ -65,7 +71,7 @@ export async function handleS3Blob (request: BlobRequest, env: Env, ctx: Executi
     const contentLength = Number.parseInt(contentLengthHeader)
     const lastModified = lastModifiedHeader !== null ? new Date(lastModifiedHeader).getTime() : Date.now()
 
-    const result = await saveBlob(env, sql, object.body, contentLength, contentType, workspace, name, lastModified)
+    const result = await saveBlob(env, db, object.body, contentLength, contentType, workspace, name, lastModified)
     return json(result)
   })
 }

workers/datalake/src/sign.ts

@@ -17,8 +17,9 @@ import { AwsClient } from 'aws4fetch'
 import { error } from 'itty-router'
 
 import { handleBlobUploaded } from './blob'
+import { type MetricsContext } from './metrics'
+import { type Storage, selectStorage } from './storage'
 import { type BlobRequest, type UUID } from './types'
-import { selectStorage, type Storage } from './storage'
 
 const S3_SIGNED_LINK_TTL = 3600
 
@@ -39,7 +40,12 @@ function getS3Client (storage: Storage): AwsClient {
   })
 }
 
-export async function handleSignCreate (request: BlobRequest, env: Env, ctx: ExecutionContext): Promise<Response> {
+export async function handleSignCreate (
+  request: BlobRequest,
+  env: Env,
+  ctx: ExecutionContext,
+  metrics: MetricsContext
+): Promise<Response> {
   const { workspace, name } = request
   const storage = selectStorage(env, workspace)
   const accountId = env.R2_ACCOUNT_ID
@@ -57,7 +63,9 @@ export async function handleSignCreate (request: BlobRequest, env: Env, ctx: Exe
   try {
     const client = getS3Client(storage)
 
-    signed = await client.sign(new Request(url, { method: 'PUT' }), { aws: { signQuery: true } })
+    signed = await metrics.with('s3.sign', () => {
+      return client.sign(new Request(url, { method: 'PUT' }), { aws: { signQuery: true } })
+    })
   } catch (err: any) {
     console.error({ error: 'failed to generate signed url', message: `${err}` })
     return error(500, 'failed to generate signed url')
@@ -73,7 +81,12 @@ export async function handleSignCreate (request: BlobRequest, env: Env, ctx: Exe
   return new Response(signed.url, { status: 200, headers })
 }
 
-export async function handleSignComplete (request: BlobRequest, env: Env, ctx: ExecutionContext): Promise<Response> {
+export async function handleSignComplete (
+  request: BlobRequest,
+  env: Env,
+  ctx: ExecutionContext,
+  metrics: MetricsContext
+): Promise<Response> {
   const { workspace, name } = request
 
   const { bucket } = selectStorage(env, workspace)
@@ -96,7 +109,7 @@ export async function handleSignComplete (request: BlobRequest, env: Env, ctx: E
   }
 
   try {
-    await handleBlobUploaded(env, ctx, workspace, name, uuid)
+    await handleBlobUploaded(env, ctx, metrics, workspace, name, uuid)
   } catch (err) {
     const message = err instanceof Error ? err.message : String(err)
     console.error({ error: message, workspace, name, uuid })

workers/datalake/wrangler.toml

@@ -95,7 +95,7 @@ r2_buckets = [
 ]
 
 hyperdrive = [
-  { binding = "HYPERDRIVE", id = "055e968f3067414eaa30467d8a9c5021" }
+  { binding = "HYPERDRIVE", id = "055e968f3067414eaa30467d8a9c5021", localConnectionString = "postgresql://root:roach@localhost:26257/datalake" }
 ]
 
 [env.dev.vars]