UBERF-11398: Fixing rate limits (#9143)

Signed-off-by: Andrey Sobolev <haiodo@gmail.com>
2025-06-11 21:11:57 +00:00 · 2025-06-02 10:03:24 +07:00 · 2025-06-02 10:03:24 +07:00 · bd6bbf8d15
commit bd6bbf8d15
parent 96c6e4f6f4
15 changed files with 312 additions and 146 deletions
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@ -261,26 +261,26 @@ jobs:
        run: |
          cd ./tests/sanity
          node ../../common/scripts/install-run-rushx.js ci
-      - name: Start profiling
-        run: |
-          cd ./tests
-          ./profile-start.sh
+      # - name: Start profiling
+      #   run: |
+      #     cd ./tests
+      #     ./profile-start.sh
      - name: Run UI tests
        run: |
          cd ./tests/sanity          
          node ../../common/scripts/install-run-rushx.js uitest
-      - name: Download profile
-        run: |
-          cd ./tests
-          ./profile-download.sh
-          npm install -g cpupro
-          ./profile-generate.sh
-      - name: Upload profiling results
-        if: always()
-        uses: actions/upload-artifact@v4
-        with:
-          name: profiling
-          path: ./tests/profiles
+      # - name: Download profile
+      #   run: |
+      #     cd ./tests
+      #     ./profile-download.sh
+      #     npm install -g cpupro
+      #     ./profile-generate.sh
+      # - name: Upload profiling results
+      #   if: always()
+      #   uses: actions/upload-artifact@v4
+      #   with:
+      #     name: profiling
+      #     path: ./tests/profiles
      - name: 'Store docker logs'
        if: always()
        run: |
--- a/common/config/rush/pnpm-lock.yaml
+++ b/common/config/rush/pnpm-lock.yaml
@ -504,7 +504,7 @@ importers:
        version: file:projects/model-communication.tgz
      '@rush-temp/model-contact':
        specifier: file:./projects/model-contact.tgz
-        version: file:projects/model-contact.tgz(encoding@0.1.13)
+        version: file:projects/model-contact.tgz
      '@rush-temp/model-controlled-documents':
        specifier: file:./projects/model-controlled-documents.tgz
        version: file:projects/model-controlled-documents.tgz
@ -1572,7 +1572,7 @@ importers:
        version: 3.0.11
      '@types/node-fetch':
        specifier: ~2.6.2
-        version: 2.6.11
+        version: 2.6.12
      '@types/nodemailer':
        specifier: ^6.4.17
        version: 6.4.17
@ -1690,9 +1690,6 @@ importers:
      cross-env:
        specifier: ~7.0.3
        version: 7.0.3
-      cross-fetch:
-        specifier: ^3.1.5
-        version: 3.1.8(encoding@0.1.13)
      crypto-js:
        specifier: ^4.2.0
        version: 4.2.0
@ -4473,7 +4470,7 @@ packages:
    version: 0.0.0

  '@rush-temp/model-contact@file:projects/model-contact.tgz':
-    resolution: {integrity: sha512-+wfUTUKjOzRKHOpcINUdbA6SGyJOYE+hxIiY2ZAL/3x6CqWb720+uhpkmNqb88ZXubhrOCT4z/nDXg29QkqL4A==, tarball: file:projects/model-contact.tgz}
+    resolution: {integrity: sha512-pUqoMMgSoArIiZmW9SL5BBapmezesVqgb6k0lCNNswPAP+UoKOChFTyqNAzzeTZ1AiLms4hvXbPEVlM1tKMKfg==, tarball: file:projects/model-contact.tgz}
    version: 0.0.0

  '@rush-temp/model-controlled-documents@file:projects/model-controlled-documents.tgz':
@ -5537,7 +5534,7 @@ packages:
    version: 0.0.0

  '@rush-temp/tool@file:projects/tool.tgz':
-    resolution: {integrity: sha512-97pzx1qZJlMkJmWU1JEKOII/g4WM7evf3NZuj+9uHAvqjtpfHJDZ7QYzA4P7tMz+DVmCNjqUxWQyCSvHz5MqJA==, tarball: file:projects/tool.tgz}
+    resolution: {integrity: sha512-M5PhT+eqpWqh6vTLuYmcQhjIYmRyoYNGc7pFOmozt4RgKb5s0V5AwfqIkPr23DiBZm7Q9aYrscp1NxXx/RObOg==, tarball: file:projects/tool.tgz}
    version: 0.0.0

  '@rush-temp/tracker-assets@file:projects/tracker-assets.tgz':
@ -6500,8 +6497,8 @@ packages:
  '@types/node-cron@3.0.11':
    resolution: {integrity: sha512-0ikrnug3/IyneSHqCBeslAhlK2aBfYek1fGo4bP4QnZPmiqSGRK+Oy7ZMisLWkesffJvQ1cqAcBnJC+8+nxIAg==}

-  '@types/node-fetch@2.6.11':
-    resolution: {integrity: sha512-24xFj9R5+rfQJLRyM56qh+wnVSYhyXC2tkoBndtY0U+vubqNsYXGjufB2nn8Q6gt0LrARwL6UBtMCSVCwl4B1g==}
+  '@types/node-fetch@2.6.12':
+    resolution: {integrity: sha512-8nneRWKCg3rMtF69nLQJnOYUcbafYeFSjqkw3jCRLsqkWFlHaoQrr5mXmofFGOx3DKn7UfmBMyov8ySvLRVldA==}

  '@types/node-forge@1.3.11':
    resolution: {integrity: sha512-FQx220y22OKNTqaByeBGqHWYz4cl94tpcxeFdvBo3wjG6XPBuZ0BNgNZRV5J5TFmmcsJ4IzsLkmGRiQbnYsBEQ==}
@ -11101,6 +11098,7 @@ packages:
  node-domexception@1.0.0:
    resolution: {integrity: sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==}
    engines: {node: '>=10.5.0'}
+    deprecated: Use your platform's native DOMException instead

  node-fetch@2.7.0:
    resolution: {integrity: sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==}
@ -16758,7 +16756,7 @@ snapshots:
    dependencies:
      '@types/jest': 29.5.12
      '@types/node': 20.11.19
-      '@types/node-fetch': 2.6.11
+      '@types/node-fetch': 2.6.12
      '@types/snappyjs': 0.7.1
      '@types/ws': 8.5.11
      '@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.6.2)
@ -18033,7 +18031,7 @@ snapshots:
    dependencies:
      '@types/jest': 29.5.12
      '@types/node': 20.11.19
-      '@types/node-fetch': 2.6.11
+      '@types/node-fetch': 2.6.12
      '@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.3.3)
      '@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.3.3)
      eslint: 8.56.0
@ -20410,11 +20408,10 @@ snapshots:
    transitivePeerDependencies:
      - supports-color

-  '@rush-temp/model-contact@file:projects/model-contact.tgz(encoding@0.1.13)':
+  '@rush-temp/model-contact@file:projects/model-contact.tgz':
    dependencies:
      '@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.3.3)
      '@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.3.3)
-      cross-fetch: 3.1.8(encoding@0.1.13)
      eslint: 8.56.0
      eslint-config-standard-with-typescript: 40.0.0(@typescript-eslint/eslint-plugin@6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.3.3))(eslint-plugin-import@2.29.1(eslint@8.56.0))(eslint-plugin-n@15.7.0(eslint@8.56.0))(eslint-plugin-promise@6.1.1(eslint@8.56.0))(eslint@8.56.0)(typescript@5.3.3)
      eslint-plugin-import: 2.29.1(eslint@8.56.0)
@ -20423,7 +20420,6 @@ snapshots:
      prettier: 3.2.5
      typescript: 5.3.3
    transitivePeerDependencies:
-      - encoding
      - supports-color

  '@rush-temp/model-controlled-documents@file:projects/model-controlled-documents.tgz':
@ -27335,6 +27331,7 @@ snapshots:
  '@rush-temp/tool@file:projects/tool.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(gcp-metadata@5.3.0(encoding@0.1.13))(snappy@7.2.2)(socks@2.8.3)':
    dependencies:
      '@elastic/elasticsearch': 7.17.14
+      '@faker-js/faker': 8.4.1
      '@types/jest': 29.5.12
      '@types/mime-types': 2.1.4
      '@types/minio': 7.0.18
@ -29089,7 +29086,7 @@ snapshots:

  '@types/node-cron@3.0.11': {}

-  '@types/node-fetch@2.6.11':
+  '@types/node-fetch@2.6.12':
    dependencies:
      '@types/node': 20.11.19
      form-data: 4.0.0
@ -35072,7 +35069,7 @@ snapshots:
  openai@4.56.0(encoding@0.1.13)(zod@3.23.8):
    dependencies:
      '@types/node': 18.19.17
-      '@types/node-fetch': 2.6.11
+      '@types/node-fetch': 2.6.12
      abort-controller: 3.0.0
      agentkeepalive: 4.5.0
      form-data-encoder: 1.7.2
--- a/dev/prod/src/main-dev.ts
+++ b/dev/prod/src/main-dev.ts
@ -20,7 +20,12 @@ import { configurePlatformDevServer } from './platform-dev'

 configurePlatform().then(() => {

-  if (process.env.CLIENT_TYPE === 'dev-server' || process.env.CLIENT_TYPE === 'dev-production' || process.env.CLIENT_TYPE === 'dev-huly' || process.env.CLIENT_TYPE === 'dev-bold') {
+  if (process.env.CLIENT_TYPE === 'dev-server' || 
+    process.env.CLIENT_TYPE === 'dev-production' || 
+    process.env.CLIENT_TYPE === 'dev-huly' || 
+    process.env.CLIENT_TYPE === 'dev-bold' ||
+    process.env.CLIENT_TYPE === 'dev-server-test'
+  ) {
    configurePlatformDevServer()
  }
  
--- a/dev/tool/package.json
+++ b/dev/tool/package.json
@ -175,6 +175,8 @@
    "utf-8-validate": "^6.0.4",
    "msgpackr": "^1.11.2",
    "msgpackr-extract": "^3.0.3",
-    "@hcengineering/kafka": "^0.6.0"
+    "@hcengineering/kafka": "^0.6.0",
+    "@hcengineering/api-client": "^0.6.0",
+    "@faker-js/faker": "^8.4.1"
  }
 }
--- a/dev/tool/src/index.ts
+++ b/dev/tool/src/index.ts
@ -34,7 +34,7 @@ import {
  createStorageBackupStorage,
  restore
 } from '@hcengineering/server-backup'
-import serverClientPlugin, { getAccountClient } from '@hcengineering/server-client'
+import serverClientPlugin, { getAccountClient, getTransactorEndpoint } from '@hcengineering/server-client'
 import {
  registerAdapterFactory,
  registerDestroyFactory,
@ -47,6 +47,7 @@ import {
 import serverToken, { generateToken } from '@hcengineering/server-token'
 import { createWorkspace, upgradeWorkspace } from '@hcengineering/workspace-service'

+import { faker } from '@faker-js/faker'
 import { getPlatformQueue } from '@hcengineering/kafka'
 import { buildStorageFromConfig, createStorageFromConfig, storageConfigFromEnv } from '@hcengineering/server-storage'
 import { program, type Command } from 'commander'
@ -56,7 +57,10 @@ import {
  AccountRole,
  MeasureMetricsContext,
  metricsToString,
+  SocialIdType,
+  systemAccountEmail,
  systemAccountUuid,
+  type AccountUuid,
  type Data,
  type Doc,
  type PersonId,
@ -93,17 +97,19 @@ import { getAccountDBUrl, getKvsUrl, getMongoDBUrl } from './__start'
 // import { fillGithubUsers, fixAccountEmails, renameAccount } from './account'
 import { changeConfiguration } from './configuration'

-import { performGithubAccountMigrations } from './github'
-import {
-  migrateCreatedModifiedBy,
-  ensureGlobalPersonsForLocalAccounts,
-  moveAccountDbFromMongoToPG,
-  migrateMergedAccounts,
-  filterMergedAccountsInMembers
-} from './db'
-import { getToolToken, getWorkspace, getWorkspaceTransactorEndpoint } from './utils'
-import { performGmailAccountMigrations } from './gmail'
 import { performCalendarAccountMigrations } from './calendar'
+import {
+  ensureGlobalPersonsForLocalAccounts,
+  filterMergedAccountsInMembers,
+  migrateCreatedModifiedBy,
+  migrateMergedAccounts,
+  moveAccountDbFromMongoToPG
+} from './db'
+import { performGithubAccountMigrations } from './github'
+import { performGmailAccountMigrations } from './gmail'
+import { getToolToken, getWorkspace, getWorkspaceTransactorEndpoint } from './utils'
+
+import { createRestClient } from '@hcengineering/api-client'

 const colorConstants = {
  colorRed: '\u001b[31m',
@ -189,7 +195,6 @@ export function devTool (
      console.error(err)
    }
    closeAccountsDb()
-    console.log(`closing database connection to '${uri}'...`)
    await shutdownMongo()
  }

@ -1659,13 +1664,43 @@ export function devTool (
  //   })
  // })

-  // program
-  // .command('generate-token <name> <workspace>')
-  // .description('generate token')
-  // .option('--admin', 'Generate token with admin access', false)
-  // .action(async (name: string, workspace: string, opt: { admin: boolean }) => {
-  //   console.log(generateToken(name, getWorkspaceId(workspace), { ...(opt.admin ? { admin: 'true' } : {}) }))
-  // })
+  program
+    .command('generate-token <name> <workspace>')
+    .description('generate token')
+    .option('--admin', 'Generate token with admin access', false)
+    .action(async (name: string, workspace: string, opt: { admin: boolean }) => {
+      await withAccountDatabase(async (db) => {
+        if (name === systemAccountEmail) {
+          name = systemAccountUuid
+        }
+        const wsByUrl = await db.workspace.findOne({ url: workspace })
+        const account = await db.socialId.findOne({ key: name })
+        console.log(
+          generateToken(account?.personUuid ?? (name as AccountUuid), wsByUrl?.uuid ?? (workspace as WorkspaceUuid), {
+            ...(opt.admin ? { admin: 'true' } : {})
+          })
+        )
+      })
+    })
+
+  program
+    .command('generate-persons <workspace>')
+    .description('generate a random persons into workspace')
+    .option('--admin', 'Generate token with admin access', false)
+    .option('--count <count>', 'Number of persons to generate', '1000')
+    .action(async (workspace: string, opt: { admin: boolean, count: string }) => {
+      const count = parseInt(opt.count)
+      const token = generateToken(systemAccountUuid, workspace as WorkspaceUuid, {
+        ...(opt.admin ? { admin: 'true', service: 'tool' } : { service: 'tool' })
+      })
+      const endpoint = await getTransactorEndpoint(token, 'external')
+      const client = createRestClient(endpoint, workspace, token)
+      for (let i = 0; i < count; i++) {
+        const email = `${faker.internet.email()}`
+        await client.ensurePerson(SocialIdType.EMAIL, email, faker.person.firstName(), faker.person.lastName())
+      }
+    })
+
  // program
  // .command('decode-token <token>')
  // .description('decode token')
--- a/models/contact/package.json
+++ b/models/contact/package.json
@ -52,7 +52,6 @@
    "@hcengineering/rank": "^0.6.4",
    "@hcengineering/ui": "^0.6.15",
    "@hcengineering/view": "^0.6.13",
-    "@hcengineering/workbench": "^0.6.16",
-    "cross-fetch": "^3.1.5"
+    "@hcengineering/workbench": "^0.6.16"
  }
 }
--- a/packages/api-client/src/rest/rest.ts
+++ b/packages/api-client/src/rest/rest.ts
@ -36,12 +36,12 @@ import {
  type TxResult,
  type WithLookup
 } from '@hcengineering/core'
-import { PlatformError, unknownError } from '@hcengineering/platform'
+import { PlatformError, type Status, unknownError } from '@hcengineering/platform'

-import type { RestClient } from './types'
 import { AuthOptions } from '../types'
-import { extractJson, withRetry } from './utils'
 import { getWorkspaceToken } from '../utils'
+import type { RestClient } from './types'
+import { extractJson, withRetry } from './utils'

 export function createRestClient (endpoint: string, workspaceId: string, token: string): RestClient {
  return new RestClientImpl(endpoint, workspaceId, token)
@ -62,6 +62,7 @@ export class RestClientImpl implements RestClient {
  endpoint: string

  slowDownTimer = 0
+  currentRateLimit: { remaining: number, limit: number } = { remaining: 1000, limit: 1000 }

  remaining: number = 1000
  limit: number = 1000
@ -103,12 +104,13 @@ export class RestClientImpl implements RestClient {
      params.append('options', JSON.stringify(options))
    }
    const requestUrl = concatLink(this.endpoint, `/api/v1/find-all/${this.workspace}?${params.toString()}`)
-    const result = await withRetry(async () => {
+    const result = await withRetry<FindResult<T> & { error?: Status }>(async () => {
      const response = await fetch(requestUrl, this.requestInit())
      if (!response.ok) {
        await this.checkRateLimits(response)
        throw new PlatformError(unknownError(response.statusText))
      }
+      this.updateRateLimit(response)
      return await extractJson<FindResult<T>>(response)
    }, isRLE)

@ -122,9 +124,9 @@ export class RestClientImpl implements RestClient {
        if (d.$lookup !== undefined) {
          for (const [k, v] of Object.entries(d.$lookup)) {
            if (!Array.isArray(v)) {
-              d.$lookup[k] = result.lookupMap[v as any]
+              ;(d as any).$lookup[k] = result.lookupMap[v]
            } else {
-              d.$lookup[k] = v.map((it) => result.lookupMap?.[it])
+              ;(d as any).$lookup[k] = v.map((it) => result.lookupMap?.[it])
            }
          }
        }
@ -140,8 +142,8 @@ export class RestClientImpl implements RestClient {
      }
      for (const [k, v] of Object.entries(query)) {
        if (typeof v === 'string' || typeof v === 'number' || typeof v === 'boolean') {
-          if (doc[k] == null) {
-            doc[k] = v
+          if ((doc as any)[k] == null) {
+            ;(doc as any)[k] = v
          }
        }
      }
@ -150,20 +152,42 @@ export class RestClientImpl implements RestClient {
    return result
  }

+  private async checkRate (): Promise<void> {
+    if (this.currentRateLimit.remaining < this.currentRateLimit.limit / 3) {
+      if (this.slowDownTimer < 50) {
+        this.slowDownTimer += 50
+      }
+      this.slowDownTimer++
+    } else if (this.slowDownTimer > 0) {
+      this.slowDownTimer--
+    }
+    if (this.slowDownTimer > 0) {
+      // We need to wait a bit to avoid ban.
+      await new Promise((resolve) => setTimeout(resolve, this.slowDownTimer))
+    }
+  }
+
+  private updateRateLimit (response: Response): void {
+    const rateLimitLimit: number = parseInt(response.headers.get('X-RateLimit-Limit') ?? '100')
+    const remaining: number = parseInt(response.headers.get('X-RateLimit-Remaining') ?? '100')
+    this.currentRateLimit = { remaining, limit: rateLimitLimit }
+  }
+
  private async checkRateLimits (response: Response): Promise<void> {
    if (response.status === 429) {
      // Extract rate limit information from headers
      const retryAfter = response.headers.get('Retry-After')
+      const retryAfterMS = response.headers.get('Retry-After-ms')
      const rateLimitReset = response.headers.get('X-RateLimit-Reset')
-      // const rateLimitLimit: string | null = response.headers.get('X-RateLimit-Limit')
+
+      this.updateRateLimit(response)
      const waitTime =
-        retryAfter != null
-          ? parseInt(retryAfter)
+        (retryAfterMS != null ? parseInt(retryAfterMS) : undefined) ??
+        (retryAfter != null
+          ? parseInt(retryAfter) * 1000
          : rateLimitReset != null
            ? new Date(parseInt(rateLimitReset)).getTime() - Date.now()
-            : 1000 // Default to 1 seconds if no headers are provided
-
-      console.warn(`Rate limit exceeded. Waiting ${Math.round((10 * waitTime) / 1000) / 10} seconds before retrying...`)
+            : 1000) // Default to 1 seconds if no headers are provided
      await new Promise((resolve) => setTimeout(resolve, waitTime))
      throw new Error(rateLimitError)
    }
@ -171,28 +195,47 @@ export class RestClientImpl implements RestClient {

  async getAccount (): Promise<Account> {
    const requestUrl = concatLink(this.endpoint, `/api/v1/account/${this.workspace}`)
-    const response = await fetch(requestUrl, this.requestInit())
-    if (!response.ok) {
-      throw new PlatformError(unknownError(response.statusText))
+    await this.checkRate()
+    const result = await withRetry<Account & { error?: Status }>(async () => {
+      const response = await fetch(requestUrl, this.requestInit())
+      if (!response.ok) {
+        await this.checkRateLimits(response)
+        throw new PlatformError(unknownError(response.statusText))
+      }
+      this.updateRateLimit(response)
+      return await extractJson<Account>(response)
+    })
+    if (result.error !== undefined) {
+      throw new PlatformError(result.error)
    }
-    return await extractJson<Account>(response)
+    return result
  }

  async getModel (): Promise<{ hierarchy: Hierarchy, model: ModelDb }> {
    const requestUrl = concatLink(this.endpoint, `/api/v1/load-model/${this.workspace}`)
-    const response = await fetch(requestUrl, this.requestInit())
-    if (!response.ok) {
-      throw new PlatformError(unknownError(response.statusText))
+    await this.checkRate()
+    const result = await withRetry<{ hierarchy: Hierarchy, model: ModelDb, error?: Status }>(async () => {
+      const response = await fetch(requestUrl, this.requestInit())
+      if (!response.ok) {
+        await this.checkRateLimits(response)
+        throw new PlatformError(unknownError(response.statusText))
+      }
+      this.updateRateLimit(response)
+
+      const modelResponse: Tx[] = await extractJson<Tx[]>(response)
+
+      const hierarchy = new Hierarchy()
+      const model = new ModelDb(hierarchy)
+
+      const ctx = new MeasureMetricsContext('loadModel', {})
+      buildModel(ctx, modelResponse, (txes: Tx[]) => txes, hierarchy, model)
+
+      return { hierarchy, model }
+    }, isRLE)
+    if (result.error !== undefined) {
+      throw new PlatformError(result.error)
    }
-    const modelResponse: Tx[] = await extractJson<Tx[]>(response)
-
-    const hierarchy = new Hierarchy()
-    const model = new ModelDb(hierarchy)
-
-    const ctx = new MeasureMetricsContext('loadModel', {})
-    buildModel(ctx, modelResponse, (txes: Tx[]) => txes, hierarchy, model)
-
-    return { hierarchy, model }
+    return result
  }

  async findOne<T extends Doc>(
@ -205,7 +248,8 @@ export class RestClientImpl implements RestClient {

  async tx (tx: Tx): Promise<TxResult> {
    const requestUrl = concatLink(this.endpoint, `/api/v1/tx/${this.workspace}`)
-    const result = await withRetry(async () => {
+    await this.checkRate()
+    const result = await withRetry<TxResult & { error?: Status }>(async () => {
      const response = await fetch(requestUrl, {
        method: 'POST',
        headers: this.jsonHeaders(),
@ -216,6 +260,7 @@ export class RestClientImpl implements RestClient {
        await this.checkRateLimits(response)
        throw new PlatformError(unknownError(response.statusText))
      }
+      this.updateRateLimit(response)
      return await extractJson<TxResult>(response)
    }, isRLE)
    if (result.error !== undefined) {
@ -225,27 +270,35 @@ export class RestClientImpl implements RestClient {
  }

  async searchFulltext (query: SearchQuery, options: SearchOptions): Promise<SearchResult> {
-    const params = new URLSearchParams()
-    params.append('query', query.query)
-    if (query.classes != null && Object.keys(query.classes).length > 0) {
-      params.append('classes', JSON.stringify(query.classes))
-    }
-    if (query.spaces != null && Object.keys(query.spaces).length > 0) {
-      params.append('spaces', JSON.stringify(query.spaces))
-    }
-    if (options.limit != null) {
-      params.append('limit', `${options.limit}`)
-    }
-    const requestUrl = concatLink(this.endpoint, `/api/v1/search-fulltext/${this.workspace}?${params.toString()}`)
-    const response = await fetch(requestUrl, {
-      method: 'GET',
-      headers: this.jsonHeaders(),
-      keepalive: true
+    const result = await withRetry<SearchResult & { error?: Status }>(async () => {
+      const params = new URLSearchParams()
+      params.append('query', query.query)
+      if (query.classes != null && Object.keys(query.classes).length > 0) {
+        params.append('classes', JSON.stringify(query.classes))
+      }
+      if (query.spaces != null && Object.keys(query.spaces).length > 0) {
+        params.append('spaces', JSON.stringify(query.spaces))
+      }
+      if (options.limit != null) {
+        params.append('limit', `${options.limit}`)
+      }
+      const requestUrl = concatLink(this.endpoint, `/api/v1/search-fulltext/${this.workspace}?${params.toString()}`)
+      const response = await fetch(requestUrl, {
+        method: 'GET',
+        headers: this.jsonHeaders(),
+        keepalive: true
+      })
+      if (!response.ok) {
+        await this.checkRateLimits(response)
+        throw new PlatformError(unknownError(response.statusText))
+      }
+      this.updateRateLimit(response)
+      return await extractJson<TxResult>(response)
    })
-    if (!response.ok) {
-      throw new PlatformError(unknownError(response.statusText))
+    if (result.error !== undefined) {
+      throw new PlatformError(result.error)
    }
-    return await extractJson<TxResult>(response)
+    return result
  }

  async ensurePerson (
@ -255,20 +308,29 @@ export class RestClientImpl implements RestClient {
    lastName: string
  ): Promise<{ uuid: PersonUuid, socialId: PersonId, localPerson: string }> {
    const requestUrl = concatLink(this.endpoint, `/api/v1/ensure-person/${this.workspace}`)
-    const response = await fetch(requestUrl, {
-      method: 'POST',
-      headers: this.jsonHeaders(),
-      keepalive: true,
-      body: JSON.stringify({
-        socialType,
-        socialValue,
-        firstName,
-        lastName
+    await this.checkRate()
+    const result = await withRetry(async () => {
+      const response = await fetch(requestUrl, {
+        method: 'POST',
+        headers: this.jsonHeaders(),
+        keepalive: true,
+        body: JSON.stringify({
+          socialType,
+          socialValue,
+          firstName,
+          lastName
+        })
      })
-    })
-    if (!response.ok) {
-      throw new PlatformError(unknownError(response.statusText))
+      if (!response.ok) {
+        await this.checkRateLimits(response)
+        throw new PlatformError(unknownError(response.statusText))
+      }
+      this.updateRateLimit(response)
+      return await extractJson<{ uuid: PersonUuid, socialId: PersonId, localPerson: string }>(response)
+    }, isRLE)
+    if (result.error !== undefined) {
+      throw new PlatformError(result.error)
    }
-    return await extractJson<{ uuid: PersonUuid, socialId: PersonId, localPerson: string }>(response)
+    return result
  }
 }
--- a/packages/core/src/index.ts
+++ b/packages/core/src/index.ts
@ -17,7 +17,7 @@ import core from './component'
 export * from './classes'
 export * from './client'
 export * from './collaboration'
-export { coreId, systemAccountUuid, systemAccount, configUserAccountUuid } from './component'
+export { coreId, systemAccountUuid, systemAccountEmail, systemAccount, configUserAccountUuid } from './component'
 export * from './hierarchy'
 export * from './measurements'
 export * from './memdb'
--- a/packages/presentation/src/utils.ts
+++ b/packages/presentation/src/utils.ts
@ -26,6 +26,7 @@ import core, {
  type AttachedDoc,
  type Class,
  type Client,
+  type ClientConnection,
  type Collection,
  type Doc,
  type DocumentQuery,
@ -33,6 +34,7 @@ import core, {
  type FindResult,
  type Hierarchy,
  type Mixin,
+  type ModelDb,
  type Obj,
  type Ref,
  type RefTo,
@ -59,7 +61,7 @@ import { get, writable } from 'svelte/store'

 import { type KeyedAttribute } from '..'
 import { OptimizeQueryMiddleware, PresentationPipelineImpl, type PresentationPipeline } from './pipeline'
-import plugin from './plugin'
+import plugin, { type ClientHook } from './plugin'

 export { reduceCalls } from '@hcengineering/core'

@ -97,7 +99,6 @@ export const uiContext = new MeasureMetricsContext('client-ui', {})
 export const pendingCreatedDocs = writable<Record<Ref<Doc>, boolean>>({})

 class UIClient extends TxOperations implements Client {
-  hook = getMetadata(plugin.metadata.ClientHook)
  constructor (
    client: Client,
    private readonly liveQuery: Client
@ -156,9 +157,6 @@ class UIClient extends TxOperations implements Client {
    query: DocumentQuery<T>,
    options?: FindOptions<T>
  ): Promise<FindResult<T>> {
-    if (this.hook !== undefined) {
-      return await this.hook.findAll(this.liveQuery, _class, query, options)
-    }
    return await this.liveQuery.findAll(_class, query, options)
  }

@ -167,17 +165,13 @@ class UIClient extends TxOperations implements Client {
    query: DocumentQuery<T>,
    options?: FindOptions<T>
  ): Promise<WithLookup<T> | undefined> {
-    if (this.hook !== undefined) {
-      return await this.hook.findOne(this.liveQuery, _class, query, options)
-    }
    return await this.liveQuery.findOne(_class, query, options)
  }

  override async tx (tx: Tx): Promise<TxResult> {
-    void this.notifyEarly(tx)
-    if (this.hook !== undefined) {
-      return await this.hook.tx(this.client, tx)
-    }
+    void this.notifyEarly(tx).catch((err) => {
+      console.error(err)
+    })
    return await this.client.tx(tx)
  }

@ -220,9 +214,6 @@ class UIClient extends TxOperations implements Client {
  }

  async searchFulltext (query: SearchQuery, options: SearchOptions): Promise<SearchResult> {
-    if (this.hook !== undefined) {
-      return await this.hook.searchFulltext(this.client, query, options)
-    }
    return await this.client.searchFulltext(query, options)
  }
 }
@ -278,6 +269,73 @@ export function addRefreshListener (r: RefreshListener): void {
  refreshListeners.add(r)
 }

+class ClientHookImpl implements Client {
+  constructor (
+    private readonly client: Client,
+    private readonly hook: ClientHook
+  ) {}
+
+  set notify (op: (...tx: Tx[]) => void) {
+    this.client.notify = op
+  }
+
+  get notify (): ((...tx: Tx[]) => void) | undefined {
+    return this.client.notify
+  }
+
+  getHierarchy (): Hierarchy {
+    return this.client.getHierarchy()
+  }
+
+  getModel (): ModelDb {
+    return this.client.getModel()
+  }
+
+  async findOne<T extends Doc>(
+    _class: Ref<Class<T>>,
+    query: DocumentQuery<T>,
+    options?: FindOptions<T>
+  ): Promise<WithLookup<T> | undefined> {
+    if (this.hook !== undefined) {
+      return await this.hook.findOne(this.client, _class, query, options)
+    }
+    return await this.client.findOne(_class, query, options)
+  }
+
+  get getConnection (): (() => ClientConnection) | undefined {
+    return this.client.getConnection
+  }
+
+  async close (): Promise<void> {
+    await this.client.close()
+  }
+
+  async findAll<T extends Doc>(
+    _class: Ref<Class<T>>,
+    query: DocumentQuery<T>,
+    options?: FindOptions<T>
+  ): Promise<FindResult<T>> {
+    if (this.hook !== undefined) {
+      return await this.hook.findAll(this.client, _class, query, options)
+    }
+    return await this.client.findAll(_class, query, options)
+  }
+
+  async tx (tx: Tx): Promise<TxResult> {
+    if (this.hook !== undefined) {
+      return await this.hook.tx(this.client, tx)
+    }
+    return await this.client.tx(tx)
+  }
+
+  async searchFulltext (query: SearchQuery, options: SearchOptions): Promise<SearchResult> {
+    if (this.hook !== undefined) {
+      return await this.hook.searchFulltext(this.client, query, options)
+    }
+    return await this.client.searchFulltext(query, options)
+  }
+}
+
 /**
 * @public
 */
@ -293,6 +351,12 @@ export async function setClient (_client: Client): Promise<void> {
    await pipeline.close()
  }

+  const hook = getMetadata(plugin.metadata.ClientHook)
+
+  if (hook !== undefined) {
+    _client = new ClientHookImpl(_client, hook)
+  }
+
  const needRefresh = liveQuery !== undefined
  rawLiveQuery = new LQ(_client)

--- a/plugins/client-resources/src/connection.ts
+++ b/plugins/client-resources/src/connection.ts
@ -310,6 +310,9 @@ class Connection implements ClientConnection {
      )
      this.currentRateLimit = resp.rateLimit
      if (this.currentRateLimit.remaining < this.currentRateLimit.limit / 3) {
+        if (this.slowDownTimer < 50) {
+          this.slowDownTimer += 50
+        }
        this.slowDownTimer++
      } else if (this.slowDownTimer > 0) {
        this.slowDownTimer--
--- a/plugins/devmodel-resources/src/index.ts
+++ b/plugins/devmodel-resources/src/index.ts
@ -102,7 +102,7 @@ export class PresentationClientHook implements ClientHook {
        ' =>model',
        client.getModel(),
        getMetadata(devmodel.metadata.DevModel),
-        platformNow() - startTime,
+        platformNowDiff(startTime),
        this.stackLine()
      )
    }
@ -129,7 +129,7 @@ export class PresentationClientHook implements ClientHook {
        ' =>model',
        client.getModel(),
        getMetadata(devmodel.metadata.DevModel),
-        platformNow() - startTime,
+        platformNowDiff(startTime),
        JSON.stringify(result).length,
        this.stackLine()
      )
--- a/pods/server/src/rpc.ts
+++ b/pods/server/src/rpc.ts
@ -65,8 +65,8 @@ function rateLimitToHeaders (rateLimit?: RateLimitInfo): OutgoingHttpHeaders {
  }
  const { remaining, limit, reset, retryAfter } = rateLimit
  return {
-    'Retry-After': `${Math.max(retryAfter ?? 0, 1)}`,
-    'Retry-After-ms': `${retryAfter ?? 0}`,
+    'Retry-After': `${Math.max(Math.round((retryAfter ?? 0) / 1000), 1)}`,
+    'Retry-After-ms': `${retryAfter ?? 1000}`,
    'X-RateLimit-Limit': `${limit}`,
    'X-RateLimit-Remaining': `${remaining}`,
    'X-RateLimit-Reset': `${reset}`
@ -188,8 +188,8 @@ export function registerRPC (app: Express, sessions: SessionManager, ctx: Measur
          ...keepAliveOptions,
          'Content-Type': 'application/json',
          'Cache-Control': 'no-cache',
-          'Retry-After': `${Math.max((retryAfter ?? 0) / 1000, 1)}`,
-          'Retry-After-ms': `${retryAfter ?? 0}`,
+          'Retry-After': `${Math.max(Math.round((retryAfter ?? 0) / 1000), 1)}`,
+          'Retry-After-ms': `${retryAfter ?? 1000}`,
          'X-RateLimit-Limit': `${limit}`,
          'X-RateLimit-Remaining': `${remaining}`,
          'X-RateLimit-Reset': `${reset}`
--- a/server-plugins/activity-resources/src/references.ts
+++ b/server-plugins/activity-resources/src/references.ts
@ -277,7 +277,7 @@ async function getCreateReferencesTxes (
      refs.push(...attrReferences)
    } else if (attr.type._class === core.class.TypeCollaborativeDoc) {
      const blobId = (createdDoc as any)[attr.name] as Ref<Blob>
-      if (blobId != null) {
+      if (blobId != null && blobId !== '') {
        try {
          const buffer = await storage.read(ctx, control.workspace, blobId)
          const markup = Buffer.concat(buffer as any).toString()
--- a/server/rpc/src/sliding.ts
+++ b/server/rpc/src/sliding.ts
@ -44,12 +44,11 @@ export class SlidingWindowRateLimitter {
      rateLimit.requests.push(now + (rateLimit.rejectedRequests > this.rateLimitMax * 2 ? this.rateLimitWindow * 5 : 0))
    }

-    if (rateLimit.requests.length > this.rateLimitMax) {
+    if (rateLimit.requests.length >= this.rateLimitMax) {
      rateLimit.rejectedRequests++

      // Find when the oldest request will exit the window
-      const someRequest = Math.round(Math.random() * rateLimit.requests.length)
-      const nextAvailableTime = rateLimit.requests[someRequest] + this.rateLimitWindow
+      const nextAvailableTime = rateLimit.requests[0] + this.rateLimitWindow

      return {
        remaining: 0,
--- a/server/server/src/sessionManager.ts
+++ b/server/server/src/sessionManager.ts
@ -1211,7 +1211,7 @@ export class TSessionManager implements SessionManager {
    requestCtx: MeasureContext,
    service: S,
    ws: ConnectionSocket,
-    operation: (ctx: ClientSessionCtx) => Promise<void>
+    operation: (ctx: ClientSessionCtx, rateLimit: RateLimitInfo | undefined) => Promise<void>
  ): Promise<RateLimitInfo | undefined> {
    const rateLimitStatus = this.limitter.checkRateLimit(service.getUser())
    // If remaining is 0, rate limit is exceeded
@ -1252,7 +1252,7 @@ export class TSessionManager implements SessionManager {
            ws,
            rateLimitStatus
          )
-          await operation(uctx)
+          await operation(uctx, rateLimitStatus)
        })
      } catch (err: any) {
        Analytics.handleError(err)