Merge branch 'develop' into unified-editor-toolbar

This commit is contained in:
Victor Ilyushchenko 2025-06-02 08:58:17 +03:00 committed by GitHub
commit 431233fc25
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
19 changed files with 352 additions and 185 deletions

View File

@ -261,26 +261,26 @@ jobs:
run: |
cd ./tests/sanity
node ../../common/scripts/install-run-rushx.js ci
- name: Start profiling
run: |
cd ./tests
./profile-start.sh
# - name: Start profiling
# run: |
# cd ./tests
# ./profile-start.sh
- name: Run UI tests
run: |
cd ./tests/sanity
node ../../common/scripts/install-run-rushx.js uitest
- name: Download profile
run: |
cd ./tests
./profile-download.sh
npm install -g cpupro
./profile-generate.sh
- name: Upload profiling results
if: always()
uses: actions/upload-artifact@v4
with:
name: profiling
path: ./tests/profiles
# - name: Download profile
# run: |
# cd ./tests
# ./profile-download.sh
# npm install -g cpupro
# ./profile-generate.sh
# - name: Upload profiling results
# if: always()
# uses: actions/upload-artifact@v4
# with:
# name: profiling
# path: ./tests/profiles
- name: 'Store docker logs'
if: always()
run: |

View File

@ -504,7 +504,7 @@ importers:
version: file:projects/model-communication.tgz
'@rush-temp/model-contact':
specifier: file:./projects/model-contact.tgz
version: file:projects/model-contact.tgz(encoding@0.1.13)
version: file:projects/model-contact.tgz
'@rush-temp/model-controlled-documents':
specifier: file:./projects/model-controlled-documents.tgz
version: file:projects/model-controlled-documents.tgz
@ -1572,7 +1572,7 @@ importers:
version: 3.0.11
'@types/node-fetch':
specifier: ~2.6.2
version: 2.6.11
version: 2.6.12
'@types/nodemailer':
specifier: ^6.4.17
version: 6.4.17
@ -1661,8 +1661,8 @@ importers:
specifier: ^1.20.3
version: 1.20.3
browserslist:
specifier: ^4.24.4
version: 4.24.4
specifier: ^4.25.0
version: 4.25.0
bson:
specifier: ^6.10.3
version: 6.10.3
@ -1690,9 +1690,6 @@ importers:
cross-env:
specifier: ~7.0.3
version: 7.0.3
cross-fetch:
specifier: ^3.1.5
version: 3.1.8(encoding@0.1.13)
crypto-js:
specifier: ^4.2.0
version: 4.2.0
@ -2174,8 +2171,8 @@ importers:
specifier: ^5.3.3
version: 5.3.3
update-browserslist-db:
specifier: ^1.1.2
version: 1.1.2(browserslist@4.24.4)
specifier: ^1.1.3
version: 1.1.3(browserslist@4.25.0)
url-loader:
specifier: ~4.1.1
version: 4.1.1(file-loader@6.2.0(webpack@5.97.1))(webpack@5.97.1)
@ -4165,7 +4162,7 @@ packages:
version: 0.0.0
'@rush-temp/desktop@file:projects/desktop.tgz':
resolution: {integrity: sha512-vuoDjTgp50vbfTGTiiDScJkMvxWVMCQ7IMhdaLWwVNIRzLKPrczq1q9GB9LYkpTDc6qWSaMFlWMCKWyzYDUo6Q==, tarball: file:projects/desktop.tgz}
resolution: {integrity: sha512-EZYvSkajswd6HkZz3/XPrgLiYvmGchLgOGdljM/CSvtzhczZb8q8ld9Pp1wDYd+GceqSVW3fNGJWLsnedN9Gmw==, tarball: file:projects/desktop.tgz}
version: 0.0.0
'@rush-temp/devmodel-resources@file:projects/devmodel-resources.tgz':
@ -4473,7 +4470,7 @@ packages:
version: 0.0.0
'@rush-temp/model-contact@file:projects/model-contact.tgz':
resolution: {integrity: sha512-+wfUTUKjOzRKHOpcINUdbA6SGyJOYE+hxIiY2ZAL/3x6CqWb720+uhpkmNqb88ZXubhrOCT4z/nDXg29QkqL4A==, tarball: file:projects/model-contact.tgz}
resolution: {integrity: sha512-pUqoMMgSoArIiZmW9SL5BBapmezesVqgb6k0lCNNswPAP+UoKOChFTyqNAzzeTZ1AiLms4hvXbPEVlM1tKMKfg==, tarball: file:projects/model-contact.tgz}
version: 0.0.0
'@rush-temp/model-controlled-documents@file:projects/model-controlled-documents.tgz':
@ -4977,7 +4974,7 @@ packages:
version: 0.0.0
'@rush-temp/prod@file:projects/prod.tgz':
resolution: {integrity: sha512-uhaxu3bqYa0aoK0C734fxnGRHIFfRWX6b61/2BFkK/rO3imXXEqEy4b/pUtvVyUJ949dxgNmDrQ+0ztF5IsG2A==, tarball: file:projects/prod.tgz}
resolution: {integrity: sha512-xgXHeJVqYvR69IysLMyje3Uhucu4QpdbjVue38HFuJbpmP5BgbTZfP3GQCFrUWGiGdtAw4P8oJne5L4qYX3jrA==, tarball: file:projects/prod.tgz}
version: 0.0.0
'@rush-temp/products-assets@file:projects/products-assets.tgz':
@ -5537,7 +5534,7 @@ packages:
version: 0.0.0
'@rush-temp/tool@file:projects/tool.tgz':
resolution: {integrity: sha512-97pzx1qZJlMkJmWU1JEKOII/g4WM7evf3NZuj+9uHAvqjtpfHJDZ7QYzA4P7tMz+DVmCNjqUxWQyCSvHz5MqJA==, tarball: file:projects/tool.tgz}
resolution: {integrity: sha512-M5PhT+eqpWqh6vTLuYmcQhjIYmRyoYNGc7pFOmozt4RgKb5s0V5AwfqIkPr23DiBZm7Q9aYrscp1NxXx/RObOg==, tarball: file:projects/tool.tgz}
version: 0.0.0
'@rush-temp/tracker-assets@file:projects/tracker-assets.tgz':
@ -6500,8 +6497,8 @@ packages:
'@types/node-cron@3.0.11':
resolution: {integrity: sha512-0ikrnug3/IyneSHqCBeslAhlK2aBfYek1fGo4bP4QnZPmiqSGRK+Oy7ZMisLWkesffJvQ1cqAcBnJC+8+nxIAg==}
'@types/node-fetch@2.6.11':
resolution: {integrity: sha512-24xFj9R5+rfQJLRyM56qh+wnVSYhyXC2tkoBndtY0U+vubqNsYXGjufB2nn8Q6gt0LrARwL6UBtMCSVCwl4B1g==}
'@types/node-fetch@2.6.12':
resolution: {integrity: sha512-8nneRWKCg3rMtF69nLQJnOYUcbafYeFSjqkw3jCRLsqkWFlHaoQrr5mXmofFGOx3DKn7UfmBMyov8ySvLRVldA==}
'@types/node-forge@1.3.11':
resolution: {integrity: sha512-FQx220y22OKNTqaByeBGqHWYz4cl94tpcxeFdvBo3wjG6XPBuZ0BNgNZRV5J5TFmmcsJ4IzsLkmGRiQbnYsBEQ==}
@ -7340,8 +7337,8 @@ packages:
browser-or-node@2.1.1:
resolution: {integrity: sha512-8CVjaLJGuSKMVTxJ2DpBl5XnlNDiT4cQFeuCJJrvJmts9YrTZDizTX7PjC2s6W4x+MBGZeEY6dGMrF04/6Hgqg==}
browserslist@4.24.4:
resolution: {integrity: sha512-KDi1Ny1gSePi1vm0q4oxSF8b4DR44GF4BbmS2YdhPLOEqd8pDviZOGH/GsmRwoWJ2+5Lr085X7naowMwKHDG1A==}
browserslist@4.25.0:
resolution: {integrity: sha512-PJ8gYKeS5e/whHBh8xrwYK+dAvEj7JXtz6uTucnMRB8OiGTsKccFekoRrjajPBHV8oOY+2tI4uxeceSimKwMFA==}
engines: {node: ^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7}
hasBin: true
@ -7482,6 +7479,9 @@ packages:
caniuse-lite@1.0.30001689:
resolution: {integrity: sha512-CmeR2VBycfa+5/jOfnp/NpWPGd06nf1XYiefUvhXFfZE4GkRc9jv+eGPS4nT558WS/8lYCzV8SlANCIPvbWP1g==}
caniuse-lite@1.0.30001720:
resolution: {integrity: sha512-Ec/2yV2nNPwb4DnTANEV99ZWwm3ZWfdlfkQbWSDDt+PsXEVYwlhPH8tdMaPunYTKKmz7AnHi2oNEi1GcmKCD8g==}
centra@2.7.0:
resolution: {integrity: sha512-PbFMgMSrmgx6uxCdm57RUos9Tc3fclMvhLSATYN39XsDV29B89zZ3KA89jmY0vwSGazyU+uerqwa6t+KaodPcg==}
@ -8486,8 +8486,8 @@ packages:
electron-store@8.2.0:
resolution: {integrity: sha512-ukLL5Bevdil6oieAOXz3CMy+OgaItMiVBg701MNlG6W5RaC0AHN7rvlqTCmeb6O7jP0Qa1KKYTE0xV0xbhF4Hw==}
electron-to-chromium@1.5.74:
resolution: {integrity: sha512-ck3//9RC+6oss/1Bh9tiAVFy5vfSKbRHAFh7Z3/eTRkEqJeWgymloShB17Vg3Z4nmDNp35vAd1BZ6CMW4Wt6Iw==}
electron-to-chromium@1.5.161:
resolution: {integrity: sha512-hwtetwfKNZo/UlwHIVBlKZVdy7o8bIZxxKs0Mv/ROPiQQQmDgdm5a+KvKtBsxM8ZjFzTaCeLoodZ8jiBE3o9rA==}
electron-updater@6.3.4:
resolution: {integrity: sha512-uZUo7p1Y53G4tl6Cgw07X1yF8Jlz6zhaL7CQJDZ1fVVkOaBfE2cWtx80avwDVi8jHp+I/FWawrMgTAeCCNIfAg==}
@ -11101,6 +11101,7 @@ packages:
node-domexception@1.0.0:
resolution: {integrity: sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==}
engines: {node: '>=10.5.0'}
deprecated: Use your platform's native DOMException instead
node-fetch@2.7.0:
resolution: {integrity: sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==}
@ -13328,8 +13329,8 @@ packages:
resolution: {integrity: sha512-ZX6U1J04K1FoSUeoX1OicAhw4d0aro2qo+L8RhJkiGTNtBNkd/Fi1Wxoc9HzcVu6HfOzm0si/N15JjxFmD1z6A==}
engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0}
update-browserslist-db@1.1.2:
resolution: {integrity: sha512-PPypAm5qvlD7XMZC3BujecnaOxwhrtoFR+Dqkk5Aa/6DssiH0ibKoketaj9w8LP7Bont1rYeoV5plxD7RTEPRg==}
update-browserslist-db@1.1.3:
resolution: {integrity: sha512-UxhIZQ+QInVdunkDAaiazvvT/+fXL5Osr0JZlJulepYu6Jd7qJtDZjlur0emRlT71EN3ScPoE7gvsuIKKNavKw==}
hasBin: true
peerDependencies:
browserslist: '>= 4.21.0'
@ -14441,7 +14442,7 @@ snapshots:
dependencies:
'@babel/compat-data': 7.23.5
'@babel/helper-validator-option': 7.23.5
browserslist: 4.24.4
browserslist: 4.25.0
lru-cache: 5.1.1
semver: 6.3.1
@ -16758,7 +16759,7 @@ snapshots:
dependencies:
'@types/jest': 29.5.12
'@types/node': 20.11.19
'@types/node-fetch': 2.6.11
'@types/node-fetch': 2.6.12
'@types/snappyjs': 0.7.1
'@types/ws': 8.5.11
'@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.6.2)
@ -18033,7 +18034,7 @@ snapshots:
dependencies:
'@types/jest': 29.5.12
'@types/node': 20.11.19
'@types/node-fetch': 2.6.11
'@types/node-fetch': 2.6.12
'@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.3.3)
'@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.3.3)
eslint: 8.56.0
@ -18175,7 +18176,7 @@ snapshots:
'@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.3.3)
'@vercel/webpack-asset-relocator-loader': 1.7.4
autoprefixer: 10.4.17(postcss@8.5.3)
browserslist: 4.24.4
browserslist: 4.25.0
commander: 8.3.0
compression-webpack-plugin: 10.0.0(webpack@5.97.1)
copy-webpack-plugin: 11.0.0(webpack@5.97.1)
@ -18217,7 +18218,7 @@ snapshots:
ts-node: 10.9.2(@types/node@20.11.19)(typescript@5.3.3)
ts-node-dev: 2.0.0(@types/node@20.11.19)(typescript@5.3.3)
typescript: 5.3.3
update-browserslist-db: 1.1.2(browserslist@4.24.4)
update-browserslist-db: 1.1.3(browserslist@4.25.0)
webpack: 5.97.1(esbuild@0.24.2)(webpack-cli@5.1.4)
webpack-bundle-analyzer: 4.10.2(bufferutil@4.0.8)(utf-8-validate@6.0.4)
webpack-cli: 5.1.4(webpack-bundle-analyzer@4.10.2(bufferutil@4.0.8)(utf-8-validate@6.0.4))(webpack-dev-server@4.15.1)(webpack@5.97.1)
@ -20410,11 +20411,10 @@ snapshots:
transitivePeerDependencies:
- supports-color
'@rush-temp/model-contact@file:projects/model-contact.tgz(encoding@0.1.13)':
'@rush-temp/model-contact@file:projects/model-contact.tgz':
dependencies:
'@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.3.3)
'@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.3.3)
cross-fetch: 3.1.8(encoding@0.1.13)
eslint: 8.56.0
eslint-config-standard-with-typescript: 40.0.0(@typescript-eslint/eslint-plugin@6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.3.3))(eslint-plugin-import@2.29.1(eslint@8.56.0))(eslint-plugin-n@15.7.0(eslint@8.56.0))(eslint-plugin-promise@6.1.1(eslint@8.56.0))(eslint@8.56.0)(typescript@5.3.3)
eslint-plugin-import: 2.29.1(eslint@8.56.0)
@ -20423,7 +20423,6 @@ snapshots:
prettier: 3.2.5
typescript: 5.3.3
transitivePeerDependencies:
- encoding
- supports-color
'@rush-temp/model-controlled-documents@file:projects/model-controlled-documents.tgz':
@ -23369,7 +23368,7 @@ snapshots:
'@sentry/svelte': 9.22.0(svelte@4.2.19)
'@types/node': 20.11.19
autoprefixer: 10.4.17(postcss@8.5.3)
browserslist: 4.24.4
browserslist: 4.25.0
compression-webpack-plugin: 10.0.0(webpack@5.97.1)
cross-env: 7.0.3
css-loader: 5.2.7(webpack@5.97.1)
@ -23393,7 +23392,7 @@ snapshots:
svgo-loader: 3.0.3
ts-loader: 9.5.1(typescript@5.3.3)(webpack@5.97.1)
typescript: 5.3.3
update-browserslist-db: 1.1.2(browserslist@4.24.4)
update-browserslist-db: 1.1.3(browserslist@4.25.0)
webpack: 5.97.1(esbuild@0.24.2)(webpack-cli@5.1.4)
webpack-bundle-analyzer: 4.10.2(bufferutil@4.0.8)(utf-8-validate@6.0.4)
webpack-cli: 5.1.4(webpack-bundle-analyzer@4.10.2(bufferutil@4.0.8)(utf-8-validate@6.0.4))(webpack-dev-server@4.15.1)(webpack@5.97.1)
@ -27335,6 +27334,7 @@ snapshots:
'@rush-temp/tool@file:projects/tool.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(gcp-metadata@5.3.0(encoding@0.1.13))(snappy@7.2.2)(socks@2.8.3)':
dependencies:
'@elastic/elasticsearch': 7.17.14
'@faker-js/faker': 8.4.1
'@types/jest': 29.5.12
'@types/mime-types': 2.1.4
'@types/minio': 7.0.18
@ -29089,7 +29089,7 @@ snapshots:
'@types/node-cron@3.0.11': {}
'@types/node-fetch@2.6.11':
'@types/node-fetch@2.6.12':
dependencies:
'@types/node': 20.11.19
form-data: 4.0.0
@ -30102,7 +30102,7 @@ snapshots:
autoprefixer@10.4.17(postcss@8.5.3):
dependencies:
browserslist: 4.24.4
browserslist: 4.25.0
caniuse-lite: 1.0.30001689
fraction.js: 4.3.7
normalize-range: 0.1.2
@ -30285,12 +30285,12 @@ snapshots:
browser-or-node@2.1.1: {}
browserslist@4.24.4:
browserslist@4.25.0:
dependencies:
caniuse-lite: 1.0.30001689
electron-to-chromium: 1.5.74
caniuse-lite: 1.0.30001720
electron-to-chromium: 1.5.161
node-releases: 2.0.19
update-browserslist-db: 1.1.2(browserslist@4.24.4)
update-browserslist-db: 1.1.3(browserslist@4.25.0)
bs-logger@0.2.6:
dependencies:
@ -30463,6 +30463,8 @@ snapshots:
caniuse-lite@1.0.30001689: {}
caniuse-lite@1.0.30001720: {}
centra@2.7.0:
dependencies:
follow-redirects: 1.15.6
@ -31559,7 +31561,7 @@ snapshots:
conf: 10.2.0
type-fest: 2.19.0
electron-to-chromium@1.5.74: {}
electron-to-chromium@1.5.161: {}
electron-updater@6.3.4:
dependencies:
@ -35072,7 +35074,7 @@ snapshots:
openai@4.56.0(encoding@0.1.13)(zod@3.23.8):
dependencies:
'@types/node': 18.19.17
'@types/node-fetch': 2.6.11
'@types/node-fetch': 2.6.12
abort-controller: 3.0.0
agentkeepalive: 4.5.0
form-data-encoder: 1.7.2
@ -37476,9 +37478,9 @@ snapshots:
escape-string-regexp: 5.0.0
path-exists: 5.0.0
update-browserslist-db@1.1.2(browserslist@4.24.4):
update-browserslist-db@1.1.3(browserslist@4.25.0):
dependencies:
browserslist: 4.24.4
browserslist: 4.25.0
escalade: 3.2.0
picocolors: 1.1.1
@ -37736,7 +37738,7 @@ snapshots:
'@webassemblyjs/wasm-edit': 1.14.1
'@webassemblyjs/wasm-parser': 1.14.1
acorn: 8.14.0
browserslist: 4.24.4
browserslist: 4.25.0
chrome-trace-event: 1.0.3
enhanced-resolve: 5.17.1
es-module-lexer: 1.4.1

View File

@ -42,8 +42,8 @@
"compression-webpack-plugin": "^10.0.0",
"html-webpack-plugin": "^5.5.0",
"fork-ts-checker-webpack-plugin": "^9.0.2",
"update-browserslist-db": "^1.1.2",
"browserslist": "^4.24.4",
"update-browserslist-db": "^1.1.3",
"browserslist": "^4.25.0",
"typescript": "^5.3.3",
"ts-node": "^10.8.0",
"ts-node-dev": "^2.0.0",

View File

@ -26,7 +26,7 @@
"@hcengineering/platform-rig": "^0.6.0",
"@types/node": "~20.11.16",
"autoprefixer": "^10.4.14",
"browserslist": "^4.24.4",
"browserslist": "^4.25.0",
"compression-webpack-plugin": "^10.0.0",
"cross-env": "~7.0.3",
"css-loader": "^5.2.1",
@ -47,7 +47,7 @@
"svgo-loader": "^3.0.0",
"ts-loader": "^9.2.5",
"typescript": "^5.3.3",
"update-browserslist-db": "^1.1.2",
"update-browserslist-db": "^1.1.3",
"webpack": "^5.97.1",
"webpack-bundle-analyzer": "^4.10.2",
"webpack-cli": "^5.1.4",

View File

@ -20,7 +20,12 @@ import { configurePlatformDevServer } from './platform-dev'
configurePlatform().then(() => {
if (process.env.CLIENT_TYPE === 'dev-server' || process.env.CLIENT_TYPE === 'dev-production' || process.env.CLIENT_TYPE === 'dev-huly' || process.env.CLIENT_TYPE === 'dev-bold') {
if (process.env.CLIENT_TYPE === 'dev-server' ||
process.env.CLIENT_TYPE === 'dev-production' ||
process.env.CLIENT_TYPE === 'dev-huly' ||
process.env.CLIENT_TYPE === 'dev-bold' ||
process.env.CLIENT_TYPE === 'dev-server-test'
) {
configurePlatformDevServer()
}

View File

@ -175,6 +175,8 @@
"utf-8-validate": "^6.0.4",
"msgpackr": "^1.11.2",
"msgpackr-extract": "^3.0.3",
"@hcengineering/kafka": "^0.6.0"
"@hcengineering/kafka": "^0.6.0",
"@hcengineering/api-client": "^0.6.0",
"@faker-js/faker": "^8.4.1"
}
}

View File

@ -34,7 +34,7 @@ import {
createStorageBackupStorage,
restore
} from '@hcengineering/server-backup'
import serverClientPlugin, { getAccountClient } from '@hcengineering/server-client'
import serverClientPlugin, { getAccountClient, getTransactorEndpoint } from '@hcengineering/server-client'
import {
registerAdapterFactory,
registerDestroyFactory,
@ -47,6 +47,7 @@ import {
import serverToken, { generateToken } from '@hcengineering/server-token'
import { createWorkspace, upgradeWorkspace } from '@hcengineering/workspace-service'
import { faker } from '@faker-js/faker'
import { getPlatformQueue } from '@hcengineering/kafka'
import { buildStorageFromConfig, createStorageFromConfig, storageConfigFromEnv } from '@hcengineering/server-storage'
import { program, type Command } from 'commander'
@ -56,7 +57,10 @@ import {
AccountRole,
MeasureMetricsContext,
metricsToString,
SocialIdType,
systemAccountEmail,
systemAccountUuid,
type AccountUuid,
type Data,
type Doc,
type PersonId,
@ -93,17 +97,19 @@ import { getAccountDBUrl, getKvsUrl, getMongoDBUrl } from './__start'
// import { fillGithubUsers, fixAccountEmails, renameAccount } from './account'
import { changeConfiguration } from './configuration'
import { performGithubAccountMigrations } from './github'
import {
migrateCreatedModifiedBy,
ensureGlobalPersonsForLocalAccounts,
moveAccountDbFromMongoToPG,
migrateMergedAccounts,
filterMergedAccountsInMembers
} from './db'
import { getToolToken, getWorkspace, getWorkspaceTransactorEndpoint } from './utils'
import { performGmailAccountMigrations } from './gmail'
import { performCalendarAccountMigrations } from './calendar'
import {
ensureGlobalPersonsForLocalAccounts,
filterMergedAccountsInMembers,
migrateCreatedModifiedBy,
migrateMergedAccounts,
moveAccountDbFromMongoToPG
} from './db'
import { performGithubAccountMigrations } from './github'
import { performGmailAccountMigrations } from './gmail'
import { getToolToken, getWorkspace, getWorkspaceTransactorEndpoint } from './utils'
import { createRestClient } from '@hcengineering/api-client'
const colorConstants = {
colorRed: '\u001b[31m',
@ -189,7 +195,6 @@ export function devTool (
console.error(err)
}
closeAccountsDb()
console.log(`closing database connection to '${uri}'...`)
await shutdownMongo()
}
@ -1659,13 +1664,43 @@ export function devTool (
// })
// })
// program
// .command('generate-token <name> <workspace>')
// .description('generate token')
// .option('--admin', 'Generate token with admin access', false)
// .action(async (name: string, workspace: string, opt: { admin: boolean }) => {
// console.log(generateToken(name, getWorkspaceId(workspace), { ...(opt.admin ? { admin: 'true' } : {}) }))
// })
program
.command('generate-token <name> <workspace>')
.description('generate token')
.option('--admin', 'Generate token with admin access', false)
.action(async (name: string, workspace: string, opt: { admin: boolean }) => {
await withAccountDatabase(async (db) => {
if (name === systemAccountEmail) {
name = systemAccountUuid
}
const wsByUrl = await db.workspace.findOne({ url: workspace })
const account = await db.socialId.findOne({ key: name })
console.log(
generateToken(account?.personUuid ?? (name as AccountUuid), wsByUrl?.uuid ?? (workspace as WorkspaceUuid), {
...(opt.admin ? { admin: 'true' } : {})
})
)
})
})
program
.command('generate-persons <workspace>')
.description('generate a random persons into workspace')
.option('--admin', 'Generate token with admin access', false)
.option('--count <count>', 'Number of persons to generate', '1000')
.action(async (workspace: string, opt: { admin: boolean, count: string }) => {
const count = parseInt(opt.count)
const token = generateToken(systemAccountUuid, workspace as WorkspaceUuid, {
...(opt.admin ? { admin: 'true', service: 'tool' } : { service: 'tool' })
})
const endpoint = await getTransactorEndpoint(token, 'external')
const client = createRestClient(endpoint, workspace, token)
for (let i = 0; i < count; i++) {
const email = `${faker.internet.email()}`
await client.ensurePerson(SocialIdType.EMAIL, email, faker.person.firstName(), faker.person.lastName())
}
})
// program
// .command('decode-token <token>')
// .description('decode token')

View File

@ -52,7 +52,6 @@
"@hcengineering/rank": "^0.6.4",
"@hcengineering/ui": "^0.6.15",
"@hcengineering/view": "^0.6.13",
"@hcengineering/workbench": "^0.6.16",
"cross-fetch": "^3.1.5"
"@hcengineering/workbench": "^0.6.16"
}
}

View File

@ -36,12 +36,12 @@ import {
type TxResult,
type WithLookup
} from '@hcengineering/core'
import { PlatformError, unknownError } from '@hcengineering/platform'
import { PlatformError, type Status, unknownError } from '@hcengineering/platform'
import type { RestClient } from './types'
import { AuthOptions } from '../types'
import { extractJson, withRetry } from './utils'
import { getWorkspaceToken } from '../utils'
import type { RestClient } from './types'
import { extractJson, withRetry } from './utils'
export function createRestClient (endpoint: string, workspaceId: string, token: string): RestClient {
return new RestClientImpl(endpoint, workspaceId, token)
@ -62,6 +62,7 @@ export class RestClientImpl implements RestClient {
endpoint: string
slowDownTimer = 0
currentRateLimit: { remaining: number, limit: number } = { remaining: 1000, limit: 1000 }
remaining: number = 1000
limit: number = 1000
@ -103,12 +104,13 @@ export class RestClientImpl implements RestClient {
params.append('options', JSON.stringify(options))
}
const requestUrl = concatLink(this.endpoint, `/api/v1/find-all/${this.workspace}?${params.toString()}`)
const result = await withRetry(async () => {
const result = await withRetry<FindResult<T> & { error?: Status }>(async () => {
const response = await fetch(requestUrl, this.requestInit())
if (!response.ok) {
await this.checkRateLimits(response)
throw new PlatformError(unknownError(response.statusText))
}
this.updateRateLimit(response)
return await extractJson<FindResult<T>>(response)
}, isRLE)
@ -122,9 +124,9 @@ export class RestClientImpl implements RestClient {
if (d.$lookup !== undefined) {
for (const [k, v] of Object.entries(d.$lookup)) {
if (!Array.isArray(v)) {
d.$lookup[k] = result.lookupMap[v as any]
;(d as any).$lookup[k] = result.lookupMap[v]
} else {
d.$lookup[k] = v.map((it) => result.lookupMap?.[it])
;(d as any).$lookup[k] = v.map((it) => result.lookupMap?.[it])
}
}
}
@ -140,8 +142,8 @@ export class RestClientImpl implements RestClient {
}
for (const [k, v] of Object.entries(query)) {
if (typeof v === 'string' || typeof v === 'number' || typeof v === 'boolean') {
if (doc[k] == null) {
doc[k] = v
if ((doc as any)[k] == null) {
;(doc as any)[k] = v
}
}
}
@ -150,20 +152,42 @@ export class RestClientImpl implements RestClient {
return result
}
private async checkRate (): Promise<void> {
if (this.currentRateLimit.remaining < this.currentRateLimit.limit / 3) {
if (this.slowDownTimer < 50) {
this.slowDownTimer += 50
}
this.slowDownTimer++
} else if (this.slowDownTimer > 0) {
this.slowDownTimer--
}
if (this.slowDownTimer > 0) {
// We need to wait a bit to avoid ban.
await new Promise((resolve) => setTimeout(resolve, this.slowDownTimer))
}
}
private updateRateLimit (response: Response): void {
const rateLimitLimit: number = parseInt(response.headers.get('X-RateLimit-Limit') ?? '100')
const remaining: number = parseInt(response.headers.get('X-RateLimit-Remaining') ?? '100')
this.currentRateLimit = { remaining, limit: rateLimitLimit }
}
private async checkRateLimits (response: Response): Promise<void> {
if (response.status === 429) {
// Extract rate limit information from headers
const retryAfter = response.headers.get('Retry-After')
const retryAfterMS = response.headers.get('Retry-After-ms')
const rateLimitReset = response.headers.get('X-RateLimit-Reset')
// const rateLimitLimit: string | null = response.headers.get('X-RateLimit-Limit')
this.updateRateLimit(response)
const waitTime =
retryAfter != null
? parseInt(retryAfter)
(retryAfterMS != null ? parseInt(retryAfterMS) : undefined) ??
(retryAfter != null
? parseInt(retryAfter) * 1000
: rateLimitReset != null
? new Date(parseInt(rateLimitReset)).getTime() - Date.now()
: 1000 // Default to 1 seconds if no headers are provided
console.warn(`Rate limit exceeded. Waiting ${Math.round((10 * waitTime) / 1000) / 10} seconds before retrying...`)
: 1000) // Default to 1 seconds if no headers are provided
await new Promise((resolve) => setTimeout(resolve, waitTime))
throw new Error(rateLimitError)
}
@ -171,28 +195,47 @@ export class RestClientImpl implements RestClient {
async getAccount (): Promise<Account> {
const requestUrl = concatLink(this.endpoint, `/api/v1/account/${this.workspace}`)
const response = await fetch(requestUrl, this.requestInit())
if (!response.ok) {
throw new PlatformError(unknownError(response.statusText))
await this.checkRate()
const result = await withRetry<Account & { error?: Status }>(async () => {
const response = await fetch(requestUrl, this.requestInit())
if (!response.ok) {
await this.checkRateLimits(response)
throw new PlatformError(unknownError(response.statusText))
}
this.updateRateLimit(response)
return await extractJson<Account>(response)
})
if (result.error !== undefined) {
throw new PlatformError(result.error)
}
return await extractJson<Account>(response)
return result
}
async getModel (): Promise<{ hierarchy: Hierarchy, model: ModelDb }> {
const requestUrl = concatLink(this.endpoint, `/api/v1/load-model/${this.workspace}`)
const response = await fetch(requestUrl, this.requestInit())
if (!response.ok) {
throw new PlatformError(unknownError(response.statusText))
await this.checkRate()
const result = await withRetry<{ hierarchy: Hierarchy, model: ModelDb, error?: Status }>(async () => {
const response = await fetch(requestUrl, this.requestInit())
if (!response.ok) {
await this.checkRateLimits(response)
throw new PlatformError(unknownError(response.statusText))
}
this.updateRateLimit(response)
const modelResponse: Tx[] = await extractJson<Tx[]>(response)
const hierarchy = new Hierarchy()
const model = new ModelDb(hierarchy)
const ctx = new MeasureMetricsContext('loadModel', {})
buildModel(ctx, modelResponse, (txes: Tx[]) => txes, hierarchy, model)
return { hierarchy, model }
}, isRLE)
if (result.error !== undefined) {
throw new PlatformError(result.error)
}
const modelResponse: Tx[] = await extractJson<Tx[]>(response)
const hierarchy = new Hierarchy()
const model = new ModelDb(hierarchy)
const ctx = new MeasureMetricsContext('loadModel', {})
buildModel(ctx, modelResponse, (txes: Tx[]) => txes, hierarchy, model)
return { hierarchy, model }
return result
}
async findOne<T extends Doc>(
@ -205,7 +248,8 @@ export class RestClientImpl implements RestClient {
async tx (tx: Tx): Promise<TxResult> {
const requestUrl = concatLink(this.endpoint, `/api/v1/tx/${this.workspace}`)
const result = await withRetry(async () => {
await this.checkRate()
const result = await withRetry<TxResult & { error?: Status }>(async () => {
const response = await fetch(requestUrl, {
method: 'POST',
headers: this.jsonHeaders(),
@ -216,6 +260,7 @@ export class RestClientImpl implements RestClient {
await this.checkRateLimits(response)
throw new PlatformError(unknownError(response.statusText))
}
this.updateRateLimit(response)
return await extractJson<TxResult>(response)
}, isRLE)
if (result.error !== undefined) {
@ -225,27 +270,35 @@ export class RestClientImpl implements RestClient {
}
async searchFulltext (query: SearchQuery, options: SearchOptions): Promise<SearchResult> {
const params = new URLSearchParams()
params.append('query', query.query)
if (query.classes != null && Object.keys(query.classes).length > 0) {
params.append('classes', JSON.stringify(query.classes))
}
if (query.spaces != null && Object.keys(query.spaces).length > 0) {
params.append('spaces', JSON.stringify(query.spaces))
}
if (options.limit != null) {
params.append('limit', `${options.limit}`)
}
const requestUrl = concatLink(this.endpoint, `/api/v1/search-fulltext/${this.workspace}?${params.toString()}`)
const response = await fetch(requestUrl, {
method: 'GET',
headers: this.jsonHeaders(),
keepalive: true
const result = await withRetry<SearchResult & { error?: Status }>(async () => {
const params = new URLSearchParams()
params.append('query', query.query)
if (query.classes != null && Object.keys(query.classes).length > 0) {
params.append('classes', JSON.stringify(query.classes))
}
if (query.spaces != null && Object.keys(query.spaces).length > 0) {
params.append('spaces', JSON.stringify(query.spaces))
}
if (options.limit != null) {
params.append('limit', `${options.limit}`)
}
const requestUrl = concatLink(this.endpoint, `/api/v1/search-fulltext/${this.workspace}?${params.toString()}`)
const response = await fetch(requestUrl, {
method: 'GET',
headers: this.jsonHeaders(),
keepalive: true
})
if (!response.ok) {
await this.checkRateLimits(response)
throw new PlatformError(unknownError(response.statusText))
}
this.updateRateLimit(response)
return await extractJson<TxResult>(response)
})
if (!response.ok) {
throw new PlatformError(unknownError(response.statusText))
if (result.error !== undefined) {
throw new PlatformError(result.error)
}
return await extractJson<TxResult>(response)
return result
}
async ensurePerson (
@ -255,20 +308,29 @@ export class RestClientImpl implements RestClient {
lastName: string
): Promise<{ uuid: PersonUuid, socialId: PersonId, localPerson: string }> {
const requestUrl = concatLink(this.endpoint, `/api/v1/ensure-person/${this.workspace}`)
const response = await fetch(requestUrl, {
method: 'POST',
headers: this.jsonHeaders(),
keepalive: true,
body: JSON.stringify({
socialType,
socialValue,
firstName,
lastName
await this.checkRate()
const result = await withRetry(async () => {
const response = await fetch(requestUrl, {
method: 'POST',
headers: this.jsonHeaders(),
keepalive: true,
body: JSON.stringify({
socialType,
socialValue,
firstName,
lastName
})
})
})
if (!response.ok) {
throw new PlatformError(unknownError(response.statusText))
if (!response.ok) {
await this.checkRateLimits(response)
throw new PlatformError(unknownError(response.statusText))
}
this.updateRateLimit(response)
return await extractJson<{ uuid: PersonUuid, socialId: PersonId, localPerson: string }>(response)
}, isRLE)
if (result.error !== undefined) {
throw new PlatformError(result.error)
}
return await extractJson<{ uuid: PersonUuid, socialId: PersonId, localPerson: string }>(response)
return result
}
}

View File

@ -17,7 +17,7 @@ import core from './component'
export * from './classes'
export * from './client'
export * from './collaboration'
export { coreId, systemAccountUuid, systemAccount, configUserAccountUuid } from './component'
export { coreId, systemAccountUuid, systemAccountEmail, systemAccount, configUserAccountUuid } from './component'
export * from './hierarchy'
export * from './measurements'
export * from './memdb'

View File

@ -26,6 +26,7 @@ import core, {
type AttachedDoc,
type Class,
type Client,
type ClientConnection,
type Collection,
type Doc,
type DocumentQuery,
@ -33,6 +34,7 @@ import core, {
type FindResult,
type Hierarchy,
type Mixin,
type ModelDb,
type Obj,
type Ref,
type RefTo,
@ -59,7 +61,7 @@ import { get, writable } from 'svelte/store'
import { type KeyedAttribute } from '..'
import { OptimizeQueryMiddleware, PresentationPipelineImpl, type PresentationPipeline } from './pipeline'
import plugin from './plugin'
import plugin, { type ClientHook } from './plugin'
export { reduceCalls } from '@hcengineering/core'
@ -97,7 +99,6 @@ export const uiContext = new MeasureMetricsContext('client-ui', {})
export const pendingCreatedDocs = writable<Record<Ref<Doc>, boolean>>({})
class UIClient extends TxOperations implements Client {
hook = getMetadata(plugin.metadata.ClientHook)
constructor (
client: Client,
private readonly liveQuery: Client
@ -156,9 +157,6 @@ class UIClient extends TxOperations implements Client {
query: DocumentQuery<T>,
options?: FindOptions<T>
): Promise<FindResult<T>> {
if (this.hook !== undefined) {
return await this.hook.findAll(this.liveQuery, _class, query, options)
}
return await this.liveQuery.findAll(_class, query, options)
}
@ -167,17 +165,13 @@ class UIClient extends TxOperations implements Client {
query: DocumentQuery<T>,
options?: FindOptions<T>
): Promise<WithLookup<T> | undefined> {
if (this.hook !== undefined) {
return await this.hook.findOne(this.liveQuery, _class, query, options)
}
return await this.liveQuery.findOne(_class, query, options)
}
override async tx (tx: Tx): Promise<TxResult> {
void this.notifyEarly(tx)
if (this.hook !== undefined) {
return await this.hook.tx(this.client, tx)
}
void this.notifyEarly(tx).catch((err) => {
console.error(err)
})
return await this.client.tx(tx)
}
@ -220,9 +214,6 @@ class UIClient extends TxOperations implements Client {
}
async searchFulltext (query: SearchQuery, options: SearchOptions): Promise<SearchResult> {
if (this.hook !== undefined) {
return await this.hook.searchFulltext(this.client, query, options)
}
return await this.client.searchFulltext(query, options)
}
}
@ -278,6 +269,73 @@ export function addRefreshListener (r: RefreshListener): void {
refreshListeners.add(r)
}
class ClientHookImpl implements Client {
constructor (
private readonly client: Client,
private readonly hook: ClientHook
) {}
set notify (op: (...tx: Tx[]) => void) {
this.client.notify = op
}
get notify (): ((...tx: Tx[]) => void) | undefined {
return this.client.notify
}
getHierarchy (): Hierarchy {
return this.client.getHierarchy()
}
getModel (): ModelDb {
return this.client.getModel()
}
async findOne<T extends Doc>(
_class: Ref<Class<T>>,
query: DocumentQuery<T>,
options?: FindOptions<T>
): Promise<WithLookup<T> | undefined> {
if (this.hook !== undefined) {
return await this.hook.findOne(this.client, _class, query, options)
}
return await this.client.findOne(_class, query, options)
}
get getConnection (): (() => ClientConnection) | undefined {
return this.client.getConnection
}
async close (): Promise<void> {
await this.client.close()
}
async findAll<T extends Doc>(
_class: Ref<Class<T>>,
query: DocumentQuery<T>,
options?: FindOptions<T>
): Promise<FindResult<T>> {
if (this.hook !== undefined) {
return await this.hook.findAll(this.client, _class, query, options)
}
return await this.client.findAll(_class, query, options)
}
async tx (tx: Tx): Promise<TxResult> {
if (this.hook !== undefined) {
return await this.hook.tx(this.client, tx)
}
return await this.client.tx(tx)
}
async searchFulltext (query: SearchQuery, options: SearchOptions): Promise<SearchResult> {
if (this.hook !== undefined) {
return await this.hook.searchFulltext(this.client, query, options)
}
return await this.client.searchFulltext(query, options)
}
}
/**
* @public
*/
@ -293,6 +351,12 @@ export async function setClient (_client: Client): Promise<void> {
await pipeline.close()
}
const hook = getMetadata(plugin.metadata.ClientHook)
if (hook !== undefined) {
_client = new ClientHookImpl(_client, hook)
}
const needRefresh = liveQuery !== undefined
rawLiveQuery = new LQ(_client)

View File

@ -310,6 +310,9 @@ class Connection implements ClientConnection {
)
this.currentRateLimit = resp.rateLimit
if (this.currentRateLimit.remaining < this.currentRateLimit.limit / 3) {
if (this.slowDownTimer < 50) {
this.slowDownTimer += 50
}
this.slowDownTimer++
} else if (this.slowDownTimer > 0) {
this.slowDownTimer--

View File

@ -102,7 +102,7 @@ export class PresentationClientHook implements ClientHook {
' =>model',
client.getModel(),
getMetadata(devmodel.metadata.DevModel),
platformNow() - startTime,
platformNowDiff(startTime),
this.stackLine()
)
}
@ -129,7 +129,7 @@ export class PresentationClientHook implements ClientHook {
' =>model',
client.getModel(),
getMetadata(devmodel.metadata.DevModel),
platformNow() - startTime,
platformNowDiff(startTime),
JSON.stringify(result).length,
this.stackLine()
)

View File

@ -65,8 +65,8 @@ function rateLimitToHeaders (rateLimit?: RateLimitInfo): OutgoingHttpHeaders {
}
const { remaining, limit, reset, retryAfter } = rateLimit
return {
'Retry-After': `${Math.max(retryAfter ?? 0, 1)}`,
'Retry-After-ms': `${retryAfter ?? 0}`,
'Retry-After': `${Math.max(Math.round((retryAfter ?? 0) / 1000), 1)}`,
'Retry-After-ms': `${retryAfter ?? 1000}`,
'X-RateLimit-Limit': `${limit}`,
'X-RateLimit-Remaining': `${remaining}`,
'X-RateLimit-Reset': `${reset}`
@ -188,8 +188,8 @@ export function registerRPC (app: Express, sessions: SessionManager, ctx: Measur
...keepAliveOptions,
'Content-Type': 'application/json',
'Cache-Control': 'no-cache',
'Retry-After': `${Math.max((retryAfter ?? 0) / 1000, 1)}`,
'Retry-After-ms': `${retryAfter ?? 0}`,
'Retry-After': `${Math.max(Math.round((retryAfter ?? 0) / 1000), 1)}`,
'Retry-After-ms': `${retryAfter ?? 1000}`,
'X-RateLimit-Limit': `${limit}`,
'X-RateLimit-Remaining': `${remaining}`,
'X-RateLimit-Reset': `${reset}`

View File

@ -277,7 +277,7 @@ async function getCreateReferencesTxes (
refs.push(...attrReferences)
} else if (attr.type._class === core.class.TypeCollaborativeDoc) {
const blobId = (createdDoc as any)[attr.name] as Ref<Blob>
if (blobId != null) {
if (blobId != null && blobId !== '') {
try {
const buffer = await storage.read(ctx, control.workspace, blobId)
const markup = Buffer.concat(buffer as any).toString()

View File

@ -14,6 +14,7 @@
// limitations under the License.
//
import card from '@hcengineering/card'
import contact, {
Channel,
Contact,
@ -38,7 +39,6 @@ import core, {
MarkupBlobRef,
Ref,
SocialIdType,
SortingOrder,
type Space,
SpaceType,
systemAccountUuid,
@ -50,10 +50,9 @@ import core, {
TxUpdateDoc,
TypedSpace
} from '@hcengineering/core'
import { makeRank } from '@hcengineering/rank'
import card from '@hcengineering/card'
import notification, { Collaborators } from '@hcengineering/notification'
import { getMetadata } from '@hcengineering/platform'
import { makeRank } from '@hcengineering/rank'
import { getAccountBySocialId, getCurrentPerson } from '@hcengineering/server-contact'
import serverCore, { TriggerControl } from '@hcengineering/server-core'
import { workbenchId } from '@hcengineering/workbench'
@ -169,13 +168,10 @@ export async function OnPersonCreate (_txes: Tx[], control: TriggerControl): Pro
const result: Tx[] = []
for (const tx of _txes) {
const ctx = tx as TxCreateDoc<Person>
const lastOne = (
await control.findAll(control.ctx, card.class.Card, {}, { sort: { rank: SortingOrder.Descending }, limit: 1 })
)[0]
const userProfileTx = control.txFactory.createTxCreateDoc<UserProfile>(contact.class.UserProfile, ctx.objectSpace, {
person: ctx.objectId,
title: formatName(ctx.attributes.name),
rank: makeRank(lastOne?.rank, undefined),
rank: makeRank(undefined, undefined),
content: '' as MarkupBlobRef,
parentInfo: [],
blobs: {}

View File

@ -44,12 +44,11 @@ export class SlidingWindowRateLimitter {
rateLimit.requests.push(now + (rateLimit.rejectedRequests > this.rateLimitMax * 2 ? this.rateLimitWindow * 5 : 0))
}
if (rateLimit.requests.length > this.rateLimitMax) {
if (rateLimit.requests.length >= this.rateLimitMax) {
rateLimit.rejectedRequests++
// Find when the oldest request will exit the window
const someRequest = Math.round(Math.random() * rateLimit.requests.length)
const nextAvailableTime = rateLimit.requests[someRequest] + this.rateLimitWindow
const nextAvailableTime = rateLimit.requests[0] + this.rateLimitWindow
return {
remaining: 0,

View File

@ -1211,7 +1211,7 @@ export class TSessionManager implements SessionManager {
requestCtx: MeasureContext,
service: S,
ws: ConnectionSocket,
operation: (ctx: ClientSessionCtx) => Promise<void>
operation: (ctx: ClientSessionCtx, rateLimit: RateLimitInfo | undefined) => Promise<void>
): Promise<RateLimitInfo | undefined> {
const rateLimitStatus = this.limitter.checkRateLimit(service.getUser())
// If remaining is 0, rate limit is exceeded
@ -1252,7 +1252,7 @@ export class TSessionManager implements SessionManager {
ws,
rateLimitStatus
)
await operation(uctx)
await operation(uctx, rateLimitStatus)
})
} catch (err: any) {
Analytics.handleError(err)

View File

@ -26,8 +26,8 @@
"compression-webpack-plugin": "^10.0.0",
"html-webpack-plugin": "^5.5.0",
"fork-ts-checker-webpack-plugin": "^9.0.2",
"update-browserslist-db": "^1.1.2",
"browserslist": "^4.24.4",
"update-browserslist-db": "^1.1.3",
"browserslist": "^4.25.0",
"esbuild": "^0.24.2",
"esbuild-loader": "^4.0.3",
"typescript": "^5.3.3",