Merge branch 'develop' into readonly-workspace-access

Signed-off-by: Anton Alexeyev <alexeyev.anton@gmail.com>
This commit is contained in:
Anton Alexeyev 2025-06-02 11:39:28 +07:00 committed by GitHub
commit 59ae6413fc
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
25 changed files with 390 additions and 181 deletions

View File

@ -261,26 +261,26 @@ jobs:
run: |
cd ./tests/sanity
node ../../common/scripts/install-run-rushx.js ci
- name: Start profiling
run: |
cd ./tests
./profile-start.sh
# - name: Start profiling
# run: |
# cd ./tests
# ./profile-start.sh
- name: Run UI tests
run: |
cd ./tests/sanity
node ../../common/scripts/install-run-rushx.js uitest
- name: Download profile
run: |
cd ./tests
./profile-download.sh
npm install -g cpupro
./profile-generate.sh
- name: Upload profiling results
if: always()
uses: actions/upload-artifact@v4
with:
name: profiling
path: ./tests/profiles
# - name: Download profile
# run: |
# cd ./tests
# ./profile-download.sh
# npm install -g cpupro
# ./profile-generate.sh
# - name: Upload profiling results
# if: always()
# uses: actions/upload-artifact@v4
# with:
# name: profiling
# path: ./tests/profiles
- name: 'Store docker logs'
if: always()
run: |

8
.vscode/launch.json vendored
View File

@ -367,9 +367,7 @@
"METRICS_CONSOLE": "true",
"ACCOUNTS_URL": "http://localhost:3000",
"MONGO_URL": "mongodb://localhost:27017",
"MINIO_ACCESS_KEY": "minioadmin",
"MINIO_SECRET_KEY": "minioadmin",
"MINIO_ENDPOINT": "localhost"
"STORAGE_CONFIG": "datalake|http://localhost:4030"
},
"runtimeVersion": "20",
"runtimeArgs": ["--nolazy", "-r", "ts-node/register"],
@ -763,8 +761,8 @@
"env": {
"PORT": "4030",
"SECRET": "secret",
"DB_URL": "",
"BUCKETS": "",
"DB_URL": "postgresql://root@localhost:26257/defaultdb?sslmode=disable",
"BUCKETS": "blobs,eu|http://localhost:9000?accessKey=minioadmin&secretKey=minioadmin",
"ACCOUNTS_URL": "http://localhost:3000",
"STATS_URL": "http://huly.local:4900",
"STREAM_URL": "http://huly.local:1080/recording"

View File

@ -504,7 +504,7 @@ importers:
version: file:projects/model-communication.tgz
'@rush-temp/model-contact':
specifier: file:./projects/model-contact.tgz
version: file:projects/model-contact.tgz(encoding@0.1.13)
version: file:projects/model-contact.tgz
'@rush-temp/model-controlled-documents':
specifier: file:./projects/model-controlled-documents.tgz
version: file:projects/model-controlled-documents.tgz
@ -1572,7 +1572,7 @@ importers:
version: 3.0.11
'@types/node-fetch':
specifier: ~2.6.2
version: 2.6.11
version: 2.6.12
'@types/nodemailer':
specifier: ^6.4.17
version: 6.4.17
@ -1690,9 +1690,6 @@ importers:
cross-env:
specifier: ~7.0.3
version: 7.0.3
cross-fetch:
specifier: ^3.1.5
version: 3.1.8(encoding@0.1.13)
crypto-js:
specifier: ^4.2.0
version: 4.2.0
@ -4473,7 +4470,7 @@ packages:
version: 0.0.0
'@rush-temp/model-contact@file:projects/model-contact.tgz':
resolution: {integrity: sha512-+wfUTUKjOzRKHOpcINUdbA6SGyJOYE+hxIiY2ZAL/3x6CqWb720+uhpkmNqb88ZXubhrOCT4z/nDXg29QkqL4A==, tarball: file:projects/model-contact.tgz}
resolution: {integrity: sha512-pUqoMMgSoArIiZmW9SL5BBapmezesVqgb6k0lCNNswPAP+UoKOChFTyqNAzzeTZ1AiLms4hvXbPEVlM1tKMKfg==, tarball: file:projects/model-contact.tgz}
version: 0.0.0
'@rush-temp/model-controlled-documents@file:projects/model-controlled-documents.tgz':
@ -5537,7 +5534,7 @@ packages:
version: 0.0.0
'@rush-temp/tool@file:projects/tool.tgz':
resolution: {integrity: sha512-97pzx1qZJlMkJmWU1JEKOII/g4WM7evf3NZuj+9uHAvqjtpfHJDZ7QYzA4P7tMz+DVmCNjqUxWQyCSvHz5MqJA==, tarball: file:projects/tool.tgz}
resolution: {integrity: sha512-M5PhT+eqpWqh6vTLuYmcQhjIYmRyoYNGc7pFOmozt4RgKb5s0V5AwfqIkPr23DiBZm7Q9aYrscp1NxXx/RObOg==, tarball: file:projects/tool.tgz}
version: 0.0.0
'@rush-temp/tracker-assets@file:projects/tracker-assets.tgz':
@ -6500,8 +6497,8 @@ packages:
'@types/node-cron@3.0.11':
resolution: {integrity: sha512-0ikrnug3/IyneSHqCBeslAhlK2aBfYek1fGo4bP4QnZPmiqSGRK+Oy7ZMisLWkesffJvQ1cqAcBnJC+8+nxIAg==}
'@types/node-fetch@2.6.11':
resolution: {integrity: sha512-24xFj9R5+rfQJLRyM56qh+wnVSYhyXC2tkoBndtY0U+vubqNsYXGjufB2nn8Q6gt0LrARwL6UBtMCSVCwl4B1g==}
'@types/node-fetch@2.6.12':
resolution: {integrity: sha512-8nneRWKCg3rMtF69nLQJnOYUcbafYeFSjqkw3jCRLsqkWFlHaoQrr5mXmofFGOx3DKn7UfmBMyov8ySvLRVldA==}
'@types/node-forge@1.3.11':
resolution: {integrity: sha512-FQx220y22OKNTqaByeBGqHWYz4cl94tpcxeFdvBo3wjG6XPBuZ0BNgNZRV5J5TFmmcsJ4IzsLkmGRiQbnYsBEQ==}
@ -11101,6 +11098,7 @@ packages:
node-domexception@1.0.0:
resolution: {integrity: sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==}
engines: {node: '>=10.5.0'}
deprecated: Use your platform's native DOMException instead
node-fetch@2.7.0:
resolution: {integrity: sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==}
@ -16758,7 +16756,7 @@ snapshots:
dependencies:
'@types/jest': 29.5.12
'@types/node': 20.11.19
'@types/node-fetch': 2.6.11
'@types/node-fetch': 2.6.12
'@types/snappyjs': 0.7.1
'@types/ws': 8.5.11
'@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.6.2)
@ -18033,7 +18031,7 @@ snapshots:
dependencies:
'@types/jest': 29.5.12
'@types/node': 20.11.19
'@types/node-fetch': 2.6.11
'@types/node-fetch': 2.6.12
'@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.3.3)
'@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.3.3)
eslint: 8.56.0
@ -20410,11 +20408,10 @@ snapshots:
transitivePeerDependencies:
- supports-color
'@rush-temp/model-contact@file:projects/model-contact.tgz(encoding@0.1.13)':
'@rush-temp/model-contact@file:projects/model-contact.tgz':
dependencies:
'@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.3.3)
'@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.3.3)
cross-fetch: 3.1.8(encoding@0.1.13)
eslint: 8.56.0
eslint-config-standard-with-typescript: 40.0.0(@typescript-eslint/eslint-plugin@6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.3.3))(eslint-plugin-import@2.29.1(eslint@8.56.0))(eslint-plugin-n@15.7.0(eslint@8.56.0))(eslint-plugin-promise@6.1.1(eslint@8.56.0))(eslint@8.56.0)(typescript@5.3.3)
eslint-plugin-import: 2.29.1(eslint@8.56.0)
@ -20423,7 +20420,6 @@ snapshots:
prettier: 3.2.5
typescript: 5.3.3
transitivePeerDependencies:
- encoding
- supports-color
'@rush-temp/model-controlled-documents@file:projects/model-controlled-documents.tgz':
@ -27335,6 +27331,7 @@ snapshots:
'@rush-temp/tool@file:projects/tool.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(gcp-metadata@5.3.0(encoding@0.1.13))(snappy@7.2.2)(socks@2.8.3)':
dependencies:
'@elastic/elasticsearch': 7.17.14
'@faker-js/faker': 8.4.1
'@types/jest': 29.5.12
'@types/mime-types': 2.1.4
'@types/minio': 7.0.18
@ -29089,7 +29086,7 @@ snapshots:
'@types/node-cron@3.0.11': {}
'@types/node-fetch@2.6.11':
'@types/node-fetch@2.6.12':
dependencies:
'@types/node': 20.11.19
form-data: 4.0.0
@ -35072,7 +35069,7 @@ snapshots:
openai@4.56.0(encoding@0.1.13)(zod@3.23.8):
dependencies:
'@types/node': 18.19.17
'@types/node-fetch': 2.6.11
'@types/node-fetch': 2.6.12
abort-controller: 3.0.0
agentkeepalive: 4.5.0
form-data-encoder: 1.7.2

View File

@ -20,7 +20,12 @@ import { configurePlatformDevServer } from './platform-dev'
configurePlatform().then(() => {
if (process.env.CLIENT_TYPE === 'dev-server' || process.env.CLIENT_TYPE === 'dev-production' || process.env.CLIENT_TYPE === 'dev-huly' || process.env.CLIENT_TYPE === 'dev-bold') {
if (process.env.CLIENT_TYPE === 'dev-server' ||
process.env.CLIENT_TYPE === 'dev-production' ||
process.env.CLIENT_TYPE === 'dev-huly' ||
process.env.CLIENT_TYPE === 'dev-bold' ||
process.env.CLIENT_TYPE === 'dev-server-test'
) {
configurePlatformDevServer()
}

View File

@ -175,6 +175,8 @@
"utf-8-validate": "^6.0.4",
"msgpackr": "^1.11.2",
"msgpackr-extract": "^3.0.3",
"@hcengineering/kafka": "^0.6.0"
"@hcengineering/kafka": "^0.6.0",
"@hcengineering/api-client": "^0.6.0",
"@faker-js/faker": "^8.4.1"
}
}

View File

@ -34,7 +34,7 @@ import {
createStorageBackupStorage,
restore
} from '@hcengineering/server-backup'
import serverClientPlugin, { getAccountClient } from '@hcengineering/server-client'
import serverClientPlugin, { getAccountClient, getTransactorEndpoint } from '@hcengineering/server-client'
import {
registerAdapterFactory,
registerDestroyFactory,
@ -47,6 +47,7 @@ import {
import serverToken, { generateToken } from '@hcengineering/server-token'
import { createWorkspace, upgradeWorkspace } from '@hcengineering/workspace-service'
import { faker } from '@faker-js/faker'
import { getPlatformQueue } from '@hcengineering/kafka'
import { buildStorageFromConfig, createStorageFromConfig, storageConfigFromEnv } from '@hcengineering/server-storage'
import { program, type Command } from 'commander'
@ -56,7 +57,10 @@ import {
AccountRole,
MeasureMetricsContext,
metricsToString,
SocialIdType,
systemAccountEmail,
systemAccountUuid,
type AccountUuid,
type Data,
type Doc,
type PersonId,
@ -93,17 +97,19 @@ import { getAccountDBUrl, getKvsUrl, getMongoDBUrl } from './__start'
// import { fillGithubUsers, fixAccountEmails, renameAccount } from './account'
import { changeConfiguration } from './configuration'
import { performGithubAccountMigrations } from './github'
import {
migrateCreatedModifiedBy,
ensureGlobalPersonsForLocalAccounts,
moveAccountDbFromMongoToPG,
migrateMergedAccounts,
filterMergedAccountsInMembers
} from './db'
import { getToolToken, getWorkspace, getWorkspaceTransactorEndpoint } from './utils'
import { performGmailAccountMigrations } from './gmail'
import { performCalendarAccountMigrations } from './calendar'
import {
ensureGlobalPersonsForLocalAccounts,
filterMergedAccountsInMembers,
migrateCreatedModifiedBy,
migrateMergedAccounts,
moveAccountDbFromMongoToPG
} from './db'
import { performGithubAccountMigrations } from './github'
import { performGmailAccountMigrations } from './gmail'
import { getToolToken, getWorkspace, getWorkspaceTransactorEndpoint } from './utils'
import { createRestClient } from '@hcengineering/api-client'
const colorConstants = {
colorRed: '\u001b[31m',
@ -189,7 +195,6 @@ export function devTool (
console.error(err)
}
closeAccountsDb()
console.log(`closing database connection to '${uri}'...`)
await shutdownMongo()
}
@ -1659,13 +1664,43 @@ export function devTool (
// })
// })
// program
// .command('generate-token <name> <workspace>')
// .description('generate token')
// .option('--admin', 'Generate token with admin access', false)
// .action(async (name: string, workspace: string, opt: { admin: boolean }) => {
// console.log(generateToken(name, getWorkspaceId(workspace), { ...(opt.admin ? { admin: 'true' } : {}) }))
// })
program
.command('generate-token <name> <workspace>')
.description('generate token')
.option('--admin', 'Generate token with admin access', false)
.action(async (name: string, workspace: string, opt: { admin: boolean }) => {
await withAccountDatabase(async (db) => {
if (name === systemAccountEmail) {
name = systemAccountUuid
}
const wsByUrl = await db.workspace.findOne({ url: workspace })
const account = await db.socialId.findOne({ key: name })
console.log(
generateToken(account?.personUuid ?? (name as AccountUuid), wsByUrl?.uuid ?? (workspace as WorkspaceUuid), {
...(opt.admin ? { admin: 'true' } : {})
})
)
})
})
program
.command('generate-persons <workspace>')
.description('generate a random persons into workspace')
.option('--admin', 'Generate token with admin access', false)
.option('--count <count>', 'Number of persons to generate', '1000')
.action(async (workspace: string, opt: { admin: boolean, count: string }) => {
const count = parseInt(opt.count)
const token = generateToken(systemAccountUuid, workspace as WorkspaceUuid, {
...(opt.admin ? { admin: 'true', service: 'tool' } : { service: 'tool' })
})
const endpoint = await getTransactorEndpoint(token, 'external')
const client = createRestClient(endpoint, workspace, token)
for (let i = 0; i < count; i++) {
const email = `${faker.internet.email()}`
await client.ensurePerson(SocialIdType.EMAIL, email, faker.person.firstName(), faker.person.lastName())
}
})
// program
// .command('decode-token <token>')
// .description('decode token')

View File

@ -52,7 +52,6 @@
"@hcengineering/rank": "^0.6.4",
"@hcengineering/ui": "^0.6.15",
"@hcengineering/view": "^0.6.13",
"@hcengineering/workbench": "^0.6.16",
"cross-fetch": "^3.1.5"
"@hcengineering/workbench": "^0.6.16"
}
}

View File

@ -36,12 +36,12 @@ import {
type TxResult,
type WithLookup
} from '@hcengineering/core'
import { PlatformError, unknownError } from '@hcengineering/platform'
import { PlatformError, type Status, unknownError } from '@hcengineering/platform'
import type { RestClient } from './types'
import { AuthOptions } from '../types'
import { extractJson, withRetry } from './utils'
import { getWorkspaceToken } from '../utils'
import type { RestClient } from './types'
import { extractJson, withRetry } from './utils'
export function createRestClient (endpoint: string, workspaceId: string, token: string): RestClient {
return new RestClientImpl(endpoint, workspaceId, token)
@ -62,6 +62,7 @@ export class RestClientImpl implements RestClient {
endpoint: string
slowDownTimer = 0
currentRateLimit: { remaining: number, limit: number } = { remaining: 1000, limit: 1000 }
remaining: number = 1000
limit: number = 1000
@ -103,12 +104,13 @@ export class RestClientImpl implements RestClient {
params.append('options', JSON.stringify(options))
}
const requestUrl = concatLink(this.endpoint, `/api/v1/find-all/${this.workspace}?${params.toString()}`)
const result = await withRetry(async () => {
const result = await withRetry<FindResult<T> & { error?: Status }>(async () => {
const response = await fetch(requestUrl, this.requestInit())
if (!response.ok) {
await this.checkRateLimits(response)
throw new PlatformError(unknownError(response.statusText))
}
this.updateRateLimit(response)
return await extractJson<FindResult<T>>(response)
}, isRLE)
@ -122,9 +124,9 @@ export class RestClientImpl implements RestClient {
if (d.$lookup !== undefined) {
for (const [k, v] of Object.entries(d.$lookup)) {
if (!Array.isArray(v)) {
d.$lookup[k] = result.lookupMap[v as any]
;(d as any).$lookup[k] = result.lookupMap[v]
} else {
d.$lookup[k] = v.map((it) => result.lookupMap?.[it])
;(d as any).$lookup[k] = v.map((it) => result.lookupMap?.[it])
}
}
}
@ -140,8 +142,8 @@ export class RestClientImpl implements RestClient {
}
for (const [k, v] of Object.entries(query)) {
if (typeof v === 'string' || typeof v === 'number' || typeof v === 'boolean') {
if (doc[k] == null) {
doc[k] = v
if ((doc as any)[k] == null) {
;(doc as any)[k] = v
}
}
}
@ -150,20 +152,42 @@ export class RestClientImpl implements RestClient {
return result
}
private async checkRate (): Promise<void> {
if (this.currentRateLimit.remaining < this.currentRateLimit.limit / 3) {
if (this.slowDownTimer < 50) {
this.slowDownTimer += 50
}
this.slowDownTimer++
} else if (this.slowDownTimer > 0) {
this.slowDownTimer--
}
if (this.slowDownTimer > 0) {
// We need to wait a bit to avoid ban.
await new Promise((resolve) => setTimeout(resolve, this.slowDownTimer))
}
}
private updateRateLimit (response: Response): void {
const rateLimitLimit: number = parseInt(response.headers.get('X-RateLimit-Limit') ?? '100')
const remaining: number = parseInt(response.headers.get('X-RateLimit-Remaining') ?? '100')
this.currentRateLimit = { remaining, limit: rateLimitLimit }
}
private async checkRateLimits (response: Response): Promise<void> {
if (response.status === 429) {
// Extract rate limit information from headers
const retryAfter = response.headers.get('Retry-After')
const retryAfterMS = response.headers.get('Retry-After-ms')
const rateLimitReset = response.headers.get('X-RateLimit-Reset')
// const rateLimitLimit: string | null = response.headers.get('X-RateLimit-Limit')
this.updateRateLimit(response)
const waitTime =
retryAfter != null
? parseInt(retryAfter)
(retryAfterMS != null ? parseInt(retryAfterMS) : undefined) ??
(retryAfter != null
? parseInt(retryAfter) * 1000
: rateLimitReset != null
? new Date(parseInt(rateLimitReset)).getTime() - Date.now()
: 1000 // Default to 1 seconds if no headers are provided
console.warn(`Rate limit exceeded. Waiting ${Math.round((10 * waitTime) / 1000) / 10} seconds before retrying...`)
: 1000) // Default to 1 seconds if no headers are provided
await new Promise((resolve) => setTimeout(resolve, waitTime))
throw new Error(rateLimitError)
}
@ -171,28 +195,47 @@ export class RestClientImpl implements RestClient {
async getAccount (): Promise<Account> {
const requestUrl = concatLink(this.endpoint, `/api/v1/account/${this.workspace}`)
const response = await fetch(requestUrl, this.requestInit())
if (!response.ok) {
throw new PlatformError(unknownError(response.statusText))
await this.checkRate()
const result = await withRetry<Account & { error?: Status }>(async () => {
const response = await fetch(requestUrl, this.requestInit())
if (!response.ok) {
await this.checkRateLimits(response)
throw new PlatformError(unknownError(response.statusText))
}
this.updateRateLimit(response)
return await extractJson<Account>(response)
})
if (result.error !== undefined) {
throw new PlatformError(result.error)
}
return await extractJson<Account>(response)
return result
}
async getModel (): Promise<{ hierarchy: Hierarchy, model: ModelDb }> {
const requestUrl = concatLink(this.endpoint, `/api/v1/load-model/${this.workspace}`)
const response = await fetch(requestUrl, this.requestInit())
if (!response.ok) {
throw new PlatformError(unknownError(response.statusText))
await this.checkRate()
const result = await withRetry<{ hierarchy: Hierarchy, model: ModelDb, error?: Status }>(async () => {
const response = await fetch(requestUrl, this.requestInit())
if (!response.ok) {
await this.checkRateLimits(response)
throw new PlatformError(unknownError(response.statusText))
}
this.updateRateLimit(response)
const modelResponse: Tx[] = await extractJson<Tx[]>(response)
const hierarchy = new Hierarchy()
const model = new ModelDb(hierarchy)
const ctx = new MeasureMetricsContext('loadModel', {})
buildModel(ctx, modelResponse, (txes: Tx[]) => txes, hierarchy, model)
return { hierarchy, model }
}, isRLE)
if (result.error !== undefined) {
throw new PlatformError(result.error)
}
const modelResponse: Tx[] = await extractJson<Tx[]>(response)
const hierarchy = new Hierarchy()
const model = new ModelDb(hierarchy)
const ctx = new MeasureMetricsContext('loadModel', {})
buildModel(ctx, modelResponse, (txes: Tx[]) => txes, hierarchy, model)
return { hierarchy, model }
return result
}
async findOne<T extends Doc>(
@ -205,7 +248,8 @@ export class RestClientImpl implements RestClient {
async tx (tx: Tx): Promise<TxResult> {
const requestUrl = concatLink(this.endpoint, `/api/v1/tx/${this.workspace}`)
const result = await withRetry(async () => {
await this.checkRate()
const result = await withRetry<TxResult & { error?: Status }>(async () => {
const response = await fetch(requestUrl, {
method: 'POST',
headers: this.jsonHeaders(),
@ -216,6 +260,7 @@ export class RestClientImpl implements RestClient {
await this.checkRateLimits(response)
throw new PlatformError(unknownError(response.statusText))
}
this.updateRateLimit(response)
return await extractJson<TxResult>(response)
}, isRLE)
if (result.error !== undefined) {
@ -225,27 +270,35 @@ export class RestClientImpl implements RestClient {
}
async searchFulltext (query: SearchQuery, options: SearchOptions): Promise<SearchResult> {
const params = new URLSearchParams()
params.append('query', query.query)
if (query.classes != null && Object.keys(query.classes).length > 0) {
params.append('classes', JSON.stringify(query.classes))
}
if (query.spaces != null && Object.keys(query.spaces).length > 0) {
params.append('spaces', JSON.stringify(query.spaces))
}
if (options.limit != null) {
params.append('limit', `${options.limit}`)
}
const requestUrl = concatLink(this.endpoint, `/api/v1/search-fulltext/${this.workspace}?${params.toString()}`)
const response = await fetch(requestUrl, {
method: 'GET',
headers: this.jsonHeaders(),
keepalive: true
const result = await withRetry<SearchResult & { error?: Status }>(async () => {
const params = new URLSearchParams()
params.append('query', query.query)
if (query.classes != null && Object.keys(query.classes).length > 0) {
params.append('classes', JSON.stringify(query.classes))
}
if (query.spaces != null && Object.keys(query.spaces).length > 0) {
params.append('spaces', JSON.stringify(query.spaces))
}
if (options.limit != null) {
params.append('limit', `${options.limit}`)
}
const requestUrl = concatLink(this.endpoint, `/api/v1/search-fulltext/${this.workspace}?${params.toString()}`)
const response = await fetch(requestUrl, {
method: 'GET',
headers: this.jsonHeaders(),
keepalive: true
})
if (!response.ok) {
await this.checkRateLimits(response)
throw new PlatformError(unknownError(response.statusText))
}
this.updateRateLimit(response)
return await extractJson<TxResult>(response)
})
if (!response.ok) {
throw new PlatformError(unknownError(response.statusText))
if (result.error !== undefined) {
throw new PlatformError(result.error)
}
return await extractJson<TxResult>(response)
return result
}
async ensurePerson (
@ -255,20 +308,29 @@ export class RestClientImpl implements RestClient {
lastName: string
): Promise<{ uuid: PersonUuid, socialId: PersonId, localPerson: string }> {
const requestUrl = concatLink(this.endpoint, `/api/v1/ensure-person/${this.workspace}`)
const response = await fetch(requestUrl, {
method: 'POST',
headers: this.jsonHeaders(),
keepalive: true,
body: JSON.stringify({
socialType,
socialValue,
firstName,
lastName
await this.checkRate()
const result = await withRetry(async () => {
const response = await fetch(requestUrl, {
method: 'POST',
headers: this.jsonHeaders(),
keepalive: true,
body: JSON.stringify({
socialType,
socialValue,
firstName,
lastName
})
})
})
if (!response.ok) {
throw new PlatformError(unknownError(response.statusText))
if (!response.ok) {
await this.checkRateLimits(response)
throw new PlatformError(unknownError(response.statusText))
}
this.updateRateLimit(response)
return await extractJson<{ uuid: PersonUuid, socialId: PersonId, localPerson: string }>(response)
}, isRLE)
if (result.error !== undefined) {
throw new PlatformError(result.error)
}
return await extractJson<{ uuid: PersonUuid, socialId: PersonId, localPerson: string }>(response)
return result
}
}

View File

@ -17,7 +17,7 @@ import core from './component'
export * from './classes'
export * from './client'
export * from './collaboration'
export { coreId, systemAccountUuid, systemAccount, configUserAccountUuid } from './component'
export { coreId, systemAccountUuid, systemAccountEmail, systemAccount, configUserAccountUuid } from './component'
export * from './hierarchy'
export * from './measurements'
export * from './memdb'

View File

@ -22,6 +22,7 @@ import core, {
type AttachedDoc,
type Class,
type Client,
type ClientConnection,
type Collection,
type Doc,
type DocumentQuery,
@ -31,6 +32,7 @@ import core, {
type Hierarchy,
MeasureMetricsContext,
type Mixin,
type ModelDb,
type Obj,
reduceCalls,
type Ref,
@ -59,8 +61,8 @@ import { onDestroy } from 'svelte'
import { get, writable } from 'svelte/store'
import { type KeyedAttribute } from '..'
import { OptimizeQueryMiddleware, type PresentationPipeline, PresentationPipelineImpl } from './pipeline'
import plugin from './plugin'
import { OptimizeQueryMiddleware, PresentationPipelineImpl, type PresentationPipeline } from './pipeline'
import plugin, { type ClientHook } from './plugin'
export { reduceCalls } from '@hcengineering/core'
@ -98,7 +100,6 @@ export const uiContext = new MeasureMetricsContext('client-ui', {})
export const pendingCreatedDocs = writable<Record<Ref<Doc>, boolean>>({})
class UIClient extends TxOperations implements Client {
hook = getMetadata(plugin.metadata.ClientHook)
constructor (
client: Client,
private readonly liveQuery: Client
@ -157,9 +158,6 @@ class UIClient extends TxOperations implements Client {
query: DocumentQuery<T>,
options?: FindOptions<T>
): Promise<FindResult<T>> {
if (this.hook !== undefined) {
return await this.hook.findAll(this.liveQuery, _class, query, options)
}
return await this.liveQuery.findAll(_class, query, options)
}
@ -168,17 +166,13 @@ class UIClient extends TxOperations implements Client {
query: DocumentQuery<T>,
options?: FindOptions<T>
): Promise<WithLookup<T> | undefined> {
if (this.hook !== undefined) {
return await this.hook.findOne(this.liveQuery, _class, query, options)
}
return await this.liveQuery.findOne(_class, query, options)
}
override async tx (tx: Tx): Promise<TxResult> {
void this.notifyEarly(tx)
if (this.hook !== undefined) {
return await this.hook.tx(this.client, tx)
}
void this.notifyEarly(tx).catch((err) => {
console.error(err)
})
return await this.client.tx(tx)
}
@ -222,9 +216,6 @@ class UIClient extends TxOperations implements Client {
}
async searchFulltext (query: SearchQuery, options: SearchOptions): Promise<SearchResult> {
if (this.hook !== undefined) {
return await this.hook.searchFulltext(this.client, query, options)
}
return await this.client.searchFulltext(query, options)
}
}
@ -280,6 +271,73 @@ export function addRefreshListener (r: RefreshListener): void {
refreshListeners.add(r)
}
class ClientHookImpl implements Client {
constructor (
private readonly client: Client,
private readonly hook: ClientHook
) {}
set notify (op: (...tx: Tx[]) => void) {
this.client.notify = op
}
get notify (): ((...tx: Tx[]) => void) | undefined {
return this.client.notify
}
getHierarchy (): Hierarchy {
return this.client.getHierarchy()
}
getModel (): ModelDb {
return this.client.getModel()
}
async findOne<T extends Doc>(
_class: Ref<Class<T>>,
query: DocumentQuery<T>,
options?: FindOptions<T>
): Promise<WithLookup<T> | undefined> {
if (this.hook !== undefined) {
return await this.hook.findOne(this.client, _class, query, options)
}
return await this.client.findOne(_class, query, options)
}
get getConnection (): (() => ClientConnection) | undefined {
return this.client.getConnection
}
async close (): Promise<void> {
await this.client.close()
}
async findAll<T extends Doc>(
_class: Ref<Class<T>>,
query: DocumentQuery<T>,
options?: FindOptions<T>
): Promise<FindResult<T>> {
if (this.hook !== undefined) {
return await this.hook.findAll(this.client, _class, query, options)
}
return await this.client.findAll(_class, query, options)
}
async tx (tx: Tx): Promise<TxResult> {
if (this.hook !== undefined) {
return await this.hook.tx(this.client, tx)
}
return await this.client.tx(tx)
}
async searchFulltext (query: SearchQuery, options: SearchOptions): Promise<SearchResult> {
if (this.hook !== undefined) {
return await this.hook.searchFulltext(this.client, query, options)
}
return await this.client.searchFulltext(query, options)
}
}
/**
* @public
*/
@ -295,6 +353,12 @@ export async function setClient (_client: Client): Promise<void> {
await pipeline.close()
}
const hook = getMetadata(plugin.metadata.ClientHook)
if (hook !== undefined) {
_client = new ClientHookImpl(_client, hook)
}
const needRefresh = liveQuery !== undefined
rawLiveQuery = new LQ(_client)

View File

@ -180,7 +180,7 @@
{:else}
{#await getBlobRef(value.file, value.name, sizeToWidth('large')) then valueRef}
<a
class="no-line"
class="no-line no-underline"
style:flex-shrink={0}
href={valueRef.src}
download={value.name}
@ -211,7 +211,7 @@
</a>
</div>
<div class="info-content flex-row-center">
{#if value.size != null}{filesize(value.size, { spacer: '' })}{/if}
{#if value.size != null && value.size !== 0}{filesize(value.size, { spacer: '' })}{/if}
<span class="actions inline-flex clear-mins ml-1 gap-1">
<span></span>
<a class="no-line colorInherit" href={valueRef.src} download={value.name} bind:this={download}>

View File

@ -310,6 +310,9 @@ class Connection implements ClientConnection {
)
this.currentRateLimit = resp.rateLimit
if (this.currentRateLimit.remaining < this.currentRateLimit.limit / 3) {
if (this.slowDownTimer < 50) {
this.slowDownTimer += 50
}
this.slowDownTimer++
} else if (this.slowDownTimer > 0) {
this.slowDownTimer--

View File

@ -102,7 +102,7 @@ export class PresentationClientHook implements ClientHook {
' =>model',
client.getModel(),
getMetadata(devmodel.metadata.DevModel),
platformNow() - startTime,
platformNowDiff(startTime),
this.stackLine()
)
}
@ -129,7 +129,7 @@ export class PresentationClientHook implements ClientHook {
' =>model',
client.getModel(),
getMetadata(devmodel.metadata.DevModel),
platformNow() - startTime,
platformNowDiff(startTime),
JSON.stringify(result).length,
this.stackLine()
)

View File

@ -103,6 +103,7 @@
$: if (state !== null && state.state === 'stopped') {
dispatch('close')
recording.set(null)
}
async function updateCameraStream (
@ -269,6 +270,8 @@
async function handleStopRecording (): Promise<void> {
await stopRecording()
camEnabled = false
micEnabled = false
}
async function handleRestartRecording (): Promise<void> {
@ -412,6 +415,8 @@
}
function onKeyDown (e: KeyboardEvent): void {
if (state?.state === 'stopping') return
if (e.key === 'Enter') {
e.preventDefault()
e.stopPropagation()
@ -455,7 +460,7 @@
>
<div class="container p-3">
{#if state !== null && state.state === 'stopped'}
<div class="placeholder flex-col-center justify-center">OK</div>
<!-- Maybe show recording result -->
{:else if mainStream === null}
<div class="placeholder flex-col-center justify-center">
{#if cameraStreamPromise === null}
@ -590,6 +595,8 @@
icon={IconStop}
iconProps={{ size: 'small' }}
label={plugin.string.Stop}
disabled={state.state === 'stopping'}
loading={state.state === 'stopping'}
noFocus
on:click={handleStopRecording}
/>

View File

@ -1,15 +1,15 @@
<script lang="ts">
import { DropdownLabels, DropdownTextItem, Label } from '@hcengineering/ui'
import { DropdownLabelsIntl, DropdownIntlItem, Label } from '@hcengineering/ui'
import plugin from '../plugin'
import { recordingCameraPosition, recordingCameraSize } from '../stores'
const sizes: DropdownTextItem[] = [
const sizes: DropdownIntlItem[] = [
{ label: plugin.string.Small, id: 'small' },
{ label: plugin.string.Medium, id: 'medium' },
{ label: plugin.string.Large, id: 'large' }
]
const poses: DropdownTextItem[] = [
const poses: DropdownIntlItem[] = [
{ label: plugin.string.TopLeft, id: 'top-left' },
{ label: plugin.string.TopRight, id: 'top-right' },
{ label: plugin.string.BottomLeft, id: 'bottom-left' },
@ -19,10 +19,8 @@
<div class="antiPopup p-4 grid">
<Label label={plugin.string.CameraSize} />
<DropdownLabels
<DropdownLabelsIntl
items={sizes}
placeholder={plugin.string.CameraSize}
enableSearch={false}
justify={'left'}
width={'100%'}
selected={$recordingCameraSize}
@ -32,10 +30,8 @@
/>
<Label label={plugin.string.CameraPos} />
<DropdownLabels
<DropdownLabelsIntl
items={poses}
placeholder={plugin.string.CameraPos}
enableSearch={false}
justify={'left'}
width={'100%'}
selected={$recordingCameraPosition}

View File

@ -96,13 +96,56 @@ export class Recorder {
return this.chunkStream
}
public stop (): void {
public async stop (): Promise<void> {
if (this.lastStartTime !== null) {
this.elapsedMs += Date.now() - this.lastStartTime
this.lastStartTime = null
}
// If the recorder is paused, we need to resume it before stopping
try {
if (this.mediaRecorder.state === 'paused') {
const resumed = new Promise<void>((resolve) => {
const timeout = setTimeout(resolve, 500)
this.mediaRecorder.addEventListener(
'resume',
() => {
clearTimeout(timeout)
resolve()
},
{ once: true }
)
})
this.mediaRecorder.resume()
await resumed
}
} catch (err) {
console.error('Recorder: error resuming MediaRecorder:', err)
}
// Ensure we wait for any pending data to be available
try {
if (this.mediaRecorder.state === 'recording') {
const dataavailable = new Promise<void>((resolve) => {
const timeout = setTimeout(resolve, 500)
this.mediaRecorder.addEventListener(
'dataavailable',
() => {
clearTimeout(timeout)
resolve()
},
{ once: true }
)
})
this.mediaRecorder.requestData()
await dataavailable
}
} catch (err) {
console.error('Recorder: error requesting data from MediaRecorder:', err)
}
try {
// After requesting data, we can safely stop the recorder
this.mediaRecorder.stop()
} catch (err) {
console.error('Recorder: error stopping MediaRecorder:', err)
@ -111,6 +154,7 @@ export class Recorder {
track.stop()
}
}
console.debug('Recorder: recording stopped, duration:', this.getRecordedTimeMs(), 'ms')
}

View File

@ -51,12 +51,11 @@ export async function stopRecording (): Promise<void> {
const current = get(recording)
const popup = get(recorder)
if (current !== null && current.state === 'recording') {
recording.set({ ...current, state: 'stopped' })
recording.set({ ...current, state: 'stopping' })
const result = await current.recorder.stop()
await current.options.onSuccess?.(result)
recording.set(null)
recording.set({ ...current, state: 'stopped' })
popup?.close()
} else {
console.warn('Recording is not in `recording` state', current)
@ -80,7 +79,7 @@ export async function restartRecording (): Promise<void> {
await current.recorder.start()
recording.set({ ...current, state: 'recording' })
} else {
console.warn('Recording is not in `paused` state', current)
console.warn('Recording not started', current)
}
}

View File

@ -67,12 +67,12 @@ export class ScreenRecorder {
}
public async stop (): Promise<RecordingResult> {
this.recorder.stop()
await this.recorder.stop()
return await this.uploader.wait()
}
public async cancel (): Promise<void> {
this.recorder.stop()
await this.recorder.stop()
await this.uploader.cancel()
}
}

View File

@ -20,7 +20,7 @@ export interface RecordingState {
recorder: ScreenRecorder
options: RecordingOptions
stream: MediaStream
state: 'recording' | 'paused' | 'stopped'
state: 'recording' | 'paused' | 'stopping' | 'stopped'
}
export interface RecordingOptions {

View File

@ -65,8 +65,8 @@ function rateLimitToHeaders (rateLimit?: RateLimitInfo): OutgoingHttpHeaders {
}
const { remaining, limit, reset, retryAfter } = rateLimit
return {
'Retry-After': `${Math.max(retryAfter ?? 0, 1)}`,
'Retry-After-ms': `${retryAfter ?? 0}`,
'Retry-After': `${Math.max(Math.round((retryAfter ?? 0) / 1000), 1)}`,
'Retry-After-ms': `${retryAfter ?? 1000}`,
'X-RateLimit-Limit': `${limit}`,
'X-RateLimit-Remaining': `${remaining}`,
'X-RateLimit-Reset': `${reset}`
@ -188,8 +188,8 @@ export function registerRPC (app: Express, sessions: SessionManager, ctx: Measur
...keepAliveOptions,
'Content-Type': 'application/json',
'Cache-Control': 'no-cache',
'Retry-After': `${Math.max((retryAfter ?? 0) / 1000, 1)}`,
'Retry-After-ms': `${retryAfter ?? 0}`,
'Retry-After': `${Math.max(Math.round((retryAfter ?? 0) / 1000), 1)}`,
'Retry-After-ms': `${retryAfter ?? 1000}`,
'X-RateLimit-Limit': `${limit}`,
'X-RateLimit-Remaining': `${remaining}`,
'X-RateLimit-Reset': `${reset}`

View File

@ -277,7 +277,7 @@ async function getCreateReferencesTxes (
refs.push(...attrReferences)
} else if (attr.type._class === core.class.TypeCollaborativeDoc) {
const blobId = (createdDoc as any)[attr.name] as Ref<Blob>
if (blobId != null) {
if (blobId != null && blobId !== '') {
try {
const buffer = await storage.read(ctx, control.workspace, blobId)
const markup = Buffer.concat(buffer as any).toString()

View File

@ -14,6 +14,7 @@
// limitations under the License.
//
import card from '@hcengineering/card'
import contact, {
Channel,
Contact,
@ -38,7 +39,6 @@ import core, {
MarkupBlobRef,
Ref,
SocialIdType,
SortingOrder,
type Space,
SpaceType,
systemAccountUuid,
@ -50,10 +50,9 @@ import core, {
TxUpdateDoc,
TypedSpace
} from '@hcengineering/core'
import { makeRank } from '@hcengineering/rank'
import card from '@hcengineering/card'
import notification, { Collaborators } from '@hcengineering/notification'
import { getMetadata } from '@hcengineering/platform'
import { makeRank } from '@hcengineering/rank'
import { getAccountBySocialId, getCurrentPerson } from '@hcengineering/server-contact'
import serverCore, { TriggerControl } from '@hcengineering/server-core'
import { workbenchId } from '@hcengineering/workbench'
@ -169,13 +168,10 @@ export async function OnPersonCreate (_txes: Tx[], control: TriggerControl): Pro
const result: Tx[] = []
for (const tx of _txes) {
const ctx = tx as TxCreateDoc<Person>
const lastOne = (
await control.findAll(control.ctx, card.class.Card, {}, { sort: { rank: SortingOrder.Descending }, limit: 1 })
)[0]
const userProfileTx = control.txFactory.createTxCreateDoc<UserProfile>(contact.class.UserProfile, ctx.objectSpace, {
person: ctx.objectId,
title: formatName(ctx.attributes.name),
rank: makeRank(lastOne?.rank, undefined),
rank: makeRank(undefined, undefined),
content: '' as MarkupBlobRef,
parentInfo: [],
blobs: {}

View File

@ -44,12 +44,11 @@ export class SlidingWindowRateLimitter {
rateLimit.requests.push(now + (rateLimit.rejectedRequests > this.rateLimitMax * 2 ? this.rateLimitWindow * 5 : 0))
}
if (rateLimit.requests.length > this.rateLimitMax) {
if (rateLimit.requests.length >= this.rateLimitMax) {
rateLimit.rejectedRequests++
// Find when the oldest request will exit the window
const someRequest = Math.round(Math.random() * rateLimit.requests.length)
const nextAvailableTime = rateLimit.requests[someRequest] + this.rateLimitWindow
const nextAvailableTime = rateLimit.requests[0] + this.rateLimitWindow
return {
remaining: 0,

View File

@ -1211,7 +1211,7 @@ export class TSessionManager implements SessionManager {
requestCtx: MeasureContext,
service: S,
ws: ConnectionSocket,
operation: (ctx: ClientSessionCtx) => Promise<void>
operation: (ctx: ClientSessionCtx, rateLimit: RateLimitInfo | undefined) => Promise<void>
): Promise<RateLimitInfo | undefined> {
const rateLimitStatus = this.limitter.checkRateLimit(service.getUser())
// If remaining is 0, rate limit is exceeded
@ -1252,7 +1252,7 @@ export class TSessionManager implements SessionManager {
ws,
rateLimitStatus
)
await operation(uctx)
await operation(uctx, rateLimitStatus)
})
} catch (err: any) {
Analytics.handleError(err)

View File

@ -70,7 +70,7 @@ export async function handleBlobGet (
res.setHeader('Content-Security-Policy', "default-src 'none';")
res.setHeader(
'Content-Disposition',
filename !== undefined ? `attachment; filename*=UTF-8''${encodeURIComponent(filename)}"` : 'attachment'
filename !== undefined ? `attachment; filename*=UTF-8''${encodeURIComponent(filename)}` : 'attachment'
)
res.setHeader('Cache-Control', blob.cacheControl ?? cacheControl)
res.setHeader('Last-Modified', new Date(blob.lastModified).toUTCString())
@ -122,7 +122,10 @@ export async function handleBlobHead (
res.setHeader('Content-Length', head.size.toString())
res.setHeader('Content-Type', head.contentType ?? '')
res.setHeader('Content-Security-Policy', "default-src 'none';")
res.setHeader('Content-Disposition', filename !== undefined ? `attachment; filename="${filename}"` : 'attachment')
res.setHeader(
'Content-Disposition',
filename !== undefined ? `attachment; filename*=UTF-8''${encodeURIComponent(filename)}` : 'attachment'
)
res.setHeader('Cache-Control', head.cacheControl ?? cacheControl)
res.setHeader('Last-Modified', new Date(head.lastModified).toUTCString())
res.setHeader('ETag', wrapETag(head.etag))