From 6156455f4911ac91ff1d9b39dad3457811a4b752 Mon Sep 17 00:00:00 2001 From: Denis Bykhov Date: Sun, 23 Feb 2025 23:27:30 +0500 Subject: [PATCH 01/16] Rewrite calendar service (#8074) Signed-off-by: Denis Bykhov --- common/config/rush/pnpm-lock.yaml | 15 +- packages/core/src/classes.ts | 10 + pods/server/package.json | 1 + pods/server/src/__start.ts | 2 + .../calendar-resources/package.json | 2 + .../calendar-resources/src/index.ts | 47 +- server-plugins/calendar/src/index.ts | 5 +- server/account/src/operations.ts | 31 +- server/account/src/types.ts | 5 - server/client/src/account.ts | 20 + server/collaborator/src/account.ts | 2 +- .../calendar/pod-calendar/src/calendar.ts | 534 ++++++------------ .../pod-calendar/src/calendarController.ts | 162 +++--- .../calendar/pod-calendar/src/googleClient.ts | 313 ++++++++++ services/calendar/pod-calendar/src/main.ts | 30 +- services/calendar/pod-calendar/src/types.ts | 21 +- services/calendar/pod-calendar/src/watch.ts | 230 ++++++++ .../pod-calendar/src/workspaceClient.ts | 138 +++-- services/github/pod-github/src/account.ts | 2 +- services/github/pod-github/src/platform.ts | 2 +- 20 files changed, 1071 insertions(+), 501 deletions(-) create mode 100644 services/calendar/pod-calendar/src/googleClient.ts create mode 100644 services/calendar/pod-calendar/src/watch.ts diff --git a/common/config/rush/pnpm-lock.yaml b/common/config/rush/pnpm-lock.yaml index 5195fe978a..eda1faf365 100644 --- a/common/config/rush/pnpm-lock.yaml +++ b/common/config/rush/pnpm-lock.yaml @@ -4241,7 +4241,7 @@ packages: version: 0.0.0 '@rush-temp/model-card@file:projects/model-card.tgz': - resolution: {integrity: sha512-7yhnpluYR/Gypgi3kKPtEVe1NpcG8YfE7S3ACF56GIqXideFnlXYWJ8PjUDQ9Ub5BHUulGYTXKsFUqcHRxQDZA==, tarball: file:projects/model-card.tgz} + resolution: {integrity: sha512-diGAGkQiQNXD27/ln6i6al/sr7X2dOUdiTx6114JOahvZngprIhzdJLdXuW44BnQHTugr4j0Sp1zOwQXKcTBVA==, tarball: file:projects/model-card.tgz} version: 0.0.0 '@rush-temp/model-chunter@file:projects/model-chunter.tgz': @@ -4353,7 +4353,7 @@ packages: version: 0.0.0 '@rush-temp/model-server-card@file:projects/model-server-card.tgz': - resolution: {integrity: sha512-SP381dOkAOymE4k9VaEP2o28yv7oJxJEol5LeYjcqHXkyN/al/v8UetEGmS4/gvuxk0Np0KZPpLnczV9O3ApUg==, tarball: file:projects/model-server-card.tgz} + resolution: {integrity: sha512-yMw0GV5uufsqDDev756PLadZksaAwUcL1eq52hijUSI4yYC1cr5wvJLW97/g/znWX6kZuHC7lAGvrBcgHf2axw==, tarball: file:projects/model-server-card.tgz} version: 0.0.0 '@rush-temp/model-server-chunter@file:projects/model-server-chunter.tgz': @@ -4589,7 +4589,7 @@ packages: version: 0.0.0 '@rush-temp/pod-calendar@file:projects/pod-calendar.tgz': - resolution: {integrity: sha512-gUk3jshnHJ0CUucxZ1Stk8oQ2DTdSVfJzFHIl/EBUISbRv/YySoXesQCnqfn1kVpqOBZdIvI/N4NRAL/yREbWw==, tarball: file:projects/pod-calendar.tgz} + resolution: {integrity: sha512-BvN38ScXSeB5Mui24Sawekl81QZMFZ47EK6KdIqRJJQwM+Qj0owWul3y/r7ZtNAfKnD9bV3AeEWaqkZ8EXYjwg==, tarball: file:projects/pod-calendar.tgz} version: 0.0.0 '@rush-temp/pod-collaborator@file:projects/pod-collaborator.tgz': @@ -4621,7 +4621,7 @@ packages: version: 0.0.0 '@rush-temp/pod-server@file:projects/pod-server.tgz': - resolution: {integrity: sha512-U7wNYw+h8a6hEV6fwXYJFStTk5noW4LAsL1G4JCJjyQ7PVzlmXZqGncFe+bq5EqgYgbOewZ2tR+T8rB8HWi9Cw==, tarball: file:projects/pod-server.tgz} + resolution: {integrity: sha512-MJ5TQobQy4kv67qy7qpQgz056fhaZw8Zs/NaXniFafG/xjstgWeIJCXNHDkKvdBm+wT6EdHgNLIN3BaRpMSDsA==, tarball: file:projects/pod-server.tgz} version: 0.0.0 '@rush-temp/pod-ses@file:projects/pod-ses.tgz': @@ -4809,7 +4809,7 @@ packages: version: 0.0.0 '@rush-temp/server-calendar-resources@file:projects/server-calendar-resources.tgz': - resolution: {integrity: sha512-Rpdy7Nx56rXrJ1apUsvlancAVa5USFSwXboZ8iexTJ4Jaebv1yf32j5LUo/fLKwpZyL9KaUHT+A4N+nJKoRVWg==, tarball: file:projects/server-calendar-resources.tgz} + resolution: {integrity: sha512-pFGwg9Q/vogm2In88l39LD/B3Da/8sVTD3BVw3Or0VJ9peXU/YJHrqkgd4+ZU3RDRf4amDVz/rIpqMqjD0rS6Q==, tarball: file:projects/server-calendar-resources.tgz} version: 0.0.0 '@rush-temp/server-calendar@file:projects/server-calendar.tgz': @@ -4817,7 +4817,7 @@ packages: version: 0.0.0 '@rush-temp/server-card-resources@file:projects/server-card-resources.tgz': - resolution: {integrity: sha512-lp3RSXAHV78ux6p3I0iSbVkfY3aB0fXDfxMhxQZfjU9sSXLOsKolgDxMaUNXVm+/WIpS6dwLk4jWHgGopH8Nmw==, tarball: file:projects/server-card-resources.tgz} + resolution: {integrity: sha512-sJdtOVlhTFawBXtSTzcx+Vs8NALsiqBfwAz9cqUBj45vobiSF97D987LPBgjhCARk2+qx8lZyUPPTUJkJdaLOQ==, tarball: file:projects/server-card-resources.tgz} version: 0.0.0 '@rush-temp/server-card@file:projects/server-card.tgz': @@ -5193,7 +5193,7 @@ packages: version: 0.0.0 '@rush-temp/text-editor-resources@file:projects/text-editor-resources.tgz': - resolution: {integrity: sha512-JAu7yul5yHwjxWkBN9p1LW7HR8vAaRlf8Ioiyc7FYAKaQ1EBV05sCCpYsVayVpmU0vU92XzQffryB8TH/dZ6+A==, tarball: file:projects/text-editor-resources.tgz} + resolution: {integrity: sha512-PkvG582CT0XfnKC+4uNE1ZnwUgT9ywmfDmVr5vgcXti04Ntx1rB9VzTqnNDFI/A7h5dFAeEbR8s08gzgO6fdrw==, tarball: file:projects/text-editor-resources.tgz} version: 0.0.0 '@rush-temp/text-editor@file:projects/text-editor.tgz': @@ -20586,6 +20586,7 @@ snapshots: '@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.3.3) '@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.3.3) cors: 2.8.5 + cross-env: 7.0.3 dotenv: 16.0.3 esbuild: 0.24.2 eslint: 8.56.0 diff --git a/packages/core/src/classes.ts b/packages/core/src/classes.ts index 766994966d..70693b1f3c 100644 --- a/packages/core/src/classes.ts +++ b/packages/core/src/classes.ts @@ -791,3 +791,13 @@ export interface BaseWorkspaceInfo { backupInfo?: BackupStatus } + +/** + * @public + */ +export type ClientWorkspaceInfo = Omit & { + lastProcessingTime?: number + attempts?: number + message?: string + workspaceId: string +} diff --git a/pods/server/package.json b/pods/server/package.json index ca61179794..7fa99af653 100644 --- a/pods/server/package.json +++ b/pods/server/package.json @@ -70,6 +70,7 @@ "@hcengineering/analytics-service": "^0.6.0", "@hcengineering/contact": "^0.6.24", "@hcengineering/notification": "^0.6.23", + "@hcengineering/server-calendar": "^0.6.0", "@hcengineering/server-notification": "^0.6.1", "@hcengineering/server-telegram": "^0.6.0", "@hcengineering/pod-telegram-bot": "^0.6.0", diff --git a/pods/server/src/__start.ts b/pods/server/src/__start.ts index 2d049cf40a..cc026823a4 100644 --- a/pods/server/src/__start.ts +++ b/pods/server/src/__start.ts @@ -10,6 +10,7 @@ import { MeasureMetricsContext, newMetrics, setOperationLogProfiling } from '@hc import { setMetadata } from '@hcengineering/platform' import { serverConfigFromEnv } from '@hcengineering/server' import serverAiBot from '@hcengineering/server-ai-bot' +import serverCalendar from '@hcengineering/server-calendar' import serverCore, { type ConnectionSocket, type Session, @@ -75,6 +76,7 @@ setMetadata(serverNotification.metadata.SesAuthToken, config.sesAuthToken) setMetadata(serverTelegram.metadata.BotUrl, process.env.TELEGRAM_BOT_URL) setMetadata(serverAiBot.metadata.SupportWorkspaceId, process.env.SUPPORT_WORKSPACE) setMetadata(serverAiBot.metadata.EndpointURL, process.env.AI_BOT_URL) +setMetadata(serverCalendar.metadata.EndpointURL, process.env.CALENDAR_URL) const { shutdown, sessionManager } = start(metricsContext, config.dbUrl, { fulltextUrl: config.fulltextUrl, diff --git a/server-plugins/calendar-resources/package.json b/server-plugins/calendar-resources/package.json index dc1718ba3c..c62140575a 100644 --- a/server-plugins/calendar-resources/package.json +++ b/server-plugins/calendar-resources/package.json @@ -39,6 +39,8 @@ "dependencies": { "@hcengineering/core": "^0.6.32", "@hcengineering/platform": "^0.6.11", + "@hcengineering/server-calendar": "^0.6.0", + "@hcengineering/server-token": "^0.6.11", "@hcengineering/calendar": "^0.6.24", "@hcengineering/contact": "^0.6.24", "@hcengineering/server-core": "^0.6.1", diff --git a/server-plugins/calendar-resources/src/index.ts b/server-plugins/calendar-resources/src/index.ts index edfc41c11c..0afb07f16f 100644 --- a/server-plugins/calendar-resources/src/index.ts +++ b/server-plugins/calendar-resources/src/index.ts @@ -17,6 +17,7 @@ import calendar, { Calendar, Event, ExternalCalendar } from '@hcengineering/cale import contactPlugin, { Contact, Person, PersonAccount } from '@hcengineering/contact' import core, { Class, + concatLink, Data, Doc, DocumentQuery, @@ -24,6 +25,7 @@ import core, { FindResult, Hierarchy, Ref, + systemAccountEmail, Tx, TxCreateDoc, TxCUD, @@ -31,9 +33,11 @@ import core, { TxRemoveDoc, TxUpdateDoc } from '@hcengineering/core' -import { getResource } from '@hcengineering/platform' +import serverCalendar from '@hcengineering/server-calendar' +import { getMetadata, getResource } from '@hcengineering/platform' import { TriggerControl } from '@hcengineering/server-core' import { getHTMLPresenter, getTextPresenter } from '@hcengineering/server-notification-resources' +import { generateToken } from '@hcengineering/server-token' /** * @public @@ -145,6 +149,9 @@ async function onEventUpdate (ctx: TxUpdateDoc, control: TriggerControl): if (Object.keys(otherOps).length === 0) return [] const event = (await control.findAll(control.ctx, calendar.class.Event, { _id: ctx.objectId }, { limit: 1 }))[0] if (event === undefined) return [] + if (ctx.modifiedBy !== core.account.System) { + void sendEventToService(event, 'update', control) + } if (event.access !== 'owner') return [] const events = await control.findAll(control.ctx, calendar.class.Event, { eventId: event.eventId }) const res: Tx[] = [] @@ -222,8 +229,43 @@ async function eventForNewParticipants ( return res } +async function sendEventToService ( + event: Event, + type: 'create' | 'update' | 'delete', + control: TriggerControl +): Promise { + const url = getMetadata(serverCalendar.metadata.EndpointURL) ?? '' + + if (url === '') { + return + } + + const workspace = control.workspace.name + + try { + await fetch(concatLink(url, '/event'), { + method: 'POST', + keepalive: true, + headers: { + Authorization: 'Bearer ' + generateToken(systemAccountEmail, control.workspace), + 'Content-Type': 'application/json' + }, + body: JSON.stringify({ + event, + workspace, + type + }) + }) + } catch (err) { + control.ctx.error('Could not send calendar event to service', { err }) + } +} + async function onEventCreate (ctx: TxCreateDoc, control: TriggerControl): Promise { const event = TxProcessor.createDoc2Doc(ctx) + if (ctx.modifiedBy !== core.account.System) { + void sendEventToService(event, 'create', control) + } if (event.access !== 'owner') return [] const res: Tx[] = [] const { _class, space, attachedTo, attachedToClass, collection, ...attr } = event @@ -265,6 +307,9 @@ async function onRemoveEvent (ctx: TxRemoveDoc, control: TriggerControl): const removed = control.removedMap.get(ctx.objectId) as Event const res: Tx[] = [] if (removed !== undefined) { + if (ctx.modifiedBy !== core.account.System) { + void sendEventToService(removed, 'delete', control) + } if (removed.access !== 'owner') return [] const current = await control.findAll(control.ctx, calendar.class.Event, { eventId: removed.eventId }) for (const cur of current) { diff --git a/server-plugins/calendar/src/index.ts b/server-plugins/calendar/src/index.ts index 804ec16bea..a7298e1b49 100644 --- a/server-plugins/calendar/src/index.ts +++ b/server-plugins/calendar/src/index.ts @@ -14,7 +14,7 @@ // limitations under the License. // -import type { Plugin, Resource } from '@hcengineering/platform' +import type { Metadata, Plugin, Resource } from '@hcengineering/platform' import { plugin } from '@hcengineering/platform' import type { ObjectDDParticipantFunc, TriggerFunc } from '@hcengineering/server-core' import { Presenter } from '@hcengineering/server-notification' @@ -28,6 +28,9 @@ export const serverCalendarId = 'server-calendar' as Plugin * @public */ export default plugin(serverCalendarId, { + metadata: { + EndpointURL: '' as Metadata + }, function: { ReminderHTMLPresenter: '' as Resource, ReminderTextPresenter: '' as Resource, diff --git a/server/account/src/operations.ts b/server/account/src/operations.ts index c265fac33f..772c009dd3 100644 --- a/server/account/src/operations.ts +++ b/server/account/src/operations.ts @@ -26,6 +26,7 @@ import contact, { import core, { AccountRole, Client, + ClientWorkspaceInfo, concatLink, Data, generateId, @@ -68,7 +69,6 @@ import type { Account, AccountDB, AccountInfo, - ClientWorkspaceInfo, Invite, LoginInfo, ObjectId, @@ -1822,6 +1822,34 @@ export async function getWorkspaceInfo ( return clientWs } +/** + * @public + */ +export async function getWorkspacesInfo ( + ctx: MeasureContext, + db: AccountDB, + branding: Branding | null, + token: string, + ids: string[] +): Promise { + const { email } = decodeToken(ctx, token) + + if (email !== systemAccountEmail) { + ctx.error('getWorkspaceInfos with wrong email', { email, token }) + throw new PlatformError(new Status(Severity.ERROR, platform.status.Forbidden, {})) + } + const query: Query = { + workspace: { $in: ids } + } + const workspaces = await ctx.with( + 'get-workspace', + {}, + async () => await db.workspace.find(query, { lastVisit: 'descending' }) + ) + + return workspaces.map(mapToClientWorkspace) +} + async function getUpgradeStatistics (db: AccountDB, region: string): Promise { return ( (await db.upgrade.findOne({ @@ -2937,6 +2965,7 @@ export function getMethods (hasSignUp: boolean = true): Record & { workspaceId: string } - /** * @public */ diff --git a/server/client/src/account.ts b/server/client/src/account.ts index b05d86f0c4..f866a2fc6f 100644 --- a/server/client/src/account.ts +++ b/server/client/src/account.ts @@ -16,6 +16,7 @@ import { AccountRole, BackupStatus, + ClientWorkspaceInfo, Doc, Ref, type BaseWorkspaceInfo, @@ -262,6 +263,25 @@ export async function getWorkspaceInfo ( return workspaceInfo.result as BaseWorkspaceInfo | undefined } +export async function getWorkspacesInfo (token: string, workspaces: string[]): Promise { + const accountsUrl = getAccoutsUrlOrFail() + const workspaceInfo = await ( + await fetch(accountsUrl, { + method: 'POST', + headers: { + Authorization: 'Bearer ' + token, + 'Content-Type': 'application/json' + }, + body: JSON.stringify({ + method: 'getWorkspacesInfo', + params: [workspaces] + }) + }) + ).json() + + return workspaceInfo.result as ClientWorkspaceInfo[] +} + export async function login (user: string, password: string, workspace: string): Promise { const accountsUrl = getAccoutsUrlOrFail() const response = await fetch(accountsUrl, { diff --git a/server/collaborator/src/account.ts b/server/collaborator/src/account.ts index 40c456b070..297eb7acb0 100644 --- a/server/collaborator/src/account.ts +++ b/server/collaborator/src/account.ts @@ -13,7 +13,7 @@ // limitations under the License. // -import { ClientWorkspaceInfo } from '@hcengineering/account' +import { ClientWorkspaceInfo } from '@hcengineering/core' import config from './config' export async function getWorkspaceInfo (token: string): Promise { diff --git a/services/calendar/pod-calendar/src/calendar.ts b/services/calendar/pod-calendar/src/calendar.ts index 9e5627cd80..3b0af5d449 100644 --- a/services/calendar/pod-calendar/src/calendar.ts +++ b/services/calendar/pod-calendar/src/calendar.ts @@ -23,7 +23,6 @@ import calendar, { } from '@hcengineering/calendar' import { Contact } from '@hcengineering/contact' import core, { - Account, AttachedData, Client, Data, @@ -32,134 +31,114 @@ import core, { DocumentUpdate, Mixin, Ref, - TxOperations, - TxUpdateDoc, - generateId + TxOperations } from '@hcengineering/core' import setting from '@hcengineering/setting' import { htmlToMarkup, markupToHTML } from '@hcengineering/text' import { deepEqual } from 'fast-equals' -import type { Credentials, OAuth2Client } from 'google-auth-library' -import { calendar_v3, google } from 'googleapis' import type { Collection, Db } from 'mongodb' -import { encode64 } from './base64' import { CalendarController } from './calendarController' -import config from './config' -import { RateLimiter } from './rateLimiter' -import type { CalendarHistory, EventHistory, EventWatch, ProjectCredentials, State, Token, User, Watch } from './types' +import type { CalendarHistory, DummyWatch, EventHistory, Token, User } from './types' import { encodeReccuring, isToken, parseRecurrenceStrings } from './utils' import type { WorkspaceClient } from './workspaceClient' - -const SCOPES = [ - 'https://www.googleapis.com/auth/calendar.calendars.readonly', - 'https://www.googleapis.com/auth/calendar.calendarlist.readonly', - 'https://www.googleapis.com/auth/calendar.events', - 'https://www.googleapis.com/auth/userinfo.email' -] -const DUMMY_RESOURCE = 'Dummy' +import { GoogleClient } from './googleClient' +import { calendar_v3 } from 'googleapis' +import { WatchController } from './watch' export class CalendarClient { - private readonly oAuth2Client: OAuth2Client private readonly calendar: calendar_v3.Calendar - private readonly tokens: Collection private readonly calendarHistories: Collection private readonly histories: Collection private readonly client: TxOperations - private me: string | undefined = undefined - private readonly watches: EventWatch[] = [] - private calendarWatch: Watch | undefined = undefined - private refreshTimer: NodeJS.Timeout | undefined = undefined + private readonly systemTxOp: TxOperations private readonly activeSync: Record = {} - private readonly rateLimiter = new RateLimiter(1000, 500) + private readonly dummyWatches: DummyWatch[] = [] + // to do< find!!!! + private readonly googleClient + + private inactiveTimer: NodeJS.Timeout isClosed: boolean = false private constructor ( - credentials: ProjectCredentials, private readonly user: User, - mongo: Db, + private readonly mongo: Db, client: Client, private readonly workspace: WorkspaceClient ) { - const { client_secret, client_id, redirect_uris } = credentials.web // eslint-disable-line - this.oAuth2Client = new google.auth.OAuth2(client_id, client_secret, redirect_uris[0]) // eslint-disable-line - this.calendar = google.calendar({ version: 'v3', auth: this.oAuth2Client }) - this.tokens = mongo.collection('tokens') + this.client = new TxOperations(client, this.user.userId) + this.systemTxOp = new TxOperations(client, core.account.System) + this.googleClient = new GoogleClient(user, mongo, this) + this.calendar = this.googleClient.calendar this.histories = mongo.collection('histories') this.calendarHistories = mongo.collection('calendarHistories') - this.client = new TxOperations(client, this.user.userId) + this.inactiveTimer = setTimeout(() => { + this.closeByTimer() + }, 60 * 1000) + } + + async cleanIntegration (): Promise { + const integration = await this.client.findOne(setting.class.Integration, { + createdBy: this.user.userId, + type: calendar.integrationType.Calendar, + value: this.user.email + }) + if (integration !== undefined) { + await this.client.update(integration, { disabled: true }) + } + this.workspace.removeClient(this.user.email) + } + + private updateTimer (): void { + clearTimeout(this.inactiveTimer) + this.inactiveTimer = setTimeout(() => { + this.closeByTimer() + }, 60 * 1000) } static async create ( - credentials: ProjectCredentials, user: User | Token, mongo: Db, client: Client, workspace: WorkspaceClient ): Promise { - const calendarClient = new CalendarClient(credentials, user, mongo, client, workspace) + const calendarClient = new CalendarClient(user, mongo, client, workspace) if (isToken(user)) { - await calendarClient.setToken(user) - await calendarClient.refreshToken() + await calendarClient.googleClient.init(user) await calendarClient.addClient() } return calendarClient } - static getAutUrl (redirectURL: string, workspace: string, userId: Ref, token: string): string { - const credentials = JSON.parse(config.Credentials) - const { client_secret, client_id, redirect_uris } = credentials.web // eslint-disable-line - const oAuth2Client = new google.auth.OAuth2(client_id, client_secret, redirect_uris[0]) // eslint-disable-line - const state: State = { - token, - redirectURL, - workspace, - userId, - email: '' - } - const authUrl = oAuth2Client.generateAuthUrl({ - access_type: 'offline', - scope: SCOPES, - state: encode64(JSON.stringify(state)) - }) - return authUrl - } - async authorize (code: string): Promise { - const token = await this.oAuth2Client.getToken(code) - await this.setToken(token.tokens) - const me = await this.getMe() - const providedScopes = token.tokens.scope?.split(' ') ?? [] - for (const scope of SCOPES) { - if (providedScopes.findIndex((p) => p === scope) === -1) { - const integrations = await this.client.findAll(setting.class.Integration, { - createdBy: this.user.userId, - type: calendar.integrationType.Calendar - }) - for (const integration of integrations.filter((p) => p.value === '')) { - await this.client.remove(integration) - } - - const updated = integrations.find((p) => p.disabled && p.value === me) - if (updated !== undefined) { - await this.client.update(updated, { - disabled: true, - error: calendar.string.NotAllPermissions - }) - } else { - await this.client.createDoc(setting.class.Integration, core.space.Workspace, { - type: calendar.integrationType.Calendar, - disabled: true, - error: calendar.string.NotAllPermissions, - value: me - }) - } - throw new Error( - `Not all scopes provided, provided: ${providedScopes.join(', ')} required: ${SCOPES.join(', ')}` - ) + this.updateTimer() + const me = await this.googleClient.authorize(code) + if (me === undefined) { + const integrations = await this.client.findAll(setting.class.Integration, { + createdBy: this.user.userId, + type: calendar.integrationType.Calendar + }) + for (const integration of integrations.filter((p) => p.value === '')) { + await this.client.remove(integration) } + + const updated = integrations.find((p) => p.disabled && p.value === me) + if (updated !== undefined) { + await this.client.update(updated, { + disabled: true, + error: calendar.string.NotAllPermissions + }) + } else { + const value = await this.googleClient.getMe() + await this.client.createDoc(setting.class.Integration, core.space.Workspace, { + type: calendar.integrationType.Calendar, + disabled: true, + error: calendar.string.NotAllPermissions, + value + }) + } + throw new Error('Not all scopes provided') } - await this.refreshToken() await this.addClient() const integrations = await this.client.findAll(setting.class.Integration, { @@ -185,32 +164,31 @@ export class CalendarClient { }) } - await this.startSync(me) - void this.syncOurEvents() + void this.syncOurEvents().then(async () => { + await this.startSync(me) + }) return me } - async signout (byError: boolean = false): Promise { + async signout (): Promise { + this.updateTimer() try { - await this.close() - await this.oAuth2Client.revokeCredentials() + this.close() + if (isToken(this.user)) { + const watch = WatchController.get(this.mongo) + await watch.unsubscribe(this.user) + } + await this.googleClient.signout() } catch {} - await this.tokens.deleteOne({ - userId: this.user.userId, - workspace: this.user.workspace - }) + const integration = await this.client.findOne(setting.class.Integration, { createdBy: this.user.userId, type: calendar.integrationType.Calendar, value: this.user.email }) if (integration !== undefined) { - if (byError) { - await this.client.update(integration, { disabled: true }) - } else { - await this.client.remove(integration) - } + await this.client.remove(integration) } this.workspace.removeClient(this.user.email) } @@ -218,194 +196,62 @@ export class CalendarClient { async startSync (me?: string): Promise { try { if (me === undefined) { - me = await this.getMe() + me = await this.googleClient.getMe() } await this.syncCalendars(me) const calendars = this.workspace.getMyCalendars(me) for (const calendar of calendars) { if (calendar.externalId !== undefined) { - void this.sync(calendar.externalId, me) + await this.sync(calendar.externalId, me) } } } catch (err) { - console.log('Start sync error', this.user.workspace, this.user.userId, err) + console.error('Start sync error', this.user.workspace, this.user.userId, err) } } async startSyncCalendar (calendar: ExternalCalendar): Promise { - const me = await this.getMe() + const me = await this.googleClient.getMe() void this.sync(calendar.externalId, me) } - async close (): Promise { - if (this.refreshTimer !== undefined) clearTimeout(this.refreshTimer) - for (const watch of this.watches) { + private closeByTimer (): void { + this.close() + this.workspace.removeClient(this.user.email) + } + + close (): void { + this.googleClient.close() + for (const watch of this.dummyWatches) { clearTimeout(watch.timer) - try { - if (watch.resourceId !== DUMMY_RESOURCE) { - await this.rateLimiter.take(1) - await this.calendar.channels.stop({ requestBody: { id: watch.channelId, resourceId: watch.resourceId } }) - } - } catch (err) { - console.log('close error', err) - } - } - if (this.calendarWatch !== undefined) { - clearTimeout(this.calendarWatch.timer) - try { - await this.rateLimiter.take(1) - await this.calendar.channels.stop({ - requestBody: { id: this.calendarWatch.channelId, resourceId: this.calendarWatch.resourceId } - }) - } catch (err) { - console.log('close error', err) - } } this.isClosed = true } - private async getMe (): Promise { - if (this.me !== undefined) { - return this.me - } - - const info = await google.oauth2({ version: 'v2', auth: this.oAuth2Client }).userinfo.get() - this.me = info.data.email ?? '' - return this.me - } - - // #region Token - - private async getCurrentToken (): Promise { - return await this.tokens.findOne({ - userId: this.user.userId, - workspace: this.user.workspace, - email: this.me ?? this.user.email - }) - } - - private async updateCurrentToken (token: Credentials): Promise { - await this.tokens.updateOne( - { - userId: this.user.userId, - workspace: this.user.workspace, - email: this.me ?? this.user.email - }, - { - $set: { - ...token - } - } - ) - } - private async addClient (): Promise { try { - const me = await this.getMe() + const me = await this.googleClient.getMe() const controller = CalendarController.getCalendarController() controller.addClient(me, this) + this.updateTimer() } catch (err) { - console.log('Add client error', this.user.workspace, this.user.userId, err) + console.error('Add client error', this.user.workspace, this.user.userId, err) } } - private async setToken (token: Credentials): Promise { - try { - this.oAuth2Client.setCredentials(token) - } catch (err: any) { - console.log('Set token error', this.user.workspace, this.user.userId, err) - await this.checkError(err) - throw err - } - } - - private async updateToken (token: Credentials): Promise { - try { - const currentToken = await this.getCurrentToken() - if (currentToken != null) { - await this.updateCurrentToken(token) - } else { - await this.tokens.insertOne({ - userId: this.user.userId, - email: this.me ?? this.user.email, - workspace: this.user.workspace, - token: this.user.token, - ...token - }) - } - } catch (err) { - console.log('update token error', this.user.workspace, this.user.userId, err) - } - } - - private async refreshToken (): Promise { - try { - const res = await this.oAuth2Client.refreshAccessToken() - await this.updateToken(res.credentials) - this.refreshTimer = setTimeout( - () => { - void this.refreshToken() - }, - 30 * 60 * 1000 - ) - } catch (err: any) { - console.log("Couldn't refresh token, error:", err) - if (err?.response?.data?.error === 'invalid_grant' || err.message === 'No refresh token is set.') { - await this.signout(true) - } else { - this.refreshTimer = setTimeout( - () => { - void this.refreshToken() - }, - 15 * 60 * 1000 - ) - } - throw err - } - } - - // #endregion - // #region Calendars - private async watchCalendar (): Promise { - try { - const current = this.calendarWatch - if (current !== undefined) { - clearTimeout(current.timer) - await this.rateLimiter.take(1) - await this.calendar.channels.stop({ requestBody: { id: current.channelId, resourceId: current.resourceId } }) - } - const channelId = generateId() - const me = await this.getMe() - const body = { id: channelId, address: config.WATCH_URL, type: 'webhook', token: `user=${me}&mode=calendar` } - await this.rateLimiter.take(1) - const res = await this.calendar.calendarList.watch({ requestBody: body }) - if (res.data.expiration != null && res.data.resourceId !== null) { - const time = Number(res.data.expiration) - new Date().getTime() - // eslint-disable-next-line - const timer = setTimeout(() => void this.watchCalendar(), time) - this.calendarWatch = { - channelId, - resourceId: res.data.resourceId ?? '', - timer - } - } - } catch (err) { - console.log('Calendar watch error', err) - } - } - async syncCalendars (me: string): Promise { const history = await this.getCalendarHistory(me) await this.calendarSync(history?.historyId) - await this.watchCalendar() + await this.googleClient.watchCalendar() } private async calendarSync (syncToken?: string, pageToken?: string): Promise { try { - await this.rateLimiter.take(1) - const res = await this.calendar.calendarList.list({ + this.updateTimer() + await this.googleClient.rateLimiter.take(1) + const res = await this.googleClient.calendar.calendarList.list({ syncToken, pageToken }) @@ -418,7 +264,7 @@ export class CalendarClient { try { await this.syncCalendar(calendar) } catch (err) { - console.log('save calendar error', JSON.stringify(event), err) + console.error('save calendar error', JSON.stringify(calendar), err) } } if (nextPageToken != null) { @@ -432,15 +278,16 @@ export class CalendarClient { await this.calendarSync() return } - console.log('Calendar sync error', this.user.workspace, this.user.userId, err) + console.error('Calendar sync error', this.user.workspace, this.user.userId, err) } } private async syncCalendar (val: calendar_v3.Schema$CalendarListEntry): Promise { if (val.id != null) { + const me = await this.googleClient.getMe() const exists = await this.client.findOne(calendar.class.ExternalCalendar, { externalId: val.id, - externalUser: this.me ?? '' + externalUser: me }) if (exists === undefined) { const data: Data = { @@ -448,7 +295,7 @@ export class CalendarClient { visibility: 'freeBusy', hidden: false, externalId: val.id, - externalUser: this.me ?? '', + externalUser: me, default: false } if (val.primary === true) { @@ -485,7 +332,7 @@ export class CalendarClient { } private async setCalendarHistoryId (historyId: string): Promise { - const me = await this.getMe() + const me = await this.googleClient.getMe() await this.calendarHistories.updateOne( { userId: this.user.userId, @@ -506,90 +353,22 @@ export class CalendarClient { // #region Events // #region Incoming - - async stopWatch (calendar: ExternalCalendar): Promise { - for (const watch of this.watches) { - if (watch.calendarId === calendar.externalId) { - clearTimeout(watch.timer) - try { - if (watch.resourceId !== DUMMY_RESOURCE) { - await this.rateLimiter.take(1) - await this.calendar.channels.stop({ requestBody: { id: watch.channelId, resourceId: watch.resourceId } }) - } - } catch (err) { - console.log('close error', err) - } - } - } - } - private async watch (calendarId: string): Promise { - try { - const index = this.watches.findIndex((p) => p.calendarId === calendarId) - if (index !== -1) { - const current = this.watches[index] - if (current !== undefined) { - clearTimeout(current.timer) - if (current.resourceId !== DUMMY_RESOURCE) { - await this.rateLimiter.take(1) - await this.calendar.channels.stop({ - requestBody: { id: current.channelId, resourceId: current.resourceId } - }) - } - } - this.watches.splice(index, 1) - } - const channelId = generateId() - const me = await this.getMe() - const body = { - id: channelId, - address: config.WATCH_URL, - type: 'webhook', - token: `user=${me}&mode=events&calendarId=${calendarId}` - } - await this.rateLimiter.take(1) - const res = await this.calendar.events.watch({ calendarId, requestBody: body }) - if (res.data.expiration != null && res.data.resourceId != null) { - const time = Number(res.data.expiration) - new Date().getTime() - // eslint-disable-next-line - const timer = setTimeout(() => void this.watch(calendarId), time) - this.watches.push({ - calendarId, - channelId, - resourceId: res.data.resourceId ?? '', - timer - }) - } - } catch (err: any) { - if (err?.errors?.[0]?.reason === 'pushNotSupportedForRequestedResource') { - await this.dummyWatch(calendarId) - } else { - console.log('Watch error', err) - await this.checkError(err) - } + if (!(await this.googleClient.watch(calendarId))) { + await this.dummyWatch(calendarId) } } - private async checkError (err: any): Promise { - if (err?.response?.data?.error === 'invalid_grant') { - await this.signout(true) - return true - } - return false - } - private async dummyWatch (calendarId: string): Promise { - const me = await this.getMe() + const me = await this.googleClient.getMe() const timer = setTimeout( () => { void this.sync(calendarId, me) }, 6 * 60 * 60 * 1000 ) - this.watches.push({ + this.dummyWatches.push({ calendarId, - channelId: DUMMY_RESOURCE, - resourceId: DUMMY_RESOURCE, timer }) } @@ -613,7 +392,7 @@ export class CalendarClient { } private async setEventHistoryId (calendarId: string, historyId: string): Promise { - const me = await this.getMe() + const me = await this.googleClient.getMe() await this.histories.updateOne( { calendarId, @@ -637,7 +416,7 @@ export class CalendarClient { private async eventsSync (calendarId: string, syncToken?: string, pageToken?: string): Promise { try { - await this.rateLimiter.take(1) + await this.googleClient.rateLimiter.take(1) const res = await this.calendar.events.list({ calendarId, syncToken, @@ -653,7 +432,7 @@ export class CalendarClient { try { await this.syncEvent(calendarId, event, res.data.accessRole ?? 'reader') } catch (err) { - console.log('save event error', JSON.stringify(event), err) + console.error('save event error', JSON.stringify(event), err) } } if (nextPageToken != null) { @@ -667,14 +446,15 @@ export class CalendarClient { await this.eventsSync(calendarId) return } - await this.checkError(err) - console.log('Event sync error', this.user.workspace, this.user.userId, err) + await this.googleClient.checkError(err) + console.error('Event sync error', this.user.workspace, this.user.userId, err) } } private async syncEvent (calendarId: string, event: calendar_v3.Schema$Event, accessRole: string): Promise { + this.updateTimer() if (event.id != null) { - const me = await this.getMe() + const me = await this.googleClient.getMe() const calendars = this.workspace.getMyCalendars(me) const _calendar = calendars.find((p) => p.externalId === event.organizer?.email) ?? @@ -695,6 +475,7 @@ export class CalendarClient { } private async updateExtEvent (event: calendar_v3.Schema$Event, current: Event): Promise { + this.updateTimer() if (event.status === 'cancelled' && current._class !== calendar.class.ReccuringInstance) { await this.client.remove(current) return @@ -747,7 +528,7 @@ export class CalendarClient { if (this.client.getHierarchy().hasMixin(current, mixin as Ref>)) { const diff = this.getDiff(attr, this.client.getHierarchy().as(current, mixin as Ref>)) if (Object.keys(diff).length > 0) { - await this.client.updateMixin( + await this.systemTxOp.updateMixin( current._id, current._class, calendar.space.Calendar, @@ -756,7 +537,7 @@ export class CalendarClient { ) } } else { - await this.client.createMixin( + await this.systemTxOp.createMixin( current._id, current._class, calendar.space.Calendar, @@ -782,7 +563,7 @@ export class CalendarClient { for (const mixin in mixins) { const attr = mixins[mixin] if (typeof attr === 'object' && Object.keys(attr).length > 0) { - await this.client.createMixin( + await this.systemTxOp.createMixin( _id, calendar.class.Event, calendar.space.Calendar, @@ -799,10 +580,11 @@ export class CalendarClient { accessRole: string, _calendar: ExternalCalendar ): Promise { + this.updateTimer() const data: AttachedData = await this.parseData(event, accessRole, _calendar._id) if (event.recurringEventId != null) { const parseRule = parseRecurrenceStrings(event.recurrence ?? []) - const id = await this.client.addCollection( + const id = await this.systemTxOp.addCollection( calendar.class.ReccuringInstance, calendar.space.Calendar, calendar.ids.NoAttached, @@ -823,7 +605,7 @@ export class CalendarClient { } else if (event.status !== 'cancelled') { if (event.recurrence != null) { const parseRule = parseRecurrenceStrings(event.recurrence) - const id = await this.client.addCollection( + const id = await this.systemTxOp.addCollection( calendar.class.ReccuringEvent, calendar.space.Calendar, calendar.ids.NoAttached, @@ -840,7 +622,7 @@ export class CalendarClient { ) await this.saveMixins(event, id) } else { - const id = await this.client.addCollection( + const id = await this.systemTxOp.addCollection( calendar.class.Event, calendar.space.Calendar, calendar.ids.NoAttached, @@ -1004,12 +786,14 @@ export class CalendarClient { } private async createRecInstance (calendarId: string, event: ReccuringInstance): Promise { - const body = this.convertBody(event) + this.updateTimer() + const me = await this.googleClient.getMe() + const body = this.convertBody(event, me) const req: calendar_v3.Params$Resource$Events$Instances = { calendarId, eventId: event.recurringEventId } - await this.rateLimiter.take(1) + await this.googleClient.rateLimiter.take(1) const instancesResp = await this.calendar.events.instances(req) const items = instancesResp.data.items const target = items?.find( @@ -1020,7 +804,7 @@ export class CalendarClient { ) if (target?.id != null) { body.id = target.id - await this.rateLimiter.take(1) + await this.googleClient.rateLimiter.take(1) await this.calendar.events.update({ calendarId, eventId: target.id, @@ -1031,14 +815,15 @@ export class CalendarClient { } async createEvent (event: Event): Promise { + const me = await this.googleClient.getMe() try { const _calendar = this.workspace.calendars.byId.get(event.calendar as Ref) if (_calendar !== undefined) { if (event._class === calendar.class.ReccuringInstance) { await this.createRecInstance(_calendar.externalId, event as ReccuringInstance) } else { - const body = this.convertBody(event) - await this.rateLimiter.take(1) + const body = this.convertBody(event, me) + await this.googleClient.rateLimiter.take(1) await this.calendar.events.insert({ calendarId: _calendar.externalId, requestBody: body @@ -1046,23 +831,24 @@ export class CalendarClient { } } } catch (err: any) { - await this.checkError(err) + await this.googleClient.checkError(err) // eslint-disable-next-line throw new Error(`Create event error, ${this.user.workspace}, ${this.user.userId}, ${event._id}, ${err?.message}`) } } - async updateEvent (event: Event, tx: TxUpdateDoc): Promise { + async updateEvent (event: Event): Promise { + const me = await this.googleClient.getMe() const _calendar = this.workspace.calendars.byId.get(event.calendar as Ref) const calendarId = _calendar?.externalId if (calendarId !== undefined) { try { - await this.rateLimiter.take(1) + await this.googleClient.rateLimiter.take(1) const current = await this.calendar.events.get({ calendarId, eventId: event.eventId }) if (current?.data !== undefined) { if (current.data.organizer?.self === true) { - const ev = this.applyUpdate(current.data, event) - await this.rateLimiter.take(1) + const ev = this.applyUpdate(current.data, event, me) + await this.googleClient.rateLimiter.take(1) await this.calendar.events.update({ calendarId, eventId: event.eventId, @@ -1074,18 +860,19 @@ export class CalendarClient { if (err.code === 404) { await this.createEvent(event) } else { - console.log('Update event error', this.user.workspace, this.user.userId, err) - await this.checkError(err) + console.error('Update event error', this.user.workspace, this.user.userId, err) + await this.googleClient.checkError(err) } } } } async remove (eventId: string, calendarId: string): Promise { + this.updateTimer() const current = await this.calendar.events.get({ calendarId, eventId }) if (current?.data !== undefined) { if (current.data.organizer?.self === true) { - await this.rateLimiter.take(1) + await this.googleClient.rateLimiter.take(1) await this.calendar.events.delete({ eventId, calendarId @@ -1101,12 +888,13 @@ export class CalendarClient { await this.remove(event.eventId, _calendar.externalId) } } catch (err) { - console.log('Remove event error', this.user.workspace, this.user.userId, err) + console.error('Remove event error', this.user.workspace, this.user.userId, err) } } async syncOurEvents (): Promise { - const me = await this.getMe() + this.updateTimer() + const me = await this.googleClient.getMe() const events = await this.client.findAll(calendar.class.Event, { access: 'owner', createdBy: this.user.userId, @@ -1119,25 +907,29 @@ export class CalendarClient { } async syncMyEvent (event: Event): Promise { + const me = await this.googleClient.getMe() if (event.access === 'owner' || event.access === 'writer') { try { const space = this.workspace.calendars.byId.get(event.calendar as Ref) - if (space !== undefined && space.externalUser === this.me) { + if (space !== undefined && space.externalUser === me) { + this.updateTimer() if (!(await this.update(event, space))) { await this.create(event, space) } } } catch (err: any) { - console.log('Sync event error', this.user.workspace, this.user.userId, event._id, err.message) + console.error('Sync event error', this.user.workspace, this.user.userId, event._id, err.message) } } } private async create (event: Event, space: ExternalCalendar): Promise { - const body = this.convertBody(event) + this.updateTimer() + const me = await this.googleClient.getMe() + const body = this.convertBody(event, me) const calendarId = space?.externalId if (calendarId !== undefined) { - await this.rateLimiter.take(1) + await this.googleClient.rateLimiter.take(1) await this.calendar.events.insert({ calendarId, requestBody: body @@ -1146,14 +938,16 @@ export class CalendarClient { } private async update (event: Event, space: ExternalCalendar): Promise { + this.updateTimer() + const me = await this.googleClient.getMe() const calendarId = space?.externalId if (calendarId !== undefined) { try { - await this.rateLimiter.take(1) + await this.googleClient.rateLimiter.take(1) const current = await this.calendar.events.get({ calendarId, eventId: event.eventId }) if (current !== undefined) { - const ev = this.applyUpdate(current.data, event) - await this.rateLimiter.take(1) + const ev = this.applyUpdate(current.data, event, me) + await this.googleClient.rateLimiter.take(1) await this.calendar.events.update({ calendarId, eventId: event.eventId, @@ -1190,7 +984,7 @@ export class CalendarClient { return res } - private convertBody (event: Event): calendar_v3.Schema$Event { + private convertBody (event: Event, me: string): calendar_v3.Schema$Event { const res: calendar_v3.Schema$Event = { start: convertDate(event.date, event.allDay, getTimezone(event)), end: convertDate(event.dueDate, event.allDay, getTimezone(event)), @@ -1229,10 +1023,10 @@ export class CalendarClient { }) } } - const attendees = this.getAttendees(event) + const attendees = this.getAttendees(event, me) if (attendees.length > 0) { res.attendees = attendees.map((p) => { - if (p === this.me) { + if (p === me) { return { email: p, responseStatus: 'accepted', self: true } } return { email: p } @@ -1251,7 +1045,7 @@ export class CalendarClient { return res } - private applyUpdate (event: calendar_v3.Schema$Event, current: Event): calendar_v3.Schema$Event { + private applyUpdate (event: calendar_v3.Schema$Event, current: Event, me: string): calendar_v3.Schema$Event { if (current.title !== event.summary) { event.summary = current.title } @@ -1276,7 +1070,7 @@ export class CalendarClient { if (current.location !== event.location) { event.location = current.location } - const attendees = this.getAttendees(current) + const attendees = this.getAttendees(current, me) if (attendees.length > 0 && event.attendees !== undefined) { for (const attendee of attendees) { if (event.attendees.findIndex((p) => p.email === attendee) === -1) { @@ -1293,11 +1087,11 @@ export class CalendarClient { return event } - private getAttendees (event: Event): string[] { + private getAttendees (event: Event, me: string): string[] { const res = new Set() for (const participant of event.participants) { const integrations = this.workspace.integrations.byContact.get(participant) ?? [] - const integration = integrations.find((p) => p === this.me) ?? integrations[0] + const integration = integrations.find((p) => p === me) ?? integrations[0] if (integration !== undefined && integration !== '') { res.add(integration) } else { diff --git a/services/calendar/pod-calendar/src/calendarController.ts b/services/calendar/pod-calendar/src/calendarController.ts index ad6783d4c9..6ef093c324 100644 --- a/services/calendar/pod-calendar/src/calendarController.ts +++ b/services/calendar/pod-calendar/src/calendarController.ts @@ -13,27 +13,35 @@ // limitations under the License. // -import { Account, isActiveMode, RateLimiter, Ref, systemAccountEmail } from '@hcengineering/core' -import { type Db } from 'mongodb' +import { Account, isActiveMode, isDeletingMode, RateLimiter, Ref, systemAccountEmail } from '@hcengineering/core' +import { Event } from '@hcengineering/calendar' +import { Collection, type Db } from 'mongodb' import { type CalendarClient } from './calendar' import config from './config' -import { type ProjectCredentials, type Token, type User } from './types' +import { type Token, type User } from './types' import { WorkspaceClient } from './workspaceClient' -import { getWorkspaceInfo } from '@hcengineering/server-client' +import { getWorkspacesInfo } from '@hcengineering/server-client' import { generateToken } from '@hcengineering/server-token' export class CalendarController { - private readonly workspaces: Map = new Map() + private readonly workspaces: Map> = new Map< + string, + WorkspaceClient | Promise + >() - private readonly credentials: ProjectCredentials + private readonly tokens: Collection private readonly clients: Map = new Map() - private readonly initLimitter = new RateLimiter(config.InitLimit) protected static _instance: CalendarController private constructor (private readonly mongo: Db) { - this.credentials = JSON.parse(config.Credentials) + this.tokens = mongo.collection('tokens') CalendarController._instance = this + setInterval(() => { + if (this.workspaces.size > 0) { + console.log('active workspaces', this.workspaces.size) + } + }, 60000) } static getCalendarController (mongo?: Db): CalendarController { @@ -45,7 +53,7 @@ export class CalendarController { } async startAll (): Promise { - const tokens = await this.mongo.collection('tokens').find().toArray() + const tokens = await this.tokens.find().toArray() const groups = new Map() console.log('start calendar service', tokens.length) for (const token of tokens) { @@ -59,69 +67,86 @@ export class CalendarController { } const limiter = new RateLimiter(config.InitLimit) - - for (const [workspace, tokens] of groups) { + const token = generateToken(systemAccountEmail, { name: '' }) + const ids = [...groups.keys()] + console.log('start workspaces', ids) + const infos = await getWorkspacesInfo(token, ids) + console.log('infos', infos) + for (const info of infos) { + const tokens = groups.get(info.workspaceId) + if (tokens === undefined) { + console.log('no tokens for workspace', info.workspaceId) + continue + } + if (isDeletingMode(info.mode)) { + if (tokens !== undefined) { + for (const token of tokens) { + await this.tokens.deleteOne({ userId: token.userId, workspace: token.workspace }) + } + } + continue + } + if (!isActiveMode(info.mode)) { + continue + } await limiter.add(async () => { - const wstok = generateToken(systemAccountEmail, { name: workspace }) - const info = await getWorkspaceInfo(wstok) - if (info === undefined) { - console.log('workspace not found', workspace) - return - } - if (!isActiveMode(info.mode)) { - console.log('workspace is not active', workspace) - return - } - const startPromise = this.startWorkspace(workspace, tokens) - const timeoutPromise = new Promise((resolve) => { - setTimeout(() => { - resolve() - }, 60000) - }) - await Promise.race([startPromise, timeoutPromise]) + console.log('start workspace', info.workspaceId) + const workspace = await this.startWorkspace(info.workspaceId, tokens) + await workspace.sync() }) } - - await limiter.waitProcessing() - console.log('Calendar service started') } - async startWorkspace (workspace: string, tokens: Token[]): Promise { + async startWorkspace (workspace: string, tokens: Token[]): Promise { const workspaceClient = await this.getWorkspaceClient(workspace) - const clients: CalendarClient[] = [] for (const token of tokens) { try { const timeout = setTimeout(() => { - console.log('init client hang', token.workspace, token.userId) + console.warn('init client hang', token.workspace, token.userId) }, 60000) - const client = await workspaceClient.createCalendarClient(token) + console.log('init client', token.workspace, token.userId) + await workspaceClient.createCalendarClient(token) clearTimeout(timeout) - clients.push(client) } catch (err) { console.error(`Couldn't create client for ${workspace} ${token.userId} ${token.email}`) } } - for (const client of clients) { - void this.initLimitter.add(async () => { - await client.startSync() - }) - } - void workspaceClient.sync() - console.log('Workspace started', workspace) + return workspaceClient } - push (email: string, mode: 'events' | 'calendar', calendarId?: string): void { - const clients = this.clients.get(email) - for (const client of clients ?? []) { + async push (email: string, mode: 'events' | 'calendar', calendarId?: string): Promise { + const tokens = await this.tokens.find({ email, access_token: { $exists: true } }).toArray() + const token = generateToken(systemAccountEmail, { name: '' }) + const workspaces = [...new Set(tokens.map((p) => p.workspace))] + const infos = await getWorkspacesInfo(token, workspaces) + for (const token of tokens) { + const info = infos.find((p) => p.workspace === token.workspace) + if (info === undefined) { + continue + } + if (isDeletingMode(info.mode)) { + await this.tokens.deleteOne({ userId: token.userId, workspace: token.workspace }) + continue + } + if (!isActiveMode(info.mode)) { + continue + } + const workspace = await this.getWorkspaceClient(token.workspace) + const calendarClient = await workspace.createCalendarClient(token) if (mode === 'calendar') { - void client.syncCalendars(email) + await calendarClient.syncCalendars(email) } if (mode === 'events' && calendarId !== undefined) { - void client.sync(calendarId, email) + await calendarClient.sync(calendarId, email) } } } + async pushEvent (workspace: string, event: Event, type: 'create' | 'update' | 'delete'): Promise { + const workspaceController = await this.getWorkspaceClient(workspace) + await workspaceController.pushEvent(event, type) + } + addClient (email: string, client: CalendarClient): void { const clients = this.clients.get(email) if (clients === undefined) { @@ -135,10 +160,12 @@ export class CalendarController { removeClient (email: string): void { const clients = this.clients.get(email) if (clients !== undefined) { - this.clients.set( - email, - clients.filter((p) => !p.isClosed) - ) + const filtered = clients.filter((p) => !p.isClosed) + if (filtered.length === 0) { + this.clients.delete(email) + } else { + this.clients.set(email, filtered) + } } } @@ -151,7 +178,7 @@ export class CalendarController { const workspaceClient = await this.getWorkspaceClient(workspace) const clients = await workspaceClient.signout(value) if (clients === 0) { - this.workspaces.delete(workspace) + this.removeWorkspace(workspace) } } @@ -160,7 +187,10 @@ export class CalendarController { } async close (): Promise { - for (const workspace of this.workspaces.values()) { + for (let workspace of this.workspaces.values()) { + if (workspace instanceof Promise) { + workspace = await workspace + } await workspace.close() } this.workspaces.clear() @@ -179,16 +209,22 @@ export class CalendarController { } private async getWorkspaceClient (workspace: string): Promise { - let res = this.workspaces.get(workspace) - if (res === undefined) { - try { - res = await WorkspaceClient.create(this.credentials, this.mongo, workspace, this) - this.workspaces.set(workspace, res) - } catch (err) { - console.error(`Couldn't create workspace worker for ${workspace}, reason: ${JSON.stringify(err)}`) - throw err + const res = this.workspaces.get(workspace) + if (res !== undefined) { + if (res instanceof Promise) { + return await res } + return res + } + try { + const client = WorkspaceClient.create(this.mongo, workspace, this) + this.workspaces.set(workspace, client) + const res = await client + this.workspaces.set(workspace, res) + return res + } catch (err) { + console.error(`Couldn't create workspace worker for ${workspace}, reason: ${JSON.stringify(err)}`) + throw err } - return res } } diff --git a/services/calendar/pod-calendar/src/googleClient.ts b/services/calendar/pod-calendar/src/googleClient.ts new file mode 100644 index 0000000000..7f4c6be317 --- /dev/null +++ b/services/calendar/pod-calendar/src/googleClient.ts @@ -0,0 +1,313 @@ +// +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. + +import type { Credentials, OAuth2Client } from 'google-auth-library' +import { calendar_v3, google } from 'googleapis' +import { ProjectCredentials, State, Token, User, Watch, WatchBase } from './types' +import config from './config' +import { encode64 } from './base64' +import { Account, generateId, Ref } from '@hcengineering/core' +import { Collection, Db } from 'mongodb' +import { RateLimiter } from './rateLimiter' +import { CalendarClient } from './calendar' + +export const DUMMY_RESOURCE = 'Dummy' + +const SCOPES = [ + 'https://www.googleapis.com/auth/calendar.calendars.readonly', + 'https://www.googleapis.com/auth/calendar.calendarlist.readonly', + 'https://www.googleapis.com/auth/calendar.events', + 'https://www.googleapis.com/auth/userinfo.email' +] + +export class GoogleClient { + private me: string | undefined = undefined + private readonly credentials: ProjectCredentials + private readonly oAuth2Client: OAuth2Client + readonly calendar: calendar_v3.Calendar + private readonly tokens: Collection + private readonly watches: Collection + + private refreshTimer: NodeJS.Timeout | undefined = undefined + + readonly rateLimiter = new RateLimiter(1000, 500) + + constructor ( + private readonly user: User, + mongo: Db, + private readonly calendarClient: CalendarClient + ) { + this.tokens = mongo.collection('tokens') + this.credentials = JSON.parse(config.Credentials) + const { client_secret, client_id, redirect_uris } = this.credentials.web // eslint-disable-line + this.oAuth2Client = new google.auth.OAuth2(client_id, client_secret, redirect_uris[0]) // eslint-disable-line + this.calendar = google.calendar({ version: 'v3', auth: this.oAuth2Client }) + this.watches = mongo.collection('watch') + } + + static getAutUrl (redirectURL: string, workspace: string, userId: Ref, token: string): string { + const credentials = JSON.parse(config.Credentials) + const { client_secret, client_id, redirect_uris } = credentials.web // eslint-disable-line + const oAuth2Client = new google.auth.OAuth2(client_id, client_secret, redirect_uris[0]) // eslint-disable-line + const state: State = { + token, + redirectURL, + workspace, + userId, + email: '' + } + const authUrl = oAuth2Client.generateAuthUrl({ + access_type: 'offline', + scope: SCOPES, + state: encode64(JSON.stringify(state)) + }) + return authUrl + } + + async signout (): Promise { + // get watch controller and unsubscibe + await this.oAuth2Client.revokeCredentials() + await this.tokens.deleteOne({ + userId: this.user.userId, + workspace: this.user.workspace + }) + } + + async init (token: Token): Promise { + await this.setToken(token) + await this.refreshToken() + } + + async authorize (code: string): Promise { + const token = await this.oAuth2Client.getToken(code) + await this.setToken(token.tokens) + const me = await this.getMe() + const providedScopes = token.tokens.scope?.split(' ') ?? [] + for (const scope of SCOPES) { + if (providedScopes.findIndex((p) => p === scope) === -1) { + console.error(`Not all scopes provided, provided: ${providedScopes.join(', ')} required: ${SCOPES.join(', ')}`) + return undefined + } + } + await this.refreshToken() + + return me + } + + close (): void { + if (this.refreshTimer !== undefined) clearTimeout(this.refreshTimer) + } + + async getMe (): Promise { + if (this.me !== undefined) { + return this.me + } + + const info = await google.oauth2({ version: 'v2', auth: this.oAuth2Client }).userinfo.get() + this.me = info.data.email ?? '' + return this.me + } + + private async setToken (token: Credentials): Promise { + try { + this.oAuth2Client.setCredentials(token) + } catch (err: any) { + console.error('Set token error', this.user.workspace, this.user.userId, err) + await this.checkError(err) + throw err + } + } + + async checkError (err: any): Promise { + if (err?.response?.data?.error === 'invalid_grant') { + await this.calendarClient.cleanIntegration() + return true + } + return false + } + + private async updateToken (token: Credentials): Promise { + try { + const currentToken = await this.getCurrentToken() + if (currentToken != null) { + await this.updateCurrentToken(token) + } else { + await this.tokens.insertOne({ + userId: this.user.userId, + email: this.me ?? this.user.email, + workspace: this.user.workspace, + token: this.user.token, + ...token + }) + } + } catch (err) { + console.error('update token error', this.user.workspace, this.user.userId, err) + } + } + + private async refreshToken (): Promise { + try { + const res = await this.oAuth2Client.refreshAccessToken() + await this.updateToken(res.credentials) + this.refreshTimer = setTimeout( + () => { + void this.refreshToken() + }, + 30 * 60 * 1000 + ) + } catch (err: any) { + console.error("Couldn't refresh token, error:", err) + if (err?.response?.data?.error === 'invalid_grant' || err.message === 'No refresh token is set.') { + await this.calendarClient.cleanIntegration() + } else { + this.refreshTimer = setTimeout( + () => { + void this.refreshToken() + }, + 15 * 60 * 1000 + ) + } + throw err + } + } + + private async getCurrentToken (): Promise { + return await this.tokens.findOne({ + userId: this.user.userId, + workspace: this.user.workspace, + email: this.me ?? this.user.email + }) + } + + private async updateCurrentToken (token: Credentials): Promise { + await this.tokens.updateOne( + { + userId: this.user.userId, + workspace: this.user.workspace, + email: this.me ?? this.user.email + }, + { + $set: { + ...token + } + } + ) + } + + async watchCalendar (): Promise { + try { + const current = await this.watches.findOne({ + userId: this.user.userId, + workspace: this.user.workspace, + calendarId: null + }) + if (current != null) { + await this.rateLimiter.take(1) + await this.calendar.channels.stop({ requestBody: { id: current.channelId, resourceId: current.resourceId } }) + } + const channelId = generateId() + const me = await this.getMe() + const body = { id: channelId, address: config.WATCH_URL, type: 'webhook', token: `user=${me}&mode=calendar` } + await this.rateLimiter.take(1) + const res = await this.calendar.calendarList.watch({ requestBody: body }) + if (res.data.expiration != null && res.data.resourceId !== null) { + if (current != null) { + await this.watches.updateOne( + { + userId: this.user.userId, + workspace: this.user.workspace, + calendarId: null + }, + { + channelId, + expired: Number.parseInt(res.data.expiration), + resourceId: res.data.resourceId ?? '' + } + ) + } else { + await this.watches.insertOne({ + calendarId: null, + channelId, + expired: Number.parseInt(res.data.expiration), + resourceId: res.data.resourceId ?? '', + userId: this.user.userId, + workspace: this.user.workspace + }) + } + } + } catch (err) { + console.error('Calendar watch error', err) + } + } + + async watch (calendarId: string): Promise { + try { + const current = await this.watches.findOne({ + userId: this.user.userId, + workspace: this.user.workspace, + calendarId + }) + if (current != null) { + await this.rateLimiter.take(1) + await this.calendar.channels.stop({ + requestBody: { id: current.channelId, resourceId: current.resourceId } + }) + } + const channelId = generateId() + const me = await this.getMe() + const body = { + id: channelId, + address: config.WATCH_URL, + type: 'webhook', + token: `user=${me}&mode=events&calendarId=${calendarId}` + } + await this.rateLimiter.take(1) + const res = await this.calendar.events.watch({ calendarId, requestBody: body }) + if (res.data.expiration != null && res.data.resourceId != null) { + if (current != null) { + await this.watches.updateOne( + { + userId: this.user.userId, + workspace: this.user.workspace, + calendarId + }, + { + channelId, + expired: Number.parseInt(res.data.expiration), + resourceId: res.data.resourceId ?? '' + } + ) + } else { + await this.watches.insertOne({ + calendarId, + channelId, + expired: Number.parseInt(res.data.expiration), + resourceId: res.data.resourceId ?? '', + userId: this.user.userId, + workspace: this.user.workspace + }) + } + } + return true + } catch (err: any) { + if (err?.errors?.[0]?.reason === 'pushNotSupportedForRequestedResource') { + return false + } else { + console.error('Watch error', err) + await this.checkError(err) + return false + } + } + } +} diff --git a/services/calendar/pod-calendar/src/main.ts b/services/calendar/pod-calendar/src/main.ts index 324b2a516b..2367745f37 100644 --- a/services/calendar/pod-calendar/src/main.ts +++ b/services/calendar/pod-calendar/src/main.ts @@ -15,7 +15,6 @@ import { type IncomingHttpHeaders } from 'http' import { decode64 } from './base64' -import { CalendarClient } from './calendar' import { CalendarController } from './calendarController' import config from './config' import { createServer, listen } from './server' @@ -24,6 +23,8 @@ import { type Endpoint, type State } from './types' import { setMetadata } from '@hcengineering/platform' import serverClient from '@hcengineering/server-client' import serverToken, { decodeToken } from '@hcengineering/server-token' +import { GoogleClient } from './googleClient' +import { WatchController } from './watch' const extractToken = (header: IncomingHttpHeaders): any => { try { @@ -41,6 +42,8 @@ export const main = async (): Promise => { const db = await getDB() const calendarController = CalendarController.getCalendarController(db) await calendarController.startAll() + const watchController = WatchController.get(db) + watchController.startCheck() const endpoints: Endpoint[] = [ { endpoint: '/signin', @@ -57,10 +60,10 @@ export const main = async (): Promise => { const { email, workspace } = decodeToken(token) const userId = await calendarController.getUserId(email, workspace.name) - const url = CalendarClient.getAutUrl(redirectURL, workspace.name, userId, token) + const url = GoogleClient.getAutUrl(redirectURL, workspace.name, userId, token) res.send(url) } catch (err) { - console.log('signin error', err) + console.error('signin error', err) res.status(500).send() } } @@ -75,7 +78,7 @@ export const main = async (): Promise => { await calendarController.newClient(state, code) res.redirect(state.redirectURL) } catch (err) { - console.log(err) + console.error(err) res.redirect(state.redirectURL) } } @@ -97,7 +100,7 @@ export const main = async (): Promise => { const { workspace } = decodeToken(token) await calendarController.signout(workspace.name, value) } catch (err) { - console.log('signout error', err) + console.error('signout error', err) } res.send() @@ -122,9 +125,23 @@ export const main = async (): Promise => { res.status(400).send({ err: "'data' is missing" }) return } - calendarController.push(data.user, data.mode as 'events' | 'calendar', data.calendarId) + void calendarController.push(data.user, data.mode as 'events' | 'calendar', data.calendarId) } + res.send() + } + }, + { + endpoint: '/event', + type: 'post', + handler: async (req, res) => { + const { event, workspace, type } = req.body + + if (event === undefined || workspace === undefined || type === undefined) { + res.status(400).send({ err: "'event' or 'workspace' or 'type' is missing" }) + return + } + void calendarController.pushEvent(workspace, event, type) res.send() } } @@ -134,6 +151,7 @@ export const main = async (): Promise => { const shutdown = (): void => { server.close(() => { + watchController.stop() void calendarController .close() .then(async () => { diff --git a/services/calendar/pod-calendar/src/types.ts b/services/calendar/pod-calendar/src/types.ts index 45b9ede9c4..3ec0e10e6a 100644 --- a/services/calendar/pod-calendar/src/types.ts +++ b/services/calendar/pod-calendar/src/types.ts @@ -18,17 +18,28 @@ import type { Account, Ref, Timestamp } from '@hcengineering/core' import type { NextFunction, Request, Response } from 'express' import type { Credentials } from 'google-auth-library' -export interface Watch { - timer: NodeJS.Timeout +export interface WatchBase { + userId: Ref + workspace: string + expired: Timestamp channelId: string resourceId: string + calendarId: string | null } -export interface EventWatch { +export interface CalendarsWatch extends WatchBase { + calendarId: null +} + +export interface EventWatch extends WatchBase { + calendarId: string +} + +export type Watch = CalendarsWatch | EventWatch + +export interface DummyWatch { timer: NodeJS.Timeout calendarId: string - channelId: string - resourceId: string } export type Token = User & Credentials diff --git a/services/calendar/pod-calendar/src/watch.ts b/services/calendar/pod-calendar/src/watch.ts new file mode 100644 index 0000000000..940343e552 --- /dev/null +++ b/services/calendar/pod-calendar/src/watch.ts @@ -0,0 +1,230 @@ +import { Collection, Db } from 'mongodb' +import { EventWatch, Token, Watch, WatchBase } from './types' +import { generateId, isActiveMode, systemAccountEmail } from '@hcengineering/core' +import { getWorkspacesInfo } from '@hcengineering/server-client' +import { generateToken } from '@hcengineering/server-token' +import config from './config' +import { Credentials, OAuth2Client } from 'google-auth-library' +import { calendar_v3, google } from 'googleapis' +import { RateLimiter } from './rateLimiter' + +export class WatchClient { + private readonly watches: Collection + private readonly oAuth2Client: OAuth2Client + private readonly calendar: calendar_v3.Calendar + private readonly user: Token + private me: string = '' + readonly rateLimiter = new RateLimiter(1000, 500) + + private constructor (mongo: Db, token: Token) { + this.user = token + this.watches = mongo.collection('watch') + const credentials = JSON.parse(config.Credentials) + const { client_secret, client_id, redirect_uris } = credentials.web // eslint-disable-line + this.oAuth2Client = new google.auth.OAuth2(client_id, client_secret, redirect_uris[0]) // eslint-disable-line + this.calendar = google.calendar({ version: 'v3', auth: this.oAuth2Client }) + } + + static async Create (mongo: Db, token: Token): Promise { + const watchClient = new WatchClient(mongo, token) + await watchClient.init(token) + return watchClient + } + + private async setToken (token: Credentials): Promise { + try { + this.oAuth2Client.setCredentials(token) + const info = await google.oauth2({ version: 'v2', auth: this.oAuth2Client }).userinfo.get() + this.me = info.data.email ?? '' + } catch (err: any) { + console.error('Set token error', this.user.workspace, this.user.userId, err) + await this.checkError(err) + throw err + } + } + + async checkError (err: any): Promise { + if (err?.response?.data?.error === 'invalid_grant') { + await this.watches.deleteMany({ userId: this.user.userId, workspace: this.user.workspace }) + } + } + + private async init (token: Token): Promise { + await this.setToken(token) + } + + async subscribe (watches: Watch[]): Promise { + for (const watch of watches) { + if (watch.calendarId == null) { + await this.watchCalendars(watch) + } else { + await this.watchCalendar(watch) + } + } + } + + async unsubscribe (watches: Watch[]): Promise { + for (const watch of watches) { + await this.unsubscribeWatch(watch) + } + } + + private async unsubscribeWatch (current: Watch): Promise { + await this.rateLimiter.take(1) + await this.calendar.channels.stop({ requestBody: { id: current.channelId, resourceId: current.resourceId } }) + } + + private async watchCalendars (current: Watch): Promise { + try { + await this.unsubscribeWatch(current) + const channelId = generateId() + const body = { id: channelId, address: config.WATCH_URL, type: 'webhook', token: `user=${this.me}&mode=calendar` } + await this.rateLimiter.take(1) + const res = await this.calendar.calendarList.watch({ requestBody: body }) + if (res.data.expiration != null && res.data.resourceId !== null) { + // eslint-disable-next-line + this.watches.updateOne( + { + userId: current.userId, + workspace: current.workspace, + calendarId: null + }, + { + $set: { + channelId, + expired: Number.parseInt(res.data.expiration), + resourceId: res.data.resourceId ?? '' + } + } + ) + } + } catch (err) { + console.error('Calendar watch error', err) + } + } + + private async watchCalendar (current: EventWatch): Promise { + try { + await this.unsubscribeWatch(current) + const channelId = generateId() + const body = { + id: channelId, + address: config.WATCH_URL, + type: 'webhook', + token: `user=${this.me}&mode=events&calendarId=${current.calendarId}` + } + await this.rateLimiter.take(1) + const res = await this.calendar.events.watch({ calendarId: current.calendarId, requestBody: body }) + if (res.data.expiration != null && res.data.resourceId != null) { + // eslint-disable-next-line + this.watches.updateOne( + { + userId: current.userId, + workspace: current.workspace, + calendarId: current.calendarId + }, + { + $set: { + channelId, + expired: Number.parseInt(res.data.expiration), + resourceId: res.data.resourceId ?? '' + } + } + ) + } + } catch (err: any) { + await this.checkError(err) + } + } +} + +// we have to refresh channels approx each week +export class WatchController { + private readonly watches: Collection + private readonly tokens: Collection + + private timer: NodeJS.Timeout | undefined = undefined + protected static _instance: WatchController + + private constructor (private readonly mongo: Db) { + this.watches = mongo.collection('watch') + this.tokens = mongo.collection('tokens') + console.log('watch started') + } + + static get (mongo: Db): WatchController { + if (WatchController._instance !== undefined) { + return WatchController._instance + } + return new WatchController(mongo) + } + + async unsubscribe (user: Token): Promise { + const allWatches = await this.watches.find({ userId: user.userId, workspae: user.workspace }).toArray() + await this.watches.deleteMany({ userId: user.userId, workspae: user.workspace }) + const token = this.tokens.findOne({ user: user.userId, workspace: user.workspace }) + if (token == null) return + const watchClient = await WatchClient.Create(this.mongo, user) + await watchClient.unsubscribe(allWatches) + } + + stop (): void { + if (this.timer !== undefined) { + clearInterval(this.timer) + } + } + + startCheck (): void { + this.timer = setInterval( + () => { + void this.checkAll() + }, + 1000 * 60 * 60 * 24 + ) + void this.checkAll() + } + + async checkAll (): Promise { + const expired = Date.now() + 24 * 60 * 60 * 1000 + const watches = await this.watches + .find({ + expired: { $lt: expired } + }) + .toArray() + console.log('watch, found for update', watches.length) + const groups = new Map() + const workspaces = new Set() + for (const watch of watches) { + workspaces.add(watch.workspace) + const key = `${watch.userId}:${watch.workspace}` + const group = groups.get(key) + if (group !== undefined) { + group.push(watch) + } else { + groups.set(key, [watch]) + } + } + const token = generateToken(systemAccountEmail, { name: '' }) + const infos = await getWorkspacesInfo(token, [...groups.keys()]) + const tokens = await this.tokens.find({ workspace: { $in: [...workspaces] } }).toArray() + for (const group of groups.values()) { + try { + const userId = group[0].userId + const workspace = group[0].workspace + const token = tokens.find((p) => p.workspace === workspace && p.userId === userId) + if (token === undefined) { + await this.watches.deleteMany({ userId, workspace }) + continue + } + const info = infos.find((p) => p.workspace === workspace) + if (info === undefined || isActiveMode(info.mode)) { + await this.watches.deleteMany({ userId, workspace }) + continue + } + const watchClient = await WatchClient.Create(this.mongo, token) + await watchClient.subscribe(group) + } catch {} + } + console.log('watch check done') + } +} diff --git a/services/calendar/pod-calendar/src/workspaceClient.ts b/services/calendar/pod-calendar/src/workspaceClient.ts index 189e39ecbb..c7531da5e9 100644 --- a/services/calendar/pod-calendar/src/workspaceClient.ts +++ b/services/calendar/pod-calendar/src/workspaceClient.ts @@ -16,6 +16,7 @@ import calendar, { Event, ExternalCalendar } from '@hcengineering/calendar' import contact, { Channel, Contact, type Employee, type PersonAccount } from '@hcengineering/contact' import core, { + RateLimiter, TxOperations, TxProcessor, systemAccountEmail, @@ -35,14 +36,20 @@ import { Collection, type Db } from 'mongodb' import { CalendarClient } from './calendar' import { CalendarController } from './calendarController' import { getClient } from './client' -import { SyncHistory, type ProjectCredentials, type User } from './types' +import { SyncHistory, Token, type User } from './types' +import config from './config' export class WorkspaceClient { private readonly txHandlers: ((...tx: Tx[]) => Promise)[] = [] - private client!: Client - private readonly clients: Map = new Map() + client!: Client + private readonly clients: Map> = new Map< + string, + CalendarClient | Promise + >() + private readonly syncHistory: Collection + private readonly tokens: Collection private channels = new Map, Channel>() private readonly calendarsByEmail = new Map() readonly calendars = { @@ -62,39 +69,45 @@ export class WorkspaceClient { } private constructor ( - private readonly credentials: ProjectCredentials, private readonly mongo: Db, private readonly workspace: string, private readonly serviceController: CalendarController ) { + this.tokens = mongo.collection('tokens') this.syncHistory = mongo.collection('syncHistories') } - static async create ( - credentials: ProjectCredentials, - mongo: Db, - workspace: string, - serviceController: CalendarController - ): Promise { - const instance = new WorkspaceClient(credentials, mongo, workspace, serviceController) + static async getSystemClient (workspace: string): Promise { + const token = generateToken(systemAccountEmail, { name: workspace }) + return await getClient(token) + } + + static async create (mongo: Db, workspace: string, serviceController: CalendarController): Promise { + const instance = new WorkspaceClient(mongo, workspace, serviceController) await instance.initClient(workspace) return instance } async createCalendarClient (user: User): Promise { const current = this.getCalendarClient(user.email) - if (current !== undefined) return current - const newClient = await CalendarClient.create(this.credentials, user, this.mongo, this.client, this) + if (current !== undefined) { + if (current instanceof Promise) { + return await current + } + return current + } + const newClient = CalendarClient.create(user, this.mongo, this.client, this) this.clients.set(user.email, newClient) - console.log('create new client', user.email, this.workspace) - return newClient + const res = await newClient + this.clients.set(user.email, res) + return res } async newCalendarClient (user: User, code: string): Promise { - const newClient = await CalendarClient.create(this.credentials, user, this.mongo, this.client, this) + const newClient = await CalendarClient.create(user, this.mongo, this.client, this) const email = await newClient.authorize(code) if (this.clients.has(email)) { - await newClient.close() + newClient.close() throw new Error('Client already exist') } this.clients.set(email, newClient) @@ -102,8 +115,11 @@ export class WorkspaceClient { } async close (): Promise { - for (const client of this.clients.values()) { - await client.close() + for (let client of this.clients.values()) { + if (client instanceof Promise) { + client = await client + } + client.close() } this.clients.clear() await this.client?.close() @@ -118,9 +134,12 @@ export class WorkspaceClient { } async signout (value: string, byError: boolean = false): Promise { - const client = this.clients.get(value) + let client = this.clients.get(value) if (client !== undefined) { - await client.signout(byError) + if (client instanceof Promise) { + client = await client + } + await client.signout() } else { const integration = await this.client.findOne(setting.class.Integration, { type: calendar.integrationType.Calendar, @@ -143,19 +162,38 @@ export class WorkspaceClient { this.clients.delete(email) this.serviceController.removeClient(email) if (this.clients.size > 0) return + void this.close() this.serviceController.removeWorkspace(this.workspace) } - private getCalendarClient (email: string): CalendarClient | undefined { + private getCalendarClient (email: string): CalendarClient | Promise | undefined { return this.clients.get(email) } - private getCalendarClientByCalendar (id: Ref): CalendarClient | undefined { + private async getCalendarClientByCalendar ( + id: Ref, + create: boolean = false + ): Promise { const calendar = this.calendars.byId.get(id) if (calendar === undefined) { - console.log("couldn't find calendar by id", id) + console.warn("couldn't find calendar by id", id) + return } - return calendar != null ? this.clients.get(calendar.externalUser) : undefined + const client = this.clients.get(calendar.externalUser) + if (client instanceof Promise) { + return await client + } + if (client === undefined && create) { + const user = await this.tokens.findOne({ + workspace: this.workspace, + access_token: { $exists: true }, + email: calendar.externalUser + }) + if (user != null) { + return await this.createCalendarClient(user) + } + } + return client } private async initClient (workspace: string): Promise { @@ -187,6 +225,15 @@ export class WorkspaceClient { async sync (): Promise { await this.getNewEvents() + const limiter = new RateLimiter(config.InitLimit) + for (let client of this.clients.values()) { + void limiter.add(async () => { + if (client instanceof Promise) { + client = await client + } + await client.startSync() + }) + } } // #region Events @@ -213,6 +260,20 @@ export class WorkspaceClient { ) } + async pushEvent (event: Event, type: 'create' | 'update' | 'delete'): Promise { + const client = await this.getCalendarClientByCalendar(event.calendar as Ref, true) + if (client === undefined) { + console.warn('Client not found', event.calendar, this.workspace) + return + } + if (type === 'delete') { + await client.removeEvent(event) + } else { + await client.syncMyEvent(event) + } + await this.updateSyncTime() + } + async getNewEvents (): Promise { const lastSync = await this.getSyncTime() const query = lastSync !== undefined ? { modifiedOn: { $gt: lastSync } } : {} @@ -220,17 +281,16 @@ export class WorkspaceClient { this.txHandlers.push(async (...tx: Tx[]) => { await this.txEventHandler(...tx) }) - console.log('receive new events', this.workspace, newEvents.length) for (const newEvent of newEvents) { - const client = this.getCalendarClientByCalendar(newEvent.calendar as Ref) + const client = await this.getCalendarClientByCalendar(newEvent.calendar as Ref) if (client === undefined) { - console.log('Client not found', newEvent.calendar, this.workspace) + console.warn('Client not found', newEvent.calendar, this.workspace) return } await client.syncMyEvent(newEvent) await this.updateSyncTime() } - console.log('all messages synced', this.workspace) + console.log('all outcoming messages synced', this.workspace) } private async txEventHandler (...txes: Tx[]): Promise { @@ -256,7 +316,7 @@ export class WorkspaceClient { if (hierarhy.isDerived(tx.objectClass, calendar.class.Event)) { const doc = TxProcessor.createDoc2Doc(tx as TxCreateDoc) if (doc.access !== 'owner') return - const client = this.getCalendarClientByCalendar(doc.calendar as Ref) + const client = await this.getCalendarClientByCalendar(doc.calendar as Ref) if (client === undefined) { return } @@ -264,7 +324,7 @@ export class WorkspaceClient { await client.createEvent(doc) await this.updateSyncTime() } catch (err) { - console.log(err) + console.error(err) } } } @@ -281,7 +341,7 @@ export class WorkspaceClient { const extracted = txes.filter((p) => p._id !== tx._id) const ev = TxProcessor.buildDoc2Doc(extracted) if (ev !== undefined) { - const oldClient = this.getCalendarClientByCalendar(ev.calendar as Ref) + const oldClient = await this.getCalendarClientByCalendar(ev.calendar as Ref) if (oldClient !== undefined) { const oldCalendar = this.calendars.byId.get(ev.calendar as Ref) if (oldCalendar !== undefined) { @@ -290,16 +350,16 @@ export class WorkspaceClient { } } } catch (err) { - console.log('Error on remove event', err) + console.error('Error on remove event', err) } try { - const client = this.getCalendarClientByCalendar(event.calendar as Ref) + const client = await this.getCalendarClientByCalendar(event.calendar as Ref) if (client !== undefined) { await client.syncMyEvent(event) } await this.updateSyncTime() } catch (err) { - console.log('Error on move event', err) + console.error('Error on move event', err) } } @@ -315,15 +375,15 @@ export class WorkspaceClient { return } if (event.access !== 'owner' && event.access !== 'writer') return - const client = this.getCalendarClientByCalendar(event.calendar as Ref) + const client = await this.getCalendarClientByCalendar(event.calendar as Ref) if (client === undefined) { return } try { - await client.updateEvent(event, tx) + await client.updateEvent(event) await this.updateSyncTime() } catch (err) { - console.log(err) + console.error(err) } } } @@ -337,7 +397,7 @@ export class WorkspaceClient { const ev = TxProcessor.buildDoc2Doc(txes) if (ev === undefined) return if (ev.access !== 'owner' && ev.access !== 'writer') return - const client = this.getCalendarClientByCalendar(ev?.calendar as Ref) + const client = await this.getCalendarClientByCalendar(ev?.calendar as Ref) if (client === undefined) { return } diff --git a/services/github/pod-github/src/account.ts b/services/github/pod-github/src/account.ts index cf013c2a71..092d0743e6 100644 --- a/services/github/pod-github/src/account.ts +++ b/services/github/pod-github/src/account.ts @@ -1,4 +1,4 @@ -import { ClientWorkspaceInfo } from '@hcengineering/account' +import { ClientWorkspaceInfo } from '@hcengineering/core' import config from './config' /** diff --git a/services/github/pod-github/src/platform.ts b/services/github/pod-github/src/platform.ts index 4de5d9befe..a9e4333c99 100644 --- a/services/github/pod-github/src/platform.ts +++ b/services/github/pod-github/src/platform.ts @@ -9,6 +9,7 @@ import core, { BrandingMap, Client, ClientConnectEvent, + ClientWorkspaceInfo, DocumentUpdate, isActiveMode, isDeletingMode, @@ -29,7 +30,6 @@ import { Installation, type InstallationCreatedEvent, type InstallationUnsuspend import { Collection } from 'mongodb' import { App, Octokit } from 'octokit' -import { ClientWorkspaceInfo } from '@hcengineering/account' import { Analytics } from '@hcengineering/analytics' import { SplitLogger } from '@hcengineering/analytics-service' import contact, { Person, PersonAccount } from '@hcengineering/contact' From 238d1a249520efc409a5b62a3802940c51820325 Mon Sep 17 00:00:00 2001 From: Andrey Sobolev Date: Mon, 24 Feb 2025 10:26:10 +0700 Subject: [PATCH 02/16] Initial rest RPC (#8076) --- common/config/rush/pnpm-lock.yaml | 6 +- packages/api-client/package.json | 6 +- packages/api-client/src/index.ts | 1 + packages/api-client/src/rest.ts | 130 ++++++++++ packages/core/src/measurements/metrics.ts | 26 ++ .../components/ServerManagerGeneral.svelte | 16 +- server/core/src/types.ts | 26 ++ server/server/src/client.ts | 50 ++-- server/server/src/sessionManager.ts | 62 ++++- server/ws/package.json | 3 +- server/ws/src/__tests__/minmodel.ts | 2 +- server/ws/src/__tests__/rest.test.ts | 225 ++++++++++++++++++ server/ws/src/rpc.ts | 182 ++++++++++++++ server/ws/src/server_http.ts | 36 ++- server/ws/src/utils.ts | 23 ++ 15 files changed, 738 insertions(+), 56 deletions(-) create mode 100644 packages/api-client/src/rest.ts create mode 100644 server/ws/src/__tests__/rest.test.ts create mode 100644 server/ws/src/rpc.ts create mode 100644 server/ws/src/utils.ts diff --git a/common/config/rush/pnpm-lock.yaml b/common/config/rush/pnpm-lock.yaml index eda1faf365..d64f04f26a 100644 --- a/common/config/rush/pnpm-lock.yaml +++ b/common/config/rush/pnpm-lock.yaml @@ -3861,7 +3861,7 @@ packages: version: 0.0.0 '@rush-temp/api-client@file:projects/api-client.tgz': - resolution: {integrity: sha512-tyhf+5DHkF+HpWZlimhdgls3Gp3L7PR0UHYEu0YH7e0UmOoZSuvAVbRbwY/6UnVG1am+xQnrs0xiZ93AHIsgBA==, tarball: file:projects/api-client.tgz} + resolution: {integrity: sha512-gq68pTrBheqcmVrkbiqYY+6p80DQHIsJvC1nIvhRfwrj3eR1ir5fVsW5t2Gs8fvN/Um8nH0MnkTUGavVrPWIdw==, tarball: file:projects/api-client.tgz} version: 0.0.0 '@rush-temp/attachment-assets@file:projects/attachment-assets.tgz': @@ -5065,7 +5065,7 @@ packages: version: 0.0.0 '@rush-temp/server-ws@file:projects/server-ws.tgz': - resolution: {integrity: sha512-9PP7kE8LXoZ+7r6OwLp26KT+JFAQtSwg5QP/awwXjcsV2t3yphDYKbdAJuVG5xlLfHW52Xn4NRbr0wFhfwS0sg==, tarball: file:projects/server-ws.tgz} + resolution: {integrity: sha512-3jjoKDZ0NAs4XRajHH6n4EgS00GwQY0Zga1JlG8XIeSPjAGphGUYbCaHQuJbcddPAYRdT632gK+119U4RvGSxw==, tarball: file:projects/server-ws.tgz} version: 0.0.0 '@rush-temp/server@file:projects/server.tgz': @@ -16130,6 +16130,7 @@ snapshots: dependencies: '@types/jest': 29.5.12 '@types/node': 20.11.19 + '@types/snappyjs': 0.7.1 '@types/ws': 8.5.11 '@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.6.2) '@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.6.2) @@ -16141,6 +16142,7 @@ snapshots: jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)) prettier: 3.2.5 simplytyped: 3.3.0(typescript@5.6.2) + snappyjs: 0.7.0 ts-jest: 29.1.2(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(jest@29.7.0(@types/node@20.11.19)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)))(typescript@5.6.2) ts-node: 10.9.2(@types/node@20.11.19)(typescript@5.6.2) typescript: 5.6.2 diff --git a/packages/api-client/package.json b/packages/api-client/package.json index 72f64b4422..e9b77fd5b6 100644 --- a/packages/api-client/package.json +++ b/packages/api-client/package.json @@ -39,7 +39,8 @@ "ts-node": "^10.8.0", "@types/node": "~20.11.16", "@types/jest": "^29.5.5", - "@types/ws": "^8.5.11" + "@types/ws": "^8.5.11", + "@types/snappyjs": "^0.7.1" }, "dependencies": { "@hcengineering/core": "^0.6.32", @@ -47,7 +48,8 @@ "@hcengineering/client-resources": "^0.6.27", "@hcengineering/collaborator-client": "^0.6.4", "@hcengineering/platform": "^0.6.11", - "@hcengineering/text": "^0.6.5" + "@hcengineering/text": "^0.6.5", + "snappyjs": "^0.7.0" }, "repository": "https://github.com/hcengineering/platform", "publishConfig": { diff --git a/packages/api-client/src/index.ts b/packages/api-client/src/index.ts index 2dd51ba8f5..911f155e01 100644 --- a/packages/api-client/src/index.ts +++ b/packages/api-client/src/index.ts @@ -17,3 +17,4 @@ export * from './client' export * from './markup/types' export * from './socket' export * from './types' +export * from './rest' diff --git a/packages/api-client/src/rest.ts b/packages/api-client/src/rest.ts new file mode 100644 index 0000000000..23648ca2d1 --- /dev/null +++ b/packages/api-client/src/rest.ts @@ -0,0 +1,130 @@ +// +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import { + type Account, + type Class, + type Doc, + type DocumentQuery, + type FindOptions, + type FindResult, + type Ref, + type Storage, + type Tx, + type TxResult, + type WithLookup, + concatLink +} from '@hcengineering/core' + +import { PlatformError, unknownError } from '@hcengineering/platform' + +import { uncompress } from 'snappyjs' + +export interface RestClient extends Storage { + getAccount: () => Promise + + findOne: ( + _class: Ref>, + query: DocumentQuery, + options?: FindOptions + ) => Promise | undefined> +} + +export async function createRestClient (endpoint: string, workspaceId: string, token: string): Promise { + return new RestClientImpl(endpoint, workspaceId, token) +} + +class RestClientImpl implements RestClient { + constructor ( + readonly endpoint: string, + readonly workspace: string, + readonly token: string + ) {} + + async findAll( + _class: Ref>, + query: DocumentQuery, + options?: FindOptions + ): Promise> { + const params = new URLSearchParams() + params.append('class', _class) + if (query !== undefined && Object.keys(query).length > 0) { + params.append('query', JSON.stringify(query)) + } + if (options !== undefined && Object.keys(options).length > 0) { + params.append('options', JSON.stringify(options)) + } + const response = await fetch(concatLink(this.endpoint, `/api/v1/find-all/${this.workspace}?${params.toString()}`), { + method: 'GET', + headers: { + 'Content-Type': 'application/json', + Authorization: 'Bearer ' + this.token, + 'accept-encoding': 'snappy, gzip' + }, + keepalive: true + }) + if (!response.ok) { + throw new PlatformError(unknownError(response.statusText)) + } + const encoding = response.headers.get('content-encoding') + if (encoding === 'snappy') { + const buffer = await response.arrayBuffer() + const decompressed = uncompress(buffer) + const decoder = new TextDecoder() + const jsonString = decoder.decode(decompressed) + return JSON.parse(jsonString) as FindResult + } + return (await response.json()) as FindResult + } + + async getAccount (): Promise { + const response = await fetch(concatLink(this.endpoint, `/api/v1/account/${this.workspace}`), { + method: 'GET', + headers: { + 'Content-Type': 'application/json', + Authorization: 'Bearer ' + this.token + }, + keepalive: true + }) + if (!response.ok) { + throw new PlatformError(unknownError(response.statusText)) + } + return (await response.json()) as Account + } + + async findOne( + _class: Ref>, + query: DocumentQuery, + options?: FindOptions + ): Promise | undefined> { + return (await this.findAll(_class, query, { ...options, limit: 1 })).shift() + } + + async tx (tx: Tx): Promise { + const response = await fetch(concatLink(this.endpoint, `/api/v1/tx/${this.workspace}`), { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Authorization: 'Bearer ' + this.token + }, + keepalive: true, + body: JSON.stringify(tx) + }) + if (!response.ok) { + throw new PlatformError(unknownError(response.statusText)) + } + return (await response.json()) as TxResult + } +} diff --git a/packages/core/src/measurements/metrics.ts b/packages/core/src/measurements/metrics.ts index 49d6591804..9b8f9d8d9a 100644 --- a/packages/core/src/measurements/metrics.ts +++ b/packages/core/src/measurements/metrics.ts @@ -233,6 +233,28 @@ function toString (name: string, m: Metrics, offset: number, length: number): st return r } +function toJson (m: Metrics): any { + const obj: any = { + $total: m.value, + $ops: m.operations + } + if (m.operations > 1) { + obj.avg = Math.round((m.value / (m.operations > 0 ? m.operations : 1)) * 100) / 100 + } + if (Object.keys(m.params).length > 0) { + obj.params = m.params + } + for (const [k, v] of Object.entries(m.measurements ?? {})) { + obj[ + `${k} ${v.value} ${v.operations} ${ + v.operations > 1 ? Math.round((v.value / (v.operations > 0 ? m.operations : 1)) * 100) / 100 : '' + }` + ] = toJson(v) + } + + return obj +} + /** * @public */ @@ -240,6 +262,10 @@ export function metricsToString (metrics: Metrics, name = 'System', length: numb return toString(name, metricsAggregate(metrics, 50), 0, length) } +export function metricsToJson (metrics: Metrics): any { + return toJson(metricsAggregate(metrics)) +} + function printMetricsParamsRows ( params: Record>, offset: number diff --git a/plugins/workbench-resources/src/components/ServerManagerGeneral.svelte b/plugins/workbench-resources/src/components/ServerManagerGeneral.svelte index 0bb5c65e2c..06584bdf2d 100644 --- a/plugins/workbench-resources/src/components/ServerManagerGeneral.svelte +++ b/plugins/workbench-resources/src/components/ServerManagerGeneral.svelte @@ -3,8 +3,7 @@ import login from '@hcengineering/login' import { getEmbeddedLabel, getMetadata } from '@hcengineering/platform' import presentation, { getClient, isAdminUser, uiContext } from '@hcengineering/presentation' - import { Button, IconArrowLeft, IconArrowRight, fetchMetadataLocalStorage, ticker } from '@hcengineering/ui' - import EditBox from '@hcengineering/ui/src/components/EditBox.svelte' + import { Button, EditBox, IconArrowLeft, IconArrowRight, fetchMetadataLocalStorage, ticker } from '@hcengineering/ui' import MetricsInfo from './statistics/MetricsInfo.svelte' const _endpoint: string = fetchMetadataLocalStorage(login.metadata.LoginEndpoint) ?? '' @@ -25,8 +24,6 @@ let avgTime = 0 - let rps = 0 - let active = 0 let opss = 0 @@ -44,7 +41,7 @@ profiling = data?.profiling ?? false }) .catch((err) => { - console.error(err) + console.error(err, time) }) } let data: any @@ -65,13 +62,8 @@ avgTime = 0 maxTime = 0 let count = commandsToSend - let ops = 0 avgTime = 0 opss = 0 - const int = setInterval(() => { - rps = ops - ops = 0 - }, 1000) const rate = new RateLimiter(commandsToSendParallel) const client = getClient() @@ -96,7 +88,6 @@ } else { avgTime = ed - st } - ops++ opss++ count-- } @@ -112,7 +103,6 @@ } await rate.waitProcessing() running = false - clearInterval(int) } async function downloadProfile (): Promise { @@ -132,7 +122,7 @@ document.body.appendChild(link) link.click() document.body.removeChild(link) - fetchStats(0) + await fetchStats(0) } let metrics: Metrics | undefined diff --git a/server/core/src/types.ts b/server/core/src/types.ts index 3ba40d0225..53e0db939d 100644 --- a/server/core/src/types.ts +++ b/server/core/src/types.ts @@ -565,6 +565,16 @@ export interface Session { ) => Promise> searchFulltext: (ctx: ClientSessionCtx, query: SearchQuery, options: SearchOptions) => Promise tx: (ctx: ClientSessionCtx, tx: Tx) => Promise + + txRaw: ( + ctx: ClientSessionCtx, + tx: Tx + ) => Promise<{ + result: TxResult + broadcastPromise: Promise + asyncsPromise: Promise | undefined + }> + loadChunk: (ctx: ClientSessionCtx, domain: Domain, idx?: number) => Promise getDomainHash: (ctx: ClientSessionCtx, domain: Domain) => Promise @@ -702,6 +712,22 @@ export interface SessionManager { request: Request, workspace: string // wsId, toWorkspaceString() ) => Promise + + createOpContext: ( + ctx: MeasureContext, + pipeline: Pipeline, + requestId: Request['id'], + service: Session, + ws: ConnectionSocket, + workspace: WorkspaceId + ) => ClientSessionCtx + + handleRPC: ( + requestCtx: MeasureContext, + service: S, + ws: ConnectionSocket, + operation: (ctx: ClientSessionCtx) => Promise + ) => Promise } /** diff --git a/server/server/src/client.ts b/server/server/src/client.ts index 052b3d970a..875ad702f4 100644 --- a/server/server/src/client.ts +++ b/server/server/src/client.ts @@ -30,7 +30,8 @@ import { type SessionData, type Timestamp, type Tx, - type TxCUD + type TxCUD, + type TxResult } from '@hcengineering/core' import { PlatformError, unknownError } from '@hcengineering/platform' import { @@ -162,7 +163,14 @@ export class ClientSession implements Session { await ctx.sendResponse(ctx.requestId, await ctx.pipeline.searchFulltext(ctx.ctx, query, options)) } - async tx (ctx: ClientSessionCtx, tx: Tx): Promise { + async txRaw ( + ctx: ClientSessionCtx, + tx: Tx + ): Promise<{ + result: TxResult + broadcastPromise: Promise + asyncsPromise: Promise | undefined + }> { this.lastRequest = Date.now() this.total.tx++ this.current.tx++ @@ -171,31 +179,45 @@ export class ClientSession implements Session { let cid = 'client_' + generateId() ctx.ctx.id = cid let onEnd = useReserveContext ? ctx.pipeline.context.adapterManager?.reserveContext?.(cid) : undefined + let result: TxResult try { - const result = await ctx.pipeline.tx(ctx.ctx, [tx]) - - // Send result immideately - await ctx.sendResponse(ctx.requestId, result) - - // We need to broadcast all collected transactions - await ctx.pipeline.handleBroadcast(ctx.ctx) + result = await ctx.pipeline.tx(ctx.ctx, [tx]) } finally { onEnd?.() } + // Send result immideately + await ctx.sendResponse(ctx.requestId, result) + + // We need to broadcast all collected transactions + const broadcastPromise = ctx.pipeline.handleBroadcast(ctx.ctx) // ok we could perform async requests if any const asyncs = (ctx.ctx.contextData as SessionData).asyncRequests ?? [] + let asyncsPromise: Promise | undefined if (asyncs.length > 0) { cid = 'client_async_' + generateId() ctx.ctx.id = cid onEnd = useReserveContext ? ctx.pipeline.context.adapterManager?.reserveContext?.(cid) : undefined - try { - for (const r of (ctx.ctx.contextData as SessionData).asyncRequests ?? []) { - await r() + const handleAyncs = async (): Promise => { + try { + for (const r of (ctx.ctx.contextData as SessionData).asyncRequests ?? []) { + await r() + } + } finally { + onEnd?.() } - } finally { - onEnd?.() } + asyncsPromise = handleAyncs() + } + + return { result, broadcastPromise, asyncsPromise } + } + + async tx (ctx: ClientSessionCtx, tx: Tx): Promise { + const { broadcastPromise, asyncsPromise } = await this.txRaw(ctx, tx) + await broadcastPromise + if (asyncsPromise !== undefined) { + await asyncsPromise } } diff --git a/server/server/src/sessionManager.ts b/server/server/src/sessionManager.ts index 2549cb7480..f74dc61bf7 100644 --- a/server/server/src/sessionManager.ts +++ b/server/server/src/sessionManager.ts @@ -79,7 +79,7 @@ export interface Timeouts { reconnectTimeout: number // Default 3 seconds } -class TSessionManager implements SessionManager { +export class TSessionManager implements SessionManager { private readonly statusPromises = new Map>() readonly workspaces = new Map() checkInterval: any @@ -941,7 +941,7 @@ class TSessionManager implements SessionManager { createOpContext ( ctx: MeasureContext, pipeline: Pipeline, - request: Request, + requestId: Request['id'], service: Session, ws: ConnectionSocket ): ClientSessionCtx { @@ -949,7 +949,7 @@ class TSessionManager implements SessionManager { return { ctx, pipeline, - requestId: request.id, + requestId, sendResponse: (reqId, msg) => sendResponse(ctx, service, ws, { id: reqId, @@ -1009,6 +1009,7 @@ class TSessionManager implements SessionManager { return } if (request.id === -2 && request.method === 'forceClose') { + // TODO: we chould allow this only for admin or system accounts let done = false const wsRef = this.workspaces.get(workspace) if (wsRef?.upgrade ?? false) { @@ -1043,7 +1044,7 @@ class TSessionManager implements SessionManager { const params = [...request.params] await ctx.with('🧨 process', {}, (callTx) => - f.apply(service, [this.createOpContext(callTx, pipeline, request, service, ws), ...params]) + f.apply(service, [this.createOpContext(callTx, pipeline, request.id, service, ws), ...params]) ) } catch (err: any) { Analytics.handleError(err) @@ -1068,6 +1069,59 @@ class TSessionManager implements SessionManager { }) } + handleRPC( + requestCtx: MeasureContext, + service: S, + ws: ConnectionSocket, + operation: (ctx: ClientSessionCtx) => Promise + ): Promise { + const userCtx = requestCtx.newChild('📞 client', {}) + + // Calculate total number of clients + const reqId = generateId() + + const st = Date.now() + return userCtx + .with('🧭 handleRPC', {}, async (ctx) => { + if (service.workspace.closing !== undefined) { + throw new Error('Workspace is closing') + } + + service.requests.set(reqId, { + id: reqId, + params: {}, + start: st + }) + + const pipeline = + service.workspace.pipeline instanceof Promise ? await service.workspace.pipeline : service.workspace.pipeline + + try { + const uctx = this.createOpContext(ctx, pipeline, reqId, service, ws) + await operation(uctx) + } catch (err: any) { + Analytics.handleError(err) + if (LOGGING_ENABLED) { + this.ctx.error('error handle request', { error: err }) + } + ws.send( + ctx, + { + id: reqId, + error: unknownError(err), + result: JSON.parse(JSON.stringify(err?.stack)) + }, + service.binaryMode, + service.useCompression + ) + } + }) + .finally(() => { + userCtx.end() + service.requests.delete(reqId) + }) + } + private async handleHello( request: Request, service: S, diff --git a/server/ws/package.json b/server/ws/package.json index 253c6da9e6..26a56b85b5 100644 --- a/server/ws/package.json +++ b/server/ws/package.json @@ -52,6 +52,7 @@ "utf-8-validate": "^6.0.4", "ws": "^8.18.0", "body-parser": "^1.20.2", - "snappy": "^7.2.2" + "snappy": "^7.2.2", + "@hcengineering/api-client": "^0.6.0" } } diff --git a/server/ws/src/__tests__/minmodel.ts b/server/ws/src/__tests__/minmodel.ts index da06698841..262c75ecb1 100644 --- a/server/ws/src/__tests__/minmodel.ts +++ b/server/ws/src/__tests__/minmodel.ts @@ -183,7 +183,7 @@ export function genMinModel (): TxCUD[] { const u1 = 'User1' as Ref const u2 = 'User2' as Ref txes.push( - createDoc(core.class.Account, { email: 'user1@site.com', role: AccountRole.User }, u1), + createDoc(core.class.Account, { email: 'user1@site.com', role: AccountRole.Owner }, u1), createDoc(core.class.Account, { email: 'user2@site.com', role: AccountRole.User }, u2), createDoc(core.class.Space, { name: 'Sp1', diff --git a/server/ws/src/__tests__/rest.test.ts b/server/ws/src/__tests__/rest.test.ts new file mode 100644 index 0000000000..6ad4530733 --- /dev/null +++ b/server/ws/src/__tests__/rest.test.ts @@ -0,0 +1,225 @@ +// +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import { generateToken } from '@hcengineering/server-token' + +import { createRestClient, type RestClient } from '@hcengineering/api-client' +import core, { + generateId, + getWorkspaceId, + Hierarchy, + MeasureMetricsContext, + ModelDb, + toFindResult, + type Class, + type Doc, + type DocumentQuery, + type Domain, + type FindOptions, + type FindResult, + type MeasureContext, + type Ref, + type Space, + type Tx, + type TxCreateDoc, + type TxResult +} from '@hcengineering/core' +import { ClientSession, startSessionManager, type TSessionManager } from '@hcengineering/server' +import { createDummyStorageAdapter, type SessionManager, type WorkspaceLoginInfo } from '@hcengineering/server-core' +import { startHttpServer } from '../server_http' +import { genMinModel } from './minmodel' + +describe('rest-server', () => { + async function getModelDb (): Promise<{ modelDb: ModelDb, hierarchy: Hierarchy, txes: Tx[] }> { + const txes = genMinModel() + const hierarchy = new Hierarchy() + for (const tx of txes) { + hierarchy.tx(tx) + } + const modelDb = new ModelDb(hierarchy) + for (const tx of txes) { + await modelDb.tx(tx) + } + return { modelDb, hierarchy, txes } + } + + let shutdown: () => Promise + let sessionManager: SessionManager + const port: number = 3330 + + beforeAll(async () => { + ;({ shutdown, sessionManager } = startSessionManager(new MeasureMetricsContext('test', {}), { + pipelineFactory: async () => { + const { modelDb, hierarchy, txes } = await getModelDb() + return { + hierarchy, + modelDb, + context: { + workspace: { + name: 'test-ws', + workspaceName: 'test-ws', + workspaceUrl: 'test-ws' + }, + hierarchy, + modelDb, + lastTx: generateId(), + lastHash: generateId(), + contextVars: {}, + branding: null + }, + handleBroadcast: async (ctx) => {}, + findAll: async ( + ctx: MeasureContext, + _class: Ref>, + query: DocumentQuery, + options?: FindOptions + ): Promise> => toFindResult(await modelDb.findAll(_class, query, options)), + tx: async (ctx: MeasureContext, tx: Tx[]): Promise<[TxResult, Tx[], string[] | undefined]> => [ + await modelDb.tx(...tx), + [], + undefined + ], + close: async () => {}, + domains: async () => hierarchy.domains(), + groupBy: async () => new Map(), + find: (ctx: MeasureContext, domain: Domain) => ({ + next: async (ctx: MeasureContext) => undefined, + close: async (ctx: MeasureContext) => {} + }), + load: async (ctx: MeasureContext, domain: Domain, docs: Ref[]) => [], + upload: async (ctx: MeasureContext, domain: Domain, docs: Doc[]) => {}, + clean: async (ctx: MeasureContext, domain: Domain, docs: Ref[]) => {}, + searchFulltext: async (ctx, query, options) => { + return { docs: [] } + }, + loadModel: async (ctx, lastModelTx, hash) => ({ + full: true, + hash: generateId(), + transactions: txes + }) + } + }, + sessionFactory: (token, workspace) => new ClientSession(token, workspace, true), + port, + brandingMap: {}, + serverFactory: startHttpServer, + accountsUrl: '', + externalStorage: createDummyStorageAdapter() + })) + jest + .spyOn(sessionManager as TSessionManager, 'getWorkspaceInfo') + .mockImplementation(async (ctx: MeasureContext, token: string): Promise => { + return { + workspaceId: 'test-ws', + workspaceUrl: 'test-ws', + workspaceName: 'Test Workspace', + uuid: 'test-ws', + createdBy: 'test-owner', + mode: 'active', + createdOn: Date.now(), + lastVisit: Date.now(), + disabled: false, + endpoint: `http://localhost:${port}`, + region: 'test-region', + targetRegion: 'test-region', + backupInfo: { + dataSize: 0, + blobsSize: 0, + backupSize: 0, + lastBackup: 0, + backups: 0 + } + } + }) + }) + afterAll(async () => { + await shutdown() + }) + + async function connect (): Promise { + const token: string = generateToken('user1@site.com', getWorkspaceId('test-ws')) + return await createRestClient(`http://localhost:${port}`, 'test-ws', token) + } + + it('get account', async () => { + const conn = await connect() + const account = await conn.getAccount() + + expect(account.email).toBe('user1@site.com') + expect(account.role).toBe('OWNER') + expect(account._id).toBe('User1') + expect(account._class).toBe('core:class:Account') + expect(account.space).toBe('core:space:Model') + expect(account.modifiedBy).toBe('core:account:System') + expect(account.createdBy).toBe('core:account:System') + expect(typeof account.modifiedOn).toBe('number') + expect(typeof account.createdOn).toBe('number') + }) + + it('find spaces', async () => { + const conn = await connect() + const spaces = await conn.findAll(core.class.Space, {}) + expect(spaces.length).toBe(2) + expect(spaces[0].name).toBe('Sp1') + expect(spaces[1].name).toBe('Sp2') + }) + + it('find avg', async () => { + const conn = await connect() + let ops = 0 + let total = 0 + const attempts = 1000 + for (let i = 0; i < attempts; i++) { + const st = performance.now() + const spaces = await conn.findAll(core.class.Space, {}) + expect(spaces.length).toBe(2) + expect(spaces[0].name).toBe('Sp1') + expect(spaces[1].name).toBe('Sp2') + const ed = performance.now() + ops++ + total += ed - st + } + const avg = total / ops + // console.log('ops:', ops, 'total:', total, 'avg:', ) + expect(ops).toEqual(attempts) + expect(avg).toBeLessThan(5) // 5ms max per operation + }) + + it('add space', async () => { + const conn = await connect() + const account = await conn.getAccount() + const tx: TxCreateDoc = { + _class: core.class.TxCreateDoc, + space: core.space.Tx, + _id: generateId(), + objectSpace: core.space.Model, + modifiedBy: account._id, + modifiedOn: Date.now(), + attributes: { + name: 'Sp3', + description: '', + private: false, + archived: false, + members: [], + autoJoin: false + }, + objectClass: core.class.Space, + objectId: generateId() + } + await conn.tx(tx) + const spaces = await conn.findAll(core.class.Space, {}) + expect(spaces.length).toBe(3) + }) +}) diff --git a/server/ws/src/rpc.ts b/server/ws/src/rpc.ts new file mode 100644 index 0000000000..2d2685ff5f --- /dev/null +++ b/server/ws/src/rpc.ts @@ -0,0 +1,182 @@ +import type { Class, Doc, MeasureContext, Ref } from '@hcengineering/core' +import type { + ClientSessionCtx, + ConnectionSocket, + PipelineFactory, + Session, + SessionManager +} from '@hcengineering/server-core' +import { decodeToken } from '@hcengineering/server-token' + +import { type Express, type Response as ExpressResponse, type Request } from 'express' +import type { OutgoingHttpHeaders } from 'http2' +import { compress } from 'snappy' +import { promisify } from 'util' +import { gzip } from 'zlib' +import { retrieveJson } from './utils' +interface RPCClientInfo { + client: ConnectionSocket + session: Session + workspaceId: string +} + +const gzipAsync = promisify(gzip) + +const sendError = (res: ExpressResponse, code: number, data: any): void => { + res.writeHead(code, { + 'Content-Type': 'application/json', + 'Cache-Control': 'no-cache', + 'keep-alive': 'timeout=5, max=1000' + }) + res.end(JSON.stringify(data)) +} + +async function sendJson (req: Request, res: ExpressResponse, result: any): Promise { + const headers: OutgoingHttpHeaders = { + 'Content-Type': 'application/json', + 'Cache-Control': 'no-cache', + 'keep-alive': 'timeout=5, max=1000' + } + let body: any = JSON.stringify(result) + + const contentEncodings: string[] = + typeof req.headers['accept-encoding'] === 'string' + ? req.headers['accept-encoding'].split(',').map((it) => it.trim()) + : req.headers['accept-encoding'] ?? [] + for (const contentEncoding of contentEncodings) { + let done = false + switch (contentEncoding) { + case 'snappy': + headers['content-encoding'] = 'snappy' + body = await compress(body) + done = true + break + case 'gzip': + headers['content-encoding'] = 'gzip' + body = await gzipAsync(body) + done = true + break + } + if (done) { + break + } + } + + res.writeHead(200, headers) + res.end(body) +} + +export function registerRPC ( + app: Express, + sessions: SessionManager, + ctx: MeasureContext, + pipelineFactory: PipelineFactory +): void { + const rpcSessions = new Map() + + async function withSession ( + req: Request, + res: ExpressResponse, + operation: (ctx: ClientSessionCtx, session: Session) => Promise + ): Promise { + if (req.params.workspaceId === undefined || req.params.workspaceId === '') { + res.writeHead(400, {}) + res.end('Missing workspace') + return + } + let token = req.headers.authorization as string + if (token === null) { + sendError(res, 401, { message: 'Missing Authorization header' }) + return + } + const workspaceId = decodeURIComponent(req.params.workspaceId) + token = token.split(' ')[1] + + const decodedToken = decodeToken(token) + if (workspaceId !== decodedToken.workspace.name) { + sendError(res, 401, { message: 'Invalid workspace' }) + return + } + + let transactorRpc = rpcSessions.get(token) + + if (transactorRpc === undefined) { + const cs: ConnectionSocket = createClosingSocket(token, rpcSessions) + const s = await sessions.addSession(ctx, cs, decodedToken, token, pipelineFactory, token) + if (!('session' in s)) { + sendError(res, 401, { + message: 'Failed to create session', + mode: 'specialError' in s ? s.specialError ?? '' : 'upgrading' + }) + return + } + transactorRpc = { session: s.session, client: cs, workspaceId: s.workspaceId } + rpcSessions.set(token, transactorRpc) + } + try { + const rpc = transactorRpc + await sessions.handleRPC(ctx, rpc.session, rpc.client, async (ctx) => { + await operation(ctx, rpc.session) + }) + } catch (err: any) { + sendError(res, 401, { message: 'Failed to execute operation', error: err.message, stack: err.stack }) + } + } + + app.get('/api/v1/ping/:workspaceId', (req, res) => { + void withSession(req, res, async (ctx, session) => { + await session.ping(ctx) + await sendJson(req, res, { pong: true }) + }) + }) + + app.get('/api/v1/find-all/:workspaceId', (req, res) => { + void withSession(req, res, async (ctx, session) => { + const _class = req.query.class as Ref> + const query = req.query.query !== undefined ? JSON.parse(req.query.query as string) : {} + const options = req.query.options !== undefined ? JSON.parse(req.query.options as string) : {} + + const result = await session.findAllRaw(ctx.ctx, ctx.pipeline, _class, query, options) + await sendJson(req, res, result) + }) + }) + + app.post('/api/v1/find-all/:workspaceId', (req, res) => { + void withSession(req, res, async (ctx, session) => { + const { _class, query, options }: any = (await retrieveJson(req)) ?? {} + + const result = await session.findAllRaw(ctx.ctx, ctx.pipeline, _class, query, options) + await sendJson(req, res, result) + }) + }) + + app.post('/api/v1/tx/:workspaceId', (req, res) => { + void withSession(req, res, async (ctx, session) => { + const tx: any = (await retrieveJson(req)) ?? {} + + const result = await session.txRaw(ctx, tx) + await sendJson(req, res, result.result) + }) + }) + app.get('/api/v1/account/:workspaceId', (req, res) => { + void withSession(req, res, async (ctx, session) => { + const result = session.getRawAccount(ctx.pipeline) + await sendJson(req, res, result) + }) + }) +} + +function createClosingSocket (rawToken: string, rpcSessions: Map): ConnectionSocket { + return { + id: rawToken, + isClosed: false, + close: () => { + rpcSessions.delete(rawToken) + }, + send: (ctx, msg, binary, compression) => {}, + sendPong: () => {}, + data: () => ({}), + readRequest: (buffer, binary) => ({ method: '', params: [], id: -1, time: Date.now() }), + checkState: () => true + } +} diff --git a/server/ws/src/server_http.ts b/server/ws/src/server_http.ts index 3bbf51baa4..82609f67d5 100644 --- a/server/ws/src/server_http.ts +++ b/server/ws/src/server_http.ts @@ -47,6 +47,8 @@ import { WebSocketServer, type RawData, type WebSocket } from 'ws' import 'bufferutil' import { compress } from 'snappy' import 'utf-8-validate' +import { registerRPC } from './rpc' +import { retrieveJson } from './utils' let profiling = false const rpcHandler = new RPCHandler() @@ -108,6 +110,7 @@ export function startHttpServer ( res.end() } }) + app.get('/api/v1/profiling', (req, res) => { try { const token = req.query.token as string @@ -297,6 +300,8 @@ export function startHttpServer ( } }) + registerRPC(app, sessions, ctx, pipelineFactory) + app.put('/api/v1/broadcast', (req, res) => { try { const token = req.query.token as string @@ -304,26 +309,19 @@ export function startHttpServer ( const ws = sessions.workspaces.get(req.query.workspace as string) if (ws !== undefined) { // push the data to body - const body: Buffer[] = [] - req - .on('data', (chunk) => { - body.push(chunk) - }) - .on('end', () => { - // on end of data, perform necessary action - try { - const data = JSON.parse(Buffer.concat(body as any).toString()) - if (Array.isArray(data)) { - sessions.broadcastAll(ws, data as Tx[]) - } else { - sessions.broadcastAll(ws, [data as unknown as Tx]) - } - res.end() - } catch (err: any) { - ctx.error('JSON parse error', { err }) - res.writeHead(400, {}) - res.end() + void retrieveJson(req) + .then((data) => { + if (Array.isArray(data)) { + sessions.broadcastAll(ws, data as Tx[]) + } else { + sessions.broadcastAll(ws, [data as unknown as Tx]) } + res.end() + }) + .catch((err) => { + ctx.error('JSON parse error', { err }) + res.writeHead(400, {}) + res.end() }) } else { res.writeHead(404, {}) diff --git a/server/ws/src/utils.ts b/server/ws/src/utils.ts new file mode 100644 index 0000000000..ceae377428 --- /dev/null +++ b/server/ws/src/utils.ts @@ -0,0 +1,23 @@ +import type { Request } from 'express' + +export function retrieveJson (req: Request): Promise { + const body: Uint8Array[] = [] + return new Promise((resolve, reject) => { + req + .on('data', (chunk: Uint8Array) => { + body.push(chunk) + }) + .on('error', (err) => { + reject(err) + }) + .on('end', () => { + // on end of data, perform necessary action + try { + const data = JSON.parse(Buffer.concat(body).toString()) + resolve(data) + } catch (err: any) { + reject(err) + } + }) + }) +} From 4d3488b81ee052ded084b1c06f073399df3cfb77 Mon Sep 17 00:00:00 2001 From: Denis Bykhov Date: Mon, 24 Feb 2025 10:37:45 +0500 Subject: [PATCH 03/16] Fix calendar push handler and remove unused client management (#8078) Signed-off-by: Denis Bykhov --- .../calendar/pod-calendar/src/calendar.ts | 16 ++---------- .../pod-calendar/src/calendarController.ts | 25 +------------------ .../pod-calendar/src/workspaceClient.ts | 1 - 3 files changed, 3 insertions(+), 39 deletions(-) diff --git a/services/calendar/pod-calendar/src/calendar.ts b/services/calendar/pod-calendar/src/calendar.ts index 3b0af5d449..8ffe987ded 100644 --- a/services/calendar/pod-calendar/src/calendar.ts +++ b/services/calendar/pod-calendar/src/calendar.ts @@ -37,7 +37,6 @@ import setting from '@hcengineering/setting' import { htmlToMarkup, markupToHTML } from '@hcengineering/text' import { deepEqual } from 'fast-equals' import type { Collection, Db } from 'mongodb' -import { CalendarController } from './calendarController' import type { CalendarHistory, DummyWatch, EventHistory, Token, User } from './types' import { encodeReccuring, isToken, parseRecurrenceStrings } from './utils' import type { WorkspaceClient } from './workspaceClient' @@ -105,7 +104,7 @@ export class CalendarClient { const calendarClient = new CalendarClient(user, mongo, client, workspace) if (isToken(user)) { await calendarClient.googleClient.init(user) - await calendarClient.addClient() + calendarClient.updateTimer() } return calendarClient } @@ -139,7 +138,7 @@ export class CalendarClient { } throw new Error('Not all scopes provided') } - await this.addClient() + this.updateTimer() const integrations = await this.client.findAll(setting.class.Integration, { createdBy: this.user.userId, @@ -228,17 +227,6 @@ export class CalendarClient { this.isClosed = true } - private async addClient (): Promise { - try { - const me = await this.googleClient.getMe() - const controller = CalendarController.getCalendarController() - controller.addClient(me, this) - this.updateTimer() - } catch (err) { - console.error('Add client error', this.user.workspace, this.user.userId, err) - } - } - // #region Calendars async syncCalendars (me: string): Promise { diff --git a/services/calendar/pod-calendar/src/calendarController.ts b/services/calendar/pod-calendar/src/calendarController.ts index 6ef093c324..9d7d15d16f 100644 --- a/services/calendar/pod-calendar/src/calendarController.ts +++ b/services/calendar/pod-calendar/src/calendarController.ts @@ -30,7 +30,6 @@ export class CalendarController { >() private readonly tokens: Collection - private readonly clients: Map = new Map() protected static _instance: CalendarController @@ -120,7 +119,7 @@ export class CalendarController { const workspaces = [...new Set(tokens.map((p) => p.workspace))] const infos = await getWorkspacesInfo(token, workspaces) for (const token of tokens) { - const info = infos.find((p) => p.workspace === token.workspace) + const info = infos.find((p) => p.workspaceId === token.workspace) if (info === undefined) { continue } @@ -147,28 +146,6 @@ export class CalendarController { await workspaceController.pushEvent(event, type) } - addClient (email: string, client: CalendarClient): void { - const clients = this.clients.get(email) - if (clients === undefined) { - this.clients.set(email, [client]) - } else { - clients.push(client) - this.clients.set(email, clients) - } - } - - removeClient (email: string): void { - const clients = this.clients.get(email) - if (clients !== undefined) { - const filtered = clients.filter((p) => !p.isClosed) - if (filtered.length === 0) { - this.clients.delete(email) - } else { - this.clients.set(email, filtered) - } - } - } - async getUserId (email: string, workspace: string): Promise> { const workspaceClient = await this.getWorkspaceClient(workspace) return await workspaceClient.getUserId(email) diff --git a/services/calendar/pod-calendar/src/workspaceClient.ts b/services/calendar/pod-calendar/src/workspaceClient.ts index c7531da5e9..ddb995e45c 100644 --- a/services/calendar/pod-calendar/src/workspaceClient.ts +++ b/services/calendar/pod-calendar/src/workspaceClient.ts @@ -160,7 +160,6 @@ export class WorkspaceClient { removeClient (email: string): void { this.clients.delete(email) - this.serviceController.removeClient(email) if (this.clients.size > 0) return void this.close() this.serviceController.removeWorkspace(this.workspace) From f98152986c18e2518580dd192fe655f0e726dc93 Mon Sep 17 00:00:00 2001 From: Alexander Onnikov Date: Mon, 24 Feb 2025 15:00:48 +0300 Subject: [PATCH 04/16] fix: filter props on component update (#8080) Signed-off-by: Alexander Onnikov --- packages/ui/src/components/Component.svelte | 6 +++--- .../src/components/AttachmentImagePreview.svelte | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/ui/src/components/Component.svelte b/packages/ui/src/components/Component.svelte index e83560e83e..4ea4888512 100644 --- a/packages/ui/src/components/Component.svelte +++ b/packages/ui/src/components/Component.svelte @@ -73,7 +73,7 @@ .then((res) => { if (current === counter) { Ctor = res - _props = props + _props = filterDefaultUndefined(props, props) loading = false } }) @@ -85,11 +85,11 @@ } else { loading = false Ctor = component - _props = props + _props = filterDefaultUndefined(props, props) } } else { Ctor = _is - _props = props + _props = filterDefaultUndefined(props, props) } } diff --git a/plugins/attachment-resources/src/components/AttachmentImagePreview.svelte b/plugins/attachment-resources/src/components/AttachmentImagePreview.svelte index 611003711b..ff29da46cc 100644 --- a/plugins/attachment-resources/src/components/AttachmentImagePreview.svelte +++ b/plugins/attachment-resources/src/components/AttachmentImagePreview.svelte @@ -43,7 +43,7 @@ $: urlSize = getUrlSize(size) function getDimensions (value: Attachment, size: AttachmentImageSize): Dimensions { - if (size === 'auto') { + if (size === 'auto' || size == null) { return { width: 'auto', height: 'auto', From c6e752c7aa21e30a820bb2f1dddfc7bda656a5d4 Mon Sep 17 00:00:00 2001 From: Victor Ilyushchenko Date: Mon, 24 Feb 2025 17:13:35 +0300 Subject: [PATCH 05/16] EQMS-1451/1453/1455: reference tweaks for QMS documents (#8083) * EQMS-1451/1453/1455: reference tweaks for QMS documents Signed-off-by: Victor Ilyushchenko * fmt Signed-off-by: Victor Ilyushchenko --------- Signed-off-by: Victor Ilyushchenko --- models/controlled-documents/src/index.ts | 14 +++- models/controlled-documents/src/plugin.ts | 6 ++ models/view/src/index.ts | 7 ++ .../controlled-documents-assets/lang/cs.json | 4 +- .../controlled-documents-assets/lang/de.json | 4 +- .../controlled-documents-assets/lang/en.json | 4 +- .../controlled-documents-assets/lang/fr.json | 4 +- .../controlled-documents-assets/lang/it.json | 4 +- .../controlled-documents-assets/lang/ru.json | 4 +- .../controlled-documents-assets/lang/zh.json | 4 +- .../components/document/EditDocContent.svelte | 11 ++- .../document/popups/ChangeDocCodePopup.svelte | 6 +- .../src/index.ts | 6 ++ .../src/plugin.ts | 5 +- .../src/utils.ts | 72 +++++++++++++++++-- plugins/controlled-documents/src/docutils.ts | 2 +- plugins/controlled-documents/src/utils.ts | 25 ++++--- .../src/components/MentionPopup.svelte | 9 +-- .../src/components/extension/reference.ts | 22 +++++- plugins/view/src/index.ts | 2 + plugins/view/src/types.ts | 7 ++ .../src/index.ts | 13 ++-- 22 files changed, 195 insertions(+), 40 deletions(-) diff --git a/models/controlled-documents/src/index.ts b/models/controlled-documents/src/index.ts index 1b15efadc0..aaf04d5ed4 100644 --- a/models/controlled-documents/src/index.ts +++ b/models/controlled-documents/src/index.ts @@ -106,7 +106,7 @@ export function createModel (builder: Builder): void { }) builder.mixin(documents.class.DocumentMeta, core.class.Class, view.mixin.ObjectTitle, { - titleProvider: documents.function.ControlledDocumentTitleProvider + titleProvider: documents.function.DocumentMetaTitleProvider }) builder.mixin(documents.class.DocumentApprovalRequest, core.class.Class, view.mixin.ObjectPresenter, { @@ -406,6 +406,10 @@ export function createModel (builder: Builder): void { presenter: documents.component.DocumentPresenter }) + builder.mixin(documents.class.ControlledDocument, core.class.Class, view.mixin.LinkProvider, { + encode: documents.function.GetControlledDocumentLinkFragment + }) + builder.mixin(documents.class.Document, core.class.Class, view.mixin.IgnoreActions, { actions: [view.action.Delete] }) @@ -741,6 +745,14 @@ export function createModel (builder: Builder): void { provider: documents.function.DocumentIdentifierProvider }) + builder.mixin(documents.class.ControlledDocument, core.class.Class, view.mixin.ReferenceObjectProvider, { + provider: documents.function.ControlledDocumentReferenceObjectProvider + }) + + builder.mixin(documents.class.ProjectDocument, core.class.Class, view.mixin.ReferenceObjectProvider, { + provider: documents.function.ProjectDocumentReferenceObjectProvider + }) + createAction( builder, { diff --git a/models/controlled-documents/src/plugin.ts b/models/controlled-documents/src/plugin.ts index fdde0039ed..e0b5eca013 100644 --- a/models/controlled-documents/src/plugin.ts +++ b/models/controlled-documents/src/plugin.ts @@ -53,6 +53,12 @@ export default mergeIds(documentsId, documents, { DocumentIdentifierProvider: '' as Resource< (client: Client, ref: Ref, doc?: T) => Promise >, + ControlledDocumentReferenceObjectProvider: '' as Resource< + (client: Client, ref: Ref, doc?: T) => Promise + >, + ProjectDocumentReferenceObjectProvider: '' as Resource< + (client: Client, ref: Ref, doc?: T) => Promise + >, Comment: '' as Resource, IsCommentVisible: '' as Resource }, diff --git a/models/view/src/index.ts b/models/view/src/index.ts index e83609c617..9c9b73d2b5 100644 --- a/models/view/src/index.ts +++ b/models/view/src/index.ts @@ -89,6 +89,7 @@ import { type ViewletDescriptor, type ViewletPreference, type ObjectIdentifier, + type ReferenceObjectProvider, type ObjectIcon, type ObjectTooltip, type AttrPresenter, @@ -253,6 +254,11 @@ export class TObjectIdentifier extends TClass implements ObjectIdentifier { provider!: Resource<(client: Client, ref: Ref, doc?: T) => Promise> } +@Mixin(view.mixin.ReferenceObjectProvider, core.class.Class) +export class TReferenceObjectProvider extends TClass implements ReferenceObjectProvider { + provider!: Resource<(client: Client, ref: Ref, doc?: T) => Promise> +} + @Mixin(view.mixin.ObjectTooltip, core.class.Class) export class TObjectTooltip extends TClass implements ObjectTooltip { provider!: Resource<(client: Client, doc?: Doc | null) => Promise> @@ -465,6 +471,7 @@ export function createModel (builder: Builder): void { TAggregation, TGroupping, TObjectIdentifier, + TReferenceObjectProvider, TObjectTooltip, TObjectIcon, TAttrPresenter, diff --git a/plugins/controlled-documents-assets/lang/cs.json b/plugins/controlled-documents-assets/lang/cs.json index 34f47aca5f..4241ef4869 100644 --- a/plugins/controlled-documents-assets/lang/cs.json +++ b/plugins/controlled-documents-assets/lang/cs.json @@ -300,7 +300,9 @@ "Obsolete": "Zastaralé", "MakeDocumentObsolete": "Označit jako zastaralé", "MakeDocumentObsoleteDialog": "Označit {count, plural, one {dokument jako zastaralý} other {dokumenty jako zastaralé}}", - "MakeDocumentObsoleteConfirm": "Opravdu chcete označit následující dokumenty jako zastaralé: {titles}?" + "MakeDocumentObsoleteConfirm": "Opravdu chcete označit následující dokumenty jako zastaralé: {titles}?", + + "LatestVersionHint": "nejnovější" }, "controlledDocStates": { "Empty": "", diff --git a/plugins/controlled-documents-assets/lang/de.json b/plugins/controlled-documents-assets/lang/de.json index 0653e4b4c0..f1866edaf9 100644 --- a/plugins/controlled-documents-assets/lang/de.json +++ b/plugins/controlled-documents-assets/lang/de.json @@ -307,7 +307,9 @@ "Obsolete": "Veraltet", "MakeDocumentObsolete": "Als veraltet markieren", "MakeDocumentObsoleteDialog": "{count, plural, one {Dokument als veraltet markieren} other {Dokumente als veraltet markieren}}", - "MakeDocumentObsoleteConfirm": "Möchten Sie die folgenden Dokumente wirklich als veraltet markieren: {titles}?" + "MakeDocumentObsoleteConfirm": "Möchten Sie die folgenden Dokumente wirklich als veraltet markieren: {titles}?", + + "LatestVersionHint": "neueste" }, "controlledDocStates": { "Empty": "", diff --git a/plugins/controlled-documents-assets/lang/en.json b/plugins/controlled-documents-assets/lang/en.json index dbd49df4b7..a98bc2d854 100644 --- a/plugins/controlled-documents-assets/lang/en.json +++ b/plugins/controlled-documents-assets/lang/en.json @@ -309,7 +309,9 @@ "CreateFolder": "Create new folder", "RenameFolder": "Rename folder", - "CreateChildFolder": "Create child folder" + "CreateChildFolder": "Create child folder", + + "LatestVersionHint": "latest" }, "controlledDocStates": { "Empty": "", diff --git a/plugins/controlled-documents-assets/lang/fr.json b/plugins/controlled-documents-assets/lang/fr.json index 9b2ee9fe3f..82b93dbd9b 100644 --- a/plugins/controlled-documents-assets/lang/fr.json +++ b/plugins/controlled-documents-assets/lang/fr.json @@ -267,7 +267,9 @@ "Obsolete": "Obsolète", "MakeDocumentObsolete": "Marquer comme obsolète", "MakeDocumentObsoleteDialog": "Marquer {count, plural, one {le document comme obsolète} other {les documents comme obsolètes}}", - "MakeDocumentObsoleteConfirm": "Voulez-vous vraiment marquer les documents suivants comme obsolètes : {titles} ?" + "MakeDocumentObsoleteConfirm": "Voulez-vous vraiment marquer les documents suivants comme obsolètes : {titles} ?", + + "LatestVersionHint": "dernier" }, "controlledDocStates": { "Empty": "", diff --git a/plugins/controlled-documents-assets/lang/it.json b/plugins/controlled-documents-assets/lang/it.json index 1ec19f9672..79b28decd0 100644 --- a/plugins/controlled-documents-assets/lang/it.json +++ b/plugins/controlled-documents-assets/lang/it.json @@ -265,7 +265,9 @@ "Obsolete": "Obsoleto", "MakeDocumentObsolete": "Segna come obsoleto", "MakeDocumentObsoleteDialog": "Segna {count, plural, one {il documento come obsoleto} other {i documenti come obsoleti}}", - "MakeDocumentObsoleteConfirm": "Vuoi davvero segnare i seguenti documenti come obsoleti: {titles}?" + "MakeDocumentObsoleteConfirm": "Vuoi davvero segnare i seguenti documenti come obsoleti: {titles}?", + + "LatestVersionHint": "ultimo" }, "controlledDocStates": { "Empty": "", diff --git a/plugins/controlled-documents-assets/lang/ru.json b/plugins/controlled-documents-assets/lang/ru.json index dd99112ec0..37fc54813b 100644 --- a/plugins/controlled-documents-assets/lang/ru.json +++ b/plugins/controlled-documents-assets/lang/ru.json @@ -309,7 +309,9 @@ "Obsolete": "Устаревший", "MakeDocumentObsolete": "Пометить как устаревшее", "MakeDocumentObsoleteDialog": "Пометить {count, plural, one {документ как устаревший} other {документы как устаревшие}}", - "MakeDocumentObsoleteConfirm": "Вы действительно хотите пометить следующие документы как устаревшие: {titles}?" + "MakeDocumentObsoleteConfirm": "Вы действительно хотите пометить следующие документы как устаревшие: {titles}?", + + "LatestVersionHint": "последняя" }, "controlledDocStates": { "Empty": "", diff --git a/plugins/controlled-documents-assets/lang/zh.json b/plugins/controlled-documents-assets/lang/zh.json index d0b07eddbe..7b4c2f584e 100644 --- a/plugins/controlled-documents-assets/lang/zh.json +++ b/plugins/controlled-documents-assets/lang/zh.json @@ -306,7 +306,9 @@ "Obsolete": "已过时", "MakeDocumentObsolete": "标记为过时", "MakeDocumentObsoleteDialog": "标记 {count, plural, one {文档为过时} other {文档为过时}}", - "MakeDocumentObsoleteConfirm": "您确定要将以下文档标记为过时吗：{titles}？" + "MakeDocumentObsoleteConfirm": "您确定要将以下文档标记为过时吗：{titles}？", + + "LatestVersionHint": "最新" }, "controlledDocStates": { "Empty": "", diff --git a/plugins/controlled-documents-resources/src/components/document/EditDocContent.svelte b/plugins/controlled-documents-resources/src/components/document/EditDocContent.svelte index 5e404a3980..fe2e87ccfd 100644 --- a/plugins/controlled-documents-resources/src/components/document/EditDocContent.svelte +++ b/plugins/controlled-documents-resources/src/components/document/EditDocContent.svelte @@ -49,10 +49,12 @@ $documentComments as documentComments, documentCommentsDisplayRequested, documentCommentsHighlightUpdated, - documentCommentsLocationNavigateRequested + documentCommentsLocationNavigateRequested, + $documentReleasedVersions as documentReleasedVersions } from '../../stores/editors/document' import DocumentTitle from './DocumentTitle.svelte' import DocumentPrintTitlePage from '../print/DocumentPrintTitlePage.svelte' + import { syncDocumentMetaTitle } from '../../utils' const client = getClient() const hierarchy = client.getHierarchy() @@ -111,14 +113,17 @@ unsubscribeNavigateToLocation() }) - const handleUpdateTitle = () => { + const handleUpdateTitle = async () => { if (!$controlledDocument || !title) { return } const titleTrimmed = title.trim() if (titleTrimmed.length > 0 && titleTrimmed !== $controlledDocument.title) { - client.update($controlledDocument, { title: titleTrimmed }) + await client.update($controlledDocument, { title: titleTrimmed }) + if ($documentReleasedVersions.length === 0 && $controlledDocument.state === DocumentState.Draft) { + await syncDocumentMetaTitle(client, $controlledDocument.attachedTo, $controlledDocument.code, titleTrimmed) + } } } diff --git a/plugins/controlled-documents-resources/src/components/document/popups/ChangeDocCodePopup.svelte b/plugins/controlled-documents-resources/src/components/document/popups/ChangeDocCodePopup.svelte index 8459e97d04..aeea262934 100644 --- a/plugins/controlled-documents-resources/src/components/document/popups/ChangeDocCodePopup.svelte +++ b/plugins/controlled-documents-resources/src/components/document/popups/ChangeDocCodePopup.svelte @@ -15,14 +15,15 @@ diff --git a/plugins/controlled-documents-resources/src/index.ts b/plugins/controlled-documents-resources/src/index.ts index 40d379be40..e7d32d0c4b 100644 --- a/plugins/controlled-documents-resources/src/index.ts +++ b/plugins/controlled-documents-resources/src/index.ts @@ -106,7 +106,10 @@ import { createTemplate, deleteFolder, documentIdentifierProvider, + controlledDocumentReferenceObjectProvider, + projectDocumentReferenceObjectProvider, getAllDocumentStates, + getControlledDocumentLinkFragment, getControlledDocumentTitle, getDocumentMetaLinkFragment, getDocumentMetaTitle, @@ -450,6 +453,7 @@ export default async (): Promise => ({ DocumentStateSort: sortDocumentStates, GetAllDocumentStates: getAllDocumentStates, GetDocumentMetaLinkFragment: getDocumentMetaLinkFragment, + GetControlledDocumentLinkFragment: getControlledDocumentLinkFragment, CanDeleteDocument: canDeleteDocument, CanArchiveDocument: canArchiveDocument, CanMakeDocumentObsolete: canMakeDocumentObsolete, @@ -457,6 +461,8 @@ export default async (): Promise => ({ CanOpenDocument: canOpenDocument, CanPrintDocument: canPrintDocument, DocumentIdentifierProvider: documentIdentifierProvider, + ControlledDocumentReferenceObjectProvider: controlledDocumentReferenceObjectProvider, + ProjectDocumentReferenceObjectProvider: projectDocumentReferenceObjectProvider, ControlledDocumentTitleProvider: getControlledDocumentTitle, DocumentMetaTitleProvider: getDocumentMetaTitle, Comment: comment, diff --git a/plugins/controlled-documents-resources/src/plugin.ts b/plugins/controlled-documents-resources/src/plugin.ts index 85a569dad6..04a794ca23 100644 --- a/plugins/controlled-documents-resources/src/plugin.ts +++ b/plugins/controlled-documents-resources/src/plugin.ts @@ -212,7 +212,9 @@ export default mergeIds(documentsId, documents, { CreateDocumentFailed: '' as IntlString, CreateDocumentTemplateFailed: '' as IntlString, - TryAgain: '' as IntlString + TryAgain: '' as IntlString, + + LatestVersionHint: '' as IntlString }, controlledDocStates: { Empty: '' as IntlString, @@ -240,6 +242,7 @@ export default mergeIds(documentsId, documents, { GetAllDocumentStates: '' as Resource<() => Promise>, GetVisibleFilters: '' as Resource<(filters: KeyFilter[], space?: Ref) => Promise>, GetDocumentMetaLinkFragment: '' as Resource<(doc: Doc, props: Record) => Promise>, + GetControlledDocumentLinkFragment: '' as Resource<(doc: Doc, props: Record) => Promise>, CanDeleteDocument: '' as Resource<(doc?: Doc | Doc[]) => Promise>, CanArchiveDocument: '' as Resource<(doc?: Doc | Doc[]) => Promise>, CanMakeDocumentObsolete: '' as Resource<(doc?: Doc | Doc[]) => Promise>, diff --git a/plugins/controlled-documents-resources/src/utils.ts b/plugins/controlled-documents-resources/src/utils.ts index d238e18a3b..bd411abe75 100644 --- a/plugins/controlled-documents-resources/src/utils.ts +++ b/plugins/controlled-documents-resources/src/utils.ts @@ -16,6 +16,7 @@ import { type Employee, type Person, type PersonAccount } from '@hcengineering/c import documents, { type ControlledDocument, type Document, + type DocumentBundle, type DocumentCategory, type DocumentComment, type DocumentMeta, @@ -27,12 +28,12 @@ import documents, { type ProjectDocument, type ProjectMeta, ControlledDocumentState, - type DocumentBundle, DocumentState, + ProjectDocumentTree, + compareDocumentVersions, emptyBundle, getDocumentName, - getFirstRank, - ProjectDocumentTree + getFirstRank } from '@hcengineering/controlled-documents' import core, { type Class, @@ -195,6 +196,18 @@ export async function getDocumentMetaLinkFragment (document: Doc): Promise { + const client = getClient() + const targetDocument = await client.findOne(documents.class.ProjectDocument, { document: document._id }) + + if (targetDocument === undefined) { + throw new Error('Cannot resolve a ProjectDocument for document ' + document._id) + } + + const project = targetDocument.project ?? documents.ids.NoProject + return getProjectDocumentLink(document, project) +} + export interface TeamPopupData { controlledDoc: ControlledDocument requestClass: Ref> @@ -707,6 +720,41 @@ export async function documentIdentifierProvider (client: Client, ref: Ref, + doc?: ControlledDocument +): Promise { + const document = doc ?? (await client.findOne(documents.class.ControlledDocument, { _id: ref })) + if (document === undefined) return + + const meta = await client.findOne(documents.class.DocumentMeta, { _id: document.attachedTo }) + if (meta === undefined) return + + let documentSeq: ControlledDocument[] = await client.findAll(documents.class.ControlledDocument, { + attachedTo: meta._id + }) + + const allowStates = [DocumentState.Draft, DocumentState.Effective] + documentSeq = documentSeq.filter((d) => allowStates.includes(d.state)).sort(compareDocumentVersions) + + const effIndex = documentSeq.findIndex((d) => d.state === DocumentState.Effective) + const docIndex = documentSeq.findIndex((d) => d._id === document._id) + + return docIndex >= 0 && (docIndex <= effIndex || effIndex < 0) ? meta : document +} + +export async function projectDocumentReferenceObjectProvider ( + client: Client, + ref: Ref, + doc?: ProjectDocument +): Promise { + const prjdoc = doc ?? (await client.findOne(documents.class.ProjectDocument, { _id: ref })) + if (prjdoc === undefined) return + + return await controlledDocumentReferenceObjectProvider(client, prjdoc.document as Ref) +} + export function documentCompareFn (doc1: Document, doc2: Document): number { return doc1.major - doc2.major !== 0 ? doc1.major - doc2.major : doc1.minor - doc2.minor } @@ -724,7 +772,7 @@ export async function getControlledDocumentTitle ( if (object === undefined) return '' - return object.title + return object.title + ` (${getDocumentVersionString(object)})` } export async function getDocumentMetaTitle ( @@ -736,7 +784,9 @@ export async function getDocumentMetaTitle ( if (object === undefined) return '' - return object.title + const hint = await translate(documentsResources.string.LatestVersionHint, {}) + + return object.title + ` (${hint})` } export const getCurrentEmployee = (): Ref | undefined => { @@ -932,3 +982,15 @@ export class DocumentHiearchyQuery { export function createDocumentHierarchyQuery (): DocumentHiearchyQuery { return new DocumentHiearchyQuery() } + +export async function syncDocumentMetaTitle ( + client: Client & TxOperations, + _id: Ref, + code: string, + title: string +): Promise { + const meta = await client.findOne(documents.class.DocumentMeta, { _id }) + if (meta !== undefined) { + await client.update(meta, { title: `${code} ${title}` }) + } +} diff --git a/plugins/controlled-documents/src/docutils.ts b/plugins/controlled-documents/src/docutils.ts index d1346c0356..9a13d9fb9f 100644 --- a/plugins/controlled-documents/src/docutils.ts +++ b/plugins/controlled-documents/src/docutils.ts @@ -166,7 +166,7 @@ export async function createControlledDocMetadata ( space, { documents: 0, - title: `${prefix}-${seqNumber} ${specTitle}` + title: `${specCode} ${specTitle}` }, metaId ) diff --git a/plugins/controlled-documents/src/utils.ts b/plugins/controlled-documents/src/utils.ts index 0e093632e1..f080cf494f 100644 --- a/plugins/controlled-documents/src/utils.ts +++ b/plugins/controlled-documents/src/utils.ts @@ -156,17 +156,25 @@ export function isFolder (doc: ProjectDocument | undefined): boolean { return doc !== undefined && doc.document === documents.ids.Folder } +function getDocumentSortSequence (doc: ControlledDocument | undefined): number[] { + return doc !== undefined ? [doc.seqNumber, doc.major, doc.minor, doc.createdOn ?? 0] : [0, 0, 0, 0] +} + +export function compareDocumentVersions ( + doc1: ControlledDocument | undefined, + doc2: ControlledDocument | undefined +): number { + const s0 = getDocumentSortSequence(doc1) + const s1 = getDocumentSortSequence(doc2) + return s0.reduce((r, v, i) => (r !== 0 ? r : s1[i] - v), 0) +} + function extractPresentableStateFromDocumentBundle (bundle: DocumentBundle, prjmeta: ProjectMeta): DocumentBundle { bundle = { ...bundle } const person = getCurrentAccount().person as Ref const documentById = toIdMap(bundle.ControlledDocument) - const getSortSequence = (prjdoc: ProjectDocument): number[] => { - const doc = documentById.get(prjdoc.document as Ref) - return doc !== undefined ? [doc.seqNumber, doc.major, doc.minor, doc.createdOn ?? 0] : [0, 0, 0, 0] - } - const prjdoc = bundle.ProjectDocument.filter((prjdoc) => { if (prjdoc.attachedTo !== prjmeta._id) return false if (isFolder(prjdoc)) return true @@ -174,9 +182,10 @@ function extractPresentableStateFromDocumentBundle (bundle: DocumentBundle, prjm const isPublicState = doc?.state === DocumentState.Effective || doc?.state === DocumentState.Archived return doc !== undefined && (isPublicState || isCollaborator(doc, person)) }).sort((a, b) => { - const s0 = getSortSequence(a) - const s1 = getSortSequence(b) - return s0.reduce((r, v, i) => (r !== 0 ? r : s1[i] - v), 0) + return compareDocumentVersions( + documentById.get(a.document as Ref), + documentById.get(b.document as Ref) + ) })[0] const doc = prjdoc !== undefined ? documentById.get(prjdoc.document as Ref) : undefined diff --git a/plugins/text-editor-resources/src/components/MentionPopup.svelte b/plugins/text-editor-resources/src/components/MentionPopup.svelte index c43b03241d..f57ea3e9dd 100644 --- a/plugins/text-editor-resources/src/components/MentionPopup.svelte +++ b/plugins/text-editor-resources/src/components/MentionPopup.svelte @@ -18,7 +18,7 @@ import presentation, { SearchResult, reduceCalls, searchFor, type SearchItem } from '@hcengineering/presentation' import { Label, ListView, resizeObserver } from '@hcengineering/ui' import { createEventDispatcher } from 'svelte' - import { getReferenceLabel } from './extension/reference' + import { getReferenceLabel, getReferenceObject } from './extension/reference' export let query: string = '' @@ -31,11 +31,12 @@ let selection = 0 async function handleSelectItem (item: SearchResultDoc): Promise { - const label = await getReferenceLabel(item.doc._class, item.doc._id) + const obj = (await getReferenceObject(item.doc._class, item.doc._id)) ?? item.doc + const label = await getReferenceLabel(obj._class, obj._id) dispatch('close', { - id: item.id, + id: obj._id, label, - objectclass: item.doc._class + objectclass: obj._class }) } diff --git a/plugins/text-editor-resources/src/components/extension/reference.ts b/plugins/text-editor-resources/src/components/extension/reference.ts index 8c1b69165a..f66c9fa3cf 100644 --- a/plugins/text-editor-resources/src/components/extension/reference.ts +++ b/plugins/text-editor-resources/src/components/extension/reference.ts @@ -344,10 +344,30 @@ export async function getReferenceLabel ( return label } +export async function getReferenceObject ( + objectclass: Ref>, + id: Ref, + doc?: T +): Promise { + const client = getClient() + const hierarchy = client.getHierarchy() + + const referenceObjectProvider = hierarchy.classHierarchyMixin( + objectclass as Ref>, + view.mixin.ReferenceObjectProvider + ) + const referenceObjectProviderFn = + referenceObjectProvider !== undefined ? await getResource(referenceObjectProvider.provider) : undefined + + return await referenceObjectProviderFn?.(client, id, doc) +} + export async function getReferenceFromUrl (urlString: string): Promise { - const target = await getTargetObjectFromUrl(urlString) + let target = await getTargetObjectFromUrl(urlString) if (target === undefined) return + target = (await getReferenceObject(target._class, target._id)) ?? target + const label = await getReferenceLabel(target._class, target._id) if (label === '') return diff --git a/plugins/view/src/index.ts b/plugins/view/src/index.ts index 5cee522138..2b567833ac 100644 --- a/plugins/view/src/index.ts +++ b/plugins/view/src/index.ts @@ -52,6 +52,7 @@ import { ObjectTitle, ObjectTooltip, ObjectValidator, + ReferenceObjectProvider, AttrPresenter, PreviewPresenter, SpaceHeader, @@ -96,6 +97,7 @@ const view = plugin(viewId, { ObjectFactory: '' as Ref>, ObjectTitle: '' as Ref>, ObjectIdentifier: '' as Ref>, + ReferenceObjectProvider: '' as Ref>, ObjectTooltip: '' as Ref>, SpaceHeader: '' as Ref>, SpaceName: '' as Ref>, diff --git a/plugins/view/src/types.ts b/plugins/view/src/types.ts index b77614f365..063be94678 100644 --- a/plugins/view/src/types.ts +++ b/plugins/view/src/types.ts @@ -290,6 +290,13 @@ export interface ObjectIdentifier extends Class { provider: Resource<(client: Client, ref: Ref, doc?: T) => Promise> } +/** + * @public + */ +export interface ReferenceObjectProvider extends Class { + provider: Resource<(client: Client, ref: Ref, doc?: T) => Promise> +} + /** * @public */ diff --git a/server-plugins/controlled-documents-resources/src/index.ts b/server-plugins/controlled-documents-resources/src/index.ts index 2349ca95ac..a078023fb2 100644 --- a/server-plugins/controlled-documents-resources/src/index.ts +++ b/server-plugins/controlled-documents-resources/src/index.ts @@ -9,31 +9,30 @@ import documents, { DocumentApprovalRequest, DocumentState, DocumentTemplate, - getDocumentId, getEffectiveDocUpdate, type DocumentRequest, type DocumentTraining } from '@hcengineering/controlled-documents' import core, { AccountRole, + combineAttributes, + Doc, DocumentQuery, Ref, SortingOrder, Tx, TxCreateDoc, + TxCUD, TxFactory, TxUpdateDoc, type Account, type RolesAssignment, - type Timestamp, - Doc, - combineAttributes, - TxCUD + type Timestamp } from '@hcengineering/core' +import { NotificationType } from '@hcengineering/notification' import { RequestStatus } from '@hcengineering/request' import { TriggerControl } from '@hcengineering/server-core' import training, { TrainingState, type TrainingRequest } from '@hcengineering/training' -import { NotificationType } from '@hcengineering/notification' async function getDocs ( control: TriggerControl, @@ -77,7 +76,7 @@ function archiveDocs (docs: ControlledDocument[], txFactory: TxFactory): Tx[] { function updateMeta (doc: ControlledDocument, txFactory: TxFactory): Tx[] { return [ txFactory.createTxUpdateDoc(doc.attachedToClass, doc.space, doc.attachedTo, { - title: `${getDocumentId(doc)} ${doc.title}` + title: `${doc.code} ${doc.title}` }) ] } From dcbeb1e04318c1481326cfaf68c7b9b60a5c0ff2 Mon Sep 17 00:00:00 2001 From: Andrey Sobolev Date: Tue, 25 Feb 2025 03:15:45 +0700 Subject: [PATCH 06/16] UBERF-9511: Allow to unarchive workspace by user request (#8084) Signed-off-by: Andrey Sobolev --- plugins/login-assets/lang/cs.json | 3 +- plugins/login-assets/lang/de.json | 3 +- plugins/login-assets/lang/en.json | 3 +- plugins/login-assets/lang/es.json | 3 +- plugins/login-assets/lang/fr.json | 3 +- plugins/login-assets/lang/it.json | 3 +- plugins/login-assets/lang/pt.json | 3 +- plugins/login-assets/lang/ru.json | 3 +- plugins/login-assets/lang/zh.json | 3 +- .../src/components/SelectWorkspace.svelte | 24 ++++++++++---- plugins/login-resources/src/index.ts | 29 +++++------------ plugins/login-resources/src/utils.ts | 32 +++++++++++++++++++ plugins/login/src/index.ts | 20 +++++++++++- plugins/workbench-resources/src/connect.ts | 12 ++++--- server/account/src/operations.ts | 4 ++- .../sanity/tests/workspace/archive.spec.ts | 6 ++-- 16 files changed, 109 insertions(+), 45 deletions(-) diff --git a/plugins/login-assets/lang/cs.json b/plugins/login-assets/lang/cs.json index 9babbe924d..afcb704c87 100644 --- a/plugins/login-assets/lang/cs.json +++ b/plugins/login-assets/lang/cs.json @@ -63,6 +63,7 @@ "PasswordMinDigits": "{count, plural, =1 {Heslo musí obsahovat alespoň # číslo} other {Heslo musí obsahovat alespoň # čísel}}", "PasswordMinUpperChars": "{count, plural, =1 {Heslo musí obsahovat alespoň # velké písmeno} other {Heslo musí obsahovat alespoň # velkých písmen}}", "PasswordMinLowerChars": "{count, plural, =1 {Heslo musí obsahovat alespoň # malé písmeno} other {Heslo musí obsahovat alespoň # malých písmen}}", - "WorkspaceIsArchived": "Pracovní prostor je archivován kvůli nečinnosti. Kontaktujte nás prosím pro obnovení..." + "WorkspaceIsArchived": "Pracovní prostor je archivován kvůli nečinnosti.", + "RestoreArchivedWorkspace": "Obnovit pracovní prostor." } } diff --git a/plugins/login-assets/lang/de.json b/plugins/login-assets/lang/de.json index 9d81a390b4..0131a5604c 100644 --- a/plugins/login-assets/lang/de.json +++ b/plugins/login-assets/lang/de.json @@ -63,6 +63,7 @@ "PasswordMinDigits": "{count, plural, =1 {Das Passwort muss mindestens # Zahl enthalten} other {Das Passwort muss mindestens # Zahlen enthalten}}", "PasswordMinUpperChars": "{count, plural, =1 {Das Passwort muss mindestens # Großbuchstaben enthalten} other {Das Passwort muss mindestens # Großbuchstaben enthalten}}", "PasswordMinLowerChars": "{count, plural, =1 {Das Passwort muss mindestens # Kleinbuchstaben enthalten} other {Das Passwort muss mindestens # Kleinbuchstaben enthalten}}", - "WorkspaceArchivedDesc": "Workspace wurde wegen Inaktivität archiviert. Bitte kontaktieren Sie uns zur Wiederherstellung..." + "WorkspaceArchivedDesc": "Workspace wurde wegen Inaktivität archiviert.", + "RestoreArchivedWorkspace": "Workspace wiederherstellen." } } diff --git a/plugins/login-assets/lang/en.json b/plugins/login-assets/lang/en.json index 5906d2ef6a..61e56429e1 100644 --- a/plugins/login-assets/lang/en.json +++ b/plugins/login-assets/lang/en.json @@ -63,6 +63,7 @@ "PasswordMinDigits": "{count, plural, =1 {Password must contain at least # number} other {Password must contain at least # numbers}}", "PasswordMinUpperChars": "{count, plural, =1 {Password must contain at least # uppercase letter} other {Password must contain at least # uppercase letters}}", "PasswordMinLowerChars": "{count, plural, =1 {Password must contain at least # lowercase letter} other {Password must contain at least # lowercase letters}}", - "WorkspaceArchivedDesc": "Workspace is archived because of being unused, Please contact us to restore..." + "WorkspaceArchivedDesc": "Workspace is archived because of being unused.", + "RestoreArchivedWorkspace": "Unarchive" } } diff --git a/plugins/login-assets/lang/es.json b/plugins/login-assets/lang/es.json index b05bfece33..19c9027562 100644 --- a/plugins/login-assets/lang/es.json +++ b/plugins/login-assets/lang/es.json @@ -63,6 +63,7 @@ "PasswordMinDigits": "{count, plural, =1 {La contraseña debe contener al menos # número} other {La contraseña debe contener al menos # números}}", "PasswordMinUpperChars": "{count, plural, =1 {La contraseña debe contener al menos # letra mayúscula} other {La contraseña debe contener al menos # letras mayúsculas}}", "PasswordMinLowerChars": "{count, plural, =1 {La contraseña debe contener al menos # letra minúscula} other {La contraseña debe contener al menos # letras minúsculas}}", - "WorkspaceArchivedDesc": "El espacio de trabajo está archivado por no estar en uso, por favor contáctenos para restaurarlo..." + "WorkspaceArchivedDesc": "El espacio de trabajo está archivado por no estar en uso.", + "RestoreArchivedWorkspace": "Restaurar" } } diff --git a/plugins/login-assets/lang/fr.json b/plugins/login-assets/lang/fr.json index 34e57a088c..70af0bdd34 100644 --- a/plugins/login-assets/lang/fr.json +++ b/plugins/login-assets/lang/fr.json @@ -63,6 +63,7 @@ "PasswordMinDigits": "{count, plural, =1 {Le mot de passe doit contenir au moins # chiffre} other {Le mot de passe doit contenir au moins # chiffres}}", "PasswordMinUpperChars": "{count, plural, =1 {Le mot de passe doit contenir au moins # lettre majuscule} other {Le mot de passe doit contenir au moins # lettres majuscules}}", "PasswordMinLowerChars": "{count, plural, =1 {Le mot de passe doit contenir au moins # lettre minuscule} other {Le mot de passe doit contenir au moins # lettres minuscules}}", - "WorkspaceArchivedDesc": "L'espace de travail est archivé en raison de son inactivité, veuillez nous contacter pour le restaurer..." + "WorkspaceArchivedDesc": "L'espace de travail est archivé en raison de son inactivité.", + "RestoreArchivedWorkspace": "Restaurer" } } diff --git a/plugins/login-assets/lang/it.json b/plugins/login-assets/lang/it.json index 34b72b087f..21958dd1c9 100644 --- a/plugins/login-assets/lang/it.json +++ b/plugins/login-assets/lang/it.json @@ -63,6 +63,7 @@ "PasswordMinDigits": "{count, plural, =1 {La password deve contenere almeno # numero} other {La password deve contenere almeno # numeri}}", "PasswordMinUpperChars": "{count, plural, =1 {La password deve contenere almeno # lettera maiuscola} other {La password deve contenere almeno # lettere maiuscole}}", "PasswordMinLowerChars": "{count, plural, =1 {La password deve contenere almeno # lettera minuscola} other {La password deve contenere almeno # lettere minuscole}}", - "WorkspaceArchivedDesc": "Il workspace è stato archiviato perché inutilizzato. Si prega di contattarci per ripristinarlo..." + "WorkspaceArchivedDesc": "Il workspace è stato archiviato perché inutilizzato.", + "RestoreArchivedWorkspace": "Un archiviato workspace" } } diff --git a/plugins/login-assets/lang/pt.json b/plugins/login-assets/lang/pt.json index b8df9dab0e..5ab6aa8b0b 100644 --- a/plugins/login-assets/lang/pt.json +++ b/plugins/login-assets/lang/pt.json @@ -63,6 +63,7 @@ "PasswordMinDigits": "{count, plural, =1 {A senha deve conter pelo menos # número} other {A senha deve conter pelo menos # números}}", "PasswordMinUpperChars": "{count, plural, =1 {A senha deve conter pelo menos # letra maiúscula} other {A senha deve conter pelo menos # letras maiúsculas}}", "PasswordMinLowerChars": "{count, plural, =1 {A senha deve conter pelo menos # letra minúscula} other {A senha deve conter pelo menos # letras minúsculas}}", - "WorkspaceArchivedDesc": "O espaço de trabalho está arquivado por estar inativo, por favor, entre em contato conosco para restaurá-lo..." + "WorkspaceArchivedDesc": "O espaço de trabalho está arquivado por estar inativo.", + "RestoreArchivedWorkspace": "Restaurar" } } diff --git a/plugins/login-assets/lang/ru.json b/plugins/login-assets/lang/ru.json index 0143a02b8e..811bea91ce 100644 --- a/plugins/login-assets/lang/ru.json +++ b/plugins/login-assets/lang/ru.json @@ -63,6 +63,7 @@ "PasswordMinDigits": "{count, plural, =1 {Пароль должен содержать минимум # цифру} other {Пароль должен содержать минимум # цифр}}", "PasswordMinUpperChars": "{count, plural, =1 {Пароль должен содержать минимум # заглавную букву} other {Пароль должен содержать минимум # заглавных букв}}", "PasswordMinLowerChars": "{count, plural, =1 {Пароль должен содержать минимум # строчную букву} other {Пароль должен содержать минимум # строчных букв}}", - "WorkspaceArchivedDesc": "Рабочее пространство архивировано из-за неиспользования, пожалуйста, свяжитесь с нами для восстановления..." + "WorkspaceArchivedDesc": "Рабочее пространство архивировано из-за неиспользования.", + "RestoreArchivedWorkspace": "Восстановить" } } diff --git a/plugins/login-assets/lang/zh.json b/plugins/login-assets/lang/zh.json index 4d6f23bfd0..4f14eb0e96 100644 --- a/plugins/login-assets/lang/zh.json +++ b/plugins/login-assets/lang/zh.json @@ -63,6 +63,7 @@ "PasswordMinDigits": "{count, plural, =1 {密码至少需要 # 个数字} other {密码至少需要 # 个数字}}", "PasswordMinUpperChars": "{count, plural, =1 {密码至少需要 # 个大写字母} other {密码至少需要 # 个大写字母}}", "PasswordMinLowerChars": "{count, plural, =1 {密码至少需要 # 个小写字母} other {密码至少需要 # 个小写字母}}", - "WorkspaceArchivedDesc": "工作区因未使用而归档，请与我们联系以恢复..." + "WorkspaceArchivedDesc": "工作区已被归档，因为未使用。", + "RestoreArchivedWorkspace": "解包" } } diff --git a/plugins/login-resources/src/components/SelectWorkspace.svelte b/plugins/login-resources/src/components/SelectWorkspace.svelte index 099da5239c..2d28355ec4 100644 --- a/plugins/login-resources/src/components/SelectWorkspace.svelte +++ b/plugins/login-resources/src/components/SelectWorkspace.svelte @@ -14,7 +14,7 @@ // limitations under the License. -->
- {#if !readonly} + {#if !readonly && allowToCreate}
diff --git a/plugins/view-resources/src/components/RelationsEditor.svelte b/plugins/view-resources/src/components/RelationsEditor.svelte index a4fe577bdf..7be498a3aa 100644 --- a/plugins/view-resources/src/components/RelationsEditor.svelte +++ b/plugins/view-resources/src/components/RelationsEditor.svelte @@ -52,7 +52,7 @@ (res) => { relationsA = res?.[0]?.$associations ?? {} }, - { associations: associationsA.map((a) => [a._id, 1]) } + { associations: associationsA.map((a) => [a._id, -1]) } ) const queryB = createQuery() @@ -62,7 +62,7 @@ (res) => { relationsB = res?.[0]?.$associations ?? {} }, - { associations: associationsB.map((a) => [a._id, -1]) } + { associations: associationsB.map((a) => [a._id, 1]) } ) diff --git a/server/postgres/src/storage.ts b/server/postgres/src/storage.ts index b6fa71ee4c..97c2207840 100644 --- a/server/postgres/src/storage.ts +++ b/server/postgres/src/storage.ts @@ -876,9 +876,13 @@ abstract class PostgresAdapterBase implements DbAdapter { } } } else if (column.startsWith('assoc_')) { + if (row[column] == null) continue const keys = column.split('_') const key = keys[keys.length - 1] - associations[key] = row[column] + const associationDomain = keys[1] + const associationSchema = getSchema(associationDomain) + const parsed = row[column].map((p: any) => parseDoc(p, associationSchema)) + associations[key] = parsed } else { joinIndex = undefined if (!map.has(row._id)) { @@ -1433,17 +1437,18 @@ abstract class PostgresAdapterBase implements DbAdapter { } const isReverse = association[1] === -1 const _class = isReverse ? assoc.classA : assoc.classB + const tagetDomain = translateDomain(this.hierarchy.getDomain(_class)) const keyA = isReverse ? 'docB' : 'docA' const keyB = isReverse ? 'docA' : 'docB' const wsId = vars.add(this.workspaceId.name, '::uuid') res.push( `(SELECT jsonb_agg(assoc.*) - FROM ${translateDomain(this.hierarchy.getDomain(_class))} AS assoc + FROM ${tagetDomain} AS assoc JOIN ${translateDomain(DOMAIN_RELATION)} as relation ON relation."${keyB}" = assoc."_id" AND relation."workspaceId" = ${wsId} WHERE relation."${keyA}" = ${translateDomain(baseDomain)}."_id" - AND assoc."workspaceId" = ${wsId}) AS assoc_${association[0]}` + AND assoc."workspaceId" = ${wsId}) AS assoc_${tagetDomain}_${association[0]}` ) } return res From c2f8eaad129a3bee3b2835c8ab165cdac31f8535 Mon Sep 17 00:00:00 2001 From: Victor Ilyushchenko Date: Wed, 26 Feb 2025 10:29:22 +0300 Subject: [PATCH 11/16] Added migration to remove QMS internal codes in DocumentMeta (#8085) * Added migration to remove QMS internal codes in DocumentMeta Signed-off-by: Victor Ilyushchenko * bump index for updated DocumentMetas Signed-off-by: Victor Ilyushchenko --------- Signed-off-by: Victor Ilyushchenko --- models/controlled-documents/src/migration.ts | 58 ++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/models/controlled-documents/src/migration.ts b/models/controlled-documents/src/migration.ts index a69e9e2f36..50124e62b4 100644 --- a/models/controlled-documents/src/migration.ts +++ b/models/controlled-documents/src/migration.ts @@ -26,6 +26,8 @@ import { type Class, type Data, type Doc, + type DocIndexState, + DOMAIN_DOC_INDEX_STATE, DOMAIN_SEQUENCE, DOMAIN_TX, generateId, @@ -51,6 +53,7 @@ import { DOMAIN_ATTACHMENT } from '@hcengineering/model-attachment' import core from '@hcengineering/model-core' import tags from '@hcengineering/tags' +import { compareDocumentVersions } from '@hcengineering/controlled-documents/src' import { makeRank } from '@hcengineering/rank' import documents, { DOMAIN_DOCUMENTS } from './index' @@ -404,6 +407,57 @@ async function migrateProjectMetaRank (client: MigrationClient): Promise { await client.bulk(DOMAIN_DOCUMENTS, operations) } +async function migrateDocumentMetaInternalCode (client: MigrationClient): Promise { + const docMetas = await client.find(DOMAIN_DOCUMENTS, { + _class: documents.class.DocumentMeta + }) + + let docs = await client.find(DOMAIN_DOCUMENTS, { + _class: documents.class.ControlledDocument + }) + + docs = docs.slice().sort(compareDocumentVersions).reverse() + const docMap = new Map, ControlledDocument>() + + for (const doc of docs) { + const curr = docMap.get(doc.attachedTo) + const metaId = doc.attachedTo + + const shouldBind = + curr === undefined || + doc.state === DocumentState.Effective || + (doc.state === DocumentState.Archived && curr.state !== DocumentState.Effective) + + if (shouldBind) docMap.set(metaId, doc) + } + + const operations: { filter: MigrationDocumentQuery, update: MigrateUpdate }[] = [] + const updatedIds = new Set>() + for (const meta of docMetas) { + const doc = docMap.get(meta._id) + if (doc === undefined) continue + + const title = `${doc.code} ${doc.title}` + if (meta.title === title) continue + + operations.push({ + filter: { _id: meta._id }, + update: { $set: { title } } + }) + updatedIds.add(meta._id) + } + + await client.bulk(DOMAIN_DOCUMENTS, operations) + await client.update( + DOMAIN_DOC_INDEX_STATE, + { + _id: { $in: Array.from(updatedIds) as any }, + objectClass: documents.class.DocumentMeta + }, + { $set: { needIndex: true } } + ) +} + export const documentsOperation: MigrateOperation = { async migrate (client: MigrationClient): Promise { await tryMigrate(client, documentsId, [ @@ -429,6 +483,10 @@ export const documentsOperation: MigrateOperation = { ) await client.move(DOMAIN_DOCUMENTS, { _class: core.class.Sequence }, DOMAIN_SEQUENCE) } + }, + { + state: 'migrateDocumentMetaInternalCode', + func: migrateDocumentMetaInternalCode } ]) }, From 9c460d63880d15c96938fcf04fc6fa4540e4f702 Mon Sep 17 00:00:00 2001 From: Andrey Sobolev Date: Wed, 26 Feb 2025 15:13:24 +0700 Subject: [PATCH 12/16] UBERF-9522: Fix memory backpressure (#8098) Signed-off-by: Andrey Sobolev --- .vscode/launch.json | 2 +- dev/tool/src/benchmark.ts | 105 +++++++++------ .../src/components/AdminWorkspaces.svelte | 14 +- pods/stats/src/stats.ts | 2 +- server/account-service/src/index.ts | 9 +- server/core/src/base.ts | 10 +- server/core/src/stats.ts | 17 ++- server/core/src/types.ts | 5 +- server/front/src/index.ts | 13 +- server/middleware/src/domainFind.ts | 8 +- server/middleware/src/queryJoin.ts | 122 ++++++++++-------- .../middleware/src/tests/queryJoiner.spec.ts | 9 +- server/server/src/blobs.ts | 9 +- server/server/src/client.ts | 4 +- server/server/src/sessionManager.ts | 33 +++-- server/server/src/stats.ts | 2 + server/server/src/utils.ts | 3 +- server/ws/src/rpc.ts | 6 +- server/ws/src/server_http.ts | 78 +++++++---- workers/transactor/src/transactor.ts | 10 +- ws-tests/tool.sh | 2 +- 21 files changed, 298 insertions(+), 165 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 77330c4067..d0977e7b7c 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -91,7 +91,7 @@ "FRONT_URL": "http://localhost:8083", "ACCOUNTS_URL": "http://localhost:3003", "MODEL_JSON": "${workspaceRoot}/models/all/bundle/model.json", - "MODEL_VERSION": "0.6.435", + "MODEL_VERSION": "0.6.436", "STATS_URL": "http://host.docker.internal:4901" }, "runtimeArgs": ["--nolazy", "-r", "ts-node/register"], diff --git a/dev/tool/src/benchmark.ts b/dev/tool/src/benchmark.ts index 7143746c0e..8d22dcb6e7 100644 --- a/dev/tool/src/benchmark.ts +++ b/dev/tool/src/benchmark.ts @@ -135,6 +135,9 @@ export async function benchmark ( } } }) + worker.on('error', (err) => { + console.error('worker error', err) + }) }) const m = newMetrics() @@ -147,6 +150,9 @@ export async function benchmark ( moment: number mem: number memTotal: number + memRSS: number + memFree: number + memArrays: number cpu: number requestTime: number operations: number @@ -158,6 +164,9 @@ export async function benchmark ( moment: 'Moment Time', mem: 'Mem', memTotal: 'Mem total', + memRSS: 'Mem RSS', + memFree: 'Mem Free', + memArrays: 'Mem Arrays', cpu: 'CPU', requestTime: 'Request time', operations: 'OPS', @@ -170,6 +179,9 @@ export async function benchmark ( let cpu: number = 0 let memUsed: number = 0 let memTotal: number = 0 + let memRSS: number = 0 + const memFree: number = 0 + let memArrays: number = 0 let elapsed = 0 let requestTime: number = 0 let operations = 0 @@ -204,6 +216,7 @@ export async function benchmark ( } } if (!found) { + console.log('no measurements found for path', path, p) return null } } @@ -211,47 +224,60 @@ export async function benchmark ( } let timer: any + let p: Promise | undefined if (isMainThread && monitorConnection !== undefined) { timer = setInterval(() => { - const st = Date.now() + const st = performance.now() try { - const fetchUrl = endpoint.replace('ws:/', 'http:/') + '/api/v1/statistics?token=' + token - void fetch(fetchUrl) - .then((res) => { - void res - .json() - .then((json) => { - memUsed = json.statistics.memoryUsed - memTotal = json.statistics.memoryTotal - cpu = json.statistics.cpuUsage - // operations = 0 - requestTime = 0 - // transfer = 0 - const r = extract( - json.metrics as Metrics, - '🧲 session', - 'client', - 'handleRequest', - 'process', - 'find-all' - ) - operations = (r?.operations ?? 0) - oldOperations - oldOperations = r?.operations ?? 0 - - requestTime = (r?.value ?? 0) / (((r?.operations as number) ?? 0) + 1) - - const tr = extract(json.metrics as Metrics, '🧲 session', '#send-data') - transfer = (tr?.value ?? 0) - oldTransfer - oldTransfer = tr?.value ?? 0 - }) - .catch((err) => { - console.log(err) - }) - }) - .catch((err) => { - console.log(err) + const fetchUrl = endpoint.replace('ws:/', 'http:/') + '/api/v1/statistics' + if (p === undefined) { + p = fetch(fetchUrl, { + headers: { + Authorization: 'Bearer ' + token + }, + keepalive: true }) + .then((res) => { + void res + .json() + .then((json) => { + memUsed = json.statistics.memoryUsed + memTotal = json.statistics.memoryTotal + memRSS = json.statistics.memoryRSS + memArrays = json.statistics.memoryArrayBuffers + cpu = json.statistics.cpuUsage + // operations = 0 + requestTime = 0 + // transfer = 0 + const r = extract( + json.metrics as Metrics, + '🧲 session', + 'client', + 'handleRequest', + 'process', + 'find-all' + ) + operations = (r?.operations ?? 0) - oldOperations + oldOperations = r?.operations ?? 0 + + requestTime = (r?.value ?? 0) / (((r?.operations as number) ?? 0) + 1) + + const tr = extract(json.metrics as Metrics, '🧲 session', 'client', '#send-data') + transfer = (tr?.value ?? 0) - oldTransfer + oldTransfer = tr?.value ?? 0 + p = undefined + }) + .catch((err) => { + console.log(err) + p = undefined + }) + }) + .catch((err) => { + console.log(err) + p = undefined + }) + } } catch (err) { console.log(err) } @@ -285,7 +311,10 @@ export async function benchmark ( moment, average: Math.round(opTime / (ops + 1)), mem: memUsed, + memRSS, memTotal, + memFree, + memArrays, cpu, requestTime, operations, @@ -360,7 +389,9 @@ export function benchmarkWorker (): void { if (!isMainThread) { parentPort?.on('message', (msg: StartMessage) => { console.log('starting worker', msg.workId) - void perform(msg) + void perform(msg).catch((err) => { + console.error('failed to perform', err) + }) }) } diff --git a/plugins/login-resources/src/components/AdminWorkspaces.svelte b/plugins/login-resources/src/components/AdminWorkspaces.svelte index 2d91fc4764..a127765c9b 100644 --- a/plugins/login-resources/src/components/AdminWorkspaces.svelte +++ b/plugins/login-resources/src/components/AdminWorkspaces.svelte @@ -12,13 +12,14 @@ type BaseWorkspaceInfo } from '@hcengineering/core' import { getEmbeddedLabel } from '@hcengineering/platform' - import { isAdminUser, MessageBox } from '@hcengineering/presentation' + import { copyTextToClipboard, isAdminUser, MessageBox } from '@hcengineering/presentation' import { Button, ButtonMenu, CheckBox, Expandable, IconArrowRight, + IconCopy, IconOpen, IconStart, IconStop, @@ -383,14 +384,21 @@
- {wsName} -
+
+
+
+ {wsName}
{workspace.createdBy} diff --git a/pods/stats/src/stats.ts b/pods/stats/src/stats.ts index 8a4bce1cac..46bf6a31fe 100644 --- a/pods/stats/src/stats.ts +++ b/pods/stats/src/stats.ts @@ -19,7 +19,7 @@ import Koa from 'koa' import bodyParser from 'koa-bodyparser' import Router from 'koa-router' -const serviceTimeout = 30000 +const serviceTimeout = 5 * 60000 interface ServiceStatisticsEx extends ServiceStatistics { lastUpdate: number // Last updated diff --git a/server/account-service/src/index.ts b/server/account-service/src/index.ts index 884289f5e9..9cb7df7832 100644 --- a/server/account-service/src/index.ts +++ b/server/account-service/src/index.ts @@ -138,7 +138,7 @@ export function serveAccount (measureCtx: MeasureContext, brandings: BrandingMap router.get('/api/v1/statistics', (req, res) => { try { - const token = req.query.token as string + const token = (req.query.token as string) ?? extractToken(req.headers) const payload = decodeToken(token) const admin = payload.extra?.admin === 'true' const data: Record = { @@ -146,8 +146,11 @@ export function serveAccount (measureCtx: MeasureContext, brandings: BrandingMap statistics: {} } data.statistics.totalClients = 0 - data.statistics.memoryUsed = Math.round((process.memoryUsage().heapUsed / 1024 / 1024) * 100) / 100 - data.statistics.memoryTotal = Math.round((process.memoryUsage().heapTotal / 1024 / 1024) * 100) / 100 + const mem = process.memoryUsage() + data.statistics.memoryUsed = Math.round((mem.heapUsed / 1024 / 1024) * 100) / 100 + data.statistics.memoryTotal = Math.round((mem.heapTotal / 1024 / 1024) * 100) / 100 + data.statistics.memoryRSS = Math.round((mem.rss / 1024 / 1024) * 100) / 100 + data.statistics.memoryArrayBuffers = Math.round((mem.arrayBuffers / 1024 / 1024) * 100) / 100 data.statistics.cpuUsage = Math.round(os.loadavg()[0] * 100) / 100 data.statistics.freeMem = Math.round((os.freemem() / 1024 / 1024) * 100) / 100 data.statistics.totalMem = Math.round((os.totalmem() / 1024 / 1024) * 100) / 100 diff --git a/server/core/src/base.ts b/server/core/src/base.ts index 571764cba4..070d1d5cff 100644 --- a/server/core/src/base.ts +++ b/server/core/src/base.ts @@ -57,7 +57,7 @@ export abstract class BaseMiddleware implements Middleware { return this.provideFindAll(ctx, _class, query, options) } - loadModel ( + provideLoadModel ( ctx: MeasureContext, lastModelTx: Timestamp, hash?: string @@ -65,6 +65,14 @@ export abstract class BaseMiddleware implements Middleware { return this.next?.loadModel(ctx, lastModelTx, hash) ?? emptyModelResult } + loadModel ( + ctx: MeasureContext, + lastModelTx: Timestamp, + hash?: string + ): Promise { + return this.provideLoadModel(ctx, lastModelTx, hash) + } + provideGroupBy( ctx: MeasureContext, domain: Domain, diff --git a/server/core/src/stats.ts b/server/core/src/stats.ts index 959155a35f..4f1853dff0 100644 --- a/server/core/src/stats.ts +++ b/server/core/src/stats.ts @@ -7,6 +7,8 @@ import os from 'os' export interface MemoryStatistics { memoryUsed: number memoryTotal: number + + memoryArrayBuffers: number memoryRSS: number freeMem: number totalMem: number @@ -56,6 +58,7 @@ export function getMemoryInfo (): MemoryStatistics { memoryUsed: Math.round((memU.heapUsed / 1024 / 1024) * 100) / 100, memoryRSS: Math.round((memU.rss / 1024 / 1024) * 100) / 100, memoryTotal: Math.round((memU.heapTotal / 1024 / 1024) * 100) / 100, + memoryArrayBuffers: Math.round((memU.arrayBuffers / 1024 / 1024) * 100) / 100, freeMem: Math.round((os.freemem() / 1024 / 1024) * 100) / 100, totalMem: Math.round((os.totalmem() / 1024 / 1024) * 100) / 100 } @@ -103,6 +106,7 @@ export function initStatisticsContext ( let oldMetricsValue = '' const serviceId = encodeURIComponent(os.hostname() + '-' + serviceName) + let prev: Promise | undefined const handleError = (err: any): void => { errorToSend++ if (errorToSend % 2 === 0) { @@ -110,6 +114,7 @@ export function initStatisticsContext ( console.error(err) } } + prev = undefined } const intTimer = setInterval(() => { @@ -128,6 +133,10 @@ export function initStatisticsContext ( } } } + if (prev !== undefined) { + // In case of high load, skip + return + } if (statsUrl !== undefined) { const token = generateToken(systemAccountEmail, { name: '' }, { service: 'true' }) const data: ServiceStatistics = { @@ -140,7 +149,7 @@ export function initStatisticsContext ( const statData = JSON.stringify(data) - void fetch( + prev = fetch( concatLink(statsUrl, '/api/v1/statistics') + `/?token=${encodeURIComponent(token)}&name=${serviceId}`, { method: 'PUT', @@ -149,7 +158,11 @@ export function initStatisticsContext ( }, body: statData } - ).catch(handleError) + ) + .catch(handleError) + .then(() => { + prev = undefined + }) } } catch (err: any) { handleError(err) diff --git a/server/core/src/types.ts b/server/core/src/types.ts index 53e0db939d..af0053c2bd 100644 --- a/server/core/src/types.ts +++ b/server/core/src/types.ts @@ -591,13 +591,15 @@ export interface ConnectionSocket { id: string isClosed: boolean close: () => void - send: (ctx: MeasureContext, msg: Response, binary: boolean, compression: boolean) => void + send: (ctx: MeasureContext, msg: Response, binary: boolean, compression: boolean) => Promise sendPong: () => void data: () => Record readRequest: (buffer: Buffer, binary: boolean) => Request + isBackpressure: () => boolean // In bytes + backpressure: (ctx: MeasureContext) => Promise checkState: () => boolean } @@ -715,6 +717,7 @@ export interface SessionManager { createOpContext: ( ctx: MeasureContext, + sendCtx: MeasureContext, pipeline: Pipeline, requestId: Request['id'], service: Session, diff --git a/server/front/src/index.ts b/server/front/src/index.ts index 904c2599c1..95cd2e433e 100644 --- a/server/front/src/index.ts +++ b/server/front/src/index.ts @@ -178,8 +178,8 @@ async function getFile ( etag: stat.etag, 'last-modified': new Date(stat.modifiedOn).toISOString(), 'cache-control': cacheControlValue, - Connection: 'keep-alive', - 'Keep-Alive': 'timeout=5' + connection: 'keep-alive', + 'keep-alive': 'timeout=5, max=1000' }) res.end() return @@ -191,8 +191,8 @@ async function getFile ( etag: stat.etag, 'last-modified': new Date(stat.modifiedOn).toISOString(), 'cache-control': cacheControlValue, - Connection: 'keep-alive', - 'Keep-Alive': 'timeout=5' + connection: 'keep-alive', + 'keep-alive': 'timeout=5, max=1000' }) res.end() return @@ -211,8 +211,8 @@ async function getFile ( Etag: stat.etag, 'Last-Modified': new Date(stat.modifiedOn).toISOString(), 'Cache-Control': cacheControlValue, - Connection: 'keep-alive', - 'Keep-Alive': 'timeout=5' + connection: 'keep-alive', + 'keep-alive': 'timeout=5, max=1000' }) dataStream.pipe(res) @@ -442,6 +442,7 @@ export function start ( if (req.method === 'HEAD') { res.writeHead(200, { 'accept-ranges': 'bytes', + connection: 'keep-alive', 'Keep-Alive': 'timeout=5', 'content-length': blobInfo.size, 'content-security-policy': "default-src 'none';", diff --git a/server/middleware/src/domainFind.ts b/server/middleware/src/domainFind.ts index cf73fdf46b..6678bd2408 100644 --- a/server/middleware/src/domainFind.ts +++ b/server/middleware/src/domainFind.ts @@ -18,6 +18,7 @@ import { type Doc, type DocumentQuery, type Domain, + type FindOptions, type FindResult, type MeasureContext, type Ref, @@ -44,6 +45,11 @@ export class DomainFindMiddleware extends BaseMiddleware implements Middleware { return middleware } + toPrintableOptions (options?: ServerFindOptions): FindOptions { + const { ctx, allowedSpaces, associations, ...opt } = options ?? {} + return opt + } + findAll( ctx: MeasureContext, _class: Ref>, @@ -65,7 +71,7 @@ export class DomainFindMiddleware extends BaseMiddleware implements Middleware { (ctx) => { return this.adapterManager.getAdapter(domain, false).findAll(ctx, _class, query, options) }, - { _class, query, options } + { _class, query, options: this.toPrintableOptions(options) } ) } diff --git a/server/middleware/src/queryJoin.ts b/server/middleware/src/queryJoin.ts index 6cc71787cc..25090fb92b 100644 --- a/server/middleware/src/queryJoin.ts +++ b/server/middleware/src/queryJoin.ts @@ -17,19 +17,23 @@ import { type Class, type Doc, DocumentQuery, - FindOptions, + type Domain, FindResult, + type LoadModelResponse, type MeasureContext, - Ref + Ref, + type SearchOptions, + type SearchQuery, + type SearchResult, + type SessionData, + type Timestamp, + type Tx } from '@hcengineering/core' -import { BaseMiddleware, Middleware, ServerFindOptions, type PipelineContext } from '@hcengineering/server-core' -import { deepEqual } from 'fast-equals' +import { BaseMiddleware, Middleware, type PipelineContext, ServerFindOptions } from '@hcengineering/server-core' interface Query { - _class: Ref> - query: DocumentQuery - result: FindResult | Promise> | undefined - options?: FindOptions + key: string + result: object | Promise | undefined callbacks: number max: number } @@ -37,27 +41,20 @@ interface Query { * @public */ export class QueryJoiner { - private readonly queries: Map>, Query[]> = new Map>, Query[]>() + private readonly queries: Map = new Map() - constructor (readonly _findAll: Middleware['findAll']) {} - - async findAll( - ctx: MeasureContext, - _class: Ref>, - query: DocumentQuery, - options?: ServerFindOptions - ): Promise> { + async query(ctx: MeasureContext, key: string, retrieve: (ctx: MeasureContext) => Promise): Promise { // Will find a query or add + 1 to callbacks - const q = this.findQuery(_class, query, options) ?? this.createQuery(_class, query, options) + const q = this.getQuery(key) try { if (q.result === undefined) { - q.result = this._findAll(ctx, _class, query, options) + q.result = retrieve(ctx) } if (q.result instanceof Promise) { q.result = await q.result } - return q.result as FindResult + return q.result as T } finally { q.callbacks-- @@ -65,46 +62,27 @@ export class QueryJoiner { } } - private findQuery( - _class: Ref>, - query: DocumentQuery, - options?: FindOptions - ): Query | undefined { - const queries = this.queries.get(_class) - if (queries === undefined) return - for (const q of queries) { - if (!deepEqual(query, q.query) || !deepEqual(options, q.options)) { - continue + private getQuery (key: string): Query { + const query = this.queries.get(key) + if (query === undefined) { + const q: Query = { + key, + result: undefined, + callbacks: 1, + max: 1 } - q.callbacks++ - q.max++ + this.queries.set(key, q) return q } - } - private createQuery(_class: Ref>, query: DocumentQuery, options?: FindOptions): Query { - const queries = this.queries.get(_class) ?? [] - const q: Query = { - _class, - query, - result: undefined, - options: options as FindOptions, - callbacks: 1, - max: 1 - } - - queries.push(q) - this.queries.set(_class, queries) - return q + query.callbacks++ + query.max++ + return query } private removeFromQueue (q: Query): void { if (q.callbacks === 0) { - const queries = this.queries.get(q._class) ?? [] - this.queries.set( - q._class, - queries.filter((it) => it !== q) - ) + this.queries.delete(q.key) } } } @@ -117,8 +95,16 @@ export class QueryJoinMiddleware extends BaseMiddleware implements Middleware { private constructor (context: PipelineContext, next?: Middleware) { super(context, next) - this.joiner = new QueryJoiner((ctx, _class, query, options) => { - return this.provideFindAll(ctx, _class, query, options) + this.joiner = new QueryJoiner() + } + + loadModel ( + ctx: MeasureContext, + lastModelTx: Timestamp, + hash?: string + ): Promise { + return this.joiner.query(ctx, `model-${lastModelTx}${hash ?? ''}`, async (ctx) => { + return await this.provideLoadModel(ctx, lastModelTx, hash) }) } @@ -136,7 +122,31 @@ export class QueryJoinMiddleware extends BaseMiddleware implements Middleware { query: DocumentQuery, options?: ServerFindOptions ): Promise> { - // Will find a query or add + 1 to callbacks - return this.joiner.findAll(ctx, _class, query, options) + const opt = { ...options } + delete opt.ctx + return this.joiner.query( + ctx, + `findAll-${_class}-${JSON.stringify(query)}-${JSON.stringify(options)}`, + async (ctx) => { + return await this.provideFindAll(ctx, _class, query, options) + } + ) + } + + groupBy( + ctx: MeasureContext, + domain: Domain, + field: string, + query?: DocumentQuery

+ ): Promise> { + return this.joiner.query(ctx, `groupBy-${domain}-${field}-${JSON.stringify(query ?? {})})`, async (ctx) => { + return await this.provideGroupBy(ctx, domain, field, query) + }) + } + + searchFulltext (ctx: MeasureContext, query: SearchQuery, options: SearchOptions): Promise { + return this.joiner.query(ctx, `searchFulltext-${JSON.stringify(query)}-${JSON.stringify(options)}`, async (ctx) => { + return await this.provideSearchFulltext(ctx, query, options) + }) } } diff --git a/server/middleware/src/tests/queryJoiner.spec.ts b/server/middleware/src/tests/queryJoiner.spec.ts index e6402ec15f..b289cd8850 100644 --- a/server/middleware/src/tests/queryJoiner.spec.ts +++ b/server/middleware/src/tests/queryJoiner.spec.ts @@ -12,14 +12,13 @@ describe('test query joiner', () => { }) return toFindResult([]) } - const joiner = new QueryJoiner(findT) + const joiner = new QueryJoiner() const ctx = new MeasureMetricsContext('test', {}) - const p1 = joiner.findAll(ctx, core.class.Class, {}) - const p2 = joiner.findAll(ctx, core.class.Class, {}) + const p1 = joiner.query(ctx, core.class.Class, (ctx) => findT(ctx, core.class.Class, {})) + const p2 = joiner.query(ctx, core.class.Class, (ctx) => findT(ctx, core.class.Class, {})) await Promise.all([p1, p2]) expect(count).toBe(1) - expect((joiner as any).queries.size).toBe(1) - expect((joiner as any).queries.get(core.class.Class).length).toBe(0) + expect((joiner as any).queries.size).toBe(0) }) }) diff --git a/server/server/src/blobs.ts b/server/server/src/blobs.ts index c55660b3a4..3bf152afb7 100644 --- a/server/server/src/blobs.ts +++ b/server/server/src/blobs.ts @@ -42,6 +42,8 @@ export async function getFile ( res.writeHead(200, { 'Content-Type': stat.contentType, Etag: stat.etag, + connection: 'keep-alive', + 'keep-alive': 'timeout=5, max=1000', 'Last-Modified': new Date(stat.modifiedOn).toISOString(), 'Cache-Control': cacheControlNoCache }) @@ -118,7 +120,9 @@ export async function getFileRange ( if (start >= size) { res.cork(() => { res.writeHead(416, { - 'Content-Range': `bytes */${size}` + 'Content-Range': `bytes */${size}`, + connection: 'keep-alive', + 'keep-alive': 'timeout=5, max=1000' }) res.end() }) @@ -139,9 +143,10 @@ export async function getFileRange ( await new Promise((resolve, reject) => { res.cork(() => { res.writeHead(206, { - Connection: 'keep-alive', 'Content-Range': `bytes ${start}-${end}/${size}`, 'Accept-Ranges': 'bytes', + connection: 'keep-alive', + 'keep-alive': 'timeout=5, max=1000', // 'Content-Length': end - start + 1, 'Content-Type': stat.contentType, Etag: stat.etag, diff --git a/server/server/src/client.ts b/server/server/src/client.ts index 875ad702f4..7bc77f0cab 100644 --- a/server/server/src/client.ts +++ b/server/server/src/client.ts @@ -234,7 +234,7 @@ export class ClientSession implements Session { } } const bevent = createBroadcastEvent(Array.from(classes)) - socket.send( + void socket.send( ctx, { result: [bevent] @@ -243,7 +243,7 @@ export class ClientSession implements Session { this.useCompression ) } else { - socket.send(ctx, { result: tx }, this.binaryMode, this.useCompression) + void socket.send(ctx, { result: tx }, this.binaryMode, this.useCompression) } } diff --git a/server/server/src/sessionManager.ts b/server/server/src/sessionManager.ts index 500eb3f5d8..c4dafb3f87 100644 --- a/server/server/src/sessionManager.ts +++ b/server/server/src/sessionManager.ts @@ -237,7 +237,7 @@ export class TSessionManager implements SessionManager { // And ping other wize s[1].session.lastPing = now if (s[1].socket.checkState()) { - s[1].socket.send( + void s[1].socket.send( workspace.context, { result: pingConst }, s[1].session.binaryMode, @@ -504,7 +504,7 @@ export class TSessionManager implements SessionManager { } if (this.timeMinutes > 0) { - ws.send(ctx, { result: this.createMaintenanceWarning() }, session.binaryMode, session.useCompression) + void ws.send(ctx, { result: this.createMaintenanceWarning() }, session.binaryMode, session.useCompression) } return { session, context: workspace.context, workspaceId: wsString } } @@ -884,7 +884,7 @@ export class TSessionManager implements SessionManager { } private sendUpgrade (ctx: MeasureContext, webSocket: ConnectionSocket, binary: boolean, compression: boolean): void { - webSocket.send( + void webSocket.send( ctx, { result: { @@ -951,6 +951,7 @@ export class TSessionManager implements SessionManager { createOpContext ( ctx: MeasureContext, + sendCtx: MeasureContext, pipeline: Pipeline, requestId: Request['id'], service: Session, @@ -962,7 +963,7 @@ export class TSessionManager implements SessionManager { pipeline, requestId, sendResponse: (reqId, msg) => - sendResponse(ctx, service, ws, { + sendResponse(sendCtx, service, ws, { id: reqId, result: msg, time: Date.now() - st, @@ -973,7 +974,7 @@ export class TSessionManager implements SessionManager { ws.sendPong() }, sendError: (reqId, msg, error: Status) => - sendResponse(ctx, service, ws, { + sendResponse(sendCtx, service, ws, { id: reqId, result: msg, error, @@ -1004,7 +1005,7 @@ export class TSessionManager implements SessionManager { requestCtx.measure('msg-receive-delta', delta) } if (service.workspace.closing !== undefined) { - ws.send( + await ws.send( ctx, { id: request.id, @@ -1033,7 +1034,7 @@ export class TSessionManager implements SessionManager { id: request.id, result: done } - ws.send(ctx, forceCloseResponse, service.binaryMode, service.useCompression) + await ws.send(ctx, forceCloseResponse, service.binaryMode, service.useCompression) return } @@ -1054,16 +1055,20 @@ export class TSessionManager implements SessionManager { try { const params = [...request.params] + if (ws.isBackpressure()) { + await ws.backpressure(ctx) + } + await ctx.with('🧨 process', {}, (callTx) => - f.apply(service, [this.createOpContext(callTx, pipeline, request.id, service, ws), ...params]) + f.apply(service, [this.createOpContext(callTx, userCtx, pipeline, request.id, service, ws), ...params]) ) } catch (err: any) { Analytics.handleError(err) if (LOGGING_ENABLED) { this.ctx.error('error handle request', { error: err, request }) } - ws.send( - ctx, + await ws.send( + userCtx, { id: request.id, error: unknownError(err), @@ -1108,15 +1113,15 @@ export class TSessionManager implements SessionManager { service.workspace.pipeline instanceof Promise ? await service.workspace.pipeline : service.workspace.pipeline try { - const uctx = this.createOpContext(ctx, pipeline, reqId, service, ws) + const uctx = this.createOpContext(ctx, userCtx, pipeline, reqId, service, ws) await operation(uctx) } catch (err: any) { Analytics.handleError(err) if (LOGGING_ENABLED) { this.ctx.error('error handle request', { error: err }) } - ws.send( - ctx, + await ws.send( + userCtx, { id: reqId, error: unknownError(err), @@ -1174,7 +1179,7 @@ export class TSessionManager implements SessionManager { account: service.getRawAccount(pipeline), useCompression: service.useCompression } - ws.send(requestCtx, helloResponse, false, false) + await ws.send(requestCtx, helloResponse, false, false) // We do not need to wait for set-status, just return session to client const _workspace = service.workspace diff --git a/server/server/src/stats.ts b/server/server/src/stats.ts index b77d2c506c..cc7c3ae280 100644 --- a/server/server/src/stats.ts +++ b/server/server/src/stats.ts @@ -43,6 +43,8 @@ export function getStatistics (ctx: MeasureContext, sessions: SessionManager, ad const memU = process.memoryUsage() data.statistics.memoryUsed = Math.round(((memU.heapUsed + memU.rss) / 1024 / 1024) * 100) / 100 data.statistics.memoryTotal = Math.round((memU.heapTotal / 1024 / 1024) * 100) / 100 + data.statistics.memoryRSS = Math.round((memU.rss / 1024 / 1024) * 100) / 100 + data.statistics.memoryArrayBuffers = Math.round((memU.arrayBuffers / 1024 / 1024) * 100) / 100 data.statistics.cpuUsage = Math.round(os.loadavg()[0] * 100) / 100 data.statistics.freeMem = Math.round((os.freemem() / 1024 / 1024) * 100) / 100 data.statistics.totalMem = Math.round((os.totalmem() / 1024 / 1024) * 100) / 100 diff --git a/server/server/src/utils.ts b/server/server/src/utils.ts index 429029d659..fe44d5402e 100644 --- a/server/server/src/utils.ts +++ b/server/server/src/utils.ts @@ -85,6 +85,5 @@ export function sendResponse ( socket: ConnectionSocket, resp: Response ): Promise { - socket.send(ctx, resp, session.binaryMode, session.useCompression) - return Promise.resolve() + return socket.send(ctx, resp, session.binaryMode, session.useCompression) } diff --git a/server/ws/src/rpc.ts b/server/ws/src/rpc.ts index 2d2685ff5f..e633cb0fc8 100644 --- a/server/ws/src/rpc.ts +++ b/server/ws/src/rpc.ts @@ -26,6 +26,7 @@ const sendError = (res: ExpressResponse, code: number, data: any): void => { res.writeHead(code, { 'Content-Type': 'application/json', 'Cache-Control': 'no-cache', + Connection: 'keep-alive', 'keep-alive': 'timeout=5, max=1000' }) res.end(JSON.stringify(data)) @@ -35,6 +36,7 @@ async function sendJson (req: Request, res: ExpressResponse, result: any): Promi const headers: OutgoingHttpHeaders = { 'Content-Type': 'application/json', 'Cache-Control': 'no-cache', + Connection: 'keep-alive', 'keep-alive': 'timeout=5, max=1000' } let body: any = JSON.stringify(result) @@ -173,7 +175,9 @@ function createClosingSocket (rawToken: string, rpcSessions: Map { rpcSessions.delete(rawToken) }, - send: (ctx, msg, binary, compression) => {}, + send: async (ctx, msg, binary, compression) => {}, + isBackpressure: () => false, + backpressure: async (ctx) => {}, sendPong: () => {}, data: () => ({}), readRequest: (buffer, binary) => ({ method: '', params: [], id: -1, time: Date.now() }), diff --git a/server/ws/src/server_http.ts b/server/ws/src/server_http.ts index 82609f67d5..d9961c4806 100644 --- a/server/ws/src/server_http.ts +++ b/server/ws/src/server_http.ts @@ -50,8 +50,12 @@ import 'utf-8-validate' import { registerRPC } from './rpc' import { retrieveJson } from './utils' +import { setImmediate } from 'timers/promises' + let profiling = false const rpcHandler = new RPCHandler() + +const backpressureSize = 100 * 1024 /** * @public * @param sessionFactory - @@ -81,7 +85,11 @@ export function startHttpServer ( const getUsers = (): any => Array.from(sessions.sessions.entries()).map(([k, v]) => v.session.getUser()) app.get('/api/v1/version', (req, res) => { - res.writeHead(200, { 'Content-Type': 'application/json' }) + res.writeHead(200, { + 'Content-Type': 'application/json', + Connection: 'keep-alive', + 'keep-alive': 'timeout=5, max=1000' + }) res.end( JSON.stringify({ version: process.env.MODEL_VERSION @@ -91,7 +99,7 @@ export function startHttpServer ( app.get('/api/v1/statistics', (req, res) => { try { - const token = req.query.token as string + const token = (req.query.token as string) ?? (req.headers.authorization ?? '').split(' ')[1] const payload = decodeToken(token) const admin = payload.extra?.admin === 'true' const jsonData = { @@ -101,7 +109,11 @@ export function startHttpServer ( profiling } const json = JSON.stringify(jsonData) - res.writeHead(200, { 'Content-Type': 'application/json' }) + res.writeHead(200, { + 'Content-Type': 'application/json', + Connection: 'keep-alive', + 'keep-alive': 'timeout=5, max=1000' + }) res.end(json) } catch (err: any) { Analytics.handleError(err) @@ -130,7 +142,7 @@ export function startHttpServer ( }) app.put('/api/v1/manage', (req, res) => { try { - const token = req.query.token as string + const token = (req.query.token as string) ?? (req.headers.authorization ?? '').split(' ')[1] const payload = decodeToken(token) if (payload.extra?.admin !== 'true' && payload.email !== systemAccountEmail) { console.warn('Non admin attempt to maintenance action', { payload }) @@ -246,7 +258,11 @@ export function startHttpServer ( { file: name, contentType } ) .then(() => { - res.writeHead(200, { 'Cache-Control': 'no-cache' }) + res.writeHead(200, { + 'Cache-Control': 'no-cache', + Connection: 'keep-alive', + 'keep-alive': 'timeout=5, max=1000' + }) res.end() }) .catch((err) => { @@ -373,7 +389,7 @@ export function startHttpServer ( void webSocketData.session.then((s) => { if ('error' in s) { if (s.specialError === 'archived') { - cs.send( + void cs.send( ctx, { id: -1, @@ -386,7 +402,7 @@ export function startHttpServer ( false ) } else if (s.specialError === 'migration') { - cs.send( + void cs.send( ctx, { id: -1, @@ -399,7 +415,7 @@ export function startHttpServer ( false ) } else { - cs.send( + void cs.send( ctx, { id: -1, error: unknownStatus(s.error.message ?? 'Unknown error'), terminate: s.terminate }, false, @@ -412,7 +428,7 @@ export function startHttpServer ( }, 1000) } if ('upgrade' in s) { - cs.send(ctx, { id: -1, result: { state: 'upgrading', stats: (s as any).upgradeInfo } }, false, false) + void cs.send(ctx, { id: -1, result: { state: 'upgrading', stats: (s as any).upgradeInfo } }, false, false) setTimeout(() => { cs.close() }, 5000) @@ -557,6 +573,17 @@ function createWebsocketClientSocket ( ws.close() ws.terminate() }, + isBackpressure: () => ws.bufferedAmount > backpressureSize, + backpressure: async (ctx) => { + if (ws.bufferedAmount < backpressureSize) { + return + } + await ctx.with('backpressure', {}, async () => { + while (ws.bufferedAmount > backpressureSize) { + await setImmediate() + } + }) + }, checkState: () => { if (ws.readyState === ws.CLOSED || ws.readyState === ws.CLOSING) { ws.terminate() @@ -577,7 +604,7 @@ function createWebsocketClientSocket ( } ws.send(pongConst) }, - send: (ctx: MeasureContext, msg, binary, _compression) => { + send: async (ctx: MeasureContext, msg, binary, _compression): Promise => { const smsg = rpcHandler.serialize(msg, binary) ctx.measure('send-data', smsg.length) @@ -586,23 +613,28 @@ function createWebsocketClientSocket ( return } - const handleErr = (err?: Error): void => { - ctx.measure('msg-send-delta', Date.now() - st) - if (err != null) { - if (!`${err.message}`.includes('WebSocket is not open')) { - ctx.error('send error', { err }) - Analytics.handleError(err) - } - } + // We need to be sure all data is send before we will send more. + if (cs.isBackpressure()) { + await cs.backpressure(ctx) } + let sendMsg = smsg if (_compression) { - void compress(smsg).then((msg: any) => { - ws.send(msg, { binary: true }, handleErr) - }) - } else { - ws.send(smsg, { binary: true }, handleErr) + sendMsg = await compress(smsg) } + await new Promise((resolve) => { + const handleErr = (err?: Error): void => { + ctx.measure('msg-send-delta', Date.now() - st) + if (err != null) { + if (!`${err.message}`.includes('WebSocket is not open')) { + ctx.error('send error', { err }) + Analytics.handleError(err) + } + } + resolve() // In any case we need to resolve. + } + ws.send(sendMsg, { binary: true }, handleErr) + }) } } return cs diff --git a/workers/transactor/src/transactor.ts b/workers/transactor/src/transactor.ts index 1b516c265e..ebad370d06 100644 --- a/workers/transactor/src/transactor.ts +++ b/workers/transactor/src/transactor.ts @@ -300,7 +300,7 @@ export class Transactor extends DurableObject { throw session.error } if ('upgrade' in session) { - cs.send( + await cs.send( this.measureCtx, { id: -1, result: { state: 'upgrading', stats: (session as any).upgradeInfo } }, false, @@ -352,6 +352,8 @@ export class Transactor extends DurableObject { } return true }, + backpressure: async (ctx) => {}, + isBackpressure: () => false, readRequest: (buffer: Buffer, binary: boolean) => { if (buffer.length === pingConst.length) { if (buffer.toString() === pingConst) { @@ -361,7 +363,7 @@ export class Transactor extends DurableObject { return rpcHandler.readRequest(buffer, binary) }, data: () => data, - send: (ctx: MeasureContext, msg, binary, _compression) => { + send: async (ctx: MeasureContext, msg, binary, _compression) => { let smsg = rpcHandler.serialize(msg, binary) ctx.measure('send-data', smsg.length) @@ -435,7 +437,9 @@ export class Transactor extends DurableObject { data: () => { return {} }, - send: (ctx: MeasureContext, msg, binary, compression) => {}, + isBackpressure: () => false, + backpressure: async (ctx) => {}, + send: async (ctx: MeasureContext, msg, binary, compression) => {}, sendPong: () => {} } return cs diff --git a/ws-tests/tool.sh b/ws-tests/tool.sh index 779df41367..c5003d0c73 100755 --- a/ws-tests/tool.sh +++ b/ws-tests/tool.sh @@ -13,4 +13,4 @@ export ELASTIC_URL=http://localhost:9201 export SERVER_SECRET=secret export DB_URL=postgresql://root@localhost:26258/defaultdb?sslmode=disable -node ${TOOL_OPTIONS} ../dev/tool/bundle/bundle.js $@ \ No newline at end of file +node ${TOOL_OPTIONS} --max-old-space-size=8096 ../dev/tool/bundle/bundle.js $@ \ No newline at end of file From 615a088ed0964922321da4e503ac8da83014b08e Mon Sep 17 00:00:00 2001 From: Andrey Sobolev Date: Wed, 26 Feb 2025 22:44:48 +0700 Subject: [PATCH 13/16] UBERF-9513: Support model operations (#8100) Signed-off-by: Andrey Sobolev --- .github/workflows/main.yml | 5 + common/config/rush/pnpm-lock.yaml | 59 +++++- packages/api-client/src/client.ts | 24 ++- packages/api-client/src/rest.ts | 130 ------------ packages/api-client/src/rest/index.ts | 18 ++ packages/api-client/src/rest/rest.ts | 199 ++++++++++++++++++ packages/api-client/src/rest/tx.ts | 101 +++++++++ packages/api-client/src/rest/types.ts | 39 ++++ packages/api-client/src/rest/utils.ts | 42 ++++ packages/core/src/client.ts | 2 +- rush.json | 5 + server/core/src/types.ts | 3 + server/middleware/src/model.ts | 20 ++ server/mongo/src/index.ts | 2 + server/rpc/src/rpc.ts | 12 +- server/server/src/client.ts | 13 ++ server/ws/package.json | 6 +- server/ws/src/__tests__/rest.test.ts | 21 +- server/ws/src/__tests__/server.test.ts | 11 +- server/ws/src/rpc.ts | 157 ++++++++++---- server/ws/src/server_http.ts | 16 ++ ws-tests/api-tests/.eslintrc.js | 7 + ws-tests/api-tests/.gitignore | 2 + ws-tests/api-tests/.npmignore | 4 + ws-tests/api-tests/config/rig.json | 5 + ws-tests/api-tests/jest.config.js | 7 + ws-tests/api-tests/package.json | 59 ++++++ ws-tests/api-tests/src/__tests__/rest.test.ts | 178 ++++++++++++++++ ws-tests/api-tests/src/index.ts | 0 ws-tests/api-tests/tsconfig.json | 10 + ws-tests/prepare.sh | 6 + ws-tests/tool-europe.sh | 19 ++ ws-tests/tool.sh | 2 +- 33 files changed, 981 insertions(+), 203 deletions(-) delete mode 100644 packages/api-client/src/rest.ts create mode 100644 packages/api-client/src/rest/index.ts create mode 100644 packages/api-client/src/rest/rest.ts create mode 100644 packages/api-client/src/rest/tx.ts create mode 100644 packages/api-client/src/rest/types.ts create mode 100644 packages/api-client/src/rest/utils.ts create mode 100644 ws-tests/api-tests/.eslintrc.js create mode 100644 ws-tests/api-tests/.gitignore create mode 100644 ws-tests/api-tests/.npmignore create mode 100644 ws-tests/api-tests/config/rig.json create mode 100644 ws-tests/api-tests/jest.config.js create mode 100644 ws-tests/api-tests/package.json create mode 100644 ws-tests/api-tests/src/__tests__/rest.test.ts create mode 100644 ws-tests/api-tests/src/index.ts create mode 100644 ws-tests/api-tests/tsconfig.json create mode 100755 ws-tests/tool-europe.sh diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d13d5a4be6..da8809c6c9 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -485,6 +485,11 @@ jobs: cd ./ws-tests export DO_CLEAN=true ./prepare.sh + - name: Run API tests + run: | + cd ./ws-tests/api-tests + node ../../common/scripts/install-run-rush.js validate --to @hcengineering/api-tests + node ../../common/scripts/install-run-rushx.js api-test --verbose - name: Install Playwright run: | cd ./ws-tests/sanity diff --git a/common/config/rush/pnpm-lock.yaml b/common/config/rush/pnpm-lock.yaml index d64f04f26a..33a41bd0c3 100644 --- a/common/config/rush/pnpm-lock.yaml +++ b/common/config/rush/pnpm-lock.yaml @@ -106,6 +106,9 @@ importers: '@rush-temp/api-client': specifier: file:./projects/api-client.tgz version: file:projects/api-client.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(bufferutil@4.0.8)(esbuild@0.24.2)(utf-8-validate@6.0.4) + '@rush-temp/api-tests': + specifier: file:./projects/api-tests.tgz + version: file:projects/api-tests.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)) '@rush-temp/attachment': specifier: file:./projects/attachment.tgz version: file:projects/attachment.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(@types/node@20.11.19)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)) @@ -1095,7 +1098,7 @@ importers: version: file:projects/tests-sanity.tgz '@rush-temp/tests-ws-sanity': specifier: file:./projects/tests-ws-sanity.tgz - version: file:projects/tests-ws-sanity.tgz + version: file:projects/tests-ws-sanity.tgz(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)) '@rush-temp/text': specifier: file:./projects/text.tgz version: file:projects/text.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(@types/node@20.11.19)(babel-jest@29.7.0(@babel/core@7.23.9))(bufferutil@4.0.8)(esbuild@0.24.2)(prosemirror-inputrules@1.4.0)(prosemirror-model@1.24.1)(prosemirror-state@1.4.3)(prosemirror-view@1.37.2)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3))(utf-8-validate@6.0.4) @@ -3864,6 +3867,10 @@ packages: resolution: {integrity: sha512-gq68pTrBheqcmVrkbiqYY+6p80DQHIsJvC1nIvhRfwrj3eR1ir5fVsW5t2Gs8fvN/Um8nH0MnkTUGavVrPWIdw==, tarball: file:projects/api-client.tgz} version: 0.0.0 + '@rush-temp/api-tests@file:projects/api-tests.tgz': + resolution: {integrity: sha512-gCHsoeILAG/lcLoL1r7XjxDLqdEcEMphk+FU2tHcqqXolwuytD2Rnd9Ko+IVta8kGBTMwCk8DiX9qdICHL7NbQ==, tarball: file:projects/api-tests.tgz} + version: 0.0.0 + '@rush-temp/attachment-assets@file:projects/attachment-assets.tgz': resolution: {integrity: sha512-C4ZvrB9y7H0bh1vbkqmKw9tIrwFAHUG7P6kPI+tjRPRKQNGAn61eiVSTpM8BCGNY/mFC5b+uMy0us0UaqvPy+A==, tarball: file:projects/attachment-assets.tgz} version: 0.0.0 @@ -4589,7 +4596,7 @@ packages: version: 0.0.0 '@rush-temp/pod-calendar@file:projects/pod-calendar.tgz': - resolution: {integrity: sha512-BvN38ScXSeB5Mui24Sawekl81QZMFZ47EK6KdIqRJJQwM+Qj0owWul3y/r7ZtNAfKnD9bV3AeEWaqkZ8EXYjwg==, tarball: file:projects/pod-calendar.tgz} + resolution: {integrity: sha512-gUk3jshnHJ0CUucxZ1Stk8oQ2DTdSVfJzFHIl/EBUISbRv/YySoXesQCnqfn1kVpqOBZdIvI/N4NRAL/yREbWw==, tarball: file:projects/pod-calendar.tgz} version: 0.0.0 '@rush-temp/pod-collaborator@file:projects/pod-collaborator.tgz': @@ -5065,7 +5072,7 @@ packages: version: 0.0.0 '@rush-temp/server-ws@file:projects/server-ws.tgz': - resolution: {integrity: sha512-3jjoKDZ0NAs4XRajHH6n4EgS00GwQY0Zga1JlG8XIeSPjAGphGUYbCaHQuJbcddPAYRdT632gK+119U4RvGSxw==, tarball: file:projects/server-ws.tgz} + resolution: {integrity: sha512-kVd/j4HPgbCYMMEqEkxRQiGqM0es47rgzzPdQvTDEFs6HtkyWxy8pbzRqA6HXJIDbNY5WLeTbtiLV37db6q33A==, tarball: file:projects/server-ws.tgz} version: 0.0.0 '@rush-temp/server@file:projects/server.tgz': @@ -5181,7 +5188,7 @@ packages: version: 0.0.0 '@rush-temp/tests-ws-sanity@file:projects/tests-ws-sanity.tgz': - resolution: {integrity: sha512-2y+gKxse3ozzKeoi1TF7Qfv7Y1fWy9Z6HiqSsEeQgGPpv396cH7ZfpSJ/qwsxO66c6iq8LUuBtzZ4AlquHfG3w==, tarball: file:projects/tests-ws-sanity.tgz} + resolution: {integrity: sha512-YLGIPrvHv+48mtZrd5T8uJpGCkoGXjBtuxMvcD+Q3CNbHGTcYLxlznkmlNXa6wPm0G6v2VrpRuOfXAuAqlYnHg==, tarball: file:projects/tests-ws-sanity.tgz} version: 0.0.0 '@rush-temp/text-core@file:projects/text-core.tgz': @@ -16161,6 +16168,41 @@ snapshots: - supports-color - utf-8-validate + '@rush-temp/api-tests@file:projects/api-tests.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3))': + dependencies: + '@types/body-parser': 1.19.5 + '@types/compression': 1.7.5 + '@types/cors': 2.8.17 + '@types/express': 4.17.21 + '@types/jest': 29.5.12 + '@types/morgan': 1.9.9 + '@types/node': 20.11.19 + '@types/ws': 8.5.11 + '@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.7.3) + '@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.7.3) + bufferutil: 4.0.8 + eslint: 8.56.0 + eslint-config-standard-with-typescript: 40.0.0(@typescript-eslint/eslint-plugin@6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.3.3))(eslint-plugin-import@2.29.1(eslint@8.56.0))(eslint-plugin-n@15.7.0(eslint@8.56.0))(eslint-plugin-promise@6.1.1(eslint@8.56.0))(eslint@8.56.0)(typescript@5.7.3) + eslint-plugin-import: 2.29.1(eslint@8.56.0) + eslint-plugin-n: 15.7.0(eslint@8.56.0) + eslint-plugin-promise: 6.1.1(eslint@8.56.0) + jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)) + prettier: 3.2.5 + snappyjs: 0.7.0 + ts-jest: 29.1.2(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(jest@29.7.0(@types/node@20.11.19)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)))(typescript@5.7.3) + typescript: 5.7.3 + utf-8-validate: 6.0.4 + ws: 8.18.0(bufferutil@4.0.8)(utf-8-validate@6.0.4) + transitivePeerDependencies: + - '@babel/core' + - '@jest/types' + - babel-jest + - babel-plugin-macros + - esbuild + - node-notifier + - supports-color + - ts-node + '@rush-temp/attachment-assets@file:projects/attachment-assets.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3))': dependencies: '@types/jest': 29.5.12 @@ -20588,7 +20630,6 @@ snapshots: '@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.3.3) '@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.3.3) cors: 2.8.5 - cross-env: 7.0.3 dotenv: 16.0.3 esbuild: 0.24.2 eslint: 8.56.0 @@ -24085,6 +24126,7 @@ snapshots: '@types/cors': 2.8.17 '@types/express': 4.17.21 '@types/jest': 29.5.12 + '@types/morgan': 1.9.9 '@types/node': 20.11.19 '@types/ws': 8.5.11 '@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.3.3) @@ -24099,6 +24141,7 @@ snapshots: eslint-plugin-promise: 6.1.1(eslint@8.56.0) express: 4.21.2 jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)) + morgan: 1.10.0 prettier: 3.2.5 snappy: 7.2.2 ts-jest: 29.1.2(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(jest@29.7.0(@types/node@20.11.19)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)))(typescript@5.3.3) @@ -24939,7 +24982,7 @@ snapshots: transitivePeerDependencies: - supports-color - '@rush-temp/tests-ws-sanity@file:projects/tests-ws-sanity.tgz': + '@rush-temp/tests-ws-sanity@file:projects/tests-ws-sanity.tgz(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3))': dependencies: '@faker-js/faker': 8.4.1 '@playwright/test': 1.49.1 @@ -24956,10 +24999,14 @@ snapshots: eslint-plugin-import: 2.29.1(eslint@8.56.0) eslint-plugin-n: 15.7.0(eslint@8.56.0) eslint-plugin-promise: 6.1.1(eslint@8.56.0) + jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)) prettier: 3.2.5 typescript: 5.7.3 transitivePeerDependencies: + - babel-plugin-macros + - node-notifier - supports-color + - ts-node '@rush-temp/text-core@file:projects/text-core.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(@types/node@20.11.19)(babel-jest@29.7.0(@babel/core@7.23.9))(bufferutil@4.0.8)(esbuild@0.24.2)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3))(utf-8-validate@6.0.4)': dependencies: diff --git a/packages/api-client/src/client.ts b/packages/api-client/src/client.ts index 61d8483b1d..99fa3f192d 100644 --- a/packages/api-client/src/client.ts +++ b/packages/api-client/src/client.ts @@ -13,6 +13,7 @@ // limitations under the License. // +import client, { clientId } from '@hcengineering/client' import { type Account, type Class, @@ -26,18 +27,17 @@ import { type ModelDb, type Ref, type Space, - type WithLookup, type TxResult, - DocumentUpdate, - TxOperations, - AttachedDoc, + type WithLookup, AttachedData, + AttachedDoc, + DocumentUpdate, Mixin, - MixinUpdate, MixinData, + MixinUpdate, + TxOperations, generateId } from '@hcengineering/core' -import client, { clientId } from '@hcengineering/client' import { addLocation, getResource } from '@hcengineering/platform' import { login, selectWorkspace } from './account' @@ -49,7 +49,7 @@ import { MarkupContent, createMarkupOperations } from './markup' -import { type PlatformClient, type ConnectOptions, WithMarkup } from './types' +import { type ConnectOptions, type PlatformClient, WithMarkup } from './types' /** * Create platform client @@ -278,11 +278,17 @@ class PlatformClientImpl implements PlatformClient { } } -async function getWorkspaceToken ( +export interface WorkspaceToken { + endpoint: string + token: string + workspaceId: string +} + +export async function getWorkspaceToken ( url: string, options: ConnectOptions, config?: ServerConfig -): Promise<{ endpoint: string, token: string, workspaceId: string }> { +): Promise { config ??= await loadServerConfig(url) let token: string diff --git a/packages/api-client/src/rest.ts b/packages/api-client/src/rest.ts deleted file mode 100644 index 23648ca2d1..0000000000 --- a/packages/api-client/src/rest.ts +++ /dev/null @@ -1,130 +0,0 @@ -// -// Copyright © 2025 Hardcore Engineering Inc. -// -// Licensed under the Eclipse Public License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. You may -// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// -// See the License for the specific language governing permissions and -// limitations under the License. -// - -import { - type Account, - type Class, - type Doc, - type DocumentQuery, - type FindOptions, - type FindResult, - type Ref, - type Storage, - type Tx, - type TxResult, - type WithLookup, - concatLink -} from '@hcengineering/core' - -import { PlatformError, unknownError } from '@hcengineering/platform' - -import { uncompress } from 'snappyjs' - -export interface RestClient extends Storage { - getAccount: () => Promise - - findOne: ( - _class: Ref>, - query: DocumentQuery, - options?: FindOptions - ) => Promise | undefined> -} - -export async function createRestClient (endpoint: string, workspaceId: string, token: string): Promise { - return new RestClientImpl(endpoint, workspaceId, token) -} - -class RestClientImpl implements RestClient { - constructor ( - readonly endpoint: string, - readonly workspace: string, - readonly token: string - ) {} - - async findAll( - _class: Ref>, - query: DocumentQuery, - options?: FindOptions - ): Promise> { - const params = new URLSearchParams() - params.append('class', _class) - if (query !== undefined && Object.keys(query).length > 0) { - params.append('query', JSON.stringify(query)) - } - if (options !== undefined && Object.keys(options).length > 0) { - params.append('options', JSON.stringify(options)) - } - const response = await fetch(concatLink(this.endpoint, `/api/v1/find-all/${this.workspace}?${params.toString()}`), { - method: 'GET', - headers: { - 'Content-Type': 'application/json', - Authorization: 'Bearer ' + this.token, - 'accept-encoding': 'snappy, gzip' - }, - keepalive: true - }) - if (!response.ok) { - throw new PlatformError(unknownError(response.statusText)) - } - const encoding = response.headers.get('content-encoding') - if (encoding === 'snappy') { - const buffer = await response.arrayBuffer() - const decompressed = uncompress(buffer) - const decoder = new TextDecoder() - const jsonString = decoder.decode(decompressed) - return JSON.parse(jsonString) as FindResult - } - return (await response.json()) as FindResult - } - - async getAccount (): Promise { - const response = await fetch(concatLink(this.endpoint, `/api/v1/account/${this.workspace}`), { - method: 'GET', - headers: { - 'Content-Type': 'application/json', - Authorization: 'Bearer ' + this.token - }, - keepalive: true - }) - if (!response.ok) { - throw new PlatformError(unknownError(response.statusText)) - } - return (await response.json()) as Account - } - - async findOne( - _class: Ref>, - query: DocumentQuery, - options?: FindOptions - ): Promise | undefined> { - return (await this.findAll(_class, query, { ...options, limit: 1 })).shift() - } - - async tx (tx: Tx): Promise { - const response = await fetch(concatLink(this.endpoint, `/api/v1/tx/${this.workspace}`), { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - Authorization: 'Bearer ' + this.token - }, - keepalive: true, - body: JSON.stringify(tx) - }) - if (!response.ok) { - throw new PlatformError(unknownError(response.statusText)) - } - return (await response.json()) as TxResult - } -} diff --git a/packages/api-client/src/rest/index.ts b/packages/api-client/src/rest/index.ts new file mode 100644 index 0000000000..4397c92ecd --- /dev/null +++ b/packages/api-client/src/rest/index.ts @@ -0,0 +1,18 @@ +// +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +export { createRestClient } from './rest' +export { createRestTxOperations } from './tx' +export * from './types' diff --git a/packages/api-client/src/rest/rest.ts b/packages/api-client/src/rest/rest.ts new file mode 100644 index 0000000000..a483c94c1f --- /dev/null +++ b/packages/api-client/src/rest/rest.ts @@ -0,0 +1,199 @@ +// +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import { + type Account, + type Class, + type Doc, + type DocumentQuery, + type FindOptions, + type FindResult, + Hierarchy, + MeasureMetricsContext, + ModelDb, + type Ref, + type SearchOptions, + type SearchQuery, + type SearchResult, + type Tx, + type TxResult, + type WithLookup, + buildModel, + concatLink +} from '@hcengineering/core' + +import { PlatformError, unknownError } from '@hcengineering/platform' + +import type { RestClient } from './types' +import { extractJson, withRetry } from './utils' + +export async function createRestClient (endpoint: string, workspaceId: string, token: string): Promise { + return new RestClientImpl(endpoint, workspaceId, token) +} + +export class RestClientImpl implements RestClient { + endpoint: string + constructor ( + endpoint: string, + readonly workspace: string, + readonly token: string + ) { + this.endpoint = endpoint.replace('ws', 'http') + } + + jsonHeaders (): Record { + return { + 'Content-Type': 'application/json', + Authorization: 'Bearer ' + this.token, + 'accept-encoding': 'snappy, gzip' + } + } + + requestInit (): RequestInit { + return { + method: 'GET', + keepalive: true, + headers: this.jsonHeaders() + } + } + + async findAll( + _class: Ref>, + query: DocumentQuery, + options?: FindOptions + ): Promise> { + const params = new URLSearchParams() + params.append('class', _class) + if (query !== undefined && Object.keys(query).length > 0) { + params.append('query', JSON.stringify(query)) + } + if (options !== undefined && Object.keys(options).length > 0) { + params.append('options', JSON.stringify(options)) + } + const requestUrl = concatLink(this.endpoint, `/api/v1/find-all/${this.workspace}?${params.toString()}`) + const result = await withRetry(async () => { + const response = await fetch(requestUrl, this.requestInit()) + if (!response.ok) { + throw new PlatformError(unknownError(response.statusText)) + } + return await extractJson>(response) + }) + + if (result.lookupMap !== undefined) { + // We need to extract lookup map to document lookups + for (const d of result) { + if (d.$lookup !== undefined) { + for (const [k, v] of Object.entries(d.$lookup)) { + if (!Array.isArray(v)) { + d.$lookup[k] = result.lookupMap[v as any] + } else { + d.$lookup[k] = v.map((it) => result.lookupMap?.[it]) + } + } + } + } + delete result.lookupMap + } + + // We need to revert deleted query simple values. + // We need to get rid of simple query parameters matched in documents + for (const doc of result) { + if (doc._class == null) { + doc._class = _class + } + for (const [k, v] of Object.entries(query)) { + if (typeof v === 'string' || typeof v === 'number' || typeof v === 'boolean') { + if (doc[k] == null) { + doc[k] = v + } + } + } + } + + return result + } + + async getAccount (): Promise { + const requestUrl = concatLink(this.endpoint, `/api/v1/account/${this.workspace}`) + const response = await fetch(requestUrl, this.requestInit()) + if (!response.ok) { + throw new PlatformError(unknownError(response.statusText)) + } + return await extractJson(response) + } + + async getModel (): Promise<{ hierarchy: Hierarchy, model: ModelDb }> { + const requestUrl = concatLink(this.endpoint, `/api/v1/load-model/${this.workspace}`) + const response = await fetch(requestUrl, this.requestInit()) + if (!response.ok) { + throw new PlatformError(unknownError(response.statusText)) + } + const modelResponse: Tx[] = await extractJson(response) + + const hierarchy = new Hierarchy() + const model = new ModelDb(hierarchy) + + const ctx = new MeasureMetricsContext('loadModel', {}) + buildModel(ctx, modelResponse, (txes: Tx[]) => txes, hierarchy, model) + + return { hierarchy, model } + } + + async findOne( + _class: Ref>, + query: DocumentQuery, + options?: FindOptions + ): Promise | undefined> { + return (await this.findAll(_class, query, { ...options, limit: 1 })).shift() + } + + async tx (tx: Tx): Promise { + const requestUrl = concatLink(this.endpoint, `/api/v1/tx/${this.workspace}`) + const response = await fetch(requestUrl, { + method: 'POST', + headers: this.jsonHeaders(), + keepalive: true, + body: JSON.stringify(tx) + }) + if (!response.ok) { + throw new PlatformError(unknownError(response.statusText)) + } + return await extractJson(response) + } + + async searchFulltext (query: SearchQuery, options: SearchOptions): Promise { + const params = new URLSearchParams() + params.append('query', query.query) + if (query.classes != null && Object.keys(query.classes).length > 0) { + params.append('classes', JSON.stringify(query.classes)) + } + if (query.spaces != null && Object.keys(query.spaces).length > 0) { + params.append('spaces', JSON.stringify(query.spaces)) + } + if (options.limit != null) { + params.append('limit', `${options.limit}`) + } + const requestUrl = concatLink(this.endpoint, `/api/v1/search-fulltext/${this.workspace}`) + const response = await fetch(requestUrl, { + method: 'GET', + headers: this.jsonHeaders(), + keepalive: true + }) + if (!response.ok) { + throw new PlatformError(unknownError(response.statusText)) + } + return await extractJson(response) + } +} diff --git a/packages/api-client/src/rest/tx.ts b/packages/api-client/src/rest/tx.ts new file mode 100644 index 0000000000..ee08900614 --- /dev/null +++ b/packages/api-client/src/rest/tx.ts @@ -0,0 +1,101 @@ +// +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import { + type Account, + type Class, + type Client, + type Doc, + type DocumentQuery, + type FindOptions, + type FindResult, + Hierarchy, + ModelDb, + type Ref, + type SearchOptions, + type SearchQuery, + type SearchResult, + toFindResult, + type Tx, + TxOperations, + type TxResult, + type WithLookup +} from '@hcengineering/core' +import { RestClientImpl } from './rest' + +export async function createRestTxOperations ( + endpoint: string, + workspaceId: string, + token: string +): Promise { + const restClient = new RestClientImpl(endpoint, workspaceId, token) + + const account = await restClient.getAccount() + const { hierarchy, model } = await restClient.getModel() + + return new TxOperations(new RestTxClient(restClient, hierarchy, model, account), account._id) +} + +class RestTxClient implements Client { + constructor ( + readonly client: RestClientImpl, + readonly hierarchy: Hierarchy, + readonly model: ModelDb, + readonly account: Account + ) {} + + close (): Promise { + return Promise.resolve() + } + + async findAll( + _class: Ref>, + query: DocumentQuery, + options?: FindOptions + ): Promise> { + const data = await this.client.findAll(_class, query, options) + const result = data.map((v) => { + return this.hierarchy.updateLookupMixin(_class, v, options) + }) + return toFindResult(result, data.total) + } + + async findOne( + _class: Ref>, + query: DocumentQuery, + options?: FindOptions + ): Promise | undefined> { + const v = await this.client.findOne(_class, query, options) + if (v === undefined) { + return + } + return this.hierarchy.updateLookupMixin(_class, v, options) + } + + getHierarchy: () => Hierarchy = () => this.hierarchy + getModel: () => ModelDb = () => this.model + + async getAccount (): Promise { + return this.account + } + + async tx (tx: Tx): Promise { + return await this.client.tx(tx) + } + + async searchFulltext (query: SearchQuery, options: SearchOptions): Promise { + return await this.client.searchFulltext(query, options) + } +} diff --git a/packages/api-client/src/rest/types.ts b/packages/api-client/src/rest/types.ts new file mode 100644 index 0000000000..b117d90c60 --- /dev/null +++ b/packages/api-client/src/rest/types.ts @@ -0,0 +1,39 @@ +// +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import { + type Account, + type Class, + type Doc, + type DocumentQuery, + type FindOptions, + type Hierarchy, + type ModelDb, + type Ref, + type Storage, + type WithLookup +} from '@hcengineering/core' + +export interface RestClient extends Storage { + getAccount: () => Promise + + findOne: ( + _class: Ref>, + query: DocumentQuery, + options?: FindOptions + ) => Promise | undefined> + + getModel: () => Promise<{ hierarchy: Hierarchy, model: ModelDb }> +} diff --git a/packages/api-client/src/rest/utils.ts b/packages/api-client/src/rest/utils.ts new file mode 100644 index 0000000000..d6b87cbc58 --- /dev/null +++ b/packages/api-client/src/rest/utils.ts @@ -0,0 +1,42 @@ +import { uncompress } from 'snappyjs' + +export async function withRetry (fn: () => Promise): Promise { + const maxRetries = 3 + let lastError: any + + for (let attempt = 0; attempt < maxRetries; attempt++) { + try { + return await fn() + } catch (err: any) { + lastError = err + if (attempt === maxRetries - 1) { + console.error('Failed to execute query', err) + throw lastError + } + await new Promise((resolve) => setTimeout(resolve, Math.pow(2, attempt) * 100)) + } + } + throw lastError +} + +function rpcJSONReceiver (key: string, value: any): any { + if (typeof value === 'object' && value !== null) { + if (value.dataType === 'TotalArray') { + return Object.assign(value.value, { total: value.total, lookupMap: value.lookupMap }) + } + } + return value +} + +export async function extractJson (response: Response): Promise { + const encoding = response.headers.get('content-encoding') + if (encoding === 'snappy') { + const buffer = await response.arrayBuffer() + const decompressed = uncompress(buffer) + const decoder = new TextDecoder() + const jsonString = decoder.decode(decompressed) + return JSON.parse(jsonString, rpcJSONReceiver) as T + } + const jsonString = await response.text() + return JSON.parse(jsonString, rpcJSONReceiver) as T +} diff --git a/packages/core/src/client.ts b/packages/core/src/client.ts index b24c37de8c..da6c8b3227 100644 --- a/packages/core/src/client.ts +++ b/packages/core/src/client.ts @@ -404,7 +404,7 @@ async function loadModel ( return { mode: 'addition', current: current.transactions, addition: result.transactions } } -function buildModel ( +export function buildModel ( ctx: MeasureContext, transactions: Tx[], modelFilter: ModelFilter | undefined, diff --git a/rush.json b/rush.json index 0b771f87ab..45c9b8868e 100644 --- a/rush.json +++ b/rush.json @@ -455,6 +455,11 @@ "projectFolder": "packages/api-client", "shouldPublish": false }, + { + "packageName": "@hcengineering/api-tests", + "projectFolder": "ws-tests/api-tests", + "shouldPublish": false + }, { "packageName": "@hcengineering/importer", "projectFolder": "packages/importer", diff --git a/server/core/src/types.ts b/server/core/src/types.ts index af0053c2bd..68c5ac9397 100644 --- a/server/core/src/types.ts +++ b/server/core/src/types.ts @@ -547,6 +547,8 @@ export interface Session { getUser: () => string loadModel: (ctx: ClientSessionCtx, lastModelTx: Timestamp, hash?: string) => Promise + + loadModelRaw: (ctx: ClientSessionCtx, lastModelTx: Timestamp, hash?: string) => Promise getAccount: (ctx: ClientSessionCtx) => Promise getRawAccount: (pipeline: Pipeline) => Account @@ -564,6 +566,7 @@ export interface Session { options?: FindOptions ) => Promise> searchFulltext: (ctx: ClientSessionCtx, query: SearchQuery, options: SearchOptions) => Promise + searchFulltextRaw: (ctx: ClientSessionCtx, query: SearchQuery, options: SearchOptions) => Promise tx: (ctx: ClientSessionCtx, tx: Tx) => Promise txRaw: ( diff --git a/server/middleware/src/model.ts b/server/middleware/src/model.ts index 2a40ef3613..0009037a41 100644 --- a/server/middleware/src/model.ts +++ b/server/middleware/src/model.ts @@ -14,12 +14,19 @@ // import core, { + type Class, type Doc, + type DocumentQuery, + type FindOptions, + type FindResult, type LoadModelResponse, type MeasureContext, + type Ref, + type SessionData, type Timestamp, type Tx, type TxCUD, + DOMAIN_MODEL, DOMAIN_TX, withContext } from '@hcengineering/core' @@ -78,6 +85,19 @@ export class ModelMiddleware extends BaseMiddleware implements Middleware { return allUserTxes.filter((it) => isUserTx(it)) } + findAll( + ctx: MeasureContext, + _class: Ref>, + query: DocumentQuery, + options?: FindOptions + ): Promise> { + const d = this.context.hierarchy.findDomain(_class) + if (d === DOMAIN_MODEL) { + return this.context.modelDb.findAll(_class, query, options) + } + return this.provideFindAll(ctx, _class, query, options) + } + async init (ctx: MeasureContext): Promise { if (this.context.adapterManager == null) { throw new PlatformError(unknownError('Adapter manager should be configured')) diff --git a/server/mongo/src/index.ts b/server/mongo/src/index.ts index 64a7113b38..f44bf62797 100644 --- a/server/mongo/src/index.ts +++ b/server/mongo/src/index.ts @@ -30,6 +30,8 @@ export function createMongoDestroyAdapter (url: string): WorkspaceDestroyAdapter const db = getWorkspaceMongoDB(dbClient, workspace) await db.dropDatabase() }) + } catch (err) { + console.error('Failed to delete workspace', err) } finally { client.close() } diff --git a/server/rpc/src/rpc.ts b/server/rpc/src/rpc.ts index 9771af11ca..8745cfee74 100644 --- a/server/rpc/src/rpc.ts +++ b/server/rpc/src/rpc.ts @@ -53,7 +53,7 @@ export interface HelloResponse extends Response { useCompression?: boolean } -function replacer (key: string, value: any): any { +export function rpcJSONReplacer (key: string, value: any): any { if (Array.isArray(value) && ((value as any).total !== undefined || (value as any).lookupMap !== undefined)) { return { dataType: 'TotalArray', @@ -66,7 +66,7 @@ function replacer (key: string, value: any): any { } } -function receiver (key: string, value: any): any { +export function rpcJSONReceiver (key: string, value: any): any { if (typeof value === 'object' && value !== null) { if (value.dataType === 'TotalArray') { return Object.assign(value.value, { total: value.total, lookupMap: value.lookupMap }) @@ -99,7 +99,7 @@ export class RPCHandler { packr = new Packr({ structuredClone: true, bundleStrings: true, copyBuffers: false }) protoSerialize (object: object, binary: boolean): any { if (!binary) { - return JSON.stringify(object, replacer) + return JSON.stringify(object, rpcJSONReplacer) } return new Uint8Array(this.packr.pack(object)) } @@ -112,7 +112,7 @@ export class RPCHandler { _data = decoder.decode(_data) } try { - return JSON.parse(_data.toString(), receiver) + return JSON.parse(_data.toString(), rpcJSONReceiver) } catch (err: any) { if (((err.message as string) ?? '').includes('Unexpected token')) { return this.packr.unpack(new Uint8Array(data)) @@ -129,7 +129,7 @@ export class RPCHandler { */ serialize (object: Request | Response, binary: boolean): any { if ((object as any).result !== undefined) { - ;(object as any).result = replacer('result', (object as any).result) + ;(object as any).result = rpcJSONReplacer('result', (object as any).result) } return this.protoSerialize(object, binary) } @@ -142,7 +142,7 @@ export class RPCHandler { readResponse(response: any, binary: boolean): Response { const data = this.protoDeserialize(response, binary) if (data.result !== undefined) { - data.result = receiver('result', data.result) + data.result = rpcJSONReceiver('result', data.result) } return data } diff --git a/server/server/src/client.ts b/server/server/src/client.ts index 7bc77f0cab..93be836495 100644 --- a/server/server/src/client.ts +++ b/server/server/src/client.ts @@ -23,10 +23,12 @@ import { type Domain, type FindOptions, type FindResult, + type LoadModelResponse, type MeasureContext, type Ref, type SearchOptions, type SearchQuery, + type SearchResult, type SessionData, type Timestamp, type Tx, @@ -106,6 +108,11 @@ export class ClientSession implements Session { await ctx.sendResponse(ctx.requestId, result) } + async loadModelRaw (ctx: ClientSessionCtx, lastModelTx: Timestamp, hash?: string): Promise { + this.includeSessionContext(ctx.ctx, ctx.pipeline) + return await ctx.ctx.with('load-model', {}, (_ctx) => ctx.pipeline.loadModel(_ctx, lastModelTx, hash)) + } + async getAccount (ctx: ClientSessionCtx): Promise { await ctx.sendResponse(ctx.requestId, this.getRawAccount(ctx.pipeline)) } @@ -163,6 +170,12 @@ export class ClientSession implements Session { await ctx.sendResponse(ctx.requestId, await ctx.pipeline.searchFulltext(ctx.ctx, query, options)) } + async searchFulltextRaw (ctx: ClientSessionCtx, query: SearchQuery, options: SearchOptions): Promise { + this.lastRequest = Date.now() + this.includeSessionContext(ctx.ctx, ctx.pipeline) + return await ctx.pipeline.searchFulltext(ctx.ctx, query, options) + } + async txRaw ( ctx: ClientSessionCtx, tx: Tx diff --git a/server/ws/package.json b/server/ws/package.json index 26a56b85b5..d124072374 100644 --- a/server/ws/package.json +++ b/server/ws/package.json @@ -36,7 +36,8 @@ "prettier": "^3.1.0", "ts-jest": "^29.1.1", "typescript": "^5.3.3", - "@types/body-parser": "~1.19.2" + "@types/body-parser": "~1.19.2", + "@types/morgan": "~1.9.9" }, "dependencies": { "@hcengineering/analytics": "^0.6.0", @@ -53,6 +54,7 @@ "ws": "^8.18.0", "body-parser": "^1.20.2", "snappy": "^7.2.2", - "@hcengineering/api-client": "^0.6.0" + "@hcengineering/api-client": "^0.6.0", + "morgan": "^1.10.0" } } diff --git a/server/ws/src/__tests__/rest.test.ts b/server/ws/src/__tests__/rest.test.ts index 6ad4530733..0e527efb40 100644 --- a/server/ws/src/__tests__/rest.test.ts +++ b/server/ws/src/__tests__/rest.test.ts @@ -15,7 +15,7 @@ import { generateToken } from '@hcengineering/server-token' -import { createRestClient, type RestClient } from '@hcengineering/api-client' +import { createRestClient, createRestTxOperations, type RestClient } from '@hcengineering/api-client' import core, { generateId, getWorkspaceId, @@ -34,12 +34,13 @@ import core, { type Space, type Tx, type TxCreateDoc, + type TxOperations, type TxResult } from '@hcengineering/core' import { ClientSession, startSessionManager, type TSessionManager } from '@hcengineering/server' import { createDummyStorageAdapter, type SessionManager, type WorkspaceLoginInfo } from '@hcengineering/server-core' import { startHttpServer } from '../server_http' -import { genMinModel } from './minmodel' +import { genMinModel, test } from './minmodel' describe('rest-server', () => { async function getModelDb (): Promise<{ modelDb: ModelDb, hierarchy: Hierarchy, txes: Tx[] }> { @@ -57,7 +58,7 @@ describe('rest-server', () => { let shutdown: () => Promise let sessionManager: SessionManager - const port: number = 3330 + const port: number = 11000 beforeAll(async () => { ;({ shutdown, sessionManager } = startSessionManager(new MeasureMetricsContext('test', {}), { @@ -153,6 +154,11 @@ describe('rest-server', () => { return await createRestClient(`http://localhost:${port}`, 'test-ws', token) } + async function connectTx (): Promise { + const token: string = generateToken('user1@site.com', getWorkspaceId('test-ws')) + return await createRestTxOperations(`http://localhost:${port}`, 'test-ws', token) + } + it('get account', async () => { const conn = await connect() const account = await conn.getAccount() @@ -222,4 +228,13 @@ describe('rest-server', () => { const spaces = await conn.findAll(core.class.Space, {}) expect(spaces.length).toBe(3) }) + + it('check-model-operations', async () => { + const conn = await connectTx() + const h = conn.getHierarchy() + const domains = h.domains() + expect(domains.length).toBe(2) + + expect(h.isDerived(test.class.TestComment, core.class.AttachedDoc)).toBe(true) + }) }) diff --git a/server/ws/src/__tests__/server.test.ts b/server/ws/src/__tests__/server.test.ts index e8524542e2..3a12a7ae6c 100644 --- a/server/ws/src/__tests__/server.test.ts +++ b/server/ws/src/__tests__/server.test.ts @@ -45,6 +45,7 @@ import { startHttpServer } from '../server_http' import { genMinModel } from './minmodel' describe('server', () => { + const port = 10000 const handler = new RPCHandler() async function getModelDb (): Promise<{ modelDb: ModelDb, hierarchy: Hierarchy }> { const txes = genMinModel() @@ -95,7 +96,7 @@ describe('server', () => { } }, sessionFactory: (token, workspace) => new ClientSession(token, workspace, true), - port: 3335, + port, brandingMap: {}, serverFactory: startHttpServer, accountsUrl: '', @@ -104,7 +105,7 @@ describe('server', () => { function connect (): WebSocket { const token: string = generateToken('', getWorkspaceId('latest')) - return new WebSocket(`ws://localhost:3335/${token}`) + return new WebSocket(`ws://localhost:${port}/${token}`) } afterAll(async () => { @@ -122,7 +123,7 @@ describe('server', () => { }) it('should not connect to server without token', (done) => { - const conn = new WebSocket('ws://localhost:3335/xyz') + const conn = new WebSocket(`ws://localhost:${port}/xyz`) conn.on('error', () => { conn.close(1000) }) @@ -206,7 +207,7 @@ describe('server', () => { } }, sessionFactory: (token, workspace) => new ClientSession(token, workspace, true), - port: 3336, + port: port + 1, brandingMap: {}, serverFactory: startHttpServer, accountsUrl: '', @@ -214,7 +215,7 @@ describe('server', () => { }) async function findClose (token: string, timeoutPromise: Promise, code: number): Promise { - const newConn = new WebSocket(`ws://localhost:3336/${token}?sessionId=s1`) + const newConn = new WebSocket(`ws://localhost:${port + 1}/${token}?sessionId=s1`) await Promise.race([ timeoutPromise, diff --git a/server/ws/src/rpc.ts b/server/ws/src/rpc.ts index e633cb0fc8..c83225e240 100644 --- a/server/ws/src/rpc.ts +++ b/server/ws/src/rpc.ts @@ -1,4 +1,13 @@ -import type { Class, Doc, MeasureContext, Ref } from '@hcengineering/core' +import core, { + TxProcessor, + type Class, + type Doc, + type MeasureContext, + type Ref, + type SearchOptions, + type SearchQuery, + type TxCUD +} from '@hcengineering/core' import type { ClientSessionCtx, ConnectionSocket, @@ -8,6 +17,8 @@ import type { } from '@hcengineering/server-core' import { decodeToken } from '@hcengineering/server-token' +import { rpcJSONReplacer } from '@hcengineering/rpc' +import { createHash } from 'crypto' import { type Express, type Response as ExpressResponse, type Request } from 'express' import type { OutgoingHttpHeaders } from 'http2' import { compress } from 'snappy' @@ -32,14 +43,31 @@ const sendError = (res: ExpressResponse, code: number, data: any): void => { res.end(JSON.stringify(data)) } -async function sendJson (req: Request, res: ExpressResponse, result: any): Promise { +async function sendJson ( + req: Request, + res: ExpressResponse, + result: any, + extraHeaders?: OutgoingHttpHeaders +): Promise { + // Calculate ETag + let body: any = JSON.stringify(result, rpcJSONReplacer) + + const etag = createHash('sha1').update(body).digest('hex') const headers: OutgoingHttpHeaders = { + ...(extraHeaders ?? {}), 'Content-Type': 'application/json', 'Cache-Control': 'no-cache', - Connection: 'keep-alive', - 'keep-alive': 'timeout=5, max=1000' + connection: 'keep-alive', + 'keep-alive': 'timeout=5, max=1000', + ETag: etag + } + + // Check if the ETag matches + if (req.headers['if-none-match'] === etag) { + res.writeHead(304, headers) + res.end() + return } - let body: any = JSON.stringify(result) const contentEncodings: string[] = typeof req.headers['accept-encoding'] === 'string' @@ -63,7 +91,7 @@ async function sendJson (req: Request, res: ExpressResponse, result: any): Promi break } } - + headers['content-length'] = body.length res.writeHead(200, headers) res.end(body) } @@ -81,41 +109,42 @@ export function registerRPC ( res: ExpressResponse, operation: (ctx: ClientSessionCtx, session: Session) => Promise ): Promise { - if (req.params.workspaceId === undefined || req.params.workspaceId === '') { - res.writeHead(400, {}) - res.end('Missing workspace') - return - } - let token = req.headers.authorization as string - if (token === null) { - sendError(res, 401, { message: 'Missing Authorization header' }) - return - } - const workspaceId = decodeURIComponent(req.params.workspaceId) - token = token.split(' ')[1] - - const decodedToken = decodeToken(token) - if (workspaceId !== decodedToken.workspace.name) { - sendError(res, 401, { message: 'Invalid workspace' }) - return - } - - let transactorRpc = rpcSessions.get(token) - - if (transactorRpc === undefined) { - const cs: ConnectionSocket = createClosingSocket(token, rpcSessions) - const s = await sessions.addSession(ctx, cs, decodedToken, token, pipelineFactory, token) - if (!('session' in s)) { - sendError(res, 401, { - message: 'Failed to create session', - mode: 'specialError' in s ? s.specialError ?? '' : 'upgrading' - }) + try { + if (req.params.workspaceId === undefined || req.params.workspaceId === '') { + res.writeHead(400, {}) + res.end('Missing workspace') return } - transactorRpc = { session: s.session, client: cs, workspaceId: s.workspaceId } - rpcSessions.set(token, transactorRpc) - } - try { + let token = req.headers.authorization as string + if (token === null) { + sendError(res, 401, { message: 'Missing Authorization header' }) + return + } + const workspaceId = decodeURIComponent(req.params.workspaceId) + token = token.split(' ')[1] + + const decodedToken = decodeToken(token) + if (workspaceId !== decodedToken.workspace.name) { + sendError(res, 401, { message: 'Invalid workspace', workspace: decodedToken.workspace.name }) + return + } + + let transactorRpc = rpcSessions.get(token) + + if (transactorRpc === undefined) { + const cs: ConnectionSocket = createClosingSocket(token, rpcSessions) + const s = await sessions.addSession(ctx, cs, decodedToken, token, pipelineFactory, token) + if (!('session' in s)) { + sendError(res, 401, { + message: 'Failed to create session', + mode: 'specialError' in s ? s.specialError ?? '' : 'upgrading' + }) + return + } + transactorRpc = { session: s.session, client: cs, workspaceId: s.workspaceId } + rpcSessions.set(token, transactorRpc) + } + const rpc = transactorRpc await sessions.handleRPC(ctx, rpc.session, rpc.client, async (ctx) => { await operation(ctx, rpc.session) @@ -128,7 +157,11 @@ export function registerRPC ( app.get('/api/v1/ping/:workspaceId', (req, res) => { void withSession(req, res, async (ctx, session) => { await session.ping(ctx) - await sendJson(req, res, { pong: true }) + await sendJson(req, res, { + pong: true, + lastTx: ctx.pipeline.context.lastTx, + lastHash: ctx.pipeline.context.lastHash + }) }) }) @@ -166,6 +199,50 @@ export function registerRPC ( await sendJson(req, res, result) }) }) + + app.get('/api/v1/load-model/:workspaceId', (req, res) => { + void withSession(req, res, async (ctx, session) => { + const lastModelTx = parseInt((req.query.lastModelTx as string) ?? '0') + const lastHash = req.query.lastHash as string + const result = await session.loadModelRaw(ctx, lastModelTx, lastHash) + const txes = Array.isArray(result) ? result : result.transactions + // we need to filter only hierarchy related txes. + const allowedClasess: Ref>[] = [ + core.class.Class, + core.class.Attribute, + core.class.Mixin, + core.class.Type, + core.class.Status, + core.class.Account, + core.class.Permission, + core.class.Space, + core.class.Tx + ] + const h = ctx.pipeline.context.hierarchy + const filtered = txes.filter( + (it) => + TxProcessor.isExtendsCUD(it._class) && + allowedClasess.some((cl) => h.isDerived((it as TxCUD).objectClass, cl)) + ) + + await sendJson(req, res, filtered) + }) + }) + + app.get('/api/v1/search-fulltext/:workspaceId', (req, res) => { + void withSession(req, res, async (ctx, session) => { + const query: SearchQuery = { + query: req.query.query as string, + classes: req.query.classes !== undefined ? JSON.parse(req.query.classes as string) : undefined, + spaces: req.query.spaces !== undefined ? JSON.parse(req.query.spaces as string) : undefined + } + const options: SearchOptions = { + limit: req.query.limit !== undefined ? parseInt(req.query.limit as string) : undefined + } + const result = await session.searchFulltextRaw(ctx, query, options) + await sendJson(req, res, result) + }) + }) } function createClosingSocket (rawToken: string, rpcSessions: Map): ConnectionSocket { diff --git a/server/ws/src/server_http.ts b/server/ws/src/server_http.ts index d9961c4806..bc117d8fb3 100644 --- a/server/ws/src/server_http.ts +++ b/server/ws/src/server_http.ts @@ -49,6 +49,7 @@ import { compress } from 'snappy' import 'utf-8-validate' import { registerRPC } from './rpc' import { retrieveJson } from './utils' +import morgan from 'morgan' import { setImmediate } from 'timers/promises' @@ -82,6 +83,21 @@ export function startHttpServer ( const app = express() app.use(cors()) + const childLogger = ctx.logger.childLogger?.('requests', { + enableConsole: 'true' + }) + const requests = ctx.newChild('requests', {}, {}, childLogger) + + class MyStream { + write (text: string): void { + requests.info(text) + } + } + + const myStream = new MyStream() + + app.use(morgan('short', { stream: myStream })) + const getUsers = (): any => Array.from(sessions.sessions.entries()).map(([k, v]) => v.session.getUser()) app.get('/api/v1/version', (req, res) => { diff --git a/ws-tests/api-tests/.eslintrc.js b/ws-tests/api-tests/.eslintrc.js new file mode 100644 index 0000000000..ce90fb9646 --- /dev/null +++ b/ws-tests/api-tests/.eslintrc.js @@ -0,0 +1,7 @@ +module.exports = { + extends: ['./node_modules/@hcengineering/platform-rig/profiles/node/eslint.config.json'], + parserOptions: { + tsconfigRootDir: __dirname, + project: './tsconfig.json' + } +} diff --git a/ws-tests/api-tests/.gitignore b/ws-tests/api-tests/.gitignore new file mode 100644 index 0000000000..4e9e09b9aa --- /dev/null +++ b/ws-tests/api-tests/.gitignore @@ -0,0 +1,2 @@ +v*.zip +src/uws \ No newline at end of file diff --git a/ws-tests/api-tests/.npmignore b/ws-tests/api-tests/.npmignore new file mode 100644 index 0000000000..e3ec093c38 --- /dev/null +++ b/ws-tests/api-tests/.npmignore @@ -0,0 +1,4 @@ +* +!/lib/** +!CHANGELOG.md +/lib/**/__tests__/ diff --git a/ws-tests/api-tests/config/rig.json b/ws-tests/api-tests/config/rig.json new file mode 100644 index 0000000000..78cc5a1733 --- /dev/null +++ b/ws-tests/api-tests/config/rig.json @@ -0,0 +1,5 @@ +{ + "$schema": "https://developer.microsoft.com/json-schemas/rig-package/rig.schema.json", + "rigPackageName": "@hcengineering/platform-rig", + "rigProfile": "node" +} diff --git a/ws-tests/api-tests/jest.config.js b/ws-tests/api-tests/jest.config.js new file mode 100644 index 0000000000..2cfd408b67 --- /dev/null +++ b/ws-tests/api-tests/jest.config.js @@ -0,0 +1,7 @@ +module.exports = { + preset: 'ts-jest', + testEnvironment: 'node', + testMatch: ['**/?(*.)+(spec|test).[jt]s?(x)'], + roots: ["./src"], + coverageReporters: ["text-summary", "html"] +} diff --git a/ws-tests/api-tests/package.json b/ws-tests/api-tests/package.json new file mode 100644 index 0000000000..482d26ec4d --- /dev/null +++ b/ws-tests/api-tests/package.json @@ -0,0 +1,59 @@ +{ + "name": "@hcengineering/api-tests", + "version": "0.6.0", + "main": "lib/index.js", + "svelte": "src/index.ts", + "types": "types/index.d.ts", + "author": "Anticrm Platform Contributors", + "template": "@hcengineering/node-package-ws", + "license": "EPL-2.0", + "scripts": { + "build": "compile", + "build:watch": "compile", + "test": "echo 'run api-test' for API-tests", + "api-test": "jest --passWithNoTests --silent --forceExit", + "format": "format src", + "_phase:build": "compile transpile src", + "_phase:test": "echo 'run api-test' for API-tests", + "_phase:format": "format src", + "_phase:validate": "compile validate" + }, + "devDependencies": { + "@hcengineering/platform-rig": "^0.6.0", + "@types/compression": "~1.7.2", + "@types/cors": "^2.8.12", + "@types/express": "^4.17.13", + "@types/jest": "^29.5.5", + "@types/node": "~20.11.16", + "@types/ws": "^8.5.11", + "@typescript-eslint/eslint-plugin": "^6.11.0", + "@typescript-eslint/parser": "^6.11.0", + "eslint": "^8.54.0", + "eslint-config-standard-with-typescript": "^40.0.0", + "eslint-plugin-import": "^2.26.0", + "eslint-plugin-n": "^15.4.0", + "eslint-plugin-promise": "^6.1.1", + "jest": "^29.7.0", + "prettier": "^3.1.0", + "ts-jest": "^29.1.1", + "typescript": "^5.3.3", + "@types/body-parser": "~1.19.2", + "@types/morgan": "~1.9.9" + }, + "dependencies": { + "@hcengineering/analytics": "^0.6.0", + "@hcengineering/core": "^0.6.32", + "@hcengineering/platform": "^0.6.11", + "@hcengineering/rpc": "^0.6.5", + "@hcengineering/server-token": "^0.6.11", + "bufferutil": "^4.0.8", + "utf-8-validate": "^6.0.4", + "ws": "^8.18.0", + "snappyjs": "^0.7.0", + "@hcengineering/api-client": "^0.6.0", + "@hcengineering/tracker": "^0.6.24", + "@hcengineering/task": "^0.6.20", + "@hcengineering/contact": "^0.6.24", + "@hcengineering/chunter": "^0.6.20" + } +} diff --git a/ws-tests/api-tests/src/__tests__/rest.test.ts b/ws-tests/api-tests/src/__tests__/rest.test.ts new file mode 100644 index 0000000000..74e1f21930 --- /dev/null +++ b/ws-tests/api-tests/src/__tests__/rest.test.ts @@ -0,0 +1,178 @@ +// +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import { + createRestClient, + createRestTxOperations, + getWorkspaceToken, + type RestClient, + type WorkspaceToken +} from '@hcengineering/api-client' +import core, { generateId, type Space, type TxCreateDoc, type TxOperations } from '@hcengineering/core' + +import chunter from '@hcengineering/chunter' +import contact from '@hcengineering/contact' + +describe('rest-api-server', () => { + const wsName = 'api-tests' + let apiWorkspace1: WorkspaceToken + let apiWorkspace2: WorkspaceToken + + beforeAll(async () => { + apiWorkspace1 = await getWorkspaceToken('http://localhost:8083', { + email: 'user1', + password: '1234', + workspace: wsName + }) + + apiWorkspace2 = await getWorkspaceToken('http://localhost:8083', { + email: 'user1', + password: '1234', + workspace: wsName + '-cr' + }) + }) + + async function connect (ws?: WorkspaceToken): Promise { + const tok = ws ?? apiWorkspace1 + return await createRestClient(tok.endpoint, tok.workspaceId, tok.token) + } + + async function connectTx (ws?: WorkspaceToken): Promise { + const tok = ws ?? apiWorkspace1 + return await createRestTxOperations(tok.endpoint, tok.workspaceId, tok.token) + } + + it('get account', async () => { + const conn = await connect() + const account = await conn.getAccount() + + expect(account.email).toBe('user1') + expect(account.role).toBe('USER') + expect(account._class).toBe(contact.class.PersonAccount) + expect(account.space).toBe(core.space.Model) + expect(account.modifiedBy).toBe(core.account.System) + expect(account.createdBy).toBe(core.account.System) + expect(typeof account.modifiedOn).toBe('number') + expect(typeof account.createdOn).toBe('number') + }) + + it('find spaces', async () => { + const conn = await connect() + const spaces = await conn.findAll(core.class.Space, {}) + expect(spaces.length).toBeGreaterThanOrEqual(20) + const personSpace = spaces.find((it) => it.name === 'Pesonal space' && it.private) + expect(personSpace).not.toBeNull() + }) + + it('find spaces limit', async () => { + const conn = await connect() + const spaces = await conn.findAll(core.class.Space, {}, { limit: 5 }) + expect(spaces.length).toBe(5) + }) + it('find spaces by-name', async () => { + const conn = await connect() + const spaces = await conn.findAll( + contact.class.PersonSpace, + { name: 'Personal space' }, + { + lookup: { + person: contact.class.Person + } + } + ) + expect(spaces.length).toBe(1) + expect(spaces[0].name).toBe('Personal space') + expect(spaces[0].$lookup?.person?.name).toBe('Appleseed,John') + }) + + it('find channels', async () => { + const conn = await connect() + const spaces = await conn.findAll(chunter.class.Channel, {}) + expect(spaces.length).toBeGreaterThanOrEqual(2) + expect(spaces.find((it) => it._id === 'chunter:space:General')).not.toBeNull() + }) + + it('find avg', async () => { + const conn = await connect() + await checkFindPerformance(conn) // 5ms max per operation + }) + + it('find avg-europe', async () => { + const conn = await connect(apiWorkspace2) + await checkFindPerformance(conn) // 5ms max per operation + }) + + it('add space', async () => { + const conn = await connect() + const account = await conn.getAccount() + const spaceName = generateId() + const tx: TxCreateDoc = { + _class: core.class.TxCreateDoc, + space: core.space.Tx, + _id: generateId(), + objectSpace: core.space.Model, + modifiedBy: account._id, + modifiedOn: Date.now(), + attributes: { + name: spaceName, + description: '', + private: false, + archived: false, + members: [], + autoJoin: false + }, + objectClass: core.class.Space, + objectId: generateId() + } + await conn.tx(tx) + const spaces = await conn.findAll(core.class.Space, {}) + expect(spaces.filter((it) => it.name === spaceName).length).toBe(1) + }) + + it('get-model', async () => { + const conn = await connect() + const { hierarchy, model } = await conn.getModel() + + const dsc = hierarchy.getDescendants(core.class.Space) + expect(dsc.length).toBe(32) + expect(model.getObject(core.class.Space)).not.toBeNull() + }) + + it('tx-client', async () => { + const conn = await connectTx() + + const employee = await conn.findAll(contact.mixin.Employee, {}, { limit: 5 }) + + expect(employee.length).toBeGreaterThanOrEqual(1) + expect(employee[0].active).toBe(true) + }) +}) +async function checkFindPerformance (conn: RestClient): Promise { + let ops = 0 + let total = 0 + const attempts = 1000 + for (let i = 0; i < attempts; i++) { + const st = performance.now() + const spaces = await conn.findAll(core.class.Space, {}) + expect(spaces.length).toBeGreaterThanOrEqual(22) + const ed = performance.now() + ops++ + total += ed - st + } + const avg = total / ops + // console.log('ops:', ops, 'total:', total, 'avg:', ) + expect(ops).toEqual(attempts) + expect(avg).toBeLessThan(5) +} diff --git a/ws-tests/api-tests/src/index.ts b/ws-tests/api-tests/src/index.ts new file mode 100644 index 0000000000..e69de29bb2 diff --git a/ws-tests/api-tests/tsconfig.json b/ws-tests/api-tests/tsconfig.json new file mode 100644 index 0000000000..f017cc597c --- /dev/null +++ b/ws-tests/api-tests/tsconfig.json @@ -0,0 +1,10 @@ +{ + "extends": "./node_modules/@hcengineering/platform-rig/profiles/node/tsconfig.json", + + "compilerOptions": { + "rootDir": "./src", + "outDir": "./lib", + "declarationDir": "./types", + "tsBuildInfoFile": ".build/build.tsbuildinfo" + } +} \ No newline at end of file diff --git a/ws-tests/prepare.sh b/ws-tests/prepare.sh index 286693708e..d26fa2154b 100755 --- a/ws-tests/prepare.sh +++ b/ws-tests/prepare.sh @@ -27,3 +27,9 @@ fi ./tool.sh confirm-email user1 ./tool.sh confirm-email user2 + + +./tool.sh create-workspace api-tests -w api-tests +./tool-europe.sh create-workspace api-tests-cr -w api-tests --region 'europe' +./tool.sh assign-workspace user1 api-tests +./tool.sh assign-workspace user1 api-tests-cr diff --git a/ws-tests/tool-europe.sh b/ws-tests/tool-europe.sh new file mode 100755 index 0000000000..78e2fc1d53 --- /dev/null +++ b/ws-tests/tool-europe.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +export MODEL_VERSION=$(node ../common/scripts/show_version.js) +export MINIO_ACCESS_KEY=minioadmin +export MINIO_SECRET_KEY=minioadmin +export MINIO_ENDPOINT=localhost:9002 +export ACCOUNTS_URL=http://localhost:3003 +export TRANSACTOR_URL=ws://localhost:3334 +# export ACCOUNT_DB_URL=postgresql://root@localhost:26258/defaultdb?sslmode=disable +export ACCOUNT_DB_URL=mongodb://localhost:27018 +export MONGO_URL=mongodb://localhost:27018 +export ELASTIC_URL=http://localhost:9201 +export SERVER_SECRET=secret +export DB_URL=postgresql://root@localhost:26258/defaultdb?sslmode=disable + +export REGION_INFO="|America;europe|" # Europe without name will not be available for creation of new workspaces. +export TRANSACTOR_URL="ws://transactor:3334;ws://localhost:3334,ws://transactor-europe:3335;ws://localhost:3335;europe," + +node ${TOOL_OPTIONS} --max-old-space-size=8096 ../dev/tool/bundle/bundle.js $@ \ No newline at end of file diff --git a/ws-tests/tool.sh b/ws-tests/tool.sh index c5003d0c73..9633107e71 100755 --- a/ws-tests/tool.sh +++ b/ws-tests/tool.sh @@ -11,6 +11,6 @@ export ACCOUNT_DB_URL=mongodb://localhost:27018 export MONGO_URL=mongodb://localhost:27018 export ELASTIC_URL=http://localhost:9201 export SERVER_SECRET=secret -export DB_URL=postgresql://root@localhost:26258/defaultdb?sslmode=disable +export DB_URL=$MONGO_URL node ${TOOL_OPTIONS} --max-old-space-size=8096 ../dev/tool/bundle/bundle.js $@ \ No newline at end of file From 2844a83c88d6df7866b757f0e18b7f46e3354318 Mon Sep 17 00:00:00 2001 From: Denis Bykhov Date: Thu, 27 Feb 2025 08:05:04 +0500 Subject: [PATCH 14/16] Fix calendar (#8104) --- services/calendar/pod-calendar/src/calendar.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/services/calendar/pod-calendar/src/calendar.ts b/services/calendar/pod-calendar/src/calendar.ts index a1e99f1169..6e70604c22 100644 --- a/services/calendar/pod-calendar/src/calendar.ts +++ b/services/calendar/pod-calendar/src/calendar.ts @@ -487,7 +487,7 @@ export class CalendarClient { private async updateExtEvent (event: calendar_v3.Schema$Event, current: Event): Promise { this.updateTimer() if (event.status === 'cancelled' && current._class !== calendar.class.ReccuringInstance) { - await this.client.remove(current) + await this.systemTxOp.remove(current) return } const data: Partial> = await this.parseUpdateData(event) @@ -502,7 +502,7 @@ export class CalendarClient { current as ReccuringInstance ) if (Object.keys(diff).length > 0) { - await this.client.update(current, diff) + await this.systemTxOp.update(current, diff) } } else { if (event.recurrence != null) { @@ -517,12 +517,12 @@ export class CalendarClient { current as ReccuringEvent ) if (Object.keys(diff).length > 0) { - await this.client.update(current, diff) + await this.systemTxOp.update(current, diff) } } else { const diff = this.getDiff(data, current) if (Object.keys(diff).length > 0) { - await this.client.update(current, diff) + await this.systemTxOp.update(current, diff) } } } From 5bb36d6eee70c5c89e894fe9856a66f31f10d0e5 Mon Sep 17 00:00:00 2001 From: Denis Bykhov Date: Thu, 27 Feb 2025 14:52:17 +0500 Subject: [PATCH 15/16] Fix join (#8105) Signed-off-by: Denis Bykhov --- .../src/components/Join.svelte | 32 +++++++------------ tests/sanity/tests/model/signin-page.ts | 1 + tests/sanity/tests/model/signup-page.ts | 14 ++------ tests/sanity/tests/workspace/create.spec.ts | 3 +- .../sanity/tests/workspace/create.spec.ts | 3 +- 5 files changed, 18 insertions(+), 35 deletions(-) diff --git a/plugins/login-resources/src/components/Join.svelte b/plugins/login-resources/src/components/Join.svelte index d97080cd1a..68a5bb6149 100644 --- a/plugins/login-resources/src/components/Join.svelte +++ b/plugins/login-resources/src/components/Join.svelte @@ -28,7 +28,7 @@ const location = getCurrentLocation() Analytics.handleEvent('invite_link_activated') - let page = 'login' + let page = 'signUp' $: fields = page === 'login' @@ -60,7 +60,7 @@ let status = OK $: action = { - i18n: login.string.Join, + i18n: page === 'login' ? login.string.Join : login.string.SignUp, func: async () => { status = new Status(Severity.INFO, login.status.ConnectingToServer, {}) @@ -96,23 +96,15 @@ } } - const signUpAction: BottomAction = { - caption: login.string.DoNotHaveAnAccount, - i18n: login.string.SignUp, - func: () => (page = 'signUp') - } - - const loginJoinAction: BottomAction = { - caption: login.string.HaveAccount, - i18n: login.string.LogIn, - func: () => (page = 'login') - } - - $: bottom = page === 'login' ? [signUpAction] : [loginJoinAction] - $: secondaryButtonLabel = page === 'login' ? login.string.SignUp : undefined - $: secondaryButtonAction = () => { - page = 'signUp' - } + $: secondaryButtonLabel = page === 'login' ? login.string.SignUp : login.string.Join + $: secondaryButtonAction = + page === 'login' + ? () => { + page = 'signUp' + } + : () => { + page = 'login' + } onMount(() => { void check() @@ -150,6 +142,6 @@ {action} {secondaryButtonLabel} {secondaryButtonAction} - bottomActions={[...bottom, loginAction, recoveryAction]} + bottomActions={[loginAction, recoveryAction]} withProviders /> diff --git a/tests/sanity/tests/model/signin-page.ts b/tests/sanity/tests/model/signin-page.ts index f72f3036bb..e2beb145f5 100644 --- a/tests/sanity/tests/model/signin-page.ts +++ b/tests/sanity/tests/model/signin-page.ts @@ -15,6 +15,7 @@ export class SignInJoinPage extends CommonPage { buttonJoin = (): Locator => this.page.locator('button', { hasText: 'Join' }) async join (data: Pick): Promise { + await this.buttonJoin().click() await this.inputEmail().fill(data.email) await this.inputPassword().fill(data.password) expect(await this.buttonJoin().isEnabled()).toBe(true) diff --git a/tests/sanity/tests/model/signup-page.ts b/tests/sanity/tests/model/signup-page.ts index 55f9911dc2..bec5d76885 100644 --- a/tests/sanity/tests/model/signup-page.ts +++ b/tests/sanity/tests/model/signup-page.ts @@ -16,7 +16,6 @@ export class SignUpPage extends CommonPage { inputNewPassword = (): Locator => this.page.locator('//div[text()="Password"]/../input') inputRepeatPassword = (): Locator => this.page.locator('//div[text()="Repeat password"]/../input') buttonSignUp = (): Locator => this.page.locator('button', { hasText: 'Sign Up' }) - buttonJoin = (): Locator => this.page.locator('button', { hasText: 'Join' }) async enterFirstName (firstName: string): Promise { await this.inputFirstName().fill(firstName) @@ -42,21 +41,14 @@ export class SignUpPage extends CommonPage { await this.buttonSignUp().click() } - async signUp (data: SignUpData, mode: 'join' | 'signup' = 'signup'): Promise { + async signUp (data: SignUpData): Promise { await this.enterFirstName(data.firstName) await this.enterLastName(data.lastName) await this.enterEmail(data.email) await this.enterPassword(data.password) await this.enterRepeatPassword(data.password) - switch (mode) { - case 'join': - expect(await this.buttonJoin().isEnabled()).toBe(true) - await this.buttonJoin().click() - break - case 'signup': - expect(await this.buttonSignUp().isEnabled()).toBe(true) - await this.buttonSignUp().click() - } + expect(await this.buttonSignUp().isEnabled()).toBe(true) + await this.buttonSignUp().click() } async checkIfSignUpButtonIsDisabled (): Promise { diff --git a/tests/sanity/tests/workspace/create.spec.ts b/tests/sanity/tests/workspace/create.spec.ts index d940020dae..cdd46e0432 100644 --- a/tests/sanity/tests/workspace/create.spec.ts +++ b/tests/sanity/tests/workspace/create.spec.ts @@ -143,9 +143,8 @@ test.describe('Workspace tests', () => { password: '1234' } - await page2.getByRole('link', { name: 'Sign Up' }).click() const signUpPage2 = new SignUpPage(page2) - await signUpPage2.signUp(newUser2, 'join') + await signUpPage2.signUp(newUser2) const leftSideMenuPage2 = new LeftSideMenuPage(page2) await leftSideMenuPage2.clickTracker() diff --git a/ws-tests/sanity/tests/workspace/create.spec.ts b/ws-tests/sanity/tests/workspace/create.spec.ts index a0ff27f3ba..17c055e987 100644 --- a/ws-tests/sanity/tests/workspace/create.spec.ts +++ b/ws-tests/sanity/tests/workspace/create.spec.ts @@ -144,9 +144,8 @@ test.describe('Workspace tests', () => { password: '1234' } - await page2.getByRole('link', { name: 'Sign Up' }).click() const signUpPage2 = new SignUpPage(page2) - await signUpPage2.signUp(newUser2, 'join') + await signUpPage2.signUp(newUser2) const leftSideMenuPage2 = new LeftSideMenuPage(page2) await leftSideMenuPage2.clickTracker() From e0ec8d420647866ae4cfd29946496b49106838ce Mon Sep 17 00:00:00 2001 From: Andrey Sobolev Date: Thu, 27 Feb 2025 17:00:00 +0700 Subject: [PATCH 16/16] QFIX: Fix REST API + few minors (#8108) Signed-off-by: Andrey Sobolev --- dev/prod/public/huly/site.webmanifest | 4 ++- dev/prod/src/analytics.ts | 4 +-- packages/api-client/src/rest/rest.ts | 2 +- server/workspace-service/src/service.ts | 28 +++++++++++++------ server/ws/src/__tests__/rest.test.ts | 12 ++++---- ws-tests/api-tests/src/__tests__/rest.test.ts | 22 +++++++-------- 6 files changed, 41 insertions(+), 31 deletions(-) diff --git a/dev/prod/public/huly/site.webmanifest b/dev/prod/public/huly/site.webmanifest index 3d240ef504..174bdb93de 100644 --- a/dev/prod/public/huly/site.webmanifest +++ b/dev/prod/public/huly/site.webmanifest @@ -5,5 +5,7 @@ { "src": "/icon-192.png", "type": "image/png", "sizes": "192x192" }, { "src": "/icon-512.png", "type": "image/png", "sizes": "512x512" }, { "src": "/icon-1024.png", "type": "image/png", "sizes": "1024x1024" } - ] + ], + "start_url": "/", + "scope": "/" } \ No newline at end of file diff --git a/dev/prod/src/analytics.ts b/dev/prod/src/analytics.ts index ac19bdf2d9..3a040258aa 100644 --- a/dev/prod/src/analytics.ts +++ b/dev/prod/src/analytics.ts @@ -3,7 +3,6 @@ // import { type AnalyticProvider, Analytics } from "@hcengineering/analytics" -import { AnalyticsCollectorProvider } from './analytics/analyticsCollector' import { PosthogAnalyticProvider } from "./analytics/posthog" import { SentryAnalyticProvider } from "./analytics/sentry" import { type Config } from "./platform" @@ -11,8 +10,7 @@ import { type Config } from "./platform" export function configureAnalytics (config: Config) { const providers: AnalyticProvider[] = [ new SentryAnalyticProvider, - new PosthogAnalyticProvider, - new AnalyticsCollectorProvider + new PosthogAnalyticProvider ] for (const provider of providers) { Analytics.init(provider, config) diff --git a/packages/api-client/src/rest/rest.ts b/packages/api-client/src/rest/rest.ts index a483c94c1f..fb2995eb9c 100644 --- a/packages/api-client/src/rest/rest.ts +++ b/packages/api-client/src/rest/rest.ts @@ -39,7 +39,7 @@ import { PlatformError, unknownError } from '@hcengineering/platform' import type { RestClient } from './types' import { extractJson, withRetry } from './utils' -export async function createRestClient (endpoint: string, workspaceId: string, token: string): Promise { +export function createRestClient (endpoint: string, workspaceId: string, token: string): RestClient { return new RestClientImpl(endpoint, workspaceId, token) } diff --git a/server/workspace-service/src/service.ts b/server/workspace-service/src/service.ts index 9971ed683f..58a3fc6fb9 100644 --- a/server/workspace-service/src/service.ts +++ b/server/workspace-service/src/service.ts @@ -70,6 +70,8 @@ import { import { buildStorageFromConfig, storageConfigFromEnv } from '@hcengineering/server-storage' import { createWorkspace, upgradeWorkspace } from './ws-operations' +const dbCleanTreshold = 256 // Cleanup workspaces if less 256mb + export interface WorkspaceOptions { errorHandler: (workspace: BaseWorkspaceInfo, error: any) => Promise force: boolean @@ -372,15 +374,19 @@ export class WorkspaceWorker { /** * If onlyDrop is true, will drop workspace from database, overwize remove only indexes and do full reindex. */ - async doCleanup (ctx: MeasureContext, workspace: BaseWorkspaceInfo, onlyDrop: boolean): Promise { + async doCleanup (ctx: MeasureContext, workspace: BaseWorkspaceInfo, cleanIndexes: boolean): Promise { const { dbUrl } = prepareTools([]) const adapter = getWorkspaceDestroyAdapter(dbUrl) await adapter.deleteWorkspace(ctx, sharedPipelineContextVars, { name: workspace.workspace, uuid: workspace.uuid }) - await this.doReindexFulltext(ctx, workspace, onlyDrop) + await this.doReindexFulltext(ctx, workspace, cleanIndexes) } - private async doReindexFulltext (ctx: MeasureContext, workspace: BaseWorkspaceInfo, onlyDrop: boolean): Promise { + private async doReindexFulltext ( + ctx: MeasureContext, + workspace: BaseWorkspaceInfo, + clearIndexes: boolean + ): Promise { if (this.fulltextUrl !== undefined) { const token = generateToken(systemAccountEmail, { name: workspace.workspace }, { service: 'workspace' }) @@ -390,7 +396,7 @@ export class WorkspaceWorker { headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ token, onlyDrop }) + body: JSON.stringify({ token, onlyDrop: clearIndexes }) }) if (!res.ok) { throw new Error(`HTTP Error ${res.status} ${res.statusText}`) @@ -491,11 +497,15 @@ export class WorkspaceWorker { // We should remove DB, not storages. await sendEvent('migrate-clean-started', 0) await this.sendTransactorMaitenance(token, { name: workspace.workspace }) - try { - await this.doCleanup(ctx, workspace, false) - } catch (err: any) { - Analytics.handleError(err) - return + + const sz = workspace.backupInfo?.backupSize ?? 0 + if (sz <= dbCleanTreshold) { + try { + await this.doCleanup(ctx, workspace, false) + } catch (err: any) { + Analytics.handleError(err) + return + } } await sendEvent('migrate-clean-done', 0) break diff --git a/server/ws/src/__tests__/rest.test.ts b/server/ws/src/__tests__/rest.test.ts index 0e527efb40..edcca6a125 100644 --- a/server/ws/src/__tests__/rest.test.ts +++ b/server/ws/src/__tests__/rest.test.ts @@ -149,9 +149,9 @@ describe('rest-server', () => { await shutdown() }) - async function connect (): Promise { + function connect (): RestClient { const token: string = generateToken('user1@site.com', getWorkspaceId('test-ws')) - return await createRestClient(`http://localhost:${port}`, 'test-ws', token) + return createRestClient(`http://localhost:${port}`, 'test-ws', token) } async function connectTx (): Promise { @@ -160,7 +160,7 @@ describe('rest-server', () => { } it('get account', async () => { - const conn = await connect() + const conn = connect() const account = await conn.getAccount() expect(account.email).toBe('user1@site.com') @@ -175,7 +175,7 @@ describe('rest-server', () => { }) it('find spaces', async () => { - const conn = await connect() + const conn = connect() const spaces = await conn.findAll(core.class.Space, {}) expect(spaces.length).toBe(2) expect(spaces[0].name).toBe('Sp1') @@ -183,7 +183,7 @@ describe('rest-server', () => { }) it('find avg', async () => { - const conn = await connect() + const conn = connect() let ops = 0 let total = 0 const attempts = 1000 @@ -204,7 +204,7 @@ describe('rest-server', () => { }) it('add space', async () => { - const conn = await connect() + const conn = connect() const account = await conn.getAccount() const tx: TxCreateDoc = { _class: core.class.TxCreateDoc, diff --git a/ws-tests/api-tests/src/__tests__/rest.test.ts b/ws-tests/api-tests/src/__tests__/rest.test.ts index 74e1f21930..fd38e7af75 100644 --- a/ws-tests/api-tests/src/__tests__/rest.test.ts +++ b/ws-tests/api-tests/src/__tests__/rest.test.ts @@ -44,9 +44,9 @@ describe('rest-api-server', () => { }) }) - async function connect (ws?: WorkspaceToken): Promise { + function connect (ws?: WorkspaceToken): RestClient { const tok = ws ?? apiWorkspace1 - return await createRestClient(tok.endpoint, tok.workspaceId, tok.token) + return createRestClient(tok.endpoint, tok.workspaceId, tok.token) } async function connectTx (ws?: WorkspaceToken): Promise { @@ -55,7 +55,7 @@ describe('rest-api-server', () => { } it('get account', async () => { - const conn = await connect() + const conn = connect() const account = await conn.getAccount() expect(account.email).toBe('user1') @@ -69,7 +69,7 @@ describe('rest-api-server', () => { }) it('find spaces', async () => { - const conn = await connect() + const conn = connect() const spaces = await conn.findAll(core.class.Space, {}) expect(spaces.length).toBeGreaterThanOrEqual(20) const personSpace = spaces.find((it) => it.name === 'Pesonal space' && it.private) @@ -77,12 +77,12 @@ describe('rest-api-server', () => { }) it('find spaces limit', async () => { - const conn = await connect() + const conn = connect() const spaces = await conn.findAll(core.class.Space, {}, { limit: 5 }) expect(spaces.length).toBe(5) }) it('find spaces by-name', async () => { - const conn = await connect() + const conn = connect() const spaces = await conn.findAll( contact.class.PersonSpace, { name: 'Personal space' }, @@ -98,24 +98,24 @@ describe('rest-api-server', () => { }) it('find channels', async () => { - const conn = await connect() + const conn = connect() const spaces = await conn.findAll(chunter.class.Channel, {}) expect(spaces.length).toBeGreaterThanOrEqual(2) expect(spaces.find((it) => it._id === 'chunter:space:General')).not.toBeNull() }) it('find avg', async () => { - const conn = await connect() + const conn = connect() await checkFindPerformance(conn) // 5ms max per operation }) it('find avg-europe', async () => { - const conn = await connect(apiWorkspace2) + const conn = connect(apiWorkspace2) await checkFindPerformance(conn) // 5ms max per operation }) it('add space', async () => { - const conn = await connect() + const conn = connect() const account = await conn.getAccount() const spaceName = generateId() const tx: TxCreateDoc = { @@ -142,7 +142,7 @@ describe('rest-api-server', () => { }) it('get-model', async () => { - const conn = await connect() + const conn = connect() const { hierarchy, model } = await conn.getModel() const dsc = hierarchy.getDescendants(core.class.Space)