diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d13d5a4be6..da8809c6c9 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -485,6 +485,11 @@ jobs: cd ./ws-tests export DO_CLEAN=true ./prepare.sh + - name: Run API tests + run: | + cd ./ws-tests/api-tests + node ../../common/scripts/install-run-rush.js validate --to @hcengineering/api-tests + node ../../common/scripts/install-run-rushx.js api-test --verbose - name: Install Playwright run: | cd ./ws-tests/sanity diff --git a/common/config/rush/pnpm-lock.yaml b/common/config/rush/pnpm-lock.yaml index d64f04f26a..33a41bd0c3 100644 --- a/common/config/rush/pnpm-lock.yaml +++ b/common/config/rush/pnpm-lock.yaml @@ -106,6 +106,9 @@ importers: '@rush-temp/api-client': specifier: file:./projects/api-client.tgz version: file:projects/api-client.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(bufferutil@4.0.8)(esbuild@0.24.2)(utf-8-validate@6.0.4) + '@rush-temp/api-tests': + specifier: file:./projects/api-tests.tgz + version: file:projects/api-tests.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)) '@rush-temp/attachment': specifier: file:./projects/attachment.tgz version: file:projects/attachment.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(@types/node@20.11.19)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)) @@ -1095,7 +1098,7 @@ importers: version: file:projects/tests-sanity.tgz '@rush-temp/tests-ws-sanity': specifier: file:./projects/tests-ws-sanity.tgz - version: file:projects/tests-ws-sanity.tgz + version: file:projects/tests-ws-sanity.tgz(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)) '@rush-temp/text': specifier: file:./projects/text.tgz version: file:projects/text.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(@types/node@20.11.19)(babel-jest@29.7.0(@babel/core@7.23.9))(bufferutil@4.0.8)(esbuild@0.24.2)(prosemirror-inputrules@1.4.0)(prosemirror-model@1.24.1)(prosemirror-state@1.4.3)(prosemirror-view@1.37.2)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3))(utf-8-validate@6.0.4) @@ -3864,6 +3867,10 @@ packages: resolution: {integrity: sha512-gq68pTrBheqcmVrkbiqYY+6p80DQHIsJvC1nIvhRfwrj3eR1ir5fVsW5t2Gs8fvN/Um8nH0MnkTUGavVrPWIdw==, tarball: file:projects/api-client.tgz} version: 0.0.0 + '@rush-temp/api-tests@file:projects/api-tests.tgz': + resolution: {integrity: sha512-gCHsoeILAG/lcLoL1r7XjxDLqdEcEMphk+FU2tHcqqXolwuytD2Rnd9Ko+IVta8kGBTMwCk8DiX9qdICHL7NbQ==, tarball: file:projects/api-tests.tgz} + version: 0.0.0 + '@rush-temp/attachment-assets@file:projects/attachment-assets.tgz': resolution: {integrity: sha512-C4ZvrB9y7H0bh1vbkqmKw9tIrwFAHUG7P6kPI+tjRPRKQNGAn61eiVSTpM8BCGNY/mFC5b+uMy0us0UaqvPy+A==, tarball: file:projects/attachment-assets.tgz} version: 0.0.0 @@ -4589,7 +4596,7 @@ packages: version: 0.0.0 '@rush-temp/pod-calendar@file:projects/pod-calendar.tgz': - resolution: {integrity: sha512-BvN38ScXSeB5Mui24Sawekl81QZMFZ47EK6KdIqRJJQwM+Qj0owWul3y/r7ZtNAfKnD9bV3AeEWaqkZ8EXYjwg==, tarball: file:projects/pod-calendar.tgz} + resolution: {integrity: sha512-gUk3jshnHJ0CUucxZ1Stk8oQ2DTdSVfJzFHIl/EBUISbRv/YySoXesQCnqfn1kVpqOBZdIvI/N4NRAL/yREbWw==, tarball: file:projects/pod-calendar.tgz} version: 0.0.0 '@rush-temp/pod-collaborator@file:projects/pod-collaborator.tgz': @@ -5065,7 +5072,7 @@ packages: version: 0.0.0 '@rush-temp/server-ws@file:projects/server-ws.tgz': - resolution: {integrity: sha512-3jjoKDZ0NAs4XRajHH6n4EgS00GwQY0Zga1JlG8XIeSPjAGphGUYbCaHQuJbcddPAYRdT632gK+119U4RvGSxw==, tarball: file:projects/server-ws.tgz} + resolution: {integrity: sha512-kVd/j4HPgbCYMMEqEkxRQiGqM0es47rgzzPdQvTDEFs6HtkyWxy8pbzRqA6HXJIDbNY5WLeTbtiLV37db6q33A==, tarball: file:projects/server-ws.tgz} version: 0.0.0 '@rush-temp/server@file:projects/server.tgz': @@ -5181,7 +5188,7 @@ packages: version: 0.0.0 '@rush-temp/tests-ws-sanity@file:projects/tests-ws-sanity.tgz': - resolution: {integrity: sha512-2y+gKxse3ozzKeoi1TF7Qfv7Y1fWy9Z6HiqSsEeQgGPpv396cH7ZfpSJ/qwsxO66c6iq8LUuBtzZ4AlquHfG3w==, tarball: file:projects/tests-ws-sanity.tgz} + resolution: {integrity: sha512-YLGIPrvHv+48mtZrd5T8uJpGCkoGXjBtuxMvcD+Q3CNbHGTcYLxlznkmlNXa6wPm0G6v2VrpRuOfXAuAqlYnHg==, tarball: file:projects/tests-ws-sanity.tgz} version: 0.0.0 '@rush-temp/text-core@file:projects/text-core.tgz': @@ -16161,6 +16168,41 @@ snapshots: - supports-color - utf-8-validate + '@rush-temp/api-tests@file:projects/api-tests.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3))': + dependencies: + '@types/body-parser': 1.19.5 + '@types/compression': 1.7.5 + '@types/cors': 2.8.17 + '@types/express': 4.17.21 + '@types/jest': 29.5.12 + '@types/morgan': 1.9.9 + '@types/node': 20.11.19 + '@types/ws': 8.5.11 + '@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.7.3) + '@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.7.3) + bufferutil: 4.0.8 + eslint: 8.56.0 + eslint-config-standard-with-typescript: 40.0.0(@typescript-eslint/eslint-plugin@6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.3.3))(eslint-plugin-import@2.29.1(eslint@8.56.0))(eslint-plugin-n@15.7.0(eslint@8.56.0))(eslint-plugin-promise@6.1.1(eslint@8.56.0))(eslint@8.56.0)(typescript@5.7.3) + eslint-plugin-import: 2.29.1(eslint@8.56.0) + eslint-plugin-n: 15.7.0(eslint@8.56.0) + eslint-plugin-promise: 6.1.1(eslint@8.56.0) + jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)) + prettier: 3.2.5 + snappyjs: 0.7.0 + ts-jest: 29.1.2(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(jest@29.7.0(@types/node@20.11.19)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)))(typescript@5.7.3) + typescript: 5.7.3 + utf-8-validate: 6.0.4 + ws: 8.18.0(bufferutil@4.0.8)(utf-8-validate@6.0.4) + transitivePeerDependencies: + - '@babel/core' + - '@jest/types' + - babel-jest + - babel-plugin-macros + - esbuild + - node-notifier + - supports-color + - ts-node + '@rush-temp/attachment-assets@file:projects/attachment-assets.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3))': dependencies: '@types/jest': 29.5.12 @@ -20588,7 +20630,6 @@ snapshots: '@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.3.3) '@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.3.3) cors: 2.8.5 - cross-env: 7.0.3 dotenv: 16.0.3 esbuild: 0.24.2 eslint: 8.56.0 @@ -24085,6 +24126,7 @@ snapshots: '@types/cors': 2.8.17 '@types/express': 4.17.21 '@types/jest': 29.5.12 + '@types/morgan': 1.9.9 '@types/node': 20.11.19 '@types/ws': 8.5.11 '@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.3.3) @@ -24099,6 +24141,7 @@ snapshots: eslint-plugin-promise: 6.1.1(eslint@8.56.0) express: 4.21.2 jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)) + morgan: 1.10.0 prettier: 3.2.5 snappy: 7.2.2 ts-jest: 29.1.2(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(jest@29.7.0(@types/node@20.11.19)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)))(typescript@5.3.3) @@ -24939,7 +24982,7 @@ snapshots: transitivePeerDependencies: - supports-color - '@rush-temp/tests-ws-sanity@file:projects/tests-ws-sanity.tgz': + '@rush-temp/tests-ws-sanity@file:projects/tests-ws-sanity.tgz(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3))': dependencies: '@faker-js/faker': 8.4.1 '@playwright/test': 1.49.1 @@ -24956,10 +24999,14 @@ snapshots: eslint-plugin-import: 2.29.1(eslint@8.56.0) eslint-plugin-n: 15.7.0(eslint@8.56.0) eslint-plugin-promise: 6.1.1(eslint@8.56.0) + jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)) prettier: 3.2.5 typescript: 5.7.3 transitivePeerDependencies: + - babel-plugin-macros + - node-notifier - supports-color + - ts-node '@rush-temp/text-core@file:projects/text-core.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(@types/node@20.11.19)(babel-jest@29.7.0(@babel/core@7.23.9))(bufferutil@4.0.8)(esbuild@0.24.2)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3))(utf-8-validate@6.0.4)': dependencies: diff --git a/packages/api-client/src/client.ts b/packages/api-client/src/client.ts index 61d8483b1d..99fa3f192d 100644 --- a/packages/api-client/src/client.ts +++ b/packages/api-client/src/client.ts @@ -13,6 +13,7 @@ // limitations under the License. // +import client, { clientId } from '@hcengineering/client' import { type Account, type Class, @@ -26,18 +27,17 @@ import { type ModelDb, type Ref, type Space, - type WithLookup, type TxResult, - DocumentUpdate, - TxOperations, - AttachedDoc, + type WithLookup, AttachedData, + AttachedDoc, + DocumentUpdate, Mixin, - MixinUpdate, MixinData, + MixinUpdate, + TxOperations, generateId } from '@hcengineering/core' -import client, { clientId } from '@hcengineering/client' import { addLocation, getResource } from '@hcengineering/platform' import { login, selectWorkspace } from './account' @@ -49,7 +49,7 @@ import { MarkupContent, createMarkupOperations } from './markup' -import { type PlatformClient, type ConnectOptions, WithMarkup } from './types' +import { type ConnectOptions, type PlatformClient, WithMarkup } from './types' /** * Create platform client @@ -278,11 +278,17 @@ class PlatformClientImpl implements PlatformClient { } } -async function getWorkspaceToken ( +export interface WorkspaceToken { + endpoint: string + token: string + workspaceId: string +} + +export async function getWorkspaceToken ( url: string, options: ConnectOptions, config?: ServerConfig -): Promise<{ endpoint: string, token: string, workspaceId: string }> { +): Promise { config ??= await loadServerConfig(url) let token: string diff --git a/packages/api-client/src/rest.ts b/packages/api-client/src/rest.ts deleted file mode 100644 index 23648ca2d1..0000000000 --- a/packages/api-client/src/rest.ts +++ /dev/null @@ -1,130 +0,0 @@ -// -// Copyright © 2025 Hardcore Engineering Inc. -// -// Licensed under the Eclipse Public License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. You may -// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// -// See the License for the specific language governing permissions and -// limitations under the License. -// - -import { - type Account, - type Class, - type Doc, - type DocumentQuery, - type FindOptions, - type FindResult, - type Ref, - type Storage, - type Tx, - type TxResult, - type WithLookup, - concatLink -} from '@hcengineering/core' - -import { PlatformError, unknownError } from '@hcengineering/platform' - -import { uncompress } from 'snappyjs' - -export interface RestClient extends Storage { - getAccount: () => Promise - - findOne: ( - _class: Ref>, - query: DocumentQuery, - options?: FindOptions - ) => Promise | undefined> -} - -export async function createRestClient (endpoint: string, workspaceId: string, token: string): Promise { - return new RestClientImpl(endpoint, workspaceId, token) -} - -class RestClientImpl implements RestClient { - constructor ( - readonly endpoint: string, - readonly workspace: string, - readonly token: string - ) {} - - async findAll( - _class: Ref>, - query: DocumentQuery, - options?: FindOptions - ): Promise> { - const params = new URLSearchParams() - params.append('class', _class) - if (query !== undefined && Object.keys(query).length > 0) { - params.append('query', JSON.stringify(query)) - } - if (options !== undefined && Object.keys(options).length > 0) { - params.append('options', JSON.stringify(options)) - } - const response = await fetch(concatLink(this.endpoint, `/api/v1/find-all/${this.workspace}?${params.toString()}`), { - method: 'GET', - headers: { - 'Content-Type': 'application/json', - Authorization: 'Bearer ' + this.token, - 'accept-encoding': 'snappy, gzip' - }, - keepalive: true - }) - if (!response.ok) { - throw new PlatformError(unknownError(response.statusText)) - } - const encoding = response.headers.get('content-encoding') - if (encoding === 'snappy') { - const buffer = await response.arrayBuffer() - const decompressed = uncompress(buffer) - const decoder = new TextDecoder() - const jsonString = decoder.decode(decompressed) - return JSON.parse(jsonString) as FindResult - } - return (await response.json()) as FindResult - } - - async getAccount (): Promise { - const response = await fetch(concatLink(this.endpoint, `/api/v1/account/${this.workspace}`), { - method: 'GET', - headers: { - 'Content-Type': 'application/json', - Authorization: 'Bearer ' + this.token - }, - keepalive: true - }) - if (!response.ok) { - throw new PlatformError(unknownError(response.statusText)) - } - return (await response.json()) as Account - } - - async findOne( - _class: Ref>, - query: DocumentQuery, - options?: FindOptions - ): Promise | undefined> { - return (await this.findAll(_class, query, { ...options, limit: 1 })).shift() - } - - async tx (tx: Tx): Promise { - const response = await fetch(concatLink(this.endpoint, `/api/v1/tx/${this.workspace}`), { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - Authorization: 'Bearer ' + this.token - }, - keepalive: true, - body: JSON.stringify(tx) - }) - if (!response.ok) { - throw new PlatformError(unknownError(response.statusText)) - } - return (await response.json()) as TxResult - } -} diff --git a/packages/api-client/src/rest/index.ts b/packages/api-client/src/rest/index.ts new file mode 100644 index 0000000000..4397c92ecd --- /dev/null +++ b/packages/api-client/src/rest/index.ts @@ -0,0 +1,18 @@ +// +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +export { createRestClient } from './rest' +export { createRestTxOperations } from './tx' +export * from './types' diff --git a/packages/api-client/src/rest/rest.ts b/packages/api-client/src/rest/rest.ts new file mode 100644 index 0000000000..a483c94c1f --- /dev/null +++ b/packages/api-client/src/rest/rest.ts @@ -0,0 +1,199 @@ +// +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import { + type Account, + type Class, + type Doc, + type DocumentQuery, + type FindOptions, + type FindResult, + Hierarchy, + MeasureMetricsContext, + ModelDb, + type Ref, + type SearchOptions, + type SearchQuery, + type SearchResult, + type Tx, + type TxResult, + type WithLookup, + buildModel, + concatLink +} from '@hcengineering/core' + +import { PlatformError, unknownError } from '@hcengineering/platform' + +import type { RestClient } from './types' +import { extractJson, withRetry } from './utils' + +export async function createRestClient (endpoint: string, workspaceId: string, token: string): Promise { + return new RestClientImpl(endpoint, workspaceId, token) +} + +export class RestClientImpl implements RestClient { + endpoint: string + constructor ( + endpoint: string, + readonly workspace: string, + readonly token: string + ) { + this.endpoint = endpoint.replace('ws', 'http') + } + + jsonHeaders (): Record { + return { + 'Content-Type': 'application/json', + Authorization: 'Bearer ' + this.token, + 'accept-encoding': 'snappy, gzip' + } + } + + requestInit (): RequestInit { + return { + method: 'GET', + keepalive: true, + headers: this.jsonHeaders() + } + } + + async findAll( + _class: Ref>, + query: DocumentQuery, + options?: FindOptions + ): Promise> { + const params = new URLSearchParams() + params.append('class', _class) + if (query !== undefined && Object.keys(query).length > 0) { + params.append('query', JSON.stringify(query)) + } + if (options !== undefined && Object.keys(options).length > 0) { + params.append('options', JSON.stringify(options)) + } + const requestUrl = concatLink(this.endpoint, `/api/v1/find-all/${this.workspace}?${params.toString()}`) + const result = await withRetry(async () => { + const response = await fetch(requestUrl, this.requestInit()) + if (!response.ok) { + throw new PlatformError(unknownError(response.statusText)) + } + return await extractJson>(response) + }) + + if (result.lookupMap !== undefined) { + // We need to extract lookup map to document lookups + for (const d of result) { + if (d.$lookup !== undefined) { + for (const [k, v] of Object.entries(d.$lookup)) { + if (!Array.isArray(v)) { + d.$lookup[k] = result.lookupMap[v as any] + } else { + d.$lookup[k] = v.map((it) => result.lookupMap?.[it]) + } + } + } + } + delete result.lookupMap + } + + // We need to revert deleted query simple values. + // We need to get rid of simple query parameters matched in documents + for (const doc of result) { + if (doc._class == null) { + doc._class = _class + } + for (const [k, v] of Object.entries(query)) { + if (typeof v === 'string' || typeof v === 'number' || typeof v === 'boolean') { + if (doc[k] == null) { + doc[k] = v + } + } + } + } + + return result + } + + async getAccount (): Promise { + const requestUrl = concatLink(this.endpoint, `/api/v1/account/${this.workspace}`) + const response = await fetch(requestUrl, this.requestInit()) + if (!response.ok) { + throw new PlatformError(unknownError(response.statusText)) + } + return await extractJson(response) + } + + async getModel (): Promise<{ hierarchy: Hierarchy, model: ModelDb }> { + const requestUrl = concatLink(this.endpoint, `/api/v1/load-model/${this.workspace}`) + const response = await fetch(requestUrl, this.requestInit()) + if (!response.ok) { + throw new PlatformError(unknownError(response.statusText)) + } + const modelResponse: Tx[] = await extractJson(response) + + const hierarchy = new Hierarchy() + const model = new ModelDb(hierarchy) + + const ctx = new MeasureMetricsContext('loadModel', {}) + buildModel(ctx, modelResponse, (txes: Tx[]) => txes, hierarchy, model) + + return { hierarchy, model } + } + + async findOne( + _class: Ref>, + query: DocumentQuery, + options?: FindOptions + ): Promise | undefined> { + return (await this.findAll(_class, query, { ...options, limit: 1 })).shift() + } + + async tx (tx: Tx): Promise { + const requestUrl = concatLink(this.endpoint, `/api/v1/tx/${this.workspace}`) + const response = await fetch(requestUrl, { + method: 'POST', + headers: this.jsonHeaders(), + keepalive: true, + body: JSON.stringify(tx) + }) + if (!response.ok) { + throw new PlatformError(unknownError(response.statusText)) + } + return await extractJson(response) + } + + async searchFulltext (query: SearchQuery, options: SearchOptions): Promise { + const params = new URLSearchParams() + params.append('query', query.query) + if (query.classes != null && Object.keys(query.classes).length > 0) { + params.append('classes', JSON.stringify(query.classes)) + } + if (query.spaces != null && Object.keys(query.spaces).length > 0) { + params.append('spaces', JSON.stringify(query.spaces)) + } + if (options.limit != null) { + params.append('limit', `${options.limit}`) + } + const requestUrl = concatLink(this.endpoint, `/api/v1/search-fulltext/${this.workspace}`) + const response = await fetch(requestUrl, { + method: 'GET', + headers: this.jsonHeaders(), + keepalive: true + }) + if (!response.ok) { + throw new PlatformError(unknownError(response.statusText)) + } + return await extractJson(response) + } +} diff --git a/packages/api-client/src/rest/tx.ts b/packages/api-client/src/rest/tx.ts new file mode 100644 index 0000000000..ee08900614 --- /dev/null +++ b/packages/api-client/src/rest/tx.ts @@ -0,0 +1,101 @@ +// +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import { + type Account, + type Class, + type Client, + type Doc, + type DocumentQuery, + type FindOptions, + type FindResult, + Hierarchy, + ModelDb, + type Ref, + type SearchOptions, + type SearchQuery, + type SearchResult, + toFindResult, + type Tx, + TxOperations, + type TxResult, + type WithLookup +} from '@hcengineering/core' +import { RestClientImpl } from './rest' + +export async function createRestTxOperations ( + endpoint: string, + workspaceId: string, + token: string +): Promise { + const restClient = new RestClientImpl(endpoint, workspaceId, token) + + const account = await restClient.getAccount() + const { hierarchy, model } = await restClient.getModel() + + return new TxOperations(new RestTxClient(restClient, hierarchy, model, account), account._id) +} + +class RestTxClient implements Client { + constructor ( + readonly client: RestClientImpl, + readonly hierarchy: Hierarchy, + readonly model: ModelDb, + readonly account: Account + ) {} + + close (): Promise { + return Promise.resolve() + } + + async findAll( + _class: Ref>, + query: DocumentQuery, + options?: FindOptions + ): Promise> { + const data = await this.client.findAll(_class, query, options) + const result = data.map((v) => { + return this.hierarchy.updateLookupMixin(_class, v, options) + }) + return toFindResult(result, data.total) + } + + async findOne( + _class: Ref>, + query: DocumentQuery, + options?: FindOptions + ): Promise | undefined> { + const v = await this.client.findOne(_class, query, options) + if (v === undefined) { + return + } + return this.hierarchy.updateLookupMixin(_class, v, options) + } + + getHierarchy: () => Hierarchy = () => this.hierarchy + getModel: () => ModelDb = () => this.model + + async getAccount (): Promise { + return this.account + } + + async tx (tx: Tx): Promise { + return await this.client.tx(tx) + } + + async searchFulltext (query: SearchQuery, options: SearchOptions): Promise { + return await this.client.searchFulltext(query, options) + } +} diff --git a/packages/api-client/src/rest/types.ts b/packages/api-client/src/rest/types.ts new file mode 100644 index 0000000000..b117d90c60 --- /dev/null +++ b/packages/api-client/src/rest/types.ts @@ -0,0 +1,39 @@ +// +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import { + type Account, + type Class, + type Doc, + type DocumentQuery, + type FindOptions, + type Hierarchy, + type ModelDb, + type Ref, + type Storage, + type WithLookup +} from '@hcengineering/core' + +export interface RestClient extends Storage { + getAccount: () => Promise + + findOne: ( + _class: Ref>, + query: DocumentQuery, + options?: FindOptions + ) => Promise | undefined> + + getModel: () => Promise<{ hierarchy: Hierarchy, model: ModelDb }> +} diff --git a/packages/api-client/src/rest/utils.ts b/packages/api-client/src/rest/utils.ts new file mode 100644 index 0000000000..d6b87cbc58 --- /dev/null +++ b/packages/api-client/src/rest/utils.ts @@ -0,0 +1,42 @@ +import { uncompress } from 'snappyjs' + +export async function withRetry (fn: () => Promise): Promise { + const maxRetries = 3 + let lastError: any + + for (let attempt = 0; attempt < maxRetries; attempt++) { + try { + return await fn() + } catch (err: any) { + lastError = err + if (attempt === maxRetries - 1) { + console.error('Failed to execute query', err) + throw lastError + } + await new Promise((resolve) => setTimeout(resolve, Math.pow(2, attempt) * 100)) + } + } + throw lastError +} + +function rpcJSONReceiver (key: string, value: any): any { + if (typeof value === 'object' && value !== null) { + if (value.dataType === 'TotalArray') { + return Object.assign(value.value, { total: value.total, lookupMap: value.lookupMap }) + } + } + return value +} + +export async function extractJson (response: Response): Promise { + const encoding = response.headers.get('content-encoding') + if (encoding === 'snappy') { + const buffer = await response.arrayBuffer() + const decompressed = uncompress(buffer) + const decoder = new TextDecoder() + const jsonString = decoder.decode(decompressed) + return JSON.parse(jsonString, rpcJSONReceiver) as T + } + const jsonString = await response.text() + return JSON.parse(jsonString, rpcJSONReceiver) as T +} diff --git a/packages/core/src/client.ts b/packages/core/src/client.ts index b24c37de8c..da6c8b3227 100644 --- a/packages/core/src/client.ts +++ b/packages/core/src/client.ts @@ -404,7 +404,7 @@ async function loadModel ( return { mode: 'addition', current: current.transactions, addition: result.transactions } } -function buildModel ( +export function buildModel ( ctx: MeasureContext, transactions: Tx[], modelFilter: ModelFilter | undefined, diff --git a/rush.json b/rush.json index 0b771f87ab..45c9b8868e 100644 --- a/rush.json +++ b/rush.json @@ -455,6 +455,11 @@ "projectFolder": "packages/api-client", "shouldPublish": false }, + { + "packageName": "@hcengineering/api-tests", + "projectFolder": "ws-tests/api-tests", + "shouldPublish": false + }, { "packageName": "@hcengineering/importer", "projectFolder": "packages/importer", diff --git a/server/core/src/types.ts b/server/core/src/types.ts index af0053c2bd..68c5ac9397 100644 --- a/server/core/src/types.ts +++ b/server/core/src/types.ts @@ -547,6 +547,8 @@ export interface Session { getUser: () => string loadModel: (ctx: ClientSessionCtx, lastModelTx: Timestamp, hash?: string) => Promise + + loadModelRaw: (ctx: ClientSessionCtx, lastModelTx: Timestamp, hash?: string) => Promise getAccount: (ctx: ClientSessionCtx) => Promise getRawAccount: (pipeline: Pipeline) => Account @@ -564,6 +566,7 @@ export interface Session { options?: FindOptions ) => Promise> searchFulltext: (ctx: ClientSessionCtx, query: SearchQuery, options: SearchOptions) => Promise + searchFulltextRaw: (ctx: ClientSessionCtx, query: SearchQuery, options: SearchOptions) => Promise tx: (ctx: ClientSessionCtx, tx: Tx) => Promise txRaw: ( diff --git a/server/middleware/src/model.ts b/server/middleware/src/model.ts index 2a40ef3613..0009037a41 100644 --- a/server/middleware/src/model.ts +++ b/server/middleware/src/model.ts @@ -14,12 +14,19 @@ // import core, { + type Class, type Doc, + type DocumentQuery, + type FindOptions, + type FindResult, type LoadModelResponse, type MeasureContext, + type Ref, + type SessionData, type Timestamp, type Tx, type TxCUD, + DOMAIN_MODEL, DOMAIN_TX, withContext } from '@hcengineering/core' @@ -78,6 +85,19 @@ export class ModelMiddleware extends BaseMiddleware implements Middleware { return allUserTxes.filter((it) => isUserTx(it)) } + findAll( + ctx: MeasureContext, + _class: Ref>, + query: DocumentQuery, + options?: FindOptions + ): Promise> { + const d = this.context.hierarchy.findDomain(_class) + if (d === DOMAIN_MODEL) { + return this.context.modelDb.findAll(_class, query, options) + } + return this.provideFindAll(ctx, _class, query, options) + } + async init (ctx: MeasureContext): Promise { if (this.context.adapterManager == null) { throw new PlatformError(unknownError('Adapter manager should be configured')) diff --git a/server/mongo/src/index.ts b/server/mongo/src/index.ts index 64a7113b38..f44bf62797 100644 --- a/server/mongo/src/index.ts +++ b/server/mongo/src/index.ts @@ -30,6 +30,8 @@ export function createMongoDestroyAdapter (url: string): WorkspaceDestroyAdapter const db = getWorkspaceMongoDB(dbClient, workspace) await db.dropDatabase() }) + } catch (err) { + console.error('Failed to delete workspace', err) } finally { client.close() } diff --git a/server/rpc/src/rpc.ts b/server/rpc/src/rpc.ts index 9771af11ca..8745cfee74 100644 --- a/server/rpc/src/rpc.ts +++ b/server/rpc/src/rpc.ts @@ -53,7 +53,7 @@ export interface HelloResponse extends Response { useCompression?: boolean } -function replacer (key: string, value: any): any { +export function rpcJSONReplacer (key: string, value: any): any { if (Array.isArray(value) && ((value as any).total !== undefined || (value as any).lookupMap !== undefined)) { return { dataType: 'TotalArray', @@ -66,7 +66,7 @@ function replacer (key: string, value: any): any { } } -function receiver (key: string, value: any): any { +export function rpcJSONReceiver (key: string, value: any): any { if (typeof value === 'object' && value !== null) { if (value.dataType === 'TotalArray') { return Object.assign(value.value, { total: value.total, lookupMap: value.lookupMap }) @@ -99,7 +99,7 @@ export class RPCHandler { packr = new Packr({ structuredClone: true, bundleStrings: true, copyBuffers: false }) protoSerialize (object: object, binary: boolean): any { if (!binary) { - return JSON.stringify(object, replacer) + return JSON.stringify(object, rpcJSONReplacer) } return new Uint8Array(this.packr.pack(object)) } @@ -112,7 +112,7 @@ export class RPCHandler { _data = decoder.decode(_data) } try { - return JSON.parse(_data.toString(), receiver) + return JSON.parse(_data.toString(), rpcJSONReceiver) } catch (err: any) { if (((err.message as string) ?? '').includes('Unexpected token')) { return this.packr.unpack(new Uint8Array(data)) @@ -129,7 +129,7 @@ export class RPCHandler { */ serialize (object: Request | Response, binary: boolean): any { if ((object as any).result !== undefined) { - ;(object as any).result = replacer('result', (object as any).result) + ;(object as any).result = rpcJSONReplacer('result', (object as any).result) } return this.protoSerialize(object, binary) } @@ -142,7 +142,7 @@ export class RPCHandler { readResponse(response: any, binary: boolean): Response { const data = this.protoDeserialize(response, binary) if (data.result !== undefined) { - data.result = receiver('result', data.result) + data.result = rpcJSONReceiver('result', data.result) } return data } diff --git a/server/server/src/client.ts b/server/server/src/client.ts index 7bc77f0cab..93be836495 100644 --- a/server/server/src/client.ts +++ b/server/server/src/client.ts @@ -23,10 +23,12 @@ import { type Domain, type FindOptions, type FindResult, + type LoadModelResponse, type MeasureContext, type Ref, type SearchOptions, type SearchQuery, + type SearchResult, type SessionData, type Timestamp, type Tx, @@ -106,6 +108,11 @@ export class ClientSession implements Session { await ctx.sendResponse(ctx.requestId, result) } + async loadModelRaw (ctx: ClientSessionCtx, lastModelTx: Timestamp, hash?: string): Promise { + this.includeSessionContext(ctx.ctx, ctx.pipeline) + return await ctx.ctx.with('load-model', {}, (_ctx) => ctx.pipeline.loadModel(_ctx, lastModelTx, hash)) + } + async getAccount (ctx: ClientSessionCtx): Promise { await ctx.sendResponse(ctx.requestId, this.getRawAccount(ctx.pipeline)) } @@ -163,6 +170,12 @@ export class ClientSession implements Session { await ctx.sendResponse(ctx.requestId, await ctx.pipeline.searchFulltext(ctx.ctx, query, options)) } + async searchFulltextRaw (ctx: ClientSessionCtx, query: SearchQuery, options: SearchOptions): Promise { + this.lastRequest = Date.now() + this.includeSessionContext(ctx.ctx, ctx.pipeline) + return await ctx.pipeline.searchFulltext(ctx.ctx, query, options) + } + async txRaw ( ctx: ClientSessionCtx, tx: Tx diff --git a/server/ws/package.json b/server/ws/package.json index 26a56b85b5..d124072374 100644 --- a/server/ws/package.json +++ b/server/ws/package.json @@ -36,7 +36,8 @@ "prettier": "^3.1.0", "ts-jest": "^29.1.1", "typescript": "^5.3.3", - "@types/body-parser": "~1.19.2" + "@types/body-parser": "~1.19.2", + "@types/morgan": "~1.9.9" }, "dependencies": { "@hcengineering/analytics": "^0.6.0", @@ -53,6 +54,7 @@ "ws": "^8.18.0", "body-parser": "^1.20.2", "snappy": "^7.2.2", - "@hcengineering/api-client": "^0.6.0" + "@hcengineering/api-client": "^0.6.0", + "morgan": "^1.10.0" } } diff --git a/server/ws/src/__tests__/rest.test.ts b/server/ws/src/__tests__/rest.test.ts index 6ad4530733..0e527efb40 100644 --- a/server/ws/src/__tests__/rest.test.ts +++ b/server/ws/src/__tests__/rest.test.ts @@ -15,7 +15,7 @@ import { generateToken } from '@hcengineering/server-token' -import { createRestClient, type RestClient } from '@hcengineering/api-client' +import { createRestClient, createRestTxOperations, type RestClient } from '@hcengineering/api-client' import core, { generateId, getWorkspaceId, @@ -34,12 +34,13 @@ import core, { type Space, type Tx, type TxCreateDoc, + type TxOperations, type TxResult } from '@hcengineering/core' import { ClientSession, startSessionManager, type TSessionManager } from '@hcengineering/server' import { createDummyStorageAdapter, type SessionManager, type WorkspaceLoginInfo } from '@hcengineering/server-core' import { startHttpServer } from '../server_http' -import { genMinModel } from './minmodel' +import { genMinModel, test } from './minmodel' describe('rest-server', () => { async function getModelDb (): Promise<{ modelDb: ModelDb, hierarchy: Hierarchy, txes: Tx[] }> { @@ -57,7 +58,7 @@ describe('rest-server', () => { let shutdown: () => Promise let sessionManager: SessionManager - const port: number = 3330 + const port: number = 11000 beforeAll(async () => { ;({ shutdown, sessionManager } = startSessionManager(new MeasureMetricsContext('test', {}), { @@ -153,6 +154,11 @@ describe('rest-server', () => { return await createRestClient(`http://localhost:${port}`, 'test-ws', token) } + async function connectTx (): Promise { + const token: string = generateToken('user1@site.com', getWorkspaceId('test-ws')) + return await createRestTxOperations(`http://localhost:${port}`, 'test-ws', token) + } + it('get account', async () => { const conn = await connect() const account = await conn.getAccount() @@ -222,4 +228,13 @@ describe('rest-server', () => { const spaces = await conn.findAll(core.class.Space, {}) expect(spaces.length).toBe(3) }) + + it('check-model-operations', async () => { + const conn = await connectTx() + const h = conn.getHierarchy() + const domains = h.domains() + expect(domains.length).toBe(2) + + expect(h.isDerived(test.class.TestComment, core.class.AttachedDoc)).toBe(true) + }) }) diff --git a/server/ws/src/__tests__/server.test.ts b/server/ws/src/__tests__/server.test.ts index e8524542e2..3a12a7ae6c 100644 --- a/server/ws/src/__tests__/server.test.ts +++ b/server/ws/src/__tests__/server.test.ts @@ -45,6 +45,7 @@ import { startHttpServer } from '../server_http' import { genMinModel } from './minmodel' describe('server', () => { + const port = 10000 const handler = new RPCHandler() async function getModelDb (): Promise<{ modelDb: ModelDb, hierarchy: Hierarchy }> { const txes = genMinModel() @@ -95,7 +96,7 @@ describe('server', () => { } }, sessionFactory: (token, workspace) => new ClientSession(token, workspace, true), - port: 3335, + port, brandingMap: {}, serverFactory: startHttpServer, accountsUrl: '', @@ -104,7 +105,7 @@ describe('server', () => { function connect (): WebSocket { const token: string = generateToken('', getWorkspaceId('latest')) - return new WebSocket(`ws://localhost:3335/${token}`) + return new WebSocket(`ws://localhost:${port}/${token}`) } afterAll(async () => { @@ -122,7 +123,7 @@ describe('server', () => { }) it('should not connect to server without token', (done) => { - const conn = new WebSocket('ws://localhost:3335/xyz') + const conn = new WebSocket(`ws://localhost:${port}/xyz`) conn.on('error', () => { conn.close(1000) }) @@ -206,7 +207,7 @@ describe('server', () => { } }, sessionFactory: (token, workspace) => new ClientSession(token, workspace, true), - port: 3336, + port: port + 1, brandingMap: {}, serverFactory: startHttpServer, accountsUrl: '', @@ -214,7 +215,7 @@ describe('server', () => { }) async function findClose (token: string, timeoutPromise: Promise, code: number): Promise { - const newConn = new WebSocket(`ws://localhost:3336/${token}?sessionId=s1`) + const newConn = new WebSocket(`ws://localhost:${port + 1}/${token}?sessionId=s1`) await Promise.race([ timeoutPromise, diff --git a/server/ws/src/rpc.ts b/server/ws/src/rpc.ts index e633cb0fc8..c83225e240 100644 --- a/server/ws/src/rpc.ts +++ b/server/ws/src/rpc.ts @@ -1,4 +1,13 @@ -import type { Class, Doc, MeasureContext, Ref } from '@hcengineering/core' +import core, { + TxProcessor, + type Class, + type Doc, + type MeasureContext, + type Ref, + type SearchOptions, + type SearchQuery, + type TxCUD +} from '@hcengineering/core' import type { ClientSessionCtx, ConnectionSocket, @@ -8,6 +17,8 @@ import type { } from '@hcengineering/server-core' import { decodeToken } from '@hcengineering/server-token' +import { rpcJSONReplacer } from '@hcengineering/rpc' +import { createHash } from 'crypto' import { type Express, type Response as ExpressResponse, type Request } from 'express' import type { OutgoingHttpHeaders } from 'http2' import { compress } from 'snappy' @@ -32,14 +43,31 @@ const sendError = (res: ExpressResponse, code: number, data: any): void => { res.end(JSON.stringify(data)) } -async function sendJson (req: Request, res: ExpressResponse, result: any): Promise { +async function sendJson ( + req: Request, + res: ExpressResponse, + result: any, + extraHeaders?: OutgoingHttpHeaders +): Promise { + // Calculate ETag + let body: any = JSON.stringify(result, rpcJSONReplacer) + + const etag = createHash('sha1').update(body).digest('hex') const headers: OutgoingHttpHeaders = { + ...(extraHeaders ?? {}), 'Content-Type': 'application/json', 'Cache-Control': 'no-cache', - Connection: 'keep-alive', - 'keep-alive': 'timeout=5, max=1000' + connection: 'keep-alive', + 'keep-alive': 'timeout=5, max=1000', + ETag: etag + } + + // Check if the ETag matches + if (req.headers['if-none-match'] === etag) { + res.writeHead(304, headers) + res.end() + return } - let body: any = JSON.stringify(result) const contentEncodings: string[] = typeof req.headers['accept-encoding'] === 'string' @@ -63,7 +91,7 @@ async function sendJson (req: Request, res: ExpressResponse, result: any): Promi break } } - + headers['content-length'] = body.length res.writeHead(200, headers) res.end(body) } @@ -81,41 +109,42 @@ export function registerRPC ( res: ExpressResponse, operation: (ctx: ClientSessionCtx, session: Session) => Promise ): Promise { - if (req.params.workspaceId === undefined || req.params.workspaceId === '') { - res.writeHead(400, {}) - res.end('Missing workspace') - return - } - let token = req.headers.authorization as string - if (token === null) { - sendError(res, 401, { message: 'Missing Authorization header' }) - return - } - const workspaceId = decodeURIComponent(req.params.workspaceId) - token = token.split(' ')[1] - - const decodedToken = decodeToken(token) - if (workspaceId !== decodedToken.workspace.name) { - sendError(res, 401, { message: 'Invalid workspace' }) - return - } - - let transactorRpc = rpcSessions.get(token) - - if (transactorRpc === undefined) { - const cs: ConnectionSocket = createClosingSocket(token, rpcSessions) - const s = await sessions.addSession(ctx, cs, decodedToken, token, pipelineFactory, token) - if (!('session' in s)) { - sendError(res, 401, { - message: 'Failed to create session', - mode: 'specialError' in s ? s.specialError ?? '' : 'upgrading' - }) + try { + if (req.params.workspaceId === undefined || req.params.workspaceId === '') { + res.writeHead(400, {}) + res.end('Missing workspace') return } - transactorRpc = { session: s.session, client: cs, workspaceId: s.workspaceId } - rpcSessions.set(token, transactorRpc) - } - try { + let token = req.headers.authorization as string + if (token === null) { + sendError(res, 401, { message: 'Missing Authorization header' }) + return + } + const workspaceId = decodeURIComponent(req.params.workspaceId) + token = token.split(' ')[1] + + const decodedToken = decodeToken(token) + if (workspaceId !== decodedToken.workspace.name) { + sendError(res, 401, { message: 'Invalid workspace', workspace: decodedToken.workspace.name }) + return + } + + let transactorRpc = rpcSessions.get(token) + + if (transactorRpc === undefined) { + const cs: ConnectionSocket = createClosingSocket(token, rpcSessions) + const s = await sessions.addSession(ctx, cs, decodedToken, token, pipelineFactory, token) + if (!('session' in s)) { + sendError(res, 401, { + message: 'Failed to create session', + mode: 'specialError' in s ? s.specialError ?? '' : 'upgrading' + }) + return + } + transactorRpc = { session: s.session, client: cs, workspaceId: s.workspaceId } + rpcSessions.set(token, transactorRpc) + } + const rpc = transactorRpc await sessions.handleRPC(ctx, rpc.session, rpc.client, async (ctx) => { await operation(ctx, rpc.session) @@ -128,7 +157,11 @@ export function registerRPC ( app.get('/api/v1/ping/:workspaceId', (req, res) => { void withSession(req, res, async (ctx, session) => { await session.ping(ctx) - await sendJson(req, res, { pong: true }) + await sendJson(req, res, { + pong: true, + lastTx: ctx.pipeline.context.lastTx, + lastHash: ctx.pipeline.context.lastHash + }) }) }) @@ -166,6 +199,50 @@ export function registerRPC ( await sendJson(req, res, result) }) }) + + app.get('/api/v1/load-model/:workspaceId', (req, res) => { + void withSession(req, res, async (ctx, session) => { + const lastModelTx = parseInt((req.query.lastModelTx as string) ?? '0') + const lastHash = req.query.lastHash as string + const result = await session.loadModelRaw(ctx, lastModelTx, lastHash) + const txes = Array.isArray(result) ? result : result.transactions + // we need to filter only hierarchy related txes. + const allowedClasess: Ref>[] = [ + core.class.Class, + core.class.Attribute, + core.class.Mixin, + core.class.Type, + core.class.Status, + core.class.Account, + core.class.Permission, + core.class.Space, + core.class.Tx + ] + const h = ctx.pipeline.context.hierarchy + const filtered = txes.filter( + (it) => + TxProcessor.isExtendsCUD(it._class) && + allowedClasess.some((cl) => h.isDerived((it as TxCUD).objectClass, cl)) + ) + + await sendJson(req, res, filtered) + }) + }) + + app.get('/api/v1/search-fulltext/:workspaceId', (req, res) => { + void withSession(req, res, async (ctx, session) => { + const query: SearchQuery = { + query: req.query.query as string, + classes: req.query.classes !== undefined ? JSON.parse(req.query.classes as string) : undefined, + spaces: req.query.spaces !== undefined ? JSON.parse(req.query.spaces as string) : undefined + } + const options: SearchOptions = { + limit: req.query.limit !== undefined ? parseInt(req.query.limit as string) : undefined + } + const result = await session.searchFulltextRaw(ctx, query, options) + await sendJson(req, res, result) + }) + }) } function createClosingSocket (rawToken: string, rpcSessions: Map): ConnectionSocket { diff --git a/server/ws/src/server_http.ts b/server/ws/src/server_http.ts index d9961c4806..bc117d8fb3 100644 --- a/server/ws/src/server_http.ts +++ b/server/ws/src/server_http.ts @@ -49,6 +49,7 @@ import { compress } from 'snappy' import 'utf-8-validate' import { registerRPC } from './rpc' import { retrieveJson } from './utils' +import morgan from 'morgan' import { setImmediate } from 'timers/promises' @@ -82,6 +83,21 @@ export function startHttpServer ( const app = express() app.use(cors()) + const childLogger = ctx.logger.childLogger?.('requests', { + enableConsole: 'true' + }) + const requests = ctx.newChild('requests', {}, {}, childLogger) + + class MyStream { + write (text: string): void { + requests.info(text) + } + } + + const myStream = new MyStream() + + app.use(morgan('short', { stream: myStream })) + const getUsers = (): any => Array.from(sessions.sessions.entries()).map(([k, v]) => v.session.getUser()) app.get('/api/v1/version', (req, res) => { diff --git a/ws-tests/api-tests/.eslintrc.js b/ws-tests/api-tests/.eslintrc.js new file mode 100644 index 0000000000..ce90fb9646 --- /dev/null +++ b/ws-tests/api-tests/.eslintrc.js @@ -0,0 +1,7 @@ +module.exports = { + extends: ['./node_modules/@hcengineering/platform-rig/profiles/node/eslint.config.json'], + parserOptions: { + tsconfigRootDir: __dirname, + project: './tsconfig.json' + } +} diff --git a/ws-tests/api-tests/.gitignore b/ws-tests/api-tests/.gitignore new file mode 100644 index 0000000000..4e9e09b9aa --- /dev/null +++ b/ws-tests/api-tests/.gitignore @@ -0,0 +1,2 @@ +v*.zip +src/uws \ No newline at end of file diff --git a/ws-tests/api-tests/.npmignore b/ws-tests/api-tests/.npmignore new file mode 100644 index 0000000000..e3ec093c38 --- /dev/null +++ b/ws-tests/api-tests/.npmignore @@ -0,0 +1,4 @@ +* +!/lib/** +!CHANGELOG.md +/lib/**/__tests__/ diff --git a/ws-tests/api-tests/config/rig.json b/ws-tests/api-tests/config/rig.json new file mode 100644 index 0000000000..78cc5a1733 --- /dev/null +++ b/ws-tests/api-tests/config/rig.json @@ -0,0 +1,5 @@ +{ + "$schema": "https://developer.microsoft.com/json-schemas/rig-package/rig.schema.json", + "rigPackageName": "@hcengineering/platform-rig", + "rigProfile": "node" +} diff --git a/ws-tests/api-tests/jest.config.js b/ws-tests/api-tests/jest.config.js new file mode 100644 index 0000000000..2cfd408b67 --- /dev/null +++ b/ws-tests/api-tests/jest.config.js @@ -0,0 +1,7 @@ +module.exports = { + preset: 'ts-jest', + testEnvironment: 'node', + testMatch: ['**/?(*.)+(spec|test).[jt]s?(x)'], + roots: ["./src"], + coverageReporters: ["text-summary", "html"] +} diff --git a/ws-tests/api-tests/package.json b/ws-tests/api-tests/package.json new file mode 100644 index 0000000000..482d26ec4d --- /dev/null +++ b/ws-tests/api-tests/package.json @@ -0,0 +1,59 @@ +{ + "name": "@hcengineering/api-tests", + "version": "0.6.0", + "main": "lib/index.js", + "svelte": "src/index.ts", + "types": "types/index.d.ts", + "author": "Anticrm Platform Contributors", + "template": "@hcengineering/node-package-ws", + "license": "EPL-2.0", + "scripts": { + "build": "compile", + "build:watch": "compile", + "test": "echo 'run api-test' for API-tests", + "api-test": "jest --passWithNoTests --silent --forceExit", + "format": "format src", + "_phase:build": "compile transpile src", + "_phase:test": "echo 'run api-test' for API-tests", + "_phase:format": "format src", + "_phase:validate": "compile validate" + }, + "devDependencies": { + "@hcengineering/platform-rig": "^0.6.0", + "@types/compression": "~1.7.2", + "@types/cors": "^2.8.12", + "@types/express": "^4.17.13", + "@types/jest": "^29.5.5", + "@types/node": "~20.11.16", + "@types/ws": "^8.5.11", + "@typescript-eslint/eslint-plugin": "^6.11.0", + "@typescript-eslint/parser": "^6.11.0", + "eslint": "^8.54.0", + "eslint-config-standard-with-typescript": "^40.0.0", + "eslint-plugin-import": "^2.26.0", + "eslint-plugin-n": "^15.4.0", + "eslint-plugin-promise": "^6.1.1", + "jest": "^29.7.0", + "prettier": "^3.1.0", + "ts-jest": "^29.1.1", + "typescript": "^5.3.3", + "@types/body-parser": "~1.19.2", + "@types/morgan": "~1.9.9" + }, + "dependencies": { + "@hcengineering/analytics": "^0.6.0", + "@hcengineering/core": "^0.6.32", + "@hcengineering/platform": "^0.6.11", + "@hcengineering/rpc": "^0.6.5", + "@hcengineering/server-token": "^0.6.11", + "bufferutil": "^4.0.8", + "utf-8-validate": "^6.0.4", + "ws": "^8.18.0", + "snappyjs": "^0.7.0", + "@hcengineering/api-client": "^0.6.0", + "@hcengineering/tracker": "^0.6.24", + "@hcengineering/task": "^0.6.20", + "@hcengineering/contact": "^0.6.24", + "@hcengineering/chunter": "^0.6.20" + } +} diff --git a/ws-tests/api-tests/src/__tests__/rest.test.ts b/ws-tests/api-tests/src/__tests__/rest.test.ts new file mode 100644 index 0000000000..74e1f21930 --- /dev/null +++ b/ws-tests/api-tests/src/__tests__/rest.test.ts @@ -0,0 +1,178 @@ +// +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import { + createRestClient, + createRestTxOperations, + getWorkspaceToken, + type RestClient, + type WorkspaceToken +} from '@hcengineering/api-client' +import core, { generateId, type Space, type TxCreateDoc, type TxOperations } from '@hcengineering/core' + +import chunter from '@hcengineering/chunter' +import contact from '@hcengineering/contact' + +describe('rest-api-server', () => { + const wsName = 'api-tests' + let apiWorkspace1: WorkspaceToken + let apiWorkspace2: WorkspaceToken + + beforeAll(async () => { + apiWorkspace1 = await getWorkspaceToken('http://localhost:8083', { + email: 'user1', + password: '1234', + workspace: wsName + }) + + apiWorkspace2 = await getWorkspaceToken('http://localhost:8083', { + email: 'user1', + password: '1234', + workspace: wsName + '-cr' + }) + }) + + async function connect (ws?: WorkspaceToken): Promise { + const tok = ws ?? apiWorkspace1 + return await createRestClient(tok.endpoint, tok.workspaceId, tok.token) + } + + async function connectTx (ws?: WorkspaceToken): Promise { + const tok = ws ?? apiWorkspace1 + return await createRestTxOperations(tok.endpoint, tok.workspaceId, tok.token) + } + + it('get account', async () => { + const conn = await connect() + const account = await conn.getAccount() + + expect(account.email).toBe('user1') + expect(account.role).toBe('USER') + expect(account._class).toBe(contact.class.PersonAccount) + expect(account.space).toBe(core.space.Model) + expect(account.modifiedBy).toBe(core.account.System) + expect(account.createdBy).toBe(core.account.System) + expect(typeof account.modifiedOn).toBe('number') + expect(typeof account.createdOn).toBe('number') + }) + + it('find spaces', async () => { + const conn = await connect() + const spaces = await conn.findAll(core.class.Space, {}) + expect(spaces.length).toBeGreaterThanOrEqual(20) + const personSpace = spaces.find((it) => it.name === 'Pesonal space' && it.private) + expect(personSpace).not.toBeNull() + }) + + it('find spaces limit', async () => { + const conn = await connect() + const spaces = await conn.findAll(core.class.Space, {}, { limit: 5 }) + expect(spaces.length).toBe(5) + }) + it('find spaces by-name', async () => { + const conn = await connect() + const spaces = await conn.findAll( + contact.class.PersonSpace, + { name: 'Personal space' }, + { + lookup: { + person: contact.class.Person + } + } + ) + expect(spaces.length).toBe(1) + expect(spaces[0].name).toBe('Personal space') + expect(spaces[0].$lookup?.person?.name).toBe('Appleseed,John') + }) + + it('find channels', async () => { + const conn = await connect() + const spaces = await conn.findAll(chunter.class.Channel, {}) + expect(spaces.length).toBeGreaterThanOrEqual(2) + expect(spaces.find((it) => it._id === 'chunter:space:General')).not.toBeNull() + }) + + it('find avg', async () => { + const conn = await connect() + await checkFindPerformance(conn) // 5ms max per operation + }) + + it('find avg-europe', async () => { + const conn = await connect(apiWorkspace2) + await checkFindPerformance(conn) // 5ms max per operation + }) + + it('add space', async () => { + const conn = await connect() + const account = await conn.getAccount() + const spaceName = generateId() + const tx: TxCreateDoc = { + _class: core.class.TxCreateDoc, + space: core.space.Tx, + _id: generateId(), + objectSpace: core.space.Model, + modifiedBy: account._id, + modifiedOn: Date.now(), + attributes: { + name: spaceName, + description: '', + private: false, + archived: false, + members: [], + autoJoin: false + }, + objectClass: core.class.Space, + objectId: generateId() + } + await conn.tx(tx) + const spaces = await conn.findAll(core.class.Space, {}) + expect(spaces.filter((it) => it.name === spaceName).length).toBe(1) + }) + + it('get-model', async () => { + const conn = await connect() + const { hierarchy, model } = await conn.getModel() + + const dsc = hierarchy.getDescendants(core.class.Space) + expect(dsc.length).toBe(32) + expect(model.getObject(core.class.Space)).not.toBeNull() + }) + + it('tx-client', async () => { + const conn = await connectTx() + + const employee = await conn.findAll(contact.mixin.Employee, {}, { limit: 5 }) + + expect(employee.length).toBeGreaterThanOrEqual(1) + expect(employee[0].active).toBe(true) + }) +}) +async function checkFindPerformance (conn: RestClient): Promise { + let ops = 0 + let total = 0 + const attempts = 1000 + for (let i = 0; i < attempts; i++) { + const st = performance.now() + const spaces = await conn.findAll(core.class.Space, {}) + expect(spaces.length).toBeGreaterThanOrEqual(22) + const ed = performance.now() + ops++ + total += ed - st + } + const avg = total / ops + // console.log('ops:', ops, 'total:', total, 'avg:', ) + expect(ops).toEqual(attempts) + expect(avg).toBeLessThan(5) +} diff --git a/ws-tests/api-tests/src/index.ts b/ws-tests/api-tests/src/index.ts new file mode 100644 index 0000000000..e69de29bb2 diff --git a/ws-tests/api-tests/tsconfig.json b/ws-tests/api-tests/tsconfig.json new file mode 100644 index 0000000000..f017cc597c --- /dev/null +++ b/ws-tests/api-tests/tsconfig.json @@ -0,0 +1,10 @@ +{ + "extends": "./node_modules/@hcengineering/platform-rig/profiles/node/tsconfig.json", + + "compilerOptions": { + "rootDir": "./src", + "outDir": "./lib", + "declarationDir": "./types", + "tsBuildInfoFile": ".build/build.tsbuildinfo" + } +} \ No newline at end of file diff --git a/ws-tests/prepare.sh b/ws-tests/prepare.sh index 286693708e..d26fa2154b 100755 --- a/ws-tests/prepare.sh +++ b/ws-tests/prepare.sh @@ -27,3 +27,9 @@ fi ./tool.sh confirm-email user1 ./tool.sh confirm-email user2 + + +./tool.sh create-workspace api-tests -w api-tests +./tool-europe.sh create-workspace api-tests-cr -w api-tests --region 'europe' +./tool.sh assign-workspace user1 api-tests +./tool.sh assign-workspace user1 api-tests-cr diff --git a/ws-tests/tool-europe.sh b/ws-tests/tool-europe.sh new file mode 100755 index 0000000000..78e2fc1d53 --- /dev/null +++ b/ws-tests/tool-europe.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +export MODEL_VERSION=$(node ../common/scripts/show_version.js) +export MINIO_ACCESS_KEY=minioadmin +export MINIO_SECRET_KEY=minioadmin +export MINIO_ENDPOINT=localhost:9002 +export ACCOUNTS_URL=http://localhost:3003 +export TRANSACTOR_URL=ws://localhost:3334 +# export ACCOUNT_DB_URL=postgresql://root@localhost:26258/defaultdb?sslmode=disable +export ACCOUNT_DB_URL=mongodb://localhost:27018 +export MONGO_URL=mongodb://localhost:27018 +export ELASTIC_URL=http://localhost:9201 +export SERVER_SECRET=secret +export DB_URL=postgresql://root@localhost:26258/defaultdb?sslmode=disable + +export REGION_INFO="|America;europe|" # Europe without name will not be available for creation of new workspaces. +export TRANSACTOR_URL="ws://transactor:3334;ws://localhost:3334,ws://transactor-europe:3335;ws://localhost:3335;europe," + +node ${TOOL_OPTIONS} --max-old-space-size=8096 ../dev/tool/bundle/bundle.js $@ \ No newline at end of file diff --git a/ws-tests/tool.sh b/ws-tests/tool.sh index c5003d0c73..9633107e71 100755 --- a/ws-tests/tool.sh +++ b/ws-tests/tool.sh @@ -11,6 +11,6 @@ export ACCOUNT_DB_URL=mongodb://localhost:27018 export MONGO_URL=mongodb://localhost:27018 export ELASTIC_URL=http://localhost:9201 export SERVER_SECRET=secret -export DB_URL=postgresql://root@localhost:26258/defaultdb?sslmode=disable +export DB_URL=$MONGO_URL node ${TOOL_OPTIONS} --max-old-space-size=8096 ../dev/tool/bundle/bundle.js $@ \ No newline at end of file