From 9672f5064e3d2127f78738d09bb345126a948afc Mon Sep 17 00:00:00 2001 From: Andrey Sobolev Date: Wed, 25 Dec 2024 11:53:01 +0700 Subject: [PATCH 1/7] UBERF-9015: Remove confusing SYSTEM_EMAIL env (#7548) Signed-off-by: Andrey Sobolev --- .vscode/launch.json | 3 --- server/account/src/operations.ts | 3 +-- services/calendar/pod-calendar/src/config.ts | 3 --- .../calendar/pod-calendar/src/workspaceClient.ts | 4 ++-- services/github/pod-github/src/client.ts | 4 ++-- services/github/pod-github/src/collaborator.ts | 4 ++-- services/github/pod-github/src/config.ts | 6 ------ services/github/pod-github/src/platform.ts | 3 ++- services/gmail/pod-gmail/src/config.ts | 3 --- services/gmail/pod-gmail/src/workspaceClient.ts | 4 ++-- services/love/src/config.ts | 3 --- services/love/src/workspaceClient.ts | 15 +++++++++++---- services/sign/pod-sign/src/config.ts | 2 -- services/sign/pod-sign/src/sign.ts | 4 ++-- services/sign/pod-sign/src/signController.ts | 5 ++--- services/telegram/pod-telegram/src/config.ts | 6 +----- services/telegram/pod-telegram/src/workspace.ts | 4 ++-- 17 files changed, 29 insertions(+), 47 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 548cf05872..94849ec1cd 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -441,7 +441,6 @@ "CLIENT_SECRET": "${env:POD_GITHUB_CLIENT_SECRET}", "PRIVATE_KEY": "${env:POD_GITHUB_PRIVATE_KEY}", "COLLABORATOR_URL": "ws://localhost:3078", - "SYSTEM_EMAIL": "anticrm@hc.engineering", "MINIO_ENDPOINT": "localhost", "MINIO_ACCESS_KEY": "minioadmin", "MINIO_SECRET_KEY": "minioadmin", @@ -463,7 +462,6 @@ "args": ["src/index.ts"], "env": { "ACCOUNTS_URL": "http://localhost:3000", - "SYSTEM_EMAIL": "anticrm@hc.engineering", "SECRET": "secret", "DOCS_RELEASE_INTERVAL": "10000", "DOCS_IN_REVIEW_CHECK_INTERVAL": "10000", @@ -523,7 +521,6 @@ "MINIO_ACCESS_KEY": "minioadmin", "MINIO_SECRET_KEY": "minioadmin", "SERVICE_ID": "sign-service", - "SYSTEM_EMAIL": "", "ACCOUNTS_URL": "http://localhost:3000", "BRANDING_PATH": "${workspaceRoot}/services/sign/pod-sign/debug/branding.json" }, diff --git a/server/account/src/operations.ts b/server/account/src/operations.ts index 0ed4f32fc2..88b76e7685 100644 --- a/server/account/src/operations.ts +++ b/server/account/src/operations.ts @@ -761,8 +761,7 @@ export async function createAcc ( const salt = randomBytes(32) const hash = password !== null ? hashWithSalt(password, salt) : null - const systemEmails = [systemAccountEmail] - if (systemEmails.includes(email)) { + if (systemAccountEmail === email) { ctx.error('system email used for account', { email }) throw new PlatformError(new Status(Severity.ERROR, platform.status.AccountAlreadyExists, { account: email })) } diff --git a/services/calendar/pod-calendar/src/config.ts b/services/calendar/pod-calendar/src/config.ts index e93fb08446..dc17770a9d 100644 --- a/services/calendar/pod-calendar/src/config.ts +++ b/services/calendar/pod-calendar/src/config.ts @@ -23,7 +23,6 @@ interface Config { Secret: string Credentials: string WATCH_URL: string - SystemEmail: string InitLimit: number } @@ -37,7 +36,6 @@ const envMap: { [key in keyof Config]: string } = { ServiceID: 'SERVICE_ID', Secret: 'SECRET', Credentials: 'Credentials', - SystemEmail: 'SYSTEM_EMAIL', WATCH_URL: 'WATCH_URL', InitLimit: 'INIT_LIMIT' } @@ -52,7 +50,6 @@ const config: Config = (() => { AccountsURL: process.env[envMap.AccountsURL], ServiceID: process.env[envMap.ServiceID] ?? 'calendar-service', Secret: process.env[envMap.Secret], - SystemEmail: process.env[envMap.SystemEmail] ?? 'anticrm@hc.engineering', Credentials: process.env[envMap.Credentials], InitLimit: parseNumber(process.env[envMap.InitLimit]) ?? 50, WATCH_URL: process.env[envMap.WATCH_URL] diff --git a/services/calendar/pod-calendar/src/workspaceClient.ts b/services/calendar/pod-calendar/src/workspaceClient.ts index 34e6475d10..189e39ecbb 100644 --- a/services/calendar/pod-calendar/src/workspaceClient.ts +++ b/services/calendar/pod-calendar/src/workspaceClient.ts @@ -18,6 +18,7 @@ import contact, { Channel, Contact, type Employee, type PersonAccount } from '@h import core, { TxOperations, TxProcessor, + systemAccountEmail, toIdMap, type Account, type Client, @@ -34,7 +35,6 @@ import { Collection, type Db } from 'mongodb' import { CalendarClient } from './calendar' import { CalendarController } from './calendarController' import { getClient } from './client' -import config from './config' import { SyncHistory, type ProjectCredentials, type User } from './types' export class WorkspaceClient { @@ -159,7 +159,7 @@ export class WorkspaceClient { } private async initClient (workspace: string): Promise { - const token = generateToken(config.SystemEmail, { name: workspace }) + const token = generateToken(systemAccountEmail, { name: workspace }) const client = await getClient(token) client.notify = (...tx: Tx[]) => { void this.txHandler(...tx) diff --git a/services/github/pod-github/src/client.ts b/services/github/pod-github/src/client.ts index 6356b23a24..11fedcde1c 100644 --- a/services/github/pod-github/src/client.ts +++ b/services/github/pod-github/src/client.ts @@ -5,7 +5,7 @@ import client, { ClientSocket } from '@hcengineering/client' import clientResources from '@hcengineering/client-resources' -import { Client, ClientConnectEvent } from '@hcengineering/core' +import { Client, ClientConnectEvent, systemAccountEmail } from '@hcengineering/core' import { setMetadata } from '@hcengineering/platform' import { getTransactorEndpoint } from '@hcengineering/server-client' import serverToken, { generateToken } from '@hcengineering/server-token' @@ -30,7 +30,7 @@ export async function createPlatformClient ( setMetadata(serverToken.metadata.Secret, config.ServerSecret) const token = generateToken( - config.SystemEmail, + systemAccountEmail, { name: workspace }, diff --git a/services/github/pod-github/src/collaborator.ts b/services/github/pod-github/src/collaborator.ts index ef6a365ea7..a9a2ee339f 100644 --- a/services/github/pod-github/src/collaborator.ts +++ b/services/github/pod-github/src/collaborator.ts @@ -4,7 +4,7 @@ // import { CollaboratorClient, getClient as getCollaboratorClient } from '@hcengineering/collaborator-client' -import { WorkspaceId } from '@hcengineering/core' +import { systemAccountEmail, WorkspaceId } from '@hcengineering/core' import { generateToken } from '@hcengineering/server-token' import config from './config' @@ -12,6 +12,6 @@ import config from './config' * @public */ export function createCollaboratorClient (workspaceId: WorkspaceId): CollaboratorClient { - const token = generateToken(config.SystemEmail, workspaceId, { mode: 'github' }) + const token = generateToken(systemAccountEmail, workspaceId, { mode: 'github' }) return getCollaboratorClient(workspaceId, token, config.CollaboratorURL) } diff --git a/services/github/pod-github/src/config.ts b/services/github/pod-github/src/config.ts index 3422220baa..88c806d7a1 100644 --- a/services/github/pod-github/src/config.ts +++ b/services/github/pod-github/src/config.ts @@ -2,13 +2,10 @@ // Copyright © 2023 Hardcore Engineering Inc. // -import { systemAccountEmail } from '@hcengineering/core' - interface Config { AccountsURL: string ServiceID: string ServerSecret: string - SystemEmail: string FrontURL: string // '*' means all workspaces @@ -36,7 +33,6 @@ const envMap: { [key in keyof Config]: string } = { AccountsURL: 'ACCOUNTS_URL', ServiceID: 'SERVICE_ID', ServerSecret: 'SERVER_SECRET', - SystemEmail: 'SYSTEM_EMAIL', FrontURL: 'FRONT_URL', AppID: 'APP_ID', @@ -62,7 +58,6 @@ const required: Array = [ 'AccountsURL', 'ServerSecret', 'ServiceID', - 'SystemEmail', 'FrontURL', 'AppID', 'ClientID', @@ -82,7 +77,6 @@ const config: Config = (() => { AccountsURL: process.env[envMap.AccountsURL], ServerSecret: process.env[envMap.ServerSecret], ServiceID: process.env[envMap.ServiceID] ?? 'github-service', - SystemEmail: process.env[envMap.SystemEmail] ?? systemAccountEmail, AllowedWorkspaces: process.env[envMap.AllowedWorkspaces]?.split(',') ?? ['*'], FrontURL: process.env[envMap.FrontURL] ?? '', diff --git a/services/github/pod-github/src/platform.ts b/services/github/pod-github/src/platform.ts index 89470f82dd..7820768f40 100644 --- a/services/github/pod-github/src/platform.ts +++ b/services/github/pod-github/src/platform.ts @@ -14,6 +14,7 @@ import core, { MeasureContext, RateLimiter, Ref, + systemAccountEmail, TxOperations } from '@hcengineering/core' import github, { GithubAuthentication, makeQuery, type GithubIntegration } from '@hcengineering/github' @@ -730,7 +731,7 @@ export class PlatformWorker { } await rateLimiter.add(async () => { const token = generateToken( - config.SystemEmail, + systemAccountEmail, { name: workspace }, diff --git a/services/gmail/pod-gmail/src/config.ts b/services/gmail/pod-gmail/src/config.ts index 3d98317309..870f660a19 100644 --- a/services/gmail/pod-gmail/src/config.ts +++ b/services/gmail/pod-gmail/src/config.ts @@ -24,7 +24,6 @@ interface Config { Secret: string Credentials: string WATCH_TOPIC_NAME: string - SystemEmail: string FooterMessage: string InitLimit: number } @@ -39,7 +38,6 @@ const envMap: { [key in keyof Config]: string } = { ServiceID: 'SERVICE_ID', Secret: 'SECRET', Credentials: 'Credentials', - SystemEmail: 'SYSTEM_EMAIL', WATCH_TOPIC_NAME: 'WATCH_TOPIC_NAME', FooterMessage: 'FOOTER_MESSAGE', InitLimit: 'INIT_LIMIT' @@ -55,7 +53,6 @@ const config: Config = (() => { AccountsURL: process.env[envMap.AccountsURL], ServiceID: process.env[envMap.ServiceID] ?? 'gmail-service', Secret: process.env[envMap.Secret], - SystemEmail: process.env[envMap.SystemEmail] ?? 'anticrm@hc.engineering', Credentials: process.env[envMap.Credentials], WATCH_TOPIC_NAME: process.env[envMap.WATCH_TOPIC_NAME], InitLimit: parseNumber(process.env[envMap.InitLimit]) ?? 50, diff --git a/services/gmail/pod-gmail/src/workspaceClient.ts b/services/gmail/pod-gmail/src/workspaceClient.ts index e40a344a2d..9e611dffc3 100644 --- a/services/gmail/pod-gmail/src/workspaceClient.ts +++ b/services/gmail/pod-gmail/src/workspaceClient.ts @@ -20,6 +20,7 @@ import core, { type Doc, MeasureContext, type Ref, + systemAccountEmail, type Tx, type TxCreateDoc, TxProcessor, @@ -31,7 +32,6 @@ import type { StorageAdapter } from '@hcengineering/server-core' import { generateToken } from '@hcengineering/server-token' import { type Db } from 'mongodb' import { getClient } from './client' -import config from './config' import { GmailClient } from './gmail' import { type Channel, type ProjectCredentials, type User } from './types' @@ -121,7 +121,7 @@ export class WorkspaceClient { } private async initClient (workspace: string): Promise { - const token = generateToken(config.SystemEmail, { name: workspace }) + const token = generateToken(systemAccountEmail, { name: workspace }) console.log('token', token, workspace) const client = await getClient(token) client.notify = (...tx: Tx[]) => { diff --git a/services/love/src/config.ts b/services/love/src/config.ts index aaab341ed8..7fee523d4a 100644 --- a/services/love/src/config.ts +++ b/services/love/src/config.ts @@ -16,7 +16,6 @@ interface Config { AccountsURL: string Port: number - SystemEmail: string ServiceID: string LiveKitHost: string @@ -42,7 +41,6 @@ const envMap: { [key in keyof Config]: string } = { StorageProviderName: 'STORAGE_PROVIDER_NAME', Secret: 'SECRET', ServiceID: 'SERVICE_ID', - SystemEmail: 'SYSTEM_EMAIL', MongoUrl: 'MONGO_URL' } @@ -59,7 +57,6 @@ const config: Config = (() => { StorageProviderName: process.env[envMap.StorageProviderName] ?? 's3', Secret: process.env[envMap.Secret], ServiceID: process.env[envMap.ServiceID] ?? 'love-service', - SystemEmail: process.env[envMap.SystemEmail] ?? 'anticrm@hc.engineering', MongoUrl: process.env[envMap.MongoUrl] } diff --git a/services/love/src/workspaceClient.ts b/services/love/src/workspaceClient.ts index 4daf1bc42b..6ee51a7d27 100644 --- a/services/love/src/workspaceClient.ts +++ b/services/love/src/workspaceClient.ts @@ -12,13 +12,20 @@ // See the License for the specific language governing permissions and // limitations under the License. -import core, { Client, Ref, TxOperations, type Blob, Data, MeasureContext } from '@hcengineering/core' +import attachment, { Attachment } from '@hcengineering/attachment' +import core, { + Client, + Data, + MeasureContext, + Ref, + systemAccountEmail, + TxOperations, + type Blob +} from '@hcengineering/core' import drive, { createFile } from '@hcengineering/drive' import love, { MeetingMinutes } from '@hcengineering/love' import { generateToken } from '@hcengineering/server-token' -import attachment, { Attachment } from '@hcengineering/attachment' import { getClient } from './client' -import config from './config' export class WorkspaceClient { private client!: TxOperations @@ -39,7 +46,7 @@ export class WorkspaceClient { } private async initClient (workspace: string): Promise { - const token = generateToken(config.SystemEmail, { name: workspace }) + const token = generateToken(systemAccountEmail, { name: workspace }) const client = await getClient(token) this.client = new TxOperations(client, core.account.System) return this.client diff --git a/services/sign/pod-sign/src/config.ts b/services/sign/pod-sign/src/config.ts index cc71f9abd4..d3d17bec1a 100644 --- a/services/sign/pod-sign/src/config.ts +++ b/services/sign/pod-sign/src/config.ts @@ -11,7 +11,6 @@ export interface Config { Port: number Secret: string ServiceID: string - SystemEmail: string BrandingPath: string } @@ -25,7 +24,6 @@ const config: Config = (() => { Port: parseNumber(process.env.PORT) ?? 4006, Secret: process.env.SECRET, ServiceID: process.env.SERVICE_ID, - SystemEmail: process.env.SYSTEM_EMAIL ?? 'anticrm@hc.engineering', BrandingPath: process.env.BRANDING_PATH ?? '' } diff --git a/services/sign/pod-sign/src/sign.ts b/services/sign/pod-sign/src/sign.ts index a0dab86df4..91b02320f0 100644 --- a/services/sign/pod-sign/src/sign.ts +++ b/services/sign/pod-sign/src/sign.ts @@ -6,7 +6,7 @@ import { P12Signer } from '@signpdf/signer-p12' import signpdf from '@signpdf/signpdf' import { PDFDocument, StandardFonts, degrees, degreesToRadians, rgb } from 'pdf-lib' -import config from './config' +import { systemAccountEmail } from '@hcengineering/core' interface Rect { x: number @@ -50,7 +50,7 @@ export async function signPDF (file: Buffer, certp12: Buffer, pwd: string, ctx: // Make it configurable when will be needed to allow signing for different reasons. const options: Options = { name: ctx.title, - contactInfo: config.SystemEmail, + contactInfo: systemAccountEmail, appName: ctx.title, reason: 'Export from the system', location: 'N/A' diff --git a/services/sign/pod-sign/src/signController.ts b/services/sign/pod-sign/src/signController.ts index f72de5bc64..d3aa96ff5e 100644 --- a/services/sign/pod-sign/src/signController.ts +++ b/services/sign/pod-sign/src/signController.ts @@ -13,11 +13,10 @@ // limitations under the License. // -import { type Client } from '@hcengineering/core' +import { systemAccountEmail, type Client } from '@hcengineering/core' import { generateToken, type Token } from '@hcengineering/server-token' import { createClient, getTransactorEndpoint } from '@hcengineering/server-client' -import config from './config' export class SignController { private readonly clients: Map = new Map() @@ -50,7 +49,7 @@ export class SignController { } private async createPlatformClient (workspace: string): Promise { - const token = generateToken(config.SystemEmail, { + const token = generateToken(systemAccountEmail, { name: workspace }) const endpoint = await getTransactorEndpoint(token) diff --git a/services/telegram/pod-telegram/src/config.ts b/services/telegram/pod-telegram/src/config.ts index 2e0c25836f..b4687813e3 100644 --- a/services/telegram/pod-telegram/src/config.ts +++ b/services/telegram/pod-telegram/src/config.ts @@ -12,7 +12,6 @@ interface Config { AccountsURL: string ServiceID: string Secret: string - SystemEmail: string } const envMap: { [key in keyof Config]: string } = { @@ -28,8 +27,7 @@ const envMap: { [key in keyof Config]: string } = { AccountsURL: 'ACCOUNTS_URL', ServiceID: 'SERVICE_ID', - Secret: 'SECRET', - SystemEmail: 'SYSTEM_EMAIL' + Secret: 'SECRET' } const defaults: Partial = { @@ -45,7 +43,6 @@ const defaults: Partial = { AccountsURL: undefined, ServiceID: 'telegram-service', - SystemEmail: 'anticrm@hc.engineering', Secret: undefined } @@ -76,7 +73,6 @@ const config = (() => { MongoURI: process.env[envMap.MongoURI], AccountsURL: process.env[envMap.AccountsURL], ServiceID: process.env[envMap.ServiceID], - SystemEmail: process.env[envMap.SystemEmail], Secret: process.env[envMap.Secret] } diff --git a/services/telegram/pod-telegram/src/workspace.ts b/services/telegram/pod-telegram/src/workspace.ts index 8a57b4bf8e..e865c23948 100644 --- a/services/telegram/pod-telegram/src/workspace.ts +++ b/services/telegram/pod-telegram/src/workspace.ts @@ -15,6 +15,7 @@ import core, { Hierarchy, MeasureContext, Ref, + systemAccountEmail, Tx, TxCreateDoc, TxCUD, @@ -31,7 +32,6 @@ import telegramP, { NewTelegramMessage } from '@hcengineering/telegram' import type { Collection } from 'mongodb' import { Api } from 'telegram' import { v4 as uuid } from 'uuid' -import config from './config' import { platformToTelegram, telegramToPlatform } from './markup' import { MsgQueue } from './queue' import type { TelegramConnectionInterface } from './telegram' @@ -151,7 +151,7 @@ export class WorkspaceWorker { lastMsgStorage: Collection, channelsStorage: Collection ): Promise { - const token = generateToken(config.SystemEmail, { name: workspace }) + const token = generateToken(systemAccountEmail, { name: workspace }) const client = await createPlatformClient(token) const worker = new WorkspaceWorker( From 118ede7388e95a69754e1c1bc5d1ada99a032b6c Mon Sep 17 00:00:00 2001 From: Chunosov Date: Wed, 25 Dec 2024 14:43:25 +0700 Subject: [PATCH 2/7] Add storage key for viewlet view options (#7545) * add storage key for viewlet view options Signed-off-by: Nikolay Chunosov * load view options from old key when there is no new one Signed-off-by: Nikolay Chunosov --------- Signed-off-by: Nikolay Chunosov --- models/recruit/src/index.ts | 12 ++++++++---- plugins/view-resources/src/viewOptions.ts | 21 +++++++++++++++------ plugins/view/src/types.ts | 1 + 3 files changed, 24 insertions(+), 10 deletions(-) diff --git a/models/recruit/src/index.ts b/models/recruit/src/index.ts index 003de31209..0d01744fff 100644 --- a/models/recruit/src/index.ts +++ b/models/recruit/src/index.ts @@ -361,7 +361,8 @@ export function createModel (builder: Builder): void { viewOptions: { groupBy: [], orderBy: [], - other: [vacancyHideArchivedOption] + other: [vacancyHideArchivedOption], + storageKey: 'vacancyViewOptions' } }, recruit.viewlet.TableVacancy @@ -502,7 +503,8 @@ export function createModel (builder: Builder): void { viewOptions: { groupBy: [], orderBy: [], - other: [applicationDoneOption, hideApplicantsFromArchivedVacanciesOption] + other: [applicationDoneOption, hideApplicantsFromArchivedVacanciesOption], + storageKey: 'applicantViewOptions' } }, recruit.viewlet.ApplicantTable @@ -559,7 +561,8 @@ export function createModel (builder: Builder): void { action: view.function.ShowEmptyGroups, label: view.string.ShowEmptyGroups } - ] + ], + storageKey: 'applicantViewOptions' } if (colors) { model.other.push(showColorsViewOption) @@ -784,7 +787,8 @@ export function createModel (builder: Builder): void { ['modifiedOn', SortingOrder.Descending], ['createdOn', SortingOrder.Descending] ], - other: [vacancyHideArchivedOption] + other: [vacancyHideArchivedOption], + storageKey: 'vacancyViewOptions' } }, recruit.viewlet.ListVacancy diff --git a/plugins/view-resources/src/viewOptions.ts b/plugins/view-resources/src/viewOptions.ts index 6ca0e42bb4..66855b0530 100644 --- a/plugins/view-resources/src/viewOptions.ts +++ b/plugins/view-resources/src/viewOptions.ts @@ -31,8 +31,11 @@ export function isDropdownType (viewOption: ViewOptionModel): viewOption is Drop return viewOption.type === 'dropdown' } -export function makeViewOptionsKey (viewlet: Ref, variant?: string): string { - const prefix = viewlet + (variant !== undefined ? `-${variant}` : '') +function makeViewOptionsKey (viewlet: Viewlet, variant?: string, ignoreViewletKey = false): string { + const prefix = + viewlet.viewOptions?.storageKey !== undefined && !ignoreViewletKey + ? viewlet.viewOptions.storageKey + : viewlet._id + (variant !== undefined ? `-${variant}` : '') const loc = getCurrentResolvedLocation() loc.fragment = undefined loc.query = undefined @@ -40,7 +43,7 @@ export function makeViewOptionsKey (viewlet: Ref, variant?: string): st } export function setViewOptions (viewlet: Viewlet, options: ViewOptions): void { - const key = makeViewOptionsKey(viewlet._id, viewlet.variant) + const key = makeViewOptionsKey(viewlet, viewlet.variant) localStorage.setItem(key, JSON.stringify(options)) setStore(key, options) } @@ -52,13 +55,19 @@ function setStore (key: string, options: ViewOptions): void { } function _getViewOptions (viewlet: Viewlet, viewOptionStore: Map): ViewOptions | null { - const key = makeViewOptionsKey(viewlet._id, viewlet.variant) + const key = makeViewOptionsKey(viewlet, viewlet.variant) const store = viewOptionStore.get(key) if (store !== undefined) { return store } - const options = localStorage.getItem(key) - if (options === null) return null + let options = localStorage.getItem(key) + if (options === null) { + const key = makeViewOptionsKey(viewlet, viewlet.variant, true) + options = localStorage.getItem(key) + if (options === null) { + return null + } + } const res = JSON.parse(options) setStore(key, res) return res diff --git a/plugins/view/src/types.ts b/plugins/view/src/types.ts index 3e343c6308..15586de1f6 100644 --- a/plugins/view/src/types.ts +++ b/plugins/view/src/types.ts @@ -794,6 +794,7 @@ export interface ViewOptionsModel { orderBy: OrderOption[] other: ViewOptionModel[] groupDepth?: number + storageKey?: string } /** From 84020b3305b2c919ea3bfb0f3b1ef20ca439d9fa Mon Sep 17 00:00:00 2001 From: Andrey Sobolev Date: Wed, 25 Dec 2024 16:08:11 +0700 Subject: [PATCH 3/7] UBERF-9017: Reduce createTable calls (#7550) Signed-off-by: Andrey Sobolev --- dev/tool/src/db.ts | 2 +- server/postgres/src/storage.ts | 13 +++++++++++-- server/postgres/src/utils.ts | 34 ++++++++++++++++++++++++---------- 3 files changed, 36 insertions(+), 13 deletions(-) diff --git a/dev/tool/src/db.ts b/dev/tool/src/db.ts index ed4db3d78a..30ecccb7dd 100644 --- a/dev/tool/src/db.ts +++ b/dev/tool/src/db.ts @@ -81,7 +81,7 @@ async function moveWorkspace ( tables = tables.filter((t) => include.has(t)) } - await createTables(new MeasureMetricsContext('', {}), pgClient, tables) + await createTables(new MeasureMetricsContext('', {}), pgClient, '', tables) const token = generateToken(systemAccountEmail, wsId) const endpoint = await getTransactorEndpoint(token, 'external') const connection = (await connect(endpoint, wsId, undefined, { diff --git a/server/postgres/src/storage.ts b/server/postgres/src/storage.ts index 32d82b533d..ba02249c4f 100644 --- a/server/postgres/src/storage.ts +++ b/server/postgres/src/storage.ts @@ -37,6 +37,7 @@ import core, { type ModelDb, type ObjQueryType, type Projection, + RateLimiter, type Ref, type ReverseLookups, type SessionData, @@ -1506,13 +1507,18 @@ interface OperationBulk { mixins: TxMixin[] } +const initRateLimit = new RateLimiter(1) + class PostgresAdapter extends PostgresAdapterBase { async init (ctx: MeasureContext, domains?: string[], excludeDomains?: string[]): Promise { let resultDomains = domains ?? this.hierarchy.domains() if (excludeDomains !== undefined) { resultDomains = resultDomains.filter((it) => !excludeDomains.includes(it)) } - await createTables(ctx, this.client, resultDomains) + const url = this.refClient.url() + await initRateLimit.exec(async () => { + await createTables(ctx, this.client, url, resultDomains) + }) this._helper.domains = new Set(resultDomains as Domain[]) } @@ -1789,7 +1795,10 @@ class PostgresAdapter extends PostgresAdapterBase { class PostgresTxAdapter extends PostgresAdapterBase implements TxAdapter { async init (ctx: MeasureContext, domains?: string[], excludeDomains?: string[]): Promise { const resultDomains = domains ?? [DOMAIN_TX, DOMAIN_MODEL_TX] - await createTables(ctx, this.client, resultDomains) + await initRateLimit.exec(async () => { + const url = this.refClient.url() + await createTables(ctx, this.client, url, resultDomains) + }) this._helper.domains = new Set(resultDomains as Domain[]) } diff --git a/server/postgres/src/utils.ts b/server/postgres/src/utils.ts index 0d985d3f0a..f05927f68c 100644 --- a/server/postgres/src/utils.ts +++ b/server/postgres/src/utils.ts @@ -72,8 +72,13 @@ export const NumericTypes = [ core.class.Collection ] -export async function createTables (ctx: MeasureContext, client: postgres.Sql, domains: string[]): Promise { - const filtered = domains.filter((d) => !loadedDomains.has(d)) +export async function createTables ( + ctx: MeasureContext, + client: postgres.Sql, + url: string, + domains: string[] +): Promise { + const filtered = domains.filter((d) => !loadedDomains.has(url + translateDomain(d))) if (filtered.length === 0) { return } @@ -90,17 +95,15 @@ export async function createTables (ctx: MeasureContext, client: postgres.Sql, d const exists = new Set(tables.map((it) => it.table_name)) await retryTxn(client, async (client) => { - await ctx.with('load-schemas', {}, () => - getTableSchema( - client, - mapped.filter((it) => exists.has(it)) - ) - ) + const domainsToLoad = mapped.filter((it) => exists.has(it)) + if (domainsToLoad.length > 0) { + await ctx.with('load-schemas', {}, () => getTableSchema(client, domainsToLoad)) + } for (const domain of mapped) { if (!exists.has(domain)) { await ctx.with('create-table', {}, () => createTable(client, domain)) } - loadedDomains.add(domain) + loadedDomains.add(url + domain) } }) } @@ -188,6 +191,8 @@ export async function shutdown (): Promise { export interface PostgresClientReference { getClient: () => Promise close: () => void + + url: () => string } class PostgresClientReferenceImpl { @@ -195,6 +200,7 @@ class PostgresClientReferenceImpl { client: postgres.Sql | Promise constructor ( + readonly connectionString: string, client: postgres.Sql | Promise, readonly onclose: () => void ) { @@ -202,6 +208,10 @@ class PostgresClientReferenceImpl { this.client = client } + url (): string { + return this.connectionString + } + async getClient (): Promise { if (this.client instanceof Promise) { this.client = await this.client @@ -233,6 +243,10 @@ export class ClientRef implements PostgresClientReference { clientRefs.set(this.id, this) } + url (): string { + return this.client.url() + } + closed = false async getClient (): Promise { if (!this.closed) { @@ -274,7 +288,7 @@ export function getDBClient (connectionString: string, database?: string): Postg ...extraOptions }) - existing = new PostgresClientReferenceImpl(sql, () => { + existing = new PostgresClientReferenceImpl(connectionString, sql, () => { connections.delete(key) }) connections.set(key, existing) From faac57091c3ea79c0dba141360d0424e9f20b4a9 Mon Sep 17 00:00:00 2001 From: Denis Tingaikin Date: Wed, 25 Dec 2024 18:31:25 +0300 Subject: [PATCH 4/7] Use RateLimiter instead of waiting for each attachment to load sequentially (#7546) Signed-off-by: denis-tingaikin --- .../src/components/AttachmentRefInput.svelte | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/plugins/attachment-resources/src/components/AttachmentRefInput.svelte b/plugins/attachment-resources/src/components/AttachmentRefInput.svelte index c8fb1f7923..1c8758b1ea 100644 --- a/plugins/attachment-resources/src/components/AttachmentRefInput.svelte +++ b/plugins/attachment-resources/src/components/AttachmentRefInput.svelte @@ -14,7 +14,7 @@ -->