From 9d79ab65aef4da346aa717ab3619be8772bfd8d2 Mon Sep 17 00:00:00 2001 From: Denis Bykhov Date: Mon, 4 Nov 2024 19:53:01 +0500 Subject: [PATCH] Use current db schema (#7094) Signed-off-by: Denis Bykhov --- dev/tool/src/db.ts | 4 +- server/postgres/src/index.ts | 2 +- server/postgres/src/schemas.ts | 149 ++++++++++++++++++++++++++++----- server/postgres/src/storage.ts | 6 +- server/postgres/src/utils.ts | 122 ++++++++++++++++++++------- 5 files changed, 222 insertions(+), 61 deletions(-) diff --git a/dev/tool/src/db.ts b/dev/tool/src/db.ts index 603d18eb93..9edff48cfc 100644 --- a/dev/tool/src/db.ts +++ b/dev/tool/src/db.ts @@ -20,7 +20,7 @@ import { import { getMongoClient, getWorkspaceMongoDB } from '@hcengineering/mongo' import { convertDoc, - createTable, + createTables, getDBClient, getDocFieldsByDomains, retryTxn, @@ -80,7 +80,7 @@ async function moveWorkspace ( tables = tables.filter((t) => include.has(t)) } - await createTable(pgClient, tables) + await createTables(pgClient, tables) const token = generateToken(systemAccountEmail, wsId) const endpoint = await getTransactorEndpoint(token, 'external') const connection = (await connect(endpoint, wsId, undefined, { diff --git a/server/postgres/src/index.ts b/server/postgres/src/index.ts index faa3c82c2c..e6c07fcc4b 100644 --- a/server/postgres/src/index.ts +++ b/server/postgres/src/index.ts @@ -14,5 +14,5 @@ // export * from './storage' -export { getDBClient, convertDoc, createTable, retryTxn } from './utils' +export { getDBClient, convertDoc, createTables, retryTxn } from './utils' export { getDocFieldsByDomains, translateDomain } from './schemas' diff --git a/server/postgres/src/schemas.ts b/server/postgres/src/schemas.ts index 1c866ec4c4..4adb6d0c0f 100644 --- a/server/postgres/src/schemas.ts +++ b/server/postgres/src/schemas.ts @@ -1,59 +1,162 @@ import { DOMAIN_DOC_INDEX_STATE, DOMAIN_SPACE, DOMAIN_TX } from '@hcengineering/core' -type DataType = 'bigint' | 'bool' | 'text' | 'text[]' +export type DataType = 'bigint' | 'bool' | 'text' | 'text[]' -type Schema = Record +export function getIndex (field: FieldSchema): string { + if (field.indexType === undefined || field.indexType === 'btree') { + return '' + } + return ` USING ${field.indexType}` +} + +export interface FieldSchema { + type: DataType + notNull: boolean + index: boolean + indexType?: 'btree' | 'gin' | 'gist' | 'brin' | 'hash' +} + +export type Schema = Record const baseSchema: Schema = { - _id: ['text', true], - _class: ['text', true], - space: ['text', true], - modifiedBy: ['text', true], - createdBy: ['text', false], - modifiedOn: ['bigint', true], - createdOn: ['bigint', false], - '%hash%': ['text', false] + _id: { + type: 'text', + notNull: true, + index: false + }, + _class: { + type: 'text', + notNull: true, + index: true + }, + space: { + type: 'text', + notNull: true, + index: true + }, + modifiedBy: { + type: 'text', + notNull: true, + index: false + }, + createdBy: { + type: 'text', + notNull: false, + index: false + }, + modifiedOn: { + type: 'bigint', + notNull: true, + index: false + }, + createdOn: { + type: 'bigint', + notNull: false, + index: false + }, + '%hash%': { + type: 'text', + notNull: false, + index: false + } } const defaultSchema: Schema = { ...baseSchema, - attachedTo: ['text', false] + attachedTo: { + type: 'text', + notNull: false, + index: true + } } const spaceSchema: Schema = { ...baseSchema, - private: ['bool', true], - members: ['text[]', true] + private: { + type: 'bool', + notNull: true, + index: true + }, + members: { + type: 'text[]', + notNull: true, + index: true, + indexType: 'gin' + } } const txSchema: Schema = { ...baseSchema, - objectSpace: ['text', true], - objectId: ['text', false] + objectSpace: { + type: 'text', + notNull: true, + index: true + }, + objectId: { + type: 'text', + notNull: false, + index: false + } } const notificationSchema: Schema = { ...baseSchema, - isViewed: ['bool', true], - archived: ['bool', true], - user: ['text', true] + isViewed: { + type: 'bool', + notNull: true, + index: true + }, + archived: { + type: 'bool', + notNull: true, + index: true + }, + user: { + type: 'text', + notNull: true, + index: true + } } const dncSchema: Schema = { ...baseSchema, - objectId: ['text', true], - objectClass: ['text', true], - user: ['text', true] + objectId: { + type: 'text', + notNull: true, + index: true + }, + objectClass: { + type: 'text', + notNull: true, + index: false + }, + user: { + type: 'text', + notNull: true, + index: true + } } const userNotificationSchema: Schema = { ...baseSchema, - user: ['text', true] + user: { + type: 'text', + notNull: true, + index: true + } } const docIndexStateSchema: Schema = { ...baseSchema, - needIndex: ['bool', true] + needIndex: { + type: 'bool', + notNull: true, + index: true + } +} + +export function addSchema (domain: string, schema: Schema): void { + domainSchemas[translateDomain(domain)] = schema } export function translateDomain (domain: string): string { diff --git a/server/postgres/src/storage.ts b/server/postgres/src/storage.ts index 9e72e4bb06..69c0321613 100644 --- a/server/postgres/src/storage.ts +++ b/server/postgres/src/storage.ts @@ -69,7 +69,7 @@ import type postgres from 'postgres' import { type ValueType } from './types' import { convertDoc, - createTable, + createTables, DBCollectionHelper, type DBDoc, escapeBackticks, @@ -1302,7 +1302,7 @@ class PostgresAdapter extends PostgresAdapterBase { if (excludeDomains !== undefined) { resultDomains = resultDomains.filter((it) => !excludeDomains.includes(it)) } - await createTable(this.client, resultDomains) + await createTables(this.client, resultDomains) this._helper.domains = new Set(resultDomains as Domain[]) } @@ -1531,7 +1531,7 @@ class PostgresAdapter extends PostgresAdapterBase { class PostgresTxAdapter extends PostgresAdapterBase implements TxAdapter { async init (domains?: string[], excludeDomains?: string[]): Promise { const resultDomains = domains ?? [DOMAIN_TX] - await createTable(this.client, resultDomains) + await createTables(this.client, resultDomains) this._helper.domains = new Set(resultDomains as Domain[]) } diff --git a/server/postgres/src/utils.ts b/server/postgres/src/utils.ts index 550a05c055..47939a2d6d 100644 --- a/server/postgres/src/utils.ts +++ b/server/postgres/src/utils.ts @@ -30,7 +30,15 @@ import core, { import { PlatformError, unknownStatus } from '@hcengineering/platform' import { type DomainHelperOperations } from '@hcengineering/server-core' import postgres from 'postgres' -import { getDocFieldsByDomains, getSchema, translateDomain } from './schemas' +import { + addSchema, + type DataType, + getDocFieldsByDomains, + getIndex, + getSchema, + type Schema, + translateDomain +} from './schemas' const connections = new Map() @@ -42,6 +50,7 @@ process.on('exit', () => { }) const clientRefs = new Map() +const loadedDomains = new Set() export async function retryTxn ( pool: postgres.Sql, @@ -53,46 +62,95 @@ export async function retryTxn ( }) } -export async function createTable (client: postgres.Sql, domains: string[]): Promise { - if (domains.length === 0) { +export async function createTables (client: postgres.Sql, domains: string[]): Promise { + const filtered = domains.filter((d) => !loadedDomains.has(d)) + if (filtered.length === 0) { return } - const mapped = domains.map((p) => translateDomain(p)) + const mapped = filtered.map((p) => translateDomain(p)) + const inArr = mapped.map((it) => `'${it}'`).join(', ') + const tables = await client.unsafe(` + SELECT table_name + FROM information_schema.tables + WHERE table_name IN (${inArr}) + `) + + const exists = new Set(tables.map((it) => it.table_name)) + await retryTxn(client, async (client) => { for (const domain of mapped) { - const schema = getSchema(domain) - const fields: string[] = [] - for (const key in schema) { - const val = schema[key] - fields.push(`"${key}" ${val[0]} ${val[1] ? 'NOT NULL' : ''}`) - } - const colums = fields.join(', ') - const res = await client.unsafe(`CREATE TABLE IF NOT EXISTS ${domain} ( - "workspaceId" text NOT NULL, - ${colums}, - data JSONB NOT NULL, - PRIMARY KEY("workspaceId", _id) - )`) - if (res.count > 0) { - if (schema.attachedTo !== undefined) { - await client.unsafe(` - CREATE INDEX ${domain}_attachedTo ON ${domain} ("attachedTo") - `) - } - await client.unsafe(` - CREATE INDEX ${domain}_class ON ${domain} (_class) - `) - await client.unsafe(` - CREATE INDEX ${domain}_space ON ${domain} (space) - `) - await client.unsafe(` - CREATE INDEX ${domain}_idxgin ON ${domain} USING GIN (data) - `) + if (exists.has(domain)) { + await getTableSchema(client, domain) + } else { + await createTable(client, domain) } + loadedDomains.add(domain) } }) } +async function getTableSchema (client: postgres.Sql, domain: string): Promise { + const res = await client.unsafe(`SELECT column_name, data_type, is_nullable + FROM information_schema.columns + WHERE table_name = '${domain}' ORDER BY ordinal_position ASC; + `) + + const schema: Schema = {} + for (const column of res) { + if (column.column_name === 'workspaceId' || column.column_name === 'data') { + continue + } + schema[column.column_name] = { + type: parseDataType(column.data_type), + notNull: column.is_nullable === 'NO', + index: false + } + } + + addSchema(domain, schema) +} + +function parseDataType (type: string): DataType { + switch (type) { + case 'text': + return 'text' + case 'bigint': + return 'bigint' + case 'boolean': + return 'bool' + case 'ARRAY': + return 'text[]' + } + return 'text' +} + +async function createTable (client: postgres.Sql, domain: string): Promise { + const schema = getSchema(domain) + const fields: string[] = [] + for (const key in schema) { + const val = schema[key] + fields.push(`"${key}" ${val.type} ${val.notNull ? 'NOT NULL' : ''}`) + } + const colums = fields.join(', ') + const res = await client.unsafe(`CREATE TABLE IF NOT EXISTS ${domain} ( + "workspaceId" text NOT NULL, + ${colums}, + data JSONB NOT NULL, + PRIMARY KEY("workspaceId", _id) + )`) + if (res.count > 0) { + for (const key in schema) { + const val = schema[key] + if (val.index) { + await client.unsafe(` + CREATE INDEX ${domain}_${key} ON ${domain} ${getIndex(val)} ("${key}") + `) + } + fields.push(`"${key}" ${val.type} ${val.notNull ? 'NOT NULL' : ''}`) + } + } +} + /** * @public */