Use current db schema (#7094)

Signed-off-by: Denis Bykhov <bykhov.denis@gmail.com>

dev/tool/src/db.ts

@@ -20,7 +20,7 @@ import {
 import { getMongoClient, getWorkspaceMongoDB } from '@hcengineering/mongo'
 import {
   convertDoc,
-  createTable,
+  createTables,
   getDBClient,
   getDocFieldsByDomains,
   retryTxn,
@@ -80,7 +80,7 @@ async function moveWorkspace (
       tables = tables.filter((t) => include.has(t))
     }
 
-    await createTable(pgClient, tables)
+    await createTables(pgClient, tables)
     const token = generateToken(systemAccountEmail, wsId)
     const endpoint = await getTransactorEndpoint(token, 'external')
     const connection = (await connect(endpoint, wsId, undefined, {

server/postgres/src/index.ts

@@ -14,5 +14,5 @@
 //
 
 export * from './storage'
-export { getDBClient, convertDoc, createTable, retryTxn } from './utils'
+export { getDBClient, convertDoc, createTables, retryTxn } from './utils'
 export { getDocFieldsByDomains, translateDomain } from './schemas'

server/postgres/src/schemas.ts

@@ -1,59 +1,162 @@
 import { DOMAIN_DOC_INDEX_STATE, DOMAIN_SPACE, DOMAIN_TX } from '@hcengineering/core'
 
-type DataType = 'bigint' | 'bool' | 'text' | 'text[]'
+export type DataType = 'bigint' | 'bool' | 'text' | 'text[]'
 
-type Schema = Record<string, [DataType, boolean]>
+export function getIndex (field: FieldSchema): string {
+  if (field.indexType === undefined || field.indexType === 'btree') {
+    return ''
+  }
+  return ` USING ${field.indexType}`
+}
+
+export interface FieldSchema {
+  type: DataType
+  notNull: boolean
+  index: boolean
+  indexType?: 'btree' | 'gin' | 'gist' | 'brin' | 'hash'
+}
+
+export type Schema = Record<string, FieldSchema>
 
 const baseSchema: Schema = {
-  _id: ['text', true],
-  _class: ['text', true],
-  space: ['text', true],
-  modifiedBy: ['text', true],
-  createdBy: ['text', false],
-  modifiedOn: ['bigint', true],
-  createdOn: ['bigint', false],
-  '%hash%': ['text', false]
+  _id: {
+    type: 'text',
+    notNull: true,
+    index: false
+  },
+  _class: {
+    type: 'text',
+    notNull: true,
+    index: true
+  },
+  space: {
+    type: 'text',
+    notNull: true,
+    index: true
+  },
+  modifiedBy: {
+    type: 'text',
+    notNull: true,
+    index: false
+  },
+  createdBy: {
+    type: 'text',
+    notNull: false,
+    index: false
+  },
+  modifiedOn: {
+    type: 'bigint',
+    notNull: true,
+    index: false
+  },
+  createdOn: {
+    type: 'bigint',
+    notNull: false,
+    index: false
+  },
+  '%hash%': {
+    type: 'text',
+    notNull: false,
+    index: false
+  }
 }
 
 const defaultSchema: Schema = {
   ...baseSchema,
-  attachedTo: ['text', false]
+  attachedTo: {
+    type: 'text',
+    notNull: false,
+    index: true
+  }
 }
 
 const spaceSchema: Schema = {
   ...baseSchema,
-  private: ['bool', true],
-  members: ['text[]', true]
+  private: {
+    type: 'bool',
+    notNull: true,
+    index: true
+  },
+  members: {
+    type: 'text[]',
+    notNull: true,
+    index: true,
+    indexType: 'gin'
+  }
 }
 
 const txSchema: Schema = {
   ...baseSchema,
-  objectSpace: ['text', true],
-  objectId: ['text', false]
+  objectSpace: {
+    type: 'text',
+    notNull: true,
+    index: true
+  },
+  objectId: {
+    type: 'text',
+    notNull: false,
+    index: false
+  }
 }
 
 const notificationSchema: Schema = {
   ...baseSchema,
-  isViewed: ['bool', true],
-  archived: ['bool', true],
-  user: ['text', true]
+  isViewed: {
+    type: 'bool',
+    notNull: true,
+    index: true
+  },
+  archived: {
+    type: 'bool',
+    notNull: true,
+    index: true
+  },
+  user: {
+    type: 'text',
+    notNull: true,
+    index: true
+  }
 }
 
 const dncSchema: Schema = {
   ...baseSchema,
-  objectId: ['text', true],
-  objectClass: ['text', true],
-  user: ['text', true]
+  objectId: {
+    type: 'text',
+    notNull: true,
+    index: true
+  },
+  objectClass: {
+    type: 'text',
+    notNull: true,
+    index: false
+  },
+  user: {
+    type: 'text',
+    notNull: true,
+    index: true
+  }
 }
 
 const userNotificationSchema: Schema = {
   ...baseSchema,
-  user: ['text', true]
+  user: {
+    type: 'text',
+    notNull: true,
+    index: true
+  }
 }
 
 const docIndexStateSchema: Schema = {
   ...baseSchema,
-  needIndex: ['bool', true]
+  needIndex: {
+    type: 'bool',
+    notNull: true,
+    index: true
+  }
+}
+
+export function addSchema (domain: string, schema: Schema): void {
+  domainSchemas[translateDomain(domain)] = schema
 }
 
 export function translateDomain (domain: string): string {

server/postgres/src/storage.ts

@@ -69,7 +69,7 @@ import type postgres from 'postgres'
 import { type ValueType } from './types'
 import {
   convertDoc,
-  createTable,
+  createTables,
   DBCollectionHelper,
   type DBDoc,
   escapeBackticks,
@@ -1302,7 +1302,7 @@ class PostgresAdapter extends PostgresAdapterBase {
     if (excludeDomains !== undefined) {
       resultDomains = resultDomains.filter((it) => !excludeDomains.includes(it))
     }
-    await createTable(this.client, resultDomains)
+    await createTables(this.client, resultDomains)
     this._helper.domains = new Set(resultDomains as Domain[])
   }
 
@@ -1531,7 +1531,7 @@ class PostgresAdapter extends PostgresAdapterBase {
 class PostgresTxAdapter extends PostgresAdapterBase implements TxAdapter {
   async init (domains?: string[], excludeDomains?: string[]): Promise<void> {
     const resultDomains = domains ?? [DOMAIN_TX]
-    await createTable(this.client, resultDomains)
+    await createTables(this.client, resultDomains)
     this._helper.domains = new Set(resultDomains as Domain[])
   }
 

server/postgres/src/utils.ts

@@ -30,7 +30,15 @@ import core, {
 import { PlatformError, unknownStatus } from '@hcengineering/platform'
 import { type DomainHelperOperations } from '@hcengineering/server-core'
 import postgres from 'postgres'
-import { getDocFieldsByDomains, getSchema, translateDomain } from './schemas'
+import {
+  addSchema,
+  type DataType,
+  getDocFieldsByDomains,
+  getIndex,
+  getSchema,
+  type Schema,
+  translateDomain
+} from './schemas'
 
 const connections = new Map<string, PostgresClientReferenceImpl>()
 
@@ -42,6 +50,7 @@ process.on('exit', () => {
 })
 
 const clientRefs = new Map<string, ClientRef>()
+const loadedDomains = new Set<string>()
 
 export async function retryTxn (
   pool: postgres.Sql,
@@ -53,46 +62,95 @@ export async function retryTxn (
   })
 }
 
-export async function createTable (client: postgres.Sql, domains: string[]): Promise<void> {
-  if (domains.length === 0) {
+export async function createTables (client: postgres.Sql, domains: string[]): Promise<void> {
+  const filtered = domains.filter((d) => !loadedDomains.has(d))
+  if (filtered.length === 0) {
     return
   }
-  const mapped = domains.map((p) => translateDomain(p))
+  const mapped = filtered.map((p) => translateDomain(p))
+  const inArr = mapped.map((it) => `'${it}'`).join(', ')
+  const tables = await client.unsafe(`
+    SELECT table_name 
+    FROM information_schema.tables 
+    WHERE table_name IN (${inArr})
+  `)
+
+  const exists = new Set(tables.map((it) => it.table_name))
+
   await retryTxn(client, async (client) => {
     for (const domain of mapped) {
-      const schema = getSchema(domain)
-      const fields: string[] = []
-      for (const key in schema) {
-        const val = schema[key]
-        fields.push(`"${key}" ${val[0]} ${val[1] ? 'NOT NULL' : ''}`)
-      }
-      const colums = fields.join(', ')
-      const res = await client.unsafe(`CREATE TABLE IF NOT EXISTS ${domain} (
-          "workspaceId" text NOT NULL,
-          ${colums}, 
-          data JSONB NOT NULL,
-          PRIMARY KEY("workspaceId", _id)
-        )`)
-      if (res.count > 0) {
-        if (schema.attachedTo !== undefined) {
-          await client.unsafe(`
-            CREATE INDEX ${domain}_attachedTo ON ${domain} ("attachedTo")
-          `)
-        }
-        await client.unsafe(`
-          CREATE INDEX ${domain}_class ON ${domain} (_class)
-        `)
-        await client.unsafe(`
-          CREATE INDEX ${domain}_space ON ${domain} (space)
-        `)
-        await client.unsafe(`
-          CREATE INDEX ${domain}_idxgin ON ${domain} USING GIN (data)
-        `)
+      if (exists.has(domain)) {
+        await getTableSchema(client, domain)
+      } else {
+        await createTable(client, domain)
       }
+      loadedDomains.add(domain)
     }
   })
 }
 
+async function getTableSchema (client: postgres.Sql, domain: string): Promise<void> {
+  const res = await client.unsafe(`SELECT column_name, data_type, is_nullable
+    FROM information_schema.columns
+    WHERE table_name = '${domain}' ORDER BY ordinal_position ASC;
+  `)
+
+  const schema: Schema = {}
+  for (const column of res) {
+    if (column.column_name === 'workspaceId' || column.column_name === 'data') {
+      continue
+    }
+    schema[column.column_name] = {
+      type: parseDataType(column.data_type),
+      notNull: column.is_nullable === 'NO',
+      index: false
+    }
+  }
+
+  addSchema(domain, schema)
+}
+
+function parseDataType (type: string): DataType {
+  switch (type) {
+    case 'text':
+      return 'text'
+    case 'bigint':
+      return 'bigint'
+    case 'boolean':
+      return 'bool'
+    case 'ARRAY':
+      return 'text[]'
+  }
+  return 'text'
+}
+
+async function createTable (client: postgres.Sql, domain: string): Promise<void> {
+  const schema = getSchema(domain)
+  const fields: string[] = []
+  for (const key in schema) {
+    const val = schema[key]
+    fields.push(`"${key}" ${val.type} ${val.notNull ? 'NOT NULL' : ''}`)
+  }
+  const colums = fields.join(', ')
+  const res = await client.unsafe(`CREATE TABLE IF NOT EXISTS ${domain} (
+      "workspaceId" text NOT NULL,
+      ${colums}, 
+      data JSONB NOT NULL,
+      PRIMARY KEY("workspaceId", _id)
+    )`)
+  if (res.count > 0) {
+    for (const key in schema) {
+      const val = schema[key]
+      if (val.index) {
+        await client.unsafe(`
+          CREATE INDEX ${domain}_${key} ON ${domain} ${getIndex(val)} ("${key}")
+        `)
+      }
+      fields.push(`"${key}" ${val.type} ${val.notNull ? 'NOT NULL' : ''}`)
+    }
+  }
+}
+
 /**
  * @public
  */