Forced update db schema

Signed-off-by: Denis Bykhov <bykhov.denis@gmail.com>

server/postgres/src/schemas.ts

@@ -1,6 +1,5 @@
 import { DOMAIN_DOC_INDEX_STATE, DOMAIN_MODEL_TX, DOMAIN_RELATION, DOMAIN_SPACE, DOMAIN_TX } from '@hcengineering/core'
-
-export type DataType = 'bigint' | 'bool' | 'text' | 'text[]'
+import { type SchemaDiff, type FieldSchema, type Schema } from './types'
 
 export function getIndex (field: FieldSchema): string {
   if (field.indexType === undefined || field.indexType === 'btree') {
@@ -9,15 +8,6 @@ export function getIndex (field: FieldSchema): string {
   return ` USING ${field.indexType}`
 }
 
-export interface FieldSchema {
-  type: DataType
-  notNull: boolean
-  index: boolean
-  indexType?: 'btree' | 'gin' | 'gist' | 'brin' | 'hash'
-}
-
-export type Schema = Record<string, FieldSchema>
-
 const baseSchema: Schema = {
   _id: {
     type: 'text',
@@ -139,6 +129,11 @@ const notificationSchema: Schema = {
     type: 'text',
     notNull: true,
     index: true
+  },
+  docNotifyContext: {
+    type: 'text',
+    notNull: false,
+    index: false
   }
 }
 
@@ -185,7 +180,7 @@ const docIndexStateSchema: Schema = {
 }
 
 const timeSchema: Schema = {
-  ...baseSchema,
+  ...defaultSchema,
   workslots: {
     type: 'bigint',
     notNull: false,
@@ -289,11 +284,98 @@ const githubLogin: Schema = {
   }
 }
 
-export function addSchema (domain: string, schema: Schema): void {
+function addSchema (domain: string, schema: Schema): void {
   domainSchemas[translateDomain(domain)] = schema
   domainSchemaFields.set(domain, createSchemaFields(schema))
 }
 
+// add schema if not forced and return migrate script if have differences
+export function setSchema (domain: string, schema: Schema): string | undefined {
+  const translated = translateDomain(domain)
+  if (forcedSchemas.includes(translated)) {
+    const diff = getSchemaDiff(translated, schema)
+    if (diff !== undefined) {
+      return migrateSchema(translated, diff)
+    }
+  }
+  addSchema(translated, schema)
+}
+
+function migrateSchema (domain: string, diff: SchemaDiff): string {
+  const queries: string[] = []
+  if (diff.remove !== undefined) {
+    for (const key in diff.remove) {
+      const field = diff.remove[key]
+      switch (field.type) {
+        case 'text':
+          queries.push(`UPDATE ${domain} SET data = jsonb_set(data, '{${key}}', to_jsonb("${key}"), true);`)
+          break
+        case 'text[]':
+          queries.push(`UPDATE ${domain} SET data = jsonb_set(data, '{${key}}', to_jsonb("${key}::text[]"), true);`)
+          break
+        case 'bigint':
+          queries.push(`UPDATE ${domain} SET data = jsonb_set(data, '{${key}}', to_jsonb("${key}"::bigint), true);`)
+          break
+        case 'bool':
+          queries.push(`UPDATE ${domain} SET data = jsonb_set(data, '{${key}}', to_jsonb("${key}"::boolean), true);`)
+          break
+      }
+      queries.push(`ALTER TABLE ${domain} DROP COLUMN "${key}"`)
+    }
+  }
+  if (diff.add !== undefined) {
+    for (const key in diff.add) {
+      const field = diff.add[key]
+      queries.push(`ALTER TABLE ${domain} ADD COLUMN "${key}" ${field.type}`)
+      queries.push('COMMIT')
+      switch (field.type) {
+        case 'text':
+          queries.push(`UPDATE ${domain} SET "${key}" = (data->>'${key}');`)
+          break
+        case 'text[]':
+          queries.push(`UPDATE ${domain} SET "${key}" = array(
+            SELECT jsonb_array_elements_text(data->'${key}')
+          )`)
+          break
+        case 'bigint':
+          queries.push(`UPDATE ${domain} SET "${key}" = (data->>'${key}')::bigint;`)
+          break
+        case 'bool':
+          queries.push(`UPDATE ${domain} SET "${key}" = (data->>'${key}')::boolean;`)
+          break
+      }
+      if (field.notNull) {
+        queries.push(`ALTER TABLE ${domain} ALTER COLUMN "${key}" SET NOT NULL`)
+      }
+    }
+  }
+  return queries.join(';')
+}
+
+function getSchemaDiff (domain: string, dbSchema: Schema): SchemaDiff | undefined {
+  const domainSchema = getSchema(domain)
+  const res: SchemaDiff = {}
+  const add: Schema = {}
+  const remove: Schema = {}
+  for (const key in domainSchema) {
+    if (dbSchema[key] === undefined) {
+      add[key] = domainSchema[key]
+    }
+  }
+  for (const key in dbSchema) {
+    if (domainSchema[key] === undefined) {
+      remove[key] = dbSchema[key]
+    }
+  }
+  if (Object.keys(add).length > 0) {
+    res.add = add
+  }
+  if (Object.keys(remove).length > 0) {
+    res.remove = remove
+  }
+  return Object.keys(res).length > 0 ? res : undefined
+}
+
 export function translateDomain (domain: string): string {
   return domain.replaceAll('-', '_')
 }
@@ -315,6 +397,8 @@ export const domainSchemas: Record<string, Schema> = {
   kanban: defaultSchema
 }
 
+const forcedSchemas: string[] = Object.keys(domainSchemas)
+
 export function getSchema (domain: string): Schema {
   return domainSchemas[translateDomain(domain)] ?? defaultSchema
 }

server/postgres/src/storage.ts

@@ -71,15 +71,8 @@ import {
 } from '@hcengineering/server-core'
 import type postgres from 'postgres'
 import { createDBClient, createGreenDBClient, type DBClient } from './client'
-import {
-  getDocFieldsByDomains,
-  getSchema,
-  getSchemaAndFields,
-  type Schema,
-  type SchemaAndFields,
-  translateDomain
-} from './schemas'
-import { type ValueType } from './types'
+import { getDocFieldsByDomains, getSchema, getSchemaAndFields, type SchemaAndFields, translateDomain } from './schemas'
+import { type Schema, type ValueType } from './types'
 import {
   convertArrayParams,
   convertDoc,

server/postgres/src/types.ts

@@ -1 +1,17 @@
 export type ValueType = 'common' | 'array' | 'dataArray'
+
+export type DataType = 'bigint' | 'bool' | 'text' | 'text[]'
+
+export interface FieldSchema {
+  type: DataType
+  notNull: boolean
+  index: boolean
+  indexType?: 'btree' | 'gin' | 'gist' | 'brin' | 'hash'
+}
+
+export type Schema = Record<string, FieldSchema>
+
+export interface SchemaDiff {
+  remove?: Schema
+  add?: Schema
+}

server/postgres/src/utils.ts

@@ -36,16 +36,15 @@ import { type DomainHelperOperations } from '@hcengineering/server-core'
 import postgres, { type Options, type ParameterOrJSON } from 'postgres'
 import type { DBClient } from './client'
 import {
-  addSchema,
-  type DataType,
   getDocFieldsByDomains,
   getIndex,
   getSchema,
   getSchemaAndFields,
-  type Schema,
   type SchemaAndFields,
+  setSchema,
   translateDomain
 } from './schemas'
+import { type Schema, type DataType } from './types'
 
 const clientRefs = new Map<string, ClientRef>()
 const loadedDomains = new Set<string>()
@@ -145,7 +144,10 @@ async function getTableSchema (client: postgres.Sql, domains: string[]): Promise
     }
   }
   for (const [domain, schema] of Object.entries(schemas)) {
-    addSchema(domain, schema)
+    const schemaMigration = setSchema(domain, schema)
+    if (schemaMigration !== undefined) {
+      await client.unsafe(schemaMigration)
+    }
   }
 }