Extract domain fields (#7075)

Signed-off-by: Denis Bykhov <bykhov.denis@gmail.com>

server/postgres/migrations/allSchema.sql

@@ -0,0 +1,37 @@
+DO $$
+DECLARE
+    tbl_name text;
+    hash_col_not_exists boolean;
+    data_col_exists boolean;
+BEGIN
+    FOR tbl_name IN 
+        SELECT table_name 
+        FROM information_schema.tables 
+        WHERE table_schema = 'public' 
+        AND table_type = 'BASE TABLE'
+    LOOP
+        EXECUTE format('
+            SELECT EXISTS (
+                SELECT 1 
+                FROM information_schema.columns 
+                WHERE table_name = %L 
+                AND column_name = ''data''
+                AND data_type = ''jsonb''
+            );', tbl_name) INTO data_col_exists;
+
+        EXECUTE format('
+            SELECT NOT EXISTS (
+                SELECT 1 
+                FROM information_schema.columns 
+                WHERE table_name = %L 
+                AND column_name = ''"%%hash%%"'' 
+            );', tbl_name) INTO hash_col_not_exists;
+
+        IF data_col_exists AND hash_col_not_exists THEN
+            EXECUTE format('
+                ALTER TABLE %I ADD COLUMN "%%hash%%" text;', tbl_name);
+            EXECUTE format('
+                UPDATE %I SET "%%hash%%" = data->>''%%data%%'';', tbl_name);
+        END IF;
+    END LOOP;
+END $$;

server/postgres/migrations/dncSchema.sql

@@ -0,0 +1,25 @@
+ALTER TABLE notification_dnc 
+ADD "objectId" text,
+ADD "objectClass" text,
+ADD "user" text;
+
+ALTER TABLE notification_dnc
+DROP COLUMN "attachedTo";
+
+UPDATE notification_dnc
+SET "objectId" = (data->>'objectId');
+
+UPDATE notification_dnc
+SET "objectClass" = (data->>'objectClass');
+
+UPDATE notification_dnc
+SET "user" = (data->>'user');
+
+ALTER TABLE notification_dnc
+ALTER COLUMN "objectId" SET NOT NULL;
+
+ALTER TABLE notification_dnc
+ALTER COLUMN "objectClass" SET NOT NULL;
+
+ALTER TABLE notification_dnc
+ALTER COLUMN "user" SET NOT NULL;

server/postgres/migrations/docIndexStateSchema.sql

@@ -0,0 +1,11 @@
+ALTER TABLE doc_index_state 
+ADD "needIndex" bool;
+
+ALTER TABLE doc_index_state
+DROP COLUMN "attachedTo";
+
+UPDATE doc_index_state
+SET "needIndex" = (data->>'needIndex')::boolean;
+
+ALTER TABLE doc_index_state
+ALTER COLUMN "needIndex" SET NOT NULL;

server/postgres/migrations/notificationSchema.sql

@@ -0,0 +1,25 @@
+ALTER TABLE notification 
+ADD "isViewed" bool,
+ADD archived bool,
+ADD "user" text;
+
+ALTER TABLE notification
+DROP COLUMN "attachedTo";
+
+UPDATE notification
+SET "isViewed" = (data->>'isViewed')::boolean;
+
+UPDATE notification
+SET "archived" = (data->>'archived')::boolean;
+
+UPDATE notification
+SET "user" = (data->>'user');
+
+ALTER TABLE notification
+ALTER COLUMN "isViewed" SET NOT NULL;
+
+ALTER TABLE notification
+ALTER COLUMN archived SET NOT NULL;
+
+ALTER TABLE notification
+ALTER COLUMN "user" SET NOT NULL;

server/postgres/migrations/txSchema.sql

@@ -0,0 +1,15 @@
+ALTER TABLE tx 
+ADD "objectSpace" text,
+ADD "objectId" text;
+
+ALTER TABLE tx
+DROP COLUMN "attachedTo";
+
+UPDATE tx
+SET "objectId" = (data->>'objectId');
+
+UPDATE tx
+SET "objectSpace" = (data->>'objectSpace');
+
+ALTER TABLE tx
+ALTER COLUMN "objectSpace" SET NOT NULL;

server/postgres/migrations/uncSchema.sql

@@ -0,0 +1,11 @@
+ALTER TABLE notification_user 
+ADD "user" text;
+
+ALTER TABLE notification_user
+DROP COLUMN "attachedTo";
+
+UPDATE notification_user
+SET "user" = (data->>'user');
+
+ALTER TABLE notification_user
+ALTER COLUMN "user" SET NOT NULL;

server/postgres/src/index.ts

@@ -14,4 +14,5 @@
 //
 
 export * from './storage'
-export { getDBClient, convertDoc, createTable, retryTxn, translateDomain, getDocFieldsByDomains } from './utils'
+export { getDBClient, convertDoc, createTable, retryTxn } from './utils'
+export { getDocFieldsByDomains, translateDomain } from './schemas'

server/postgres/src/schemas.ts

@@ -1,10 +1,10 @@
-import { DOMAIN_SPACE } from '@hcengineering/core'
+import { DOMAIN_DOC_INDEX_STATE, DOMAIN_SPACE, DOMAIN_TX } from '@hcengineering/core'
 
 type DataType = 'bigint' | 'bool' | 'text' | 'text[]'
 
 type Schema = Record<string, [DataType, boolean]>
 
-export const defaultSchema: Schema = {
+const baseSchema: Schema = {
   _id: ['text', true],
   _class: ['text', true],
   space: ['text', true],
@@ -12,25 +12,68 @@ export const defaultSchema: Schema = {
   createdBy: ['text', false],
   modifiedOn: ['bigint', true],
   createdOn: ['bigint', false],
+  '%hash%': ['text', false]
+}
+
+const defaultSchema: Schema = {
+  ...baseSchema,
   attachedTo: ['text', false]
 }
 
-export const spaceSchema: Schema = {
-  _id: ['text', true],
-  _class: ['text', true],
-  space: ['text', true],
-  modifiedBy: ['text', true],
-  createdBy: ['text', false],
-  modifiedOn: ['bigint', true],
-  createdOn: ['bigint', false],
+const spaceSchema: Schema = {
+  ...baseSchema,
   private: ['bool', true],
   members: ['text[]', true]
 }
 
+const txSchema: Schema = {
+  ...baseSchema,
+  objectSpace: ['text', true],
+  objectId: ['text', false]
+}
+
+const notificationSchema: Schema = {
+  ...baseSchema,
+  isViewed: ['bool', true],
+  archived: ['bool', true],
+  user: ['text', true]
+}
+
+const dncSchema: Schema = {
+  ...baseSchema,
+  objectId: ['text', true],
+  objectClass: ['text', true],
+  user: ['text', true]
+}
+
+const userNotificationSchema: Schema = {
+  ...baseSchema,
+  user: ['text', true]
+}
+
+const docIndexStateSchema: Schema = {
+  ...baseSchema,
+  needIndex: ['bool', true]
+}
+
+export function translateDomain (domain: string): string {
+  return domain.replaceAll('-', '_')
+}
+
 export const domainSchemas: Record<string, Schema> = {
-  [DOMAIN_SPACE]: spaceSchema
+  [DOMAIN_SPACE]: spaceSchema,
+  [DOMAIN_TX]: txSchema,
+  [translateDomain(DOMAIN_DOC_INDEX_STATE)]: docIndexStateSchema,
+  notification: notificationSchema,
+  [translateDomain('notification-dnc')]: dncSchema,
+  [translateDomain('notification-user')]: userNotificationSchema
 }
 
 export function getSchema (domain: string): Schema {
-  return domainSchemas[domain] ?? defaultSchema
+  return domainSchemas[translateDomain(domain)] ?? defaultSchema
+}
+
+export function getDocFieldsByDomains (domain: string): string[] {
+  const schema = domainSchemas[translateDomain(domain)] ?? defaultSchema
+  return Object.keys(schema)
 }

server/postgres/src/storage.ts

@@ -74,7 +74,6 @@ import {
   type DBDoc,
   escapeBackticks,
   getDBClient,
-  getDocFieldsByDomains,
   inferType,
   isDataField,
   isOwner,
@@ -82,9 +81,9 @@ import {
   parseDoc,
   parseDocWithProjection,
   parseUpdate,
-  type PostgresClientReference,
-  translateDomain
+  type PostgresClientReference
 } from './utils'
+import { getDocFieldsByDomains, translateDomain } from './schemas'
 
 abstract class PostgresAdapterBase implements DbAdapter {
   protected readonly _helper: DBCollectionHelper
@@ -1550,7 +1549,7 @@ class PostgresTxAdapter extends PostgresAdapterBase implements TxAdapter {
 
   async getModel (ctx: MeasureContext): Promise<Tx[]> {
     const res = await this
-      .client`SELECT * FROM ${this.client(translateDomain(DOMAIN_TX))} WHERE "workspaceId" = ${this.workspaceId.name} AND data->>'objectSpace' = ${core.space.Model} ORDER BY _id ASC, "modifiedOn" ASC`
+      .client`SELECT * FROM ${this.client(translateDomain(DOMAIN_TX))} WHERE "workspaceId" = ${this.workspaceId.name} AND "objectSpace" = ${core.space.Model} ORDER BY _id ASC, "modifiedOn" ASC`
 
     const model = res.map((p) => parseDoc<Tx>(p as any))
     // We need to put all core.account.System transactions first

server/postgres/src/utils.ts

@@ -30,7 +30,7 @@ import core, {
 import { PlatformError, unknownStatus } from '@hcengineering/platform'
 import { type DomainHelperOperations } from '@hcengineering/server-core'
 import postgres from 'postgres'
-import { defaultSchema, domainSchemas, getSchema } from './schemas'
+import { getDocFieldsByDomains, getSchema, translateDomain } from './schemas'
 
 const connections = new Map<string, PostgresClientReferenceImpl>()
 
@@ -329,10 +329,6 @@ export class DBCollectionHelper implements DomainHelperOperations {
   }
 }
 
-export function translateDomain (domain: string): string {
-  return domain.replaceAll('-', '_')
-}
-
 export function parseDocWithProjection<T extends Doc> (doc: DBDoc, projection: Projection<T> | undefined): T {
   const { workspaceId, data, ...rest } = doc
   for (const key in rest) {
@@ -397,11 +393,6 @@ export function isDataField (domain: string, field: string): boolean {
   return !getDocFieldsByDomains(domain).includes(field)
 }
 
-export function getDocFieldsByDomains (domain: string): string[] {
-  const schema = domainSchemas[domain] ?? defaultSchema
-  return Object.keys(schema)
-}
-
 export interface JoinProps {
   table: string // table to join
   path: string // _id.roles, attachedTo.attachedTo, space...

server/tool/src/index.ts

@@ -33,7 +33,6 @@ import core, {
   WorkspaceId,
   WorkspaceIdWithUrl,
   type Client,
-  type Doc,
   type Ref,
   type WithLookup
 } from '@hcengineering/core'
@@ -112,15 +111,16 @@ export async function initModel (
 
   try {
     logger.log('creating database...', workspaceId)
-    await adapter.upload(ctx, DOMAIN_TX, [
-      {
-        _class: core.class.Tx,
-        _id: 'first-tx' as Ref<Doc>,
-        modifiedBy: core.account.System,
-        modifiedOn: Date.now(),
-        space: core.space.DerivedTx
-      }
-    ])
+    const firstTx: Tx = {
+      _class: core.class.Tx,
+      _id: 'first-tx' as Ref<Tx>,
+      modifiedBy: core.account.System,
+      modifiedOn: Date.now(),
+      space: core.space.DerivedTx,
+      objectSpace: core.space.DerivedTx
+    }
+
+    await adapter.upload(ctx, DOMAIN_TX, [firstTx])
 
     await progress(30)
 

tests/restore-workspace.sh

@@ -3,7 +3,7 @@
 # Restore workspace contents in mongo/elastic
 ./tool.sh backup-restore ./sanity-ws sanity-ws
 
-./tool.sh upgrade-workspace sanity-ws --indexes
+./tool.sh upgrade-workspace sanity-ws --indexes --force
 
 # Re-assign user to workspace.
 ./tool.sh assign-workspace user1 sanity-ws

tests/update-snapshot.sh

@@ -1,9 +1,10 @@
 
 export MINIO_ACCESS_KEY=minioadmin
 export MINIO_SECRET_KEY=minioadmin
-export MINIO_ENDPOINT=localhost:9000
-export MONGO_URL=mongodb://localhost:27017
-export ELASTIC_URL=http://localhost:9200
+export MINIO_ENDPOINT=localhost:9002
+export ACCOUNTS_URL=http://localhost:3003
+export MONGO_URL=mongodb://localhost:27018
+export ELASTIC_URL=http://localhost:9201
 export SERVER_SECRET=secret
 
 location="${1:-./sanity-ws}"