From c675f4549142201cbe07ca5b0e66ab95dc70590b Mon Sep 17 00:00:00 2001 From: Alexey Zinoviev Date: Sat, 3 Aug 2024 10:10:44 +0400 Subject: [PATCH 1/8] uberf-7764: improve space permissions query (#6236) Signed-off-by: Alexey Zinoviev --- plugins/view-resources/src/utils.ts | 107 +++++++++++++++++----------- 1 file changed, 64 insertions(+), 43 deletions(-) diff --git a/plugins/view-resources/src/utils.ts b/plugins/view-resources/src/utils.ts index 7f5ba88114..df58df7436 100644 --- a/plugins/view-resources/src/utils.ts +++ b/plugins/view-resources/src/utils.ts @@ -1495,59 +1495,80 @@ export const permissionsStore = writable({ ap: {}, whitelist: new Set() }) + +const spaceTypesQuery = createQuery(true) const permissionsQuery = createQuery(true) +type TargetClassesProjection = Record>, number> -permissionsQuery.query(core.class.Space, {}, (res) => { - const whitelistedSpaces = new Set>() - const permissionsBySpace: PermissionsBySpace = {} - const accountsByPermission: AccountsByPermission = {} - const client = getClient() - const hierarchy = client.getHierarchy() - const me = getCurrentAccount() +spaceTypesQuery.query(core.class.SpaceType, {}, (types) => { + const targetClasses = types.reduce((acc, st) => { + acc[st.targetClass] = 1 + return acc + }, {}) - for (const s of res) { - if (hierarchy.isDerived(s._class, core.class.TypedSpace)) { - const type = client.getModel().findAllSync(core.class.SpaceType, { _id: (s as TypedSpace).type })[0] - const mixin = type?.targetClass + permissionsQuery.query( + core.class.Space, + {}, + (res) => { + const whitelistedSpaces = new Set>() + const permissionsBySpace: PermissionsBySpace = {} + const accountsByPermission: AccountsByPermission = {} + const client = getClient() + const hierarchy = client.getHierarchy() + const me = getCurrentAccount() - if (mixin === undefined) { - permissionsBySpace[s._id] = new Set() - accountsByPermission[s._id] = {} - continue - } + for (const s of res) { + if (hierarchy.isDerived(s._class, core.class.TypedSpace)) { + const type = client.getModel().findAllSync(core.class.SpaceType, { _id: (s as TypedSpace).type })[0] + const mixin = type?.targetClass - const asMixin = hierarchy.as(s, mixin) - const roles = client.getModel().findAllSync(core.class.Role, { attachedTo: type._id }) - const myRoles = roles.filter((r) => ((asMixin as any)[r._id] ?? []).includes(me._id)) - permissionsBySpace[s._id] = new Set(myRoles.flatMap((r) => r.permissions)) - - accountsByPermission[s._id] = {} - - for (const role of roles) { - const assignment: Array> = (asMixin as any)[role._id] ?? [] - - if (assignment.length === 0) { - continue - } - - for (const permissionId of role.permissions) { - if (accountsByPermission[s._id][permissionId] === undefined) { - accountsByPermission[s._id][permissionId] = new Set() + if (mixin === undefined) { + permissionsBySpace[s._id] = new Set() + accountsByPermission[s._id] = {} + continue } - assignment.forEach((acc) => accountsByPermission[s._id][permissionId].add(acc)) + const asMixin = hierarchy.as(s, mixin) + const roles = client.getModel().findAllSync(core.class.Role, { attachedTo: type._id }) + const myRoles = roles.filter((r) => ((asMixin as any)[r._id] ?? []).includes(me._id)) + permissionsBySpace[s._id] = new Set(myRoles.flatMap((r) => r.permissions)) + + accountsByPermission[s._id] = {} + + for (const role of roles) { + const assignment: Array> = (asMixin as any)[role._id] ?? [] + + if (assignment.length === 0) { + continue + } + + for (const permissionId of role.permissions) { + if (accountsByPermission[s._id][permissionId] === undefined) { + accountsByPermission[s._id][permissionId] = new Set() + } + + assignment.forEach((acc) => accountsByPermission[s._id][permissionId].add(acc)) + } + } + } else { + whitelistedSpaces.add(s._id) } } - } else { - whitelistedSpaces.add(s._id) - } - } - permissionsStore.set({ - ps: permissionsBySpace, - ap: accountsByPermission, - whitelist: whitelistedSpaces - }) + permissionsStore.set({ + ps: permissionsBySpace, + ap: accountsByPermission, + whitelist: whitelistedSpaces + }) + }, + { + projection: { + _id: 1, + type: 1, + ...targetClasses + } as any + } + ) }) export function getCollaborationUser (): CollaborationUser { From 7bf2a7c8d1600614474d5dda256d853f8445a964 Mon Sep 17 00:00:00 2001 From: Kristina Date: Sat, 3 Aug 2024 10:11:06 +0400 Subject: [PATCH 2/8] Remove slow trigger (#6240) Signed-off-by: Kristina Fefelova --- desktop-package/package.json | 2 +- desktop/package.json | 2 +- models/server-notification/src/index.ts | 8 --- .../src/activityMessagesUtils.ts | 2 +- plugins/chunter-resources/src/utils.ts | 12 ++++- .../notification-resources/src/index.ts | 53 ------------------- server-plugins/notification/src/index.ts | 1 - 7 files changed, 14 insertions(+), 66 deletions(-) diff --git a/desktop-package/package.json b/desktop-package/package.json index 0b7e3cb98d..2291d5c053 100644 --- a/desktop-package/package.json +++ b/desktop-package/package.json @@ -1,6 +1,6 @@ { "name": "desktop", - "version": "0.6.266", + "version": "0.6.271", "main": "dist/main/electron.js", "author": "Hardcore Engineering ", "template": "@hcengineering/default-package", diff --git a/desktop/package.json b/desktop/package.json index 686f32fe53..b6aa591571 100644 --- a/desktop/package.json +++ b/desktop/package.json @@ -1,6 +1,6 @@ { "name": "@hcengineering/desktop", - "version": "0.6.266", + "version": "0.6.271", "main": "dist/main/electron.js", "template": "@hcengineering/webpack-package", "scripts": { diff --git a/models/server-notification/src/index.ts b/models/server-notification/src/index.ts index 6fc68eab55..d6e371b5a4 100644 --- a/models/server-notification/src/index.ts +++ b/models/server-notification/src/index.ts @@ -72,14 +72,6 @@ export function createModel (builder: Builder): void { TNotificationProviderResources ) - builder.createDoc(serverCore.class.Trigger, core.space.Model, { - trigger: serverNotification.trigger.OnActivityNotificationViewed, - txMatch: { - _class: core.class.TxUpdateDoc, - objectClass: notification.class.ActivityInboxNotification - } - }) - builder.createDoc(serverCore.class.Trigger, core.space.Model, { trigger: serverNotification.trigger.OnAttributeCreate, txMatch: { diff --git a/plugins/activity-resources/src/activityMessagesUtils.ts b/plugins/activity-resources/src/activityMessagesUtils.ts index 72d7ca9644..4d503fd869 100644 --- a/plugins/activity-resources/src/activityMessagesUtils.ts +++ b/plugins/activity-resources/src/activityMessagesUtils.ts @@ -579,7 +579,7 @@ export function isActivityMessage (message?: Doc): message is ActivityMessage { return getClient().getHierarchy().isDerived(message._class, activity.class.ActivityMessage) } -export function isReactionMessage (message?: ActivityMessage): boolean { +export function isReactionMessage (message?: ActivityMessage): message is DocUpdateMessage { if (message === undefined) { return false } diff --git a/plugins/chunter-resources/src/utils.ts b/plugins/chunter-resources/src/utils.ts index ea07a1bc0b..05baaf95ce 100644 --- a/plugins/chunter-resources/src/utils.ts +++ b/plugins/chunter-resources/src/utils.ts @@ -46,6 +46,7 @@ import { getClient } from '@hcengineering/presentation' import { type AnySvelteComponent } from '@hcengineering/ui' import { classIcon, getDocLinkTitle, getDocTitle } from '@hcengineering/view-resources' import { get, writable, type Unsubscriber } from 'svelte/store' +import { isReactionMessage } from '@hcengineering/activity-resources' import ChannelIcon from './components/ChannelIcon.svelte' import DirectIcon from './components/DirectIcon.svelte' @@ -439,7 +440,16 @@ export async function readChannelMessages ( const allIds = getAllIds(messages).filter((id) => !readMessages.has(id)) const notifications = get(inboxClient.activityInboxNotifications) - .filter(({ _id, attachedTo }) => allIds.includes(attachedTo)) + .filter(({ attachedTo, $lookup, isViewed }) => { + if (isViewed) return false + const includes = allIds.includes(attachedTo) + if (includes) return true + const msg = $lookup?.attachedTo + if (isReactionMessage(msg)) { + return allIds.includes(msg.attachedTo as Ref) + } + return false + }) .map((n) => n._id) const relatedMentions = get(inboxClient.otherInboxNotifications) diff --git a/server-plugins/notification-resources/src/index.ts b/server-plugins/notification-resources/src/index.ts index a3113231f4..2d6012e6b3 100644 --- a/server-plugins/notification-resources/src/index.ts +++ b/server-plugins/notification-resources/src/index.ts @@ -1516,58 +1516,6 @@ export async function removeDocInboxNotifications (_id: Ref, co ) } -async function OnActivityNotificationViewed ( - tx: TxUpdateDoc, - control: TriggerControl -): Promise { - if (tx.objectClass !== notification.class.ActivityInboxNotification || tx.operations.isViewed !== true) { - return [] - } - - const inboxNotification = ( - await control.findAll( - notification.class.ActivityInboxNotification, - { - _id: tx.objectId as Ref - }, - { projection: { _id: 1, attachedTo: 1, user: 1 } } - ) - )[0] - - if (inboxNotification === undefined) { - return [] - } - - // Read reactions notifications when message is read - const { attachedTo, user } = inboxNotification - - const reactionMessages = await control.findAll( - activity.class.DocUpdateMessage, - { - attachedTo, - objectClass: activity.class.Reaction - }, - { projection: { _id: 1 } } - ) - - if (reactionMessages.length === 0) { - return [] - } - - const reactionNotifications = await control.findAll( - notification.class.ActivityInboxNotification, - { - attachedTo: { $in: reactionMessages.map(({ _id }) => _id) }, - user - }, - { projection: { _id: 1, _class: 1, space: 1 } } - ) - - return reactionNotifications.map(({ _id, _class, space }) => - control.txFactory.createTxUpdateDoc(_class, space, _id, { isViewed: true }) - ) -} - export async function getCollaborators ( doc: Doc, control: TriggerControl, @@ -1659,7 +1607,6 @@ export default async () => ({ trigger: { OnAttributeCreate, OnAttributeUpdate, - OnActivityNotificationViewed, OnDocRemove }, function: { diff --git a/server-plugins/notification/src/index.ts b/server-plugins/notification/src/index.ts index a6561a3760..331cf45822 100644 --- a/server-plugins/notification/src/index.ts +++ b/server-plugins/notification/src/index.ts @@ -180,7 +180,6 @@ export default plugin(serverNotificationId, { OnAttributeCreate: '' as Resource, OnAttributeUpdate: '' as Resource, OnReactionChanged: '' as Resource, - OnActivityNotificationViewed: '' as Resource, OnDocRemove: '' as Resource }, function: { From f95f2b1dda7dc5c4599c2615c729fc0b8b7f71ad Mon Sep 17 00:00:00 2001 From: Alexander Onnikov Date: Mon, 5 Aug 2024 11:45:38 +0700 Subject: [PATCH 3/8] fix: properly update uppy state (#6252) Signed-off-by: Alexander Onnikov --- .../src/components/FileUploadStatusBar.svelte | 40 ++++++------------- .../components/FileUploadStatusPopup.svelte | 29 +++++--------- 2 files changed, 24 insertions(+), 45 deletions(-) diff --git a/plugins/uploader-resources/src/components/FileUploadStatusBar.svelte b/plugins/uploader-resources/src/components/FileUploadStatusBar.svelte index 9e0f45a3db..a7070e0df7 100644 --- a/plugins/uploader-resources/src/components/FileUploadStatusBar.svelte +++ b/plugins/uploader-resources/src/components/FileUploadStatusBar.svelte @@ -28,44 +28,30 @@ let state: UppyState = upload.uppy.getState() let progress: number = state.totalProgress + $: state = upload.uppy.getState() + $: progress = state.totalProgress $: files = Object.values(state.files) $: filesTotal = files.length $: filesComplete = files.filter((p) => p.progress?.uploadComplete).length - function handleProgress (totalProgress: number): void { - progress = totalProgress - } - - function handleUploadProgress (): void { - state = upload.uppy.getState() - } - - function handleUploadSuccess (): void { - state = upload.uppy.getState() - } - - function handleFileRemoved (): void { - state = upload.uppy.getState() - } - - function handleError (): void { + function updateState (): void { state = upload.uppy.getState() } onMount(() => { - upload.uppy.on('error', handleError) - upload.uppy.on('progress', handleProgress) - upload.uppy.on('upload-progress', handleUploadProgress) - upload.uppy.on('upload-success', handleUploadSuccess) - upload.uppy.on('file-removed', handleFileRemoved) + upload.uppy.on('error', updateState) + upload.uppy.on('progress', updateState) + upload.uppy.on('upload-progress', updateState) + upload.uppy.on('upload-success', updateState) + upload.uppy.on('file-removed', updateState) }) onDestroy(() => { - upload.uppy.off('error', handleError) - upload.uppy.off('progress', handleProgress) - upload.uppy.off('upload-progress', handleUploadProgress) - upload.uppy.off('upload-success', handleUploadSuccess) - upload.uppy.off('file-removed', handleFileRemoved) + upload.uppy.off('error', updateState) + upload.uppy.off('progress', updateState) + upload.uppy.off('upload-progress', updateState) + upload.uppy.off('upload-success', updateState) + upload.uppy.off('file-removed', updateState) }) function handleClick (ev: MouseEvent): void { diff --git a/plugins/uploader-resources/src/components/FileUploadStatusPopup.svelte b/plugins/uploader-resources/src/components/FileUploadStatusPopup.svelte index 1a23c485cb..717a8c3508 100644 --- a/plugins/uploader-resources/src/components/FileUploadStatusPopup.svelte +++ b/plugins/uploader-resources/src/components/FileUploadStatusPopup.svelte @@ -40,28 +40,21 @@ let state: UppyState = upload.uppy.getState() + $: state = upload.uppy.getState() $: files = Object.values(state.files) $: capabilities = state.capabilities ?? {} $: individualCancellation = 'individualCancellation' in capabilities && capabilities.individualCancellation + function updateState (): void { + state = upload.uppy.getState() + } + function handleComplete (): void { if (upload.uppy.getState().error === undefined) { dispatch('close') } } - function handleUploadError (): void { - state = upload.uppy.getState() - } - - function handleUploadProgress (): void { - state = upload.uppy.getState() - } - - function handleUploadSuccess (): void { - state = upload.uppy.getState() - } - function handleFileRemoved (): void { state = upload.uppy.getState() const files = upload.uppy.getFiles() @@ -84,18 +77,18 @@ onMount(() => { upload.uppy.on('complete', handleComplete) - upload.uppy.on('upload-error', handleUploadError) - upload.uppy.on('upload-progress', handleUploadProgress) - upload.uppy.on('upload-success', handleUploadSuccess) upload.uppy.on('file-removed', handleFileRemoved) + upload.uppy.on('upload-error', updateState) + upload.uppy.on('upload-progress', updateState) + upload.uppy.on('upload-success', updateState) }) onDestroy(() => { upload.uppy.off('complete', handleComplete) - upload.uppy.off('upload-error', handleUploadError) - upload.uppy.off('upload-progress', handleUploadProgress) - upload.uppy.off('upload-success', handleUploadSuccess) upload.uppy.off('file-removed', handleFileRemoved) + upload.uppy.off('upload-error', updateState) + upload.uppy.off('upload-progress', updateState) + upload.uppy.off('upload-success', updateState) }) function getFileError (file: UppyFile): string | undefined { From 0076c4f76c664a0de862c6f0e59df469daaefcba Mon Sep 17 00:00:00 2001 From: Alexander Onnikov Date: Mon, 5 Aug 2024 12:34:51 +0700 Subject: [PATCH 4/8] fix: remove provider from preview config (#6253) Signed-off-by: Alexander Onnikov --- server/front/src/starter.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/front/src/starter.ts b/server/front/src/starter.ts index 6ba2d4ae47..8ba776e4bf 100644 --- a/server/front/src/starter.ts +++ b/server/front/src/starter.ts @@ -109,7 +109,7 @@ export function startFront (ctx: MeasureContext, extraConfig?: Record Date: Mon, 5 Aug 2024 08:36:08 +0300 Subject: [PATCH 5/8] Added tests with backlink checks (#6251) Signed-off-by: Alexander Platov --- tests/sanity/tests/chat/chat.spec.ts | 27 +++++++++++++ tests/sanity/tests/model/channel-page.ts | 18 +++++++-- tests/sanity/tests/model/common-page.ts | 10 ++++- .../model/tracker/issues-details-page.ts | 1 + .../sanity/tests/model/tracker/issues-page.ts | 9 +++-- tests/sanity/tests/model/tracker/types.ts | 1 + tests/sanity/tests/tracker/mentions.spec.ts | 38 +++++++++++++++++++ 7 files changed, 95 insertions(+), 9 deletions(-) diff --git a/tests/sanity/tests/chat/chat.spec.ts b/tests/sanity/tests/chat/chat.spec.ts index 4e29c88d34..58abaf09a3 100644 --- a/tests/sanity/tests/chat/chat.spec.ts +++ b/tests/sanity/tests/chat/chat.spec.ts @@ -188,6 +188,7 @@ test.describe('channel tests', () => { await channelPage.clickChannelTab() await channelPage.clickOnUser(data.lastName + ' ' + data.firstName) await channelPage.addMemberToChannel(newUser2.lastName + ' ' + newUser2.firstName) + await channelPage.pressEscape() await leftSideMenuPageSecond.clickChunter() await channelPageSecond.checkIfChannelDefaultExist(true, data.channelName) await channelPageSecond.clickChannelTab() @@ -409,4 +410,30 @@ test.describe('channel tests', () => { await channelPage.addMemberToChannelPreview(newUser2.lastName + ' ' + newUser2.firstName) await page2.close() }) + + test('Checking backlinks in the Chat', async ({ browser, page }) => { + await leftSideMenuPage.openProfileMenu() + await leftSideMenuPage.inviteToWorkspace() + await leftSideMenuPage.getInviteLink() + const linkText = await page.locator('.antiPopup .link').textContent() + await leftSideMenuPage.clickOnCloseInvite() + const page2 = await browser.newPage() + const leftSideMenuPageSecond = new LeftSideMenuPage(page2) + const channelPageSecond = new ChannelPage(page2) + await api.createAccount(newUser2.email, newUser2.password, newUser2.firstName, newUser2.lastName) + await page2.goto(linkText ?? '') + const joinPage = new SignInJoinPage(page2) + await joinPage.join(newUser2) + + await leftSideMenuPage.clickChunter() + await channelPage.clickChannel('general') + const mentionName = `${newUser2.lastName} ${newUser2.firstName}` + await channelPage.sendMention(mentionName) + await channelPage.checkMessageExist(`@${mentionName}`, true, `@${mentionName}`) + + await leftSideMenuPageSecond.clickChunter() + await channelPageSecond.clickChannel('general') + await channelPageSecond.checkMessageExist(`@${mentionName}`, true, `@${mentionName}`) + await page2.close() + }) }) diff --git a/tests/sanity/tests/model/channel-page.ts b/tests/sanity/tests/model/channel-page.ts index 7226e1c78a..9991085cc1 100644 --- a/tests/sanity/tests/model/channel-page.ts +++ b/tests/sanity/tests/model/channel-page.ts @@ -1,15 +1,19 @@ import { expect, type Locator, type Page } from '@playwright/test' +import { CommonPage } from './common-page' -export class ChannelPage { +export class ChannelPage extends CommonPage { readonly page: Page constructor (page: Page) { + super(page) this.page = page } readonly inputMessage = (): Locator => this.page.locator('div[class~="text-editor-view"]') readonly buttonSendMessage = (): Locator => this.page.locator('g#Send') - readonly textMessage = (messageText: string): Locator => this.page.getByText(messageText) + readonly textMessage = (messageText: string): Locator => + this.page.locator('.hulyComponent .activityMessage', { hasText: messageText }) + readonly channelName = (channel: string): Locator => this.page.getByText('general random').getByText(channel) readonly channelTab = (): Locator => this.page.getByRole('link', { name: 'Channels' }).getByRole('button') readonly channelTable = (): Locator => this.page.getByRole('table') @@ -71,6 +75,12 @@ export class ChannelPage { await this.buttonSendMessage().click() } + async sendMention (message: string): Promise { + await this.inputMessage().fill(`@${message}`) + await this.selectMention(message) + await this.buttonSendMessage().click() + } + async clickOnOpenChannelDetails (): Promise { await this.openChannelDetails().click() } @@ -232,9 +242,9 @@ export class ChannelPage { async checkMessageExist (message: string, messageExists: boolean, messageText: string): Promise { if (messageExists) { - await expect(this.textMessage(messageText).filter({ hasText: message })).toBeVisible() + await expect(this.textMessage(messageText)).toBeVisible() } else { - await expect(this.textMessage(messageText).filter({ hasText: message })).toBeHidden() + await expect(this.textMessage(messageText)).toBeHidden() } } diff --git a/tests/sanity/tests/model/common-page.ts b/tests/sanity/tests/model/common-page.ts index 0ee1643926..cd94426f64 100644 --- a/tests/sanity/tests/model/common-page.ts +++ b/tests/sanity/tests/model/common-page.ts @@ -38,7 +38,7 @@ export class CommonPage { historyBoxButtonFirst = (): Locator => this.page.locator('div.history-box button:first-child') inboxNotyButton = (): Locator => this.page.locator('button[id$="Inbox"] > div.noty') mentionPopupListItem = (mentionName: string): Locator => - this.page.locator('form.mentionPoup div.list-item span.name', { hasText: mentionName }) + this.page.locator('form.mentionPoup div.list-item', { hasText: mentionName }) hulyPopupRowButton = (name: string): Locator => this.page.locator('div.hulyPopup-container button.hulyPopup-row', { hasText: name }) @@ -298,4 +298,12 @@ export class CommonPage { async openRowInTableByText (text: string): Promise { await this.linesFromTable(text).locator('a', { hasText: text }).click() } + + async checkRowsInListExist (text: string, count: number = 1): Promise { + await expect(this.linesFromList(text)).toHaveCount(count) + } + + async pressEscape (): Promise { + await this.page.keyboard.press('Escape') + } } diff --git a/tests/sanity/tests/model/tracker/issues-details-page.ts b/tests/sanity/tests/model/tracker/issues-details-page.ts index 051b39fc1f..e1e77c44a3 100644 --- a/tests/sanity/tests/model/tracker/issues-details-page.ts +++ b/tests/sanity/tests/model/tracker/issues-details-page.ts @@ -10,6 +10,7 @@ export class IssuesDetailsPage extends CommonTrackerPage { this.page = page } + readonly issueTitle = (): Locator => this.page.locator('div.hulyHeader-container div.title') readonly inputTitle = (): Locator => this.page.locator('div.popupPanel-body input[type="text"]') readonly inputDescription = (): Locator => this.page.locator('div.popupPanel-body div.textInput div.tiptap') readonly textIdentifier = (): Locator => this.page.locator('div.title.not-active') diff --git a/tests/sanity/tests/model/tracker/issues-page.ts b/tests/sanity/tests/model/tracker/issues-page.ts index 2c409e39eb..869789e494 100644 --- a/tests/sanity/tests/model/tracker/issues-page.ts +++ b/tests/sanity/tests/model/tracker/issues-page.ts @@ -19,6 +19,7 @@ export class IssuesPage extends CommonTrackerPage { inputPopupCreateNewIssueDescription = (): Locator => this.page.locator('form[id="tracker:string:NewIssue"] div.tiptap') + buttonPopupCreateNewIssueProject = (): Locator => this.page.locator('[id="space\\.selector"]') buttonPopupCreateNewIssueStatus = (): Locator => this.page.locator('form[id="tracker:string:NewIssue"] div#status-editor button') @@ -279,10 +280,6 @@ export class IssuesPage extends CommonTrackerPage { await this.modifiedDateMenuItem().click() } - async pressEscape (): Promise { - await this.page.keyboard.press('Escape') - } - async clickEstimationContainer (): Promise { await this.estimationContainer().click() } @@ -423,6 +420,10 @@ export class IssuesPage extends CommonTrackerPage { async fillNewIssueForm (data: NewIssue): Promise { await this.inputPopupCreateNewIssueTitle().fill(data.title) await this.inputPopupCreateNewIssueDescription().fill(data.description) + if (data.projectName != null) { + await this.buttonPopupCreateNewIssueProject().click() + await this.selectMenuItem(this.page, data.projectName) + } if (data.status != null) { await this.buttonPopupCreateNewIssueStatus().click() await this.selectFromDropdown(this.page, data.status) diff --git a/tests/sanity/tests/model/tracker/types.ts b/tests/sanity/tests/model/tracker/types.ts index 0c7267d203..a99378ca4f 100644 --- a/tests/sanity/tests/model/tracker/types.ts +++ b/tests/sanity/tests/model/tracker/types.ts @@ -1,6 +1,7 @@ export interface NewIssue extends Issue { title: string description: string + projectName?: string } export interface Issue { diff --git a/tests/sanity/tests/tracker/mentions.spec.ts b/tests/sanity/tests/tracker/mentions.spec.ts index d8659fec32..535ee1c5b3 100644 --- a/tests/sanity/tests/tracker/mentions.spec.ts +++ b/tests/sanity/tests/tracker/mentions.spec.ts @@ -3,6 +3,7 @@ import { generateId, PlatformSetting, PlatformURI } from '../utils' import { LeftSideMenuPage } from '../model/left-side-menu-page' import { IssuesPage } from '../model/tracker/issues-page' import { IssuesDetailsPage } from '../model/tracker/issues-details-page' +import { TrackerNavigationMenuPage } from '../model/tracker/tracker-navigation-menu-page' import { NewIssue } from '../model/tracker/types' import { EmployeeDetailsPage } from '../model/contacts/employee-details-page' @@ -92,4 +93,41 @@ test.describe('Mentions issue tests', () => { lastName: mentionName.split(' ')[0] }) }) + + test('Checking backlinks in different spaces', async ({ page }) => { + const backlinkIssueDefault: NewIssue = { + title: `Issue for Default project-${generateId()}`, + description: 'Description', + projectName: 'Default' + } + const backlinkIssueSecond: NewIssue = { + title: `Issue for Second project-${generateId()}`, + description: 'Description', + projectName: 'Second Project' + } + await leftSideMenuPage.clickTracker() + await issuesPage.createNewIssue(backlinkIssueDefault) + await issuesPage.createNewIssue(backlinkIssueSecond) + const issuesNavigationPage = new TrackerNavigationMenuPage(page) + await issuesNavigationPage.issuesLinkForProject(backlinkIssueDefault.projectName ?? '').click() + await issuesPage.clickModelSelectorAll() + await issuesPage.searchIssueByName(backlinkIssueDefault.title) + await issuesPage.checkRowsInListExist(backlinkIssueDefault.title) + const defaultId = await issuesPage.getIssueId(backlinkIssueDefault.title) + await issuesNavigationPage.issuesLinkForProject(backlinkIssueSecond.projectName ?? '').click() + await issuesPage.clickModelSelectorAll() + await issuesPage.searchIssueByName(backlinkIssueSecond.title) + await issuesPage.checkRowsInListExist(backlinkIssueSecond.title) + const secondId = await issuesPage.getIssueId(backlinkIssueSecond.title) + await issuesPage.openIssueByName(backlinkIssueSecond.title) + await issuesDetailsPage.addMentions(defaultId) + await issuesDetailsPage.checkCommentExist(`@${defaultId}`) + await issuesDetailsPage.openLinkFromActivitiesByText(`@${defaultId}`) + await issuesDetailsPage.checkIssue(backlinkIssueDefault) + await issuesDetailsPage.addMentions(secondId) + await issuesDetailsPage.checkCommentExist(`@${secondId}`) + await issuesDetailsPage.openLinkFromActivitiesByText(`@${secondId}`) + await issuesDetailsPage.checkIssue(backlinkIssueSecond) + await issuesDetailsPage.clickCloseIssueButton() + }) }) From 576027f98b8f35a7c925e57fcc0b4b07f5555b53 Mon Sep 17 00:00:00 2001 From: Andrey Sobolev Date: Mon, 5 Aug 2024 12:37:13 +0700 Subject: [PATCH 6/8] UBERF-7796: Rework index creation logic (#6246) Signed-off-by: Andrey Sobolev --- .vscode/launch.json | 1 + common/scripts/version.txt | 2 +- models/core/src/core.ts | 4 +- models/core/src/index.ts | 21 +- server/core/src/adapter.ts | 25 ++- server/core/src/indexer/indexer.ts | 39 +++- server/core/src/mem.ts | 14 +- server/core/src/server/domainHelper.ts | 29 +-- server/core/src/server/index.ts | 6 +- server/core/src/server/storage.ts | 122 +++++++---- server/core/src/types.ts | 3 + server/elastic/src/backup.ts | 21 +- server/mongo/src/storage.ts | 245 ++++++++++++----------- server/mongo/src/utils.ts | 10 +- server/server-pipeline/src/indexing.ts | 13 +- server/server-storage/src/blobStorage.ts | 11 +- server/tool/src/index.ts | 12 +- tests/restore-local.sh | 2 +- tests/restore-workspace.sh | 2 +- 19 files changed, 330 insertions(+), 252 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 7c1bf86405..829d0722da 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -41,6 +41,7 @@ "METRICS_FILE": "${workspaceRoot}/metrics.txt", // Show metrics in console evert 30 seconds., "STORAGE_CONFIG": "minio|localhost?accessKey=minioadmin&secretKey=minioadmin", "SERVER_SECRET": "secret", + "ENABLE_CONSOLE": "true", "COLLABORATOR_URL": "ws://localhost:3078", "COLLABORATOR_API_URL": "http://localhost:3078", "REKONI_URL": "http://localhost:4004", diff --git a/common/scripts/version.txt b/common/scripts/version.txt index 2974978539..17e55ee553 100644 --- a/common/scripts/version.txt +++ b/common/scripts/version.txt @@ -1 +1 @@ -"0.6.271" \ No newline at end of file +"0.6.274" \ No newline at end of file diff --git a/models/core/src/core.ts b/models/core/src/core.ts index 783c86ab83..2f3bc0a777 100644 --- a/models/core/src/core.ts +++ b/models/core/src/core.ts @@ -329,13 +329,13 @@ export class TDocIndexState extends TDoc implements DocIndexState { attributes!: Record @Prop(TypeBoolean(), getEmbeddedLabel('Removed')) - @Index(IndexKind.Indexed) + // @Index(IndexKind.Indexed) @Hidden() removed!: boolean // States for different stages @Prop(TypeRecord(), getEmbeddedLabel('Stages')) - @Index(IndexKind.Indexed) + // @Index(IndexKind.Indexed) @Hidden() stages!: Record diff --git a/models/core/src/index.ts b/models/core/src/index.ts index 088e98a8d4..f4a97a5818 100644 --- a/models/core/src/index.ts +++ b/models/core/src/index.ts @@ -27,7 +27,6 @@ import { type AttachedDoc, type Class, type Doc, - type DocIndexState, type IndexingConfiguration, type TxCollectionCUD } from '@hcengineering/core' @@ -284,8 +283,10 @@ export function createModel (builder: Builder): void { builder.createDoc(core.class.DomainIndexConfiguration, core.space.Model, { domain: DOMAIN_DOC_INDEX_STATE, + indexes: [{ keys: { removed: 1 }, filter: { removed: true } }], disabled: [ { attachedToClass: 1 }, + { objectClass: 1 }, { stages: 1 }, { generationId: 1 }, { space: 1 }, @@ -298,24 +299,6 @@ export function createModel (builder: Builder): void { skip: ['stages.'] }) - builder.mixin, IndexingConfiguration>>( - core.class.DocIndexState, - core.class.Class, - core.mixin.IndexConfiguration, - { - indexes: [ - { - keys: { - _class: 1, - stages: 1, - _id: 1, - modifiedOn: 1 - } - } - ] - } - ) - builder.mixin(core.class.Space, core.class.Class, core.mixin.FullTextSearchContext, { childProcessingAllowed: false }) diff --git a/server/core/src/adapter.ts b/server/core/src/adapter.ts index b3ea060214..578cecd513 100644 --- a/server/core/src/adapter.ts +++ b/server/core/src/adapter.ts @@ -23,7 +23,6 @@ import { type FindOptions, type FindResult, type Hierarchy, - type IndexingConfiguration, type MeasureContext, type ModelDb, type Ref, @@ -38,19 +37,23 @@ import type { ServerFindOptions } from './types' export interface DomainHelperOperations { create: (domain: Domain) => Promise exists: (domain: Domain) => boolean + + listDomains: () => Promise> createIndex: (domain: Domain, value: string | FieldIndexConfig, options?: { name: string }) => Promise dropIndex: (domain: Domain, name: string) => Promise listIndexes: (domain: Domain) => Promise<{ name: string }[]> - hasDocuments: (domain: Domain, count: number) => Promise + + // Could return 0 even if it has documents + estimatedCount: (domain: Domain) => Promise } export interface DomainHelper { checkDomain: ( ctx: MeasureContext, domain: Domain, - forceCreate: boolean, + documents: number, operations: DomainHelperOperations - ) => Promise + ) => Promise } export interface RawDBAdapterStream { @@ -87,15 +90,20 @@ export interface RawDBAdapter { close: () => Promise } +export type DbAdapterHandler = ( + domain: Domain, + event: 'add' | 'update' | 'delete' | 'read', + count: number, + time: number, + helper: DomainHelperOperations +) => void /** * @public */ export interface DbAdapter { init?: () => Promise - helper?: () => DomainHelperOperations - createIndexes: (domain: Domain, config: Pick, 'indexes'>) => Promise - removeOldIndex: (domain: Domain, deletePattern: RegExp[], keepPattern: RegExp[]) => Promise + helper: () => DomainHelperOperations close: () => Promise findAll: ( @@ -116,6 +124,9 @@ export interface DbAdapter { // Bulk update operations update: (ctx: MeasureContext, domain: Domain, operations: Map, DocumentUpdate>) => Promise + + // Allow to register a handler to listen for domain operations + on?: (handler: DbAdapterHandler) => void } /** diff --git a/server/core/src/indexer/indexer.ts b/server/core/src/indexer/indexer.ts index cca74d4277..40889eb95f 100644 --- a/server/core/src/indexer/indexer.ts +++ b/server/core/src/indexer/indexer.ts @@ -349,23 +349,36 @@ export class FullTextIndexPipeline implements FullTextPipeline { keepPattern.push(new RegExp(st.stageId)) } } + const helper = this.storage.helper() if (deletePattern.length > 0) { - await this.storage.removeOldIndex(DOMAIN_DOC_INDEX_STATE, deletePattern, keepPattern) + try { + const existingIndexes = await helper.listIndexes(DOMAIN_DOC_INDEX_STATE) + for (const existingIndex of existingIndexes) { + if (existingIndex.name !== undefined) { + const name: string = existingIndex.name + if (deletePattern.some((it) => it.test(name)) && !keepPattern.some((it) => it.test(name))) { + await helper.dropIndex(DOMAIN_DOC_INDEX_STATE, name) + } + } + } + } catch (err: any) { + console.error(err) + } } for (const st of this.stages) { if (this.cancelling) { return } - await this.storage.createIndexes(DOMAIN_DOC_INDEX_STATE, { - indexes: [ - { - keys: { - ['stages.' + st.stageId]: 1 - } + await this.storage.helper().createIndex( + DOMAIN_DOC_INDEX_STATE, + { + keys: { + ['stages.' + st.stageId]: 1 } - ] - }) + }, + { name: 'stages.' + st.stageId + '_1' } + ) } } @@ -481,7 +494,9 @@ export class FullTextIndexPipeline implements FullTextPipeline { async (ctx) => await this.storage.findAll(ctx, core.class.DocIndexState, q, { sort: { modifiedOn: SortingOrder.Descending }, - limit: globalIndexer.processingSize + limit: globalIndexer.processingSize, + skipClass: true, + skipSpace: true }) ) const toRemove: DocIndexState[] = [] @@ -594,7 +609,9 @@ export class FullTextIndexPipeline implements FullTextPipeline { _id: 1, stages: 1, objectClass: 1 - } + }, + skipSpace: true, + skipClass: true } ) diff --git a/server/core/src/mem.ts b/server/core/src/mem.ts index f3c0dcd43b..06dcdcdfa5 100644 --- a/server/core/src/mem.ts +++ b/server/core/src/mem.ts @@ -32,7 +32,7 @@ import core, { type TxResult, type WorkspaceId } from '@hcengineering/core' -import { type DbAdapter } from './adapter' +import { type DbAdapter, type DomainHelperOperations } from './adapter' /** * @public @@ -49,6 +49,18 @@ export class DummyDbAdapter implements DbAdapter { async init (): Promise {} + helper (): DomainHelperOperations { + return { + create: async () => {}, + exists: () => true, + listDomains: async () => new Set(), + createIndex: async () => {}, + dropIndex: async () => {}, + listIndexes: async () => [], + estimatedCount: async () => 0 + } + } + async createIndexes (domain: Domain, config: Pick, 'indexes'>): Promise {} async removeOldIndex (domain: Domain, deletePattern: RegExp[], keepPattern: RegExp[]): Promise {} diff --git a/server/core/src/server/domainHelper.ts b/server/core/src/server/domainHelper.ts index 15955291c3..b0347f9319 100644 --- a/server/core/src/server/domainHelper.ts +++ b/server/core/src/server/domainHelper.ts @@ -74,41 +74,24 @@ export class DomainIndexHelperImpl implements DomainHelper { } /** - * return false if and only if domain underline structures are not required. + * Check if some indexes need to be created for domain. */ async checkDomain ( ctx: MeasureContext, domain: Domain, - forceCreate: boolean, + documents: number, operations: DomainHelperOperations - ): Promise { + ): Promise { const domainInfo = this.domains.get(domain) const cfg = this.domainConfigurations.find((it) => it.domain === domain) - let exists = operations.exists(domain) - const hasDocuments = exists && (await operations.hasDocuments(domain, 1)) - // Drop collection if it exists and should not exists or doesn't have documents. - if (exists && (cfg?.disableCollection === true || (!hasDocuments && !forceCreate))) { - // We do not need this collection - return false - } - - if (forceCreate && !exists) { - await operations.create(domain) - ctx.info('collection will be created', domain) - exists = true - } - if (!exists) { - // Do not need to create, since not force and no documents. - return false - } const bb: (string | FieldIndexConfig)[] = [] const added = new Set() try { - const has50Documents = await operations.hasDocuments(domain, 50) + const has50Documents = documents > 50 const allIndexes = (await operations.listIndexes(domain)).filter((it) => it.name !== '_id_') - ctx.info('check indexes', { domain, has50Documents }) + ctx.info('check indexes', { domain, has50Documents, documents }) if (has50Documents) { for (const vv of [...(domainInfo?.values() ?? []), ...(cfg?.indexes ?? [])]) { try { @@ -188,7 +171,5 @@ export class DomainIndexHelperImpl implements DomainHelper { if (bb.length > 0) { ctx.info('created indexes', { domain, bb }) } - - return true } } diff --git a/server/core/src/server/index.ts b/server/core/src/server/index.ts index c44d93b6b7..8065f4b20f 100644 --- a/server/core/src/server/index.ts +++ b/server/core/src/server/index.ts @@ -167,7 +167,7 @@ export async function createServerStorage ( const domainHelper = new DomainIndexHelperImpl(metrics, hierarchy, modelDb, conf.workspace) - return new TServerStorage( + const serverStorage = new TServerStorage( conf.domains, conf.defaultAdapter, adapters, @@ -184,6 +184,10 @@ export async function createServerStorage ( model, domainHelper ) + await ctx.with('init-domain-info', {}, async () => { + await serverStorage.initDomainInfo() + }) + return serverStorage } /** diff --git a/server/core/src/server/storage.ts b/server/core/src/server/storage.ts index 71a05827f2..9e320e291c 100644 --- a/server/core/src/server/storage.ts +++ b/server/core/src/server/storage.ts @@ -79,6 +79,11 @@ import type { } from '../types' import { SessionContextImpl, createBroadcastEvent } from '../utils' +interface DomainInfo { + exists: boolean + documents: number +} + export class TServerStorage implements ServerStorage { private readonly fulltext: FullTextIndex hierarchy: Hierarchy @@ -92,14 +97,8 @@ export class TServerStorage implements ServerStorage { liveQuery: LQ branding: Branding | null - domainInfo = new Map< - Domain, - { - exists: boolean - checkPromise: Promise | undefined - lastCheck: number - } - >() + domainInfo = new Map() + statsCtx: MeasureContext emptyAdapter = new DummyDbAdapter() @@ -126,6 +125,71 @@ export class TServerStorage implements ServerStorage { this.branding = options.branding this.setModel(model) + this.statsCtx = metrics.newChild('stats-' + this.workspaceId.name, {}) + } + + async initDomainInfo (): Promise { + const adapterDomains = new Map>() + for (const d of this.hierarchy.domains()) { + // We need to init domain info + const info = this.getDomainInfo(d) + const adapter = this.adapters.get(d) ?? this.adapters.get(this.defaultAdapter) + if (adapter !== undefined) { + const h = adapter.helper?.() + if (h !== undefined) { + const dbDomains = adapterDomains.get(adapter) ?? (await h.listDomains()) + adapterDomains.set(adapter, dbDomains) + const dbIdIndex = dbDomains.has(d) + info.exists = dbIdIndex !== undefined + if (info.exists) { + info.documents = await h.estimatedCount(d) + } + } else { + info.exists = true + } + } else { + info.exists = false + } + } + for (const adapter of this.adapters.values()) { + adapter.on?.((domain, event, count, time, helper) => { + const info = this.getDomainInfo(domain) + const oldDocuments = info.documents + switch (event) { + case 'add': + info.documents += count + break + case 'update': + break + case 'delete': + info.documents -= count + break + case 'read': + break + } + + if (oldDocuments < 50 && info.documents > 50) { + // We have more 50 documents, we need to check for indexes + void this.domainHelper.checkDomain(this.metrics, domain, info.documents, helper) + } + if (oldDocuments > 50 && info.documents < 50) { + // We have more 50 documents, we need to check for indexes + void this.domainHelper.checkDomain(this.metrics, domain, info.documents, helper) + } + }) + } + } + + private getDomainInfo (domain: Domain): DomainInfo { + let info = this.domainInfo.get(domain) + if (info === undefined) { + info = { + documents: -1, + exists: false + } + this.domainInfo.set(domain, info) + } + return info } private newCastClient (hierarchy: Hierarchy, modelDb: ModelDb, metrics: MeasureContext): Client { @@ -172,13 +236,6 @@ export class TServerStorage implements ServerStorage { async close (): Promise { await this.fulltext.close() - for (const [domain, info] of this.domainInfo.entries()) { - if (info.checkPromise !== undefined) { - this.metrics.info('wait for check domain', { domain }) - // We need to be sure we wait for check to be complete - await info.checkPromise - } - } for (const o of this.adapters.values()) { await o.close() } @@ -193,36 +250,13 @@ export class TServerStorage implements ServerStorage { throw new Error('adapter not provided: ' + name) } - const helper = adapter.helper?.() - if (helper !== undefined) { - let info = this.domainInfo.get(domain) - if (info == null) { - // For first time, lets assume all is fine - info = { - exists: true, - lastCheck: Date.now(), - checkPromise: undefined - } - this.domainInfo.set(domain, info) - return adapter - } - if (Date.now() - info.lastCheck > 5 * 60 * 1000) { - // Re-check every 5 minutes - const exists = helper.exists(domain) - // We will create necessary indexes if required, and not touch collection if not required. - info = { - exists, - lastCheck: Date.now(), - checkPromise: this.domainHelper.checkDomain(this.metrics, domain, requireExists, helper) - } - this.domainInfo.set(domain, info) - } - if (!info.exists && !requireExists) { - return this.emptyAdapter - } - // If we require it exists, it will be exists - info.exists = true + const info = this.getDomainInfo(domain) + + if (!info.exists && !requireExists) { + return this.emptyAdapter } + // If we require it exists, it will be exists + info.exists = true return adapter } diff --git a/server/core/src/types.ts b/server/core/src/types.ts index 99f4ebc2d6..a2fa30fcbb 100644 --- a/server/core/src/types.ts +++ b/server/core/src/types.ts @@ -51,6 +51,9 @@ import { type StorageAdapter } from './storage' export interface ServerFindOptions extends FindOptions { domain?: Domain // Allow to find for Doc's in specified domain only. prefix?: string + + skipClass?: boolean + skipSpace?: boolean } /** * @public diff --git a/server/elastic/src/backup.ts b/server/elastic/src/backup.ts index 53755ab1a5..2ac433ab84 100644 --- a/server/elastic/src/backup.ts +++ b/server/elastic/src/backup.ts @@ -33,7 +33,7 @@ import { WorkspaceId } from '@hcengineering/core' import { getMetadata } from '@hcengineering/platform' -import serverCore, { DbAdapter } from '@hcengineering/server-core' +import serverCore, { DbAdapter, type DomainHelperOperations } from '@hcengineering/server-core' function getIndexName (): string { return getMetadata(serverCore.metadata.ElasticIndexName) ?? 'storage_index' @@ -61,7 +61,24 @@ class ElasticDataAdapter implements DbAdapter { this.getDocId = (fulltext) => fulltext.slice(0, -1 * (this.workspaceString.length + 1)) as Ref } - async groupBy(ctx: MeasureContext, domain: Domain, field: string): Promise> { + helper (): DomainHelperOperations { + return { + create: async () => {}, + exists: () => true, + listDomains: async () => new Set(), + createIndex: async () => {}, + dropIndex: async () => {}, + listIndexes: async () => [], + estimatedCount: async () => 0 + } + } + + async groupBy( + ctx: MeasureContext, + domain: Domain, + field: string, + query?: DocumentQuery + ): Promise> { return new Set() } diff --git a/server/mongo/src/storage.ts b/server/mongo/src/storage.ts index c1ee4ec32c..a011c93566 100644 --- a/server/mongo/src/storage.ts +++ b/server/mongo/src/storage.ts @@ -37,7 +37,6 @@ import core, { type FindResult, type FullParamsType, type Hierarchy, - type IndexingConfiguration, type Lookup, type MeasureContext, type Mixin, @@ -65,6 +64,7 @@ import { estimateDocSize, updateHashForDoc, type DbAdapter, + type DbAdapterHandler, type DomainHelperOperations, type ServerFindOptions, type StorageAdapter, @@ -76,7 +76,6 @@ import { type AbstractCursor, type AnyBulkWriteOperation, type Collection, - type CreateIndexesOptions, type Db, type Document, type Filter, @@ -131,6 +130,18 @@ abstract class MongoAdapterBase implements DbAdapter { findRateLimit = new RateLimiter(parseInt(process.env.FIND_RLIMIT ?? '1000')) rateLimit = new RateLimiter(parseInt(process.env.TX_RLIMIT ?? '5')) + handlers: DbAdapterHandler[] = [] + + on (handler: DbAdapterHandler): void { + this.handlers.push(handler) + } + + handleEvent (domain: Domain, event: 'add' | 'update' | 'delete' | 'read', count: number, time: number): void { + for (const handler of this.handlers) { + handler(domain, event, count, time, this._db) + } + } + constructor ( protected readonly db: Db, protected readonly hierarchy: Hierarchy, @@ -151,45 +162,6 @@ abstract class MongoAdapterBase implements DbAdapter { return this._db } - async createIndexes (domain: Domain, config: Pick, 'indexes'>): Promise { - for (const value of config.indexes) { - try { - if (typeof value === 'string') { - await this.collection(domain).createIndex(value) - } else { - const opt: CreateIndexesOptions = {} - if (value.filter !== undefined) { - opt.partialFilterExpression = value.filter - } else if (value.sparse === true) { - opt.sparse = true - } - await this.collection(domain).createIndex(value.keys, opt) - } - } catch (err: any) { - console.error('failed to create index', domain, value, err) - } - } - } - - async removeOldIndex (domain: Domain, deletePattern: RegExp[], keepPattern: RegExp[]): Promise { - try { - const existingIndexes = await this.collection(domain).indexes() - for (const existingIndex of existingIndexes) { - if (existingIndex.name !== undefined) { - const name: string = existingIndex.name - if ( - deletePattern.some((it) => it.test(name)) && - (existingIndex.sparse === true || !keepPattern.some((it) => it.test(name))) - ) { - await this.collection(domain).dropIndex(name) - } - } - } - } catch (err: any) { - console.error(err) - } - } - async tx (ctx: MeasureContext, ...tx: Tx[]): Promise { return [] } @@ -198,7 +170,11 @@ abstract class MongoAdapterBase implements DbAdapter { this.client.close() } - private translateQuery(clazz: Ref>, query: DocumentQuery): Filter { + private translateQuery( + clazz: Ref>, + query: DocumentQuery, + options?: ServerFindOptions + ): Filter { const translated: any = {} for (const key in query) { const value = (query as any)[key] @@ -213,6 +189,13 @@ abstract class MongoAdapterBase implements DbAdapter { } translated[tkey] = value } + if (options?.skipSpace === true) { + delete translated.space + } + if (options?.skipClass === true) { + delete translated._class + return translated + } const baseClass = this.hierarchy.getBaseClass(clazz) if (baseClass !== core.class.Doc) { const classes = this.hierarchy.getDescendants(baseClass).filter((it) => !this.hierarchy.isMixin(it)) @@ -473,12 +456,15 @@ abstract class MongoAdapterBase implements DbAdapter { private async findWithPipeline( ctx: MeasureContext, + domain: Domain, clazz: Ref>, query: DocumentQuery, - options?: ServerFindOptions + options: ServerFindOptions, + stTime: number ): Promise> { + const st = Date.now() const pipeline: any[] = [] - const match = { $match: this.translateQuery(clazz, query) } + const match = { $match: this.translateQuery(clazz, query, options) } const slowPipeline = isLookupQuery(query) || isLookupSort(options?.sort) const steps = await ctx.with('get-lookups', {}, async () => await this.getLookups(clazz, options?.lookup)) if (slowPipeline) { @@ -506,9 +492,6 @@ abstract class MongoAdapterBase implements DbAdapter { pipeline.push({ $project: projection }) } - // const domain = this.hierarchy.getDomain(clazz) - const domain = options?.domain ?? this.hierarchy.getDomain(clazz) - const cursor = this.collection(domain).aggregate>(pipeline) let result: WithLookup[] = [] let total = options?.total === true ? 0 : -1 @@ -558,6 +541,17 @@ abstract class MongoAdapterBase implements DbAdapter { ) total = arr?.[0]?.total ?? 0 } + const edTime = Date.now() + if (edTime - stTime > 1000 || st - stTime > 1000) { + ctx.error('aggregate', { + time: edTime - stTime, + clazz, + query: cutObjectArray(query), + options, + queueTime: st - stTime + }) + } + this.handleEvent(domain, 'read', result.length, edTime - st) return toFindResult(this.stripHash(result) as T[], total) } @@ -643,7 +637,12 @@ abstract class MongoAdapterBase implements DbAdapter { } @withContext('groupBy') - async groupBy(ctx: MeasureContext, domain: Domain, field: string): Promise> { + async groupBy( + ctx: MeasureContext, + domain: Domain, + field: string, + query?: DocumentQuery + ): Promise> { const result = await ctx.with( 'groupBy', { domain }, @@ -651,6 +650,7 @@ abstract class MongoAdapterBase implements DbAdapter { const coll = this.collection(domain) const grResult = await coll .aggregate([ + ...(query !== undefined ? [{ $match: query }] : []), { $group: { _id: '$' + field @@ -716,20 +716,20 @@ abstract class MongoAdapterBase implements DbAdapter { const stTime = Date.now() return await this.findRateLimit.exec(async () => { const st = Date.now() + const domain = options?.domain ?? this.hierarchy.getDomain(_class) const result = await this.collectOps( ctx, - this.hierarchy.findDomain(_class), + domain, 'find', async (ctx) => { - const domain = options?.domain ?? this.hierarchy.getDomain(_class) if ( options != null && (options?.lookup != null || this.isEnumSort(_class, options) || this.isRulesSort(options)) ) { - return await this.findWithPipeline(ctx, _class, query, options) + return await this.findWithPipeline(ctx, domain, _class, query, options, stTime) } const coll = this.collection(domain) - const mongoQuery = this.translateQuery(_class, query) + const mongoQuery = this.translateQuery(_class, query, options) if (options?.limit === 1) { // Skip sort/projection/etc. @@ -825,6 +825,7 @@ abstract class MongoAdapterBase implements DbAdapter { queueTime: st - stTime }) } + this.handleEvent(domain, 'read', result.length, edTime - st) return result }) } @@ -1122,7 +1123,6 @@ class MongoAdapter extends MongoAdapterBase { }) await this.rateLimit.exec(async () => { - const domains: Promise[] = [] for (const [domain, txs] of byDomain) { if (domain === undefined) { continue @@ -1146,75 +1146,80 @@ class MongoAdapter extends MongoAdapterBase { ) { continue } - domains.push( - this.collectOps( - ctx, - domain, - 'tx', - async (ctx) => { - const coll = this.db.collection(domain) + await this.collectOps( + ctx, + domain, + 'tx', + async (ctx) => { + const coll = this.db.collection(domain) - // Minir optimizations - // Add Remove optimization + // Minir optimizations + // Add Remove optimization - if (domainBulk.add.length > 0) { - await ctx.with('insertMany', {}, async () => { - await coll.insertMany(domainBulk.add, { ordered: false }) - }) - } - if (domainBulk.update.size > 0) { - // Extract similar update to update many if possible - // TODO: - await ctx.with('updateMany-bulk', {}, async () => { - await coll.bulkWrite( - Array.from(domainBulk.update.entries()).map((it) => ({ - updateOne: { - filter: { _id: it[0] }, - update: { - $set: it[1] - } - } - })), - { - ordered: false - } - ) - }) - } - if (domainBulk.bulkOperations.length > 0) { - await ctx.with('bulkWrite', {}, async () => { - await coll.bulkWrite(domainBulk.bulkOperations, { - ordered: false - }) - }) - } - if (domainBulk.findUpdate.size > 0) { - await ctx.with('find-result', {}, async () => { - const docs = await coll.find({ _id: { $in: Array.from(domainBulk.findUpdate) } }).toArray() - result.push(...docs) - }) - } - - if (domainBulk.raw.length > 0) { - await ctx.with('raw', {}, async () => { - for (const r of domainBulk.raw) { - result.push({ object: await r() }) - } - }) - } - }, - { - domain, - add: domainBulk.add.length, - update: domainBulk.update.size, - bulk: domainBulk.bulkOperations.length, - find: domainBulk.findUpdate.size, - raw: domainBulk.raw.length + if (domainBulk.add.length > 0) { + await ctx.with('insertMany', {}, async () => { + const st = Date.now() + const result = await coll.insertMany(domainBulk.add, { ordered: false }) + this.handleEvent(domain, 'add', result.insertedCount, Date.now() - st) + }) } - ) + if (domainBulk.update.size > 0) { + // Extract similar update to update many if possible + // TODO: + await ctx.with('updateMany-bulk', {}, async () => { + const st = Date.now() + const result = await coll.bulkWrite( + Array.from(domainBulk.update.entries()).map((it) => ({ + updateOne: { + filter: { _id: it[0] }, + update: { + $set: it[1] + } + } + })), + { + ordered: false + } + ) + this.handleEvent(domain, 'update', result.modifiedCount, Date.now() - st) + }) + } + if (domainBulk.bulkOperations.length > 0) { + await ctx.with('bulkWrite', {}, async () => { + const st = Date.now() + const result = await coll.bulkWrite(domainBulk.bulkOperations, { + ordered: false + }) + this.handleEvent(domain, 'update', result.modifiedCount, Date.now() - st) + }) + } + if (domainBulk.findUpdate.size > 0) { + await ctx.with('find-result', {}, async () => { + const st = Date.now() + const docs = await coll.find({ _id: { $in: Array.from(domainBulk.findUpdate) } }).toArray() + result.push(...docs) + this.handleEvent(domain, 'read', docs.length, Date.now() - st) + }) + } + + if (domainBulk.raw.length > 0) { + await ctx.with('raw', {}, async () => { + for (const r of domainBulk.raw) { + result.push({ object: await r() }) + } + }) + } + }, + { + domain, + add: domainBulk.add.length, + update: domainBulk.update.size, + bulk: domainBulk.bulkOperations.length, + find: domainBulk.findUpdate.size, + raw: domainBulk.raw.length + } ) } - await Promise.all(domains) }) return result } @@ -1395,6 +1400,7 @@ class MongoAdapter extends MongoAdapterBase { if (tx.retrieve === true) { bulk.raw.push(async () => { + const st = Date.now() const res = await this.collection(domain).findOneAndUpdate( { _id: tx.objectId }, { @@ -1407,6 +1413,9 @@ class MongoAdapter extends MongoAdapterBase { } as unknown as UpdateFilter, { returnDocument: 'after', includeResultMetadata: true } ) + const dnow = Date.now() - st + this.handleEvent(domain, 'read', 1, dnow) + this.handleEvent(domain, 'update', 1, dnow) return res.value as TxResult }) } else { @@ -1459,6 +1468,7 @@ class MongoTxAdapter extends MongoAdapterBase implements TxAdapter { if (tx.length === 0) { return [] } + const st = Date.now() await this.collectOps( ctx, DOMAIN_TX, @@ -1468,6 +1478,7 @@ class MongoTxAdapter extends MongoAdapterBase implements TxAdapter { }, { tx: tx.length } ) + this.handleEvent(DOMAIN_TX, 'add', tx.length, Date.now() - st) return [] } diff --git a/server/mongo/src/utils.ts b/server/mongo/src/utils.ts index 4410a07f45..9e32881cf0 100644 --- a/server/mongo/src/utils.ts +++ b/server/mongo/src/utils.ts @@ -158,6 +158,11 @@ export class DBCollectionHelper implements DomainHelperOperations { collections = new Map>() constructor (readonly db: Db) {} + async listDomains (): Promise> { + const collections = await this.db.listCollections({}, { nameOnly: true }).toArray() + return new Set(collections.map((it) => it.name as unknown as Domain)) + } + async init (domain?: Domain): Promise { if (domain === undefined) { // Init existing collecfions @@ -224,7 +229,8 @@ export class DBCollectionHelper implements DomainHelperOperations { return await this.collection(domain).listIndexes().toArray() } - async hasDocuments (domain: Domain, count: number): Promise { - return (await this.collection(domain).countDocuments({}, { limit: count })) >= count + async estimatedCount (domain: Domain): Promise { + const c = this.collection(domain) + return await c.estimatedDocumentCount() } } diff --git a/server/server-pipeline/src/indexing.ts b/server/server-pipeline/src/indexing.ts index e79988b1d0..26a84b26df 100644 --- a/server/server-pipeline/src/indexing.ts +++ b/server/server-pipeline/src/indexing.ts @@ -1,6 +1,5 @@ /* eslint-disable @typescript-eslint/unbound-method */ import { type Branding, type MeasureContext, type WorkspaceId } from '@hcengineering/core' -import { OpenAIEmbeddingsStage } from '@hcengineering/openai' import { CollaborativeContentRetrievalStage } from '@hcengineering/server-collaboration' import { ContentRetrievalStage, @@ -65,14 +64,14 @@ export function createIndexStages ( const pushStage = new FullTextPushStage(storage, adapter, workspace, branding) stages.push(pushStage) - // OpenAI prepare stage - const openAIStage = new OpenAIEmbeddingsStage(adapter, workspace) - // We depend on all available stages. - openAIStage.require = stages.map((it) => it.stageId) + // // OpenAI prepare stage + // const openAIStage = new OpenAIEmbeddingsStage(adapter, workspace) + // // We depend on all available stages. + // openAIStage.require = stages.map((it) => it.stageId) - openAIStage.updateSummary(summaryStage) + // openAIStage.updateSummary(summaryStage) - stages.push(openAIStage) + // stages.push(openAIStage) return stages } diff --git a/server/server-storage/src/blobStorage.ts b/server/server-storage/src/blobStorage.ts index 818ded0357..ef46bd51fe 100644 --- a/server/server-storage/src/blobStorage.ts +++ b/server/server-storage/src/blobStorage.ts @@ -34,7 +34,12 @@ import core, { } from '@hcengineering/core' import { createMongoAdapter } from '@hcengineering/mongo' import { PlatformError, unknownError } from '@hcengineering/platform' -import { DbAdapter, StorageAdapter, type StorageAdapterEx } from '@hcengineering/server-core' +import { + DbAdapter, + StorageAdapter, + type DomainHelperOperations, + type StorageAdapterEx +} from '@hcengineering/server-core' class StorageBlobAdapter implements DbAdapter { constructor ( @@ -53,6 +58,10 @@ class StorageBlobAdapter implements DbAdapter { return await this.blobAdapter.findAll(ctx, _class, query, options) } + helper (): DomainHelperOperations { + return this.blobAdapter.helper() + } + async groupBy(ctx: MeasureContext, domain: Domain, field: string): Promise> { return await this.blobAdapter.groupBy(ctx, domain, field) } diff --git a/server/tool/src/index.ts b/server/tool/src/index.ts index 0f1f434660..36d54b0d79 100644 --- a/server/tool/src/index.ts +++ b/server/tool/src/index.ts @@ -516,17 +516,7 @@ async function createUpdateIndexes ( if (domain === DOMAIN_MODEL || domain === DOMAIN_TRANSIENT || domain === DOMAIN_BENCHMARK) { continue } - const result = await domainHelper.checkDomain(ctx, domain, false, dbHelper) - if (!result && dbHelper.exists(domain)) { - try { - logger.log('dropping domain', { domain }) - if ((await db.collection(domain).countDocuments({})) === 0) { - await db.dropCollection(domain) - } - } catch (err) { - logger.error('error: failed to delete collection', { domain, err }) - } - } + await domainHelper.checkDomain(ctx, domain, await dbHelper.estimatedCount(domain), dbHelper) completed++ await progress((100 / allDomains.length) * completed) } diff --git a/tests/restore-local.sh b/tests/restore-local.sh index 10c3993701..df4b077ec8 100755 --- a/tests/restore-local.sh +++ b/tests/restore-local.sh @@ -9,7 +9,7 @@ export SERVER_SECRET=secret # Restore workspace contents in mongo/elastic ./tool-local.sh backup-restore ./sanity-ws sanity-ws -./tool-local.sh upgrade-workspace sanity-ws +./tool-local.sh upgrade-workspace sanity-ws --indexes # Re-assign user to workspace. ./tool-local.sh assign-workspace user1 sanity-ws diff --git a/tests/restore-workspace.sh b/tests/restore-workspace.sh index 4eec408b3c..0fb8ac485b 100755 --- a/tests/restore-workspace.sh +++ b/tests/restore-workspace.sh @@ -3,7 +3,7 @@ # Restore workspace contents in mongo/elastic ./tool.sh backup-restore ./sanity-ws sanity-ws -./tool.sh upgrade-workspace sanity-ws +./tool.sh upgrade-workspace sanity-ws --indexes # Re-assign user to workspace. ./tool.sh assign-workspace user1 sanity-ws From 8667b7482d9f4f9e324a9fe9594ba1ff5ce8a1b4 Mon Sep 17 00:00:00 2001 From: Andrey Sobolev Date: Mon, 5 Aug 2024 12:37:34 +0700 Subject: [PATCH 7/8] UBERF-7794: Restore related issues control (#6244) Signed-off-by: Andrey Sobolev --- models/lead/src/index.ts | 14 ++++++- models/recruit/src/index.ts | 14 ++++++- plugins/lead-resources/package.json | 1 + .../src/components/KanbanCard.svelte | 39 ++++++++++++++++--- .../src/components/KanbanCard.svelte | 2 +- 5 files changed, 62 insertions(+), 8 deletions(-) diff --git a/models/lead/src/index.ts b/models/lead/src/index.ts index b1d9057dd0..7e9e6ed71d 100644 --- a/models/lead/src/index.ts +++ b/models/lead/src/index.ts @@ -227,6 +227,12 @@ export function createModel (builder: Builder): void { 'status', 'attachments', 'comments', + { + key: '', + label: tracker.string.RelatedIssues, + presenter: tracker.component.RelatedIssueSelector, + displayProps: { key: 'related', suffix: true } + }, 'modifiedOn', { key: '$lookup.attachedTo.$lookup.channels', @@ -294,6 +300,12 @@ export function createModel (builder: Builder): void { }, { key: 'attachments', displayProps: { key: 'attachments', suffix: true } }, { key: 'comments', displayProps: { key: 'comments', suffix: true } }, + { + key: '', + label: tracker.string.RelatedIssues, + presenter: tracker.component.RelatedIssueSelector, + displayProps: { key: 'related', suffix: true } + }, { key: '', displayProps: { grow: true } }, { key: '$lookup.attachedTo.$lookup.channels', @@ -441,7 +453,7 @@ export function createModel (builder: Builder): void { groupDepth: 1 }, options: lookupLeadOptions, - config: ['attachedTo', 'attachments', 'comments', 'dueDate', 'assignee'], + config: ['attachedTo', 'status', 'attachments', 'comments', 'dueDate', 'assignee'], configOptions: { strict: true } diff --git a/models/recruit/src/index.ts b/models/recruit/src/index.ts index 50c7823292..bce4f9a7b7 100644 --- a/models/recruit/src/index.ts +++ b/models/recruit/src/index.ts @@ -21,6 +21,7 @@ import calendar from '@hcengineering/model-calendar' import chunter from '@hcengineering/model-chunter' import contact from '@hcengineering/model-contact' import core from '@hcengineering/model-core' +import gmail from '@hcengineering/model-gmail' import { generateClassNotificationTypes } from '@hcengineering/model-notification' import presentation from '@hcengineering/model-presentation' import tags from '@hcengineering/model-tags' @@ -33,7 +34,6 @@ import { type IntlString } from '@hcengineering/platform' import { recruitId, type Applicant } from '@hcengineering/recruit' import setting from '@hcengineering/setting' import { type KeyBinding, type ViewOptionModel, type ViewOptionsModel } from '@hcengineering/view' -import gmail from '@hcengineering/model-gmail' import recruit from './plugin' import { createReviewModel, reviewTableConfig, reviewTableOptions } from './review' @@ -475,6 +475,12 @@ export function createModel (builder: Builder): void { 'status', 'attachments', 'comments', + { + key: '', + label: tracker.string.RelatedIssues, + presenter: tracker.component.RelatedIssueSelector, + displayProps: { key: 'related', suffix: true } + }, 'modifiedOn', '$lookup.space.company', { @@ -604,6 +610,12 @@ export function createModel (builder: Builder): void { }, { key: 'attachments', displayProps: { key: 'attachments', suffix: true } }, { key: 'comments', displayProps: { key: 'comments', suffix: true } }, + { + key: '', + label: tracker.string.RelatedIssues, + presenter: tracker.component.RelatedIssueSelector, + displayProps: { key: 'related', suffix: true } + }, { key: '', displayProps: { grow: true } }, { key: '$lookup.space.company', diff --git a/plugins/lead-resources/package.json b/plugins/lead-resources/package.json index 5a813e9a9a..12879fb8f4 100644 --- a/plugins/lead-resources/package.json +++ b/plugins/lead-resources/package.json @@ -52,6 +52,7 @@ "@hcengineering/presentation": "^0.6.3", "@hcengineering/task": "^0.6.20", "@hcengineering/task-resources": "^0.6.0", + "@hcengineering/tracker": "^0.6.24", "@hcengineering/text-editor-resources": "^0.6.0", "@hcengineering/ui": "^0.6.15", "@hcengineering/view": "^0.6.13", diff --git a/plugins/lead-resources/src/components/KanbanCard.svelte b/plugins/lead-resources/src/components/KanbanCard.svelte index 854fbde4ca..e1c01959d3 100644 --- a/plugins/lead-resources/src/components/KanbanCard.svelte +++ b/plugins/lead-resources/src/components/KanbanCard.svelte @@ -23,16 +23,18 @@ import notification from '@hcengineering/notification' import { getClient } from '@hcengineering/presentation' import task from '@hcengineering/task' - import { AssigneePresenter } from '@hcengineering/task-resources' + import { AssigneePresenter, StateRefPresenter } from '@hcengineering/task-resources' import { ActionIcon, Component, DueDatePresenter, IconMoreH } from '@hcengineering/ui' import { BuildModelKey } from '@hcengineering/view' import { enabledConfig, openDoc, showMenu, statusStore } from '@hcengineering/view-resources' + import tracker from '@hcengineering/tracker' import lead from '../plugin' import LeadPresenter from './LeadPresenter.svelte' export let object: WithLookup export let config: (string | BuildModelKey)[] + export let groupByKey: string const client = getClient() const assigneeAttribute = client.getHierarchy().getAttribute(lead.class.Lead, 'assignee') @@ -69,8 +71,21 @@ {/if} - {#if enabledConfig(config, 'dueDate')} -
+
+ {#if groupByKey !== 'status' && enabledConfig(config, 'status')} + { + client.update(object, { status }) + }} + /> + {/if} + + {#if enabledConfig(config, 'dueDate')} -
- {/if} + {/if} +
@@ -105,3 +120,17 @@ {/if}
+ + diff --git a/plugins/recruit-resources/src/components/KanbanCard.svelte b/plugins/recruit-resources/src/components/KanbanCard.svelte index 988c631dd8..a8422fae8f 100644 --- a/plugins/recruit-resources/src/components/KanbanCard.svelte +++ b/plugins/recruit-resources/src/components/KanbanCard.svelte @@ -111,7 +111,7 @@ }} /> {/if} - + {#if enabledConfig(config, 'dueDate')} Date: Mon, 5 Aug 2024 13:03:25 +0700 Subject: [PATCH 8/8] UBERF-7800: Space improvements (#6250) Signed-off-by: Andrey Sobolev --- server/middleware/src/spaceSecurity.ts | 105 +++++++++++-------------- 1 file changed, 47 insertions(+), 58 deletions(-) diff --git a/server/middleware/src/spaceSecurity.ts b/server/middleware/src/spaceSecurity.ts index 1ecacebb91..c5985fd9ea 100644 --- a/server/middleware/src/spaceSecurity.ts +++ b/server/middleware/src/spaceSecurity.ts @@ -325,43 +325,6 @@ export class SpaceSecurityMiddleware extends BaseMiddleware implements Middlewar return res } - private async getTxTargets (ctx: SessionContext, tx: Tx): Promise { - const h = this.storage.hierarchy - let targets: string[] | undefined - - if (TxProcessor.isExtendsCUD(tx._class)) { - const account = await getUser(this.storage, ctx) - if (tx.objectSpace === (account._id as string)) { - targets = [account.email, systemAccountEmail] - } else if ([...this.systemSpaces, ...this.mainSpaces].includes(tx.objectSpace)) { - return - } else { - const cudTx = tx as TxCUD - const isSpace = h.isDerived(cudTx.objectClass, core.class.Space) - - if (isSpace) { - return undefined - } - - const space = this.spacesMap.get(tx.objectSpace) - - if (space !== undefined) { - targets = await this.getTargets(space.members) - if (!isOwner(account, ctx)) { - const allowed = this.getAllAllowedSpaces(account, true) - if (allowed === undefined || !allowed.includes(tx.objectSpace)) { - throw new PlatformError(new Status(Severity.ERROR, platform.status.Forbidden, {})) - } - } else if (!targets.includes(account.email)) { - targets.push(account.email) - } - } - } - } - - return targets - } - private async processTxSpaceDomain (tx: TxCUD): Promise { const actualTx = TxProcessor.extractTx(tx) if (actualTx._class === core.class.TxCreateDoc) { @@ -450,15 +413,10 @@ export class SpaceSecurityMiddleware extends BaseMiddleware implements Middlewar return isData ? res : [...res, ...this.publicSpaces] } - async loadDomainSpaces (ctx: MeasureContext, domain: Domain): Promise>> { - const field = this.getKey(domain) - return await this.storage.groupBy>(ctx, domain, field) - } - async getDomainSpaces (domain: Domain): Promise>> { let domainSpaces = this._domainSpaces.get(domain) if (domainSpaces === undefined) { - const p = this.loadDomainSpaces(this.spaceMeasureCtx, domain) + const p = this.storage.groupBy>(this.spaceMeasureCtx, domain, this.getKey(domain)) this._domainSpaces.set(domain, p) domainSpaces = await p this._domainSpaces.set(domain, domainSpaces) @@ -466,9 +424,17 @@ export class SpaceSecurityMiddleware extends BaseMiddleware implements Middlewar return domainSpaces instanceof Promise ? await domainSpaces : domainSpaces } - private async filterByDomain (domain: Domain, spaces: Ref[]): Promise[]> { + private async filterByDomain ( + domain: Domain, + spaces: Ref[] + ): Promise<{ result: Ref[], allDomainSpaces: boolean, domainSpaces: Set> }> { const domainSpaces = await this.getDomainSpaces(domain) - return spaces.filter((p) => domainSpaces.has(p)) + const result = spaces.filter((p) => domainSpaces.has(p)) + return { + result: spaces.filter((p) => domainSpaces.has(p)), + allDomainSpaces: result.length === domainSpaces.size, + domainSpaces + } } private async mergeQuery( @@ -476,19 +442,33 @@ export class SpaceSecurityMiddleware extends BaseMiddleware implements Middlewar query: ObjQueryType, domain: Domain, isSpace: boolean - ): Promise> { + ): Promise | undefined> { const spaces = await this.filterByDomain(domain, this.getAllAllowedSpaces(account, !isSpace)) if (query == null) { - return { $in: spaces } + if (spaces.allDomainSpaces) { + return undefined + } + return { $in: spaces.result } } if (typeof query === 'string') { - if (!spaces.includes(query)) { + if (!spaces.result.includes(query)) { return { $in: [] } } } else if (query.$in != null) { - query.$in = query.$in.filter((p) => spaces.includes(p)) + query.$in = query.$in.filter((p) => spaces.result.includes(p)) + if (query.$in.length === spaces.domainSpaces.size) { + // all domain spaces + delete query.$in + } } else { - query.$in = spaces + if (spaces.allDomainSpaces) { + delete query.$in + } else { + query.$in = spaces.result + } + } + if (Object.keys(query).length === 0) { + return undefined } return query } @@ -515,22 +495,31 @@ export class SpaceSecurityMiddleware extends BaseMiddleware implements Middlewar if (!isOwner(account, ctx) || !isSpace) { if (query[field] !== undefined) { const res = await this.mergeQuery(account, query[field], domain, isSpace) - ;(newQuery as any)[field] = res - if (typeof res === 'object') { - if (Array.isArray(res.$in) && res.$in.length === 1 && Object.keys(res).length === 1) { - ;(newQuery as any)[field] = res.$in[0] + if (res === undefined) { + // eslint-disable-next-line @typescript-eslint/no-dynamic-delete + delete (query as any)[field] + } else { + ;(newQuery as any)[field] = res + if (typeof res === 'object') { + if (Array.isArray(res.$in) && res.$in.length === 1 && Object.keys(res).length === 1) { + ;(newQuery as any)[field] = res.$in[0] + } } } } else { const spaces = await this.filterByDomain(domain, this.getAllAllowedSpaces(account, !isSpace)) - if (spaces.length === 1) { - ;(newQuery as any)[field] = spaces[0] + if (spaces.allDomainSpaces) { + // eslint-disable-next-line @typescript-eslint/no-dynamic-delete + delete (newQuery as any)[field] + } else if (spaces.result.length === 1) { + ;(newQuery as any)[field] = spaces.result[0] } else { - ;(newQuery as any)[field] = { $in: spaces } + ;(newQuery as any)[field] = { $in: spaces.result } } } } } + const findResult = await this.provideFindAll(ctx, _class, newQuery, options) if (!isOwner(account, ctx) && account.role !== AccountRole.DocGuest) { if (options?.lookup !== undefined) { @@ -564,7 +553,7 @@ export class SpaceSecurityMiddleware extends BaseMiddleware implements Middlewar } passedDomains.add(domain) const spaces = await this.filterByDomain(domain, allSpaces) - for (const space of spaces) { + for (const space of spaces.result) { res.add(space) } }