diff --git a/.vscode/launch.json b/.vscode/launch.json index eb10412154..c1f2a92066 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -432,7 +432,7 @@ "request": "launch", "args": ["src/index.ts"], "env": { - "PORT": "4007", + "PORT": "4017", "SECRET": "secret", "MONGO_URL": "mongodb://localhost:27017", "MINIO_ENDPOINT": "localhost", diff --git a/dev/tool/src/storage.ts b/dev/tool/src/storage.ts index 7cd974d9a7..394cc4729f 100644 --- a/dev/tool/src/storage.ts +++ b/dev/tool/src/storage.ts @@ -14,9 +14,9 @@ // import { type Attachment } from '@hcengineering/attachment' -import { type Blob, type MeasureContext, type WorkspaceId, RateLimiter } from '@hcengineering/core' +import { type Blob, type MeasureContext, type Ref, type WorkspaceId, RateLimiter } from '@hcengineering/core' import { DOMAIN_ATTACHMENT } from '@hcengineering/model-attachment' -import { type StorageAdapter, type StorageAdapterEx } from '@hcengineering/server-core' +import { type ListBlobResult, type StorageAdapter, type StorageAdapterEx } from '@hcengineering/server-core' import { type Db } from 'mongodb' import { PassThrough } from 'stream' @@ -143,6 +143,20 @@ async function processAdapter ( const iterator = await source.listStream(ctx, workspaceId) + const targetIterator = await target.listStream(ctx, workspaceId) + + const targetBlobs = new Map, ListBlobResult>() + + while (true) { + const part = await targetIterator.next() + for (const p of part) { + targetBlobs.set(p._id, p) + } + if (part.length === 0) { + break + } + } + const toRemove: string[] = [] try { while (true) { @@ -150,18 +164,18 @@ async function processAdapter ( if (dataBulk.length === 0) break for (const data of dataBulk) { - let targetBlob = await target.stat(ctx, workspaceId, data._id) - const sourceBlob = await source.stat(ctx, workspaceId, data._id) - - if (sourceBlob === undefined) { - console.error('blob not found', data._id) - continue - } + let targetBlob: Blob | ListBlobResult | undefined = targetBlobs.get(data._id) if (targetBlob !== undefined) { - console.log('Target blob already exists', targetBlob._id, targetBlob.contentType) + console.log('Target blob already exists', targetBlob._id) } if (targetBlob === undefined) { + const sourceBlob = await source.stat(ctx, workspaceId, data._id) + + if (sourceBlob === undefined) { + console.error('blob not found', data._id) + continue + } await rateLimiter.exec(async () => { try { await retryOnFailure( @@ -170,7 +184,12 @@ async function processAdapter ( async () => { await processFile(ctx, source, target, workspaceId, sourceBlob) // We need to sync and update aggregator table for now. - await exAdapter.syncBlobFromStorage(ctx, workspaceId, sourceBlob._id, exAdapter.defaultAdapter) + targetBlob = await exAdapter.syncBlobFromStorage( + ctx, + workspaceId, + sourceBlob._id, + exAdapter.defaultAdapter + ) }, 50 ) @@ -181,23 +200,14 @@ async function processAdapter ( console.error('failed to process blob', data._id, err) } }) - } - if (targetBlob === undefined) { - targetBlob = await target.stat(ctx, workspaceId, data._id) + if (targetBlob !== undefined && 'size' in targetBlob && (targetBlob as Blob).size === sourceBlob.size) { + // We could safely delete source blob + toRemove.push(sourceBlob._id) + } + processedBytes += sourceBlob.size } - - if ( - targetBlob !== undefined && - targetBlob.size === sourceBlob.size && - targetBlob.contentType === sourceBlob.contentType - ) { - // We could safely delete source blob - toRemove.push(sourceBlob._id) - } - processedCnt += 1 - processedBytes += sourceBlob.size if (processedCnt % 100 === 0) { await rateLimiter.waitProcessing() diff --git a/packages/storage/src/index.ts b/packages/storage/src/index.ts index 5a1a850d42..4390e23f75 100644 --- a/packages/storage/src/index.ts +++ b/packages/storage/src/index.ts @@ -14,6 +14,7 @@ // import { type Blob, type MeasureContext, type StorageIterator, type WorkspaceId } from '@hcengineering/core' +import { PlatformError, unknownError } from '@hcengineering/platform' import { type Readable } from 'stream' export type ListBlobResult = Omit @@ -77,7 +78,7 @@ export interface StorageAdapterEx extends StorageAdapter { workspaceId: WorkspaceId, objectName: string, provider?: string - ) => Promise + ) => Promise find: (ctx: MeasureContext, workspaceId: WorkspaceId) => StorageIterator } @@ -87,7 +88,9 @@ export interface StorageAdapterEx extends StorageAdapter { */ export class DummyStorageAdapter implements StorageAdapter, StorageAdapterEx { defaultAdapter: string = '' - async syncBlobFromStorage (ctx: MeasureContext, workspaceId: WorkspaceId, objectName: string): Promise {} + async syncBlobFromStorage (ctx: MeasureContext, workspaceId: WorkspaceId, objectName: string): Promise { + throw new PlatformError(unknownError('Method not implemented')) + } async initialize (ctx: MeasureContext, workspaceId: WorkspaceId): Promise {} diff --git a/plugins/chunter-resources/src/components/ChatWidgetTab.svelte b/plugins/chunter-resources/src/components/ChatWidgetTab.svelte index 5bfc57a6db..da4e207ef2 100644 --- a/plugins/chunter-resources/src/components/ChatWidgetTab.svelte +++ b/plugins/chunter-resources/src/components/ChatWidgetTab.svelte @@ -87,7 +87,7 @@ ): void { } widgetsState.delete(widget) - Analytics.handleEvent('workbench.CloseSidebarWidget', { widget }) + Analytics.handleEvent(WorkbenchEvents.SidebarCloseWidget, { widget }) if (state.widget === widget) { sidebarStore.set({ ...state, @@ -162,7 +162,7 @@ export async function closeWidgetTab (widget: Widget, tab: string): Promise it.id !== tab) const closedTab = tabs.find((it) => it.id === tab) - Analytics.handleEvent('workbench.CloseSidebarWidget', { widget: widget._id, tab }) + Analytics.handleEvent(WorkbenchEvents.SidebarCloseWidget, { widget: widget._id, tab: closedTab?.name }) if (widget.onTabClose !== undefined && closedTab !== undefined) { const fn = await getResource(widget.onTabClose) @@ -208,7 +208,7 @@ export function openWidgetTab (widget: Ref, tab: string): void { if (newTab === undefined) return widgetsState.set(widget, { ...widgetState, tab }) - Analytics.handleEvent('workbench.OpenSidebarWidget', { widget, tab }) + Analytics.handleEvent(WorkbenchEvents.SidebarOpenWidget, { widget, tab: newTab?.name }) sidebarStore.set({ ...state, widgetsState @@ -243,7 +243,7 @@ export function createWidgetTab (widget: Widget, tab: WidgetTab, newTab = false) tab: tab.id }) - Analytics.handleEvent('workbench.OpenSidebarWidget', { widget: widget._id, tab: tab.id }) + Analytics.handleEvent(WorkbenchEvents.SidebarOpenWidget, { widget: widget._id, tab: tab.name }) sidebarStore.set({ ...state, widget: widget._id, diff --git a/plugins/workbench/src/analytics.ts b/plugins/workbench/src/analytics.ts index 5c83dc185c..82cceda572 100644 --- a/plugins/workbench/src/analytics.ts +++ b/plugins/workbench/src/analytics.ts @@ -1,4 +1,6 @@ export enum WorkbenchEvents { DocumentationOpened = 'workbench.help.DocumentationOpened', - KeyboardShortcutsOpened = 'workbench.help.KeyboardShortcutsOpened' + KeyboardShortcutsOpened = 'workbench.help.KeyboardShortcutsOpened', + SidebarCloseWidget = 'workbench.sidebar.CloseWidget', + SidebarOpenWidget = 'workbench.sidebar.OpenWidget' } diff --git a/plugins/workbench/src/index.ts b/plugins/workbench/src/index.ts index e34c555b7c..2bdf78decf 100644 --- a/plugins/workbench/src/index.ts +++ b/plugins/workbench/src/index.ts @@ -87,7 +87,6 @@ export interface WidgetPreference extends Preference { export interface WidgetTab { id: string name?: string - nameIntl?: IntlString icon?: Asset | AnySvelteComponent iconComponent?: AnyComponent iconProps?: Record diff --git a/server/s3/src/index.ts b/server/s3/src/index.ts index 1314ebd98e..9eaacc2440 100644 --- a/server/s3/src/index.ts +++ b/server/s3/src/index.ts @@ -299,7 +299,7 @@ export class S3Service implements StorageAdapter { version: result.VersionId ?? null } } catch (err: any) { - ctx.error('no object found', { error: err, objectName, workspaceId: workspaceId.name }) + ctx.warn('no object found', { error: err, objectName, workspaceId: workspaceId.name }) } } diff --git a/server/server-storage/src/aggregator.ts b/server/server-storage/src/aggregator.ts index b17b63a43c..7ad1c60b0b 100644 --- a/server/server-storage/src/aggregator.ts +++ b/server/server-storage/src/aggregator.ts @@ -51,7 +51,7 @@ export class AggregatorStorageAdapter implements StorageAdapter, StorageAdapterE workspaceId: WorkspaceId, objectName: string, providerId?: string - ): Promise { + ): Promise { let current: Blob | undefined = ( await this.dbAdapter.find(ctx, workspaceId, DOMAIN_BLOB, { _id: objectName as Ref }, { limit: 1 }) ).shift() @@ -74,6 +74,9 @@ export class AggregatorStorageAdapter implements StorageAdapter, StorageAdapterE } await this.dbAdapter.upload(ctx, workspaceId, DOMAIN_BLOB, [stat]) // TODO: We need to send notification about Blob is changed. + return stat + } else { + throw new NoSuchKeyError('No such blob found') } } diff --git a/services/ai-bot/pod-ai-bot/src/workspaceClient.ts b/services/ai-bot/pod-ai-bot/src/workspaceClient.ts index 2e9071b74a..bd425f0e1a 100644 --- a/services/ai-bot/pod-ai-bot/src/workspaceClient.ts +++ b/services/ai-bot/pod-ai-bot/src/workspaceClient.ts @@ -130,27 +130,27 @@ export class WorkspaceClient { this.ctx.info('Upload avatar file', { workspace: this.workspace }) try { - await this.checkPersonData(client) - const stat = fs.statSync(config.AvatarPath) const lastModified = stat.mtime.getTime() - if ( + const isAlreadyUploaded = this.info !== undefined && this.info.avatarPath === config.AvatarPath && this.info.avatarLastModified === lastModified - ) { - this.ctx.info('Avatar file already uploaded', { workspace: this.workspace, path: config.AvatarPath }) - return - } - const data = fs.readFileSync(config.AvatarPath) + if (!isAlreadyUploaded) { + const data = fs.readFileSync(config.AvatarPath) - await this.blobClient.upload(this.ctx, config.AvatarName, data.length, config.AvatarContentType, data) - await this.controller.updateAvatarInfo(this.workspace, config.AvatarPath, lastModified) - this.ctx.info('Uploaded avatar file', { workspace: this.workspace, path: config.AvatarPath }) + await this.blobClient.upload(this.ctx, config.AvatarName, data.length, config.AvatarContentType, data) + await this.controller.updateAvatarInfo(this.workspace, config.AvatarPath, lastModified) + this.ctx.info('Avatar file uploaded successfully', { workspace: this.workspace, path: config.AvatarPath }) + } else { + this.ctx.info('Avatar file already uploaded', { workspace: this.workspace, path: config.AvatarPath }) + } } catch (e) { this.ctx.error('Failed to upload avatar file', { e }) } + + await this.checkPersonData(client) } private async tryLogin (): Promise { diff --git a/services/analytics-collector/pod-analytics-collector/package.json b/services/analytics-collector/pod-analytics-collector/package.json index 7198911f2d..1b9ea5cb50 100644 --- a/services/analytics-collector/pod-analytics-collector/package.json +++ b/services/analytics-collector/pod-analytics-collector/package.json @@ -57,6 +57,8 @@ "@hcengineering/analytics-collector": "^0.6.0", "@hcengineering/analytics-collector-assets": "^0.6.0", "@hcengineering/analytics-service": "^0.6.0", + "@hcengineering/calendar": "^0.6.24", + "@hcengineering/calendar-assets": "^0.6.22", "@hcengineering/chunter": "^0.6.20", "@hcengineering/chunter-assets": "^0.6.18", "@hcengineering/client": "^0.6.18", diff --git a/services/analytics-collector/pod-analytics-collector/src/format.ts b/services/analytics-collector/pod-analytics-collector/src/format.ts index b44286d3bc..629f963fad 100644 --- a/services/analytics-collector/pod-analytics-collector/src/format.ts +++ b/services/analytics-collector/pod-analytics-collector/src/format.ts @@ -23,14 +23,19 @@ import notification, { notificationId } from '@hcengineering/notification' import recruit, { recruitId } from '@hcengineering/recruit' import time, { timeId } from '@hcengineering/time' import tracker, { trackerId } from '@hcengineering/tracker' -import { Class, Doc, Hierarchy, Markup, Ref } from '@hcengineering/core' +import workbench, { WorkbenchEvents } from '@hcengineering/workbench' +import { Class, Doc, Hierarchy, Markup, Ref, TxOperations } from '@hcengineering/core' import { MarkupNode, MarkupNodeType, MarkupMark, MarkupMarkType } from '@hcengineering/text' import { translate } from '@hcengineering/platform' -export async function eventToMarkup (event: AnalyticEvent, hierarchy: Hierarchy): Promise { +export async function eventToMarkup ( + event: AnalyticEvent, + hierarchy: Hierarchy, + client: TxOperations +): Promise { switch (event.event) { case AnalyticEventType.CustomEvent: - return formatCustomEvent(event) + return await formatCustomEvent(event, client) case AnalyticEventType.Error: return await formatErrorEvent(event) case AnalyticEventType.Navigation: @@ -67,10 +72,15 @@ function toText (text: string, display: 'normal' | 'bold' | 'code' = 'normal'): return { type: MarkupNodeType.text, text, marks } } -function formatCustomEvent (event: AnalyticEvent): string | undefined { +async function formatCustomEvent (event: AnalyticEvent, client: TxOperations): Promise { const text = event.params.event as string | undefined if (text === undefined || text === '') return + if (eventsToSkip.includes(event.params.event)) return + if (sidebarEvents.includes(text)) { + return await formatSidebarEvent(text, event.params, client) + } + const paramsTexts = [] for (const key in event.params) { @@ -85,6 +95,41 @@ function formatCustomEvent (event: AnalyticEvent): string | undefined { return toMarkup([toText(text + ' '), toText(paramsTexts.join(', '), 'code')]) } +async function formatSidebarEvent ( + event: string, + params: Record, + client: TxOperations +): Promise { + let text = event + switch (event) { + case WorkbenchEvents.SidebarOpenWidget: + text = 'open widget' + break + case WorkbenchEvents.SidebarCloseWidget: + text = 'close widget' + break + default: + break + } + const paramsTexts = [] + + if (params.widget !== undefined) { + const widget = client.getModel().findAllSync(workbench.class.Widget, { _id: params.widget })[0] + if (widget !== undefined) { + const widgetName = await translate(widget.label, {}) + paramsTexts.push(`widget: ${widgetName}`) + } else { + paramsTexts.push(`widget: ${params.widget}`) + } + } + + if (params.tab !== undefined) { + paramsTexts.push(`tab: ${params.tab}`) + } + + return toMarkup([toText('Sidebar: ', 'bold'), toText(text + ' '), toText(paramsTexts.join(', '), 'code')]) +} + async function formatErrorEvent (event: AnalyticEvent): Promise { const error = event.params.error @@ -427,3 +472,17 @@ export function getOnboardingMessage (email: string, workspace: string, name: st return toMarkup(nodes) } + +const eventsToSkip = [ + 'Fetch workspace', + 'Create Tab', + 'Update Tab', + 'document.Opened', + 'Create Message', + 'chunter.MessageCreated', + 'Create Time', + 'Create Tag', + 'SetCollectionItems' +] + +const sidebarEvents = [WorkbenchEvents.SidebarOpenWidget, WorkbenchEvents.SidebarCloseWidget] as string[] diff --git a/services/analytics-collector/pod-analytics-collector/src/loaders.ts b/services/analytics-collector/pod-analytics-collector/src/loaders.ts index bb9ad3a056..1c028922ac 100644 --- a/services/analytics-collector/pod-analytics-collector/src/loaders.ts +++ b/services/analytics-collector/pod-analytics-collector/src/loaders.ts @@ -14,6 +14,7 @@ // import { analyticsCollectorId } from '@hcengineering/analytics-collector' +import { calendarId } from '@hcengineering/calendar' import { chunterId } from '@hcengineering/chunter' import { contactId } from '@hcengineering/contact' import { coreId } from '@hcengineering/core' @@ -32,6 +33,7 @@ import { viewId } from '@hcengineering/view' import { workbenchId } from '@hcengineering/workbench' import analyticsCollectorEn from '@hcengineering/analytics-collector-assets/lang/en.json' +import calendarEn from '@hcengineering/calendar-assets/lang/en.json' import chunterEn from '@hcengineering/chunter-assets/lang/en.json' import contactEn from '@hcengineering/contact-assets/lang/en.json' import coreEng from '@hcengineering/core/lang/en.json' @@ -57,12 +59,14 @@ export function registerLoaders (): void { addStringsLoader(platformId, async (lang: string) => platformEng) addStringsLoader(analyticsCollectorId, async (lang: string) => analyticsCollectorEn) + addStringsLoader(calendarId, async (lang: string) => calendarEn) addStringsLoader(chunterId, async (lang: string) => chunterEn) addStringsLoader(contactId, async (lang: string) => contactEn) addStringsLoader(documentId, async (lang: string) => documentEn) addStringsLoader(driveId, async (lang: string) => driveEn) addStringsLoader(hrId, async (lang: string) => hrEn) addStringsLoader(leadId, async (lang: string) => leadEn) + addStringsLoader(loveId, async (lang: string) => loveEn) addStringsLoader(notificationId, async (lang: string) => notificationEn) addStringsLoader(preferenceId, async (lang: string) => preferenceEn) addStringsLoader(recruitId, async (lang: string) => recruitEn) @@ -71,5 +75,4 @@ export function registerLoaders (): void { addStringsLoader(trackerId, async (lang: string) => trackerEn) addStringsLoader(viewId, async (lang: string) => viewEn) addStringsLoader(workbenchId, async (lang: string) => workbenchEn) - addStringsLoader(loveId, async (lang: string) => loveEn) } diff --git a/services/analytics-collector/pod-analytics-collector/src/supportWsClient.ts b/services/analytics-collector/pod-analytics-collector/src/supportWsClient.ts index 46bb3ee282..83f10029a5 100644 --- a/services/analytics-collector/pod-analytics-collector/src/supportWsClient.ts +++ b/services/analytics-collector/pod-analytics-collector/src/supportWsClient.ts @@ -245,7 +245,7 @@ export class SupportWsClient extends WorkspaceClient { const hierarchy = client.getHierarchy() for (const event of events) { - const markup = await eventToMarkup(event, hierarchy) + const markup = await eventToMarkup(event, hierarchy, client) if (markup === undefined) { continue diff --git a/services/analytics-collector/pod-analytics-collector/src/workspaceClient.ts b/services/analytics-collector/pod-analytics-collector/src/workspaceClient.ts index a2795eadf1..66c503f9e0 100644 --- a/services/analytics-collector/pod-analytics-collector/src/workspaceClient.ts +++ b/services/analytics-collector/pod-analytics-collector/src/workspaceClient.ts @@ -23,8 +23,6 @@ export class WorkspaceClient { client: Client | undefined opClient: Promise | TxOperations - initializePromise: Promise | undefined = undefined - constructor ( readonly ctx: MeasureContext, readonly workspace: WorkspaceId