diff --git a/models/love/src/index.ts b/models/love/src/index.ts index 9457c67b8b..729fbf08d7 100644 --- a/models/love/src/index.ts +++ b/models/love/src/index.ts @@ -58,7 +58,8 @@ import { TypeDate, TypeRef, TypeString, - UX + UX, + TypeBoolean } from '@hcengineering/model' import calendar, { TEvent } from '@hcengineering/model-calendar' import core, { TAttachedDoc, TDoc } from '@hcengineering/model-core' @@ -106,9 +107,14 @@ export class TRoom extends TDoc implements Room { x!: number y!: number - language!: RoomLanguage - startWithTranscription!: boolean - startWithRecording!: boolean + @Prop(TypeString(), love.string.Language, { editor: love.component.RoomLanguageEditor }) + language!: RoomLanguage + + @Prop(TypeBoolean(), love.string.StartWithTranscription) + startWithTranscription!: boolean + + @Prop(TypeBoolean(), love.string.StartWithRecording) + startWithRecording!: boolean @Prop(Collection(attachment.class.Attachment), attachment.string.Attachments, { shortLabel: attachment.string.Files }) attachments?: number diff --git a/models/love/src/plugin.ts b/models/love/src/plugin.ts index ef026ba0cf..1829a21dfd 100644 --- a/models/love/src/plugin.ts +++ b/models/love/src/plugin.ts @@ -28,7 +28,8 @@ export default mergeIds(loveId, love, { Settings: '' as AnyComponent, LoveWidget: '' as AnyComponent, MeetingWidget: '' as AnyComponent, - WidgetSwitcher: '' as AnyComponent + WidgetSwitcher: '' as AnyComponent, + RoomLanguageEditor: '' as AnyComponent }, app: { Love: '' as Ref diff --git a/plugins/activity-resources/src/components/doc-update-message/DocUpdateMessageContent.svelte b/plugins/activity-resources/src/components/doc-update-message/DocUpdateMessageContent.svelte index 222306696f..12374e95e9 100644 --- a/plugins/activity-resources/src/components/doc-update-message/DocUpdateMessageContent.svelte +++ b/plugins/activity-resources/src/components/doc-update-message/DocUpdateMessageContent.svelte @@ -63,6 +63,7 @@ {#each createMessages as valueMessage, index} , _class: Ref>): Promise { - const isRemoved = await checkIsObjectRemoved(client, _id, _class) + async function loadObject (_id: Ref, _class: Ref>, attachedTo: Ref): Promise { + const isRemoved = attachedTo === _id ? false : await checkIsObjectRemoved(client, _id, _class) if (isRemoved) { object = await buildRemovedDoc(client, _id, _class) @@ -64,7 +65,7 @@ } } - $: void loadObject(objectId, objectClass) + $: void loadObject(objectId, objectClass, attachedTo) function getPanelComponent (object: Doc, objectPanel?: ObjectPanel): AnyComponent { if (objectPanel !== undefined) { diff --git a/plugins/chunter-resources/src/components/chat/Chat.svelte b/plugins/chunter-resources/src/components/chat/Chat.svelte index 986b3364b9..7e203db249 100644 --- a/plugins/chunter-resources/src/components/chat/Chat.svelte +++ b/plugins/chunter-resources/src/components/chat/Chat.svelte @@ -148,7 +148,7 @@ object = detail.object } - openChannel(selectedData.id, selectedData._class) + openChannel(selectedData.id, selectedData._class, undefined, true) } defineSeparators('chat', [ diff --git a/plugins/chunter-resources/src/components/chat/navigator/ChatNavItem.svelte b/plugins/chunter-resources/src/components/chat/navigator/ChatNavItem.svelte index 8ee6715691..dc332b4e66 100644 --- a/plugins/chunter-resources/src/components/chat/navigator/ChatNavItem.svelte +++ b/plugins/chunter-resources/src/components/chat/navigator/ChatNavItem.svelte @@ -83,7 +83,7 @@ group: 'edit', action: async () => { const id = await getObjectLinkId(linkProviders, object._id, object._class, object) - openChannel(id, object._class) + openChannel(id, object._class, undefined, true) } }) diff --git a/plugins/document-resources/src/components/DocumentEditor.svelte b/plugins/document-resources/src/components/DocumentEditor.svelte index 547023115e..8c5766c6e7 100644 --- a/plugins/document-resources/src/components/DocumentEditor.svelte +++ b/plugins/document-resources/src/components/DocumentEditor.svelte @@ -77,5 +77,6 @@ onExtensions={handleExtensions} on:update on:open-document + on:loaded bind:this={collabEditor} /> diff --git a/plugins/document-resources/src/components/EditDoc.svelte b/plugins/document-resources/src/components/EditDoc.svelte index b1814d3a60..8bdd24d4ef 100644 --- a/plugins/document-resources/src/components/EditDoc.svelte +++ b/plugins/document-resources/src/components/EditDoc.svelte @@ -15,7 +15,6 @@ // --> @@ -303,15 +287,6 @@ {/if} - {#if asideId} - {@const asideComponent = navigatorModel?.aside ?? currentApplication?.aside} - {#if asideComponent !== undefined} - -
- -
- {/if} - {/if}
diff --git a/plugins/love-assets/lang/cs.json b/plugins/love-assets/lang/cs.json index 54f410c561..6c86bad07f 100644 --- a/plugins/love-assets/lang/cs.json +++ b/plugins/love-assets/lang/cs.json @@ -78,6 +78,7 @@ "Status": "Stav", "Active": "Aktivní", "Finished": "Dokončeno", - "StartWithRecording": "Začít s nahráváním" + "StartWithRecording": "Začít s nahráváním", + "Language": "Jazyk" } } \ No newline at end of file diff --git a/plugins/love-assets/lang/en.json b/plugins/love-assets/lang/en.json index cfa427d098..22cc2923a9 100644 --- a/plugins/love-assets/lang/en.json +++ b/plugins/love-assets/lang/en.json @@ -78,6 +78,7 @@ "Status": "Status", "Active": "Active", "Finished": "Finished", - "StartWithRecording": "Start with recording" + "StartWithRecording": "Start with recording", + "Language": "Language" } } diff --git a/plugins/love-assets/lang/es.json b/plugins/love-assets/lang/es.json index 58b63972be..8a50a54be6 100644 --- a/plugins/love-assets/lang/es.json +++ b/plugins/love-assets/lang/es.json @@ -78,6 +78,7 @@ "Status": "Estado", "Active": "Activo", "Finished": "Terminado", - "StartWithRecording": "Iniciar con grabación" + "StartWithRecording": "Iniciar con grabación", + "Language": "Idioma" } } diff --git a/plugins/love-assets/lang/fr.json b/plugins/love-assets/lang/fr.json index 736318a6a3..1c93c733a0 100644 --- a/plugins/love-assets/lang/fr.json +++ b/plugins/love-assets/lang/fr.json @@ -78,6 +78,7 @@ "Status": "Statut", "Active": "Actif", "Finished": "Terminé", - "StartWithRecording": "Démarrer avec l'enregistrement" + "StartWithRecording": "Démarrer avec l'enregistrement", + "Language": "Langue" } } \ No newline at end of file diff --git a/plugins/love-assets/lang/it.json b/plugins/love-assets/lang/it.json index 7463abbfb0..aa569589fd 100644 --- a/plugins/love-assets/lang/it.json +++ b/plugins/love-assets/lang/it.json @@ -78,6 +78,7 @@ "Status": "Stato", "Active": "Attivo", "Finished": "Finito", - "StartWithRecording": "Inizia con la registrazione" + "StartWithRecording": "Inizia con la registrazione", + "Language": "Lingua" } } diff --git a/plugins/love-assets/lang/pt.json b/plugins/love-assets/lang/pt.json index 97412fb5c5..28491c92a8 100644 --- a/plugins/love-assets/lang/pt.json +++ b/plugins/love-assets/lang/pt.json @@ -78,6 +78,7 @@ "Status": "Estado", "Active": "Ativo", "Finished": "Finalizado", - "StartWithRecording": "Começar com gravação" + "StartWithRecording": "Começar com gravação", + "Language": "Idioma" } } diff --git a/plugins/love-assets/lang/ru.json b/plugins/love-assets/lang/ru.json index 124e1647d1..95ecdf8082 100644 --- a/plugins/love-assets/lang/ru.json +++ b/plugins/love-assets/lang/ru.json @@ -78,6 +78,7 @@ "Status": "Статус", "Active": "Активно", "Finished": "Завершено", - "StartWithRecording": "Начинать с записью" + "StartWithRecording": "Начинать с записью", + "Language": "Язык" } } diff --git a/plugins/love-assets/lang/zh.json b/plugins/love-assets/lang/zh.json index 4902ff5397..a665270a73 100644 --- a/plugins/love-assets/lang/zh.json +++ b/plugins/love-assets/lang/zh.json @@ -78,6 +78,7 @@ "Status": "状态", "Active": "活动", "Finished": "已完成", - "StartWithRecording": "开始录制" + "StartWithRecording": "开始录制", + "Language": "语言" } } diff --git a/plugins/love-resources/src/components/FloorAttributePresenter.svelte b/plugins/love-resources/src/components/FloorAttributePresenter.svelte index a98dc53262..5dc48a6f65 100644 --- a/plugins/love-resources/src/components/FloorAttributePresenter.svelte +++ b/plugins/love-resources/src/components/FloorAttributePresenter.svelte @@ -29,7 +29,7 @@ {#if inline} {:else} -
+
{floor.name}
{/if} diff --git a/plugins/love-resources/src/components/RoomLanguage.svelte b/plugins/love-resources/src/components/RoomLanguage.svelte index 9bcac4bed6..350ef30e27 100644 --- a/plugins/love-resources/src/components/RoomLanguage.svelte +++ b/plugins/love-resources/src/components/RoomLanguage.svelte @@ -18,10 +18,14 @@ import { languagesDisplayData } from '../types' export let room: Room + export let withLabel = false $: lang = room.language {languagesDisplayData[lang].emoji ?? languagesDisplayData.en.emoji} + {#if withLabel} + {languagesDisplayData[lang].label ?? languagesDisplayData.en.label} + {/if} diff --git a/plugins/love-resources/src/components/RoomLanguageEditor.svelte b/plugins/love-resources/src/components/RoomLanguageEditor.svelte new file mode 100644 index 0000000000..274724a88e --- /dev/null +++ b/plugins/love-resources/src/components/RoomLanguageEditor.svelte @@ -0,0 +1,67 @@ + + +{#if object} + +{/if} diff --git a/plugins/love-resources/src/index.ts b/plugins/love-resources/src/index.ts index c06f50ae41..fd308ddb50 100644 --- a/plugins/love-resources/src/index.ts +++ b/plugins/love-resources/src/index.ts @@ -23,6 +23,7 @@ import PanelControlBar from './components/PanelControlBar.svelte' import RoomPresenter from './components/RoomPresenter.svelte' import MeetingMinutesDocEditor from './components/MeetingMinutesDocEditor.svelte' import MeetingMinutesStatusPresenter from './components/MeetingMinutesStatusPresenter.svelte' +import RoomLanguageEditor from './components/RoomLanguageEditor.svelte' import { copyGuestLink, @@ -59,7 +60,8 @@ export default async (): Promise => ({ PanelControlBar, RoomPresenter, MeetingMinutesDocEditor, - MeetingMinutesStatusPresenter + MeetingMinutesStatusPresenter, + RoomLanguageEditor }, function: { CreateMeeting: createMeeting, diff --git a/plugins/love-resources/src/plugin.ts b/plugins/love-resources/src/plugin.ts index b74e0c9cd1..90cd754245 100644 --- a/plugins/love-resources/src/plugin.ts +++ b/plugins/love-resources/src/plugin.ts @@ -94,6 +94,7 @@ export default mergeIds(loveId, love, { KnockAction: '' as IntlString, Select: '' as IntlString, ChooseShare: '' as IntlString, - MoreOptions: '' as IntlString + MoreOptions: '' as IntlString, + Language: '' as IntlString } }) diff --git a/plugins/text-editor-resources/src/components/CollaborativeTextEditor.svelte b/plugins/text-editor-resources/src/components/CollaborativeTextEditor.svelte index cb43f5641e..b8650476f0 100644 --- a/plugins/text-editor-resources/src/components/CollaborativeTextEditor.svelte +++ b/plugins/text-editor-resources/src/components/CollaborativeTextEditor.svelte @@ -140,6 +140,10 @@ void localProvider.loaded.then(() => (localSynced = true)) void remoteProvider.loaded.then(() => (remoteSynced = true)) + void Promise.all([localProvider.loaded, remoteProvider.loaded]).then(() => { + dispatch('loaded') + }) + let editor: Editor let element: HTMLElement let textToolbarElement: HTMLElement diff --git a/plugins/text-editor-resources/src/components/CollaboratorEditor.svelte b/plugins/text-editor-resources/src/components/CollaboratorEditor.svelte index ef86f08098..fc969bef1a 100644 --- a/plugins/text-editor-resources/src/components/CollaboratorEditor.svelte +++ b/plugins/text-editor-resources/src/components/CollaboratorEditor.svelte @@ -103,6 +103,7 @@ on:update on:open-document on:blur + on:loaded on:focus={handleFocus} />
diff --git a/plugins/text-editor-resources/src/components/StyledTextBox.svelte b/plugins/text-editor-resources/src/components/StyledTextBox.svelte index fc27ac1e32..4541ee88d1 100644 --- a/plugins/text-editor-resources/src/components/StyledTextBox.svelte +++ b/plugins/text-editor-resources/src/components/StyledTextBox.svelte @@ -30,7 +30,8 @@ import { ImageUploadExtension } from './extension/imageUploadExt' import { InlineCommandsExtension } from './extension/inlineCommands' import { type FileAttachFunction } from './extension/types' - import { completionConfig, inlineCommandsConfig } from './extensions' + import { completionConfig, InlineCommandId, inlineCommandsConfig } from './extensions' + import { MermaidExtension, mermaidOptions } from './extension/mermaid' export let label: IntlString | undefined = undefined export let content: Markup @@ -192,6 +193,7 @@ } extensions.push( imageUploadPlugin, + MermaidExtension.configure(mermaidOptions), FocusExtension.configure({ onCanBlur: (value: boolean) => (canBlur = value), onFocus: handleFocus }) ) if (enableEmojiReplace) { @@ -199,10 +201,12 @@ } if (enableInlineCommands) { + const excludedInlineCommands: InlineCommandId[] = ['drawing-board', 'todo-list'] + + if (attachFile === undefined) excludedInlineCommands.push('image') + extensions.push( - InlineCommandsExtension.configure( - inlineCommandsConfig(handleCommandSelected, attachFile === undefined ? ['image'] : []) - ) + InlineCommandsExtension.configure(inlineCommandsConfig(handleCommandSelected, excludedInlineCommands)) ) } @@ -226,11 +230,13 @@ } case 'code-block': editor.editorHandler.insertCodeBlock(pos) - break case 'separator-line': editor.editorHandler.insertSeparatorLine() break + case 'mermaid': + editor.getEditor()?.commands.insertContentAt(pos, { type: 'mermaid' }) + break } } diff --git a/plugins/text-editor-resources/src/components/StyledTextEditor.svelte b/plugins/text-editor-resources/src/components/StyledTextEditor.svelte index 82c4b84209..9c760e903a 100644 --- a/plugins/text-editor-resources/src/components/StyledTextEditor.svelte +++ b/plugins/text-editor-resources/src/components/StyledTextEditor.svelte @@ -17,10 +17,9 @@ import { IntlString } from '@hcengineering/platform' import { EmptyMarkup } from '@hcengineering/text' import { Button, type ButtonSize, Scroller } from '@hcengineering/ui' - import { AnyExtension, mergeAttributes } from '@tiptap/core' + import { AnyExtension, mergeAttributes, type Editor } from '@tiptap/core' import { createEventDispatcher } from 'svelte' import textEditor, { RefAction, TextEditorHandler, TextFormatCategory } from '@hcengineering/text-editor' - import { defaultRefActions, getModelRefActions } from './editor/actions' import TextEditor from './TextEditor.svelte' @@ -63,6 +62,9 @@ export function insertText (text: string): void { editor?.insertText(text) } + export function getEditor (): Editor | undefined { + return editor?.getEditor() + } $: varsStyle = maxHeight === 'card' diff --git a/plugins/text-editor-resources/src/components/TextEditor.svelte b/plugins/text-editor-resources/src/components/TextEditor.svelte index ff018f894e..e670bee1ce 100644 --- a/plugins/text-editor-resources/src/components/TextEditor.svelte +++ b/plugins/text-editor-resources/src/components/TextEditor.svelte @@ -90,6 +90,9 @@ editor.commands.clearContent(true) } } + export function getEditor (): Editor { + return editor + } export function insertText (text: string): void { editor?.commands.insertContent(text) diff --git a/plugins/text-editor-resources/src/components/extensions.ts b/plugins/text-editor-resources/src/components/extensions.ts index 96bdeb673e..5ed841d04b 100644 --- a/plugins/text-editor-resources/src/components/extensions.ts +++ b/plugins/text-editor-resources/src/components/extensions.ts @@ -130,7 +130,15 @@ export const completionConfig: Partial = { } } -const inlineCommandsIds = ['image', 'table', 'code-block', 'separator-line', 'todo-list'] as const +const inlineCommandsIds = [ + 'image', + 'table', + 'code-block', + 'separator-line', + 'todo-list', + 'drawing-board', + 'mermaid' +] as const export type InlineCommandId = (typeof inlineCommandsIds)[number] /** diff --git a/plugins/tracker-resources/src/index.ts b/plugins/tracker-resources/src/index.ts index 086587f69e..b46bea9337 100644 --- a/plugins/tracker-resources/src/index.ts +++ b/plugins/tracker-resources/src/index.ts @@ -229,7 +229,7 @@ async function move (issues: Issue | Issue[]): Promise { async function editWorkflowStatuses (project: Project): Promise { const loc = getCurrentLocation() loc.path[2] = settingId - loc.path[3] = 'statuses' + loc.path[3] = 'spaceTypes' loc.path[4] = project.type navigate(loc) } diff --git a/plugins/view-resources/src/components/ParentsNavigator.svelte b/plugins/view-resources/src/components/ParentsNavigator.svelte index 51f46a4898..bc505ef18b 100644 --- a/plugins/view-resources/src/components/ParentsNavigator.svelte +++ b/plugins/view-resources/src/components/ParentsNavigator.svelte @@ -13,7 +13,7 @@ // limitations under the License. --> -{#await getParents(element) then parents} - -{/await} + diff --git a/plugins/view-resources/src/utils.ts b/plugins/view-resources/src/utils.ts index 8bf9723798..01124c437a 100644 --- a/plugins/view-resources/src/utils.ts +++ b/plugins/view-resources/src/utils.ts @@ -752,9 +752,18 @@ export function categorizeFields ( export function makeViewletKey (loc?: Location): string { loc = loc != null ? { path: loc.path } : getCurrentResolvedLocation() - loc.fragment = undefined loc.query = undefined + if (loc.fragment != null && loc.fragment !== '') { + const props = decodeURIComponent(loc.fragment).split('|') + if (props.length >= 3) { + const [panel, , _class] = props + + return 'viewlet' + '#' + encodeURIComponent([panel, _class].join('|')) + } + } + + loc.fragment = undefined return 'viewlet' + locationToUrl(loc) } diff --git a/plugins/workbench-resources/src/components/Workbench.svelte b/plugins/workbench-resources/src/components/Workbench.svelte index 172131dc21..62e0dcb8ef 100644 --- a/plugins/workbench-resources/src/components/Workbench.svelte +++ b/plugins/workbench-resources/src/components/Workbench.svelte @@ -136,7 +136,6 @@ let currentSpecial: string | undefined let currentQuery: Record | undefined let specialComponent: SpecialNavModel | undefined - let asideId: string | undefined let currentFragment: string | undefined = '' let currentApplication: Application | undefined @@ -374,8 +373,6 @@ loc.path[4] = currentSpecial } else if (loc.path[3] === resolved.defaultLocation.path[3]) { loc.path[4] = resolved.defaultLocation.path[4] - } else { - loc.path[4] = asideId as string } } else { loc.path.length = 4 @@ -512,10 +509,6 @@ } } - if (special !== currentSpecial && (navigatorModel?.aside || currentApplication?.aside)) { - asideId = special - } - if (app !== undefined) { localStorage.setItem(`${locationStorageKeyId}_${app}`, originalLoc) } @@ -589,21 +582,12 @@ specialComponent = undefined // eslint-disable-next-line no-fallthrough case 3: - asideId = undefined if (currentSpace !== undefined) { specialComponent = undefined } } } - function closeAside (): void { - const loc = getLocation() - loc.path.length = 4 - asideId = undefined - checkOnHide() - navigate(loc) - } - async function updateSpace (spaceId?: Ref): Promise { if (spaceId === currentSpace) return clear(2) @@ -620,14 +604,11 @@ function setSpaceSpecial (spaceSpecial: string | undefined): void { if (currentSpecial !== undefined && spaceSpecial === currentSpecial) return - if (asideId !== undefined && spaceSpecial === asideId) return clear(3) if (spaceSpecial === undefined) return specialComponent = getSpecialComponent(spaceSpecial) if (specialComponent !== undefined) { currentSpecial = spaceSpecial - } else if (navigatorModel?.aside !== undefined || currentApplication?.aside !== undefined) { - asideId = spaceSpecial } } @@ -647,7 +628,6 @@ } } - let aside: HTMLElement let cover: HTMLElement let workbenchWidth: number = $deviceInfo.docWidth @@ -762,14 +742,6 @@ person && client.getHierarchy().hasMixin(person, contact.mixin.Employee) ? !client.getHierarchy().as(person, contact.mixin.Employee).active : false - - let asideComponent: AnyComponent | undefined - - $: if (asideId !== undefined && navigatorModel !== undefined) { - asideComponent = navigatorModel?.aside ?? currentApplication?.aside - } else { - asideComponent = undefined - } {#if person && deactivated && !isAdminUser()} @@ -1001,11 +973,8 @@ {:else if specialComponent} {/if}
- {#if asideComponent !== undefined} - -
- -
- {/if} {#if !$deviceInfo.aside.float} {#if $sidebarStore.variant === SidebarVariant.EXPANDED} diff --git a/plugins/workbench-resources/src/utils.ts b/plugins/workbench-resources/src/utils.ts index dffc7b52d1..c36a0d9dc1 100644 --- a/plugins/workbench-resources/src/utils.ts +++ b/plugins/workbench-resources/src/utils.ts @@ -192,8 +192,7 @@ export async function buildNavModel ( const newSpaces = (nm.spaces ?? []).filter((it) => !spaces.some((sp) => sp.id === it.id)) newNavModel = { spaces: [...spaces, ...newSpaces], - specials: [...(newNavModel?.specials ?? []), ...(nm.specials ?? [])], - aside: newNavModel?.aside ?? nm?.aside + specials: [...(newNavModel?.specials ?? []), ...(nm.specials ?? [])] } } } diff --git a/plugins/workbench/src/index.ts b/plugins/workbench/src/index.ts index 86b377ba68..15a8da28f3 100644 --- a/plugins/workbench/src/index.ts +++ b/plugins/workbench/src/index.ts @@ -50,7 +50,6 @@ export interface Application extends Doc { // Also attached ApplicationNavModel will be joined after this one main. navigatorModel?: NavigatorModel - aside?: AnyComponent locationResolver?: Resource<(loc: Location) => Promise> locationDataResolver?: Resource<(loc: Location) => Promise> @@ -130,7 +129,6 @@ export interface ApplicationNavModel extends Doc { spaces?: SpacesNavModel[] specials?: SpecialNavModel[] - aside?: AnyComponent } /** @@ -163,7 +161,6 @@ export interface SpacesNavModel { export interface NavigatorModel { spaces: SpacesNavModel[] specials?: SpecialNavModel[] - aside?: AnyComponent } /** diff --git a/server/core/src/types.ts b/server/core/src/types.ts index afb621593b..628f989f98 100644 --- a/server/core/src/types.ts +++ b/server/core/src/types.ts @@ -62,6 +62,9 @@ export interface ServerFindOptions extends FindOptions { domain: Domain } + // using for join query security + allowedSpaces?: Ref[] + // Optional measure context, for server side operations ctx?: MeasureContext } diff --git a/server/middleware/src/queryJoin.ts b/server/middleware/src/queryJoin.ts index 891de6008c..35fa53f3cc 100644 --- a/server/middleware/src/queryJoin.ts +++ b/server/middleware/src/queryJoin.ts @@ -22,7 +22,7 @@ import { type MeasureContext, Ref } from '@hcengineering/core' -import { BaseMiddleware, Middleware, type PipelineContext } from '@hcengineering/server-core' +import { BaseMiddleware, Middleware, ServerFindOptions, type PipelineContext } from '@hcengineering/server-core' import { deepEqual } from 'fast-equals' interface Query { @@ -45,7 +45,7 @@ export class QueryJoiner { ctx: MeasureContext, _class: Ref>, query: DocumentQuery, - options?: FindOptions + options?: ServerFindOptions ): Promise> { // Will find a query or add + 1 to callbacks const q = this.findQuery(_class, query, options) ?? this.createQuery(_class, query, options) @@ -131,7 +131,7 @@ export class QueryJoinMiddleware extends BaseMiddleware implements Middleware { ctx: MeasureContext, _class: Ref>, query: DocumentQuery, - options?: FindOptions + options?: ServerFindOptions ): Promise> { // Will find a query or add + 1 to callbacks return this.joiner.findAll(ctx, _class, query, options) diff --git a/server/middleware/src/spaceSecurity.ts b/server/middleware/src/spaceSecurity.ts index 2ce09da102..6c792db300 100644 --- a/server/middleware/src/spaceSecurity.ts +++ b/server/middleware/src/spaceSecurity.ts @@ -21,7 +21,6 @@ import core, { Doc, DocumentQuery, Domain, - FindOptions, FindResult, LookupData, MeasureContext, @@ -47,7 +46,13 @@ import core, { type SessionData } from '@hcengineering/core' import platform, { PlatformError, Severity, Status } from '@hcengineering/platform' -import { BaseMiddleware, Middleware, TxMiddlewareResult, type PipelineContext } from '@hcengineering/server-core' +import { + BaseMiddleware, + Middleware, + ServerFindOptions, + TxMiddlewareResult, + type PipelineContext +} from '@hcengineering/server-core' import { isOwner, isSystem } from './utils' type SpaceWithMembers = Pick @@ -416,6 +421,14 @@ export class SpaceSecurityMiddleware extends BaseMiddleware implements Middlewar } } + ctx.contextData.broadcast.targets.spaceSec = (tx) => { + const space = this.spacesMap.get(tx.objectSpace) + if (space === undefined) return undefined + if (this.systemSpaces.has(space._id) || this.mainSpaces.includes(space._id)) return undefined + + return space.members.length === 0 ? undefined : this.getTargets(space?.members) + } + await this.next?.handleBroadcast(ctx) } @@ -497,7 +510,7 @@ export class SpaceSecurityMiddleware extends BaseMiddleware implements Middlewar ctx: MeasureContext, _class: Ref>, query: DocumentQuery, - options?: FindOptions + options?: ServerFindOptions ): Promise> { await this.init(ctx) @@ -509,12 +522,7 @@ export class SpaceSecurityMiddleware extends BaseMiddleware implements Middlewar let clientFilterSpaces: Set> | undefined - if ( - !this.skipFindCheck && - !isSystem(account, ctx) && - account.role !== AccountRole.DocGuest && - domain !== DOMAIN_MODEL - ) { + if (!isSystem(account, ctx) && account.role !== AccountRole.DocGuest && domain !== DOMAIN_MODEL) { if (!isOwner(account, ctx) || !isSpace) { if (query[field] !== undefined) { const res = await this.mergeQuery(ctx, account, query[field], domain, isSpace) @@ -536,6 +544,11 @@ export class SpaceSecurityMiddleware extends BaseMiddleware implements Middlewar delete (newQuery as any)[field] } else if (spaces.result.length === 1) { ;(newQuery as any)[field] = spaces.result[0] + if (options !== undefined) { + options.allowedSpaces = spaces.result + } else { + options = { allowedSpaces: spaces.result } + } } else { // Check if spaces > 85% of all domain spaces, in this case return all and filter on client. if (spaces.result.length / spaces.domainSpaces.size > 0.85 && options?.limit === undefined) { @@ -543,17 +556,22 @@ export class SpaceSecurityMiddleware extends BaseMiddleware implements Middlewar delete newQuery.space } else { ;(newQuery as any)[field] = { $in: spaces.result } + if (options !== undefined) { + options.allowedSpaces = spaces.result + } else { + options = { allowedSpaces: spaces.result } + } } } } } } - let findResult = await this.provideFindAll(ctx, _class, newQuery, options) + let findResult = await this.provideFindAll(ctx, _class, !this.skipFindCheck ? newQuery : query, options) if (clientFilterSpaces !== undefined) { const cfs = clientFilterSpaces findResult = toFindResult( - findResult.filter((it) => cfs.has(it.space)), + findResult.filter((it) => cfs.has((it as any)[field])), findResult.total, findResult.lookupMap ) diff --git a/workers/datalake/src/blob.ts b/workers/datalake/src/blob.ts index 7f118e3a90..3fd6fcd16f 100644 --- a/workers/datalake/src/blob.ts +++ b/workers/datalake/src/blob.ts @@ -14,14 +14,15 @@ // import { error, json } from 'itty-router' -import postgres from 'postgres' -import * as db from './db' +import { type Sql } from 'postgres' +import db, { withPostgres } from './db' import { cacheControl, hashLimit } from './const' import { toUUID } from './encodings' import { getSha256 } from './hash' import { selectStorage } from './storage' import { type BlobRequest, type WorkspaceRequest, type UUID } from './types' import { copyVideo, deleteVideo } from './video' +import { measure, LoggedCache } from './measure' interface BlobMetadata { lastModified: number @@ -38,20 +39,22 @@ export function getBlobURL (request: Request, workspace: string, name: string): export async function handleBlobGet (request: BlobRequest, env: Env, ctx: ExecutionContext): Promise { const { workspace, name } = request - const sql = postgres(env.HYPERDRIVE.connectionString) - const { bucket } = selectStorage(env, workspace) - - const blob = await db.getBlob(sql, { workspace, name }) - if (blob === null || blob.deleted) { - return error(404) - } - - const cache = caches.default + const cache = new LoggedCache(caches.default) const cached = await cache.match(request) if (cached !== undefined) { + console.log({ message: 'cache hit' }) return cached } + const { bucket } = selectStorage(env, workspace) + + const blob = await withPostgres(env, ctx, (sql) => { + return db.getBlob(sql, { workspace, name }) + }) + if (blob === null || blob.deleted) { + return error(404) + } + const range = request.headers.has('Range') ? request.headers : undefined const object = await bucket.get(blob.filename, { range }) if (object === null) { @@ -67,6 +70,7 @@ export async function handleBlobGet (request: BlobRequest, env: Env, ctx: Execut const status = length !== undefined && length < object.size ? 206 : 200 const response = new Response(object?.body, { headers, status }) + if (response.status === 200) { ctx.waitUntil(cache.put(request, response.clone())) } @@ -77,10 +81,11 @@ export async function handleBlobGet (request: BlobRequest, env: Env, ctx: Execut export async function handleBlobHead (request: BlobRequest, env: Env, ctx: ExecutionContext): Promise { const { workspace, name } = request - const sql = postgres(env.HYPERDRIVE.connectionString) const { bucket } = selectStorage(env, workspace) - const blob = await db.getBlob(sql, { workspace, name }) + const blob = await withPostgres(env, ctx, (sql) => { + return db.getBlob(sql, { workspace, name }) + }) if (blob === null || blob.deleted) { return error(404) } @@ -97,10 +102,10 @@ export async function handleBlobHead (request: BlobRequest, env: Env, ctx: Execu export async function handleBlobDelete (request: BlobRequest, env: Env, ctx: ExecutionContext): Promise { const { workspace, name } = request - const sql = postgres(env.HYPERDRIVE.connectionString) - try { - await Promise.all([db.deleteBlob(sql, { workspace, name }), deleteVideo(env, workspace, name)]) + await withPostgres(env, ctx, (sql) => { + return Promise.all([db.deleteBlob(sql, { workspace, name }), deleteVideo(env, workspace, name)]) + }) return new Response(null, { status: 204 }) } catch (err: any) { @@ -110,7 +115,11 @@ export async function handleBlobDelete (request: BlobRequest, env: Env, ctx: Exe } } -export async function handleUploadFormData (request: WorkspaceRequest, env: Env): Promise { +export async function handleUploadFormData ( + request: WorkspaceRequest, + env: Env, + ctx: ExecutionContext +): Promise { const contentType = request.headers.get('Content-Type') if (contentType === null || !contentType.includes('multipart/form-data')) { console.error({ error: 'expected multipart/form-data' }) @@ -119,11 +128,9 @@ export async function handleUploadFormData (request: WorkspaceRequest, env: Env) const { workspace } = request - const sql = postgres(env.HYPERDRIVE.connectionString) - let formData: FormData try { - formData = await request.formData() + formData = await measure('fetch formdata', () => request.formData()) } catch (err: any) { const message = err instanceof Error ? err.message : String(err) console.error({ error: 'failed to parse form data', message }) @@ -139,7 +146,9 @@ export async function handleUploadFormData (request: WorkspaceRequest, env: Env) files.map(async ([file, key]) => { const { name, type, lastModified } = file try { - const metadata = await saveBlob(env, sql, file.stream(), file.size, type, workspace, name, lastModified) + const metadata = await withPostgres(env, ctx, (sql) => { + return saveBlob(env, sql, file.stream(), file.size, type, workspace, name, lastModified) + }) // TODO this probably should happen via queue, let it be here for now if (type.startsWith('video/')) { @@ -161,7 +170,7 @@ export async function handleUploadFormData (request: WorkspaceRequest, env: Env) export async function saveBlob ( env: Env, - sql: postgres.Sql, + sql: Sql, stream: ReadableStream, size: number, type: string, @@ -179,6 +188,7 @@ export async function saveBlob ( const hash = await getSha256(hashStream) const data = await db.getData(sql, { hash, location }) + if (data !== null) { // Lucky boy, nothing to upload, use existing blob await db.createBlob(sql, { workspace, name, hash, location }) @@ -212,8 +222,13 @@ export async function saveBlob ( } } -export async function handleBlobUploaded (env: Env, workspace: string, name: string, filename: UUID): Promise { - const sql = postgres(env.HYPERDRIVE.connectionString) +export async function handleBlobUploaded ( + env: Env, + ctx: ExecutionContext, + workspace: string, + name: string, + filename: UUID +): Promise { const { location, bucket } = selectStorage(env, workspace) const object = await bucket.head(filename) @@ -223,16 +238,20 @@ export async function handleBlobUploaded (env: Env, workspace: string, name: str const hash = object.checksums.md5 !== undefined ? digestToUUID(object.checksums.md5) : (crypto.randomUUID() as UUID) - const data = await db.getData(sql, { hash, location }) - if (data !== null) { - await Promise.all([bucket.delete(filename), db.createBlob(sql, { workspace, name, hash, location })]) - } else { - const size = object.size - const type = object.httpMetadata.contentType ?? 'application/octet-stream' + await withPostgres(env, ctx, async (sql) => { + const data = await db.getData(sql, { hash, location }) + if (data !== null) { + await Promise.all([bucket.delete(filename), db.createBlob(sql, { workspace, name, hash, location })]) + } else { + const size = object.size + const type = object.httpMetadata?.contentType ?? 'application/octet-stream' - await db.createData(sql, { hash, location, filename, type, size }) - await db.createBlob(sql, { workspace, name, hash, location }) - } + await sql.begin((sql) => [ + db.createData(sql, { hash, location, filename, type, size }), + db.createBlob(sql, { workspace, name, hash, location }) + ]) + } + }) } async function uploadLargeFile ( diff --git a/workers/datalake/src/db.ts b/workers/datalake/src/db.ts index 725c21c62c..6aad3e3be8 100644 --- a/workers/datalake/src/db.ts +++ b/workers/datalake/src/db.ts @@ -13,7 +13,8 @@ // limitations under the License. // -import type postgres from 'postgres' +import postgres from 'postgres' +import { measure, measureSync } from './measure' import { type Location, type UUID } from './types' export interface BlobDataId { @@ -42,63 +43,94 @@ export interface BlobRecordWithFilename extends BlobRecord { filename: string } -export async function getData (sql: postgres.Sql, dataId: BlobDataId): Promise { - const { hash, location } = dataId - - const rows = await sql` - SELECT hash, location, filename, size, type - FROM blob.data - WHERE hash = ${hash} AND location = ${location} - ` - - if (rows.length > 0) { - return rows[0] +export async function withPostgres ( + env: Env, + ctx: ExecutionContext, + fn: (sql: postgres.Sql) => Promise +): Promise { + const sql = measureSync('db.connect', () => { + return postgres(env.HYPERDRIVE.connectionString) + }) + try { + return await fn(sql) + } finally { + measureSync('db.close', () => { + ctx.waitUntil(sql.end({ timeout: 0 })) + }) } - - return null } -export async function createData (sql: postgres.Sql, data: BlobDataRecord): Promise { - const { hash, location, filename, size, type } = data - - await sql` - UPSERT INTO blob.data (hash, location, filename, size, type) - VALUES (${hash}, ${location}, ${filename}, ${size}, ${type}) - ` +export interface BlobDB { + getData: (sql: postgres.Sql, dataId: BlobDataId) => Promise + createData: (sql: postgres.Sql, data: BlobDataRecord) => Promise + getBlob: (sql: postgres.Sql, blobId: BlobId) => Promise + createBlob: (sql: postgres.Sql, blob: Omit) => Promise + deleteBlob: (sql: postgres.Sql, blob: BlobId) => Promise } -export async function getBlob (sql: postgres.Sql, blobId: BlobId): Promise { - const { workspace, name } = blobId +const db: BlobDB = { + async getData (sql: postgres.Sql, dataId: BlobDataId): Promise { + const { hash, location } = dataId + const rows = await sql` + SELECT hash, location, filename, size, type + FROM blob.data + WHERE hash = ${hash} AND location = ${location} + ` + return rows.length > 0 ? rows[0] : null + }, - const rows = await sql` - SELECT b.workspace, b.name, b.hash, b.location, b.deleted, d.filename - FROM blob.blob AS b - JOIN blob.data AS d ON b.hash = d.hash AND b.location = d.location - WHERE b.workspace = ${workspace} AND b.name = ${name} - ` + async createData (sql: postgres.Sql, data: BlobDataRecord): Promise { + const { hash, location, filename, size, type } = data - if (rows.length > 0) { - return rows[0] + await sql` + UPSERT INTO blob.data (hash, location, filename, size, type) + VALUES (${hash}, ${location}, ${filename}, ${size}, ${type}) + ` + }, + + async getBlob (sql: postgres.Sql, blobId: BlobId): Promise { + const { workspace, name } = blobId + + const rows = await sql` + SELECT b.workspace, b.name, b.hash, b.location, b.deleted, d.filename + FROM blob.blob AS b + JOIN blob.data AS d ON b.hash = d.hash AND b.location = d.location + WHERE b.workspace = ${workspace} AND b.name = ${name} + ` + + if (rows.length > 0) { + return rows[0] + } + + return null + }, + + async createBlob (sql: postgres.Sql, blob: Omit): Promise { + const { workspace, name, hash, location } = blob + + await sql` + UPSERT INTO blob.blob (workspace, name, hash, location, deleted) + VALUES (${workspace}, ${name}, ${hash}, ${location}, false) + ` + }, + + async deleteBlob (sql: postgres.Sql, blob: BlobId): Promise { + const { workspace, name } = blob + + await sql` + UPDATE blob.blob + SET deleted = true + WHERE workspace = ${workspace} AND name = ${name} + ` } - - return null } -export async function createBlob (sql: postgres.Sql, blob: Omit): Promise { - const { workspace, name, hash, location } = blob - - await sql` - UPSERT INTO blob.blob (workspace, name, hash, location, deleted) - VALUES (${workspace}, ${name}, ${hash}, ${location}, false) - ` +export const measuredDb: BlobDB = { + getData: (sql, dataId) => measure('db.getData', () => db.getData(sql, dataId)), + createData: (sql, data) => measure('db.createData', () => db.createData(sql, data)), + getBlob: (sql, blobId) => measure('db.getBlob', () => db.getBlob(sql, blobId)), + createBlob: (sql, blob) => measure('db.createBlob', () => db.createBlob(sql, blob)), + deleteBlob: (sql, blob) => measure('db.deleteBlob', () => db.deleteBlob(sql, blob)) } -export async function deleteBlob (sql: postgres.Sql, blob: BlobId): Promise { - const { workspace, name } = blob - - await sql` - UPDATE blob.blob - SET deleted = true - WHERE workspace = ${workspace} AND name = ${name} - ` -} +export default measuredDb diff --git a/workers/datalake/src/index.ts b/workers/datalake/src/index.ts index d61b35e70f..4dcf33df06 100644 --- a/workers/datalake/src/index.ts +++ b/workers/datalake/src/index.ts @@ -18,6 +18,7 @@ import { type IRequestStrict, type RequestHandler, Router, error, html } from 'i import { handleBlobDelete, handleBlobGet, handleBlobHead, handleUploadFormData } from './blob' import { cors } from './cors' +import { LoggedKVNamespace, LoggedR2Bucket, requestTimeAfter, requestTimeBefore } from './measure' import { handleImageGet } from './image' import { handleS3Blob } from './s3' import { handleVideoMetaGet } from './video' @@ -35,8 +36,8 @@ const { preflight, corsify } = cors({ }) const router = Router({ - before: [preflight], - finally: [corsify] + before: [preflight, requestTimeBefore], + finally: [corsify, requestTimeAfter] }) const withWorkspace: RequestHandler = (request: WorkspaceRequest) => { @@ -87,6 +88,19 @@ router .all('*', () => error(404)) export default class DatalakeWorker extends WorkerEntrypoint { + constructor (ctx: ExecutionContext, env: Env) { + env = { + ...env, + datalake_blobs: new LoggedKVNamespace(env.datalake_blobs), + DATALAKE_APAC: new LoggedR2Bucket(env.DATALAKE_APAC), + DATALAKE_EEUR: new LoggedR2Bucket(env.DATALAKE_EEUR), + DATALAKE_WEUR: new LoggedR2Bucket(env.DATALAKE_WEUR), + DATALAKE_ENAM: new LoggedR2Bucket(env.DATALAKE_ENAM), + DATALAKE_WNAM: new LoggedR2Bucket(env.DATALAKE_WNAM) + } + super(ctx, env) + } + async fetch (request: Request): Promise { return await router.fetch(request, this.env, this.ctx).catch(error) } diff --git a/workers/datalake/src/measure.ts b/workers/datalake/src/measure.ts new file mode 100644 index 0000000000..385f3b44e8 --- /dev/null +++ b/workers/datalake/src/measure.ts @@ -0,0 +1,177 @@ +// +// Copyright © 2024 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the 'License'); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an 'AS IS' BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import { type IRequest, type ResponseHandler, type RequestHandler } from 'itty-router' + +export async function measure (label: string, fn: () => Promise): Promise { + const start = performance.now() + try { + return await fn() + } finally { + const duration = performance.now() - start + console.log({ stage: label, duration }) + } +} + +export function measureSync (label: string, fn: () => T): T { + const start = performance.now() + try { + return fn() + } finally { + const duration = performance.now() - start + console.log({ stage: label, duration }) + } +} + +export const requestTimeBefore: RequestHandler = async (request: IRequest) => { + request.startTime = performance.now() +} + +export const requestTimeAfter: ResponseHandler = async (response: Response, request: IRequest) => { + const duration = performance.now() - request.startTime + console.log({ stage: 'total', duration }) +} + +export class LoggedR2Bucket implements R2Bucket { + constructor (private readonly bucket: R2Bucket) {} + + async head (key: string): Promise { + return await measure('r2.head', () => this.bucket.head(key)) + } + + async get ( + key: string, + options?: R2GetOptions & { + onlyIf?: R2Conditional | Headers + } + ): Promise { + return await measure('r2.get', () => this.bucket.get(key, options)) + } + + async put ( + key: string, + value: ReadableStream | ArrayBuffer | ArrayBufferView | string | null | Blob, + options?: R2PutOptions & { + onlyIf?: R2Conditional | Headers + } + ): Promise { + return await measure('r2.put', () => this.bucket.put(key, value, options)) + } + + async createMultipartUpload (key: string, options?: R2MultipartOptions): Promise { + return await measure('r2.createMultipartUpload', () => this.bucket.createMultipartUpload(key, options)) + } + + resumeMultipartUpload (key: string, uploadId: string): R2MultipartUpload { + return measureSync('r2.resumeMultipartUpload', () => this.bucket.resumeMultipartUpload(key, uploadId)) + } + + async delete (keys: string | string[]): Promise { + await measure('r2.delete', () => this.bucket.delete(keys)) + } + + async list (options?: R2ListOptions): Promise { + return await measure('r2.list', () => this.bucket.list(options)) + } +} + +export class LoggedKVNamespace implements KVNamespace { + constructor (private readonly kv: KVNamespace) {} + + get (key: string, options?: Partial>): Promise + get (key: string, type: 'text'): Promise + get(key: string, type: 'json'): Promise + get (key: string, type: 'arrayBuffer'): Promise + get (key: string, type: 'stream'): Promise + get (key: string, options?: KVNamespaceGetOptions<'text'>): Promise + get(key: string, options?: KVNamespaceGetOptions<'json'>): Promise + get (key: string, options?: KVNamespaceGetOptions<'arrayBuffer'>): Promise + get (key: string, options?: KVNamespaceGetOptions<'stream'>): Promise + async get (key: string, options?: any): Promise { + return await measure('kv.get', () => this.kv.get(key, options)) + } + + getWithMetadata( + key: string, + options?: Partial> + ): Promise> + getWithMetadata( + key: string, + type: 'text' + ): Promise> + getWithMetadata( + key: string, + type: 'json' + ): Promise> + getWithMetadata( + key: string, + type: 'arrayBuffer' + ): Promise> + getWithMetadata( + key: string, + type: 'stream' + ): Promise> + getWithMetadata( + key: string, + options?: KVNamespaceGetOptions<'text'> + ): Promise> + getWithMetadata( + key: string, + options?: KVNamespaceGetOptions<'json'> + ): Promise> + getWithMetadata( + key: string, + options?: KVNamespaceGetOptions<'arrayBuffer'> + ): Promise> + getWithMetadata( + key: string, + options?: KVNamespaceGetOptions<'stream'> + ): Promise> + async getWithMetadata (key: string, options?: any): Promise { + return await measure('kv.getWithMetadata', () => this.kv.getWithMetadata(key, options)) + } + + async list(options?: KVNamespaceListOptions): Promise> { + return await measure('kv.list', () => this.kv.list(options)) + } + + async put ( + key: string, + value: string | ArrayBuffer | ArrayBufferView | ReadableStream, + options?: KVNamespacePutOptions + ): Promise { + await measure('kv.put', () => this.kv.put(key, value)) + } + + async delete (key: string): Promise { + await measure('kv.delete', () => this.kv.delete(key)) + } +} + +export class LoggedCache implements Cache { + constructor (private readonly cache: Cache) {} + + async match (request: RequestInfo, options?: CacheQueryOptions): Promise { + return await measure('cache.match', () => this.cache.match(request, options)) + } + + async delete (request: RequestInfo, options?: CacheQueryOptions): Promise { + return await measure('cache.delete', () => this.cache.delete(request, options)) + } + + async put (request: RequestInfo, response: Response): Promise { + await measure('cache.put', () => this.cache.put(request, response)) + } +} diff --git a/workers/datalake/src/multipart.ts b/workers/datalake/src/multipart.ts index c842bbe522..6b5558a816 100644 --- a/workers/datalake/src/multipart.ts +++ b/workers/datalake/src/multipart.ts @@ -14,8 +14,7 @@ // import { error, json } from 'itty-router' -import postgres from 'postgres' -import * as db from './db' +import db, { withPostgres } from './db' import { cacheControl } from './const' import { toUUID } from './encodings' import { selectStorage } from './storage' @@ -85,8 +84,6 @@ export async function handleMultipartUploadComplete ( env: Env, ctx: ExecutionContext ): Promise { - const sql = postgres(env.HYPERDRIVE.connectionString) - const { workspace, name } = request const multipartKey = request.query?.key @@ -108,17 +105,19 @@ export async function handleMultipartUploadComplete ( const size = object.size ?? 0 const filename = multipartKey as UUID - const data = await db.getData(sql, { hash, location }) - if (data !== null) { - // blob already exists - await Promise.all([bucket.delete(filename), db.createBlob(sql, { workspace, name, hash, location })]) - } else { - // Otherwise register a new hash and blob - await sql.begin((sql) => [ - db.createData(sql, { hash, location, filename, type, size }), - db.createBlob(sql, { workspace, name, hash, location }) - ]) - } + await withPostgres(env, ctx, async (sql) => { + const data = await db.getData(sql, { hash, location }) + if (data !== null) { + // blob already exists + await Promise.all([bucket.delete(filename), db.createBlob(sql, { workspace, name, hash, location })]) + } else { + // Otherwise register a new hash and blob + await sql.begin((sql) => [ + db.createData(sql, { hash, location, filename, type, size }), + db.createBlob(sql, { workspace, name, hash, location }) + ]) + } + }) return new Response(null, { status: 204 }) } diff --git a/workers/datalake/src/s3.ts b/workers/datalake/src/s3.ts index 0c8f4f46ac..062b7714e3 100644 --- a/workers/datalake/src/s3.ts +++ b/workers/datalake/src/s3.ts @@ -15,8 +15,7 @@ import { AwsClient } from 'aws4fetch' import { error, json } from 'itty-router' -import postgres from 'postgres' -import * as db from './db' +import db, { withPostgres } from './db' import { saveBlob } from './blob' import { type BlobRequest } from './types' @@ -38,34 +37,35 @@ function getS3Client (payload: S3UploadPayload): AwsClient { export async function handleS3Blob (request: BlobRequest, env: Env, ctx: ExecutionContext): Promise { const { workspace, name } = request - const sql = postgres(env.HYPERDRIVE.connectionString) const payload = await request.json() const client = getS3Client(payload) - // Ensure the blob does not exist - const blob = await db.getBlob(sql, { workspace, name }) - if (blob !== null) { - return new Response(null, { status: 200 }) - } + return await withPostgres(env, ctx, async (sql) => { + // Ensure the blob does not exist + const blob = await db.getBlob(sql, { workspace, name }) + if (blob !== null) { + return new Response(null, { status: 200 }) + } - const object = await client.fetch(payload.url) - if (!object.ok || object.status !== 200) { - return error(object.status) - } + const object = await client.fetch(payload.url) + if (!object.ok || object.status !== 200) { + return error(object.status) + } - if (object.body === null) { - return error(400) - } + if (object.body === null) { + return error(400) + } - const contentType = object.headers.get('content-type') ?? 'application/octet-stream' - const contentLengthHeader = object.headers.get('content-length') ?? '0' - const lastModifiedHeader = object.headers.get('last-modified') + const contentType = object.headers.get('content-type') ?? 'application/octet-stream' + const contentLengthHeader = object.headers.get('content-length') ?? '0' + const lastModifiedHeader = object.headers.get('last-modified') - const contentLength = Number.parseInt(contentLengthHeader) - const lastModified = lastModifiedHeader !== null ? new Date(lastModifiedHeader).getTime() : Date.now() + const contentLength = Number.parseInt(contentLengthHeader) + const lastModified = lastModifiedHeader !== null ? new Date(lastModifiedHeader).getTime() : Date.now() - const result = await saveBlob(env, sql, object.body, contentLength, contentType, workspace, name, lastModified) - return json(result) + const result = await saveBlob(env, sql, object.body, contentLength, contentType, workspace, name, lastModified) + return json(result) + }) } diff --git a/workers/datalake/src/sign.ts b/workers/datalake/src/sign.ts index 28f97f8f52..1ac7c6206c 100644 --- a/workers/datalake/src/sign.ts +++ b/workers/datalake/src/sign.ts @@ -96,7 +96,7 @@ export async function handleSignComplete (request: BlobRequest, env: Env, ctx: E } try { - await handleBlobUploaded(env, workspace, name, uuid) + await handleBlobUploaded(env, ctx, workspace, name, uuid) } catch (err) { const message = err instanceof Error ? err.message : String(err) console.error({ error: message, workspace, name, uuid }) diff --git a/workers/datalake/wrangler.toml b/workers/datalake/wrangler.toml index f578e1f574..0b6989c4b0 100644 --- a/workers/datalake/wrangler.toml +++ b/workers/datalake/wrangler.toml @@ -1,7 +1,7 @@ #:schema node_modules/wrangler/config-schema.json name = "datalake-worker" main = "src/index.ts" -compatibility_date = "2024-07-01" +compatibility_date = "2024-09-23" compatibility_flags = ["nodejs_compat"] keep_vars = true