Merge remote-tracking branch 'origin/develop' into staging

Signed-off-by: Andrey Sobolev <haiodo@gmail.com>
This commit is contained in:
Andrey Sobolev 2024-12-10 12:03:05 +07:00
commit 227ec717e6
No known key found for this signature in database
GPG Key ID: BD80F68D68D8F7F2
45 changed files with 597 additions and 258 deletions

View File

@ -58,7 +58,8 @@ import {
TypeDate,
TypeRef,
TypeString,
UX
UX,
TypeBoolean
} from '@hcengineering/model'
import calendar, { TEvent } from '@hcengineering/model-calendar'
import core, { TAttachedDoc, TDoc } from '@hcengineering/model-core'
@ -106,9 +107,14 @@ export class TRoom extends TDoc implements Room {
x!: number
y!: number
language!: RoomLanguage
startWithTranscription!: boolean
startWithRecording!: boolean
@Prop(TypeString(), love.string.Language, { editor: love.component.RoomLanguageEditor })
language!: RoomLanguage
@Prop(TypeBoolean(), love.string.StartWithTranscription)
startWithTranscription!: boolean
@Prop(TypeBoolean(), love.string.StartWithRecording)
startWithRecording!: boolean
@Prop(Collection(attachment.class.Attachment), attachment.string.Attachments, { shortLabel: attachment.string.Files })
attachments?: number

View File

@ -28,7 +28,8 @@ export default mergeIds(loveId, love, {
Settings: '' as AnyComponent,
LoveWidget: '' as AnyComponent,
MeetingWidget: '' as AnyComponent,
WidgetSwitcher: '' as AnyComponent
WidgetSwitcher: '' as AnyComponent,
RoomLanguageEditor: '' as AnyComponent
},
app: {
Love: '' as Ref<Doc>

View File

@ -63,6 +63,7 @@
{#each createMessages as valueMessage, index}
<DocUpdateMessageObjectValue
attachedTo={valueMessage.attachedTo}
objectClass={valueMessage.objectClass}
objectId={valueMessage.objectId}
action={valueMessage.action}
@ -74,6 +75,7 @@
{/each}
{#each removeMessages as valueMessage, index}
<DocUpdateMessageObjectValue
attachedTo={valueMessage.attachedTo}
objectClass={valueMessage.objectClass}
objectId={valueMessage.objectId}
action={valueMessage.action}
@ -87,6 +89,7 @@
{@const len = valueMessages.length}
{#each valueMessages as valueMessage, index}
<DocUpdateMessageObjectValue
attachedTo={valueMessage.attachedTo}
objectClass={valueMessage.objectClass}
objectId={valueMessage.objectId}
action={valueMessage.action}

View File

@ -26,6 +26,7 @@
isAttachedDoc
} from '@hcengineering/view-resources'
export let attachedTo: DisplayDocUpdateMessage['attachedTo']
export let objectClass: DisplayDocUpdateMessage['objectClass']
export let objectId: DisplayDocUpdateMessage['objectId']
export let action: DisplayDocUpdateMessage['action']
@ -51,8 +52,8 @@
return await getDocLinkTitle(client, object._id, object._class, object)
}
async function loadObject (_id: Ref<Doc>, _class: Ref<Class<Doc>>): Promise<void> {
const isRemoved = await checkIsObjectRemoved(client, _id, _class)
async function loadObject (_id: Ref<Doc>, _class: Ref<Class<Doc>>, attachedTo: Ref<Doc>): Promise<void> {
const isRemoved = attachedTo === _id ? false : await checkIsObjectRemoved(client, _id, _class)
if (isRemoved) {
object = await buildRemovedDoc(client, _id, _class)
@ -64,7 +65,7 @@
}
}
$: void loadObject(objectId, objectClass)
$: void loadObject(objectId, objectClass, attachedTo)
function getPanelComponent (object: Doc, objectPanel?: ObjectPanel): AnyComponent {
if (objectPanel !== undefined) {

View File

@ -148,7 +148,7 @@
object = detail.object
}
openChannel(selectedData.id, selectedData._class)
openChannel(selectedData.id, selectedData._class, undefined, true)
}
defineSeparators('chat', [

View File

@ -83,7 +83,7 @@
group: 'edit',
action: async () => {
const id = await getObjectLinkId(linkProviders, object._id, object._class, object)
openChannel(id, object._class)
openChannel(id, object._class, undefined, true)
}
})

View File

@ -77,5 +77,6 @@
onExtensions={handleExtensions}
on:update
on:open-document
on:loaded
bind:this={collabEditor}
/>

View File

@ -15,7 +15,6 @@
//
-->
<script lang="ts">
import activity from '@hcengineering/activity'
import attachment, { Attachment } from '@hcengineering/attachment'
import core, { Doc, Ref, WithLookup, generateId, type Blob } from '@hcengineering/core'
import { Document, DocumentEvents } from '@hcengineering/document'
@ -59,7 +58,6 @@
import DocumentEditor from './DocumentEditor.svelte'
import DocumentPresenter from './DocumentPresenter.svelte'
import DocumentTitle from './DocumentTitle.svelte'
import Activity from './sidebar/Activity.svelte'
import History from './sidebar/History.svelte'
import References from './sidebar/References.svelte'
@ -88,11 +86,14 @@
let headings: Heading[] = []
let loadedDocumentContent = false
const notificationClient = getResource(notification.function.GetInboxNotificationsClient).then((res) => res())
$: read(_id)
function read (_id: Ref<Doc>): void {
if (lastId !== _id) {
loadedDocumentContent = false
const prev = lastId
lastId = _id
void notificationClient.then((client) => client.readDoc(prev))
@ -200,11 +201,6 @@
id: 'references',
icon: document.icon.References,
showTooltip: { label: document.string.Backlinks, direction: 'bottom' }
},
{
id: 'activity',
icon: activity.icon.Activity,
showTooltip: { label: activity.string.Activity, direction: 'bottom' }
}
]
let selectedAside: string | boolean = false
@ -248,8 +244,8 @@
{#if doc !== undefined}
<Panel
withoutActivity={!loadedDocumentContent}
object={doc}
withoutActivity
allowClose={!embedded}
isAside={true}
customAside={aside}
@ -367,7 +363,7 @@
{readonly}
boundary={content}
overflow={'none'}
editorAttributes={{ style: 'padding: 0 2em 30vh; margin: 0 -2em;' }}
editorAttributes={{ style: 'padding: 0 2em 2em; margin: 0 -2em; min-height: 30vh' }}
attachFile={async (file) => {
return await createEmbedding(file)
}}
@ -381,6 +377,9 @@
navigate(location)
}
}}
on:loaded={() => {
loadedDocumentContent = true
}}
bind:this={editor}
/>
{/key}
@ -390,8 +389,6 @@
<svelte:fragment slot="aside">
{#if selectedAside === 'references'}
<References doc={doc._id} />
{:else if selectedAside === 'activity'}
<Activity value={doc} />
{:else if selectedAside === 'history'}
<History value={doc} {readonly} />
{/if}

View File

@ -115,7 +115,6 @@
let currentSpace: Ref<Space> | undefined
let currentSpecial: string | undefined
let specialComponent: SpecialNavModel | undefined
let asideId: string | undefined
let currentFragment: string | undefined = ''
let currentApplication: Application | undefined
@ -124,13 +123,10 @@
function setSpaceSpecial (spaceSpecial: string | undefined): void {
if (currentSpecial !== undefined && spaceSpecial === currentSpecial) return
if (asideId !== undefined && spaceSpecial === asideId) return
if (spaceSpecial === undefined) return
specialComponent = getSpecialComponent(spaceSpecial)
if (specialComponent !== undefined) {
currentSpecial = spaceSpecial
} else if (navigatorModel?.aside !== undefined || currentApplication?.aside !== undefined) {
asideId = spaceSpecial
}
}
@ -175,10 +171,6 @@
}
}
if (special !== currentSpecial && (navigatorModel?.aside || currentApplication?.aside)) {
asideId = special
}
if (fragment !== currentFragment) {
currentFragment = fragment
if (fragment != null && fragment.trim().length > 0) {
@ -250,13 +242,6 @@
}
}
function closeAside (): void {
const loc = getLocation()
loc.path.length = 4
asideId = undefined
navigate(loc)
}
async function getWindowTitle (loc: Location): Promise<string | undefined> {
if (loc.fragment == null) return
const hierarchy = client.getHierarchy()
@ -277,7 +262,6 @@
defineSeparators('workbenchGuest', workbenchGuestSeparators)
let aside: HTMLElement
let cover: HTMLElement
</script>
@ -303,15 +287,6 @@
<SpaceView {currentSpace} {currentView} />
{/if}
</div>
{#if asideId}
{@const asideComponent = navigatorModel?.aside ?? currentApplication?.aside}
{#if asideComponent !== undefined}
<Separator name={'workbenchGuest'} index={0} />
<div class="antiPanel-component antiComponent aside" bind:this={aside}>
<Component is={asideComponent} props={{ currentSpace, _id: asideId }} on:close={closeAside} />
</div>
{/if}
{/if}
</div>
</div>
<div bind:this={cover} class="cover" />

View File

@ -78,6 +78,7 @@
"Status": "Stav",
"Active": "Aktivní",
"Finished": "Dokončeno",
"StartWithRecording": "Začít s nahráváním"
"StartWithRecording": "Začít s nahráváním",
"Language": "Jazyk"
}
}

View File

@ -78,6 +78,7 @@
"Status": "Status",
"Active": "Active",
"Finished": "Finished",
"StartWithRecording": "Start with recording"
"StartWithRecording": "Start with recording",
"Language": "Language"
}
}

View File

@ -78,6 +78,7 @@
"Status": "Estado",
"Active": "Activo",
"Finished": "Terminado",
"StartWithRecording": "Iniciar con grabación"
"StartWithRecording": "Iniciar con grabación",
"Language": "Idioma"
}
}

View File

@ -78,6 +78,7 @@
"Status": "Statut",
"Active": "Actif",
"Finished": "Terminé",
"StartWithRecording": "Démarrer avec l'enregistrement"
"StartWithRecording": "Démarrer avec l'enregistrement",
"Language": "Langue"
}
}

View File

@ -78,6 +78,7 @@
"Status": "Stato",
"Active": "Attivo",
"Finished": "Finito",
"StartWithRecording": "Inizia con la registrazione"
"StartWithRecording": "Inizia con la registrazione",
"Language": "Lingua"
}
}

View File

@ -78,6 +78,7 @@
"Status": "Estado",
"Active": "Ativo",
"Finished": "Finalizado",
"StartWithRecording": "Começar com gravação"
"StartWithRecording": "Começar com gravação",
"Language": "Idioma"
}
}

View File

@ -78,6 +78,7 @@
"Status": "Статус",
"Active": "Активно",
"Finished": "Завершено",
"StartWithRecording": "Начинать с записью"
"StartWithRecording": "Начинать с записью",
"Language": "Язык"
}
}

View File

@ -78,6 +78,7 @@
"Status": "状态",
"Active": "活动",
"Finished": "已完成",
"StartWithRecording": "开始录制"
"StartWithRecording": "开始录制",
"Language": "语言"
}
}

View File

@ -29,7 +29,7 @@
{#if inline}
<ObjectMention object={floor} />
{:else}
<div class="flex-presenter overflow-label sm-tool-icon">
<div class="flex-presenter overflow-label sm-tool-icon ml-3">
{floor.name}
</div>
{/if}

View File

@ -18,10 +18,14 @@
import { languagesDisplayData } from '../types'
export let room: Room
export let withLabel = false
$: lang = room.language
</script>
<span title={languagesDisplayData[lang].label ?? languagesDisplayData.en.label}>
{languagesDisplayData[lang].emoji ?? languagesDisplayData.en.emoji}
{#if withLabel}
<span class="ml-1">{languagesDisplayData[lang].label ?? languagesDisplayData.en.label}</span>
{/if}
</span>

View File

@ -0,0 +1,67 @@
<script lang="ts">
import { Room, RoomLanguage } from '@hcengineering/love'
import { getEmbeddedLabel } from '@hcengineering/platform'
import RoomLanguageComponent from './RoomLanguage.svelte'
import {
Button,
type ButtonKind,
type ButtonSize,
DropdownIntlItem,
DropdownLabelsPopupIntl,
eventToHTMLElement,
showPopup
} from '@hcengineering/ui'
import { languagesDisplayData } from '../types'
import LanguageIcon from './LanguageIcon.svelte'
export let value: RoomLanguage = 'en'
export let object: Room | undefined = undefined
export let onChange: (value: any) => void
export let disabled: boolean = false
export let kind: ButtonKind = 'no-border'
export let size: ButtonSize = 'small'
export let justify: 'left' | 'center' = 'center'
export let width: string | undefined = 'fit-content'
let shown: boolean = false
let items: DropdownIntlItem[] = []
$: items = Object.entries(languagesDisplayData).map(([lang, data]) => ({
id: lang,
label: getEmbeddedLabel(data.label),
icon: LanguageIcon,
iconProps: { lang }
}))
function showLanguagesPopup (ev: MouseEvent): void {
shown = true
showPopup(DropdownLabelsPopupIntl, { items, selected: value }, eventToHTMLElement(ev), async (result) => {
if (result != null && result !== '') {
value = result
onChange(value)
shown = false
}
})
}
</script>
{#if object}
<Button
{kind}
{size}
{justify}
{width}
{disabled}
on:click={(ev) => {
if (!shown && !disabled) {
showLanguagesPopup(ev)
}
}}
>
<svelte:fragment slot="content">
<div class="pointer-events-none"><RoomLanguageComponent room={object} withLabel /></div>
</svelte:fragment>
</Button>
{/if}

View File

@ -23,6 +23,7 @@ import PanelControlBar from './components/PanelControlBar.svelte'
import RoomPresenter from './components/RoomPresenter.svelte'
import MeetingMinutesDocEditor from './components/MeetingMinutesDocEditor.svelte'
import MeetingMinutesStatusPresenter from './components/MeetingMinutesStatusPresenter.svelte'
import RoomLanguageEditor from './components/RoomLanguageEditor.svelte'
import {
copyGuestLink,
@ -59,7 +60,8 @@ export default async (): Promise<Resources> => ({
PanelControlBar,
RoomPresenter,
MeetingMinutesDocEditor,
MeetingMinutesStatusPresenter
MeetingMinutesStatusPresenter,
RoomLanguageEditor
},
function: {
CreateMeeting: createMeeting,

View File

@ -94,6 +94,7 @@ export default mergeIds(loveId, love, {
KnockAction: '' as IntlString,
Select: '' as IntlString,
ChooseShare: '' as IntlString,
MoreOptions: '' as IntlString
MoreOptions: '' as IntlString,
Language: '' as IntlString
}
})

View File

@ -140,6 +140,10 @@
void localProvider.loaded.then(() => (localSynced = true))
void remoteProvider.loaded.then(() => (remoteSynced = true))
void Promise.all([localProvider.loaded, remoteProvider.loaded]).then(() => {
dispatch('loaded')
})
let editor: Editor
let element: HTMLElement
let textToolbarElement: HTMLElement

View File

@ -103,6 +103,7 @@
on:update
on:open-document
on:blur
on:loaded
on:focus={handleFocus}
/>
</div>

View File

@ -30,7 +30,8 @@
import { ImageUploadExtension } from './extension/imageUploadExt'
import { InlineCommandsExtension } from './extension/inlineCommands'
import { type FileAttachFunction } from './extension/types'
import { completionConfig, inlineCommandsConfig } from './extensions'
import { completionConfig, InlineCommandId, inlineCommandsConfig } from './extensions'
import { MermaidExtension, mermaidOptions } from './extension/mermaid'
export let label: IntlString | undefined = undefined
export let content: Markup
@ -192,6 +193,7 @@
}
extensions.push(
imageUploadPlugin,
MermaidExtension.configure(mermaidOptions),
FocusExtension.configure({ onCanBlur: (value: boolean) => (canBlur = value), onFocus: handleFocus })
)
if (enableEmojiReplace) {
@ -199,10 +201,12 @@
}
if (enableInlineCommands) {
const excludedInlineCommands: InlineCommandId[] = ['drawing-board', 'todo-list']
if (attachFile === undefined) excludedInlineCommands.push('image')
extensions.push(
InlineCommandsExtension.configure(
inlineCommandsConfig(handleCommandSelected, attachFile === undefined ? ['image'] : [])
)
InlineCommandsExtension.configure(inlineCommandsConfig(handleCommandSelected, excludedInlineCommands))
)
}
@ -226,11 +230,13 @@
}
case 'code-block':
editor.editorHandler.insertCodeBlock(pos)
break
case 'separator-line':
editor.editorHandler.insertSeparatorLine()
break
case 'mermaid':
editor.getEditor()?.commands.insertContentAt(pos, { type: 'mermaid' })
break
}
}

View File

@ -17,10 +17,9 @@
import { IntlString } from '@hcengineering/platform'
import { EmptyMarkup } from '@hcengineering/text'
import { Button, type ButtonSize, Scroller } from '@hcengineering/ui'
import { AnyExtension, mergeAttributes } from '@tiptap/core'
import { AnyExtension, mergeAttributes, type Editor } from '@tiptap/core'
import { createEventDispatcher } from 'svelte'
import textEditor, { RefAction, TextEditorHandler, TextFormatCategory } from '@hcengineering/text-editor'
import { defaultRefActions, getModelRefActions } from './editor/actions'
import TextEditor from './TextEditor.svelte'
@ -63,6 +62,9 @@
export function insertText (text: string): void {
editor?.insertText(text)
}
export function getEditor (): Editor | undefined {
return editor?.getEditor()
}
$: varsStyle =
maxHeight === 'card'

View File

@ -90,6 +90,9 @@
editor.commands.clearContent(true)
}
}
export function getEditor (): Editor {
return editor
}
export function insertText (text: string): void {
editor?.commands.insertContent(text)

View File

@ -130,7 +130,15 @@ export const completionConfig: Partial<CompletionOptions> = {
}
}
const inlineCommandsIds = ['image', 'table', 'code-block', 'separator-line', 'todo-list'] as const
const inlineCommandsIds = [
'image',
'table',
'code-block',
'separator-line',
'todo-list',
'drawing-board',
'mermaid'
] as const
export type InlineCommandId = (typeof inlineCommandsIds)[number]
/**

View File

@ -229,7 +229,7 @@ async function move (issues: Issue | Issue[]): Promise<void> {
async function editWorkflowStatuses (project: Project): Promise<void> {
const loc = getCurrentLocation()
loc.path[2] = settingId
loc.path[3] = 'statuses'
loc.path[3] = 'spaceTypes'
loc.path[4] = project.type
navigate(loc)
}

View File

@ -13,7 +13,7 @@
// limitations under the License.
-->
<script lang="ts">
import { AttachedDoc, Doc } from '@hcengineering/core'
import { AttachedDoc, Doc, Ref, Class } from '@hcengineering/core'
import { getClient } from '@hcengineering/presentation'
import { isAttachedDoc } from '../utils'
import DocsNavigator from './DocsNavigator.svelte'
@ -26,19 +26,31 @@
return 'parent' in doc && doc.parent != null
}
async function getParents (doc: Doc | AttachedDoc): Promise<readonly Doc[]> {
if (!isAttachedDoc(doc) && !hasParent(doc)) {
function getParentId (doc: Doc | AttachedDoc): Ref<Doc> {
return isAttachedDoc(doc) ? doc.attachedTo : (doc as any).parent
}
function getParentClass (doc: Doc | AttachedDoc): Ref<Class<Doc>> {
return isAttachedDoc(doc) ? doc.attachedToClass : doc._class
}
function withParent (doc: Doc | AttachedDoc): boolean {
return isAttachedDoc(doc) || hasParent(doc)
}
async function getParents (_id: Ref<Doc>, _class: Ref<Class<Doc>>, showParents: boolean): Promise<readonly Doc[]> {
if (!showParents) {
return []
}
const parents: Doc[] = []
let currentDoc: Doc | undefined = doc
let currentDoc: Doc | undefined = element
while (currentDoc && (isAttachedDoc(currentDoc) || hasParent(currentDoc))) {
const parent: Doc | undefined = isAttachedDoc(currentDoc)
? await client.findOne(currentDoc.attachedToClass, { _id: currentDoc.attachedTo })
: await client.findOne(currentDoc._class, { _id: (currentDoc as any).parent })
while (currentDoc && withParent(currentDoc)) {
const _id = getParentId(currentDoc)
const _class = getParentClass(currentDoc)
const parent: Doc | undefined = await client.findOne(_class, { _id })
if (parent) {
currentDoc = parent
@ -50,8 +62,15 @@
return parents.reverse()
}
let parents: readonly Doc[] = []
$: parentId = getParentId(element)
$: parentClass = getParentClass(element)
$: showParents = withParent(element)
$: getParents(parentId, parentClass, showParents).then((res) => {
parents = res
})
</script>
{#await getParents(element) then parents}
<DocsNavigator elements={parents} />
{/await}
<DocsNavigator elements={parents} />

View File

@ -752,9 +752,18 @@ export function categorizeFields (
export function makeViewletKey (loc?: Location): string {
loc = loc != null ? { path: loc.path } : getCurrentResolvedLocation()
loc.fragment = undefined
loc.query = undefined
if (loc.fragment != null && loc.fragment !== '') {
const props = decodeURIComponent(loc.fragment).split('|')
if (props.length >= 3) {
const [panel, , _class] = props
return 'viewlet' + '#' + encodeURIComponent([panel, _class].join('|'))
}
}
loc.fragment = undefined
return 'viewlet' + locationToUrl(loc)
}

View File

@ -136,7 +136,6 @@
let currentSpecial: string | undefined
let currentQuery: Record<string, string | null> | undefined
let specialComponent: SpecialNavModel | undefined
let asideId: string | undefined
let currentFragment: string | undefined = ''
let currentApplication: Application | undefined
@ -374,8 +373,6 @@
loc.path[4] = currentSpecial
} else if (loc.path[3] === resolved.defaultLocation.path[3]) {
loc.path[4] = resolved.defaultLocation.path[4]
} else {
loc.path[4] = asideId as string
}
} else {
loc.path.length = 4
@ -512,10 +509,6 @@
}
}
if (special !== currentSpecial && (navigatorModel?.aside || currentApplication?.aside)) {
asideId = special
}
if (app !== undefined) {
localStorage.setItem(`${locationStorageKeyId}_${app}`, originalLoc)
}
@ -589,21 +582,12 @@
specialComponent = undefined
// eslint-disable-next-line no-fallthrough
case 3:
asideId = undefined
if (currentSpace !== undefined) {
specialComponent = undefined
}
}
}
function closeAside (): void {
const loc = getLocation()
loc.path.length = 4
asideId = undefined
checkOnHide()
navigate(loc)
}
async function updateSpace (spaceId?: Ref<Space>): Promise<void> {
if (spaceId === currentSpace) return
clear(2)
@ -620,14 +604,11 @@
function setSpaceSpecial (spaceSpecial: string | undefined): void {
if (currentSpecial !== undefined && spaceSpecial === currentSpecial) return
if (asideId !== undefined && spaceSpecial === asideId) return
clear(3)
if (spaceSpecial === undefined) return
specialComponent = getSpecialComponent(spaceSpecial)
if (specialComponent !== undefined) {
currentSpecial = spaceSpecial
} else if (navigatorModel?.aside !== undefined || currentApplication?.aside !== undefined) {
asideId = spaceSpecial
}
}
@ -647,7 +628,6 @@
}
}
let aside: HTMLElement
let cover: HTMLElement
let workbenchWidth: number = $deviceInfo.docWidth
@ -762,14 +742,6 @@
person && client.getHierarchy().hasMixin(person, contact.mixin.Employee)
? !client.getHierarchy().as(person, contact.mixin.Employee).active
: false
let asideComponent: AnyComponent | undefined
$: if (asideId !== undefined && navigatorModel !== undefined) {
asideComponent = navigatorModel?.aside ?? currentApplication?.aside
} else {
asideComponent = undefined
}
</script>
{#if person && deactivated && !isAdminUser()}
@ -1001,11 +973,8 @@
<Component
is={currentApplication.component}
props={{
currentSpace,
asideId,
asideComponent: currentApplication?.aside
currentSpace
}}
on:close={closeAside}
/>
{:else if specialComponent}
<Component
@ -1035,12 +1004,6 @@
<SpaceView {currentSpace} {currentView} {createItemDialog} {createItemLabel} />
{/if}
</div>
{#if asideComponent !== undefined}
<Separator name={'workbench'} index={1} color={'transparent'} separatorSize={0} short />
<div class="antiPanel-component antiComponent aside" bind:this={aside}>
<Component is={asideComponent} props={{ currentSpace, _id: asideId }} on:close={closeAside} />
</div>
{/if}
</div>
{#if !$deviceInfo.aside.float}
{#if $sidebarStore.variant === SidebarVariant.EXPANDED}

View File

@ -192,8 +192,7 @@ export async function buildNavModel (
const newSpaces = (nm.spaces ?? []).filter((it) => !spaces.some((sp) => sp.id === it.id))
newNavModel = {
spaces: [...spaces, ...newSpaces],
specials: [...(newNavModel?.specials ?? []), ...(nm.specials ?? [])],
aside: newNavModel?.aside ?? nm?.aside
specials: [...(newNavModel?.specials ?? []), ...(nm.specials ?? [])]
}
}
}

View File

@ -50,7 +50,6 @@ export interface Application extends Doc {
// Also attached ApplicationNavModel will be joined after this one main.
navigatorModel?: NavigatorModel
aside?: AnyComponent
locationResolver?: Resource<(loc: Location) => Promise<ResolvedLocation | undefined>>
locationDataResolver?: Resource<(loc: Location) => Promise<LocationData>>
@ -130,7 +129,6 @@ export interface ApplicationNavModel extends Doc {
spaces?: SpacesNavModel[]
specials?: SpecialNavModel[]
aside?: AnyComponent
}
/**
@ -163,7 +161,6 @@ export interface SpacesNavModel {
export interface NavigatorModel {
spaces: SpacesNavModel[]
specials?: SpecialNavModel[]
aside?: AnyComponent
}
/**

View File

@ -62,6 +62,9 @@ export interface ServerFindOptions<T extends Doc> extends FindOptions<T> {
domain: Domain
}
// using for join query security
allowedSpaces?: Ref<Space>[]
// Optional measure context, for server side operations
ctx?: MeasureContext
}

View File

@ -22,7 +22,7 @@ import {
type MeasureContext,
Ref
} from '@hcengineering/core'
import { BaseMiddleware, Middleware, type PipelineContext } from '@hcengineering/server-core'
import { BaseMiddleware, Middleware, ServerFindOptions, type PipelineContext } from '@hcengineering/server-core'
import { deepEqual } from 'fast-equals'
interface Query {
@ -45,7 +45,7 @@ export class QueryJoiner {
ctx: MeasureContext,
_class: Ref<Class<T>>,
query: DocumentQuery<T>,
options?: FindOptions<T>
options?: ServerFindOptions<T>
): Promise<FindResult<T>> {
// Will find a query or add + 1 to callbacks
const q = this.findQuery(_class, query, options) ?? this.createQuery(_class, query, options)
@ -131,7 +131,7 @@ export class QueryJoinMiddleware extends BaseMiddleware implements Middleware {
ctx: MeasureContext,
_class: Ref<Class<T>>,
query: DocumentQuery<T>,
options?: FindOptions<T>
options?: ServerFindOptions<T>
): Promise<FindResult<T>> {
// Will find a query or add + 1 to callbacks
return this.joiner.findAll(ctx, _class, query, options)

View File

@ -21,7 +21,6 @@ import core, {
Doc,
DocumentQuery,
Domain,
FindOptions,
FindResult,
LookupData,
MeasureContext,
@ -47,7 +46,13 @@ import core, {
type SessionData
} from '@hcengineering/core'
import platform, { PlatformError, Severity, Status } from '@hcengineering/platform'
import { BaseMiddleware, Middleware, TxMiddlewareResult, type PipelineContext } from '@hcengineering/server-core'
import {
BaseMiddleware,
Middleware,
ServerFindOptions,
TxMiddlewareResult,
type PipelineContext
} from '@hcengineering/server-core'
import { isOwner, isSystem } from './utils'
type SpaceWithMembers = Pick<Space, '_id' | 'members' | 'private' | '_class'>
@ -416,6 +421,14 @@ export class SpaceSecurityMiddleware extends BaseMiddleware implements Middlewar
}
}
ctx.contextData.broadcast.targets.spaceSec = (tx) => {
const space = this.spacesMap.get(tx.objectSpace)
if (space === undefined) return undefined
if (this.systemSpaces.has(space._id) || this.mainSpaces.includes(space._id)) return undefined
return space.members.length === 0 ? undefined : this.getTargets(space?.members)
}
await this.next?.handleBroadcast(ctx)
}
@ -497,7 +510,7 @@ export class SpaceSecurityMiddleware extends BaseMiddleware implements Middlewar
ctx: MeasureContext<SessionData>,
_class: Ref<Class<T>>,
query: DocumentQuery<T>,
options?: FindOptions<T>
options?: ServerFindOptions<T>
): Promise<FindResult<T>> {
await this.init(ctx)
@ -509,12 +522,7 @@ export class SpaceSecurityMiddleware extends BaseMiddleware implements Middlewar
let clientFilterSpaces: Set<Ref<Space>> | undefined
if (
!this.skipFindCheck &&
!isSystem(account, ctx) &&
account.role !== AccountRole.DocGuest &&
domain !== DOMAIN_MODEL
) {
if (!isSystem(account, ctx) && account.role !== AccountRole.DocGuest && domain !== DOMAIN_MODEL) {
if (!isOwner(account, ctx) || !isSpace) {
if (query[field] !== undefined) {
const res = await this.mergeQuery(ctx, account, query[field], domain, isSpace)
@ -536,6 +544,11 @@ export class SpaceSecurityMiddleware extends BaseMiddleware implements Middlewar
delete (newQuery as any)[field]
} else if (spaces.result.length === 1) {
;(newQuery as any)[field] = spaces.result[0]
if (options !== undefined) {
options.allowedSpaces = spaces.result
} else {
options = { allowedSpaces: spaces.result }
}
} else {
// Check if spaces > 85% of all domain spaces, in this case return all and filter on client.
if (spaces.result.length / spaces.domainSpaces.size > 0.85 && options?.limit === undefined) {
@ -543,17 +556,22 @@ export class SpaceSecurityMiddleware extends BaseMiddleware implements Middlewar
delete newQuery.space
} else {
;(newQuery as any)[field] = { $in: spaces.result }
if (options !== undefined) {
options.allowedSpaces = spaces.result
} else {
options = { allowedSpaces: spaces.result }
}
}
}
}
}
}
let findResult = await this.provideFindAll(ctx, _class, newQuery, options)
let findResult = await this.provideFindAll(ctx, _class, !this.skipFindCheck ? newQuery : query, options)
if (clientFilterSpaces !== undefined) {
const cfs = clientFilterSpaces
findResult = toFindResult(
findResult.filter((it) => cfs.has(it.space)),
findResult.filter((it) => cfs.has((it as any)[field])),
findResult.total,
findResult.lookupMap
)

View File

@ -14,14 +14,15 @@
//
import { error, json } from 'itty-router'
import postgres from 'postgres'
import * as db from './db'
import { type Sql } from 'postgres'
import db, { withPostgres } from './db'
import { cacheControl, hashLimit } from './const'
import { toUUID } from './encodings'
import { getSha256 } from './hash'
import { selectStorage } from './storage'
import { type BlobRequest, type WorkspaceRequest, type UUID } from './types'
import { copyVideo, deleteVideo } from './video'
import { measure, LoggedCache } from './measure'
interface BlobMetadata {
lastModified: number
@ -38,20 +39,22 @@ export function getBlobURL (request: Request, workspace: string, name: string):
export async function handleBlobGet (request: BlobRequest, env: Env, ctx: ExecutionContext): Promise<Response> {
const { workspace, name } = request
const sql = postgres(env.HYPERDRIVE.connectionString)
const { bucket } = selectStorage(env, workspace)
const blob = await db.getBlob(sql, { workspace, name })
if (blob === null || blob.deleted) {
return error(404)
}
const cache = caches.default
const cache = new LoggedCache(caches.default)
const cached = await cache.match(request)
if (cached !== undefined) {
console.log({ message: 'cache hit' })
return cached
}
const { bucket } = selectStorage(env, workspace)
const blob = await withPostgres(env, ctx, (sql) => {
return db.getBlob(sql, { workspace, name })
})
if (blob === null || blob.deleted) {
return error(404)
}
const range = request.headers.has('Range') ? request.headers : undefined
const object = await bucket.get(blob.filename, { range })
if (object === null) {
@ -67,6 +70,7 @@ export async function handleBlobGet (request: BlobRequest, env: Env, ctx: Execut
const status = length !== undefined && length < object.size ? 206 : 200
const response = new Response(object?.body, { headers, status })
if (response.status === 200) {
ctx.waitUntil(cache.put(request, response.clone()))
}
@ -77,10 +81,11 @@ export async function handleBlobGet (request: BlobRequest, env: Env, ctx: Execut
export async function handleBlobHead (request: BlobRequest, env: Env, ctx: ExecutionContext): Promise<Response> {
const { workspace, name } = request
const sql = postgres(env.HYPERDRIVE.connectionString)
const { bucket } = selectStorage(env, workspace)
const blob = await db.getBlob(sql, { workspace, name })
const blob = await withPostgres(env, ctx, (sql) => {
return db.getBlob(sql, { workspace, name })
})
if (blob === null || blob.deleted) {
return error(404)
}
@ -97,10 +102,10 @@ export async function handleBlobHead (request: BlobRequest, env: Env, ctx: Execu
export async function handleBlobDelete (request: BlobRequest, env: Env, ctx: ExecutionContext): Promise<Response> {
const { workspace, name } = request
const sql = postgres(env.HYPERDRIVE.connectionString)
try {
await Promise.all([db.deleteBlob(sql, { workspace, name }), deleteVideo(env, workspace, name)])
await withPostgres(env, ctx, (sql) => {
return Promise.all([db.deleteBlob(sql, { workspace, name }), deleteVideo(env, workspace, name)])
})
return new Response(null, { status: 204 })
} catch (err: any) {
@ -110,7 +115,11 @@ export async function handleBlobDelete (request: BlobRequest, env: Env, ctx: Exe
}
}
export async function handleUploadFormData (request: WorkspaceRequest, env: Env): Promise<Response> {
export async function handleUploadFormData (
request: WorkspaceRequest,
env: Env,
ctx: ExecutionContext
): Promise<Response> {
const contentType = request.headers.get('Content-Type')
if (contentType === null || !contentType.includes('multipart/form-data')) {
console.error({ error: 'expected multipart/form-data' })
@ -119,11 +128,9 @@ export async function handleUploadFormData (request: WorkspaceRequest, env: Env)
const { workspace } = request
const sql = postgres(env.HYPERDRIVE.connectionString)
let formData: FormData
try {
formData = await request.formData()
formData = await measure('fetch formdata', () => request.formData())
} catch (err: any) {
const message = err instanceof Error ? err.message : String(err)
console.error({ error: 'failed to parse form data', message })
@ -139,7 +146,9 @@ export async function handleUploadFormData (request: WorkspaceRequest, env: Env)
files.map(async ([file, key]) => {
const { name, type, lastModified } = file
try {
const metadata = await saveBlob(env, sql, file.stream(), file.size, type, workspace, name, lastModified)
const metadata = await withPostgres(env, ctx, (sql) => {
return saveBlob(env, sql, file.stream(), file.size, type, workspace, name, lastModified)
})
// TODO this probably should happen via queue, let it be here for now
if (type.startsWith('video/')) {
@ -161,7 +170,7 @@ export async function handleUploadFormData (request: WorkspaceRequest, env: Env)
export async function saveBlob (
env: Env,
sql: postgres.Sql,
sql: Sql,
stream: ReadableStream,
size: number,
type: string,
@ -179,6 +188,7 @@ export async function saveBlob (
const hash = await getSha256(hashStream)
const data = await db.getData(sql, { hash, location })
if (data !== null) {
// Lucky boy, nothing to upload, use existing blob
await db.createBlob(sql, { workspace, name, hash, location })
@ -212,8 +222,13 @@ export async function saveBlob (
}
}
export async function handleBlobUploaded (env: Env, workspace: string, name: string, filename: UUID): Promise<void> {
const sql = postgres(env.HYPERDRIVE.connectionString)
export async function handleBlobUploaded (
env: Env,
ctx: ExecutionContext,
workspace: string,
name: string,
filename: UUID
): Promise<void> {
const { location, bucket } = selectStorage(env, workspace)
const object = await bucket.head(filename)
@ -223,16 +238,20 @@ export async function handleBlobUploaded (env: Env, workspace: string, name: str
const hash = object.checksums.md5 !== undefined ? digestToUUID(object.checksums.md5) : (crypto.randomUUID() as UUID)
const data = await db.getData(sql, { hash, location })
if (data !== null) {
await Promise.all([bucket.delete(filename), db.createBlob(sql, { workspace, name, hash, location })])
} else {
const size = object.size
const type = object.httpMetadata.contentType ?? 'application/octet-stream'
await withPostgres(env, ctx, async (sql) => {
const data = await db.getData(sql, { hash, location })
if (data !== null) {
await Promise.all([bucket.delete(filename), db.createBlob(sql, { workspace, name, hash, location })])
} else {
const size = object.size
const type = object.httpMetadata?.contentType ?? 'application/octet-stream'
await db.createData(sql, { hash, location, filename, type, size })
await db.createBlob(sql, { workspace, name, hash, location })
}
await sql.begin((sql) => [
db.createData(sql, { hash, location, filename, type, size }),
db.createBlob(sql, { workspace, name, hash, location })
])
}
})
}
async function uploadLargeFile (

View File

@ -13,7 +13,8 @@
// limitations under the License.
//
import type postgres from 'postgres'
import postgres from 'postgres'
import { measure, measureSync } from './measure'
import { type Location, type UUID } from './types'
export interface BlobDataId {
@ -42,63 +43,94 @@ export interface BlobRecordWithFilename extends BlobRecord {
filename: string
}
export async function getData (sql: postgres.Sql, dataId: BlobDataId): Promise<BlobDataRecord | null> {
const { hash, location } = dataId
const rows = await sql<BlobDataRecord[]>`
SELECT hash, location, filename, size, type
FROM blob.data
WHERE hash = ${hash} AND location = ${location}
`
if (rows.length > 0) {
return rows[0]
export async function withPostgres<T> (
env: Env,
ctx: ExecutionContext,
fn: (sql: postgres.Sql) => Promise<T>
): Promise<T> {
const sql = measureSync('db.connect', () => {
return postgres(env.HYPERDRIVE.connectionString)
})
try {
return await fn(sql)
} finally {
measureSync('db.close', () => {
ctx.waitUntil(sql.end({ timeout: 0 }))
})
}
return null
}
export async function createData (sql: postgres.Sql, data: BlobDataRecord): Promise<void> {
const { hash, location, filename, size, type } = data
await sql`
UPSERT INTO blob.data (hash, location, filename, size, type)
VALUES (${hash}, ${location}, ${filename}, ${size}, ${type})
`
export interface BlobDB {
getData: (sql: postgres.Sql, dataId: BlobDataId) => Promise<BlobDataRecord | null>
createData: (sql: postgres.Sql, data: BlobDataRecord) => Promise<void>
getBlob: (sql: postgres.Sql, blobId: BlobId) => Promise<BlobRecordWithFilename | null>
createBlob: (sql: postgres.Sql, blob: Omit<BlobRecord, 'filename' | 'deleted'>) => Promise<void>
deleteBlob: (sql: postgres.Sql, blob: BlobId) => Promise<void>
}
export async function getBlob (sql: postgres.Sql, blobId: BlobId): Promise<BlobRecordWithFilename | null> {
const { workspace, name } = blobId
const db: BlobDB = {
async getData (sql: postgres.Sql, dataId: BlobDataId): Promise<BlobDataRecord | null> {
const { hash, location } = dataId
const rows = await sql<BlobDataRecord[]>`
SELECT hash, location, filename, size, type
FROM blob.data
WHERE hash = ${hash} AND location = ${location}
`
return rows.length > 0 ? rows[0] : null
},
const rows = await sql<BlobRecordWithFilename[]>`
SELECT b.workspace, b.name, b.hash, b.location, b.deleted, d.filename
FROM blob.blob AS b
JOIN blob.data AS d ON b.hash = d.hash AND b.location = d.location
WHERE b.workspace = ${workspace} AND b.name = ${name}
`
async createData (sql: postgres.Sql, data: BlobDataRecord): Promise<void> {
const { hash, location, filename, size, type } = data
if (rows.length > 0) {
return rows[0]
await sql`
UPSERT INTO blob.data (hash, location, filename, size, type)
VALUES (${hash}, ${location}, ${filename}, ${size}, ${type})
`
},
async getBlob (sql: postgres.Sql, blobId: BlobId): Promise<BlobRecordWithFilename | null> {
const { workspace, name } = blobId
const rows = await sql<BlobRecordWithFilename[]>`
SELECT b.workspace, b.name, b.hash, b.location, b.deleted, d.filename
FROM blob.blob AS b
JOIN blob.data AS d ON b.hash = d.hash AND b.location = d.location
WHERE b.workspace = ${workspace} AND b.name = ${name}
`
if (rows.length > 0) {
return rows[0]
}
return null
},
async createBlob (sql: postgres.Sql, blob: Omit<BlobRecord, 'filename' | 'deleted'>): Promise<void> {
const { workspace, name, hash, location } = blob
await sql`
UPSERT INTO blob.blob (workspace, name, hash, location, deleted)
VALUES (${workspace}, ${name}, ${hash}, ${location}, false)
`
},
async deleteBlob (sql: postgres.Sql, blob: BlobId): Promise<void> {
const { workspace, name } = blob
await sql`
UPDATE blob.blob
SET deleted = true
WHERE workspace = ${workspace} AND name = ${name}
`
}
return null
}
export async function createBlob (sql: postgres.Sql, blob: Omit<BlobRecord, 'filename' | 'deleted'>): Promise<void> {
const { workspace, name, hash, location } = blob
await sql`
UPSERT INTO blob.blob (workspace, name, hash, location, deleted)
VALUES (${workspace}, ${name}, ${hash}, ${location}, false)
`
export const measuredDb: BlobDB = {
getData: (sql, dataId) => measure('db.getData', () => db.getData(sql, dataId)),
createData: (sql, data) => measure('db.createData', () => db.createData(sql, data)),
getBlob: (sql, blobId) => measure('db.getBlob', () => db.getBlob(sql, blobId)),
createBlob: (sql, blob) => measure('db.createBlob', () => db.createBlob(sql, blob)),
deleteBlob: (sql, blob) => measure('db.deleteBlob', () => db.deleteBlob(sql, blob))
}
export async function deleteBlob (sql: postgres.Sql, blob: BlobId): Promise<void> {
const { workspace, name } = blob
await sql`
UPDATE blob.blob
SET deleted = true
WHERE workspace = ${workspace} AND name = ${name}
`
}
export default measuredDb

View File

@ -18,6 +18,7 @@ import { type IRequestStrict, type RequestHandler, Router, error, html } from 'i
import { handleBlobDelete, handleBlobGet, handleBlobHead, handleUploadFormData } from './blob'
import { cors } from './cors'
import { LoggedKVNamespace, LoggedR2Bucket, requestTimeAfter, requestTimeBefore } from './measure'
import { handleImageGet } from './image'
import { handleS3Blob } from './s3'
import { handleVideoMetaGet } from './video'
@ -35,8 +36,8 @@ const { preflight, corsify } = cors({
})
const router = Router<IRequestStrict, [Env, ExecutionContext], Response>({
before: [preflight],
finally: [corsify]
before: [preflight, requestTimeBefore],
finally: [corsify, requestTimeAfter]
})
const withWorkspace: RequestHandler<WorkspaceRequest> = (request: WorkspaceRequest) => {
@ -87,6 +88,19 @@ router
.all('*', () => error(404))
export default class DatalakeWorker extends WorkerEntrypoint<Env> {
constructor (ctx: ExecutionContext, env: Env) {
env = {
...env,
datalake_blobs: new LoggedKVNamespace(env.datalake_blobs),
DATALAKE_APAC: new LoggedR2Bucket(env.DATALAKE_APAC),
DATALAKE_EEUR: new LoggedR2Bucket(env.DATALAKE_EEUR),
DATALAKE_WEUR: new LoggedR2Bucket(env.DATALAKE_WEUR),
DATALAKE_ENAM: new LoggedR2Bucket(env.DATALAKE_ENAM),
DATALAKE_WNAM: new LoggedR2Bucket(env.DATALAKE_WNAM)
}
super(ctx, env)
}
async fetch (request: Request): Promise<Response> {
return await router.fetch(request, this.env, this.ctx).catch(error)
}

View File

@ -0,0 +1,177 @@
//
// Copyright © 2024 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the 'License');
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an 'AS IS' BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
import { type IRequest, type ResponseHandler, type RequestHandler } from 'itty-router'
export async function measure<T> (label: string, fn: () => Promise<T>): Promise<T> {
const start = performance.now()
try {
return await fn()
} finally {
const duration = performance.now() - start
console.log({ stage: label, duration })
}
}
export function measureSync<T> (label: string, fn: () => T): T {
const start = performance.now()
try {
return fn()
} finally {
const duration = performance.now() - start
console.log({ stage: label, duration })
}
}
export const requestTimeBefore: RequestHandler<IRequest> = async (request: IRequest) => {
request.startTime = performance.now()
}
export const requestTimeAfter: ResponseHandler<Response> = async (response: Response, request: IRequest) => {
const duration = performance.now() - request.startTime
console.log({ stage: 'total', duration })
}
export class LoggedR2Bucket implements R2Bucket {
constructor (private readonly bucket: R2Bucket) {}
async head (key: string): Promise<R2Object | null> {
return await measure('r2.head', () => this.bucket.head(key))
}
async get (
key: string,
options?: R2GetOptions & {
onlyIf?: R2Conditional | Headers
}
): Promise<R2ObjectBody | null> {
return await measure('r2.get', () => this.bucket.get(key, options))
}
async put (
key: string,
value: ReadableStream | ArrayBuffer | ArrayBufferView | string | null | Blob,
options?: R2PutOptions & {
onlyIf?: R2Conditional | Headers
}
): Promise<R2Object> {
return await measure('r2.put', () => this.bucket.put(key, value, options))
}
async createMultipartUpload (key: string, options?: R2MultipartOptions): Promise<R2MultipartUpload> {
return await measure('r2.createMultipartUpload', () => this.bucket.createMultipartUpload(key, options))
}
resumeMultipartUpload (key: string, uploadId: string): R2MultipartUpload {
return measureSync('r2.resumeMultipartUpload', () => this.bucket.resumeMultipartUpload(key, uploadId))
}
async delete (keys: string | string[]): Promise<void> {
await measure('r2.delete', () => this.bucket.delete(keys))
}
async list (options?: R2ListOptions): Promise<R2Objects> {
return await measure('r2.list', () => this.bucket.list(options))
}
}
export class LoggedKVNamespace implements KVNamespace {
constructor (private readonly kv: KVNamespace) {}
get (key: string, options?: Partial<KVNamespaceGetOptions<undefined>>): Promise<string | null>
get (key: string, type: 'text'): Promise<string | null>
get<ExpectedValue = unknown>(key: string, type: 'json'): Promise<ExpectedValue | null>
get (key: string, type: 'arrayBuffer'): Promise<ArrayBuffer | null>
get (key: string, type: 'stream'): Promise<ReadableStream | null>
get (key: string, options?: KVNamespaceGetOptions<'text'>): Promise<string | null>
get<ExpectedValue = unknown>(key: string, options?: KVNamespaceGetOptions<'json'>): Promise<ExpectedValue | null>
get (key: string, options?: KVNamespaceGetOptions<'arrayBuffer'>): Promise<ArrayBuffer | null>
get (key: string, options?: KVNamespaceGetOptions<'stream'>): Promise<ReadableStream | null>
async get (key: string, options?: any): Promise<any> {
return await measure('kv.get', () => this.kv.get(key, options))
}
getWithMetadata<Metadata = unknown>(
key: string,
options?: Partial<KVNamespaceGetOptions<undefined>>
): Promise<KVNamespaceGetWithMetadataResult<string, Metadata>>
getWithMetadata<Metadata = unknown>(
key: string,
type: 'text'
): Promise<KVNamespaceGetWithMetadataResult<string, Metadata>>
getWithMetadata<ExpectedValue = unknown, Metadata = unknown>(
key: string,
type: 'json'
): Promise<KVNamespaceGetWithMetadataResult<ExpectedValue, Metadata>>
getWithMetadata<Metadata = unknown>(
key: string,
type: 'arrayBuffer'
): Promise<KVNamespaceGetWithMetadataResult<ArrayBuffer, Metadata>>
getWithMetadata<Metadata = unknown>(
key: string,
type: 'stream'
): Promise<KVNamespaceGetWithMetadataResult<ReadableStream, Metadata>>
getWithMetadata<Metadata = unknown>(
key: string,
options?: KVNamespaceGetOptions<'text'>
): Promise<KVNamespaceGetWithMetadataResult<string, Metadata>>
getWithMetadata<ExpectedValue = unknown, Metadata = unknown>(
key: string,
options?: KVNamespaceGetOptions<'json'>
): Promise<KVNamespaceGetWithMetadataResult<ExpectedValue, Metadata>>
getWithMetadata<Metadata = unknown>(
key: string,
options?: KVNamespaceGetOptions<'arrayBuffer'>
): Promise<KVNamespaceGetWithMetadataResult<ArrayBuffer, Metadata>>
getWithMetadata<Metadata = unknown>(
key: string,
options?: KVNamespaceGetOptions<'stream'>
): Promise<KVNamespaceGetWithMetadataResult<ReadableStream, Metadata>>
async getWithMetadata (key: string, options?: any): Promise<any> {
return await measure('kv.getWithMetadata', () => this.kv.getWithMetadata(key, options))
}
async list<Metadata = unknown>(options?: KVNamespaceListOptions): Promise<KVNamespaceListResult<Metadata, string>> {
return await measure('kv.list', () => this.kv.list(options))
}
async put (
key: string,
value: string | ArrayBuffer | ArrayBufferView | ReadableStream,
options?: KVNamespacePutOptions
): Promise<void> {
await measure('kv.put', () => this.kv.put(key, value))
}
async delete (key: string): Promise<void> {
await measure('kv.delete', () => this.kv.delete(key))
}
}
export class LoggedCache implements Cache {
constructor (private readonly cache: Cache) {}
async match (request: RequestInfo, options?: CacheQueryOptions): Promise<Response | undefined> {
return await measure('cache.match', () => this.cache.match(request, options))
}
async delete (request: RequestInfo, options?: CacheQueryOptions): Promise<boolean> {
return await measure('cache.delete', () => this.cache.delete(request, options))
}
async put (request: RequestInfo, response: Response): Promise<void> {
await measure('cache.put', () => this.cache.put(request, response))
}
}

View File

@ -14,8 +14,7 @@
//
import { error, json } from 'itty-router'
import postgres from 'postgres'
import * as db from './db'
import db, { withPostgres } from './db'
import { cacheControl } from './const'
import { toUUID } from './encodings'
import { selectStorage } from './storage'
@ -85,8 +84,6 @@ export async function handleMultipartUploadComplete (
env: Env,
ctx: ExecutionContext
): Promise<Response> {
const sql = postgres(env.HYPERDRIVE.connectionString)
const { workspace, name } = request
const multipartKey = request.query?.key
@ -108,17 +105,19 @@ export async function handleMultipartUploadComplete (
const size = object.size ?? 0
const filename = multipartKey as UUID
const data = await db.getData(sql, { hash, location })
if (data !== null) {
// blob already exists
await Promise.all([bucket.delete(filename), db.createBlob(sql, { workspace, name, hash, location })])
} else {
// Otherwise register a new hash and blob
await sql.begin((sql) => [
db.createData(sql, { hash, location, filename, type, size }),
db.createBlob(sql, { workspace, name, hash, location })
])
}
await withPostgres(env, ctx, async (sql) => {
const data = await db.getData(sql, { hash, location })
if (data !== null) {
// blob already exists
await Promise.all([bucket.delete(filename), db.createBlob(sql, { workspace, name, hash, location })])
} else {
// Otherwise register a new hash and blob
await sql.begin((sql) => [
db.createData(sql, { hash, location, filename, type, size }),
db.createBlob(sql, { workspace, name, hash, location })
])
}
})
return new Response(null, { status: 204 })
}

View File

@ -15,8 +15,7 @@
import { AwsClient } from 'aws4fetch'
import { error, json } from 'itty-router'
import postgres from 'postgres'
import * as db from './db'
import db, { withPostgres } from './db'
import { saveBlob } from './blob'
import { type BlobRequest } from './types'
@ -38,34 +37,35 @@ function getS3Client (payload: S3UploadPayload): AwsClient {
export async function handleS3Blob (request: BlobRequest, env: Env, ctx: ExecutionContext): Promise<Response> {
const { workspace, name } = request
const sql = postgres(env.HYPERDRIVE.connectionString)
const payload = await request.json<S3UploadPayload>()
const client = getS3Client(payload)
// Ensure the blob does not exist
const blob = await db.getBlob(sql, { workspace, name })
if (blob !== null) {
return new Response(null, { status: 200 })
}
return await withPostgres(env, ctx, async (sql) => {
// Ensure the blob does not exist
const blob = await db.getBlob(sql, { workspace, name })
if (blob !== null) {
return new Response(null, { status: 200 })
}
const object = await client.fetch(payload.url)
if (!object.ok || object.status !== 200) {
return error(object.status)
}
const object = await client.fetch(payload.url)
if (!object.ok || object.status !== 200) {
return error(object.status)
}
if (object.body === null) {
return error(400)
}
if (object.body === null) {
return error(400)
}
const contentType = object.headers.get('content-type') ?? 'application/octet-stream'
const contentLengthHeader = object.headers.get('content-length') ?? '0'
const lastModifiedHeader = object.headers.get('last-modified')
const contentType = object.headers.get('content-type') ?? 'application/octet-stream'
const contentLengthHeader = object.headers.get('content-length') ?? '0'
const lastModifiedHeader = object.headers.get('last-modified')
const contentLength = Number.parseInt(contentLengthHeader)
const lastModified = lastModifiedHeader !== null ? new Date(lastModifiedHeader).getTime() : Date.now()
const contentLength = Number.parseInt(contentLengthHeader)
const lastModified = lastModifiedHeader !== null ? new Date(lastModifiedHeader).getTime() : Date.now()
const result = await saveBlob(env, sql, object.body, contentLength, contentType, workspace, name, lastModified)
return json(result)
const result = await saveBlob(env, sql, object.body, contentLength, contentType, workspace, name, lastModified)
return json(result)
})
}

View File

@ -96,7 +96,7 @@ export async function handleSignComplete (request: BlobRequest, env: Env, ctx: E
}
try {
await handleBlobUploaded(env, workspace, name, uuid)
await handleBlobUploaded(env, ctx, workspace, name, uuid)
} catch (err) {
const message = err instanceof Error ? err.message : String(err)
console.error({ error: message, workspace, name, uuid })

View File

@ -1,7 +1,7 @@
#:schema node_modules/wrangler/config-schema.json
name = "datalake-worker"
main = "src/index.ts"
compatibility_date = "2024-07-01"
compatibility_date = "2024-09-23"
compatibility_flags = ["nodejs_compat"]
keep_vars = true