Do auth when importing files from Notion (#6510)

* Do auth when importing files from Notion

* Remove accidental change and fix linter issues

dev/tool/src/index.ts

@@ -46,7 +46,14 @@ import {
   createStorageBackupStorage,
   restore
 } from '@hcengineering/server-backup'
-import serverClientPlugin, { BlobClient, createClient, getTransactorEndpoint } from '@hcengineering/server-client'
+import serverClientPlugin, {
+  BlobClient,
+  createClient,
+  getTransactorEndpoint,
+  getUserWorkspaces,
+  login,
+  selectWorkspace
+} from '@hcengineering/server-client'
 import serverToken, { decodeToken, generateToken } from '@hcengineering/server-token'
 import toolPlugin, { connect, FileModelLogger } from '@hcengineering/server-tool'
 import path from 'path'
@@ -65,14 +72,14 @@ import core, {
   systemAccountEmail,
   TxOperations,
   versionToString,
-  type WorkspaceIdWithUrl,
   type Client as CoreClient,
   type Data,
   type Doc,
   type Ref,
   type Tx,
   type Version,
-  type WorkspaceId
+  type WorkspaceId,
+  concatLink
 } from '@hcengineering/core'
 import { consoleModelLogger, type MigrateOperation } from '@hcengineering/model'
 import contact from '@hcengineering/model-contact'
@@ -98,7 +105,7 @@ import { fixJsonMarkup, migrateMarkup } from './markup'
 import { fixMixinForeignAttributes, showMixinForeignAttributes } from './mixin'
 import { fixAccountEmails, renameAccount } from './renameAccount'
 import { moveFiles, syncFiles } from './storage'
-import { importNotion, importToTeamspace } from './notion'
+import { importNotion } from './notion'
 
 const colorConstants = {
   colorRed: '\u001b[31m',
@@ -152,6 +159,15 @@ export function devTool (
     return elasticUrl
   }
 
+  function getFrontUrl (): string {
+    const frontUrl = process.env.FRONT_URL
+    if (frontUrl === undefined) {
+      console.error('please provide front url')
+      process.exit(1)
+    }
+    return frontUrl
+  }
+
   const initWS = process.env.INIT_WORKSPACE
   if (initWS !== undefined) {
     setMetadata(toolPlugin.metadata.InitWorkspace, initWS)
@@ -212,84 +228,77 @@ export function devTool (
       })
     })
 
-  // import-notion /home/anna/work/notion/pages/exported --workspace workspace
+  // import-notion-with-teamspaces /home/anna/work/notion/pages/exported --workspace workspace
   program
-    .command('import-notion <dir>')
+    .command('import-notion-with-teamspaces <dir>')
     .description('import extracted archive exported from Notion as "Markdown & CSV"')
+    .requiredOption('-u, --user <user>', 'user')
+    .requiredOption('-pw, --password <password>', 'password')
     .requiredOption('-ws, --workspace <workspace>', 'workspace where the documents should be imported to')
     .action(async (dir: string, cmd) => {
-      if (cmd.workspace === '') return
+      await importFromNotion(dir, cmd.user, cmd.password, cmd.workspace)
-
-      const { mongodbUri } = prepareTools()
-
-      await withDatabase(mongodbUri, async (db) => {
-        const ws = await getWorkspaceById(db, cmd.workspace)
-        if (ws === null) {
-          console.log('Workspace not found: ', cmd.workspace)
-          return
-        }
-
-        const wsUrl: WorkspaceIdWithUrl = {
-          name: ws.workspace,
-          workspaceName: ws.workspaceName ?? '',
-          workspaceUrl: ws.workspaceUrl ?? ''
-        }
-
-        await withStorage(mongodbUri, async (storageAdapter) => {
-          const token = generateToken(systemAccountEmail, { name: ws.workspace })
-          const endpoint = await getTransactorEndpoint(token, 'external')
-          const connection = (await connect(endpoint, wsUrl, undefined, {
-            mode: 'backup'
-          })) as unknown as CoreClient
-          const client = new TxOperations(connection, core.account.System)
-
-          await importNotion(toolCtx, client, storageAdapter, dir, wsUrl)
-
-          await connection.close()
-        })
-      })
     })
 
   // import-notion-to-teamspace /home/anna/work/notion/pages/exported --workspace workspace --teamspace notion
   program
     .command('import-notion-to-teamspace <dir>')
     .description('import extracted archive exported from Notion as "Markdown & CSV"')
+    .requiredOption('-u, --user <user>', 'user')
+    .requiredOption('-pw, --password <password>', 'password')
     .requiredOption('-ws, --workspace <workspace>', 'workspace where the documents should be imported to')
-    .requiredOption('-ts, --teamspace <teamspace>', 'teamspace where the documents should be imported to')
+    .requiredOption('-ts, --teamspace <teamspace>', 'new teamspace name where the documents should be imported to')
     .action(async (dir: string, cmd) => {
-      if (cmd.workspace === '') return
+      await importFromNotion(dir, cmd.user, cmd.password, cmd.workspace, cmd.teamspace)
-      if (cmd.teamspace === '') return
-
-      const { mongodbUri } = prepareTools()
-
-      await withDatabase(mongodbUri, async (db) => {
-        const ws = await getWorkspaceById(db, cmd.workspace)
-        if (ws === null) {
-          console.log('Workspace not found: ', cmd.workspace)
-          return
-        }
-
-        const wsUrl: WorkspaceIdWithUrl = {
-          name: ws.workspace,
-          workspaceName: ws.workspaceName ?? '',
-          workspaceUrl: ws.workspaceUrl ?? ''
-        }
-
-        await withStorage(mongodbUri, async (storageAdapter) => {
-          const token = generateToken(systemAccountEmail, { name: ws.workspace })
-          const endpoint = await getTransactorEndpoint(token, 'external')
-          const connection = (await connect(endpoint, wsUrl, undefined, {
-            mode: 'backup'
-          })) as unknown as CoreClient
-          const client = new TxOperations(connection, core.account.System)
-
-          await importToTeamspace(toolCtx, client, storageAdapter, dir, wsUrl, cmd.teamspace)
-
-          await connection.close()
-        })
-      })
     })
 
+  async function importFromNotion (
+    dir: string,
+    user: string,
+    password: string,
+    workspace: string,
+    teamspace?: string
+  ): Promise<void> {
+    if (workspace === '' || user === '' || password === '' || teamspace === '') {
+      return
+    }
+
+    const userToken = await login(user, password, workspace)
+    const allWorkspaces = await getUserWorkspaces(userToken)
+    const workspaces = allWorkspaces.filter((ws) => ws.workspace === workspace)
+    if (workspaces.length < 1) {
+      console.log('Workspace not found: ', workspace)
+      return
+    }
+    const selectedWs = await selectWorkspace(userToken, workspaces[0].workspace)
+    console.log(selectedWs)
+
+    function uploader (token: string) {
+      return (id: string, data: any) => {
+        return fetch(concatLink(getFrontUrl(), '/files'), {
+          method: 'POST',
+          headers: {
+            Authorization: 'Bearer ' + token
+          },
+          body: data
+        })
+      }
+    }
+
+    const connection = (await connect(
+      selectedWs.endpoint,
+      {
+        name: selectedWs.workspaceId
+      },
+      undefined,
+      {
+        mode: 'backup'
+      }
+    )) as unknown as CoreClient
+    const client = new TxOperations(connection, core.account.System)
+    await importNotion(client, uploader(selectedWs.token), dir, teamspace)
+    await connection.close()
+  }
+
   program
     .command('reset-account <email>')
     .description('create user and corresponding account in master database')

dev/tool/src/notion.ts

@@ -2,15 +2,13 @@ import {
   generateId,
   type AttachedData,
   type Ref,
-  type WorkspaceIdWithUrl,
   makeCollaborativeDoc,
-  type MeasureMetricsContext,
   type TxOperations,
-  type Blob
+  type Blob,
+  collaborativeDocParse
 } from '@hcengineering/core'
-import { saveCollaborativeDoc } from '@hcengineering/collaboration'
+import { yDocToBuffer } from '@hcengineering/collaboration'
 import document, { type Document, type Teamspace } from '@hcengineering/document'
-import { type StorageAdapter } from '@hcengineering/server-core'
 import {
   MarkupMarkType,
   type MarkupNode,
@@ -58,12 +56,13 @@ enum NOTION_MD_LINK_TYPES {
   UNKNOWN
 }
 
+export type FileUploader = (id: string, data: any) => Promise<any>
+
 export async function importNotion (
-  ctx: MeasureMetricsContext,
   client: TxOperations,
-  storage: StorageAdapter,
+  uploadFile: FileUploader,
   dir: string,
-  ws: WorkspaceIdWithUrl
+  teamspace?: string
 ): Promise<void> {
   const files = await getFilesForImport(dir)
 
@@ -74,13 +73,17 @@ export async function importNotion (
   console.log(fileMetaMap)
   console.log(documentMetaMap)
 
-  const spaceIdMap = await createTeamspaces(fileMetaMap, client)
+  if (teamspace === undefined) {
-  if (spaceIdMap.size === 0) {
+    const spaceIdMap = await createTeamspaces(fileMetaMap, client)
-    console.error('No teamspaces found in directory: ', dir)
+    if (spaceIdMap.size === 0) {
-    return
+      console.error('No teamspaces found in directory: ', dir)
+      return
+    }
+    await importFiles(client, uploadFile, fileMetaMap, documentMetaMap, spaceIdMap)
+  } else {
+    const spaceId = await createTeamspace(teamspace, client)
+    await importFilesToSpace(client, uploadFile, fileMetaMap, documentMetaMap, spaceId)
   }
-
-  await importFiles(ctx, client, storage, fileMetaMap, documentMetaMap, spaceIdMap, ws)
 }
 
 async function getFilesForImport (dir: string): Promise<Dirent[]> {
@@ -91,28 +94,6 @@ async function getFilesForImport (dir: string): Promise<Dirent[]> {
   return files
 }
 
-export async function importToTeamspace (
-  ctx: MeasureMetricsContext,
-  client: TxOperations,
-  storage: StorageAdapter,
-  dir: string,
-  ws: WorkspaceIdWithUrl,
-  teamspace: string
-): Promise<void> {
-  const files = await getFilesForImport(dir)
-
-  const fileMetaMap = new Map<string, FileMetadata>()
-  const documentMetaMap = new Map<string, DocumentMetadata>()
-
-  await collectMetadata(dir, files, fileMetaMap, documentMetaMap)
-  console.log(fileMetaMap)
-  console.log(documentMetaMap)
-
-  const spaceId = await createTeamspace(teamspace, client)
-
-  await importFilesToSpace(ctx, client, storage, fileMetaMap, documentMetaMap, spaceId, ws)
-}
-
 async function collectMetadata (
   root: string,
   files: Dirent[],
@@ -205,31 +186,27 @@ async function createTeamspace (name: string, client: TxOperations): Promise<Ref
 }
 
 async function importFilesToSpace (
-  ctx: MeasureMetricsContext,
   client: TxOperations,
-  storage: StorageAdapter,
+  uploadFile: FileUploader,
   fileMetaMap: Map<string, FileMetadata>,
   documentMetaMap: Map<string, DocumentMetadata>,
-  spaceId: Ref<Teamspace>,
+  spaceId: Ref<Teamspace>
-  ws: WorkspaceIdWithUrl
 ): Promise<void> {
   for (const [notionId, fileMeta] of fileMetaMap) {
     if (!fileMeta.isFolder) {
       const docMeta = documentMetaMap.get(notionId)
       if (docMeta === undefined) throw new Error('Cannot find metadata for entry: ' + fileMeta.fileName)
-      await importFile(ctx, client, storage, fileMeta, docMeta, spaceId, documentMetaMap, ws)
+      await importFile(client, uploadFile, fileMeta, docMeta, spaceId, documentMetaMap)
     }
   }
 }
 
 async function importFiles (
-  ctx: MeasureMetricsContext,
   client: TxOperations,
-  storage: StorageAdapter,
+  uploadFile: FileUploader,
   fileMetaMap: Map<string, FileMetadata>,
   documentMetaMap: Map<string, DocumentMetadata>,
-  spaceIdMap: Map<string, Ref<Teamspace>>,
+  spaceIdMap: Map<string, Ref<Teamspace>>
-  ws: WorkspaceIdWithUrl
 ): Promise<void> {
   for (const [notionId, fileMeta] of fileMetaMap) {
     if (!fileMeta.isFolder) {
@@ -241,20 +218,18 @@ async function importFiles (
         throw new Error('Teamspace not found for document: ' + docMeta.name)
       }
 
-      await importFile(ctx, client, storage, fileMeta, docMeta, spaceId, documentMetaMap, ws)
+      await importFile(client, uploadFile, fileMeta, docMeta, spaceId, documentMetaMap)
     }
   }
 }
 
 async function importFile (
-  ctx: MeasureMetricsContext,
   client: TxOperations,
-  storage: StorageAdapter,
+  uploadFile: FileUploader,
   fileMeta: FileMetadata,
   docMeta: DocumentMetadata,
   spaceId: Ref<Teamspace>,
-  documentMetaMap: Map<string, DocumentMetadata>,
+  documentMetaMap: Map<string, DocumentMetadata>
-  ws: WorkspaceIdWithUrl
 ): Promise<void> {
   await new Promise<void>((resolve, reject) => {
     if (fileMeta.isFolder) throw new Error('Importing folder entry is not supported: ' + fileMeta.fileName)
@@ -268,7 +243,7 @@ async function importFile (
           notionParentId !== undefined && notionParentId !== '' ? documentMetaMap.get(notionParentId) : undefined
 
         const processFileData = getDataProcessor(fileMeta, docMeta)
-        processFileData(ctx, client, storage, ws, data, docMeta, spaceId, parentMeta, documentMetaMap)
+        processFileData(client, uploadFile, data, docMeta, spaceId, parentMeta, documentMetaMap)
           .then(() => {
             console.log('IMPORT SUCCEED:', docMeta.name)
             console.log('------------------------------------------------------------------')
@@ -292,10 +267,8 @@ async function importFile (
 }
 
 type DataProcessor = (
-  ctx: MeasureMetricsContext,
   client: TxOperations,
-  storage: StorageAdapter,
+  uploadFile: FileUploader,
-  ws: WorkspaceIdWithUrl,
   data: Buffer,
   docMeta: DocumentMetadata,
   space: Ref<Teamspace>,
@@ -328,10 +301,8 @@ function getDataProcessor (fileMeta: FileMetadata, docMeta: DocumentMetadata): D
 }
 
 async function createDBPageWithAttachments (
-  ctx: MeasureMetricsContext,
   client: TxOperations,
-  storage: StorageAdapter,
+  uploadFile: FileUploader,
-  ws: WorkspaceIdWithUrl,
   data: Buffer,
   docMeta: DocumentMetadata,
   space: Ref<Teamspace>,
@@ -380,14 +351,12 @@ async function createDBPageWithAttachments (
     size: docMeta.size
   }
 
-  await importAttachment(ctx, client, storage, ws, data, attachment, space, dbPage)
+  await importAttachment(client, uploadFile, data, attachment, space, dbPage)
 }
 
 async function importDBAttachment (
-  ctx: MeasureMetricsContext,
   client: TxOperations,
-  storage: StorageAdapter,
+  uploadFile: FileUploader,
-  ws: WorkspaceIdWithUrl,
   data: Buffer,
   docMeta: DocumentMetadata,
   space: Ref<Teamspace>,
@@ -413,14 +382,12 @@ async function importDBAttachment (
     mimeType: docMeta.mimeType,
     size: docMeta.size
   }
-  await importAttachment(ctx, client, storage, ws, data, attachment, space, dbPage)
+  await importAttachment(client, uploadFile, data, attachment, space, dbPage)
 }
 
 async function importAttachment (
-  ctx: MeasureMetricsContext,
   client: TxOperations,
-  storage: StorageAdapter,
+  uploadFile: FileUploader,
-  ws: WorkspaceIdWithUrl,
   data: Buffer,
   docMeta: DocumentMetadata,
   space: Ref<Teamspace>,
@@ -433,7 +400,17 @@ async function importAttachment (
 
   const size = docMeta.size ?? 0
   const type = docMeta.mimeType ?? DEFAULT_ATTACHMENT_MIME_TYPE
-  await storage.put(ctx, ws, docMeta.id, data, type, size)
+
+  const form = new FormData()
+  const file = new File([new Blob([data])], docMeta.name)
+  form.append('file', file, docMeta.id)
+  form.append('type', type)
+  form.append('size', size.toString())
+  form.append('name', docMeta.name)
+  form.append('id', docMeta.id)
+  form.append('data', new Blob([data])) // ?
+
+  await uploadFile(docMeta.id, form)
 
   const attachedData: AttachedData<Attachment> = {
     file: docMeta.id as Ref<Blob>,
@@ -455,10 +432,8 @@ async function importAttachment (
 }
 
 async function importPageDocument (
-  ctx: MeasureMetricsContext,
   client: TxOperations,
-  storage: StorageAdapter,
+  uploadFile: FileUploader,
-  ws: WorkspaceIdWithUrl,
   data: Buffer,
   docMeta: DocumentMetadata,
   space: Ref<Teamspace>,
@@ -474,7 +449,19 @@ async function importPageDocument (
   const id = docMeta.id as Ref<Document>
   const collabId = makeCollaborativeDoc(id, 'content')
   const yDoc = jsonToYDocNoSchema(json, 'content')
-  await saveCollaborativeDoc(storage, ws, collabId, yDoc, ctx)
+  const { documentId } = collaborativeDocParse(collabId)
+  const buffer = yDocToBuffer(yDoc)
+
+  const form = new FormData()
+  const file = new File([new Blob([buffer])], docMeta.name)
+  form.append('file', file, documentId)
+  form.append('type', 'application/ydoc')
+  form.append('size', buffer.length.toString())
+  form.append('name', docMeta.name)
+  form.append('id', docMeta.id)
+  form.append('data', new Blob([buffer])) // ?
+
+  await uploadFile(docMeta.id, form)
 
   const parentId = parentMeta?.id ?? document.ids.NoParent
 

server/client/src/account.ts

@@ -18,11 +18,19 @@ import { getMetadata, PlatformError, unknownError } from '@hcengineering/platfor
 
 import plugin from './plugin'
 
+export interface WorkspaceLoginInfo extends LoginInfo {
+  workspace: string
+  workspaceId: string
+}
+export interface LoginInfo {
+  token: string
+  endpoint: string
+  confirmed: boolean
+  email: string
+}
+
 export async function listAccountWorkspaces (token: string): Promise<BaseWorkspaceInfo[]> {
-  const accountsUrl = getMetadata(plugin.metadata.Endpoint)
+  const accountsUrl = getAccoutsUrlOrFail()
-  if (accountsUrl == null) {
-    throw new PlatformError(unknownError('No account endpoint specified'))
-  }
   const workspaces = await (
     await fetch(accountsUrl, {
       method: 'POST',
@@ -44,11 +52,7 @@ export async function getTransactorEndpoint (
   kind: 'internal' | 'external' = 'internal',
   timeout: number = -1
 ): Promise<string> {
-  const accountsUrl = getMetadata(plugin.metadata.Endpoint)
+  const accountsUrl = getAccoutsUrlOrFail()
-  if (accountsUrl == null) {
-    throw new PlatformError(unknownError('No account endpoint specified'))
-  }
-
   const st = Date.now()
   while (true) {
     try {
@@ -86,11 +90,7 @@ export async function getPendingWorkspace (
   version: Data<Version>,
   operation: 'create' | 'upgrade' | 'all'
 ): Promise<BaseWorkspaceInfo | undefined> {
-  const accountsUrl = getMetadata(plugin.metadata.Endpoint)
+  const accountsUrl = getAccoutsUrlOrFail()
-  if (accountsUrl == null) {
-    throw new PlatformError(unknownError('No account endpoint specified'))
-  }
-
   const workspaces = await (
     await fetch(accountsUrl, {
       method: 'POST',
@@ -115,10 +115,7 @@ export async function updateWorkspaceInfo (
   progress: number,
   message?: string
 ): Promise<void> {
-  const accountsUrl = getMetadata(plugin.metadata.Endpoint)
+  const accountsUrl = getAccoutsUrlOrFail()
-  if (accountsUrl == null) {
-    throw new PlatformError(unknownError('No account endpoint specified'))
-  }
   await (
     await fetch(accountsUrl, {
       method: 'POST',
@@ -139,11 +136,7 @@ export async function workerHandshake (
   version: Data<Version>,
   operation: 'create' | 'upgrade' | 'all'
 ): Promise<void> {
-  const accountsUrl = getMetadata(plugin.metadata.Endpoint)
+  const accountsUrl = getAccoutsUrlOrFail()
-  if (accountsUrl == null) {
-    throw new PlatformError(unknownError('No account endpoint specified'))
-  }
-
   await fetch(accountsUrl, {
     method: 'POST',
     headers: {
@@ -157,10 +150,7 @@ export async function workerHandshake (
 }
 
 export async function getWorkspaceInfo (token: string): Promise<BaseWorkspaceInfo | undefined> {
-  const accountsUrl = getMetadata(plugin.metadata.Endpoint)
+  const accountsUrl = getAccoutsUrlOrFail()
-  if (accountsUrl == null) {
-    throw new PlatformError(unknownError('No account endpoint specified'))
-  }
   const workspaceInfo = await (
     await fetch(accountsUrl, {
       method: 'POST',
@@ -177,3 +167,63 @@ export async function getWorkspaceInfo (token: string): Promise<BaseWorkspaceInf
 
   return workspaceInfo.result as BaseWorkspaceInfo | undefined
 }
+
+export async function login (user: string, password: string, workspace: string): Promise<string> {
+  const accountsUrl = getAccoutsUrlOrFail()
+  const response = await fetch(accountsUrl, {
+    method: 'POST',
+    headers: {
+      'Content-Type': 'application/json'
+    },
+    body: JSON.stringify({
+      method: 'login',
+      params: [user, password, workspace]
+    })
+  })
+
+  const result = await response.json()
+  const { token } = result.result
+  return token
+}
+
+export async function getUserWorkspaces (token: string): Promise<BaseWorkspaceInfo[]> {
+  const accountsUrl = getAccoutsUrlOrFail()
+  const response = await fetch(accountsUrl, {
+    method: 'POST',
+    headers: {
+      Authorization: 'Bearer ' + token,
+      'Content-Type': 'application/json'
+    },
+    body: JSON.stringify({
+      method: 'getUserWorkspaces',
+      params: []
+    })
+  })
+  const result = await response.json()
+  return (result.result as BaseWorkspaceInfo[]) ?? []
+}
+
+export async function selectWorkspace (token: string, workspace: string): Promise<WorkspaceLoginInfo> {
+  const accountsUrl = getAccoutsUrlOrFail()
+  const response = await fetch(accountsUrl, {
+    method: 'POST',
+    headers: {
+      'Content-Type': 'application/json',
+      Authorization: 'Bearer ' + token
+    },
+    body: JSON.stringify({
+      method: 'selectWorkspace',
+      params: [workspace, 'external']
+    })
+  })
+  const result = await response.json()
+  return result.result as WorkspaceLoginInfo
+}
+
+function getAccoutsUrlOrFail (): string {
+  const accountsUrl = getMetadata(plugin.metadata.Endpoint)
+  if (accountsUrl == null) {
+    throw new PlatformError(unknownError('No account endpoint specified'))
+  }
+  return accountsUrl
+}