From f16d109ed529404003d49262a32e5c703be5ad80 Mon Sep 17 00:00:00 2001 From: akhismat Date: Tue, 10 Sep 2024 18:58:47 +0700 Subject: [PATCH] Do auth when importing files from Notion (#6510) * Do auth when importing files from Notion * Remove accidental change and fix linter issues --- dev/tool/src/index.ts | 145 +++++++++++++++++++---------------- dev/tool/src/notion.ts | 127 ++++++++++++++---------------- server/client/src/account.ts | 104 ++++++++++++++++++------- 3 files changed, 211 insertions(+), 165 deletions(-) diff --git a/dev/tool/src/index.ts b/dev/tool/src/index.ts index c2c345f089..4fcc7c2690 100644 --- a/dev/tool/src/index.ts +++ b/dev/tool/src/index.ts @@ -46,7 +46,14 @@ import { createStorageBackupStorage, restore } from '@hcengineering/server-backup' -import serverClientPlugin, { BlobClient, createClient, getTransactorEndpoint } from '@hcengineering/server-client' +import serverClientPlugin, { + BlobClient, + createClient, + getTransactorEndpoint, + getUserWorkspaces, + login, + selectWorkspace +} from '@hcengineering/server-client' import serverToken, { decodeToken, generateToken } from '@hcengineering/server-token' import toolPlugin, { connect, FileModelLogger } from '@hcengineering/server-tool' import path from 'path' @@ -65,14 +72,14 @@ import core, { systemAccountEmail, TxOperations, versionToString, - type WorkspaceIdWithUrl, type Client as CoreClient, type Data, type Doc, type Ref, type Tx, type Version, - type WorkspaceId + type WorkspaceId, + concatLink } from '@hcengineering/core' import { consoleModelLogger, type MigrateOperation } from '@hcengineering/model' import contact from '@hcengineering/model-contact' @@ -98,7 +105,7 @@ import { fixJsonMarkup, migrateMarkup } from './markup' import { fixMixinForeignAttributes, showMixinForeignAttributes } from './mixin' import { fixAccountEmails, renameAccount } from './renameAccount' import { moveFiles, syncFiles } from './storage' -import { importNotion, importToTeamspace } from './notion' +import { importNotion } from './notion' const colorConstants = { colorRed: '\u001b[31m', @@ -152,6 +159,15 @@ export function devTool ( return elasticUrl } + function getFrontUrl (): string { + const frontUrl = process.env.FRONT_URL + if (frontUrl === undefined) { + console.error('please provide front url') + process.exit(1) + } + return frontUrl + } + const initWS = process.env.INIT_WORKSPACE if (initWS !== undefined) { setMetadata(toolPlugin.metadata.InitWorkspace, initWS) @@ -212,84 +228,77 @@ export function devTool ( }) }) - // import-notion /home/anna/work/notion/pages/exported --workspace workspace + // import-notion-with-teamspaces /home/anna/work/notion/pages/exported --workspace workspace program - .command('import-notion ') + .command('import-notion-with-teamspaces ') .description('import extracted archive exported from Notion as "Markdown & CSV"') + .requiredOption('-u, --user ', 'user') + .requiredOption('-pw, --password ', 'password') .requiredOption('-ws, --workspace ', 'workspace where the documents should be imported to') .action(async (dir: string, cmd) => { - if (cmd.workspace === '') return - - const { mongodbUri } = prepareTools() - - await withDatabase(mongodbUri, async (db) => { - const ws = await getWorkspaceById(db, cmd.workspace) - if (ws === null) { - console.log('Workspace not found: ', cmd.workspace) - return - } - - const wsUrl: WorkspaceIdWithUrl = { - name: ws.workspace, - workspaceName: ws.workspaceName ?? '', - workspaceUrl: ws.workspaceUrl ?? '' - } - - await withStorage(mongodbUri, async (storageAdapter) => { - const token = generateToken(systemAccountEmail, { name: ws.workspace }) - const endpoint = await getTransactorEndpoint(token, 'external') - const connection = (await connect(endpoint, wsUrl, undefined, { - mode: 'backup' - })) as unknown as CoreClient - const client = new TxOperations(connection, core.account.System) - - await importNotion(toolCtx, client, storageAdapter, dir, wsUrl) - - await connection.close() - }) - }) + await importFromNotion(dir, cmd.user, cmd.password, cmd.workspace) }) // import-notion-to-teamspace /home/anna/work/notion/pages/exported --workspace workspace --teamspace notion program .command('import-notion-to-teamspace ') .description('import extracted archive exported from Notion as "Markdown & CSV"') + .requiredOption('-u, --user ', 'user') + .requiredOption('-pw, --password ', 'password') .requiredOption('-ws, --workspace ', 'workspace where the documents should be imported to') - .requiredOption('-ts, --teamspace ', 'teamspace where the documents should be imported to') + .requiredOption('-ts, --teamspace ', 'new teamspace name where the documents should be imported to') .action(async (dir: string, cmd) => { - if (cmd.workspace === '') return - if (cmd.teamspace === '') return - - const { mongodbUri } = prepareTools() - - await withDatabase(mongodbUri, async (db) => { - const ws = await getWorkspaceById(db, cmd.workspace) - if (ws === null) { - console.log('Workspace not found: ', cmd.workspace) - return - } - - const wsUrl: WorkspaceIdWithUrl = { - name: ws.workspace, - workspaceName: ws.workspaceName ?? '', - workspaceUrl: ws.workspaceUrl ?? '' - } - - await withStorage(mongodbUri, async (storageAdapter) => { - const token = generateToken(systemAccountEmail, { name: ws.workspace }) - const endpoint = await getTransactorEndpoint(token, 'external') - const connection = (await connect(endpoint, wsUrl, undefined, { - mode: 'backup' - })) as unknown as CoreClient - const client = new TxOperations(connection, core.account.System) - - await importToTeamspace(toolCtx, client, storageAdapter, dir, wsUrl, cmd.teamspace) - - await connection.close() - }) - }) + await importFromNotion(dir, cmd.user, cmd.password, cmd.workspace, cmd.teamspace) }) + async function importFromNotion ( + dir: string, + user: string, + password: string, + workspace: string, + teamspace?: string + ): Promise { + if (workspace === '' || user === '' || password === '' || teamspace === '') { + return + } + + const userToken = await login(user, password, workspace) + const allWorkspaces = await getUserWorkspaces(userToken) + const workspaces = allWorkspaces.filter((ws) => ws.workspace === workspace) + if (workspaces.length < 1) { + console.log('Workspace not found: ', workspace) + return + } + const selectedWs = await selectWorkspace(userToken, workspaces[0].workspace) + console.log(selectedWs) + + function uploader (token: string) { + return (id: string, data: any) => { + return fetch(concatLink(getFrontUrl(), '/files'), { + method: 'POST', + headers: { + Authorization: 'Bearer ' + token + }, + body: data + }) + } + } + + const connection = (await connect( + selectedWs.endpoint, + { + name: selectedWs.workspaceId + }, + undefined, + { + mode: 'backup' + } + )) as unknown as CoreClient + const client = new TxOperations(connection, core.account.System) + await importNotion(client, uploader(selectedWs.token), dir, teamspace) + await connection.close() + } + program .command('reset-account ') .description('create user and corresponding account in master database') diff --git a/dev/tool/src/notion.ts b/dev/tool/src/notion.ts index c99c0f0bc7..b1d23efcb7 100644 --- a/dev/tool/src/notion.ts +++ b/dev/tool/src/notion.ts @@ -2,15 +2,13 @@ import { generateId, type AttachedData, type Ref, - type WorkspaceIdWithUrl, makeCollaborativeDoc, - type MeasureMetricsContext, type TxOperations, - type Blob + type Blob, + collaborativeDocParse } from '@hcengineering/core' -import { saveCollaborativeDoc } from '@hcengineering/collaboration' +import { yDocToBuffer } from '@hcengineering/collaboration' import document, { type Document, type Teamspace } from '@hcengineering/document' -import { type StorageAdapter } from '@hcengineering/server-core' import { MarkupMarkType, type MarkupNode, @@ -58,12 +56,13 @@ enum NOTION_MD_LINK_TYPES { UNKNOWN } +export type FileUploader = (id: string, data: any) => Promise + export async function importNotion ( - ctx: MeasureMetricsContext, client: TxOperations, - storage: StorageAdapter, + uploadFile: FileUploader, dir: string, - ws: WorkspaceIdWithUrl + teamspace?: string ): Promise { const files = await getFilesForImport(dir) @@ -74,13 +73,17 @@ export async function importNotion ( console.log(fileMetaMap) console.log(documentMetaMap) - const spaceIdMap = await createTeamspaces(fileMetaMap, client) - if (spaceIdMap.size === 0) { - console.error('No teamspaces found in directory: ', dir) - return + if (teamspace === undefined) { + const spaceIdMap = await createTeamspaces(fileMetaMap, client) + if (spaceIdMap.size === 0) { + console.error('No teamspaces found in directory: ', dir) + return + } + await importFiles(client, uploadFile, fileMetaMap, documentMetaMap, spaceIdMap) + } else { + const spaceId = await createTeamspace(teamspace, client) + await importFilesToSpace(client, uploadFile, fileMetaMap, documentMetaMap, spaceId) } - - await importFiles(ctx, client, storage, fileMetaMap, documentMetaMap, spaceIdMap, ws) } async function getFilesForImport (dir: string): Promise { @@ -91,28 +94,6 @@ async function getFilesForImport (dir: string): Promise { return files } -export async function importToTeamspace ( - ctx: MeasureMetricsContext, - client: TxOperations, - storage: StorageAdapter, - dir: string, - ws: WorkspaceIdWithUrl, - teamspace: string -): Promise { - const files = await getFilesForImport(dir) - - const fileMetaMap = new Map() - const documentMetaMap = new Map() - - await collectMetadata(dir, files, fileMetaMap, documentMetaMap) - console.log(fileMetaMap) - console.log(documentMetaMap) - - const spaceId = await createTeamspace(teamspace, client) - - await importFilesToSpace(ctx, client, storage, fileMetaMap, documentMetaMap, spaceId, ws) -} - async function collectMetadata ( root: string, files: Dirent[], @@ -205,31 +186,27 @@ async function createTeamspace (name: string, client: TxOperations): Promise, documentMetaMap: Map, - spaceId: Ref, - ws: WorkspaceIdWithUrl + spaceId: Ref ): Promise { for (const [notionId, fileMeta] of fileMetaMap) { if (!fileMeta.isFolder) { const docMeta = documentMetaMap.get(notionId) if (docMeta === undefined) throw new Error('Cannot find metadata for entry: ' + fileMeta.fileName) - await importFile(ctx, client, storage, fileMeta, docMeta, spaceId, documentMetaMap, ws) + await importFile(client, uploadFile, fileMeta, docMeta, spaceId, documentMetaMap) } } } async function importFiles ( - ctx: MeasureMetricsContext, client: TxOperations, - storage: StorageAdapter, + uploadFile: FileUploader, fileMetaMap: Map, documentMetaMap: Map, - spaceIdMap: Map>, - ws: WorkspaceIdWithUrl + spaceIdMap: Map> ): Promise { for (const [notionId, fileMeta] of fileMetaMap) { if (!fileMeta.isFolder) { @@ -241,20 +218,18 @@ async function importFiles ( throw new Error('Teamspace not found for document: ' + docMeta.name) } - await importFile(ctx, client, storage, fileMeta, docMeta, spaceId, documentMetaMap, ws) + await importFile(client, uploadFile, fileMeta, docMeta, spaceId, documentMetaMap) } } } async function importFile ( - ctx: MeasureMetricsContext, client: TxOperations, - storage: StorageAdapter, + uploadFile: FileUploader, fileMeta: FileMetadata, docMeta: DocumentMetadata, spaceId: Ref, - documentMetaMap: Map, - ws: WorkspaceIdWithUrl + documentMetaMap: Map ): Promise { await new Promise((resolve, reject) => { if (fileMeta.isFolder) throw new Error('Importing folder entry is not supported: ' + fileMeta.fileName) @@ -268,7 +243,7 @@ async function importFile ( notionParentId !== undefined && notionParentId !== '' ? documentMetaMap.get(notionParentId) : undefined const processFileData = getDataProcessor(fileMeta, docMeta) - processFileData(ctx, client, storage, ws, data, docMeta, spaceId, parentMeta, documentMetaMap) + processFileData(client, uploadFile, data, docMeta, spaceId, parentMeta, documentMetaMap) .then(() => { console.log('IMPORT SUCCEED:', docMeta.name) console.log('------------------------------------------------------------------') @@ -292,10 +267,8 @@ async function importFile ( } type DataProcessor = ( - ctx: MeasureMetricsContext, client: TxOperations, - storage: StorageAdapter, - ws: WorkspaceIdWithUrl, + uploadFile: FileUploader, data: Buffer, docMeta: DocumentMetadata, space: Ref, @@ -328,10 +301,8 @@ function getDataProcessor (fileMeta: FileMetadata, docMeta: DocumentMetadata): D } async function createDBPageWithAttachments ( - ctx: MeasureMetricsContext, client: TxOperations, - storage: StorageAdapter, - ws: WorkspaceIdWithUrl, + uploadFile: FileUploader, data: Buffer, docMeta: DocumentMetadata, space: Ref, @@ -380,14 +351,12 @@ async function createDBPageWithAttachments ( size: docMeta.size } - await importAttachment(ctx, client, storage, ws, data, attachment, space, dbPage) + await importAttachment(client, uploadFile, data, attachment, space, dbPage) } async function importDBAttachment ( - ctx: MeasureMetricsContext, client: TxOperations, - storage: StorageAdapter, - ws: WorkspaceIdWithUrl, + uploadFile: FileUploader, data: Buffer, docMeta: DocumentMetadata, space: Ref, @@ -413,14 +382,12 @@ async function importDBAttachment ( mimeType: docMeta.mimeType, size: docMeta.size } - await importAttachment(ctx, client, storage, ws, data, attachment, space, dbPage) + await importAttachment(client, uploadFile, data, attachment, space, dbPage) } async function importAttachment ( - ctx: MeasureMetricsContext, client: TxOperations, - storage: StorageAdapter, - ws: WorkspaceIdWithUrl, + uploadFile: FileUploader, data: Buffer, docMeta: DocumentMetadata, space: Ref, @@ -433,7 +400,17 @@ async function importAttachment ( const size = docMeta.size ?? 0 const type = docMeta.mimeType ?? DEFAULT_ATTACHMENT_MIME_TYPE - await storage.put(ctx, ws, docMeta.id, data, type, size) + + const form = new FormData() + const file = new File([new Blob([data])], docMeta.name) + form.append('file', file, docMeta.id) + form.append('type', type) + form.append('size', size.toString()) + form.append('name', docMeta.name) + form.append('id', docMeta.id) + form.append('data', new Blob([data])) // ? + + await uploadFile(docMeta.id, form) const attachedData: AttachedData = { file: docMeta.id as Ref, @@ -455,10 +432,8 @@ async function importAttachment ( } async function importPageDocument ( - ctx: MeasureMetricsContext, client: TxOperations, - storage: StorageAdapter, - ws: WorkspaceIdWithUrl, + uploadFile: FileUploader, data: Buffer, docMeta: DocumentMetadata, space: Ref, @@ -474,7 +449,19 @@ async function importPageDocument ( const id = docMeta.id as Ref const collabId = makeCollaborativeDoc(id, 'content') const yDoc = jsonToYDocNoSchema(json, 'content') - await saveCollaborativeDoc(storage, ws, collabId, yDoc, ctx) + const { documentId } = collaborativeDocParse(collabId) + const buffer = yDocToBuffer(yDoc) + + const form = new FormData() + const file = new File([new Blob([buffer])], docMeta.name) + form.append('file', file, documentId) + form.append('type', 'application/ydoc') + form.append('size', buffer.length.toString()) + form.append('name', docMeta.name) + form.append('id', docMeta.id) + form.append('data', new Blob([buffer])) // ? + + await uploadFile(docMeta.id, form) const parentId = parentMeta?.id ?? document.ids.NoParent diff --git a/server/client/src/account.ts b/server/client/src/account.ts index 94b765c647..0d20f61a34 100644 --- a/server/client/src/account.ts +++ b/server/client/src/account.ts @@ -18,11 +18,19 @@ import { getMetadata, PlatformError, unknownError } from '@hcengineering/platfor import plugin from './plugin' +export interface WorkspaceLoginInfo extends LoginInfo { + workspace: string + workspaceId: string +} +export interface LoginInfo { + token: string + endpoint: string + confirmed: boolean + email: string +} + export async function listAccountWorkspaces (token: string): Promise { - const accountsUrl = getMetadata(plugin.metadata.Endpoint) - if (accountsUrl == null) { - throw new PlatformError(unknownError('No account endpoint specified')) - } + const accountsUrl = getAccoutsUrlOrFail() const workspaces = await ( await fetch(accountsUrl, { method: 'POST', @@ -44,11 +52,7 @@ export async function getTransactorEndpoint ( kind: 'internal' | 'external' = 'internal', timeout: number = -1 ): Promise { - const accountsUrl = getMetadata(plugin.metadata.Endpoint) - if (accountsUrl == null) { - throw new PlatformError(unknownError('No account endpoint specified')) - } - + const accountsUrl = getAccoutsUrlOrFail() const st = Date.now() while (true) { try { @@ -86,11 +90,7 @@ export async function getPendingWorkspace ( version: Data, operation: 'create' | 'upgrade' | 'all' ): Promise { - const accountsUrl = getMetadata(plugin.metadata.Endpoint) - if (accountsUrl == null) { - throw new PlatformError(unknownError('No account endpoint specified')) - } - + const accountsUrl = getAccoutsUrlOrFail() const workspaces = await ( await fetch(accountsUrl, { method: 'POST', @@ -115,10 +115,7 @@ export async function updateWorkspaceInfo ( progress: number, message?: string ): Promise { - const accountsUrl = getMetadata(plugin.metadata.Endpoint) - if (accountsUrl == null) { - throw new PlatformError(unknownError('No account endpoint specified')) - } + const accountsUrl = getAccoutsUrlOrFail() await ( await fetch(accountsUrl, { method: 'POST', @@ -139,11 +136,7 @@ export async function workerHandshake ( version: Data, operation: 'create' | 'upgrade' | 'all' ): Promise { - const accountsUrl = getMetadata(plugin.metadata.Endpoint) - if (accountsUrl == null) { - throw new PlatformError(unknownError('No account endpoint specified')) - } - + const accountsUrl = getAccoutsUrlOrFail() await fetch(accountsUrl, { method: 'POST', headers: { @@ -157,10 +150,7 @@ export async function workerHandshake ( } export async function getWorkspaceInfo (token: string): Promise { - const accountsUrl = getMetadata(plugin.metadata.Endpoint) - if (accountsUrl == null) { - throw new PlatformError(unknownError('No account endpoint specified')) - } + const accountsUrl = getAccoutsUrlOrFail() const workspaceInfo = await ( await fetch(accountsUrl, { method: 'POST', @@ -177,3 +167,63 @@ export async function getWorkspaceInfo (token: string): Promise { + const accountsUrl = getAccoutsUrlOrFail() + const response = await fetch(accountsUrl, { + method: 'POST', + headers: { + 'Content-Type': 'application/json' + }, + body: JSON.stringify({ + method: 'login', + params: [user, password, workspace] + }) + }) + + const result = await response.json() + const { token } = result.result + return token +} + +export async function getUserWorkspaces (token: string): Promise { + const accountsUrl = getAccoutsUrlOrFail() + const response = await fetch(accountsUrl, { + method: 'POST', + headers: { + Authorization: 'Bearer ' + token, + 'Content-Type': 'application/json' + }, + body: JSON.stringify({ + method: 'getUserWorkspaces', + params: [] + }) + }) + const result = await response.json() + return (result.result as BaseWorkspaceInfo[]) ?? [] +} + +export async function selectWorkspace (token: string, workspace: string): Promise { + const accountsUrl = getAccoutsUrlOrFail() + const response = await fetch(accountsUrl, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Authorization: 'Bearer ' + token + }, + body: JSON.stringify({ + method: 'selectWorkspace', + params: [workspace, 'external'] + }) + }) + const result = await response.json() + return result.result as WorkspaceLoginInfo +} + +function getAccoutsUrlOrFail (): string { + const accountsUrl = getMetadata(plugin.metadata.Endpoint) + if (accountsUrl == null) { + throw new PlatformError(unknownError('No account endpoint specified')) + } + return accountsUrl +}