diff --git a/dev/tool/src/index.ts b/dev/tool/src/index.ts
index c2c345f089..4fcc7c2690 100644
--- a/dev/tool/src/index.ts
+++ b/dev/tool/src/index.ts
@@ -46,7 +46,14 @@ import {
createStorageBackupStorage,
restore
} from '@hcengineering/server-backup'
-import serverClientPlugin, { BlobClient, createClient, getTransactorEndpoint } from '@hcengineering/server-client'
+import serverClientPlugin, {
+ BlobClient,
+ createClient,
+ getTransactorEndpoint,
+ getUserWorkspaces,
+ login,
+ selectWorkspace
+} from '@hcengineering/server-client'
import serverToken, { decodeToken, generateToken } from '@hcengineering/server-token'
import toolPlugin, { connect, FileModelLogger } from '@hcengineering/server-tool'
import path from 'path'
@@ -65,14 +72,14 @@ import core, {
systemAccountEmail,
TxOperations,
versionToString,
- type WorkspaceIdWithUrl,
type Client as CoreClient,
type Data,
type Doc,
type Ref,
type Tx,
type Version,
- type WorkspaceId
+ type WorkspaceId,
+ concatLink
} from '@hcengineering/core'
import { consoleModelLogger, type MigrateOperation } from '@hcengineering/model'
import contact from '@hcengineering/model-contact'
@@ -98,7 +105,7 @@ import { fixJsonMarkup, migrateMarkup } from './markup'
import { fixMixinForeignAttributes, showMixinForeignAttributes } from './mixin'
import { fixAccountEmails, renameAccount } from './renameAccount'
import { moveFiles, syncFiles } from './storage'
-import { importNotion, importToTeamspace } from './notion'
+import { importNotion } from './notion'
const colorConstants = {
colorRed: '\u001b[31m',
@@ -152,6 +159,15 @@ export function devTool (
return elasticUrl
}
+ function getFrontUrl (): string {
+ const frontUrl = process.env.FRONT_URL
+ if (frontUrl === undefined) {
+ console.error('please provide front url')
+ process.exit(1)
+ }
+ return frontUrl
+ }
+
const initWS = process.env.INIT_WORKSPACE
if (initWS !== undefined) {
setMetadata(toolPlugin.metadata.InitWorkspace, initWS)
@@ -212,84 +228,77 @@ export function devTool (
})
})
- // import-notion /home/anna/work/notion/pages/exported --workspace workspace
+ // import-notion-with-teamspaces /home/anna/work/notion/pages/exported --workspace workspace
program
- .command('import-notion
')
+ .command('import-notion-with-teamspaces ')
.description('import extracted archive exported from Notion as "Markdown & CSV"')
+ .requiredOption('-u, --user ', 'user')
+ .requiredOption('-pw, --password ', 'password')
.requiredOption('-ws, --workspace ', 'workspace where the documents should be imported to')
.action(async (dir: string, cmd) => {
- if (cmd.workspace === '') return
-
- const { mongodbUri } = prepareTools()
-
- await withDatabase(mongodbUri, async (db) => {
- const ws = await getWorkspaceById(db, cmd.workspace)
- if (ws === null) {
- console.log('Workspace not found: ', cmd.workspace)
- return
- }
-
- const wsUrl: WorkspaceIdWithUrl = {
- name: ws.workspace,
- workspaceName: ws.workspaceName ?? '',
- workspaceUrl: ws.workspaceUrl ?? ''
- }
-
- await withStorage(mongodbUri, async (storageAdapter) => {
- const token = generateToken(systemAccountEmail, { name: ws.workspace })
- const endpoint = await getTransactorEndpoint(token, 'external')
- const connection = (await connect(endpoint, wsUrl, undefined, {
- mode: 'backup'
- })) as unknown as CoreClient
- const client = new TxOperations(connection, core.account.System)
-
- await importNotion(toolCtx, client, storageAdapter, dir, wsUrl)
-
- await connection.close()
- })
- })
+ await importFromNotion(dir, cmd.user, cmd.password, cmd.workspace)
})
// import-notion-to-teamspace /home/anna/work/notion/pages/exported --workspace workspace --teamspace notion
program
.command('import-notion-to-teamspace ')
.description('import extracted archive exported from Notion as "Markdown & CSV"')
+ .requiredOption('-u, --user ', 'user')
+ .requiredOption('-pw, --password ', 'password')
.requiredOption('-ws, --workspace ', 'workspace where the documents should be imported to')
- .requiredOption('-ts, --teamspace ', 'teamspace where the documents should be imported to')
+ .requiredOption('-ts, --teamspace ', 'new teamspace name where the documents should be imported to')
.action(async (dir: string, cmd) => {
- if (cmd.workspace === '') return
- if (cmd.teamspace === '') return
-
- const { mongodbUri } = prepareTools()
-
- await withDatabase(mongodbUri, async (db) => {
- const ws = await getWorkspaceById(db, cmd.workspace)
- if (ws === null) {
- console.log('Workspace not found: ', cmd.workspace)
- return
- }
-
- const wsUrl: WorkspaceIdWithUrl = {
- name: ws.workspace,
- workspaceName: ws.workspaceName ?? '',
- workspaceUrl: ws.workspaceUrl ?? ''
- }
-
- await withStorage(mongodbUri, async (storageAdapter) => {
- const token = generateToken(systemAccountEmail, { name: ws.workspace })
- const endpoint = await getTransactorEndpoint(token, 'external')
- const connection = (await connect(endpoint, wsUrl, undefined, {
- mode: 'backup'
- })) as unknown as CoreClient
- const client = new TxOperations(connection, core.account.System)
-
- await importToTeamspace(toolCtx, client, storageAdapter, dir, wsUrl, cmd.teamspace)
-
- await connection.close()
- })
- })
+ await importFromNotion(dir, cmd.user, cmd.password, cmd.workspace, cmd.teamspace)
})
+ async function importFromNotion (
+ dir: string,
+ user: string,
+ password: string,
+ workspace: string,
+ teamspace?: string
+ ): Promise {
+ if (workspace === '' || user === '' || password === '' || teamspace === '') {
+ return
+ }
+
+ const userToken = await login(user, password, workspace)
+ const allWorkspaces = await getUserWorkspaces(userToken)
+ const workspaces = allWorkspaces.filter((ws) => ws.workspace === workspace)
+ if (workspaces.length < 1) {
+ console.log('Workspace not found: ', workspace)
+ return
+ }
+ const selectedWs = await selectWorkspace(userToken, workspaces[0].workspace)
+ console.log(selectedWs)
+
+ function uploader (token: string) {
+ return (id: string, data: any) => {
+ return fetch(concatLink(getFrontUrl(), '/files'), {
+ method: 'POST',
+ headers: {
+ Authorization: 'Bearer ' + token
+ },
+ body: data
+ })
+ }
+ }
+
+ const connection = (await connect(
+ selectedWs.endpoint,
+ {
+ name: selectedWs.workspaceId
+ },
+ undefined,
+ {
+ mode: 'backup'
+ }
+ )) as unknown as CoreClient
+ const client = new TxOperations(connection, core.account.System)
+ await importNotion(client, uploader(selectedWs.token), dir, teamspace)
+ await connection.close()
+ }
+
program
.command('reset-account ')
.description('create user and corresponding account in master database')
diff --git a/dev/tool/src/notion.ts b/dev/tool/src/notion.ts
index c99c0f0bc7..b1d23efcb7 100644
--- a/dev/tool/src/notion.ts
+++ b/dev/tool/src/notion.ts
@@ -2,15 +2,13 @@ import {
generateId,
type AttachedData,
type Ref,
- type WorkspaceIdWithUrl,
makeCollaborativeDoc,
- type MeasureMetricsContext,
type TxOperations,
- type Blob
+ type Blob,
+ collaborativeDocParse
} from '@hcengineering/core'
-import { saveCollaborativeDoc } from '@hcengineering/collaboration'
+import { yDocToBuffer } from '@hcengineering/collaboration'
import document, { type Document, type Teamspace } from '@hcengineering/document'
-import { type StorageAdapter } from '@hcengineering/server-core'
import {
MarkupMarkType,
type MarkupNode,
@@ -58,12 +56,13 @@ enum NOTION_MD_LINK_TYPES {
UNKNOWN
}
+export type FileUploader = (id: string, data: any) => Promise
+
export async function importNotion (
- ctx: MeasureMetricsContext,
client: TxOperations,
- storage: StorageAdapter,
+ uploadFile: FileUploader,
dir: string,
- ws: WorkspaceIdWithUrl
+ teamspace?: string
): Promise {
const files = await getFilesForImport(dir)
@@ -74,13 +73,17 @@ export async function importNotion (
console.log(fileMetaMap)
console.log(documentMetaMap)
- const spaceIdMap = await createTeamspaces(fileMetaMap, client)
- if (spaceIdMap.size === 0) {
- console.error('No teamspaces found in directory: ', dir)
- return
+ if (teamspace === undefined) {
+ const spaceIdMap = await createTeamspaces(fileMetaMap, client)
+ if (spaceIdMap.size === 0) {
+ console.error('No teamspaces found in directory: ', dir)
+ return
+ }
+ await importFiles(client, uploadFile, fileMetaMap, documentMetaMap, spaceIdMap)
+ } else {
+ const spaceId = await createTeamspace(teamspace, client)
+ await importFilesToSpace(client, uploadFile, fileMetaMap, documentMetaMap, spaceId)
}
-
- await importFiles(ctx, client, storage, fileMetaMap, documentMetaMap, spaceIdMap, ws)
}
async function getFilesForImport (dir: string): Promise {
@@ -91,28 +94,6 @@ async function getFilesForImport (dir: string): Promise {
return files
}
-export async function importToTeamspace (
- ctx: MeasureMetricsContext,
- client: TxOperations,
- storage: StorageAdapter,
- dir: string,
- ws: WorkspaceIdWithUrl,
- teamspace: string
-): Promise {
- const files = await getFilesForImport(dir)
-
- const fileMetaMap = new Map()
- const documentMetaMap = new Map()
-
- await collectMetadata(dir, files, fileMetaMap, documentMetaMap)
- console.log(fileMetaMap)
- console.log(documentMetaMap)
-
- const spaceId = await createTeamspace(teamspace, client)
-
- await importFilesToSpace(ctx, client, storage, fileMetaMap, documentMetaMap, spaceId, ws)
-}
-
async function collectMetadata (
root: string,
files: Dirent[],
@@ -205,31 +186,27 @@ async function createTeamspace (name: string, client: TxOperations): Promise[,
documentMetaMap: Map,
- spaceId: Ref,
- ws: WorkspaceIdWithUrl
+ spaceId: Ref
): Promise {
for (const [notionId, fileMeta] of fileMetaMap) {
if (!fileMeta.isFolder) {
const docMeta = documentMetaMap.get(notionId)
if (docMeta === undefined) throw new Error('Cannot find metadata for entry: ' + fileMeta.fileName)
- await importFile(ctx, client, storage, fileMeta, docMeta, spaceId, documentMetaMap, ws)
+ await importFile(client, uploadFile, fileMeta, docMeta, spaceId, documentMetaMap)
}
}
}
async function importFiles (
- ctx: MeasureMetricsContext,
client: TxOperations,
- storage: StorageAdapter,
+ uploadFile: FileUploader,
fileMetaMap: Map,
documentMetaMap: Map,
- spaceIdMap: Map>,
- ws: WorkspaceIdWithUrl
+ spaceIdMap: Map>
): Promise {
for (const [notionId, fileMeta] of fileMetaMap) {
if (!fileMeta.isFolder) {
@@ -241,20 +218,18 @@ async function importFiles (
throw new Error('Teamspace not found for document: ' + docMeta.name)
}
- await importFile(ctx, client, storage, fileMeta, docMeta, spaceId, documentMetaMap, ws)
+ await importFile(client, uploadFile, fileMeta, docMeta, spaceId, documentMetaMap)
}
}
}
async function importFile (
- ctx: MeasureMetricsContext,
client: TxOperations,
- storage: StorageAdapter,
+ uploadFile: FileUploader,
fileMeta: FileMetadata,
docMeta: DocumentMetadata,
spaceId: Ref,
- documentMetaMap: Map,
- ws: WorkspaceIdWithUrl
+ documentMetaMap: Map
): Promise {
await new Promise((resolve, reject) => {
if (fileMeta.isFolder) throw new Error('Importing folder entry is not supported: ' + fileMeta.fileName)
@@ -268,7 +243,7 @@ async function importFile (
notionParentId !== undefined && notionParentId !== '' ? documentMetaMap.get(notionParentId) : undefined
const processFileData = getDataProcessor(fileMeta, docMeta)
- processFileData(ctx, client, storage, ws, data, docMeta, spaceId, parentMeta, documentMetaMap)
+ processFileData(client, uploadFile, data, docMeta, spaceId, parentMeta, documentMetaMap)
.then(() => {
console.log('IMPORT SUCCEED:', docMeta.name)
console.log('------------------------------------------------------------------')
@@ -292,10 +267,8 @@ async function importFile (
}
type DataProcessor = (
- ctx: MeasureMetricsContext,
client: TxOperations,
- storage: StorageAdapter,
- ws: WorkspaceIdWithUrl,
+ uploadFile: FileUploader,
data: Buffer,
docMeta: DocumentMetadata,
space: Ref,
@@ -328,10 +301,8 @@ function getDataProcessor (fileMeta: FileMetadata, docMeta: DocumentMetadata): D
}
async function createDBPageWithAttachments (
- ctx: MeasureMetricsContext,
client: TxOperations,
- storage: StorageAdapter,
- ws: WorkspaceIdWithUrl,
+ uploadFile: FileUploader,
data: Buffer,
docMeta: DocumentMetadata,
space: Ref,
@@ -380,14 +351,12 @@ async function createDBPageWithAttachments (
size: docMeta.size
}
- await importAttachment(ctx, client, storage, ws, data, attachment, space, dbPage)
+ await importAttachment(client, uploadFile, data, attachment, space, dbPage)
}
async function importDBAttachment (
- ctx: MeasureMetricsContext,
client: TxOperations,
- storage: StorageAdapter,
- ws: WorkspaceIdWithUrl,
+ uploadFile: FileUploader,
data: Buffer,
docMeta: DocumentMetadata,
space: Ref,
@@ -413,14 +382,12 @@ async function importDBAttachment (
mimeType: docMeta.mimeType,
size: docMeta.size
}
- await importAttachment(ctx, client, storage, ws, data, attachment, space, dbPage)
+ await importAttachment(client, uploadFile, data, attachment, space, dbPage)
}
async function importAttachment (
- ctx: MeasureMetricsContext,
client: TxOperations,
- storage: StorageAdapter,
- ws: WorkspaceIdWithUrl,
+ uploadFile: FileUploader,
data: Buffer,
docMeta: DocumentMetadata,
space: Ref,
@@ -433,7 +400,17 @@ async function importAttachment (
const size = docMeta.size ?? 0
const type = docMeta.mimeType ?? DEFAULT_ATTACHMENT_MIME_TYPE
- await storage.put(ctx, ws, docMeta.id, data, type, size)
+
+ const form = new FormData()
+ const file = new File([new Blob([data])], docMeta.name)
+ form.append('file', file, docMeta.id)
+ form.append('type', type)
+ form.append('size', size.toString())
+ form.append('name', docMeta.name)
+ form.append('id', docMeta.id)
+ form.append('data', new Blob([data])) // ?
+
+ await uploadFile(docMeta.id, form)
const attachedData: AttachedData = {
file: docMeta.id as Ref,
@@ -455,10 +432,8 @@ async function importAttachment (
}
async function importPageDocument (
- ctx: MeasureMetricsContext,
client: TxOperations,
- storage: StorageAdapter,
- ws: WorkspaceIdWithUrl,
+ uploadFile: FileUploader,
data: Buffer,
docMeta: DocumentMetadata,
space: Ref,
@@ -474,7 +449,19 @@ async function importPageDocument (
const id = docMeta.id as Ref
const collabId = makeCollaborativeDoc(id, 'content')
const yDoc = jsonToYDocNoSchema(json, 'content')
- await saveCollaborativeDoc(storage, ws, collabId, yDoc, ctx)
+ const { documentId } = collaborativeDocParse(collabId)
+ const buffer = yDocToBuffer(yDoc)
+
+ const form = new FormData()
+ const file = new File([new Blob([buffer])], docMeta.name)
+ form.append('file', file, documentId)
+ form.append('type', 'application/ydoc')
+ form.append('size', buffer.length.toString())
+ form.append('name', docMeta.name)
+ form.append('id', docMeta.id)
+ form.append('data', new Blob([buffer])) // ?
+
+ await uploadFile(docMeta.id, form)
const parentId = parentMeta?.id ?? document.ids.NoParent
diff --git a/server/client/src/account.ts b/server/client/src/account.ts
index 94b765c647..0d20f61a34 100644
--- a/server/client/src/account.ts
+++ b/server/client/src/account.ts
@@ -18,11 +18,19 @@ import { getMetadata, PlatformError, unknownError } from '@hcengineering/platfor
import plugin from './plugin'
+export interface WorkspaceLoginInfo extends LoginInfo {
+ workspace: string
+ workspaceId: string
+}
+export interface LoginInfo {
+ token: string
+ endpoint: string
+ confirmed: boolean
+ email: string
+}
+
export async function listAccountWorkspaces (token: string): Promise {
- const accountsUrl = getMetadata(plugin.metadata.Endpoint)
- if (accountsUrl == null) {
- throw new PlatformError(unknownError('No account endpoint specified'))
- }
+ const accountsUrl = getAccoutsUrlOrFail()
const workspaces = await (
await fetch(accountsUrl, {
method: 'POST',
@@ -44,11 +52,7 @@ export async function getTransactorEndpoint (
kind: 'internal' | 'external' = 'internal',
timeout: number = -1
): Promise {
- const accountsUrl = getMetadata(plugin.metadata.Endpoint)
- if (accountsUrl == null) {
- throw new PlatformError(unknownError('No account endpoint specified'))
- }
-
+ const accountsUrl = getAccoutsUrlOrFail()
const st = Date.now()
while (true) {
try {
@@ -86,11 +90,7 @@ export async function getPendingWorkspace (
version: Data,
operation: 'create' | 'upgrade' | 'all'
): Promise {
- const accountsUrl = getMetadata(plugin.metadata.Endpoint)
- if (accountsUrl == null) {
- throw new PlatformError(unknownError('No account endpoint specified'))
- }
-
+ const accountsUrl = getAccoutsUrlOrFail()
const workspaces = await (
await fetch(accountsUrl, {
method: 'POST',
@@ -115,10 +115,7 @@ export async function updateWorkspaceInfo (
progress: number,
message?: string
): Promise {
- const accountsUrl = getMetadata(plugin.metadata.Endpoint)
- if (accountsUrl == null) {
- throw new PlatformError(unknownError('No account endpoint specified'))
- }
+ const accountsUrl = getAccoutsUrlOrFail()
await (
await fetch(accountsUrl, {
method: 'POST',
@@ -139,11 +136,7 @@ export async function workerHandshake (
version: Data,
operation: 'create' | 'upgrade' | 'all'
): Promise {
- const accountsUrl = getMetadata(plugin.metadata.Endpoint)
- if (accountsUrl == null) {
- throw new PlatformError(unknownError('No account endpoint specified'))
- }
-
+ const accountsUrl = getAccoutsUrlOrFail()
await fetch(accountsUrl, {
method: 'POST',
headers: {
@@ -157,10 +150,7 @@ export async function workerHandshake (
}
export async function getWorkspaceInfo (token: string): Promise {
- const accountsUrl = getMetadata(plugin.metadata.Endpoint)
- if (accountsUrl == null) {
- throw new PlatformError(unknownError('No account endpoint specified'))
- }
+ const accountsUrl = getAccoutsUrlOrFail()
const workspaceInfo = await (
await fetch(accountsUrl, {
method: 'POST',
@@ -177,3 +167,63 @@ export async function getWorkspaceInfo (token: string): Promise {
+ const accountsUrl = getAccoutsUrlOrFail()
+ const response = await fetch(accountsUrl, {
+ method: 'POST',
+ headers: {
+ 'Content-Type': 'application/json'
+ },
+ body: JSON.stringify({
+ method: 'login',
+ params: [user, password, workspace]
+ })
+ })
+
+ const result = await response.json()
+ const { token } = result.result
+ return token
+}
+
+export async function getUserWorkspaces (token: string): Promise {
+ const accountsUrl = getAccoutsUrlOrFail()
+ const response = await fetch(accountsUrl, {
+ method: 'POST',
+ headers: {
+ Authorization: 'Bearer ' + token,
+ 'Content-Type': 'application/json'
+ },
+ body: JSON.stringify({
+ method: 'getUserWorkspaces',
+ params: []
+ })
+ })
+ const result = await response.json()
+ return (result.result as BaseWorkspaceInfo[]) ?? []
+}
+
+export async function selectWorkspace (token: string, workspace: string): Promise {
+ const accountsUrl = getAccoutsUrlOrFail()
+ const response = await fetch(accountsUrl, {
+ method: 'POST',
+ headers: {
+ 'Content-Type': 'application/json',
+ Authorization: 'Bearer ' + token
+ },
+ body: JSON.stringify({
+ method: 'selectWorkspace',
+ params: [workspace, 'external']
+ })
+ })
+ const result = await response.json()
+ return result.result as WorkspaceLoginInfo
+}
+
+function getAccoutsUrlOrFail (): string {
+ const accountsUrl = getMetadata(plugin.metadata.Endpoint)
+ if (accountsUrl == null) {
+ throw new PlatformError(unknownError('No account endpoint specified'))
+ }
+ return accountsUrl
+}
]