Do auth when importing files from Notion (#6510)

* Do auth when importing files from Notion

* Remove accidental change and fix linter issues
This commit is contained in:
akhismat 2024-09-10 18:58:47 +07:00 committed by GitHub
parent 7ad23c3443
commit f16d109ed5
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 211 additions and 165 deletions

View File

@ -46,7 +46,14 @@ import {
createStorageBackupStorage,
restore
} from '@hcengineering/server-backup'
import serverClientPlugin, { BlobClient, createClient, getTransactorEndpoint } from '@hcengineering/server-client'
import serverClientPlugin, {
BlobClient,
createClient,
getTransactorEndpoint,
getUserWorkspaces,
login,
selectWorkspace
} from '@hcengineering/server-client'
import serverToken, { decodeToken, generateToken } from '@hcengineering/server-token'
import toolPlugin, { connect, FileModelLogger } from '@hcengineering/server-tool'
import path from 'path'
@ -65,14 +72,14 @@ import core, {
systemAccountEmail,
TxOperations,
versionToString,
type WorkspaceIdWithUrl,
type Client as CoreClient,
type Data,
type Doc,
type Ref,
type Tx,
type Version,
type WorkspaceId
type WorkspaceId,
concatLink
} from '@hcengineering/core'
import { consoleModelLogger, type MigrateOperation } from '@hcengineering/model'
import contact from '@hcengineering/model-contact'
@ -98,7 +105,7 @@ import { fixJsonMarkup, migrateMarkup } from './markup'
import { fixMixinForeignAttributes, showMixinForeignAttributes } from './mixin'
import { fixAccountEmails, renameAccount } from './renameAccount'
import { moveFiles, syncFiles } from './storage'
import { importNotion, importToTeamspace } from './notion'
import { importNotion } from './notion'
const colorConstants = {
colorRed: '\u001b[31m',
@ -152,6 +159,15 @@ export function devTool (
return elasticUrl
}
function getFrontUrl (): string {
const frontUrl = process.env.FRONT_URL
if (frontUrl === undefined) {
console.error('please provide front url')
process.exit(1)
}
return frontUrl
}
const initWS = process.env.INIT_WORKSPACE
if (initWS !== undefined) {
setMetadata(toolPlugin.metadata.InitWorkspace, initWS)
@ -212,84 +228,77 @@ export function devTool (
})
})
// import-notion /home/anna/work/notion/pages/exported --workspace workspace
// import-notion-with-teamspaces /home/anna/work/notion/pages/exported --workspace workspace
program
.command('import-notion <dir>')
.command('import-notion-with-teamspaces <dir>')
.description('import extracted archive exported from Notion as "Markdown & CSV"')
.requiredOption('-u, --user <user>', 'user')
.requiredOption('-pw, --password <password>', 'password')
.requiredOption('-ws, --workspace <workspace>', 'workspace where the documents should be imported to')
.action(async (dir: string, cmd) => {
if (cmd.workspace === '') return
const { mongodbUri } = prepareTools()
await withDatabase(mongodbUri, async (db) => {
const ws = await getWorkspaceById(db, cmd.workspace)
if (ws === null) {
console.log('Workspace not found: ', cmd.workspace)
return
}
const wsUrl: WorkspaceIdWithUrl = {
name: ws.workspace,
workspaceName: ws.workspaceName ?? '',
workspaceUrl: ws.workspaceUrl ?? ''
}
await withStorage(mongodbUri, async (storageAdapter) => {
const token = generateToken(systemAccountEmail, { name: ws.workspace })
const endpoint = await getTransactorEndpoint(token, 'external')
const connection = (await connect(endpoint, wsUrl, undefined, {
mode: 'backup'
})) as unknown as CoreClient
const client = new TxOperations(connection, core.account.System)
await importNotion(toolCtx, client, storageAdapter, dir, wsUrl)
await connection.close()
})
})
await importFromNotion(dir, cmd.user, cmd.password, cmd.workspace)
})
// import-notion-to-teamspace /home/anna/work/notion/pages/exported --workspace workspace --teamspace notion
program
.command('import-notion-to-teamspace <dir>')
.description('import extracted archive exported from Notion as "Markdown & CSV"')
.requiredOption('-u, --user <user>', 'user')
.requiredOption('-pw, --password <password>', 'password')
.requiredOption('-ws, --workspace <workspace>', 'workspace where the documents should be imported to')
.requiredOption('-ts, --teamspace <teamspace>', 'teamspace where the documents should be imported to')
.requiredOption('-ts, --teamspace <teamspace>', 'new teamspace name where the documents should be imported to')
.action(async (dir: string, cmd) => {
if (cmd.workspace === '') return
if (cmd.teamspace === '') return
const { mongodbUri } = prepareTools()
await withDatabase(mongodbUri, async (db) => {
const ws = await getWorkspaceById(db, cmd.workspace)
if (ws === null) {
console.log('Workspace not found: ', cmd.workspace)
return
}
const wsUrl: WorkspaceIdWithUrl = {
name: ws.workspace,
workspaceName: ws.workspaceName ?? '',
workspaceUrl: ws.workspaceUrl ?? ''
}
await withStorage(mongodbUri, async (storageAdapter) => {
const token = generateToken(systemAccountEmail, { name: ws.workspace })
const endpoint = await getTransactorEndpoint(token, 'external')
const connection = (await connect(endpoint, wsUrl, undefined, {
mode: 'backup'
})) as unknown as CoreClient
const client = new TxOperations(connection, core.account.System)
await importToTeamspace(toolCtx, client, storageAdapter, dir, wsUrl, cmd.teamspace)
await connection.close()
})
})
await importFromNotion(dir, cmd.user, cmd.password, cmd.workspace, cmd.teamspace)
})
async function importFromNotion (
dir: string,
user: string,
password: string,
workspace: string,
teamspace?: string
): Promise<void> {
if (workspace === '' || user === '' || password === '' || teamspace === '') {
return
}
const userToken = await login(user, password, workspace)
const allWorkspaces = await getUserWorkspaces(userToken)
const workspaces = allWorkspaces.filter((ws) => ws.workspace === workspace)
if (workspaces.length < 1) {
console.log('Workspace not found: ', workspace)
return
}
const selectedWs = await selectWorkspace(userToken, workspaces[0].workspace)
console.log(selectedWs)
function uploader (token: string) {
return (id: string, data: any) => {
return fetch(concatLink(getFrontUrl(), '/files'), {
method: 'POST',
headers: {
Authorization: 'Bearer ' + token
},
body: data
})
}
}
const connection = (await connect(
selectedWs.endpoint,
{
name: selectedWs.workspaceId
},
undefined,
{
mode: 'backup'
}
)) as unknown as CoreClient
const client = new TxOperations(connection, core.account.System)
await importNotion(client, uploader(selectedWs.token), dir, teamspace)
await connection.close()
}
program
.command('reset-account <email>')
.description('create user and corresponding account in master database')

View File

@ -2,15 +2,13 @@ import {
generateId,
type AttachedData,
type Ref,
type WorkspaceIdWithUrl,
makeCollaborativeDoc,
type MeasureMetricsContext,
type TxOperations,
type Blob
type Blob,
collaborativeDocParse
} from '@hcengineering/core'
import { saveCollaborativeDoc } from '@hcengineering/collaboration'
import { yDocToBuffer } from '@hcengineering/collaboration'
import document, { type Document, type Teamspace } from '@hcengineering/document'
import { type StorageAdapter } from '@hcengineering/server-core'
import {
MarkupMarkType,
type MarkupNode,
@ -58,12 +56,13 @@ enum NOTION_MD_LINK_TYPES {
UNKNOWN
}
export type FileUploader = (id: string, data: any) => Promise<any>
export async function importNotion (
ctx: MeasureMetricsContext,
client: TxOperations,
storage: StorageAdapter,
uploadFile: FileUploader,
dir: string,
ws: WorkspaceIdWithUrl
teamspace?: string
): Promise<void> {
const files = await getFilesForImport(dir)
@ -74,13 +73,17 @@ export async function importNotion (
console.log(fileMetaMap)
console.log(documentMetaMap)
const spaceIdMap = await createTeamspaces(fileMetaMap, client)
if (spaceIdMap.size === 0) {
console.error('No teamspaces found in directory: ', dir)
return
if (teamspace === undefined) {
const spaceIdMap = await createTeamspaces(fileMetaMap, client)
if (spaceIdMap.size === 0) {
console.error('No teamspaces found in directory: ', dir)
return
}
await importFiles(client, uploadFile, fileMetaMap, documentMetaMap, spaceIdMap)
} else {
const spaceId = await createTeamspace(teamspace, client)
await importFilesToSpace(client, uploadFile, fileMetaMap, documentMetaMap, spaceId)
}
await importFiles(ctx, client, storage, fileMetaMap, documentMetaMap, spaceIdMap, ws)
}
async function getFilesForImport (dir: string): Promise<Dirent[]> {
@ -91,28 +94,6 @@ async function getFilesForImport (dir: string): Promise<Dirent[]> {
return files
}
export async function importToTeamspace (
ctx: MeasureMetricsContext,
client: TxOperations,
storage: StorageAdapter,
dir: string,
ws: WorkspaceIdWithUrl,
teamspace: string
): Promise<void> {
const files = await getFilesForImport(dir)
const fileMetaMap = new Map<string, FileMetadata>()
const documentMetaMap = new Map<string, DocumentMetadata>()
await collectMetadata(dir, files, fileMetaMap, documentMetaMap)
console.log(fileMetaMap)
console.log(documentMetaMap)
const spaceId = await createTeamspace(teamspace, client)
await importFilesToSpace(ctx, client, storage, fileMetaMap, documentMetaMap, spaceId, ws)
}
async function collectMetadata (
root: string,
files: Dirent[],
@ -205,31 +186,27 @@ async function createTeamspace (name: string, client: TxOperations): Promise<Ref
}
async function importFilesToSpace (
ctx: MeasureMetricsContext,
client: TxOperations,
storage: StorageAdapter,
uploadFile: FileUploader,
fileMetaMap: Map<string, FileMetadata>,
documentMetaMap: Map<string, DocumentMetadata>,
spaceId: Ref<Teamspace>,
ws: WorkspaceIdWithUrl
spaceId: Ref<Teamspace>
): Promise<void> {
for (const [notionId, fileMeta] of fileMetaMap) {
if (!fileMeta.isFolder) {
const docMeta = documentMetaMap.get(notionId)
if (docMeta === undefined) throw new Error('Cannot find metadata for entry: ' + fileMeta.fileName)
await importFile(ctx, client, storage, fileMeta, docMeta, spaceId, documentMetaMap, ws)
await importFile(client, uploadFile, fileMeta, docMeta, spaceId, documentMetaMap)
}
}
}
async function importFiles (
ctx: MeasureMetricsContext,
client: TxOperations,
storage: StorageAdapter,
uploadFile: FileUploader,
fileMetaMap: Map<string, FileMetadata>,
documentMetaMap: Map<string, DocumentMetadata>,
spaceIdMap: Map<string, Ref<Teamspace>>,
ws: WorkspaceIdWithUrl
spaceIdMap: Map<string, Ref<Teamspace>>
): Promise<void> {
for (const [notionId, fileMeta] of fileMetaMap) {
if (!fileMeta.isFolder) {
@ -241,20 +218,18 @@ async function importFiles (
throw new Error('Teamspace not found for document: ' + docMeta.name)
}
await importFile(ctx, client, storage, fileMeta, docMeta, spaceId, documentMetaMap, ws)
await importFile(client, uploadFile, fileMeta, docMeta, spaceId, documentMetaMap)
}
}
}
async function importFile (
ctx: MeasureMetricsContext,
client: TxOperations,
storage: StorageAdapter,
uploadFile: FileUploader,
fileMeta: FileMetadata,
docMeta: DocumentMetadata,
spaceId: Ref<Teamspace>,
documentMetaMap: Map<string, DocumentMetadata>,
ws: WorkspaceIdWithUrl
documentMetaMap: Map<string, DocumentMetadata>
): Promise<void> {
await new Promise<void>((resolve, reject) => {
if (fileMeta.isFolder) throw new Error('Importing folder entry is not supported: ' + fileMeta.fileName)
@ -268,7 +243,7 @@ async function importFile (
notionParentId !== undefined && notionParentId !== '' ? documentMetaMap.get(notionParentId) : undefined
const processFileData = getDataProcessor(fileMeta, docMeta)
processFileData(ctx, client, storage, ws, data, docMeta, spaceId, parentMeta, documentMetaMap)
processFileData(client, uploadFile, data, docMeta, spaceId, parentMeta, documentMetaMap)
.then(() => {
console.log('IMPORT SUCCEED:', docMeta.name)
console.log('------------------------------------------------------------------')
@ -292,10 +267,8 @@ async function importFile (
}
type DataProcessor = (
ctx: MeasureMetricsContext,
client: TxOperations,
storage: StorageAdapter,
ws: WorkspaceIdWithUrl,
uploadFile: FileUploader,
data: Buffer,
docMeta: DocumentMetadata,
space: Ref<Teamspace>,
@ -328,10 +301,8 @@ function getDataProcessor (fileMeta: FileMetadata, docMeta: DocumentMetadata): D
}
async function createDBPageWithAttachments (
ctx: MeasureMetricsContext,
client: TxOperations,
storage: StorageAdapter,
ws: WorkspaceIdWithUrl,
uploadFile: FileUploader,
data: Buffer,
docMeta: DocumentMetadata,
space: Ref<Teamspace>,
@ -380,14 +351,12 @@ async function createDBPageWithAttachments (
size: docMeta.size
}
await importAttachment(ctx, client, storage, ws, data, attachment, space, dbPage)
await importAttachment(client, uploadFile, data, attachment, space, dbPage)
}
async function importDBAttachment (
ctx: MeasureMetricsContext,
client: TxOperations,
storage: StorageAdapter,
ws: WorkspaceIdWithUrl,
uploadFile: FileUploader,
data: Buffer,
docMeta: DocumentMetadata,
space: Ref<Teamspace>,
@ -413,14 +382,12 @@ async function importDBAttachment (
mimeType: docMeta.mimeType,
size: docMeta.size
}
await importAttachment(ctx, client, storage, ws, data, attachment, space, dbPage)
await importAttachment(client, uploadFile, data, attachment, space, dbPage)
}
async function importAttachment (
ctx: MeasureMetricsContext,
client: TxOperations,
storage: StorageAdapter,
ws: WorkspaceIdWithUrl,
uploadFile: FileUploader,
data: Buffer,
docMeta: DocumentMetadata,
space: Ref<Teamspace>,
@ -433,7 +400,17 @@ async function importAttachment (
const size = docMeta.size ?? 0
const type = docMeta.mimeType ?? DEFAULT_ATTACHMENT_MIME_TYPE
await storage.put(ctx, ws, docMeta.id, data, type, size)
const form = new FormData()
const file = new File([new Blob([data])], docMeta.name)
form.append('file', file, docMeta.id)
form.append('type', type)
form.append('size', size.toString())
form.append('name', docMeta.name)
form.append('id', docMeta.id)
form.append('data', new Blob([data])) // ?
await uploadFile(docMeta.id, form)
const attachedData: AttachedData<Attachment> = {
file: docMeta.id as Ref<Blob>,
@ -455,10 +432,8 @@ async function importAttachment (
}
async function importPageDocument (
ctx: MeasureMetricsContext,
client: TxOperations,
storage: StorageAdapter,
ws: WorkspaceIdWithUrl,
uploadFile: FileUploader,
data: Buffer,
docMeta: DocumentMetadata,
space: Ref<Teamspace>,
@ -474,7 +449,19 @@ async function importPageDocument (
const id = docMeta.id as Ref<Document>
const collabId = makeCollaborativeDoc(id, 'content')
const yDoc = jsonToYDocNoSchema(json, 'content')
await saveCollaborativeDoc(storage, ws, collabId, yDoc, ctx)
const { documentId } = collaborativeDocParse(collabId)
const buffer = yDocToBuffer(yDoc)
const form = new FormData()
const file = new File([new Blob([buffer])], docMeta.name)
form.append('file', file, documentId)
form.append('type', 'application/ydoc')
form.append('size', buffer.length.toString())
form.append('name', docMeta.name)
form.append('id', docMeta.id)
form.append('data', new Blob([buffer])) // ?
await uploadFile(docMeta.id, form)
const parentId = parentMeta?.id ?? document.ids.NoParent

View File

@ -18,11 +18,19 @@ import { getMetadata, PlatformError, unknownError } from '@hcengineering/platfor
import plugin from './plugin'
export interface WorkspaceLoginInfo extends LoginInfo {
workspace: string
workspaceId: string
}
export interface LoginInfo {
token: string
endpoint: string
confirmed: boolean
email: string
}
export async function listAccountWorkspaces (token: string): Promise<BaseWorkspaceInfo[]> {
const accountsUrl = getMetadata(plugin.metadata.Endpoint)
if (accountsUrl == null) {
throw new PlatformError(unknownError('No account endpoint specified'))
}
const accountsUrl = getAccoutsUrlOrFail()
const workspaces = await (
await fetch(accountsUrl, {
method: 'POST',
@ -44,11 +52,7 @@ export async function getTransactorEndpoint (
kind: 'internal' | 'external' = 'internal',
timeout: number = -1
): Promise<string> {
const accountsUrl = getMetadata(plugin.metadata.Endpoint)
if (accountsUrl == null) {
throw new PlatformError(unknownError('No account endpoint specified'))
}
const accountsUrl = getAccoutsUrlOrFail()
const st = Date.now()
while (true) {
try {
@ -86,11 +90,7 @@ export async function getPendingWorkspace (
version: Data<Version>,
operation: 'create' | 'upgrade' | 'all'
): Promise<BaseWorkspaceInfo | undefined> {
const accountsUrl = getMetadata(plugin.metadata.Endpoint)
if (accountsUrl == null) {
throw new PlatformError(unknownError('No account endpoint specified'))
}
const accountsUrl = getAccoutsUrlOrFail()
const workspaces = await (
await fetch(accountsUrl, {
method: 'POST',
@ -115,10 +115,7 @@ export async function updateWorkspaceInfo (
progress: number,
message?: string
): Promise<void> {
const accountsUrl = getMetadata(plugin.metadata.Endpoint)
if (accountsUrl == null) {
throw new PlatformError(unknownError('No account endpoint specified'))
}
const accountsUrl = getAccoutsUrlOrFail()
await (
await fetch(accountsUrl, {
method: 'POST',
@ -139,11 +136,7 @@ export async function workerHandshake (
version: Data<Version>,
operation: 'create' | 'upgrade' | 'all'
): Promise<void> {
const accountsUrl = getMetadata(plugin.metadata.Endpoint)
if (accountsUrl == null) {
throw new PlatformError(unknownError('No account endpoint specified'))
}
const accountsUrl = getAccoutsUrlOrFail()
await fetch(accountsUrl, {
method: 'POST',
headers: {
@ -157,10 +150,7 @@ export async function workerHandshake (
}
export async function getWorkspaceInfo (token: string): Promise<BaseWorkspaceInfo | undefined> {
const accountsUrl = getMetadata(plugin.metadata.Endpoint)
if (accountsUrl == null) {
throw new PlatformError(unknownError('No account endpoint specified'))
}
const accountsUrl = getAccoutsUrlOrFail()
const workspaceInfo = await (
await fetch(accountsUrl, {
method: 'POST',
@ -177,3 +167,63 @@ export async function getWorkspaceInfo (token: string): Promise<BaseWorkspaceInf
return workspaceInfo.result as BaseWorkspaceInfo | undefined
}
export async function login (user: string, password: string, workspace: string): Promise<string> {
const accountsUrl = getAccoutsUrlOrFail()
const response = await fetch(accountsUrl, {
method: 'POST',
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify({
method: 'login',
params: [user, password, workspace]
})
})
const result = await response.json()
const { token } = result.result
return token
}
export async function getUserWorkspaces (token: string): Promise<BaseWorkspaceInfo[]> {
const accountsUrl = getAccoutsUrlOrFail()
const response = await fetch(accountsUrl, {
method: 'POST',
headers: {
Authorization: 'Bearer ' + token,
'Content-Type': 'application/json'
},
body: JSON.stringify({
method: 'getUserWorkspaces',
params: []
})
})
const result = await response.json()
return (result.result as BaseWorkspaceInfo[]) ?? []
}
export async function selectWorkspace (token: string, workspace: string): Promise<WorkspaceLoginInfo> {
const accountsUrl = getAccoutsUrlOrFail()
const response = await fetch(accountsUrl, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
Authorization: 'Bearer ' + token
},
body: JSON.stringify({
method: 'selectWorkspace',
params: [workspace, 'external']
})
})
const result = await response.json()
return result.result as WorkspaceLoginInfo
}
function getAccoutsUrlOrFail (): string {
const accountsUrl = getMetadata(plugin.metadata.Endpoint)
if (accountsUrl == null) {
throw new PlatformError(unknownError('No account endpoint specified'))
}
return accountsUrl
}