fix: get rid of node-fetch (#9150)

Signed-off-by: Alexander Onnikov <Alexander.Onnikov@xored.com>
2025-06-09 09:20:54 +00:00 · 2025-06-02 16:43:51 +07:00 · 2025-06-02 16:43:51 +07:00 · d5919f3de9
commit d5919f3de9
parent cfafaaf025
5 changed files with 57 additions and 53 deletions
--- a/common/config/rush/pnpm-lock.yaml
+++ b/common/config/rush/pnpm-lock.yaml
@ -135,7 +135,7 @@ importers:
        version: file:projects/analytics-service.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(@types/node@22.15.29)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(ts-node@10.9.2(@types/node@22.15.29)(typescript@5.3.3))
      '@rush-temp/api-client':
        specifier: file:./projects/api-client.tgz
-        version: file:projects/api-client.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(bufferutil@4.0.8)(encoding@0.1.13)(esbuild@0.24.2)(utf-8-validate@6.0.4)
+        version: file:projects/api-client.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(bufferutil@4.0.8)(esbuild@0.24.2)(utf-8-validate@6.0.4)
      '@rush-temp/api-tests':
        specifier: file:./projects/api-tests.tgz
        version: file:projects/api-tests.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(ts-node@10.9.2(@types/node@22.15.29)(typescript@5.3.3))
@ -255,7 +255,7 @@ importers:
        version: file:projects/core.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(@types/node@22.15.29)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(ts-node@10.9.2(@types/node@22.15.29)(typescript@5.3.3))
      '@rush-temp/datalake':
        specifier: file:./projects/datalake.tgz
-        version: file:projects/datalake.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(encoding@0.1.13)(esbuild@0.24.2)
+        version: file:projects/datalake.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)
      '@rush-temp/desktop':
        specifier: file:./projects/desktop.tgz
        version: file:projects/desktop.tgz(@babel/core@7.23.9)(bufferutil@4.0.8)(sass@1.71.1)(utf-8-validate@6.0.4)
@ -1570,9 +1570,6 @@ importers:
      '@types/node-cron':
        specifier: ^3.0.11
        version: 3.0.11
-      '@types/node-fetch':
-        specifier: ~2.6.2
-        version: 2.6.12
      '@types/nodemailer':
        specifier: ^6.4.17
        version: 6.4.17
@ -1981,9 +1978,6 @@ importers:
      node-cron:
        specifier: ^3.0.3
        version: 3.0.3
-      node-fetch:
-        specifier: ^2.6.6
-        version: 2.7.0(encoding@0.1.13)
      node-forge:
        specifier: ^1.3.1
        version: 1.3.1
@ -16749,11 +16743,10 @@ snapshots:
      - supports-color
      - ts-node

-  '@rush-temp/api-client@file:projects/api-client.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(bufferutil@4.0.8)(encoding@0.1.13)(esbuild@0.24.2)(utf-8-validate@6.0.4)':
+  '@rush-temp/api-client@file:projects/api-client.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(bufferutil@4.0.8)(esbuild@0.24.2)(utf-8-validate@6.0.4)':
    dependencies:
      '@types/jest': 29.5.12
      '@types/node': 22.15.29
-      '@types/node-fetch': 2.6.12
      '@types/snappyjs': 0.7.1
      '@types/ws': 8.5.11
      '@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.6.2)
@ -16763,9 +16756,7 @@ snapshots:
      eslint-plugin-import: 2.29.1(eslint@8.56.0)
      eslint-plugin-n: 15.7.0(eslint@8.56.0)
      eslint-plugin-promise: 6.1.1(eslint@8.56.0)
-      form-data: 4.0.0
-      jest: 29.7.0(@types/node@22.15.29)(ts-node@10.9.2(@types/node@22.15.29)(typescript@5.3.3))
-      node-fetch: 2.7.0(encoding@0.1.13)
+      jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3))
      prettier: 3.2.5
      simplytyped: 3.3.0(typescript@5.6.2)
      snappyjs: 0.7.0
@ -16782,7 +16773,6 @@ snapshots:
      - babel-jest
      - babel-plugin-macros
      - bufferutil
-      - encoding
      - esbuild
      - node-notifier
      - supports-color
@ -18024,11 +18014,10 @@ snapshots:
      - supports-color
      - ts-node

-  '@rush-temp/datalake@file:projects/datalake.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(encoding@0.1.13)(esbuild@0.24.2)':
+  '@rush-temp/datalake@file:projects/datalake.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)':
    dependencies:
      '@types/jest': 29.5.12
      '@types/node': 22.15.29
-      '@types/node-fetch': 2.6.12
      '@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.3.3)
      '@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.3.3)
      eslint: 8.56.0
@ -18036,9 +18025,7 @@ snapshots:
      eslint-plugin-import: 2.29.1(eslint@8.56.0)
      eslint-plugin-n: 15.7.0(eslint@8.56.0)
      eslint-plugin-promise: 6.1.1(eslint@8.56.0)
-      form-data: 4.0.0
-      jest: 29.7.0(@types/node@22.15.29)(ts-node@10.9.2(@types/node@22.15.29)(typescript@5.3.3))
-      node-fetch: 2.7.0(encoding@0.1.13)
+      jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3))
      prettier: 3.2.5
      ts-jest: 29.1.2(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(jest@29.7.0(@types/node@22.15.29)(ts-node@10.9.2(@types/node@22.15.29)(typescript@5.3.3)))(typescript@5.3.3)
      ts-node: 10.9.2(@types/node@22.15.29)(typescript@5.3.3)
@ -18050,7 +18037,6 @@ snapshots:
      - '@swc/wasm'
      - babel-jest
      - babel-plugin-macros
-      - encoding
      - esbuild
      - node-notifier
      - supports-color
--- a/packages/api-client/package.json
+++ b/packages/api-client/package.json
@ -47,7 +47,6 @@
    "@types/node": "^22.15.29",
    "@types/jest": "^29.5.5",
    "@types/ws": "^8.5.11",
-    "@types/node-fetch": "~2.6.2",
    "@types/snappyjs": "^0.7.1"
  },
  "dependencies": {
@ -59,8 +58,6 @@
    "@hcengineering/platform": "^0.6.11",
    "@hcengineering/text": "^0.6.5",
    "@hcengineering/text-markdown": "^0.6.0",
-    "form-data": "^4.0.0",
-    "node-fetch": "^2.6.6",
    "snappyjs": "^0.7.0"
  },
  "repository": "https://github.com/hcengineering/platform",
--- a/packages/api-client/src/storage/client.ts
+++ b/packages/api-client/src/storage/client.ts
@ -14,14 +14,12 @@
 //

 import core, { concatLink, WorkspaceUuid, Blob, Ref } from '@hcengineering/core'
-import FormData from 'form-data'
 import { Readable } from 'stream'
 import { StorageClient } from './types'
 import { loadServerConfig, ServerConfig } from '../config'
 import { NetworkError, NotFoundError, StorageError } from './error'
 import { AuthOptions } from '../types'
 import { getWorkspaceToken } from '../utils'
-import nodeFetch from 'node-fetch'

 interface ObjectMetadata {
  name: string
@ -102,16 +100,13 @@ export class StorageClientImpl implements StorageClient {
  }

  async put (objectName: string, stream: Readable | Buffer | string, contentType: string, size?: number): Promise<Blob> {
+    const buffer = await toBuffer(stream)
+    const file = new File([buffer], objectName, { type: contentType })
    const formData = new FormData()
-    const options: FormData.AppendOptions = {
-      filename: objectName,
-      contentType,
-      knownLength: size
-    }
-    formData.append('file', stream, options)
+    formData.append('file', file)
    let response
    try {
-      response = await nodeFetch(this.uploadUrl, {
+      response = await fetch(this.uploadUrl, {
        method: 'POST',
        body: formData,
        headers: { ...this.headers }
@ -168,6 +163,22 @@ export class StorageClientImpl implements StorageClient {
  }
 }

+async function toBuffer (data: Buffer | string | Readable): Promise<Buffer> {
+  if (Buffer.isBuffer(data)) {
+    return data
+  } else if (typeof data === 'string') {
+    return Buffer.from(data)
+  } else if (data instanceof Readable) {
+    const chunks: Buffer[] = []
+    for await (const chunk of data) {
+      chunks.push(chunk)
+    }
+    return Buffer.concat(chunks as any)
+  } else {
+    throw new TypeError('Unsupported data type')
+  }
+}
+
 async function wrappedFetch (url: string | URL, init?: RequestInit): Promise<Response> {
  let response: Response
  try {
--- a/server/datalake/package.json
+++ b/server/datalake/package.json
@ -32,15 +32,12 @@
    "jest": "^29.7.0",
    "ts-jest": "^29.1.1",
    "@types/jest": "^29.5.5",
-    "@types/node-fetch": "~2.6.2",
    "ts-node": "^10.8.0"
  },
  "dependencies": {
    "@hcengineering/core": "^0.6.32",
    "@hcengineering/platform": "^0.6.11",
    "@hcengineering/server-core": "^0.6.1",
-    "@hcengineering/server-token": "^0.6.11",
-    "node-fetch": "^2.6.6",
-    "form-data": "^4.0.0"
+    "@hcengineering/server-token": "^0.6.11"
  }
 }
--- a/server/datalake/src/client.ts
+++ b/server/datalake/src/client.ts
@ -14,8 +14,6 @@
 //

 import { type MeasureContext, type WorkspaceUuid, concatLink } from '@hcengineering/core'
-import FormData from 'form-data'
-import fetch, { type RequestInfo, type RequestInit, type Response } from 'node-fetch'
 import { Readable } from 'stream'

 import { DatalakeError, NetworkError, NotFoundError } from './error'
@ -130,7 +128,7 @@ export class DatalakeClient {
      throw new DatalakeError('Missing response body')
    }

-    return Readable.from(response.body)
+    return Readable.fromWeb(response.body as any)
  }

  async getPartialObject (
@ -162,7 +160,7 @@ export class DatalakeClient {
      throw new DatalakeError('Missing response body')
    }

-    return Readable.from(response.body)
+    return Readable.fromWeb(response.body as any)
  }

  async statObject (
@ -258,21 +256,18 @@ export class DatalakeClient {
    const path = `/upload/form-data/${workspace}`
    const url = concatLink(this.endpoint, path)

+    const buffer = await toBuffer(stream)
+    const file = new File([buffer], objectName, { type: params.type, lastModified: params.lastModified })
+
    const form = new FormData()
-    const options: FormData.AppendOptions = {
-      filename: objectName,
-      contentType: params.type,
-      knownLength: params.size,
-      header: {
-        'Last-Modified': params.lastModified
-      }
-    }
-    form.append('file', stream, options)
+    form.append('file', file)

    const response = await fetchSafe(ctx, url, {
      method: 'POST',
-      body: form,
-      headers: { ...this.headers }
+      body: form as unknown as BodyInit,
+      headers: {
+        ...this.headers
+      }
    })

    const result = (await response.json()) as BlobUploadResult[]
@ -404,13 +399,15 @@ export class DatalakeClient {
    objectName: string,
    multipart: MultipartUpload,
    partNumber: number,
-    body: Readable | Buffer | string
+    data: Readable | Buffer | string
  ): Promise<MultipartUploadPart> {
    const path = `/upload/multipart/${workspace}/${encodeURIComponent(objectName)}/part`
    const url = new URL(concatLink(this.endpoint, path))
    url.searchParams.set('uploadId', multipart.uploadId)
    url.searchParams.set('partNumber', partNumber.toString())

+    const body = data instanceof Readable ? (Readable.toWeb(data) as ReadableStream) : data
+
    try {
      const response = await fetchSafe(ctx, url, {
        method: 'PUT',
@ -470,6 +467,22 @@ export class DatalakeClient {
  }
 }

+async function toBuffer (data: Buffer | string | Readable): Promise<Buffer> {
+  if (Buffer.isBuffer(data)) {
+    return data
+  } else if (typeof data === 'string') {
+    return Buffer.from(data)
+  } else if (data instanceof Readable) {
+    const chunks: Buffer[] = []
+    for await (const chunk of data) {
+      chunks.push(chunk)
+    }
+    return Buffer.concat(chunks as any)
+  } else {
+    throw new TypeError('Unsupported data type')
+  }
+}
+
 async function * getChunks (data: Buffer | string | Readable, chunkSize: number): AsyncGenerator<Buffer> {
  if (Buffer.isBuffer(data)) {
    let offset = 0
@ -497,7 +510,7 @@ async function * getChunks (data: Buffer | string | Readable, chunkSize: number)
  }
 }

-async function fetchSafe (ctx: MeasureContext, url: RequestInfo, init?: RequestInit): Promise<Response> {
+async function fetchSafe (ctx: MeasureContext, url: string | URL, init?: RequestInit): Promise<Response> {
  let response
  try {
    response = await fetch(url, init)