wieerwill/platform
1
0
Fork 0
mirror of https://github.com/hcengineering/platform.git synced 2025-05-18 13:37:59 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

fix: remove datalake extra retries on 404 (#8962)

Browse Source 
Signed-off-by: Alexander Onnikov <Alexander.Onnikov@xored.com>
This commit is contained in:
Alexander Onnikov 2025-05-16 14:59:07 +07:00 committed by GitHub
parent b5da594b5e
commit 4aef00c462
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 25 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
server/datalake/src/client.ts
View File

@ -111,16 +111,17 @@ export class DatalakeClient {
return (await response.json()) as ListObjectOutput
}
async getObject (ctx: MeasureContext, workspace: WorkspaceUuid, objectName: string): Promise<Readable> {
async getObject (ctx: MeasureContext, workspace: WorkspaceUuid, objectName: string): Promise<Readable | undefined> {
const url = this.getObjectUrl(ctx, workspace, objectName)
let response
try {
response = await fetchSafe(ctx, url, { headers: { ...this.headers } })
} catch (err: any) {
if (err.name !== 'NotFoundError') {
console.error('failed to get object', { workspace, objectName, err })
if (err.name === 'NotFoundError') {
return undefined
}
console.error('failed to get object', { workspace, objectName, err })
throw err
}
@ -138,7 +139,7 @@ export class DatalakeClient {
objectName: string,
offset: number,
length?: number
): Promise<Readable> {
): Promise<Readable | undefined> {
const url = this.getObjectUrl(ctx, workspace, objectName)
const headers = {
...this.headers,
@ -149,9 +150,10 @@ export class DatalakeClient {
try {
response = await fetchSafe(ctx, url, { headers })
} catch (err: any) {
if (err.name !== 'NotFoundError') {
console.error('failed to get partial object', { workspace, objectName, err })
if (err.name === 'NotFoundError') {
return undefined
}
console.error('failed to get partial object', { workspace, objectName, err })
throw err
}

20
server/datalake/src/index.ts
View File

@ -33,6 +33,7 @@ import {
import { generateToken } from '@hcengineering/server-token'
import { type Readable } from 'stream'
import { type UploadObjectParams, DatalakeClient } from './client'
import { NotFoundError } from './error'
export { DatalakeClient }
@ -168,7 +169,11 @@ export class DatalakeService implements StorageAdapter {
@withContext('get')
async get (ctx: MeasureContext, wsIds: WorkspaceIds, objectName: string): Promise<Readable> {
return await this.retry(ctx, () => this.client.getObject(ctx, wsIds.uuid, objectName))
const object = await this.retry(ctx, () => this.client.getObject(ctx, wsIds.uuid, objectName))
if (object === undefined) {
throw new NotFoundError()
}
return object
}
@withContext('put')
@ -199,8 +204,11 @@ export class DatalakeService implements StorageAdapter {
@withContext('read')
async read (ctx: MeasureContext, wsIds: WorkspaceIds, objectName: string): Promise<Buffer[]> {
const data = await this.retry(ctx, () => this.client.getObject(ctx, wsIds.uuid, objectName))
const chunks: Buffer[] = []
if (data === undefined) {
throw new NotFoundError()
}
const chunks: Buffer[] = []
for await (const chunk of data) {
chunks.push(chunk)
}
@ -216,7 +224,13 @@ export class DatalakeService implements StorageAdapter {
offset: number,
length?: number
): Promise<Readable> {
return await this.retry(ctx, () => this.client.getPartialObject(ctx, wsIds.uuid, objectName, offset, length))
const object = await this.retry(ctx, () =>
this.client.getPartialObject(ctx, wsIds.uuid, objectName, offset, length)
)
if (object === undefined) {
throw new NotFoundError()
}
return object
}
async getUrl (ctx: MeasureContext, wsIds: WorkspaceIds, objectName: string): Promise<string> {