From afbe37d7efae9a51afa388a93aa083862e53b1e3 Mon Sep 17 00:00:00 2001
From: Andrey Sobolev <haiodo@users.noreply.github.com>
Date: Mon, 16 Sep 2024 15:04:05 +0700
Subject: [PATCH] UBERF-8122: Fix backup service (#6577)
Signed-off-by: Andrey Sobolev <haiodo@gmail.com>
---
pods/backup/package.json | 1 +
server/backup/src/service.ts | 4 ++--
server/client/src/blob.ts | 7 ++++++-
server/front/src/index.ts | 14 ++++++++++++--
server/server-storage/src/starter.ts | 6 +++---
server/ws/src/blobs.ts | 7 ++++++-
6 files changed, 30 insertions(+), 9 deletions(-)
diff --git a/pods/backup/package.json b/pods/backup/package.json
index 13f45b9e88..f3c10469ef 100644
--- a/pods/backup/package.json
+++ b/pods/backup/package.json
@@ -16,6 +16,7 @@
"_phase:docker-staging": "rushx docker:staging",
"bundle": "mkdir -p bundle && esbuild src/index.ts --bundle --sourcemap=inline --platform=node --external:*.node --external:bufferutil --external:snappy > bundle/bundle.js",
"docker:build": "../../common/scripts/docker_build.sh hardcoreeng/backup",
+ "docker:tbuild": "docker build -t hardcoreeng/backup . --platform=linux/amd64 && ../../common/scripts/docker_tag_push.sh hardcoreeng/backup",
"docker:staging": "../../common/scripts/docker_tag.sh hardcoreeng/backup staging",
"docker:push": "../../common/scripts/docker_tag.sh hardcoreeng/backup",
"run-local": "cross-env ACCOUNTS_URL=http://localhost:3000/ SECRET=secret MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin MINIO_ENDPOINT=localhost BUCKET_NAME=backups INTERVAL=30 ts-node src/index.ts",
diff --git a/server/backup/src/service.ts b/server/backup/src/service.ts
index 6443e40f15..ecaf26d10d 100644
--- a/server/backup/src/service.ts
+++ b/server/backup/src/service.ts
@@ -106,7 +106,7 @@ class BackupWorker {
})
const childLogger = rootCtx.logger.childLogger?.(ws.workspace, {
workspace: ws.workspace,
- enableConsole: 'false'
+ enableConsole: 'true'
})
const ctx = rootCtx.newChild(ws.workspace, { workspace: ws.workspace }, {}, childLogger)
try {
@@ -124,7 +124,7 @@ class BackupWorker {
const pipeline = await this.pipelineFactory(ctx, wsUrl, true, () => {}, null)
await ctx.with('backup', { workspace: ws.workspace }, async (ctx) => {
- await backup(ctx, ws.endpoint, getWorkspaceId(ws.workspace), storage, {
+ await backup(ctx, '', getWorkspaceId(ws.workspace), storage, {
skipDomains: [],
force: false,
recheck: false,
diff --git a/server/client/src/blob.ts b/server/client/src/blob.ts
index fcf05d0e10..febeda76bb 100644
--- a/server/client/src/blob.ts
+++ b/server/client/src/blob.ts
@@ -173,7 +173,12 @@ export class BlobClient {
}
break
} catch (err: any) {
- if (err?.code === 'NoSuchKey') {
+ if (
+ err?.code === 'NoSuchKey' ||
+ err?.code === 'NotFound' ||
+ err?.message === 'No such key' ||
+ err?.Code === 'NoSuchKey'
+ ) {
ctx.info('No such key', { name })
return
}
diff --git a/server/front/src/index.ts b/server/front/src/index.ts
index 028e8fa87e..03e3d63040 100644
--- a/server/front/src/index.ts
+++ b/server/front/src/index.ts
@@ -135,7 +135,12 @@ async function getFileRange (
})
})
} catch (err: any) {
- if (err?.code === 'NoSuchKey' || err?.code === 'NotFound') {
+ if (
+ err?.code === 'NoSuchKey' ||
+ err?.code === 'NotFound' ||
+ err?.message === 'No such key' ||
+ err?.Code === 'NoSuchKey'
+ ) {
ctx.info('No such key', { workspace: workspace.name, uuid })
res.status(404).send()
return
@@ -459,7 +464,12 @@ export function start (
)
}
} catch (error: any) {
- if (error?.code === 'NoSuchKey' || error?.code === 'NotFound' || error?.message === 'No such key') {
+ if (
+ error?.code === 'NoSuchKey' ||
+ error?.code === 'NotFound' ||
+ error?.message === 'No such key' ||
+ error?.Code === 'NoSuchKey'
+ ) {
ctx.error('No such storage key', {
file: req.query.file,
workspace: payload?.workspace,
diff --git a/server/server-storage/src/starter.ts b/server/server-storage/src/starter.ts
index 61c0c40b4f..97c769cac3 100644
--- a/server/server-storage/src/starter.ts
+++ b/server/server-storage/src/starter.ts
@@ -1,4 +1,4 @@
-import { type DatalakeConfig, DatalakeService } from '@hcengineering/datalake'
+import { DatalakeService, type DatalakeConfig } from '@hcengineering/datalake'
import { MinioConfig, MinioService, addMinioFallback } from '@hcengineering/minio'
import { createRawMongoDBAdapter } from '@hcengineering/mongo'
import { S3Service, type S3Config } from '@hcengineering/s3'
@@ -30,10 +30,10 @@ import {
*/
-export function storageConfigFromEnv (configEnv: string = 'STORAGE_CONFIG'): StorageConfiguration {
+export function storageConfigFromEnv (configEnv?: string): StorageConfiguration {
const storageConfig: StorageConfiguration = { default: '', storages: [] }
- const storageEnv = process.env[configEnv]
+ const storageEnv = configEnv ?? process.env.STORAGE_CONFIG
if (storageEnv !== undefined) {
parseStorageEnv(storageEnv, storageConfig)
}
diff --git a/server/ws/src/blobs.ts b/server/ws/src/blobs.ts
index a7d6d08b7d..c55660b3a4 100644
--- a/server/ws/src/blobs.ts
+++ b/server/ws/src/blobs.ts
@@ -167,7 +167,12 @@ export async function getFileRange (
})
})
} catch (err: any) {
- if (err?.code === 'NoSuchKey' || err?.code === 'NotFound') {
+ if (
+ err?.code === 'NoSuchKey' ||
+ err?.code === 'NotFound' ||
+ err?.message === 'No such key' ||
+ err?.Code === 'NoSuchKey'
+ ) {
ctx.info('No such key', { workspace: workspace.name, uuid })
res.cork(() => {
res.status(404)