wieerwill/platform
1
0
Fork 0
mirror of https://github.com/hcengineering/platform.git synced 2025-05-31 04:38:02 +00:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

UBERF-8100: Fix backup (#6558)

Browse Source 
Signed-off-by: Andrey Sobolev <haiodo@gmail.com>
This commit is contained in:
Andrey Sobolev 2024-09-13 17:25:53 +07:00 committed by GitHub
parent 939103a116
commit 0a32f8885b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 25 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
pods/server/Dockerfile
View File

@ -16,4 +16,4 @@ COPY bundle/bundle.js.map ./
EXPOSE 8080 EXPOSE 8080
ENV UWS_HTTP_MAX_HEADERS_SIZE 32768 ENV UWS_HTTP_MAX_HEADERS_SIZE 32768
CMD node --enable-source-maps --inspect=0.0.0.0:9229 bundle.js CMD node bundle.js

4
pods/server/package.json
View File

@ -8,8 +8,8 @@
"template": "@hcengineering/node-package", "template": "@hcengineering/node-package",
"license": "EPL-2.0", "license": "EPL-2.0",
"scripts": { "scripts": {
"start": "rush bundle --to @hcengineering/pod-server && cross-env NODE_ENV=production ELASTIC_INDEX_NAME=local_storage_index MODEL_VERSION=$(node ../../common/scripts/show_version.js) ACCOUNTS_URL=http://localhost:3000 REKONI_URL=http://localhost:4004 MONGO_URL=mongodb://localhost:27017 ELASTIC_URL=http://localhost:9200 FRONT_URL=http://localhost:8087 UPLOAD_URL=/upload MINIO_ENDPOINT=localhost MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin METRICS_CONSOLE=true SERVER_SECRET=secret OPERATION_PROFILING=false node --inspect --enable-source-maps bundle/bundle.js", "start": "rush bundle --to @hcengineering/pod-server && cross-env NODE_ENV=production ELASTIC_INDEX_NAME=local_storage_index MODEL_VERSION=$(node ../../common/scripts/show_version.js) ACCOUNTS_URL=http://localhost:3000 REKONI_URL=http://localhost:4004 MONGO_URL=mongodb://localhost:27017 ELASTIC_URL=http://localhost:9200 FRONT_URL=http://localhost:8087 UPLOAD_URL=/upload MINIO_ENDPOINT=localhost MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin METRICS_CONSOLE=true SERVER_SECRET=secret OPERATION_PROFILING=false node bundle/bundle.js",
"start-u": "rush bundle --to @hcengineering/pod-server && cp ./node_modules/@hcengineering/uws/lib/*.node ./bundle/ && cross-env NODE_ENV=production SERVER_PROVIDER=uweb ELASTIC_INDEX_NAME=local_storage_index MODEL_VERSION=$(node ../../common/scripts/show_version.js) ACCOUNTS_URL=http://localhost:3000 REKONI_URL=http://localhost:4004 MONGO_URL=mongodb://localhost:27017 ELASTIC_URL=http://localhost:9200 FRONT_URL=http://localhost:8087 UPLOAD_URL=/upload MINIO_ENDPOINT=localhost MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin METRICS_CONSOLE=true SERVER_SECRET=secret node --inspect bundle/bundle.js", "start-u": "rush bundle --to @hcengineering/pod-server && cp ./node_modules/@hcengineering/uws/lib/*.node ./bundle/ && cross-env NODE_ENV=production SERVER_PROVIDER=uweb ELASTIC_INDEX_NAME=local_storage_index MODEL_VERSION=$(node ../../common/scripts/show_version.js) ACCOUNTS_URL=http://localhost:3000 REKONI_URL=http://localhost:4004 MONGO_URL=mongodb://localhost:27017 ELASTIC_URL=http://localhost:9200 FRONT_URL=http://localhost:8087 UPLOAD_URL=/upload MINIO_ENDPOINT=localhost MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin METRICS_CONSOLE=true SERVER_SECRET=secret node bundle/bundle.js",
"start-flame": "rush bundle --to @hcengineering/pod-server && cross-env NODE_ENV=production ELASTIC_INDEX_NAME=local_storage_index MODEL_VERSION=$(node ../../common/scripts/show_version.js) ACCOUNTS_URL=http://localhost:3000 REKONI_URL=http://localhost:4004 MONGO_URL=mongodb://localhost:27017 ELASTIC_URL=http://localhost:9200 FRONT_URL=http://localhost:8087 UPLOAD_URL=/upload MINIO_ENDPOINT=localhost MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin METRICS_CONSOLE=true SERVER_SECRET=secret clinic flame --dest ./out -- node --nolazy -r ts-node/register --enable-source-maps src/__start.ts", "start-flame": "rush bundle --to @hcengineering/pod-server && cross-env NODE_ENV=production ELASTIC_INDEX_NAME=local_storage_index MODEL_VERSION=$(node ../../common/scripts/show_version.js) ACCOUNTS_URL=http://localhost:3000 REKONI_URL=http://localhost:4004 MONGO_URL=mongodb://localhost:27017 ELASTIC_URL=http://localhost:9200 FRONT_URL=http://localhost:8087 UPLOAD_URL=/upload MINIO_ENDPOINT=localhost MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin METRICS_CONSOLE=true SERVER_SECRET=secret clinic flame --dest ./out -- node --nolazy -r ts-node/register --enable-source-maps src/__start.ts",
"build": "compile", "build": "compile",
"_phase:bundle": "rushx bundle", "_phase:bundle": "rushx bundle",

20
server/backup/src/backup.ts
View File

@ -636,11 +636,7 @@ export async function backup (
// Load all digest from collection. // Load all digest from collection.
while (true) { while (true) {
try { try {
const currentChunk = await ctx.with( const currentChunk = await ctx.with('loadChunk', {}, () => connection.loadChunk(domain, idx, options.recheck))
'loadChunk',
{},
async () => await connection.loadChunk(domain, idx, options.recheck)
)
idx = currentChunk.idx idx = currentChunk.idx
let needRetrieve: Ref<Doc>[] = [] let needRetrieve: Ref<Doc>[] = []
@ -1201,10 +1197,22 @@ export async function restore (
workspace: workspaceId.name workspace: workspaceId.name
}) })
const doTrim = (s: string | undefined): string | undefined => {
if (s == null) {
return s
}
if (s.startsWith('"') && s.endsWith('"')) {
return s.slice(1, s.length - 1)
}
return s
}
// Let's find difference // Let's find difference
const docsToAdd = new Map( const docsToAdd = new Map(
Array.from(changeset.entries()).filter( Array.from(changeset.entries()).filter(
([it]) => !serverChangeset.has(it) || (serverChangeset.has(it) && serverChangeset.get(it) !== changeset.get(it)) ([it]) =>
!serverChangeset.has(it) ||
(serverChangeset.has(it) && doTrim(serverChangeset.get(it)) !== doTrim(changeset.get(it)))
) )
) )
const docsToRemove = Array.from(serverChangeset.keys()).filter((it) => !changeset.has(it)) const docsToRemove = Array.from(serverChangeset.keys()).filter((it) => !changeset.has(it))

15
server/core/src/storage.ts
View File

@ -9,7 +9,6 @@ import {
type StorageIterator, type StorageIterator,
type WorkspaceId type WorkspaceId
} from '@hcengineering/core' } from '@hcengineering/core'
import { estimateDocSize } from './utils'
export * from '@hcengineering/storage' export * from '@hcengineering/storage'
@ -77,7 +76,7 @@ export class BackupClientOps {
break break
} }
size += estimateDocSize(doc) size += doc.size
docs.push(doc) docs.push(doc)
} }
@ -99,15 +98,15 @@ export class BackupClientOps {
}) })
} }
async loadDocs (ctx: MeasureContext, domain: Domain, docs: Ref<Doc>[]): Promise<Doc[]> { loadDocs (ctx: MeasureContext, domain: Domain, docs: Ref<Doc>[]): Promise<Doc[]> {
return await this.storage.load(ctx, domain, docs) return this.storage.load(ctx, domain, docs)
} }
async upload (ctx: MeasureContext, domain: Domain, docs: Doc[]): Promise<void> { upload (ctx: MeasureContext, domain: Domain, docs: Doc[]): Promise<void> {
await this.storage.upload(ctx, domain, docs) return this.storage.upload(ctx, domain, docs)
} }
async clean (ctx: MeasureContext, domain: Domain, docs: Ref<Doc>[]): Promise<void> { clean (ctx: MeasureContext, domain: Domain, docs: Ref<Doc>[]): Promise<void> {
await this.storage.clean(ctx, domain, docs) return this.storage.clean(ctx, domain, docs)
} }
} }

2
server/mongo/src/storage.ts
View File

@ -1000,7 +1000,7 @@ abstract class MongoAdapterBase implements DbAdapter {
await coll.bulkWrite( await coll.bulkWrite(
Array.from(bulkUpdate.entries()).map((it) => ({ Array.from(bulkUpdate.entries()).map((it) => ({
updateOne: { updateOne: {
filter: { _id: it[0] }, filter: { _id: it[0], '%hash%': null },
update: { $set: { '%hash%': it[1] } } update: { $set: { '%hash%': it[1] } }
} }
})) }))