From b5b4d23629d42d34ccf3dbea079d8a27e33b1224 Mon Sep 17 00:00:00 2001 From: Andrey Sobolev Date: Tue, 15 Oct 2024 12:04:39 +0700 Subject: [PATCH] UBERF-8469: Fix exit from github service (#6921) Signed-off-by: Andrey Sobolev --- services/github/pod-github/package.json | 4 ++-- services/github/pod-github/run.sh | 13 +++++++++++++ services/github/pod-github/src/index.ts | 15 +++++++++++---- services/github/pod-github/src/platform.ts | 1 + services/github/pod-github/src/server.ts | 9 +++++++-- .../github/pod-github/src/sync/comments.ts | 6 ++++++ .../github/pod-github/src/sync/issueBase.ts | 2 +- services/github/pod-github/src/sync/issues.ts | 9 +++++++++ .../github/pod-github/src/sync/projects.ts | 6 ++++++ .../pod-github/src/sync/pullrequests.ts | 6 ++++++ services/github/pod-github/src/sync/users.ts | 6 ++++++ services/github/pod-github/src/types.ts | 2 ++ services/github/pod-github/src/worker.ts | 19 +++++++++++++++++++ 13 files changed, 89 insertions(+), 9 deletions(-) create mode 100755 services/github/pod-github/run.sh diff --git a/services/github/pod-github/package.json b/services/github/pod-github/package.json index 39de621832..3dc820dff0 100644 --- a/services/github/pod-github/package.json +++ b/services/github/pod-github/package.json @@ -17,12 +17,12 @@ "_phase:bundle": "rushx bundle", "_phase:docker-build": "rushx docker:build", "_phase:docker-staging": "rushx docker:staging", - "bundle": "mkdir -p bundle && esbuild src/index.ts --bundle --platform=node --outfile=bundle/bundle.js --log-level=error --sourcemap=external", + "bundle": "mkdir -p bundle && esbuild src/index.ts --keep-names --bundle --platform=node --outfile=bundle/bundle.js --log-level=error --sourcemap=external", "docker:build": "../../../common/scripts/docker_build.sh hardcoreeng/github", "docker:staging": "../../../common/scripts/docker_tag.sh hardcoreeng/github staging", "docker:push": "../../../common/scripts/docker_tag.sh hardcoreeng/github", "docker:tbuild": "rush bundle --to @hcengineering/pod-github && docker build -t hardcoreeng/github . --platform=linux/amd64 && ../../../common/scripts/docker_tag_push.sh hardcoreeng/github", - "run-local": "cross-env APP_ID=$(cat ../../../../uberflow_private/appid) PRIVATE_KEY=\"$(cat ../../../../uberflow_private/private-key.pem)\" CLIENT_ID=$(cat ../../../../uberflow_private/client-id) CLIENT_SECRET=$(cat ../../../../uberflow_private/client-secret) SERVER_SECRET=secret ACCOUNTS_URL=http://localhost:3000/ COLLABORATOR_URL=http://localhost:3078 MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin MINIO_ENDPOINT=localhost ts-node src/index.ts", + "run-local": "ruh.sh", "format": "format src", "_phase:build": "compile transpile src", "_phase:test": "jest --passWithNoTests --silent", diff --git a/services/github/pod-github/run.sh b/services/github/pod-github/run.sh new file mode 100755 index 0000000000..ebc9faf4f9 --- /dev/null +++ b/services/github/pod-github/run.sh @@ -0,0 +1,13 @@ +export APP_ID="$POD_GITHUB_APPID" +export CLIENT_ID="$POD_GITHUB_CLIENTID" +export CLIENT_SECRET="$POD_GITHUB_CLIENT_SECRET" +export PRIVATE_KEY="$POD_GITHUB_PRIVATE_KEY" +export SERVER_SECRET=secret +export ACCOUNTS_URL=http://localhost:3000 +export COLLABORATOR_URL=http://localhost:3078 +export MINIO_ACCESS_KEY=minioadminchmo +export MINIO_SECRET_KEY=minioadmin +export MINIO_ENDPOINT=localhost +export MONGO_URL=mongodb://localhost:27017 +rush bundle --to @hcengineering/pod-github +node $@ bundle/bundle.js \ No newline at end of file diff --git a/services/github/pod-github/src/index.ts b/services/github/pod-github/src/index.ts index fa038ef824..e154398a93 100644 --- a/services/github/pod-github/src/index.ts +++ b/services/github/pod-github/src/index.ts @@ -2,14 +2,14 @@ // Copyright © 2023 Hardcore Engineering Inc. // -import { MeasureMetricsContext, metricsToString, newMetrics } from '@hcengineering/core' +import { Analytics } from '@hcengineering/analytics' import { SplitLogger, configureAnalytics } from '@hcengineering/analytics-service' +import { MeasureMetricsContext, metricsToString, newMetrics } from '@hcengineering/core' +import { loadBrandingMap } from '@hcengineering/server-core' import { writeFile } from 'fs/promises' import { join } from 'path' import config from './config' import { start } from './server' -import { Analytics } from '@hcengineering/analytics' -import { loadBrandingMap } from '@hcengineering/server-core' // Load and inc startID, to have easy logs. @@ -39,11 +39,18 @@ const intTimer = setInterval(() => { } }, 30000) -void start(metricsContext, loadBrandingMap(config.BrandingPath)) +let doOnClose: () => Promise = async () => {} + +void start(metricsContext, loadBrandingMap(config.BrandingPath)).then((r) => { + doOnClose = r +}) const onClose = (): void => { clearInterval(intTimer) metricsContext.info('Closed') + void doOnClose().then((r) => { + process.exit(0) + }) } process.on('uncaughtException', (e) => { diff --git a/services/github/pod-github/src/platform.ts b/services/github/pod-github/src/platform.ts index f525fdf803..6b1c6ed307 100644 --- a/services/github/pod-github/src/platform.ts +++ b/services/github/pod-github/src/platform.ts @@ -96,6 +96,7 @@ export class PlatformWorker { async close (): Promise { this.canceled = true + clearInterval(this.periodicSyncInterval) await Promise.all( [...this.clients.values()].map(async (worker) => { await worker.close() diff --git a/services/github/pod-github/src/server.ts b/services/github/pod-github/src/server.ts index b0d8d0d9ae..df41cd0a25 100644 --- a/services/github/pod-github/src/server.ts +++ b/services/github/pod-github/src/server.ts @@ -18,7 +18,7 @@ import { decodeToken } from '@hcengineering/server-token' /** * @public */ -export async function start (ctx: MeasureContext, brandingMap: BrandingMap): Promise { +export async function start (ctx: MeasureContext, brandingMap: BrandingMap): Promise<() => Promise> { // Create an authenticated Octokit client authenticated as a GitHub App ctx.info('Running Huly Github integration', { appId: config.AppID, clientID: config.ClientID }) @@ -184,8 +184,13 @@ export async function start (ctx: MeasureContext, brandingMap: BrandingMap): Pro } }) - app.listen(port, () => { + const server = app.listen(port, () => { ctx.info(`Server is listening for events at: ${localWebhookUrl}`) ctx.info('Press Ctrl + C to quit.') }) + + return async () => { + await worker.close() + server.close() + } } diff --git a/services/github/pod-github/src/sync/comments.ts b/services/github/pod-github/src/sync/comments.ts index cf525d0903..3492c30599 100644 --- a/services/github/pod-github/src/sync/comments.ts +++ b/services/github/pod-github/src/sync/comments.ts @@ -457,6 +457,9 @@ export class CommentSyncManager implements DocSyncManager { repositories: GithubIntegrationRepository[] ): Promise { for (const repo of repositories) { + if (this.provider.isClosing()) { + break + } const syncKey = `${repo._id}:comment` if (repo.githubProject === undefined || !repo.enabled || integration.synchronized.has(syncKey)) { if (!repo.enabled) { @@ -487,6 +490,9 @@ export class CommentSyncManager implements DocSyncManager { }) try { for await (const data of i) { + if (this.provider.isClosing()) { + break + } const comments: CommentExternalData[] = data.data as any this.ctx.info('retrieve comments for', { repo: repo.name, diff --git a/services/github/pod-github/src/sync/issueBase.ts b/services/github/pod-github/src/sync/issueBase.ts index f2e9c9278a..0ff45f0f96 100644 --- a/services/github/pod-github/src/sync/issueBase.ts +++ b/services/github/pod-github/src/sync/issueBase.ts @@ -815,7 +815,7 @@ export abstract class IssueSyncManagerBase { // Collect field update. for (const [k, v] of Object.entries(platformUpdate)) { - const mapping = target.mappings.find((it) => it.name === k) + const mapping = target.mappings.filter((it) => it != null).find((it) => it.name === k) if (mapping === undefined) { continue } diff --git a/services/github/pod-github/src/sync/issues.ts b/services/github/pod-github/src/sync/issues.ts index c981910720..290ce7ed15 100644 --- a/services/github/pod-github/src/sync/issues.ts +++ b/services/github/pod-github/src/sync/issues.ts @@ -985,6 +985,9 @@ export class IssueSyncManager extends IssueSyncManagerBase implements DocSyncMan let partsize = 50 try { while (true) { + if (this.provider.isClosing()) { + break + } const idsPart = ids.splice(0, partsize) if (idsPart.length === 0) { break @@ -1080,6 +1083,9 @@ export class IssueSyncManager extends IssueSyncManagerBase implements DocSyncMan repositories: GithubIntegrationRepository[] ): Promise { for (const repo of repositories) { + if (this.provider.isClosing()) { + break + } const prj = projects.find((it) => repo.githubProject === it._id) if (prj === undefined) { continue @@ -1128,6 +1134,9 @@ export class IssueSyncManager extends IssueSyncManagerBase implements DocSyncMan ) try { for await (const data of i) { + if (this.provider.isClosing()) { + break + } const issues: IssueExternalData[] = data.repository.issues.nodes if (issues.some((issue) => issue.url === undefined && Object.keys(issue).length === 0)) { this.ctx.error('empty document content', { diff --git a/services/github/pod-github/src/sync/projects.ts b/services/github/pod-github/src/sync/projects.ts index 6833820794..4748b1e41a 100644 --- a/services/github/pod-github/src/sync/projects.ts +++ b/services/github/pod-github/src/sync/projects.ts @@ -376,6 +376,9 @@ export class ProjectsSyncManager implements DocSyncManager { repositories: GithubIntegrationRepository[] ): Promise { for (const prj of projects) { + if (this.provider.isClosing()) { + break + } // Wait global project sync await integration.syncLock.get(prj._id) @@ -449,6 +452,9 @@ export class ProjectsSyncManager implements DocSyncManager { (it) => it.space === prj._id ) for (const m of milestones) { + if (this.provider.isClosing()) { + break + } try { let { projectStructure, wasUpdates } = await this.ctx.withLog( 'update project structure', diff --git a/services/github/pod-github/src/sync/pullrequests.ts b/services/github/pod-github/src/sync/pullrequests.ts index 8ef9fd275d..51d3e78d54 100644 --- a/services/github/pod-github/src/sync/pullrequests.ts +++ b/services/github/pod-github/src/sync/pullrequests.ts @@ -1433,6 +1433,9 @@ export class PullRequestSyncManager extends IssueSyncManagerBase implements DocS repositories: GithubIntegrationRepository[] ): Promise { for (const repo of repositories) { + if (this.provider.isClosing()) { + break + } const prj = projects.find((it) => repo.githubProject === it._id) if (prj === undefined) { continue @@ -1518,6 +1521,9 @@ export class PullRequestSyncManager extends IssueSyncManagerBase implements DocS } ) for await (const data of pullRequestIterator) { + if (this.provider.isClosing()) { + break + } const issues: PullRequestExternalData[] = data.repository.pullRequests.nodes this.ctx.info('retrieve pull requests for', { repo: repo.name, diff --git a/services/github/pod-github/src/sync/users.ts b/services/github/pod-github/src/sync/users.ts index 94f5d43b72..b018dce4fa 100644 --- a/services/github/pod-github/src/sync/users.ts +++ b/services/github/pod-github/src/sync/users.ts @@ -60,6 +60,9 @@ export class UsersSyncManager implements DocSyncManager { repositories: GithubIntegrationRepository[] ): Promise { for (const repo of repositories) { + if (this.provider.isClosing()) { + break + } const syncKey = `${repo._id}:users` if ( repo.githubProject === undefined || @@ -107,6 +110,9 @@ export class UsersSyncManager implements DocSyncManager { ) try { for await (const data of assignableUsersIterator) { + if (this.provider.isClosing()) { + break + } const users: UserInfo[] = data.repository[key]?.nodes ?? [] for (const d of users) { if (d.login !== undefined) { diff --git a/services/github/pod-github/src/types.ts b/services/github/pod-github/src/types.ts index 13d33177b5..f4e24fddc9 100644 --- a/services/github/pod-github/src/types.ts +++ b/services/github/pod-github/src/types.ts @@ -129,6 +129,8 @@ export interface IntegrationManager { getProjectRepositories: (space: Ref) => Promise getRepositoryById: (ref?: Ref | null) => Promise + + isClosing: () => boolean } export type ExternalSyncField = 'externalVersion' | 'derivedVersion' diff --git a/services/github/pod-github/src/worker.ts b/services/github/pod-github/src/worker.ts index db5f974876..1ef7acdc63 100644 --- a/services/github/pod-github/src/worker.ts +++ b/services/github/pod-github/src/worker.ts @@ -117,11 +117,18 @@ export class GithubWorker implements IntegrationManager { personMapper: UsersSyncManager + isClosing (): boolean { + return this.closing + } + async close (): Promise { clearInterval(this.periodicTimer) this.closing = true + this.ctx.warn('Closing', { workspace: this.workspace.name }) + this.triggerSync() await this.syncPromise + this.ctx.warn('ClosingDone', { workspace: this.workspace.name }) await this.client.close() } @@ -1347,6 +1354,9 @@ export class GithubWorker implements IntegrationManager { } private async waitChanges (): Promise { + if (this.closing) { + return + } if (this.triggerRequests > 0 || this.updateRequests > 0) { this.ctx.info('Trigger check pending:', { requests: this.triggerRequests, @@ -1404,6 +1414,9 @@ export class GithubWorker implements IntegrationManager { async _performFullSync (): Promise { // Wait previous active sync for (const integration of this.integrations.values()) { + if (this.closing) { + break + } await this.ctx.withLog( 'external sync', { installation: integration.installationName, workspace: this.workspace.name }, @@ -1442,6 +1455,9 @@ export class GithubWorker implements IntegrationManager { // Cleanup broken synchronized documents while (true) { + if (this.closing) { + break + } const withError = await derivedClient.findAll( github.class.DocSyncInfo, { error: { $ne: null }, url: null }, @@ -1458,6 +1474,9 @@ export class GithubWorker implements IntegrationManager { } for (const { _class, mapper } of this.mappers) { + if (this.closing) { + break + } await this.ctx.withLog( 'external sync', { _class: _class.join(', '), workspace: this.workspace.name },