diff --git a/dev/tool/src/fulltext.ts b/dev/tool/src/fulltext.ts new file mode 100644 index 0000000000..fc31673916 --- /dev/null +++ b/dev/tool/src/fulltext.ts @@ -0,0 +1,33 @@ +// +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import { type MeasureContext } from '@hcengineering/core' + +export async function reindexWorkspace (ctx: MeasureContext, fulltextUrl: string, token: string): Promise { + try { + const res = await fetch(fulltextUrl + '/api/v1/reindex', { + method: 'PUT', + headers: { + 'Content-Type': 'application/json' + }, + body: JSON.stringify({ token }) + }) + if (!res.ok) { + throw new Error(`HTTP Error ${res.status} ${res.statusText}`) + } + } catch (err: any) { + ctx.error('failed to reset index', { err }) + } +} diff --git a/dev/tool/src/index.ts b/dev/tool/src/index.ts index 4095fac8cd..a6dac6c52b 100644 --- a/dev/tool/src/index.ts +++ b/dev/tool/src/index.ts @@ -77,7 +77,7 @@ import { buildStorageFromConfig, createStorageFromConfig, storageConfigFromEnv } import { program, type Command } from 'commander' import { addControlledDocumentRank } from './qms' import { clearTelegramHistory } from './telegram' -import { diffWorkspace, recreateElastic, updateField } from './workspace' +import { diffWorkspace, updateField } from './workspace' import core, { AccountRole, @@ -149,6 +149,7 @@ import { fixMixinForeignAttributes, showMixinForeignAttributes } from './mixin' import { fixAccountEmails, renameAccount } from './renameAccount' import { copyToDatalake, moveFiles, showLostFiles } from './storage' import { createPostgresTxAdapter, createPostgresAdapter, createPostgreeDestroyAdapter } from '@hcengineering/postgres' +import { reindexWorkspace } from './fulltext' const colorConstants = { colorRed: '\u001b[31m', @@ -1925,27 +1926,43 @@ export function devTool ( ) program - .command('recreate-elastic-indexes-mongo ') - .description('reindex workspace to elastic') + .command('fulltext-reindex ') + .description('reindex workspace') .action(async (workspace: string) => { - const mongodbUri = getMongoDBUrl() + const fulltextUrl = process.env.FULLTEXT_URL + if (fulltextUrl === undefined) { + console.error('please provide FULLTEXT_URL') + process.exit(1) + } + const wsid = getWorkspaceId(workspace) - await recreateElastic(mongodbUri, wsid) + const token = generateToken(systemAccountEmail, wsid) + + console.log('reindex workspace', workspace) + await reindexWorkspace(toolCtx, fulltextUrl, token) + console.log('done', workspace) }) program - .command('recreate-all-elastic-indexes-mongo') - .description('reindex elastic') + .command('fulltext-reindex-all') + .description('reindex workspaces') .action(async () => { - const { dbUrl } = prepareTools() - const mongodbUri = getMongoDBUrl() + const fulltextUrl = process.env.FULLTEXT_URL + if (fulltextUrl === undefined) { + console.error('please provide FULLTEXT_URL') + process.exit(1) + } await withAccountDatabase(async (db) => { const workspaces = await listWorkspacesRaw(db) workspaces.sort((a, b) => b.lastVisit - a.lastVisit) for (const workspace of workspaces) { const wsid = getWorkspaceId(workspace.workspace) - await recreateElastic(mongodbUri ?? dbUrl, wsid) + const token = generateToken(systemAccountEmail, wsid) + + console.log('reindex workspace', workspace) + await reindexWorkspace(toolCtx, fulltextUrl, token) + console.log('done', workspace) } }) }) diff --git a/dev/tool/src/workspace.ts b/dev/tool/src/workspace.ts index 9852311668..d704a8d309 100644 --- a/dev/tool/src/workspace.ts +++ b/dev/tool/src/workspace.ts @@ -20,7 +20,6 @@ import core, { type Class, type Client as CoreClient, type Doc, - DOMAIN_DOC_INDEX_STATE, DOMAIN_TX, type Ref, type Tx, @@ -96,16 +95,3 @@ export async function updateField ( await connection.close() } } - -export async function recreateElastic (mongoUrl: string, workspaceId: WorkspaceId): Promise { - const client = getMongoClient(mongoUrl) - const _client = await client.getClient() - try { - const db = getWorkspaceMongoDB(_client, workspaceId) - await db - .collection(DOMAIN_DOC_INDEX_STATE) - .updateMany({ _class: core.class.DocIndexState }, { $set: { needIndex: true } }) - } finally { - client.close() - } -} diff --git a/pods/fulltext/src/server.ts b/pods/fulltext/src/server.ts index 93fc5e6d31..605a28c37e 100644 --- a/pods/fulltext/src/server.ts +++ b/pods/fulltext/src/server.ts @@ -161,6 +161,14 @@ class WorkspaceIndexer { return result } + async reindex (): Promise { + await this.fulltext.cancel() + await this.fulltext.clearIndex() + await this.fulltext.startIndexing(() => { + this.lastUpdate = Date.now() + }) + } + async close (): Promise { await this.fulltext.cancel() await this.pipeline.close() @@ -188,6 +196,10 @@ interface Search { fullTextLimit: number } +interface Reindex { + token: string +} + export async function startIndexer ( ctx: MeasureContext, opt: { @@ -391,6 +403,26 @@ export async function startIndexer ( } }) + router.put('/api/v1/reindex', async (req, res) => { + try { + const request = req.request.body as Reindex + const decoded = decodeToken(request.token) // Just to be safe + req.body = {} + + ctx.info('reindex', { workspace: decoded.workspace }) + const indexer = await getIndexer(ctx, decoded.workspace, request.token, true) + if (indexer !== undefined) { + indexer.lastUpdate = Date.now() + await indexer.reindex() + } + } catch (err: any) { + Analytics.handleError(err) + console.error(err) + req.res.writeHead(404, {}) + req.res.end() + } + }) + app.use(router.routes()).use(router.allowedMethods()) const server = app.listen(opt.port, () => { diff --git a/server/indexer/src/indexer/indexer.ts b/server/indexer/src/indexer/indexer.ts index 0777e7445c..60fca5e5d7 100644 --- a/server/indexer/src/indexer/indexer.ts +++ b/server/indexer/src/indexer/indexer.ts @@ -165,6 +165,7 @@ export class FullTextIndexPipeline implements FullTextPipeline { triggerIndexing = (): void => {} async startIndexing (indexing: () => void): Promise { + this.cancelling = false this.verify = this.verifyWorkspace(this.metrics, indexing) void this.verify.then(() => { this.indexing = this.doIndexing(indexing) @@ -282,6 +283,19 @@ export class FullTextIndexPipeline implements FullTextPipeline { } } + async clearIndex (): Promise { + const ctx = this.metrics + const migrations = await this.storage.findAll(ctx, core.class.MigrationState, { + plugin: coreId, + state: { + $in: ['verify-indexes-v2', 'full-text-indexer-v4', 'full-text-structure-v4'] + } + }) + + const refs = migrations.map((it) => it._id) + await this.storage.clean(ctx, DOMAIN_MIGRATION, refs) + } + broadcastClasses = new Set>>() broadcasts: number = 0