wieerwill/platform
1
0
Fork 0
mirror of https://github.com/hcengineering/platform.git synced 2025-04-03 06:21:28 +00:00
Code Issues Actions 7 Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/develop' into staging

Browse Source 
Signed-off-by: Andrey Sobolev <haiodo@gmail.com>
This commit is contained in:
Andrey Sobolev 2024-10-14 23:28:06 +07:00
commit be1116f1ec
No known key found for this signature in database
GPG Key ID: BD80F68D68D8F7F2
13 changed files with 118 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

73
dev/tool/src/clean.ts
View File

@ -14,13 +14,13 @@
//
import { getAccountDB, listWorkspacesRaw } from '@hcengineering/account'
import attachment from '@hcengineering/attachment'
import calendar from '@hcengineering/calendar'
import chunter, { type ChatMessage } from '@hcengineering/chunter'
import { loadCollaborativeDoc, saveCollaborativeDoc, yDocToBuffer } from '@hcengineering/collaboration'
import contact from '@hcengineering/contact'
import core, {
type ArrOf,
type AttachedDoc,
type BackupClient,
type Class,
ClassifierKind,
@ -45,11 +45,14 @@ import core, {
SortingOrder,
type Status,
type StatusCategory,
type Tx,
type TxCUD,
type TxCollectionCUD,
type TxCreateDoc,
type TxMixin,
TxOperations,
TxProcessor,
type TxRemoveDoc,
type TxUpdateDoc,
type WorkspaceId,
generateId,
@ -93,30 +96,6 @@ export async function cleanWorkspace (
const hierarchy = ops.getHierarchy()
const attachments = await ops.findAll(attachment.class.Attachment, {})
const contacts = await ops.findAll(contact.class.Contact, {})
const files = new Set(
attachments.map((it) => it.file as string).concat(contacts.map((it) => it.avatar).filter((it) => it) as string[])
)
const minioList = await storageAdapter.listStream(ctx, workspaceId)
const toClean: string[] = []
while (true) {
const mvFiles = await minioList.next()
if (mvFiles.length === 0) {
break
}
for (const mv of mvFiles) {
if (!files.has(mv._id)) {
toClean.push(mv._id)
}
}
}
await storageAdapter.remove(ctx, workspaceId, toClean)
if (opt.recruit) {
const contacts = await ops.findAll(recruit.mixin.Candidate, {})
console.log('removing Talents', contacts.length)
@ -134,13 +113,6 @@ export async function cleanWorkspace (
const t2 = Date.now()
console.log('remove time:', t2 - t, filter.length)
}
// const vacancies = await ops.findAll(recruit.class.Vacancy, {})
// console.log('removing vacancies', vacancies.length)
// for (const c of vacancies) {
// console.log('Remove', c.name)
// await ops.remove(c)
// }
}
if (opt.tracker) {
@ -166,12 +138,39 @@ export async function cleanWorkspace (
const db = getWorkspaceMongoDB(_client, workspaceId)
if (opt.removedTx) {
const txes = await db.collection(DOMAIN_TX).find({}).toArray()
let processed = 0
const iterator = db.collection(DOMAIN_TX).find({})
while (true) {
const txes: Tx[] = []
for (const tx of txes) {
if (tx._class === core.class.TxRemoveDoc) {
// We need to remove all update and create operations for document
await db.collection(DOMAIN_TX).deleteMany({ objectId: tx.objectId })
const doc = await iterator.next()
if (doc == null) {
break
}
txes.push(doc as unknown as Tx)
if (iterator.bufferedCount() > 0) {
txes.push(...(iterator.readBufferedDocuments() as unknown as Tx[]))
}
for (const tx of txes) {
if (tx._class === core.class.TxRemoveDoc) {
// We need to remove all update and create operations for document
await db.collection(DOMAIN_TX).deleteMany({ objectId: (tx as TxRemoveDoc<Doc>).objectId })
processed++
}
if (
tx._class === core.class.TxCollectionCUD &&
(tx as TxCollectionCUD<Doc, AttachedDoc>).tx._class === core.class.TxRemoveDoc
) {
// We need to remove all update and create operations for document
await db.collection(DOMAIN_TX).deleteMany({
'tx.objectId': ((tx as TxCollectionCUD<Doc, AttachedDoc>).tx as TxRemoveDoc<Doc>).objectId
})
processed++
}
}
if (processed % 1000 === 0) {
console.log('processed', processed)
}
}
}

6
models/activity/src/migration.ts
View File

@ -221,6 +221,12 @@ export const activityOperation: MigrateOperation = {
{
state: 'migrate-activity-markup',
func: migrateActivityMarkup
},
{
state: 'fix-rename-backups',
func: async (client: MigrationClient): Promise<void> => {
await client.update(DOMAIN_ACTIVITY, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } })
}
}
])
},

19
models/attachment/src/migration.ts
View File

@ -13,9 +13,24 @@
// limitations under the License.
//
import { type MigrateOperation, type MigrationClient, type MigrationUpgradeClient } from '@hcengineering/model'
import {
tryMigrate,
type MigrateOperation,
type MigrationClient,
type MigrationUpgradeClient
} from '@hcengineering/model'
import { attachmentId, DOMAIN_ATTACHMENT } from '.'
export const attachmentOperation: MigrateOperation = {
async migrate (client: MigrationClient): Promise<void> {},
async migrate (client: MigrationClient): Promise<void> {
await tryMigrate(client, attachmentId, [
{
state: 'fix-rename-backups',
func: async (client: MigrationClient): Promise<void> => {
await client.update(DOMAIN_ATTACHMENT, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } })
}
}
])
},
async upgrade (state: Map<string, Set<string>>, client: () => Promise<MigrationUpgradeClient>): Promise<void> {}
}

6
models/chunter/src/migration.ts
View File

@ -362,6 +362,12 @@ export const chunterOperation: MigrateOperation = {
'attributeUpdates.attrKey': 'members'
})
}
},
{
state: 'fix-rename-backups',
func: async (client: MigrationClient): Promise<void> => {
await client.update(DOMAIN_CHUNTER, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } })
}
}
])
},

11
models/contact/src/migration.ts
View File

@ -1,5 +1,6 @@
//
import { AvatarType, type Contact, type Person, type PersonSpace } from '@hcengineering/contact'
import {
type Class,
type Doc,
@ -24,9 +25,8 @@ import {
import activity, { DOMAIN_ACTIVITY } from '@hcengineering/model-activity'
import core, { DOMAIN_SPACE } from '@hcengineering/model-core'
import { DOMAIN_VIEW } from '@hcengineering/model-view'
import { AvatarType, type Contact, type Person, type PersonSpace } from '@hcengineering/contact'
import contact, { contactId, DOMAIN_CONTACT } from './index'
import contact, { contactId, DOMAIN_CHANNEL, DOMAIN_CONTACT } from './index'
async function createEmployeeEmail (client: TxOperations): Promise<void> {
const employees = await client.findAll(contact.mixin.Employee, {})
@ -300,6 +300,13 @@ export const contactOperation: MigrateOperation = {
{
state: 'create-person-spaces-v1',
func: createPersonSpaces
},
{
state: 'fix-rename-backups',
func: async (client: MigrationClient): Promise<void> => {
await client.update(DOMAIN_CONTACT, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } })
await client.update(DOMAIN_CHANNEL, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } })
}
}
])
},

7
models/core/src/migration.ts
View File

@ -301,6 +301,13 @@ export const coreOperation: MigrateOperation = {
{
state: 'collaborative-content-to-storage',
func: migrateCollaborativeContentToStorage
},
{
state: 'fix-rename-backups',
func: async (client: MigrationClient): Promise<void> => {
await client.update(DOMAIN_TX, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } })
await client.update(DOMAIN_SPACE, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } })
}
}
])
},

6
models/document/src/migration.ts
View File

@ -228,6 +228,12 @@ export const documentOperation: MigrateOperation = {
{
state: 'renameFields',
func: renameFields
},
{
state: 'fix-rename-backups',
func: async (client: MigrationClient): Promise<void> => {
await client.update(DOMAIN_DOCUMENT, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } })
}
}
])
},

6
models/drive/src/migration.ts
View File

@ -132,6 +132,12 @@ export const driveOperation: MigrateOperation = {
{
state: 'renameFields',
func: renameFields
},
{
state: 'fix-rename-backups',
func: async (client: MigrationClient): Promise<void> => {
await client.update(DOMAIN_DRIVE, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } })
}
}
])
},

6
models/notification/src/migration.ts
View File

@ -393,6 +393,12 @@ export const notificationOperation: MigrateOperation = {
{ hidden: false }
)
}
},
{
state: 'fix-rename-backups',
func: async (client: MigrationClient): Promise<void> => {
await client.update(DOMAIN_DOC_NOTIFY, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } })
}
}
])

6
models/task/src/migration.ts
View File

@ -584,6 +584,12 @@ export const taskOperation: MigrateOperation = {
func: async (client: MigrationClient) => {
await migrateSpace(client, task.space.Sequence, core.space.Workspace, [DOMAIN_KANBAN])
}
},
{
state: 'fix-rename-backups',
func: async (client: MigrationClient): Promise<void> => {
await client.update(DOMAIN_TASK, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } })
}
}
])
},

6
models/time/src/migration.ts
View File

@ -172,6 +172,12 @@ export const timeOperation: MigrateOperation = {
func: async (client) => {
await fillProps(client)
}
},
{
state: 'fix-rename-backups',
func: async (client: MigrationClient): Promise<void> => {
await client.update(DOMAIN_TIME, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } })
}
}
])
},

6
models/view/src/migration.ts
View File

@ -86,6 +86,12 @@ export const viewOperation: MigrateOperation = {
{
state: 'remove-done-state-filter',
func: removeDoneStateFilter
},
{
state: 'fix-rename-backups',
func: async (client: MigrationClient): Promise<void> => {
await client.update(DOMAIN_VIEW, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } })
}
}
])
},

1
server/mongo/src/storage.ts
View File

@ -251,6 +251,7 @@ abstract class MongoAdapterBase implements DbAdapter {
operations: DocumentUpdate<T>
): Promise<void> {
if (isOperator(operations)) {
await this.db.collection(domain).updateMany(this.translateRawQuery(query), { $set: { '%hash%': null } })
await this.db
.collection(domain)
.updateMany(this.translateRawQuery(query), { ...operations } as unknown as UpdateFilter<Document>)