mirror of
https://github.com/hcengineering/platform.git
synced 2025-05-10 09:22:23 +00:00
Qfix: remove dev migrations (#8651)
This commit is contained in:
parent
d450a7397f
commit
a7422a627d
@ -43,8 +43,6 @@ import {
|
||||
import { htmlToMarkup } from '@hcengineering/text'
|
||||
import {
|
||||
getAccountUuidByOldAccount,
|
||||
getAccountUuidBySocialKey,
|
||||
getSocialIdBySocialKey,
|
||||
getSocialIdFromOldAccount,
|
||||
getSocialKeyByOldAccount
|
||||
} from '@hcengineering/model-core'
|
||||
@ -365,151 +363,6 @@ async function migrateAccountsInDocUpdates (client: MigrationClient): Promise<vo
|
||||
ctx.info('finished processing activity doc updates ', {})
|
||||
}
|
||||
|
||||
/**
|
||||
* Migrates social ids to new accounts where needed.
|
||||
* Should only be applied to staging where old accounts have already been migrated to social ids.
|
||||
* REMOVE IT BEFORE MERGING TO PRODUCTION
|
||||
* @param client
|
||||
* @returns
|
||||
*/
|
||||
async function migrateSocialIdsInDocUpdates (client: MigrationClient): Promise<void> {
|
||||
const ctx = new MeasureMetricsContext('activity migrateSocialIdsInDocUpdates', {})
|
||||
const accountUuidBySocialKey = new Map<string, AccountUuid | null>()
|
||||
ctx.info('processing activity doc updates ', {})
|
||||
|
||||
async function getUpdatedVal (oldVal: string): Promise<any> {
|
||||
return (await getAccountUuidBySocialKey(client, oldVal, accountUuidBySocialKey)) ?? oldVal
|
||||
}
|
||||
|
||||
async function migrateField<P extends keyof DocAttributeUpdates> (
|
||||
au: DocAttributeUpdates,
|
||||
update: MigrateUpdate<DocUpdateMessage>['attributeUpdates'],
|
||||
field: P
|
||||
): Promise<void> {
|
||||
const oldValue = au?.[field]
|
||||
if (oldValue == null) return
|
||||
|
||||
let changed = false
|
||||
let newValue: any
|
||||
if (Array.isArray(oldValue)) {
|
||||
newValue = []
|
||||
for (const a of oldValue as any[]) {
|
||||
const newA = a != null ? await getUpdatedVal(a) : a
|
||||
if (newA !== a) {
|
||||
changed = true
|
||||
}
|
||||
newValue.push(newA)
|
||||
}
|
||||
} else {
|
||||
newValue = await getUpdatedVal(oldValue)
|
||||
if (newValue !== oldValue) {
|
||||
changed = true
|
||||
}
|
||||
}
|
||||
|
||||
if (changed) {
|
||||
if (update == null) throw new Error('update is null')
|
||||
|
||||
update[field] = newValue
|
||||
}
|
||||
}
|
||||
|
||||
const iterator = await client.traverse(DOMAIN_ACTIVITY, {
|
||||
_class: activity.class.DocUpdateMessage,
|
||||
action: 'update',
|
||||
'attributeUpdates.attrClass': 'core:class:TypePersonId',
|
||||
'attributeUpdates.attrKey': { $in: ['members', 'owners', 'user'] }
|
||||
})
|
||||
|
||||
try {
|
||||
let processed = 0
|
||||
while (true) {
|
||||
const docs = await iterator.next(200)
|
||||
if (docs === null || docs.length === 0) {
|
||||
break
|
||||
}
|
||||
|
||||
const operations: {
|
||||
filter: MigrationDocumentQuery<DocUpdateMessage>
|
||||
update: MigrateUpdate<DocUpdateMessage>
|
||||
}[] = []
|
||||
|
||||
for (const doc of docs) {
|
||||
const dum = doc as DocUpdateMessage
|
||||
if (dum.attributeUpdates == null) continue
|
||||
const update: any = { attributeUpdates: { ...dum.attributeUpdates } }
|
||||
|
||||
await migrateField(dum.attributeUpdates, update.attributeUpdates, 'added')
|
||||
await migrateField(dum.attributeUpdates, update.attributeUpdates, 'prevValue')
|
||||
await migrateField(dum.attributeUpdates, update.attributeUpdates, 'removed')
|
||||
await migrateField(dum.attributeUpdates, update.attributeUpdates, 'set')
|
||||
|
||||
update.attributeUpdates.attrClass = core.class.TypeAccountUuid
|
||||
|
||||
operations.push({
|
||||
filter: { _id: dum._id },
|
||||
update
|
||||
})
|
||||
}
|
||||
|
||||
if (operations.length > 0) {
|
||||
await client.bulk(DOMAIN_ACTIVITY, operations)
|
||||
}
|
||||
|
||||
processed += docs.length
|
||||
ctx.info('...processed', { count: processed })
|
||||
}
|
||||
} finally {
|
||||
await iterator.close()
|
||||
}
|
||||
|
||||
ctx.info('finished processing activity doc updates ', {})
|
||||
}
|
||||
|
||||
async function migrateSocialKeysToSocialIds (client: MigrationClient): Promise<void> {
|
||||
const ctx = new MeasureMetricsContext('activity migrateSocialKeysToSocialIds', {})
|
||||
|
||||
ctx.info('processing activity reactions ', {})
|
||||
const socialIdBySocialKey = new Map<string, PersonId | null>()
|
||||
const iterator = await client.traverse(DOMAIN_REACTION, { _class: activity.class.Reaction })
|
||||
|
||||
try {
|
||||
let processed = 0
|
||||
while (true) {
|
||||
const docs = await iterator.next(200)
|
||||
if (docs === null || docs.length === 0) {
|
||||
break
|
||||
}
|
||||
|
||||
const operations: { filter: MigrationDocumentQuery<Doc>, update: MigrateUpdate<Doc> }[] = []
|
||||
|
||||
for (const doc of docs) {
|
||||
const reaction = doc as Reaction
|
||||
const newCreateBy =
|
||||
(await getSocialIdBySocialKey(client, reaction.createBy, socialIdBySocialKey)) ?? reaction.createBy
|
||||
|
||||
if (newCreateBy === reaction.createBy) continue
|
||||
|
||||
operations.push({
|
||||
filter: { _id: doc._id },
|
||||
update: {
|
||||
createBy: newCreateBy
|
||||
}
|
||||
})
|
||||
processed++
|
||||
}
|
||||
|
||||
if (operations.length > 0) {
|
||||
await client.bulk(DOMAIN_REACTION, operations)
|
||||
ctx.info('...processed', { count: processed })
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
await iterator.close()
|
||||
}
|
||||
ctx.info('finished processing activity reactions ', {})
|
||||
}
|
||||
|
||||
export const activityOperation: MigrateOperation = {
|
||||
async migrate (client: MigrationClient, mode): Promise<void> {
|
||||
await tryMigrate(mode, client, activityId, [
|
||||
@ -592,18 +445,6 @@ export const activityOperation: MigrateOperation = {
|
||||
state: 'accounts-in-doc-updates-v2',
|
||||
mode: 'upgrade',
|
||||
func: migrateAccountsInDocUpdates
|
||||
},
|
||||
// ONLY FOR STAGING. REMOVE IT BEFORE MERGING TO PRODUCTION
|
||||
{
|
||||
state: 'social-ids-in-doc-updates',
|
||||
mode: 'upgrade',
|
||||
func: migrateSocialIdsInDocUpdates
|
||||
},
|
||||
// ONLY FOR STAGING. REMOVE IT BEFORE MERGING TO PRODUCTION
|
||||
{
|
||||
state: 'social-keys-to-social-ids-v2',
|
||||
mode: 'upgrade',
|
||||
func: migrateSocialKeysToSocialIds
|
||||
}
|
||||
])
|
||||
},
|
||||
|
@ -166,66 +166,6 @@ async function fillAccountUuids (client: MigrationClient): Promise<void> {
|
||||
}
|
||||
}
|
||||
|
||||
async function fillSocialIdentitiesIds (client: MigrationClient): Promise<void> {
|
||||
const ctx = new MeasureMetricsContext('contact fillSocialIdentitiesIds', {})
|
||||
ctx.info('filling social identities genenrated ids...')
|
||||
const socialIdBySocialKey = new Map<string, PersonId | null>()
|
||||
const iterator = await client.traverse<SocialIdentity>(DOMAIN_CHANNEL, { _class: contact.class.SocialIdentity })
|
||||
let count = 0
|
||||
|
||||
try {
|
||||
let newSids: SocialIdentity[] = []
|
||||
let newSidIds = new Set<Ref<SocialIdentity>>()
|
||||
let deleteSids: Ref<SocialIdentity>[] = []
|
||||
|
||||
while (true) {
|
||||
const socialIdentities = await iterator.next(200)
|
||||
if (socialIdentities === null || socialIdentities.length === 0) {
|
||||
break
|
||||
}
|
||||
|
||||
for (const socialIdentity of socialIdentities) {
|
||||
const socialId = await getSocialIdBySocialKey(client, socialIdentity.key, socialIdBySocialKey)
|
||||
|
||||
if (socialId == null || socialId === socialIdentity._id) continue
|
||||
|
||||
const socialIdRef = socialId as SocialIdentityRef
|
||||
// Some old data might contain duplicate accounts for github users
|
||||
// so need to filter just in case
|
||||
if (!newSidIds.has(socialIdRef)) {
|
||||
newSidIds.add(socialIdRef)
|
||||
newSids.push({
|
||||
...socialIdentity,
|
||||
_id: socialIdRef
|
||||
})
|
||||
}
|
||||
|
||||
deleteSids.push(socialIdentity._id)
|
||||
count++
|
||||
|
||||
if (newSids.length > 50) {
|
||||
await client.create(DOMAIN_CHANNEL, newSids)
|
||||
await client.deleteMany(DOMAIN_CHANNEL, { _id: { $in: deleteSids } })
|
||||
newSids = []
|
||||
newSidIds = new Set()
|
||||
deleteSids = []
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (newSids.length > 0) {
|
||||
await client.create(DOMAIN_CHANNEL, newSids)
|
||||
await client.deleteMany(DOMAIN_CHANNEL, { _id: { $in: deleteSids } })
|
||||
newSids = []
|
||||
newSidIds = new Set()
|
||||
deleteSids = []
|
||||
}
|
||||
ctx.info('finished filling social identities genenrated ids. Updated count: ', { count })
|
||||
} finally {
|
||||
await iterator.close()
|
||||
}
|
||||
}
|
||||
|
||||
async function assignWorkspaceRoles (client: MigrationClient): Promise<void> {
|
||||
const ctx = new MeasureMetricsContext('contact assignWorkspaceRoles', {})
|
||||
ctx.info('assigning workspace roles...')
|
||||
@ -606,12 +546,6 @@ export const contactOperation: MigrateOperation = {
|
||||
mode: 'upgrade',
|
||||
func: assignEmployeeRoles
|
||||
},
|
||||
// ONLY FOR STAGING. REMOVE IT BEFORE MERGING TO PRODUCTION
|
||||
{
|
||||
state: 'fill-social-identities-ids-v2',
|
||||
mode: 'upgrade',
|
||||
func: fillSocialIdentitiesIds
|
||||
},
|
||||
{
|
||||
state: 'create-user-profiles',
|
||||
mode: 'upgrade',
|
||||
|
@ -707,197 +707,6 @@ export async function getUniqueAccountsFromOldAccounts (
|
||||
return Array.from(accounts)
|
||||
}
|
||||
|
||||
/**
|
||||
* Migrates social keys to new accounts where needed.
|
||||
* Should only be applied to staging where old accounts have already been migrated to social keys.
|
||||
* REMOVE IT BEFORE MERGING TO PRODUCTION
|
||||
* @param client
|
||||
* @returns
|
||||
*/
|
||||
async function migrateSpaceMembersToAccountUuids (client: MigrationClient): Promise<void> {
|
||||
const ctx = new MeasureMetricsContext('core migrateSpaceMembersToAccountUuids', {})
|
||||
const hierarchy = client.hierarchy
|
||||
const accountUuidBySocialKey = new Map<string, AccountUuid | null>()
|
||||
|
||||
const spaceTypes = client.model.findAllSync(core.class.SpaceType, {})
|
||||
const spaceTypesById = toIdMap(spaceTypes)
|
||||
const roles = client.model.findAllSync(core.class.Role, {})
|
||||
const rolesBySpaceType = new Map<Ref<SpaceType>, Role[]>()
|
||||
for (const role of roles) {
|
||||
const spaceType = role.attachedTo
|
||||
if (spaceType === undefined) continue
|
||||
if (rolesBySpaceType.has(spaceType)) {
|
||||
rolesBySpaceType.get(spaceType)?.push(role)
|
||||
} else {
|
||||
rolesBySpaceType.set(spaceType, [role])
|
||||
}
|
||||
}
|
||||
|
||||
ctx.info('processing spaces members, owners and roles assignment', {})
|
||||
let processedSpaces = 0
|
||||
const spacesIterator = await client.traverse(DOMAIN_SPACE, {})
|
||||
|
||||
try {
|
||||
while (true) {
|
||||
const spaces = await spacesIterator.next(200)
|
||||
if (spaces === null || spaces.length === 0) {
|
||||
break
|
||||
}
|
||||
|
||||
const operations: { filter: MigrationDocumentQuery<Space>, update: MigrateUpdate<Space> }[] = []
|
||||
|
||||
for (const s of spaces) {
|
||||
if (!hierarchy.isDerived(s._class, core.class.Space)) continue
|
||||
const space = s as Space
|
||||
const update: MigrateUpdate<Space> = {
|
||||
members: await getUniqueAccounts(client, space.members, accountUuidBySocialKey),
|
||||
owners: await getUniqueAccounts(client, space.owners ?? [], accountUuidBySocialKey)
|
||||
}
|
||||
|
||||
const type = spaceTypesById.get((space as TypedSpace).type)
|
||||
|
||||
if (type !== undefined) {
|
||||
const mixin = hierarchy.as(space, type.targetClass)
|
||||
if (mixin !== undefined) {
|
||||
const roles = rolesBySpaceType.get(type._id)
|
||||
|
||||
for (const role of roles ?? []) {
|
||||
const oldAssignees: PersonId[] | undefined = (mixin as any)[role._id]
|
||||
if (oldAssignees != null && oldAssignees.length > 0) {
|
||||
const newAssignees = await getUniqueAccounts(client, oldAssignees, accountUuidBySocialKey)
|
||||
|
||||
update[`${type.targetClass}`] = {
|
||||
[role._id]: newAssignees
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
operations.push({
|
||||
filter: { _id: space._id },
|
||||
update
|
||||
})
|
||||
}
|
||||
|
||||
if (operations.length > 0) {
|
||||
await client.bulk(DOMAIN_SPACE, operations)
|
||||
}
|
||||
|
||||
processedSpaces += spaces.length
|
||||
ctx.info('...spaces processed', { count: processedSpaces })
|
||||
}
|
||||
|
||||
ctx.info('finished processing spaces members, owners and roles assignment', { processedSpaces })
|
||||
} finally {
|
||||
await spacesIterator.close()
|
||||
}
|
||||
|
||||
ctx.info('processing space types members', {})
|
||||
let updatedSpaceTypes = 0
|
||||
for (const spaceType of spaceTypes) {
|
||||
if (spaceType.members === undefined || spaceType.members.length === 0) continue
|
||||
|
||||
const newMembers = await getUniqueAccounts(
|
||||
client,
|
||||
spaceType.members as unknown as PersonId[],
|
||||
accountUuidBySocialKey
|
||||
)
|
||||
const tx: TxUpdateDoc<SpaceType> = {
|
||||
_id: generateId(),
|
||||
_class: core.class.TxUpdateDoc,
|
||||
space: core.space.Tx,
|
||||
objectId: spaceType._id,
|
||||
objectClass: spaceType._class,
|
||||
objectSpace: spaceType.space,
|
||||
operations: {
|
||||
members: newMembers
|
||||
},
|
||||
modifiedOn: Date.now(),
|
||||
createdBy: core.account.ConfigUser,
|
||||
createdOn: Date.now(),
|
||||
modifiedBy: core.account.ConfigUser
|
||||
}
|
||||
|
||||
await client.create(DOMAIN_MODEL_TX, tx)
|
||||
updatedSpaceTypes++
|
||||
}
|
||||
ctx.info('finished processing space types members', { totalSpaceTypes: spaceTypes.length, updatedSpaceTypes })
|
||||
}
|
||||
|
||||
/**
|
||||
* Migrates social keys to social ids where needed.
|
||||
* Should only be applied to staging where old accounts have already been migrated to social keys.
|
||||
* REMOVE IT BEFORE MERGING TO PRODUCTION
|
||||
* @param client
|
||||
* @returns
|
||||
*/
|
||||
async function migrateCreatedByToGenSocialIds (client: MigrationClient): Promise<void> {
|
||||
const ctx = new MeasureMetricsContext('core migrateCreatedByToGenSocialIds', {})
|
||||
const socialIdBySocialKey = new Map<string, PersonId | null>()
|
||||
|
||||
ctx.info('migrating createdBy and modifiedBy')
|
||||
function chunkArray<T> (array: T[], chunkSize: number): T[][] {
|
||||
const chunks: T[][] = []
|
||||
for (let i = 0; i < array.length; i += chunkSize) {
|
||||
chunks.push(array.slice(i, i + chunkSize))
|
||||
}
|
||||
return chunks
|
||||
}
|
||||
|
||||
for (const domain of client.hierarchy.domains()) {
|
||||
ctx.info('processing domain ', { domain })
|
||||
const operations: { filter: MigrationDocumentQuery<Doc>, update: MigrateUpdate<Doc> }[] = []
|
||||
const groupByCreated = await client.groupBy<any, Doc>(domain, 'createdBy', {})
|
||||
const groupByModified = await client.groupBy<any, Doc>(domain, 'modifiedBy', {})
|
||||
|
||||
for (const socialKey of groupByCreated.keys()) {
|
||||
if (socialKey == null) continue
|
||||
const socialId = await getSocialIdBySocialKey(client, socialKey, socialIdBySocialKey)
|
||||
if (socialId == null || socialKey === socialId) continue
|
||||
|
||||
operations.push({
|
||||
filter: { createdBy: socialKey },
|
||||
update: {
|
||||
createdBy: socialId
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
for (const socialKey of groupByModified.keys()) {
|
||||
if (socialKey == null) continue
|
||||
const socialId = await getSocialIdBySocialKey(client, socialKey, socialIdBySocialKey)
|
||||
if (socialId == null || socialKey === socialId) continue
|
||||
|
||||
operations.push({
|
||||
filter: { modifiedBy: socialKey },
|
||||
update: {
|
||||
modifiedBy: socialId
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
if (operations.length > 0) {
|
||||
const operationsChunks = chunkArray(operations, 40)
|
||||
ctx.info('chunks to process ', { total: operationsChunks.length })
|
||||
let processed = 0
|
||||
for (const operationsChunk of operationsChunks) {
|
||||
if (operationsChunk.length === 0) continue
|
||||
|
||||
await client.bulk(domain, operationsChunk)
|
||||
processed++
|
||||
if (operationsChunks.length > 1) {
|
||||
ctx.info('processed chunk', { processed, of: operationsChunks.length })
|
||||
}
|
||||
}
|
||||
} else {
|
||||
ctx.info('no social keys to migrate')
|
||||
}
|
||||
}
|
||||
|
||||
ctx.info('finished migrating createdBy and modifiedBy')
|
||||
}
|
||||
|
||||
async function processMigrateJsonForDomain (
|
||||
ctx: MeasureContext,
|
||||
domain: Domain,
|
||||
@ -1112,18 +921,6 @@ export const coreOperation: MigrateOperation = {
|
||||
state: 'accounts-to-social-ids',
|
||||
mode: 'upgrade',
|
||||
func: migrateAccounts
|
||||
},
|
||||
// ONLY FOR STAGING. REMOVE IT BEFORE MERGING TO PRODUCTION
|
||||
{
|
||||
state: 'created-by-to-account-uuids',
|
||||
mode: 'upgrade',
|
||||
func: migrateSpaceMembersToAccountUuids
|
||||
},
|
||||
// ONLY FOR STAGING. REMOVE IT BEFORE MERGING TO PRODUCTION
|
||||
{
|
||||
state: 'created-by-to-gen-social-ids-v2',
|
||||
mode: 'upgrade',
|
||||
func: migrateCreatedByToGenSocialIds
|
||||
}
|
||||
])
|
||||
},
|
||||
|
@ -46,11 +46,8 @@ import { DOMAIN_PREFERENCE } from '@hcengineering/preference'
|
||||
import {
|
||||
DOMAIN_SPACE,
|
||||
getSocialKeyByOldAccount,
|
||||
getUniqueAccounts,
|
||||
getAccountUuidBySocialKey,
|
||||
getAccountUuidByOldAccount,
|
||||
getUniqueAccountsFromOldAccounts,
|
||||
getSocialIdBySocialKey,
|
||||
getSocialIdFromOldAccount
|
||||
} from '@hcengineering/model-core'
|
||||
import { DOMAIN_DOC_NOTIFY, DOMAIN_NOTIFICATION, DOMAIN_USER_NOTIFY } from './index'
|
||||
@ -258,6 +255,7 @@ async function migrateAccounts (client: MigrationClient): Promise<void> {
|
||||
|
||||
ctx.info('processing collaborators ', {})
|
||||
for (const domain of client.hierarchy.domains()) {
|
||||
if (['tx'].includes(domain)) continue
|
||||
ctx.info('processing domain ', { domain })
|
||||
let processed = 0
|
||||
const iterator = await client.traverse(domain, {})
|
||||
@ -445,232 +443,6 @@ async function migrateAccounts (client: MigrationClient): Promise<void> {
|
||||
ctx.info('finished processing doc notify contexts ', {})
|
||||
}
|
||||
|
||||
/**
|
||||
* Migrates social ids to new accounts where needed.
|
||||
* Should only be applied to staging where old accounts have already been migrated to social ids.
|
||||
* REMOVE IT BEFORE MERGING TO PRODUCTION
|
||||
* @param client
|
||||
* @returns
|
||||
*/
|
||||
async function migrateSocialIdsToAccountUuids (client: MigrationClient): Promise<void> {
|
||||
const ctx = new MeasureMetricsContext('notification migrateSocialIdsToAccountUuids', {})
|
||||
const hierarchy = client.hierarchy
|
||||
const accountUuidBySocialKey = new Map<string, AccountUuid | null>()
|
||||
|
||||
ctx.info('processing collaborators ', {})
|
||||
for (const domain of client.hierarchy.domains()) {
|
||||
ctx.info('processing domain ', { domain })
|
||||
let processed = 0
|
||||
const iterator = await client.traverse(domain, {})
|
||||
|
||||
try {
|
||||
while (true) {
|
||||
const docs = await iterator.next(200)
|
||||
if (docs === null || docs.length === 0) {
|
||||
break
|
||||
}
|
||||
|
||||
const operations: { filter: MigrationDocumentQuery<Doc>, update: MigrateUpdate<Doc> }[] = []
|
||||
|
||||
for (const doc of docs) {
|
||||
const mixin = hierarchy.as(doc, notification.mixin.Collaborators)
|
||||
const oldCollaborators = mixin.collaborators as unknown as PersonId[]
|
||||
|
||||
if (oldCollaborators === undefined || oldCollaborators.length === 0) continue
|
||||
|
||||
const newCollaborators = await getUniqueAccounts(client, oldCollaborators, accountUuidBySocialKey)
|
||||
|
||||
operations.push({
|
||||
filter: { _id: doc._id },
|
||||
update: {
|
||||
[`${notification.mixin.Collaborators}`]: {
|
||||
collaborators: newCollaborators
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
if (operations.length > 0) {
|
||||
await client.bulk(domain, operations)
|
||||
}
|
||||
|
||||
processed += docs.length
|
||||
ctx.info('...processed', { count: processed })
|
||||
}
|
||||
|
||||
ctx.info('finished processing domain ', { domain, processed })
|
||||
} finally {
|
||||
await iterator.close()
|
||||
}
|
||||
}
|
||||
ctx.info('finished processing collaborators ', {})
|
||||
|
||||
ctx.info('processing notifications fields ', {})
|
||||
function chunkArray<T> (array: T[], chunkSize: number): T[][] {
|
||||
const chunks: T[][] = []
|
||||
for (let i = 0; i < array.length; i += chunkSize) {
|
||||
chunks.push(array.slice(i, i + chunkSize))
|
||||
}
|
||||
return chunks
|
||||
}
|
||||
|
||||
const operations: { filter: MigrationDocumentQuery<Doc>, update: MigrateUpdate<Doc> }[] = []
|
||||
const groupByUser = await client.groupBy<PersonId, Doc>(DOMAIN_NOTIFICATION, 'user', {
|
||||
_class: {
|
||||
$in: [
|
||||
notification.class.DocNotifyContext,
|
||||
notification.class.BrowserNotification,
|
||||
notification.class.PushSubscription,
|
||||
notification.class.InboxNotification,
|
||||
notification.class.ActivityInboxNotification,
|
||||
notification.class.CommonInboxNotification
|
||||
]
|
||||
}
|
||||
})
|
||||
|
||||
for (const socialId of groupByUser.keys()) {
|
||||
if (socialId == null) continue
|
||||
const account = await getAccountUuidBySocialKey(client, socialId, accountUuidBySocialKey)
|
||||
|
||||
if (account == null || (account as unknown as PersonId) === socialId) continue
|
||||
|
||||
operations.push({
|
||||
filter: {
|
||||
user: socialId,
|
||||
_class: {
|
||||
$in: [
|
||||
notification.class.DocNotifyContext,
|
||||
notification.class.BrowserNotification,
|
||||
notification.class.PushSubscription,
|
||||
notification.class.InboxNotification,
|
||||
notification.class.ActivityInboxNotification,
|
||||
notification.class.CommonInboxNotification
|
||||
]
|
||||
}
|
||||
},
|
||||
update: {
|
||||
user: account
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
if (operations.length > 0) {
|
||||
const operationsChunks = chunkArray(operations, 40)
|
||||
let processed = 0
|
||||
for (const operationsChunk of operationsChunks) {
|
||||
if (operationsChunk.length === 0) continue
|
||||
|
||||
await client.bulk(DOMAIN_NOTIFICATION, operationsChunk)
|
||||
processed++
|
||||
if (operationsChunks.length > 1) {
|
||||
ctx.info('processed chunk', { processed, of: operationsChunks.length })
|
||||
}
|
||||
}
|
||||
} else {
|
||||
ctx.info('no user social ids to migrate')
|
||||
}
|
||||
|
||||
ctx.info('finished processing notifications fields ', {})
|
||||
|
||||
ctx.info('processing doc notify contexts ', {})
|
||||
// If there's more than one DNC for a user it's not a problem.
|
||||
// We'll migrate all of them but only one will be used going further.
|
||||
// Also, it's only possible on front so we don't need to worry about it.
|
||||
const dncIterator = await client.traverse<DocNotifyContext>(DOMAIN_DOC_NOTIFY, {
|
||||
_class: notification.class.DocNotifyContext
|
||||
})
|
||||
try {
|
||||
let processed = 0
|
||||
while (true) {
|
||||
const docs = await dncIterator.next(200)
|
||||
if (docs === null || docs.length === 0) {
|
||||
break
|
||||
}
|
||||
|
||||
const operations: {
|
||||
filter: MigrationDocumentQuery<DocNotifyContext>
|
||||
update: MigrateUpdate<DocNotifyContext>
|
||||
}[] = []
|
||||
|
||||
for (const doc of docs) {
|
||||
const oldUser: any = doc.user
|
||||
const newUser = await getAccountUuidBySocialKey(client, oldUser, accountUuidBySocialKey)
|
||||
|
||||
if (newUser != null && newUser !== oldUser) {
|
||||
operations.push({
|
||||
filter: { _id: doc._id },
|
||||
update: {
|
||||
user: newUser
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
if (operations.length > 0) {
|
||||
await client.bulk(DOMAIN_DOC_NOTIFY, operations)
|
||||
}
|
||||
|
||||
processed += docs.length
|
||||
ctx.info('...processed', { count: processed })
|
||||
}
|
||||
} finally {
|
||||
await dncIterator.close()
|
||||
}
|
||||
ctx.info('finished processing doc notify contexts ', {})
|
||||
}
|
||||
|
||||
async function migrateSocialKeysToSocialIds (client: MigrationClient): Promise<void> {
|
||||
const ctx = new MeasureMetricsContext('notification migrateSocialKeysToSocialIds', {})
|
||||
ctx.info('processing browser notifications sender ids ', {})
|
||||
const socialIdBySocialKey = new Map<string, PersonId | null>()
|
||||
function chunkArray<T> (array: T[], chunkSize: number): T[][] {
|
||||
const chunks: T[][] = []
|
||||
for (let i = 0; i < array.length; i += chunkSize) {
|
||||
chunks.push(array.slice(i, i + chunkSize))
|
||||
}
|
||||
return chunks
|
||||
}
|
||||
|
||||
const operations: { filter: MigrationDocumentQuery<Doc>, update: MigrateUpdate<Doc> }[] = []
|
||||
const groupBySenderId = await client.groupBy<any, Doc>(DOMAIN_NOTIFICATION, 'senderId', {
|
||||
_class: notification.class.BrowserNotification
|
||||
})
|
||||
|
||||
for (const socialKey of groupBySenderId.keys()) {
|
||||
if (socialKey == null) continue
|
||||
const socialId = (await getSocialIdBySocialKey(client, socialKey, socialIdBySocialKey)) ?? socialKey
|
||||
if (socialId == null || socialKey === socialId) continue
|
||||
|
||||
operations.push({
|
||||
filter: {
|
||||
senderId: socialKey,
|
||||
_class: notification.class.BrowserNotification
|
||||
},
|
||||
update: {
|
||||
senderId: socialId
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
if (operations.length > 0) {
|
||||
const operationsChunks = chunkArray(operations, 40)
|
||||
let processed = 0
|
||||
for (const operationsChunk of operationsChunks) {
|
||||
if (operationsChunk.length === 0) continue
|
||||
|
||||
await client.bulk(DOMAIN_NOTIFICATION, operationsChunk)
|
||||
processed++
|
||||
if (operationsChunks.length > 1) {
|
||||
ctx.info('processed chunk', { processed, of: operationsChunks.length })
|
||||
}
|
||||
}
|
||||
} else {
|
||||
ctx.info('no social keys to migrate')
|
||||
}
|
||||
|
||||
ctx.info('finished processing browser notifications sender ids ', {})
|
||||
}
|
||||
|
||||
export async function migrateSettings (client: MigrationClient): Promise<void> {
|
||||
await client.update(
|
||||
DOMAIN_PREFERENCE,
|
||||
@ -937,26 +709,10 @@ export const notificationOperation: MigrateOperation = {
|
||||
await client.update(DOMAIN_DOC_NOTIFY, { space: core.space.Space }, { space: core.space.Workspace })
|
||||
}
|
||||
},
|
||||
// {
|
||||
// state: 'migrate-notifications-object',
|
||||
// func: migrateNotificationsObject
|
||||
// },
|
||||
{
|
||||
state: 'accounts-to-social-ids',
|
||||
mode: 'upgrade',
|
||||
func: migrateAccounts
|
||||
},
|
||||
// ONLY FOR STAGING. REMOVE IT BEFORE MERGING TO PRODUCTION
|
||||
{
|
||||
state: 'migrate-social-ids-to-account-uuids',
|
||||
mode: 'upgrade',
|
||||
func: migrateSocialIdsToAccountUuids
|
||||
},
|
||||
// ONLY FOR STAGING. REMOVE IT BEFORE MERGING TO PRODUCTION
|
||||
{
|
||||
state: 'migrate-social-keys-to-social-ids-v2',
|
||||
mode: 'upgrade',
|
||||
func: migrateSocialKeysToSocialIds
|
||||
}
|
||||
])
|
||||
},
|
||||
|
@ -24,11 +24,7 @@ import {
|
||||
type MigrationUpgradeClient
|
||||
} from '@hcengineering/model'
|
||||
import setting, { type Integration, settingId } from '@hcengineering/setting'
|
||||
import {
|
||||
getSocialKeyByOldAccount,
|
||||
getUniqueAccounts,
|
||||
getUniqueAccountsFromOldAccounts
|
||||
} from '@hcengineering/model-core'
|
||||
import { getSocialKeyByOldAccount, getUniqueAccountsFromOldAccounts } from '@hcengineering/model-core'
|
||||
|
||||
import { DOMAIN_SETTING } from '.'
|
||||
|
||||
@ -89,58 +85,6 @@ async function migrateAccounts (client: MigrationClient): Promise<void> {
|
||||
ctx.info('finished processing setting integration shared ', {})
|
||||
}
|
||||
|
||||
/**
|
||||
* Migrates social ids to new accounts where needed.
|
||||
* Should only be applied to staging where old accounts have already been migrated to social ids.
|
||||
* REMOVE IT BEFORE MERGING TO PRODUCTION
|
||||
* @param client
|
||||
* @returns
|
||||
*/
|
||||
async function migrateSocialIdsToAccountUuids (client: MigrationClient): Promise<void> {
|
||||
const ctx = new MeasureMetricsContext('setting migrateAccounts', {})
|
||||
const accountUuidBySocialKey = new Map<string, AccountUuid | null>()
|
||||
|
||||
ctx.info('processing setting integration shared ', {})
|
||||
const iterator = await client.traverse(DOMAIN_SETTING, { _class: setting.class.Integration })
|
||||
|
||||
try {
|
||||
let processed = 0
|
||||
while (true) {
|
||||
const docs = await iterator.next(200)
|
||||
if (docs === null || docs.length === 0) {
|
||||
break
|
||||
}
|
||||
|
||||
const operations: { filter: MigrationDocumentQuery<Integration>, update: MigrateUpdate<Integration> }[] = []
|
||||
|
||||
for (const doc of docs) {
|
||||
const integration = doc as Integration
|
||||
|
||||
if (integration.shared === undefined || integration.shared.length === 0) continue
|
||||
|
||||
const newShared = await getUniqueAccounts(client, integration.shared, accountUuidBySocialKey)
|
||||
|
||||
operations.push({
|
||||
filter: { _id: integration._id },
|
||||
update: {
|
||||
shared: newShared
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
if (operations.length > 0) {
|
||||
await client.bulk(DOMAIN_SETTING, operations)
|
||||
}
|
||||
|
||||
processed += docs.length
|
||||
ctx.info('...processed', { count: processed })
|
||||
}
|
||||
} finally {
|
||||
await iterator.close()
|
||||
}
|
||||
ctx.info('finished processing setting integration shared ', {})
|
||||
}
|
||||
|
||||
export const settingOperation: MigrateOperation = {
|
||||
async migrate (client: MigrationClient, mode): Promise<void> {
|
||||
await tryMigrate(mode, client, settingId, [
|
||||
@ -155,12 +99,6 @@ export const settingOperation: MigrateOperation = {
|
||||
state: 'accounts-to-social-ids',
|
||||
mode: 'upgrade',
|
||||
func: migrateAccounts
|
||||
},
|
||||
// ONLY FOR STAGING. REMOVE IT BEFORE MERGING TO PRODUCTION
|
||||
{
|
||||
state: 'migrate-social-ids-to-account-uuids',
|
||||
mode: 'upgrade',
|
||||
func: migrateSocialIdsToAccountUuids
|
||||
}
|
||||
])
|
||||
},
|
||||
|
Loading…
Reference in New Issue
Block a user