mirror of
https://github.com/hcengineering/platform.git
synced 2025-06-09 09:20:54 +00:00
uberf-8425: close migration iterators properly
Signed-off-by: Alexey Zinoviev <alexey.zinoviev@xored.com> Signed-off-by: Alexey Zinoviev <alexey.zinoviev@xored.com>
This commit is contained in:
parent
d3cd5da17e
commit
ff43872a41
@ -195,13 +195,13 @@ async function assignEmployeeRoles (client: MigrationClient): Promise<void> {
|
||||
client.logger.log('assigning roles to employees...', {})
|
||||
|
||||
const wsMembers = await client.accountClient.getWorkspaceMembers()
|
||||
const persons = await client.traverse<Person>(DOMAIN_CONTACT, {
|
||||
const personsIterator = await client.traverse<Person>(DOMAIN_CONTACT, {
|
||||
_class: contact.class.Person
|
||||
})
|
||||
|
||||
try {
|
||||
while (true) {
|
||||
const docs = await persons.next(50)
|
||||
const docs = await personsIterator.next(50)
|
||||
if (docs === null || docs?.length === 0) {
|
||||
break
|
||||
}
|
||||
@ -236,7 +236,7 @@ async function assignEmployeeRoles (client: MigrationClient): Promise<void> {
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
await persons.close()
|
||||
await personsIterator.close()
|
||||
client.logger.log('finished assigning roles to employees...', {})
|
||||
}
|
||||
}
|
||||
@ -402,7 +402,7 @@ async function ensureGlobalPersonsForLocalAccounts (client: MigrationClient): Pr
|
||||
async function createUserProfiles (client: MigrationClient): Promise<void> {
|
||||
client.logger.log('creating user profiles for persons...', {})
|
||||
|
||||
const persons = await client.traverse<Person>(DOMAIN_CONTACT, {
|
||||
const personsIterator = await client.traverse<Person>(DOMAIN_CONTACT, {
|
||||
_class: contact.class.Person,
|
||||
profile: { $exists: false }
|
||||
})
|
||||
@ -418,7 +418,7 @@ async function createUserProfiles (client: MigrationClient): Promise<void> {
|
||||
|
||||
try {
|
||||
while (true) {
|
||||
const docs = await persons.next(200)
|
||||
const docs = await personsIterator.next(200)
|
||||
if (docs === null || docs?.length === 0) {
|
||||
break
|
||||
}
|
||||
@ -452,7 +452,7 @@ async function createUserProfiles (client: MigrationClient): Promise<void> {
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
await persons.close()
|
||||
await personsIterator.close()
|
||||
client.logger.log('finished creating user profiles for persons...', {})
|
||||
}
|
||||
}
|
||||
@ -460,27 +460,31 @@ async function createUserProfiles (client: MigrationClient): Promise<void> {
|
||||
async function fixSocialIdCase (client: MigrationClient): Promise<void> {
|
||||
client.logger.log('Fixing social id case...', {})
|
||||
|
||||
const socialIds = await client.traverse<SocialIdentity>(DOMAIN_CHANNEL, {
|
||||
const socialIdsIterator = await client.traverse<SocialIdentity>(DOMAIN_CHANNEL, {
|
||||
_class: contact.class.SocialIdentity
|
||||
})
|
||||
|
||||
let updated = 0
|
||||
while (true) {
|
||||
const docs = await socialIds.next(200)
|
||||
if (docs === null || docs?.length === 0) {
|
||||
break
|
||||
}
|
||||
|
||||
for (const d of docs) {
|
||||
const newKey = d.key.toLowerCase()
|
||||
const newVal = d.value.toLowerCase()
|
||||
if (newKey !== d.key || newVal !== d.value) {
|
||||
await client.update(DOMAIN_CHANNEL, { _id: d._id }, { key: newKey, value: newVal })
|
||||
updated++
|
||||
try {
|
||||
while (true) {
|
||||
const docs = await socialIdsIterator.next(200)
|
||||
if (docs === null || docs?.length === 0) {
|
||||
break
|
||||
}
|
||||
|
||||
for (const d of docs) {
|
||||
const newKey = d.key.toLowerCase()
|
||||
const newVal = d.value.toLowerCase()
|
||||
if (newKey !== d.key || newVal !== d.value) {
|
||||
await client.update(DOMAIN_CHANNEL, { _id: d._id }, { key: newKey, value: newVal })
|
||||
updated++
|
||||
}
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
await socialIdsIterator.close()
|
||||
client.logger.log('Finished fixing social id case. Total updated:', { updated })
|
||||
}
|
||||
client.logger.log('Finished fixing social id case. Total updated:', { updated })
|
||||
}
|
||||
|
||||
export const contactOperation: MigrateOperation = {
|
||||
|
@ -270,31 +270,35 @@ export async function migrateBackupMixins (client: MigrationClient): Promise<voi
|
||||
|
||||
const txIterator = await client.traverse<TxMixin<Doc, AttachedDoc>>(DOMAIN_TX, { _class: core.class.TxMixin })
|
||||
|
||||
while (true) {
|
||||
const mixinOps = await txIterator.next(500)
|
||||
if (mixinOps === null || mixinOps.length === 0) break
|
||||
const _classes = groupByArray(mixinOps, (it) => it.objectClass)
|
||||
try {
|
||||
while (true) {
|
||||
const mixinOps = await txIterator.next(500)
|
||||
if (mixinOps === null || mixinOps.length === 0) break
|
||||
const _classes = groupByArray(mixinOps, (it) => it.objectClass)
|
||||
|
||||
for (const [_class, ops] of _classes.entries()) {
|
||||
const domain = hierarchy.findDomain(_class)
|
||||
if (domain === undefined) continue
|
||||
let docs = await client.find(domain, { _id: { $in: ops.map((it) => it.objectId) } })
|
||||
for (const [_class, ops] of _classes.entries()) {
|
||||
const domain = hierarchy.findDomain(_class)
|
||||
if (domain === undefined) continue
|
||||
let docs = await client.find(domain, { _id: { $in: ops.map((it) => it.objectId) } })
|
||||
|
||||
docs = docs.filter((it) => {
|
||||
// Check if mixin is last operation by modifiedOn
|
||||
const mops = ops.filter((mi) => mi.objectId === it._id)
|
||||
if (mops.length === 0) return false
|
||||
return mops.some((mi) => mi.modifiedOn === it.modifiedOn && mi.modifiedBy === it.modifiedBy)
|
||||
})
|
||||
docs = docs.filter((it) => {
|
||||
// Check if mixin is last operation by modifiedOn
|
||||
const mops = ops.filter((mi) => mi.objectId === it._id)
|
||||
if (mops.length === 0) return false
|
||||
return mops.some((mi) => mi.modifiedOn === it.modifiedOn && mi.modifiedBy === it.modifiedBy)
|
||||
})
|
||||
|
||||
if (docs.length > 0) {
|
||||
// Check if docs has mixins from list
|
||||
const toUpdate = docs.filter((it) => hierarchy.findAllMixins(it).length > 0)
|
||||
if (toUpdate.length > 0) {
|
||||
await client.update(domain, { _id: { $in: toUpdate.map((it) => it._id) } }, { '%hash%': curHash })
|
||||
if (docs.length > 0) {
|
||||
// Check if docs has mixins from list
|
||||
const toUpdate = docs.filter((it) => hierarchy.findAllMixins(it).length > 0)
|
||||
if (toUpdate.length > 0) {
|
||||
await client.update(domain, { _id: { $in: toUpdate.map((it) => it._id) } }, { '%hash%': curHash })
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
await txIterator.close()
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -160,7 +160,7 @@ async function migrateFixMissingDocSyncInfo (client: MigrationClient): Promise<v
|
||||
[github.mixin.GithubProject]: { $exists: true }
|
||||
})
|
||||
for (const p of projects) {
|
||||
const issues = await client.traverse<Issue>(
|
||||
const issuesIterator = await client.traverse<Issue>(
|
||||
DOMAIN_TASK,
|
||||
{
|
||||
_class: tracker.class.Issue,
|
||||
@ -178,52 +178,56 @@ async function migrateFixMissingDocSyncInfo (client: MigrationClient): Promise<v
|
||||
}
|
||||
)
|
||||
let counter = 0
|
||||
while (true) {
|
||||
const docs = await issues.next(1000)
|
||||
if (docs === null || docs.length === 0) {
|
||||
break
|
||||
}
|
||||
const infos = await client.find(
|
||||
DOMAIN_GITHUB,
|
||||
{
|
||||
_class: github.class.DocSyncInfo,
|
||||
_id: { $in: docs.map((it) => it._id as unknown as Ref<DocSyncInfo>) }
|
||||
},
|
||||
{
|
||||
projection: {
|
||||
_id: 1
|
||||
}
|
||||
try {
|
||||
while (true) {
|
||||
const docs = await issuesIterator.next(1000)
|
||||
if (docs === null || docs.length === 0) {
|
||||
break
|
||||
}
|
||||
)
|
||||
const infoIds = toIdMap(infos)
|
||||
let repository: Ref<GithubIntegrationRepository> | null = null
|
||||
for (const issue of docs) {
|
||||
if (!infoIds.has(issue._id)) {
|
||||
if (client.hierarchy.hasMixin(issue, github.mixin.GithubIssue)) {
|
||||
repository = client.hierarchy.as(issue, github.mixin.GithubIssue).repository
|
||||
}
|
||||
counter++
|
||||
// Missing
|
||||
await client.create<DocSyncInfo>(DOMAIN_GITHUB, {
|
||||
const infos = await client.find(
|
||||
DOMAIN_GITHUB,
|
||||
{
|
||||
_class: github.class.DocSyncInfo,
|
||||
_id: issue._id as any,
|
||||
url: '',
|
||||
githubNumber: 0,
|
||||
repository,
|
||||
objectClass: issue._class,
|
||||
externalVersion: '#', // We need to put this one to handle new documents.
|
||||
needSync: '',
|
||||
derivedVersion: '',
|
||||
attachedTo: issue.attachedTo ?? tracker.ids.NoParent,
|
||||
space: issue.space,
|
||||
modifiedBy: issue.modifiedBy,
|
||||
modifiedOn: issue.modifiedOn
|
||||
})
|
||||
_id: { $in: docs.map((it) => it._id as unknown as Ref<DocSyncInfo>) }
|
||||
},
|
||||
{
|
||||
projection: {
|
||||
_id: 1
|
||||
}
|
||||
}
|
||||
)
|
||||
const infoIds = toIdMap(infos)
|
||||
let repository: Ref<GithubIntegrationRepository> | null = null
|
||||
for (const issue of docs) {
|
||||
if (!infoIds.has(issue._id)) {
|
||||
if (client.hierarchy.hasMixin(issue, github.mixin.GithubIssue)) {
|
||||
repository = client.hierarchy.as(issue, github.mixin.GithubIssue).repository
|
||||
}
|
||||
counter++
|
||||
// Missing
|
||||
await client.create<DocSyncInfo>(DOMAIN_GITHUB, {
|
||||
_class: github.class.DocSyncInfo,
|
||||
_id: issue._id as any,
|
||||
url: '',
|
||||
githubNumber: 0,
|
||||
repository,
|
||||
objectClass: issue._class,
|
||||
externalVersion: '#', // We need to put this one to handle new documents.
|
||||
needSync: '',
|
||||
derivedVersion: '',
|
||||
attachedTo: issue.attachedTo ?? tracker.ids.NoParent,
|
||||
space: issue.space,
|
||||
modifiedBy: issue.modifiedBy,
|
||||
modifiedOn: issue.modifiedOn
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if (counter > 0) {
|
||||
console.log('Created', counter, 'DocSyncInfos')
|
||||
} finally {
|
||||
await issuesIterator.close()
|
||||
if (counter > 0) {
|
||||
console.log('Created', counter, 'DocSyncInfos')
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user