mirror of
https://github.com/hcengineering/platform.git
synced 2025-06-11 21:11:57 +00:00
Merge branch 'develop' of https://github.com/hcengineering/platform into staging-new
Signed-off-by: Artem Savchenko <armisav@gmail.com>
This commit is contained in:
commit
bbdd4ee5ff
2
.vscode/launch.json
vendored
2
.vscode/launch.json
vendored
@ -312,7 +312,7 @@
|
||||
"MINIO_ACCESS_KEY": "minioadmin",
|
||||
"MINIO_SECRET_KEY": "minioadmin",
|
||||
"MINIO_ENDPOINT": "localhost",
|
||||
"MODEL_VERSION": "0.7.75",
|
||||
"MODEL_VERSION": "0.7.110",
|
||||
"WS_OPERATION": "all+backup",
|
||||
"BACKUP_STORAGE": "minio|minio?accessKey=minioadmin&secretKey=minioadmin",
|
||||
"BACKUP_BUCKET": "dev-backups",
|
||||
|
@ -26,7 +26,6 @@ import core, {
|
||||
type Doc,
|
||||
type Domain,
|
||||
groupByArray,
|
||||
MeasureMetricsContext,
|
||||
type PersonId,
|
||||
type Ref,
|
||||
type Space
|
||||
@ -203,12 +202,11 @@ async function migrateActivityMarkup (client: MigrationClient): Promise<void> {
|
||||
}
|
||||
|
||||
async function migrateAccountsToSocialIds (client: MigrationClient): Promise<void> {
|
||||
const ctx = new MeasureMetricsContext('activity migrateAccountsToSocialIds', {})
|
||||
const socialKeyByAccount = await getSocialKeyByOldAccount(client)
|
||||
const socialIdBySocialKey = new Map<string, PersonId | null>()
|
||||
const socialIdByOldAccount = new Map<string, PersonId | null>()
|
||||
|
||||
ctx.info('processing activity reactions ', {})
|
||||
client.logger.log('processing activity reactions ', {})
|
||||
const iterator = await client.traverse(DOMAIN_REACTION, { _class: activity.class.Reaction })
|
||||
|
||||
try {
|
||||
@ -247,12 +245,12 @@ async function migrateAccountsToSocialIds (client: MigrationClient): Promise<voi
|
||||
}
|
||||
|
||||
processed += docs.length
|
||||
ctx.info('...processed', { count: processed })
|
||||
client.logger.log('...processed', { count: processed })
|
||||
}
|
||||
} finally {
|
||||
await iterator.close()
|
||||
}
|
||||
ctx.info('finished processing activity reactions ', {})
|
||||
client.logger.log('finished processing activity reactions ', {})
|
||||
}
|
||||
|
||||
/**
|
||||
@ -262,10 +260,9 @@ async function migrateAccountsToSocialIds (client: MigrationClient): Promise<voi
|
||||
* @returns
|
||||
*/
|
||||
async function migrateAccountsInDocUpdates (client: MigrationClient): Promise<void> {
|
||||
const ctx = new MeasureMetricsContext('activity migrateAccountsToSocialIds', {})
|
||||
const socialKeyByAccount = await getSocialKeyByOldAccount(client)
|
||||
const accountUuidBySocialKey = new Map<string, AccountUuid | null>()
|
||||
ctx.info('processing activity doc updates ', {})
|
||||
client.logger.log('processing activity doc updates ', {})
|
||||
|
||||
function getUpdatedClass (attrKey: string): string {
|
||||
return ['members', 'owners', 'user'].includes(attrKey) ? core.class.TypeAccountUuid : core.class.TypePersonId
|
||||
@ -354,13 +351,13 @@ async function migrateAccountsInDocUpdates (client: MigrationClient): Promise<vo
|
||||
}
|
||||
|
||||
processed += docs.length
|
||||
ctx.info('...processed', { count: processed })
|
||||
client.logger.log('...processed', { count: processed })
|
||||
}
|
||||
} finally {
|
||||
await iterator.close()
|
||||
}
|
||||
|
||||
ctx.info('finished processing activity doc updates ', {})
|
||||
client.logger.log('finished processing activity doc updates ', {})
|
||||
}
|
||||
|
||||
export const activityOperation: MigrateOperation = {
|
||||
|
@ -14,14 +14,7 @@
|
||||
//
|
||||
|
||||
import { type Calendar, calendarId, type Event, type ReccuringEvent } from '@hcengineering/calendar'
|
||||
import core, {
|
||||
type AccountUuid,
|
||||
type Doc,
|
||||
MeasureMetricsContext,
|
||||
type Ref,
|
||||
type Space,
|
||||
toIdMap
|
||||
} from '@hcengineering/core'
|
||||
import core, { type AccountUuid, type Doc, type Ref, type Space, toIdMap } from '@hcengineering/core'
|
||||
import {
|
||||
createDefaultSpace,
|
||||
type MigrateOperation,
|
||||
@ -41,7 +34,6 @@ function getCalendarId (val: string): Ref<Calendar> {
|
||||
}
|
||||
|
||||
async function migrateAccountsToSocialIds (client: MigrationClient): Promise<void> {
|
||||
const ctx = new MeasureMetricsContext('calendar migrateAccountsToSocialIds', {})
|
||||
const hierarchy = client.hierarchy
|
||||
const socialKeyByAccount = await getSocialKeyByOldAccount(client)
|
||||
const eventClasses = hierarchy.getDescendants(calendar.class.Event)
|
||||
@ -50,19 +42,19 @@ async function migrateAccountsToSocialIds (client: MigrationClient): Promise<voi
|
||||
_class: calendar.class.Calendar
|
||||
})
|
||||
|
||||
ctx.info('processing internal calendars')
|
||||
client.logger.log('processing internal calendars', {})
|
||||
|
||||
for (const calendar of calendars) {
|
||||
const id = calendar._id
|
||||
if (!id.endsWith('_calendar')) {
|
||||
ctx.warn('Wrong calendar id format', { calendar: calendar._id })
|
||||
client.logger.error('Wrong calendar id format', { calendar: calendar._id })
|
||||
continue
|
||||
}
|
||||
|
||||
const account = id.substring(0, id.length - 9)
|
||||
const socialId = socialKeyByAccount[account]
|
||||
if (socialId === undefined) {
|
||||
ctx.warn('no socialId for account', { account })
|
||||
client.logger.error('no socialId for account', { account })
|
||||
continue
|
||||
}
|
||||
|
||||
@ -97,7 +89,7 @@ async function migrateAccountsToSocialIds (client: MigrationClient): Promise<voi
|
||||
const account = id.substring(0, id.length - 9)
|
||||
const socialId = socialKeyByAccount[account]
|
||||
if (socialId === undefined) {
|
||||
ctx.warn('no socialId for account', { account })
|
||||
client.logger.error('no socialId for account', { account })
|
||||
continue
|
||||
}
|
||||
|
||||
@ -114,17 +106,16 @@ async function migrateAccountsToSocialIds (client: MigrationClient): Promise<voi
|
||||
}
|
||||
|
||||
processedEvents += events.length
|
||||
ctx.info('...processed events', { count: processedEvents })
|
||||
client.logger.log('...processed events', { count: processedEvents })
|
||||
}
|
||||
|
||||
ctx.info('finished processing events')
|
||||
client.logger.log('finished processing events', {})
|
||||
} finally {
|
||||
await eventsIterator.close()
|
||||
}
|
||||
}
|
||||
|
||||
async function migrateSocialIdsToAccountUuids (client: MigrationClient): Promise<void> {
|
||||
const ctx = new MeasureMetricsContext('calendar migrateSocialIdsToAccountUuids', {})
|
||||
const hierarchy = client.hierarchy
|
||||
const accountUuidBySocialKey = new Map<string, AccountUuid | null>()
|
||||
|
||||
@ -134,19 +125,19 @@ async function migrateSocialIdsToAccountUuids (client: MigrationClient): Promise
|
||||
_class: calendar.class.Calendar
|
||||
})
|
||||
|
||||
ctx.info('processing internal calendars')
|
||||
client.logger.log('processing internal calendars', {})
|
||||
|
||||
for (const calendar of calendars) {
|
||||
const id = calendar._id
|
||||
if (!id.endsWith('_calendar')) {
|
||||
ctx.warn('Wrong calendar id format', { calendar: calendar._id })
|
||||
client.logger.error('Wrong calendar id format', { calendar: calendar._id })
|
||||
continue
|
||||
}
|
||||
|
||||
const socialKey = id.substring(0, id.length - 9)
|
||||
const accountUuid = await getAccountUuidBySocialKey(client, socialKey, accountUuidBySocialKey)
|
||||
if (accountUuid == null) {
|
||||
ctx.warn('no account uuid for social key', { socialKey })
|
||||
client.logger.error('no account uuid for social key', { socialKey })
|
||||
continue
|
||||
}
|
||||
|
||||
@ -181,7 +172,7 @@ async function migrateSocialIdsToAccountUuids (client: MigrationClient): Promise
|
||||
const socialKey = id.substring(0, id.length - 9)
|
||||
const accountUuid = await getAccountUuidBySocialKey(client, socialKey, accountUuidBySocialKey)
|
||||
if (accountUuid == null) {
|
||||
ctx.warn('no account uuid for social key', { socialKey })
|
||||
client.logger.error('no account uuid for social key', { socialKey })
|
||||
continue
|
||||
}
|
||||
|
||||
@ -198,10 +189,10 @@ async function migrateSocialIdsToAccountUuids (client: MigrationClient): Promise
|
||||
}
|
||||
|
||||
processedEvents += events.length
|
||||
ctx.info('...processed events', { count: processedEvents })
|
||||
client.logger.log('...processed events', { count: processedEvents })
|
||||
}
|
||||
|
||||
ctx.info('finished processing events')
|
||||
client.logger.log('finished processing events', {})
|
||||
} finally {
|
||||
await eventsIterator.close()
|
||||
}
|
||||
|
@ -22,7 +22,6 @@ import {
|
||||
DOMAIN_TX,
|
||||
generateId,
|
||||
type MarkupBlobRef,
|
||||
MeasureMetricsContext,
|
||||
type PersonId,
|
||||
type PersonUuid,
|
||||
type Ref,
|
||||
@ -114,8 +113,7 @@ async function getOldPersonAccounts (
|
||||
}
|
||||
|
||||
async function fillAccountUuids (client: MigrationClient): Promise<void> {
|
||||
const ctx = new MeasureMetricsContext('contact fillAccountUuids', {})
|
||||
ctx.info('filling account uuids...')
|
||||
client.logger.log('filling account uuids...', {})
|
||||
const iterator = await client.traverse<Person>(DOMAIN_CONTACT, { _class: contact.class.Person })
|
||||
|
||||
try {
|
||||
@ -170,8 +168,7 @@ async function fillAccountUuids (client: MigrationClient): Promise<void> {
|
||||
}
|
||||
|
||||
async function assignWorkspaceRoles (client: MigrationClient): Promise<void> {
|
||||
const ctx = new MeasureMetricsContext('contact assignWorkspaceRoles', {})
|
||||
ctx.info('assigning workspace roles...')
|
||||
client.logger.log('assigning workspace roles...', {})
|
||||
const oldPersonAccounts = await getOldPersonAccounts(client)
|
||||
for (const { person, email, role } of oldPersonAccounts) {
|
||||
// check it's an active employee
|
||||
@ -187,16 +184,15 @@ async function assignWorkspaceRoles (client: MigrationClient): Promise<void> {
|
||||
try {
|
||||
await client.accountClient.updateWorkspaceRoleBySocialKey(buildSocialIdString(socialKey), role)
|
||||
} catch (err: any) {
|
||||
ctx.error('Failed to update workspace role', { email, ...socialKey, role, err })
|
||||
client.logger.error('Failed to update workspace role', { email, ...socialKey, role, err })
|
||||
}
|
||||
}
|
||||
|
||||
ctx.info('finished assigning workspace roles', { users: oldPersonAccounts.length })
|
||||
client.logger.log('finished assigning workspace roles', { users: oldPersonAccounts.length })
|
||||
}
|
||||
|
||||
async function assignEmployeeRoles (client: MigrationClient): Promise<void> {
|
||||
const ctx = new MeasureMetricsContext('contact assignEmployeeRoles', {})
|
||||
ctx.info('assigning roles to employees...')
|
||||
client.logger.log('assigning roles to employees...', {})
|
||||
|
||||
const wsMembers = await client.accountClient.getWorkspaceMembers()
|
||||
const persons = await client.traverse<Person>(DOMAIN_CONTACT, {
|
||||
@ -241,13 +237,12 @@ async function assignEmployeeRoles (client: MigrationClient): Promise<void> {
|
||||
}
|
||||
} finally {
|
||||
await persons.close()
|
||||
ctx.info('finished assigning roles to employees...')
|
||||
client.logger.log('finished assigning roles to employees...', {})
|
||||
}
|
||||
}
|
||||
|
||||
async function createSocialIdentities (client: MigrationClient): Promise<void> {
|
||||
const ctx = new MeasureMetricsContext('createSocialIdentities', {})
|
||||
ctx.info('processing person accounts ', {})
|
||||
client.logger.log('processing person accounts ', {})
|
||||
|
||||
const socialIdBySocialKey = new Map<string, PersonId | null>()
|
||||
const personAccountsTxes: any[] = await client.find<TxCUD<Doc>>(DOMAIN_MODEL_TX, {
|
||||
@ -287,8 +282,7 @@ async function createSocialIdentities (client: MigrationClient): Promise<void> {
|
||||
}
|
||||
|
||||
async function migrateMergedAccounts (client: MigrationClient): Promise<void> {
|
||||
const ctx = new MeasureMetricsContext('migrateMergedAccounts', {})
|
||||
ctx.info('migrating merged person accounts ', {})
|
||||
client.logger.log('migrating merged person accounts ', {})
|
||||
const accountsByPerson = new Map<string, any[]>()
|
||||
const personAccountsTxes: any[] = await client.find<TxCUD<Doc>>(DOMAIN_MODEL_TX, {
|
||||
objectClass: 'contact:class:PersonAccount' as Ref<Class<Doc>>
|
||||
@ -363,14 +357,13 @@ async function migrateMergedAccounts (client: MigrationClient): Promise<void> {
|
||||
await client.accountClient.addSocialIdToPerson(primaryAccount, addTarget.type, addTarget.value, false)
|
||||
}
|
||||
} catch (err: any) {
|
||||
ctx.error('Failed to merge accounts for person', { person, oldAccounts, err })
|
||||
client.logger.error('Failed to merge accounts for person', { person, oldAccounts, err })
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function ensureGlobalPersonsForLocalAccounts (client: MigrationClient): Promise<void> {
|
||||
const ctx = new MeasureMetricsContext('contact ensureGlobalPersonsForLocalAccounts', {})
|
||||
ctx.info('ensuring global persons for local accounts ', {})
|
||||
client.logger.log('ensuring global persons for local accounts ', {})
|
||||
|
||||
const personAccountsTxes: any[] = await client.find<TxCUD<Doc>>(DOMAIN_MODEL_TX, {
|
||||
objectClass: 'contact:class:PersonAccount' as Ref<Class<Doc>>
|
||||
@ -393,16 +386,21 @@ async function ensureGlobalPersonsForLocalAccounts (client: MigrationClient): Pr
|
||||
await client.accountClient.ensurePerson(socialIdKey.type, socialIdKey.value, effectiveFirstName, lastName)
|
||||
count++
|
||||
} catch (err: any) {
|
||||
ctx.error('Failed to ensure person', { socialIdKey, email: pAcc.email, firstName, lastName, effectiveFirstName })
|
||||
client.logger.error('Failed to ensure person', {
|
||||
socialIdKey,
|
||||
email: pAcc.email,
|
||||
firstName,
|
||||
lastName,
|
||||
effectiveFirstName
|
||||
})
|
||||
console.error(err)
|
||||
}
|
||||
}
|
||||
ctx.info('finished ensuring global persons for local accounts. Total persons ensured: ', { count })
|
||||
client.logger.log('finished ensuring global persons for local accounts. Total persons ensured: ', { count })
|
||||
}
|
||||
|
||||
async function createUserProfiles (client: MigrationClient): Promise<void> {
|
||||
const ctx = new MeasureMetricsContext('contact createUserProfiles', {})
|
||||
ctx.info('creating user profiles for persons...')
|
||||
client.logger.log('creating user profiles for persons...', {})
|
||||
|
||||
const persons = await client.traverse<Person>(DOMAIN_CONTACT, {
|
||||
_class: contact.class.Person,
|
||||
@ -455,13 +453,12 @@ async function createUserProfiles (client: MigrationClient): Promise<void> {
|
||||
}
|
||||
} finally {
|
||||
await persons.close()
|
||||
ctx.info('finished creating user profiles for persons...')
|
||||
client.logger.log('finished creating user profiles for persons...', {})
|
||||
}
|
||||
}
|
||||
|
||||
async function fixSocialIdCase (client: MigrationClient): Promise<void> {
|
||||
const ctx = new MeasureMetricsContext('contact fixSocialIdCase', {})
|
||||
ctx.info('Fixing social id case...')
|
||||
client.logger.log('Fixing social id case...', {})
|
||||
|
||||
const socialIds = await client.traverse<SocialIdentity>(DOMAIN_CHANNEL, {
|
||||
_class: contact.class.SocialIdentity
|
||||
@ -483,7 +480,7 @@ async function fixSocialIdCase (client: MigrationClient): Promise<void> {
|
||||
}
|
||||
}
|
||||
}
|
||||
ctx.info('Finished fixing social id case. Total updated:', { updated })
|
||||
client.logger.log('Finished fixing social id case. Total updated:', { updated })
|
||||
}
|
||||
|
||||
export const contactOperation: MigrateOperation = {
|
||||
|
@ -32,7 +32,6 @@ import {
|
||||
DOMAIN_TX,
|
||||
generateId,
|
||||
makeDocCollabId,
|
||||
MeasureMetricsContext,
|
||||
type Ref,
|
||||
SortingOrder,
|
||||
toIdMap,
|
||||
@ -280,7 +279,6 @@ async function migrateSpaceTypes (client: MigrationClient): Promise<void> {
|
||||
}
|
||||
|
||||
async function migrateDocSections (client: MigrationClient): Promise<void> {
|
||||
const ctx = new MeasureMetricsContext('migrate_doc_sections', {})
|
||||
const storage = client.storageAdapter
|
||||
|
||||
const targetDocuments = await client.find<ControlledDocument>(DOMAIN_DOCUMENTS, {
|
||||
@ -303,7 +301,7 @@ async function migrateDocSections (client: MigrationClient): Promise<void> {
|
||||
// Migrate sections headers + content
|
||||
try {
|
||||
const collabId = makeDocCollabId(document, 'content')
|
||||
const ydoc = await loadCollabYdoc(ctx, storage, client.wsIds, collabId)
|
||||
const ydoc = await loadCollabYdoc(client.ctx, storage, client.wsIds, collabId)
|
||||
if (ydoc === undefined) {
|
||||
// no content, ignore
|
||||
continue
|
||||
@ -349,9 +347,9 @@ async function migrateDocSections (client: MigrationClient): Promise<void> {
|
||||
}
|
||||
})
|
||||
|
||||
await saveCollabYdoc(ctx, storage, client.wsIds, collabId, ydoc)
|
||||
await saveCollabYdoc(client.ctx, storage, client.wsIds, collabId, ydoc)
|
||||
} catch (err) {
|
||||
ctx.error('error collaborative document content migration', { error: err, document: document.title })
|
||||
client.logger.error('error collaborative document content migration', { error: err, document: document.title })
|
||||
}
|
||||
|
||||
attachmentsOps.push({
|
||||
|
@ -27,7 +27,6 @@ import core, {
|
||||
makeCollabJsonId,
|
||||
makeCollabYdocId,
|
||||
makeDocCollabId,
|
||||
MeasureMetricsContext,
|
||||
RateLimiter,
|
||||
SocialIdType,
|
||||
systemAccountUuid,
|
||||
@ -40,7 +39,6 @@ import core, {
|
||||
type Class,
|
||||
type Doc,
|
||||
type Domain,
|
||||
type MeasureContext,
|
||||
type PersonId,
|
||||
type Ref,
|
||||
type Role,
|
||||
@ -169,7 +167,6 @@ async function migrateStatusTransactions (client: MigrationClient): Promise<void
|
||||
}
|
||||
|
||||
async function migrateCollaborativeContentToStorage (client: MigrationClient): Promise<void> {
|
||||
const ctx = new MeasureMetricsContext('migrate_content', {})
|
||||
const storageAdapter = client.storageAdapter
|
||||
|
||||
const hierarchy = client.hierarchy
|
||||
@ -190,8 +187,8 @@ async function migrateCollaborativeContentToStorage (client: MigrationClient): P
|
||||
|
||||
const iterator = await client.traverse(domain, query)
|
||||
try {
|
||||
ctx.info('processing', { _class })
|
||||
await processMigrateContentFor(ctx, domain, attributes, client, storageAdapter, iterator)
|
||||
client.logger.log('processing', { _class })
|
||||
await processMigrateContentFor(domain, attributes, client, storageAdapter, iterator)
|
||||
} finally {
|
||||
await iterator.close()
|
||||
}
|
||||
@ -199,7 +196,6 @@ async function migrateCollaborativeContentToStorage (client: MigrationClient): P
|
||||
}
|
||||
|
||||
async function processMigrateContentFor (
|
||||
ctx: MeasureContext,
|
||||
domain: Domain,
|
||||
attributes: AnyAttribute[],
|
||||
client: MigrationClient,
|
||||
@ -239,9 +235,9 @@ async function processMigrateContentFor (
|
||||
if (value != null && value.startsWith('{')) {
|
||||
try {
|
||||
const buffer = Buffer.from(value)
|
||||
await storageAdapter.put(ctx, client.wsIds, blobId, buffer, 'application/json', buffer.length)
|
||||
await storageAdapter.put(client.ctx, client.wsIds, blobId, buffer, 'application/json', buffer.length)
|
||||
} catch (err) {
|
||||
ctx.error('failed to process document', { _class: doc._class, _id: doc._id, err })
|
||||
client.logger.error('failed to process document', { _class: doc._class, _id: doc._id, err })
|
||||
}
|
||||
|
||||
update[attributeName] = blobId
|
||||
@ -263,7 +259,7 @@ async function processMigrateContentFor (
|
||||
}
|
||||
|
||||
processed += docs.length
|
||||
ctx.info('...processed', { count: processed })
|
||||
client.logger.log('...processed', { count: processed })
|
||||
}
|
||||
}
|
||||
|
||||
@ -303,7 +299,6 @@ export async function migrateBackupMixins (client: MigrationClient): Promise<voi
|
||||
}
|
||||
|
||||
async function migrateCollaborativeDocsToJson (client: MigrationClient): Promise<void> {
|
||||
const ctx = new MeasureMetricsContext('migrateCollaborativeDocsToJson', {})
|
||||
const storageAdapter = client.storageAdapter
|
||||
|
||||
const hierarchy = client.hierarchy
|
||||
@ -324,8 +319,8 @@ async function migrateCollaborativeDocsToJson (client: MigrationClient): Promise
|
||||
|
||||
const iterator = await client.traverse(domain, query)
|
||||
try {
|
||||
ctx.info('processing', { _class })
|
||||
await processMigrateJsonForDomain(ctx, domain, attributes, client, storageAdapter, iterator)
|
||||
client.logger.log('processing', { _class })
|
||||
await processMigrateJsonForDomain(domain, attributes, client, storageAdapter, iterator)
|
||||
} finally {
|
||||
await iterator.close()
|
||||
}
|
||||
@ -398,13 +393,12 @@ export function getSocialKeyByOldEmail (rawEmail: string): SocialKey {
|
||||
* @returns
|
||||
*/
|
||||
async function migrateAccounts (client: MigrationClient): Promise<void> {
|
||||
const ctx = new MeasureMetricsContext('core migrateAccounts', {})
|
||||
const hierarchy = client.hierarchy
|
||||
const socialKeyByAccount = await getSocialKeyByOldAccount(client)
|
||||
const socialIdBySocialKey = new Map<string, PersonId | null>()
|
||||
const socialIdByOldAccount = new Map<string, PersonId | null>()
|
||||
|
||||
ctx.info('migrating createdBy and modifiedBy')
|
||||
client.logger.log('migrating createdBy and modifiedBy', {})
|
||||
function chunkArray<T> (array: T[], chunkSize: number): T[][] {
|
||||
const chunks: T[][] = []
|
||||
for (let i = 0; i < array.length; i += chunkSize) {
|
||||
@ -414,7 +408,7 @@ async function migrateAccounts (client: MigrationClient): Promise<void> {
|
||||
}
|
||||
|
||||
for (const domain of client.hierarchy.domains()) {
|
||||
ctx.info('processing domain ', { domain })
|
||||
client.logger.log('processing domain ', { domain })
|
||||
const operations: { filter: MigrationDocumentQuery<Doc>, update: MigrateUpdate<Doc> }[] = []
|
||||
const groupByCreated = await client.groupBy<any, Doc>(domain, 'createdBy', {})
|
||||
const groupByModified = await client.groupBy<any, Doc>(domain, 'modifiedBy', {})
|
||||
@ -459,7 +453,7 @@ async function migrateAccounts (client: MigrationClient): Promise<void> {
|
||||
|
||||
if (operations.length > 0) {
|
||||
const operationsChunks = chunkArray(operations, 40)
|
||||
ctx.info('chunks to process ', { total: operationsChunks.length })
|
||||
client.logger.log('chunks to process ', { total: operationsChunks.length })
|
||||
let processed = 0
|
||||
for (const operationsChunk of operationsChunks) {
|
||||
if (operationsChunk.length === 0) continue
|
||||
@ -467,15 +461,15 @@ async function migrateAccounts (client: MigrationClient): Promise<void> {
|
||||
await client.bulk(domain, operationsChunk)
|
||||
processed++
|
||||
if (operationsChunks.length > 1) {
|
||||
ctx.info('processed chunk', { processed, of: operationsChunks.length })
|
||||
client.logger.log('processed chunk', { processed, of: operationsChunks.length })
|
||||
}
|
||||
}
|
||||
} else {
|
||||
ctx.info('no user accounts to migrate')
|
||||
client.logger.log('no user accounts to migrate', {})
|
||||
}
|
||||
}
|
||||
|
||||
ctx.info('finished migrating createdBy and modifiedBy')
|
||||
client.logger.log('finished migrating createdBy and modifiedBy', {})
|
||||
|
||||
const spaceTypes = client.model.findAllSync(core.class.SpaceType, {})
|
||||
const spaceTypesById = toIdMap(spaceTypes)
|
||||
@ -493,7 +487,7 @@ async function migrateAccounts (client: MigrationClient): Promise<void> {
|
||||
|
||||
const accountUuidBySocialKey = new Map<string, AccountUuid | null>()
|
||||
|
||||
ctx.info('processing spaces members, owners and roles assignment', {})
|
||||
client.logger.log('processing spaces members, owners and roles assignment', {})
|
||||
let processedSpaces = 0
|
||||
const spacesIterator = await client.traverse(DOMAIN_SPACE, {})
|
||||
|
||||
@ -563,15 +557,15 @@ async function migrateAccounts (client: MigrationClient): Promise<void> {
|
||||
}
|
||||
|
||||
processedSpaces += spaces.length
|
||||
ctx.info('...spaces processed', { count: processedSpaces })
|
||||
client.logger.log('...spaces processed', { count: processedSpaces })
|
||||
}
|
||||
|
||||
ctx.info('finished processing spaces members, owners and roles assignment', { processedSpaces })
|
||||
client.logger.log('finished processing spaces members, owners and roles assignment', { processedSpaces })
|
||||
} finally {
|
||||
await spacesIterator.close()
|
||||
}
|
||||
|
||||
ctx.info('processing space types members', {})
|
||||
client.logger.log('processing space types members', {})
|
||||
let updatedSpaceTypes = 0
|
||||
for (const spaceType of spaceTypes) {
|
||||
if (spaceType.members === undefined || spaceType.members.length === 0) continue
|
||||
@ -601,7 +595,10 @@ async function migrateAccounts (client: MigrationClient): Promise<void> {
|
||||
await client.create(DOMAIN_MODEL_TX, tx)
|
||||
updatedSpaceTypes++
|
||||
}
|
||||
ctx.info('finished processing space types members', { totalSpaceTypes: spaceTypes.length, updatedSpaceTypes })
|
||||
client.logger.log('finished processing space types members', {
|
||||
totalSpaceTypes: spaceTypes.length,
|
||||
updatedSpaceTypes
|
||||
})
|
||||
}
|
||||
|
||||
export async function getAccountUuidBySocialKey (
|
||||
@ -746,7 +743,6 @@ export async function getUniqueAccountsFromOldAccounts (
|
||||
}
|
||||
|
||||
async function processMigrateJsonForDomain (
|
||||
ctx: MeasureContext,
|
||||
domain: Domain,
|
||||
attributes: AnyAttribute[],
|
||||
client: MigrationClient,
|
||||
@ -767,7 +763,7 @@ async function processMigrateJsonForDomain (
|
||||
|
||||
for (const doc of docs) {
|
||||
await rateLimiter.add(async () => {
|
||||
const update = await processMigrateJsonForDoc(ctx, doc, attributes, client, storageAdapter)
|
||||
const update = await processMigrateJsonForDoc(doc, attributes, client, storageAdapter)
|
||||
if (Object.keys(update).length > 0) {
|
||||
operations.push({ filter: { _id: doc._id }, update })
|
||||
}
|
||||
@ -781,12 +777,11 @@ async function processMigrateJsonForDomain (
|
||||
}
|
||||
|
||||
processed += docs.length
|
||||
ctx.info('...processed', { count: processed })
|
||||
client.logger.log('...processed', { count: processed })
|
||||
}
|
||||
}
|
||||
|
||||
async function processMigrateJsonForDoc (
|
||||
ctx: MeasureContext,
|
||||
doc: Doc,
|
||||
attributes: AnyAttribute[],
|
||||
client: MigrationClient,
|
||||
@ -813,7 +808,7 @@ async function processMigrateJsonForDoc (
|
||||
if (value.startsWith('{')) {
|
||||
// For some reason we have documents that are already markups
|
||||
const jsonId = await retry(5, async () => {
|
||||
return await saveCollabJson(ctx, storageAdapter, wsIds, collabId, value)
|
||||
return await saveCollabJson(client.ctx, storageAdapter, wsIds, collabId, value)
|
||||
})
|
||||
|
||||
update[attributeName] = jsonId
|
||||
@ -835,17 +830,22 @@ async function processMigrateJsonForDoc (
|
||||
const ydocId = makeCollabYdocId(collabId)
|
||||
if (ydocId !== currentYdocId) {
|
||||
await retry(5, async () => {
|
||||
const stat = await storageAdapter.stat(ctx, wsIds, currentYdocId)
|
||||
const stat = await storageAdapter.stat(client.ctx, wsIds, currentYdocId)
|
||||
if (stat !== undefined) {
|
||||
const data = await storageAdapter.read(ctx, wsIds, currentYdocId)
|
||||
const data = await storageAdapter.read(client.ctx, wsIds, currentYdocId)
|
||||
const buffer = Buffer.concat(data as any)
|
||||
await storageAdapter.put(ctx, wsIds, ydocId, buffer, 'application/ydoc', buffer.length)
|
||||
await storageAdapter.put(client.ctx, wsIds, ydocId, buffer, 'application/ydoc', buffer.length)
|
||||
}
|
||||
})
|
||||
}
|
||||
} catch (err) {
|
||||
const error = err instanceof Error ? err.message : String(err)
|
||||
ctx.warn('failed to process collaborative doc', { workspace: wsIds.uuid, collabId, currentYdocId, error })
|
||||
client.logger.error('failed to process collaborative doc', {
|
||||
workspace: wsIds.uuid,
|
||||
collabId,
|
||||
currentYdocId,
|
||||
error
|
||||
})
|
||||
}
|
||||
|
||||
const unset = update.$unset ?? {}
|
||||
|
@ -17,7 +17,6 @@ import {
|
||||
DOMAIN_MODEL_TX,
|
||||
DOMAIN_TX,
|
||||
makeDocCollabId,
|
||||
MeasureMetricsContext,
|
||||
type Ref,
|
||||
SortingOrder,
|
||||
type Class,
|
||||
@ -184,7 +183,6 @@ async function renameFields (client: MigrationClient): Promise<void> {
|
||||
}
|
||||
|
||||
async function renameFieldsRevert (client: MigrationClient): Promise<void> {
|
||||
const ctx = new MeasureMetricsContext('renameFieldsRevert', {})
|
||||
const storage = client.storageAdapter
|
||||
|
||||
type ExDocument = Document & {
|
||||
@ -209,7 +207,7 @@ async function renameFieldsRevert (client: MigrationClient): Promise<void> {
|
||||
|
||||
try {
|
||||
const collabId = makeDocCollabId(document, 'content')
|
||||
const ydoc = await loadCollabYdoc(ctx, storage, client.wsIds, collabId)
|
||||
const ydoc = await loadCollabYdoc(client.ctx, storage, client.wsIds, collabId)
|
||||
if (ydoc === undefined) {
|
||||
continue
|
||||
}
|
||||
@ -220,9 +218,9 @@ async function renameFieldsRevert (client: MigrationClient): Promise<void> {
|
||||
|
||||
yDocCopyXmlField(ydoc, 'description', 'content')
|
||||
|
||||
await saveCollabYdoc(ctx, storage, client.wsIds, collabId, ydoc)
|
||||
await saveCollabYdoc(client.ctx, storage, client.wsIds, collabId, ydoc)
|
||||
} catch (err) {
|
||||
ctx.error('error document content migration', { error: err, document: document.title })
|
||||
client.logger.error('error document content migration', { error: err, document: document.title })
|
||||
}
|
||||
}
|
||||
|
||||
@ -245,7 +243,6 @@ async function renameFieldsRevert (client: MigrationClient): Promise<void> {
|
||||
}
|
||||
|
||||
async function restoreContentField (client: MigrationClient): Promise<void> {
|
||||
const ctx = new MeasureMetricsContext('restoreContentField', {})
|
||||
const storage = client.storageAdapter
|
||||
|
||||
const documents = await client.find<Document>(DOMAIN_DOCUMENT, {
|
||||
@ -256,9 +253,9 @@ async function restoreContentField (client: MigrationClient): Promise<void> {
|
||||
for (const document of documents) {
|
||||
try {
|
||||
const collabId = makeDocCollabId(document, 'content')
|
||||
const ydoc = await loadCollabYdoc(ctx, storage, client.wsIds, collabId)
|
||||
const ydoc = await loadCollabYdoc(client.ctx, storage, client.wsIds, collabId)
|
||||
if (ydoc === undefined) {
|
||||
ctx.error('document content not found', { document: document.title })
|
||||
client.logger.error('document content not found', { document: document.title })
|
||||
continue
|
||||
}
|
||||
|
||||
@ -270,13 +267,13 @@ async function restoreContentField (client: MigrationClient): Promise<void> {
|
||||
if (ydoc.share.has('')) {
|
||||
yDocCopyXmlField(ydoc, '', 'content')
|
||||
if (ydoc.share.has('content')) {
|
||||
await saveCollabYdoc(ctx, storage, client.wsIds, collabId, ydoc)
|
||||
await saveCollabYdoc(client.ctx, storage, client.wsIds, collabId, ydoc)
|
||||
} else {
|
||||
ctx.error('document content still not found', { document: document.title })
|
||||
client.logger.error('document content still not found', { document: document.title })
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
ctx.error('error document content migration', { error: err, document: document.title })
|
||||
client.logger.error('error document content migration', { error: err, document: document.title })
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -292,10 +289,9 @@ async function migrateRanks (client: MigrationClient): Promise<void> {
|
||||
}
|
||||
|
||||
async function migrateAccountsToSocialIds (client: MigrationClient): Promise<void> {
|
||||
const ctx = new MeasureMetricsContext('document migrateAccountsToSocialIds', {})
|
||||
const socialKeyByAccount = await getSocialKeyByOldAccount(client)
|
||||
|
||||
ctx.info('processing document lockedBy ', {})
|
||||
client.logger.log('processing document lockedBy ', {})
|
||||
const iterator = await client.traverse(DOMAIN_DOCUMENT, { _class: document.class.Document })
|
||||
|
||||
try {
|
||||
@ -328,19 +324,18 @@ async function migrateAccountsToSocialIds (client: MigrationClient): Promise<voi
|
||||
}
|
||||
|
||||
processed += docs.length
|
||||
ctx.info('...processed', { count: processed })
|
||||
client.logger.log('...processed', { count: processed })
|
||||
}
|
||||
} finally {
|
||||
await iterator.close()
|
||||
}
|
||||
ctx.info('finished processing document lockedBy ', {})
|
||||
client.logger.log('finished processing document lockedBy ', {})
|
||||
}
|
||||
|
||||
async function migrateSocialIdsToGlobalAccounts (client: MigrationClient): Promise<void> {
|
||||
const ctx = new MeasureMetricsContext('document migrateSocialIdsToGlobalAccounts', {})
|
||||
const accountUuidBySocialKey = new Map<string, AccountUuid | null>()
|
||||
|
||||
ctx.info('processing document lockedBy ', {})
|
||||
client.logger.log('processing document lockedBy ', {})
|
||||
const iterator = await client.traverse(DOMAIN_DOCUMENT, { _class: document.class.Document })
|
||||
|
||||
try {
|
||||
@ -375,12 +370,12 @@ async function migrateSocialIdsToGlobalAccounts (client: MigrationClient): Promi
|
||||
}
|
||||
|
||||
processed += docs.length
|
||||
ctx.info('...processed', { count: processed })
|
||||
client.logger.log('...processed', { count: processed })
|
||||
}
|
||||
} finally {
|
||||
await iterator.close()
|
||||
}
|
||||
ctx.info('finished processing document lockedBy ', {})
|
||||
client.logger.log('finished processing document lockedBy ', {})
|
||||
}
|
||||
|
||||
async function removeOldClasses (client: MigrationClient): Promise<void> {
|
||||
|
@ -17,7 +17,6 @@ import chunter from '@hcengineering/chunter'
|
||||
import contact, { type PersonSpace } from '@hcengineering/contact'
|
||||
import core, {
|
||||
DOMAIN_TX,
|
||||
MeasureMetricsContext,
|
||||
type PersonId,
|
||||
type Class,
|
||||
type Doc,
|
||||
@ -247,17 +246,16 @@ export async function migrateDuplicateContexts (client: MigrationClient): Promis
|
||||
* @returns
|
||||
*/
|
||||
async function migrateAccounts (client: MigrationClient): Promise<void> {
|
||||
const ctx = new MeasureMetricsContext('notification migrateAccounts', {})
|
||||
const hierarchy = client.hierarchy
|
||||
const socialKeyByAccount = await getSocialKeyByOldAccount(client)
|
||||
const socialIdBySocialKey = new Map<string, PersonId | null>()
|
||||
const socialIdByOldAccount = new Map<string, PersonId | null>()
|
||||
const accountUuidByOldAccount = new Map<string, AccountUuid | null>()
|
||||
|
||||
ctx.info('processing collaborators ', {})
|
||||
client.logger.log('processing collaborators ', {})
|
||||
for (const domain of client.hierarchy.domains()) {
|
||||
if (['tx'].includes(domain)) continue
|
||||
ctx.info('processing domain ', { domain })
|
||||
client.logger.log('processing domain ', { domain })
|
||||
let processed = 0
|
||||
const iterator = await client.traverse(domain, {})
|
||||
|
||||
@ -298,17 +296,17 @@ async function migrateAccounts (client: MigrationClient): Promise<void> {
|
||||
}
|
||||
|
||||
processed += docs.length
|
||||
ctx.info('...processed', { count: processed })
|
||||
client.logger.log('...processed', { count: processed })
|
||||
}
|
||||
|
||||
ctx.info('finished processing domain ', { domain, processed })
|
||||
client.logger.log('finished processing domain ', { domain, processed })
|
||||
} finally {
|
||||
await iterator.close()
|
||||
}
|
||||
}
|
||||
ctx.info('finished processing collaborators ', {})
|
||||
client.logger.log('finished processing collaborators ', {})
|
||||
|
||||
ctx.info('processing notifications fields ', {})
|
||||
client.logger.log('processing notifications fields ', {})
|
||||
function chunkArray<T> (array: T[], chunkSize: number): T[][] {
|
||||
const chunks: T[][] = []
|
||||
for (let i = 0; i < array.length; i += chunkSize) {
|
||||
@ -387,16 +385,16 @@ async function migrateAccounts (client: MigrationClient): Promise<void> {
|
||||
await client.bulk(DOMAIN_NOTIFICATION, operationsChunk)
|
||||
processed++
|
||||
if (operationsChunks.length > 1) {
|
||||
ctx.info('processed chunk', { processed, of: operationsChunks.length })
|
||||
client.logger.log('processed chunk', { processed, of: operationsChunks.length })
|
||||
}
|
||||
}
|
||||
} else {
|
||||
ctx.info('no user accounts to migrate')
|
||||
client.logger.log('no user accounts to migrate', {})
|
||||
}
|
||||
|
||||
ctx.info('finished processing notifications fields ', {})
|
||||
client.logger.log('finished processing notifications fields ', {})
|
||||
|
||||
ctx.info('processing doc notify contexts ', {})
|
||||
client.logger.log('processing doc notify contexts ', {})
|
||||
const dncIterator = await client.traverse<DocNotifyContext>(DOMAIN_DOC_NOTIFY, {
|
||||
_class: notification.class.DocNotifyContext
|
||||
})
|
||||
@ -432,14 +430,14 @@ async function migrateAccounts (client: MigrationClient): Promise<void> {
|
||||
}
|
||||
|
||||
processed += docs.length
|
||||
ctx.info('...processed', { count: processed })
|
||||
client.logger.log('...processed', { count: processed })
|
||||
}
|
||||
} finally {
|
||||
await dncIterator.close()
|
||||
}
|
||||
ctx.info('finished processing doc notify contexts ', {})
|
||||
client.logger.log('finished processing doc notify contexts ', {})
|
||||
|
||||
ctx.info('processing push subscriptions ', {})
|
||||
client.logger.log('processing push subscriptions ', {})
|
||||
const psIterator = await client.traverse<PushSubscription>(DOMAIN_USER_NOTIFY, {
|
||||
_class: notification.class.PushSubscription
|
||||
})
|
||||
@ -475,12 +473,12 @@ async function migrateAccounts (client: MigrationClient): Promise<void> {
|
||||
}
|
||||
|
||||
processed += docs.length
|
||||
ctx.info('...processed', { count: processed })
|
||||
client.logger.log('...processed', { count: processed })
|
||||
}
|
||||
} finally {
|
||||
await psIterator.close()
|
||||
}
|
||||
ctx.info('finished processing push subscriptions ', {})
|
||||
client.logger.log('finished processing push subscriptions ', {})
|
||||
}
|
||||
|
||||
export async function migrateSettings (client: MigrationClient): Promise<void> {
|
||||
|
@ -13,7 +13,7 @@
|
||||
// limitations under the License.
|
||||
//
|
||||
|
||||
import core, { type AccountUuid, MeasureMetricsContext, type Ref, type Space } from '@hcengineering/core'
|
||||
import core, { type AccountUuid, type Ref, type Space } from '@hcengineering/core'
|
||||
import {
|
||||
migrateSpace,
|
||||
type MigrateUpdate,
|
||||
@ -35,11 +35,10 @@ import { DOMAIN_SETTING } from '.'
|
||||
* @returns
|
||||
*/
|
||||
async function migrateAccounts (client: MigrationClient): Promise<void> {
|
||||
const ctx = new MeasureMetricsContext('setting migrateAccounts', {})
|
||||
const socialKeyByAccount = await getSocialKeyByOldAccount(client)
|
||||
const accountUuidByOldAccount = new Map<string, AccountUuid | null>()
|
||||
|
||||
ctx.info('processing setting integration shared ', {})
|
||||
client.logger.log('processing setting integration shared ', {})
|
||||
const iterator = await client.traverse(DOMAIN_SETTING, { _class: setting.class.Integration })
|
||||
|
||||
try {
|
||||
@ -77,12 +76,12 @@ async function migrateAccounts (client: MigrationClient): Promise<void> {
|
||||
}
|
||||
|
||||
processed += docs.length
|
||||
ctx.info('...processed', { count: processed })
|
||||
client.logger.log('...processed', { count: processed })
|
||||
}
|
||||
} finally {
|
||||
await iterator.close()
|
||||
}
|
||||
ctx.info('finished processing setting integration shared ', {})
|
||||
client.logger.log('finished processing setting integration shared ', {})
|
||||
}
|
||||
|
||||
export const settingOperation: MigrateOperation = {
|
||||
|
@ -24,7 +24,7 @@ import {
|
||||
import { DOMAIN_PREFERENCE } from '@hcengineering/preference'
|
||||
import view, { type Filter, type FilteredView, type ViewletPreference, viewId } from '@hcengineering/view'
|
||||
import { getSocialIdFromOldAccount, getSocialKeyByOldAccount, getUniqueAccounts } from '@hcengineering/model-core'
|
||||
import core, { type AccountUuid, MeasureMetricsContext, type PersonId } from '@hcengineering/core'
|
||||
import core, { type AccountUuid, type PersonId } from '@hcengineering/core'
|
||||
|
||||
import { DOMAIN_VIEW } from '.'
|
||||
|
||||
@ -82,10 +82,9 @@ async function removeDoneStateFilter (client: MigrationClient): Promise<void> {
|
||||
}
|
||||
|
||||
async function migrateAccountsToSocialIds (client: MigrationClient): Promise<void> {
|
||||
const ctx = new MeasureMetricsContext('view migrateAccountsToSocialIds', {})
|
||||
const socialKeyByAccount = await getSocialKeyByOldAccount(client)
|
||||
|
||||
ctx.info('processing view filtered view users ', {})
|
||||
client.logger.log('processing view filtered view users ', {})
|
||||
const iterator = await client.traverse(DOMAIN_VIEW, { _class: view.class.FilteredView })
|
||||
|
||||
try {
|
||||
@ -118,19 +117,18 @@ async function migrateAccountsToSocialIds (client: MigrationClient): Promise<voi
|
||||
}
|
||||
|
||||
processed += docs.length
|
||||
ctx.info('...processed', { count: processed })
|
||||
client.logger.log('...processed', { count: processed })
|
||||
}
|
||||
} finally {
|
||||
await iterator.close()
|
||||
}
|
||||
ctx.info('finished processing view filtered view users ', {})
|
||||
client.logger.log('finished processing view filtered view users ', {})
|
||||
}
|
||||
|
||||
async function migrateSocialIdsToGlobalAccounts (client: MigrationClient): Promise<void> {
|
||||
const ctx = new MeasureMetricsContext('view migrateSocialIdsToGlobalAccounts', {})
|
||||
const accountUuidBySocialKey = new Map<string, AccountUuid | null>()
|
||||
|
||||
ctx.info('processing view filtered view users ', {})
|
||||
client.logger.log('processing view filtered view users ', {})
|
||||
const iterator = await client.traverse(DOMAIN_VIEW, { _class: view.class.FilteredView })
|
||||
|
||||
try {
|
||||
@ -163,22 +161,21 @@ async function migrateSocialIdsToGlobalAccounts (client: MigrationClient): Promi
|
||||
}
|
||||
|
||||
processed += docs.length
|
||||
ctx.info('...processed', { count: processed })
|
||||
client.logger.log('...processed', { count: processed })
|
||||
}
|
||||
} finally {
|
||||
await iterator.close()
|
||||
}
|
||||
ctx.info('finished processing view filtered view users ', {})
|
||||
client.logger.log('finished processing view filtered view users ', {})
|
||||
}
|
||||
|
||||
async function migrateAccsInSavedFilters (client: MigrationClient): Promise<void> {
|
||||
const ctx = new MeasureMetricsContext('view migrateAccsInSavedFilters', {})
|
||||
const hierarchy = client.hierarchy
|
||||
const socialKeyByAccount = await getSocialKeyByOldAccount(client)
|
||||
const socialIdBySocialKey = new Map<string, PersonId | null>()
|
||||
const socialIdByOldAccount = new Map<string, PersonId | null>()
|
||||
|
||||
ctx.info('processing view filtered view accounts in filters ', {})
|
||||
client.logger.log('processing view filtered view accounts in filters ', {})
|
||||
const affectedViews = await client.find<FilteredView>(DOMAIN_VIEW, {
|
||||
_class: view.class.FilteredView,
|
||||
filters: { $regex: '%core:class:Account%' }
|
||||
@ -243,7 +240,7 @@ async function migrateAccsInSavedFilters (client: MigrationClient): Promise<void
|
||||
}
|
||||
}
|
||||
|
||||
ctx.info('finished processing view filtered view accounts in filters ', {})
|
||||
client.logger.log('finished processing view filtered view accounts in filters ', {})
|
||||
}
|
||||
|
||||
export const viewOperation: MigrateOperation = {
|
||||
|
@ -20,7 +20,7 @@ import {
|
||||
} from '@hcengineering/model'
|
||||
import { DOMAIN_PREFERENCE } from '@hcengineering/preference'
|
||||
import workbench, { type WorkbenchTab } from '@hcengineering/workbench'
|
||||
import core, { type AccountUuid, DOMAIN_TX, MeasureMetricsContext } from '@hcengineering/core'
|
||||
import core, { type AccountUuid, DOMAIN_TX } from '@hcengineering/core'
|
||||
import { getAccountUuidBySocialKey, getSocialKeyByOldAccount } from '@hcengineering/model-core'
|
||||
|
||||
import { workbenchId } from '.'
|
||||
@ -30,8 +30,7 @@ async function removeTabs (client: MigrationClient): Promise<void> {
|
||||
}
|
||||
|
||||
async function migrateTabsToSocialIds (client: MigrationClient): Promise<void> {
|
||||
const ctx = new MeasureMetricsContext('workbench migrateTabsToSocialIds', {})
|
||||
ctx.info('migrating workbench tabs to social ids...')
|
||||
client.logger.log('migrating workbench tabs to social ids...', {})
|
||||
const socialKeyByAccount = await getSocialKeyByOldAccount(client)
|
||||
const tabs = await client.find<WorkbenchTab>(DOMAIN_PREFERENCE, { _class: workbench.class.WorkbenchTab })
|
||||
for (const tab of tabs) {
|
||||
@ -40,12 +39,11 @@ async function migrateTabsToSocialIds (client: MigrationClient): Promise<void> {
|
||||
await client.update(DOMAIN_PREFERENCE, { _id: tab._id }, { attachedTo: newAttachedTo })
|
||||
}
|
||||
}
|
||||
ctx.info('migrating workbench tabs to social ids completed...')
|
||||
client.logger.log('migrating workbench tabs to social ids completed...', {})
|
||||
}
|
||||
|
||||
async function migrateSocialIdsToGlobalAccounts (client: MigrationClient): Promise<void> {
|
||||
const ctx = new MeasureMetricsContext('workbench migrateSocialIdsToGlobalAccounts', {})
|
||||
ctx.info('migrating workbench tabs to global accounts...')
|
||||
client.logger.log('migrating workbench tabs to global accounts...', {})
|
||||
const accountUuidBySocialKey = new Map<string, AccountUuid | null>()
|
||||
|
||||
const tabs = await client.find<WorkbenchTab>(DOMAIN_PREFERENCE, { _class: workbench.class.WorkbenchTab })
|
||||
@ -55,7 +53,7 @@ async function migrateSocialIdsToGlobalAccounts (client: MigrationClient): Promi
|
||||
await client.update(DOMAIN_PREFERENCE, { _id: tab._id }, { attachedTo: newAttachedTo })
|
||||
}
|
||||
}
|
||||
ctx.info('migrating workbench tabs to global accounts completed...')
|
||||
client.logger.log('migrating workbench tabs to global accounts completed...', {})
|
||||
}
|
||||
|
||||
export const workbenchOperation: MigrateOperation = {
|
||||
|
@ -191,6 +191,10 @@ export class MeasureMetricsContext implements MeasureContext {
|
||||
end (): void {
|
||||
this.done()
|
||||
}
|
||||
|
||||
getParams (): ParamsType {
|
||||
return this.params
|
||||
}
|
||||
}
|
||||
|
||||
export class NoMetricsContext implements MeasureContext {
|
||||
@ -260,6 +264,10 @@ export class NoMetricsContext implements MeasureContext {
|
||||
}
|
||||
|
||||
end (): void {}
|
||||
|
||||
getParams (): ParamsType {
|
||||
return {}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
|
@ -100,6 +100,7 @@ export interface MeasureContext<Q = any> {
|
||||
logger: MeasureLogger
|
||||
|
||||
parent?: MeasureContext
|
||||
getParams: () => ParamsType
|
||||
|
||||
measure: (name: string, value: number, override?: boolean) => void
|
||||
|
||||
|
@ -11,6 +11,7 @@ import core, {
|
||||
Domain,
|
||||
FindOptions,
|
||||
Hierarchy,
|
||||
MeasureContext,
|
||||
MigrationState,
|
||||
ModelDb,
|
||||
ObjQueryType,
|
||||
@ -118,6 +119,8 @@ export interface MigrationClient {
|
||||
wsIds: WorkspaceIds
|
||||
|
||||
reindex: (domain: Domain, classes: Ref<Class<Doc>>[]) => Promise<void>
|
||||
readonly logger: ModelLogger
|
||||
readonly ctx: MeasureContext
|
||||
}
|
||||
|
||||
/**
|
||||
@ -174,10 +177,10 @@ export async function tryMigrate (
|
||||
if (states.has(migration.state)) continue
|
||||
if (migration.mode == null || migration.mode === mode) {
|
||||
try {
|
||||
console.log('running migration', plugin, migration.state)
|
||||
client.logger.log('running migration', { plugin, state: migration.state })
|
||||
await migration.func(client, mode)
|
||||
} catch (err: any) {
|
||||
console.error(err)
|
||||
client.logger.error('Failed to run migration', { plugin, state: migration.state, err })
|
||||
Analytics.handleError(err)
|
||||
continue
|
||||
}
|
||||
|
1
pods/external/services.d/stream.service
vendored
Normal file
1
pods/external/services.d/stream.service
vendored
Normal file
@ -0,0 +1 @@
|
||||
stream hardcoreeng/service_stream:0.1.0
|
@ -140,7 +140,7 @@ export function start (
|
||||
ctx.newChild('💬 communication api', {}),
|
||||
workspace.uuid,
|
||||
dbUrl,
|
||||
broadcastSessions
|
||||
broadcastSessions as any // FIXME when communication will be inside the repo
|
||||
)
|
||||
}
|
||||
|
||||
|
@ -268,6 +268,7 @@ export async function upgradeModel (
|
||||
|
||||
const { hierarchy, modelDb, model } = await buildModel(ctx, newModel)
|
||||
const { migrateClient: preMigrateClient } = await prepareMigrationClient(
|
||||
ctx,
|
||||
pipeline,
|
||||
hierarchy,
|
||||
modelDb,
|
||||
@ -302,6 +303,7 @@ export async function upgradeModel (
|
||||
})
|
||||
|
||||
const { migrateClient, migrateState } = await prepareMigrationClient(
|
||||
ctx,
|
||||
pipeline,
|
||||
hierarchy,
|
||||
modelDb,
|
||||
@ -390,6 +392,7 @@ export async function upgradeModel (
|
||||
}
|
||||
|
||||
async function prepareMigrationClient (
|
||||
ctx: MeasureContext,
|
||||
pipeline: Pipeline,
|
||||
hierarchy: Hierarchy,
|
||||
model: ModelDb,
|
||||
@ -410,7 +413,8 @@ async function prepareMigrationClient (
|
||||
storageAdapter,
|
||||
accountClient,
|
||||
wsIds,
|
||||
queue
|
||||
queue,
|
||||
ctx
|
||||
)
|
||||
const states = await migrateClient.find<MigrationState>(DOMAIN_MIGRATION, { _class: core.class.MigrationState })
|
||||
const sts = Array.from(groupByArray(states, (it) => it.plugin).entries())
|
||||
|
@ -36,13 +36,14 @@ export class MigrateClientImpl implements MigrationClient {
|
||||
readonly storageAdapter: StorageAdapter,
|
||||
readonly accountClient: AccountClient,
|
||||
readonly wsIds: WorkspaceIds,
|
||||
readonly queue: PlatformQueueProducer<QueueWorkspaceMessage>
|
||||
readonly queue: PlatformQueueProducer<QueueWorkspaceMessage>,
|
||||
ctx?: MeasureContext
|
||||
) {
|
||||
if (this.pipeline.context.lowLevelStorage === undefined) {
|
||||
throw new Error('lowLevelStorage is not defined')
|
||||
}
|
||||
this.lowLevel = this.pipeline.context.lowLevelStorage
|
||||
this.ctx = new MeasureMetricsContext('migrateClient', {})
|
||||
this.ctx = ctx ?? new MeasureMetricsContext('migrateClient', {})
|
||||
}
|
||||
|
||||
migrateState = new Map<string, Set<string>>()
|
||||
|
@ -19,6 +19,7 @@ import {
|
||||
isArchivingMode,
|
||||
isMigrationMode,
|
||||
isRestoringMode,
|
||||
MeasureMetricsContext,
|
||||
systemAccountUuid,
|
||||
type BrandingMap,
|
||||
type Data,
|
||||
@ -38,6 +39,7 @@ import {
|
||||
import { generateToken } from '@hcengineering/server-token'
|
||||
import { FileModelLogger, prepareTools } from '@hcengineering/server-tool'
|
||||
import path from 'path'
|
||||
import { randomUUID } from 'crypto'
|
||||
|
||||
import { Analytics } from '@hcengineering/analytics'
|
||||
import {
|
||||
@ -188,14 +190,14 @@ export class WorkspaceWorker {
|
||||
await this.doSleep(ctx, opt)
|
||||
} else {
|
||||
void this.exec(async () => {
|
||||
await ctx
|
||||
.with('workspaceOperation', { mode: workspace.mode }, (ctx) =>
|
||||
this.doWorkspaceOperation(ctx, workspace, opt)
|
||||
)
|
||||
.catch((err) => {
|
||||
Analytics.handleError(err)
|
||||
ctx.error('error', { err })
|
||||
})
|
||||
const job = randomUUID().slice(-8)
|
||||
const opContext = new MeasureMetricsContext(`ws op with job ${job}`, { job })
|
||||
try {
|
||||
await this.doWorkspaceOperation(opContext, workspace, opt)
|
||||
} catch (err: any) {
|
||||
Analytics.handleError(err)
|
||||
ctx.error('error', { err })
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@ -234,6 +236,7 @@ export class WorkspaceWorker {
|
||||
|
||||
ctx.info('---CREATING----', {
|
||||
workspace: ws.uuid,
|
||||
mode: ws.mode,
|
||||
version: this.version,
|
||||
region: this.region
|
||||
})
|
||||
@ -339,6 +342,7 @@ export class WorkspaceWorker {
|
||||
|
||||
ctx.info('---UPGRADING----', {
|
||||
workspace: ws.uuid,
|
||||
mode: ws.mode,
|
||||
workspaceVersion,
|
||||
requestedVersion: this.version,
|
||||
region: this.region
|
||||
|
@ -50,7 +50,7 @@ export async function createWorkspace (
|
||||
) => Promise<void>,
|
||||
external: boolean = false
|
||||
): Promise<void> {
|
||||
const childLogger = ctx.newChild('createWorkspace', {}, {})
|
||||
const childLogger = ctx.newChild('createWorkspace', ctx.getParams(), {})
|
||||
const ctxModellogger: ModelLogger = {
|
||||
log: (msg, data) => {
|
||||
childLogger.info(msg, data)
|
||||
|
@ -40,7 +40,8 @@ describe('AttachmentHandler', () => {
|
||||
error: jest.fn(),
|
||||
info: jest.fn(),
|
||||
warn: jest.fn(),
|
||||
end: jest.fn()
|
||||
end: jest.fn(),
|
||||
getParams: jest.fn()
|
||||
}
|
||||
const mockWorkspaceLoginInfo: WorkspaceLoginInfo = {
|
||||
endpoint: 'wss://test-endpoint.com',
|
||||
|
@ -83,7 +83,7 @@ async function wait (sec: number): Promise<void> {
|
||||
|
||||
export class GmailClient {
|
||||
private readonly account: AccountUuid
|
||||
private email: string | undefined = undefined
|
||||
private email: string
|
||||
private readonly tokenStorage: TokenStorage
|
||||
private readonly client: TxOperations
|
||||
private watchTimer: NodeJS.Timeout | undefined = undefined
|
||||
@ -374,7 +374,7 @@ export class GmailClient {
|
||||
const profile = await this.gmail.getProfile({
|
||||
userId: 'me'
|
||||
})
|
||||
this.email = profile.data.emailAddress ?? undefined
|
||||
this.email = profile.data.emailAddress ?? this.email
|
||||
if (this.email !== undefined) return this.email
|
||||
await wait(5)
|
||||
}
|
||||
@ -394,7 +394,7 @@ export class GmailClient {
|
||||
try {
|
||||
this.ctx.info('Register client', { socialId: this.socialId._id, email: this.email })
|
||||
const controller = GmailController.getGmailController()
|
||||
controller.addClient(this.socialId._id, this.user.workspace, this)
|
||||
controller.addClient(this.socialId._id, this.user.workspace, this.email, this)
|
||||
} catch (err) {
|
||||
this.ctx.error('Add client error', {
|
||||
workspaceUuid: this.user.workspace,
|
||||
|
@ -22,6 +22,7 @@ import {
|
||||
type PersonId
|
||||
} from '@hcengineering/core'
|
||||
import type { StorageAdapter } from '@hcengineering/server-core'
|
||||
import { normalizeEmail } from '@hcengineering/mail-common'
|
||||
|
||||
import { decode64 } from './base64'
|
||||
import config from './config'
|
||||
@ -44,6 +45,8 @@ export class GmailController {
|
||||
Map<WorkspaceUuid, GmailClient>
|
||||
>()
|
||||
|
||||
private readonly personIdByEmail = new Map<string, PersonId>()
|
||||
|
||||
private readonly initLimitter = new RateLimiter(config.InitLimit)
|
||||
private readonly authProvider
|
||||
|
||||
@ -162,22 +165,30 @@ export class GmailController {
|
||||
push (message: string): void {
|
||||
const data = JSON.parse(decode64(message))
|
||||
const email = data.emailAddress
|
||||
const clients = this.clients.get(email)
|
||||
if (clients === undefined) {
|
||||
this.ctx.info('No clients found', { email })
|
||||
const socialId = this.personIdByEmail.get(normalizeEmail(email))
|
||||
if (socialId === undefined) {
|
||||
this.ctx.warn('No socialId found for email', { email })
|
||||
return
|
||||
}
|
||||
|
||||
const clients = this.clients.get(socialId)
|
||||
if (clients === undefined) {
|
||||
this.ctx.info('No clients found', { email, socialId })
|
||||
return
|
||||
}
|
||||
this.ctx.info('Processing push', { clients: clients.size, email })
|
||||
for (const client of clients.values()) {
|
||||
void client.sync()
|
||||
}
|
||||
}
|
||||
|
||||
addClient (socialId: PersonId, workspace: WorkspaceUuid, client: GmailClient): void {
|
||||
addClient (socialId: PersonId, workspace: WorkspaceUuid, email: string, client: GmailClient): void {
|
||||
let userClients = this.clients.get(socialId)
|
||||
if (userClients === undefined) {
|
||||
userClients = new Map<WorkspaceUuid, GmailClient>()
|
||||
this.clients.set(socialId, userClients)
|
||||
}
|
||||
this.personIdByEmail.set(normalizeEmail(email), socialId)
|
||||
|
||||
const existingClient = userClients.get(workspace)
|
||||
if (existingClient != null) {
|
||||
|
@ -163,8 +163,8 @@ export const main = async (): Promise<void> => {
|
||||
gmailController.push(data)
|
||||
|
||||
res.send()
|
||||
} catch (err) {
|
||||
ctx.error('Push request failed', { message: JSON.stringify(err) })
|
||||
} catch (err: any) {
|
||||
ctx.error('Push request failed', { message: err.message })
|
||||
res.status(500).send()
|
||||
}
|
||||
}
|
||||
|
@ -18,6 +18,7 @@ import chat from '@hcengineering/chat'
|
||||
import mail from '@hcengineering/mail'
|
||||
import { PersonSpace } from '@hcengineering/contact'
|
||||
import { SyncMutex } from './mutex'
|
||||
import { normalizeEmail } from './utils'
|
||||
|
||||
const createMutex = new SyncMutex()
|
||||
|
||||
@ -191,7 +192,3 @@ export const ChannelCacheFactory = {
|
||||
return ChannelCacheFactory.instances.size
|
||||
}
|
||||
}
|
||||
|
||||
function normalizeEmail (email: string): string {
|
||||
return email.toLowerCase().trim()
|
||||
}
|
||||
|
@ -127,3 +127,7 @@ export function parseNameFromEmailHeader (headerValue: string | undefined): Emai
|
||||
lastName: ''
|
||||
}
|
||||
}
|
||||
|
||||
export function normalizeEmail (email: string): string {
|
||||
return email.toLowerCase().trim()
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user