// // Copyright © 2024 Anticrm Platform Contributors. // // Licensed under the Eclipse Public License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. You may // obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // // See the License for the specific language governing permissions and // limitations under the License. // import { type WorkspaceIds, type Blob, type MeasureContext, type StorageIterator, type WorkspaceDataId } from '@hcengineering/core' import { PlatformError, unknownError } from '@hcengineering/platform' import { type Readable } from 'stream' export type ListBlobResult = Omit export interface UploadedObjectInfo { etag: string versionId: string | null } export interface BlobStorageIterator { next: () => Promise close: () => Promise } export interface BucketInfo { name: string delete: () => Promise list: () => Promise } export interface StorageAdapter { initialize: (ctx: MeasureContext, wsIds: WorkspaceIds) => Promise close: () => Promise exists: (ctx: MeasureContext, wsIds: WorkspaceIds) => Promise make: (ctx: MeasureContext, wsIds: WorkspaceIds) => Promise delete: (ctx: MeasureContext, wsIds: WorkspaceIds) => Promise listBuckets: (ctx: MeasureContext) => Promise remove: (ctx: MeasureContext, wsIds: WorkspaceIds, objectNames: string[]) => Promise listStream: (ctx: MeasureContext, wsIds: WorkspaceIds) => Promise stat: (ctx: MeasureContext, wsIds: WorkspaceIds, objectName: string) => Promise get: (ctx: MeasureContext, wsIds: WorkspaceIds, objectName: string) => Promise put: ( ctx: MeasureContext, wsIds: WorkspaceIds, objectName: string, stream: Readable | Buffer | string, contentType: string, size?: number ) => Promise read: (ctx: MeasureContext, wsIds: WorkspaceIds, name: string) => Promise partial: ( ctx: MeasureContext, wsIds: WorkspaceIds, objectName: string, offset: number, length?: number ) => Promise getUrl: (ctx: MeasureContext, wsIds: WorkspaceIds, objectName: string) => Promise } export interface NamedStorageAdapter { name: string adapter: StorageAdapter } export interface StorageAdapterEx extends StorageAdapter { adapters?: NamedStorageAdapter[] find: (ctx: MeasureContext, wsIds: WorkspaceIds) => StorageIterator } /** * Ad dummy storage adapter for tests */ export class DummyStorageAdapter implements StorageAdapter, StorageAdapterEx { defaultAdapter: string = '' async syncBlobFromStorage (ctx: MeasureContext, wsIds: WorkspaceIds, objectName: string): Promise { throw new PlatformError(unknownError('Method not implemented')) } async initialize (ctx: MeasureContext, wsIds: WorkspaceIds): Promise {} async close (): Promise {} async exists (ctx: MeasureContext, wsIds: WorkspaceIds): Promise { return false } find (ctx: MeasureContext, wsIds: WorkspaceIds): StorageIterator { return { next: async (ctx) => [], close: async (ctx) => {} } } async listBuckets (ctx: MeasureContext): Promise { return [] } async make (ctx: MeasureContext, wsIds: WorkspaceIds): Promise {} async delete (ctx: MeasureContext, wsIds: WorkspaceIds): Promise {} async remove (ctx: MeasureContext, wsIds: WorkspaceIds, objectNames: string[]): Promise {} async list (ctx: MeasureContext, wsIds: WorkspaceIds): Promise { return [] } async listStream (ctx: MeasureContext, wsIds: WorkspaceIds): Promise { return { next: async (): Promise => { return [] }, close: async () => {} } } async stat (ctx: MeasureContext, wsIds: WorkspaceIds, name: string): Promise { return undefined } async get (ctx: MeasureContext, wsIds: WorkspaceIds, name: string): Promise { throw new Error('not implemented') } async partial ( ctx: MeasureContext, wsIds: WorkspaceIds, objectName: string, offset: number, length?: number | undefined ): Promise { throw new Error('not implemented') } async read (ctx: MeasureContext, wsIds: WorkspaceIds, name: string): Promise { throw new Error('not implemented') } async put ( ctx: MeasureContext, wsIds: WorkspaceIds, objectName: string, stream: string | Readable | Buffer, contentType: string, size?: number | undefined ): Promise { throw new Error('not implemented') } async getUrl (ctx: MeasureContext, wsIds: WorkspaceIds, objectName: string): Promise { throw new Error('not implemented') } } export function createDummyStorageAdapter (): StorageAdapter { return new DummyStorageAdapter() } export async function removeAllObjects ( ctx: MeasureContext, storage: StorageAdapter, wsIds: WorkspaceIds ): Promise { ctx.warn('removing all objects from workspace', wsIds) // We need to list all files and delete them const iterator = await storage.listStream(ctx, wsIds) let bulk: string[] = [] while (true) { const objs = await iterator.next() if (objs.length === 0) { break } for (const obj of objs) { bulk.push(obj._id) if (bulk.length > 50) { await storage.remove(ctx, wsIds, bulk) bulk = [] } } } if (bulk.length > 0) { await storage.remove(ctx, wsIds, bulk) bulk = [] } await iterator.close() } export async function objectsToArray ( ctx: MeasureContext, storage: StorageAdapter, wsIds: WorkspaceIds ): Promise { // We need to list all files and delete them const iterator = await storage.listStream(ctx, wsIds) const bulk: ListBlobResult[] = [] while (true) { const obj = await iterator.next() if (obj.length === 0) { break } bulk.push(...obj) } await iterator.close() return bulk } export function getDataId (wsIds: WorkspaceIds): WorkspaceDataId { return wsIds.dataId ?? (wsIds.uuid as unknown as WorkspaceDataId) }