// // Copyright © 2024 Anticrm Platform Contributors. // // Licensed under the Eclipse Public License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. You may // obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // // See the License for the specific language governing permissions and // limitations under the License. // import { type Blob, type MeasureContext, type StorageIterator, type WorkspaceId } from '@hcengineering/core' import { type Readable } from 'stream' export type ListBlobResult = Omit export interface UploadedObjectInfo { etag: string versionId: string | null } export interface BlobStorageIterator { next: () => Promise close: () => Promise } export interface BucketInfo { name: string delete: () => Promise list: () => Promise } export interface StorageAdapter { initialize: (ctx: MeasureContext, workspaceId: WorkspaceId) => Promise close: () => Promise exists: (ctx: MeasureContext, workspaceId: WorkspaceId) => Promise make: (ctx: MeasureContext, workspaceId: WorkspaceId) => Promise delete: (ctx: MeasureContext, workspaceId: WorkspaceId) => Promise listBuckets: (ctx: MeasureContext) => Promise remove: (ctx: MeasureContext, workspaceId: WorkspaceId, objectNames: string[]) => Promise listStream: (ctx: MeasureContext, workspaceId: WorkspaceId, prefix?: string) => Promise stat: (ctx: MeasureContext, workspaceId: WorkspaceId, objectName: string) => Promise get: (ctx: MeasureContext, workspaceId: WorkspaceId, objectName: string) => Promise put: ( ctx: MeasureContext, workspaceId: WorkspaceId, objectName: string, stream: Readable | Buffer | string, contentType: string, size?: number ) => Promise read: (ctx: MeasureContext, workspaceId: WorkspaceId, name: string) => Promise partial: ( ctx: MeasureContext, workspaceId: WorkspaceId, objectName: string, offset: number, length?: number ) => Promise getUrl: (ctx: MeasureContext, workspaceId: WorkspaceId, objectName: string) => Promise } export interface StorageAdapterEx extends StorageAdapter { defaultAdapter: string adapters?: Map syncBlobFromStorage: ( ctx: MeasureContext, workspaceId: WorkspaceId, objectName: string, provider?: string ) => Promise find: (ctx: MeasureContext, workspaceId: WorkspaceId) => StorageIterator } /** * Ad dummy storage adapter for tests */ export class DummyStorageAdapter implements StorageAdapter, StorageAdapterEx { defaultAdapter: string = '' async syncBlobFromStorage (ctx: MeasureContext, workspaceId: WorkspaceId, objectName: string): Promise {} async initialize (ctx: MeasureContext, workspaceId: WorkspaceId): Promise {} async close (): Promise {} async exists (ctx: MeasureContext, workspaceId: WorkspaceId): Promise { return false } find (ctx: MeasureContext, workspaceId: WorkspaceId): StorageIterator { return { next: async (ctx) => undefined, close: async (ctx) => {} } } async listBuckets (ctx: MeasureContext): Promise { return [] } async make (ctx: MeasureContext, workspaceId: WorkspaceId): Promise {} async delete (ctx: MeasureContext, workspaceId: WorkspaceId): Promise {} async remove (ctx: MeasureContext, workspaceId: WorkspaceId, objectNames: string[]): Promise {} async list (ctx: MeasureContext, workspaceId: WorkspaceId, prefix?: string | undefined): Promise { return [] } async listStream ( ctx: MeasureContext, workspaceId: WorkspaceId, prefix?: string | undefined ): Promise { return { next: async (): Promise => { return undefined }, close: async () => {} } } async stat (ctx: MeasureContext, workspaceId: WorkspaceId, name: string): Promise { return undefined } async get (ctx: MeasureContext, workspaceId: WorkspaceId, name: string): Promise { throw new Error('not implemented') } async partial ( ctx: MeasureContext, workspaceId: WorkspaceId, objectName: string, offset: number, length?: number | undefined ): Promise { throw new Error('not implemented') } async read (ctx: MeasureContext, workspaceId: WorkspaceId, name: string): Promise { throw new Error('not implemented') } async put ( ctx: MeasureContext, workspaceId: WorkspaceId, objectName: string, stream: string | Readable | Buffer, contentType: string, size?: number | undefined ): Promise { throw new Error('not implemented') } async getUrl (ctx: MeasureContext, workspaceId: WorkspaceId, objectName: string): Promise { throw new Error('not implemented') } } export function createDummyStorageAdapter (): StorageAdapter { return new DummyStorageAdapter() } export async function removeAllObjects ( ctx: MeasureContext, storage: StorageAdapter, workspaceId: WorkspaceId ): Promise { ctx.warn('removing all objects from workspace', { workspaceId }) // We need to list all files and delete them const iterator = await storage.listStream(ctx, workspaceId) let bulk: string[] = [] while (true) { const obj = await iterator.next() if (obj === undefined) { break } bulk.push(obj.storageId) if (bulk.length > 50) { await storage.remove(ctx, workspaceId, bulk) bulk = [] } } if (bulk.length > 0) { await storage.remove(ctx, workspaceId, bulk) bulk = [] } await iterator.close() } export async function objectsToArray ( ctx: MeasureContext, storage: StorageAdapter, workspaceId: WorkspaceId, prefix?: string ): Promise { // We need to list all files and delete them const iterator = await storage.listStream(ctx, workspaceId, prefix) const bulk: ListBlobResult[] = [] while (true) { const obj = await iterator.next() if (obj === undefined) { break } bulk.push(obj) } await iterator.close() return bulk }