UBERF-7665: Fix OOM in sharp (#6138)

Signed-off-by: Andrey Sobolev <haiodo@gmail.com>

packages/presentation/src/preview.ts

@@ -52,11 +52,7 @@ export function parsePreviewConfig (config?: string): PreviewConfig | undefined
     if (c === '') {
       continue // Skip empty lines
     }
-    const vars = c.split('|')
+    const [provider, url, contentTypes] = c.split('|').map((it) => it.trim())
-    let [provider, url, formats, contentTypes] = c.split('|').map((it) => it.trim())
-    if (vars.length === 3) {
-      contentTypes = formats // Backward compatibility, since formats are obsolete
-    }
     const p: ProviderPreviewConfig = {
       providerId: provider,
       previewUrl: url,

plugins/view-resources/src/components/viewer/ImageViewer.svelte

@@ -44,7 +44,6 @@
     style:max-width={width}
     style:max-height={height}
     src={blobRef.src}
-    srcset={blobRef.srcset}
     alt={name}
     style:height={loading ? '0' : ''}
   />

server/front/src/index.ts

@@ -31,7 +31,9 @@ import sharp from 'sharp'
 import { v4 as uuid } from 'uuid'
 import { preConditions } from './utils'
 
-import fs from 'fs'
+import fs, { createReadStream, mkdtempSync } from 'fs'
+import { rm, writeFile } from 'fs/promises'
+import { tmpdir } from 'os'
 
 const cacheControlValue = 'public, max-age=365d'
 const cacheControlNoCache = 'public, no-store, no-cache, must-revalidate, max-age=0'
@@ -126,12 +128,11 @@ async function getFileRange (
           dataStream.on('error', (err) => {
             ctx.error('error receive stream', { workspace: workspace.name, uuid, error: err })
             Analytics.handleError(err)
+
             res.end()
+            dataStream.destroy()
             reject(err)
           })
-          dataStream.on('close', () => {
-            res.end()
-          })
         })
       } catch (err: any) {
         if (err?.code === 'NoSuchKey' || err?.code === 'NotFound') {
@@ -169,7 +170,8 @@ async function getFile (
       'content-type': stat.contentType,
       etag: stat.etag,
       'last-modified': new Date(stat.modifiedOn).toISOString(),
-      'cache-control': cacheControlValue
+      'cache-control': cacheControlValue,
+      Connection: 'keep-alive'
     })
     res.end()
     return
@@ -180,7 +182,8 @@ async function getFile (
       'content-type': stat.contentType,
       etag: stat.etag,
       'last-modified': new Date(stat.modifiedOn).toISOString(),
-      'cache-control': cacheControlValue
+      'cache-control': cacheControlValue,
+      Connection: 'keep-alive'
     })
     res.end()
     return
@@ -196,7 +199,8 @@ async function getFile (
           'Content-Type': stat.contentType,
           Etag: stat.etag,
           'Last-Modified': new Date(stat.modifiedOn).toISOString(),
-          'Cache-Control': cacheControlValue
+          'Cache-Control': cacheControlValue,
+          Connection: 'keep-alive'
         })
 
         dataStream.pipe(res)
@@ -209,6 +213,9 @@ async function getFile (
           dataStream.on('error', function (err) {
             Analytics.handleError(err)
             ctx.error('error', { err })
+
+            res.end()
+            dataStream.destroy()
             reject(err)
           })
         })
@@ -249,17 +256,21 @@ export function start (
 ): () => void {
   const app = express()
 
+  const tempFileDir = mkdtempSync(join(tmpdir(), 'front-'))
+  let temoFileIndex = 0
+
   app.use(cors())
   app.use(
     fileUpload({
-      useTempFiles: true
+      useTempFiles: true,
+      tempFileDir
     })
   )
   app.use(bp.json())
   app.use(bp.urlencoded({ extended: true }))
 
   const childLogger = ctx.logger.childLogger?.('requests', {
-    enableConsole: 'false'
+    enableConsole: 'true'
   })
   const requests = ctx.newChild('requests', {}, {}, childLogger)
 
@@ -383,7 +394,7 @@ export function start (
           )
 
           if (blobInfo === undefined) {
-            ctx.error('No such key', { file: uuid, workspace: payload.workspace })
+            ctx.error('No such key', { file: uuid, workspace: payload.workspace.name })
             res.status(404).send()
             return
           }
@@ -417,12 +428,14 @@ export function start (
 
           const size = req.query.size !== undefined ? parseInt(req.query.size as string) : undefined
           const accept = req.headers.accept
-          if (accept !== undefined && isImage && blobInfo.contentType !== 'image/gif') {
+          if (accept !== undefined && isImage && blobInfo.contentType !== 'image/gif' && size !== undefined) {
             blobInfo = await ctx.with(
               'resize',
               {},
               async (ctx) =>
-                await getGeneratePreview(ctx, blobInfo as PlatformBlob, size ?? -1, uuid, config, payload, accept)
+                await getGeneratePreview(ctx, blobInfo as PlatformBlob, size, uuid, config, payload, accept, () =>
+                  join(tempFileDir, `${++temoFileIndex}`)
+                )
             )
           }
 
@@ -746,7 +759,8 @@ async function getGeneratePreview (
   uuid: string,
   config: { storageAdapter: StorageAdapter },
   payload: Token,
-  accept: string
+  accept: string,
+  tempFile: () => string
 ): Promise<PlatformBlob> {
   if (size === undefined) {
     return blob
@@ -769,6 +783,14 @@ async function getGeneratePreview (
     return blob
   }
 
+  if (size === -1) {
+    size = 2048
+  }
+
+  if (size > 2048) {
+    size = 2048
+  }
+
   const sizeId = uuid + `%preview%${size}${format !== 'jpeg' ? format : ''}`
 
   const d = await config.storageAdapter.stat(ctx, payload.workspace, sizeId)
@@ -778,54 +800,61 @@ async function getGeneratePreview (
     // We have cached small document, let's proceed with it.
     return d
   } else {
-    let data: Buffer
+    const files: string[] = []
     try {
       // Let's get data and resize it
-      data = Buffer.concat(await config.storageAdapter.read(ctx, payload.workspace, uuid))
+      const fname = tempFile()
-
+      files.push(fname)
-      let pipeline = sharp(data)
+      await writeFile(fname, await config.storageAdapter.get(ctx, payload.workspace, uuid))
 
+      let pipeline = sharp(fname)
       sharp.cache(false)
 
-      // const metadata = await pipeline.metadata()
+      pipeline = pipeline.resize({
-
+        width: size,
-      if (size !== -1) {
+        fit: 'cover',
-        pipeline = pipeline.resize({
+        withoutEnlargement: true
-          width: size,
+      })
-          fit: 'cover',
-          withoutEnlargement: true
-        })
-      }
 
       let contentType = 'image/jpeg'
       switch (format) {
         case 'jpeg':
-          pipeline = pipeline.jpeg({})
+          pipeline = pipeline.jpeg({
+            progressive: true
+          })
           contentType = 'image/jpeg'
           break
         case 'avif':
           pipeline = pipeline.avif({
-            quality: size !== undefined && size < 128 ? undefined : 85
+            lossless: false,
+            effort: 0
           })
           contentType = 'image/avif'
           break
         case 'heif':
           pipeline = pipeline.heif({
-            quality: size !== undefined && size < 128 ? undefined : 80
+            effort: 0
           })
           contentType = 'image/heif'
           break
         case 'webp':
-          pipeline = pipeline.webp()
+          pipeline = pipeline.webp({
+            effort: 0
+          })
           contentType = 'image/webp'
           break
         case 'png':
-          pipeline = pipeline.png()
+          pipeline = pipeline.png({
+            effort: 0
+          })
           contentType = 'image/png'
           break
       }
 
-      const dataBuff = await pipeline.toBuffer()
+      const outFile = tempFile()
+      files.push(outFile)
+
+      const dataBuff = await ctx.with('resize', { contentType }, async () => await pipeline.toFile(outFile))
       pipeline.destroy()
 
       // Add support of avif as well.
@@ -833,14 +862,14 @@ async function getGeneratePreview (
         ctx,
         payload.workspace,
         sizeId,
-        dataBuff,
+        createReadStream(outFile),
         contentType,
-        dataBuff.length
+        dataBuff.size
       )
       return {
         ...blob,
         _id: sizeId as Ref<PlatformBlob>,
-        size: dataBuff.length,
+        size: dataBuff.size,
         contentType,
         etag: upload.etag,
         storageId: sizeId
@@ -858,6 +887,10 @@ async function getGeneratePreview (
 
       // Return original in case of error
       return blob
+    } finally {
+      for (const f of files) {
+        await rm(f)
+      }
     }
   }
 }