Remember indexing error (#2493)

Signed-off-by: Andrey Sobolev <haiodo@gmail.com>

dev/prod/webpack.config.js

@@ -59,11 +59,6 @@ module.exports = {
     chunkFilename: '[name].[contenthash].js',
     publicPath: '/'
   },
-  cache: {
-    type: 'filesystem',
-    allowCollectingMemory: true,
-    cacheLocation: path.resolve(__dirname, '../../common/temp/webpack_cache'),
-  },
   module: {
     rules: [
       {

pods/server/Dockerfile

@@ -5,4 +5,4 @@ WORKDIR /usr/src/app
 COPY bundle.js ./
 
 EXPOSE 8080
-CMD [ "node", "bundle.js" ]
+CMD [ "node", "--enable-source-maps", "bundle.js" ]

pods/server/package.json

@@ -8,7 +8,7 @@
     "start": "cross-env MONGO_URL=mongodb://localhost:27017 ELASTIC_URL=http://localhost:9200 FRONT_URL=http://localhost:8087 MINIO_ENDPOINT=localhost MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin METRICS_CONSOLE=true SERVER_SECRET=secret ts-node src/__start.ts",
     "build": "heft build",
     "lint:fix": "eslint --fix src",
-    "bundle": "esbuild src/__start.ts --bundle --platform=node > bundle.js",
+    "bundle": "esbuild src/__start.ts --bundle --sourcemap=inline --minify --platform=node > bundle.js",
     "docker:build": "docker build -t hardcoreeng/transactor .",
     "docker:staging": "../../common/scripts/docker_tag.sh hardcoreeng/transactor staging",
     "docker:push": "../../common/scripts/docker_tag.sh hardcoreeng/transactor",

server/core/src/indexer/content.ts

@@ -151,6 +151,13 @@ export class ContentRetrievalStage implements FullTextPipelineStage {
         }
       }
     } catch (err: any) {
+      const wasError = (doc as any).error !== undefined
+
+      await pipeline.update(doc._id, false, { [docKey('error')]: JSON.stringify({ message: err.message, err }) }, {})
+      if (wasError) {
+        return
+      }
+      // Print error only first time, and update it in doc index
       console.error(err)
       return
     }

server/openai/src/openai.ts

@@ -238,6 +238,13 @@ export class OpenAIEmbeddingsStage implements FullTextPipelineStage {
         if (err.message === 'Response code 401 (Unauthorized)') {
           this.unauthorized = true
         }
+        const wasError = doc.attributes.error !== undefined
+
+        await pipeline.update(doc._id, false, { [docKey('error')]: JSON.stringify(err) }, {})
+        if (wasError) {
+          continue
+        }
+        // Print error only first time, and update it in doc index
         console.error(err)
         continue
       }

server/translate/src/retranslate.ts

@@ -200,6 +200,13 @@ export class LibRetranslateStage implements TranslationStage {
         }
       }
     } catch (err: any) {
+      const wasError = doc.attributes.error !== undefined
+
+      await pipeline.update(doc._id, false, { [docKey('error')]: JSON.stringify(err) }, {})
+      if (wasError) {
+        return
+      }
+      // Print error only first time, and update it in doc index
       console.error(err)
       return
     }