Update queue-worker.ts

apps/api/src/services/queue-worker.ts

@@ -17,7 +17,7 @@ getWebScraperQueue().process(
         current_url: "",
       });
       const start = Date.now();
-      
+
       const { success, message, docs } = await startWebScraperPipeline({ job });
       const end = Date.now();
       const timeTakenInSeconds = (end - start) / 1000;
@@ -74,6 +74,19 @@ getWebScraperQueue().process(
           "Something went wrong... Contact help@mendable.ai or try again." /* etc... */,
       };
       await callWebhook(job.data.team_id, data);
+      await logJob({
+        success: false,
+        message: typeof error === 'string' ? error : (error.message ?? "Something went wrong... Contact help@mendable.ai"),
+        num_docs: 0,
+        docs: [],
+        time_taken: 0,
+        team_id: job.data.team_id,
+        mode: "crawl",
+        url: job.data.url,
+        crawlerOptions: job.data.crawlerOptions,
+        pageOptions: job.data.pageOptions,
+        origin: job.data.origin,
+      });
       done(null, data);
     }
   }