Merge branch 'main' into cjp/contributors-guide-and

apps/api/src/services/queue-worker.ts

@@ -74,6 +74,19 @@ getWebScraperQueue().process(
           "Something went wrong... Contact help@mendable.ai or try again." /* etc... */,
       };
       await callWebhook(job.data.team_id, data);
+      await logJob({
+        success: false,
+        message: typeof error === 'string' ? error : (error.message ?? "Something went wrong... Contact help@mendable.ai"),
+        num_docs: 0,
+        docs: [],
+        time_taken: 0,
+        team_id: job.data.team_id,
+        mode: "crawl",
+        url: job.data.url,
+        crawlerOptions: job.data.crawlerOptions,
+        pageOptions: job.data.pageOptions,
+        origin: job.data.origin,
+      });
       done(null, data);
     }
   }