Merge pull request #242 from mendableai/nsc/partial-data-changes

Partial Data Sliding window of 50

apps/api/src/lib/entities.ts

@@ -77,6 +77,8 @@ export class Document {
   provider?: string;
   warning?: string;
 
+  index?: number;
+
   constructor(data: Partial<Document>) {
     if (!data.content) {
       throw new Error("Missing required fields");

apps/api/src/main/runWebScraper.ts

@@ -19,6 +19,9 @@ export async function startWebScraperPipeline({
     inProgress: (progress) => {
       if (progress.currentDocument) {
         partialDocs.push(progress.currentDocument);
+        if (partialDocs.length > 50) {
+          partialDocs = partialDocs.slice(-50);
+        }
         job.progress({ ...progress, partialDocs: partialDocs });
       }
     },

apps/api/src/scraper/WebScraper/index.ts

@@ -72,7 +72,7 @@ export class WebScraperDataProvider {
               total: totalUrls,
               status: "SCRAPING",
               currentDocumentUrl: url,
-              currentDocument: result,
+              currentDocument: { ...result, index: processedUrls },
             });
           }