github/v-firecrawl
github/v-firecrawl
0
Fork 0
Code Actions

Merge pull request #242 from mendableai/nsc/partial-data-changes

Browse Source 
Partial Data Sliding window of 50
This commit is contained in:
Nicolas 2024-06-05 11:17:21 -07:00 committed by GitHub
commit 88a5286495
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 6 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
apps/api/src/lib/entities.ts
View File

@ -77,6 +77,8 @@ export class Document {
provider?: string; provider?: string;
warning?: string; warning?: string;
index?: number;
constructor(data: Partial<Document>) { constructor(data: Partial<Document>) {
if (!data.content) { if (!data.content) {
throw new Error("Missing required fields"); throw new Error("Missing required fields");

3
apps/api/src/main/runWebScraper.ts
View File

@ -19,6 +19,9 @@ export async function startWebScraperPipeline({
inProgress: (progress) => { inProgress: (progress) => {
if (progress.currentDocument) { if (progress.currentDocument) {
partialDocs.push(progress.currentDocument); partialDocs.push(progress.currentDocument);
if (partialDocs.length > 50) {
partialDocs = partialDocs.slice(-50);
}
job.progress({ ...progress, partialDocs: partialDocs }); job.progress({ ...progress, partialDocs: partialDocs });
} }
}, },

2
apps/api/src/scraper/WebScraper/index.ts
View File

@ -72,7 +72,7 @@ export class WebScraperDataProvider {
total: totalUrls, total: totalUrls,
status: "SCRAPING", status: "SCRAPING",
currentDocumentUrl: url, currentDocumentUrl: url,
currentDocument: result, currentDocument: { ...result, index: processedUrls },
}); });
} }