github/v-firecrawl
github/v-firecrawl
0
Fork 0
Code Actions

fixing document response

Browse Source
This commit is contained in:
rafaelsideguide 2024-04-18 14:12:39 -03:00
parent 704a059448
commit c4cc4b9262
1 changed files with 3 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
apps/api/src/scraper/WebScraper/index.ts
View File

@ -93,8 +93,7 @@ export class WebScraperDataProvider {
pdfDocuments.push({ pdfDocuments.push({
content: pdfContent, content: pdfContent,
metadata: { sourceURL: pdfLink }, metadata: { sourceURL: pdfLink },
provider: "web", provider: "web-scraper"
type: "text",
}); });
} }
links = links.filter((link) => !link.endsWith(".pdf")); links = links.filter((link) => !link.endsWith(".pdf"));
@ -157,8 +156,7 @@ export class WebScraperDataProvider {
pdfDocuments.push({ pdfDocuments.push({
content: pdfContent, content: pdfContent,
metadata: { sourceURL: pdfLink }, metadata: { sourceURL: pdfLink },
provider: "web", provider: "web-scraper"
type: "text",
}); });
} }
@ -188,8 +186,7 @@ export class WebScraperDataProvider {
pdfDocuments.push({ pdfDocuments.push({
content: pdfContent, content: pdfContent,
metadata: { sourceURL: pdfLink }, metadata: { sourceURL: pdfLink },
provider: "web", provider: "web-scraper"
type: "text",
}); });
} }
links = links.filter((link) => !link.endsWith(".pdf")); links = links.filter((link) => !link.endsWith(".pdf"));