github/v-firecrawl
github/v-firecrawl
0
Fork 0
Code Actions

Nick: a bit faster

Browse Source
This commit is contained in:
Nicolas 2024-04-19 15:13:17 -07:00
parent c5cb268b61
commit 5b93799149
1 changed files with 13 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
apps/api/src/scraper/WebScraper/index.ts
View File

@ -157,19 +157,23 @@ export class WebScraperDataProvider {
} }
if (this.mode === "single_urls") { if (this.mode === "single_urls") {
let pdfLinks = this.urls.filter((link) => isUrlAPdf({url: link, fastMode: false}));
let pdfDocuments: Document[] = []; let pdfDocuments: Document[] = [];
for (let pdfLink of pdfLinks) { let nonPdfUrls: string[] = [];
const pdfContent = await fetchAndProcessPdf(pdfLink); for (let url of this.urls) {
if (isUrlAPdf({url: url, fastMode: false})) {
const pdfContent = await fetchAndProcessPdf(url);
pdfDocuments.push({ pdfDocuments.push({
content: pdfContent, content: pdfContent,
metadata: { sourceURL: pdfLink }, metadata: { sourceURL: url },
provider: "web-scraper" provider: "web-scraper"
}); });
} else {
nonPdfUrls.push(url);
}
} }
let documents = await this.convertUrlsToDocuments( let documents = await this.convertUrlsToDocuments(
this.urls.filter((link) => !isUrlAPdf({url: link, fastMode: true})), nonPdfUrls,
inProgress inProgress
); );