github/v-firecrawl
github/v-firecrawl
0
Fork 0
Code Actions

Nick: disabling in crawl and sitemap for now

Browse Source
This commit is contained in:
Nicolas 2024-04-19 13:12:08 -07:00
parent 140529c609
commit 43cfcec326
2 changed files with 11 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
apps/api/src/scraper/WebScraper/index.ts
View File

@ -88,7 +88,7 @@ export class WebScraperDataProvider {
})); }));
} }
let pdfLinks = links.filter((link) => isUrlAPdf(link)); let pdfLinks = links.filter((link) => isUrlAPdf({url: link, fastMode: true}));
let pdfDocuments: Document[] = []; let pdfDocuments: Document[] = [];
for (let pdfLink of pdfLinks) { for (let pdfLink of pdfLinks) {
const pdfContent = await fetchAndProcessPdf(pdfLink); const pdfContent = await fetchAndProcessPdf(pdfLink);
@ -98,7 +98,7 @@ export class WebScraperDataProvider {
provider: "web-scraper" provider: "web-scraper"
}); });
} }
links = links.filter((link) => !isUrlAPdf(link)); links = links.filter((link) => !isUrlAPdf({url: link, fastMode: true}));
let documents = await this.convertUrlsToDocuments(links, inProgress); let documents = await this.convertUrlsToDocuments(links, inProgress);
documents = await this.getSitemapData(this.urls[0], documents); documents = await this.getSitemapData(this.urls[0], documents);
@ -157,7 +157,7 @@ export class WebScraperDataProvider {
} }
if (this.mode === "single_urls") { if (this.mode === "single_urls") {
let pdfLinks = this.urls.filter((link) => isUrlAPdf(link)); let pdfLinks = this.urls.filter((link) => isUrlAPdf({url: link, fastMode: false}));
let pdfDocuments: Document[] = []; let pdfDocuments: Document[] = [];
for (let pdfLink of pdfLinks) { for (let pdfLink of pdfLinks) {
const pdfContent = await fetchAndProcessPdf(pdfLink); const pdfContent = await fetchAndProcessPdf(pdfLink);
@ -169,7 +169,7 @@ export class WebScraperDataProvider {
} }
let documents = await this.convertUrlsToDocuments( let documents = await this.convertUrlsToDocuments(
this.urls.filter((link) => !isUrlAPdf(link)), this.urls.filter((link) => !isUrlAPdf({url: link, fastMode: true})),
inProgress inProgress
); );
@ -193,7 +193,7 @@ export class WebScraperDataProvider {
} }
if (this.mode === "sitemap") { if (this.mode === "sitemap") {
let links = await getLinksFromSitemap(this.urls[0]); let links = await getLinksFromSitemap(this.urls[0]);
let pdfLinks = links.filter((link) => isUrlAPdf(link)); let pdfLinks = links.filter((link) => isUrlAPdf({url: link, fastMode: true}));
let pdfDocuments: Document[] = []; let pdfDocuments: Document[] = [];
for (let pdfLink of pdfLinks) { for (let pdfLink of pdfLinks) {
const pdfContent = await fetchAndProcessPdf(pdfLink); const pdfContent = await fetchAndProcessPdf(pdfLink);
@ -203,7 +203,7 @@ export class WebScraperDataProvider {
provider: "web-scraper" provider: "web-scraper"
}); });
} }
links = links.filter((link) => !isUrlAPdf(link)); links = links.filter((link) => !isUrlAPdf({url: link, fastMode: true}));
let documents = await this.convertUrlsToDocuments( let documents = await this.convertUrlsToDocuments(
links.slice(0, this.limit), links.slice(0, this.limit),

6
apps/api/src/scraper/WebScraper/utils/pdfProcessor.ts
View File

@ -112,11 +112,15 @@ async function processPdf(file: string){
// console.log(e); // console.log(e);
// }) // })
export async function isUrlAPdf(url: string): Promise<boolean> { export async function isUrlAPdf({url, fastMode}: {url: string, fastMode: boolean}): Promise<boolean> {
try { try {
if (url.endsWith('.pdf')) { if (url.endsWith('.pdf')) {
return true; return true;
} }
// If fast mode is enabled, we skip the HEAD request and return false
if (fastMode) {
return false;
}
const response = await fetch(url, { method: 'HEAD' }); const response = await fetch(url, { method: 'HEAD' });
const contentType = response.headers.get('Content-Type'); const contentType = response.headers.get('Content-Type');
return contentType !== null && contentType.includes('application/pdf'); return contentType !== null && contentType.includes('application/pdf');