github/v-firecrawl
github/v-firecrawl
0
Fork 0
Code Actions

Update single_url.ts

Browse Source
This commit is contained in:
Nicolas 2024-04-28 12:44:00 -07:00
parent d8ee4e90d6
commit 68838c9e0d
1 changed files with 9 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
apps/api/src/scraper/WebScraper/single_url.ts
View File

@ -20,12 +20,17 @@ export async function generateRequestParams(
headers: { "ScrapingService-Request": "TRUE" }, headers: { "ScrapingService-Request": "TRUE" },
}; };
try {
const urlKey = new URL(url).hostname; const urlKey = new URL(url).hostname;
if (urlSpecificParams.hasOwnProperty(urlKey)) { if (urlSpecificParams.hasOwnProperty(urlKey)) {
return { ...defaultParams, ...urlSpecificParams[urlKey] }; return { ...defaultParams, ...urlSpecificParams[urlKey] };
} else { } else {
return defaultParams; return defaultParams;
} }
} catch (error) {
console.error(`Error generating URL key: ${error}`);
return defaultParams;
}
} }
export async function scrapWithCustomFirecrawl( export async function scrapWithCustomFirecrawl(
url: string, url: string,