diff --git a/apps/api/src/scraper/WebScraper/custom/handleCustomScraping.ts b/apps/api/src/scraper/WebScraper/custom/handleCustomScraping.ts
index 1301757..33e0783 100644
--- a/apps/api/src/scraper/WebScraper/custom/handleCustomScraping.ts
+++ b/apps/api/src/scraper/WebScraper/custom/handleCustomScraping.ts
@@ -3,7 +3,7 @@ import { fetchAndProcessPdf } from "../utils/pdfProcessor";
export async function handleCustomScraping(
text: string,
url: string
-): Promise<{ scraper: string; url: string; wait_after_load?: number } | null> {
+): Promise<{ scraper: string; url: string; waitAfterLoad: number, pageOptions?: { scrollXPaths?: string[] } } | null> {
// Check for Readme Docs special case
if (text.includes(',
options?: any
): Promise {
@@ -68,6 +69,7 @@ export async function scrapWithFireEngine(
wait: waitParam,
screenshot: screenshotParam,
headers: headers,
+ pageOptions: pageOptions
}),
});
@@ -334,7 +336,7 @@ export async function scrapSingleUrl(
if (customScraperResult){
switch (customScraperResult.scraper) {
case "fire-engine":
- customScrapedContent = await scrapWithFireEngine(customScraperResult.url, customScraperResult.wait_after_load)
+ customScrapedContent = await scrapWithFireEngine(customScraperResult.url, customScraperResult.waitAfterLoad, false, customScraperResult.pageOptions)
case "pdf":
customScrapedContent = { html: await fetchAndProcessPdf(customScraperResult.url), screenshot }
}