diff --git a/apps/api/src/scraper/WebScraper/custom/handleCustomScraping.ts b/apps/api/src/scraper/WebScraper/custom/handleCustomScraping.ts index 1301757..33e0783 100644 --- a/apps/api/src/scraper/WebScraper/custom/handleCustomScraping.ts +++ b/apps/api/src/scraper/WebScraper/custom/handleCustomScraping.ts @@ -3,7 +3,7 @@ import { fetchAndProcessPdf } from "../utils/pdfProcessor"; export async function handleCustomScraping( text: string, url: string -): Promise<{ scraper: string; url: string; wait_after_load?: number } | null> { +): Promise<{ scraper: string; url: string; waitAfterLoad: number, pageOptions?: { scrollXPaths?: string[] } } | null> { // Check for Readme Docs special case if (text.includes(', options?: any ): Promise { @@ -68,6 +69,7 @@ export async function scrapWithFireEngine( wait: waitParam, screenshot: screenshotParam, headers: headers, + pageOptions: pageOptions }), }); @@ -334,7 +336,7 @@ export async function scrapSingleUrl( if (customScraperResult){ switch (customScraperResult.scraper) { case "fire-engine": - customScrapedContent = await scrapWithFireEngine(customScraperResult.url, customScraperResult.wait_after_load) + customScrapedContent = await scrapWithFireEngine(customScraperResult.url, customScraperResult.waitAfterLoad, false, customScraperResult.pageOptions) case "pdf": customScrapedContent = { html: await fetchAndProcessPdf(customScraperResult.url), screenshot } }