diff --git a/apps/api/src/scraper/WebScraper/custom/handleCustomScraping.ts b/apps/api/src/scraper/WebScraper/custom/handleCustomScraping.ts index 5f6c34f..c4a9c26 100644 --- a/apps/api/src/scraper/WebScraper/custom/handleCustomScraping.ts +++ b/apps/api/src/scraper/WebScraper/custom/handleCustomScraping.ts @@ -1,7 +1,7 @@ export async function handleCustomScraping( text: string, url: string -): Promise<{ scraper: string; url: string; wait_after_load: number } | null> { +): Promise<{ scraper: string; url: string; waitAfterLoad: number, pageOptions?: { scrollXPaths?: string[] } } | null> { // Check for Readme Docs special case if (text.includes(', options?: any ): Promise { @@ -68,6 +69,7 @@ export async function scrapWithFireEngine( wait: waitParam, screenshot: screenshotParam, headers: headers, + pageOptions: pageOptions }), }); @@ -332,7 +334,7 @@ export async function scrapSingleUrl( const customScraperResult = await handleCustomScraping(text, url); if(customScraperResult){ - customScrapedContent = await scrapWithFireEngine(customScraperResult.url, customScraperResult.wait_after_load) + customScrapedContent = await scrapWithFireEngine(customScraperResult.url, customScraperResult.waitAfterLoad, false, customScraperResult.pageOptions) } if (customScrapedContent) {