Nick: improvements

apps/api/.env.example

@@ -33,4 +33,6 @@ STRIPE_PRICE_ID_STANDARD=
 STRIPE_PRICE_ID_SCALE=
 
 HYPERDX_API_KEY=
-HDX_NODE_BETA_MODE=1
+HDX_NODE_BETA_MODE=1
+
+FIRE_ENGINE_BETA_URL= # set if you'd like to use the fire engine closed beta

apps/api/src/lib/load-testing-example.ts

@@ -0,0 +1,42 @@
+import { scrapWithFireEngine } from "../../src/scraper/WebScraper/single_url";
+
+const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
+
+const scrapInBatches = async (
+  urls: string[],
+  batchSize: number,
+  delayMs: number
+) => {
+  let successCount = 0;
+  let errorCount = 0;
+
+  for (let i = 0; i < urls.length; i += batchSize) {
+    const batch = urls
+      .slice(i, i + batchSize)
+      .map((url) => scrapWithFireEngine(url));
+    try {
+      const results = await Promise.all(batch);
+      results.forEach((data, index) => {
+        if (data.trim() === "") {
+          errorCount++;
+        } else {
+          successCount++;
+          console.log(
+            `Scraping result ${i + index + 1}:`,
+            data.trim().substring(0, 20) + "..."
+          );
+        }
+      });
+    } catch (error) {
+      console.error("Error during scraping:", error);
+    }
+    await delay(delayMs);
+  }
+
+  console.log(`Total successful scrapes: ${successCount}`);
+  console.log(`Total errored scrapes: ${errorCount}`);
+};
+function run() {
+  const urls = Array.from({ length: 200 }, () => "https://scrapethissite.com");
+  scrapInBatches(urls, 10, 1000);
+}

apps/api/src/scraper/WebScraper/single_url.ts

@@ -10,6 +10,15 @@ import { fetchAndProcessPdf } from "./utils/pdfProcessor";
 
 dotenv.config();
 
+const baseScrapers = [
+  "fire-engine",
+  "scrapingBee",
+  "playwright",
+  "scrapingBeeLoad",
+  "fetch",
+] as const;
+
+
 export async function generateRequestParams(
   url: string,
   wait_browser: string = "domcontentloaded",
@@ -33,15 +42,39 @@ export async function generateRequestParams(
     return defaultParams;
   }
 }
-export async function scrapWithCustomFirecrawl(
+export async function scrapWithFireEngine(
   url: string,
   options?: any
 ): Promise<string> {
   try {
-    // TODO: merge the custom firecrawl scraper into mono-repo when ready
+    const reqParams = await generateRequestParams(url);
-    return null;
+    const wait_playwright = reqParams["params"]?.wait ?? 0;
+
+    const response = await fetch(process.env.FIRE_ENGINE_BETA_URL+ "/scrape", {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+      },
+      body: JSON.stringify({ url: url, wait: wait_playwright }),
+    });
+
+    if (!response.ok) {
+      console.error(
+        `[Fire-Engine] Error fetching url: ${url} with status: ${response.status}`
+      );
+      return "";
+    }
+
+    const contentType = response.headers['content-type'];
+    if (contentType && contentType.includes('application/pdf')) {
+      return fetchAndProcessPdf(url);
+    } else {
+      const data = await response.json();
+      const html = data.content;
+      return html ?? "";
+    }
   } catch (error) {
-    console.error(`Error scraping with custom firecrawl-scraper: ${error}`);
+    console.error(`Error scraping with Fire Engine: ${error}`);
     return "";
   }
 }
@@ -63,7 +96,7 @@ export async function scrapWithScrapingBee(
 
     if (response.status !== 200 && response.status !== 404) {
       console.error(
-        `Scraping bee error in ${url} with status code ${response.status}`
+        `[ScrapingBee] Error fetching url: ${url} with status code ${response.status}`
       );
       return "";
     }
@@ -77,7 +110,7 @@ export async function scrapWithScrapingBee(
       return text;
     }
   } catch (error) {
-    console.error(`Error scraping with Scraping Bee: ${error}`);
+    console.error(`[ScrapingBee] Error fetching url: ${url} -> ${error}`);
     return "";
   }
 }
@@ -97,7 +130,7 @@ export async function scrapWithPlaywright(url: string): Promise<string> {
 
     if (!response.ok) {
       console.error(
-        `Error fetching w/ playwright server -> URL: ${url} with status: ${response.status}`
+        `[Playwright] Error fetching url: ${url} with status: ${response.status}`
       );
       return "";
     }
@@ -111,11 +144,18 @@ export async function scrapWithPlaywright(url: string): Promise<string> {
       return html ?? "";
     }
   } catch (error) {
-    console.error(`Error scraping with Puppeteer: ${error}`);
+    console.error(`Error scraping with Playwright: ${error}`);
     return "";
   }
 }
 
+function getScrapingFallbackOrder(defaultScraper?: string) {
+  const fireEngineScraper = process.env.FIRE_ENGINE_BETA_URL ? ["fire-engine"] : [];
+  const uniqueScrapers = new Set(defaultScraper ? [defaultScraper, ...fireEngineScraper, ...baseScrapers] : [...fireEngineScraper, ...baseScrapers]);
+  const scrapersInOrder = Array.from(uniqueScrapers);
+  return scrapersInOrder as typeof baseScrapers[number][];
+}
+
 export async function scrapSingleUrl(
   urlToScrap: string,
   pageOptions: PageOptions = { onlyMainContent: true, includeHtml: false },
@@ -137,17 +177,12 @@ export async function scrapSingleUrl(
 
   const attemptScraping = async (
     url: string,
-    method:
+    method: typeof baseScrapers[number]
-      | "firecrawl-scraper"
-      | "scrapingBee"
-      | "playwright"
-      | "scrapingBeeLoad"
-      | "fetch"
   ) => {
     let text = "";
     switch (method) {
-      case "firecrawl-scraper":
+      case "fire-engine":
-        text = await scrapWithCustomFirecrawl(url);
+        text = await scrapWithFireEngine(url);
         break;
       case "scrapingBee":
         if (process.env.SCRAPING_BEE_API_KEY) {
@@ -205,15 +240,7 @@ export async function scrapSingleUrl(
       console.error(`Invalid URL key, trying: ${urlToScrap}`);
     }
     const defaultScraper = urlSpecificParams[urlKey]?.defaultScraper ?? "";
-    const scrapersInOrder = defaultScraper
+    const scrapersInOrder = getScrapingFallbackOrder(defaultScraper) 
-      ? [
-          defaultScraper,
-          "scrapingBee",
-          "playwright",
-          "scrapingBeeLoad",
-          "fetch",
-        ]
-      : ["scrapingBee", "playwright", "scrapingBeeLoad", "fetch"];
 
     for (const scraper of scrapersInOrder) {
       // If exists text coming from crawler, use it
@@ -225,7 +252,10 @@ export async function scrapSingleUrl(
       }
       [text, html] = await attemptScraping(urlToScrap, scraper);
       if (text && text.trim().length >= 100) break;
-      console.log(`Falling back to ${scraper}`);
+      const nextScraperIndex = scrapersInOrder.indexOf(scraper) + 1;
+      if (nextScraperIndex < scrapersInOrder.length) {
+        console.info(`Falling back to ${scrapersInOrder[nextScraperIndex]}`);
+      }
     }
 
     if (!text) {

apps/api/src/scraper/WebScraper/utils/custom/website_params.ts

@@ -63,7 +63,7 @@ export const urlSpecificParams = {
       },
   },
   "ycombinator.com":{
-    defaultScraper: "playwright",
+    defaultScraper: "fire-engine",
     params: {
         wait_browser: "networkidle2",
         block_resources: false,