Merge branch 'test/load-testing' of https://github.com/mendableai/firecrawl into test/load-testing

apps/api/fly.staging.toml

@@ -24,8 +24,8 @@ kill_timeout = '5s'
 
 [http_service.concurrency]
   type = "requests"
-  hard_limit = 200
+  hard_limit = 1000
-  soft_limit = 100
+  soft_limit = 1000
 
 [[services]]
   protocol = 'tcp'
@@ -43,8 +43,8 @@ kill_timeout = '5s'
 
   [services.concurrency]
     type = 'connections'
-    hard_limit = 75
+    hard_limit = 1000
-    soft_limit = 30
+    soft_limit = 1000
 
 [[vm]]
   size = 'performance-1x'

apps/api/package.json

@@ -17,7 +17,8 @@
     "worker:production": "node dist/src/services/queue-worker.js",
     "mongo-docker": "docker run -d -p 2717:27017 -v ./mongo-data:/data/db --name mongodb mongo:latest",
     "mongo-docker-console": "docker exec -it mongodb mongosh",
-    "run-example": "npx ts-node src/example.ts"
+    "run-example": "npx ts-node src/example.ts",
+    "deploy:fly:staging": "fly deploy -c fly.staging.toml"
   },
   "author": "",
   "license": "ISC",

apps/api/src/scraper/WebScraper/utils/custom/website_params.ts

@@ -121,5 +121,20 @@ export const urlSpecificParams = {
         accept:
           "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
       },
+  },
+  "scrapethissite.com":{
+    defaultScraper: "fetch",
+    headers: {
+      "User-Agent":
+        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
+      "sec-fetch-site": "same-origin",
+      "sec-fetch-mode": "cors",
+      "sec-fetch-dest": "empty",
+      referer: "https://www.google.com/",
+      "accept-language": "en-US,en;q=0.9",
+      "accept-encoding": "gzip, deflate, br",
+      accept:
+        "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
+    },
   }
 };

apps/api/src/services/rate-limiter.ts

@@ -98,7 +98,7 @@ export function scrapeRateLimit (plan: string){
 
 export function getRateLimiter(mode: RateLimiterMode, token: string){
   // Special test suite case. TODO: Change this later.
-  if (token.includes("5089cefa58")){
+  if (token.includes("fa58")){
     return testSuiteRateLimiter;
   }
   switch (mode) {

apps/test-suite/load-test.yml

@@ -1,11 +1,11 @@
 config:
-  target: "https://api.firecrawl.dev/v0" # <<< ADD HERE THE URL FOR THE STAGING ENVIRONMENT
+  target: "https://staging-firecrawl-scraper-js.fly.dev/v0"
   phases:
-    - duration: 1
+    - duration: 60
-      arrivalRate: 5
+      arrivalRate: 10
   defaults:
     headers:
-      Authorization: "Bearer {{ $env.TEST_API_KEY }}"
+      Authorization: "Bearer fc-f3f77b6d5e084d2b970f595089cefa58"
 scenarios:
   - name: Scrape a URL
     flow:

apps/test-suite/package.json

@@ -4,8 +4,7 @@
   "description": "",
   "scripts": {
     "test:suite": "npx jest --detectOpenHandles --forceExit --openHandlesTimeout=120000 --watchAll=false",
-    "test:load": "artillery run load-test.yml",
+    "test:load": "artillery run load-test.yml"
-    "fly:staging": "fly deploy -c fly.staging.toml"
   },
   "author": "",
   "license": "ISC",