From d667e1417bee6a484a38aa20fa9ffb560874baf5 Mon Sep 17 00:00:00 2001
From: rafaelsideguide <150964962+rafaelsideguide@users.noreply.github.com>
Date: Fri, 17 May 2024 19:09:19 -0300
Subject: [PATCH] added fly staging load test

- being rate limited. Need to add the token to the rate-limit functions
---
 apps/api/fly.staging.toml                         |  8 ++++----
 apps/api/package.json                             |  3 ++-
 .../WebScraper/utils/custom/website_params.ts     | 15 +++++++++++++++
 apps/api/src/services/rate-limiter.ts             |  2 +-
 apps/test-suite/load-test.yml                     |  8 ++++----
 apps/test-suite/package.json                      |  3 +--
 6 files changed, 27 insertions(+), 12 deletions(-)

diff --git a/apps/api/fly.staging.toml b/apps/api/fly.staging.toml
index d5ad9ef..2e0a38f 100644
--- a/apps/api/fly.staging.toml
+++ b/apps/api/fly.staging.toml
@@ -24,8 +24,8 @@ kill_timeout = '5s'
 
 [http_service.concurrency]
   type = "requests"
-  hard_limit = 200
-  soft_limit = 100
+  hard_limit = 1000
+  soft_limit = 1000
 
 [[services]]
   protocol = 'tcp'
@@ -43,8 +43,8 @@ kill_timeout = '5s'
 
   [services.concurrency]
     type = 'connections'
-    hard_limit = 75
-    soft_limit = 30
+    hard_limit = 1000
+    soft_limit = 1000
 
 [[vm]]
   size = 'performance-4x'
diff --git a/apps/api/package.json b/apps/api/package.json
index a79e3dc..9c68e92 100644
--- a/apps/api/package.json
+++ b/apps/api/package.json
@@ -17,7 +17,8 @@
     "worker:production": "node dist/src/services/queue-worker.js",
     "mongo-docker": "docker run -d -p 2717:27017 -v ./mongo-data:/data/db --name mongodb mongo:latest",
     "mongo-docker-console": "docker exec -it mongodb mongosh",
-    "run-example": "npx ts-node src/example.ts"
+    "run-example": "npx ts-node src/example.ts",
+    "deploy:fly:staging": "fly deploy -c fly.staging.toml"
   },
   "author": "",
   "license": "ISC",
diff --git a/apps/api/src/scraper/WebScraper/utils/custom/website_params.ts b/apps/api/src/scraper/WebScraper/utils/custom/website_params.ts
index 5f8be9f..763a857 100644
--- a/apps/api/src/scraper/WebScraper/utils/custom/website_params.ts
+++ b/apps/api/src/scraper/WebScraper/utils/custom/website_params.ts
@@ -121,5 +121,20 @@ export const urlSpecificParams = {
         accept:
           "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
       },
+  },
+  "scrapethissite.com":{
+    defaultScraper: "fetch",
+    headers: {
+      "User-Agent":
+        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
+      "sec-fetch-site": "same-origin",
+      "sec-fetch-mode": "cors",
+      "sec-fetch-dest": "empty",
+      referer: "https://www.google.com/",
+      "accept-language": "en-US,en;q=0.9",
+      "accept-encoding": "gzip, deflate, br",
+      accept:
+        "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
+    },
   }
 };
diff --git a/apps/api/src/services/rate-limiter.ts b/apps/api/src/services/rate-limiter.ts
index 6139702..fcfd137 100644
--- a/apps/api/src/services/rate-limiter.ts
+++ b/apps/api/src/services/rate-limiter.ts
@@ -98,7 +98,7 @@ export function scrapeRateLimit (plan: string){
 
 export function getRateLimiter(mode: RateLimiterMode, token: string){
   // Special test suite case. TODO: Change this later.
-  if (token.includes("5089cefa58")){
+  if (token.includes("fa58")){
     return testSuiteRateLimiter;
   }
   switch (mode) {
diff --git a/apps/test-suite/load-test.yml b/apps/test-suite/load-test.yml
index 5b881ef..76366bd 100644
--- a/apps/test-suite/load-test.yml
+++ b/apps/test-suite/load-test.yml
@@ -1,11 +1,11 @@
 config:
-  target: "https://api.firecrawl.dev/v0" # <<< ADD HERE THE URL FOR THE STAGING ENVIRONMENT
+  target: "https://staging-firecrawl-scraper-js.fly.dev/v0"
   phases:
-    - duration: 1
-      arrivalRate: 5
+    - duration: 60
+      arrivalRate: 10
   defaults:
     headers:
-      Authorization: "Bearer {{ $env.TEST_API_KEY }}"
+      Authorization: "Bearer fc-f3f77b6d5e084d2b970f595089cefa58"
 scenarios:
   - name: Scrape a URL
     flow:
diff --git a/apps/test-suite/package.json b/apps/test-suite/package.json
index 6fb3c4e..9ccb660 100644
--- a/apps/test-suite/package.json
+++ b/apps/test-suite/package.json
@@ -4,8 +4,7 @@
   "description": "",
   "scripts": {
     "test:suite": "npx jest --detectOpenHandles --forceExit --openHandlesTimeout=120000 --watchAll=false",
-    "test:load": "artillery run load-test.yml",
-    "fly:staging": "fly deploy -c fly.staging.toml"
+    "test:load": "artillery run load-test.yml"
   },
   "author": "",
   "license": "ISC",