diff --git a/apps/api/fly.staging.toml b/apps/api/fly.staging.toml index d5ad9ef..2e0a38f 100644 --- a/apps/api/fly.staging.toml +++ b/apps/api/fly.staging.toml @@ -24,8 +24,8 @@ kill_timeout = '5s' [http_service.concurrency] type = "requests" - hard_limit = 200 - soft_limit = 100 + hard_limit = 1000 + soft_limit = 1000 [[services]] protocol = 'tcp' @@ -43,8 +43,8 @@ kill_timeout = '5s' [services.concurrency] type = 'connections' - hard_limit = 75 - soft_limit = 30 + hard_limit = 1000 + soft_limit = 1000 [[vm]] size = 'performance-4x' diff --git a/apps/api/package.json b/apps/api/package.json index a79e3dc..9c68e92 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -17,7 +17,8 @@ "worker:production": "node dist/src/services/queue-worker.js", "mongo-docker": "docker run -d -p 2717:27017 -v ./mongo-data:/data/db --name mongodb mongo:latest", "mongo-docker-console": "docker exec -it mongodb mongosh", - "run-example": "npx ts-node src/example.ts" + "run-example": "npx ts-node src/example.ts", + "deploy:fly:staging": "fly deploy -c fly.staging.toml" }, "author": "", "license": "ISC", diff --git a/apps/api/src/scraper/WebScraper/utils/custom/website_params.ts b/apps/api/src/scraper/WebScraper/utils/custom/website_params.ts index 5f8be9f..763a857 100644 --- a/apps/api/src/scraper/WebScraper/utils/custom/website_params.ts +++ b/apps/api/src/scraper/WebScraper/utils/custom/website_params.ts @@ -121,5 +121,20 @@ export const urlSpecificParams = { accept: "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9", }, + }, + "scrapethissite.com":{ + defaultScraper: "fetch", + headers: { + "User-Agent": + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36", + "sec-fetch-site": "same-origin", + "sec-fetch-mode": "cors", + "sec-fetch-dest": "empty", + referer: "https://www.google.com/", + "accept-language": "en-US,en;q=0.9", + "accept-encoding": "gzip, deflate, br", + accept: + "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9", + }, } }; diff --git a/apps/api/src/services/rate-limiter.ts b/apps/api/src/services/rate-limiter.ts index 6139702..fcfd137 100644 --- a/apps/api/src/services/rate-limiter.ts +++ b/apps/api/src/services/rate-limiter.ts @@ -98,7 +98,7 @@ export function scrapeRateLimit (plan: string){ export function getRateLimiter(mode: RateLimiterMode, token: string){ // Special test suite case. TODO: Change this later. - if (token.includes("5089cefa58")){ + if (token.includes("fa58")){ return testSuiteRateLimiter; } switch (mode) { diff --git a/apps/test-suite/load-test.yml b/apps/test-suite/load-test.yml index 5b881ef..76366bd 100644 --- a/apps/test-suite/load-test.yml +++ b/apps/test-suite/load-test.yml @@ -1,11 +1,11 @@ config: - target: "https://api.firecrawl.dev/v0" # <<< ADD HERE THE URL FOR THE STAGING ENVIRONMENT + target: "https://staging-firecrawl-scraper-js.fly.dev/v0" phases: - - duration: 1 - arrivalRate: 5 + - duration: 60 + arrivalRate: 10 defaults: headers: - Authorization: "Bearer {{ $env.TEST_API_KEY }}" + Authorization: "Bearer fc-f3f77b6d5e084d2b970f595089cefa58" scenarios: - name: Scrape a URL flow: diff --git a/apps/test-suite/package.json b/apps/test-suite/package.json index 6fb3c4e..9ccb660 100644 --- a/apps/test-suite/package.json +++ b/apps/test-suite/package.json @@ -4,8 +4,7 @@ "description": "", "scripts": { "test:suite": "npx jest --detectOpenHandles --forceExit --openHandlesTimeout=120000 --watchAll=false", - "test:load": "artillery run load-test.yml", - "fly:staging": "fly deploy -c fly.staging.toml" + "test:load": "artillery run load-test.yml" }, "author": "", "license": "ISC",