diff --git a/apps/api/.env.example b/apps/api/.env.example index d91799a..1ba5ffe 100644 --- a/apps/api/.env.example +++ b/apps/api/.env.example @@ -16,6 +16,8 @@ SUPABASE_SERVICE_TOKEN= # Other Optionals TEST_API_KEY= # use if you've set up authentication and want to test with a real API key +RATE_LIMIT_TEST_API_KEY_SCRAPE= # set if you'd like to test the scraping rate limit +RATE_LIMIT_TEST_API_KEY_CRAWL= # set if you'd like to test the crawling rate limit SCRAPING_BEE_API_KEY= #Set if you'd like to use scraping Be to handle JS blocking OPENAI_API_KEY= # add for LLM dependednt features (image alt generation, etc.) BULL_AUTH_KEY= # diff --git a/apps/api/src/__tests__/e2e_withAuth/index.test.ts b/apps/api/src/__tests__/e2e_withAuth/index.test.ts index 0e2caeb..352d762 100644 --- a/apps/api/src/__tests__/e2e_withAuth/index.test.ts +++ b/apps/api/src/__tests__/e2e_withAuth/index.test.ts @@ -518,4 +518,65 @@ describe("E2E Tests for API Routes", () => { expect(response.body).toHaveProperty("isProduction"); }); }); + + describe("Rate Limiter", () => { + it("should return 429 when rate limit is exceeded for preview token", async () => { + for (let i = 0; i < 5; i++) { + const response = await request(TEST_URL) + .post("/v0/scrape") + .set("Authorization", `Bearer this_is_just_a_preview_token`) + .set("Content-Type", "application/json") + .send({ url: "https://firecrawl.dev" }); + + expect(response.statusCode).toBe(200); + } + const response = await request(TEST_URL) + .post("/v0/scrape") + .set("Authorization", `Bearer this_is_just_a_preview_token`) + .set("Content-Type", "application/json") + .send({ url: "https://firecrawl.dev" }); + + expect(response.statusCode).toBe(429); + }, 60000); + }); + + it("should return 429 when rate limit is exceeded for API key", async () => { + for (let i = 0; i < parseInt(process.env.RATE_LIMIT_TEST_API_KEY_SCRAPE); i++) { + const response = await request(TEST_URL) + .post("/v0/scrape") + .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`) + .set("Content-Type", "application/json") + .send({ url: "https://firecrawl.dev" }); + + expect(response.statusCode).toBe(200); + } + + const response = await request(TEST_URL) + .post("/v0/scrape") + .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`) + .set("Content-Type", "application/json") + .send({ url: "https://firecrawl.dev" }); + + expect(response.statusCode).toBe(429); + }, 60000); + + it("should return 429 when rate limit is exceeded for API key", async () => { + for (let i = 0; i < parseInt(process.env.RATE_LIMIT_TEST_API_KEY_CRAWL); i++) { + const response = await request(TEST_URL) + .post("/v0/crawl") + .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`) + .set("Content-Type", "application/json") + .send({ url: "https://firecrawl.dev" }); + + expect(response.statusCode).toBe(200); + } + + const response = await request(TEST_URL) + .post("/v0/crawl") + .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`) + .set("Content-Type", "application/json") + .send({ url: "https://firecrawl.dev" }); + + expect(response.statusCode).toBe(429); + }, 60000); }); diff --git a/apps/api/src/controllers/auth.ts b/apps/api/src/controllers/auth.ts index 524f440..ff751ef 100644 --- a/apps/api/src/controllers/auth.ts +++ b/apps/api/src/controllers/auth.ts @@ -41,8 +41,8 @@ export async function supaAuthenticateUser( let normalizedApi: string; if (token == "this_is_just_a_preview_token") { - rateLimiter = await getRateLimiter(RateLimiterMode.Preview, token); - } else { + rateLimiter = getRateLimiter(RateLimiterMode.Preview, token); + } else { normalizedApi = parseApi(token); const { data, error } = await supabase_service.rpc( @@ -103,7 +103,7 @@ export async function supaAuthenticateUser( } try { - rateLimiter.consume(iptoken); + await rateLimiter.consume(iptoken); } catch (rateLimiterRes) { console.error(rateLimiterRes); return {