Added e2e tests

apps/api/.env.example

@@ -16,6 +16,8 @@ SUPABASE_SERVICE_TOKEN=
 
 # Other Optionals
 TEST_API_KEY= # use if you've set up authentication and want to test with a real API key
+RATE_LIMIT_TEST_API_KEY_SCRAPE= # set if you'd like to test the scraping rate limit
+RATE_LIMIT_TEST_API_KEY_CRAWL= # set if you'd like to test the crawling rate limit
 SCRAPING_BEE_API_KEY= #Set if you'd like to use scraping Be to handle JS blocking
 OPENAI_API_KEY= # add for LLM dependednt features (image alt generation, etc.)
 BULL_AUTH_KEY= #

apps/api/src/__tests__/e2e_withAuth/index.test.ts

@@ -518,4 +518,65 @@ describe("E2E Tests for API Routes", () => {
       expect(response.body).toHaveProperty("isProduction");
     });
   });
+
+  describe("Rate Limiter", () => {
+    it("should return 429 when rate limit is exceeded for preview token", async () => {
+      for (let i = 0; i < 5; i++) {
+        const response = await request(TEST_URL)
+          .post("/v0/scrape")
+          .set("Authorization", `Bearer this_is_just_a_preview_token`)
+          .set("Content-Type", "application/json")
+          .send({ url: "https://firecrawl.dev" });
+
+        expect(response.statusCode).toBe(200);
+      }
+      const response = await request(TEST_URL)
+        .post("/v0/scrape")
+        .set("Authorization", `Bearer this_is_just_a_preview_token`)
+        .set("Content-Type", "application/json")
+        .send({ url: "https://firecrawl.dev" });
+
+      expect(response.statusCode).toBe(429);
+    }, 60000);
+  });
+
+  it("should return 429 when rate limit is exceeded for API key", async () => {
+    for (let i = 0; i < parseInt(process.env.RATE_LIMIT_TEST_API_KEY_SCRAPE); i++) {
+      const response = await request(TEST_URL)
+        .post("/v0/scrape")
+        .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+        .set("Content-Type", "application/json")
+        .send({ url: "https://firecrawl.dev" });
+
+      expect(response.statusCode).toBe(200);
+    }
+
+    const response = await request(TEST_URL)
+      .post("/v0/scrape")
+      .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+      .set("Content-Type", "application/json")
+      .send({ url: "https://firecrawl.dev" });
+
+    expect(response.statusCode).toBe(429);
+  }, 60000);
+
+  it("should return 429 when rate limit is exceeded for API key", async () => {
+    for (let i = 0; i < parseInt(process.env.RATE_LIMIT_TEST_API_KEY_CRAWL); i++) {
+      const response = await request(TEST_URL)
+        .post("/v0/crawl")
+        .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+        .set("Content-Type", "application/json")
+        .send({ url: "https://firecrawl.dev" });
+
+      expect(response.statusCode).toBe(200);
+    }
+
+    const response = await request(TEST_URL)
+      .post("/v0/crawl")
+      .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+      .set("Content-Type", "application/json")
+      .send({ url: "https://firecrawl.dev" });
+
+    expect(response.statusCode).toBe(429);
+  }, 60000);
 });

apps/api/src/controllers/auth.ts

@@ -41,8 +41,8 @@ export async function supaAuthenticateUser(
   let normalizedApi: string;
 
   if (token == "this_is_just_a_preview_token") {
-    rateLimiter = await getRateLimiter(RateLimiterMode.Preview, token);
+    rateLimiter = getRateLimiter(RateLimiterMode.Preview, token);
-  } else {    
+  } else {
     normalizedApi = parseApi(token);
 
     const { data, error } = await supabase_service.rpc(
@@ -103,7 +103,7 @@ export async function supaAuthenticateUser(
   }
 
   try {
-    rateLimiter.consume(iptoken);
+    await rateLimiter.consume(iptoken);
   } catch (rateLimiterRes) {
     console.error(rateLimiterRes);
     return {