Added idempotency key to crawl route

apps/api/src/__tests__/e2e_withAuth/index.test.ts

@@ -1,6 +1,7 @@
 import request from "supertest";
 import { app } from "../../index";
 import dotenv from "dotenv";
+import { v4 as uuidv4 } from "uuid";
 
 dotenv.config();
 
@@ -145,6 +146,30 @@ describe("E2E Tests for API Routes", () => {
         /^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$/
       );
     });
+    it('should prevent duplicate requests using the same idempotency key', async () => {
+      const uniqueIdempotencyKey = uuidv4();
+  
+      // First request with the idempotency key
+      const firstResponse = await request(TEST_URL)
+        .post('/v0/crawl')
+        .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+        .set("Content-Type", "application/json")
+        .set("x-idempotency-key", uniqueIdempotencyKey)
+        .send({ url: 'https://mendable.ai' });
+  
+        expect(firstResponse.statusCode).toBe(200);
+  
+      // Second request with the same idempotency key
+      const secondResponse = await request(TEST_URL)
+        .post('/v0/crawl')
+        .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+        .set("Content-Type", "application/json")
+        .set("x-idempotency-key", uniqueIdempotencyKey)
+        .send({ url: 'https://mendable.ai' });
+  
+      expect(secondResponse.statusCode).toBe(409);
+      expect(secondResponse.body.error).toBe('Idempotency key already used');
+    });
 
     // Additional tests for insufficient credits?
   });

apps/api/src/controllers/crawl.ts

@@ -7,6 +7,8 @@ import { RateLimiterMode } from "../../src/types";
 import { addWebScraperJob } from "../../src/services/queue-jobs";
 import { isUrlBlocked } from "../../src/scraper/WebScraper/utils/blocklist";
 import { logCrawl } from "../../src/services/logging/crawl_log";
+import { validateIdempotencyKey } from "../../src/services/idempotency/validate";
+import { createIdempotencyKey } from "../../src/services/idempotency/create";
 
 export async function crawlController(req: Request, res: Response) {
   try {
@@ -19,6 +21,14 @@ export async function crawlController(req: Request, res: Response) {
       return res.status(status).json({ error });
     }
 
+    if (req.headers["x-idempotency-key"]) {
+      const isIdempotencyValid = await validateIdempotencyKey(req);
+      if (!isIdempotencyValid) {
+        return res.status(409).json({ error: "Idempotency key already used" });
+      }
+      createIdempotencyKey(req);
+    }
+
     const { success: creditsCheckSuccess, message: creditsCheckMessage } =
       await checkTeamCredits(team_id, 1);
     if (!creditsCheckSuccess) {

apps/api/src/services/idempotency/create.ts

@@ -0,0 +1,22 @@
+import { Request } from "express";
+import { supabase_service } from "../supabase";
+
+export async function createIdempotencyKey(
+  req: Request,
+): Promise<string> {
+  const idempotencyKey = req.headers['x-idempotency-key'] as string;
+  if (!idempotencyKey) {
+    throw new Error("No idempotency key provided in the request headers.");
+  }
+
+  const { data, error } = await supabase_service
+    .from("idempotency_keys")
+    .insert({ key: idempotencyKey });
+
+  if (error) {
+    console.error("Failed to create idempotency key:", error);
+    throw error;
+  }
+
+  return idempotencyKey;
+}

apps/api/src/services/idempotency/validate.ts

@@ -0,0 +1,27 @@
+import { Request } from "express";
+import { supabase_service } from "../supabase";
+
+export async function validateIdempotencyKey(
+  req: Request,
+): Promise<boolean> {
+  const idempotencyKey = req.headers['x-idempotency-key'];
+  if (!idempotencyKey) {
+    // // not returning for missing idempotency key for now
+    return true;
+  }
+
+  const { data, error } = await supabase_service
+    .from("idempotency_keys")
+    .select("key")
+    .eq("key", idempotencyKey);
+  
+  if (error) {
+    console.error(error);
+  }
+
+  if (!data || data.length === 0) {
+    return true;
+  }
+
+  return false;
+}