bugfix on idempotency key check

apps/api/src/controllers/crawl.ts

@@ -26,7 +26,12 @@ export async function crawlController(req: Request, res: Response) {
       if (!isIdempotencyValid) {
         return res.status(409).json({ error: "Idempotency key already used" });
       }
+      try {
         createIdempotencyKey(req);
+      } catch (error) {
+        console.error(error);
+        return res.status(500).json({ error: error.message });
+      }
     }
 
     const { success: creditsCheckSuccess, message: creditsCheckMessage } =

apps/api/src/services/idempotency/validate.ts

@@ -1,5 +1,6 @@
 import { Request } from "express";
 import { supabase_service } from "../supabase";
+import { validate as isUuid } from 'uuid';
 
 export async function validateIdempotencyKey(
   req: Request,
@@ -9,6 +10,10 @@ export async function validateIdempotencyKey(
     // // not returning for missing idempotency key for now
     return true;
   }
+  if (!isUuid(idempotencyKey)) {
+    console.error("Invalid idempotency key provided in the request headers.");
+    return false;
+  }
 
   const { data, error } = await supabase_service
     .from("idempotency_keys")

apps/js-sdk/example.js

@@ -1,8 +1,10 @@
+import { v4 as uuidv4 } from 'uuid';
 import FirecrawlApp from '@mendable/firecrawl-js';
 
 const app = new FirecrawlApp({apiKey: "YOUR_API_KEY"});
 
-const crawlResult = await app.crawlUrl('mendable.ai', {crawlerOptions: {excludes: ['blog/*'], limit: 5}}, false);
+const idempotencyKey = uuidv4(); // optional
+const crawlResult = await app.crawlUrl('mendable.ai', {crawlerOptions: {excludes: ['blog/*'], limit: 5}}, false, 2, idempotencyKey);
 console.log(crawlResult)
 
 const jobId = await crawlResult['jobId'];

apps/js-sdk/firecrawl/build/index.js

@@ -8,8 +8,6 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
     });
 };
 import axios from 'axios';
-import dotenv from 'dotenv';
-dotenv.config();
 /**
  * Main class for interacting with the Firecrawl API.
  */
@@ -19,7 +17,7 @@ export default class FirecrawlApp {
      * @param {FirecrawlAppConfig} config - Configuration options for the FirecrawlApp instance.
      */
     constructor({ apiKey = null }) {
-        this.apiKey = apiKey || process.env.FIRECRAWL_API_KEY || '';
+        this.apiKey = apiKey || '';
         if (!this.apiKey) {
             throw new Error('No API key provided');
         }
@@ -104,11 +102,12 @@ export default class FirecrawlApp {
      * @param {Params | null} params - Additional parameters for the crawl request.
      * @param {boolean} waitUntilDone - Whether to wait for the crawl job to complete.
      * @param {number} timeout - Timeout in seconds for job status checks.
+     * @param {string} idempotencyKey - Optional idempotency key for the request.
      * @returns {Promise<CrawlResponse | any>} The response from the crawl operation.
      */
     crawlUrl(url_1) {
-        return __awaiter(this, arguments, void 0, function* (url, params = null, waitUntilDone = true, timeout = 2) {
+        return __awaiter(this, arguments, void 0, function* (url, params = null, waitUntilDone = true, timeout = 2, idempotencyKey) {
-            const headers = this.prepareHeaders();
+            const headers = this.prepareHeaders(idempotencyKey);
             let jsonData = { url };
             if (params) {
                 jsonData = Object.assign(Object.assign({}, jsonData), params);
@@ -162,11 +161,8 @@ export default class FirecrawlApp {
      * Prepares the headers for an API request.
      * @returns {AxiosRequestHeaders} The prepared headers.
      */
-    prepareHeaders() {
+    prepareHeaders(idempotencyKey) {
-        return {
+        return Object.assign({ 'Content-Type': 'application/json', 'Authorization': `Bearer ${this.apiKey}` }, (idempotencyKey ? { 'x-idempotency-key': idempotencyKey } : {}));
-            'Content-Type': 'application/json',
-            'Authorization': `Bearer ${this.apiKey}`,
-        };
     }
     /**
      * Sends a POST request to the specified URL.

apps/js-sdk/firecrawl/src/index.ts

@@ -141,10 +141,11 @@ export default class FirecrawlApp {
    * @param {Params | null} params - Additional parameters for the crawl request.
    * @param {boolean} waitUntilDone - Whether to wait for the crawl job to complete.
    * @param {number} timeout - Timeout in seconds for job status checks.
+   * @param {string} idempotencyKey - Optional idempotency key for the request.
    * @returns {Promise<CrawlResponse | any>} The response from the crawl operation.
    */
-  async crawlUrl(url: string, params: Params | null = null, waitUntilDone: boolean = true, timeout: number = 2): Promise<CrawlResponse | any> {
+  async crawlUrl(url: string, params: Params | null = null, waitUntilDone: boolean = true, timeout: number = 2, idempotencyKey?: string): Promise<CrawlResponse | any> {
-    const headers = this.prepareHeaders();
+    const headers = this.prepareHeaders(idempotencyKey);
     let jsonData: Params = { url };
     if (params) {
       jsonData = { ...jsonData, ...params };
@@ -192,11 +193,12 @@ export default class FirecrawlApp {
    * Prepares the headers for an API request.
    * @returns {AxiosRequestHeaders} The prepared headers.
    */
-  prepareHeaders(): AxiosRequestHeaders {
+  prepareHeaders(idempotencyKey?: string): AxiosRequestHeaders {
     return {
       'Content-Type': 'application/json',
       'Authorization': `Bearer ${this.apiKey}`,
-    } as AxiosRequestHeaders;
+      ...(idempotencyKey ? { 'x-idempotency-key': idempotencyKey } : {}),
+    } as AxiosRequestHeaders & { 'x-idempotency-key'?: string };
   }
 
   /**

apps/js-sdk/firecrawl/types/index.d.ts

@@ -76,9 +76,10 @@ export default class FirecrawlApp {
      * @param {Params | null} params - Additional parameters for the crawl request.
      * @param {boolean} waitUntilDone - Whether to wait for the crawl job to complete.
      * @param {number} timeout - Timeout in seconds for job status checks.
+     * @param {string} idempotencyKey - Optional idempotency key for the request.
      * @returns {Promise<CrawlResponse | any>} The response from the crawl operation.
      */
-    crawlUrl(url: string, params?: Params | null, waitUntilDone?: boolean, timeout?: number): Promise<CrawlResponse | any>;
+    crawlUrl(url: string, params?: Params | null, waitUntilDone?: boolean, timeout?: number, idempotencyKey?: string): Promise<CrawlResponse | any>;
     /**
      * Checks the status of a crawl job using the Firecrawl API.
      * @param {string} jobId - The job ID of the crawl operation.
@@ -89,7 +90,7 @@ export default class FirecrawlApp {
      * Prepares the headers for an API request.
      * @returns {AxiosRequestHeaders} The prepared headers.
      */
-    prepareHeaders(): AxiosRequestHeaders;
+    prepareHeaders(idempotencyKey?: string): AxiosRequestHeaders;
     /**
      * Sends a POST request to the specified URL.
      * @param {string} url - The URL to send the request to.

apps/js-sdk/package-lock.json

@@ -10,7 +10,8 @@
       "license": "ISC",
       "dependencies": {
         "@mendable/firecrawl-js": "^0.0.15",
-        "axios": "^1.6.8"
+        "axios": "^1.6.8",
+        "uuid": "^9.0.1"
       }
     },
     "node_modules/@mendable/firecrawl-js": {
@@ -122,6 +123,18 @@
       "version": "1.1.0",
       "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz",
       "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg=="
+    },
+    "node_modules/uuid": {
+      "version": "9.0.1",
+      "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz",
+      "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==",
+      "funding": [
+        "https://github.com/sponsors/broofa",
+        "https://github.com/sponsors/ctavan"
+      ],
+      "bin": {
+        "uuid": "dist/bin/uuid"
+      }
     }
   }
 }

apps/js-sdk/package.json

@@ -12,6 +12,7 @@
   "license": "ISC",
   "dependencies": {
     "@mendable/firecrawl-js": "^0.0.15",
-    "axios": "^1.6.8"
+    "axios": "^1.6.8",
+    "uuid": "^9.0.1"
   }
 }