diff --git a/apps/api/src/controllers/crawl.ts b/apps/api/src/controllers/crawl.ts index 8d57354..5345b4f 100644 --- a/apps/api/src/controllers/crawl.ts +++ b/apps/api/src/controllers/crawl.ts @@ -26,7 +26,12 @@ export async function crawlController(req: Request, res: Response) { if (!isIdempotencyValid) { return res.status(409).json({ error: "Idempotency key already used" }); } - createIdempotencyKey(req); + try { + createIdempotencyKey(req); + } catch (error) { + console.error(error); + return res.status(500).json({ error: error.message }); + } } const { success: creditsCheckSuccess, message: creditsCheckMessage } = diff --git a/apps/api/src/services/idempotency/validate.ts b/apps/api/src/services/idempotency/validate.ts index ef43739..ad6f2c4 100644 --- a/apps/api/src/services/idempotency/validate.ts +++ b/apps/api/src/services/idempotency/validate.ts @@ -1,5 +1,6 @@ import { Request } from "express"; import { supabase_service } from "../supabase"; +import { validate as isUuid } from 'uuid'; export async function validateIdempotencyKey( req: Request, @@ -9,6 +10,10 @@ export async function validateIdempotencyKey( // // not returning for missing idempotency key for now return true; } + if (!isUuid(idempotencyKey)) { + console.error("Invalid idempotency key provided in the request headers."); + return false; + } const { data, error } = await supabase_service .from("idempotency_keys") diff --git a/apps/js-sdk/example.js b/apps/js-sdk/example.js index 7077b4c..e61457a 100644 --- a/apps/js-sdk/example.js +++ b/apps/js-sdk/example.js @@ -1,8 +1,10 @@ +import { v4 as uuidv4 } from 'uuid'; import FirecrawlApp from '@mendable/firecrawl-js'; const app = new FirecrawlApp({apiKey: "YOUR_API_KEY"}); -const crawlResult = await app.crawlUrl('mendable.ai', {crawlerOptions: {excludes: ['blog/*'], limit: 5}}, false); +const idempotencyKey = uuidv4(); // optional +const crawlResult = await app.crawlUrl('mendable.ai', {crawlerOptions: {excludes: ['blog/*'], limit: 5}}, false, 2, idempotencyKey); console.log(crawlResult) const jobId = await crawlResult['jobId']; diff --git a/apps/js-sdk/firecrawl/build/index.js b/apps/js-sdk/firecrawl/build/index.js index 9d8237b..b93c277 100644 --- a/apps/js-sdk/firecrawl/build/index.js +++ b/apps/js-sdk/firecrawl/build/index.js @@ -8,8 +8,6 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge }); }; import axios from 'axios'; -import dotenv from 'dotenv'; -dotenv.config(); /** * Main class for interacting with the Firecrawl API. */ @@ -19,7 +17,7 @@ export default class FirecrawlApp { * @param {FirecrawlAppConfig} config - Configuration options for the FirecrawlApp instance. */ constructor({ apiKey = null }) { - this.apiKey = apiKey || process.env.FIRECRAWL_API_KEY || ''; + this.apiKey = apiKey || ''; if (!this.apiKey) { throw new Error('No API key provided'); } @@ -104,11 +102,12 @@ export default class FirecrawlApp { * @param {Params | null} params - Additional parameters for the crawl request. * @param {boolean} waitUntilDone - Whether to wait for the crawl job to complete. * @param {number} timeout - Timeout in seconds for job status checks. + * @param {string} idempotencyKey - Optional idempotency key for the request. * @returns {Promise} The response from the crawl operation. */ crawlUrl(url_1) { - return __awaiter(this, arguments, void 0, function* (url, params = null, waitUntilDone = true, timeout = 2) { - const headers = this.prepareHeaders(); + return __awaiter(this, arguments, void 0, function* (url, params = null, waitUntilDone = true, timeout = 2, idempotencyKey) { + const headers = this.prepareHeaders(idempotencyKey); let jsonData = { url }; if (params) { jsonData = Object.assign(Object.assign({}, jsonData), params); @@ -162,11 +161,8 @@ export default class FirecrawlApp { * Prepares the headers for an API request. * @returns {AxiosRequestHeaders} The prepared headers. */ - prepareHeaders() { - return { - 'Content-Type': 'application/json', - 'Authorization': `Bearer ${this.apiKey}`, - }; + prepareHeaders(idempotencyKey) { + return Object.assign({ 'Content-Type': 'application/json', 'Authorization': `Bearer ${this.apiKey}` }, (idempotencyKey ? { 'x-idempotency-key': idempotencyKey } : {})); } /** * Sends a POST request to the specified URL. diff --git a/apps/js-sdk/firecrawl/src/index.ts b/apps/js-sdk/firecrawl/src/index.ts index aea15f8..67ff67c 100644 --- a/apps/js-sdk/firecrawl/src/index.ts +++ b/apps/js-sdk/firecrawl/src/index.ts @@ -141,10 +141,11 @@ export default class FirecrawlApp { * @param {Params | null} params - Additional parameters for the crawl request. * @param {boolean} waitUntilDone - Whether to wait for the crawl job to complete. * @param {number} timeout - Timeout in seconds for job status checks. + * @param {string} idempotencyKey - Optional idempotency key for the request. * @returns {Promise} The response from the crawl operation. */ - async crawlUrl(url: string, params: Params | null = null, waitUntilDone: boolean = true, timeout: number = 2): Promise { - const headers = this.prepareHeaders(); + async crawlUrl(url: string, params: Params | null = null, waitUntilDone: boolean = true, timeout: number = 2, idempotencyKey?: string): Promise { + const headers = this.prepareHeaders(idempotencyKey); let jsonData: Params = { url }; if (params) { jsonData = { ...jsonData, ...params }; @@ -192,11 +193,12 @@ export default class FirecrawlApp { * Prepares the headers for an API request. * @returns {AxiosRequestHeaders} The prepared headers. */ - prepareHeaders(): AxiosRequestHeaders { + prepareHeaders(idempotencyKey?: string): AxiosRequestHeaders { return { 'Content-Type': 'application/json', 'Authorization': `Bearer ${this.apiKey}`, - } as AxiosRequestHeaders; + ...(idempotencyKey ? { 'x-idempotency-key': idempotencyKey } : {}), + } as AxiosRequestHeaders & { 'x-idempotency-key'?: string }; } /** diff --git a/apps/js-sdk/firecrawl/types/index.d.ts b/apps/js-sdk/firecrawl/types/index.d.ts index 7f79d64..9828f22 100644 --- a/apps/js-sdk/firecrawl/types/index.d.ts +++ b/apps/js-sdk/firecrawl/types/index.d.ts @@ -76,9 +76,10 @@ export default class FirecrawlApp { * @param {Params | null} params - Additional parameters for the crawl request. * @param {boolean} waitUntilDone - Whether to wait for the crawl job to complete. * @param {number} timeout - Timeout in seconds for job status checks. + * @param {string} idempotencyKey - Optional idempotency key for the request. * @returns {Promise} The response from the crawl operation. */ - crawlUrl(url: string, params?: Params | null, waitUntilDone?: boolean, timeout?: number): Promise; + crawlUrl(url: string, params?: Params | null, waitUntilDone?: boolean, timeout?: number, idempotencyKey?: string): Promise; /** * Checks the status of a crawl job using the Firecrawl API. * @param {string} jobId - The job ID of the crawl operation. @@ -89,7 +90,7 @@ export default class FirecrawlApp { * Prepares the headers for an API request. * @returns {AxiosRequestHeaders} The prepared headers. */ - prepareHeaders(): AxiosRequestHeaders; + prepareHeaders(idempotencyKey?: string): AxiosRequestHeaders; /** * Sends a POST request to the specified URL. * @param {string} url - The URL to send the request to. diff --git a/apps/js-sdk/package-lock.json b/apps/js-sdk/package-lock.json index 363f301..aea3322 100644 --- a/apps/js-sdk/package-lock.json +++ b/apps/js-sdk/package-lock.json @@ -10,7 +10,8 @@ "license": "ISC", "dependencies": { "@mendable/firecrawl-js": "^0.0.15", - "axios": "^1.6.8" + "axios": "^1.6.8", + "uuid": "^9.0.1" } }, "node_modules/@mendable/firecrawl-js": { @@ -122,6 +123,18 @@ "version": "1.1.0", "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==" + }, + "node_modules/uuid": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", + "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==", + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "bin": { + "uuid": "dist/bin/uuid" + } } } } diff --git a/apps/js-sdk/package.json b/apps/js-sdk/package.json index 563e1e3..c0ac6f1 100644 --- a/apps/js-sdk/package.json +++ b/apps/js-sdk/package.json @@ -12,6 +12,7 @@ "license": "ISC", "dependencies": { "@mendable/firecrawl-js": "^0.0.15", - "axios": "^1.6.8" + "axios": "^1.6.8", + "uuid": "^9.0.1" } }