github/v-firecrawl
github/v-firecrawl
0
Fork 0
Code Actions

bugfix on idempotency key check

Browse Source
This commit is contained in:
rafaelsideguide 2024-05-23 11:47:04 -03:00
parent 3f460af6c5
commit 184e4678f1
8 changed files with 45 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
apps/api/src/controllers/crawl.ts
View File

@ -26,7 +26,12 @@ export async function crawlController(req: Request, res: Response) {
if (!isIdempotencyValid) {
return res.status(409).json({ error: "Idempotency key already used" });
}
try {
createIdempotencyKey(req);
} catch (error) {
console.error(error);
return res.status(500).json({ error: error.message });
}
}
const { success: creditsCheckSuccess, message: creditsCheckMessage } =

5
apps/api/src/services/idempotency/validate.ts
View File

@ -1,5 +1,6 @@
import { Request } from "express";
import { supabase_service } from "../supabase";
import { validate as isUuid } from 'uuid';
export async function validateIdempotencyKey(
req: Request,
@ -9,6 +10,10 @@ export async function validateIdempotencyKey(
// // not returning for missing idempotency key for now
return true;
}
if (!isUuid(idempotencyKey)) {
console.error("Invalid idempotency key provided in the request headers.");
return false;
}
const { data, error } = await supabase_service
.from("idempotency_keys")

4
apps/js-sdk/example.js
View File

@ -1,8 +1,10 @@
import { v4 as uuidv4 } from 'uuid';
import FirecrawlApp from '@mendable/firecrawl-js';
const app = new FirecrawlApp({apiKey: "YOUR_API_KEY"});
const crawlResult = await app.crawlUrl('mendable.ai', {crawlerOptions: {excludes: ['blog/*'], limit: 5}}, false);
const idempotencyKey = uuidv4(); // optional
const crawlResult = await app.crawlUrl('mendable.ai', {crawlerOptions: {excludes: ['blog/*'], limit: 5}}, false, 2, idempotencyKey);
console.log(crawlResult)
const jobId = await crawlResult['jobId'];

16
apps/js-sdk/firecrawl/build/index.js
View File

@ -8,8 +8,6 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
});
};
import axios from 'axios';
import dotenv from 'dotenv';
dotenv.config();
/**
* Main class for interacting with the Firecrawl API.
*/
@ -19,7 +17,7 @@ export default class FirecrawlApp {
* @param {FirecrawlAppConfig} config - Configuration options for the FirecrawlApp instance.
*/
constructor({ apiKey = null }) {
this.apiKey = apiKey || process.env.FIRECRAWL_API_KEY || '';
this.apiKey = apiKey || '';
if (!this.apiKey) {
throw new Error('No API key provided');
}
@ -104,11 +102,12 @@ export default class FirecrawlApp {
* @param {Params | null} params - Additional parameters for the crawl request.
* @param {boolean} waitUntilDone - Whether to wait for the crawl job to complete.
* @param {number} timeout - Timeout in seconds for job status checks.
* @param {string} idempotencyKey - Optional idempotency key for the request.
* @returns {Promise<CrawlResponse | any>} The response from the crawl operation.
*/
crawlUrl(url_1) {
return __awaiter(this, arguments, void 0, function* (url, params = null, waitUntilDone = true, timeout = 2) {
const headers = this.prepareHeaders();
return __awaiter(this, arguments, void 0, function* (url, params = null, waitUntilDone = true, timeout = 2, idempotencyKey) {
const headers = this.prepareHeaders(idempotencyKey);
let jsonData = { url };
if (params) {
jsonData = Object.assign(Object.assign({}, jsonData), params);
@ -162,11 +161,8 @@ export default class FirecrawlApp {
* Prepares the headers for an API request.
* @returns {AxiosRequestHeaders} The prepared headers.
*/
prepareHeaders() {
return {
'Content-Type': 'application/json',
'Authorization': `Bearer ${this.apiKey}`,
};
prepareHeaders(idempotencyKey) {
return Object.assign({ 'Content-Type': 'application/json', 'Authorization': `Bearer ${this.apiKey}` }, (idempotencyKey ? { 'x-idempotency-key': idempotencyKey } : {}));
}
/**
* Sends a POST request to the specified URL.

10
apps/js-sdk/firecrawl/src/index.ts
View File

@ -141,10 +141,11 @@ export default class FirecrawlApp {
* @param {Params | null} params - Additional parameters for the crawl request.
* @param {boolean} waitUntilDone - Whether to wait for the crawl job to complete.
* @param {number} timeout - Timeout in seconds for job status checks.
* @param {string} idempotencyKey - Optional idempotency key for the request.
* @returns {Promise<CrawlResponse | any>} The response from the crawl operation.
*/
async crawlUrl(url: string, params: Params | null = null, waitUntilDone: boolean = true, timeout: number = 2): Promise<CrawlResponse | any> {
const headers = this.prepareHeaders();
async crawlUrl(url: string, params: Params | null = null, waitUntilDone: boolean = true, timeout: number = 2, idempotencyKey?: string): Promise<CrawlResponse | any> {
const headers = this.prepareHeaders(idempotencyKey);
let jsonData: Params = { url };
if (params) {
jsonData = { ...jsonData, ...params };
@ -192,11 +193,12 @@ export default class FirecrawlApp {
* Prepares the headers for an API request.
* @returns {AxiosRequestHeaders} The prepared headers.
*/
prepareHeaders(): AxiosRequestHeaders {
prepareHeaders(idempotencyKey?: string): AxiosRequestHeaders {
return {
'Content-Type': 'application/json',
'Authorization': `Bearer ${this.apiKey}`,
} as AxiosRequestHeaders;
...(idempotencyKey ? { 'x-idempotency-key': idempotencyKey } : {}),
} as AxiosRequestHeaders & { 'x-idempotency-key'?: string };
}
/**

5
apps/js-sdk/firecrawl/types/index.d.ts vendored
View File

@ -76,9 +76,10 @@ export default class FirecrawlApp {
* @param {Params | null} params - Additional parameters for the crawl request.
* @param {boolean} waitUntilDone - Whether to wait for the crawl job to complete.
* @param {number} timeout - Timeout in seconds for job status checks.
* @param {string} idempotencyKey - Optional idempotency key for the request.
* @returns {Promise<CrawlResponse | any>} The response from the crawl operation.
*/
crawlUrl(url: string, params?: Params | null, waitUntilDone?: boolean, timeout?: number): Promise<CrawlResponse | any>;
crawlUrl(url: string, params?: Params | null, waitUntilDone?: boolean, timeout?: number, idempotencyKey?: string): Promise<CrawlResponse | any>;
/**
* Checks the status of a crawl job using the Firecrawl API.
* @param {string} jobId - The job ID of the crawl operation.
@ -89,7 +90,7 @@ export default class FirecrawlApp {
* Prepares the headers for an API request.
* @returns {AxiosRequestHeaders} The prepared headers.
*/
prepareHeaders(): AxiosRequestHeaders;
prepareHeaders(idempotencyKey?: string): AxiosRequestHeaders;
/**
* Sends a POST request to the specified URL.
* @param {string} url - The URL to send the request to.

15
apps/js-sdk/package-lock.json generated
View File

@ -10,7 +10,8 @@
"license": "ISC",
"dependencies": {
"@mendable/firecrawl-js": "^0.0.15",
"axios": "^1.6.8"
"axios": "^1.6.8",
"uuid": "^9.0.1"
}
},
"node_modules/@mendable/firecrawl-js": {
@ -122,6 +123,18 @@
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz",
"integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg=="
},
"node_modules/uuid": {
"version": "9.0.1",
"resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz",
"integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==",
"funding": [
"https://github.com/sponsors/broofa",
"https://github.com/sponsors/ctavan"
],
"bin": {
"uuid": "dist/bin/uuid"
}
}
}
}

3
apps/js-sdk/package.json
View File

@ -12,6 +12,7 @@
"license": "ISC",
"dependencies": {
"@mendable/firecrawl-js": "^0.0.15",
"axios": "^1.6.8"
"axios": "^1.6.8",
"uuid": "^9.0.1"
}
}