diff --git a/apps/js-sdk/firecrawl/README.md b/apps/js-sdk/firecrawl/README.md index 085e865..d916bf7 100644 --- a/apps/js-sdk/firecrawl/README.md +++ b/apps/js-sdk/firecrawl/README.md @@ -176,6 +176,11 @@ async function checkStatusExample(jobId) { checkStatusExample('your_job_id_here'); ``` +## Running Locally +To use the SDK when running Firecrawl locally, you can change the initial Firecrawl app instance to: +```js +const app = new FirecrawlApp({ apiKey: "YOUR_API_KEY", apiUrl: "http://localhost:3002" }); +``` ## Error Handling diff --git a/apps/js-sdk/firecrawl/build/index.js b/apps/js-sdk/firecrawl/build/index.js index b418513..e54e532 100644 --- a/apps/js-sdk/firecrawl/build/index.js +++ b/apps/js-sdk/firecrawl/build/index.js @@ -18,9 +18,9 @@ export default class FirecrawlApp { * Initializes a new instance of the FirecrawlApp class. * @param {FirecrawlAppConfig} config - Configuration options for the FirecrawlApp instance. */ - constructor({ apiKey = null }) { - this.apiUrl = "https://api.firecrawl.dev"; + constructor({ apiKey = null, apiUrl = null }) { this.apiKey = apiKey || ""; + this.apiUrl = apiUrl || "https://api.firecrawl.dev"; if (!this.apiKey) { throw new Error("No API key provided"); } diff --git a/apps/js-sdk/firecrawl/package-lock.json b/apps/js-sdk/firecrawl/package-lock.json index b1cebde..7094cc9 100644 --- a/apps/js-sdk/firecrawl/package-lock.json +++ b/apps/js-sdk/firecrawl/package-lock.json @@ -1,12 +1,12 @@ { "name": "@mendable/firecrawl-js", - "version": "0.0.22", + "version": "0.0.26", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@mendable/firecrawl-js", - "version": "0.0.22", + "version": "0.0.26", "license": "MIT", "dependencies": { "axios": "^1.6.8", diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.json index 15d8034..b162882 100644 --- a/apps/js-sdk/firecrawl/package.json +++ b/apps/js-sdk/firecrawl/package.json @@ -1,6 +1,6 @@ { "name": "@mendable/firecrawl-js", - "version": "0.0.27", + "version": "0.0.28", "description": "JavaScript SDK for Firecrawl API", "main": "build/index.js", "types": "types/index.d.ts", diff --git a/apps/js-sdk/firecrawl/src/index.ts b/apps/js-sdk/firecrawl/src/index.ts index 5028a20..5ee9043 100644 --- a/apps/js-sdk/firecrawl/src/index.ts +++ b/apps/js-sdk/firecrawl/src/index.ts @@ -73,62 +73,63 @@ export interface FirecrawlDocument { * Response interface for scraping operations. */ export interface ScrapeResponse { - success: boolean; - data?: FirecrawlDocument; - error?: string; + success: boolean; + data?: FirecrawlDocument; + error?: string; } /** -* Response interface for searching operations. -*/ + * Response interface for searching operations. + */ export interface SearchResponse { - success: boolean; - data?: FirecrawlDocument[]; - error?: string; + success: boolean; + data?: FirecrawlDocument[]; + error?: string; } /** -* Response interface for crawling operations. -*/ + * Response interface for crawling operations. + */ export interface CrawlResponse { - success: boolean; - jobId?: string; - data?: FirecrawlDocument[]; - error?: string; + success: boolean; + jobId?: string; + data?: FirecrawlDocument[]; + error?: string; } /** -* Response interface for job status checks. -*/ + * Response interface for job status checks. + */ export interface JobStatusResponse { - success: boolean; - status: string; - jobId?: string; - data?: FirecrawlDocument[]; - partial_data?: FirecrawlDocument[]; - error?: string; + success: boolean; + status: string; + jobId?: string; + data?: FirecrawlDocument[]; + partial_data?: FirecrawlDocument[]; + error?: string; } /** - * Generic parameter interface. - */ + * Generic parameter interface. + */ export interface Params { - [key: string]: any; - extractorOptions?: { - extractionSchema: z.ZodSchema | any; - mode?: "llm-extraction"; - extractionPrompt?: string; - }; + [key: string]: any; + extractorOptions?: { + extractionSchema: z.ZodSchema | any; + mode?: "llm-extraction"; + extractionPrompt?: string; + }; } /** * Main class for interacting with the Firecrawl API. */ export default class FirecrawlApp { private apiKey: string; - private apiUrl: string = "https://api.firecrawl.dev"; + private apiUrl: string; /** * Initializes a new instance of the FirecrawlApp class. * @param {FirecrawlAppConfig} config - Configuration options for the FirecrawlApp instance. */ - constructor({ apiKey = null }: FirecrawlAppConfig) { + constructor({ apiKey = null, apiUrl = null }: FirecrawlAppConfig) { this.apiKey = apiKey || ""; + this.apiUrl = apiUrl || "https://api.firecrawl.dev"; if (!this.apiKey) { throw new Error("No API key provided"); } @@ -168,7 +169,7 @@ export default class FirecrawlApp { const response: AxiosResponse = await axios.post( this.apiUrl + "/v0/scrape", jsonData, - { headers }, + { headers } ); if (response.status === 200) { const responseData = response.data; @@ -287,7 +288,9 @@ export default class FirecrawlApp { success: true, status: response.data.status, data: response.data.data, - partial_data: !response.data.data ? response.data.partial_data : undefined, + partial_data: !response.data.data + ? response.data.partial_data + : undefined, }; } else { this.handleError(response, "check crawl status"); @@ -308,10 +311,10 @@ export default class FirecrawlApp { */ prepareHeaders(idempotencyKey?: string): AxiosRequestHeaders { return { - 'Content-Type': 'application/json', - 'Authorization': `Bearer ${this.apiKey}`, - ...(idempotencyKey ? { 'x-idempotency-key': idempotencyKey } : {}), - } as AxiosRequestHeaders & { 'x-idempotency-key'?: string }; + "Content-Type": "application/json", + Authorization: `Bearer ${this.apiKey}`, + ...(idempotencyKey ? { "x-idempotency-key": idempotencyKey } : {}), + } as AxiosRequestHeaders & { "x-idempotency-key"?: string }; } /** @@ -373,7 +376,9 @@ export default class FirecrawlApp { if (checkInterval < 2) { checkInterval = 2; } - await new Promise((resolve) => setTimeout(resolve, checkInterval * 1000)); // Wait for the specified timeout before checking again + await new Promise((resolve) => + setTimeout(resolve, checkInterval * 1000) + ); // Wait for the specified timeout before checking again } else { throw new Error( `Crawl job failed or was stopped. Status: ${statusData.status}` diff --git a/apps/js-sdk/firecrawl/types/index.d.ts b/apps/js-sdk/firecrawl/types/index.d.ts index 1a7cd1f..9e6f18c 100644 --- a/apps/js-sdk/firecrawl/types/index.d.ts +++ b/apps/js-sdk/firecrawl/types/index.d.ts @@ -121,7 +121,7 @@ export default class FirecrawlApp { * Initializes a new instance of the FirecrawlApp class. * @param {FirecrawlAppConfig} config - Configuration options for the FirecrawlApp instance. */ - constructor({ apiKey }: FirecrawlAppConfig); + constructor({ apiKey, apiUrl }: FirecrawlAppConfig); /** * Scrapes a URL using the Firecrawl API. * @param {string} url - The URL to scrape. diff --git a/apps/js-sdk/package-lock.json b/apps/js-sdk/package-lock.json index c59a371..2bf3f00 100644 --- a/apps/js-sdk/package-lock.json +++ b/apps/js-sdk/package-lock.json @@ -11,10 +11,8 @@ "dependencies": { "@mendable/firecrawl-js": "^0.0.19", "axios": "^1.6.8", - "dotenv": "^16.4.5", "ts-node": "^10.9.2", "typescript": "^5.4.5", - "uuid": "^9.0.1", "zod": "^3.23.8" }, "devDependencies": { @@ -452,15 +450,6 @@ "resolved": "https://registry.npmjs.org/@tsconfig/node16/-/node16-1.0.4.tgz", "integrity": "sha512-vxhUy4J8lyeyinH7Azl1pdd43GJhZH/tP2weN8TntQblOY+A0XbT8DJk1/oCPuOOyg/Ja757rG0CgHcWC8OfMA==" }, - "node_modules/@types/node": { - "version": "20.12.11", - "resolved": "https://registry.npmjs.org/@types/node/-/node-20.12.11.tgz", - "integrity": "sha512-vDg9PZ/zi+Nqp6boSOT7plNuthRugEKixDv5sFTIpkE89MmNtEArAShI4mxuX2+UrLEe9pxC1vm2cjm9YlWbJw==", - "peer": true, - "dependencies": { - "undici-types": "~5.26.4" - } - }, "node_modules/acorn": { "version": "8.11.3", "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.11.3.tgz", @@ -532,17 +521,6 @@ "node": ">=0.3.1" } }, - "node_modules/dotenv": { - "version": "16.4.5", - "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.4.5.tgz", - "integrity": "sha512-ZmdL2rui+eB2YwhsWzjInR8LldtZHGDoQ1ugH85ppHKwpUHL7j7rN0Ti9NCnGiQbhaZ11FpR+7ao1dNsmduNUg==", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://dotenvx.com" - } - }, "node_modules/esbuild": { "version": "0.20.2", "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.20.2.tgz", @@ -750,24 +728,6 @@ "node": ">=14.17" } }, - "node_modules/undici-types": { - "version": "5.26.5", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", - "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==", - "peer": true - }, - "node_modules/uuid": { - "version": "9.0.1", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", - "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==", - "funding": [ - "https://github.com/sponsors/broofa", - "https://github.com/sponsors/ctavan" - ], - "bin": { - "uuid": "dist/bin/uuid" - } - }, "node_modules/v8-compile-cache-lib": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz",