From 93f30986729657a55a7de06434556002dbb111b5 Mon Sep 17 00:00:00 2001 From: rafaelsideguide <150964962+rafaelsideguide@users.noreply.github.com> Date: Tue, 4 Jun 2024 14:54:54 -0300 Subject: [PATCH] build files --- apps/js-sdk/firecrawl/build/index.js | 32 +++++++++++++++----------- apps/js-sdk/firecrawl/types/index.d.ts | 9 +++++--- 2 files changed, 25 insertions(+), 16 deletions(-) diff --git a/apps/js-sdk/firecrawl/build/index.js b/apps/js-sdk/firecrawl/build/index.js index 76edfe5..b418513 100644 --- a/apps/js-sdk/firecrawl/build/index.js +++ b/apps/js-sdk/firecrawl/build/index.js @@ -19,6 +19,7 @@ export default class FirecrawlApp { * @param {FirecrawlAppConfig} config - Configuration options for the FirecrawlApp instance. */ constructor({ apiKey = null }) { + this.apiUrl = "https://api.firecrawl.dev"; this.apiKey = apiKey || ""; if (!this.apiKey) { throw new Error("No API key provided"); @@ -47,7 +48,7 @@ export default class FirecrawlApp { jsonData = Object.assign(Object.assign({}, jsonData), { extractorOptions: Object.assign(Object.assign({}, params.extractorOptions), { extractionSchema: schema, mode: params.extractorOptions.mode || "llm-extraction" }) }); } try { - const response = yield axios.post("https://api.firecrawl.dev/v0/scrape", jsonData, { headers }); + const response = yield axios.post(this.apiUrl + "/v0/scrape", jsonData, { headers }); if (response.status === 200) { const responseData = response.data; if (responseData.success) { @@ -84,7 +85,7 @@ export default class FirecrawlApp { jsonData = Object.assign(Object.assign({}, jsonData), params); } try { - const response = yield axios.post("https://api.firecrawl.dev/v0/search", jsonData, { headers }); + const response = yield axios.post(this.apiUrl + "/v0/search", jsonData, { headers }); if (response.status === 200) { const responseData = response.data; if (responseData.success) { @@ -109,23 +110,23 @@ export default class FirecrawlApp { * @param {string} url - The URL to crawl. * @param {Params | null} params - Additional parameters for the crawl request. * @param {boolean} waitUntilDone - Whether to wait for the crawl job to complete. - * @param {number} timeout - Timeout in seconds for job status checks. + * @param {number} pollInterval - Time in seconds for job status checks. * @param {string} idempotencyKey - Optional idempotency key for the request. * @returns {Promise} The response from the crawl operation. */ crawlUrl(url_1) { - return __awaiter(this, arguments, void 0, function* (url, params = null, waitUntilDone = true, timeout = 2, idempotencyKey) { + return __awaiter(this, arguments, void 0, function* (url, params = null, waitUntilDone = true, pollInterval = 2, idempotencyKey) { const headers = this.prepareHeaders(idempotencyKey); let jsonData = { url }; if (params) { jsonData = Object.assign(Object.assign({}, jsonData), params); } try { - const response = yield this.postRequest("https://api.firecrawl.dev/v0/crawl", jsonData, headers); + const response = yield this.postRequest(this.apiUrl + "/v0/crawl", jsonData, headers); if (response.status === 200) { const jobId = response.data.jobId; if (waitUntilDone) { - return this.monitorJobStatus(jobId, headers, timeout); + return this.monitorJobStatus(jobId, headers, pollInterval); } else { return { success: true, jobId }; @@ -151,9 +152,14 @@ export default class FirecrawlApp { return __awaiter(this, void 0, void 0, function* () { const headers = this.prepareHeaders(); try { - const response = yield this.getRequest(`https://api.firecrawl.dev/v0/crawl/status/${jobId}`, headers); + const response = yield this.getRequest(this.apiUrl + `/v0/crawl/status/${jobId}`, headers); if (response.status === 200) { - return response.data; + return { + success: true, + status: response.data.status, + data: response.data.data, + partial_data: !response.data.data ? response.data.partial_data : undefined, + }; } else { this.handleError(response, "check crawl status"); @@ -202,10 +208,10 @@ export default class FirecrawlApp { * @param {number} timeout - Timeout in seconds for job status checks. * @returns {Promise} The final job status or data. */ - monitorJobStatus(jobId, headers, timeout) { + monitorJobStatus(jobId, headers, checkInterval) { return __awaiter(this, void 0, void 0, function* () { while (true) { - const statusResponse = yield this.getRequest(`https://api.firecrawl.dev/v0/crawl/status/${jobId}`, headers); + const statusResponse = yield this.getRequest(this.apiUrl + `/v0/crawl/status/${jobId}`, headers); if (statusResponse.status === 200) { const statusData = statusResponse.data; if (statusData.status === "completed") { @@ -217,10 +223,10 @@ export default class FirecrawlApp { } } else if (["active", "paused", "pending", "queued"].includes(statusData.status)) { - if (timeout < 2) { - timeout = 2; + if (checkInterval < 2) { + checkInterval = 2; } - yield new Promise((resolve) => setTimeout(resolve, timeout * 1000)); // Wait for the specified timeout before checking again + yield new Promise((resolve) => setTimeout(resolve, checkInterval * 1000)); // Wait for the specified timeout before checking again } else { throw new Error(`Crawl job failed or was stopped. Status: ${statusData.status}`); diff --git a/apps/js-sdk/firecrawl/types/index.d.ts b/apps/js-sdk/firecrawl/types/index.d.ts index cc186e8..52a7d1e 100644 --- a/apps/js-sdk/firecrawl/types/index.d.ts +++ b/apps/js-sdk/firecrawl/types/index.d.ts @@ -5,6 +5,7 @@ import { z } from "zod"; */ export interface FirecrawlAppConfig { apiKey?: string | null; + apiUrl?: string | null; } /** * Generic parameter interface. @@ -50,6 +51,7 @@ export interface JobStatusResponse { status: string; jobId?: string; data?: any; + partial_data?: any; error?: string; } /** @@ -57,6 +59,7 @@ export interface JobStatusResponse { */ export default class FirecrawlApp { private apiKey; + private apiUrl; /** * Initializes a new instance of the FirecrawlApp class. * @param {FirecrawlAppConfig} config - Configuration options for the FirecrawlApp instance. @@ -81,11 +84,11 @@ export default class FirecrawlApp { * @param {string} url - The URL to crawl. * @param {Params | null} params - Additional parameters for the crawl request. * @param {boolean} waitUntilDone - Whether to wait for the crawl job to complete. - * @param {number} timeout - Timeout in seconds for job status checks. + * @param {number} pollInterval - Time in seconds for job status checks. * @param {string} idempotencyKey - Optional idempotency key for the request. * @returns {Promise} The response from the crawl operation. */ - crawlUrl(url: string, params?: Params | null, waitUntilDone?: boolean, timeout?: number, idempotencyKey?: string): Promise; + crawlUrl(url: string, params?: Params | null, waitUntilDone?: boolean, pollInterval?: number, idempotencyKey?: string): Promise; /** * Checks the status of a crawl job using the Firecrawl API. * @param {string} jobId - The job ID of the crawl operation. @@ -119,7 +122,7 @@ export default class FirecrawlApp { * @param {number} timeout - Timeout in seconds for job status checks. * @returns {Promise} The final job status or data. */ - monitorJobStatus(jobId: string, headers: AxiosRequestHeaders, timeout: number): Promise; + monitorJobStatus(jobId: string, headers: AxiosRequestHeaders, checkInterval: number): Promise; /** * Handles errors from API responses. * @param {AxiosResponse} response - The response from the API.