github/v-firecrawl
github/v-firecrawl
0
Fork 0
Code Actions

build files

Browse Source
This commit is contained in:
rafaelsideguide 2024-06-04 14:54:54 -03:00
parent 64a4338ff0
commit 93f3098672
2 changed files with 25 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
apps/js-sdk/firecrawl/build/index.js
View File

@ -19,6 +19,7 @@ export default class FirecrawlApp {
* @param {FirecrawlAppConfig} config - Configuration options for the FirecrawlApp instance. * @param {FirecrawlAppConfig} config - Configuration options for the FirecrawlApp instance.
*/ */
constructor({ apiKey = null }) { constructor({ apiKey = null }) {
this.apiUrl = "https://api.firecrawl.dev";
this.apiKey = apiKey || ""; this.apiKey = apiKey || "";
if (!this.apiKey) { if (!this.apiKey) {
throw new Error("No API key provided"); throw new Error("No API key provided");
@ -47,7 +48,7 @@ export default class FirecrawlApp {
jsonData = Object.assign(Object.assign({}, jsonData), { extractorOptions: Object.assign(Object.assign({}, params.extractorOptions), { extractionSchema: schema, mode: params.extractorOptions.mode || "llm-extraction" }) }); jsonData = Object.assign(Object.assign({}, jsonData), { extractorOptions: Object.assign(Object.assign({}, params.extractorOptions), { extractionSchema: schema, mode: params.extractorOptions.mode || "llm-extraction" }) });
} }
try { try {
const response = yield axios.post("https://api.firecrawl.dev/v0/scrape", jsonData, { headers }); const response = yield axios.post(this.apiUrl + "/v0/scrape", jsonData, { headers });
if (response.status === 200) { if (response.status === 200) {
const responseData = response.data; const responseData = response.data;
if (responseData.success) { if (responseData.success) {
@ -84,7 +85,7 @@ export default class FirecrawlApp {
jsonData = Object.assign(Object.assign({}, jsonData), params); jsonData = Object.assign(Object.assign({}, jsonData), params);
} }
try { try {
const response = yield axios.post("https://api.firecrawl.dev/v0/search", jsonData, { headers }); const response = yield axios.post(this.apiUrl + "/v0/search", jsonData, { headers });
if (response.status === 200) { if (response.status === 200) {
const responseData = response.data; const responseData = response.data;
if (responseData.success) { if (responseData.success) {
@ -109,23 +110,23 @@ export default class FirecrawlApp {
* @param {string} url - The URL to crawl. * @param {string} url - The URL to crawl.
* @param {Params | null} params - Additional parameters for the crawl request. * @param {Params | null} params - Additional parameters for the crawl request.
* @param {boolean} waitUntilDone - Whether to wait for the crawl job to complete. * @param {boolean} waitUntilDone - Whether to wait for the crawl job to complete.
* @param {number} timeout - Timeout in seconds for job status checks. * @param {number} pollInterval - Time in seconds for job status checks.
* @param {string} idempotencyKey - Optional idempotency key for the request. * @param {string} idempotencyKey - Optional idempotency key for the request.
* @returns {Promise<CrawlResponse | any>} The response from the crawl operation. * @returns {Promise<CrawlResponse | any>} The response from the crawl operation.
*/ */
crawlUrl(url_1) { crawlUrl(url_1) {
return __awaiter(this, arguments, void 0, function* (url, params = null, waitUntilDone = true, timeout = 2, idempotencyKey) { return __awaiter(this, arguments, void 0, function* (url, params = null, waitUntilDone = true, pollInterval = 2, idempotencyKey) {
const headers = this.prepareHeaders(idempotencyKey); const headers = this.prepareHeaders(idempotencyKey);
let jsonData = { url }; let jsonData = { url };
if (params) { if (params) {
jsonData = Object.assign(Object.assign({}, jsonData), params); jsonData = Object.assign(Object.assign({}, jsonData), params);
} }
try { try {
const response = yield this.postRequest("https://api.firecrawl.dev/v0/crawl", jsonData, headers); const response = yield this.postRequest(this.apiUrl + "/v0/crawl", jsonData, headers);
if (response.status === 200) { if (response.status === 200) {
const jobId = response.data.jobId; const jobId = response.data.jobId;
if (waitUntilDone) { if (waitUntilDone) {
return this.monitorJobStatus(jobId, headers, timeout); return this.monitorJobStatus(jobId, headers, pollInterval);
} }
else { else {
return { success: true, jobId }; return { success: true, jobId };
@ -151,9 +152,14 @@ export default class FirecrawlApp {
return __awaiter(this, void 0, void 0, function* () { return __awaiter(this, void 0, void 0, function* () {
const headers = this.prepareHeaders(); const headers = this.prepareHeaders();
try { try {
const response = yield this.getRequest(`https://api.firecrawl.dev/v0/crawl/status/${jobId}`, headers); const response = yield this.getRequest(this.apiUrl + `/v0/crawl/status/${jobId}`, headers);
if (response.status === 200) { if (response.status === 200) {
return response.data; return {
success: true,
status: response.data.status,
data: response.data.data,
partial_data: !response.data.data ? response.data.partial_data : undefined,
};
} }
else { else {
this.handleError(response, "check crawl status"); this.handleError(response, "check crawl status");
@ -202,10 +208,10 @@ export default class FirecrawlApp {
* @param {number} timeout - Timeout in seconds for job status checks. * @param {number} timeout - Timeout in seconds for job status checks.
* @returns {Promise<any>} The final job status or data. * @returns {Promise<any>} The final job status or data.
*/ */
monitorJobStatus(jobId, headers, timeout) { monitorJobStatus(jobId, headers, checkInterval) {
return __awaiter(this, void 0, void 0, function* () { return __awaiter(this, void 0, void 0, function* () {
while (true) { while (true) {
const statusResponse = yield this.getRequest(`https://api.firecrawl.dev/v0/crawl/status/${jobId}`, headers); const statusResponse = yield this.getRequest(this.apiUrl + `/v0/crawl/status/${jobId}`, headers);
if (statusResponse.status === 200) { if (statusResponse.status === 200) {
const statusData = statusResponse.data; const statusData = statusResponse.data;
if (statusData.status === "completed") { if (statusData.status === "completed") {
@ -217,10 +223,10 @@ export default class FirecrawlApp {
} }
} }
else if (["active", "paused", "pending", "queued"].includes(statusData.status)) { else if (["active", "paused", "pending", "queued"].includes(statusData.status)) {
if (timeout < 2) { if (checkInterval < 2) {
timeout = 2; checkInterval = 2;
} }
yield new Promise((resolve) => setTimeout(resolve, timeout * 1000)); // Wait for the specified timeout before checking again yield new Promise((resolve) => setTimeout(resolve, checkInterval * 1000)); // Wait for the specified timeout before checking again
} }
else { else {
throw new Error(`Crawl job failed or was stopped. Status: ${statusData.status}`); throw new Error(`Crawl job failed or was stopped. Status: ${statusData.status}`);

9
apps/js-sdk/firecrawl/types/index.d.ts vendored
View File

@ -5,6 +5,7 @@ import { z } from "zod";
*/ */
export interface FirecrawlAppConfig { export interface FirecrawlAppConfig {
apiKey?: string | null; apiKey?: string | null;
apiUrl?: string | null;
} }
/** /**
* Generic parameter interface. * Generic parameter interface.
@ -50,6 +51,7 @@ export interface JobStatusResponse {
status: string; status: string;
jobId?: string; jobId?: string;
data?: any; data?: any;
partial_data?: any;
error?: string; error?: string;
} }
/** /**
@ -57,6 +59,7 @@ export interface JobStatusResponse {
*/ */
export default class FirecrawlApp { export default class FirecrawlApp {
private apiKey; private apiKey;
private apiUrl;
/** /**
* Initializes a new instance of the FirecrawlApp class. * Initializes a new instance of the FirecrawlApp class.
* @param {FirecrawlAppConfig} config - Configuration options for the FirecrawlApp instance. * @param {FirecrawlAppConfig} config - Configuration options for the FirecrawlApp instance.
@ -81,11 +84,11 @@ export default class FirecrawlApp {
* @param {string} url - The URL to crawl. * @param {string} url - The URL to crawl.
* @param {Params | null} params - Additional parameters for the crawl request. * @param {Params | null} params - Additional parameters for the crawl request.
* @param {boolean} waitUntilDone - Whether to wait for the crawl job to complete. * @param {boolean} waitUntilDone - Whether to wait for the crawl job to complete.
* @param {number} timeout - Timeout in seconds for job status checks. * @param {number} pollInterval - Time in seconds for job status checks.
* @param {string} idempotencyKey - Optional idempotency key for the request. * @param {string} idempotencyKey - Optional idempotency key for the request.
* @returns {Promise<CrawlResponse | any>} The response from the crawl operation. * @returns {Promise<CrawlResponse | any>} The response from the crawl operation.
*/ */
crawlUrl(url: string, params?: Params | null, waitUntilDone?: boolean, timeout?: number, idempotencyKey?: string): Promise<CrawlResponse | any>; crawlUrl(url: string, params?: Params | null, waitUntilDone?: boolean, pollInterval?: number, idempotencyKey?: string): Promise<CrawlResponse | any>;
/** /**
* Checks the status of a crawl job using the Firecrawl API. * Checks the status of a crawl job using the Firecrawl API.
* @param {string} jobId - The job ID of the crawl operation. * @param {string} jobId - The job ID of the crawl operation.
@ -119,7 +122,7 @@ export default class FirecrawlApp {
* @param {number} timeout - Timeout in seconds for job status checks. * @param {number} timeout - Timeout in seconds for job status checks.
* @returns {Promise<any>} The final job status or data. * @returns {Promise<any>} The final job status or data.
*/ */
monitorJobStatus(jobId: string, headers: AxiosRequestHeaders, timeout: number): Promise<any>; monitorJobStatus(jobId: string, headers: AxiosRequestHeaders, checkInterval: number): Promise<any>;
/** /**
* Handles errors from API responses. * Handles errors from API responses.
* @param {AxiosResponse} response - The response from the API. * @param {AxiosResponse} response - The response from the API.