Added local host support for the javascript SDK
This commit is contained in:
parent
a20d002a6b
commit
e5ffda1eec
@ -176,6 +176,11 @@ async function checkStatusExample(jobId) {
|
|||||||
checkStatusExample('your_job_id_here');
|
checkStatusExample('your_job_id_here');
|
||||||
```
|
```
|
||||||
|
|
||||||
|
## Running Locally
|
||||||
|
To use the SDK when running Firecrawl locally, you can change the initial Firecrawl app instance to:
|
||||||
|
```js
|
||||||
|
const app = new FirecrawlApp({ apiKey: "YOUR_API_KEY", apiUrl: "http://localhost:3002" });
|
||||||
|
```
|
||||||
|
|
||||||
## Error Handling
|
## Error Handling
|
||||||
|
|
||||||
|
@ -18,9 +18,9 @@ export default class FirecrawlApp {
|
|||||||
* Initializes a new instance of the FirecrawlApp class.
|
* Initializes a new instance of the FirecrawlApp class.
|
||||||
* @param {FirecrawlAppConfig} config - Configuration options for the FirecrawlApp instance.
|
* @param {FirecrawlAppConfig} config - Configuration options for the FirecrawlApp instance.
|
||||||
*/
|
*/
|
||||||
constructor({ apiKey = null }) {
|
constructor({ apiKey = null, apiUrl = null }) {
|
||||||
this.apiUrl = "https://api.firecrawl.dev";
|
|
||||||
this.apiKey = apiKey || "";
|
this.apiKey = apiKey || "";
|
||||||
|
this.apiUrl = apiUrl || "https://api.firecrawl.dev";
|
||||||
if (!this.apiKey) {
|
if (!this.apiKey) {
|
||||||
throw new Error("No API key provided");
|
throw new Error("No API key provided");
|
||||||
}
|
}
|
||||||
|
4
apps/js-sdk/firecrawl/package-lock.json
generated
4
apps/js-sdk/firecrawl/package-lock.json
generated
@ -1,12 +1,12 @@
|
|||||||
{
|
{
|
||||||
"name": "@mendable/firecrawl-js",
|
"name": "@mendable/firecrawl-js",
|
||||||
"version": "0.0.22",
|
"version": "0.0.26",
|
||||||
"lockfileVersion": 3,
|
"lockfileVersion": 3,
|
||||||
"requires": true,
|
"requires": true,
|
||||||
"packages": {
|
"packages": {
|
||||||
"": {
|
"": {
|
||||||
"name": "@mendable/firecrawl-js",
|
"name": "@mendable/firecrawl-js",
|
||||||
"version": "0.0.22",
|
"version": "0.0.26",
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"axios": "^1.6.8",
|
"axios": "^1.6.8",
|
||||||
|
@ -5,346 +5,347 @@ import { zodToJsonSchema } from "zod-to-json-schema";
|
|||||||
* Configuration interface for FirecrawlApp.
|
* Configuration interface for FirecrawlApp.
|
||||||
*/
|
*/
|
||||||
export interface FirecrawlAppConfig {
|
export interface FirecrawlAppConfig {
|
||||||
apiKey?: string | null;
|
apiKey?: string | null;
|
||||||
apiUrl?: string | null;
|
apiUrl?: string | null;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Generic parameter interface.
|
* Generic parameter interface.
|
||||||
*/
|
*/
|
||||||
export interface Params {
|
export interface Params {
|
||||||
[key: string]: any;
|
[key: string]: any;
|
||||||
extractorOptions?: {
|
extractorOptions?: {
|
||||||
extractionSchema: z.ZodSchema | any;
|
extractionSchema: z.ZodSchema | any;
|
||||||
mode?: "llm-extraction";
|
mode?: "llm-extraction";
|
||||||
extractionPrompt?: string;
|
extractionPrompt?: string;
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Response interface for scraping operations.
|
* Response interface for scraping operations.
|
||||||
*/
|
*/
|
||||||
export interface ScrapeResponse {
|
export interface ScrapeResponse {
|
||||||
success: boolean;
|
success: boolean;
|
||||||
data?: any;
|
data?: any;
|
||||||
error?: string;
|
error?: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Response interface for searching operations.
|
* Response interface for searching operations.
|
||||||
*/
|
*/
|
||||||
export interface SearchResponse {
|
export interface SearchResponse {
|
||||||
success: boolean;
|
success: boolean;
|
||||||
data?: any;
|
data?: any;
|
||||||
error?: string;
|
error?: string;
|
||||||
}
|
}
|
||||||
/**
|
/**
|
||||||
* Response interface for crawling operations.
|
* Response interface for crawling operations.
|
||||||
*/
|
*/
|
||||||
export interface CrawlResponse {
|
export interface CrawlResponse {
|
||||||
success: boolean;
|
success: boolean;
|
||||||
jobId?: string;
|
jobId?: string;
|
||||||
data?: any;
|
data?: any;
|
||||||
error?: string;
|
error?: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Response interface for job status checks.
|
* Response interface for job status checks.
|
||||||
*/
|
*/
|
||||||
export interface JobStatusResponse {
|
export interface JobStatusResponse {
|
||||||
success: boolean;
|
success: boolean;
|
||||||
status: string;
|
status: string;
|
||||||
jobId?: string;
|
jobId?: string;
|
||||||
data?: any;
|
data?: any;
|
||||||
partial_data?: any,
|
partial_data?: any,
|
||||||
error?: string;
|
error?: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Main class for interacting with the Firecrawl API.
|
* Main class for interacting with the Firecrawl API.
|
||||||
*/
|
*/
|
||||||
export default class FirecrawlApp {
|
export default class FirecrawlApp {
|
||||||
private apiKey: string;
|
private apiKey: string;
|
||||||
private apiUrl: string = "https://api.firecrawl.dev";
|
private apiUrl: string;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Initializes a new instance of the FirecrawlApp class.
|
* Initializes a new instance of the FirecrawlApp class.
|
||||||
* @param {FirecrawlAppConfig} config - Configuration options for the FirecrawlApp instance.
|
* @param {FirecrawlAppConfig} config - Configuration options for the FirecrawlApp instance.
|
||||||
*/
|
*/
|
||||||
constructor({ apiKey = null }: FirecrawlAppConfig) {
|
constructor({ apiKey = null, apiUrl = null }: FirecrawlAppConfig) {
|
||||||
this.apiKey = apiKey || "";
|
this.apiKey = apiKey || "";
|
||||||
if (!this.apiKey) {
|
this.apiUrl = apiUrl || "https://api.firecrawl.dev"
|
||||||
throw new Error("No API key provided");
|
if (!this.apiKey) {
|
||||||
}
|
throw new Error("No API key provided");
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Scrapes a URL using the Firecrawl API.
|
|
||||||
* @param {string} url - The URL to scrape.
|
|
||||||
* @param {Params | null} params - Additional parameters for the scrape request.
|
|
||||||
* @returns {Promise<ScrapeResponse>} The response from the scrape operation.
|
|
||||||
*/
|
|
||||||
async scrapeUrl(
|
|
||||||
url: string,
|
|
||||||
params: Params | null = null
|
|
||||||
): Promise<ScrapeResponse> {
|
|
||||||
const headers: AxiosRequestHeaders = {
|
|
||||||
"Content-Type": "application/json",
|
|
||||||
Authorization: `Bearer ${this.apiKey}`,
|
|
||||||
} as AxiosRequestHeaders;
|
|
||||||
let jsonData: Params = { url, ...params };
|
|
||||||
if (params?.extractorOptions?.extractionSchema) {
|
|
||||||
let schema = params.extractorOptions.extractionSchema;
|
|
||||||
// Check if schema is an instance of ZodSchema to correctly identify Zod schemas
|
|
||||||
if (schema instanceof z.ZodSchema) {
|
|
||||||
schema = zodToJsonSchema(schema);
|
|
||||||
}
|
|
||||||
jsonData = {
|
|
||||||
...jsonData,
|
|
||||||
extractorOptions: {
|
|
||||||
...params.extractorOptions,
|
|
||||||
extractionSchema: schema,
|
|
||||||
mode: params.extractorOptions.mode || "llm-extraction",
|
|
||||||
},
|
|
||||||
};
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
const response: AxiosResponse = await axios.post(
|
|
||||||
this.apiUrl + "/v0/scrape",
|
|
||||||
jsonData,
|
|
||||||
{ headers },
|
|
||||||
);
|
|
||||||
if (response.status === 200) {
|
|
||||||
const responseData = response.data;
|
|
||||||
if (responseData.success) {
|
|
||||||
return responseData;
|
|
||||||
} else {
|
|
||||||
throw new Error(`Failed to scrape URL. Error: ${responseData.error}`);
|
|
||||||
}
|
}
|
||||||
} else {
|
|
||||||
this.handleError(response, "scrape URL");
|
|
||||||
}
|
|
||||||
} catch (error: any) {
|
|
||||||
throw new Error(error.message);
|
|
||||||
}
|
}
|
||||||
return { success: false, error: "Internal server error." };
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Searches for a query using the Firecrawl API.
|
* Scrapes a URL using the Firecrawl API.
|
||||||
* @param {string} query - The query to search for.
|
* @param {string} url - The URL to scrape.
|
||||||
* @param {Params | null} params - Additional parameters for the search request.
|
* @param {Params | null} params - Additional parameters for the scrape request.
|
||||||
* @returns {Promise<SearchResponse>} The response from the search operation.
|
* @returns {Promise<ScrapeResponse>} The response from the scrape operation.
|
||||||
*/
|
*/
|
||||||
async search(
|
async scrapeUrl(
|
||||||
query: string,
|
url: string,
|
||||||
params: Params | null = null
|
params: Params | null = null
|
||||||
): Promise<SearchResponse> {
|
): Promise<ScrapeResponse> {
|
||||||
const headers: AxiosRequestHeaders = {
|
const headers: AxiosRequestHeaders = {
|
||||||
"Content-Type": "application/json",
|
"Content-Type": "application/json",
|
||||||
Authorization: `Bearer ${this.apiKey}`,
|
Authorization: `Bearer ${this.apiKey}`,
|
||||||
} as AxiosRequestHeaders;
|
} as AxiosRequestHeaders;
|
||||||
let jsonData: Params = { query };
|
let jsonData: Params = { url, ...params };
|
||||||
if (params) {
|
if (params?.extractorOptions?.extractionSchema) {
|
||||||
jsonData = { ...jsonData, ...params };
|
let schema = params.extractorOptions.extractionSchema;
|
||||||
}
|
// Check if schema is an instance of ZodSchema to correctly identify Zod schemas
|
||||||
try {
|
if (schema instanceof z.ZodSchema) {
|
||||||
const response: AxiosResponse = await axios.post(
|
schema = zodToJsonSchema(schema);
|
||||||
this.apiUrl + "/v0/search",
|
}
|
||||||
jsonData,
|
jsonData = {
|
||||||
{ headers }
|
...jsonData,
|
||||||
);
|
extractorOptions: {
|
||||||
if (response.status === 200) {
|
...params.extractorOptions,
|
||||||
const responseData = response.data;
|
extractionSchema: schema,
|
||||||
if (responseData.success) {
|
mode: params.extractorOptions.mode || "llm-extraction",
|
||||||
return responseData;
|
},
|
||||||
} else {
|
};
|
||||||
throw new Error(`Failed to search. Error: ${responseData.error}`);
|
|
||||||
}
|
}
|
||||||
} else {
|
try {
|
||||||
this.handleError(response, "search");
|
const response: AxiosResponse = await axios.post(
|
||||||
}
|
this.apiUrl + "/v0/scrape",
|
||||||
} catch (error: any) {
|
jsonData,
|
||||||
throw new Error(error.message);
|
{ headers },
|
||||||
}
|
);
|
||||||
return { success: false, error: "Internal server error." };
|
if (response.status === 200) {
|
||||||
}
|
const responseData = response.data;
|
||||||
|
if (responseData.success) {
|
||||||
/**
|
return responseData;
|
||||||
* Initiates a crawl job for a URL using the Firecrawl API.
|
} else {
|
||||||
* @param {string} url - The URL to crawl.
|
throw new Error(`Failed to scrape URL. Error: ${responseData.error}`);
|
||||||
* @param {Params | null} params - Additional parameters for the crawl request.
|
}
|
||||||
* @param {boolean} waitUntilDone - Whether to wait for the crawl job to complete.
|
} else {
|
||||||
* @param {number} pollInterval - Time in seconds for job status checks.
|
this.handleError(response, "scrape URL");
|
||||||
* @param {string} idempotencyKey - Optional idempotency key for the request.
|
}
|
||||||
* @returns {Promise<CrawlResponse | any>} The response from the crawl operation.
|
} catch (error: any) {
|
||||||
*/
|
throw new Error(error.message);
|
||||||
async crawlUrl(
|
|
||||||
url: string,
|
|
||||||
params: Params | null = null,
|
|
||||||
waitUntilDone: boolean = true,
|
|
||||||
pollInterval: number = 2,
|
|
||||||
idempotencyKey?: string
|
|
||||||
): Promise<CrawlResponse | any> {
|
|
||||||
const headers = this.prepareHeaders(idempotencyKey);
|
|
||||||
let jsonData: Params = { url };
|
|
||||||
if (params) {
|
|
||||||
jsonData = { ...jsonData, ...params };
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
const response: AxiosResponse = await this.postRequest(
|
|
||||||
this.apiUrl + "/v0/crawl",
|
|
||||||
jsonData,
|
|
||||||
headers
|
|
||||||
);
|
|
||||||
if (response.status === 200) {
|
|
||||||
const jobId: string = response.data.jobId;
|
|
||||||
if (waitUntilDone) {
|
|
||||||
return this.monitorJobStatus(jobId, headers, pollInterval);
|
|
||||||
} else {
|
|
||||||
return { success: true, jobId };
|
|
||||||
}
|
}
|
||||||
} else {
|
return { success: false, error: "Internal server error." };
|
||||||
this.handleError(response, "start crawl job");
|
|
||||||
}
|
|
||||||
} catch (error: any) {
|
|
||||||
console.log(error);
|
|
||||||
throw new Error(error.message);
|
|
||||||
}
|
}
|
||||||
return { success: false, error: "Internal server error." };
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Checks the status of a crawl job using the Firecrawl API.
|
* Searches for a query using the Firecrawl API.
|
||||||
* @param {string} jobId - The job ID of the crawl operation.
|
* @param {string} query - The query to search for.
|
||||||
* @returns {Promise<JobStatusResponse>} The response containing the job status.
|
* @param {Params | null} params - Additional parameters for the search request.
|
||||||
*/
|
* @returns {Promise<SearchResponse>} The response from the search operation.
|
||||||
async checkCrawlStatus(jobId: string): Promise<JobStatusResponse> {
|
*/
|
||||||
const headers: AxiosRequestHeaders = this.prepareHeaders();
|
async search(
|
||||||
try {
|
query: string,
|
||||||
const response: AxiosResponse = await this.getRequest(
|
params: Params | null = null
|
||||||
this.apiUrl + `/v0/crawl/status/${jobId}`,
|
): Promise<SearchResponse> {
|
||||||
headers
|
const headers: AxiosRequestHeaders = {
|
||||||
);
|
"Content-Type": "application/json",
|
||||||
if (response.status === 200) {
|
Authorization: `Bearer ${this.apiKey}`,
|
||||||
|
} as AxiosRequestHeaders;
|
||||||
|
let jsonData: Params = { query };
|
||||||
|
if (params) {
|
||||||
|
jsonData = { ...jsonData, ...params };
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
const response: AxiosResponse = await axios.post(
|
||||||
|
this.apiUrl + "/v0/search",
|
||||||
|
jsonData,
|
||||||
|
{ headers }
|
||||||
|
);
|
||||||
|
if (response.status === 200) {
|
||||||
|
const responseData = response.data;
|
||||||
|
if (responseData.success) {
|
||||||
|
return responseData;
|
||||||
|
} else {
|
||||||
|
throw new Error(`Failed to search. Error: ${responseData.error}`);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
this.handleError(response, "search");
|
||||||
|
}
|
||||||
|
} catch (error: any) {
|
||||||
|
throw new Error(error.message);
|
||||||
|
}
|
||||||
|
return { success: false, error: "Internal server error." };
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Initiates a crawl job for a URL using the Firecrawl API.
|
||||||
|
* @param {string} url - The URL to crawl.
|
||||||
|
* @param {Params | null} params - Additional parameters for the crawl request.
|
||||||
|
* @param {boolean} waitUntilDone - Whether to wait for the crawl job to complete.
|
||||||
|
* @param {number} pollInterval - Time in seconds for job status checks.
|
||||||
|
* @param {string} idempotencyKey - Optional idempotency key for the request.
|
||||||
|
* @returns {Promise<CrawlResponse | any>} The response from the crawl operation.
|
||||||
|
*/
|
||||||
|
async crawlUrl(
|
||||||
|
url: string,
|
||||||
|
params: Params | null = null,
|
||||||
|
waitUntilDone: boolean = true,
|
||||||
|
pollInterval: number = 2,
|
||||||
|
idempotencyKey?: string
|
||||||
|
): Promise<CrawlResponse | any> {
|
||||||
|
const headers = this.prepareHeaders(idempotencyKey);
|
||||||
|
let jsonData: Params = { url };
|
||||||
|
if (params) {
|
||||||
|
jsonData = { ...jsonData, ...params };
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
const response: AxiosResponse = await this.postRequest(
|
||||||
|
this.apiUrl + "/v0/crawl",
|
||||||
|
jsonData,
|
||||||
|
headers
|
||||||
|
);
|
||||||
|
if (response.status === 200) {
|
||||||
|
const jobId: string = response.data.jobId;
|
||||||
|
if (waitUntilDone) {
|
||||||
|
return this.monitorJobStatus(jobId, headers, pollInterval);
|
||||||
|
} else {
|
||||||
|
return { success: true, jobId };
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
this.handleError(response, "start crawl job");
|
||||||
|
}
|
||||||
|
} catch (error: any) {
|
||||||
|
console.log(error);
|
||||||
|
throw new Error(error.message);
|
||||||
|
}
|
||||||
|
return { success: false, error: "Internal server error." };
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Checks the status of a crawl job using the Firecrawl API.
|
||||||
|
* @param {string} jobId - The job ID of the crawl operation.
|
||||||
|
* @returns {Promise<JobStatusResponse>} The response containing the job status.
|
||||||
|
*/
|
||||||
|
async checkCrawlStatus(jobId: string): Promise<JobStatusResponse> {
|
||||||
|
const headers: AxiosRequestHeaders = this.prepareHeaders();
|
||||||
|
try {
|
||||||
|
const response: AxiosResponse = await this.getRequest(
|
||||||
|
this.apiUrl + `/v0/crawl/status/${jobId}`,
|
||||||
|
headers
|
||||||
|
);
|
||||||
|
if (response.status === 200) {
|
||||||
|
return {
|
||||||
|
success: true,
|
||||||
|
status: response.data.status,
|
||||||
|
data: response.data.data,
|
||||||
|
partial_data: !response.data.data ? response.data.partial_data : undefined,
|
||||||
|
};
|
||||||
|
} else {
|
||||||
|
this.handleError(response, "check crawl status");
|
||||||
|
}
|
||||||
|
} catch (error: any) {
|
||||||
|
throw new Error(error.message);
|
||||||
|
}
|
||||||
return {
|
return {
|
||||||
success: true,
|
success: false,
|
||||||
status: response.data.status,
|
status: "unknown",
|
||||||
data: response.data.data,
|
error: "Internal server error.",
|
||||||
partial_data: !response.data.data ? response.data.partial_data : undefined,
|
|
||||||
};
|
};
|
||||||
} else {
|
|
||||||
this.handleError(response, "check crawl status");
|
|
||||||
}
|
|
||||||
} catch (error: any) {
|
|
||||||
throw new Error(error.message);
|
|
||||||
}
|
}
|
||||||
return {
|
|
||||||
success: false,
|
|
||||||
status: "unknown",
|
|
||||||
error: "Internal server error.",
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Prepares the headers for an API request.
|
* Prepares the headers for an API request.
|
||||||
* @returns {AxiosRequestHeaders} The prepared headers.
|
* @returns {AxiosRequestHeaders} The prepared headers.
|
||||||
*/
|
*/
|
||||||
prepareHeaders(idempotencyKey?: string): AxiosRequestHeaders {
|
prepareHeaders(idempotencyKey?: string): AxiosRequestHeaders {
|
||||||
return {
|
return {
|
||||||
'Content-Type': 'application/json',
|
'Content-Type': 'application/json',
|
||||||
'Authorization': `Bearer ${this.apiKey}`,
|
'Authorization': `Bearer ${this.apiKey}`,
|
||||||
...(idempotencyKey ? { 'x-idempotency-key': idempotencyKey } : {}),
|
...(idempotencyKey ? { 'x-idempotency-key': idempotencyKey } : {}),
|
||||||
} as AxiosRequestHeaders & { 'x-idempotency-key'?: string };
|
} as AxiosRequestHeaders & { 'x-idempotency-key'?: string };
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Sends a POST request to the specified URL.
|
* Sends a POST request to the specified URL.
|
||||||
* @param {string} url - The URL to send the request to.
|
* @param {string} url - The URL to send the request to.
|
||||||
* @param {Params} data - The data to send in the request.
|
* @param {Params} data - The data to send in the request.
|
||||||
* @param {AxiosRequestHeaders} headers - The headers for the request.
|
* @param {AxiosRequestHeaders} headers - The headers for the request.
|
||||||
* @returns {Promise<AxiosResponse>} The response from the POST request.
|
* @returns {Promise<AxiosResponse>} The response from the POST request.
|
||||||
*/
|
*/
|
||||||
postRequest(
|
postRequest(
|
||||||
url: string,
|
url: string,
|
||||||
data: Params,
|
data: Params,
|
||||||
headers: AxiosRequestHeaders
|
headers: AxiosRequestHeaders
|
||||||
): Promise<AxiosResponse> {
|
): Promise<AxiosResponse> {
|
||||||
return axios.post(url, data, { headers });
|
return axios.post(url, data, { headers });
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Sends a GET request to the specified URL.
|
* Sends a GET request to the specified URL.
|
||||||
* @param {string} url - The URL to send the request to.
|
* @param {string} url - The URL to send the request to.
|
||||||
* @param {AxiosRequestHeaders} headers - The headers for the request.
|
* @param {AxiosRequestHeaders} headers - The headers for the request.
|
||||||
* @returns {Promise<AxiosResponse>} The response from the GET request.
|
* @returns {Promise<AxiosResponse>} The response from the GET request.
|
||||||
*/
|
*/
|
||||||
getRequest(
|
getRequest(
|
||||||
url: string,
|
url: string,
|
||||||
headers: AxiosRequestHeaders
|
headers: AxiosRequestHeaders
|
||||||
): Promise<AxiosResponse> {
|
): Promise<AxiosResponse> {
|
||||||
return axios.get(url, { headers });
|
return axios.get(url, { headers });
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Monitors the status of a crawl job until completion or failure.
|
* Monitors the status of a crawl job until completion or failure.
|
||||||
* @param {string} jobId - The job ID of the crawl operation.
|
* @param {string} jobId - The job ID of the crawl operation.
|
||||||
* @param {AxiosRequestHeaders} headers - The headers for the request.
|
* @param {AxiosRequestHeaders} headers - The headers for the request.
|
||||||
* @param {number} timeout - Timeout in seconds for job status checks.
|
* @param {number} timeout - Timeout in seconds for job status checks.
|
||||||
* @returns {Promise<any>} The final job status or data.
|
* @returns {Promise<any>} The final job status or data.
|
||||||
*/
|
*/
|
||||||
async monitorJobStatus(
|
async monitorJobStatus(
|
||||||
jobId: string,
|
jobId: string,
|
||||||
headers: AxiosRequestHeaders,
|
headers: AxiosRequestHeaders,
|
||||||
checkInterval: number
|
checkInterval: number
|
||||||
): Promise<any> {
|
): Promise<any> {
|
||||||
while (true) {
|
while (true) {
|
||||||
const statusResponse: AxiosResponse = await this.getRequest(
|
const statusResponse: AxiosResponse = await this.getRequest(
|
||||||
this.apiUrl + `/v0/crawl/status/${jobId}`,
|
this.apiUrl + `/v0/crawl/status/${jobId}`,
|
||||||
headers
|
headers
|
||||||
);
|
);
|
||||||
if (statusResponse.status === 200) {
|
if (statusResponse.status === 200) {
|
||||||
const statusData = statusResponse.data;
|
const statusData = statusResponse.data;
|
||||||
if (statusData.status === "completed") {
|
if (statusData.status === "completed") {
|
||||||
if ("data" in statusData) {
|
if ("data" in statusData) {
|
||||||
return statusData.data;
|
return statusData.data;
|
||||||
} else {
|
} else {
|
||||||
throw new Error("Crawl job completed but no data was returned");
|
throw new Error("Crawl job completed but no data was returned");
|
||||||
}
|
}
|
||||||
} else if (
|
} else if (
|
||||||
["active", "paused", "pending", "queued"].includes(statusData.status)
|
["active", "paused", "pending", "queued"].includes(statusData.status)
|
||||||
) {
|
) {
|
||||||
if (checkInterval < 2) {
|
if (checkInterval < 2) {
|
||||||
checkInterval = 2;
|
checkInterval = 2;
|
||||||
}
|
}
|
||||||
await new Promise((resolve) => setTimeout(resolve, checkInterval * 1000)); // Wait for the specified timeout before checking again
|
await new Promise((resolve) => setTimeout(resolve, checkInterval * 1000)); // Wait for the specified timeout before checking again
|
||||||
} else {
|
} else {
|
||||||
throw new Error(
|
throw new Error(
|
||||||
`Crawl job failed or was stopped. Status: ${statusData.status}`
|
`Crawl job failed or was stopped. Status: ${statusData.status}`
|
||||||
);
|
);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
this.handleError(statusResponse, "check crawl status");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
} else {
|
|
||||||
this.handleError(statusResponse, "check crawl status");
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Handles errors from API responses.
|
* Handles errors from API responses.
|
||||||
* @param {AxiosResponse} response - The response from the API.
|
* @param {AxiosResponse} response - The response from the API.
|
||||||
* @param {string} action - The action being performed when the error occurred.
|
* @param {string} action - The action being performed when the error occurred.
|
||||||
*/
|
*/
|
||||||
handleError(response: AxiosResponse, action: string): void {
|
handleError(response: AxiosResponse, action: string): void {
|
||||||
if ([402, 408, 409, 500].includes(response.status)) {
|
if ([402, 408, 409, 500].includes(response.status)) {
|
||||||
const errorMessage: string =
|
const errorMessage: string =
|
||||||
response.data.error || "Unknown error occurred";
|
response.data.error || "Unknown error occurred";
|
||||||
throw new Error(
|
throw new Error(
|
||||||
`Failed to ${action}. Status code: ${response.status}. Error: ${errorMessage}`
|
`Failed to ${action}. Status code: ${response.status}. Error: ${errorMessage}`
|
||||||
);
|
);
|
||||||
} else {
|
} else {
|
||||||
throw new Error(
|
throw new Error(
|
||||||
`Unexpected error occurred while trying to ${action}. Status code: ${response.status}`
|
`Unexpected error occurred while trying to ${action}. Status code: ${response.status}`
|
||||||
);
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
2
apps/js-sdk/firecrawl/types/index.d.ts
vendored
2
apps/js-sdk/firecrawl/types/index.d.ts
vendored
@ -64,7 +64,7 @@ export default class FirecrawlApp {
|
|||||||
* Initializes a new instance of the FirecrawlApp class.
|
* Initializes a new instance of the FirecrawlApp class.
|
||||||
* @param {FirecrawlAppConfig} config - Configuration options for the FirecrawlApp instance.
|
* @param {FirecrawlAppConfig} config - Configuration options for the FirecrawlApp instance.
|
||||||
*/
|
*/
|
||||||
constructor({ apiKey }: FirecrawlAppConfig);
|
constructor({ apiKey, apiUrl }: FirecrawlAppConfig);
|
||||||
/**
|
/**
|
||||||
* Scrapes a URL using the Firecrawl API.
|
* Scrapes a URL using the Firecrawl API.
|
||||||
* @param {string} url - The URL to scrape.
|
* @param {string} url - The URL to scrape.
|
||||||
|
40
apps/js-sdk/package-lock.json
generated
40
apps/js-sdk/package-lock.json
generated
@ -11,10 +11,8 @@
|
|||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@mendable/firecrawl-js": "^0.0.19",
|
"@mendable/firecrawl-js": "^0.0.19",
|
||||||
"axios": "^1.6.8",
|
"axios": "^1.6.8",
|
||||||
"dotenv": "^16.4.5",
|
|
||||||
"ts-node": "^10.9.2",
|
"ts-node": "^10.9.2",
|
||||||
"typescript": "^5.4.5",
|
"typescript": "^5.4.5",
|
||||||
"uuid": "^9.0.1",
|
|
||||||
"zod": "^3.23.8"
|
"zod": "^3.23.8"
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
@ -452,15 +450,6 @@
|
|||||||
"resolved": "https://registry.npmjs.org/@tsconfig/node16/-/node16-1.0.4.tgz",
|
"resolved": "https://registry.npmjs.org/@tsconfig/node16/-/node16-1.0.4.tgz",
|
||||||
"integrity": "sha512-vxhUy4J8lyeyinH7Azl1pdd43GJhZH/tP2weN8TntQblOY+A0XbT8DJk1/oCPuOOyg/Ja757rG0CgHcWC8OfMA=="
|
"integrity": "sha512-vxhUy4J8lyeyinH7Azl1pdd43GJhZH/tP2weN8TntQblOY+A0XbT8DJk1/oCPuOOyg/Ja757rG0CgHcWC8OfMA=="
|
||||||
},
|
},
|
||||||
"node_modules/@types/node": {
|
|
||||||
"version": "20.12.11",
|
|
||||||
"resolved": "https://registry.npmjs.org/@types/node/-/node-20.12.11.tgz",
|
|
||||||
"integrity": "sha512-vDg9PZ/zi+Nqp6boSOT7plNuthRugEKixDv5sFTIpkE89MmNtEArAShI4mxuX2+UrLEe9pxC1vm2cjm9YlWbJw==",
|
|
||||||
"peer": true,
|
|
||||||
"dependencies": {
|
|
||||||
"undici-types": "~5.26.4"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"node_modules/acorn": {
|
"node_modules/acorn": {
|
||||||
"version": "8.11.3",
|
"version": "8.11.3",
|
||||||
"resolved": "https://registry.npmjs.org/acorn/-/acorn-8.11.3.tgz",
|
"resolved": "https://registry.npmjs.org/acorn/-/acorn-8.11.3.tgz",
|
||||||
@ -532,17 +521,6 @@
|
|||||||
"node": ">=0.3.1"
|
"node": ">=0.3.1"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/dotenv": {
|
|
||||||
"version": "16.4.5",
|
|
||||||
"resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.4.5.tgz",
|
|
||||||
"integrity": "sha512-ZmdL2rui+eB2YwhsWzjInR8LldtZHGDoQ1ugH85ppHKwpUHL7j7rN0Ti9NCnGiQbhaZ11FpR+7ao1dNsmduNUg==",
|
|
||||||
"engines": {
|
|
||||||
"node": ">=12"
|
|
||||||
},
|
|
||||||
"funding": {
|
|
||||||
"url": "https://dotenvx.com"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"node_modules/esbuild": {
|
"node_modules/esbuild": {
|
||||||
"version": "0.20.2",
|
"version": "0.20.2",
|
||||||
"resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.20.2.tgz",
|
"resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.20.2.tgz",
|
||||||
@ -750,24 +728,6 @@
|
|||||||
"node": ">=14.17"
|
"node": ">=14.17"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/undici-types": {
|
|
||||||
"version": "5.26.5",
|
|
||||||
"resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz",
|
|
||||||
"integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==",
|
|
||||||
"peer": true
|
|
||||||
},
|
|
||||||
"node_modules/uuid": {
|
|
||||||
"version": "9.0.1",
|
|
||||||
"resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz",
|
|
||||||
"integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==",
|
|
||||||
"funding": [
|
|
||||||
"https://github.com/sponsors/broofa",
|
|
||||||
"https://github.com/sponsors/ctavan"
|
|
||||||
],
|
|
||||||
"bin": {
|
|
||||||
"uuid": "dist/bin/uuid"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"node_modules/v8-compile-cache-lib": {
|
"node_modules/v8-compile-cache-lib": {
|
||||||
"version": "3.0.1",
|
"version": "3.0.1",
|
||||||
"resolved": "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz",
|
"resolved": "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz",
|
||||||
|
Loading…
Reference in New Issue
Block a user