github/v-firecrawl
github/v-firecrawl
0
Fork 0
Code Actions

bugfixing. already on main

Browse Source
This commit is contained in:
rafaelsideguide 2024-06-04 11:05:50 -03:00
parent d6762386f8
commit 6920ec8a61
5 changed files with 940 additions and 776 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
apps/api/package.json
View File

@ -29,6 +29,7 @@
"@types/cors": "^2.8.13", "@types/cors": "^2.8.13",
"@types/express": "^4.17.17", "@types/express": "^4.17.17",
"@types/jest": "^29.5.12", "@types/jest": "^29.5.12",
"@types/node": "^20.14.1",
"body-parser": "^1.20.1", "body-parser": "^1.20.1",
"express": "^4.18.2", "express": "^4.18.2",
"jest": "^29.6.3", "jest": "^29.6.3",

1706
apps/api/pnpm-lock.yaml
View File

File diff suppressed because it is too large Load Diff

4
apps/api/src/scraper/WebScraper/index.ts
View File

@ -84,13 +84,15 @@ export class WebScraperDataProvider {
const job = await getWebScraperQueue().getJob(this.bullJobId); const job = await getWebScraperQueue().getJob(this.bullJobId);
const jobStatus = await job.getState(); const jobStatus = await job.getState();
if (jobStatus === "failed") { if (jobStatus === "failed") {
throw new Error( console.error(
"Job has failed or has been cancelled by the user. Stopping the job..." "Job has failed or has been cancelled by the user. Stopping the job..."
); );
return [] as Document[];
} }
} }
} catch (error) { } catch (error) {
console.error(error); console.error(error);
return [] as Document[];
} }
} }
return results.filter((result) => result !== null) as Document[]; return results.filter((result) => result !== null) as Document[];

3
apps/api/src/services/queue-service.ts
View File

@ -1,6 +1,7 @@
import Queue from "bull"; import Queue from "bull";
import { Queue as BullQueue } from "bull";
let webScraperQueue; let webScraperQueue: BullQueue;
export function getWebScraperQueue() { export function getWebScraperQueue() {
if (!webScraperQueue) { if (!webScraperQueue) {

2
apps/api/src/services/rate-limiter.ts
View File

@ -54,7 +54,7 @@ export const crawlStatusRateLimiter = createRateLimiter("crawl-status", RATE_LIM
export const testSuiteRateLimiter = createRateLimiter("test-suite", RATE_LIMITS.testSuite); export const testSuiteRateLimiter = createRateLimiter("test-suite", RATE_LIMITS.testSuite);
export function getRateLimiter(mode: RateLimiterMode, token: string, plan?: string) { export function getRateLimiter(mode: RateLimiterMode, token: string, plan?: string) {
if (token.includes("5089cefa58") || token.includes("6254cf9")) { if (token.includes("a01ccae") || token.includes("6254cf9")) {
return testSuiteRateLimiter; return testSuiteRateLimiter;
} }