github/v-firecrawl
github/v-firecrawl
0
Fork 0
Code Actions

Nick:

Browse Source
This commit is contained in:
Nicolas 2024-04-20 19:04:27 -07:00
parent 5b8aed26dd
commit 4543c57e4e
3 changed files with 9 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
apps/api/.env.local
View File

@ -1,3 +1,4 @@
ENV=
NUM_WORKERS_PER_QUEUE=8 NUM_WORKERS_PER_QUEUE=8
PORT= PORT=
HOST= HOST=

15
apps/api/src/controllers/scrape.ts
View File

@ -1,11 +1,10 @@
import { Request, Response } from "express"; import { Request, Response } from "express";
import { WebScraperDataProvider } from "../../src/scraper/WebScraper"; import { WebScraperDataProvider } from "../scraper/WebScraper";
import { billTeam } from "../../src/services/billing/credit_billing"; import { billTeam, checkTeamCredits } from "../services/billing/credit_billing";
import { checkTeamCredits } from "../../src/services/billing/credit_billing";
import { authenticateUser } from "./auth"; import { authenticateUser } from "./auth";
import { RateLimiterMode } from "../../src/types"; import { RateLimiterMode } from "../types";
import { logJob } from "../../src/services/logging/log_job"; import { logJob } from "../services/logging/log_job";
import { Document } from "../../src/lib/entities"; import { Document } from "../lib/entities";
export async function scrapeHelper( export async function scrapeHelper(
req: Request, req: Request,
@ -16,7 +15,7 @@ export async function scrapeHelper(
success: boolean; success: boolean;
error?: string; error?: string;
data?: Document; data?: Document;
returnCode?: number; returnCode: number;
}> { }> {
const url = req.body.url; const url = req.body.url;
if (!url) { if (!url) {
@ -103,7 +102,7 @@ export async function scrapeController(req: Request, res: Response) {
crawlerOptions: crawlerOptions, crawlerOptions: crawlerOptions,
pageOptions: pageOptions, pageOptions: pageOptions,
}); });
return res.json(result); return res.status(result.returnCode).json(result);
} catch (error) { } catch (error) {
console.error(error); console.error(error);
return res.status(500).json({ error: error.message }); return res.status(500).json({ error: error.message });

8
apps/api/src/index.ts
View File

@ -3,13 +3,7 @@ import bodyParser from "body-parser";
import cors from "cors"; import cors from "cors";
import "dotenv/config"; import "dotenv/config";
import { getWebScraperQueue } from "./services/queue-service"; import { getWebScraperQueue } from "./services/queue-service";
import { addWebScraperJob } from "./services/queue-jobs"; import { redisClient } from "./services/rate-limiter";
import { supabase_service } from "./services/supabase";
import { WebScraperDataProvider } from "./scraper/WebScraper";
import { billTeam, checkTeamCredits } from "./services/billing/credit_billing";
import { getRateLimiter, redisClient } from "./services/rate-limiter";
import { parseApi } from "./lib/parseApi";
import { RateLimiterMode } from "./types";
import { v0Router } from "./routes/v0"; import { v0Router } from "./routes/v0";
const { createBullBoard } = require("@bull-board/api"); const { createBullBoard } = require("@bull-board/api");