diff --git a/README.md b/README.md index 6ed41b6..e3a1094 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ Self-host. To self-host refer to guide [here](https://github.com/mendableai/fire ### API Key To use the API, you need to sign up on [Firecrawl](https://firecrawl.com) and get an API key. - + ### Crawling Used to crawl a URL and all accessible subpages. This submits a crawl job and returns a job ID to check the status of the crawl. diff --git a/apps/api/requests.http b/apps/api/requests.http index ab112e9..7b3df11 100644 --- a/apps/api/requests.http +++ b/apps/api/requests.http @@ -1,6 +1,7 @@ ### Crawl Website -POST http://localhost:3002/v0/crawl HTTP/1.1 -Authorization: Bearer +POST http://localhost:3002/v0/scrape HTTP/1.1 +Authorization: Bearer fc-a6a2d63aed2b46a9946d2a7207efed4d +content-type: application/json { "url":"https://docs.mendable.ai" diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts index a039703..7198988 100644 --- a/apps/api/src/index.ts +++ b/apps/api/src/index.ts @@ -8,6 +8,7 @@ import { supabase_service } from "./services/supabase"; import { WebScraperDataProvider } from "./scraper/WebScraper"; import { billTeam, checkTeamCredits } from "./services/billing/credit_billing"; import { getRateLimiter, redisClient } from "./services/rate-limiter"; +import { parseApi } from "./lib/parseApi"; const { createBullBoard } = require("@bull-board/api"); const { BullAdapter } = require("@bull-board/api/bullAdapter"); @@ -71,11 +72,13 @@ async function authenticateUser(req, res, mode?: string): Promise { if (token === "this_is_just_a_preview_token" && mode === "scrape") { return "preview"; } + + const normalizedApi = parseApi(token); // make sure api key is valid, based on the api_keys table in supabase const { data, error } = await supabase_service .from("api_keys") .select("*") - .eq("key", token); + .eq("key", normalizedApi); if (error || !data || data.length === 0) { return res.status(401).json({ error: "Unauthorized: Invalid token" }); } @@ -312,7 +315,7 @@ redisClient.connect(); export function startServer(port = DEFAULT_PORT) { const server = app.listen(Number(port), HOST, () => { console.log(`Server listening on port ${port}`); - console.log(`For the UI, open http://${HOST}:${port}/admin/queues`); + console.log(`For the UI, open http://${HOST}:${port}/admin/${process.env.BULL_AUTH_KEY}/queues`); console.log(""); console.log("1. Make sure Redis is running on port 6379 by default"); console.log( @@ -326,7 +329,7 @@ if (require.main === module) { startServer(); } -// Use this as a health check that way we dont destroy the server +// Use this as a "health check" that way we dont destroy the server app.get(`/admin/${process.env.BULL_AUTH_KEY}/queues`, async (req, res) => { try { const webScraperQueue = getWebScraperQueue(); diff --git a/apps/api/src/lib/parseApi.ts b/apps/api/src/lib/parseApi.ts new file mode 100644 index 0000000..8e1a354 --- /dev/null +++ b/apps/api/src/lib/parseApi.ts @@ -0,0 +1,21 @@ +export function parseApi(api: string) { + // Handle older versions of the API that don't have the fc- prefix + if (!api.startsWith("fc-")) { + return api; + } + + // remove the fc- prefix + // re add all the dashes based on the uuidv4 format + // 3d478a29-6e59-403e-85c7-94aba81ffd2a + const uuid = api + .replace(/^fc-/, "") + .replace(/(.{8})(.{4})(.{4})(.{4})(.{12})/, "$1-$2-$3-$4-$5"); + return uuid; +} + +console.log(parseApi("fc-a6a2d63aed2b46a9946d2a7207efed4d")) + +export function uuidToFcUuid(uuid: string) { + const uuidWithoutDashes = uuid.replace(/-/g, ""); + return `fc-${uuidWithoutDashes}`; +}