github/v-firecrawl
github/v-firecrawl
0
Fork 0
Code Actions

Nick: tests

Browse Source
This commit is contained in:
Nicolas 2024-04-21 11:27:31 -07:00
parent 5cdbf3a0ac
commit 898d729a84
4 changed files with 334 additions and 171 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

210
apps/api/src/__tests__/e2e/index.test.ts
View File

@ -1,189 +1,197 @@
import request from 'supertest'; import request from "supertest";
import { app } from '../../index'; import { app } from "../../index";
import dotenv from 'dotenv'; import dotenv from "dotenv";
dotenv.config(); dotenv.config();
// const TEST_URL = 'http://localhost:3002' // const TEST_URL = 'http://localhost:3002'
const TEST_URL = 'http://127.0.0.1:3002' const TEST_URL = "http://127.0.0.1:3002";
describe("E2E Tests for API Routes", () => {
beforeAll(() => {
process.env.USE_DB_AUTHENTICATION = "true";
});
afterAll(() => {
delete process.env.USE_DB_AUTHENTICATION;
describe('E2E Tests for API Routes', () => { });
describe('GET /', () => { describe("GET /", () => {
it('should return Hello, world! message', async () => { it("should return Hello, world! message", async () => {
const response = await request(TEST_URL).get("/");
const response = await request(TEST_URL).get('/');
expect(response.statusCode).toBe(200); expect(response.statusCode).toBe(200);
expect(response.text).toContain('SCRAPERS-JS: Hello, world! Fly.io'); expect(response.text).toContain("SCRAPERS-JS: Hello, world! Fly.io");
}); });
}); });
describe('GET /test', () => { describe("GET /test", () => {
it('should return Hello, world! message', async () => { it("should return Hello, world! message", async () => {
const response = await request(TEST_URL).get("/test");
const response = await request(TEST_URL).get('/test');
expect(response.statusCode).toBe(200); expect(response.statusCode).toBe(200);
expect(response.text).toContain('Hello, world!'); expect(response.text).toContain("Hello, world!");
}); });
}); });
describe('POST /v0/scrape', () => { describe("POST /v0/scrape", () => {
it('should require authorization', async () => { it("should require authorization", async () => {
const response = await request(app).post('/v0/scrape'); const response = await request(app).post("/v0/scrape");
expect(response.statusCode).toBe(401); expect(response.statusCode).toBe(401);
}); });
it('should return an error response with an invalid API key', async () => { it("should return an error response with an invalid API key", async () => {
const response = await request(TEST_URL) const response = await request(TEST_URL)
.post('/v0/scrape') .post("/v0/scrape")
.set('Authorization', `Bearer invalid-api-key`) .set("Authorization", `Bearer invalid-api-key`)
.set('Content-Type', 'application/json') .set("Content-Type", "application/json")
.send({ url: 'https://firecrawl.dev' }); .send({ url: "https://firecrawl.dev" });
expect(response.statusCode).toBe(401); expect(response.statusCode).toBe(401);
}); });
it('should return a successful response with a valid preview token', async () => { it("should return a successful response with a valid preview token", async () => {
const response = await request(TEST_URL) const response = await request(TEST_URL)
.post('/v0/scrape') .post("/v0/scrape")
.set('Authorization', `Bearer this_is_just_a_preview_token`) .set("Authorization", `Bearer this_is_just_a_preview_token`)
.set('Content-Type', 'application/json') .set("Content-Type", "application/json")
.send({ url: 'https://firecrawl.dev' }); .send({ url: "https://firecrawl.dev" });
expect(response.statusCode).toBe(200); expect(response.statusCode).toBe(200);
}, 10000); // 10 seconds timeout }, 10000); // 10 seconds timeout
it('should return a successful response with a valid API key', async () => { it("should return a successful response with a valid API key", async () => {
const response = await request(TEST_URL) const response = await request(TEST_URL)
.post('/v0/scrape') .post("/v0/scrape")
.set('Authorization', `Bearer ${process.env.TEST_API_KEY}`) .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
.set('Content-Type', 'application/json') .set("Content-Type", "application/json")
.send({ url: 'https://firecrawl.dev' }); .send({ url: "https://firecrawl.dev" });
expect(response.statusCode).toBe(200); expect(response.statusCode).toBe(200);
expect(response.body).toHaveProperty('data'); expect(response.body).toHaveProperty("data");
expect(response.body.data).toHaveProperty('content'); expect(response.body.data).toHaveProperty("content");
expect(response.body.data).toHaveProperty('markdown'); expect(response.body.data).toHaveProperty("markdown");
expect(response.body.data).toHaveProperty('metadata'); expect(response.body.data).toHaveProperty("metadata");
expect(response.body.data.content).toContain('🔥 FireCrawl'); expect(response.body.data.content).toContain("🔥 FireCrawl");
}, 30000); // 30 seconds timeout }, 30000); // 30 seconds timeout
}); });
describe('POST /v0/crawl', () => { describe("POST /v0/crawl", () => {
it('should require authorization', async () => { it("should require authorization", async () => {
const response = await request(TEST_URL).post('/v0/crawl'); const response = await request(TEST_URL).post("/v0/crawl");
expect(response.statusCode).toBe(401); expect(response.statusCode).toBe(401);
}); });
it('should return an error response with an invalid API key', async () => { it("should return an error response with an invalid API key", async () => {
const response = await request(TEST_URL) const response = await request(TEST_URL)
.post('/v0/crawl') .post("/v0/crawl")
.set('Authorization', `Bearer invalid-api-key`) .set("Authorization", `Bearer invalid-api-key`)
.set('Content-Type', 'application/json') .set("Content-Type", "application/json")
.send({ url: 'https://firecrawl.dev' }); .send({ url: "https://firecrawl.dev" });
expect(response.statusCode).toBe(401); expect(response.statusCode).toBe(401);
}); });
it('should return a successful response with a valid API key', async () => { it("should return a successful response with a valid API key", async () => {
const response = await request(TEST_URL) const response = await request(TEST_URL)
.post('/v0/crawl') .post("/v0/crawl")
.set('Authorization', `Bearer ${process.env.TEST_API_KEY}`) .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
.set('Content-Type', 'application/json') .set("Content-Type", "application/json")
.send({ url: 'https://firecrawl.dev' }); .send({ url: "https://firecrawl.dev" });
expect(response.statusCode).toBe(200); expect(response.statusCode).toBe(200);
expect(response.body).toHaveProperty('jobId'); expect(response.body).toHaveProperty("jobId");
expect(response.body.jobId).toMatch(/^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$/); expect(response.body.jobId).toMatch(
/^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$/
);
}); });
// Additional tests for insufficient credits? // Additional tests for insufficient credits?
}); });
describe('POST /v0/crawlWebsitePreview', () => { describe("POST /v0/crawlWebsitePreview", () => {
it('should require authorization', async () => { it("should require authorization", async () => {
const response = await request(TEST_URL).post('/v0/crawlWebsitePreview'); const response = await request(TEST_URL).post(
"/v0/crawlWebsitePreview"
);
expect(response.statusCode).toBe(401); expect(response.statusCode).toBe(401);
}); });
it('should return an error response with an invalid API key', async () => { it("should return an error response with an invalid API key", async () => {
const response = await request(TEST_URL) const response = await request(TEST_URL)
.post('/v0/crawlWebsitePreview') .post("/v0/crawlWebsitePreview")
.set('Authorization', `Bearer invalid-api-key`) .set("Authorization", `Bearer invalid-api-key`)
.set('Content-Type', 'application/json') .set("Content-Type", "application/json")
.send({ url: 'https://firecrawl.dev' }); .send({ url: "https://firecrawl.dev" });
expect(response.statusCode).toBe(401); expect(response.statusCode).toBe(401);
}); });
it('should return a successful response with a valid API key', async () => { it("should return a successful response with a valid API key", async () => {
const response = await request(TEST_URL) const response = await request(TEST_URL)
.post('/v0/crawlWebsitePreview') .post("/v0/crawlWebsitePreview")
.set('Authorization', `Bearer this_is_just_a_preview_token`) .set("Authorization", `Bearer this_is_just_a_preview_token`)
.set('Content-Type', 'application/json') .set("Content-Type", "application/json")
.send({ url: 'https://firecrawl.dev' }); .send({ url: "https://firecrawl.dev" });
expect(response.statusCode).toBe(200); expect(response.statusCode).toBe(200);
expect(response.body).toHaveProperty('jobId'); expect(response.body).toHaveProperty("jobId");
expect(response.body.jobId).toMatch(/^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$/); expect(response.body.jobId).toMatch(
/^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$/
);
}); });
}); });
describe('GET /v0/crawl/status/:jobId', () => { describe("GET /v0/crawl/status/:jobId", () => {
it('should require authorization', async () => { it("should require authorization", async () => {
const response = await request(TEST_URL).get('/v0/crawl/status/123'); const response = await request(TEST_URL).get("/v0/crawl/status/123");
expect(response.statusCode).toBe(401); expect(response.statusCode).toBe(401);
}); });
it('should return an error response with an invalid API key', async () => { it("should return an error response with an invalid API key", async () => {
const response = await request(TEST_URL) const response = await request(TEST_URL)
.get('/v0/crawl/status/123') .get("/v0/crawl/status/123")
.set('Authorization', `Bearer invalid-api-key`); .set("Authorization", `Bearer invalid-api-key`);
expect(response.statusCode).toBe(401); expect(response.statusCode).toBe(401);
}); });
it('should return Job not found for invalid job ID', async () => { it("should return Job not found for invalid job ID", async () => {
const response = await request(TEST_URL) const response = await request(TEST_URL)
.get('/v0/crawl/status/invalidJobId') .get("/v0/crawl/status/invalidJobId")
.set('Authorization', `Bearer ${process.env.TEST_API_KEY}`); .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);
expect(response.statusCode).toBe(404); expect(response.statusCode).toBe(404);
}); });
it('should return a successful response for a valid crawl job', async () => { it("should return a successful response for a valid crawl job", async () => {
const crawlResponse = await request(TEST_URL) const crawlResponse = await request(TEST_URL)
.post('/v0/crawl') .post("/v0/crawl")
.set('Authorization', `Bearer ${process.env.TEST_API_KEY}`) .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
.set('Content-Type', 'application/json') .set("Content-Type", "application/json")
.send({ url: 'https://firecrawl.dev' }); .send({ url: "https://firecrawl.dev" });
expect(crawlResponse.statusCode).toBe(200); expect(crawlResponse.statusCode).toBe(200);
const response = await request(TEST_URL) const response = await request(TEST_URL)
.get(`/v0/crawl/status/${crawlResponse.body.jobId}`) .get(`/v0/crawl/status/${crawlResponse.body.jobId}`)
.set('Authorization', `Bearer ${process.env.TEST_API_KEY}`); .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);
expect(response.statusCode).toBe(200); expect(response.statusCode).toBe(200);
expect(response.body).toHaveProperty('status'); expect(response.body).toHaveProperty("status");
expect(response.body.status).toBe('active'); expect(response.body.status).toBe("active");
// wait for 30 seconds // wait for 30 seconds
await new Promise((r) => setTimeout(r, 30000)); await new Promise((r) => setTimeout(r, 30000));
const completedResponse = await request(TEST_URL) const completedResponse = await request(TEST_URL)
.get(`/v0/crawl/status/${crawlResponse.body.jobId}`) .get(`/v0/crawl/status/${crawlResponse.body.jobId}`)
.set('Authorization', `Bearer ${process.env.TEST_API_KEY}`); .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);
expect(completedResponse.statusCode).toBe(200); expect(completedResponse.statusCode).toBe(200);
expect(completedResponse.body).toHaveProperty('status'); expect(completedResponse.body).toHaveProperty("status");
expect(completedResponse.body.status).toBe('completed'); expect(completedResponse.body.status).toBe("completed");
expect(completedResponse.body).toHaveProperty('data'); expect(completedResponse.body).toHaveProperty("data");
expect(completedResponse.body.data[0]).toHaveProperty('content'); expect(completedResponse.body.data[0]).toHaveProperty("content");
expect(completedResponse.body.data[0]).toHaveProperty('markdown'); expect(completedResponse.body.data[0]).toHaveProperty("markdown");
expect(completedResponse.body.data[0]).toHaveProperty('metadata'); expect(completedResponse.body.data[0]).toHaveProperty("metadata");
expect(completedResponse.body.data[0].content).toContain('🔥 FireCrawl'); expect(completedResponse.body.data[0].content).toContain(
"🔥 FireCrawl"
);
}, 60000); // 60 seconds }, 60000); // 60 seconds
}); });
describe('GET /is-production', () => { describe("GET /is-production", () => {
it('should return the production status', async () => { it("should return the production status", async () => {
const response = await request(TEST_URL).get('/is-production'); const response = await request(TEST_URL).get("/is-production");
expect(response.statusCode).toBe(200); expect(response.statusCode).toBe(200);
expect(response.body).toHaveProperty('isProduction'); expect(response.body).toHaveProperty("isProduction");
}); });
}); });
}); });

156
apps/api/src/__tests__/e2e_noAuth/index.test.ts Normal file
View File

@ -0,0 +1,156 @@
import request from "supertest";
import { app } from "../../index";
import dotenv from "dotenv";
const fs = require("fs");
const path = require("path");
dotenv.config();
const TEST_URL = "http://127.0.0.1:3002";
describe("E2E Tests for API Routes with No Authentication", () => {
let originalEnv: NodeJS.ProcessEnv;
// save original process.env
beforeAll(() => {
originalEnv = { ...process.env };
process.env.USE_DB_AUTHENTICATION = "false";
process.env.SUPABASE_ANON_TOKEN = "";
process.env.SUPABASE_URL = "";
process.env.SUPABASE_SERVICE_TOKEN = "";
process.env.SCRAPING_BEE_API_KEY = "";
process.env.OPENAI_API_KEY = "";
process.env.BULL_AUTH_KEY = "";
process.env.LOGTAIL_KEY = "";
process.env.PLAYWRIGHT_MICROSERVICE_URL = "";
process.env.LLAMAPARSE_API_KEY = "";
process.env.TEST_API_KEY = "";
});
// restore original process.env
afterAll(() => {
process.env = originalEnv;
});
describe("GET /", () => {
it("should return Hello, world! message", async () => {
const response = await request(TEST_URL).get("/");
expect(response.statusCode).toBe(200);
expect(response.text).toContain("SCRAPERS-JS: Hello, world! Fly.io");
});
});
describe("GET /test", () => {
it("should return Hello, world! message", async () => {
const response = await request(TEST_URL).get("/test");
expect(response.statusCode).toBe(200);
expect(response.text).toContain("Hello, world!");
});
});
describe("POST /v0/scrape", () => {
it("should not require authorization", async () => {
const response = await request(TEST_URL).post("/v0/scrape");
expect(response.statusCode).not.toBe(401);
});
it("should return a successful response", async () => {
const response = await request(TEST_URL)
.post("/v0/scrape")
.set("Content-Type", "application/json")
.send({ url: "https://firecrawl.dev" });
expect(response.statusCode).toBe(200);
}, 10000); // 10 seconds timeout
});
describe("POST /v0/crawl", () => {
it("should not require authorization", async () => {
const response = await request(TEST_URL).post("/v0/crawl");
expect(response.statusCode).not.toBe(401);
});
it("should return a successful response", async () => {
const response = await request(TEST_URL)
.post("/v0/crawl")
.set("Content-Type", "application/json")
.send({ url: "https://firecrawl.dev" });
expect(response.statusCode).toBe(200);
expect(response.body).toHaveProperty("jobId");
expect(response.body.jobId).toMatch(
/^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$/
);
});
});
describe("POST /v0/crawlWebsitePreview", () => {
it("should not require authorization", async () => {
const response = await request(TEST_URL).post("/v0/crawlWebsitePreview");
expect(response.statusCode).not.toBe(401);
});
it("should return a successful response", async () => {
const response = await request(TEST_URL)
.post("/v0/crawlWebsitePreview")
.set("Content-Type", "application/json")
.send({ url: "https://firecrawl.dev" });
expect(response.statusCode).toBe(200);
expect(response.body).toHaveProperty("jobId");
expect(response.body.jobId).toMatch(
/^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$/
);
});
});
describe("GET /v0/crawl/status/:jobId", () => {
it("should not require authorization", async () => {
const response = await request(TEST_URL).get("/v0/crawl/status/123");
expect(response.statusCode).not.toBe(401);
});
it("should return Job not found for invalid job ID", async () => {
const response = await request(TEST_URL).get(
"/v0/crawl/status/invalidJobId"
);
expect(response.statusCode).toBe(404);
});
it("should return a successful response for a valid crawl job", async () => {
const crawlResponse = await request(TEST_URL)
.post("/v0/crawl")
.set("Content-Type", "application/json")
.send({ url: "https://firecrawl.dev" });
expect(crawlResponse.statusCode).toBe(200);
const response = await request(TEST_URL).get(
`/v0/crawl/status/${crawlResponse.body.jobId}`
);
expect(response.statusCode).toBe(200);
expect(response.body).toHaveProperty("status");
expect(response.body.status).toBe("active");
// wait for 30 seconds
await new Promise((r) => setTimeout(r, 30000));
const completedResponse = await request(TEST_URL).get(
`/v0/crawl/status/${crawlResponse.body.jobId}`
);
expect(completedResponse.statusCode).toBe(200);
expect(completedResponse.body).toHaveProperty("status");
expect(completedResponse.body.status).toBe("completed");
expect(completedResponse.body).toHaveProperty("data");
expect(completedResponse.body.data[0]).toHaveProperty("content");
expect(completedResponse.body.data[0]).toHaveProperty("markdown");
expect(completedResponse.body.data[0]).toHaveProperty("metadata");
expect(completedResponse.body.data[0].content).toContain("🔥 FireCrawl");
}, 60000); // 60 seconds
});
describe("GET /is-production", () => {
it("should return the production status", async () => {
const response = await request(TEST_URL).get("/is-production");
expect(response.statusCode).toBe(200);
expect(response.body).toHaveProperty("isProduction");
});
});
});

1
apps/api/src/controllers/crawl.ts
View File

@ -8,7 +8,6 @@ import { addWebScraperJob } from "../../src/services/queue-jobs";
export async function crawlController(req: Request, res: Response) { export async function crawlController(req: Request, res: Response) {
try { try {
console.log("hello");
const { success, team_id, error, status } = await authenticateUser( const { success, team_id, error, status } = await authenticateUser(
req, req,
res, res,

2
apps/api/src/index.ts
View File

@ -5,7 +5,6 @@ import "dotenv/config";
import { getWebScraperQueue } from "./services/queue-service"; import { getWebScraperQueue } from "./services/queue-service";
import { redisClient } from "./services/rate-limiter"; import { redisClient } from "./services/rate-limiter";
import { v0Router } from "./routes/v0"; import { v0Router } from "./routes/v0";
const { createBullBoard } = require("@bull-board/api"); const { createBullBoard } = require("@bull-board/api");
const { BullAdapter } = require("@bull-board/api/bullAdapter"); const { BullAdapter } = require("@bull-board/api/bullAdapter");
const { ExpressAdapter } = require("@bull-board/express"); const { ExpressAdapter } = require("@bull-board/express");
@ -48,6 +47,7 @@ const DEFAULT_PORT = process.env.PORT ?? 3002;
const HOST = process.env.HOST ?? "localhost"; const HOST = process.env.HOST ?? "localhost";
redisClient.connect(); redisClient.connect();
export function startServer(port = DEFAULT_PORT) { export function startServer(port = DEFAULT_PORT) {
const server = app.listen(Number(port), HOST, () => { const server = app.listen(Number(port), HOST, () => {
console.log(`Server listening on port ${port}`); console.log(`Server listening on port ${port}`);