0

Nick: tests

This commit is contained in:
Nicolas 2024-04-21 11:27:31 -07:00
parent 5cdbf3a0ac
commit 898d729a84
4 changed files with 334 additions and 171 deletions

View File

@ -1,189 +1,197 @@
import request from 'supertest';
import { app } from '../../index';
import dotenv from 'dotenv';
import request from "supertest";
import { app } from "../../index";
import dotenv from "dotenv";
dotenv.config();
// const TEST_URL = 'http://localhost:3002'
const TEST_URL = 'http://127.0.0.1:3002'
const TEST_URL = "http://127.0.0.1:3002";
describe('E2E Tests for API Routes', () => {
describe('GET /', () => {
it('should return Hello, world! message', async () => {
const response = await request(TEST_URL).get('/');
expect(response.statusCode).toBe(200);
expect(response.text).toContain('SCRAPERS-JS: Hello, world! Fly.io');
});
});
describe('GET /test', () => {
it('should return Hello, world! message', async () => {
const response = await request(TEST_URL).get('/test');
expect(response.statusCode).toBe(200);
expect(response.text).toContain('Hello, world!');
});
});
describe('POST /v0/scrape', () => {
it('should require authorization', async () => {
const response = await request(app).post('/v0/scrape');
expect(response.statusCode).toBe(401);
describe("E2E Tests for API Routes", () => {
beforeAll(() => {
process.env.USE_DB_AUTHENTICATION = "true";
});
it('should return an error response with an invalid API key', async () => {
const response = await request(TEST_URL)
.post('/v0/scrape')
.set('Authorization', `Bearer invalid-api-key`)
.set('Content-Type', 'application/json')
.send({ url: 'https://firecrawl.dev' });
expect(response.statusCode).toBe(401);
afterAll(() => {
delete process.env.USE_DB_AUTHENTICATION;
});
it('should return a successful response with a valid preview token', async () => {
const response = await request(TEST_URL)
.post('/v0/scrape')
.set('Authorization', `Bearer this_is_just_a_preview_token`)
.set('Content-Type', 'application/json')
.send({ url: 'https://firecrawl.dev' });
expect(response.statusCode).toBe(200);
}, 10000); // 10 seconds timeout
describe("GET /", () => {
it("should return Hello, world! message", async () => {
const response = await request(TEST_URL).get("/");
it('should return a successful response with a valid API key', async () => {
const response = await request(TEST_URL)
.post('/v0/scrape')
.set('Authorization', `Bearer ${process.env.TEST_API_KEY}`)
.set('Content-Type', 'application/json')
.send({ url: 'https://firecrawl.dev' });
expect(response.statusCode).toBe(200);
expect(response.body).toHaveProperty('data');
expect(response.body.data).toHaveProperty('content');
expect(response.body.data).toHaveProperty('markdown');
expect(response.body.data).toHaveProperty('metadata');
expect(response.body.data.content).toContain('🔥 FireCrawl');
}, 30000); // 30 seconds timeout
});
describe('POST /v0/crawl', () => {
it('should require authorization', async () => {
const response = await request(TEST_URL).post('/v0/crawl');
expect(response.statusCode).toBe(401);
});
it('should return an error response with an invalid API key', async () => {
const response = await request(TEST_URL)
.post('/v0/crawl')
.set('Authorization', `Bearer invalid-api-key`)
.set('Content-Type', 'application/json')
.send({ url: 'https://firecrawl.dev' });
expect(response.statusCode).toBe(401);
});
it('should return a successful response with a valid API key', async () => {
const response = await request(TEST_URL)
.post('/v0/crawl')
.set('Authorization', `Bearer ${process.env.TEST_API_KEY}`)
.set('Content-Type', 'application/json')
.send({ url: 'https://firecrawl.dev' });
expect(response.statusCode).toBe(200);
expect(response.body).toHaveProperty('jobId');
expect(response.body.jobId).toMatch(/^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$/);
});
// Additional tests for insufficient credits?
});
describe('POST /v0/crawlWebsitePreview', () => {
it('should require authorization', async () => {
const response = await request(TEST_URL).post('/v0/crawlWebsitePreview');
expect(response.statusCode).toBe(401);
});
it('should return an error response with an invalid API key', async () => {
const response = await request(TEST_URL)
.post('/v0/crawlWebsitePreview')
.set('Authorization', `Bearer invalid-api-key`)
.set('Content-Type', 'application/json')
.send({ url: 'https://firecrawl.dev' });
expect(response.statusCode).toBe(401);
});
it('should return a successful response with a valid API key', async () => {
const response = await request(TEST_URL)
.post('/v0/crawlWebsitePreview')
.set('Authorization', `Bearer this_is_just_a_preview_token`)
.set('Content-Type', 'application/json')
.send({ url: 'https://firecrawl.dev' });
expect(response.statusCode).toBe(200);
expect(response.body).toHaveProperty('jobId');
expect(response.body.jobId).toMatch(/^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$/);
});
});
describe('GET /v0/crawl/status/:jobId', () => {
it('should require authorization', async () => {
const response = await request(TEST_URL).get('/v0/crawl/status/123');
expect(response.statusCode).toBe(401);
});
it('should return an error response with an invalid API key', async () => {
const response = await request(TEST_URL)
.get('/v0/crawl/status/123')
.set('Authorization', `Bearer invalid-api-key`);
expect(response.statusCode).toBe(401);
});
it('should return Job not found for invalid job ID', async () => {
const response = await request(TEST_URL)
.get('/v0/crawl/status/invalidJobId')
.set('Authorization', `Bearer ${process.env.TEST_API_KEY}`);
expect(response.statusCode).toBe(404);
});
it('should return a successful response for a valid crawl job', async () => {
const crawlResponse = await request(TEST_URL)
.post('/v0/crawl')
.set('Authorization', `Bearer ${process.env.TEST_API_KEY}`)
.set('Content-Type', 'application/json')
.send({ url: 'https://firecrawl.dev' });
expect(crawlResponse.statusCode).toBe(200);
const response = await request(TEST_URL)
.get(`/v0/crawl/status/${crawlResponse.body.jobId}`)
.set('Authorization', `Bearer ${process.env.TEST_API_KEY}`);
expect(response.statusCode).toBe(200);
expect(response.body).toHaveProperty('status');
expect(response.body.status).toBe('active');
expect(response.text).toContain("SCRAPERS-JS: Hello, world! Fly.io");
});
});
describe("GET /test", () => {
it("should return Hello, world! message", async () => {
const response = await request(TEST_URL).get("/test");
expect(response.statusCode).toBe(200);
expect(response.text).toContain("Hello, world!");
});
});
describe("POST /v0/scrape", () => {
it("should require authorization", async () => {
const response = await request(app).post("/v0/scrape");
expect(response.statusCode).toBe(401);
});
it("should return an error response with an invalid API key", async () => {
const response = await request(TEST_URL)
.post("/v0/scrape")
.set("Authorization", `Bearer invalid-api-key`)
.set("Content-Type", "application/json")
.send({ url: "https://firecrawl.dev" });
expect(response.statusCode).toBe(401);
});
it("should return a successful response with a valid preview token", async () => {
const response = await request(TEST_URL)
.post("/v0/scrape")
.set("Authorization", `Bearer this_is_just_a_preview_token`)
.set("Content-Type", "application/json")
.send({ url: "https://firecrawl.dev" });
expect(response.statusCode).toBe(200);
}, 10000); // 10 seconds timeout
it("should return a successful response with a valid API key", async () => {
const response = await request(TEST_URL)
.post("/v0/scrape")
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({ url: "https://firecrawl.dev" });
expect(response.statusCode).toBe(200);
expect(response.body).toHaveProperty("data");
expect(response.body.data).toHaveProperty("content");
expect(response.body.data).toHaveProperty("markdown");
expect(response.body.data).toHaveProperty("metadata");
expect(response.body.data.content).toContain("🔥 FireCrawl");
}, 30000); // 30 seconds timeout
});
describe("POST /v0/crawl", () => {
it("should require authorization", async () => {
const response = await request(TEST_URL).post("/v0/crawl");
expect(response.statusCode).toBe(401);
});
it("should return an error response with an invalid API key", async () => {
const response = await request(TEST_URL)
.post("/v0/crawl")
.set("Authorization", `Bearer invalid-api-key`)
.set("Content-Type", "application/json")
.send({ url: "https://firecrawl.dev" });
expect(response.statusCode).toBe(401);
});
it("should return a successful response with a valid API key", async () => {
const response = await request(TEST_URL)
.post("/v0/crawl")
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({ url: "https://firecrawl.dev" });
expect(response.statusCode).toBe(200);
expect(response.body).toHaveProperty("jobId");
expect(response.body.jobId).toMatch(
/^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$/
);
});
// Additional tests for insufficient credits?
});
describe("POST /v0/crawlWebsitePreview", () => {
it("should require authorization", async () => {
const response = await request(TEST_URL).post(
"/v0/crawlWebsitePreview"
);
expect(response.statusCode).toBe(401);
});
it("should return an error response with an invalid API key", async () => {
const response = await request(TEST_URL)
.post("/v0/crawlWebsitePreview")
.set("Authorization", `Bearer invalid-api-key`)
.set("Content-Type", "application/json")
.send({ url: "https://firecrawl.dev" });
expect(response.statusCode).toBe(401);
});
it("should return a successful response with a valid API key", async () => {
const response = await request(TEST_URL)
.post("/v0/crawlWebsitePreview")
.set("Authorization", `Bearer this_is_just_a_preview_token`)
.set("Content-Type", "application/json")
.send({ url: "https://firecrawl.dev" });
expect(response.statusCode).toBe(200);
expect(response.body).toHaveProperty("jobId");
expect(response.body.jobId).toMatch(
/^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$/
);
});
});
describe("GET /v0/crawl/status/:jobId", () => {
it("should require authorization", async () => {
const response = await request(TEST_URL).get("/v0/crawl/status/123");
expect(response.statusCode).toBe(401);
});
it("should return an error response with an invalid API key", async () => {
const response = await request(TEST_URL)
.get("/v0/crawl/status/123")
.set("Authorization", `Bearer invalid-api-key`);
expect(response.statusCode).toBe(401);
});
it("should return Job not found for invalid job ID", async () => {
const response = await request(TEST_URL)
.get("/v0/crawl/status/invalidJobId")
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);
expect(response.statusCode).toBe(404);
});
it("should return a successful response for a valid crawl job", async () => {
const crawlResponse = await request(TEST_URL)
.post("/v0/crawl")
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({ url: "https://firecrawl.dev" });
expect(crawlResponse.statusCode).toBe(200);
const response = await request(TEST_URL)
.get(`/v0/crawl/status/${crawlResponse.body.jobId}`)
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);
expect(response.statusCode).toBe(200);
expect(response.body).toHaveProperty("status");
expect(response.body.status).toBe("active");
// wait for 30 seconds
await new Promise((r) => setTimeout(r, 30000));
const completedResponse = await request(TEST_URL)
.get(`/v0/crawl/status/${crawlResponse.body.jobId}`)
.set('Authorization', `Bearer ${process.env.TEST_API_KEY}`);
.get(`/v0/crawl/status/${crawlResponse.body.jobId}`)
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);
expect(completedResponse.statusCode).toBe(200);
expect(completedResponse.body).toHaveProperty('status');
expect(completedResponse.body.status).toBe('completed');
expect(completedResponse.body).toHaveProperty('data');
expect(completedResponse.body.data[0]).toHaveProperty('content');
expect(completedResponse.body.data[0]).toHaveProperty('markdown');
expect(completedResponse.body.data[0]).toHaveProperty('metadata');
expect(completedResponse.body.data[0].content).toContain('🔥 FireCrawl');
}, 60000); // 60 seconds
});
expect(completedResponse.body).toHaveProperty("status");
expect(completedResponse.body.status).toBe("completed");
expect(completedResponse.body).toHaveProperty("data");
expect(completedResponse.body.data[0]).toHaveProperty("content");
expect(completedResponse.body.data[0]).toHaveProperty("markdown");
expect(completedResponse.body.data[0]).toHaveProperty("metadata");
expect(completedResponse.body.data[0].content).toContain(
"🔥 FireCrawl"
);
}, 60000); // 60 seconds
});
describe('GET /is-production', () => {
it('should return the production status', async () => {
const response = await request(TEST_URL).get('/is-production');
expect(response.statusCode).toBe(200);
expect(response.body).toHaveProperty('isProduction');
describe("GET /is-production", () => {
it("should return the production status", async () => {
const response = await request(TEST_URL).get("/is-production");
expect(response.statusCode).toBe(200);
expect(response.body).toHaveProperty("isProduction");
});
});
});
});

View File

@ -0,0 +1,156 @@
import request from "supertest";
import { app } from "../../index";
import dotenv from "dotenv";
const fs = require("fs");
const path = require("path");
dotenv.config();
const TEST_URL = "http://127.0.0.1:3002";
describe("E2E Tests for API Routes with No Authentication", () => {
let originalEnv: NodeJS.ProcessEnv;
// save original process.env
beforeAll(() => {
originalEnv = { ...process.env };
process.env.USE_DB_AUTHENTICATION = "false";
process.env.SUPABASE_ANON_TOKEN = "";
process.env.SUPABASE_URL = "";
process.env.SUPABASE_SERVICE_TOKEN = "";
process.env.SCRAPING_BEE_API_KEY = "";
process.env.OPENAI_API_KEY = "";
process.env.BULL_AUTH_KEY = "";
process.env.LOGTAIL_KEY = "";
process.env.PLAYWRIGHT_MICROSERVICE_URL = "";
process.env.LLAMAPARSE_API_KEY = "";
process.env.TEST_API_KEY = "";
});
// restore original process.env
afterAll(() => {
process.env = originalEnv;
});
describe("GET /", () => {
it("should return Hello, world! message", async () => {
const response = await request(TEST_URL).get("/");
expect(response.statusCode).toBe(200);
expect(response.text).toContain("SCRAPERS-JS: Hello, world! Fly.io");
});
});
describe("GET /test", () => {
it("should return Hello, world! message", async () => {
const response = await request(TEST_URL).get("/test");
expect(response.statusCode).toBe(200);
expect(response.text).toContain("Hello, world!");
});
});
describe("POST /v0/scrape", () => {
it("should not require authorization", async () => {
const response = await request(TEST_URL).post("/v0/scrape");
expect(response.statusCode).not.toBe(401);
});
it("should return a successful response", async () => {
const response = await request(TEST_URL)
.post("/v0/scrape")
.set("Content-Type", "application/json")
.send({ url: "https://firecrawl.dev" });
expect(response.statusCode).toBe(200);
}, 10000); // 10 seconds timeout
});
describe("POST /v0/crawl", () => {
it("should not require authorization", async () => {
const response = await request(TEST_URL).post("/v0/crawl");
expect(response.statusCode).not.toBe(401);
});
it("should return a successful response", async () => {
const response = await request(TEST_URL)
.post("/v0/crawl")
.set("Content-Type", "application/json")
.send({ url: "https://firecrawl.dev" });
expect(response.statusCode).toBe(200);
expect(response.body).toHaveProperty("jobId");
expect(response.body.jobId).toMatch(
/^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$/
);
});
});
describe("POST /v0/crawlWebsitePreview", () => {
it("should not require authorization", async () => {
const response = await request(TEST_URL).post("/v0/crawlWebsitePreview");
expect(response.statusCode).not.toBe(401);
});
it("should return a successful response", async () => {
const response = await request(TEST_URL)
.post("/v0/crawlWebsitePreview")
.set("Content-Type", "application/json")
.send({ url: "https://firecrawl.dev" });
expect(response.statusCode).toBe(200);
expect(response.body).toHaveProperty("jobId");
expect(response.body.jobId).toMatch(
/^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$/
);
});
});
describe("GET /v0/crawl/status/:jobId", () => {
it("should not require authorization", async () => {
const response = await request(TEST_URL).get("/v0/crawl/status/123");
expect(response.statusCode).not.toBe(401);
});
it("should return Job not found for invalid job ID", async () => {
const response = await request(TEST_URL).get(
"/v0/crawl/status/invalidJobId"
);
expect(response.statusCode).toBe(404);
});
it("should return a successful response for a valid crawl job", async () => {
const crawlResponse = await request(TEST_URL)
.post("/v0/crawl")
.set("Content-Type", "application/json")
.send({ url: "https://firecrawl.dev" });
expect(crawlResponse.statusCode).toBe(200);
const response = await request(TEST_URL).get(
`/v0/crawl/status/${crawlResponse.body.jobId}`
);
expect(response.statusCode).toBe(200);
expect(response.body).toHaveProperty("status");
expect(response.body.status).toBe("active");
// wait for 30 seconds
await new Promise((r) => setTimeout(r, 30000));
const completedResponse = await request(TEST_URL).get(
`/v0/crawl/status/${crawlResponse.body.jobId}`
);
expect(completedResponse.statusCode).toBe(200);
expect(completedResponse.body).toHaveProperty("status");
expect(completedResponse.body.status).toBe("completed");
expect(completedResponse.body).toHaveProperty("data");
expect(completedResponse.body.data[0]).toHaveProperty("content");
expect(completedResponse.body.data[0]).toHaveProperty("markdown");
expect(completedResponse.body.data[0]).toHaveProperty("metadata");
expect(completedResponse.body.data[0].content).toContain("🔥 FireCrawl");
}, 60000); // 60 seconds
});
describe("GET /is-production", () => {
it("should return the production status", async () => {
const response = await request(TEST_URL).get("/is-production");
expect(response.statusCode).toBe(200);
expect(response.body).toHaveProperty("isProduction");
});
});
});

View File

@ -8,7 +8,6 @@ import { addWebScraperJob } from "../../src/services/queue-jobs";
export async function crawlController(req: Request, res: Response) {
try {
console.log("hello");
const { success, team_id, error, status } = await authenticateUser(
req,
res,

View File

@ -5,7 +5,6 @@ import "dotenv/config";
import { getWebScraperQueue } from "./services/queue-service";
import { redisClient } from "./services/rate-limiter";
import { v0Router } from "./routes/v0";
const { createBullBoard } = require("@bull-board/api");
const { BullAdapter } = require("@bull-board/api/bullAdapter");
const { ExpressAdapter } = require("@bull-board/express");
@ -48,6 +47,7 @@ const DEFAULT_PORT = process.env.PORT ?? 3002;
const HOST = process.env.HOST ?? "localhost";
redisClient.connect();
export function startServer(port = DEFAULT_PORT) {
const server = app.listen(Number(port), HOST, () => {
console.log(`Server listening on port ${port}`);