github/v-firecrawl
github/v-firecrawl
0
Fork 0
Code Actions

Merge branch 'mendableai:main' into main

Browse Source
This commit is contained in:
Romain Bruyère 2024-06-03 21:29:33 +02:00 committed by GitHub
commit 4987f901d1
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 8 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
apps/api/src/__tests__/e2e_withAuth/index.test.ts
View File

@ -68,7 +68,7 @@ describe("E2E Tests for API Routes", () => {
.set("Content-Type", "application/json") .set("Content-Type", "application/json")
.send({ url: "https://firecrawl.dev" }); .send({ url: "https://firecrawl.dev" });
expect(response.statusCode).toBe(200); expect(response.statusCode).toBe(200);
}, 10000); // 10 seconds timeout }, 30000); // 30 seconds timeout
it("should return a successful response with a valid API key", async () => { it("should return a successful response with a valid API key", async () => {
const response = await request(TEST_URL) const response = await request(TEST_URL)

4
apps/api/src/scraper/WebScraper/single_url.ts
View File

@ -282,8 +282,8 @@ export async function scrapSingleUrl(
// Check for custom scraping conditions // Check for custom scraping conditions
const customScrapedContent = await handleCustomScraping(text, url); const customScrapedContent = await handleCustomScraping(text, url);
if (customScrapedContent) { if (customScrapedContent) {
text = customScrapedContent[0]; text = customScrapedContent.html;
screenshot = customScrapedContent[1]; screenshot = customScrapedContent.screenshot;
} }
//* TODO: add an optional to return markdown or structured/extracted content //* TODO: add an optional to return markdown or structured/extracted content

10
apps/js-sdk/firecrawl/src/__tests__/e2e_withAuth/index.test.ts
View File

@ -30,7 +30,7 @@ describe('FirecrawlApp E2E Tests', () => {
const response = await app.scrapeUrl('https://firecrawl.dev'); const response = await app.scrapeUrl('https://firecrawl.dev');
expect(response).not.toBeNull(); expect(response).not.toBeNull();
expect(response.data.content).toContain("🔥 Firecrawl"); expect(response.data.content).toContain("🔥 Firecrawl");
}, 10000); // 10 seconds timeout }, 30000); // 30 seconds timeout
test('should return successful response for valid scrape', async () => { test('should return successful response for valid scrape', async () => {
const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL }); const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
@ -40,7 +40,7 @@ describe('FirecrawlApp E2E Tests', () => {
expect(response.data).toHaveProperty('markdown'); expect(response.data).toHaveProperty('markdown');
expect(response.data).toHaveProperty('metadata'); expect(response.data).toHaveProperty('metadata');
expect(response.data).not.toHaveProperty('html'); expect(response.data).not.toHaveProperty('html');
}, 10000); // 10 seconds timeout }, 30000); // 30 seconds timeout
test('should return successful response with valid API key and include HTML', async () => { test('should return successful response with valid API key and include HTML', async () => {
const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL }); const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
@ -49,7 +49,7 @@ describe('FirecrawlApp E2E Tests', () => {
expect(response.data.content).toContain("🔥 Firecrawl"); expect(response.data.content).toContain("🔥 Firecrawl");
expect(response.data.markdown).toContain("🔥 Firecrawl"); expect(response.data.markdown).toContain("🔥 Firecrawl");
expect(response.data.html).toContain("<h1"); expect(response.data.html).toContain("<h1");
}, 10000); // 10 seconds timeout }, 30000); // 30 seconds timeout
test('should return successful response for valid scrape with PDF file', async () => { test('should return successful response for valid scrape with PDF file', async () => {
const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL }); const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
@ -99,12 +99,12 @@ describe('FirecrawlApp E2E Tests', () => {
expect(response).not.toBeNull(); expect(response).not.toBeNull();
expect(response.jobId).toBeDefined(); expect(response.jobId).toBeDefined();
await new Promise(resolve => setTimeout(resolve, 10000)); // wait for 10 seconds await new Promise(resolve => setTimeout(resolve, 30000)); // wait for 30 seconds
const statusResponse = await app.checkCrawlStatus(response.jobId); const statusResponse = await app.checkCrawlStatus(response.jobId);
expect(statusResponse).not.toBeNull(); expect(statusResponse).not.toBeNull();
expect(statusResponse.status).toBe('completed'); expect(statusResponse.status).toBe('completed');
expect(statusResponse.data.length).toBeGreaterThan(0); expect(statusResponse.data.length).toBeGreaterThan(0);
}, 30000); // 30 seconds timeout }, 35000); // 35 seconds timeout
test('should return successful response for search', async () => { test('should return successful response for search', async () => {
const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL }); const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });