From 1fc3a1514976496388a05b6e97072c7682feeb71 Mon Sep 17 00:00:00 2001 From: rafaelsideguide <150964962+rafaelsideguide@users.noreply.github.com> Date: Mon, 3 Jun 2024 15:24:40 -0300 Subject: [PATCH 1/3] Update single_url.ts --- apps/api/src/scraper/WebScraper/single_url.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/api/src/scraper/WebScraper/single_url.ts b/apps/api/src/scraper/WebScraper/single_url.ts index c7f9469..567a329 100644 --- a/apps/api/src/scraper/WebScraper/single_url.ts +++ b/apps/api/src/scraper/WebScraper/single_url.ts @@ -282,8 +282,8 @@ export async function scrapSingleUrl( // Check for custom scraping conditions const customScrapedContent = await handleCustomScraping(text, url); if (customScrapedContent) { - text = customScrapedContent[0]; - screenshot = customScrapedContent[1]; + text = customScrapedContent.html; + screenshot = customScrapedContent.screenshot; } //* TODO: add an optional to return markdown or structured/extracted content From c1aed1360e4ca8f4ba73a3c156983fcbcfccf72a Mon Sep 17 00:00:00 2001 From: rafaelsideguide <150964962+rafaelsideguide@users.noreply.github.com> Date: Mon, 3 Jun 2024 15:51:07 -0300 Subject: [PATCH 2/3] Update index.test.ts --- apps/api/src/__tests__/e2e_withAuth/index.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/api/src/__tests__/e2e_withAuth/index.test.ts b/apps/api/src/__tests__/e2e_withAuth/index.test.ts index 8346839..e81d18e 100644 --- a/apps/api/src/__tests__/e2e_withAuth/index.test.ts +++ b/apps/api/src/__tests__/e2e_withAuth/index.test.ts @@ -68,7 +68,7 @@ describe("E2E Tests for API Routes", () => { .set("Content-Type", "application/json") .send({ url: "https://firecrawl.dev" }); expect(response.statusCode).toBe(200); - }, 10000); // 10 seconds timeout + }, 30000); // 30 seconds timeout it("should return a successful response with a valid API key", async () => { const response = await request(TEST_URL) From 4100cc92230b221d6af2e3e428a5421522656884 Mon Sep 17 00:00:00 2001 From: rafaelsideguide <150964962+rafaelsideguide@users.noreply.github.com> Date: Mon, 3 Jun 2024 16:29:16 -0300 Subject: [PATCH 3/3] Update index.test.ts --- .../firecrawl/src/__tests__/e2e_withAuth/index.test.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/apps/js-sdk/firecrawl/src/__tests__/e2e_withAuth/index.test.ts b/apps/js-sdk/firecrawl/src/__tests__/e2e_withAuth/index.test.ts index c9db6a9..10da000 100644 --- a/apps/js-sdk/firecrawl/src/__tests__/e2e_withAuth/index.test.ts +++ b/apps/js-sdk/firecrawl/src/__tests__/e2e_withAuth/index.test.ts @@ -30,7 +30,7 @@ describe('FirecrawlApp E2E Tests', () => { const response = await app.scrapeUrl('https://firecrawl.dev'); expect(response).not.toBeNull(); expect(response.data.content).toContain("🔥 Firecrawl"); - }, 10000); // 10 seconds timeout + }, 30000); // 30 seconds timeout test('should return successful response for valid scrape', async () => { const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL }); @@ -40,7 +40,7 @@ describe('FirecrawlApp E2E Tests', () => { expect(response.data).toHaveProperty('markdown'); expect(response.data).toHaveProperty('metadata'); expect(response.data).not.toHaveProperty('html'); - }, 10000); // 10 seconds timeout + }, 30000); // 30 seconds timeout test('should return successful response with valid API key and include HTML', async () => { const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL }); @@ -49,7 +49,7 @@ describe('FirecrawlApp E2E Tests', () => { expect(response.data.content).toContain("🔥 Firecrawl"); expect(response.data.markdown).toContain("🔥 Firecrawl"); expect(response.data.html).toContain(" { const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL }); @@ -99,12 +99,12 @@ describe('FirecrawlApp E2E Tests', () => { expect(response).not.toBeNull(); expect(response.jobId).toBeDefined(); - await new Promise(resolve => setTimeout(resolve, 10000)); // wait for 10 seconds + await new Promise(resolve => setTimeout(resolve, 30000)); // wait for 30 seconds const statusResponse = await app.checkCrawlStatus(response.jobId); expect(statusResponse).not.toBeNull(); expect(statusResponse.status).toBe('completed'); expect(statusResponse.data.length).toBeGreaterThan(0); - }, 30000); // 30 seconds timeout + }, 35000); // 35 seconds timeout test('should return successful response for search', async () => { const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });