From d7c797d0f3ff48a3066cf26d59f88a4ddb5f0ed0 Mon Sep 17 00:00:00 2001 From: rafaelsideguide <150964962+rafaelsideguide@users.noreply.github.com> Date: Thu, 18 Apr 2024 16:49:31 -0300 Subject: [PATCH 01/13] adding env secrets --- .github/workflows/ci.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 258542e..69699ed 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,4 +36,9 @@ jobs: run: npx jest -- src/__tests__/e2e/index.test.ts working-directory: ./apps/api env: - REDIS_URL: redis://localhost:6379 \ No newline at end of file + REDIS_URL: redis://localhost:6379 + FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }} + SUPABASE_URL: ${{ secrets.SUPABASE_URL }} + SUPABASE_SERVICE_TOKEN: ${{ secrets.SUPABASE_SERVICE_TOKEN }} + TEST_API_KEY: ${{ secrets.TEST_API_KEY }} + From dcccfab4cdd4a06a8d27523f7063eb33e80c52af Mon Sep 17 00:00:00 2001 From: rafaelsideguide <150964962+rafaelsideguide@users.noreply.github.com> Date: Thu, 18 Apr 2024 16:55:49 -0300 Subject: [PATCH 02/13] adding other env secrets --- .github/workflows/ci.yml | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 69699ed..731b30d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,9 +36,21 @@ jobs: run: npx jest -- src/__tests__/e2e/index.test.ts working-directory: ./apps/api env: - REDIS_URL: redis://localhost:6379 + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + BULL_AUTH_KEY: ${{ secrets.BULL_AUTH_KEY }} FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }} - SUPABASE_URL: ${{ secrets.SUPABASE_URL }} + HOST: ${{ secrets.HOST }} + LLAMAPARSE_API_KEY: ${{ secrets.LLAMAPARSE_API_KEY }} + LOGTAIL_KEY: ${{ secrets.LOGTAIL_KEY }} + NUM_WORKERS_PER_QUEUE: ${{ secrets.NUM_WORKERS_PER_QUEUE }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + PLAYWRIGHT_MICROSERVICE_URL: ${{ secrets.PLAYWRIGHT_MICROSERVICE_URL }} + PORT: ${{ secrets.PORT }} + REDIS_URL: ${{ secrets.REDIS_URL }} + SCRAPING_BEE_API_KEY: ${{ secrets.SCRAPING_BEE_API_KEY }} + SUPABASE_ANON_TOKEN: ${{ secrets.SUPABASE_ANON_TOKEN }} SUPABASE_SERVICE_TOKEN: ${{ secrets.SUPABASE_SERVICE_TOKEN }} + SUPABASE_URL: ${{ secrets.SUPABASE_URL }} TEST_API_KEY: ${{ secrets.TEST_API_KEY }} + From 2ca00d81866063aef302fa0960d68889c4aee9c6 Mon Sep 17 00:00:00 2001 From: rafaelsideguide <150964962+rafaelsideguide@users.noreply.github.com> Date: Thu, 18 Apr 2024 17:03:52 -0300 Subject: [PATCH 03/13] adding openHandlesTimeout --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 731b30d..6017a54 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -33,7 +33,7 @@ jobs: run: npm run workers & working-directory: ./apps/api - name: Run E2E tests - run: npx jest -- src/__tests__/e2e/index.test.ts + run: npx jest --openHandlesTimeout=60000 -- src/__tests__/e2e/index.test.ts working-directory: ./apps/api env: ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} From dae024adc17386118a5a9e7d93b9e5fe244af9c9 Mon Sep 17 00:00:00 2001 From: rafaelsideguide <150964962+rafaelsideguide@users.noreply.github.com> Date: Thu, 18 Apr 2024 17:08:48 -0300 Subject: [PATCH 04/13] fixing env position --- .github/workflows/ci.yml | 36 +++++++++++++++++++----------------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6017a54..86ccc12 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,6 +6,24 @@ on: # schedule: # - cron: '0 */4 * * *' +env: + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + BULL_AUTH_KEY: ${{ secrets.BULL_AUTH_KEY }} + FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }} + HOST: ${{ secrets.HOST }} + LLAMAPARSE_API_KEY: ${{ secrets.LLAMAPARSE_API_KEY }} + LOGTAIL_KEY: ${{ secrets.LOGTAIL_KEY }} + NUM_WORKERS_PER_QUEUE: ${{ secrets.NUM_WORKERS_PER_QUEUE }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + PLAYWRIGHT_MICROSERVICE_URL: ${{ secrets.PLAYWRIGHT_MICROSERVICE_URL }} + PORT: ${{ secrets.PORT }} + REDIS_URL: ${{ secrets.REDIS_URL }} + SCRAPING_BEE_API_KEY: ${{ secrets.SCRAPING_BEE_API_KEY }} + SUPABASE_ANON_TOKEN: ${{ secrets.SUPABASE_ANON_TOKEN }} + SUPABASE_SERVICE_TOKEN: ${{ secrets.SUPABASE_SERVICE_TOKEN }} + SUPABASE_URL: ${{ secrets.SUPABASE_URL }} + TEST_API_KEY: ${{ secrets.TEST_API_KEY }} + jobs: pre-deploy: name: Pre-deploy checks @@ -35,22 +53,6 @@ jobs: - name: Run E2E tests run: npx jest --openHandlesTimeout=60000 -- src/__tests__/e2e/index.test.ts working-directory: ./apps/api - env: - ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} - BULL_AUTH_KEY: ${{ secrets.BULL_AUTH_KEY }} - FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }} - HOST: ${{ secrets.HOST }} - LLAMAPARSE_API_KEY: ${{ secrets.LLAMAPARSE_API_KEY }} - LOGTAIL_KEY: ${{ secrets.LOGTAIL_KEY }} - NUM_WORKERS_PER_QUEUE: ${{ secrets.NUM_WORKERS_PER_QUEUE }} - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - PLAYWRIGHT_MICROSERVICE_URL: ${{ secrets.PLAYWRIGHT_MICROSERVICE_URL }} - PORT: ${{ secrets.PORT }} - REDIS_URL: ${{ secrets.REDIS_URL }} - SCRAPING_BEE_API_KEY: ${{ secrets.SCRAPING_BEE_API_KEY }} - SUPABASE_ANON_TOKEN: ${{ secrets.SUPABASE_ANON_TOKEN }} - SUPABASE_SERVICE_TOKEN: ${{ secrets.SUPABASE_SERVICE_TOKEN }} - SUPABASE_URL: ${{ secrets.SUPABASE_URL }} - TEST_API_KEY: ${{ secrets.TEST_API_KEY }} + From efbb4e8905782dbe347763750091d72215ac239c Mon Sep 17 00:00:00 2001 From: rafaelsideguide <150964962+rafaelsideguide@users.noreply.github.com> Date: Thu, 18 Apr 2024 17:18:15 -0300 Subject: [PATCH 05/13] fixing jest parameters --- .github/workflows/ci.yml | 2 +- apps/api/src/__tests__/e2e/index.test.ts | 23 ++++++++++++----------- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 86ccc12..8c85782 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -51,7 +51,7 @@ jobs: run: npm run workers & working-directory: ./apps/api - name: Run E2E tests - run: npx jest --openHandlesTimeout=60000 -- src/__tests__/e2e/index.test.ts + run: npx jest --openHandlesTimeout=60000 --watchAll=false -- src/__tests__/e2e/index.test.ts working-directory: ./apps/api diff --git a/apps/api/src/__tests__/e2e/index.test.ts b/apps/api/src/__tests__/e2e/index.test.ts index ce11207..3bc3e99 100644 --- a/apps/api/src/__tests__/e2e/index.test.ts +++ b/apps/api/src/__tests__/e2e/index.test.ts @@ -144,19 +144,20 @@ describe('E2E Tests for API Routes', () => { expect(response.body).toHaveProperty('status'); expect(response.body.status).toBe('active'); - setTimeout(async () => { - const response = await request(TEST_URL) + // wait for 30 seconds + await new Promise((r) => setTimeout(r, 30000)); + + const completedResponse = await request(TEST_URL) .get(`/v0/crawl/status/${crawlResponse.body.jobId}`) .set('Authorization', `Bearer ${process.env.TEST_API_KEY}`); - expect(response.statusCode).toBe(200); - expect(response.body).toHaveProperty('status'); - expect(response.body.status).toBe('completed'); - expect(response.body).toHaveProperty('data'); - expect(response.body.data).toHaveProperty('content'); - expect(response.body.data).toHaveProperty('markdown'); - expect(response.body.data).toHaveProperty('metadata'); - expect(response.body.data.content).toContain('🔥 FireCrawl'); - }, 30000); // 30 seconds + expect(completedResponse.statusCode).toBe(200); + expect(completedResponse.body).toHaveProperty('status'); + expect(completedResponse.body.status).toBe('completed'); + expect(completedResponse.body).toHaveProperty('data'); + expect(completedResponse.body.data).toHaveProperty('content'); + expect(completedResponse.body.data).toHaveProperty('markdown'); + expect(completedResponse.body.data).toHaveProperty('metadata'); + expect(completedResponse.body.data.content).toContain('🔥 FireCrawl'); }, 60000); // 60 seconds }); From 3f833737f3cafe502bd394fde524aa8ac0bfd77c Mon Sep 17 00:00:00 2001 From: rafaelsideguide <150964962+rafaelsideguide@users.noreply.github.com> Date: Thu, 18 Apr 2024 17:25:25 -0300 Subject: [PATCH 06/13] fixing test --- apps/api/src/__tests__/e2e/index.test.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/apps/api/src/__tests__/e2e/index.test.ts b/apps/api/src/__tests__/e2e/index.test.ts index 3bc3e99..608d887 100644 --- a/apps/api/src/__tests__/e2e/index.test.ts +++ b/apps/api/src/__tests__/e2e/index.test.ts @@ -154,10 +154,10 @@ describe('E2E Tests for API Routes', () => { expect(completedResponse.body).toHaveProperty('status'); expect(completedResponse.body.status).toBe('completed'); expect(completedResponse.body).toHaveProperty('data'); - expect(completedResponse.body.data).toHaveProperty('content'); - expect(completedResponse.body.data).toHaveProperty('markdown'); - expect(completedResponse.body.data).toHaveProperty('metadata'); - expect(completedResponse.body.data.content).toContain('🔥 FireCrawl'); + expect(completedResponse.body.data[0]).toHaveProperty('content'); + expect(completedResponse.body.data[0]).toHaveProperty('markdown'); + expect(completedResponse.body.data[0]).toHaveProperty('metadata'); + expect(completedResponse.body.data[0].content).toContain('🔥 FireCrawl'); }, 60000); // 60 seconds }); From ca8c8b87dcf242da776af24ded2762a1ae12b8d6 Mon Sep 17 00:00:00 2001 From: rafaelsideguide <150964962+rafaelsideguide@users.noreply.github.com> Date: Thu, 18 Apr 2024 17:29:06 -0300 Subject: [PATCH 07/13] fixing workflow --- .github/workflows/ci.yml | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8c85782..042d0c3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -47,12 +47,17 @@ jobs: - name: Start the application run: npm start & working-directory: ./apps/api + id: start_app - name: Start workers run: npm run workers & working-directory: ./apps/api + id: start_workers - name: Run E2E tests - run: npx jest --openHandlesTimeout=60000 --watchAll=false -- src/__tests__/e2e/index.test.ts + run: | + npx jest --openHandlesTimeout=60000 --watchAll=false -- src/__tests__/e2e/index.test.ts working-directory: ./apps/api - - - + - name: Kill background processes + if: always() + run: | + kill $(jobs -p) + working-directory: ./apps/api \ No newline at end of file From 4f1179db99daf45a1f7d25e92beb735b99eb3b9d Mon Sep 17 00:00:00 2001 From: rafaelsideguide <150964962+rafaelsideguide@users.noreply.github.com> Date: Thu, 18 Apr 2024 17:32:28 -0300 Subject: [PATCH 08/13] fixing workflow --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 042d0c3..eafdf6f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -54,7 +54,7 @@ jobs: id: start_workers - name: Run E2E tests run: | - npx jest --openHandlesTimeout=60000 --watchAll=false -- src/__tests__/e2e/index.test.ts + npx jest --detectOpenHandles --forceExit --openHandlesTimeout=60000 --watchAll=false -- src/__tests__/e2e/index.test.ts working-directory: ./apps/api - name: Kill background processes if: always() From 4018d7ca17ec13642d196954dbba006a9cc28d0e Mon Sep 17 00:00:00 2001 From: rafaelsideguide <150964962+rafaelsideguide@users.noreply.github.com> Date: Thu, 18 Apr 2024 17:35:00 -0300 Subject: [PATCH 09/13] fixing workflow --- .github/workflows/ci.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index eafdf6f..d039085 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -55,9 +55,4 @@ jobs: - name: Run E2E tests run: | npx jest --detectOpenHandles --forceExit --openHandlesTimeout=60000 --watchAll=false -- src/__tests__/e2e/index.test.ts - working-directory: ./apps/api - - name: Kill background processes - if: always() - run: | - kill $(jobs -p) working-directory: ./apps/api \ No newline at end of file From dab0568c435e145cae69c47127477ed2e3478f2c Mon Sep 17 00:00:00 2001 From: rafaelsideguide <150964962+rafaelsideguide@users.noreply.github.com> Date: Thu, 18 Apr 2024 17:38:12 -0300 Subject: [PATCH 10/13] testing tests --- apps/api/src/__tests__/e2e/index.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/api/src/__tests__/e2e/index.test.ts b/apps/api/src/__tests__/e2e/index.test.ts index 608d887..b2af6f0 100644 --- a/apps/api/src/__tests__/e2e/index.test.ts +++ b/apps/api/src/__tests__/e2e/index.test.ts @@ -74,7 +74,7 @@ describe('E2E Tests for API Routes', () => { .set('Content-Type', 'application/json') .send({ url: 'https://firecrawl.dev' }); expect(response.statusCode).toBe(200); - expect(response.body).toHaveProperty('jobId'); + expect(response.body).toHaveProperty('asudhaopsidjp'); expect(response.body.jobId).toMatch(/^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$/); }); From c627d2217921e1e75ba992f2c8b82f03334ed35f Mon Sep 17 00:00:00 2001 From: rafaelsideguide <150964962+rafaelsideguide@users.noreply.github.com> Date: Thu, 18 Apr 2024 17:41:23 -0300 Subject: [PATCH 11/13] all working now --- apps/api/src/__tests__/e2e/index.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/api/src/__tests__/e2e/index.test.ts b/apps/api/src/__tests__/e2e/index.test.ts index b2af6f0..608d887 100644 --- a/apps/api/src/__tests__/e2e/index.test.ts +++ b/apps/api/src/__tests__/e2e/index.test.ts @@ -74,7 +74,7 @@ describe('E2E Tests for API Routes', () => { .set('Content-Type', 'application/json') .send({ url: 'https://firecrawl.dev' }); expect(response.statusCode).toBe(200); - expect(response.body).toHaveProperty('asudhaopsidjp'); + expect(response.body).toHaveProperty('jobId'); expect(response.body.jobId).toMatch(/^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$/); }); From 1c8ffc9faa5c9f612070cc9ac0b4e7d2dffe0f34 Mon Sep 17 00:00:00 2001 From: rafaelsideguide <150964962+rafaelsideguide@users.noreply.github.com> Date: Thu, 18 Apr 2024 17:44:33 -0300 Subject: [PATCH 12/13] fixing deploy workflow --- .github/workflows/fly.yml | 50 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/.github/workflows/fly.yml b/.github/workflows/fly.yml index df29d1b..3bb215b 100644 --- a/.github/workflows/fly.yml +++ b/.github/workflows/fly.yml @@ -6,10 +6,60 @@ on: # schedule: # - cron: '0 */4 * * *' +env: + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + BULL_AUTH_KEY: ${{ secrets.BULL_AUTH_KEY }} + FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }} + HOST: ${{ secrets.HOST }} + LLAMAPARSE_API_KEY: ${{ secrets.LLAMAPARSE_API_KEY }} + LOGTAIL_KEY: ${{ secrets.LOGTAIL_KEY }} + NUM_WORKERS_PER_QUEUE: ${{ secrets.NUM_WORKERS_PER_QUEUE }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + PLAYWRIGHT_MICROSERVICE_URL: ${{ secrets.PLAYWRIGHT_MICROSERVICE_URL }} + PORT: ${{ secrets.PORT }} + REDIS_URL: ${{ secrets.REDIS_URL }} + SCRAPING_BEE_API_KEY: ${{ secrets.SCRAPING_BEE_API_KEY }} + SUPABASE_ANON_TOKEN: ${{ secrets.SUPABASE_ANON_TOKEN }} + SUPABASE_SERVICE_TOKEN: ${{ secrets.SUPABASE_SERVICE_TOKEN }} + SUPABASE_URL: ${{ secrets.SUPABASE_URL }} + TEST_API_KEY: ${{ secrets.TEST_API_KEY }} + jobs: + pre-deploy: + name: Pre-deploy checks + runs-on: ubuntu-latest + services: + redis: + image: redis + ports: + - 6379:6379 + steps: + - uses: actions/checkout@v3 + - name: Set up Node.js + uses: actions/setup-node@v3 + with: + node-version: '20' + - name: Install pnpm + run: npm install -g pnpm + - name: Install dependencies + run: pnpm install + working-directory: ./apps/api + - name: Start the application + run: npm start & + working-directory: ./apps/api + id: start_app + - name: Start workers + run: npm run workers & + working-directory: ./apps/api + id: start_workers + - name: Run E2E tests + run: | + npx jest --detectOpenHandles --forceExit --openHandlesTimeout=60000 --watchAll=false -- src/__tests__/e2e/index.test.ts + working-directory: ./apps/api deploy: name: Deploy app runs-on: ubuntu-latest + needs: pre-deploy steps: - uses: actions/checkout@v3 - name: Change directory From 70497804268d4624f974072f2f3a2370ef6cb902 Mon Sep 17 00:00:00 2001 From: rafaelsideguide <150964962+rafaelsideguide@users.noreply.github.com> Date: Thu, 18 Apr 2024 17:47:15 -0300 Subject: [PATCH 13/13] adding all tests --- .github/workflows/ci.yml | 2 +- .github/workflows/fly.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d039085..b9a5b79 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -54,5 +54,5 @@ jobs: id: start_workers - name: Run E2E tests run: | - npx jest --detectOpenHandles --forceExit --openHandlesTimeout=60000 --watchAll=false -- src/__tests__/e2e/index.test.ts + npx jest --detectOpenHandles --forceExit --openHandlesTimeout=120000 --watchAll=false working-directory: ./apps/api \ No newline at end of file diff --git a/.github/workflows/fly.yml b/.github/workflows/fly.yml index 3bb215b..fe042c6 100644 --- a/.github/workflows/fly.yml +++ b/.github/workflows/fly.yml @@ -54,7 +54,7 @@ jobs: id: start_workers - name: Run E2E tests run: | - npx jest --detectOpenHandles --forceExit --openHandlesTimeout=60000 --watchAll=false -- src/__tests__/e2e/index.test.ts + npx jest --detectOpenHandles --forceExit --openHandlesTimeout=120000 --watchAll=false working-directory: ./apps/api deploy: name: Deploy app