Merge branch 'main' of https://github.com/mendableai/firecrawl

.github/workflows/ci.yml

@@ -6,6 +6,24 @@ on:
   # schedule:
   #   - cron: '0 */4 * * *'
 
+env:
+  ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+  BULL_AUTH_KEY: ${{ secrets.BULL_AUTH_KEY }}
+  FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }}
+  HOST: ${{ secrets.HOST }}
+  LLAMAPARSE_API_KEY: ${{ secrets.LLAMAPARSE_API_KEY }}
+  LOGTAIL_KEY: ${{ secrets.LOGTAIL_KEY }}
+  NUM_WORKERS_PER_QUEUE: ${{ secrets.NUM_WORKERS_PER_QUEUE }}
+  OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+  PLAYWRIGHT_MICROSERVICE_URL: ${{ secrets.PLAYWRIGHT_MICROSERVICE_URL }}
+  PORT: ${{ secrets.PORT }}
+  REDIS_URL: ${{ secrets.REDIS_URL }}
+  SCRAPING_BEE_API_KEY: ${{ secrets.SCRAPING_BEE_API_KEY }}
+  SUPABASE_ANON_TOKEN: ${{ secrets.SUPABASE_ANON_TOKEN }}
+  SUPABASE_SERVICE_TOKEN: ${{ secrets.SUPABASE_SERVICE_TOKEN }}
+  SUPABASE_URL: ${{ secrets.SUPABASE_URL }}
+  TEST_API_KEY: ${{ secrets.TEST_API_KEY }}
+
 jobs:
   pre-deploy:
     name: Pre-deploy checks
@@ -29,11 +47,12 @@ jobs:
       - name: Start the application
         run: npm start &
         working-directory: ./apps/api
+        id: start_app
       - name: Start workers
         run: npm run workers &
         working-directory: ./apps/api
+        id: start_workers
       - name: Run E2E tests
-        run: npx jest -- src/__tests__/e2e/index.test.ts
+        run: |
-        working-directory: ./apps/api
+          npx jest --detectOpenHandles --forceExit --openHandlesTimeout=120000 --watchAll=false
-        env:
+        working-directory: ./apps/api
-          REDIS_URL: redis://localhost:6379

.github/workflows/fly.yml

@@ -6,10 +6,60 @@ on:
   # schedule:
   #   - cron: '0 */4 * * *'
 
+env:
+  ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+  BULL_AUTH_KEY: ${{ secrets.BULL_AUTH_KEY }}
+  FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }}
+  HOST: ${{ secrets.HOST }}
+  LLAMAPARSE_API_KEY: ${{ secrets.LLAMAPARSE_API_KEY }}
+  LOGTAIL_KEY: ${{ secrets.LOGTAIL_KEY }}
+  NUM_WORKERS_PER_QUEUE: ${{ secrets.NUM_WORKERS_PER_QUEUE }}
+  OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+  PLAYWRIGHT_MICROSERVICE_URL: ${{ secrets.PLAYWRIGHT_MICROSERVICE_URL }}
+  PORT: ${{ secrets.PORT }}
+  REDIS_URL: ${{ secrets.REDIS_URL }}
+  SCRAPING_BEE_API_KEY: ${{ secrets.SCRAPING_BEE_API_KEY }}
+  SUPABASE_ANON_TOKEN: ${{ secrets.SUPABASE_ANON_TOKEN }}
+  SUPABASE_SERVICE_TOKEN: ${{ secrets.SUPABASE_SERVICE_TOKEN }}
+  SUPABASE_URL: ${{ secrets.SUPABASE_URL }}
+  TEST_API_KEY: ${{ secrets.TEST_API_KEY }}
+
 jobs:
+  pre-deploy:
+    name: Pre-deploy checks
+    runs-on: ubuntu-latest
+    services:
+      redis:
+        image: redis
+        ports:
+          - 6379:6379
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Node.js
+        uses: actions/setup-node@v3
+        with:
+          node-version: '20'
+      - name: Install pnpm
+        run: npm install -g pnpm
+      - name: Install dependencies
+        run: pnpm install
+        working-directory: ./apps/api
+      - name: Start the application
+        run: npm start &
+        working-directory: ./apps/api
+        id: start_app
+      - name: Start workers
+        run: npm run workers &
+        working-directory: ./apps/api
+        id: start_workers
+      - name: Run E2E tests
+        run: |
+          npx jest --detectOpenHandles --forceExit --openHandlesTimeout=120000 --watchAll=false
+        working-directory: ./apps/api
   deploy:
     name: Deploy app
     runs-on: ubuntu-latest
+    needs: pre-deploy
     steps:
       - uses: actions/checkout@v3
       - name: Change directory

apps/api/src/__tests__/e2e/index.test.ts

@@ -152,19 +152,20 @@ describe('E2E Tests for API Routes', () => {
         expect(response.body).toHaveProperty('status');
         expect(response.body.status).toBe('active');
 
-      setTimeout(async () => {
+        // wait for 30 seconds
-        const response = await request(TEST_URL)
+        await new Promise((r) => setTimeout(r, 30000));
+
+        const completedResponse = await request(TEST_URL)
         .get(`/v0/crawl/status/${crawlResponse.body.jobId}`)
         .set('Authorization', `Bearer ${process.env.TEST_API_KEY}`);
-        expect(response.statusCode).toBe(200);
+        expect(completedResponse.statusCode).toBe(200);
-        expect(response.body).toHaveProperty('status');
+        expect(completedResponse.body).toHaveProperty('status');
-        expect(response.body.status).toBe('completed');
+        expect(completedResponse.body.status).toBe('completed');
-        expect(response.body).toHaveProperty('data');
+        expect(completedResponse.body).toHaveProperty('data');
-        expect(response.body.data).toHaveProperty('content');
+        expect(completedResponse.body.data[0]).toHaveProperty('content');
-        expect(response.body.data).toHaveProperty('markdown');
+        expect(completedResponse.body.data[0]).toHaveProperty('markdown');
-        expect(response.body.data).toHaveProperty('metadata');
+        expect(completedResponse.body.data[0]).toHaveProperty('metadata');
-        expect(response.body.data.content).toContain('🔥 FireCrawl');
+        expect(completedResponse.body.data[0].content).toContain('🔥 FireCrawl');
-      }, 30000); // 30 seconds
     }, 60000); // 60 seconds
   });