From af9e6c94a00d90cde54c8332049aeee22fdc7c69 Mon Sep 17 00:00:00 2001
From: rafaelsideguide <150964962+rafaelsideguide@users.noreply.github.com>
Date: Thu, 6 Jun 2024 10:53:55 -0300
Subject: [PATCH 01/23] Create publish-python-sdk.yml

---
 .github/workflows/publish-python-sdk.yml | 49 ++++++++++++++++++++++++
 1 file changed, 49 insertions(+)
 create mode 100644 .github/workflows/publish-python-sdk.yml

diff --git a/.github/workflows/publish-python-sdk.yml b/.github/workflows/publish-python-sdk.yml
new file mode 100644
index 0000000..814526d
--- /dev/null
+++ b/.github/workflows/publish-python-sdk.yml
@@ -0,0 +1,49 @@
+name: Publish Python SDK
+
+on:
+  pull_request:
+    branches:
+      - main
+
+env:
+  PYPI_USERNAME: ${{ secrets.PYPI_USERNAME }}
+  PYPI_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
+
+jobs:
+  build-and-publish:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.x'
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install setuptools wheel twine build requests packaging
+
+      - name: Check if version is incremented
+        id: check_version
+        run: |
+          VERSION_INCREMENTED=$(python .github/scripts/check_version_has_incremented.py python ./apps/python-sdk/firecrawl firecrawl-py)
+          echo "VERSION_INCREMENTED=$VERSION_INCREMENTED" >> $GITHUB_ENV
+
+      - name: Build the package
+        if: env.VERSION_INCREMENTED == 'true'
+        run: |
+          python -m build
+
+      - name: Publish to PyPI
+        if: env.VERSION_INCREMENTED == 'true'
+        env:
+          TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
+          TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
+        run: |
+          echo "Publishing to PyPI"
+#          twine upload dist/*
+

From fcea2d30b8e74539524affdd4c08325822c5bcf1 Mon Sep 17 00:00:00 2001
From: rafaelsideguide <150964962+rafaelsideguide@users.noreply.github.com>
Date: Thu, 6 Jun 2024 11:00:29 -0300
Subject: [PATCH 02/23] for test purposes

---
 .github/workflows/publish-python-sdk.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.github/workflows/publish-python-sdk.yml b/.github/workflows/publish-python-sdk.yml
index 814526d..c67d111 100644
--- a/.github/workflows/publish-python-sdk.yml
+++ b/.github/workflows/publish-python-sdk.yml
@@ -1,7 +1,7 @@
 name: Publish Python SDK
 
 on:
-  pull_request:
+  push:
     branches:
       - main
 
@@ -45,5 +45,4 @@ jobs:
           TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
         run: |
           echo "Publishing to PyPI"
-#          twine upload dist/*
 

From 5bf7b9d82f14f7f1bf8825d94436d82b32ba7ebb Mon Sep 17 00:00:00 2001
From: rafaelsideguide <150964962+rafaelsideguide@users.noreply.github.com>
Date: Thu, 6 Jun 2024 11:04:14 -0300
Subject: [PATCH 03/23] Update publish-python-sdk.yml

---
 .github/workflows/publish-python-sdk.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.github/workflows/publish-python-sdk.yml b/.github/workflows/publish-python-sdk.yml
index c67d111..7b0d73f 100644
--- a/.github/workflows/publish-python-sdk.yml
+++ b/.github/workflows/publish-python-sdk.yml
@@ -1,7 +1,7 @@
 name: Publish Python SDK
 
 on:
-  push:
+  pull_request:
     branches:
       - main
 
@@ -45,4 +45,3 @@ jobs:
           TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
         run: |
           echo "Publishing to PyPI"
-

From bebd13fea8216f54a3a356d578014cdbb4d25d5e Mon Sep 17 00:00:00 2001
From: rafaelsideguide <150964962+rafaelsideguide@users.noreply.github.com>
Date: Thu, 6 Jun 2024 11:10:42 -0300
Subject: [PATCH 04/23] Update publish-python-sdk.yml

---
 .github/workflows/publish-python-sdk.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/publish-python-sdk.yml b/.github/workflows/publish-python-sdk.yml
index 7b0d73f..e729b6f 100644
--- a/.github/workflows/publish-python-sdk.yml
+++ b/.github/workflows/publish-python-sdk.yml
@@ -32,6 +32,7 @@ jobs:
         run: |
           VERSION_INCREMENTED=$(python .github/scripts/check_version_has_incremented.py python ./apps/python-sdk/firecrawl firecrawl-py)
           echo "VERSION_INCREMENTED=$VERSION_INCREMENTED" >> $GITHUB_ENV
+          echo "Published Version: ${{ env.VERSION_INCREMENTED }}"
 
       - name: Build the package
         if: env.VERSION_INCREMENTED == 'true'

From f9643bfa659f2922e4626b91142a941769fd5fa6 Mon Sep 17 00:00:00 2001
From: rafaelsideguide <150964962+rafaelsideguide@users.noreply.github.com>
Date: Thu, 6 Jun 2024 11:15:27 -0300
Subject: [PATCH 05/23] getting a bug from backslashes

---
 .github/scripts/check_version_has_incremented.py | 12 +-----------
 1 file changed, 1 insertion(+), 11 deletions(-)

diff --git a/.github/scripts/check_version_has_incremented.py b/.github/scripts/check_version_has_incremented.py
index 345d088..4300563 100644
--- a/.github/scripts/check_version_has_incremented.py
+++ b/.github/scripts/check_version_has_incremented.py
@@ -1,24 +1,14 @@
 """
-checks local verions against published verions.
+checks local versions against published versions.
 
 # Usage:
 
-Unix:
 python .github/scripts/check_version_has_incremented.py js ./apps/js-sdk/firecrawl @mendable/firecrawl-js 
-
-Windows:
-python .github\scripts\check_version_has_incremented.py js .\apps\js-sdk\firecrawl @mendable/firecrawl-js 
-
 Local version: 0.0.22
 Published version: 0.0.21
 true
 
-Unix:
 python .github/scripts/check_version_has_incremented.py python ./apps/python-sdk/firecrawl firecrawl-py 
-
-Windows:
-python .github\scripts\check_version_has_incremented.py python .\apps\python-sdk\firecrawl firecrawl-py             
-
 Local version: 0.0.11
 Published version: 0.0.11
 false

From 2744e3d4b99e557b579e95dc26331d7fe9463510 Mon Sep 17 00:00:00 2001
From: rafaelsideguide <150964962+rafaelsideguide@users.noreply.github.com>
Date: Thu, 6 Jun 2024 11:20:21 -0300
Subject: [PATCH 06/23] Update publish-python-sdk.yml

---
 .github/workflows/publish-python-sdk.yml | 18 +++++++++++++-----
 1 file changed, 13 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/publish-python-sdk.yml b/.github/workflows/publish-python-sdk.yml
index e729b6f..9ba9f08 100644
--- a/.github/workflows/publish-python-sdk.yml
+++ b/.github/workflows/publish-python-sdk.yml
@@ -27,12 +27,20 @@ jobs:
           python -m pip install --upgrade pip
           pip install setuptools wheel twine build requests packaging
 
-      - name: Check if version is incremented
-        id: check_version
+          - name: Run version check script
+          id: version_check_script
+          run: |
+            VERSION_INCREMENTED=$(python .github/scripts/check_version_has_incremented.py python ./apps/python-sdk/firecrawl firecrawl-py)
+            echo "VERSION_INCREMENTED=$VERSION_INCREMENTED" >> $GITHUB_ENV
+  
+      - name: Debug output of version check
         run: |
-          VERSION_INCREMENTED=$(python .github/scripts/check_version_has_incremented.py python ./apps/python-sdk/firecrawl firecrawl-py)
-          echo "VERSION_INCREMENTED=$VERSION_INCREMENTED" >> $GITHUB_ENV
-          echo "Published Version: ${{ env.VERSION_INCREMENTED }}"
+          echo "Output of VERSION_INCREMENTED: '${{ env.VERSION_INCREMENTED }}'"
+  
+      - name: Set published version environment variable
+        if: env.VERSION_INCREMENTED == 'true'
+        run: |
+            echo "Published Version: ${{ env.VERSION_INCREMENTED }}"
 
       - name: Build the package
         if: env.VERSION_INCREMENTED == 'true'

From a939f46f3a6577f4c0e74d7b04514770990c51ab Mon Sep 17 00:00:00 2001
From: rafaelsideguide <150964962+rafaelsideguide@users.noreply.github.com>
Date: Thu, 6 Jun 2024 11:21:45 -0300
Subject: [PATCH 07/23] Update publish-python-sdk.yml

---
 .github/workflows/publish-python-sdk.yml | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/publish-python-sdk.yml b/.github/workflows/publish-python-sdk.yml
index 9ba9f08..2244651 100644
--- a/.github/workflows/publish-python-sdk.yml
+++ b/.github/workflows/publish-python-sdk.yml
@@ -27,11 +27,11 @@ jobs:
           python -m pip install --upgrade pip
           pip install setuptools wheel twine build requests packaging
 
-          - name: Run version check script
-          id: version_check_script
-          run: |
-            VERSION_INCREMENTED=$(python .github/scripts/check_version_has_incremented.py python ./apps/python-sdk/firecrawl firecrawl-py)
-            echo "VERSION_INCREMENTED=$VERSION_INCREMENTED" >> $GITHUB_ENV
+      - name: Run version check script
+        id: version_check_script
+        run: |
+          VERSION_INCREMENTED=$(python .github/scripts/check_version_has_incremented.py python ./apps/python-sdk/firecrawl firecrawl-py)
+          echo "VERSION_INCREMENTED=$VERSION_INCREMENTED" >> $GITHUB_ENV
   
       - name: Debug output of version check
         run: |

From eb1dd54e98882974b42ad680183ede222b9ade85 Mon Sep 17 00:00:00 2001
From: rafaelsideguide <150964962+rafaelsideguide@users.noreply.github.com>
Date: Thu, 6 Jun 2024 11:29:51 -0300
Subject: [PATCH 08/23] Update publish-python-sdk.yml

---
 .github/workflows/publish-python-sdk.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/publish-python-sdk.yml b/.github/workflows/publish-python-sdk.yml
index 2244651..442ac4a 100644
--- a/.github/workflows/publish-python-sdk.yml
+++ b/.github/workflows/publish-python-sdk.yml
@@ -38,17 +38,17 @@ jobs:
           echo "Output of VERSION_INCREMENTED: '${{ env.VERSION_INCREMENTED }}'"
   
       - name: Set published version environment variable
-        if: env.VERSION_INCREMENTED == 'true'
+        if: ${{ env.VERSION_INCREMENTED == 'true' }}
         run: |
-            echo "Published Version: ${{ env.VERSION_INCREMENTED }}"
+          echo "Published Version: ${{ env.VERSION_INCREMENTED }}"
 
       - name: Build the package
-        if: env.VERSION_INCREMENTED == 'true'
+        if: ${{ env.VERSION_INCREMENTED == 'true' }}
         run: |
           python -m build
 
       - name: Publish to PyPI
-        if: env.VERSION_INCREMENTED == 'true'
+        if: ${{ env.VERSION_INCREMENTED == 'true' }}
         env:
           TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
           TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}

From 3e091b0c57d8e74a7df6db91d21c866a839ba241 Mon Sep 17 00:00:00 2001
From: rafaelsideguide <150964962+rafaelsideguide@users.noreply.github.com>
Date: Thu, 6 Jun 2024 11:33:11 -0300
Subject: [PATCH 09/23] Update publish-python-sdk.yml

---
 .github/workflows/publish-python-sdk.yml | 40 ++++++++++++------------
 1 file changed, 20 insertions(+), 20 deletions(-)

diff --git a/.github/workflows/publish-python-sdk.yml b/.github/workflows/publish-python-sdk.yml
index 442ac4a..2044b89 100644
--- a/.github/workflows/publish-python-sdk.yml
+++ b/.github/workflows/publish-python-sdk.yml
@@ -32,25 +32,25 @@ jobs:
         run: |
           VERSION_INCREMENTED=$(python .github/scripts/check_version_has_incremented.py python ./apps/python-sdk/firecrawl firecrawl-py)
           echo "VERSION_INCREMENTED=$VERSION_INCREMENTED" >> $GITHUB_ENV
-  
-      - name: Debug output of version check
-        run: |
-          echo "Output of VERSION_INCREMENTED: '${{ env.VERSION_INCREMENTED }}'"
-  
-      - name: Set published version environment variable
-        if: ${{ env.VERSION_INCREMENTED == 'true' }}
-        run: |
-          echo "Published Version: ${{ env.VERSION_INCREMENTED }}"
 
-      - name: Build the package
-        if: ${{ env.VERSION_INCREMENTED == 'true' }}
-        run: |
-          python -m build
+      # - name: Debug output of version check
+      #   run: |
+      #     echo "Output of VERSION_INCREMENTED: '${{ env.VERSION_INCREMENTED }}'"
+  
+      # - name: Set published version environment variable
+      #   if: ${{ env.VERSION_INCREMENTED == 'true' }}
+      #   run: |
+      #     echo "Published Version: ${{ env.VERSION_INCREMENTED }}"
 
-      - name: Publish to PyPI
-        if: ${{ env.VERSION_INCREMENTED == 'true' }}
-        env:
-          TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
-          TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
-        run: |
-          echo "Publishing to PyPI"
+      # - name: Build the package
+      #   if: ${{ env.VERSION_INCREMENTED == 'true' }}
+      #   run: |
+      #     python -m build
+
+      # - name: Publish to PyPI
+      #   if: ${{ env.VERSION_INCREMENTED == 'true' }}
+      #   env:
+      #     TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
+      #     TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
+      #   run: |
+      #     echo "Publishing to PyPI"

From 06aa426ac9dba2ea7e0a9b8c3b94a7fe0b22959b Mon Sep 17 00:00:00 2001
From: rafaelsideguide <150964962+rafaelsideguide@users.noreply.github.com>
Date: Thu, 6 Jun 2024 11:36:48 -0300
Subject: [PATCH 10/23] Update check_version_has_incremented.py

---
 .github/scripts/check_version_has_incremented.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/scripts/check_version_has_incremented.py b/.github/scripts/check_version_has_incremented.py
index 4300563..e437c93 100644
--- a/.github/scripts/check_version_has_incremented.py
+++ b/.github/scripts/check_version_has_incremented.py
@@ -78,8 +78,8 @@ if __name__ == "__main__":
         raise ValueError("Invalid package type. Use 'python' or 'js'.")
 
     # Print versions for debugging
-    print(f"Local version: {current_version}")
-    print(f"Published version: {published_version}")
+    # print(f"Local version: {current_version}")
+    # print(f"Published version: {published_version}")
 
     # Compare versions and print result
     if is_version_incremented(current_version, published_version):

From 5e17d2f691eb6541ede3420acfd949b745fc262b Mon Sep 17 00:00:00 2001
From: rafaelsideguide <150964962+rafaelsideguide@users.noreply.github.com>
Date: Thu, 6 Jun 2024 11:38:16 -0300
Subject: [PATCH 11/23] Update publish-python-sdk.yml

---
 .github/workflows/publish-python-sdk.yml | 36 ++++++++++++------------
 1 file changed, 18 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/publish-python-sdk.yml b/.github/workflows/publish-python-sdk.yml
index 2044b89..4835dc5 100644
--- a/.github/workflows/publish-python-sdk.yml
+++ b/.github/workflows/publish-python-sdk.yml
@@ -33,24 +33,24 @@ jobs:
           VERSION_INCREMENTED=$(python .github/scripts/check_version_has_incremented.py python ./apps/python-sdk/firecrawl firecrawl-py)
           echo "VERSION_INCREMENTED=$VERSION_INCREMENTED" >> $GITHUB_ENV
 
-      # - name: Debug output of version check
-      #   run: |
-      #     echo "Output of VERSION_INCREMENTED: '${{ env.VERSION_INCREMENTED }}'"
+      - name: Debug output of version check
+        run: |
+          echo "Output of VERSION_INCREMENTED: '${{ env.VERSION_INCREMENTED }}'"
   
-      # - name: Set published version environment variable
-      #   if: ${{ env.VERSION_INCREMENTED == 'true' }}
-      #   run: |
-      #     echo "Published Version: ${{ env.VERSION_INCREMENTED }}"
+      - name: Set published version environment variable
+        if: ${{ env.VERSION_INCREMENTED == 'true' }}
+        run: |
+          echo "Published Version: ${{ env.VERSION_INCREMENTED }}"
 
-      # - name: Build the package
-      #   if: ${{ env.VERSION_INCREMENTED == 'true' }}
-      #   run: |
-      #     python -m build
+      - name: Build the package
+        if: ${{ env.VERSION_INCREMENTED == 'true' }}
+        run: |
+          python -m build
 
-      # - name: Publish to PyPI
-      #   if: ${{ env.VERSION_INCREMENTED == 'true' }}
-      #   env:
-      #     TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
-      #     TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
-      #   run: |
-      #     echo "Publishing to PyPI"
+      - name: Publish to PyPI
+        if: ${{ env.VERSION_INCREMENTED == 'true' }}
+        env:
+          TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
+          TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
+        run: |
+          echo "Publishing to PyPI"

From bc1c1e505355110bc643627961c8ee16077c235d Mon Sep 17 00:00:00 2001
From: rafaelsideguide <150964962+rafaelsideguide@users.noreply.github.com>
Date: Thu, 6 Jun 2024 11:41:01 -0300
Subject: [PATCH 12/23] updating version to check if it runs

---
 apps/python-sdk/firecrawl/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/apps/python-sdk/firecrawl/__init__.py b/apps/python-sdk/firecrawl/__init__.py
index 6899915..b82039e 100644
--- a/apps/python-sdk/firecrawl/__init__.py
+++ b/apps/python-sdk/firecrawl/__init__.py
@@ -1,3 +1,3 @@
 from .firecrawl import FirecrawlApp
 
-__version__ = "0.0.11"
+__version__ = "0.0.13"

From 63f0214d60ed4da2281225ca813cc4a5b0d3fe36 Mon Sep 17 00:00:00 2001
From: rafaelsideguide <150964962+rafaelsideguide@users.noreply.github.com>
Date: Thu, 6 Jun 2024 11:43:44 -0300
Subject: [PATCH 13/23] Update publish-python-sdk.yml

---
 .github/workflows/publish-python-sdk.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/publish-python-sdk.yml b/.github/workflows/publish-python-sdk.yml
index 4835dc5..46cfd68 100644
--- a/.github/workflows/publish-python-sdk.yml
+++ b/.github/workflows/publish-python-sdk.yml
@@ -46,6 +46,7 @@ jobs:
         if: ${{ env.VERSION_INCREMENTED == 'true' }}
         run: |
           python -m build
+        working-directory: ./apps/python-sdk
 
       - name: Publish to PyPI
         if: ${{ env.VERSION_INCREMENTED == 'true' }}

From b3cae4c8588e7d4ccd0d1e42d653775110f958ae Mon Sep 17 00:00:00 2001
From: rafaelsideguide <150964962+rafaelsideguide@users.noreply.github.com>
Date: Thu, 6 Jun 2024 13:27:31 -0300
Subject: [PATCH 14/23] adding js and testing twine

---
 .github/workflows/publish-js-sdk.yml     | 48 ++++++++++++++++++++++++
 .github/workflows/publish-python-sdk.yml | 13 ++-----
 apps/js-sdk/firecrawl/package.json       |  2 +-
 3 files changed, 52 insertions(+), 11 deletions(-)
 create mode 100644 .github/workflows/publish-js-sdk.yml

diff --git a/.github/workflows/publish-js-sdk.yml b/.github/workflows/publish-js-sdk.yml
new file mode 100644
index 0000000..bfd61fa
--- /dev/null
+++ b/.github/workflows/publish-js-sdk.yml
@@ -0,0 +1,48 @@
+name: Publish Package to npm
+
+on:
+  pull_request:
+    branches:
+      - main
+
+env:
+  NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
+
+jobs:
+  build-and-publish:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Node.js
+        uses: actions/setup-node@v3
+        with:
+          node-version: "20"
+
+      - name: Install pnpm
+        run: npm install -g pnpm
+
+      - name: Install python for running version check script
+        run: |
+          python -m pip install --upgrade pip
+          pip install setuptools wheel requests packaging
+
+      - name: Install dependencies for JavaScript SDK
+        run: pnpm install
+        working-directory: ./apps/js-sdk/firecrawl
+        
+      - name: Run version check script
+        id: version_check_script
+        run: |
+          VERSION_INCREMENTED=$(python .github/scripts/check_version_has_incremented.py js ./apps/js-sdk/firecrawl @mendable/firecrawl-js)
+          echo "VERSION_INCREMENTED=$VERSION_INCREMENTED" >> $GITHUB_ENV
+
+      - name: Build and publish to npm
+        if: ${{ env.VERSION_INCREMENTED == 'true' }}
+        env:
+          NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
+        run: |
+          npm run publish
+        working-directory: ./apps/js-sdk/firecrawl
+
+
diff --git a/.github/workflows/publish-python-sdk.yml b/.github/workflows/publish-python-sdk.yml
index 46cfd68..612a968 100644
--- a/.github/workflows/publish-python-sdk.yml
+++ b/.github/workflows/publish-python-sdk.yml
@@ -33,15 +33,6 @@ jobs:
           VERSION_INCREMENTED=$(python .github/scripts/check_version_has_incremented.py python ./apps/python-sdk/firecrawl firecrawl-py)
           echo "VERSION_INCREMENTED=$VERSION_INCREMENTED" >> $GITHUB_ENV
 
-      - name: Debug output of version check
-        run: |
-          echo "Output of VERSION_INCREMENTED: '${{ env.VERSION_INCREMENTED }}'"
-  
-      - name: Set published version environment variable
-        if: ${{ env.VERSION_INCREMENTED == 'true' }}
-        run: |
-          echo "Published Version: ${{ env.VERSION_INCREMENTED }}"
-
       - name: Build the package
         if: ${{ env.VERSION_INCREMENTED == 'true' }}
         run: |
@@ -54,4 +45,6 @@ jobs:
           TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
           TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
         run: |
-          echo "Publishing to PyPI"
+          twine upload dist/*
+        working-directory: ./apps/python-sdk
+
diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.json
index d3581b3..3c2d604 100644
--- a/apps/js-sdk/firecrawl/package.json
+++ b/apps/js-sdk/firecrawl/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@mendable/firecrawl-js",
-  "version": "0.0.23",
+  "version": "0.0.24",
   "description": "JavaScript SDK for Firecrawl API",
   "main": "build/index.js",
   "types": "types/index.d.ts",

From 0a3846dc3f29a2ea9fe5df2346a3cd055508c5ae Mon Sep 17 00:00:00 2001
From: rafaelsideguide <150964962+rafaelsideguide@users.noreply.github.com>
Date: Thu, 6 Jun 2024 13:49:22 -0300
Subject: [PATCH 15/23] testing

---
 .github/workflows/publish-python-sdk.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/publish-python-sdk.yml b/.github/workflows/publish-python-sdk.yml
index 612a968..dd02787 100644
--- a/.github/workflows/publish-python-sdk.yml
+++ b/.github/workflows/publish-python-sdk.yml
@@ -1,4 +1,4 @@
-name: Publish Python SDK
+name: Publish Python SDK 
 
 on:
   pull_request:

From afc402144d5d5da5c23949550c406923472f3308 Mon Sep 17 00:00:00 2001
From: rafaelsideguide <150964962+rafaelsideguide@users.noreply.github.com>
Date: Thu, 6 Jun 2024 13:54:21 -0300
Subject: [PATCH 16/23] Update publish-js-sdk.yml

---
 .github/workflows/publish-js-sdk.yml | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/publish-js-sdk.yml b/.github/workflows/publish-js-sdk.yml
index bfd61fa..7593675 100644
--- a/.github/workflows/publish-js-sdk.yml
+++ b/.github/workflows/publish-js-sdk.yml
@@ -1,4 +1,4 @@
-name: Publish Package to npm
+name: Publish JavaScript SDK
 
 on:
   pull_request:
@@ -17,7 +17,10 @@ jobs:
       - name: Set up Node.js
         uses: actions/setup-node@v3
         with:
-          node-version: "20"
+          node-version: '20'
+          registry-url: 'https://registry.npmjs.org/'
+          scope: '@mendable'
+          always-auth: true
 
       - name: Install pnpm
         run: npm install -g pnpm
@@ -44,5 +47,3 @@ jobs:
         run: |
           npm run publish
         working-directory: ./apps/js-sdk/firecrawl
-
-

From fb758fa05e42e541121e3738a4315030aab58c69 Mon Sep 17 00:00:00 2001
From: rafaelsideguide <150964962+rafaelsideguide@users.noreply.github.com>
Date: Thu, 6 Jun 2024 14:01:16 -0300
Subject: [PATCH 17/23] go

---
 .github/workflows/publish-js-sdk.yml | 2 +-
 apps/js-sdk/firecrawl/package.json   | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/publish-js-sdk.yml b/.github/workflows/publish-js-sdk.yml
index 7593675..651d33b 100644
--- a/.github/workflows/publish-js-sdk.yml
+++ b/.github/workflows/publish-js-sdk.yml
@@ -45,5 +45,5 @@ jobs:
         env:
           NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
         run: |
-          npm run publish
+          npm run build-and-publish
         working-directory: ./apps/js-sdk/firecrawl
diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.json
index 3c2d604..8f9e470 100644
--- a/apps/js-sdk/firecrawl/package.json
+++ b/apps/js-sdk/firecrawl/package.json
@@ -1,13 +1,13 @@
 {
   "name": "@mendable/firecrawl-js",
-  "version": "0.0.24",
+  "version": "0.0.25",
   "description": "JavaScript SDK for Firecrawl API",
   "main": "build/index.js",
   "types": "types/index.d.ts",
   "type": "module",
   "scripts": {
     "build": "tsc",
-    "publish": "npm run build && npm publish --access public",
+    "build-and-publish": "npm run build && npm publish --access public",
     "publish-beta": "npm run build && npm publish --access public --tag beta",
     "test": "jest src/__tests__/**/*.test.ts"
   },

From 8132f22c73762e8092df0fe172817397ff2f833e Mon Sep 17 00:00:00 2001
From: rafaelsideguide <150964962+rafaelsideguide@users.noreply.github.com>
Date: Thu, 6 Jun 2024 15:36:20 -0300
Subject: [PATCH 18/23] nice

---
 .github/workflows/fly.yml                     | 127 +++++++++++--
 .github/workflows/publish-js-sdk.yml          |   5 +-
 .github/workflows/publish-python-sdk.yml      |   5 +-
 .../src/__tests__/e2e_withAuth/index.test.ts  | 170 +++++++++---------
 .../src/__tests__/e2e_withAuth/index.test.ts  |  60 +++----
 .../firecrawl/__tests__/e2e_withAuth/test.py  |  22 +--
 6 files changed, 245 insertions(+), 144 deletions(-)

diff --git a/.github/workflows/fly.yml b/.github/workflows/fly.yml
index 3950c2e..957a707 100644
--- a/.github/workflows/fly.yml
+++ b/.github/workflows/fly.yml
@@ -3,8 +3,6 @@ on:
   push:
     branches:
       - main
-  schedule:
-    - cron: '0 */2 * * *'
 
 env:
   ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
@@ -25,9 +23,12 @@ env:
   SUPABASE_SERVICE_TOKEN: ${{ secrets.SUPABASE_SERVICE_TOKEN }}
   SUPABASE_URL: ${{ secrets.SUPABASE_URL }}
   TEST_API_KEY: ${{ secrets.TEST_API_KEY }}
+  PYPI_USERNAME: ${{ secrets.PYPI_USERNAME }}
+  PYPI_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
+  NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
 
 jobs:
-  pre-deploy:
+  pre-deploy-e2e-tests:
     name: Pre-deploy checks
     runs-on: ubuntu-latest
     services:
@@ -61,7 +62,7 @@ jobs:
 
   pre-deploy-test-suite:
     name: Test Suite
-    needs: pre-deploy
+    needs: pre-deploy-e2e-tests
     runs-on: ubuntu-latest
     services:
       redis:
@@ -94,19 +95,37 @@ jobs:
         run: |
           npm run test
         working-directory: ./apps/test-suite
-      - name: Set up Python ${{ matrix.python-version }}  
-        uses: actions/setup-python@v4  
-        with:  
-          python-version: ${{ matrix.python-version }}
-      - name: Install Python dependencies  
-        run: |  
-          python -m pip install --upgrade pip  
+
+  python-sdk-tests:
+    name: Python SDK Tests
+    needs: pre-deploy-e2e-tests
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.x'
+      - name: Install Python dependencies
+        run: |
+          python -m pip install --upgrade pip
           pip install -r requirements.txt
         working-directory: ./apps/python-sdk
       - name: Run E2E tests for Python SDK
-        run: |  
-          pytest firecrawl/__tests__/e2e_withAuth/test.py 
+        run: |
+          pytest firecrawl/__tests__/e2e_withAuth/test.py
         working-directory: ./apps/python-sdk
+
+  js-sdk-tests:
+    name: JavaScript SDK Tests
+    needs: pre-deploy-e2e-tests
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Node.js
+        uses: actions/setup-node@v3
+        with:
+          node-version: "20"
       - name: Install dependencies for JavaScript SDK
         run: pnpm install
         working-directory: ./apps/js-sdk/firecrawl
@@ -117,7 +136,7 @@ jobs:
   deploy:
     name: Deploy app
     runs-on: ubuntu-latest
-    needs: pre-deploy-test-suite
+    needs: [pre-deploy-test-suite, python-sdk-tests, js-sdk-tests]
     steps:
       - uses: actions/checkout@v3
       - name: Change directory
@@ -126,3 +145,83 @@ jobs:
       - run: flyctl deploy ./apps/api --remote-only -a firecrawl-scraper-js
         env:
           FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }}
+
+  build-and-publish-python-sdk:
+    runs-on: ubuntu-latest
+    needs: deploy
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.x'
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install setuptools wheel twine build requests packaging
+
+      - name: Run version check script
+        id: version_check_script
+        run: |
+          PYTHON_SDK_VERSION_INCREMENTED=$(python .github/scripts/check_version_has_incremented.py python ./apps/python-sdk/firecrawl firecrawl-py)
+          echo "PYTHON_SDK_VERSION_INCREMENTED=$PYTHON_SDK_VERSION_INCREMENTED" >> $GITHUB_ENV
+
+      - name: Build the package
+        if: ${{ env.PYTHON_SDK_VERSION_INCREMENTED == 'true' }}
+        run: |
+          python -m build
+        working-directory: ./apps/python-sdk
+
+      - name: Publish to PyPI
+        if: ${{ env.VERSION_INCREMENTED == 'true' }}
+        env:
+          TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
+          TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
+        run: |
+          twine upload dist/*
+        working-directory: ./apps/python-sdk
+
+  build-and-publish-js-sdk:
+    runs-on: ubuntu-latest
+    needs: deploy
+
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Node.js
+        uses: actions/setup-node@v3
+        with:
+          node-version: '20'
+          registry-url: 'https://registry.npmjs.org/'
+          scope: '@mendable'
+          always-auth: true
+
+      - name: Install pnpm
+        run: npm install -g pnpm
+
+      - name: Install python for running version check script
+        run: |
+          python -m pip install --upgrade pip
+          pip install setuptools wheel requests packaging
+
+      - name: Install dependencies for JavaScript SDK
+        run: pnpm install
+        working-directory: ./apps/js-sdk/firecrawl
+        
+      - name: Run version check script
+        id: version_check_script
+        run: |
+          VERSION_INCREMENTED=$(python .github/scripts/check_version_has_incremented.py js ./apps/js-sdk/firecrawl @mendable/firecrawl-js)
+          echo "VERSION_INCREMENTED=$VERSION_INCREMENTED" >> $GITHUB_ENV
+
+      - name: Build and publish to npm
+        if: ${{ env.VERSION_INCREMENTED == 'true' }}
+        env:
+          NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
+        run: |
+          npm run build-and-publish
+        working-directory: ./apps/js-sdk/firecrawl
+      
\ No newline at end of file
diff --git a/.github/workflows/publish-js-sdk.yml b/.github/workflows/publish-js-sdk.yml
index 651d33b..c02a654 100644
--- a/.github/workflows/publish-js-sdk.yml
+++ b/.github/workflows/publish-js-sdk.yml
@@ -1,9 +1,6 @@
 name: Publish JavaScript SDK
 
-on:
-  pull_request:
-    branches:
-      - main
+on: []
 
 env:
   NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
diff --git a/.github/workflows/publish-python-sdk.yml b/.github/workflows/publish-python-sdk.yml
index dd02787..6d86f1e 100644
--- a/.github/workflows/publish-python-sdk.yml
+++ b/.github/workflows/publish-python-sdk.yml
@@ -1,9 +1,6 @@
 name: Publish Python SDK 
 
-on:
-  pull_request:
-    branches:
-      - main
+on: []
 
 env:
   PYPI_USERNAME: ${{ secrets.PYPI_USERNAME }}
diff --git a/apps/api/src/__tests__/e2e_withAuth/index.test.ts b/apps/api/src/__tests__/e2e_withAuth/index.test.ts
index f015acd..69e5316 100644
--- a/apps/api/src/__tests__/e2e_withAuth/index.test.ts
+++ b/apps/api/src/__tests__/e2e_withAuth/index.test.ts
@@ -17,7 +17,7 @@ describe("E2E Tests for API Routes", () => {
     delete process.env.USE_DB_AUTHENTICATION;
   });
   describe("GET /", () => {
-    it("should return Hello, world! message", async () => {
+    it.concurrent("should return Hello, world! message", async () => {
       const response = await request(TEST_URL).get("/");
 
       expect(response.statusCode).toBe(200);
@@ -26,7 +26,7 @@ describe("E2E Tests for API Routes", () => {
   });
 
   describe("GET /test", () => {
-    it("should return Hello, world! message", async () => {
+    it.concurrent("should return Hello, world! message", async () => {
       const response = await request(TEST_URL).get("/test");
       expect(response.statusCode).toBe(200);
       expect(response.text).toContain("Hello, world!");
@@ -34,12 +34,12 @@ describe("E2E Tests for API Routes", () => {
   });
 
   describe("POST /v0/scrape", () => {
-    it("should require authorization", async () => {
+    it.concurrent("should require authorization", async () => {
       const response = await request(app).post("/v0/scrape");
       expect(response.statusCode).toBe(401);
     });
 
-    it("should return an error response with an invalid API key", async () => {
+    it.concurrent("should return an error response with an invalid API key", async () => {
       const response = await request(TEST_URL)
         .post("/v0/scrape")
         .set("Authorization", `Bearer invalid-api-key`)
@@ -48,7 +48,7 @@ describe("E2E Tests for API Routes", () => {
       expect(response.statusCode).toBe(401);
     });
 
-    it("should return an error for a blocklisted URL", async () => {
+    it.concurrent("should return an error for a blocklisted URL", async () => {
       const blocklistedUrl = "https://facebook.com/fake-test";
       const response = await request(TEST_URL)
         .post("/v0/scrape")
@@ -61,37 +61,38 @@ describe("E2E Tests for API Routes", () => {
       );
     });
 
-    it("should return a successful response with a valid preview token", async () => {
-      const response = await request(TEST_URL)
-        .post("/v0/scrape")
-        .set("Authorization", `Bearer this_is_just_a_preview_token`)
-        .set("Content-Type", "application/json")
-        .send({ url: "https://roastmywebsite.ai" });
-      expect(response.statusCode).toBe(200);
-    }, 30000); // 30 seconds timeout
+    // tested on rate limit test
+    // it.concurrent("should return a successful response with a valid preview token", async () => {
+    //   const response = await request(TEST_URL)
+    //     .post("/v0/scrape")
+    //     .set("Authorization", `Bearer this_is_just_a_preview_token`)
+    //     .set("Content-Type", "application/json")
+    //     .send({ url: "https://roastmywebsite.ai" });
+    //   expect(response.statusCode).toBe(200);
+    // }, 30000); // 30 seconds timeout
 
-    it("should return a successful response with a valid API key", async () => {
+    it.concurrent("should return a successful response with a valid API key", async () => {
       const response = await request(TEST_URL)
         .post("/v0/scrape")
         .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
         .set("Content-Type", "application/json")
-        .send({ url: "https://firecrawl.dev" });
+        .send({ url: "https://roastmywebsite.ai" });
       expect(response.statusCode).toBe(200);
       expect(response.body).toHaveProperty("data");
       expect(response.body.data).toHaveProperty("content");
       expect(response.body.data).toHaveProperty("markdown");
       expect(response.body.data).toHaveProperty("metadata");
       expect(response.body.data).not.toHaveProperty("html");
-      expect(response.body.data.content).toContain("🔥 Firecrawl");
+      expect(response.body.data.content).toContain("_Roast_");
     }, 30000); // 30 seconds timeout
 
-    it("should return a successful response with a valid API key and includeHtml set to true", async () => {
+    it.concurrent("should return a successful response with a valid API key and includeHtml set to true", async () => {
       const response = await request(TEST_URL)
         .post("/v0/scrape")
         .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
         .set("Content-Type", "application/json")
         .send({
-          url: "https://firecrawl.dev",
+          url: "https://roastmywebsite.ai",
           pageOptions: { includeHtml: true },
         });
       expect(response.statusCode).toBe(200);
@@ -100,12 +101,12 @@ describe("E2E Tests for API Routes", () => {
       expect(response.body.data).toHaveProperty("markdown");
       expect(response.body.data).toHaveProperty("html");
       expect(response.body.data).toHaveProperty("metadata");
-      expect(response.body.data.content).toContain("🔥 Firecrawl");
-      expect(response.body.data.markdown).toContain("🔥 Firecrawl");
+      expect(response.body.data.content).toContain("_Roast_");
+      expect(response.body.data.markdown).toContain("_Roast_");
       expect(response.body.data.html).toContain("<h1");
     }, 30000); // 30 seconds timeout
     
-   it('should return a successful response for a valid scrape with PDF file', async () => {
+   it.concurrent('should return a successful response for a valid scrape with PDF file', async () => {
       const response = await request(TEST_URL)
         .post('/v0/scrape')
         .set('Authorization', `Bearer ${process.env.TEST_API_KEY}`)
@@ -120,7 +121,7 @@ describe("E2E Tests for API Routes", () => {
       expect(response.body.data.content).toContain('We present spectrophotometric observations of the Broad Line Radio Galaxy');
     }, 60000); // 60 seconds
   
-    it('should return a successful response for a valid scrape with PDF file without explicit .pdf extension', async () => {
+    it.concurrent('should return a successful response for a valid scrape with PDF file without explicit .pdf extension', async () => {
       const response = await request(TEST_URL)
         .post('/v0/scrape')
         .set('Authorization', `Bearer ${process.env.TEST_API_KEY}`)
@@ -136,7 +137,7 @@ describe("E2E Tests for API Routes", () => {
     }, 60000); // 60 seconds
 
     // TODO: add this test back once we nail the waitFor option to be more deterministic
-    // it("should return a successful response with a valid API key and waitFor option", async () => {
+    // it.concurrent("should return a successful response with a valid API key and waitFor option", async () => {
     //   const startTime = Date.now();
     //   const response = await request(TEST_URL)
     //     .post("/v0/scrape")
@@ -158,12 +159,12 @@ describe("E2E Tests for API Routes", () => {
   });
 
   describe("POST /v0/crawl", () => {
-    it("should require authorization", async () => {
+    it.concurrent("should require authorization", async () => {
       const response = await request(TEST_URL).post("/v0/crawl");
       expect(response.statusCode).toBe(401);
     });
 
-    it("should return an error response with an invalid API key", async () => {
+    it.concurrent("should return an error response with an invalid API key", async () => {
       const response = await request(TEST_URL)
         .post("/v0/crawl")
         .set("Authorization", `Bearer invalid-api-key`)
@@ -172,7 +173,7 @@ describe("E2E Tests for API Routes", () => {
       expect(response.statusCode).toBe(401);
     });
 
-    it("should return an error for a blocklisted URL", async () => {
+    it.concurrent("should return an error for a blocklisted URL", async () => {
       const blocklistedUrl = "https://twitter.com/fake-test";
       const response = await request(TEST_URL)
         .post("/v0/crawl")
@@ -185,7 +186,7 @@ describe("E2E Tests for API Routes", () => {
       );
     });
 
-    it("should return a successful response with a valid API key for crawl", async () => {
+    it.concurrent("should return a successful response with a valid API key for crawl", async () => {
       const response = await request(TEST_URL)
         .post("/v0/crawl")
         .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
@@ -197,7 +198,7 @@ describe("E2E Tests for API Routes", () => {
         /^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$/
       );
     });
-    it('should prevent duplicate requests using the same idempotency key', async () => {
+    it.concurrent('should prevent duplicate requests using the same idempotency key', async () => {
       const uniqueIdempotencyKey = uuidv4();
   
       // First request with the idempotency key
@@ -222,7 +223,7 @@ describe("E2E Tests for API Routes", () => {
       expect(secondResponse.body.error).toBe('Idempotency key already used');
     });
 
-    it("should return a successful response with a valid API key and valid includes option", async () => {
+    it.concurrent("should return a successful response with a valid API key and valid includes option", async () => {
       const crawlResponse = await request(TEST_URL)
         .post("/v0/crawl")
         .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
@@ -259,7 +260,6 @@ describe("E2E Tests for API Routes", () => {
       );
       expect(urls.length).toBeGreaterThan(5);
       urls.forEach((url: string) => {
-        console.log({url})
         expect(url.startsWith("https://www.mendable.ai/blog/")).toBeTruthy();
       });
       
@@ -273,7 +273,7 @@ describe("E2E Tests for API Routes", () => {
       expect(completedResponse.body.data[0].content).toContain("Mendable");
     }, 60000); // 60 seconds
 
-    it("should return a successful response with a valid API key and valid excludes option", async () => {
+    it.concurrent("should return a successful response with a valid API key and valid excludes option", async () => {
       const crawlResponse = await request(TEST_URL)
         .post("/v0/crawl")
         .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
@@ -314,7 +314,7 @@ describe("E2E Tests for API Routes", () => {
       });
     }, 90000); // 90 seconds
 
-    it("should return a successful response with a valid API key and limit to 3", async () => {
+    it.concurrent("should return a successful response with a valid API key and limit to 3", async () => {
       const crawlResponse = await request(TEST_URL)
         .post("/v0/crawl")
         .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
@@ -354,7 +354,7 @@ describe("E2E Tests for API Routes", () => {
       expect(completedResponse.body.data[0].content).toContain("Mendable");
     }, 60000); // 60 seconds
   
-    it("should return a successful response with max depth option for a valid crawl job", async () => {
+    it.concurrent("should return a successful response with max depth option for a valid crawl job", async () => {
       const crawlResponse = await request(TEST_URL)
         .post("/v0/crawl")
         .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
@@ -396,7 +396,7 @@ describe("E2E Tests for API Routes", () => {
       });
     }, 120000);
 
-    // it("should return a successful response with a valid API key and valid limit option", async () => {
+    // it.concurrent("should return a successful response with a valid API key and valid limit option", async () => {
     //   const crawlResponse = await request(TEST_URL)
     //     .post("/v0/crawl")
     //     .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
@@ -441,13 +441,13 @@ describe("E2E Tests for API Routes", () => {
     //   expect(completedResponse.body.data[0].content).not.toContain("main menu");
     // }, 60000); // 60 seconds
 
-    it("should return a successful response for a valid crawl job with includeHtml set to true option", async () => {
+    it.concurrent("should return a successful response for a valid crawl job with includeHtml set to true option", async () => {
       const crawlResponse = await request(TEST_URL)
         .post("/v0/crawl")
         .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
         .set("Content-Type", "application/json")
         .send({
-          url: "https://firecrawl.dev",
+          url: "https://roastmywebsite.ai",
           pageOptions: { includeHtml: true },
         });
       expect(crawlResponse.statusCode).toBe(200);
@@ -486,19 +486,19 @@ describe("E2E Tests for API Routes", () => {
       // 120 seconds
       expect(completedResponse.body.data[0]).toHaveProperty("html");
       expect(completedResponse.body.data[0]).toHaveProperty("metadata");
-      expect(completedResponse.body.data[0].content).toContain("🔥 Firecrawl");
-      expect(completedResponse.body.data[0].markdown).toContain("Firecrawl");
+      expect(completedResponse.body.data[0].content).toContain("_Roast_");
+      expect(completedResponse.body.data[0].markdown).toContain("_Roast_");
       expect(completedResponse.body.data[0].html).toContain("<h1");
     }, 60000);
   });
 
   describe("POST /v0/crawlWebsitePreview", () => {
-    it("should require authorization", async () => {
+    it.concurrent("should require authorization", async () => {
       const response = await request(TEST_URL).post("/v0/crawlWebsitePreview");
       expect(response.statusCode).toBe(401);
     });
 
-    it("should return an error response with an invalid API key", async () => {
+    it.concurrent("should return an error response with an invalid API key", async () => {
       const response = await request(TEST_URL)
         .post("/v0/crawlWebsitePreview")
         .set("Authorization", `Bearer invalid-api-key`)
@@ -507,7 +507,7 @@ describe("E2E Tests for API Routes", () => {
       expect(response.statusCode).toBe(401);
     });
 
-    // it("should return an error for a blocklisted URL", async () => {
+    // it.concurrent("should return an error for a blocklisted URL", async () => {
     //   const blocklistedUrl = "https://instagram.com/fake-test";
     //   const response = await request(TEST_URL)
     //     .post("/v0/crawlWebsitePreview")
@@ -519,7 +519,7 @@ describe("E2E Tests for API Routes", () => {
     //   expect(response.body.error).toContain("Firecrawl currently does not support social media scraping due to policy restrictions. We're actively working on building support for it.");
     // });
 
-    it("should return a timeout error when scraping takes longer than the specified timeout", async () => {
+    it.concurrent("should return a timeout error when scraping takes longer than the specified timeout", async () => {
       const response = await request(TEST_URL)
         .post("/v0/scrape")
         .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
@@ -529,7 +529,7 @@ describe("E2E Tests for API Routes", () => {
       expect(response.statusCode).toBe(408);
     }, 3000); 
 
-    it("should return a successful response with a valid API key for crawlWebsitePreview", async () => {
+    it.concurrent("should return a successful response with a valid API key for crawlWebsitePreview", async () => {
       const response = await request(TEST_URL)
         .post("/v0/crawlWebsitePreview")
         .set("Authorization", `Bearer this_is_just_a_preview_token`)
@@ -544,12 +544,12 @@ describe("E2E Tests for API Routes", () => {
   });
 
   describe("POST /v0/search", () => {
-    it("should require authorization", async () => {
+    it.concurrent("should require authorization", async () => {
       const response = await request(TEST_URL).post("/v0/search");
       expect(response.statusCode).toBe(401);
     });
 
-    it("should return an error response with an invalid API key", async () => {
+    it.concurrent("should return an error response with an invalid API key", async () => {
       const response = await request(TEST_URL)
         .post("/v0/search")
         .set("Authorization", `Bearer invalid-api-key`)
@@ -558,7 +558,7 @@ describe("E2E Tests for API Routes", () => {
       expect(response.statusCode).toBe(401);
     });
 
-    it("should return a successful response with a valid API key for search", async () => {
+    it.concurrent("should return a successful response with a valid API key for search", async () => {
       const response = await request(TEST_URL)
         .post("/v0/search")
         .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
@@ -572,31 +572,31 @@ describe("E2E Tests for API Routes", () => {
   });
 
   describe("GET /v0/crawl/status/:jobId", () => {
-    it("should require authorization", async () => {
+    it.concurrent("should require authorization", async () => {
       const response = await request(TEST_URL).get("/v0/crawl/status/123");
       expect(response.statusCode).toBe(401);
     });
 
-    it("should return an error response with an invalid API key", async () => {
+    it.concurrent("should return an error response with an invalid API key", async () => {
       const response = await request(TEST_URL)
         .get("/v0/crawl/status/123")
         .set("Authorization", `Bearer invalid-api-key`);
       expect(response.statusCode).toBe(401);
     });
 
-    it("should return Job not found for invalid job ID", async () => {
+    it.concurrent("should return Job not found for invalid job ID", async () => {
       const response = await request(TEST_URL)
         .get("/v0/crawl/status/invalidJobId")
         .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);
       expect(response.statusCode).toBe(404);
     });
 
-    it("should return a successful crawl status response for a valid crawl job", async () => {
+    it.concurrent("should return a successful crawl status response for a valid crawl job", async () => {
       const crawlResponse = await request(TEST_URL)
         .post("/v0/crawl")
         .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
         .set("Content-Type", "application/json")
-        .send({ url: "https://firecrawl.dev" });
+        .send({ url: "https://roastmywebsite.ai" });
       expect(crawlResponse.statusCode).toBe(200);
 
       let isCompleted = false;
@@ -622,10 +622,10 @@ describe("E2E Tests for API Routes", () => {
       expect(completedResponse.body.data[0]).toHaveProperty("content");
       expect(completedResponse.body.data[0]).toHaveProperty("markdown");
       expect(completedResponse.body.data[0]).toHaveProperty("metadata");
-      expect(completedResponse.body.data[0].content).toContain("🔥 Firecrawl");
-    }, 60000); // 60 seconds
+      expect(completedResponse.body.data[0].content).toContain("_Roast_");
+    }, 120000); // 120 seconds
     
-    it('should return a successful response for a valid crawl job with PDF files without explicit .pdf extension', async () => {
+    it.concurrent('should return a successful response for a valid crawl job with PDF files without explicit .pdf extension', async () => {
       const crawlResponse = await request(TEST_URL)
         .post('/v0/crawl')
         .set('Authorization', `Bearer ${process.env.TEST_API_KEY}`)
@@ -660,9 +660,9 @@ describe("E2E Tests for API Routes", () => {
             })
           ])
         );
-    }, 60000); // 60 seconds
+    }, 120000); // 120 seconds
 
-    it("should return a successful response with max depth option for a valid crawl job", async () => {
+    it.concurrent("should return a successful response with max depth option for a valid crawl job", async () => {
       const crawlResponse = await request(TEST_URL)
         .post("/v0/crawl")
         .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
@@ -707,13 +707,13 @@ describe("E2E Tests for API Routes", () => {
       });
     }, 120000);
 
-    it("should return a successful response for a valid crawl job with includeHtml set to true option", async () => {
+    it.concurrent("should return a successful response for a valid crawl job with includeHtml set to true option", async () => {
       const crawlResponse = await request(TEST_URL)
         .post("/v0/crawl")
         .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
         .set("Content-Type", "application/json")
         .send({
-          url: "https://firecrawl.dev",
+          url: "https://roastmywebsite.ai",
           pageOptions: { includeHtml: true },
         });
       expect(crawlResponse.statusCode).toBe(200);
@@ -725,12 +725,23 @@ describe("E2E Tests for API Routes", () => {
       expect(response.body).toHaveProperty("status");
       expect(response.body.status).toBe("active");
 
-      // wait for 30 seconds
-      await new Promise((r) => setTimeout(r, 30000));
+      let isFinished = false;
+      let completedResponse;
 
-      const completedResponse = await request(TEST_URL)
-        .get(`/v0/crawl/status/${crawlResponse.body.jobId}`)
-        .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);
+      while (!isFinished) {
+        const response = await request(TEST_URL)
+          .get(`/v0/crawl/status/${crawlResponse.body.jobId}`)
+          .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);
+        expect(response.statusCode).toBe(200);
+        expect(response.body).toHaveProperty("status");
+
+        if (response.body.status === "completed") {
+          isFinished = true;
+          completedResponse = response;
+        } else {
+          await new Promise((r) => setTimeout(r, 1000)); // Wait for 1 second before checking again
+        }
+      }
 
       expect(completedResponse.statusCode).toBe(200);
       expect(completedResponse.body).toHaveProperty("status");
@@ -739,17 +750,14 @@ describe("E2E Tests for API Routes", () => {
       expect(completedResponse.body.data[0]).toHaveProperty("content");
       expect(completedResponse.body.data[0]).toHaveProperty("markdown");
       expect(completedResponse.body.data[0]).toHaveProperty("metadata");
-
-      // 120 seconds
       expect(completedResponse.body.data[0]).toHaveProperty("html");
-      expect(completedResponse.body.data[0]).toHaveProperty("metadata");
-      expect(completedResponse.body.data[0].content).toContain("🔥 Firecrawl");
-      expect(completedResponse.body.data[0].markdown).toContain("Firecrawl");
+      expect(completedResponse.body.data[0].content).toContain("_Roast_");
+      expect(completedResponse.body.data[0].markdown).toContain("_Roast_");
       expect(completedResponse.body.data[0].html).toContain("<h1");
     }, 60000);
   }); // 60 seconds
 
-  it("If someone cancels a crawl job, it should turn into failed status", async () => {
+  it.concurrent("If someone cancels a crawl job, it should turn into failed status", async () => {
     const crawlResponse = await request(TEST_URL)
       .post("/v0/crawl")
       .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
@@ -785,7 +793,7 @@ describe("E2E Tests for API Routes", () => {
   }, 60000); // 60 seconds
 
   describe("POST /v0/scrape with LLM Extraction", () => {
-    it("should extract data using LLM extraction mode", async () => {
+    it.concurrent("should extract data using LLM extraction mode", async () => {
       const response = await request(TEST_URL)
         .post("/v0/scrape")
         .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
@@ -836,7 +844,7 @@ describe("E2E Tests for API Routes", () => {
   });
 
   // describe("POST /v0/scrape for Top 100 Companies", () => {
-  //   it("should extract data for the top 100 companies", async () => {
+  //   it.concurrent("should extract data for the top 100 companies", async () => {
   //     const response = await request(TEST_URL)
   //       .post("/v0/scrape")
   //       .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
@@ -894,7 +902,7 @@ describe("E2E Tests for API Routes", () => {
   // });
 
   describe("POST /v0/crawl with fast mode", () => {
-    it("should complete the crawl under 20 seconds", async () => {
+    it.concurrent("should complete the crawl under 20 seconds", async () => {
       const startTime = Date.now();
 
       const crawlResponse = await request(TEST_URL)
@@ -927,10 +935,10 @@ describe("E2E Tests for API Routes", () => {
         }
       }
 
-      const endTime = Date.now();
-      const timeElapsed = (endTime - startTime) / 1000; // Convert to seconds
+      // const endTime = Date.now();
+      // const timeElapsed = (endTime - startTime) / 1000; // Convert to seconds
 
-      console.log(`Time elapsed: ${timeElapsed} seconds`);
+      // console.log(`Time elapsed: ${timeElapsed} seconds`);
 
       expect(statusResponse.body.status).toBe("completed");
       expect(statusResponse.body).toHaveProperty("data");
@@ -945,7 +953,7 @@ describe("E2E Tests for API Routes", () => {
       
     }, 20000);
 
-    // it("should complete the crawl in more than 10 seconds", async () => {
+    // it.concurrent("should complete the crawl in more than 10 seconds", async () => {
     //   const startTime = Date.now();
 
     //   const crawlResponse = await request(TEST_URL)
@@ -995,7 +1003,7 @@ describe("E2E Tests for API Routes", () => {
   });
 
   describe("GET /is-production", () => {
-    it("should return the production status", async () => {
+    it.concurrent("should return the production status", async () => {
       const response = await request(TEST_URL).get("/is-production");
       expect(response.statusCode).toBe(200);
       expect(response.body).toHaveProperty("isProduction");
@@ -1003,8 +1011,8 @@ describe("E2E Tests for API Routes", () => {
   });
 
   describe("Rate Limiter", () => {
-    it("should return 429 when rate limit is exceeded for preview token", async () => {
-      for (let i = 0; i < 4; i++) {
+    it.concurrent("should return 429 when rate limit is exceeded for preview token", async () => {
+      for (let i = 0; i < 5; i++) {
         const response = await request(TEST_URL)
           .post("/v0/scrape")
           .set("Authorization", `Bearer this_is_just_a_preview_token`)
@@ -1023,7 +1031,7 @@ describe("E2E Tests for API Routes", () => {
     }, 60000);
   });
 
-  // it("should return 429 when rate limit is exceeded for API key", async () => {
+  // it.concurrent("should return 429 when rate limit is exceeded for API key", async () => {
   //   for (let i = 0; i < parseInt(process.env.RATE_LIMIT_TEST_API_KEY_SCRAPE); i++) {
   //     const response = await request(TEST_URL)
   //       .post("/v0/scrape")
@@ -1043,7 +1051,7 @@ describe("E2E Tests for API Routes", () => {
   //   expect(response.statusCode).toBe(429);
   // }, 60000);
 
-  // it("should return 429 when rate limit is exceeded for API key", async () => {
+  // it.concurrent("should return 429 when rate limit is exceeded for API key", async () => {
   //   for (let i = 0; i < parseInt(process.env.RATE_LIMIT_TEST_API_KEY_CRAWL); i++) {
   //     const response = await request(TEST_URL)
   //       .post("/v0/crawl")
diff --git a/apps/js-sdk/firecrawl/src/__tests__/e2e_withAuth/index.test.ts b/apps/js-sdk/firecrawl/src/__tests__/e2e_withAuth/index.test.ts
index 16b7ab7..c7dde69 100644
--- a/apps/js-sdk/firecrawl/src/__tests__/e2e_withAuth/index.test.ts
+++ b/apps/js-sdk/firecrawl/src/__tests__/e2e_withAuth/index.test.ts
@@ -8,94 +8,94 @@ const TEST_API_KEY = process.env.TEST_API_KEY;
 const API_URL = process.env.API_URL;
 
 describe('FirecrawlApp E2E Tests', () => {
-  test('should throw error for no API key', () => {
+  test.concurrent('should throw error for no API key', () => {
     expect(() => {
       new FirecrawlApp({ apiKey: null, apiUrl: API_URL });
     }).toThrow("No API key provided");
   });
 
-  test('should throw error for invalid API key on scrape', async () => {
+  test.concurrent('should throw error for invalid API key on scrape', async () => {
     const invalidApp = new FirecrawlApp({ apiKey: "invalid_api_key", apiUrl: API_URL });
-    await expect(invalidApp.scrapeUrl('https://firecrawl.dev')).rejects.toThrow("Request failed with status code 401");
+    await expect(invalidApp.scrapeUrl('https://roastmywebsite.ai')).rejects.toThrow("Request failed with status code 401");
   });
 
-  test('should throw error for blocklisted URL on scrape', async () => {
+  test.concurrent('should throw error for blocklisted URL on scrape', async () => {
     const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
     const blocklistedUrl = "https://facebook.com/fake-test";
     await expect(app.scrapeUrl(blocklistedUrl)).rejects.toThrow("Request failed with status code 403");
   });
 
-  test('should return successful response with valid preview token', async () => {
+  test.concurrent('should return successful response with valid preview token', async () => {
     const app = new FirecrawlApp({ apiKey: "this_is_just_a_preview_token", apiUrl: API_URL });
-    const response = await app.scrapeUrl('https://firecrawl.dev');
+    const response = await app.scrapeUrl('https://roastmywebsite.ai');
     expect(response).not.toBeNull();
-    expect(response.data.content).toContain("🔥 Firecrawl");
+    expect(response.data.content).toContain("_Roast_");
   }, 30000); // 30 seconds timeout
 
-  test('should return successful response for valid scrape', async () => {
+  test.concurrent('should return successful response for valid scrape', async () => {
     const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
-    const response = await app.scrapeUrl('https://firecrawl.dev');
+    const response = await app.scrapeUrl('https://roastmywebsite.ai');
     expect(response).not.toBeNull();
-    expect(response.data.content).toContain("🔥 Firecrawl");
+    expect(response.data.content).toContain("_Roast_");
     expect(response.data).toHaveProperty('markdown');
     expect(response.data).toHaveProperty('metadata');
     expect(response.data).not.toHaveProperty('html');
   }, 30000); // 30 seconds timeout
 
-  test('should return successful response with valid API key and include HTML', async () => {
+  test.concurrent('should return successful response with valid API key and include HTML', async () => {
     const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
-    const response = await app.scrapeUrl('https://firecrawl.dev', { pageOptions: { includeHtml: true } });
+    const response = await app.scrapeUrl('https://roastmywebsite.ai', { pageOptions: { includeHtml: true } });
     expect(response).not.toBeNull();
-    expect(response.data.content).toContain("🔥 Firecrawl");
-    expect(response.data.markdown).toContain("🔥 Firecrawl");
+    expect(response.data.content).toContain("_Roast_");
+    expect(response.data.markdown).toContain("_Roast_");
     expect(response.data.html).toContain("<h1");
   }, 30000); // 30 seconds timeout
 
-  test('should return successful response for valid scrape with PDF file', async () => {
+  test.concurrent('should return successful response for valid scrape with PDF file', async () => {
     const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
     const response = await app.scrapeUrl('https://arxiv.org/pdf/astro-ph/9301001.pdf');
     expect(response).not.toBeNull();
     expect(response.data.content).toContain('We present spectrophotometric observations of the Broad Line Radio Galaxy');
   }, 30000); // 30 seconds timeout
 
-  test('should return successful response for valid scrape with PDF file without explicit extension', async () => {
+  test.concurrent('should return successful response for valid scrape with PDF file without explicit extension', async () => {
     const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
     const response = await app.scrapeUrl('https://arxiv.org/pdf/astro-ph/9301001');
     expect(response).not.toBeNull();
     expect(response.data.content).toContain('We present spectrophotometric observations of the Broad Line Radio Galaxy');
   }, 30000); // 30 seconds timeout
 
-  test('should throw error for invalid API key on crawl', async () => {
+  test.concurrent('should throw error for invalid API key on crawl', async () => {
     const invalidApp = new FirecrawlApp({ apiKey: "invalid_api_key", apiUrl: API_URL });
-    await expect(invalidApp.crawlUrl('https://firecrawl.dev')).rejects.toThrow("Request failed with status code 401");
+    await expect(invalidApp.crawlUrl('https://roastmywebsite.ai')).rejects.toThrow("Request failed with status code 401");
   });
 
-  test('should throw error for blocklisted URL on crawl', async () => {
+  test.concurrent('should throw error for blocklisted URL on crawl', async () => {
     const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
     const blocklistedUrl = "https://twitter.com/fake-test";
     await expect(app.crawlUrl(blocklistedUrl)).rejects.toThrow("Request failed with status code 403");
   });
 
-  test('should return successful response for crawl and wait for completion', async () => {
+  test.concurrent('should return successful response for crawl and wait for completion', async () => {
     const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
-    const response = await app.crawlUrl('https://firecrawl.dev', { crawlerOptions: { excludes: ['blog/*'] } }, true, 30);
+    const response = await app.crawlUrl('https://roastmywebsite.ai', { crawlerOptions: { excludes: ['blog/*'] } }, true, 30);
     expect(response).not.toBeNull();
-    expect(response[0].content).toContain("🔥 Firecrawl");
+    expect(response[0].content).toContain("_Roast_");
   }, 60000); // 60 seconds timeout
 
-  test('should handle idempotency key for crawl', async () => {
+  test.concurrent('should handle idempotency key for crawl', async () => {
     const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
     const uniqueIdempotencyKey = uuidv4();
-    const response = await app.crawlUrl('https://firecrawl.dev', { crawlerOptions: { excludes: ['blog/*'] } }, false, 2, uniqueIdempotencyKey);
+    const response = await app.crawlUrl('https://roastmywebsite.ai', { crawlerOptions: { excludes: ['blog/*'] } }, false, 2, uniqueIdempotencyKey);
     expect(response).not.toBeNull();
     expect(response.jobId).toBeDefined();
 
-    await expect(app.crawlUrl('https://firecrawl.dev', { crawlerOptions: { excludes: ['blog/*'] } }, true, 2, uniqueIdempotencyKey)).rejects.toThrow("Request failed with status code 409");
+    await expect(app.crawlUrl('https://roastmywebsite.ai', { crawlerOptions: { excludes: ['blog/*'] } }, true, 2, uniqueIdempotencyKey)).rejects.toThrow("Request failed with status code 409");
   });
 
-  test('should check crawl status', async () => {
+  test.concurrent('should check crawl status', async () => {
     const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
-    const response = await app.crawlUrl('https://firecrawl.dev', { crawlerOptions: { excludes: ['blog/*'] } }, false);
+    const response = await app.crawlUrl('https://roastmywebsite.ai', { crawlerOptions: { excludes: ['blog/*'] } }, false);
     expect(response).not.toBeNull();
     expect(response.jobId).toBeDefined();
 
@@ -115,7 +115,7 @@ describe('FirecrawlApp E2E Tests', () => {
     expect(statusResponse.data.length).toBeGreaterThan(0);
   }, 35000); // 35 seconds timeout
 
-  test('should return successful response for search', async () => {
+  test.concurrent('should return successful response for search', async () => {
     const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
     const response = await app.search("test query");
     expect(response).not.toBeNull();
@@ -123,12 +123,12 @@ describe('FirecrawlApp E2E Tests', () => {
     expect(response.data.length).toBeGreaterThan(2);
   }, 30000); // 30 seconds timeout
 
-  test('should throw error for invalid API key on search', async () => {
+  test.concurrent('should throw error for invalid API key on search', async () => {
     const invalidApp = new FirecrawlApp({ apiKey: "invalid_api_key", apiUrl: API_URL });
     await expect(invalidApp.search("test query")).rejects.toThrow("Request failed with status code 401");
   });
 
-  test('should perform LLM extraction', async () => {
+  test.concurrent('should perform LLM extraction', async () => {
     const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
     const response = await app.scrapeUrl("https://mendable.ai", {
       extractorOptions: {
diff --git a/apps/python-sdk/firecrawl/__tests__/e2e_withAuth/test.py b/apps/python-sdk/firecrawl/__tests__/e2e_withAuth/test.py
index 86ce1f9..90a6498 100644
--- a/apps/python-sdk/firecrawl/__tests__/e2e_withAuth/test.py
+++ b/apps/python-sdk/firecrawl/__tests__/e2e_withAuth/test.py
@@ -38,31 +38,31 @@ def test_blocklisted_url():
 
 def test_successful_response_with_valid_preview_token():
     app = FirecrawlApp(api_url=API_URL, api_key="this_is_just_a_preview_token")
-    response = app.scrape_url('https://firecrawl.dev')
+    response = app.scrape_url('https://roastmywebsite.ai')
     assert response is not None
     assert 'content' in response
-    assert "🔥 Firecrawl" in response['content']
+    assert "_Roast_" in response['content']
 
 def test_scrape_url_e2e():
     app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY)
-    response = app.scrape_url('https://firecrawl.dev')
+    response = app.scrape_url('https://roastmywebsite.ai')
     assert response is not None
     assert 'content' in response
     assert 'markdown' in response
     assert 'metadata' in response
     assert 'html' not in response
-    assert "🔥 Firecrawl" in response['content']
+    assert "_Roast_" in response['content']
 
 def test_successful_response_with_valid_api_key_and_include_html():
     app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY)
-    response = app.scrape_url('https://firecrawl.dev', {'pageOptions': {'includeHtml': True}})
+    response = app.scrape_url('https://roastmywebsite.ai', {'pageOptions': {'includeHtml': True}})
     assert response is not None
     assert 'content' in response
     assert 'markdown' in response
     assert 'html' in response
     assert 'metadata' in response
-    assert "🔥 Firecrawl" in response['content']
-    assert "🔥 Firecrawl" in response['markdown']
+    assert "_Roast_" in response['content']
+    assert "_Roast_" in response['markdown']
     assert "<h1" in response['html']
 
 def test_successful_response_for_valid_scrape_with_pdf_file():
@@ -97,20 +97,20 @@ def test_should_return_error_for_blocklisted_url():
 
 def test_crawl_url_wait_for_completion_e2e():
     app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY)
-    response = app.crawl_url('https://firecrawl.dev', {'crawlerOptions': {'excludes': ['blog/*']}}, True)
+    response = app.crawl_url('https://roastmywebsite.ai', {'crawlerOptions': {'excludes': ['blog/*']}}, True)
     assert response is not None
     assert len(response) > 0
     assert 'content' in response[0]
-    assert "🔥 Firecrawl" in response[0]['content']
+    assert "_Roast_" in response[0]['content']
 
 def test_crawl_url_with_idempotency_key_e2e():
     app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY)
     uniqueIdempotencyKey = str(uuid4())
-    response = app.crawl_url('https://firecrawl.dev', {'crawlerOptions': {'excludes': ['blog/*']}}, True, 2, uniqueIdempotencyKey)
+    response = app.crawl_url('https://roastmywebsite.ai', {'crawlerOptions': {'excludes': ['blog/*']}}, True, 2, uniqueIdempotencyKey)
     assert response is not None
     assert len(response) > 0
     assert 'content' in response[0]
-    assert "🔥 Firecrawl" in response[0]['content']
+    assert "_Roast_" in response[0]['content']
 
     with pytest.raises(Exception) as excinfo:
         app.crawl_url('https://firecrawl.dev', {'crawlerOptions': {'excludes': ['blog/*']}}, True, 2, uniqueIdempotencyKey)

From b3ad42caf3da6b7125fecb726244b73f1d4ab9e9 Mon Sep 17 00:00:00 2001
From: rafaelsideguide <150964962+rafaelsideguide@users.noreply.github.com>
Date: Thu, 6 Jun 2024 15:46:40 -0300
Subject: [PATCH 19/23] updated workflows

---
 .github/workflows/js-sdk.yml     | 6 ++----
 .github/workflows/python-sdk.yml | 6 ++----
 2 files changed, 4 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/js-sdk.yml b/.github/workflows/js-sdk.yml
index 3c914cc..c84bb8b 100644
--- a/.github/workflows/js-sdk.yml
+++ b/.github/workflows/js-sdk.yml
@@ -1,9 +1,7 @@
 name: Run JavaScript SDK E2E Tests
   
-on:
-  pull_request:
-    branches:
-      - main
+on: []
+
 env:
   ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
   BULL_AUTH_KEY: ${{ secrets.BULL_AUTH_KEY }}
diff --git a/.github/workflows/python-sdk.yml b/.github/workflows/python-sdk.yml
index 1308cde..2744988 100644
--- a/.github/workflows/python-sdk.yml
+++ b/.github/workflows/python-sdk.yml
@@ -1,9 +1,7 @@
 name: Run Python SDK E2E Tests
   
-on:
-  pull_request:
-    branches:
-      - main
+on: []
+
 env:
   ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
   BULL_AUTH_KEY: ${{ secrets.BULL_AUTH_KEY }}

From 82e0ed4cd3a921dd71d185248f1c954817d615e2 Mon Sep 17 00:00:00 2001
From: rafaelsideguide <150964962+rafaelsideguide@users.noreply.github.com>
Date: Thu, 6 Jun 2024 16:33:27 -0300
Subject: [PATCH 20/23] Update index.test.ts

---
 .../src/__tests__/e2e_withAuth/index.test.ts  | 26 +++++++++----------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/apps/api/src/__tests__/e2e_withAuth/index.test.ts b/apps/api/src/__tests__/e2e_withAuth/index.test.ts
index 69e5316..62c3bfa 100644
--- a/apps/api/src/__tests__/e2e_withAuth/index.test.ts
+++ b/apps/api/src/__tests__/e2e_withAuth/index.test.ts
@@ -529,18 +529,18 @@ describe("E2E Tests for API Routes", () => {
       expect(response.statusCode).toBe(408);
     }, 3000); 
 
-    it.concurrent("should return a successful response with a valid API key for crawlWebsitePreview", async () => {
-      const response = await request(TEST_URL)
-        .post("/v0/crawlWebsitePreview")
-        .set("Authorization", `Bearer this_is_just_a_preview_token`)
-        .set("Content-Type", "application/json")
-        .send({ url: "https://firecrawl.dev" });
-      expect(response.statusCode).toBe(200);
-      expect(response.body).toHaveProperty("jobId");
-      expect(response.body.jobId).toMatch(
-        /^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$/
-      );
-    });
+    // it.concurrent("should return a successful response with a valid API key for crawlWebsitePreview", async () => {
+    //   const response = await request(TEST_URL)
+    //     .post("/v0/crawlWebsitePreview")
+    //     .set("Authorization", `Bearer this_is_just_a_preview_token`)
+    //     .set("Content-Type", "application/json")
+    //     .send({ url: "https://firecrawl.dev" });
+    //   expect(response.statusCode).toBe(200);
+    //   expect(response.body).toHaveProperty("jobId");
+    //   expect(response.body.jobId).toMatch(
+    //     /^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$/
+    //   );
+    // });
   });
 
   describe("POST /v0/search", () => {
@@ -1012,7 +1012,7 @@ describe("E2E Tests for API Routes", () => {
 
   describe("Rate Limiter", () => {
     it.concurrent("should return 429 when rate limit is exceeded for preview token", async () => {
-      for (let i = 0; i < 5; i++) {
+      for (let i = 0; i < 4; i++) {
         const response = await request(TEST_URL)
           .post("/v0/scrape")
           .set("Authorization", `Bearer this_is_just_a_preview_token`)

From 7b9b668b95bddb8c0c9f170f65dfd3b8d3b8dd10 Mon Sep 17 00:00:00 2001
From: rafaelsideguide <150964962+rafaelsideguide@users.noreply.github.com>
Date: Thu, 6 Jun 2024 16:36:51 -0300
Subject: [PATCH 21/23] Update index.test.ts

---
 apps/api/src/__tests__/e2e_withAuth/index.test.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/apps/api/src/__tests__/e2e_withAuth/index.test.ts b/apps/api/src/__tests__/e2e_withAuth/index.test.ts
index 62c3bfa..5cac4ad 100644
--- a/apps/api/src/__tests__/e2e_withAuth/index.test.ts
+++ b/apps/api/src/__tests__/e2e_withAuth/index.test.ts
@@ -1012,7 +1012,7 @@ describe("E2E Tests for API Routes", () => {
 
   describe("Rate Limiter", () => {
     it.concurrent("should return 429 when rate limit is exceeded for preview token", async () => {
-      for (let i = 0; i < 4; i++) {
+      for (let i = 0; i < 5; i++) {
         const response = await request(TEST_URL)
           .post("/v0/scrape")
           .set("Authorization", `Bearer this_is_just_a_preview_token`)

From cd7f9abcecf532aea164c0a3030dee27f547c12a Mon Sep 17 00:00:00 2001
From: rafaelsideguide <150964962+rafaelsideguide@users.noreply.github.com>
Date: Thu, 6 Jun 2024 16:44:46 -0300
Subject: [PATCH 22/23] Update index.test.ts

---
 apps/api/src/__tests__/e2e_withAuth/index.test.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/apps/api/src/__tests__/e2e_withAuth/index.test.ts b/apps/api/src/__tests__/e2e_withAuth/index.test.ts
index 5cac4ad..9360003 100644
--- a/apps/api/src/__tests__/e2e_withAuth/index.test.ts
+++ b/apps/api/src/__tests__/e2e_withAuth/index.test.ts
@@ -1028,7 +1028,7 @@ describe("E2E Tests for API Routes", () => {
         .send({ url: "https://www.scrapethissite.com" });
 
       expect(response.statusCode).toBe(429);
-    }, 60000);
+    }, 90000);
   });
 
   // it.concurrent("should return 429 when rate limit is exceeded for API key", async () => {

From f5318ea7d7f5833c99dee33fb257859069440a39 Mon Sep 17 00:00:00 2001
From: rafaelsideguide <150964962+rafaelsideguide@users.noreply.github.com>
Date: Thu, 6 Jun 2024 16:50:20 -0300
Subject: [PATCH 23/23] Update index.test.ts

---
 apps/api/src/__tests__/e2e_withAuth/index.test.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/apps/api/src/__tests__/e2e_withAuth/index.test.ts b/apps/api/src/__tests__/e2e_withAuth/index.test.ts
index 9360003..f619254 100644
--- a/apps/api/src/__tests__/e2e_withAuth/index.test.ts
+++ b/apps/api/src/__tests__/e2e_withAuth/index.test.ts
@@ -705,7 +705,7 @@ describe("E2E Tests for API Routes", () => {
         const depth = new URL(url).pathname.split("/").filter(Boolean).length;
         expect(depth).toBeLessThanOrEqual(1);
       });
-    }, 120000);
+    }, 180000);
 
     it.concurrent("should return a successful response for a valid crawl job with includeHtml set to true option", async () => {
       const crawlResponse = await request(TEST_URL)