Merge pull request #236 from JakobStadlhuber/doc/kubernetes

Add Kubernetes configuration for Firecrawl deployment

SELF_HOST.md

@@ -29,3 +29,6 @@ docker compose up
 
 
 This will run a local instance of Firecrawl which can be accessed at `http://localhost:3002`.
+
+# Install Firecrawl on a Kubernetes Cluster (Simple Version)
+Read the [examples/k8n/README.md](examples/k8n/README.md) for instructions on how to install Firecrawl on a Kubernetes Cluster.

examples/k8n/README.md

@@ -0,0 +1,41 @@
+# Install Firecrawl on a Kubernetes Cluster (Simple Version)
+# Before installing
+1. Set [secret.yaml](secret.yaml) and [configmap.yaml](configmap.yaml) and do not check in secrets
+2. Build Docker images, and host it in your Docker Registry (replace the target registry with your own)
+   1. API (which is also used as a worker image)
+      1. ```bash
+         docker build -t ghcr.io/winkk-dev/firecrawl:latest ../../apps/api
+         docker push ghcr.io/winkk-dev/firecrawl:latest
+         ```
+   2. Playwright 
+      1. ```bash
+            docker build -t ghcr.io/winkk-dev/firecrawl-playwright:latest ../../apps/playwright-service
+            docker push ghcr.io/winkk-dev/firecrawl-playwright:latest
+         ```
+3. Replace the image in [worker.yaml](worker.yaml), [api.yaml](api.yaml) and [playwright-service.yaml](playwright-service.yaml)
+
+## Install
+```bash
+kubectl apply -f configmap.yaml
+kubectl apply -f secret.yaml
+kubectl apply -f playwright-service.yaml
+kubectl apply -f api.yaml
+kubectl apply -f worker.yaml
+kubectl apply -f redis.yaml
+```
+
+
+# Port Forwarding for Testing
+```bash
+kubectl port-forward svc/api 3002:3002 -n dev
+```
+
+# Delete Firecrawl
+```bash
+kubectl delete -f configmap.yaml
+kubectl delete -f secret.yaml
+kubectl delete -f playwright-service.yaml
+kubectl delete -f api.yaml
+kubectl delete -f worker.yaml
+kubectl delete -f redis.yaml
+```

examples/k8n/api.yaml

@@ -0,0 +1,39 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: api
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: api
+  template:
+    metadata:
+      labels:
+        app: api
+    spec:
+      imagePullSecrets:
+        - name: docker-registry-secret
+      containers:
+      - name: api
+        image: ghcr.io/winkk-dev/firecrawl:latest
+        args:  [ "pnpm", "run", "start:production" ]
+        ports:
+        - containerPort: 3002
+        envFrom:
+        - configMapRef:
+            name: firecrawl-config
+        - secretRef:
+            name: firecrawl-secret
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: api
+spec:
+  selector:
+    app: api
+  ports:
+    - protocol: TCP
+      port: 3002
+      targetPort: 3002

examples/k8n/configmap.yaml

@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: firecrawl-config
+data:
+  NUM_WORKERS_PER_QUEUE: "8"
+  PORT: "3002"
+  HOST: "0.0.0.0"
+  REDIS_URL: "redis://redis:6379"
+  PLAYWRIGHT_MICROSERVICE_URL: "http://playwright-service:3000"
+  USE_DB_AUTHENTICATION: "false"
+  SUPABASE_ANON_TOKEN: ""
+  SUPABASE_URL: ""
+  SUPABASE_SERVICE_TOKEN: ""

examples/k8n/playwright-service.yaml

@@ -0,0 +1,36 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: playwright-service
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: playwright-service
+  template:
+    metadata:
+      labels:
+        app: playwright-service
+    spec:
+      imagePullSecrets:
+        - name: docker-registry-secret
+      containers:
+      - name: playwright-service
+        image: ghcr.io/winkk-dev/firecrawl-playwright:latest
+        ports:
+        - containerPort: 3000
+        envFrom:
+        - configMapRef:
+            name: firecrawl-config
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: playwright-service
+spec:
+  selector:
+    app: playwright-service
+  ports:
+    - protocol: TCP
+      port: 3000
+      targetPort: 3000

examples/k8n/redis.yaml

@@ -0,0 +1,30 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: redis
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: redis
+  template:
+    metadata:
+      labels:
+        app: redis
+    spec:
+      containers:
+      - name: redis
+        image: redis:alpine
+        args: ["redis-server", "--bind", "0.0.0.0"]
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: redis
+spec:
+  selector:
+    app: redis
+  ports:
+    - protocol: TCP
+      port: 6379
+      targetPort: 6379

examples/k8n/secret.yaml

@@ -0,0 +1,20 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: firecrawl-secret
+type: Opaque
+data:
+  OPENAI_API_KEY: ""
+  SLACK_WEBHOOK_URL: ""
+  SERPER_API_KEY: ""
+  LLAMAPARSE_API_KEY: ""
+  LOGTAIL_KEY: ""
+  BULL_AUTH_KEY: ""
+  TEST_API_KEY: ""
+  POSTHOG_API_KEY: ""
+  POSTHOG_HOST: ""
+  SCRAPING_BEE_API_KEY: ""
+  STRIPE_PRICE_ID_STANDARD: ""
+  STRIPE_PRICE_ID_SCALE: ""
+  HYPERDX_API_KEY: ""
+  FIRE_ENGINE_BETA_URL: ""

examples/k8n/worker.yaml

@@ -0,0 +1,24 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: worker
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: worker
+  template:
+    metadata:
+      labels:
+        app: worker
+    spec:
+      imagePullSecrets:
+        - name: docker-registry-secret
+      containers:
+      - name: worker
+        image: ghcr.io/winkk-dev/firecrawl:latest
+        envFrom:
+        - configMapRef:
+            name: firecrawl-config
+        - secretRef:
+            name: firecrawl-secret