diff --git a/SELF_HOST.md b/SELF_HOST.md index ff5ee04..f74e050 100644 --- a/SELF_HOST.md +++ b/SELF_HOST.md @@ -29,3 +29,6 @@ docker compose up This will run a local instance of Firecrawl which can be accessed at `http://localhost:3002`. + +# Install Firecrawl on a Kubernetes Cluster (Simple Version) +Read the [examples/k8n/README.md](examples/k8n/README.md) for instructions on how to install Firecrawl on a Kubernetes Cluster. \ No newline at end of file diff --git a/examples/k8n/README.md b/examples/k8n/README.md new file mode 100644 index 0000000..f874d82 --- /dev/null +++ b/examples/k8n/README.md @@ -0,0 +1,41 @@ +# Install Firecrawl on a Kubernetes Cluster (Simple Version) +# Before installing +1. Set [secret.yaml](secret.yaml) and [configmap.yaml](configmap.yaml) and do not check in secrets +2. Build Docker images, and host it in your Docker Registry (replace the target registry with your own) + 1. API (which is also used as a worker image) + 1. ```bash + docker build -t ghcr.io/winkk-dev/firecrawl:latest ../../apps/api + docker push ghcr.io/winkk-dev/firecrawl:latest + ``` + 2. Playwright + 1. ```bash + docker build -t ghcr.io/winkk-dev/firecrawl-playwright:latest ../../apps/playwright-service + docker push ghcr.io/winkk-dev/firecrawl-playwright:latest + ``` +3. Replace the image in [worker.yaml](worker.yaml), [api.yaml](api.yaml) and [playwright-service.yaml](playwright-service.yaml) + +## Install +```bash +kubectl apply -f configmap.yaml +kubectl apply -f secret.yaml +kubectl apply -f playwright-service.yaml +kubectl apply -f api.yaml +kubectl apply -f worker.yaml +kubectl apply -f redis.yaml +``` + + +# Port Forwarding for Testing +```bash +kubectl port-forward svc/api 3002:3002 -n dev +``` + +# Delete Firecrawl +```bash +kubectl delete -f configmap.yaml +kubectl delete -f secret.yaml +kubectl delete -f playwright-service.yaml +kubectl delete -f api.yaml +kubectl delete -f worker.yaml +kubectl delete -f redis.yaml +``` \ No newline at end of file diff --git a/examples/k8n/api.yaml b/examples/k8n/api.yaml new file mode 100644 index 0000000..cdc69c3 --- /dev/null +++ b/examples/k8n/api.yaml @@ -0,0 +1,39 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: api +spec: + replicas: 1 + selector: + matchLabels: + app: api + template: + metadata: + labels: + app: api + spec: + imagePullSecrets: + - name: docker-registry-secret + containers: + - name: api + image: ghcr.io/winkk-dev/firecrawl:latest + args: [ "pnpm", "run", "start:production" ] + ports: + - containerPort: 3002 + envFrom: + - configMapRef: + name: firecrawl-config + - secretRef: + name: firecrawl-secret +--- +apiVersion: v1 +kind: Service +metadata: + name: api +spec: + selector: + app: api + ports: + - protocol: TCP + port: 3002 + targetPort: 3002 diff --git a/examples/k8n/configmap.yaml b/examples/k8n/configmap.yaml new file mode 100644 index 0000000..b415d56 --- /dev/null +++ b/examples/k8n/configmap.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: firecrawl-config +data: + NUM_WORKERS_PER_QUEUE: "8" + PORT: "3002" + HOST: "0.0.0.0" + REDIS_URL: "redis://redis:6379" + PLAYWRIGHT_MICROSERVICE_URL: "http://playwright-service:3000" + USE_DB_AUTHENTICATION: "false" + SUPABASE_ANON_TOKEN: "" + SUPABASE_URL: "" + SUPABASE_SERVICE_TOKEN: "" diff --git a/examples/k8n/playwright-service.yaml b/examples/k8n/playwright-service.yaml new file mode 100644 index 0000000..ce79425 --- /dev/null +++ b/examples/k8n/playwright-service.yaml @@ -0,0 +1,36 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: playwright-service +spec: + replicas: 1 + selector: + matchLabels: + app: playwright-service + template: + metadata: + labels: + app: playwright-service + spec: + imagePullSecrets: + - name: docker-registry-secret + containers: + - name: playwright-service + image: ghcr.io/winkk-dev/firecrawl-playwright:latest + ports: + - containerPort: 3000 + envFrom: + - configMapRef: + name: firecrawl-config +--- +apiVersion: v1 +kind: Service +metadata: + name: playwright-service +spec: + selector: + app: playwright-service + ports: + - protocol: TCP + port: 3000 + targetPort: 3000 diff --git a/examples/k8n/redis.yaml b/examples/k8n/redis.yaml new file mode 100644 index 0000000..774d371 --- /dev/null +++ b/examples/k8n/redis.yaml @@ -0,0 +1,30 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: redis +spec: + replicas: 1 + selector: + matchLabels: + app: redis + template: + metadata: + labels: + app: redis + spec: + containers: + - name: redis + image: redis:alpine + args: ["redis-server", "--bind", "0.0.0.0"] +--- +apiVersion: v1 +kind: Service +metadata: + name: redis +spec: + selector: + app: redis + ports: + - protocol: TCP + port: 6379 + targetPort: 6379 diff --git a/examples/k8n/secret.yaml b/examples/k8n/secret.yaml new file mode 100644 index 0000000..2be9632 --- /dev/null +++ b/examples/k8n/secret.yaml @@ -0,0 +1,20 @@ +apiVersion: v1 +kind: Secret +metadata: + name: firecrawl-secret +type: Opaque +data: + OPENAI_API_KEY: "" + SLACK_WEBHOOK_URL: "" + SERPER_API_KEY: "" + LLAMAPARSE_API_KEY: "" + LOGTAIL_KEY: "" + BULL_AUTH_KEY: "" + TEST_API_KEY: "" + POSTHOG_API_KEY: "" + POSTHOG_HOST: "" + SCRAPING_BEE_API_KEY: "" + STRIPE_PRICE_ID_STANDARD: "" + STRIPE_PRICE_ID_SCALE: "" + HYPERDX_API_KEY: "" + FIRE_ENGINE_BETA_URL: "" diff --git a/examples/k8n/worker.yaml b/examples/k8n/worker.yaml new file mode 100644 index 0000000..2b3b2e7 --- /dev/null +++ b/examples/k8n/worker.yaml @@ -0,0 +1,24 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: worker +spec: + replicas: 1 + selector: + matchLabels: + app: worker + template: + metadata: + labels: + app: worker + spec: + imagePullSecrets: + - name: docker-registry-secret + containers: + - name: worker + image: ghcr.io/winkk-dev/firecrawl:latest + envFrom: + - configMapRef: + name: firecrawl-config + - secretRef: + name: firecrawl-secret