Ollama: Production Deployment

# /etc/nginx/conf.d/ollama-lb.conf

upstream ollama_cluster {
    least_conn;  # Route to least busy node

    server ollama-node1:11434 max_fails=3 fail_timeout=30s;
    server ollama-node2:11434 max_fails=3 fail_timeout=30s;
    server ollama-node3:11434 max_fails=3 fail_timeout=30s;
}

server {
    listen 443 ssl;
    server_name ai.example.com;

    ssl_certificate     /etc/ssl/certs/ai.example.com.crt;
    ssl_certificate_key /etc/ssl/private/ai.example.com.key;

    location /v1/ {
        # API Key authentication
        if ($http_x_api_key != "your-secret-key") {
            return 401 '{"error": "Unauthorized"}';
        }

        proxy_pass http://ollama_cluster;
        proxy_set_header Host $host;
        proxy_set_header X-Real-IP $remote_addr;
        proxy_connect_timeout 5s;
        proxy_read_timeout 120s;

        # Rate limiting
        limit_req zone=api burst=20 nodelay;
    }

    # Health check endpoint
    location /health {
        proxy_pass http://ollama_cluster/api/tags;
    }
}

// Execution successful

#!/bin/bash
# Ollama cluster health monitor

NODES=("ollama-node1:11434" "ollama-node2:11434" "ollama-node3:11434")
HEALTH_URL="/api/tags"
ALERT_EMAIL="ops@example.com"

check_node() {
    local node=$1
    if curl -sf --connect-timeout 3 "http://$node$HEALTH_URL" > /dev/null; then
        echo "UP"
    else
        echo "DOWN"
    fi
}

while true; do
    for node in "${NODES[@]}"; do
        status=$(check_node "$node")
        if [ "$status" = "DOWN" ]; then
            echo "ALERT: $node is DOWN at $(date)" >> /var/log/ollama_health.log
            # Send alert
            echo "Ollama node $node is DOWN" | mail -s "OLLAMA ALERT" "$ALERT_EMAIL" 2>/dev/null
        fi
    done
    sleep 10
done

NAME                    ID              SIZE    
llama3.2:latest        a80...          2.0 GB  
mistral:latest         61...           4.1 GB

import subprocess
import time
from typing import Optional

class AutoScaler:
    def __init__(self, scale_up_threshold: float = 5.0,
                 scale_down_threshold: float = 2.0,
                 cooldown: int = 300):
        self.scale_up_threshold = scale_up_threshold
        self.scale_down_threshold = scale_down_threshold
        self.cooldown = cooldown
        self.last_scale_time = 0

    def get_avg_latency(self) -> Optional[float]:
        try:
            result = subprocess.run(
                ["curl", "-sf", "http://localhost:11434/api/chat", "-d",
                 '{"model":"qwen2.5","messages":[{"role":"user","content":"hi"}],"stream":false}'],
                capture_output=True, text=True, timeout=10
            )
            if result.returncode == 0:
                import json
                data = json.loads(result.stdout)
                duration_ns = data.get("total_duration", 0)
                return duration_ns / 1e9
        except Exception:
            pass
        return None

    def check_and_scale(self):
        latency = self.get_avg_latency()
        if latency is None:
            return

        now = time.time()
        if now - self.last_scale_time < self.cooldown:
            return

        if latency > self.scale_up_threshold:
            print(f"High latency ({latency:.1f}s), scaling up...")
            self._scale_up()
            self.last_scale_time = now
        elif latency < self.scale_down_threshold:
            print(f"Low latency ({latency:.1f}s), scaling down...")
            self._scale_down()
            self.last_scale_time = now

    def _scale_up(self):
        # Add new Ollama node (e.g., via Docker or cloud API)
        print("Adding Ollama node...")

    def _scale_down(self):
        # Remove idle Ollama node
        print("Removing idle Ollama node...")

# Usage
# scaler = AutoScaler()
# while True:
#     scaler.check_and_scale()
#     time.sleep(60)

Adding Ollama node...
Removing idle Ollama node...

# ollama-deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: ollama
  labels:
    app: ollama
spec:
  replicas: 3
  selector:
    matchLabels:
      app: ollama
  template:
    metadata:
      labels:
        app: ollama
    spec:
      nodeSelector:
        gpu: "true"
      tolerations:
        - key: nvidia.com/gpu
          operator: Exists
          effect: NoSchedule
      containers:
        - name: ollama
          image: ollama/ollama
          ports:
            - containerPort: 11434
          resources:
            requests:
              nvidia.com/gpu: 1
              memory: "8Gi"
            limits:
              nvidia.com/gpu: 1
              memory: "16Gi"
          env:
            - name: OLLAMA_HOST
              value: "0.0.0.0:11434"
            - name: OLLAMA_NUM_PARALLEL
              value: "4"
            - name: OLLAMA_KEEP_ALIVE
              value: "30m"
          volumeMounts:
            - name: model-storage
              mountPath: /root/.ollama
          livenessProbe:
            httpGet:
              path: /api/tags
              port: 11434
            initialDelaySeconds: 30
            periodSeconds: 10
          readinessProbe:
            httpGet:
              path: /api/tags
              port: 11434
            initialDelaySeconds: 10
            periodSeconds: 5
      volumes:
        - name: model-storage
          persistentVolumeClaim:
            claimName: ollama-models-pvc
---
apiVersion: v1
kind: Service
metadata:
  name: ollama-svc
spec:
  selector:
    app: ollama
  ports:
    - port: 11434
      targetPort: 11434
  type: ClusterIP

K8s Deployment created successfully, Pods running normally

HPA auto-scaling configured successfully, cluster resource monitoring enabled

# ollama-hpa.yaml
apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
  name: ollama-hpa
spec:
  scaleTargetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: ollama
  minReplicas: 2
  maxReplicas: 6
  metrics:
    - type: Resource
      resource:
        name: nvidia.com/gpu
        target:
          type: Utilization
          averageUtilization: 80
  behavior:
    scaleUp:
      stabilizationWindowSeconds: 60
      policies:
        - type: Pods
          value: 1
          periodSeconds: 60
    scaleDown:
      stabilizationWindowSeconds: 300
      policies:
        - type: Pods
          value: 1
          periodSeconds: 300

# HPA created successfully
kubectl get hpa ollama-hpa
# NAME          REFERENCE            TARGETS         MINPODS   MAXPODS   REPLICAS   AGE
# ollama-hpa   Deployment/ollama    45%/80%         2         6         3          5m

# ============================================
# Comprehensive: Production deployment checklist
# Full verification before going live
# ============================================

PRODUCTION_CHECKLIST = {
    "infrastructure": [
        ("Ollama nodes: 3+ replicas running", "CRITICAL"),
        ("GPU drivers: installed and verified", "CRITICAL"),
        ("Model storage: shared NFS/S3 mounted", "CRITICAL"),
        ("Network: internal VLAN for Ollama traffic", "HIGH"),
        ("DNS: ai.example.com resolves to LB", "HIGH"),
    ],
    "security": [
        ("Ollama bound to 127.0.0.1 on each node", "CRITICAL"),
        ("Nginx SSL certificate valid", "CRITICAL"),
        ("API Key authentication enabled", "CRITICAL"),
        ("Rate limiting configured (60 req/min)", "HIGH"),
        ("WAF rules in place", "MEDIUM"),
    ],
    "high_availability": [
        ("Health checks configured (Nginx + app layer)", "CRITICAL"),
        ("Failover tested: kill 1 node, service continues", "CRITICAL"),
        ("Load balancer: least_conn algorithm", "HIGH"),
        ("Session affinity: disabled (stateless)", "HIGH"),
    ],
    "monitoring": [
        ("Prometheus scraping all nodes", "HIGH"),
        ("Grafana dashboards created", "HIGH"),
        ("Alert rules: GPU OOM, high latency, service down", "CRITICAL"),
        ("Log aggregation: Loki or ELK", "MEDIUM"),
    ],
    "disaster_recovery": [
        ("Config in Git (Modelfile, Compose, K8s)", "HIGH"),
        ("Model backup: GGUF files on S3/NFS", "HIGH"),
        ("Chroma data backup: daily snapshot", "HIGH"),
        ("Recovery drill: tested within last 30 days", "MEDIUM"),
    ],
    "performance": [
        ("Baseline benchmark recorded", "HIGH"),
        ("OLLAMA_NUM_PARALLEL configured", "HIGH"),
        ("OLLAMA_KEEP_ALIVE=30m set", "MEDIUM"),
        ("num_ctx per endpoint optimized", "MEDIUM"),
    ]
}

def run_checklist() -> str:
    report = ["# Production Deployment Checklist\n"]
    total = 0
    checked = 0

    for category, items in PRODUCTION_CHECKLIST.items():
        report.append(f"\n## {category.replace('_', ' ').title()}")
        for item, priority in items:
            total += 1
            report.append(f"- [ ] [{priority}] {item}")

    report.append(f"\n---\nTotal items: {total}")
    return "\n".join(report)

print(run_checklist())