Ollama: المراقبة والتسجيل

# Prometheus scrapes Ollama metrics
# Grafana visualizes real-time dashboards
# Alertmanager sends notifications when anomalies detected

# Linux: real-time Ollama logs
sudo journalctl -u ollama -f --no-pager

# Filter for errors
sudo journalctl -u ollama | grep -i "error\|fail\|oom\|panic"

# Docker: real-time container logs
docker logs -f ollama --tail 100

# Enable debug logging
sudo systemctl edit ollama
[Service]
Environment="OLLAMA_DEBUG=1"
sudo systemctl restart ollama

# Check for GPU issues in logs
sudo journalctl -u ollama --since "1 hour ago" | grep -i "cuda\|gpu\|memory"

# Ollama command executed successfully

import ollama
import time
import json
from datetime import datetime
from pathlib import Path

class OllamaMetrics:
    def __init__(self, log_file: str = "ollama_metrics.jsonl"):
        self.log_file = Path(log_file)

    def record_request(self, model: str, prompt: str) -> dict:
        start = time.time()
        try:
            response = ollama.chat(
                model=model,
                messages=[{"role": "user", "content": prompt}],
                stream=False
            )
            duration = time.time() - start
            content = response["message"]["content"]
            metrics = {
                "timestamp": datetime.now().isoformat(),
                "model": model,
                "duration_s": round(duration, 3),
                "response_length": len(content),
                "estimated_tokens": len(content) // 4,
                "tok_per_s": round(len(content) / 4 / duration, 1),
                "status": "success"
            }
        except Exception as e:
            duration = time.time() - start
            metrics = {
                "timestamp": datetime.now().isoformat(),
                "model": model,
                "duration_s": round(duration, 3),
                "status": "error",
                "error": str(e)
            }

        with open(self.log_file, "a") as f:
            f.write(json.dumps(metrics) + "\n")
        return metrics

    def get_stats(self, minutes: int = 30) -> dict:
        """Calculate stats from recent metrics."""
        cutoff = datetime.now().timestamp() - minutes * 60
        durations = []
        errors = 0
        total = 0

        if not self.log_file.exists():
            return {}

        with open(self.log_file) as f:
            for line in f:
                data = json.loads(line)
                ts = datetime.fromisoformat(data["timestamp"]).timestamp()
                if ts >= cutoff:
                    total += 1
                    durations.append(data.get("duration_s", 0))
                    if data.get("status") == "error":
                        errors += 1

        if total == 0:
            return {}

        return {
            "total_requests": total,
            "error_rate": round(errors / total * 100, 1),
            "avg_duration_s": round(sum(durations) / len(durations), 2),
            "max_duration_s": round(max(durations), 2),
            "p95_duration_s": round(sorted(durations)[int(len(durations) * 0.95)], 2)
        }

# Usage
metrics = OllamaMetrics()
metrics.record_request("qwen2.5", "Hello")
print(metrics.get_stats())

# Function defined successfully

# docker-compose.monitoring.yml
version: "3.8"

services:
  prometheus:
    image: prom/prometheus
    container_name: prometheus
    ports: ["9090:9090"]
    volumes:
      - ./prometheus.yml:/etc/prometheus/prometheus.yml
      - prometheus_data:/prometheus
    restart: unless-stopped

  grafana:
    image: grafana/grafana
    container_name: grafana
    ports: ["3001:3000"]
    volumes:
      - grafana_data:/var/lib/grafana
    environment:
      - GF_SECURITY_ADMIN_PASSWORD=admin
    depends_on: [prometheus]
    restart: unless-stopped

  node-exporter:
    image: prom/node-exporter
    container_name: node-exporter
    ports: ["9100:9100"]
    volumes:
      - /proc:/host/proc:ro
      - /sys:/host/sys:ro
    restart: unless-stopped

volumes:
  prometheus_data:
  grafana_data:

# prometheus.yml
global:
  scrape_interval: 15s

scrape_configs:
  - job_name: "node"
    static_configs:
      - targets: ["node-exporter:9100"]

  - job_name: "ollama_custom"
    static_configs:
      - targets: ["host.docker.internal:8000"]  # Custom exporter

Monitoring stack deployed successfully, Prometheus/Grafana services running normally

# alert_rules.yml
groups:
  - name: ollama_alerts
    rules:
      - alert: OllamaHighLatency
        expr: histogram_quantile(0.95, rate(ollama_request_duration_seconds_bucket[5m])) > 10
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Ollama P95 latency above 10s"
          description: "Current P95: {{ $value }}s"

      - alert: OllamaServiceDown
        expr: up{job="ollama_custom"} == 0
        for: 2m
        labels:
          severity: critical
        annotations:
          summary: "Ollama service is down"
          description: "No response from Ollama for 2 minutes"

      - alert: HighGPUMemory
        expr: DCGM_FI_DEV_FB_USED / DCGM_FI_DEV_FB_TOTAL > 0.95
        for: 2m
        labels:
          severity: critical
        annotations:
          summary: "GPU memory usage above 95%"

# Alert rules loaded successfully
kubectl exec -it prometheus-0 -- wget -qO- http://localhost:9090/api/v1/rules | head
# {"status":"success","data":{"groups":[{"name":"ollama_alerts","rules":[...]}]}}

#!/bin/bash
# Slow query diagnosis script

LOG="ollama_metrics.jsonl"

echo "=== Slow Query Report (last 60 minutes) ==="
echo ""

if [ -f "$LOG" ]; then
    # Find requests with duration > 5 seconds
    python3 -c "
import json, sys
from datetime import datetime, timedelta

cutoff = datetime.now() - timedelta(hours=1)
slow = []
with open('$LOG') as f:
    for line in f:
        data = json.loads(line)
        ts = datetime.fromisoformat(data['timestamp'])
        if ts >= cutoff and data.get('duration_s', 0) > 5:
            slow.append(data)

if slow:
    print(f'Found {len(slow)} slow requests (>5s):')
    for s in slow[:10]:
        print(f\"  {s['timestamp']}: {s['duration_s']}s, model={s.get('model','?')}, status={s.get('status','?')}\")
else:
    print('No slow requests found in the last hour.')
"
else
    echo "No metrics log found. Enable OllamaMetrics first."
fi

echo ""
echo "=== Current GPU Status ==="
nvidia-smi --query-gpu=utilization.gpu,memory.used,memory.total --format=csv 2>/dev/null || echo "No GPU detected"

NAME                    ID              SIZE    
llama3.2:latest        a80...          2.0 GB  
mistral:latest         61...           4.1 GB

# ============================================
# Comprehensive: Monitoring dashboard config
# Prometheus + Grafana + Custom exporter
# ============================================

import json
from http.server import HTTPServer, BaseHTTPRequestHandler
import ollama
import time
import threading

# Custom Prometheus exporter for Ollama
class OllamaExporter(BaseHTTPRequestHandler):
    metrics = {
        "ollama_request_duration_seconds": [],
        "ollama_requests_total": 0,
        "ollama_errors_total": 0,
        "ollama_active_models": 0,
    }

    def do_GET(self):
        if self.path == "/metrics":
            self.send_response(200)
            self.send_header("Content-Type", "text/plain")
            self.end_headers()

            output = []
            output.append(f"# HELP ollama_requests_total Total requests")
            output.append(f"# TYPE ollama_requests_total counter")
            output.append(f"ollama_requests_total {self.metrics['ollama_requests_total']}")

            output.append(f"# HELP ollama_errors_total Total errors")
            output.append(f"# TYPE ollama_errors_total counter")
            output.append(f"ollama_errors_total {self.metrics['ollama_errors_total']}")

            output.append(f"# HELP ollama_request_duration_seconds Request duration")
            output.append(f"# TYPE ollama_request_duration_seconds histogram")
            if self.metrics["ollama_request_duration_seconds"]:
                durations = self.metrics["ollama_request_duration_seconds"][-100:]
                avg = sum(durations) / len(durations)
                output.append(f'ollama_request_duration_seconds_avg {avg:.3f}')
                output.append(f'ollama_request_duration_seconds_max {max(durations):.3f}')

            # Active models
            try:
                models = ollama.list()
                active = len(models.get("models", []))
                output.append(f"# HELP ollama_active_models Number of loaded models")
                output.append(f"# TYPE ollama_active_models gauge")
                output.append(f"ollama_active_models {active}")
            except Exception:
                pass

            self.wfile.write("\n".join(output).encode())
        elif self.path == "/health":
            self.send_response(200)
            self.end_headers()
            self.wfile.write(b"OK")
        else:
            self.send_response(404)
            self.end_headers()

    @classmethod
    def record_request(cls, duration: float, is_error: bool = False):
        cls.metrics["ollama_requests_total"] += 1
        cls.metrics["ollama_request_duration_seconds"].append(duration)
        if is_error:
            cls.metrics["ollama_errors_total"] += 1

def run_exporter(port: int = 8000):
    server = HTTPServer(("0.0.0.0", port), OllamaExporter)
    print(f"Ollama exporter running on :{port}/metrics")
    server.serve_forever()

# Start exporter in background thread
# threading.Thread(target=run_exporter, daemon=True).start()
# Then configure Prometheus to scrape http://localhost:8000/metrics