Ollama: GPU and CUDA Configuration

# Verify GPU detection
ollama run --verbose llama3.2 "test"

# Before (CPU): eval speed: 5.0 tokens/s
# After (GPU):  eval speed: 55.0 tokens/s

VRAM needed (GB) ≈ Parameters (B) × Quantization bytes × 1.2 (overhead)

Examples:
  7B Q4_K_M:  7 × 0.5 bytes × 1.2 ≈ 4.2 GB
  8B Q4_K_M:  8 × 0.5 bytes × 1.2 ≈ 4.8 GB
  70B Q4_K_M: 70 × 0.5 bytes × 1.2 ≈ 42 GB

# Step 1: Check GPU is visible
nvidia-smi --query-gpu=name,memory.total,driver_version --format=csv
# NVIDIA GeForce RTX 4090, 24576 MiB, 550.54.15

# Step 2: Verify CUDA version
nvidia-smi | grep "CUDA Version"
# CUDA Version: 12.4

# Step 3: Test Ollama GPU acceleration
ollama run --verbose llama3.2 "Hello"
# Look for: "load duration" (should be < 1s with GPU)
# Look for: "eval speed" (should be 30+ tokens/s with GPU)

# Step 4: Check which device Ollama is using
OLLAMA_DEBUG=1 ollama run llama3.2 "test" 2>&1 | grep -i cuda
# cuda: detected devices [0]

I'm a helpful AI assistant running locally on your machine...

# On Windows: verify NVIDIA driver installed
nvidia-smi

# Enter WSL2
wsl

# Inside WSL2: verify GPU passthrough
nvidia-smi
# Should show same GPU as Windows

# Install Ollama in WSL2
curl -fsSL https://ollama.com/install.sh | sh

# Test GPU inference
ollama run --verbose llama3.2 "Hello"
# eval speed should show GPU-level performance

// Execution successful

# Verify ROCm is working
rocminfo | grep "gfx"
# Should show gfx1100 (RDNA3) or gfx1030 (RDNA2)

# Ollama auto-detects AMD GPU on Linux
ollama run --verbose llama3.2 "Hello"
# Should show: using AMD GPU

# On macOS: Metal acceleration is automatic
ollama run --verbose llama3.2 "Hello"

# Check Metal usage in logs
# Look for: "using Metal" in debug output
OLLAMA_DEBUG=1 ollama run llama3.2 "test" 2>&1 | grep -i metal

# Monitor GPU usage
# Open Activity Monitor → GPU tab
# Or use: sudo powermetrics --samplers gpu_power -i 1000

I'm a helpful AI assistant running locally on your machine...

# Use 2 GPUs for inference
export OLLAMA_NUM_GPU=2
ollama serve

# Or specify which GPUs to use
export CUDA_VISIBLE_DEVICES=0,1
ollama serve

# Verify multi-GPU usage
nvidia-smi  # Both GPUs should show memory usage during inference

# Run a large model that needs 2 GPUs
ollama run llama3.1:70b "Explain quantum computing"

I'm a helpful AI assistant running locally on your machine...

#!/bin/bash
# GPU memory diagnostic script

echo "=== GPU Memory Report ==="
echo ""

nvidia-smi --query-gpu=index,name,memory.total,memory.used,memory.free --format=csv

echo ""
echo "=== Model Memory Estimation ==="

estimate_vram() {
    local params=$1
    local quant=$2
    local overhead=1.2
    local vram=$(echo "scale=1; $params * $quant * $overhead" | bc)
    echo "  ${params}B model (${quant}B/param): ~${vram}GB VRAM needed"
}

estimate_vram 3 0.5   # Q4_K_M ~0.5 bytes/param
estimate_vram 8 0.5   # Q4_K_M
estimate_vram 8 1.0   # Q8_0
estimate_vram 70 0.5  # Q4_K_M
estimate_vram 70 1.0  # Q8_0

echo ""
echo "=== Recommendation ==="
total_vram=$(nvidia-smi --query-gpu=memory.total --format=csv,noheader,nounits | awk '{sum+=$1} END {print sum}')
echo "Total VRAM: ${total_vram}MB (~$((total_vram/1024))GB)"

if [ "$total_vram" -gt 40000 ]; then
    echo "Can run: 70B Q4_K_M (recommended) or 8B Q8_0"
elif [ "$total_vram" -gt 8000 ]; then
    echo "Can run: 8B Q4_K_M (recommended)"
elif [ "$total_vram" -gt 4000 ]; then
    echo "Can run: 3B Q4_K_M"
else
    echo "GPU VRAM too low. Use CPU-only mode with 16GB+ RAM."
fi

# Command executed successfully

# ============================================
# Comprehensive: GPU deployment decision engine
# Analyzes hardware and recommends model config
# ============================================

import subprocess
import json
from dataclasses import dataclass
from typing import Optional

@dataclass
class GPUInfo:
    name: str
    vram_mb: int
    index: int

@dataclass
class ModelRecommendation:
    model: str
    quantization: str
    vram_needed_gb: float
    fits: bool
    strategy: str

def detect_gpus() -> list[GPUInfo]:
    """Detect available GPUs via nvidia-smi."""
    try:
        result = subprocess.run(
            ["nvidia-smi", "--query-gpu=index,name,memory.total",
             "--format=csv,noheader,nounits"],
            capture_output=True, text=True
        )
        gpus = []
        for line in result.stdout.strip().split("\n"):
            if line:
                parts = [p.strip() for p in line.split(",")]
                gpus.append(GPUInfo(
                    index=int(parts[0]),
                    name=parts[1],
                    vram_mb=int(float(parts[2]))
                ))
        return gpus
    except Exception:
        return []

def estimate_vram(params_b: float, quant_bytes: float) -> float:
    """Estimate VRAM needed for a model."""
    return params_b * quant_bytes * 1.2

def recommend_models(total_vram_gb: float) -> list[ModelRecommendation]:
    """Recommend models based on available VRAM."""
    models = [
        ("llama3.2:3b", 3.2, 0.5, "Q4_K_M"),
        ("llama3.1:8b", 8.0, 0.5, "Q4_K_M"),
        ("llama3.1:8b", 8.0, 1.0, "Q8_0"),
        ("llama3.1:70b", 70.0, 0.5, "Q4_K_M"),
    ]
    recs = []
    for name, params, qbytes, quant in models:
        needed = estimate_vram(params, qbytes)
        fits = needed <= total_vram_gb
        if fits:
            strategy = "Full GPU offload"
        elif needed <= total_vram_gb * 1.5:
            strategy = "Hybrid (partial GPU + CPU)"
        else:
            strategy = "CPU only or multi-GPU required"
        recs.append(ModelRecommendation(
            model=name, quantization=quant,
            vram_needed_gb=round(needed, 1),
            fits=fits, strategy=strategy
        ))
    return recs

# Main analysis
if __name__ == "__main__":
    gpus = detect_gpus()
    if not gpus:
        print("No NVIDIA GPU detected. Using CPU-only mode.")
        print("Recommended: llama3.2:3b with 8GB+ RAM")
    else:
        total_vram = sum(g.vram_mb for g in gpus) / 1024
        print(f"Detected {len(gpus)} GPU(s):")
        for g in gpus:
            print(f"  GPU {g.index}: {g.name} ({g.vram_mb}MB)")
        print(f"Total VRAM: {total_vram:.1f}GB")
        print()
        print("Model Recommendations:")
        for r in recommend_models(total_vram):
            status = "✅" if r.fits else "❌"
            print(f"  {status} {r.model} ({r.quantization}): "
                  f"{r.vram_needed_gb}GB - {r.strategy}")