Ollama: Model Quantization and Optimization

# Compare model sizes
ollama show llama3.1:8b        # Q4_K_M: ~5GB
ollama show llama3.1:8b-q8_0   # Q8_0:   ~8GB

# Default pull uses optimal quantization (usually Q4_K_M)
ollama pull llama3.1:8b

# Show quantization details
ollama show llama3.1:8b
# Look for: quantization: Q4_K_M

# Compare sizes across quantizations
ollama list | grep llama
# llama3.1:8b          4.9 GB   (Q4_K_M)

NAME                    ID              SIZE    
llama3.2:latest        a80...          2.0 GB  
mistral:latest         61...           4.1 GB

# Pull specific quantization
ollama pull llama3.1:8b-q4_K_M   # 4-bit, ~5GB
ollama pull llama3.1:8b-q5_K_M   # 5-bit, ~5.7GB
ollama pull llama3.1:8b-q8_0     # 8-bit, ~8GB

# Not all models have all variants
# Check available tags on ollama.com/library/model-name

NAME                    ID              SIZE    
llama3.2:latest        a80...          2.0 GB  
mistral:latest         61...           4.1 GB

# Step 1: Download GGUF file from HuggingFace
wget "https://huggingface.co/TheBloke/Mistral-7B-Instruct-v0.2-GGUF/resolve/main/mistral-7b-instruct-v0.2.Q4_K_M.gguf"

# Step 2: Create Modelfile
cat > Modelfile <<EOF
FROM ./mistral-7b-instruct-v0.2.Q4_K_M.gguf
SYSTEM You are a helpful AI assistant.
PARAMETER temperature 0.7
PARAMETER num_ctx 4096
TEMPLATE """[INST] {{ if .System }}{{ .System }}{{ end }}
{{ .Prompt }} [/INST]
"""
EOF

# Step 3: Create model in Ollama
ollama create my-mistral -f Modelfile

# Step 4: Run
ollama run my-mistral "Explain quantum computing in simple terms"

I'm a helpful AI assistant running locally on your machine...

import ollama
import time
from statistics import mean

def compare_quantizations(model_base: str, quantizations: list[str],
                          test_prompt: str, runs: int = 3):
    """Compare output quality and speed across quantizations."""
    results = {}
    for quant in quantizations:
        model_name = f"{model_base}:{quant}" if ":" not in model_base else model_base
        try:
            # Warm up
            ollama.chat(model=model_name,
                        messages=[{"role": "user", "content": "warm up"}],
                        stream=False)

            speeds = []
            responses = []
            for _ in range(runs):
                start = time.time()
                resp = ollama.chat(
                    model=model_name,
                    messages=[{"role": "user", "content": test_prompt}],
                    stream=False
                )
                elapsed = time.time() - start
                speeds.append(len(resp["message"]["content"]) / elapsed)
                responses.append(resp["message"]["content"])

            results[quant] = {
                "avg_speed": round(mean(speeds), 1),
                "sample_response": responses[0][:200]
            }
        except Exception as e:
            results[quant] = {"error": str(e)}

    return results

# Compare (if you have multiple quantizations pulled)
# results = compare_quantizations(
#     "llama3.1", ["q4_K_M", "q8_0"],
#     "Explain machine learning in 3 sentences"
# )
# for quant, data in results.items():
#     print(f"\n{quant}:")
#     for k, v in data.items():
#         print(f"  {k}: {v}")

# Function defined successfully

def recommend_quantization(vram_gb: float, use_case: str) -> dict:
    """Recommend optimal quantization based on VRAM and use case."""
    recommendations = {
        "chat": {
            "precision_priority": ("Q8_0", "Best quality, needs more VRAM"),
            "balanced": ("Q4_K_M", "Best quality/size ratio"),
            "size_priority": ("Q3_K_M", "Smallest usable quantization")
        },
        "code": {
            "precision_priority": ("Q8_0", "Code needs high precision"),
            "balanced": ("Q5_K_M", "Good precision for code tasks"),
            "size_priority": ("Q4_K_M", "Minimal quality loss for code")
        },
        "classification": {
            "precision_priority": ("Q5_K_M", "Overkill for classification"),
            "balanced": ("Q4_K_M", "More than enough"),
            "size_priority": ("Q2_K", "Classification is robust to quantization")
        }
    }

    # Determine priority based on VRAM
    if vram_gb >= 12:
        priority = "precision_priority"
    elif vram_gb >= 8:
        priority = "balanced"
    else:
        priority = "size_priority"

    quant, reason = recommendations.get(use_case, recommendations["chat"])[priority]

    # Model size recommendation
    if vram_gb >= 40:
        model_size = "70B"
    elif vram_gb >= 8:
        model_size = "8B"
    else:
        model_size = "3B"

    return {
        "vram_gb": vram_gb,
        "use_case": use_case,
        "priority": priority,
        "quantization": quant,
        "model_size": model_size,
        "reason": reason
    }

# Example usage
for vram in [4, 8, 24]:
    rec = recommend_quantization(vram, "chat")
    print(f"VRAM {vram}GB: {rec['model_size']} {rec['quantization']} ({rec['reason']})")

# Function defined successfully

# ============================================
# Comprehensive: Quantization decision engine
# Evaluates quality, speed, and VRAM tradeoffs
# ============================================

import ollama
import time
import json
from pathlib import Path

class QuantizationAdvisor:
    def __init__(self, vram_gb: float, use_case: str = "chat"):
        self.vram_gb = vram_gb
        self.use_case = use_case
        self.results = {}

    def estimate_model_fit(self, params_b: float, quant: str) -> dict:
        """Estimate if a model+quantization fits in available VRAM."""
        quant_bytes = {
            "FP16": 2.0, "Q8_0": 1.0, "Q5_K_M": 0.625,
            "Q4_K_M": 0.5, "Q3_K_M": 0.375, "Q2_K": 0.25
        }
        bytes_per_param = quant_bytes.get(quant, 0.5)
        vram_needed = params_b * bytes_per_param * 1.2  # 20% overhead
        fits = vram_needed <= self.vram_gb
        return {
            "params": f"{params_b}B",
            "quantization": quant,
            "vram_needed_gb": round(vram_needed, 1),
            "vram_available_gb": self.vram_gb,
            "fits": fits,
            "headroom_gb": round(self.vram_gb - vram_needed, 1)
        }

    def generate_recommendations(self) -> list[dict]:
        """Generate all viable model+quantization combinations."""
        models = [
            (3.2, "llama3.2"), (8.0, "llama3.1:8b"), (70.0, "llama3.1:70b")
        ]
        quants = ["Q4_K_M", "Q5_K_M", "Q8_0"]

        viable = []
        for params, name in models:
            for quant in quants:
                fit = self.estimate_model_fit(params, quant)
                if fit["fits"]:
                    viable.append({
                        "model": name,
                        **fit
                    })

        # Sort by params descending (prefer larger model)
        viable.sort(key=lambda x: x["params"], reverse=True)
        return viable

    def best_recommendation(self) -> dict:
        """Return the single best recommendation."""
        viable = self.generate_recommendations()
        if not viable:
            return {"error": "No model fits in available VRAM. Use CPU mode."}

        # Prefer largest model, then highest quantization
        best = viable[0]
        return {
            "recommended_model": best["model"],
            "recommended_quantization": best["quantization"],
            "vram_used": f"{best['vram_needed_gb']}GB",
            "headroom": f"{best['headroom_gb']}GB",
            "reason": f"Largest model that fits {self.vram_gb}GB VRAM with Q4_K_M+"
        }

# Usage
if __name__ == "__main__":
    advisor = QuantizationAdvisor(vram_gb=8, use_case="chat")
    print("=== Quantization Recommendations (8GB VRAM) ===")
    for rec in advisor.generate_recommendations():
        print(f"  {rec['model']} {rec['quantization']}: "
              f"{rec['vram_needed_gb']}GB (headroom: {rec['headroom_gb']}GB)")

    print(f"\nBest: {advisor.best_recommendation()}")