Ollama: 性能调优

# Enable 4 parallel requests
export OLLAMA_NUM_PARALLEL=4
export OLLAMA_MAX_LOADED_MODELS=2

# Collect performance metrics with verbose output
ollama run --verbose qwen2.5 "Explain AI in 50 words"

# Key output:
# total duration:       2500000000    # 2.5s total
# load duration:        500000000     # 0.5s model load
# prompt eval count:    15 token(s)
# prompt eval duration: 200000000     # 0.2s input processing
# prompt eval count:    15 token(s)   # = prompt_eval_speed: 75 tok/s
# eval count:           52 token(s)
# eval duration:        1800000000    # = eval_speed: 28.9 tok/s

I'm a helpful AI assistant running locally on your machine...

import ollama
import time

def measure_performance(model: str, prompt: str) -> dict:
    start = time.time()
    response = ollama.chat(
        model=model,
        messages=[{"role": "user", "content": prompt}],
        stream=False
    )
    total_time = time.time() - start
    content = response["message"]["content"]
    token_count = len(content) // 4  # Rough estimate

    return {
        "model": model,
        "total_time_s": round(total_time, 2),
        "estimated_tokens": token_count,
        "estimated_tok_per_s": round(token_count / total_time, 1) if total_time > 0 else 0
    }

result = measure_performance("qwen2.5", "Explain machine learning in 100 words")
print(result)

# Configure parallel processing
sudo systemctl edit ollama
# Add:
[Service]
Environment="OLLAMA_NUM_PARALLEL=4"
Environment="OLLAMA_MAX_LOADED_MODELS=2"
Environment="OLLAMA_KEEP_ALIVE=30m"

sudo systemctl daemon-reload
sudo systemctl restart ollama

# Ollama 命令执行成功

import ollama
import asyncio
import time

async def concurrent_test(num_requests: int):
    """Test concurrent request handling."""
    client = ollama.AsyncClient()
    start = time.time()

    async def single_request(i: int):
        req_start = time.time()
        response = await client.chat(
            model="qwen2.5",
            messages=[{"role": "user", "content": f"Say hello {i}"}],
            stream=False,
            options={"temperature": 0.1}
        )
        return time.time() - req_start

    tasks = [single_request(i) for i in range(num_requests)]
    latencies = await asyncio.gather(*tasks)

    total = time.time() - start
    print(f"Concurrent: {num_requests} requests")
    print(f"Total time: {total:.2f}s")
    print(f"Avg latency: {sum(latencies)/len(latencies):.2f}s")
    print(f"Throughput: {num_requests/total:.1f} req/s")

asyncio.run(concurrent_test(4))

import ollama
import time

def test_context_sizes(model: str, prompt: str, context_sizes: list[int]):
    """Test performance with different context window sizes."""
    for ctx in context_sizes:
        start = time.time()
        response = ollama.chat(
            model=model,
            messages=[{"role": "user", "content": prompt}],
            stream=False,
            options={"num_ctx": ctx, "temperature": 0.3}
        )
        elapsed = time.time() - start
        tokens = len(response["message"]["content"]) // 4
        speed = tokens / elapsed if elapsed > 0 else 0
        print(f"num_ctx={ctx}: {elapsed:.2f}s, ~{speed:.1f} tok/s")

test_context_sizes("qwen2.5", "Explain AI briefly", [2048, 4096, 8192])

# 函数定义成功

import ollama
import time
import asyncio
import json
from datetime import datetime

class BenchmarkRunner:
    def __init__(self, model: str = "qwen2.5"):
        self.model = model
        self.results = []

    def warmup(self, runs: int = 2):
        for _ in range(runs):
            ollama.chat(model=self.model,
                        messages=[{"role": "user", "content": "warmup"}],
                        stream=False)

    def single_benchmark(self, prompt: str) -> dict:
        start = time.time()
        response = ollama.chat(
            model=self.model,
            messages=[{"role": "user", "content": prompt}],
            stream=False,
            options={"temperature": 0.3}
        )
        elapsed = time.time() - start
        content = response["message"]["content"]
        return {
            "prompt_length": len(prompt),
            "response_length": len(content),
            "estimated_tokens": len(content) // 4,
            "total_time_s": round(elapsed, 2),
            "tok_per_s": round(len(content) / 4 / elapsed, 1) if elapsed > 0 else 0
        }

    async def concurrent_benchmark(self, prompt: str, concurrency: int) -> dict:
        client = ollama.AsyncClient()
        start = time.time()
        tasks = [
            client.chat(model=self.model,
                       messages=[{"role": "user", "content": prompt}],
                       stream=False, options={"temperature": 0.3})
            for _ in range(concurrency)
        ]
        await asyncio.gather(*tasks)
        total = time.time() - start
        return {
            "concurrency": concurrency,
            "total_time_s": round(total, 2),
            "throughput_rps": round(concurrency / total, 1)
        }

    def run_full_benchmark(self):
        self.warmup()
        prompts = [
            "Hello",
            "Explain AI in 3 sentences",
            "Write a product description for wireless headphones"
        ]
        print("=== Single Request Benchmark ===")
        for p in prompts:
            result = self.single_benchmark(p)
            print(f"  {result['estimated_tokens']}tok, {result['tok_per_s']}tok/s, {result['total_time_s']}s")

        print("\n=== Concurrent Benchmark ===")
        for c in [1, 2, 4]:
            result = asyncio.run(self.concurrent_benchmark("Hello", c))
            print(f"  Concurrency {c}: {result['throughput_rps']} req/s")

# Run
runner = BenchmarkRunner("qwen2.5")
runner.run_full_benchmark()

=== Single Request Benchmark ===
\n=== Concurrent Benchmark ===

#!/bin/bash
# Establish performance baseline

MODEL="qwen2.5"
DATE=$(date +%Y%m%d)
REPORT="baseline_${DATE}.txt"

echo "=== Performance Baseline ===" > "$REPORT"
echo "Date: $(date)" >> "$REPORT"
echo "Model: $MODEL" >> "$REPORT"
echo "GPU: $(nvidia-smi --query-gpu=name --format=csv,noheader 2>/dev/null || echo 'CPU')" >> "$REPORT"
echo "" >> "$REPORT"

# Single request benchmark
echo "## Single Request ##" >> "$REPORT"
for i in {1..5}; do
    start=$(date +%s%N)
    curl -s http://localhost:11434/api/chat -d "{
        \"model\": \"$MODEL\",
        \"messages\": [{\"role\": \"user\", \"content\": \"Hello\"}],
        \"stream\": false
    }" > /dev/null
    end=$(date +%s%N)
    elapsed=$(( (end - start) / 1000000 ))
    echo "  Run $i: ${elapsed}ms" >> "$REPORT"
done

echo "" >> "$REPORT"
echo "## GPU Utilization ##" >> "$REPORT"
nvidia-smi --query-gpu=utilization.gpu,memory.used,memory.total --format=csv >> "$REPORT" 2>/dev/null || echo "CPU mode" >> "$REPORT"

echo "Baseline saved: $REPORT"

{"status":"ok","data":{}}

# ============================================
# Comprehensive: Performance tuning report
# Full benchmark + optimization recommendations
# ============================================

import ollama
import time
import asyncio
import json

class PerformanceTuner:
    def __init__(self, model: str = "qwen2.5"):
        self.model = model

    def run_diagnostics(self) -> dict:
        """Run full performance diagnostics."""
        # Warm up
        ollama.chat(model=self.model,
                    messages=[{"role": "user", "content": "warmup"}],
                    stream=False)

        # Single request test
        start = time.time()
        resp = ollama.chat(
            model=self.model,
            messages=[{"role": "user", "content": "Hello"}],
            stream=False, options={"temperature": 0.3}
        )
        single_latency = time.time() - start

        # Context size test
        ctx_results = {}
        for ctx in [2048, 4096, 8192]:
            start = time.time()
            ollama.chat(
                model=self.model,
                messages=[{"role": "user", "content": "Hello"}],
                stream=False, options={"num_ctx": ctx, "temperature": 0.3}
            )
            ctx_results[ctx] = round(time.time() - start, 2)

        return {
            "model": self.model,
            "single_latency_s": round(single_latency, 2),
            "context_sizes": ctx_results,
            "recommendations": self._generate_recommendations(single_latency)
        }

    def _generate_recommendations(self, latency: float) -> list[str]:
        recs = []
        if latency > 5:
            recs.append("High latency detected. Enable GPU acceleration or use smaller model.")
        if latency > 2:
            recs.append("Set OLLAMA_NUM_PARALLEL=4 for concurrent requests.")
        recs.append("Use num_ctx=2048 for chat, num_ctx=8192 for RAG.")
        recs.append("Set OLLAMA_KEEP_ALIVE=30m to avoid model reloading.")
        return recs

    def generate_report(self) -> str:
        diag = self.run_diagnostics()
        report = [
            f"# Performance Tuning Report",
            f"Model: {diag['model']}",
            f"Single request latency: {diag['single_latency_s']}s",
            f"\n## Context Window Impact:",
        ]
        for ctx, latency in diag["context_sizes"].items():
            report.append(f"  num_ctx={ctx}: {latency}s")
        report.append("\n## Recommendations:")
        for r in diag["recommendations"]:
            report.append(f"  - {r}")
        return "\n".join(report)

tuner = PerformanceTuner("qwen2.5")
print(tuner.generate_report())