Ollama: Multimodal Models

ollama run llava "What UI issues do you see in this image?" /path/to/screenshot.png

# Pull the multimodal model
ollama pull llava

# Basic image description (CLI with file path)
ollama run llava "Describe this image in detail" /path/to/photo.jpg

# Without image: text-only mode
ollama run llava "What is machine learning?"

I'm a helpful AI assistant running locally on your machine...

# Start interactive session with image
ollama run llava

>>> .image /path/to/product-photo.jpg
>>> What product is shown and what are its key features?

# Output: The image shows a pair of premium wireless headphones...
# Key features include:
# 1. Over-ear design with memory foam cushions
# 2. Active noise cancellation
# 3. USB-C charging port visible on the left ear cup

>>> .image /path/to/chart.png
>>> Summarize the data trends shown in this chart

# Output: The chart displays quarterly revenue from 2023-2024...

I'm a helpful AI assistant running locally on your machine...

# Convert image to Base64 and call API
IMAGE_BASE64=$(base64 -w 0 /path/to/photo.jpg)

curl http://localhost:11434/api/chat -d "{
  \"model\": \"llava\",
  \"messages\": [{
    \"role\": \"user\",
    \"content\": \"Describe this image\",
    \"images\": [\"$IMAGE_BASE64\"]
  }],
  \"stream\": false
}"

{"status":"ok","data":{}}

import ollama
import base64
from pathlib import Path

def analyze_image(image_path: str, prompt: str, model: str = "llava") -> str:
    """Analyze an image using a multimodal model."""
    # Read and encode image
    image_data = base64.b64encode(
        Path(image_path).read_bytes()
    ).decode("utf-8")

    response = ollama.chat(
        model=model,
        messages=[{
            "role": "user",
            "content": prompt,
            "images": [image_data]
        }],
        stream=False
    )
    return response["message"]["content"]

# Usage
result = analyze_image(
    "/path/to/screenshot.png",
    "List all UI issues you can identify in this screenshot"
)
print(result)

# Function defined successfully

import ollama
import base64
from pathlib import Path
import json

def analyze_ui_screenshot(image_path: str) -> dict:
    """Analyze a UI screenshot and return structured feedback."""
    image_data = base64.b64encode(
        Path(image_path).read_bytes()
    ).decode("utf-8")

    system_prompt = """You are a UI/UX expert reviewing application screenshots.
Analyze the screenshot and provide feedback in this JSON format:
{
  "issues": [{"type": "layout|color|text|accessibility", "description": "...", "severity": "high|medium|low"}],
  "summary": "Brief overall assessment"
}"""

    response = ollama.chat(
        model="llava-llama3",
        messages=[
            {"role": "system", "content": system_prompt},
            {"role": "user", "content": "Review this UI screenshot for issues",
             "images": [image_data]}
        ],
        stream=False,
        format="json",
        options={"temperature": 0.3}
    )

    try:
        return json.loads(response["message"]["content"])
    except json.JSONDecodeError:
        return {"raw": response["message"]["content"]}

# Usage
feedback = analyze_ui_screenshot("/path/to/app-screenshot.png")
print(json.dumps(feedback, indent=2))

# Function defined successfully

# ============================================
# Comprehensive: Multimodal customer service
# Image analysis + text chat for e-commerce
# ============================================

import ollama
import base64
from pathlib import Path
from dataclasses import dataclass
from typing import Optional

@dataclass
class MultimodalSupport:
    chat_model: str = "qwen2.5"
    vision_model: str = "llava"
    temperature: float = 0.3

    def handle_product_query(
        self, image_path: str, question: str
    ) -> str:
        """Answer product questions using image + text."""
        image_data = base64.b64encode(
            Path(image_path).read_bytes()
        ).decode("utf-8")

        response = ollama.chat(
            model=self.vision_model,
            messages=[{
                "role": "user",
                "content": f"Answer this question about the product shown: {question}",
                "images": [image_data]
            }],
            stream=False,
            options={"temperature": self.temperature}
        )
        return response["message"]["content"]

    def handle_damage_report(
        self, image_path: str, order_id: str
    ) -> str:
        """Process a damage report with photo evidence."""
        image_data = base64.b64encode(
            Path(image_path).read_bytes()
        ).decode("utf-8")

        # Step 1: Analyze damage from image
        damage_desc = ollama.chat(
            model=self.vision_model,
            messages=[{
                "role": "user",
                "content": "Describe the damage visible in this product photo. Be specific about the type and severity.",
                "images": [image_data]
            }],
            stream=False,
            options={"temperature": 0.2}
        )["message"]["content"]

        # Step 2: Generate response using chat model
        response = ollama.chat(
            model=self.chat_model,
            messages=[
                {"role": "system", "content": "You are an e-commerce customer service agent handling damage reports."},
                {"role": "user", "content": f"Order {order_id} has damage: {damage_desc}. Generate a polite response acknowledging the damage and explaining the refund/replacement process."}
            ],
            stream=False,
            options={"temperature": self.temperature}
        )
        return response["message"]["content"]

    def extract_text_from_image(
        self, image_path: str
    ) -> str:
        """OCR-like text extraction from document images."""
        image_data = base64.b64encode(
            Path(image_path).read_bytes()
        ).decode("utf-8")

        response = ollama.chat(
            model="minicpm-v",
            messages=[{
                "role": "user",
                "content": "Extract all visible text from this image. Preserve the original layout and formatting.",
                "images": [image_data]
            }],
            stream=False,
            options={"temperature": 0.1}
        )
        return response["message"]["content"]

# Usage
if __name__ == "__main__":
    service = MultimodalSupport()

    print("=== Damage Report ===")
    # print(service.handle_damage_report("damaged_item.jpg", "#12345"))

    print("=== Product Query ===")
    # print(service.handle_product_query("headphones.jpg", "Does this have noise cancellation?"))

    print("=== OCR Extraction ===")
    # print(service.extract_text_from_image("receipt.png"))