Model Servers

name: my-agent
runtime: python3
module: agent.py
entrypoint: handler

# Model server configuration
engine: ollama           # ollama or vllm
model: mistral           # Model name

# Install Ollama
brew install ollama

# Pull a model
ollama pull mistral

# Orpheus will start it automatically

# Install Ollama
curl -fsSL https://ollama.com/install.sh | sh

# Pull a model
ollama pull mistral

name: my-agent
runtime: python3
module: agent.py
entrypoint: handler

engine: ollama
model: mistral

import os
import requests

MODEL_URL = os.environ.get("MODEL_URL", "http://localhost:11434")

def handler(input_data):
    response = requests.post(
        f"{MODEL_URL}/api/generate",
        json={"model": "mistral", "prompt": input_data["query"]}
    )
    return {"response": response.json()["response"]}

pip install vllm

name: my-agent
runtime: python3
module: agent.py
entrypoint: handler

engine: vllm
model: mistralai/Mistral-7B-Instruct-v0.2

2s → 4s → 8s → 16s → 32s → 60s (max)

# View agent stats (includes service status)
orpheus stats my-agent

orpheus_service_up{agent="my-agent"} 1
orpheus_service_uptime_seconds{agent="my-agent"} 3600

sudo journalctl -u orpheusd | grep my-agent