Common Issues

# Check if request is stuck
orpheus execlog list <agent> --status STARTED

# Check queue metrics
curl http://localhost:7777/v1/stats

# Restart daemon to clear stuck workers
sudo systemctl restart orpheusd  # Linux
# or
orpheus vm restart  # macOS

# In agent.yaml - increase timeout if legitimate
timeout: 120  # Allow more time for slow operations

# Check queue depth
curl http://localhost:7777/v1/agents/<agent>/stats

# agent.yaml
scaling:
  queue_size: 100  # Was 50

scaling:
  max_workers: 20  # Was 10

import time
for attempt in range(5):
    try:
        response = orpheus.run(agent, input)
        break
    except QueueFullError:
        time.sleep(pow(2, attempt))  # Exponential backoff: 1s, 2s, 4s, 8s, 16s

# Check timeout configuration
orpheus config show <agent> | grep timeout

# Check task duration distribution
orpheus execlog list <agent> --limit 100 | grep duration

# agent.yaml
timeout: 120  # Allow 2 minutes

# Add progress logging to identify bottleneck
logger.info("Starting LLM call...")
result = llm.call(prompt)  # Identify slow operations
logger.info("LLM call completed")

# Process in chunks to stay within timeout
for chunk in chunks(large_data, size=100):
    process(chunk)

# Check agent logs for syscall errors
sudo journalctl -u orpheusd | grep <agent> | grep -i "permission\|eperm"

# Instead of mounting (blocked):
import os
os.mount("/dev/sda", "/mnt", "ext4")  # ✗ Blocked by seccomp

# Use provided mounts:
with open("/workspace/data.txt", "w") as f:  # ✓ Allowed
    f.write(data)

# Check logs for spawn failures
sudo journalctl -u orpheusd | grep "circuit breaker"
sudo journalctl -u orpheusd | grep "spawn failed"

# Circuit breaker will automatically retry after 60s
# Monitor logs:
sudo journalctl -u orpheusd | tail -f

# Check if image exists
ls ~/.orpheus/images/<agent>/rootfs

# Check runc works
runc --version

# Manually test spawn
orpheus test-spawn <agent>

sudo systemctl restart orpheusd

my-agent/
├── agent.yaml    # Required
└── agent.py

# agent.yaml
entrypoint: handler

# agent.py
def handler(input_data):  # Must match
    ...

# Check status
orpheus status

# Start daemon (macOS)
orpheus vm start

# Start daemon (Linux)
sudo systemctl start orpheusd

# Check memory limit
orpheus config show <agent> | grep memory

# agent.yaml
memory: 1024  # Increase to 1GB

# Stream large files instead of loading all at once
with open("large_file.txt", "r") as f:
    for line in f:  # Iterate, don't load all
        process(line)

# agent.yaml
scaling:
  min_workers: 3  # Keep 3 workers always running
  max_workers: 10

scaling:
  min_workers: 1

# Wrong (lost on restart)
with open('/tmp/data.json', 'w') as f: ...

# Right (persists forever)
with open('/workspace/data.json', 'w') as f: ...

# Check number of containers
ps aux | grep runc | wc -l

# Check worker count
orpheus status

# agent.yaml
scaling:
  max_workers: 10  # Reduce from higher value

scaling:
  scale_down_threshold: 0.8  # Scale down earlier

# Find what's using the port
lsof -i :7777

# Kill it or use different port
orpheusd --tcp-bind :7778