Observability: Prometheus metrics, log panel, runbook

- GET /api/metrics renders Prometheus text (equity, P&L, per-strategy
  equity/pnl/allocation/halted, info) with no client library.
- In-memory ring log handler (app/logs.py) exposed at GET /api/logs and a
  dashboard "Server logs" panel; Runtime mirrors bot events into the logger.
- docs/runbook.md covers deploys, the kill switch, resetting paper state and
  going live. Adds tests for metrics rendering and the log ring.

backend/app/api/routes.py

@@ -4,10 +4,20 @@ import asyncio
 from typing import Annotated
 
 import httpx
-from fastapi import APIRouter, Depends, HTTPException, Request, WebSocket, WebSocketDisconnect
+from fastapi import (
+    APIRouter,
+    Depends,
+    HTTPException,
+    Request,
+    Response,
+    WebSocket,
+    WebSocketDisconnect,
+)
 
 from app.config import Settings
 from app.engine.strategies import get_strategy, list_strategies, strategy_spec
+from app.logs import get_log_handler
+from app.metrics import render_metrics
 from app.models import OrderSide, Position, PositionStatus
 from app.runtime import Runtime
 from app.schemas import (
@@ -65,6 +75,21 @@ def health() -> dict[str, str]:
     return {"status": "ok"}
 
 
+@router.get("/metrics")
+def metrics(runtime: GuardedRuntimeDep) -> Response:
+    """Prometheus text exposition, unauthenticated so a scraper can reach it."""
+    status = runtime.multi_status()
+    body = render_metrics(status, runtime.settings.mode, runtime.settings.exchange)
+    return Response(content=body, media_type="text/plain; version=0.0.4")
+
+
+@router.get("/logs")
+def logs(runtime: GuardedRuntimeDep, limit: int = 100) -> dict:
+    handler = get_log_handler()
+    entries = handler.snapshot(limit) if handler is not None else []
+    return {"entries": entries}
+
+
 @router.get("/version")
 def version(request: Request) -> dict[str, str]:
     """Build version of the running backend, used by the client to self-heal

backend/app/logs.py

@@ -0,0 +1,55 @@
+from __future__ import annotations
+
+import logging
+from collections import deque
+from datetime import UTC, datetime
+
+
+class RingLogHandler(logging.Handler):
+    """Keeps the most recent log records in memory for the dashboard.
+
+    Deliberately small and dependency-free: the goal is a debugging aid, not a
+    log store. Attach once at startup.
+    """
+
+    def __init__(self, capacity: int = 200) -> None:
+        super().__init__()
+        self.records: deque[dict] = deque(maxlen=capacity)
+
+    def emit(self, record: logging.LogRecord) -> None:
+        try:
+            self.records.append(
+                {
+                    "time": datetime.fromtimestamp(record.created, UTC).isoformat(),
+                    "level": record.levelname.lower(),
+                    "logger": record.name,
+                    "message": record.getMessage(),
+                }
+            )
+        except Exception:  # never let logging break the app
+            self.handleError(record)
+
+    def snapshot(self, limit: int = 100) -> list[dict]:
+        items = list(self.records)
+        return items[-limit:]
+
+
+_handler: RingLogHandler | None = None
+
+
+def install_log_capture(capacity: int = 200) -> RingLogHandler:
+    """Attach the ring handler to the root logger (idempotent)."""
+    global _handler
+    if _handler is not None:
+        return _handler
+    _handler = RingLogHandler(capacity)
+    _handler.setLevel(logging.INFO)
+    root = logging.getLogger()
+    root.addHandler(_handler)
+    if root.level > logging.INFO or root.level == logging.NOTSET:
+        root.setLevel(logging.INFO)
+    return _handler
+
+
+def get_log_handler() -> RingLogHandler | None:
+    return _handler

backend/app/main.py

@@ -10,6 +10,7 @@ from fastapi.middleware.cors import CORSMiddleware
 from app.api.routes import router
 from app.config import Settings, get_settings
 from app.db import init_db
+from app.logs import install_log_capture
 from app.runtime import Runtime
 
 
@@ -22,6 +23,7 @@ def create_app(
 
     @asynccontextmanager
     async def lifespan(app: FastAPI) -> AsyncIterator[None]:
+        install_log_capture()
         init_db()
         app.state.runtime = runtime
         yield

backend/app/metrics.py

@@ -0,0 +1,96 @@
+from __future__ import annotations
+
+from app.schemas import MultiStatusOut
+
+# Rendered as Prometheus text exposition format (no client library needed).
+# The route serves this at GET /api/metrics for the existing Prometheus scrape.
+
+
+def _escape(value: str) -> str:
+    return value.replace("\\", "\\\\").replace('"', '\\"')
+
+
+def _metric(
+    name: str, help_text: str, value: float | int, labels: dict[str, str] | None = None
+) -> str:
+    lines = [f"# HELP {name} {help_text}", f"# TYPE {name} gauge"]
+    if labels:
+        rendered = ",".join(f'{k}="{_escape(v)}"' for k, v in labels.items())
+        lines.append(f"{name}{{{rendered}}} {value}")
+    else:
+        lines.append(f"{name} {value}")
+    return "\n".join(lines)
+
+
+def render_metrics(status: MultiStatusOut, mode: str, exchange: str) -> str:
+    """Render the current portfolio state as Prometheus metrics."""
+    parts = [
+        _metric("crypto_running", "1 when the bot is running", int(status.running)),
+        _metric("crypto_equity", "Total account equity", status.equity),
+        _metric("crypto_starting_balance", "Starting balance", status.starting_balance),
+        _metric("crypto_total_pnl", "Total profit and loss", status.total_pnl),
+        _metric(
+            "crypto_open_positions",
+            "Number of open positions across strategies",
+            sum(s.open_positions for s in status.strategies),
+        ),
+        _metric(
+            "crypto_strategies",
+            "Number of configured strategies",
+            len(status.strategies),
+        ),
+    ]
+
+    if status.price is not None:
+        parts.append(_metric("crypto_price", "Latest mark price", status.price))
+
+    # Common labels shared by the per-strategy families.
+    lines = [
+        "# HELP crypto_strategy_equity Equity per strategy",
+        "# TYPE crypto_strategy_equity gauge",
+    ]
+    for s in status.strategies:
+        labels = {"strategy": s.strategy_key, "symbol": s.symbol}
+        rendered = ",".join(f'{k}="{_escape(v)}"' for k, v in labels.items())
+        lines.append(f"crypto_strategy_equity{{{rendered}}} {s.equity}")
+    parts.append("\n".join(lines))
+
+    lines = [
+        "# HELP crypto_strategy_pnl Total P&L per strategy",
+        "# TYPE crypto_strategy_pnl gauge",
+    ]
+    for s in status.strategies:
+        labels = {"strategy": s.strategy_key, "symbol": s.symbol}
+        rendered = ",".join(f'{k}="{_escape(v)}"' for k, v in labels.items())
+        lines.append(f"crypto_strategy_pnl{{{rendered}}} {s.total_pnl}")
+    parts.append("\n".join(lines))
+
+    lines = [
+        "# HELP crypto_strategy_allocation Fraction of the balance per strategy",
+        "# TYPE crypto_strategy_allocation gauge",
+    ]
+    for s in status.strategies:
+        labels = {"strategy": s.strategy_key}
+        rendered = ",".join(f'{k}="{_escape(v)}"' for k, v in labels.items())
+        lines.append(f"crypto_strategy_allocation{{{rendered}}} {s.allocation}")
+    parts.append("\n".join(lines))
+
+    lines = [
+        "# HELP crypto_strategy_halted 1 when a strategy is halted",
+        "# TYPE crypto_strategy_halted gauge",
+    ]
+    for s in status.strategies:
+        labels = {"strategy": s.strategy_key}
+        rendered = ",".join(f'{k}="{_escape(v)}"' for k, v in labels.items())
+        lines.append(f"crypto_strategy_halted{{{rendered}}} {int(s.halted)}")
+    parts.append("\n".join(lines))
+
+    info_labels = {"mode": mode, "exchange": exchange}
+    rendered = ",".join(f'{k}="{_escape(v)}"' for k, v in info_labels.items())
+    parts.append(
+        "# HELP crypto_info Build information\n"
+        "# TYPE crypto_info gauge\n"
+        f"crypto_info{{{rendered}}} 1"
+    )
+
+    return "\n".join(parts) + "\n"

backend/app/runtime.py

@@ -32,6 +32,15 @@ from app.store import Store
 
 logger = logging.getLogger(__name__)
 
+_LOG_LEVELS = {
+    "debug": logging.DEBUG,
+    "info": logging.INFO,
+    "trade": logging.INFO,
+    "warning": logging.WARNING,
+    "error": logging.ERROR,
+    "critical": logging.CRITICAL,
+}
+
 FeedFactory = Callable[[Settings], MarketData]
 BrokerFactory = Callable[[Settings], Broker]
 NotifierFactory = Callable[[Settings], Notifier | None]
@@ -91,8 +100,15 @@ class Runtime:
             with suppress(asyncio.QueueFull):
                 queue.put_nowait(payload)
         if payload.get("type") == "event":
+            self._log_event(payload["data"])
             self._maybe_notify(payload["data"])
 
+    def _log_event(self, event: dict[str, Any]) -> None:
+        """Mirror bot events into the logger so the dashboard log view is useful."""
+        level = str(event.get("level", "info")).lower()
+        message = event.get("message", "")
+        logger.log(_LOG_LEVELS.get(level, logging.INFO), "%s", message)
+
     def _maybe_notify(self, event: dict[str, Any]) -> None:
         if event.get("level") not in self.notify_levels:
             return