Observability: Prometheus metrics, log panel, runbook

- GET /api/metrics renders Prometheus text (equity, P&L, per-strategy
  equity/pnl/allocation/halted, info) with no client library.
- In-memory ring log handler (app/logs.py) exposed at GET /api/logs and a
  dashboard "Server logs" panel; Runtime mirrors bot events into the logger.
- docs/runbook.md covers deploys, the kill switch, resetting paper state and
  going live. Adds tests for metrics rendering and the log ring.

AGENTS.md

@@ -136,6 +136,17 @@ the single and multi start paths are covered) and surfaced via
 call `check_live_ready` first. `LIVE_CONFIRMATION` must equal
 `"I UNDERSTAND THE RISK"` — keep it a phrase, not a boolean.
 
+## Observability
+
+- `app/metrics.py` renders Prometheus text at `GET /api/metrics` straight from
+  the manager status — no client library. Keep new series out of hot loops.
+- `app/logs.py` keeps a small in-memory ring of log records, exposed at
+  `GET /api/logs` and shown in the dashboard. `Runtime._log_event` mirrors bot
+  events into the logger so the panel is useful (uvicorn's own loggers do not
+  propagate to the root handler).
+- `docs/runbook.md` is the operations guide; update it when deploy or
+  recovery steps change.
+
 ## Risk controls & notifications
 
 - `app/engine/risk.py` (`RiskManager`) tracks daily loss, drawdown, position cap

README.md

@@ -220,6 +220,8 @@ Bundled strategies: `sma_cross`, `ema_cross`, `macd_trend`,
 | `POST` | `/api/bot/start-many` | Start several strategies with per-strategy allocations |
 | `GET` | `/api/bot/status` | Per-strategy status for a multi-strategy run |
 | `GET` | `/api/live-check` | Go-live preflight (blockers for live trading) |
+| `GET` | `/api/metrics` | Prometheus text exposition |
+| `GET` | `/api/logs` | Recent backend log lines (in-memory ring) |
 | `POST` | `/api/bot/stop` | Stop the bot |
 | `POST` | `/api/bot/kill` | Kill switch: flatten + halt |
 | `POST` | `/api/backtest` | Backtest a strategy on historical candles |
@@ -234,6 +236,16 @@ Bundled strategies: `sma_cross`, `ema_cross`, `macd_trend`,
 | `DELETE` | `/api/presets/{id}` | Delete a preset |
 | `WS` | `/api/ws` | Live status + event stream |
 
+## Observability
+
+- `GET /api/metrics` exposes Prometheus metrics (`crypto_equity`,
+  `crypto_total_pnl`, per-strategy `crypto_strategy_equity`/`_pnl`/`_allocation`/
+  `_halted`, `crypto_info`). Point the existing Prometheus scrape at it.
+- The dashboard's **Server logs** panel shows recent backend log lines
+  (in-memory ring buffer), useful alongside **Activity**.
+- See [docs/runbook.md](docs/runbook.md) for operations (deploys, kill switch,
+  resetting paper state, going live).
+
 ## Tests & lint
 
 ```bash

backend/app/api/routes.py

@@ -4,10 +4,20 @@ import asyncio
 from typing import Annotated
 
 import httpx
-from fastapi import APIRouter, Depends, HTTPException, Request, WebSocket, WebSocketDisconnect
+from fastapi import (
+    APIRouter,
+    Depends,
+    HTTPException,
+    Request,
+    Response,
+    WebSocket,
+    WebSocketDisconnect,
+)
 
 from app.config import Settings
 from app.engine.strategies import get_strategy, list_strategies, strategy_spec
+from app.logs import get_log_handler
+from app.metrics import render_metrics
 from app.models import OrderSide, Position, PositionStatus
 from app.runtime import Runtime
 from app.schemas import (
@@ -65,6 +75,21 @@ def health() -> dict[str, str]:
     return {"status": "ok"}
 
 
+@router.get("/metrics")
+def metrics(runtime: GuardedRuntimeDep) -> Response:
+    """Prometheus text exposition, unauthenticated so a scraper can reach it."""
+    status = runtime.multi_status()
+    body = render_metrics(status, runtime.settings.mode, runtime.settings.exchange)
+    return Response(content=body, media_type="text/plain; version=0.0.4")
+
+
+@router.get("/logs")
+def logs(runtime: GuardedRuntimeDep, limit: int = 100) -> dict:
+    handler = get_log_handler()
+    entries = handler.snapshot(limit) if handler is not None else []
+    return {"entries": entries}
+
+
 @router.get("/version")
 def version(request: Request) -> dict[str, str]:
     """Build version of the running backend, used by the client to self-heal

backend/app/logs.py

@@ -0,0 +1,55 @@
+from __future__ import annotations
+
+import logging
+from collections import deque
+from datetime import UTC, datetime
+
+
+class RingLogHandler(logging.Handler):
+    """Keeps the most recent log records in memory for the dashboard.
+
+    Deliberately small and dependency-free: the goal is a debugging aid, not a
+    log store. Attach once at startup.
+    """
+
+    def __init__(self, capacity: int = 200) -> None:
+        super().__init__()
+        self.records: deque[dict] = deque(maxlen=capacity)
+
+    def emit(self, record: logging.LogRecord) -> None:
+        try:
+            self.records.append(
+                {
+                    "time": datetime.fromtimestamp(record.created, UTC).isoformat(),
+                    "level": record.levelname.lower(),
+                    "logger": record.name,
+                    "message": record.getMessage(),
+                }
+            )
+        except Exception:  # never let logging break the app
+            self.handleError(record)
+
+    def snapshot(self, limit: int = 100) -> list[dict]:
+        items = list(self.records)
+        return items[-limit:]
+
+
+_handler: RingLogHandler | None = None
+
+
+def install_log_capture(capacity: int = 200) -> RingLogHandler:
+    """Attach the ring handler to the root logger (idempotent)."""
+    global _handler
+    if _handler is not None:
+        return _handler
+    _handler = RingLogHandler(capacity)
+    _handler.setLevel(logging.INFO)
+    root = logging.getLogger()
+    root.addHandler(_handler)
+    if root.level > logging.INFO or root.level == logging.NOTSET:
+        root.setLevel(logging.INFO)
+    return _handler
+
+
+def get_log_handler() -> RingLogHandler | None:
+    return _handler

backend/app/main.py

@@ -10,6 +10,7 @@ from fastapi.middleware.cors import CORSMiddleware
 from app.api.routes import router
 from app.config import Settings, get_settings
 from app.db import init_db
+from app.logs import install_log_capture
 from app.runtime import Runtime
 
 
@@ -22,6 +23,7 @@ def create_app(
 
     @asynccontextmanager
     async def lifespan(app: FastAPI) -> AsyncIterator[None]:
+        install_log_capture()
         init_db()
         app.state.runtime = runtime
         yield

backend/app/metrics.py

@@ -0,0 +1,96 @@
+from __future__ import annotations
+
+from app.schemas import MultiStatusOut
+
+# Rendered as Prometheus text exposition format (no client library needed).
+# The route serves this at GET /api/metrics for the existing Prometheus scrape.
+
+
+def _escape(value: str) -> str:
+    return value.replace("\\", "\\\\").replace('"', '\\"')
+
+
+def _metric(
+    name: str, help_text: str, value: float | int, labels: dict[str, str] | None = None
+) -> str:
+    lines = [f"# HELP {name} {help_text}", f"# TYPE {name} gauge"]
+    if labels:
+        rendered = ",".join(f'{k}="{_escape(v)}"' for k, v in labels.items())
+        lines.append(f"{name}{{{rendered}}} {value}")
+    else:
+        lines.append(f"{name} {value}")
+    return "\n".join(lines)
+
+
+def render_metrics(status: MultiStatusOut, mode: str, exchange: str) -> str:
+    """Render the current portfolio state as Prometheus metrics."""
+    parts = [
+        _metric("crypto_running", "1 when the bot is running", int(status.running)),
+        _metric("crypto_equity", "Total account equity", status.equity),
+        _metric("crypto_starting_balance", "Starting balance", status.starting_balance),
+        _metric("crypto_total_pnl", "Total profit and loss", status.total_pnl),
+        _metric(
+            "crypto_open_positions",
+            "Number of open positions across strategies",
+            sum(s.open_positions for s in status.strategies),
+        ),
+        _metric(
+            "crypto_strategies",
+            "Number of configured strategies",
+            len(status.strategies),
+        ),
+    ]
+
+    if status.price is not None:
+        parts.append(_metric("crypto_price", "Latest mark price", status.price))
+
+    # Common labels shared by the per-strategy families.
+    lines = [
+        "# HELP crypto_strategy_equity Equity per strategy",
+        "# TYPE crypto_strategy_equity gauge",
+    ]
+    for s in status.strategies:
+        labels = {"strategy": s.strategy_key, "symbol": s.symbol}
+        rendered = ",".join(f'{k}="{_escape(v)}"' for k, v in labels.items())
+        lines.append(f"crypto_strategy_equity{{{rendered}}} {s.equity}")
+    parts.append("\n".join(lines))
+
+    lines = [
+        "# HELP crypto_strategy_pnl Total P&L per strategy",
+        "# TYPE crypto_strategy_pnl gauge",
+    ]
+    for s in status.strategies:
+        labels = {"strategy": s.strategy_key, "symbol": s.symbol}
+        rendered = ",".join(f'{k}="{_escape(v)}"' for k, v in labels.items())
+        lines.append(f"crypto_strategy_pnl{{{rendered}}} {s.total_pnl}")
+    parts.append("\n".join(lines))
+
+    lines = [
+        "# HELP crypto_strategy_allocation Fraction of the balance per strategy",
+        "# TYPE crypto_strategy_allocation gauge",
+    ]
+    for s in status.strategies:
+        labels = {"strategy": s.strategy_key}
+        rendered = ",".join(f'{k}="{_escape(v)}"' for k, v in labels.items())
+        lines.append(f"crypto_strategy_allocation{{{rendered}}} {s.allocation}")
+    parts.append("\n".join(lines))
+
+    lines = [
+        "# HELP crypto_strategy_halted 1 when a strategy is halted",
+        "# TYPE crypto_strategy_halted gauge",
+    ]
+    for s in status.strategies:
+        labels = {"strategy": s.strategy_key}
+        rendered = ",".join(f'{k}="{_escape(v)}"' for k, v in labels.items())
+        lines.append(f"crypto_strategy_halted{{{rendered}}} {int(s.halted)}")
+    parts.append("\n".join(lines))
+
+    info_labels = {"mode": mode, "exchange": exchange}
+    rendered = ",".join(f'{k}="{_escape(v)}"' for k, v in info_labels.items())
+    parts.append(
+        "# HELP crypto_info Build information\n"
+        "# TYPE crypto_info gauge\n"
+        f"crypto_info{{{rendered}}} 1"
+    )
+
+    return "\n".join(parts) + "\n"

backend/app/runtime.py

@@ -32,6 +32,15 @@ from app.store import Store
 
 logger = logging.getLogger(__name__)
 
+_LOG_LEVELS = {
+    "debug": logging.DEBUG,
+    "info": logging.INFO,
+    "trade": logging.INFO,
+    "warning": logging.WARNING,
+    "error": logging.ERROR,
+    "critical": logging.CRITICAL,
+}
+
 FeedFactory = Callable[[Settings], MarketData]
 BrokerFactory = Callable[[Settings], Broker]
 NotifierFactory = Callable[[Settings], Notifier | None]
@@ -91,8 +100,15 @@ class Runtime:
             with suppress(asyncio.QueueFull):
                 queue.put_nowait(payload)
         if payload.get("type") == "event":
+            self._log_event(payload["data"])
             self._maybe_notify(payload["data"])
 
+    def _log_event(self, event: dict[str, Any]) -> None:
+        """Mirror bot events into the logger so the dashboard log view is useful."""
+        level = str(event.get("level", "info")).lower()
+        message = event.get("message", "")
+        logger.log(_LOG_LEVELS.get(level, logging.INFO), "%s", message)
+
     def _maybe_notify(self, event: dict[str, Any]) -> None:
         if event.get("level") not in self.notify_levels:
             return

backend/tests/test_observability.py

@@ -0,0 +1,107 @@
+from __future__ import annotations
+
+import logging
+
+from app.logs import RingLogHandler
+from app.metrics import render_metrics
+from app.schemas import MultiStatusOut, StrategyStatusOut
+
+
+def make_status(**overrides) -> MultiStatusOut:
+    strategy = StrategyStatusOut(
+        strategy_key="sma_cross",
+        strategy_name="SMA Crossover",
+        params={},
+        symbol="BTC/USDT",
+        timeframe="5m",
+        allocation=0.5,
+        cash=5000,
+        equity=5100,
+        realized_pnl=100,
+        unrealized_pnl=0,
+        total_pnl=100,
+        open_positions=1,
+        halted=False,
+        trades_today=2,
+    )
+    base = {
+        "running": True,
+        "mode": "paper",
+        "exchange": "binance",
+        "price": 100.0,
+        "starting_balance": 10000,
+        "equity": 10100,
+        "total_pnl": 100,
+        "strategies": [strategy],
+    }
+    base.update(overrides)
+    return MultiStatusOut(**base)
+
+
+def test_render_metrics_contains_core_series() -> None:
+    body = render_metrics(make_status(), "paper", "binance")
+    assert "crypto_running 1" in body
+    assert "crypto_equity 10100" in body
+    assert "crypto_total_pnl 100" in body
+    assert "crypto_open_positions 1" in body
+    assert 'crypto_strategy_equity{strategy="sma_cross",symbol="BTC/USDT"} 5100' in body
+    assert 'crypto_strategy_allocation{strategy="sma_cross"} 0.5' in body
+    assert 'crypto_info{mode="paper",exchange="binance"} 1' in body
+
+
+def test_render_metrics_omits_price_when_none() -> None:
+    body = render_metrics(make_status(price=None), "paper", "binance")
+    assert "crypto_price" not in body
+
+
+def test_ring_log_handler_captures() -> None:
+    handler = RingLogHandler(capacity=5)
+    logger = logging.getLogger("test.ring")
+    logger.addHandler(handler)
+    logger.setLevel(logging.INFO)
+    try:
+        logger.info("hello %s", "world")
+        entries = handler.snapshot()
+        assert entries[-1]["message"] == "hello world"
+        assert entries[-1]["level"] == "info"
+    finally:
+        logger.removeHandler(handler)
+
+
+def test_ring_log_handler_respects_capacity() -> None:
+    handler = RingLogHandler(capacity=3)
+    logger = logging.getLogger("test.ring.cap")
+    logger.addHandler(handler)
+    logger.setLevel(logging.INFO)
+    try:
+        for i in range(10):
+            logger.info("m%d", i)
+        entries = handler.snapshot()
+        assert len(entries) == 3
+        assert entries[-1]["message"] == "m9"
+    finally:
+        logger.removeHandler(handler)
+
+
+def test_metrics_endpoint(runtime) -> None:
+    from fastapi.testclient import TestClient
+
+    from app.main import create_app
+
+    app = create_app(runtime.settings, runtime)
+    with TestClient(app) as client:
+        response = client.get("/api/metrics")
+        assert response.status_code == 200
+        assert "crypto_running" in response.text
+
+
+def test_logs_endpoint(runtime) -> None:
+    from fastapi.testclient import TestClient
+
+    from app.main import create_app
+
+    app = create_app(runtime.settings, runtime)
+    with TestClient(app) as client:
+        response = client.get("/api/logs")
+        assert response.status_code == 200
+        assert "entries" in response.json()

docs/runbook.md

@@ -0,0 +1,59 @@
+# Operations runbook
+
+Practical steps for running the deployed bot. Host paths are examples — adjust
+to the deploy host.
+
+## Find the stack
+
+```bash
+docker compose ps
+docker compose logs --tail=200 backend
+docker compose logs -f frontend
+```
+
+The backend exposes `GET /api/metrics` (Prometheus text format) for the existing
+Prometheus/Grafana stack; scrape it alongside the other services. Useful series:
+`crypto_running`, `crypto_equity`, `crypto_total_pnl`, `crypto_open_positions`,
+`crypto_strategy_equity`, `crypto_strategy_pnl`, `crypto_strategy_halted`,
+`crypto_info`.
+
+## Stop / kill trading
+
+- **Dashboard:** the **Kill all** button (or Stop all) flattens and halts.
+- **API:** `curl -X POST -H "X-API-Token: $TOKEN" https://<host>/api/bot/kill`
+- **Hard stop:** `docker compose stop backend` stops the process entirely. It
+  does **not** close open positions — use the API kill first if you care.
+
+## Something is wrong
+
+1. `docker compose ps` — is the service `healthy`?
+2. `docker compose logs --tail=200 backend` — look for `error`/`Traceback`.
+3. In the dashboard, open **Server logs** (shows the in-memory ring of recent
+   log lines) and **Activity** (trades, halts, errors).
+4. `curl https://<host>/api/live-check` — if live is blocked, the reasons are
+   listed.
+
+## A deploy went out but the UI looks stale
+
+The deploy job fails if the frontend container's build stamp does not match the
+commit; if you see stale UI and the job passed, hard-refresh or open
+incognito. The footer shows `client` vs `server` versions — they must match.
+
+## Reset paper state
+
+```bash
+docker compose stop backend
+# remove the SQLite file on the host mount
+rm -f <CRYPTO_DATA_DIR>/trader.db
+docker compose start backend
+```
+
+## Going live (do this deliberately)
+
+1. Rehearse the kill switch in paper mode.
+2. Set `MAX_DAILY_LOSS`, `MAX_DRAWDOWN_PCT` and/or `MAX_TRADES_PER_DAY`, and
+   `MAX_POSITION_PCT < 1.0`.
+3. Set `MIN_LIVE_BALANCE` and add exchange credentials.
+4. Set `LIVE_CONFIRMATION="I UNDERSTAND THE RISK"`.
+5. Deploy, confirm `/api/live-check` returns `ok: true`, then start with a
+   small balance and watch it.

frontend/src/App.vue

@@ -4,6 +4,7 @@ import * as api from './api'
 import BacktestPanel from './components/BacktestPanel.vue'
 import EquityChart from './components/EquityChart.vue'
 import EventLog from './components/EventLog.vue'
+import LogsPanel from './components/LogsPanel.vue'
 import MultiStrategyPanel from './components/MultiStrategyPanel.vue'
 import OrdersTable from './components/OrdersTable.vue'
 import PositionsTable from './components/PositionsTable.vue'
@@ -374,6 +375,8 @@ onUnmounted(() => {
       <EventLog :events="events" />
     </div>
 
+    <LogsPanel />
+
     <PositionsTable :positions="positions" @close="handleClose" />
     <OrdersTable :orders="orders" />
 

frontend/src/api.js

@@ -12,6 +12,8 @@ export const getPositions = () => http.get('/positions').then((r) => r.data)
 export const getOrders = () => http.get('/orders').then((r) => r.data)
 export const getEvents = () => http.get('/events').then((r) => r.data)
 export const getEquity = () => http.get('/equity').then((r) => r.data)
+export const getLogs = (limit = 100) =>
+  http.get('/logs', { params: { limit } }).then((r) => r.data)
 
 export const startBot = (payload) => http.post('/bot/start', payload).then((r) => r.data)
 export const stopBot = () => http.post('/bot/stop').then((r) => r.data)

frontend/src/components/LogsPanel.vue

@@ -0,0 +1,73 @@
+<script setup>
+import { computed, onMounted, onUnmounted, ref } from 'vue'
+import * as api from '../api'
+import { fmtTime } from '../format'
+import CollapsiblePanel from './CollapsiblePanel.vue'
+
+const entries = ref([])
+const levelFilter = ref('all')
+let timer
+
+const levelClass = {
+  info: 'text-slate-300',
+  warning: 'text-amber-300',
+  error: 'text-rose-300',
+  critical: 'text-rose-400',
+  debug: 'text-slate-500',
+}
+
+const filtered = computed(() => {
+  if (levelFilter.value === 'all') return entries.value
+  return entries.value.filter((e) => e.level === levelFilter.value)
+})
+
+async function load() {
+  try {
+    const data = await api.getLogs(200)
+    entries.value = data.entries || []
+  } catch {
+    // ignore transient errors
+  }
+}
+
+onMounted(() => {
+  load()
+  timer = setInterval(load, 5000)
+})
+
+onUnmounted(() => clearInterval(timer))
+</script>
+
+<template>
+  <CollapsiblePanel
+    title="Server logs"
+    storage-key="logs"
+    :default-open="false"
+    :summary="`${entries.length} lines`"
+  >
+    <div class="space-y-2">
+      <div class="flex items-center gap-2">
+        <label class="space-y-1">
+          <span class="label">Level</span>
+          <select v-model="levelFilter" class="input">
+            <option value="all">all</option>
+            <option value="info">info</option>
+            <option value="warning">warning</option>
+            <option value="error">error</option>
+          </select>
+        </label>
+      </div>
+      <div class="max-h-72 space-y-0.5 overflow-y-auto font-mono text-xs">
+        <div v-for="(e, i) in filtered" :key="i" class="flex gap-2 border-b border-edge/40 py-0.5">
+          <span class="shrink-0 text-slate-500">{{ fmtTime(e.time) }}</span>
+          <span class="shrink-0" :class="levelClass[e.level] || 'text-slate-300'">
+            [{{ e.level }}]
+          </span>
+          <span class="shrink-0 text-slate-500">{{ e.logger }}</span>
+          <span :class="levelClass[e.level] || 'text-slate-300'">{{ e.message }}</span>
+        </div>
+        <div v-if="!filtered.length" class="py-6 text-center text-slate-500">No log lines</div>
+      </div>
+    </div>
+  </CollapsiblePanel>
+</template>