diff --git a/AGENTS.md b/AGENTS.md index 2d9568c..b8c972b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -136,6 +136,17 @@ the single and multi start paths are covered) and surfaced via call `check_live_ready` first. `LIVE_CONFIRMATION` must equal `"I UNDERSTAND THE RISK"` — keep it a phrase, not a boolean. +## Observability + +- `app/metrics.py` renders Prometheus text at `GET /api/metrics` straight from + the manager status — no client library. Keep new series out of hot loops. +- `app/logs.py` keeps a small in-memory ring of log records, exposed at + `GET /api/logs` and shown in the dashboard. `Runtime._log_event` mirrors bot + events into the logger so the panel is useful (uvicorn's own loggers do not + propagate to the root handler). +- `docs/runbook.md` is the operations guide; update it when deploy or + recovery steps change. + ## Risk controls & notifications - `app/engine/risk.py` (`RiskManager`) tracks daily loss, drawdown, position cap diff --git a/README.md b/README.md index ee79d5f..e5bca00 100644 --- a/README.md +++ b/README.md @@ -220,6 +220,8 @@ Bundled strategies: `sma_cross`, `ema_cross`, `macd_trend`, | `POST` | `/api/bot/start-many` | Start several strategies with per-strategy allocations | | `GET` | `/api/bot/status` | Per-strategy status for a multi-strategy run | | `GET` | `/api/live-check` | Go-live preflight (blockers for live trading) | +| `GET` | `/api/metrics` | Prometheus text exposition | +| `GET` | `/api/logs` | Recent backend log lines (in-memory ring) | | `POST` | `/api/bot/stop` | Stop the bot | | `POST` | `/api/bot/kill` | Kill switch: flatten + halt | | `POST` | `/api/backtest` | Backtest a strategy on historical candles | @@ -234,6 +236,16 @@ Bundled strategies: `sma_cross`, `ema_cross`, `macd_trend`, | `DELETE` | `/api/presets/{id}` | Delete a preset | | `WS` | `/api/ws` | Live status + event stream | +## Observability + +- `GET /api/metrics` exposes Prometheus metrics (`crypto_equity`, + `crypto_total_pnl`, per-strategy `crypto_strategy_equity`/`_pnl`/`_allocation`/ + `_halted`, `crypto_info`). Point the existing Prometheus scrape at it. +- The dashboard's **Server logs** panel shows recent backend log lines + (in-memory ring buffer), useful alongside **Activity**. +- See [docs/runbook.md](docs/runbook.md) for operations (deploys, kill switch, + resetting paper state, going live). + ## Tests & lint ```bash diff --git a/backend/app/api/routes.py b/backend/app/api/routes.py index 0a5dcb5..1275211 100644 --- a/backend/app/api/routes.py +++ b/backend/app/api/routes.py @@ -4,10 +4,20 @@ import asyncio from typing import Annotated import httpx -from fastapi import APIRouter, Depends, HTTPException, Request, WebSocket, WebSocketDisconnect +from fastapi import ( + APIRouter, + Depends, + HTTPException, + Request, + Response, + WebSocket, + WebSocketDisconnect, +) from app.config import Settings from app.engine.strategies import get_strategy, list_strategies, strategy_spec +from app.logs import get_log_handler +from app.metrics import render_metrics from app.models import OrderSide, Position, PositionStatus from app.runtime import Runtime from app.schemas import ( @@ -65,6 +75,21 @@ def health() -> dict[str, str]: return {"status": "ok"} +@router.get("/metrics") +def metrics(runtime: GuardedRuntimeDep) -> Response: + """Prometheus text exposition, unauthenticated so a scraper can reach it.""" + status = runtime.multi_status() + body = render_metrics(status, runtime.settings.mode, runtime.settings.exchange) + return Response(content=body, media_type="text/plain; version=0.0.4") + + +@router.get("/logs") +def logs(runtime: GuardedRuntimeDep, limit: int = 100) -> dict: + handler = get_log_handler() + entries = handler.snapshot(limit) if handler is not None else [] + return {"entries": entries} + + @router.get("/version") def version(request: Request) -> dict[str, str]: """Build version of the running backend, used by the client to self-heal diff --git a/backend/app/logs.py b/backend/app/logs.py new file mode 100644 index 0000000..819d357 --- /dev/null +++ b/backend/app/logs.py @@ -0,0 +1,55 @@ +from __future__ import annotations + +import logging +from collections import deque +from datetime import UTC, datetime + + +class RingLogHandler(logging.Handler): + """Keeps the most recent log records in memory for the dashboard. + + Deliberately small and dependency-free: the goal is a debugging aid, not a + log store. Attach once at startup. + """ + + def __init__(self, capacity: int = 200) -> None: + super().__init__() + self.records: deque[dict] = deque(maxlen=capacity) + + def emit(self, record: logging.LogRecord) -> None: + try: + self.records.append( + { + "time": datetime.fromtimestamp(record.created, UTC).isoformat(), + "level": record.levelname.lower(), + "logger": record.name, + "message": record.getMessage(), + } + ) + except Exception: # never let logging break the app + self.handleError(record) + + def snapshot(self, limit: int = 100) -> list[dict]: + items = list(self.records) + return items[-limit:] + + +_handler: RingLogHandler | None = None + + +def install_log_capture(capacity: int = 200) -> RingLogHandler: + """Attach the ring handler to the root logger (idempotent).""" + global _handler + if _handler is not None: + return _handler + _handler = RingLogHandler(capacity) + _handler.setLevel(logging.INFO) + root = logging.getLogger() + root.addHandler(_handler) + if root.level > logging.INFO or root.level == logging.NOTSET: + root.setLevel(logging.INFO) + return _handler + + +def get_log_handler() -> RingLogHandler | None: + return _handler diff --git a/backend/app/main.py b/backend/app/main.py index 774569c..dcedd1d 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -10,6 +10,7 @@ from fastapi.middleware.cors import CORSMiddleware from app.api.routes import router from app.config import Settings, get_settings from app.db import init_db +from app.logs import install_log_capture from app.runtime import Runtime @@ -22,6 +23,7 @@ def create_app( @asynccontextmanager async def lifespan(app: FastAPI) -> AsyncIterator[None]: + install_log_capture() init_db() app.state.runtime = runtime yield diff --git a/backend/app/metrics.py b/backend/app/metrics.py new file mode 100644 index 0000000..d2b166b --- /dev/null +++ b/backend/app/metrics.py @@ -0,0 +1,96 @@ +from __future__ import annotations + +from app.schemas import MultiStatusOut + +# Rendered as Prometheus text exposition format (no client library needed). +# The route serves this at GET /api/metrics for the existing Prometheus scrape. + + +def _escape(value: str) -> str: + return value.replace("\\", "\\\\").replace('"', '\\"') + + +def _metric( + name: str, help_text: str, value: float | int, labels: dict[str, str] | None = None +) -> str: + lines = [f"# HELP {name} {help_text}", f"# TYPE {name} gauge"] + if labels: + rendered = ",".join(f'{k}="{_escape(v)}"' for k, v in labels.items()) + lines.append(f"{name}{{{rendered}}} {value}") + else: + lines.append(f"{name} {value}") + return "\n".join(lines) + + +def render_metrics(status: MultiStatusOut, mode: str, exchange: str) -> str: + """Render the current portfolio state as Prometheus metrics.""" + parts = [ + _metric("crypto_running", "1 when the bot is running", int(status.running)), + _metric("crypto_equity", "Total account equity", status.equity), + _metric("crypto_starting_balance", "Starting balance", status.starting_balance), + _metric("crypto_total_pnl", "Total profit and loss", status.total_pnl), + _metric( + "crypto_open_positions", + "Number of open positions across strategies", + sum(s.open_positions for s in status.strategies), + ), + _metric( + "crypto_strategies", + "Number of configured strategies", + len(status.strategies), + ), + ] + + if status.price is not None: + parts.append(_metric("crypto_price", "Latest mark price", status.price)) + + # Common labels shared by the per-strategy families. + lines = [ + "# HELP crypto_strategy_equity Equity per strategy", + "# TYPE crypto_strategy_equity gauge", + ] + for s in status.strategies: + labels = {"strategy": s.strategy_key, "symbol": s.symbol} + rendered = ",".join(f'{k}="{_escape(v)}"' for k, v in labels.items()) + lines.append(f"crypto_strategy_equity{{{rendered}}} {s.equity}") + parts.append("\n".join(lines)) + + lines = [ + "# HELP crypto_strategy_pnl Total P&L per strategy", + "# TYPE crypto_strategy_pnl gauge", + ] + for s in status.strategies: + labels = {"strategy": s.strategy_key, "symbol": s.symbol} + rendered = ",".join(f'{k}="{_escape(v)}"' for k, v in labels.items()) + lines.append(f"crypto_strategy_pnl{{{rendered}}} {s.total_pnl}") + parts.append("\n".join(lines)) + + lines = [ + "# HELP crypto_strategy_allocation Fraction of the balance per strategy", + "# TYPE crypto_strategy_allocation gauge", + ] + for s in status.strategies: + labels = {"strategy": s.strategy_key} + rendered = ",".join(f'{k}="{_escape(v)}"' for k, v in labels.items()) + lines.append(f"crypto_strategy_allocation{{{rendered}}} {s.allocation}") + parts.append("\n".join(lines)) + + lines = [ + "# HELP crypto_strategy_halted 1 when a strategy is halted", + "# TYPE crypto_strategy_halted gauge", + ] + for s in status.strategies: + labels = {"strategy": s.strategy_key} + rendered = ",".join(f'{k}="{_escape(v)}"' for k, v in labels.items()) + lines.append(f"crypto_strategy_halted{{{rendered}}} {int(s.halted)}") + parts.append("\n".join(lines)) + + info_labels = {"mode": mode, "exchange": exchange} + rendered = ",".join(f'{k}="{_escape(v)}"' for k, v in info_labels.items()) + parts.append( + "# HELP crypto_info Build information\n" + "# TYPE crypto_info gauge\n" + f"crypto_info{{{rendered}}} 1" + ) + + return "\n".join(parts) + "\n" diff --git a/backend/app/runtime.py b/backend/app/runtime.py index dd584b8..4c5fcb1 100644 --- a/backend/app/runtime.py +++ b/backend/app/runtime.py @@ -32,6 +32,15 @@ from app.store import Store logger = logging.getLogger(__name__) +_LOG_LEVELS = { + "debug": logging.DEBUG, + "info": logging.INFO, + "trade": logging.INFO, + "warning": logging.WARNING, + "error": logging.ERROR, + "critical": logging.CRITICAL, +} + FeedFactory = Callable[[Settings], MarketData] BrokerFactory = Callable[[Settings], Broker] NotifierFactory = Callable[[Settings], Notifier | None] @@ -91,8 +100,15 @@ class Runtime: with suppress(asyncio.QueueFull): queue.put_nowait(payload) if payload.get("type") == "event": + self._log_event(payload["data"]) self._maybe_notify(payload["data"]) + def _log_event(self, event: dict[str, Any]) -> None: + """Mirror bot events into the logger so the dashboard log view is useful.""" + level = str(event.get("level", "info")).lower() + message = event.get("message", "") + logger.log(_LOG_LEVELS.get(level, logging.INFO), "%s", message) + def _maybe_notify(self, event: dict[str, Any]) -> None: if event.get("level") not in self.notify_levels: return diff --git a/backend/tests/test_observability.py b/backend/tests/test_observability.py new file mode 100644 index 0000000..3ddcb8e --- /dev/null +++ b/backend/tests/test_observability.py @@ -0,0 +1,107 @@ +from __future__ import annotations + +import logging + +from app.logs import RingLogHandler +from app.metrics import render_metrics +from app.schemas import MultiStatusOut, StrategyStatusOut + + +def make_status(**overrides) -> MultiStatusOut: + strategy = StrategyStatusOut( + strategy_key="sma_cross", + strategy_name="SMA Crossover", + params={}, + symbol="BTC/USDT", + timeframe="5m", + allocation=0.5, + cash=5000, + equity=5100, + realized_pnl=100, + unrealized_pnl=0, + total_pnl=100, + open_positions=1, + halted=False, + trades_today=2, + ) + base = { + "running": True, + "mode": "paper", + "exchange": "binance", + "price": 100.0, + "starting_balance": 10000, + "equity": 10100, + "total_pnl": 100, + "strategies": [strategy], + } + base.update(overrides) + return MultiStatusOut(**base) + + +def test_render_metrics_contains_core_series() -> None: + body = render_metrics(make_status(), "paper", "binance") + assert "crypto_running 1" in body + assert "crypto_equity 10100" in body + assert "crypto_total_pnl 100" in body + assert "crypto_open_positions 1" in body + assert 'crypto_strategy_equity{strategy="sma_cross",symbol="BTC/USDT"} 5100' in body + assert 'crypto_strategy_allocation{strategy="sma_cross"} 0.5' in body + assert 'crypto_info{mode="paper",exchange="binance"} 1' in body + + +def test_render_metrics_omits_price_when_none() -> None: + body = render_metrics(make_status(price=None), "paper", "binance") + assert "crypto_price" not in body + + +def test_ring_log_handler_captures() -> None: + handler = RingLogHandler(capacity=5) + logger = logging.getLogger("test.ring") + logger.addHandler(handler) + logger.setLevel(logging.INFO) + try: + logger.info("hello %s", "world") + entries = handler.snapshot() + assert entries[-1]["message"] == "hello world" + assert entries[-1]["level"] == "info" + finally: + logger.removeHandler(handler) + + +def test_ring_log_handler_respects_capacity() -> None: + handler = RingLogHandler(capacity=3) + logger = logging.getLogger("test.ring.cap") + logger.addHandler(handler) + logger.setLevel(logging.INFO) + try: + for i in range(10): + logger.info("m%d", i) + entries = handler.snapshot() + assert len(entries) == 3 + assert entries[-1]["message"] == "m9" + finally: + logger.removeHandler(handler) + + +def test_metrics_endpoint(runtime) -> None: + from fastapi.testclient import TestClient + + from app.main import create_app + + app = create_app(runtime.settings, runtime) + with TestClient(app) as client: + response = client.get("/api/metrics") + assert response.status_code == 200 + assert "crypto_running" in response.text + + +def test_logs_endpoint(runtime) -> None: + from fastapi.testclient import TestClient + + from app.main import create_app + + app = create_app(runtime.settings, runtime) + with TestClient(app) as client: + response = client.get("/api/logs") + assert response.status_code == 200 + assert "entries" in response.json() diff --git a/docs/runbook.md b/docs/runbook.md new file mode 100644 index 0000000..8d2b8d5 --- /dev/null +++ b/docs/runbook.md @@ -0,0 +1,59 @@ +# Operations runbook + +Practical steps for running the deployed bot. Host paths are examples — adjust +to the deploy host. + +## Find the stack + +```bash +docker compose ps +docker compose logs --tail=200 backend +docker compose logs -f frontend +``` + +The backend exposes `GET /api/metrics` (Prometheus text format) for the existing +Prometheus/Grafana stack; scrape it alongside the other services. Useful series: +`crypto_running`, `crypto_equity`, `crypto_total_pnl`, `crypto_open_positions`, +`crypto_strategy_equity`, `crypto_strategy_pnl`, `crypto_strategy_halted`, +`crypto_info`. + +## Stop / kill trading + +- **Dashboard:** the **Kill all** button (or Stop all) flattens and halts. +- **API:** `curl -X POST -H "X-API-Token: $TOKEN" https:///api/bot/kill` +- **Hard stop:** `docker compose stop backend` stops the process entirely. It + does **not** close open positions — use the API kill first if you care. + +## Something is wrong + +1. `docker compose ps` — is the service `healthy`? +2. `docker compose logs --tail=200 backend` — look for `error`/`Traceback`. +3. In the dashboard, open **Server logs** (shows the in-memory ring of recent + log lines) and **Activity** (trades, halts, errors). +4. `curl https:///api/live-check` — if live is blocked, the reasons are + listed. + +## A deploy went out but the UI looks stale + +The deploy job fails if the frontend container's build stamp does not match the +commit; if you see stale UI and the job passed, hard-refresh or open +incognito. The footer shows `client` vs `server` versions — they must match. + +## Reset paper state + +```bash +docker compose stop backend +# remove the SQLite file on the host mount +rm -f /trader.db +docker compose start backend +``` + +## Going live (do this deliberately) + +1. Rehearse the kill switch in paper mode. +2. Set `MAX_DAILY_LOSS`, `MAX_DRAWDOWN_PCT` and/or `MAX_TRADES_PER_DAY`, and + `MAX_POSITION_PCT < 1.0`. +3. Set `MIN_LIVE_BALANCE` and add exchange credentials. +4. Set `LIVE_CONFIRMATION="I UNDERSTAND THE RISK"`. +5. Deploy, confirm `/api/live-check` returns `ok: true`, then start with a + small balance and watch it. diff --git a/frontend/src/App.vue b/frontend/src/App.vue index 9421c8c..eb71018 100644 --- a/frontend/src/App.vue +++ b/frontend/src/App.vue @@ -4,6 +4,7 @@ import * as api from './api' import BacktestPanel from './components/BacktestPanel.vue' import EquityChart from './components/EquityChart.vue' import EventLog from './components/EventLog.vue' +import LogsPanel from './components/LogsPanel.vue' import MultiStrategyPanel from './components/MultiStrategyPanel.vue' import OrdersTable from './components/OrdersTable.vue' import PositionsTable from './components/PositionsTable.vue' @@ -374,6 +375,8 @@ onUnmounted(() => { + + diff --git a/frontend/src/api.js b/frontend/src/api.js index e3e6c3c..ad4ccaa 100644 --- a/frontend/src/api.js +++ b/frontend/src/api.js @@ -12,6 +12,8 @@ export const getPositions = () => http.get('/positions').then((r) => r.data) export const getOrders = () => http.get('/orders').then((r) => r.data) export const getEvents = () => http.get('/events').then((r) => r.data) export const getEquity = () => http.get('/equity').then((r) => r.data) +export const getLogs = (limit = 100) => + http.get('/logs', { params: { limit } }).then((r) => r.data) export const startBot = (payload) => http.post('/bot/start', payload).then((r) => r.data) export const stopBot = () => http.post('/bot/stop').then((r) => r.data) diff --git a/frontend/src/components/LogsPanel.vue b/frontend/src/components/LogsPanel.vue new file mode 100644 index 0000000..d9af2ad --- /dev/null +++ b/frontend/src/components/LogsPanel.vue @@ -0,0 +1,73 @@ + + +