Observability: Prometheus metrics, log panel, runbook
- GET /api/metrics renders Prometheus text (equity, P&L, per-strategy equity/pnl/allocation/halted, info) with no client library. - In-memory ring log handler (app/logs.py) exposed at GET /api/logs and a dashboard "Server logs" panel; Runtime mirrors bot events into the logger. - docs/runbook.md covers deploys, the kill switch, resetting paper state and going live. Adds tests for metrics rendering and the log ring.
This commit is contained in:
1 parent
c605a0f316
commit
ac92dd5ac4
12 files changed
+462
-1
No files matched your search
@@ -0,0 +1,96 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from app.schemas import MultiStatusOut
|
||||
|
||||
# Rendered as Prometheus text exposition format (no client library needed).
|
||||
# The route serves this at GET /api/metrics for the existing Prometheus scrape.
|
||||
|
||||
|
||||
def _escape(value: str) -> str:
|
||||
return value.replace("\\", "\\\\").replace('"', '\\"')
|
||||
|
||||
|
||||
def _metric(
|
||||
name: str, help_text: str, value: float | int, labels: dict[str, str] | None = None
|
||||
) -> str:
|
||||
lines = [f"# HELP {name} {help_text}", f"# TYPE {name} gauge"]
|
||||
if labels:
|
||||
rendered = ",".join(f'{k}="{_escape(v)}"' for k, v in labels.items())
|
||||
lines.append(f"{name}{{{rendered}}} {value}")
|
||||
else:
|
||||
lines.append(f"{name} {value}")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def render_metrics(status: MultiStatusOut, mode: str, exchange: str) -> str:
|
||||
"""Render the current portfolio state as Prometheus metrics."""
|
||||
parts = [
|
||||
_metric("crypto_running", "1 when the bot is running", int(status.running)),
|
||||
_metric("crypto_equity", "Total account equity", status.equity),
|
||||
_metric("crypto_starting_balance", "Starting balance", status.starting_balance),
|
||||
_metric("crypto_total_pnl", "Total profit and loss", status.total_pnl),
|
||||
_metric(
|
||||
"crypto_open_positions",
|
||||
"Number of open positions across strategies",
|
||||
sum(s.open_positions for s in status.strategies),
|
||||
),
|
||||
_metric(
|
||||
"crypto_strategies",
|
||||
"Number of configured strategies",
|
||||
len(status.strategies),
|
||||
),
|
||||
]
|
||||
|
||||
if status.price is not None:
|
||||
parts.append(_metric("crypto_price", "Latest mark price", status.price))
|
||||
|
||||
# Common labels shared by the per-strategy families.
|
||||
lines = [
|
||||
"# HELP crypto_strategy_equity Equity per strategy",
|
||||
"# TYPE crypto_strategy_equity gauge",
|
||||
]
|
||||
for s in status.strategies:
|
||||
labels = {"strategy": s.strategy_key, "symbol": s.symbol}
|
||||
rendered = ",".join(f'{k}="{_escape(v)}"' for k, v in labels.items())
|
||||
lines.append(f"crypto_strategy_equity{{{rendered}}} {s.equity}")
|
||||
parts.append("\n".join(lines))
|
||||
|
||||
lines = [
|
||||
"# HELP crypto_strategy_pnl Total P&L per strategy",
|
||||
"# TYPE crypto_strategy_pnl gauge",
|
||||
]
|
||||
for s in status.strategies:
|
||||
labels = {"strategy": s.strategy_key, "symbol": s.symbol}
|
||||
rendered = ",".join(f'{k}="{_escape(v)}"' for k, v in labels.items())
|
||||
lines.append(f"crypto_strategy_pnl{{{rendered}}} {s.total_pnl}")
|
||||
parts.append("\n".join(lines))
|
||||
|
||||
lines = [
|
||||
"# HELP crypto_strategy_allocation Fraction of the balance per strategy",
|
||||
"# TYPE crypto_strategy_allocation gauge",
|
||||
]
|
||||
for s in status.strategies:
|
||||
labels = {"strategy": s.strategy_key}
|
||||
rendered = ",".join(f'{k}="{_escape(v)}"' for k, v in labels.items())
|
||||
lines.append(f"crypto_strategy_allocation{{{rendered}}} {s.allocation}")
|
||||
parts.append("\n".join(lines))
|
||||
|
||||
lines = [
|
||||
"# HELP crypto_strategy_halted 1 when a strategy is halted",
|
||||
"# TYPE crypto_strategy_halted gauge",
|
||||
]
|
||||
for s in status.strategies:
|
||||
labels = {"strategy": s.strategy_key}
|
||||
rendered = ",".join(f'{k}="{_escape(v)}"' for k, v in labels.items())
|
||||
lines.append(f"crypto_strategy_halted{{{rendered}}} {int(s.halted)}")
|
||||
parts.append("\n".join(lines))
|
||||
|
||||
info_labels = {"mode": mode, "exchange": exchange}
|
||||
rendered = ",".join(f'{k}="{_escape(v)}"' for k, v in info_labels.items())
|
||||
parts.append(
|
||||
"# HELP crypto_info Build information\n"
|
||||
"# TYPE crypto_info gauge\n"
|
||||
f"crypto_info{{{rendered}}} 1"
|
||||
)
|
||||
|
||||
return "\n".join(parts) + "\n"
|
||||
Reference in new issue
Block a user