feat(server): 新增 LLM 成本追踪与熔断降级机制
This commit is contained in:
@@ -0,0 +1,555 @@
|
||||
"""LLM 成本追踪器 + 增强降级策略 (MVP-3 / D34)
|
||||
|
||||
职责
|
||||
----
|
||||
|
||||
1. 记录每次 LLM 调用的 token / 费用 / 延迟，提供按时段聚合的统计
|
||||
2. 实现"硬熔断"降级策略: 当达到日预算 / 错误率阈值时自动禁用 LLM
|
||||
3. 提供成本仪表盘所需的导出接口（供 ``api/llm.py`` 暴露给前端）
|
||||
|
||||
设计要点
|
||||
--------
|
||||
|
||||
* **职责分离**：成本追踪本身只负责"记账 + 状态机"，不直接决定能否调用 LLM；
|
||||
调用方 (例如检测管道) 在每次调用前调用 ``can_call()``，根据结果决定降级
|
||||
* **滑动窗口**：使用按 UTC 日历日的桶 (``deque``) 保存最近 N 天的统计，
|
||||
避免长时间运行内存无限增长
|
||||
* **熔断**：达到错误率阈值 (默认连续 ``error_threshold`` 次调用失败率 >50%)
|
||||
自动进入 ``CIRCUIT_OPEN`` 状态，``cooldown_seconds`` 后自动半开尝试恢复
|
||||
* **Token 估算**：未拿到真实 usage 时按"图像 + prompt 长度"做粗估，
|
||||
保证账单不会因为 provider 不返回 usage 而归零
|
||||
|
||||
接入方式
|
||||
--------
|
||||
|
||||
::
|
||||
|
||||
tracker = LLMCostTracker(daily_budget_usd=5.0)
|
||||
if not tracker.can_call():
|
||||
return None # 降级处理
|
||||
result = await llm_service.analyze_with_fallback(req)
|
||||
tracker.record(result, request=req)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import time
|
||||
from collections import deque
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, Deque, Dict, List, Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 常量 / 数据结构
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
# 不同 provider 的近似定价 (USD per 1M tokens)，仅供估算
|
||||
DEFAULT_PRICING: Dict[str, Dict[str, float]] = {
|
||||
# OpenAI GPT-4o
|
||||
"openai": {"prompt": 5.0, "completion": 15.0, "image": 0.005},
|
||||
# 豆包 / Qwen / GLM 国内 VLM 大致价位（仅估算）
|
||||
"qwen": {"prompt": 0.8, "completion": 2.0, "image": 0.001},
|
||||
"glm": {"prompt": 1.0, "completion": 3.0, "image": 0.001},
|
||||
"doubao": {"prompt": 0.8, "completion": 2.0, "image": 0.001},
|
||||
# mock provider 不计费
|
||||
"mock": {"prompt": 0.0, "completion": 0.0, "image": 0.0},
|
||||
# 兜底
|
||||
"default": {"prompt": 1.0, "completion": 3.0, "image": 0.002},
|
||||
}
|
||||
|
||||
|
||||
class CircuitState:
|
||||
CLOSED = "closed" # 正常
|
||||
OPEN = "open" # 熔断
|
||||
HALF_OPEN = "half_open" # 试探恢复
|
||||
|
||||
|
||||
@dataclass
|
||||
class CallRecord:
|
||||
"""单次 LLM 调用记录。"""
|
||||
|
||||
timestamp: float
|
||||
provider: str
|
||||
model: str
|
||||
success: bool
|
||||
confirmed: Optional[bool]
|
||||
latency_ms: float
|
||||
prompt_tokens: int
|
||||
completion_tokens: int
|
||||
image_count: int
|
||||
cost_usd: float
|
||||
error: Optional[str] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class _DailyBucket:
|
||||
"""按 UTC 日历日聚合的桶。"""
|
||||
|
||||
date: str
|
||||
call_count: int = 0
|
||||
success_count: int = 0
|
||||
failure_count: int = 0
|
||||
total_prompt_tokens: int = 0
|
||||
total_completion_tokens: int = 0
|
||||
total_image_count: int = 0
|
||||
total_cost_usd: float = 0.0
|
||||
total_latency_ms: float = 0.0
|
||||
|
||||
def add(self, record: CallRecord) -> None:
|
||||
self.call_count += 1
|
||||
if record.success:
|
||||
self.success_count += 1
|
||||
else:
|
||||
self.failure_count += 1
|
||||
self.total_prompt_tokens += record.prompt_tokens
|
||||
self.total_completion_tokens += record.completion_tokens
|
||||
self.total_image_count += record.image_count
|
||||
self.total_cost_usd += record.cost_usd
|
||||
self.total_latency_ms += record.latency_ms
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
return {
|
||||
"date": self.date,
|
||||
"call_count": self.call_count,
|
||||
"success_count": self.success_count,
|
||||
"failure_count": self.failure_count,
|
||||
"prompt_tokens": self.total_prompt_tokens,
|
||||
"completion_tokens": self.total_completion_tokens,
|
||||
"image_count": self.total_image_count,
|
||||
"cost_usd": round(self.total_cost_usd, 6),
|
||||
"avg_latency_ms": round(
|
||||
self.total_latency_ms / self.call_count, 2
|
||||
)
|
||||
if self.call_count
|
||||
else 0.0,
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# LLMCostTracker
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class LLMCostTracker:
|
||||
"""LLM 成本追踪器 + 熔断降级。
|
||||
|
||||
Args:
|
||||
daily_budget_usd: 日预算，0 表示不限
|
||||
history_days: 保留多少天的日级统计
|
||||
recent_window: 用于错误率计算的最近调用窗口
|
||||
error_rate_threshold: 触发熔断的错误率阈值 [0, 1]
|
||||
cooldown_seconds: 熔断进入半开状态的冷却时间
|
||||
pricing_overrides: 自定义价格表 (覆盖默认)
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
daily_budget_usd: float = 0.0,
|
||||
history_days: int = 7,
|
||||
recent_window: int = 20,
|
||||
error_rate_threshold: float = 0.5,
|
||||
cooldown_seconds: float = 60.0,
|
||||
pricing_overrides: Optional[Dict[str, Dict[str, float]]] = None,
|
||||
) -> None:
|
||||
if daily_budget_usd < 0:
|
||||
raise ValueError("daily_budget_usd 必须 >= 0")
|
||||
if history_days < 1:
|
||||
raise ValueError("history_days 必须 >= 1")
|
||||
if recent_window < 1:
|
||||
raise ValueError("recent_window 必须 >= 1")
|
||||
if not 0.0 < error_rate_threshold <= 1.0:
|
||||
raise ValueError("error_rate_threshold 必须在 (0, 1]")
|
||||
if cooldown_seconds < 0:
|
||||
raise ValueError("cooldown_seconds 必须 >= 0")
|
||||
|
||||
self.daily_budget_usd = daily_budget_usd
|
||||
self.history_days = history_days
|
||||
self.recent_window = recent_window
|
||||
self.error_rate_threshold = error_rate_threshold
|
||||
self.cooldown_seconds = cooldown_seconds
|
||||
|
||||
# 合并定价
|
||||
self._pricing: Dict[str, Dict[str, float]] = {
|
||||
k: dict(v) for k, v in DEFAULT_PRICING.items()
|
||||
}
|
||||
if pricing_overrides:
|
||||
for provider, table in pricing_overrides.items():
|
||||
self._pricing.setdefault(provider, {})
|
||||
self._pricing[provider].update(table)
|
||||
|
||||
# 状态
|
||||
self._buckets: Deque[_DailyBucket] = deque(maxlen=history_days)
|
||||
self._recent: Deque[bool] = deque(maxlen=recent_window)
|
||||
self._records: Deque[CallRecord] = deque(maxlen=200)
|
||||
self._circuit_state = CircuitState.CLOSED
|
||||
self._circuit_opened_at: Optional[float] = None
|
||||
self._manually_disabled = False
|
||||
self._lock = asyncio.Lock()
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# 准入检查
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def can_call(self, now: Optional[float] = None) -> bool:
|
||||
"""同步判断当前是否允许调用 LLM。
|
||||
|
||||
触发拒绝的条件按优先级:
|
||||
1. 手动禁用
|
||||
2. 日预算超限
|
||||
3. 熔断器处于 OPEN 且未到冷却结束
|
||||
"""
|
||||
|
||||
if self._manually_disabled:
|
||||
return False
|
||||
if self._budget_exhausted():
|
||||
return False
|
||||
|
||||
ts = now if now is not None else time.time()
|
||||
if self._circuit_state == CircuitState.OPEN:
|
||||
if (
|
||||
self._circuit_opened_at is not None
|
||||
and (ts - self._circuit_opened_at) >= self.cooldown_seconds
|
||||
):
|
||||
# 进入半开，允许一次试探
|
||||
self._circuit_state = CircuitState.HALF_OPEN
|
||||
logger.info("LLMCostTracker 熔断器进入 HALF_OPEN")
|
||||
return True
|
||||
return False
|
||||
return True
|
||||
|
||||
def reason_for_block(self, now: Optional[float] = None) -> Optional[str]:
|
||||
"""返回当前不可调用的原因 (调试用)，None 表示可以调用。"""
|
||||
|
||||
if self._manually_disabled:
|
||||
return "manually_disabled"
|
||||
if self._budget_exhausted():
|
||||
return f"daily_budget_exceeded({self._today_cost():.4f}/{self.daily_budget_usd})"
|
||||
ts = now if now is not None else time.time()
|
||||
if self._circuit_state == CircuitState.OPEN:
|
||||
if (
|
||||
self._circuit_opened_at is not None
|
||||
and (ts - self._circuit_opened_at) < self.cooldown_seconds
|
||||
):
|
||||
return "circuit_open"
|
||||
return None
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# 记录
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def record(
|
||||
self,
|
||||
result: Any,
|
||||
prompt_tokens: Optional[int] = None,
|
||||
completion_tokens: Optional[int] = None,
|
||||
image_count: int = 1,
|
||||
provider_override: Optional[str] = None,
|
||||
) -> CallRecord:
|
||||
"""记录一次 LLM 调用。
|
||||
|
||||
``result`` 可以是 ``LLMAnalysisResult`` 实例或鸭子类型 (含 confirmed
|
||||
/ provider / model / latency_ms / error / metadata)。
|
||||
"""
|
||||
|
||||
provider = provider_override or getattr(result, "provider", "default")
|
||||
model = getattr(result, "model", "")
|
||||
success = getattr(result, "error", None) is None and getattr(
|
||||
result, "confirmed", None
|
||||
) is not None
|
||||
latency_ms = float(getattr(result, "latency_ms", 0.0) or 0.0)
|
||||
|
||||
usage = self._extract_usage(result)
|
||||
prompt_tokens = (
|
||||
int(prompt_tokens)
|
||||
if prompt_tokens is not None
|
||||
else int(usage.get("prompt_tokens") or 0)
|
||||
)
|
||||
completion_tokens = (
|
||||
int(completion_tokens)
|
||||
if completion_tokens is not None
|
||||
else int(usage.get("completion_tokens") or 0)
|
||||
)
|
||||
|
||||
# token 估算兜底
|
||||
if prompt_tokens == 0 and completion_tokens == 0 and provider != "mock":
|
||||
prompt_tokens = self._estimate_prompt_tokens(image_count)
|
||||
completion_tokens = self._estimate_completion_tokens(result)
|
||||
|
||||
cost = self._calc_cost(provider, prompt_tokens, completion_tokens, image_count)
|
||||
|
||||
record = CallRecord(
|
||||
timestamp=time.time(),
|
||||
provider=provider,
|
||||
model=model,
|
||||
success=success,
|
||||
confirmed=getattr(result, "confirmed", None),
|
||||
latency_ms=latency_ms,
|
||||
prompt_tokens=prompt_tokens,
|
||||
completion_tokens=completion_tokens,
|
||||
image_count=image_count,
|
||||
cost_usd=cost,
|
||||
error=getattr(result, "error", None),
|
||||
)
|
||||
|
||||
self._append_record(record)
|
||||
self._update_circuit(record)
|
||||
return record
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# 控制
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def disable(self) -> None:
|
||||
"""手动禁用 LLM 调用 (运维场景)。"""
|
||||
|
||||
self._manually_disabled = True
|
||||
logger.warning("LLMCostTracker 已被手动禁用")
|
||||
|
||||
def enable(self) -> None:
|
||||
"""手动重新启用。"""
|
||||
|
||||
self._manually_disabled = False
|
||||
self._circuit_state = CircuitState.CLOSED
|
||||
self._circuit_opened_at = None
|
||||
logger.info("LLMCostTracker 已重新启用")
|
||||
|
||||
def reset(self) -> None:
|
||||
"""清空全部统计 (谨慎使用)。"""
|
||||
|
||||
self._buckets.clear()
|
||||
self._recent.clear()
|
||||
self._records.clear()
|
||||
self._circuit_state = CircuitState.CLOSED
|
||||
self._circuit_opened_at = None
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# 导出
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
@property
|
||||
def is_enabled(self) -> bool:
|
||||
return not self._manually_disabled
|
||||
|
||||
@property
|
||||
def circuit_state(self) -> str:
|
||||
return self._circuit_state
|
||||
|
||||
def daily_summary(self) -> List[Dict[str, Any]]:
|
||||
return [bucket.to_dict() for bucket in self._buckets]
|
||||
|
||||
def today_summary(self) -> Dict[str, Any]:
|
||||
bucket = self._today_bucket(create=False)
|
||||
if bucket is None:
|
||||
return {
|
||||
"date": _today_str(),
|
||||
"call_count": 0,
|
||||
"cost_usd": 0.0,
|
||||
"budget_usd": self.daily_budget_usd,
|
||||
"budget_used_ratio": 0.0,
|
||||
}
|
||||
data = bucket.to_dict()
|
||||
data["budget_usd"] = self.daily_budget_usd
|
||||
data["budget_used_ratio"] = (
|
||||
data["cost_usd"] / self.daily_budget_usd
|
||||
if self.daily_budget_usd > 0
|
||||
else 0.0
|
||||
)
|
||||
return data
|
||||
|
||||
def recent_records(self, limit: int = 20) -> List[Dict[str, Any]]:
|
||||
records = list(self._records)[-limit:]
|
||||
return [
|
||||
{
|
||||
"timestamp": r.timestamp,
|
||||
"provider": r.provider,
|
||||
"model": r.model,
|
||||
"success": r.success,
|
||||
"confirmed": r.confirmed,
|
||||
"latency_ms": round(r.latency_ms, 2),
|
||||
"prompt_tokens": r.prompt_tokens,
|
||||
"completion_tokens": r.completion_tokens,
|
||||
"image_count": r.image_count,
|
||||
"cost_usd": round(r.cost_usd, 6),
|
||||
"error": r.error,
|
||||
}
|
||||
for r in reversed(records)
|
||||
]
|
||||
|
||||
def stats(self) -> Dict[str, Any]:
|
||||
recent_total = len(self._recent)
|
||||
recent_failed = sum(1 for ok in self._recent if not ok)
|
||||
recent_error_rate = (
|
||||
recent_failed / recent_total if recent_total > 0 else 0.0
|
||||
)
|
||||
|
||||
return {
|
||||
"enabled": self.is_enabled,
|
||||
"circuit_state": self._circuit_state,
|
||||
"circuit_opened_at": self._circuit_opened_at,
|
||||
"block_reason": self.reason_for_block(),
|
||||
"today": self.today_summary(),
|
||||
"history_days": self.history_days,
|
||||
"recent_window": self.recent_window,
|
||||
"recent_error_rate": round(recent_error_rate, 4),
|
||||
"error_rate_threshold": self.error_rate_threshold,
|
||||
}
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# 内部
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _append_record(self, record: CallRecord) -> None:
|
||||
bucket = self._today_bucket(create=True)
|
||||
assert bucket is not None
|
||||
bucket.add(record)
|
||||
self._recent.append(record.success)
|
||||
self._records.append(record)
|
||||
|
||||
def _today_bucket(self, *, create: bool) -> Optional[_DailyBucket]:
|
||||
today = _today_str()
|
||||
if self._buckets and self._buckets[-1].date == today:
|
||||
return self._buckets[-1]
|
||||
if not create:
|
||||
for b in self._buckets:
|
||||
if b.date == today:
|
||||
return b
|
||||
return None
|
||||
bucket = _DailyBucket(date=today)
|
||||
self._buckets.append(bucket)
|
||||
return bucket
|
||||
|
||||
def _today_cost(self) -> float:
|
||||
bucket = self._today_bucket(create=False)
|
||||
return bucket.total_cost_usd if bucket else 0.0
|
||||
|
||||
def _budget_exhausted(self) -> bool:
|
||||
if self.daily_budget_usd <= 0:
|
||||
return False
|
||||
return self._today_cost() >= self.daily_budget_usd
|
||||
|
||||
def _update_circuit(self, record: CallRecord) -> None:
|
||||
# HALF_OPEN: 一次结果决定回到 CLOSED 还是 OPEN
|
||||
if self._circuit_state == CircuitState.HALF_OPEN:
|
||||
if record.success:
|
||||
self._circuit_state = CircuitState.CLOSED
|
||||
self._circuit_opened_at = None
|
||||
logger.info("LLMCostTracker 熔断器恢复 CLOSED")
|
||||
else:
|
||||
self._circuit_state = CircuitState.OPEN
|
||||
self._circuit_opened_at = time.time()
|
||||
logger.warning("LLMCostTracker 半开试探失败，回到 OPEN")
|
||||
return
|
||||
|
||||
# CLOSED: 检查最近窗口的错误率
|
||||
if (
|
||||
self._circuit_state == CircuitState.CLOSED
|
||||
and len(self._recent) >= max(3, self.recent_window // 2)
|
||||
):
|
||||
failures = sum(1 for ok in self._recent if not ok)
|
||||
error_rate = failures / len(self._recent)
|
||||
if error_rate >= self.error_rate_threshold:
|
||||
self._circuit_state = CircuitState.OPEN
|
||||
self._circuit_opened_at = time.time()
|
||||
logger.warning(
|
||||
"LLMCostTracker 触发熔断: 最近 %d 次调用错误率 %.2f >= %.2f",
|
||||
len(self._recent),
|
||||
error_rate,
|
||||
self.error_rate_threshold,
|
||||
)
|
||||
|
||||
def _calc_cost(
|
||||
self,
|
||||
provider: str,
|
||||
prompt_tokens: int,
|
||||
completion_tokens: int,
|
||||
image_count: int,
|
||||
) -> float:
|
||||
rates = self._pricing.get(provider) or self._pricing["default"]
|
||||
cost = (
|
||||
prompt_tokens / 1_000_000.0 * rates.get("prompt", 0.0)
|
||||
+ completion_tokens / 1_000_000.0 * rates.get("completion", 0.0)
|
||||
+ image_count * rates.get("image", 0.0)
|
||||
)
|
||||
return max(0.0, cost)
|
||||
|
||||
@staticmethod
|
||||
def _estimate_prompt_tokens(image_count: int) -> int:
|
||||
# 经验值：每张图 ~ 200 token，prompt 模板 ~ 120 token
|
||||
return 120 + 200 * max(0, image_count)
|
||||
|
||||
@staticmethod
|
||||
def _estimate_completion_tokens(result: Any) -> int:
|
||||
reasoning = getattr(result, "reasoning", "") or ""
|
||||
# 粗略 1.3 字符 = 1 token
|
||||
return max(16, int(len(reasoning) / 1.3))
|
||||
|
||||
@staticmethod
|
||||
def _extract_usage(result: Any) -> Dict[str, Any]:
|
||||
"""从结果对象上尽力获取 usage (provider 可能放 metadata 里)。"""
|
||||
|
||||
usage = getattr(result, "usage", None)
|
||||
if isinstance(usage, dict):
|
||||
return usage
|
||||
metadata = getattr(result, "metadata", None)
|
||||
if isinstance(metadata, dict):
|
||||
md_usage = metadata.get("usage")
|
||||
if isinstance(md_usage, dict):
|
||||
return md_usage
|
||||
return {}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 全局单例
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
_global_tracker: Optional[LLMCostTracker] = None
|
||||
|
||||
|
||||
def init_global_tracker(
|
||||
daily_budget_usd: float = 0.0,
|
||||
history_days: int = 7,
|
||||
recent_window: int = 20,
|
||||
error_rate_threshold: float = 0.5,
|
||||
cooldown_seconds: float = 60.0,
|
||||
) -> LLMCostTracker:
|
||||
"""初始化并返回全局成本追踪器。"""
|
||||
|
||||
global _global_tracker
|
||||
_global_tracker = LLMCostTracker(
|
||||
daily_budget_usd=daily_budget_usd,
|
||||
history_days=history_days,
|
||||
recent_window=recent_window,
|
||||
error_rate_threshold=error_rate_threshold,
|
||||
cooldown_seconds=cooldown_seconds,
|
||||
)
|
||||
return _global_tracker
|
||||
|
||||
|
||||
def get_global_tracker() -> Optional[LLMCostTracker]:
|
||||
return _global_tracker
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 工具
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _today_str() -> str:
|
||||
return datetime.now(tz=timezone.utc).strftime("%Y-%m-%d")
|
||||
|
||||
|
||||
__all__ = [
|
||||
"LLMCostTracker",
|
||||
"CallRecord",
|
||||
"CircuitState",
|
||||
"DEFAULT_PRICING",
|
||||
"init_global_tracker",
|
||||
"get_global_tracker",
|
||||
]
|
||||
Reference in New Issue
Block a user