quant-trader-service/training/trader_training/diagnostics.py

from __future__ import annotations

import logging
from typing import Any

import numpy as np
import pandas as pd

from trader_training.io_utils import read_json, read_parquet, run_root, write_json, write_text
from trader_training.pm import _pm_frame, _price_plan_context, _simulate_open_trades, _threshold_candidates, _thresholds_from_config, _trade_metrics, default_pm_config
from trader_training.schemas import FIT_SPLIT, LATEST_STRESS_SPLIT, TUNE_SPLIT, VALIDATION_LOCKED_SPLIT


DIAGNOSTIC_SPLITS = (FIT_SPLIT, TUNE_SPLIT, VALIDATION_LOCKED_SPLIT, LATEST_STRESS_SPLIT)
PM_EVAL_SPLITS = (TUNE_SPLIT, VALIDATION_LOCKED_SPLIT, LATEST_STRESS_SPLIT)


def diagnose_training_run(args: Any) -> None:
    root = run_root(args)
    label_summary = _label_summary(root)
    pm_summary = _pm_summary(root)
    payload = {
        "run_id": args.run_id,
        "label_summary": label_summary,
        "pm_summary": pm_summary,
        "conclusion": _diagnostic_conclusion(pm_summary),
    }
    write_json(root / "diagnostics" / "training_failure_diagnostics.json", _jsonable(payload))
    write_text(root / "diagnostics" / "training_failure_diagnostics.md", _markdown_report(payload))
    logging.info(
        "trader.training.diagnostics_written runId=%s conclusion=%s path=%s",
        args.run_id,
        payload["conclusion"]["status"],
        root / "diagnostics" / "training_failure_diagnostics.md",
    )


def _label_summary(root) -> dict[str, Any]:
    direction = read_parquet(root / "label" / "direction_labels.parquet")
    entry = read_parquet(root / "label" / "entry_labels.parquet")
    summary: dict[str, Any] = {}
    for split_id in DIAGNOSTIC_SPLITS:
        direction_split = direction[direction["split_id"].eq(split_id)].copy()
        entry_split = entry[entry["split_id"].eq(split_id)].copy()
        item: dict[str, Any] = {"direction": {}, "entry": {}}
        if not direction_split.empty:
            item["direction"] = {
                "rows": len(direction_split),
                "label_ratio": direction_split["direction_label"].value_counts(normalize=True).round(6).to_dict(),
                "future_return_bps_quantile": _quantiles(direction_split["future_return_bps"], (0.01, 0.05, 0.25, 0.5, 0.75, 0.95, 0.99)),
            }
        if not entry_split.empty:
            grouped = entry_split.groupby("side", observed=False)
            item["entry"] = {
                "rows": len(entry_split),
                "target_rate_by_side": grouped["entry_target"].mean().round(6).to_dict(),
                "edge_mean_by_side": grouped["expected_net_edge_bps"].mean().round(6).to_dict(),
                "edge_quantile_by_side": {
                    str(side): _quantiles(group["expected_net_edge_bps"], (0.05, 0.5, 0.95))
                    for side, group in grouped
                },
            }
        summary[split_id] = item
    return summary


def _pm_summary(root) -> dict[str, Any]:
    summary: dict[str, Any] = {}
    config_path = root / "pm-search" / "position_manager_config.json"
    config = read_json(config_path)["config"] if config_path.is_file() else default_pm_config()
    thresholds = _thresholds_from_config(config)
    price_plan = _price_plan_context(root)
    for split_id in PM_EVAL_SPLITS:
        frame = _pm_frame(root, split_id)
        selected_trades = _simulate_open_trades(frame, thresholds, config, price_plan)
        item = {
            "rows": len(frame),
            "score_distribution": _score_distribution(frame),
            "active_thresholds": thresholds,
            "gate_funnel": _gate_funnel(frame, thresholds),
            "selected_trade_metrics": _trade_metrics(selected_trades),
            "relaxed_variants": _relaxed_variants(frame),
            "top_bucket_edge": _top_bucket_edge(frame),
            "grid_search_any_trade": _grid_trade_summary(frame),
        }
        summary[split_id] = item
    return summary


def _score_distribution(frame: pd.DataFrame) -> dict[str, dict[str, float]]:
    columns = [
        "long_prob",
        "short_prob",
        "long_entry_prob",
        "short_entry_prob",
        "market_risk_prob",
        "pred_long_expected_net_edge_bps",
        "pred_short_expected_net_edge_bps",
        "model_pred_long_expected_net_edge_bps",
        "model_pred_short_expected_net_edge_bps",
        "actual_long_expected_net_edge_bps",
        "actual_short_expected_net_edge_bps",
    ]
    return {column: _quantiles(frame[column], (0.0, 0.05, 0.5, 0.95, 1.0)) for column in columns if column in frame.columns}


def _gate_funnel(frame: pd.DataFrame, thresholds: dict[str, float]) -> dict[str, Any]:
    direction_margin = (frame["long_prob"] - frame["short_prob"]).abs()
    long_steps = {
        f"long_prob > {thresholds['long_open_prob']}": frame["long_prob"] > thresholds["long_open_prob"],
        f"abs(long_prob - short_prob) > {thresholds['min_direction_margin']}": direction_margin > thresholds["min_direction_margin"],
        f"long_entry_prob > {thresholds['min_entry_prob']}": frame["long_entry_prob"] > thresholds["min_entry_prob"],
        f"market_risk_prob < {thresholds['max_market_risk_prob']}": frame["market_risk_prob"] < thresholds["max_market_risk_prob"],
        f"pred_long_expected_net_edge_bps > {thresholds['min_expected_edge_bps']}": frame["pred_long_expected_net_edge_bps"] > thresholds["min_expected_edge_bps"],
    }
    short_steps = {
        f"short_prob > {thresholds['short_open_prob']}": frame["short_prob"] > thresholds["short_open_prob"],
        f"abs(long_prob - short_prob) > {thresholds['min_direction_margin']}": direction_margin > thresholds["min_direction_margin"],
        f"short_entry_prob > {thresholds['min_entry_prob']}": frame["short_entry_prob"] > thresholds["min_entry_prob"],
        f"market_risk_prob < {thresholds['max_market_risk_prob']}": frame["market_risk_prob"] < thresholds["max_market_risk_prob"],
        f"pred_short_expected_net_edge_bps > {thresholds['min_expected_edge_bps']}": frame["pred_short_expected_net_edge_bps"] > thresholds["min_expected_edge_bps"],
    }
    return {
        "thresholds": thresholds,
        "long": _cumulative_gate_counts(long_steps, len(frame)),
        "short": _cumulative_gate_counts(short_steps, len(frame)),
    }


def _cumulative_gate_counts(steps: dict[str, pd.Series], total_rows: int) -> dict[str, Any]:
    mask = np.ones(total_rows, dtype=bool)
    cumulative = []
    single = {}
    for name, step in steps.items():
        values = step.to_numpy(dtype=bool)
        single[name] = int(values.sum())
        mask &= values
        cumulative.append({"gate": name, "rows_after_gate": int(mask.sum())})
    return {"single_gate_pass": single, "cumulative": cumulative}


def _relaxed_variants(frame: pd.DataFrame) -> dict[str, Any]:
    variants = {
        "no_risk_no_edge": {"prob": 0.54, "entry": 0.50, "margin": 0.02, "risk": 1.0, "edge": -99.0},
        "rare_entry_low_prob": {"prob": 0.50, "entry": 0.03, "margin": 0.02, "risk": 0.98, "edge": 0.0},
        "entry_only_55": {"prob": 0.0, "entry": 0.55, "margin": -99.0, "risk": 1.0, "edge": -99.0},
        "direction_only_54": {"prob": 0.54, "entry": 0.0, "margin": 0.02, "risk": 1.0, "edge": -99.0},
        "very_loose": {"prob": 0.50, "entry": 0.45, "margin": 0.0, "risk": 1.0, "edge": -99.0},
    }
    result: dict[str, Any] = {}
    for name, thresholds in variants.items():
        trades = _variant_trades(frame, thresholds)
        result[name] = _plain_trade_metrics(trades)
    return result


def _variant_trades(frame: pd.DataFrame, thresholds: dict[str, float]) -> pd.DataFrame:
    long_mask = (
        (frame["long_prob"] >= thresholds["prob"])
        & ((frame["long_prob"] - frame["short_prob"]) >= thresholds["margin"])
        & (frame["long_entry_prob"] >= thresholds["entry"])
        & (frame["market_risk_prob"] <= thresholds["risk"])
        & (frame["pred_long_expected_net_edge_bps"] >= thresholds["edge"])
    )
    short_mask = (
        (frame["short_prob"] >= thresholds["prob"])
        & ((frame["short_prob"] - frame["long_prob"]) >= thresholds["margin"])
        & (frame["short_entry_prob"] >= thresholds["entry"])
        & (frame["market_risk_prob"] <= thresholds["risk"])
        & (frame["pred_short_expected_net_edge_bps"] >= thresholds["edge"])
    )
    long = frame.loc[long_mask].copy()
    long["side"] = "LONG"
    long["actual_edge_bps"] = long["long_trade_net_edge_bps"]
    short = frame.loc[short_mask].copy()
    short["side"] = "SHORT"
    short["actual_edge_bps"] = short["short_trade_net_edge_bps"]
    return pd.concat([long, short], ignore_index=True)


def _plain_trade_metrics(trades: pd.DataFrame) -> dict[str, Any]:
    if trades.empty:
        return {"rows": 0, "win_rate": 0.0, "avg_actual_edge_bps": 0.0}
    return {
        "rows": len(trades),
        "win_rate": float((trades["actual_edge_bps"].astype(float) > 0).mean()),
        "avg_actual_edge_bps": float(trades["actual_edge_bps"].astype(float).mean()),
        "side_counts": trades["side"].value_counts().to_dict(),
    }


def _top_bucket_edge(frame: pd.DataFrame) -> dict[str, Any]:
    side = np.where(frame["long_prob"] >= frame["short_prob"], "LONG", "SHORT")
    side_prob = np.where(side == "LONG", frame["long_prob"], frame["short_prob"])
    side_edge = np.where(side == "LONG", frame["long_trade_net_edge_bps"], frame["short_trade_net_edge_bps"])
    direction_frame = pd.DataFrame({"score": side_prob, "actual_edge_bps": side_edge, "side": side})
    direction_top = {}
    for fraction in (0.01, 0.02, 0.05, 0.10):
        top = direction_frame.sort_values("score", ascending=False).head(max(1, int(len(direction_frame) * fraction)))
        direction_top[str(fraction)] = _plain_trade_metrics(top.rename(columns={"actual_edge_bps": "actual_edge_bps"}))
    return {
        "direction_top_score": direction_top,
        "long_entry_prob_deciles": _decile_edge(frame, "long_entry_prob", "actual_long_expected_net_edge_bps", "long_entry_target"),
        "short_entry_prob_deciles": _decile_edge(frame, "short_entry_prob", "actual_short_expected_net_edge_bps", "short_entry_target"),
    }


def _decile_edge(frame: pd.DataFrame, score_col: str, edge_col: str, target_col: str) -> list[dict[str, Any]]:
    sample = frame[[score_col, edge_col, target_col]].dropna().copy()
    if sample.empty:
        return []
    sample["bucket"] = pd.qcut(sample[score_col].rank(method="first"), 10, labels=False) + 1
    rows = []
    for bucket, group in sample.groupby("bucket", observed=False):
        rows.append(
            {
                "bucket": int(bucket),
                "rows": len(group),
                "score_min": float(group[score_col].min()),
                "score_max": float(group[score_col].max()),
                "hit_rate": float(group[target_col].mean()),
                "avg_actual_edge_bps": float(group[edge_col].mean()),
            }
        )
    return rows


def _grid_trade_summary(frame: pd.DataFrame) -> dict[str, Any]:
    nonzero = 0
    best_by_count = None
    best_metrics = None
    for thresholds in _threshold_candidates():
        trades = _simulate_open_trades(frame, thresholds)
        metrics = _trade_metrics(trades)
        if metrics["trade_count"] > 0:
            nonzero += 1
        if best_metrics is None or metrics["trade_count"] > best_metrics["trade_count"]:
            best_by_count = thresholds
            best_metrics = metrics
    return {
        "candidate_count": len(_threshold_candidates()),
        "candidates_with_trade": nonzero,
        "best_by_trade_count": best_by_count,
        "best_metrics": best_metrics,
    }


def _diagnostic_conclusion(pm_summary: dict[str, Any]) -> dict[str, Any]:
    validation = pm_summary.get(VALIDATION_LOCKED_SPLIT, {}).get("selected_trade_metrics", {})
    stress = pm_summary.get(LATEST_STRESS_SPLIT, {}).get("selected_trade_metrics", {})
    if validation.get("trade_count", 0) == 0:
        return {
            "status": "NO_VALIDATION_TRADE",
            "plain_reason": "当前 PM 阈值在验证集没有选出交易，主要要看挡单漏斗。",
            "next_action": "先看 Direction、Risk、Entry 哪个门槛挡住，再做阈值实验。",
        }
    if validation.get("avg_weighted_edge_bps", 0.0) <= 0 and stress.get("avg_weighted_edge_bps", 0.0) <= 0:
        return {
            "status": "PRICE_PLAN_OR_ENTRY_NOT_TRADABLE",
            "plain_reason": "按固定止盈止损真实收益算，验证集和压力集选出来的交易平均都不赚钱。",
            "next_action": "优先重新搜索价格计划，再重建 Entry 标签和模型；不要只放松 PM 阈值。",
        }
    return {
        "status": "NEEDS_MANUAL_REVIEW",
        "plain_reason": "没有发现单一硬挡板，需要人工继续看各模型分数和回测明细。",
        "next_action": "查看 diagnostics 报告中的漏斗和放松阈值结果。",
    }


def _quantiles(series: pd.Series, points: tuple[float, ...]) -> dict[str, float]:
    values = pd.to_numeric(series, errors="coerce").replace([np.inf, -np.inf], np.nan).dropna()
    if values.empty:
        return {}
    result = values.quantile(list(points)).round(6).to_dict()
    return {str(key): float(value) for key, value in result.items()}


def _markdown_report(payload: dict[str, Any]) -> str:
    lines = [
        "# Trader Training Failure Diagnostics",
        "",
        f"- run_id: `{payload['run_id']}`",
        f"- status: `{payload['conclusion']['status']}`",
        f"- 结论: {payload['conclusion']['plain_reason']}",
        f"- 下一步: {payload['conclusion']['next_action']}",
        "",
        "## 标签分布",
        "",
    ]
    for split_id, item in payload["label_summary"].items():
        direction = item.get("direction", {})
        entry = item.get("entry", {})
        lines.append(f"### {split_id}")
        lines.append("")
        if direction:
            lines.append(f"- Direction 行数: {direction['rows']}")
            lines.append(f"- Direction 标签比例: `{direction['label_ratio']}`")
            lines.append(f"- 45 分钟未来收益分位: `{direction['future_return_bps_quantile']}`")
        if entry:
            lines.append(f"- Entry 行数: {entry['rows']}")
            lines.append(f"- Entry 命中率: `{entry['target_rate_by_side']}`")
            lines.append(f"- Entry 平均净收益: `{entry['edge_mean_by_side']}`")
        lines.append("")

    lines.extend(["## PM 挡单漏斗", ""])
    for split_id, item in payload["pm_summary"].items():
        lines.append(f"### {split_id}")
        lines.append("")
        lines.append(f"- 样本数: {item['rows']}")
        lines.append(f"- 当前阈值: `{item['active_thresholds']}`")
        lines.append(f"- 当前阈值选中交易: `{item['selected_trade_metrics']}`")
        lines.append(f"- 网格里有交易的候选数: {item['grid_search_any_trade']['candidates_with_trade']} / {item['grid_search_any_trade']['candidate_count']}")
        lines.append("")
        for side in ("long", "short"):
            lines.append(f"#### {side.upper()}")
            lines.append("")
            rows = item["gate_funnel"][side]["cumulative"]
            lines.append("| 条件 | 剩余样本 |")
            lines.append("| --- | ---: |")
            for row in rows:
                lines.append(f"| {row['gate']} | {row['rows_after_gate']} |")
            lines.append("")
        lines.append("#### 放松条件后的结果")
        lines.append("")
        lines.append("| 方案 | 样本数 | 胜率 | 平均真实净收益bps |")
        lines.append("| --- | ---: | ---: | ---: |")
        for name, metrics in item["relaxed_variants"].items():
            lines.append(
                f"| {name} | {metrics['rows']} | {metrics['win_rate']:.4f} | {metrics['avg_actual_edge_bps']:.4f} |"
            )
        lines.append("")
    return "\n".join(lines) + "\n"


def _jsonable(value: Any) -> Any:
    if isinstance(value, dict):
        return {str(key): _jsonable(item) for key, item in value.items()}
    if isinstance(value, list):
        return [_jsonable(item) for item in value]
    if isinstance(value, tuple):
        return [_jsonable(item) for item in value]
    if isinstance(value, (np.integer,)):
        return int(value)
    if isinstance(value, (np.floating,)):
        return float(value)
    if isinstance(value, np.ndarray):
        return value.tolist()
    return value