from __future__ import annotations import logging from typing import Any import numpy as np import pandas as pd from trader_training.io_utils import read_json, read_parquet, run_root, write_json, write_text from trader_training.pm import _pm_frame, _price_plan_context, _simulate_open_trades, _threshold_candidates, _thresholds_from_config, _trade_metrics, default_pm_config from trader_training.schemas import FIT_SPLIT, LATEST_STRESS_SPLIT, TUNE_SPLIT, VALIDATION_LOCKED_SPLIT DIAGNOSTIC_SPLITS = (FIT_SPLIT, TUNE_SPLIT, VALIDATION_LOCKED_SPLIT, LATEST_STRESS_SPLIT) PM_EVAL_SPLITS = (TUNE_SPLIT, VALIDATION_LOCKED_SPLIT, LATEST_STRESS_SPLIT) def diagnose_training_run(args: Any) -> None: root = run_root(args) label_summary = _label_summary(root) pm_summary = _pm_summary(root) payload = { "run_id": args.run_id, "label_summary": label_summary, "pm_summary": pm_summary, "conclusion": _diagnostic_conclusion(pm_summary), } write_json(root / "diagnostics" / "training_failure_diagnostics.json", _jsonable(payload)) write_text(root / "diagnostics" / "training_failure_diagnostics.md", _markdown_report(payload)) logging.info( "trader.training.diagnostics_written runId=%s conclusion=%s path=%s", args.run_id, payload["conclusion"]["status"], root / "diagnostics" / "training_failure_diagnostics.md", ) def _label_summary(root) -> dict[str, Any]: direction = read_parquet(root / "dataset" / "direction_train.parquet") entry = read_parquet(root / "dataset" / "entry_train.parquet") summary: dict[str, Any] = {} for split_id in DIAGNOSTIC_SPLITS: direction_split = direction[direction["split_id"].eq(split_id)].copy() entry_split = entry[entry["split_id"].eq(split_id)].copy() item: dict[str, Any] = {"direction": {}, "entry": {}} if not direction_split.empty: item["direction"] = { "source": "dataset/direction_train.parquet", "rows": len(direction_split), "label_ratio": _direction_target_ratio(direction_split), "future_return_bps_quantile": _quantiles(direction_split["future_return_bps"], (0.01, 0.05, 0.25, 0.5, 0.75, 0.95, 0.99)), } if not entry_split.empty: required = { "long_entry_target", "short_entry_target", "long_actual_plan_net_edge_bps", "short_actual_plan_net_edge_bps", } missing = sorted(required - set(entry_split.columns)) if missing: raise ValueError(f"entry_train is missing columns required by diagnostics: {missing}") item["entry"] = { "source": "dataset/entry_train.parquet", "rows": len(entry_split), "target_rate_by_side": { "LONG": float(entry_split["long_entry_target"].astype(float).mean()), "SHORT": float(entry_split["short_entry_target"].astype(float).mean()), }, "edge_column": "actual_plan_net_edge_bps", "edge_mean_by_side": { "LONG": float(entry_split["long_actual_plan_net_edge_bps"].astype(float).mean()), "SHORT": float(entry_split["short_actual_plan_net_edge_bps"].astype(float).mean()), }, "edge_quantile_by_side": { "LONG": _quantiles(entry_split["long_actual_plan_net_edge_bps"], (0.05, 0.5, 0.95)), "SHORT": _quantiles(entry_split["short_actual_plan_net_edge_bps"], (0.05, 0.5, 0.95)), }, } summary[split_id] = item return summary def _direction_target_ratio(frame: pd.DataFrame) -> dict[str, float]: required = {"long_target", "short_target", "neutral_target"} missing = sorted(required - set(frame.columns)) if missing: raise ValueError(f"direction_train is missing target columns required by diagnostics: {missing}") rows = len(frame) if rows == 0: return {"LONG": 0.0, "SHORT": 0.0, "NEUTRAL": 0.0} return { "LONG": float(frame["long_target"].astype(float).mean()), "SHORT": float(frame["short_target"].astype(float).mean()), "NEUTRAL": float(frame["neutral_target"].astype(float).mean()), } def _pm_summary(root) -> dict[str, Any]: summary: dict[str, Any] = {} config_path = root / "pm-search" / "position_manager_config.json" config = read_json(config_path)["config"] if config_path.is_file() else default_pm_config() thresholds = _thresholds_from_config(config) price_plan = _price_plan_context(root) for split_id in PM_EVAL_SPLITS: frame = _pm_frame(root, split_id) selected_trades = _simulate_open_trades(frame, thresholds, config, price_plan) item = { "rows": len(frame), "score_distribution": _score_distribution(frame), "active_thresholds": thresholds, "gate_funnel": _gate_funnel(frame, thresholds), "selected_trade_metrics": _trade_metrics(selected_trades), "relaxed_variants": _relaxed_variants(frame), "top_bucket_edge": _top_bucket_edge(frame), "grid_search_any_trade": _grid_trade_summary(frame), } summary[split_id] = item return summary def _score_distribution(frame: pd.DataFrame) -> dict[str, dict[str, float]]: columns = [ "long_prob", "short_prob", "long_entry_prob", "short_entry_prob", "market_risk_prob", "pred_long_expected_net_edge_bps", "pred_short_expected_net_edge_bps", "model_pred_long_expected_net_edge_bps", "model_pred_short_expected_net_edge_bps", "actual_long_plan_edge_bps", "actual_short_plan_edge_bps", ] return {column: _quantiles(frame[column], (0.0, 0.05, 0.5, 0.95, 1.0)) for column in columns if column in frame.columns} def _gate_funnel(frame: pd.DataFrame, thresholds: dict[str, float]) -> dict[str, Any]: direction_margin = (frame["long_prob"] - frame["short_prob"]).abs() long_steps = { f"long_prob > {thresholds['long_open_prob']}": frame["long_prob"] > thresholds["long_open_prob"], f"abs(long_prob - short_prob) > {thresholds['min_direction_margin']}": direction_margin > thresholds["min_direction_margin"], f"long_entry_prob > {thresholds['min_entry_prob']}": frame["long_entry_prob"] > thresholds["min_entry_prob"], f"market_risk_prob < {thresholds['max_market_risk_prob']}": frame["market_risk_prob"] < thresholds["max_market_risk_prob"], f"pred_long_expected_net_edge_bps > {thresholds['min_expected_edge_bps']}": frame["pred_long_expected_net_edge_bps"] > thresholds["min_expected_edge_bps"], } short_steps = { f"short_prob > {thresholds['short_open_prob']}": frame["short_prob"] > thresholds["short_open_prob"], f"abs(long_prob - short_prob) > {thresholds['min_direction_margin']}": direction_margin > thresholds["min_direction_margin"], f"short_entry_prob > {thresholds['min_entry_prob']}": frame["short_entry_prob"] > thresholds["min_entry_prob"], f"market_risk_prob < {thresholds['max_market_risk_prob']}": frame["market_risk_prob"] < thresholds["max_market_risk_prob"], f"pred_short_expected_net_edge_bps > {thresholds['min_expected_edge_bps']}": frame["pred_short_expected_net_edge_bps"] > thresholds["min_expected_edge_bps"], } return { "thresholds": thresholds, "long": _cumulative_gate_counts(long_steps, len(frame)), "short": _cumulative_gate_counts(short_steps, len(frame)), } def _cumulative_gate_counts(steps: dict[str, pd.Series], total_rows: int) -> dict[str, Any]: mask = np.ones(total_rows, dtype=bool) cumulative = [] single = {} for name, step in steps.items(): values = step.to_numpy(dtype=bool) single[name] = int(values.sum()) mask &= values cumulative.append({"gate": name, "rows_after_gate": int(mask.sum())}) return {"single_gate_pass": single, "cumulative": cumulative} def _relaxed_variants(frame: pd.DataFrame) -> dict[str, Any]: variants = { "entry_30_positive_edge": {"prob": 0.50, "entry": 0.30, "margin": 0.02, "risk": 0.65, "edge": 3.0}, "entry_50_positive_edge": {"prob": 0.50, "entry": 0.50, "margin": 0.02, "risk": 0.65, "edge": 3.0}, "entry_70_positive_edge": {"prob": 0.50, "entry": 0.70, "margin": 0.02, "risk": 0.65, "edge": 3.0}, "direction_only_control": {"prob": 0.54, "entry": 0.0, "margin": 0.02, "risk": 1.0, "edge": -99.0}, } result: dict[str, Any] = {} for name, thresholds in variants.items(): trades = _variant_trades(frame, thresholds) result[name] = _plain_trade_metrics(trades) return result def _variant_trades(frame: pd.DataFrame, thresholds: dict[str, float]) -> pd.DataFrame: long_mask = ( (frame["long_prob"] >= thresholds["prob"]) & ((frame["long_prob"] - frame["short_prob"]) >= thresholds["margin"]) & (frame["long_entry_prob"] >= thresholds["entry"]) & (frame["market_risk_prob"] <= thresholds["risk"]) & (frame["pred_long_expected_net_edge_bps"] >= thresholds["edge"]) ) short_mask = ( (frame["short_prob"] >= thresholds["prob"]) & ((frame["short_prob"] - frame["long_prob"]) >= thresholds["margin"]) & (frame["short_entry_prob"] >= thresholds["entry"]) & (frame["market_risk_prob"] <= thresholds["risk"]) & (frame["pred_short_expected_net_edge_bps"] >= thresholds["edge"]) ) long = frame.loc[long_mask].copy() long["side"] = "LONG" long["actual_edge_bps"] = long["long_trade_net_edge_bps"] short = frame.loc[short_mask].copy() short["side"] = "SHORT" short["actual_edge_bps"] = short["short_trade_net_edge_bps"] return pd.concat([long, short], ignore_index=True) def _plain_trade_metrics(trades: pd.DataFrame) -> dict[str, Any]: if trades.empty: return {"rows": 0, "win_rate": 0.0, "avg_actual_edge_bps": 0.0} return { "rows": len(trades), "win_rate": float((trades["actual_edge_bps"].astype(float) > 0).mean()), "avg_actual_edge_bps": float(trades["actual_edge_bps"].astype(float).mean()), "side_counts": trades["side"].value_counts().to_dict(), } def _top_bucket_edge(frame: pd.DataFrame) -> dict[str, Any]: side = np.where(frame["long_prob"] >= frame["short_prob"], "LONG", "SHORT") side_prob = np.where(side == "LONG", frame["long_prob"], frame["short_prob"]) side_edge = np.where(side == "LONG", frame["long_trade_net_edge_bps"], frame["short_trade_net_edge_bps"]) direction_frame = pd.DataFrame({"score": side_prob, "actual_edge_bps": side_edge, "side": side}) direction_top = {} for fraction in (0.01, 0.02, 0.05, 0.10): top = direction_frame.sort_values("score", ascending=False).head(max(1, int(len(direction_frame) * fraction))) direction_top[str(fraction)] = _plain_trade_metrics(top.rename(columns={"actual_edge_bps": "actual_edge_bps"})) return { "direction_top_score": direction_top, "long_entry_prob_deciles": _decile_edge(frame, "long_entry_prob", "actual_long_plan_edge_bps", "long_entry_target"), "short_entry_prob_deciles": _decile_edge(frame, "short_entry_prob", "actual_short_plan_edge_bps", "short_entry_target"), } def _decile_edge(frame: pd.DataFrame, score_col: str, edge_col: str, target_col: str) -> list[dict[str, Any]]: sample = frame[[score_col, edge_col, target_col]].dropna().copy() if sample.empty: return [] sample["bucket"] = pd.qcut(sample[score_col].rank(method="first"), 10, labels=False) + 1 rows = [] for bucket, group in sample.groupby("bucket", observed=False): rows.append( { "bucket": int(bucket), "rows": len(group), "score_min": float(group[score_col].min()), "score_max": float(group[score_col].max()), "hit_rate": float(group[target_col].mean()), "avg_actual_edge_bps": float(group[edge_col].mean()), } ) return rows def _grid_trade_summary(frame: pd.DataFrame) -> dict[str, Any]: nonzero = 0 best_by_count = None best_metrics = None for thresholds in _threshold_candidates(): trades = _simulate_open_trades(frame, thresholds) metrics = _trade_metrics(trades) if metrics["trade_count"] > 0: nonzero += 1 if best_metrics is None or metrics["trade_count"] > best_metrics["trade_count"]: best_by_count = thresholds best_metrics = metrics return { "candidate_count": len(_threshold_candidates()), "candidates_with_trade": nonzero, "best_by_trade_count": best_by_count, "best_metrics": best_metrics, } def _diagnostic_conclusion(pm_summary: dict[str, Any]) -> dict[str, Any]: validation = pm_summary.get(VALIDATION_LOCKED_SPLIT, {}).get("selected_trade_metrics", {}) stress = pm_summary.get(LATEST_STRESS_SPLIT, {}).get("selected_trade_metrics", {}) if validation.get("trade_count", 0) == 0: return { "status": "NO_VALIDATION_TRADE", "plain_reason": "当前 PM 阈值在验证集没有选出交易，主要要看挡单漏斗。", "next_action": "先看 Direction、Risk、Entry 哪个门槛挡住，再做阈值实验。", } if validation.get("avg_weighted_edge_bps", 0.0) <= 0 and stress.get("avg_weighted_edge_bps", 0.0) <= 0: return { "status": "PRICE_PLAN_OR_ENTRY_NOT_TRADABLE", "plain_reason": "按当前价格计划真实收益算，验证集和压力集选出来的交易平均都不赚钱。", "next_action": "优先重新搜索价格计划，再重建 Entry 标签和模型；不要只放松 PM 阈值。", } return { "status": "NEEDS_MANUAL_REVIEW", "plain_reason": "没有发现单一硬挡板，需要人工继续看各模型分数和回测明细。", "next_action": "查看 diagnostics 报告中的漏斗和放松阈值结果。", } def _quantiles(series: pd.Series, points: tuple[float, ...]) -> dict[str, float]: values = pd.to_numeric(series, errors="coerce").replace([np.inf, -np.inf], np.nan).dropna() if values.empty: return {} result = values.quantile(list(points)).round(6).to_dict() return {str(key): float(value) for key, value in result.items()} def _markdown_report(payload: dict[str, Any]) -> str: lines = [ "# Trader Training Failure Diagnostics", "", f"- run_id: `{payload['run_id']}`", f"- status: `{payload['conclusion']['status']}`", f"- 结论: {payload['conclusion']['plain_reason']}", f"- 下一步: {payload['conclusion']['next_action']}", "", "## 标签分布", "", ] for split_id, item in payload["label_summary"].items(): direction = item.get("direction", {}) entry = item.get("entry", {}) lines.append(f"### {split_id}") lines.append("") if direction: lines.append(f"- Direction 行数: {direction['rows']}") lines.append(f"- Direction 来源: `{direction['source']}`") lines.append(f"- Direction 标签比例: `{direction['label_ratio']}`") lines.append(f"- 45 分钟未来收益分位: `{direction['future_return_bps_quantile']}`") if entry: lines.append(f"- Entry 行数: {entry['rows']}") lines.append(f"- Entry 来源: `{entry['source']}`") lines.append(f"- Entry 命中率: `{entry['target_rate_by_side']}`") lines.append(f"- Entry 平均净收益: `{entry['edge_mean_by_side']}`") lines.append("") lines.extend(["## PM 挡单漏斗", ""]) for split_id, item in payload["pm_summary"].items(): lines.append(f"### {split_id}") lines.append("") lines.append(f"- 样本数: {item['rows']}") lines.append(f"- 当前阈值: `{item['active_thresholds']}`") lines.append(f"- 当前阈值选中交易: `{item['selected_trade_metrics']}`") lines.append(f"- 网格里有交易的候选数: {item['grid_search_any_trade']['candidates_with_trade']} / {item['grid_search_any_trade']['candidate_count']}") lines.extend(_score_distribution_markdown(item["score_distribution"])) lines.append("") for side in ("long", "short"): lines.append(f"#### {side.upper()}") lines.append("") rows = item["gate_funnel"][side]["cumulative"] lines.append("| 条件 | 剩余样本 |") lines.append("| --- | ---: |") for row in rows: lines.append(f"| {row['gate']} | {row['rows_after_gate']} |") lines.append("") lines.append("#### 放松条件后的结果") lines.append("") lines.append("| 方案 | 样本数 | 胜率 | 平均真实净收益bps |") lines.append("| --- | ---: | ---: | ---: |") for name, metrics in item["relaxed_variants"].items(): lines.append( f"| {name} | {metrics['rows']} | {metrics['win_rate']:.4f} | {metrics['avg_actual_edge_bps']:.4f} |" ) lines.append("") return "\n".join(lines) + "\n" def _score_distribution_markdown(distribution: dict[str, dict[str, float]]) -> list[str]: watched_columns = [ "long_prob", "short_prob", "long_entry_prob", "short_entry_prob", "market_risk_prob", "pred_long_expected_net_edge_bps", "pred_short_expected_net_edge_bps", ] lines = ["", "#### 分数分布", "", "| 字段 | 最小 | 5% | 中位数 | 95% | 最大 |", "| --- | ---: | ---: | ---: | ---: | ---: |"] for column in watched_columns: quantiles = distribution.get(column) if not quantiles: continue lines.append( "| " + column + f" | {quantiles.get('0.0', 0.0):.4f}" + f" | {quantiles.get('0.05', 0.0):.4f}" + f" | {quantiles.get('0.5', 0.0):.4f}" + f" | {quantiles.get('0.95', 0.0):.4f}" + f" | {quantiles.get('1.0', 0.0):.4f} |" ) return lines def _jsonable(value: Any) -> Any: if isinstance(value, dict): return {str(key): _jsonable(item) for key, item in value.items()} if isinstance(value, list): return [_jsonable(item) for item in value] if isinstance(value, tuple): return [_jsonable(item) for item in value] if isinstance(value, (np.integer,)): return int(value) if isinstance(value, (np.floating,)): return float(value) if isinstance(value, np.ndarray): return value.tolist() return value