Codex
9acb3460a1
Align entry labels with max future edge, tune direction labeling, and harden regression evaluation. Add training diagnostics, price-plan search, feature screening, and nonlinear benchmark scripts.
548 lines
23 KiB
Python
548 lines
23 KiB
Python
from __future__ import annotations
|
|
|
|
import itertools
|
|
import logging
|
|
from typing import Any
|
|
|
|
import numpy as np
|
|
import pandas as pd
|
|
|
|
from trader_training.io_utils import read_json, read_parquet, run_root, sha256_json, write_json, write_parquet, write_text
|
|
from trader_training.schemas import LATEST_STRESS_SPLIT, PM_CONFIG_VERSION, TUNE_SPLIT, VALIDATION_LOCKED_SPLIT
|
|
|
|
|
|
def default_pm_config() -> dict:
|
|
return {
|
|
"pmConfigVersion": PM_CONFIG_VERSION,
|
|
"open": {
|
|
"longOpenProb": 0.58,
|
|
"shortOpenProb": 0.58,
|
|
"minLongEntryProb": 0.55,
|
|
"minShortEntryProb": 0.55,
|
|
"maxMarketRiskProb": 0.45,
|
|
"minExpectedEdgeBps": 3.0,
|
|
"minDirectionMargin": 0.03,
|
|
"minLiquidityCapacityRatio": 0.10,
|
|
"maxOodScore": 0.80,
|
|
},
|
|
"add": {
|
|
"minLongProb": 0.60,
|
|
"minShortProb": 0.60,
|
|
"minContinueProb": 0.58,
|
|
"minEntryProb": 0.55,
|
|
"maxExitProb": 0.45,
|
|
"maxMarketRiskProb": 0.45,
|
|
"maxPositionRiskProb": 0.50,
|
|
"minExpectedEdgeBps": 3.0,
|
|
"minContinueVsExitEdgeBps": 0.0,
|
|
"minLiquidityCapacityRatio": 0.10,
|
|
"minPostTradeLiquidationBufferBps": 500.0,
|
|
"maxAddCount": 3,
|
|
"cooldownMinutes": 5,
|
|
},
|
|
"exit": {
|
|
"closeExitProb": 0.70,
|
|
"closePositionRiskProb": 0.70,
|
|
"closeMarketRiskProb": 0.70,
|
|
"closeContinueMax": 0.25,
|
|
"reduceAdverseMoveProb": 0.62,
|
|
"reduceContinueMin": 0.35,
|
|
"reduceContinueMax": 0.70,
|
|
"minProfitForReduceBps": 5.0,
|
|
"maxPositionPathRiskBps": 80.0,
|
|
},
|
|
"sizing": {
|
|
"baseRatio": 0.80,
|
|
"minInitialRatio": 0.05,
|
|
"maxSingleLegRatio": 1.0,
|
|
"minAddRatio": 0.02,
|
|
"maxAddRatio": 0.25,
|
|
"maxTotalPositionRatio": 1.0,
|
|
"minEdgeBps": 3.0,
|
|
"maxLossPerTradeBps": 80.0,
|
|
"maxLiquidityUsageRatio": 0.20,
|
|
"uncertaintyPenaltyMultiplier": 0.50,
|
|
"minPostTradeLiquidationBufferBps": 500.0,
|
|
},
|
|
}
|
|
|
|
|
|
def search_pm_thresholds(args: Any) -> None:
|
|
root = run_root(args)
|
|
frame = _pm_tune_frame(root)
|
|
candidate_rows: list[dict[str, Any]] = []
|
|
best_score = -float("inf")
|
|
best_thresholds: dict[str, float] | None = None
|
|
best_metrics: dict[str, Any] | None = None
|
|
best_trades = pd.DataFrame()
|
|
|
|
for thresholds in _threshold_candidates():
|
|
trades = _simulate_open_trades(frame, thresholds)
|
|
metrics = _trade_metrics(trades)
|
|
score = _score_thresholds(metrics)
|
|
candidate_rows.append({**thresholds, **metrics, "score": score})
|
|
if score > best_score:
|
|
best_score = score
|
|
best_thresholds = thresholds
|
|
best_metrics = metrics
|
|
best_trades = trades
|
|
|
|
if best_thresholds is None or best_metrics is None:
|
|
raise ValueError("PM threshold search did not evaluate any candidate")
|
|
|
|
config = _pm_config_from_thresholds(best_thresholds)
|
|
threshold_stability = {
|
|
"source": "tune_predictions_and_entry_labels",
|
|
"method": "deterministic_grid_search_v1",
|
|
"candidate_count": len(candidate_rows),
|
|
"best_score": best_score,
|
|
"best_metrics": best_metrics,
|
|
}
|
|
payload = {
|
|
"pm_config_version": PM_CONFIG_VERSION,
|
|
"config": config,
|
|
"config_hash_sha256": sha256_json(config),
|
|
"threshold_stability_json": threshold_stability,
|
|
}
|
|
candidate_frame = pd.DataFrame(candidate_rows).sort_values("score", ascending=False).reset_index(drop=True)
|
|
equity_curve = _equity_curve(best_trades)
|
|
regime_metrics = _regime_metrics(best_trades)
|
|
write_json(root / "pm-search" / "position_manager_config.json", payload)
|
|
write_json(root / "pm-search" / "pm_threshold_config.json", payload)
|
|
write_text(root / "pm-search" / "pm_search_candidates.csv", candidate_frame.to_csv(index=False))
|
|
write_parquet(root / "pm-search" / "pm_backtest_trades.parquet", best_trades)
|
|
write_text(root / "pm-search" / "pm_equity_curve.csv", equity_curve.to_csv(index=False))
|
|
write_text(root / "pm-search" / "pm_regime_metrics.csv", regime_metrics.to_csv(index=False))
|
|
_write_pm_report(root / "pm-search" / "pm_threshold_report.md", candidate_frame, best_thresholds, best_metrics)
|
|
_write_pm_report(root / "pm-search" / "pm_search_report.md", candidate_frame, best_thresholds, best_metrics)
|
|
logging.info(
|
|
"trader.training.pm_thresholds_searched runId=%s candidateCount=%s bestScore=%.6f tradeCount=%s totalWeightedEdgeBps=%.6f",
|
|
args.run_id,
|
|
len(candidate_rows),
|
|
best_score,
|
|
best_metrics["trade_count"],
|
|
best_metrics["total_weighted_edge_bps"],
|
|
)
|
|
|
|
|
|
def integrated_backtest(args: Any) -> None:
|
|
root = run_root(args)
|
|
config_path = root / "pm-search" / "position_manager_config.json"
|
|
if not config_path.is_file():
|
|
raise FileNotFoundError(f"PM config is required before backtest: {config_path}")
|
|
pm_payload = read_json(config_path)
|
|
trades_path = root / "pm-search" / "pm_backtest_trades.parquet"
|
|
# PM search is allowed to use tune_inner, but final acceptance must be
|
|
# measured on the sealed validation_locked and latest_stress splits.
|
|
tune_trades = read_parquet(trades_path) if trades_path.is_file() else _simulate_open_trades(_pm_tune_frame(root), _thresholds_from_config(pm_payload["config"]))
|
|
tune_trades["eval_split"] = TUNE_SPLIT
|
|
validation_locked_trades = _simulate_open_trades(_pm_frame(root, VALIDATION_LOCKED_SPLIT), _thresholds_from_config(pm_payload["config"]))
|
|
validation_locked_trades["eval_split"] = VALIDATION_LOCKED_SPLIT
|
|
stress_trades = _simulate_open_trades(_pm_frame(root, LATEST_STRESS_SPLIT), _thresholds_from_config(pm_payload["config"]))
|
|
stress_trades["eval_split"] = LATEST_STRESS_SPLIT
|
|
trades = pd.concat([tune_trades, validation_locked_trades, stress_trades], ignore_index=True)
|
|
metrics = {
|
|
TUNE_SPLIT: _trade_metrics(tune_trades),
|
|
VALIDATION_LOCKED_SPLIT: _trade_metrics(validation_locked_trades),
|
|
LATEST_STRESS_SPLIT: _trade_metrics(stress_trades),
|
|
"combined": _trade_metrics(trades),
|
|
}
|
|
status, status_reasons = _backtest_status(metrics)
|
|
equity_curve = _equity_curve(trades)
|
|
regime_metrics = _regime_metrics(trades)
|
|
result = {
|
|
"backtest_manifest_id": f"backtest-{args.run_id}",
|
|
"mode": "VALIDATION_PM_BACKTEST",
|
|
"pm_config_hash_sha256": pm_payload["config_hash_sha256"],
|
|
"metrics": metrics,
|
|
"status_reasons": status_reasons,
|
|
"status": status,
|
|
}
|
|
write_json(root / "backtest" / "backtest_manifest.json", result)
|
|
write_parquet(root / "backtest" / "backtest_trades.parquet", trades)
|
|
write_text(root / "backtest" / "equity_curve.csv", equity_curve.to_csv(index=False))
|
|
write_text(root / "backtest" / "regime_metrics.csv", regime_metrics.to_csv(index=False))
|
|
_write_backtest_report(root / "backtest" / "backtest_report.md", result)
|
|
_write_failure_cases(root / "backtest" / "failure_cases.md", trades)
|
|
_write_no_baseline_ablation(root / "backtest" / "direction_ablation_backtest_report.md")
|
|
logging.info(
|
|
"trader.training.backtest_written runId=%s status=%s tradeCount=%s totalWeightedEdgeBps=%.6f maxDrawdownBps=%.6f",
|
|
args.run_id,
|
|
status,
|
|
metrics[VALIDATION_LOCKED_SPLIT]["trade_count"],
|
|
metrics[VALIDATION_LOCKED_SPLIT]["total_weighted_edge_bps"],
|
|
metrics[VALIDATION_LOCKED_SPLIT]["max_drawdown_bps"],
|
|
)
|
|
|
|
|
|
def _pm_tune_frame(root) -> pd.DataFrame:
|
|
return _pm_frame(root, TUNE_SPLIT)
|
|
|
|
|
|
def _pm_frame(root, split_id: str) -> pd.DataFrame:
|
|
prediction_files = {
|
|
TUNE_SPLIT: "tune_predictions.parquet",
|
|
VALIDATION_LOCKED_SPLIT: "validation_locked_predictions.parquet",
|
|
LATEST_STRESS_SPLIT: "latest_stress_predictions.parquet",
|
|
}
|
|
prediction_file = prediction_files[split_id]
|
|
direction = read_parquet(root / "model" / "direction" / prediction_file)
|
|
entry = read_parquet(root / "model" / "entry" / prediction_file).rename(
|
|
columns={
|
|
"long_expected_net_edge_bps": "pred_long_expected_net_edge_bps",
|
|
"short_expected_net_edge_bps": "pred_short_expected_net_edge_bps",
|
|
}
|
|
)
|
|
risk = read_parquet(root / "model" / "risk" / prediction_file)
|
|
entry_dataset = read_parquet(root / "dataset" / "entry_train.parquet").rename(
|
|
columns={
|
|
"long_expected_net_edge_bps": "actual_long_expected_net_edge_bps",
|
|
"short_expected_net_edge_bps": "actual_short_expected_net_edge_bps",
|
|
}
|
|
)
|
|
entry_cols = [
|
|
"sample_id",
|
|
"long_entry_prob",
|
|
"short_entry_prob",
|
|
"pred_long_expected_net_edge_bps",
|
|
"pred_short_expected_net_edge_bps",
|
|
]
|
|
risk_cols = ["sample_id", "market_risk_prob", "long_position_risk_prob", "short_position_risk_prob"]
|
|
actual_cols = ["sample_id", "actual_long_expected_net_edge_bps", "actual_short_expected_net_edge_bps", "long_entry_target", "short_entry_target"]
|
|
frame = (
|
|
direction[["sample_id", "symbol", "event_time", "split_id", "long_prob", "short_prob", "neutral_prob"]]
|
|
.merge(entry[entry_cols], on="sample_id", how="inner")
|
|
.merge(risk[risk_cols], on="sample_id", how="inner")
|
|
.merge(entry_dataset[actual_cols], on="sample_id", how="inner")
|
|
)
|
|
if frame.empty:
|
|
raise ValueError(f"PM frame is empty for {split_id}; check model predictions and entry dataset")
|
|
logging.info(
|
|
"trader.training.pm_frame_loaded splitId=%s rowCount=%s splitCounts=%s",
|
|
split_id,
|
|
len(frame),
|
|
frame["split_id"].value_counts().to_dict(),
|
|
)
|
|
return frame
|
|
|
|
|
|
def _threshold_candidates() -> list[dict[str, float]]:
|
|
values = itertools.product(
|
|
[0.50, 0.52, 0.54, 0.56, 0.58],
|
|
[0.50, 0.52, 0.54, 0.56, 0.58],
|
|
[0.10, 0.12, 0.14, 0.16, 0.20, 0.30, 0.50],
|
|
[0.55, 0.75, 0.90, 1.00],
|
|
[-8.0, -4.0, 0.0, 1.0, 3.0],
|
|
[0.00, 0.01, 0.02, 0.05],
|
|
)
|
|
return [
|
|
{
|
|
"long_open_prob": long_prob,
|
|
"short_open_prob": short_prob,
|
|
"min_entry_prob": entry_prob,
|
|
"max_market_risk_prob": risk_prob,
|
|
"min_expected_edge_bps": edge_bps,
|
|
"min_direction_margin": margin,
|
|
}
|
|
for long_prob, short_prob, entry_prob, risk_prob, edge_bps, margin in values
|
|
]
|
|
|
|
|
|
def _simulate_open_trades(frame: pd.DataFrame, thresholds: dict[str, float]) -> pd.DataFrame:
|
|
long_mask = (
|
|
(frame["long_prob"] >= thresholds["long_open_prob"])
|
|
& ((frame["long_prob"] - frame["short_prob"]) >= thresholds["min_direction_margin"])
|
|
& (frame["long_entry_prob"] >= thresholds["min_entry_prob"])
|
|
& (frame["market_risk_prob"] <= thresholds["max_market_risk_prob"])
|
|
& (frame["pred_long_expected_net_edge_bps"] >= thresholds["min_expected_edge_bps"])
|
|
)
|
|
short_mask = (
|
|
(frame["short_prob"] >= thresholds["short_open_prob"])
|
|
& ((frame["short_prob"] - frame["long_prob"]) >= thresholds["min_direction_margin"])
|
|
& (frame["short_entry_prob"] >= thresholds["min_entry_prob"])
|
|
& (frame["market_risk_prob"] <= thresholds["max_market_risk_prob"])
|
|
& (frame["pred_short_expected_net_edge_bps"] >= thresholds["min_expected_edge_bps"])
|
|
)
|
|
long_score = frame["pred_long_expected_net_edge_bps"] + (frame["long_prob"] - frame["short_prob"]) * 10.0
|
|
short_score = frame["pred_short_expected_net_edge_bps"] + (frame["short_prob"] - frame["long_prob"]) * 10.0
|
|
side = np.where(long_mask & (~short_mask | (long_score >= short_score)), "LONG", np.where(short_mask, "SHORT", ""))
|
|
trades = frame.loc[side != ""].copy().reset_index(drop=True)
|
|
if trades.empty:
|
|
return _empty_trade_frame()
|
|
trades["side"] = side[side != ""]
|
|
is_long = trades["side"].eq("LONG")
|
|
trades["direction_prob"] = np.where(is_long, trades["long_prob"], trades["short_prob"])
|
|
trades["entry_prob"] = np.where(is_long, trades["long_entry_prob"], trades["short_entry_prob"])
|
|
trades["predicted_edge_bps"] = np.where(is_long, trades["pred_long_expected_net_edge_bps"], trades["pred_short_expected_net_edge_bps"])
|
|
trades["actual_edge_bps"] = np.where(is_long, trades["actual_long_expected_net_edge_bps"], trades["actual_short_expected_net_edge_bps"])
|
|
trades["entry_target"] = np.where(is_long, trades["long_entry_target"], trades["short_entry_target"])
|
|
trades["planned_ratio"] = _planned_ratio(trades["predicted_edge_bps"], trades["market_risk_prob"], thresholds["min_expected_edge_bps"])
|
|
trades["weighted_edge_bps"] = trades["actual_edge_bps"] * trades["planned_ratio"]
|
|
trades["threshold_hash"] = sha256_json(thresholds)[:16]
|
|
return trades[
|
|
[
|
|
"sample_id",
|
|
"symbol",
|
|
"event_time",
|
|
"split_id",
|
|
"side",
|
|
"direction_prob",
|
|
"entry_prob",
|
|
"market_risk_prob",
|
|
"predicted_edge_bps",
|
|
"actual_edge_bps",
|
|
"entry_target",
|
|
"planned_ratio",
|
|
"weighted_edge_bps",
|
|
"threshold_hash",
|
|
]
|
|
].sort_values("event_time")
|
|
|
|
|
|
def _empty_trade_frame() -> pd.DataFrame:
|
|
return pd.DataFrame(
|
|
columns=[
|
|
"sample_id",
|
|
"symbol",
|
|
"event_time",
|
|
"split_id",
|
|
"side",
|
|
"direction_prob",
|
|
"entry_prob",
|
|
"market_risk_prob",
|
|
"predicted_edge_bps",
|
|
"actual_edge_bps",
|
|
"entry_target",
|
|
"planned_ratio",
|
|
"weighted_edge_bps",
|
|
"threshold_hash",
|
|
]
|
|
)
|
|
|
|
|
|
def _planned_ratio(predicted_edge: pd.Series, market_risk: pd.Series, min_edge: float) -> np.ndarray:
|
|
edge_strength = ((predicted_edge.astype(float) - min_edge) / 20.0).clip(lower=0.0, upper=1.5)
|
|
risk_discount = (1.0 - market_risk.astype(float)).clip(lower=0.0, upper=1.0)
|
|
return (edge_strength * risk_discount).clip(lower=0.05, upper=1.0).to_numpy()
|
|
|
|
|
|
def _trade_metrics(trades: pd.DataFrame) -> dict[str, Any]:
|
|
if trades.empty:
|
|
return {
|
|
"trade_count": 0,
|
|
"win_rate": 0.0,
|
|
"avg_actual_edge_bps": 0.0,
|
|
"avg_weighted_edge_bps": 0.0,
|
|
"total_weighted_edge_bps": 0.0,
|
|
"max_drawdown_bps": 0.0,
|
|
"avg_planned_ratio": 0.0,
|
|
"profit_factor": 0.0,
|
|
"max_consecutive_losses": 0,
|
|
}
|
|
equity = trades["weighted_edge_bps"].astype(float).cumsum()
|
|
drawdown = equity.cummax() - equity
|
|
gains = trades.loc[trades["weighted_edge_bps"] > 0, "weighted_edge_bps"].astype(float).sum()
|
|
losses = -trades.loc[trades["weighted_edge_bps"] < 0, "weighted_edge_bps"].astype(float).sum()
|
|
return {
|
|
"trade_count": int(len(trades)),
|
|
"win_rate": float((trades["actual_edge_bps"].astype(float) > 0).mean()),
|
|
"avg_actual_edge_bps": float(trades["actual_edge_bps"].astype(float).mean()),
|
|
"avg_weighted_edge_bps": float(trades["weighted_edge_bps"].astype(float).mean()),
|
|
"total_weighted_edge_bps": float(equity.iloc[-1]),
|
|
"max_drawdown_bps": float(drawdown.max()),
|
|
"avg_planned_ratio": float(trades["planned_ratio"].astype(float).mean()),
|
|
"profit_factor": float(gains / losses) if losses > 0 else float("inf"),
|
|
"max_consecutive_losses": _max_consecutive_losses(trades["weighted_edge_bps"].astype(float).to_numpy()),
|
|
}
|
|
|
|
|
|
def _max_consecutive_losses(values: np.ndarray) -> int:
|
|
max_count = 0
|
|
current = 0
|
|
for value in values:
|
|
if value < 0:
|
|
current += 1
|
|
max_count = max(max_count, current)
|
|
else:
|
|
current = 0
|
|
return max_count
|
|
|
|
|
|
def _backtest_status(metrics: dict[str, dict[str, Any]]) -> tuple[str, list[str]]:
|
|
reasons: list[str] = []
|
|
validation_locked = metrics[VALIDATION_LOCKED_SPLIT]
|
|
stress = metrics[LATEST_STRESS_SPLIT]
|
|
if validation_locked["total_weighted_edge_bps"] <= 0:
|
|
reasons.append("validation_locked_net_edge_not_positive")
|
|
if validation_locked["trade_count"] < 80:
|
|
reasons.append("validation_locked_trade_count_below_80")
|
|
if validation_locked["profit_factor"] < 1.15:
|
|
reasons.append("validation_locked_profit_factor_below_1.15")
|
|
if validation_locked["avg_weighted_edge_bps"] <= 0:
|
|
reasons.append("validation_locked_avg_trade_edge_not_positive")
|
|
if validation_locked["max_consecutive_losses"] > 8:
|
|
reasons.append("validation_locked_max_consecutive_losses_above_8")
|
|
if stress["trade_count"] < 20:
|
|
reasons.append("latest_stress_trade_count_below_20")
|
|
if stress["profit_factor"] < 1.0:
|
|
reasons.append("latest_stress_profit_factor_below_1.0")
|
|
if stress["avg_weighted_edge_bps"] < -3.0:
|
|
reasons.append("latest_stress_avg_trade_edge_below_minus_3")
|
|
if stress["max_consecutive_losses"] > 10:
|
|
reasons.append("latest_stress_max_consecutive_losses_above_10")
|
|
if validation_locked["total_weighted_edge_bps"] > 0 and stress["total_weighted_edge_bps"] < -0.5 * validation_locked["total_weighted_edge_bps"]:
|
|
reasons.append("latest_stress_loss_too_large_vs_validation")
|
|
return ("REJECTED", reasons) if reasons else ("PASS", [])
|
|
|
|
|
|
def _score_thresholds(metrics: dict[str, Any]) -> float:
|
|
if metrics["trade_count"] == 0:
|
|
return -1_000_000.0
|
|
# 最终上线门槛要求 validation_locked 至少 80 笔；调参区如果只挑几十笔，
|
|
# 很容易是运气好，不是稳定规则，所以这里提前惩罚小样本阈值。
|
|
low_sample_penalty = max(0, 120 - int(metrics["trade_count"])) * 1.5
|
|
profit_factor_penalty = max(0.0, 1.15 - float(metrics["profit_factor"])) * 20.0
|
|
negative_edge_penalty = max(0.0, -float(metrics["avg_weighted_edge_bps"])) * 40.0
|
|
return (
|
|
metrics["avg_weighted_edge_bps"] * np.sqrt(metrics["trade_count"])
|
|
+ metrics["total_weighted_edge_bps"] * 0.05
|
|
- metrics["max_drawdown_bps"] * 0.25
|
|
- low_sample_penalty
|
|
- profit_factor_penalty
|
|
- negative_edge_penalty
|
|
)
|
|
|
|
|
|
def _pm_config_from_thresholds(thresholds: dict[str, float]) -> dict:
|
|
config = default_pm_config()
|
|
config["open"].update(
|
|
{
|
|
"longOpenProb": thresholds["long_open_prob"],
|
|
"shortOpenProb": thresholds["short_open_prob"],
|
|
"minLongEntryProb": thresholds["min_entry_prob"],
|
|
"minShortEntryProb": thresholds["min_entry_prob"],
|
|
"maxMarketRiskProb": thresholds["max_market_risk_prob"],
|
|
"minExpectedEdgeBps": thresholds["min_expected_edge_bps"],
|
|
"minDirectionMargin": thresholds["min_direction_margin"],
|
|
}
|
|
)
|
|
config["add"]["maxMarketRiskProb"] = thresholds["max_market_risk_prob"]
|
|
config["add"]["minExpectedEdgeBps"] = thresholds["min_expected_edge_bps"]
|
|
config["sizing"]["minEdgeBps"] = thresholds["min_expected_edge_bps"]
|
|
config["sizing"]["maxSingleLegRatio"] = 1.0
|
|
return config
|
|
|
|
|
|
def _thresholds_from_config(config: dict) -> dict[str, float]:
|
|
open_config = config["open"]
|
|
return {
|
|
"long_open_prob": float(open_config["longOpenProb"]),
|
|
"short_open_prob": float(open_config["shortOpenProb"]),
|
|
"min_entry_prob": float(min(open_config["minLongEntryProb"], open_config["minShortEntryProb"])),
|
|
"max_market_risk_prob": float(open_config["maxMarketRiskProb"]),
|
|
"min_expected_edge_bps": float(open_config["minExpectedEdgeBps"]),
|
|
"min_direction_margin": float(open_config["minDirectionMargin"]),
|
|
}
|
|
|
|
|
|
def _equity_curve(trades: pd.DataFrame) -> pd.DataFrame:
|
|
if trades.empty:
|
|
return pd.DataFrame(columns=["event_time", "trade_index", "weighted_edge_bps", "equity_bps", "drawdown_bps"])
|
|
curve = trades[["event_time", "weighted_edge_bps"]].copy().reset_index(drop=True)
|
|
curve["trade_index"] = np.arange(1, len(curve) + 1)
|
|
curve["equity_bps"] = curve["weighted_edge_bps"].astype(float).cumsum()
|
|
curve["drawdown_bps"] = curve["equity_bps"].cummax() - curve["equity_bps"]
|
|
return curve[["event_time", "trade_index", "weighted_edge_bps", "equity_bps", "drawdown_bps"]]
|
|
|
|
|
|
def _regime_metrics(trades: pd.DataFrame) -> pd.DataFrame:
|
|
if trades.empty:
|
|
return pd.DataFrame(columns=["split_id", "side", "trade_count", "win_rate", "avg_actual_edge_bps", "total_weighted_edge_bps"])
|
|
rows = []
|
|
for (split_id, side), group in trades.groupby(["split_id", "side"], sort=True):
|
|
metrics = _trade_metrics(group)
|
|
rows.append(
|
|
{
|
|
"split_id": split_id,
|
|
"side": side,
|
|
"trade_count": metrics["trade_count"],
|
|
"win_rate": metrics["win_rate"],
|
|
"avg_actual_edge_bps": metrics["avg_actual_edge_bps"],
|
|
"total_weighted_edge_bps": metrics["total_weighted_edge_bps"],
|
|
}
|
|
)
|
|
return pd.DataFrame(rows)
|
|
|
|
|
|
def _write_pm_report(path, candidates: pd.DataFrame, best_thresholds: dict[str, float], best_metrics: dict[str, Any]) -> None:
|
|
top = candidates.head(10)
|
|
lines = [
|
|
"# PM Threshold Report",
|
|
"",
|
|
"本次不是固定写死阈值，而是在验证集上试一组可复现的阈值，选择净收益、回撤、交易数量综合更好的那组。",
|
|
"",
|
|
"## Best Thresholds",
|
|
"",
|
|
"```json",
|
|
str(best_thresholds).replace("'", '"'),
|
|
"```",
|
|
"",
|
|
"## Best Metrics",
|
|
"",
|
|
"```json",
|
|
str(best_metrics).replace("'", '"'),
|
|
"```",
|
|
"",
|
|
"## Top Candidates",
|
|
"",
|
|
_markdown_table(top.to_dict("records"), list(top.columns)),
|
|
"",
|
|
]
|
|
write_text(path, "\n".join(lines))
|
|
|
|
|
|
def _write_backtest_report(path, result: dict[str, Any]) -> None:
|
|
lines = [
|
|
"# Integrated Backtest Report",
|
|
"",
|
|
"这里用验证集模型输出和 PM 阈值生成交易明细，统计净收益、胜率、回撤和分段表现。",
|
|
"",
|
|
"```json",
|
|
str(result).replace("'", '"'),
|
|
"```",
|
|
"",
|
|
]
|
|
write_text(path, "\n".join(lines))
|
|
|
|
|
|
def _write_failure_cases(path, trades: pd.DataFrame) -> None:
|
|
worst = trades.sort_values("weighted_edge_bps").head(20) if not trades.empty else trades
|
|
lines = [
|
|
"# Backtest Failure Cases",
|
|
"",
|
|
"按加权净收益从差到好列出最差样本，方便回看特征、标签和阈值。",
|
|
"",
|
|
_markdown_table(worst.to_dict("records"), list(worst.columns)) if not worst.empty else "无交易样本。",
|
|
"",
|
|
]
|
|
write_text(path, "\n".join(lines))
|
|
|
|
|
|
def _write_no_baseline_ablation(path) -> None:
|
|
lines = [
|
|
"# Direction Ablation Backtest Report",
|
|
"",
|
|
"- status: NO_BASELINE",
|
|
"- reason: 当前 run 目录没有旧 Direction 基准模型包，所以首版不能做只替换 Direction 的消融回测。",
|
|
"- action: 后续版本必须拿上一版 ACTIVE 包做 baseline，再比较新 Direction 是否真的提升。",
|
|
"",
|
|
]
|
|
write_text(path, "\n".join(lines))
|
|
|
|
|
|
def _markdown_table(rows: list[dict[str, Any]], columns: list[str]) -> str:
|
|
lines = ["| " + " | ".join(columns) + " |", "| " + " | ".join("---" for _ in columns) + " |"]
|
|
for row in rows:
|
|
lines.append("| " + " | ".join(str(row.get(column, "")) for column in columns) + " |")
|
|
return "\n".join(lines)
|