Tighten state Continue verdict checks
This commit is contained in:
@@ -111,6 +111,26 @@ class StateContinueExperimentTest(unittest.TestCase):
|
|||||||
self.assertEqual("NOT_READY_FOR_FORMAL_CHAIN", verdict["status"])
|
self.assertEqual("NOT_READY_FOR_FORMAL_CHAIN", verdict["status"])
|
||||||
self.assertTrue(any("above 0.97" in reason for reason in verdict["reasons"]))
|
self.assertTrue(any("above 0.97" in reason for reason in verdict["reasons"]))
|
||||||
|
|
||||||
|
def test_verdict_reports_when_state_features_do_not_beat_market_only(self) -> None:
|
||||||
|
results = {}
|
||||||
|
for side in ("long", "short"):
|
||||||
|
results[f"{side}_market_only"] = {
|
||||||
|
"validation_locked": {"continue_auc": 0.64, "edge_mae_vs_constant_ratio": 0.965},
|
||||||
|
"latest_stress": {"continue_auc": 0.65, "edge_mae_vs_constant_ratio": 0.964},
|
||||||
|
"regressor_converged": True,
|
||||||
|
}
|
||||||
|
results[f"{side}_market_plus_state"] = {
|
||||||
|
"validation_locked": {"continue_auc": 0.63, "edge_mae_vs_constant_ratio": 0.975},
|
||||||
|
"latest_stress": {"continue_auc": 0.66, "edge_mae_vs_constant_ratio": 0.963},
|
||||||
|
"regressor_converged": True,
|
||||||
|
}
|
||||||
|
|
||||||
|
verdict = _verdict(results)
|
||||||
|
|
||||||
|
self.assertEqual("NOT_READY_FOR_FORMAL_CHAIN", verdict["status"])
|
||||||
|
self.assertTrue(any("continue_auc not better than market_only" in reason for reason in verdict["reasons"]))
|
||||||
|
self.assertTrue(any("edge_mae_vs_constant_ratio not better than market_only" in reason for reason in verdict["reasons"]))
|
||||||
|
|
||||||
def test_train_side_models_supports_ridge_regressor_diagnostic(self) -> None:
|
def test_train_side_models_supports_ridge_regressor_diagnostic(self) -> None:
|
||||||
rows = []
|
rows = []
|
||||||
for split_id in ("fit_inner", "tune_inner", "validation_locked", "latest_stress"):
|
for split_id in ("fit_inner", "tune_inner", "validation_locked", "latest_stress"):
|
||||||
|
|||||||
@@ -578,17 +578,21 @@ def _verdict(results: dict[str, Any]) -> dict[str, Any]:
|
|||||||
base_auc = base_metric.get("continue_auc")
|
base_auc = base_metric.get("continue_auc")
|
||||||
plus_mae = plus_metric.get("edge_mae_vs_constant_ratio")
|
plus_mae = plus_metric.get("edge_mae_vs_constant_ratio")
|
||||||
base_mae = base_metric.get("edge_mae_vs_constant_ratio")
|
base_mae = base_metric.get("edge_mae_vs_constant_ratio")
|
||||||
if plus_auc is None or plus_auc < 0.60:
|
auc_ok = plus_auc is not None and plus_auc >= 0.60
|
||||||
|
auc_beats_market_only = base_auc is None or (plus_auc is not None and plus_auc > base_auc)
|
||||||
|
if not auc_ok:
|
||||||
reasons.append(f"{side} {split_id} continue_auc below 0.60: {plus_auc}")
|
reasons.append(f"{side} {split_id} continue_auc below 0.60: {plus_auc}")
|
||||||
elif base_auc is not None and plus_auc <= base_auc:
|
if not auc_beats_market_only:
|
||||||
reasons.append(f"{side} {split_id} continue_auc not better than market_only: {plus_auc} <= {base_auc}")
|
reasons.append(f"{side} {split_id} continue_auc not better than market_only: {plus_auc} <= {base_auc}")
|
||||||
else:
|
if auc_ok and auc_beats_market_only:
|
||||||
passed_checks.append(f"{side} {split_id} continue_auc")
|
passed_checks.append(f"{side} {split_id} continue_auc")
|
||||||
if plus_mae is None or plus_mae > 0.97:
|
mae_ok = plus_mae is not None and plus_mae <= 0.97
|
||||||
|
mae_beats_market_only = base_mae is None or (plus_mae is not None and plus_mae < base_mae)
|
||||||
|
if not mae_ok:
|
||||||
reasons.append(f"{side} {split_id} edge_mae_vs_constant_ratio above 0.97: {plus_mae}")
|
reasons.append(f"{side} {split_id} edge_mae_vs_constant_ratio above 0.97: {plus_mae}")
|
||||||
elif base_mae is not None and plus_mae >= base_mae:
|
if not mae_beats_market_only:
|
||||||
reasons.append(f"{side} {split_id} edge_mae_vs_constant_ratio not better than market_only: {plus_mae} >= {base_mae}")
|
reasons.append(f"{side} {split_id} edge_mae_vs_constant_ratio not better than market_only: {plus_mae} >= {base_mae}")
|
||||||
else:
|
if mae_ok and mae_beats_market_only:
|
||||||
passed_checks.append(f"{side} {split_id} edge_mae_vs_constant_ratio")
|
passed_checks.append(f"{side} {split_id} edge_mae_vs_constant_ratio")
|
||||||
return {
|
return {
|
||||||
"status": "PASS_TO_FORMAL_CHAIN" if not reasons else "NOT_READY_FOR_FORMAL_CHAIN",
|
"status": "PASS_TO_FORMAL_CHAIN" if not reasons else "NOT_READY_FOR_FORMAL_CHAIN",
|
||||||
|
|||||||
Reference in New Issue
Block a user