davidpomerenke commited on
Commit
4c601bb
·
verified ·
1 Parent(s): 594d28a

Upload from GitHub Actions: models: drop kimi-k2.6; exclude egregiously-failing models after one run

Browse files

- Remove moonshotai/kimi-k2.6 from important_models: ~58% errors and so
heavily rate-limited that a single model ran >50min (it ate a run's 6h
budget). kimi-k2 represents Moonshot.
- Tiered strikes: a model failing >= AUTO_BLOCKLIST_IMMEDIATE_FAIL_PCT (80%)
is excluded after ONE bad run instead of waiting out the 2-run grace — it's
broken, not transiently rate-limited. The grace still protects the borderline
50-80% band where a provider outage is plausible.

Files changed (1) hide show
  1. evals/models.py +19 -2
evals/models.py CHANGED
@@ -70,7 +70,8 @@ important_models = [
70
  "deepseek/deepseek-v4-pro",
71
  "x-ai/grok-4.20",
72
  "mistralai/mistral-medium-3.5",
73
- "moonshotai/kimi-k2.6",
 
74
  "google/gemini-3.1-flash-lite",
75
  ]
76
 
@@ -586,6 +587,12 @@ AUTO_BLOCKLIST_FAIL_PCT_THRESHOLD = 50.0
586
  # Strikes are persisted to HF (survives ephemeral CI runners) and maintained by
587
  # update_blocklist_strikes(); a model that recovers in any run drops back to 0.
588
  AUTO_BLOCKLIST_MIN_RUNS = 2
 
 
 
 
 
 
589
 
590
 
591
  def compute_model_health(detailed=None) -> pd.DataFrame:
@@ -659,11 +666,21 @@ def update_blocklist_strikes(detailed=None) -> pd.DataFrame:
659
  except Exception:
660
  prior_map = {}
661
  fail_map = dict(zip(bad["model"], bad["failed_pct"]))
 
 
 
 
 
 
 
 
 
 
662
  strikes_df = pd.DataFrame(
663
  [
664
  {
665
  "model": m,
666
- "strikes": int(prior_map.get(m, 0)) + 1,
667
  "failed_pct": round(float(fail_map[m]), 1),
668
  }
669
  for m in sorted(fail_map)
 
70
  "deepseek/deepseek-v4-pro",
71
  "x-ai/grok-4.20",
72
  "mistralai/mistral-medium-3.5",
73
+ # "moonshotai/kimi-k2.6", # removed 2026-06: ~58% errors + heavily
74
+ # rate-limited (one model ran >50min); kimi-k2 represents Moonshot instead.
75
  "google/gemini-3.1-flash-lite",
76
  ]
77
 
 
587
  # Strikes are persisted to HF (survives ephemeral CI runners) and maintained by
588
  # update_blocklist_strikes(); a model that recovers in any run drops back to 0.
589
  AUTO_BLOCKLIST_MIN_RUNS = 2
590
+ # Fast path: a model failing this badly is broken/unusable, not transiently
591
+ # rate-limited — exclude it after a SINGLE bad run rather than waiting out the
592
+ # grace period, so we don't keep spending on it. The grace only protects the
593
+ # borderline band (FAIL_PCT_THRESHOLD .. IMMEDIATE_FAIL_PCT) where a provider
594
+ # outage is a plausible explanation.
595
+ AUTO_BLOCKLIST_IMMEDIATE_FAIL_PCT = 80.0
596
 
597
 
598
  def compute_model_health(detailed=None) -> pd.DataFrame:
 
666
  except Exception:
667
  prior_map = {}
668
  fail_map = dict(zip(bad["model"], bad["failed_pct"]))
669
+
670
+ def _strikes_for(model, fail_pct):
671
+ # Egregious failure → jump straight to the exclusion threshold (no
672
+ # grace). Otherwise increment so the borderline band needs MIN_RUNS
673
+ # consecutive bad runs.
674
+ incremented = int(prior_map.get(model, 0)) + 1
675
+ if fail_pct >= AUTO_BLOCKLIST_IMMEDIATE_FAIL_PCT:
676
+ return max(incremented, AUTO_BLOCKLIST_MIN_RUNS)
677
+ return incremented
678
+
679
  strikes_df = pd.DataFrame(
680
  [
681
  {
682
  "model": m,
683
+ "strikes": _strikes_for(m, fail_map[m]),
684
  "failed_pct": round(float(fail_map[m]), 1),
685
  }
686
  for m in sorted(fail_map)