Spaces:
Running
Running
Upload from GitHub Actions: models: drop kimi-k2.6; exclude egregiously-failing models after one run
Browse files- Remove moonshotai/kimi-k2.6 from important_models: ~58% errors and so
heavily rate-limited that a single model ran >50min (it ate a run's 6h
budget). kimi-k2 represents Moonshot.
- Tiered strikes: a model failing >= AUTO_BLOCKLIST_IMMEDIATE_FAIL_PCT (80%)
is excluded after ONE bad run instead of waiting out the 2-run grace — it's
broken, not transiently rate-limited. The grace still protects the borderline
50-80% band where a provider outage is plausible.
- evals/models.py +19 -2
evals/models.py
CHANGED
|
@@ -70,7 +70,8 @@ important_models = [
|
|
| 70 |
"deepseek/deepseek-v4-pro",
|
| 71 |
"x-ai/grok-4.20",
|
| 72 |
"mistralai/mistral-medium-3.5",
|
| 73 |
-
"moonshotai/kimi-k2.6",
|
|
|
|
| 74 |
"google/gemini-3.1-flash-lite",
|
| 75 |
]
|
| 76 |
|
|
@@ -586,6 +587,12 @@ AUTO_BLOCKLIST_FAIL_PCT_THRESHOLD = 50.0
|
|
| 586 |
# Strikes are persisted to HF (survives ephemeral CI runners) and maintained by
|
| 587 |
# update_blocklist_strikes(); a model that recovers in any run drops back to 0.
|
| 588 |
AUTO_BLOCKLIST_MIN_RUNS = 2
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 589 |
|
| 590 |
|
| 591 |
def compute_model_health(detailed=None) -> pd.DataFrame:
|
|
@@ -659,11 +666,21 @@ def update_blocklist_strikes(detailed=None) -> pd.DataFrame:
|
|
| 659 |
except Exception:
|
| 660 |
prior_map = {}
|
| 661 |
fail_map = dict(zip(bad["model"], bad["failed_pct"]))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 662 |
strikes_df = pd.DataFrame(
|
| 663 |
[
|
| 664 |
{
|
| 665 |
"model": m,
|
| 666 |
-
"strikes":
|
| 667 |
"failed_pct": round(float(fail_map[m]), 1),
|
| 668 |
}
|
| 669 |
for m in sorted(fail_map)
|
|
|
|
| 70 |
"deepseek/deepseek-v4-pro",
|
| 71 |
"x-ai/grok-4.20",
|
| 72 |
"mistralai/mistral-medium-3.5",
|
| 73 |
+
# "moonshotai/kimi-k2.6", # removed 2026-06: ~58% errors + heavily
|
| 74 |
+
# rate-limited (one model ran >50min); kimi-k2 represents Moonshot instead.
|
| 75 |
"google/gemini-3.1-flash-lite",
|
| 76 |
]
|
| 77 |
|
|
|
|
| 587 |
# Strikes are persisted to HF (survives ephemeral CI runners) and maintained by
|
| 588 |
# update_blocklist_strikes(); a model that recovers in any run drops back to 0.
|
| 589 |
AUTO_BLOCKLIST_MIN_RUNS = 2
|
| 590 |
+
# Fast path: a model failing this badly is broken/unusable, not transiently
|
| 591 |
+
# rate-limited — exclude it after a SINGLE bad run rather than waiting out the
|
| 592 |
+
# grace period, so we don't keep spending on it. The grace only protects the
|
| 593 |
+
# borderline band (FAIL_PCT_THRESHOLD .. IMMEDIATE_FAIL_PCT) where a provider
|
| 594 |
+
# outage is a plausible explanation.
|
| 595 |
+
AUTO_BLOCKLIST_IMMEDIATE_FAIL_PCT = 80.0
|
| 596 |
|
| 597 |
|
| 598 |
def compute_model_health(detailed=None) -> pd.DataFrame:
|
|
|
|
| 666 |
except Exception:
|
| 667 |
prior_map = {}
|
| 668 |
fail_map = dict(zip(bad["model"], bad["failed_pct"]))
|
| 669 |
+
|
| 670 |
+
def _strikes_for(model, fail_pct):
|
| 671 |
+
# Egregious failure → jump straight to the exclusion threshold (no
|
| 672 |
+
# grace). Otherwise increment so the borderline band needs MIN_RUNS
|
| 673 |
+
# consecutive bad runs.
|
| 674 |
+
incremented = int(prior_map.get(model, 0)) + 1
|
| 675 |
+
if fail_pct >= AUTO_BLOCKLIST_IMMEDIATE_FAIL_PCT:
|
| 676 |
+
return max(incremented, AUTO_BLOCKLIST_MIN_RUNS)
|
| 677 |
+
return incremented
|
| 678 |
+
|
| 679 |
strikes_df = pd.DataFrame(
|
| 680 |
[
|
| 681 |
{
|
| 682 |
"model": m,
|
| 683 |
+
"strikes": _strikes_for(m, fail_map[m]),
|
| 684 |
"failed_pct": round(float(fail_map[m]), 1),
|
| 685 |
}
|
| 686 |
for m in sorted(fail_map)
|