davidpomerenke commited on
Commit
f2add9e
·
verified ·
1 Parent(s): 4e0fc02

Upload from GitHub Actions: discovery: filter out voice/ASR/vision/build endpoints

Browse files

Auto-discovery was surfacing non-text endpoints (microsoft/mai-voice-2,
nvidia/parakeet ASR, glm-4.5v / glm-5v-turbo vision, x-ai/grok-build).
Add voice/parakeet/canary/build skip rules and a regex for the bare-"v"
vision suffix that the -vision/-vl tags miss.

Files changed (1) hide show
  1. evals/models.py +8 -0
evals/models.py CHANGED
@@ -176,6 +176,7 @@ _DISCOVERY_SKIP_TAGS = (
176
  "-preview", "-beta", "-experimental", ":free", "-latest",
177
  "-vision", "-vl", "-image", "-audio", "-tts", "-stt", "-embed",
178
  "-asr", "-transcribe", "-search", "rerank", "-ocr", "-edit",
 
179
  "coder", "codex", "devstral", "codestral",
180
  "-thinking", "-reasoning", "-think", "-deep-research", "deepresearch",
181
  "-multi-agent", "safeguard",
@@ -184,11 +185,16 @@ _DISCOVERY_SKIP_TAGS = (
184
  # Skip these whole product families (named non-text models).
185
  _DISCOVERY_SKIP_PRODUCTS = (
186
  "whisper", "voxtral", "chirp", "kokoro", "orpheus", "zonos", "csm-",
 
187
  "sora", "veo-", "wan-", "seedance", "seedream", "flux.", "imagine",
188
  "kling", "hailuo", "riverflow", "recraft", "morph-",
189
  "bge-", "gte-", "e5-", "multilingual-e5",
190
  )
191
 
 
 
 
 
192
 
193
  @cache
194
  def discover_new_models(date: date) -> list[str]:
@@ -224,6 +230,8 @@ def discover_new_models(date: date) -> list[str]:
224
  slug_lower = slug.lower()
225
  if any(prod in slug_lower for prod in _DISCOVERY_SKIP_PRODUCTS):
226
  continue
 
 
227
  if not m.get("endpoint"):
228
  continue
229
  if m["endpoint"].get("is_free"):
 
176
  "-preview", "-beta", "-experimental", ":free", "-latest",
177
  "-vision", "-vl", "-image", "-audio", "-tts", "-stt", "-embed",
178
  "-asr", "-transcribe", "-search", "rerank", "-ocr", "-edit",
179
+ "-voice", "voice", "-build", # voice/agent-build endpoints, not general text LLMs
180
  "coder", "codex", "devstral", "codestral",
181
  "-thinking", "-reasoning", "-think", "-deep-research", "deepresearch",
182
  "-multi-agent", "safeguard",
 
185
  # Skip these whole product families (named non-text models).
186
  _DISCOVERY_SKIP_PRODUCTS = (
187
  "whisper", "voxtral", "chirp", "kokoro", "orpheus", "zonos", "csm-",
188
+ "parakeet", "canary", # NVIDIA ASR speech models
189
  "sora", "veo-", "wan-", "seedance", "seedream", "flux.", "imagine",
190
  "kling", "hailuo", "riverflow", "recraft", "morph-",
191
  "bge-", "gte-", "e5-", "multilingual-e5",
192
  )
193
 
194
+ # Vision variants often tack a bare "v" onto the version (glm-4.5v, glm-5v-turbo)
195
+ # rather than a "-vision"/"-vl" tag, so the substring filter above misses them.
196
+ _VISION_SUFFIX_RE = re.compile(r"\d+(\.\d+)?v(-|$)")
197
+
198
 
199
  @cache
200
  def discover_new_models(date: date) -> list[str]:
 
230
  slug_lower = slug.lower()
231
  if any(prod in slug_lower for prod in _DISCOVERY_SKIP_PRODUCTS):
232
  continue
233
+ if _VISION_SUFFIX_RE.search(slug_lower):
234
+ continue # bare-"v" vision variant (e.g. glm-4.5v, glm-5v-turbo)
235
  if not m.get("endpoint"):
236
  continue
237
  if m["endpoint"].get("is_free"):