Spaces:
Running
Running
tarekziade HF Staff
Fleet tags for model types with several MICs; CI results from the MIC failures dashboard (part 2)
1ba85a2 verified | """The `tmic shrink` pages under shrink/: one per fleet fixture, and the map of model_type -> pages.""" | |
| import json | |
| import re | |
| import os | |
| import subprocess | |
| import sys | |
| from concurrent.futures import ThreadPoolExecutor | |
| from pathlib import Path | |
| from huggingface_hub import hf_hub_download | |
| # Relative and same-frame: the Hub shows the Space in a sandboxed iframe that blocks target="_top". | |
| BACK = '<p class="meta"><a href="../index.html">← MIC coverage of Transformers model types</a></p>\n' | |
| CI_MARK = "<!--ci-->" | |
| CI_LINE = re.compile(r'<p class="meta"><!--ci-->.*?</p>\n') | |
| OLD_BACK = re.compile(r'<p class="meta"><a href="[^"]*"[^>]*>← MIC coverage[^<]*</a></p>\n') | |
| PAGE = """ | |
| import json, sys, warnings | |
| warnings.filterwarnings("ignore") | |
| from transformers_mic import shrink_view | |
| pin, out = sys.argv[1], sys.argv[2] | |
| report = shrink_view.analyze(shrink_view.load(pin)) | |
| open(out, "w").write(shrink_view.render(report)) | |
| open(out.replace(".html", ".json"), "w").write(json.dumps({"fixture": pin, "findings": shrink_view.verdict(report)})) | |
| """ | |
| def stale(fixtures, pages): | |
| """Pins whose page is missing or was rendered from another revision.""" | |
| out = [] | |
| for repo, revision in fixtures: | |
| record = pages / f"{repo.split('/')[-1]}.json" | |
| if not record.exists() or json.loads(record.read_text()).get("fixture") != f"{repo}@{revision}": | |
| out.append(f"{repo}@{revision}") | |
| return out | |
| def render(pins, pages, mic, python=sys.executable, workers=3): | |
| """Render pages with a Python whose transformers knows every fixture's model type (tmic from `mic`).""" | |
| env = {**os.environ, "HF_HUB_DISABLE_PROGRESS_BARS": "1", "PYTHONPATH": f"{mic}/src"} | |
| def one(pin): | |
| out = pages / f"{pin.split('/')[-1].split('@')[0]}.html" | |
| done = subprocess.run([python, "-c", PAGE, pin, str(out)], env=env, capture_output=True, text=True, timeout=600) | |
| return f"{pin}: {done.stderr.strip().splitlines()[-1]}" if done.returncode else "" | |
| with ThreadPoolExecutor(workers) as pool: | |
| return [line for line in pool.map(one, pins) if line] | |
| def ci_line(name, outcome): | |
| """One line on the fixture's results on Transformers main, linking the dashboard.""" | |
| from results import DAYS, FAILURES, fixture_url | |
| now, fails = outcome or (0, 0) | |
| state = ( | |
| f"failing now on main ({fails} failed checks in {DAYS} days)" | |
| if now | |
| else f"passing now; {fails} failed checks in {DAYS} days" | |
| if fails | |
| else f"no failure on main in {DAYS} days" | |
| ) | |
| return ( | |
| f'<p class="meta">{CI_MARK}CI: <a href="{fixture_url(name)}" target="_blank" rel="noopener">{state}</a>' | |
| f' · <a href="{FAILURES}" target="_blank" rel="noopener">MIC failures dashboard</a></p>\n' | |
| ) | |
| def link_back(pages, outcomes=None): | |
| """Put a link to the coverage page, and the fixture's CI results, at the top of every shrink page.""" | |
| outcomes = outcomes or {} | |
| for page in pages.glob("*.html"): | |
| text = CI_LINE.sub("", OLD_BACK.sub("", page.read_text())) | |
| text = text.replace("<main>\n<h1>", f"<main>\n{BACK}<h1>", 1) | |
| meta_end = text.index("</p>\n", text.index("</h1>")) + len("</p>\n") # the pins line under the title | |
| page.write_text(text[:meta_end] + ci_line(page.stem, outcomes.get(page.stem)) + text[meta_end:]) | |
| def links(fixtures, pages, fleets=None): | |
| """fixtures: (repo, revision) pins -> {model_type: [[page name, findings, fleet, source], ...]} | |
| fleets: {repo: "control" | "prototypes"}; a model type can have a hand-written control | |
| contract and a tmic prototype, or fixtures built from different checkpoints. | |
| """ | |
| fleets = fleets or {} | |
| out = {} | |
| for repo, revision in fixtures: | |
| name = repo.split("/")[-1] | |
| page, record = pages / f"{name}.html", pages / f"{name}.json" | |
| if not page.exists(): | |
| continue | |
| findings = len(json.loads(record.read_text()).get("findings", [])) if record.exists() else 0 | |
| model_type = json.loads(Path(hf_hub_download(repo, "config.json", revision=revision)).read_text())["model_type"] | |
| build = json.loads(Path(hf_hub_download(repo, "build.json", revision=revision)).read_text()) | |
| entry = [name, findings, fleets.get(repo, ""), (build.get("built_from") or {}).get("repo", "")] | |
| if entry not in out.setdefault(model_type, []): | |
| out[model_type].append(entry) | |
| return {k: sorted(v) for k, v in sorted(out.items())} | |