Add automatic post-training scientific reports and model card
Browse files- src/finalize_cars_run.py +98 -0
src/finalize_cars_run.py
ADDED
|
@@ -0,0 +1,98 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Publish final model-card summaries after the supervised Cars run completes."""
|
| 3 |
+
import csv,fcntl,json,os,shutil,time
|
| 4 |
+
from datetime import datetime,timezone
|
| 5 |
+
from pathlib import Path
|
| 6 |
+
from huggingface_hub import HfApi
|
| 7 |
+
RUN_ID="cars_issynth_20261025_seed0_support1"
|
| 8 |
+
BASE=Path("/marimo/IsSynth-Cars")
|
| 9 |
+
PUB=BASE/"public"
|
| 10 |
+
RUN=BASE/"runs"/RUN_ID
|
| 11 |
+
REPO="DanielNguyen121/IsSynth-Cars"
|
| 12 |
+
|
| 13 |
+
def write_json(path,obj):
|
| 14 |
+
path.parent.mkdir(parents=True,exist_ok=True)
|
| 15 |
+
tmp=path.with_suffix(path.suffix+".tmp")
|
| 16 |
+
tmp.write_text(json.dumps(obj,indent=2,sort_keys=True)+"\n")
|
| 17 |
+
os.replace(tmp,path)
|
| 18 |
+
|
| 19 |
+
def finalize():
|
| 20 |
+
final=json.loads((RUN/"results/final_metrics.json").read_text())
|
| 21 |
+
history=json.loads((RUN/"logs/history.json").read_text())
|
| 22 |
+
metrics=PUB/"metrics";metrics.mkdir(parents=True,exist_ok=True)
|
| 23 |
+
reports=PUB/"reports";reports.mkdir(parents=True,exist_ok=True)
|
| 24 |
+
plots=PUB/"plots";plots.mkdir(parents=True,exist_ok=True)
|
| 25 |
+
shutil.copy2(RUN/"logs/metrics.csv",metrics/"epoch_metrics.csv")
|
| 26 |
+
write_json(metrics/"test_metrics.json",final)
|
| 27 |
+
per=json.loads((RUN/"results/per_class_metrics.json").read_text())
|
| 28 |
+
with (metrics/"per_class_results.csv").open("w",newline="") as f:
|
| 29 |
+
writer=csv.writer(f);writer.writerow(["class_name","precision","recall","f1_score","support"])
|
| 30 |
+
for name,row in per.items():
|
| 31 |
+
if isinstance(row,dict) and name not in {"macro avg","weighted avg"}:
|
| 32 |
+
writer.writerow([name,row.get("precision"),row.get("recall"),row.get("f1-score"),row.get("support")])
|
| 33 |
+
for image in (RUN/"plots").glob("*.png"):
|
| 34 |
+
shutil.copy2(image,plots/image.name)
|
| 35 |
+
import matplotlib
|
| 36 |
+
matplotlib.use("Agg")
|
| 37 |
+
import matplotlib.pyplot as plt
|
| 38 |
+
recalls=[(name,float(row.get("recall",0.0))) for name,row in per.items() if isinstance(row,dict) and name not in {"macro avg","weighted avg"}]
|
| 39 |
+
fig,ax=plt.subplots(figsize=(8,4))
|
| 40 |
+
ax.hist([value for _,value in recalls],bins=20,color="steelblue")
|
| 41 |
+
ax.set_xlabel("Per-class test recall (class accuracy)");ax.set_ylabel("Number of classes")
|
| 42 |
+
fig.tight_layout();fig.savefig(plots/"class_accuracy_distribution.png",dpi=150);plt.close(fig)
|
| 43 |
+
worst=sorted(recalls,key=lambda row:row[1])[:20]
|
| 44 |
+
fig,ax=plt.subplots(figsize=(10,7))
|
| 45 |
+
ax.barh([name for name,_ in worst[::-1]],[value for _,value in worst[::-1]])
|
| 46 |
+
ax.set_xlabel("Test recall");ax.set_title("20 lowest-accuracy Cars classes")
|
| 47 |
+
fig.tight_layout();fig.savefig(plots/"per_class_accuracy.png",dpi=150);plt.close(fig)
|
| 48 |
+
gpu=RUN/"metrics/gpu.csv"
|
| 49 |
+
peak_gpu_used=None
|
| 50 |
+
if gpu.exists():
|
| 51 |
+
rows=list(csv.DictReader(gpu.open()))
|
| 52 |
+
if rows:
|
| 53 |
+
peak_gpu_used=max(float(row["memory_used_mib"]) for row in rows)/1024
|
| 54 |
+
import matplotlib
|
| 55 |
+
matplotlib.use("Agg")
|
| 56 |
+
import matplotlib.pyplot as plt
|
| 57 |
+
x=range(len(rows))
|
| 58 |
+
fig,ax=plt.subplots(figsize=(9,4))
|
| 59 |
+
ax.plot(x,[float(row["gpu_util_percent"]) for row in rows],label="GPU utilization %")
|
| 60 |
+
ax2=ax.twinx()
|
| 61 |
+
ax2.plot(x,[float(row["memory_used_mib"])/1024 for row in rows],color="orange",label="VRAM GiB")
|
| 62 |
+
ax.set_xlabel("30-second sample index");ax.set_ylabel("Utilization %");ax2.set_ylabel("VRAM GiB")
|
| 63 |
+
fig.tight_layout();fig.savefig(plots/"gpu_trends.png",dpi=150);plt.close(fig)
|
| 64 |
+
steps=history[-1]["global_step"] if history else 0
|
| 65 |
+
report=f"""# IsSynth-Cars final results\n\n- Status: {final['status']}\n- Run ID: `{RUN_ID}`\n- Best epoch (zero based): {final['best_epoch']}\n- Best validation accuracy: {final['best_validation_accuracy']:.6f}\n- Final test top-1 accuracy: {final['test_accuracy']:.6f}\n- Final test macro-F1: {final['test_macro_f1']:.6f}\n- Epochs: {len(history)}; optimizer steps: {steps}\n- Training duration: {final['training_duration_seconds']:.1f} seconds\n- Real few-shot / synthetic / validation / test samples: 3136 / 12544 / 1631 / 8041\n- Classes: 196; model seed: 0; data-selection seed: 1\n- Effective optimizer batch size: 64; AdamW learning rate: 1e-4\n- Peak sampled total device memory: {peak_gpu_used if peak_gpu_used is not None else 'unavailable'} GiB\n\nThe test set was evaluated once after loading the best validation checkpoint. Four exact-byte validation/test duplicate identifiers were excluded from validation only. Original image bytes are not published here.\n"""
|
| 66 |
+
(reports/"final_results.md").write_text(report)
|
| 67 |
+
(reports/"reproducibility.md").write_text("# Reproducibility\n\nSee `configs/cars_baseline.yaml`, `configs/config_provenance.json`, `manifests/`, `src/README.md`, and `runs/"+RUN_ID+"/checkpoints/`. The original Stanford Cars archives must be obtained locally for authorized research and verified against stored SHA-256 values; this repository does not redistribute them. The latest checkpoint contains optimizer, AMP scaler, epoch, RNG, and adapter state for epoch-boundary recovery.\n")
|
| 68 |
+
(reports/"scientific_comparison.md").write_text("# Scientific comparison\n\nThe official source's active execution block is Aircraft and selects checkpoints using test accuracy. This Cars run reuses the validated Caltech infrastructure but selects by a 1,631-image validation set, then tests once. It uses the authoritative Cars seed-1 leak-free 16-shot support and model seed 0. Four validation images duplicated in the official test set were excluded. Text prototype computation follows the official `@torch.no_grad()` rule. These differences prevent a claim of exact paper-level reproduction.\n")
|
| 69 |
+
state={"status":"COMPLETED","run_id":RUN_ID,"current_epoch":40,"global_step":steps,"best_epoch":final["best_epoch"],"best_validation_accuracy":final["best_validation_accuracy"],"final_test_accuracy":final["test_accuracy"],"public_image_redistribution":"UNVERIFIED_NO_UPLOAD","completed_at_utc":datetime.now(timezone.utc).isoformat()}
|
| 70 |
+
for name in ["run_state.json","latest_status.json"]:write_json(PUB/"status"/name,state)
|
| 71 |
+
(PUB/"README.md").write_text("---\ntags:\n- image-classification\n- research\n---\n# IsSynth-Cars\n\n**Completed.** Final Stanford Cars test top-1 accuracy: "+f"{final['test_accuracy']:.6f}"+". Best validation accuracy: "+f"{final['best_validation_accuracy']:.6f}"+".\n\nSee `reports/final_results.md`, `reports/scientific_comparison.md`, `configs/cars_baseline.yaml`, `manifests/`, and `runs/"+RUN_ID+"/` for checkpoints, metrics, and reproducibility. Original Stanford Cars image bytes are not redistributed.\n")
|
| 72 |
+
api=HfApi(token=os.environ.get("HF_TOKEN"))
|
| 73 |
+
last=None
|
| 74 |
+
for attempt in range(4):
|
| 75 |
+
try:
|
| 76 |
+
api.upload_folder(folder_path=str(PUB),repo_id=REPO,repo_type="model",commit_message=f"Finalize {RUN_ID} test results")
|
| 77 |
+
required=["reports/final_results.md","metrics/test_metrics.json","plots/confusion_matrix.png","status/latest_status.json"]
|
| 78 |
+
if not all(api.file_exists(REPO,name,repo_type="model") for name in required):
|
| 79 |
+
raise RuntimeError("Remote final artifacts missing")
|
| 80 |
+
print("FINAL_ARTIFACTS_VERIFIED",RUN_ID,flush=True)
|
| 81 |
+
return
|
| 82 |
+
except Exception as error:
|
| 83 |
+
last=error;time.sleep(min(60,2**attempt*5))
|
| 84 |
+
raise RuntimeError(f"Final artifact sync failed: {last}")
|
| 85 |
+
|
| 86 |
+
if __name__=="__main__":
|
| 87 |
+
RUN.mkdir(parents=True,exist_ok=True)
|
| 88 |
+
with (RUN/"finalizer.lock").open("w") as lock:
|
| 89 |
+
try:fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
| 90 |
+
except BlockingIOError:raise SystemExit(0)
|
| 91 |
+
while True:
|
| 92 |
+
status_file=RUN/"logs/status.json"
|
| 93 |
+
status=json.loads(status_file.read_text()).get("status") if status_file.exists() else None
|
| 94 |
+
if status=="COMPLETED" and (RUN/"results/final_metrics.json").exists():
|
| 95 |
+
finalize();break
|
| 96 |
+
if status=="FAILED":
|
| 97 |
+
print("RUN_FAILED_NO_FINAL_REPORT",flush=True);break
|
| 98 |
+
time.sleep(60)
|