DanielNguyen121 commited on
Commit
73be028
·
verified ·
1 Parent(s): 24b1962

Add automatic post-training scientific reports and model card

Browse files
Files changed (1) hide show
  1. src/finalize_cars_run.py +98 -0
src/finalize_cars_run.py ADDED
@@ -0,0 +1,98 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ """Publish final model-card summaries after the supervised Cars run completes."""
3
+ import csv,fcntl,json,os,shutil,time
4
+ from datetime import datetime,timezone
5
+ from pathlib import Path
6
+ from huggingface_hub import HfApi
7
+ RUN_ID="cars_issynth_20261025_seed0_support1"
8
+ BASE=Path("/marimo/IsSynth-Cars")
9
+ PUB=BASE/"public"
10
+ RUN=BASE/"runs"/RUN_ID
11
+ REPO="DanielNguyen121/IsSynth-Cars"
12
+
13
+ def write_json(path,obj):
14
+ path.parent.mkdir(parents=True,exist_ok=True)
15
+ tmp=path.with_suffix(path.suffix+".tmp")
16
+ tmp.write_text(json.dumps(obj,indent=2,sort_keys=True)+"\n")
17
+ os.replace(tmp,path)
18
+
19
+ def finalize():
20
+ final=json.loads((RUN/"results/final_metrics.json").read_text())
21
+ history=json.loads((RUN/"logs/history.json").read_text())
22
+ metrics=PUB/"metrics";metrics.mkdir(parents=True,exist_ok=True)
23
+ reports=PUB/"reports";reports.mkdir(parents=True,exist_ok=True)
24
+ plots=PUB/"plots";plots.mkdir(parents=True,exist_ok=True)
25
+ shutil.copy2(RUN/"logs/metrics.csv",metrics/"epoch_metrics.csv")
26
+ write_json(metrics/"test_metrics.json",final)
27
+ per=json.loads((RUN/"results/per_class_metrics.json").read_text())
28
+ with (metrics/"per_class_results.csv").open("w",newline="") as f:
29
+ writer=csv.writer(f);writer.writerow(["class_name","precision","recall","f1_score","support"])
30
+ for name,row in per.items():
31
+ if isinstance(row,dict) and name not in {"macro avg","weighted avg"}:
32
+ writer.writerow([name,row.get("precision"),row.get("recall"),row.get("f1-score"),row.get("support")])
33
+ for image in (RUN/"plots").glob("*.png"):
34
+ shutil.copy2(image,plots/image.name)
35
+ import matplotlib
36
+ matplotlib.use("Agg")
37
+ import matplotlib.pyplot as plt
38
+ recalls=[(name,float(row.get("recall",0.0))) for name,row in per.items() if isinstance(row,dict) and name not in {"macro avg","weighted avg"}]
39
+ fig,ax=plt.subplots(figsize=(8,4))
40
+ ax.hist([value for _,value in recalls],bins=20,color="steelblue")
41
+ ax.set_xlabel("Per-class test recall (class accuracy)");ax.set_ylabel("Number of classes")
42
+ fig.tight_layout();fig.savefig(plots/"class_accuracy_distribution.png",dpi=150);plt.close(fig)
43
+ worst=sorted(recalls,key=lambda row:row[1])[:20]
44
+ fig,ax=plt.subplots(figsize=(10,7))
45
+ ax.barh([name for name,_ in worst[::-1]],[value for _,value in worst[::-1]])
46
+ ax.set_xlabel("Test recall");ax.set_title("20 lowest-accuracy Cars classes")
47
+ fig.tight_layout();fig.savefig(plots/"per_class_accuracy.png",dpi=150);plt.close(fig)
48
+ gpu=RUN/"metrics/gpu.csv"
49
+ peak_gpu_used=None
50
+ if gpu.exists():
51
+ rows=list(csv.DictReader(gpu.open()))
52
+ if rows:
53
+ peak_gpu_used=max(float(row["memory_used_mib"]) for row in rows)/1024
54
+ import matplotlib
55
+ matplotlib.use("Agg")
56
+ import matplotlib.pyplot as plt
57
+ x=range(len(rows))
58
+ fig,ax=plt.subplots(figsize=(9,4))
59
+ ax.plot(x,[float(row["gpu_util_percent"]) for row in rows],label="GPU utilization %")
60
+ ax2=ax.twinx()
61
+ ax2.plot(x,[float(row["memory_used_mib"])/1024 for row in rows],color="orange",label="VRAM GiB")
62
+ ax.set_xlabel("30-second sample index");ax.set_ylabel("Utilization %");ax2.set_ylabel("VRAM GiB")
63
+ fig.tight_layout();fig.savefig(plots/"gpu_trends.png",dpi=150);plt.close(fig)
64
+ steps=history[-1]["global_step"] if history else 0
65
+ report=f"""# IsSynth-Cars final results\n\n- Status: {final['status']}\n- Run ID: `{RUN_ID}`\n- Best epoch (zero based): {final['best_epoch']}\n- Best validation accuracy: {final['best_validation_accuracy']:.6f}\n- Final test top-1 accuracy: {final['test_accuracy']:.6f}\n- Final test macro-F1: {final['test_macro_f1']:.6f}\n- Epochs: {len(history)}; optimizer steps: {steps}\n- Training duration: {final['training_duration_seconds']:.1f} seconds\n- Real few-shot / synthetic / validation / test samples: 3136 / 12544 / 1631 / 8041\n- Classes: 196; model seed: 0; data-selection seed: 1\n- Effective optimizer batch size: 64; AdamW learning rate: 1e-4\n- Peak sampled total device memory: {peak_gpu_used if peak_gpu_used is not None else 'unavailable'} GiB\n\nThe test set was evaluated once after loading the best validation checkpoint. Four exact-byte validation/test duplicate identifiers were excluded from validation only. Original image bytes are not published here.\n"""
66
+ (reports/"final_results.md").write_text(report)
67
+ (reports/"reproducibility.md").write_text("# Reproducibility\n\nSee `configs/cars_baseline.yaml`, `configs/config_provenance.json`, `manifests/`, `src/README.md`, and `runs/"+RUN_ID+"/checkpoints/`. The original Stanford Cars archives must be obtained locally for authorized research and verified against stored SHA-256 values; this repository does not redistribute them. The latest checkpoint contains optimizer, AMP scaler, epoch, RNG, and adapter state for epoch-boundary recovery.\n")
68
+ (reports/"scientific_comparison.md").write_text("# Scientific comparison\n\nThe official source's active execution block is Aircraft and selects checkpoints using test accuracy. This Cars run reuses the validated Caltech infrastructure but selects by a 1,631-image validation set, then tests once. It uses the authoritative Cars seed-1 leak-free 16-shot support and model seed 0. Four validation images duplicated in the official test set were excluded. Text prototype computation follows the official `@torch.no_grad()` rule. These differences prevent a claim of exact paper-level reproduction.\n")
69
+ state={"status":"COMPLETED","run_id":RUN_ID,"current_epoch":40,"global_step":steps,"best_epoch":final["best_epoch"],"best_validation_accuracy":final["best_validation_accuracy"],"final_test_accuracy":final["test_accuracy"],"public_image_redistribution":"UNVERIFIED_NO_UPLOAD","completed_at_utc":datetime.now(timezone.utc).isoformat()}
70
+ for name in ["run_state.json","latest_status.json"]:write_json(PUB/"status"/name,state)
71
+ (PUB/"README.md").write_text("---\ntags:\n- image-classification\n- research\n---\n# IsSynth-Cars\n\n**Completed.** Final Stanford Cars test top-1 accuracy: "+f"{final['test_accuracy']:.6f}"+". Best validation accuracy: "+f"{final['best_validation_accuracy']:.6f}"+".\n\nSee `reports/final_results.md`, `reports/scientific_comparison.md`, `configs/cars_baseline.yaml`, `manifests/`, and `runs/"+RUN_ID+"/` for checkpoints, metrics, and reproducibility. Original Stanford Cars image bytes are not redistributed.\n")
72
+ api=HfApi(token=os.environ.get("HF_TOKEN"))
73
+ last=None
74
+ for attempt in range(4):
75
+ try:
76
+ api.upload_folder(folder_path=str(PUB),repo_id=REPO,repo_type="model",commit_message=f"Finalize {RUN_ID} test results")
77
+ required=["reports/final_results.md","metrics/test_metrics.json","plots/confusion_matrix.png","status/latest_status.json"]
78
+ if not all(api.file_exists(REPO,name,repo_type="model") for name in required):
79
+ raise RuntimeError("Remote final artifacts missing")
80
+ print("FINAL_ARTIFACTS_VERIFIED",RUN_ID,flush=True)
81
+ return
82
+ except Exception as error:
83
+ last=error;time.sleep(min(60,2**attempt*5))
84
+ raise RuntimeError(f"Final artifact sync failed: {last}")
85
+
86
+ if __name__=="__main__":
87
+ RUN.mkdir(parents=True,exist_ok=True)
88
+ with (RUN/"finalizer.lock").open("w") as lock:
89
+ try:fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
90
+ except BlockingIOError:raise SystemExit(0)
91
+ while True:
92
+ status_file=RUN/"logs/status.json"
93
+ status=json.loads(status_file.read_text()).get("status") if status_file.exists() else None
94
+ if status=="COMPLETED" and (RUN/"results/final_metrics.json").exists():
95
+ finalize();break
96
+ if status=="FAILED":
97
+ print("RUN_FAILED_NO_FINAL_REPORT",flush=True);break
98
+ time.sleep(60)