| |
| """Simple status server for the overnight ablation run. |
| Serves monitor dashboards for whichever training is currently active. |
| """ |
|
|
| import http.server |
| import json |
| import os |
| import time |
|
|
| VAP_DIR = "/home/deployer/laion/Voice-Acting-Pipeline" |
|
|
| ABLATIONS = [ |
| ("A_nat_quality", "finetune_output/ablation_nat_quality", 8768), |
| ("B_nat_centroid", "finetune_output/ablation_nat_centroid", 8769), |
| ("C_nat_quality_speaker", "finetune_output/ablation_nat_quality_speaker", 8770), |
| ("D_nat_speaker", "finetune_output/ablation_nat_speaker", 8771), |
| ] |
|
|
|
|
| class StatusHandler(http.server.BaseHTTPRequestHandler): |
| def do_GET(self): |
| if self.path == "/" or self.path == "/status": |
| self.send_response(200) |
| self.send_header("Content-Type", "text/html") |
| self.end_headers() |
|
|
| html = """<!DOCTYPE html> |
| <html><head><meta charset="utf-8"><title>Ablation Study Status</title> |
| <meta http-equiv="refresh" content="30"> |
| <style> |
| body{font-family:system-ui;background:#0d1117;color:#c9d1d9;padding:20px} |
| h1{color:#58a6ff} |
| .card{background:#161b22;border:1px solid #30363d;border-radius:8px;padding:15px;margin:10px 0} |
| .active{border-color:#3fb950} |
| .done{border-color:#8b949e} |
| .pending{border-color:#30363d;opacity:0.5} |
| .name{font-size:1.2em;font-weight:bold;color:#f0883e} |
| .status{font-size:0.9em;color:#8b949e;margin-top:5px} |
| .metrics{font-family:monospace;font-size:0.85em;margin-top:8px;color:#c9d1d9} |
| a{color:#58a6ff} |
| </style></head><body> |
| <h1>Ablation Study — Overnight Runner</h1> |
| <p style="color:#8b949e">Auto-refreshes every 30s. """ + time.strftime("%Y-%m-%d %H:%M:%S UTC") + """</p> |
| """ |
| for name, output_dir, port in ABLATIONS: |
| status_path = os.path.join(VAP_DIR, output_dir, "status.json") |
| metrics_path = os.path.join(VAP_DIR, output_dir, "metrics.jsonl") |
|
|
| if os.path.exists(status_path): |
| try: |
| with open(status_path) as f: |
| st = json.load(f) |
| step = st.get("step", 0) |
| total = st.get("total_steps", 234) |
| loss = st.get("flow_loss", st.get("loss", "?")) |
| best = st.get("best_loss", "?") |
| elapsed = st.get("elapsed_sec", 0) |
| eta = st.get("eta_sec", 0) |
| pct = int(100 * step / total) if total else 0 |
|
|
| if step >= total: |
| cls = "done" |
| status_text = f"COMPLETED — {step}/{total} steps in {elapsed/60:.0f}min" |
| else: |
| age = time.time() - os.path.getmtime(status_path) |
| if age < 120: |
| cls = "active" |
| status_text = f"RUNNING — {step}/{total} ({pct}%) — ETA {eta/60:.0f}min" |
| else: |
| cls = "done" |
| status_text = f"COMPLETED — {step}/{total} steps" |
|
|
| metrics_text = f"flow_loss={loss} best_loss={best} elapsed={elapsed/60:.0f}min" |
|
|
| |
| if os.path.exists(metrics_path): |
| try: |
| with open(metrics_path) as mf: |
| lines = mf.readlines() |
| if lines: |
| last = json.loads(lines[-1]) |
| nat = last.get("naturalness_reward") |
| if nat is not None: |
| metrics_text += f" nat={nat:.4f}" |
| except Exception: |
| pass |
|
|
| html += f"""<div class="card {cls}"> |
| <div class="name">{name}</div> |
| <div class="status">{status_text}</div> |
| <div class="metrics">{metrics_text}</div> |
| <div style="margin-top:5px"><a href="http://localhost:{port}" target="_blank">Monitor Dashboard (port {port})</a></div> |
| </div>""" |
| except Exception as e: |
| html += f'<div class="card done"><div class="name">{name}</div><div class="status">Error reading status: {e}</div></div>' |
| else: |
| html += f'<div class="card pending"><div class="name">{name}</div><div class="status">PENDING — not started yet</div></div>' |
|
|
| |
| eval_log = "/tmp/ablation_eval.log" |
| if os.path.exists(eval_log): |
| html += '<div class="card active"><div class="name">Evaluation</div>' |
| try: |
| with open(eval_log) as f: |
| lines = f.readlines() |
| last_lines = lines[-5:] if lines else ["No output yet"] |
| html += '<div class="metrics">' + "<br>".join(l.strip() for l in last_lines) + '</div>' |
| except Exception: |
| html += '<div class="status">Running...</div>' |
| html += '</div>' |
|
|
| |
| html += '<div class="card"><div class="name">Overall Log (last 10 lines)</div><div class="metrics">' |
| try: |
| with open("/tmp/ablation_overnight.log") as f: |
| lines = f.readlines() |
| for line in lines[-10:]: |
| html += line.strip() + "<br>" |
| except Exception: |
| html += "Log not available yet" |
| html += '</div></div>' |
|
|
| html += "</body></html>" |
| self.wfile.write(html.encode()) |
| else: |
| self.send_response(404) |
| self.end_headers() |
|
|
| def log_message(self, format, *args): |
| pass |
|
|
|
|
| if __name__ == "__main__": |
| server = http.server.HTTPServer(("0.0.0.0", 8775), StatusHandler) |
| print("Status server on http://0.0.0.0:8775") |
| server.serve_forever() |
|
|
