code/scripts/ablation_status_server.py

#!/usr/bin/env python3
"""Simple status server for the overnight ablation run.
Serves monitor dashboards for whichever training is currently active.
"""

import http.server
import json
import os
import time

VAP_DIR = "/home/deployer/laion/Voice-Acting-Pipeline"

ABLATIONS = [
    ("A_nat_quality", "finetune_output/ablation_nat_quality", 8768),
    ("B_nat_centroid", "finetune_output/ablation_nat_centroid", 8769),
    ("C_nat_quality_speaker", "finetune_output/ablation_nat_quality_speaker", 8770),
    ("D_nat_speaker", "finetune_output/ablation_nat_speaker", 8771),
]


class StatusHandler(http.server.BaseHTTPRequestHandler):
    def do_GET(self):
        if self.path == "/" or self.path == "/status":
            self.send_response(200)
            self.send_header("Content-Type", "text/html")
            self.end_headers()

            html = """<!DOCTYPE html>
<html><head><meta charset="utf-8"><title>Ablation Study Status</title>
<meta http-equiv="refresh" content="30">
<style>
body{font-family:system-ui;background:#0d1117;color:#c9d1d9;padding:20px}
h1{color:#58a6ff}
.card{background:#161b22;border:1px solid #30363d;border-radius:8px;padding:15px;margin:10px 0}
.active{border-color:#3fb950}
.done{border-color:#8b949e}
.pending{border-color:#30363d;opacity:0.5}
.name{font-size:1.2em;font-weight:bold;color:#f0883e}
.status{font-size:0.9em;color:#8b949e;margin-top:5px}
.metrics{font-family:monospace;font-size:0.85em;margin-top:8px;color:#c9d1d9}
a{color:#58a6ff}
</style></head><body>
<h1>Ablation Study — Overnight Runner</h1>
<p style="color:#8b949e">Auto-refreshes every 30s. """ + time.strftime("%Y-%m-%d %H:%M:%S UTC") + """</p>
"""
            for name, output_dir, port in ABLATIONS:
                status_path = os.path.join(VAP_DIR, output_dir, "status.json")
                metrics_path = os.path.join(VAP_DIR, output_dir, "metrics.jsonl")

                if os.path.exists(status_path):
                    try:
                        with open(status_path) as f:
                            st = json.load(f)
                        step = st.get("step", 0)
                        total = st.get("total_steps", 234)
                        loss = st.get("flow_loss", st.get("loss", "?"))
                        best = st.get("best_loss", "?")
                        elapsed = st.get("elapsed_sec", 0)
                        eta = st.get("eta_sec", 0)
                        pct = int(100 * step / total) if total else 0

                        if step >= total:
                            cls = "done"
                            status_text = f"COMPLETED — {step}/{total} steps in {elapsed/60:.0f}min"
                        else:
                            age = time.time() - os.path.getmtime(status_path)
                            if age < 120:
                                cls = "active"
                                status_text = f"RUNNING — {step}/{total} ({pct}%) — ETA {eta/60:.0f}min"
                            else:
                                cls = "done"
                                status_text = f"COMPLETED — {step}/{total} steps"

                        metrics_text = f"flow_loss={loss}  best_loss={best}  elapsed={elapsed/60:.0f}min"

                        # Get last naturalness from metrics
                        if os.path.exists(metrics_path):
                            try:
                                with open(metrics_path) as mf:
                                    lines = mf.readlines()
                                if lines:
                                    last = json.loads(lines[-1])
                                    nat = last.get("naturalness_reward")
                                    if nat is not None:
                                        metrics_text += f"  nat={nat:.4f}"
                            except Exception:
                                pass

                        html += f"""<div class="card {cls}">
<div class="name">{name}</div>
<div class="status">{status_text}</div>
<div class="metrics">{metrics_text}</div>
<div style="margin-top:5px"><a href="http://localhost:{port}" target="_blank">Monitor Dashboard (port {port})</a></div>
</div>"""
                    except Exception as e:
                        html += f'<div class="card done"><div class="name">{name}</div><div class="status">Error reading status: {e}</div></div>'
                else:
                    html += f'<div class="card pending"><div class="name">{name}</div><div class="status">PENDING — not started yet</div></div>'

            # Check for eval
            eval_log = "/tmp/ablation_eval.log"
            if os.path.exists(eval_log):
                html += '<div class="card active"><div class="name">Evaluation</div>'
                try:
                    with open(eval_log) as f:
                        lines = f.readlines()
                    last_lines = lines[-5:] if lines else ["No output yet"]
                    html += '<div class="metrics">' + "<br>".join(l.strip() for l in last_lines) + '</div>'
                except Exception:
                    html += '<div class="status">Running...</div>'
                html += '</div>'

            # Check for overall log
            html += '<div class="card"><div class="name">Overall Log (last 10 lines)</div><div class="metrics">'
            try:
                with open("/tmp/ablation_overnight.log") as f:
                    lines = f.readlines()
                for line in lines[-10:]:
                    html += line.strip() + "<br>"
            except Exception:
                html += "Log not available yet"
            html += '</div></div>'

            html += "</body></html>"
            self.wfile.write(html.encode())
        else:
            self.send_response(404)
            self.end_headers()

    def log_message(self, format, *args):
        pass  # Suppress logs


if __name__ == "__main__":
    server = http.server.HTTPServer(("0.0.0.0", 8775), StatusHandler)
    print("Status server on http://0.0.0.0:8775")
    server.serve_forever()