"""Flatten stored job and test-run metrics into one export row. Every image is kept. A failed pass flag or a no-conflict image is a row, not a reason to drop it. """ from __future__ import annotations import csv import io import json from typing import Any, Dict, Iterable, List, Optional COLUMNS = [ "source", "id", "filename", "created_at", "cvd_type", "severity", "de_before", "de_after", "de_improvement", "conflicts_found", "conflicts_resolved", "conflict_resolution_rate", "new_conflicts", "unresolved_conflicts", "new_conflict_pairs", "unresolved_pairs", "naturalness", "no_conflict", "pass_de", "pass_resolution", "pass_naturalness", ] def _metrics(value: Any) -> dict: if isinstance(value, str): try: parsed = json.loads(value) except (TypeError, ValueError): return {} return parsed if isinstance(parsed, dict) else {} return dict(value) if isinstance(value, dict) else {} def _num(value: Any, digits: int = 6) -> Optional[float]: if value is None or value == "": return None try: return round(float(value), digits) except (TypeError, ValueError): return None def _int(value: Any, default: int = 0) -> int: if value is None or value == "": return default try: return int(value) except (TypeError, ValueError): return default def _pairs(raw: Any) -> List[List[int]]: out = [] for pair in raw or []: if isinstance(pair, (list, tuple)) and len(pair) == 2: out.append([_int(pair[0]), _int(pair[1])]) return out def _pair_text(pairs: List[List[int]]) -> str: return ";".join(f"{a}-{b}" for a, b in pairs) def _flag(metrics: dict, metrics_key: str, row: dict, row_key: str) -> bool: if metrics_key in metrics: return bool(metrics.get(metrics_key)) if row_key in row and row.get(row_key) is not None: return bool(row.get(row_key)) return False def metric_row(source: str, row: dict) -> dict: """One spreadsheet row. Caller decides which records to pass; this never filters.""" metrics = _metrics(row.get("metrics")) diag = metrics.get("conflict_diagnostics") or {} if not isinstance(diag, dict): diag = {} found = metrics.get("conflicts_found") if found is None: found = metrics.get("n_conflicts_total") found_n = _int(found, 0) resolved = metrics.get("n_conflicts_resolved") resolved_pairs = _pairs(diag.get("resolved_pairs")) if resolved is None: resolved_n = len(resolved_pairs) else: resolved_n = _int(resolved, 0) new_pairs = _pairs(diag.get("new_pairs")) unresolved_pairs = _pairs(diag.get("unresolved_pairs")) no_conflict = bool(metrics.get("already_accessible")) or found_n == 0 record_id = row.get("job_id") or row.get("run_id") or row.get("id") or "" return { "source": source, "id": str(record_id), "filename": row.get("filename") or "", "created_at": row.get("created_at") or "", "cvd_type": row.get("cvd_type") or "", "severity": _num(row.get("severity")), "de_before": _num(metrics.get("de_before_mean")), "de_after": _num(metrics.get("de_after_mean")), "de_improvement": _num( metrics.get("delta_e_improvement", metrics.get("de_improvement")) ), "conflicts_found": found_n, "conflicts_resolved": resolved_n, "conflict_resolution_rate": _num(metrics.get("conflict_resolution_rate")), "new_conflicts": len(new_pairs), "unresolved_conflicts": len(unresolved_pairs), "new_conflict_pairs": _pair_text(new_pairs), "unresolved_pairs": _pair_text(unresolved_pairs), "naturalness": _num(metrics.get("naturalness_preservation")), "no_conflict": no_conflict, "pass_de": _flag(metrics, "pass_de_improvement", row, "pass_de"), "pass_resolution": _flag(metrics, "pass_resolution_rate", row, "pass_resolution"), "pass_naturalness": _flag(metrics, "pass_naturalness", row, "pass_naturalness"), } def to_csv(rows: Iterable[dict]) -> str: buf = io.StringIO() writer = csv.DictWriter(buf, fieldnames=COLUMNS, lineterminator="\n", extrasaction="ignore") writer.writeheader() for row in rows: flat = {} for key in COLUMNS: value = row.get(key, "") if isinstance(value, bool): flat[key] = "true" if value else "false" elif value is None: flat[key] = "" elif isinstance(value, str) and value.lstrip().startswith(("=", "+", "-", "@")): # Uploaded filenames must remain text when opened in a spreadsheet. flat[key] = "'" + value else: flat[key] = value writer.writerow(flat) return buf.getvalue() def to_json(rows: Iterable[dict]) -> str: return json.dumps(list(rows), indent=2)