| """ |
| make_report.py -- assemble Results/ into report.pdf. |
| |
| Everything numerical and every figure is pulled straight from Results/metrics |
| and Results/plots, so the report can never drift from the code that produced it. |
| The prose comes from report_notes.md: each `## key` section there is dropped |
| into the matching slot below. Slots with no notes are rendered as a visible |
| TODO box, so nothing can be silently left blank before submission. |
| |
| Usage |
| ----- |
| python make_report.py |
| python make_report.py --notes report_notes.md --out report.pdf |
| """ |
|
|
| from __future__ import annotations |
|
|
| import argparse |
| import json |
| import re |
| from pathlib import Path |
|
|
| from reportlab.lib import colors |
| from reportlab.lib.enums import TA_JUSTIFY |
| from reportlab.lib.pagesizes import A4 |
| from reportlab.lib.styles import ParagraphStyle, getSampleStyleSheet |
| from reportlab.lib.units import cm |
| from reportlab.platypus import (Image, PageBreak, Paragraph, SimpleDocTemplate, |
| Spacer, Table, TableStyle) |
|
|
| ROOT = Path(__file__).resolve().parent |
| METRICS = ROOT / "Results" / "metrics" |
| PLOTS = ROOT / "Results" / "plots" |
| OVERLAYS = ROOT / "Results" / "overlays" |
| SEED = 16 |
| TAG = f"seed{SEED}" |
|
|
| styles = getSampleStyleSheet() |
| S_TITLE = ParagraphStyle("t", parent=styles["Title"], fontSize=20, spaceAfter=6) |
| S_SUB = ParagraphStyle("s", parent=styles["Normal"], fontSize=11, |
| textColor=colors.HexColor("#555555"), alignment=1) |
| S_H1 = ParagraphStyle("h1", parent=styles["Heading1"], fontSize=15, spaceBefore=14, |
| textColor=colors.HexColor("#1a3d6d")) |
| S_H2 = ParagraphStyle("h2", parent=styles["Heading2"], fontSize=12.5, spaceBefore=10, |
| textColor=colors.HexColor("#24527a")) |
| S_BODY = ParagraphStyle("b", parent=styles["BodyText"], fontSize=9.5, leading=13, |
| alignment=TA_JUSTIFY) |
| S_CAP = ParagraphStyle("c", parent=styles["Normal"], fontSize=8, |
| textColor=colors.HexColor("#666666"), alignment=1, spaceAfter=8) |
| S_TODO = ParagraphStyle("todo", parent=S_BODY, backColor=colors.HexColor("#fff3cd"), |
| borderColor=colors.HexColor("#d39e00"), borderWidth=0.7, |
| borderPadding=5, textColor=colors.HexColor("#664d03")) |
|
|
|
|
| |
| def load_json(name: str): |
| p = METRICS / name |
| if not p.exists(): |
| return None |
| try: |
| return json.loads(p.read_text()) |
| except Exception: |
| return None |
|
|
|
|
| def find_json(pattern: str): |
| """Return {tag: parsed_json} for every metrics file matching a glob.""" |
| out = {} |
| for p in sorted(METRICS.glob(pattern)): |
| try: |
| out[p.stem] = json.loads(p.read_text()) |
| except Exception: |
| pass |
| return out |
|
|
|
|
| def parse_notes(path: Path) -> dict: |
| if not path.exists(): |
| return {} |
| text = path.read_text() |
| parts = re.split(r"^##\s+(.+)$", text, flags=re.MULTILINE) |
| notes = {} |
| for i in range(1, len(parts), 2): |
| key = parts[i].strip().lower() |
| body = parts[i + 1].strip() |
| if body and not body.startswith("<!--"): |
| notes[key] = body |
| return notes |
|
|
|
|
| NOTES: dict = {} |
|
|
|
|
| def prose(key: str, hint: str): |
| body = NOTES.get(key.lower()) |
| if body: |
| chunks = [c.strip() for c in body.split("\n\n") if c.strip()] |
| return [Paragraph(c.replace("\n", " "), S_BODY) for c in chunks] + [Spacer(1, 4)] |
| return [Paragraph(f"<b>TODO ({key}):</b> {hint}", S_TODO), Spacer(1, 6)] |
|
|
|
|
| _FIG_CACHE = ROOT / "Results" / ".report_figs" |
| MAX_FIG_PX = 1700 |
|
|
|
|
| def _downsampled(path: Path) -> Path: |
| """Re-encode a large PNG to a bounded-size JPEG for embedding.""" |
| from PIL import Image as PILImage |
|
|
| _FIG_CACHE.mkdir(parents=True, exist_ok=True) |
| out = _FIG_CACHE / (Path(path).stem + ".jpg") |
| if out.exists() and out.stat().st_mtime >= Path(path).stat().st_mtime: |
| return out |
| with PILImage.open(path) as im: |
| im = im.convert("RGB") |
| if max(im.size) > MAX_FIG_PX: |
| scale = MAX_FIG_PX / max(im.size) |
| im = im.resize((int(im.width * scale), int(im.height * scale)), |
| PILImage.LANCZOS) |
| im.save(out, "JPEG", quality=82, optimize=True) |
| return out |
|
|
|
|
| def fig(path: Path, caption: str, width=16.0 * cm): |
| if not Path(path).exists(): |
| return [Paragraph(f"<i>[missing figure: {Path(path).name} — run the " |
| f"corresponding script]</i>", S_CAP)] |
| from PIL import Image as PILImage |
|
|
| path = _downsampled(Path(path)) |
| with PILImage.open(path) as im: |
| w, h = im.size |
| height = width * h / w |
| max_h = 20.0 * cm |
| if height > max_h: |
| width *= max_h / height |
| height = max_h |
| return [Image(str(path), width=width, height=height), Paragraph(caption, S_CAP)] |
|
|
|
|
| def table(rows, col_widths=None, font=7.6, header=True): |
| if not rows: |
| return [] |
| t = Table(rows, colWidths=col_widths, repeatRows=1 if header else 0, hAlign="LEFT") |
| style = [ |
| ("FONTSIZE", (0, 0), (-1, -1), font), |
| ("GRID", (0, 0), (-1, -1), 0.35, colors.HexColor("#bbbbbb")), |
| ("VALIGN", (0, 0), (-1, -1), "MIDDLE"), |
| ("LEFTPADDING", (0, 0), (-1, -1), 3), |
| ("RIGHTPADDING", (0, 0), (-1, -1), 3), |
| ("TOPPADDING", (0, 0), (-1, -1), 2), |
| ("BOTTOMPADDING", (0, 0), (-1, -1), 2), |
| ] |
| if header: |
| style += [("BACKGROUND", (0, 0), (-1, 0), colors.HexColor("#1a3d6d")), |
| ("TEXTCOLOR", (0, 0), (-1, 0), colors.white), |
| ("FONTNAME", (0, 0), (-1, 0), "Helvetica-Bold")] |
| style += [("ROWBACKGROUNDS", (0, 1), (-1, -1), |
| [colors.white, colors.HexColor("#f2f5f9")])] |
| t.setStyle(TableStyle(style)) |
| return [t, Spacer(1, 8)] |
|
|
|
|
| def kv_table(d: dict, keys=None, fmt="{:.4f}"): |
| keys = keys or list(d.keys()) |
| rows = [["metric", "value"]] |
| for k in keys: |
| if k not in d or isinstance(d[k], (dict, list)): |
| continue |
| v = d[k] |
| rows.append([k, fmt.format(v) if isinstance(v, float) else str(v)]) |
| return table(rows, col_widths=[8 * cm, 5 * cm]) |
|
|
|
|
| |
| def build(out_path: Path, notes_path: Path): |
| global NOTES |
| NOTES = parse_notes(notes_path) |
| doc = SimpleDocTemplate(str(out_path), pagesize=A4, |
| leftMargin=1.9 * cm, rightMargin=1.9 * cm, |
| topMargin=1.7 * cm, bottomMargin=1.7 * cm, |
| title="Topics in AI for Healthcare - Assignment 2") |
| st: list = [] |
|
|
| |
| st += [Spacer(1, 3 * cm), |
| Paragraph("Topics in AI for Healthcare — Assignment 2", S_TITLE), |
| Paragraph("Chest X-ray classification, lung segmentation, Grad-CAM analysis " |
| "and polyp detection", S_SUB), Spacer(1, 0.7 * cm)] |
| st += table([["Roll number", NOTES.get("roll number", "«fill in report_notes.md»")], |
| ["Random seed", f"{SEED} (last two digits of the roll number)"], |
| ["Datasets", "COVID-19 Radiography Database; Kvasir-SEG"], |
| ["Split files", f"splits/covid_classification_{TAG}.csv, " |
| f"covid_segmentation_{TAG}.csv, kvasir_detection_{TAG}.csv"]], |
| col_widths=[4.2 * cm, 11.5 * cm], font=9, header=False) |
| st += [PageBreak()] |
|
|
| |
| |
| |
| st += [Paragraph("Part A — COVID-19 Radiography Database", S_H1)] |
| st += [Paragraph("A.1 Data preparation and exploratory analysis", S_H2)] |
|
|
| audit = load_json(f"A_data_audit_{TAG}.json") |
| if audit: |
| rows = [["check", "result"]] |
| for k in ["n_images_indexed", "n_images_readable", "n_images_with_mask", |
| "n_missing_masks", "n_unreadable_images", "n_unreadable_masks", |
| "n_empty_masks", "n_non_binary_masks", "n_exact_duplicate_groups", |
| "n_perceptual_duplicate_groups", "n_image_mask_size_mismatch"]: |
| if k in audit: |
| rows.append([k.replace("_", " "), str(audit[k])]) |
| det = audit.get("details", {}) |
| rows.append(["modal image size", str(det.get("modal_image_size"))]) |
| rows.append(["images at a non-modal size", str(det.get("n_unusual_image_sizes"))]) |
| st += table(rows, col_widths=[9 * cm, 5 * cm]) |
|
|
| sc = audit.get("split_counts", {}).get("classification", {}) |
| if sc: |
| classes = sorted({c for col in sc.values() for c in col}) |
| hdr = ["class"] + list(sc.keys()) |
| rows = [hdr] + [[c] + [str(sc[s].get(c, 0)) for s in sc] for c in classes] |
| st += [Paragraph("Frozen split — images per class", S_CAP)] |
| st += table(rows) |
|
|
| eda = audit.get("eda", {}) |
| if "imbalance_ratio_max_over_min" in eda: |
| st += [Paragraph(f"Class-imbalance ratio (largest / smallest class): " |
| f"<b>{eda['imbalance_ratio_max_over_min']:.2f}</b>", S_BODY), |
| Spacer(1, 4)] |
| maf = eda.get("mask_area_fraction", {}).get("overall") |
| if maf: |
| st += [Paragraph("Lung-mask area as a fraction of the image", S_CAP)] |
| st += kv_table(maf) |
|
|
| for f, cap in [(PLOTS / f"A1_class_distribution_{TAG}.png", |
| "Figure A1 — class distribution across the frozen splits."), |
| (PLOTS / f"A2_resolution_distribution_{TAG}.png", |
| "Figure A2 — image resolution distribution."), |
| (PLOTS / f"A3_intensity_summary_{TAG}.png", |
| "Figure A3 — intensity summaries by class."), |
| (PLOTS / f"A4_mask_area_distribution_{TAG}.png", |
| "Figure A4 — lung-mask area distribution.")]: |
| st += fig(f, cap) |
| st += fig(OVERLAYS / f"A5_seed_selected_examples_{TAG}.png", |
| "Figure A5 — seed-selected originals, preprocessed images and mask overlays.", |
| width=13 * cm) |
| st += prose("a1 discussion", |
| "Discuss acquisition marks, borders, cropping differences and class " |
| "imbalance visible in the figures above, and say how they shaped your " |
| "preprocessing.") |
| st += [PageBreak()] |
|
|
| |
| st += [Paragraph("A.2 Task 1 — Image classification", S_H2)] |
| trains = find_json("train_cls_*.json") |
| if trains: |
| rows = [["run", "epochs", "optimiser", "lr", "batch", "img", "augment", |
| "imbalance", "best val macro-F1", "min"]] |
| for tag, m in trains.items(): |
| c = m.get("config", {}) |
| rows.append([m.get("tag", tag), str(m.get("epochs_run", "")), |
| "AdamW", f"{c.get('lr', '')}", str(c.get("bs", "")), |
| str(c.get("img_size", "")), str(c.get("augment", "")), |
| str(c.get("imbalance", "")), |
| f"{m.get('best_val_macro_f1', float('nan')):.4f}", |
| str(m.get("minutes", ""))]) |
| st += [Paragraph("Training configuration and validation result of every run", S_CAP)] |
| st += table(rows, font=6.6) |
|
|
| evals = find_json("eval_cls_*_test.json") |
| if evals: |
| rows = [["run", "acc", "bal-acc", "macro-F1", "AUROC", "ms/img"]] |
| for tag, m in evals.items(): |
| rows.append([m.get("tag", tag), f"{m['accuracy']:.4f}", |
| f"{m['balanced_accuracy']:.4f}", f"{m['macro_f1']:.4f}", |
| f"{m.get('auroc_macro_ovr', float('nan')):.4f}", |
| f"{m['timing']['ms_per_image']:.1f}"]) |
| st += [Paragraph("Test-split headline metrics", S_CAP)] |
| st += table(rows, font=7.4) |
|
|
| for tag, m in evals.items(): |
| name = m.get("tag", tag) |
| st += [Paragraph(f"Per-class results — {name}", S_CAP)] |
| rows = [["class", "precision", "recall", "F1", "support", "AUROC"]] |
| auc = m.get("auroc_per_class", {}) |
| for c, v in m["per_class"].items(): |
| rows.append([c, f"{v['precision']:.3f}", f"{v['recall']:.3f}", |
| f"{v['f1']:.3f}", str(v["support"]), |
| f"{auc.get(c, float('nan')):.3f}"]) |
| st += table(rows, font=7.4) |
| st += fig(PLOTS / f"cm_{name}_test.png", |
| f"Confusion matrix — {name}.", width=10 * cm) |
| st += fig(PLOTS / f"roc_{name}_test.png", |
| f"ROC curves (one-vs-rest) — {name}.", width=10 * cm) |
|
|
| comp = METRICS / f"comparison_test_{TAG}.csv" |
| if comp.exists(): |
| import csv |
| rows = list(csv.reader(comp.open())) |
| st += [Paragraph("Ablation and model comparison (test split)", S_CAP)] |
| st += table([[f"{c[:14]}" for c in rows[0]]] + |
| [[f"{v[:14]}" for v in r] for r in rows[1:]], font=6.4) |
| st += prose("a2 ablation", |
| "State which single clinically meaningful preprocessing change you ran " |
| "(lung crop / border mask / grayscale-vs-RGB / augmentation strength), " |
| "why it is clinically meaningful, and what it did to each model.") |
|
|
| |
| for p in sorted(METRICS.glob("errors_cls_*.csv")): |
| import csv |
| rows = list(csv.reader(p.open())) |
| if len(rows) < 2: |
| continue |
| name = p.stem.replace("errors_", "") |
| keep = [0, 1, 2, 3, 4] |
| st += [Paragraph(f"Test-error analysis — {name}", S_CAP)] |
| st += table([[rows[0][i] for i in keep]] + |
| [[r[i] for i in keep] for r in rows[1:]], font=7) |
| st += [Paragraph("Per-error quantitative evidence (from errors_*.csv, column " |
| "<i>evidence</i>) must be paired with your written explanation " |
| "below.", S_CAP)] |
| st += fig(OVERLAYS / f"errors_{name}.png", f"Misclassified test images — {name}.") |
| st += prose("a2 errors", |
| "For each of the eight errors per model, explain the failure using the " |
| "image and the preprocessing pipeline (exposure, border marks, lung field " |
| "size, overlapping appearance of Lung Opacity and Viral Pneumonia, etc.).") |
| st += prose("a2 conclusion", |
| "Which model performed better, and did the chest-X-ray-pretrained " |
| "backbone help? Support the answer with the metric table and the errors.") |
| st += [PageBreak()] |
|
|
| |
| st += [Paragraph("A.3 Task 2 — Lung segmentation", S_H2)] |
| segtr = find_json("train_seg_*.json") |
| for tag, m in segtr.items(): |
| mp = m.get("mask_preprocessing", {}) |
| if mp: |
| st += [Paragraph("Mask preprocessing", S_CAP)] |
| st += table([["setting", "value"]] + [[k.replace("_", " "), str(v)] |
| for k, v in mp.items()], |
| col_widths=[5 * cm, 10.5 * cm], font=7.4) |
| for tag, m in find_json("eval_seg_*_test.json").items(): |
| name = m.get("tag", tag) |
| st += [Paragraph(f"Test metrics — {name} ({m['n_images']} images, " |
| f"threshold {m['threshold']})", S_CAP)] |
| st += kv_table(m["overall"], ["dice", "iou", "pixel_accuracy", "sensitivity", |
| "specificity", "precision", "hd95", "assd", |
| "boundary_f1"]) |
| bc = m.get("by_radiograph_class", {}) |
| if bc: |
| rows = [["radiograph class", "Dice", "IoU", "sensitivity", "specificity", |
| "HD95 (px)", "boundary-F1"]] |
| for c, v in bc.items(): |
| rows.append([c, f"{v['dice']:.4f}", f"{v['iou']:.4f}", |
| f"{v['sensitivity']:.4f}", f"{v['specificity']:.4f}", |
| f"{v.get('hd95', float('nan')):.2f}", |
| f"{v.get('boundary_f1', float('nan')):.4f}"]) |
| st += [Paragraph("Segmentation quality by radiograph class", S_CAP)] |
| st += table(rows, font=7.4) |
| cd = m.get("class_difference_test", {}) |
| if "anova_p" in cd: |
| st += [Paragraph(f"One-way ANOVA on Dice across the four classes: " |
| f"F = {cd['anova_F']:.2f}, p = {cd['anova_p']:.2e} " |
| f"(Kruskal–Wallis p = {cd['kruskal_p']:.2e}).", S_BODY), |
| Spacer(1, 4)] |
| st += fig(PLOTS / f"seg_dice_by_class_{name}.png", |
| "Dice by radiograph class and overall Dice distribution.") |
| st += fig(OVERLAYS / f"seg_overlays_{name}_test.png", |
| "Seed-selected overlays (green = ground truth, red = prediction), " |
| "including failure cases.") |
| st += prose("a3 discussion", |
| "Comment on where the model fails (apex, costophrenic angles, dense " |
| "consolidation) and whether Dice differs across the four radiograph " |
| "classes, citing the ANOVA above.") |
| st += [PageBreak()] |
|
|
| |
| st += [Paragraph("A.4 Task 3 — Grad-CAM versus lung segmentation", S_H2)] |
| gc = load_json(f"gradcam_summary_{TAG}.json") |
| if gc: |
| bym = gc.get("by_model", {}) |
| metrics = ["inside_fraction", "inside_outside_ratio", "iou_top20", |
| "iou_thr50", "peak_inside", "accuracy_on_subset"] |
| rows = [["overlap metric"] + list(bym.keys())] |
| for k in metrics: |
| if all(k in v for v in bym.values()): |
| rows.append([k.replace("_", " ")] + [f"{v[k]:.4f}" for v in bym.values()]) |
| st += [Paragraph(f"Attention-inside-lung metrics on {gc['n_images']} " |
| f"seed-selected test images", S_CAP)] |
| st += table(rows, col_widths=[6.5 * cm, 4 * cm, 4 * cm], font=7.6) |
|
|
| pt = gc.get("paired_tests", {}) |
| rows = [["metric", "XRV mean", "student mean", "difference", "Wilcoxon p"]] |
| for k, v in pt.items(): |
| if isinstance(v, dict) and "p_value" in v: |
| rows.append([k.replace("_", " "), f"{v['xrv_mean']:.4f}", |
| f"{v['student_mean']:.4f}", |
| f"{v['difference_xrv_minus_student']:+.4f}", |
| f"{v['p_value']:.4f}"]) |
| if len(rows) > 1: |
| st += [Paragraph("Paired comparison (same images, both models)", S_CAP)] |
| st += table(rows, font=7.4) |
|
|
| bmc = gc.get("by_model_and_correctness", {}) |
| if bmc: |
| rows = [["subset", "n", "inside fraction", "in/out ratio", "IoU top-20%"]] |
| for k, v in bmc.items(): |
| rows.append([k, str(v.get("n", "")), f"{v['inside_fraction']:.4f}", |
| f"{v['inside_outside_ratio']:.4f}", f"{v['iou_top20']:.4f}"]) |
| st += [Paragraph("Split by correct versus incorrect prediction", S_CAP)] |
| st += table(rows, font=7.4) |
| st += fig(PLOTS / f"gradcam_overlap_summary_{TAG}.png", |
| "Distribution of attention-inside-lung metrics for both models.") |
| st += fig(OVERLAYS / f"gradcam_lungmask_{TAG}.png", |
| "Grad-CAM maps overlaid with the lung mask (green contour) for both models, " |
| "including correct and incorrect predictions.", width=13.5 * cm) |
| st += prose("a4 discussion", |
| "Does attention behaviour match the classification metrics? Comment on " |
| "whether either model is looking outside the lung fields, and what that " |
| "implies about shortcut learning.") |
| st += [PageBreak()] |
|
|
| |
| |
| |
| st += [Paragraph("Part B — Kvasir-SEG polyp detection", S_H1)] |
| st += [Paragraph("B.1 Data preparation and annotation checks", S_H2)] |
| baud = load_json(f"B_data_audit_{TAG}.json") |
| if baud: |
| rows = [["check", "result"]] |
| for k, v in baud.items(): |
| if k.startswith("n_") or k.startswith("median_"): |
| rows.append([k.replace("_", " "), |
| f"{v:.4f}" if isinstance(v, float) else str(v)]) |
| st += table(rows, col_widths=[9 * cm, 5 * cm]) |
| stt = baud.get("split_table", {}) |
| if stt: |
| rows = [["split", "images", "boxes", "mean boxes/image"]] |
| for s in ["train", "val", "test"]: |
| if s in stt.get("images", {}): |
| rows.append([s, str(stt["images"][s]), str(stt["boxes"][s]), |
| f"{stt['mean_boxes_per_image'][s]:.3f}"]) |
| st += [Paragraph("Frozen detection split", S_CAP)] |
| st += table(rows) |
| st += fig(OVERLAYS / f"B3_kvasir_annotations_{TAG}.png", |
| "Figure B3 — seed-selected image + mask + box overlays.") |
|
|
| st += [Paragraph("B.2 Exploratory analysis", S_H2)] |
| if baud: |
| e = baud.get("eda", {}) |
| rows = [["statistic", "value"]] |
| for k in ["n_images", "n_boxes_total", "mean_boxes_per_image", |
| "n_images_multi_box", "n_images_with_zero_boxes", |
| "n_distinct_resolutions", "foreground_pixel_fraction_mean"]: |
| if k in e: |
| v = e[k] |
| rows.append([k.replace("_", " "), |
| f"{v:.4f}" if isinstance(v, float) else str(v)]) |
| for k in ["small_lt_1pct", "medium_1_to_5pct", "large_ge_5pct"]: |
| if k in e.get("size_buckets", {}): |
| rows.append([f"boxes, {k.replace('_', ' ')}", str(e["size_buckets"][k])]) |
| st += table(rows, col_widths=[9 * cm, 5 * cm]) |
| for key, label in [("box_width_px", "box width (px)"), |
| ("box_height_px", "box height (px)"), |
| ("box_area_fraction", "box area fraction"), |
| ("aspect_ratio", "aspect ratio (w/h)")]: |
| d = e.get(key) |
| if d: |
| st += [Paragraph(label, S_CAP)] |
| st += table([["mean", "std", "min", "p05", "median", "p95", "max"], |
| [f"{d['mean']:.3f}", f"{d['std']:.3f}", f"{d['min']:.3f}", |
| f"{d['p05']:.3f}", f"{d['median']:.3f}", |
| f"{d['p95']:.3f}", f"{d['max']:.3f}"]], font=7.4) |
| if "class_imbalance_note" in e: |
| st += [Paragraph(e["class_imbalance_note"], S_BODY), Spacer(1, 4)] |
| st += fig(PLOTS / f"B1_kvasir_eda_{TAG}.png", "Figure B1 — detection EDA.") |
| st += fig(PLOTS / f"B2_kvasir_split_distribution_{TAG}.png", |
| "Figure B2 — polyp size distribution per split.") |
| st += prose("b2 discussion", |
| "Explain how these findings drove your input resolution, augmentation " |
| "policy and evaluation choices (e.g. the size-stratified recall buckets).") |
| st += [PageBreak()] |
|
|
| st += [Paragraph("B.3 Model architecture, training and loss ablation", S_H2)] |
| for tag, m in find_json("train_det_*.json").items(): |
| r = m.get("training_recipe", {}) |
| if r: |
| st += [Paragraph(f"Training recipe — {m.get('tag', tag)}", S_CAP)] |
| st += table([["setting", "value"]] + |
| [[k.replace("_", " "), str(v)] for k, v in r.items()], |
| col_widths=[4.6 * cm, 11 * cm], font=7.2) |
| st += fig(PLOTS / f"curves_{m.get('tag', tag)}.png", |
| f"Training curves — {m.get('tag', tag)}.") |
|
|
| abl = METRICS / f"ablation_comparison_test_{TAG}.csv" |
| if abl.exists(): |
| import csv |
| rows = list(csv.reader(abl.open())) |
| st += [Paragraph("Loss-function ablation on the frozen test split", S_CAP)] |
| st += table([[c[:13] for c in rows[0]]] + [[v[:13] for v in r] for r in rows[1:]], |
| font=6.2) |
| st += prose("b3 ablation", |
| "Explain the imbalance the ablation targets, and report how focal loss " |
| "moved recall, precision, false negatives, false positives, mAP and " |
| "small-polyp recall relative to the default loss.") |
| st += [PageBreak()] |
|
|
| st += [Paragraph("B.4 Evaluation and error analysis", S_H2)] |
| for tag, m in find_json("eval_det_*_test.json").items(): |
| name = m.get("tag", tag) |
| op = m["operating_point"] |
| st += [Paragraph(f"{name} — confidence threshold {m['confidence_threshold_used']:.2f} " |
| f"({m['confidence_threshold_selection']})", S_CAP)] |
| st += table([["mAP@0.50", "mAP@0.75", "mAP@0.95", "mAP@[.50:.95]", "precision", |
| "recall", "F1", "mean IoU", "ms/img"], |
| [f"{m['mAP@0.50']:.4f}", f"{m['mAP@0.75']:.4f}", |
| f"{m['mAP@0.95']:.4f}", f"{m['mAP@[.50:.05:.95]']:.4f}", |
| f"{op['precision']:.4f}", f"{op['recall']:.4f}", f"{op['f1']:.4f}", |
| f"{op['mean_iou_matched']:.4f}", |
| f"{m['timing']['ms_per_image']:.1f}"]], font=7) |
| sz = m.get("recall_by_polyp_size", {}) |
| if sz: |
| st += table([["polyp size"] + list(sz.keys()), |
| ["ground-truth boxes"] + [str(v["n_gt"]) for v in sz.values()], |
| ["recall"] + [f"{v['recall']:.4f}" for v in sz.values()]], font=7.4) |
| fp = m.get("failure_patterns", {}) |
| if fp: |
| rows = [["failure pattern", "count"]] |
| for k, v in fp.items(): |
| if isinstance(v, dict): |
| for k2, v2 in v.items(): |
| rows.append([f"{k.replace('_', ' ')} — {k2}", str(v2)]) |
| else: |
| rows.append([k.replace("_", " "), str(v)]) |
| st += table(rows, col_widths=[9.5 * cm, 3 * cm], font=7.4) |
| st += fig(PLOTS / f"pr_sweep_{name}.png", |
| "Precision–recall curve and confidence-threshold sweep.") |
| st += fig(OVERLAYS / f"det_examples_{name}.png", |
| "Seed-selected test predictions with confidence and IoU " |
| "(green = ground truth, red = matched detection, orange = false positive).") |
| st += prose("b4 failures", |
| "Summarise the dominant failure patterns and tie each one back to a " |
| "specific exploratory finding (size distribution, multi-box images, " |
| "specular highlights, position bias).") |
| st += [PageBreak()] |
|
|
| |
| st += [Paragraph("Analysis and discussion", S_H1)] |
| st += [Paragraph("Comparing classification, segmentation and detection", S_H2)] |
| st += prose("analysis tasks", |
| "Which task was most sensitive to preprocessing, and which evaluation " |
| "metric was most informative? Justify with numbers from the tables above.") |
| st += [Paragraph("TorchXRayVision DenseNet versus the student DenseNet", S_H2)] |
| st += prose("analysis models", |
| "Bring together the test metrics, the Grad-CAM overlap statistics, the " |
| "ablation deltas and the error examples into a single verdict.") |
| st += [Paragraph("Limitations", S_H2)] |
| st += prose("limitations", |
| "Cover dataset quality (source mixing, duplicates, burned-in markers), " |
| "annotation quality (mask provenance, single-annotator boxes), split " |
| "design (no patient-level identifier, so leakage cannot be fully excluded) " |
| "and generalisation beyond these two datasets.") |
| st += [Paragraph("Reproducibility", S_H2)] |
| st += [Paragraph( |
| f"Every number and figure in this report was produced with seed {SEED} from the " |
| f"frozen split files in <font face='Courier'>splits/</font>. The exact commands " |
| f"are listed in <font face='Courier'>README.md</font> and " |
| f"<font face='Courier'>run_all.sh</font>; regenerating this document is " |
| f"<font face='Courier'>python make_report.py</font>.", S_BODY)] |
|
|
| doc.build(st) |
| print(f"[saved] {out_path}") |
| missing = [p for p in NOTES_KEYS if p not in NOTES] |
| if missing: |
| print(f"[note] {len(missing)} narrative section(s) still marked TODO in the PDF: " |
| + ", ".join(missing)) |
|
|
|
|
| NOTES_KEYS = ["roll number", "a1 discussion", "a2 ablation", "a2 errors", "a2 conclusion", |
| "a3 discussion", "a4 discussion", "b2 discussion", "b3 ablation", |
| "b4 failures", "analysis tasks", "analysis models", "limitations"] |
|
|
|
|
| def main(): |
| ap = argparse.ArgumentParser(description="Build report.pdf from Results/") |
| ap.add_argument("--notes", default=str(ROOT / "report_notes.md")) |
| ap.add_argument("--out", default=str(ROOT / "report.pdf")) |
| args = ap.parse_args() |
| build(Path(args.out), Path(args.notes)) |
|
|
|
|
| if __name__ == "__main__": |
| main() |
|
|
