Spaces:
Running on Zero
Running on Zero
File size: 3,672 Bytes
143710c 7b4e05b 143710c 7b4e05b 143710c 7b4e05b 143710c 7b4e05b 143710c 7b4e05b 143710c 7b4e05b 143710c 7b4e05b 143710c 7b4e05b 143710c 7b4e05b 143710c 7b4e05b 143710c 7b4e05b 143710c 7b4e05b | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 | """Evaluate a trained model on the test set: BLEU/ROUGE/METEOR + qualitative examples.
Usage:
python -m src.evaluation.evaluate --config configs/base_resnet_lstm.yaml --decoding greedy
python -m src.evaluation.evaluate --config configs/base_resnet_lstm.yaml --decoding beam --beam-width 3
Reads:
<paths.processed_dir>/test.csv
<paths.models_dir>/<run_name>/best_model.pt
<paths.processed_dir>/vocab.json
Writes (filenames include the decoding mode so greedy/beam results don't overwrite each other):
<paths.models_dir>/<run_name>/eval_metrics_<decoding>.json
<paths.models_dir>/<run_name>/eval_qualitative_<decoding>.md
"""
from __future__ import annotations
import argparse
import json
from pathlib import Path
import pandas as pd
from tqdm import tqdm
from src.evaluation.metrics import compute_all_metrics
from src.inference.predict import Predictor
from src.utils.config import load_config
def evaluate(
config: dict,
device: str = "cuda",
decoding: str = "greedy",
beam_width: int = 3,
n_qualitative: int = 12,
) -> None:
processed_dir = Path(config["paths"]["processed_dir"])
run_dir = Path(config["paths"]["models_dir"]) / config["run_name"]
predictor = Predictor(
checkpoint_path=run_dir / "best_model.pt",
vocab_path=processed_dir / "vocab.json",
device=device,
)
predictor.decoding = decoding
predictor.beam_width = beam_width
test_df = pd.read_csv(processed_dir / "test.csv")
grouped = test_df.groupby("image")["caption"].apply(list).reset_index()
print(f"Evaluating on {len(grouped)} unique test images (decoding={decoding}, beam_width={beam_width})")
images_dir = Path(config["paths"]["raw_images_dir"])
hypotheses: list[str] = []
references: list[list[str]] = []
per_image_results = []
for _, row in tqdm(grouped.iterrows(), total=len(grouped), desc="Generating captions"):
image_filename = row["image"]
refs = row["caption"]
caption = predictor.predict(images_dir / image_filename)
hypotheses.append(caption)
references.append(refs)
per_image_results.append(
{"image": image_filename, "generated": caption, "references": refs}
)
metrics = compute_all_metrics(hypotheses, references)
print(f"\n=== Evaluation Metrics (decoding={decoding}) ===")
for name, value in metrics.items():
print(f"{name}: {value:.4f}")
metrics_path = run_dir / f"eval_metrics_{decoding}.json"
with open(metrics_path, "w") as f:
json.dump({"decoding": decoding, "beam_width": beam_width, **metrics}, f, indent=2)
print(f"\nSaved metrics to {metrics_path}")
qual_path = run_dir / f"eval_qualitative_{decoding}.md"
with open(qual_path, "w") as f:
f.write(f"# Qualitative Evaluation -- {config['run_name']} ({decoding} decoding)\n\n")
f.write("| Image | Generated | References |\n|---|---|---|\n")
for item in per_image_results[:n_qualitative]:
refs_joined = "<br>".join(item["references"])
f.write(f"| {item['image']} | {item['generated']} | {refs_joined} |\n")
print(f"Saved qualitative examples to {qual_path}")
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("--config", required=True)
parser.add_argument("--device", default="cuda")
parser.add_argument("--decoding", default="greedy", choices=["greedy", "beam"])
parser.add_argument("--beam-width", type=int, default=3)
args = parser.parse_args()
config = load_config(args.config)
evaluate(config, device=args.device, decoding=args.decoding, beam_width=args.beam_width) |