File size: 3,533 Bytes
93cd35a | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 | #!/usr/bin/env python
"""Compute generative perplexity + unigram entropy on a JSONL of generated texts.
Decoupled from sampling so PPL eval can run on a single GPU after generation
(useful when the sampling GPU is too small to hold both the ELF model and the
PPL model simultaneously). Reads the `{"id": ..., "generated": ...}` JSONL that
`test_generation_uncond` writes to `<output_dir>/<run_name>/all_generated_*.jsonl`.
Example:
python scripts/eval_ppl.py \
--input outputs/elf_l-owt/sde-steps64-cfg1-sccfg3-ts_logit_normal-gamma1.0-uncond/all_generated_3_57051.jsonl
# Custom PPL model / batch size
python scripts/eval_ppl.py --input <path>.jsonl --model gpt2-large --batch_size 16
"""
import argparse
import json
import os
import sys
REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
SRC = os.path.join(REPO_ROOT, "src")
if SRC not in sys.path:
sys.path.insert(0, SRC)
from utils.metrics_utils import Metrics
def load_samples(path: str):
if not os.path.isfile(path):
raise FileNotFoundError(path)
samples = []
with open(path, "r", encoding="utf-8") as f:
for line in f:
line = line.strip()
if not line:
continue
obj = json.loads(line)
text = obj.get("generated", obj.get("text", ""))
samples.append(text)
return samples
def parse_args():
p = argparse.ArgumentParser(description="Compute Gen. PPL + entropy from a JSONL of generated texts.")
p.add_argument("--input", type=str, required=True,
help="Path to all_generated_*.jsonl produced by eval.py.")
p.add_argument("--model", type=str, default="gpt2-large",
help="HF causal-LM used for the PPL likelihood (default: gpt2-large).")
p.add_argument("--batch_size", type=int, default=16,
help="Batch size for PPL forward passes (default: 16).")
p.add_argument("--max_length", type=int, default=1024,
help="Max sequence length for retokenized samples.")
p.add_argument("--output", type=str, default=None,
help="Optional path for metrics jsonl. Defaults to <input_dir>/ppl_metrics.jsonl.")
return p.parse_args()
def main():
args = parse_args()
samples = load_samples(args.input)
nonempty = [s for s in samples if isinstance(s, str) and s.strip()]
skipped = len(samples) - len(nonempty)
print(f"Loaded {len(samples)} samples from {args.input} ({skipped} empty skipped)")
if not nonempty:
print("No non-empty samples; nothing to evaluate.")
return
metrics = Metrics(
gen_ppl_eval_model_name_or_path=args.model,
eval_ppl_batch_size=args.batch_size,
eval_context_size=args.max_length,
)
results = metrics.record_generative_perplexity(
text_samples=nonempty, max_length=args.max_length, retokenize=True,
)
print(f"Perplexity: {results['ppl']:.4f}")
print(f"Mean Entropy: {results['mean_entropy']:.4f}")
out_path = args.output or os.path.join(os.path.dirname(args.input), "ppl_metrics.jsonl")
record = {
"input": args.input,
"ppl_model": args.model,
"num_samples": len(nonempty),
"ppl": results["ppl"],
"mean_entropy": results["mean_entropy"],
}
with open(out_path, "a", encoding="utf-8") as f:
f.write(json.dumps(record, ensure_ascii=False) + "\n")
print(f"Appended metrics to {out_path}")
if __name__ == "__main__":
main()
|