| |
| """Audit chronological TRAIN vulnerable/fixed pairs for pairwise objectives. |
| |
| This is a data-structure diagnostic only. It does not read VALID/TEST labels |
| or train a model. Similarity uses sets and is not a semantic equivalence test. |
| """ |
| from __future__ import annotations |
|
|
| import argparse |
| import hashlib |
| import json |
| import re |
| import statistics |
| from collections import Counter |
| from pathlib import Path |
|
|
| import pandas as pd |
|
|
|
|
| def sha256(path: Path) -> str: |
| h = hashlib.sha256() |
| with path.open("rb") as handle: |
| for block in iter(lambda: handle.read(1 << 20), b""): |
| h.update(block) |
| return h.hexdigest() |
|
|
|
|
| def audit(corpus_path: Path, train_split: Path) -> dict: |
| pair_ids = set(json.loads(train_split.read_text())) |
| corpus = pd.read_parquet(corpus_path) |
| train = corpus[corpus["pair_id"].isin(pair_ids)] |
| group_sizes = train.groupby("pair_id").size() |
| bad_pairs = [] |
| label_sets = Counter() |
| line_jaccards, token_jaccards, line_deltas = [], [], [] |
| exact_equal = nonempty_both = 0 |
| token_re = re.compile(r"[A-Za-z_][A-Za-z0-9_]*|0[xX][0-9a-fA-F]+|\d+|[^\s]") |
|
|
| for pair_id, group in train.groupby("pair_id", sort=False): |
| labels = sorted(int(x) for x in group["target"]) |
| label_sets[tuple(labels)] += 1 |
| if len(group) != 2 or labels != [0, 1]: |
| bad_pairs.append({"pair_id": pair_id, "n_rows": len(group), "labels": labels}) |
| continue |
|
|
| row = group.iloc[0] |
| vuln = str(row["func_vuln"] or "") |
| fixed = str(row["func_patch"] or "") |
| vuln_lines = {line.strip() for line in vuln.splitlines() if line.strip()} |
| fixed_lines = {line.strip() for line in fixed.splitlines() if line.strip()} |
| line_union = vuln_lines | fixed_lines |
| line_jaccards.append(len(vuln_lines & fixed_lines) / len(line_union) if line_union else 1.0) |
| line_deltas.append(len(vuln_lines ^ fixed_lines)) |
| vuln_tokens = set(token_re.findall(vuln[:200_000])) |
| fixed_tokens = set(token_re.findall(fixed[:200_000])) |
| token_union = vuln_tokens | fixed_tokens |
| token_jaccards.append(len(vuln_tokens & fixed_tokens) / len(token_union) |
| if token_union else 1.0) |
| exact_equal += int(vuln == fixed) |
| nonempty_both += int(bool(vuln.strip() and fixed.strip())) |
|
|
| n_pairs = len(pair_ids) |
| if len(train) != 2 * n_pairs: |
| |
| raise ValueError(f"expected {2*n_pairs} rows for {n_pairs} pair IDs, found {len(train)}") |
|
|
| def summary(values: list[float]) -> dict: |
| ordered = sorted(values) |
| return {"mean": statistics.mean(values), "median": statistics.median(values), |
| "p10": ordered[int(0.10 * len(ordered))], |
| "p90": ordered[int(0.90 * len(ordered))]} |
|
|
| return { |
| "analysis": "chronological TRAIN pair structure for exploratory pairwise objective", |
| "split": str(train_split), |
| "test_touched": False, |
| "train_pair_ids": n_pairs, |
| "train_rows": len(train), |
| "pair_group_sizes": {str(k): int(v) for k, v in group_sizes.value_counts().items()}, |
| "label_pair_counts": {"/".join(map(str, k)): v for k, v in label_sets.items()}, |
| "invalid_pair_count": len(bad_pairs), |
| "invalid_pair_examples": bad_pairs[:10], |
| "code_pair_similarity": { |
| "exact_function_equal_count": exact_equal, |
| "both_code_nonempty_fraction": nonempty_both / n_pairs, |
| "normalized_line_set_jaccard": summary(line_jaccards), |
| "normalized_token_set_jaccard": summary(token_jaccards), |
| "line_set_symmetric_difference": { |
| "median": statistics.median(line_deltas), |
| "p90": sorted(line_deltas)[int(0.90 * len(line_deltas))], |
| "fraction_at_most_two": sum(x <= 2 for x in line_deltas) / n_pairs, |
| }, |
| }, |
| "method_caveat": ( |
| "Line/token set Jaccard ignores token order, repeated tokens, semantics, and patch correctness; " |
| "it only establishes structural relatedness, not that pairwise supervision will help." |
| ), |
| "input_sha256": {str(corpus_path): sha256(corpus_path), |
| str(train_split): sha256(train_split)}, |
| } |
|
|
|
|
| def main() -> None: |
| parser = argparse.ArgumentParser() |
| parser.add_argument("--corpus", type=Path, default=Path("data/processed/corpus.parquet")) |
| parser.add_argument("--train-split", type=Path, |
| default=Path("data/splits/chronological/train.json")) |
| parser.add_argument("--output", type=Path, |
| default=Path("results/phase6/pairwise_train_data_audit.json")) |
| args = parser.parse_args() |
| report = audit(args.corpus, args.train_split) |
| args.output.parent.mkdir(parents=True, exist_ok=True) |
| args.output.write_text(json.dumps(report, indent=2) + "\n") |
| print(json.dumps(report, indent=2)) |
|
|
|
|
| if __name__ == "__main__": |
| main() |
|
|
