Buckets:
| """Score the LLM answers in this bucket against the human-annotated records. | |
| Computes RDA, MLS and IRC (Levels 1, 2 and 3 of the scorer) for every record and | |
| model, prints each model's means over the 100 records (Table I of the SecPat | |
| paper), and compares every record with the bucket's own evaluation.json. | |
| The scorer is the package `secpat-scoring` | |
| (https://github.com/anonymousspottedtoad/secpat_evaluation_metrics). Install it with | |
| the package it depends on: | |
| pip install \ | |
| "secpat-interaction-order @ git+https://github.com/anonymousspottedtoad/secpat_interaction_recovery.git@v0.1.0" \ | |
| "secpat-scoring @ git+https://github.com/anonymousspottedtoad/secpat_evaluation_metrics.git@v0.23.0" | |
| IRC compares the interaction sequence that the LLM returned | |
| (experiment_outputs/<model>/<language>/<record>/recovery.json) with the | |
| human-verified one (human_annotated/interactions/), both narrowed to the roles that | |
| the LLM and the ground truth both detected. It therefore needs neither Joern nor the | |
| CPGs in cpg/. MLS reads the source files in codebase/. | |
| A record that a model was never run on (16 per model, whose one-shot prompt exceeded | |
| the input limit) has no recovery.json and counts as 0 on every metric, as in the paper. | |
| Usage: | |
| python scripts/score.py <bucket folder> [--models MODEL ...] [--out scores.json] | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| from collections import Counter | |
| from pathlib import Path | |
| from secpat_interaction_order import DetectionResult, Interaction | |
| from secpat_scoring import EvaluationRequest, evaluate | |
| from secpat_scoring.workspace import build_index, is_blocked | |
| MODELS = { | |
| "gpt-6.1-sol": "GPT-6.1", | |
| "muse-spark-1.3-contributor": "Muse Spark", | |
| "deepseek-v4-flash-0731": "DeepSeek-V4-Flash", | |
| "glm-5.3-flash": "GLM-5.3-Flash", | |
| } | |
| METRICS = {"level1": "RDA", "level2": "MLS", "level3": "IRC"} | |
| # The scorer reads the field names of the original release. | |
| ROW_NAMES = {"role": "patternRole", "symbol": "entity"} | |
| # How secpatrec names a codebase's language from the files the roles cite. | |
| EXTENSIONS = { | |
| ".py": "python", ".java": "java", ".kt": "kotlin", ".kts": "kotlin", | |
| ".js": "javascript", ".jsx": "javascript", ".mjs": "javascript", ".cjs": "javascript", | |
| ".ts": "typescript", ".tsx": "typescript", ".go": "go", ".c": "c", ".h": "c", | |
| ".cpp": "cpp", ".cc": "cpp", ".cxx": "cpp", ".hpp": "cpp", ".cs": "csharp", | |
| ".php": "php", ".rb": "ruby", | |
| } | |
| def read(path: Path): | |
| return json.loads(path.read_text(encoding="utf-8")) | |
| def rows(mappings: list[dict]) -> list[dict]: | |
| return [{ROW_NAMES.get(k, k): v for k, v in row.items()} for row in mappings] | |
| def language_of(codebase: Path, cited: list[str]) -> str: | |
| """The language of the cited files, else the most common one in the codebase.""" | |
| counts = Counter(EXTENSIONS[Path(f.strip()).suffix.lower()] | |
| for f in cited if Path((f or "").strip()).suffix.lower() in EXTENSIONS) | |
| if not counts: | |
| counts = Counter(EXTENSIONS[p.suffix.lower()] for p in codebase.rglob("*") | |
| if p.is_file() and p.suffix.lower() in EXTENSIONS | |
| and not is_blocked(p, codebase)) | |
| return counts.most_common(1)[0][0] if counts else "" | |
| def stored_sequence(sequence: dict | None, mappings: dict[str, dict]) -> DetectionResult: | |
| """A stored interaction sequence, kept to the interactions whose two roles are | |
| both in `mappings` (the roles both sides detected), in its stored order.""" | |
| sequence = sequence or {} | |
| if not sequence.get("available", False): | |
| return DetectionResult(reason=sequence.get("reason") or "no interaction sequence") | |
| kept, seen = [], set() | |
| for row in sorted(sequence.get("interactions") or [], key=lambda r: r.get("order", 0)): | |
| pair = (row.get("from", ""), row.get("to", "")) | |
| if pair in seen or pair[0] not in mappings or pair[1] not in mappings: | |
| continue | |
| seen.add(pair) | |
| kept.append(Interaction(source=pair[0], target=pair[1], file=row.get("file", ""), | |
| line=int(row.get("line") or 0), | |
| mechanism=row.get("mechanism") or "stored")) | |
| return DetectionResult(interactions=kept, entry_role=kept[0].source if kept else "", | |
| entry_rule="stored") | |
| def score(bucket: Path, run: dict, model: str, indexes: dict) -> dict[str, float] | None: | |
| """The record's three scores for this model, or None if the model never ran on it.""" | |
| language, slug = run["language"], f"{run['pattern']}__{run['repo']}" | |
| answer = bucket / "experiment_outputs" / model / language / slug / "recovery.json" | |
| if not answer.is_file(): | |
| return None | |
| answer = read(answer) | |
| annotated = bucket / "human_annotated" | |
| truth_rows = rows(read(annotated / "mappings" / language / f"{slug}.json")["role_mappings"]) | |
| truth_sequence = read(annotated / "interactions" / language / f"{slug}.json")["interaction_sequence"] | |
| answer_rows = rows(answer.get("role_mappings", [])) | |
| pattern = read(bucket / "security_patterns" / run["pattern"] / "04_pattern.json") | |
| codebase = bucket / "codebase" / language / run["repo"] | |
| lang = language_of(codebase, [r.get("file", "") for r in truth_rows + answer_rows]) | |
| if (codebase, lang) not in indexes: | |
| indexes[(codebase, lang)] = build_index(codebase, lang) | |
| struct = indexes[(codebase, lang)] | |
| if not struct: | |
| raise SystemExit(f"{slug}: the source index of {codebase} is empty; check that " | |
| "secpat-scoring and its tree-sitter grammars installed.") | |
| answer_ids = {id(row) for row in answer_rows} | |
| def sequence_for(mappings, behaviour): | |
| # The scorer asks once for each side, with that side's own row objects. | |
| if mappings and all(id(row) in answer_ids for row in mappings.values()): | |
| return stored_sequence(answer.get("interaction_sequence"), mappings) | |
| return stored_sequence(truth_sequence, mappings) | |
| report = evaluate(EvaluationRequest( | |
| ground_truth_rows=truth_rows, recovery_result_rows=answer_rows, | |
| pattern=pattern, struct=struct, detect_order=sequence_for, | |
| )).to_dict() | |
| return {m["key"]: (float(m["score"]) if m["available"] else 0.0) | |
| for m in report["metrics"] if m["key"] in METRICS} | |
| def main() -> None: | |
| parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) | |
| parser.add_argument("bucket", type=Path, help="the downloaded human_verified_bucket folder") | |
| parser.add_argument("--models", nargs="+", choices=list(MODELS), default=list(MODELS)) | |
| parser.add_argument("--out", type=Path, default=Path("scores.json"), | |
| help="every record's scores, as JSON (default: scores.json)") | |
| args = parser.parse_args() | |
| runs = read(args.bucket / "manifest.json")["runs"] | |
| indexes: dict = {} | |
| every: dict = {} | |
| table = [] | |
| for model in args.models: | |
| stored = {} | |
| for language in ("java", "python"): | |
| for record in read(args.bucket / "experiment_outputs" / model / language / "evaluation.json"): | |
| stored[record["slug"]] = {m["key"]: float(m["score"] or 0.0) | |
| for m in record["metrics"] if m["key"] in METRICS} | |
| print(f"Scoring {MODELS[model]} on {len(runs)} records", flush=True) | |
| scores, not_run = {}, 0 | |
| for run in runs: | |
| slug = f"{run['pattern']}__{run['repo']}" | |
| result = score(args.bucket, run, model, indexes) | |
| if result is None: | |
| not_run += 1 | |
| result = {key: 0.0 for key in METRICS} | |
| scores[slug] = result | |
| means = {key: sum(s[key] for s in scores.values()) / len(scores) for key in METRICS} | |
| # The scorer rounds to four decimals, the bucket's file sometimes not. | |
| same = sum(1 for slug, s in scores.items() | |
| if all(abs(s[k] - stored.get(slug, {}).get(k, -1)) < 1e-4 for k in METRICS)) | |
| table.append((MODELS[model], means, len(scores), not_run, same)) | |
| every[model] = scores | |
| args.out.write_text(json.dumps(every, indent=1) + "\n", encoding="utf-8") | |
| print(f"\n{'Model':<20}{'RDA':>7}{'MLS':>7}{'IRC':>7}{'records':>9}{'not run':>9}" | |
| f"{'same as evaluation.json':>25}") | |
| for label, means, n, not_run, same in table: | |
| print(f"{label:<20}{means['level1']:>7.3f}{means['level2']:>7.3f}{means['level3']:>7.3f}" | |
| f"{n:>9}{not_run:>9}{f'{same}/{n}':>25}") | |
| print(f"\nEvery record's scores: {args.out}") | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 8.65 kB
- Xet hash:
- cb9a4b4520f0df17a1e1cc86527432b8dbe46b7ce8a4db21842a76f25148cbef
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.