Buckets:

download
raw
8.65 kB
"""Score the LLM answers in this bucket against the human-annotated records.
Computes RDA, MLS and IRC (Levels 1, 2 and 3 of the scorer) for every record and
model, prints each model's means over the 100 records (Table I of the SecPat
paper), and compares every record with the bucket's own evaluation.json.
The scorer is the package `secpat-scoring`
(https://github.com/anonymousspottedtoad/secpat_evaluation_metrics). Install it with
the package it depends on:
pip install \
"secpat-interaction-order @ git+https://github.com/anonymousspottedtoad/secpat_interaction_recovery.git@v0.1.0" \
"secpat-scoring @ git+https://github.com/anonymousspottedtoad/secpat_evaluation_metrics.git@v0.23.0"
IRC compares the interaction sequence that the LLM returned
(experiment_outputs/<model>/<language>/<record>/recovery.json) with the
human-verified one (human_annotated/interactions/), both narrowed to the roles that
the LLM and the ground truth both detected. It therefore needs neither Joern nor the
CPGs in cpg/. MLS reads the source files in codebase/.
A record that a model was never run on (16 per model, whose one-shot prompt exceeded
the input limit) has no recovery.json and counts as 0 on every metric, as in the paper.
Usage:
python scripts/score.py <bucket folder> [--models MODEL ...] [--out scores.json]
"""
from __future__ import annotations
import argparse
import json
from collections import Counter
from pathlib import Path
from secpat_interaction_order import DetectionResult, Interaction
from secpat_scoring import EvaluationRequest, evaluate
from secpat_scoring.workspace import build_index, is_blocked
MODELS = {
"gpt-6.1-sol": "GPT-6.1",
"muse-spark-1.3-contributor": "Muse Spark",
"deepseek-v4-flash-0731": "DeepSeek-V4-Flash",
"glm-5.3-flash": "GLM-5.3-Flash",
}
METRICS = {"level1": "RDA", "level2": "MLS", "level3": "IRC"}
# The scorer reads the field names of the original release.
ROW_NAMES = {"role": "patternRole", "symbol": "entity"}
# How secpatrec names a codebase's language from the files the roles cite.
EXTENSIONS = {
".py": "python", ".java": "java", ".kt": "kotlin", ".kts": "kotlin",
".js": "javascript", ".jsx": "javascript", ".mjs": "javascript", ".cjs": "javascript",
".ts": "typescript", ".tsx": "typescript", ".go": "go", ".c": "c", ".h": "c",
".cpp": "cpp", ".cc": "cpp", ".cxx": "cpp", ".hpp": "cpp", ".cs": "csharp",
".php": "php", ".rb": "ruby",
}
def read(path: Path):
return json.loads(path.read_text(encoding="utf-8"))
def rows(mappings: list[dict]) -> list[dict]:
return [{ROW_NAMES.get(k, k): v for k, v in row.items()} for row in mappings]
def language_of(codebase: Path, cited: list[str]) -> str:
"""The language of the cited files, else the most common one in the codebase."""
counts = Counter(EXTENSIONS[Path(f.strip()).suffix.lower()]
for f in cited if Path((f or "").strip()).suffix.lower() in EXTENSIONS)
if not counts:
counts = Counter(EXTENSIONS[p.suffix.lower()] for p in codebase.rglob("*")
if p.is_file() and p.suffix.lower() in EXTENSIONS
and not is_blocked(p, codebase))
return counts.most_common(1)[0][0] if counts else ""
def stored_sequence(sequence: dict | None, mappings: dict[str, dict]) -> DetectionResult:
"""A stored interaction sequence, kept to the interactions whose two roles are
both in `mappings` (the roles both sides detected), in its stored order."""
sequence = sequence or {}
if not sequence.get("available", False):
return DetectionResult(reason=sequence.get("reason") or "no interaction sequence")
kept, seen = [], set()
for row in sorted(sequence.get("interactions") or [], key=lambda r: r.get("order", 0)):
pair = (row.get("from", ""), row.get("to", ""))
if pair in seen or pair[0] not in mappings or pair[1] not in mappings:
continue
seen.add(pair)
kept.append(Interaction(source=pair[0], target=pair[1], file=row.get("file", ""),
line=int(row.get("line") or 0),
mechanism=row.get("mechanism") or "stored"))
return DetectionResult(interactions=kept, entry_role=kept[0].source if kept else "",
entry_rule="stored")
def score(bucket: Path, run: dict, model: str, indexes: dict) -> dict[str, float] | None:
"""The record's three scores for this model, or None if the model never ran on it."""
language, slug = run["language"], f"{run['pattern']}__{run['repo']}"
answer = bucket / "experiment_outputs" / model / language / slug / "recovery.json"
if not answer.is_file():
return None
answer = read(answer)
annotated = bucket / "human_annotated"
truth_rows = rows(read(annotated / "mappings" / language / f"{slug}.json")["role_mappings"])
truth_sequence = read(annotated / "interactions" / language / f"{slug}.json")["interaction_sequence"]
answer_rows = rows(answer.get("role_mappings", []))
pattern = read(bucket / "security_patterns" / run["pattern"] / "04_pattern.json")
codebase = bucket / "codebase" / language / run["repo"]
lang = language_of(codebase, [r.get("file", "") for r in truth_rows + answer_rows])
if (codebase, lang) not in indexes:
indexes[(codebase, lang)] = build_index(codebase, lang)
struct = indexes[(codebase, lang)]
if not struct:
raise SystemExit(f"{slug}: the source index of {codebase} is empty; check that "
"secpat-scoring and its tree-sitter grammars installed.")
answer_ids = {id(row) for row in answer_rows}
def sequence_for(mappings, behaviour):
# The scorer asks once for each side, with that side's own row objects.
if mappings and all(id(row) in answer_ids for row in mappings.values()):
return stored_sequence(answer.get("interaction_sequence"), mappings)
return stored_sequence(truth_sequence, mappings)
report = evaluate(EvaluationRequest(
ground_truth_rows=truth_rows, recovery_result_rows=answer_rows,
pattern=pattern, struct=struct, detect_order=sequence_for,
)).to_dict()
return {m["key"]: (float(m["score"]) if m["available"] else 0.0)
for m in report["metrics"] if m["key"] in METRICS}
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
parser.add_argument("bucket", type=Path, help="the downloaded human_verified_bucket folder")
parser.add_argument("--models", nargs="+", choices=list(MODELS), default=list(MODELS))
parser.add_argument("--out", type=Path, default=Path("scores.json"),
help="every record's scores, as JSON (default: scores.json)")
args = parser.parse_args()
runs = read(args.bucket / "manifest.json")["runs"]
indexes: dict = {}
every: dict = {}
table = []
for model in args.models:
stored = {}
for language in ("java", "python"):
for record in read(args.bucket / "experiment_outputs" / model / language / "evaluation.json"):
stored[record["slug"]] = {m["key"]: float(m["score"] or 0.0)
for m in record["metrics"] if m["key"] in METRICS}
print(f"Scoring {MODELS[model]} on {len(runs)} records", flush=True)
scores, not_run = {}, 0
for run in runs:
slug = f"{run['pattern']}__{run['repo']}"
result = score(args.bucket, run, model, indexes)
if result is None:
not_run += 1
result = {key: 0.0 for key in METRICS}
scores[slug] = result
means = {key: sum(s[key] for s in scores.values()) / len(scores) for key in METRICS}
# The scorer rounds to four decimals, the bucket's file sometimes not.
same = sum(1 for slug, s in scores.items()
if all(abs(s[k] - stored.get(slug, {}).get(k, -1)) < 1e-4 for k in METRICS))
table.append((MODELS[model], means, len(scores), not_run, same))
every[model] = scores
args.out.write_text(json.dumps(every, indent=1) + "\n", encoding="utf-8")
print(f"\n{'Model':<20}{'RDA':>7}{'MLS':>7}{'IRC':>7}{'records':>9}{'not run':>9}"
f"{'same as evaluation.json':>25}")
for label, means, n, not_run, same in table:
print(f"{label:<20}{means['level1']:>7.3f}{means['level2']:>7.3f}{means['level3']:>7.3f}"
f"{n:>9}{not_run:>9}{f'{same}/{n}':>25}")
print(f"\nEvery record's scores: {args.out}")
if __name__ == "__main__":
main()

Xet Storage Details

Size:
8.65 kB
·
Xet hash:
cb9a4b4520f0df17a1e1cc86527432b8dbe46b7ce8a4db21842a76f25148cbef

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.