tracegrpo / scripts /build_ood_set.py
Dhruv1000's picture
Add files using upload-large-folder tool
867597e verified
Raw History Blame Contribute Delete
13.7 kB
"""Phase 1 §3.1.4 CLI: mines the post-cutoff OOD CVE set.
Queries NVD for CVEs published on/after --since, extracts GitHub commit
references, fetches each commit's changed C/C++ files via the GitHub API,
isolates single-function vuln/patch pairs, and (optionally) runs the
contamination screen (prompts the backbone model with the CVE id and logs —
never auto-drops — responses that look memorized).
**Deviation from the literal spec text, documented here and in DECISIONS.md**:
spec §3.1.4 says "clone repo at parent commit -> extract changed function(s)".
We instead fetch just the two file versions (at the commit's parent SHA and
at the commit SHA) via the GitHub Contents API. Same end result — isolated
before/after function text — without a full repo clone, which matters given
this machine's tight disk budget (tracegrpo/ENVIRONMENT.md) and because nothing
downstream in this script needs the rest of the repository.
Resume-safe and cache-backed: every NVD/GitHub response is disk-cached
(src/data/http_cache.py) keyed by its request, and every CVE already present
in --log is skipped on restart — killing and re-running this script costs at
most the one CVE in flight, never re-does finished work, and a killed run's
partial progress (out/log files) is exactly what the next run resumes from.
Requires GITHUB_TOKEN (same one used by fetch_commit_dates.py) and shares its
5000/hour authenticated quota — the two should not run with large concurrent
volume without accounting for that; this script backs off automatically when
GitHub reports near-zero remaining quota (src/data/http_cache.py).
"""
from __future__ import annotations
import argparse
import base64
import json
import os
import time
from datetime import datetime, timedelta, timezone
from pathlib import Path
import requests
from src.data.http_cache import cached_get_ex
from src.data.ood_mining import extract_github_commit_refs, extract_single_function_change, is_c_cpp_file
NVD_API = "https://services.nvd.nist.gov/rest/json/cves/2.0"
CACHE_ROOT = Path("data/raw/ood/api_cache")
def _nvd_session() -> requests.Session:
s = requests.Session()
api_key = os.environ.get("NVD_API_KEY")
if api_key:
s.headers.update({"apiKey": api_key})
return s
def _github_session() -> requests.Session:
token = os.environ.get("GITHUB_TOKEN")
if not token:
raise SystemExit("GITHUB_TOKEN env var required")
s = requests.Session()
s.headers.update({"Authorization": f"token {token}", "Accept": "application/vnd.github+json"})
return s
def nvd_date_windows(since: str, window_days: int = 120):
"""NVD's CVE API 2.0 rejects any pubStartDate/pubEndDate range over 120
days (confirmed empirically: a 365-day range 404s with "Date range
cannot exceed 120 days"), so we must page through time in <=120-day
windows, not just via startIndex within one query."""
start = datetime.strptime(since, "%Y-%m-%d").replace(tzinfo=timezone.utc)
now = datetime.now(timezone.utc)
while start < now:
end = min(start + timedelta(days=window_days - 1), now)
yield start, end
start = end + timedelta(seconds=1)
def fetch_nvd_cves(
session: requests.Session, cache_dir: Path, pub_start: datetime, pub_end: datetime,
start_index: int, results_per_page: int = 200,
) -> tuple[dict | None, bool]:
params = {
"pubStartDate": pub_start.strftime("%Y-%m-%dT%H:%M:%S.000"),
"pubEndDate": pub_end.strftime("%Y-%m-%dT%H:%M:%S.999"),
"startIndex": start_index,
"resultsPerPage": results_per_page,
}
cache_key = f"nvd:{params['pubStartDate']}:{params['pubEndDate']}:{start_index}:{results_per_page}"
return cached_get_ex(session, NVD_API, cache_dir / "nvd", cache_key, params=params)
def get_commit_detail(gh: requests.Session, cache_dir: Path, owner: str, repo: str, sha: str) -> tuple[dict | None, bool]:
url = f"https://api.github.com/repos/{owner}/{repo}/commits/{sha}"
return cached_get_ex(gh, url, cache_dir / "gh_commit", f"commit:{owner}/{repo}:{sha}")
def get_file_content(
gh: requests.Session, cache_dir: Path, owner: str, repo: str, path: str, ref: str
) -> tuple[str | None, bool]:
url = f"https://api.github.com/repos/{owner}/{repo}/contents/{path}"
data, cached = cached_get_ex(gh, url, cache_dir / "gh_content", f"content:{owner}/{repo}:{path}:{ref}", params={"ref": ref})
if data is None or data.get("encoding") != "base64":
return None, cached
try:
return base64.b64decode(data["content"]).decode("utf-8", errors="replace"), cached
except Exception:
return None, cached
def run_contamination_screen(model, cve_id: str) -> str:
"""spec §3.1.4: "prompt the backbone model with the CVE id and ask it to
describe the bug; log responses; flag (do not auto-drop)". Caller passes
an already-loaded vLLM `LLM` instance."""
from vllm import SamplingParams
prompt = f"Describe the security vulnerability {cve_id} in detail, including the root cause and affected code."
outputs = model.generate([prompt], SamplingParams(temperature=0.0, max_tokens=256))
return outputs[0].outputs[0].text
def _load_processed_cve_ids(log_path: Path) -> set[str]:
processed: set[str] = set()
if log_path.exists():
with open(log_path) as f:
for line in f:
line = line.strip()
if line:
processed.add(json.loads(line)["cve_id"])
return processed
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--since", default="2025-01-01")
parser.add_argument("--target-count", type=int, default=200)
parser.add_argument(
"--max-cves-to-scan", type=int, default=20_000,
help="Direct github.com/.../commit/<sha> references are rare in NVD's reference "
"list (empirically ~0-a few per 50 CVEs scanned) — reaching --target-count "
"pairs requires scanning many thousand CVEs.",
)
parser.add_argument("--out", default="data/raw/ood/ood_pairs.jsonl")
parser.add_argument("--log", default="data/raw/ood/collection_log.jsonl")
parser.add_argument("--cache-dir", default=str(CACHE_ROOT))
parser.add_argument("--run-contamination-screen", action="store_true", default=False)
parser.add_argument("--contamination-model", default="Qwen/Qwen3-4B")
args = parser.parse_args()
gh = _github_session()
nvd = _nvd_session()
cache_dir = Path(args.cache_dir)
out_path = Path(args.out)
log_path = Path(args.log)
out_path.parent.mkdir(parents=True, exist_ok=True)
already_processed = _load_processed_cve_ids(log_path)
n_already_accepted = sum(
1 for line in (out_path.read_text().splitlines() if out_path.exists() else [])
)
print(f"resuming: {len(already_processed)} CVEs already processed, "
f"{n_already_accepted} pairs already accepted")
contamination_model = None
if args.run_contamination_screen:
from vllm import LLM
contamination_model = LLM(model=args.contamination_model, dtype="bfloat16", gpu_memory_utilization=0.85)
nvd_has_key = bool(os.environ.get("NVD_API_KEY"))
nvd_delay_s = 0.6 if nvd_has_key else 6.5 # unauthenticated: 5 req/30s; authenticated: 50 req/30s
windows = list(nvd_date_windows(args.since))
window_idx = 0
start_index = 0
n_scanned = 0
n_accepted = n_already_accepted
out_f = open(out_path, "a")
log_f = open(log_path, "a")
def log(cve_id: str, status: str, **extra):
row = {"cve_id": cve_id, "status": status, **extra}
log_f.write(json.dumps(row) + "\n")
log_f.flush()
try:
while n_accepted < args.target_count and n_scanned < args.max_cves_to_scan and window_idx < len(windows):
pub_start, pub_end = windows[window_idx]
page, page_cached = fetch_nvd_cves(nvd, cache_dir, pub_start, pub_end, start_index)
if page is None:
print(f"WARNING: NVD fetch failed for window {pub_start}-{pub_end} @ {start_index}, skipping")
window_idx += 1
start_index = 0
continue
if not page_cached:
time.sleep(nvd_delay_s) # only throttle on an actual network call, never on a cache hit
vulns = page.get("vulnerabilities", [])
if not vulns:
window_idx += 1
start_index = 0
continue
for item in vulns:
cve = item["cve"]
cve_id = cve["id"]
if cve_id in already_processed:
continue
n_scanned += 1
refs = [r["url"] for r in cve.get("references", [])]
commit_refs = extract_github_commit_refs(refs)
if not commit_refs:
log(cve_id, "rejected", reason="no_github_commit_ref")
continue
accepted_for_this_cve = False
for ref in commit_refs:
detail, detail_cached = get_commit_detail(gh, cache_dir, ref.owner, ref.repo, ref.sha)
if not detail_cached:
time.sleep(0.05)
if detail is None:
log(cve_id, "rejected", reason="commit_fetch_failed",
repo=f"{ref.owner}/{ref.repo}", sha=ref.sha)
continue
parents = detail.get("parents", [])
if not parents:
log(cve_id, "rejected", reason="no_parent_commit",
repo=f"{ref.owner}/{ref.repo}", sha=ref.sha)
continue
parent_sha = parents[0]["sha"]
c_cpp_files = [f for f in detail.get("files", []) if is_c_cpp_file(f.get("filename", ""))]
if not c_cpp_files:
log(cve_id, "rejected", reason="no_c_cpp_files",
repo=f"{ref.owner}/{ref.repo}", sha=ref.sha)
continue
if len(c_cpp_files) > 1:
log(cve_id, "rejected", reason="multi_file_change",
repo=f"{ref.owner}/{ref.repo}", sha=ref.sha, n_files=len(c_cpp_files))
continue
path = c_cpp_files[0]["filename"]
before, before_cached = get_file_content(gh, cache_dir, ref.owner, ref.repo, path, parent_sha)
after, after_cached = get_file_content(gh, cache_dir, ref.owner, ref.repo, path, ref.sha)
if not (before_cached and after_cached):
time.sleep(0.05)
if before is None or after is None:
log(cve_id, "rejected", reason="file_content_fetch_failed",
repo=f"{ref.owner}/{ref.repo}", sha=ref.sha, path=path)
continue
result = extract_single_function_change(before, after)
if not result.accepted:
log(cve_id, "rejected", reason=result.reason,
repo=f"{ref.owner}/{ref.repo}", sha=ref.sha, path=path)
continue
row = {
"sample_id": f"ood:{cve_id}:{ref.sha}:{result.func_name}",
"source": "ood",
"project": ref.repo,
"cve": cve_id,
"cwe": next(
(d["description"][0]["value"] for d in cve.get("weaknesses", [])
if d.get("description")), None
),
"commit_id": ref.sha,
"func_vuln": result.func_before,
"func_patch": result.func_after,
"commit_date": cve.get("published"),
"pair_id": f"ood:{cve_id}:{ref.sha}",
}
if args.run_contamination_screen and contamination_model is not None:
row["contamination_screen_response"] = run_contamination_screen(contamination_model, cve_id)
out_f.write(json.dumps(row) + "\n")
out_f.flush()
n_accepted += 1
log(cve_id, "accepted", repo=f"{ref.owner}/{ref.repo}", sha=ref.sha)
accepted_for_this_cve = True
break # one accepted pair per CVE is enough
if not accepted_for_this_cve:
pass # already logged a rejection per attempted ref above
already_processed.add(cve_id)
if n_accepted >= args.target_count:
break
time.sleep(0.05)
start_index += len(vulns)
if page.get("totalResults", 0) <= start_index:
window_idx += 1
start_index = 0
if n_scanned % 500 == 0 and n_scanned > 0:
print(f"scanned {n_scanned} new CVEs this run, {n_accepted} pairs accepted total")
finally:
out_f.close()
log_f.close()
print(f"done: scanned {n_scanned} new CVEs this run, {n_accepted} pairs accepted total")
if n_accepted < args.target_count:
print(f"WARNING: target was {args.target_count} pairs, got {n_accepted}. "
f"Re-run this script to continue (it resumes automatically); see {log_path} for rejection reasons.")
if __name__ == "__main__":
main()