"""Phase 1 §3.1.4 CLI: mines the post-cutoff OOD CVE set. Queries NVD for CVEs published on/after --since, extracts GitHub commit references, fetches each commit's changed C/C++ files via the GitHub API, isolates single-function vuln/patch pairs, and (optionally) runs the contamination screen (prompts the backbone model with the CVE id and logs — never auto-drops — responses that look memorized). **Deviation from the literal spec text, documented here and in DECISIONS.md**: spec §3.1.4 says "clone repo at parent commit -> extract changed function(s)". We instead fetch just the two file versions (at the commit's parent SHA and at the commit SHA) via the GitHub Contents API. Same end result — isolated before/after function text — without a full repo clone, which matters given this machine's tight disk budget (tracegrpo/ENVIRONMENT.md) and because nothing downstream in this script needs the rest of the repository. Resume-safe and cache-backed: every NVD/GitHub response is disk-cached (src/data/http_cache.py) keyed by its request, and every CVE already present in --log is skipped on restart — killing and re-running this script costs at most the one CVE in flight, never re-does finished work, and a killed run's partial progress (out/log files) is exactly what the next run resumes from. Requires GITHUB_TOKEN (same one used by fetch_commit_dates.py) and shares its 5000/hour authenticated quota — the two should not run with large concurrent volume without accounting for that; this script backs off automatically when GitHub reports near-zero remaining quota (src/data/http_cache.py). """ from __future__ import annotations import argparse import base64 import json import os import time from datetime import datetime, timedelta, timezone from pathlib import Path import requests from src.data.http_cache import cached_get_ex from src.data.ood_mining import extract_github_commit_refs, extract_single_function_change, is_c_cpp_file NVD_API = "https://services.nvd.nist.gov/rest/json/cves/2.0" CACHE_ROOT = Path("data/raw/ood/api_cache") def _nvd_session() -> requests.Session: s = requests.Session() api_key = os.environ.get("NVD_API_KEY") if api_key: s.headers.update({"apiKey": api_key}) return s def _github_session() -> requests.Session: token = os.environ.get("GITHUB_TOKEN") if not token: raise SystemExit("GITHUB_TOKEN env var required") s = requests.Session() s.headers.update({"Authorization": f"token {token}", "Accept": "application/vnd.github+json"}) return s def nvd_date_windows(since: str, window_days: int = 120): """NVD's CVE API 2.0 rejects any pubStartDate/pubEndDate range over 120 days (confirmed empirically: a 365-day range 404s with "Date range cannot exceed 120 days"), so we must page through time in <=120-day windows, not just via startIndex within one query.""" start = datetime.strptime(since, "%Y-%m-%d").replace(tzinfo=timezone.utc) now = datetime.now(timezone.utc) while start < now: end = min(start + timedelta(days=window_days - 1), now) yield start, end start = end + timedelta(seconds=1) def fetch_nvd_cves( session: requests.Session, cache_dir: Path, pub_start: datetime, pub_end: datetime, start_index: int, results_per_page: int = 200, ) -> tuple[dict | None, bool]: params = { "pubStartDate": pub_start.strftime("%Y-%m-%dT%H:%M:%S.000"), "pubEndDate": pub_end.strftime("%Y-%m-%dT%H:%M:%S.999"), "startIndex": start_index, "resultsPerPage": results_per_page, } cache_key = f"nvd:{params['pubStartDate']}:{params['pubEndDate']}:{start_index}:{results_per_page}" return cached_get_ex(session, NVD_API, cache_dir / "nvd", cache_key, params=params) def get_commit_detail(gh: requests.Session, cache_dir: Path, owner: str, repo: str, sha: str) -> tuple[dict | None, bool]: url = f"https://api.github.com/repos/{owner}/{repo}/commits/{sha}" return cached_get_ex(gh, url, cache_dir / "gh_commit", f"commit:{owner}/{repo}:{sha}") def get_file_content( gh: requests.Session, cache_dir: Path, owner: str, repo: str, path: str, ref: str ) -> tuple[str | None, bool]: url = f"https://api.github.com/repos/{owner}/{repo}/contents/{path}" data, cached = cached_get_ex(gh, url, cache_dir / "gh_content", f"content:{owner}/{repo}:{path}:{ref}", params={"ref": ref}) if data is None or data.get("encoding") != "base64": return None, cached try: return base64.b64decode(data["content"]).decode("utf-8", errors="replace"), cached except Exception: return None, cached def run_contamination_screen(model, cve_id: str) -> str: """spec §3.1.4: "prompt the backbone model with the CVE id and ask it to describe the bug; log responses; flag (do not auto-drop)". Caller passes an already-loaded vLLM `LLM` instance.""" from vllm import SamplingParams prompt = f"Describe the security vulnerability {cve_id} in detail, including the root cause and affected code." outputs = model.generate([prompt], SamplingParams(temperature=0.0, max_tokens=256)) return outputs[0].outputs[0].text def _load_processed_cve_ids(log_path: Path) -> set[str]: processed: set[str] = set() if log_path.exists(): with open(log_path) as f: for line in f: line = line.strip() if line: processed.add(json.loads(line)["cve_id"]) return processed def main(): parser = argparse.ArgumentParser() parser.add_argument("--since", default="2025-01-01") parser.add_argument("--target-count", type=int, default=200) parser.add_argument( "--max-cves-to-scan", type=int, default=20_000, help="Direct github.com/.../commit/ references are rare in NVD's reference " "list (empirically ~0-a few per 50 CVEs scanned) — reaching --target-count " "pairs requires scanning many thousand CVEs.", ) parser.add_argument("--out", default="data/raw/ood/ood_pairs.jsonl") parser.add_argument("--log", default="data/raw/ood/collection_log.jsonl") parser.add_argument("--cache-dir", default=str(CACHE_ROOT)) parser.add_argument("--run-contamination-screen", action="store_true", default=False) parser.add_argument("--contamination-model", default="Qwen/Qwen3-4B") args = parser.parse_args() gh = _github_session() nvd = _nvd_session() cache_dir = Path(args.cache_dir) out_path = Path(args.out) log_path = Path(args.log) out_path.parent.mkdir(parents=True, exist_ok=True) already_processed = _load_processed_cve_ids(log_path) n_already_accepted = sum( 1 for line in (out_path.read_text().splitlines() if out_path.exists() else []) ) print(f"resuming: {len(already_processed)} CVEs already processed, " f"{n_already_accepted} pairs already accepted") contamination_model = None if args.run_contamination_screen: from vllm import LLM contamination_model = LLM(model=args.contamination_model, dtype="bfloat16", gpu_memory_utilization=0.85) nvd_has_key = bool(os.environ.get("NVD_API_KEY")) nvd_delay_s = 0.6 if nvd_has_key else 6.5 # unauthenticated: 5 req/30s; authenticated: 50 req/30s windows = list(nvd_date_windows(args.since)) window_idx = 0 start_index = 0 n_scanned = 0 n_accepted = n_already_accepted out_f = open(out_path, "a") log_f = open(log_path, "a") def log(cve_id: str, status: str, **extra): row = {"cve_id": cve_id, "status": status, **extra} log_f.write(json.dumps(row) + "\n") log_f.flush() try: while n_accepted < args.target_count and n_scanned < args.max_cves_to_scan and window_idx < len(windows): pub_start, pub_end = windows[window_idx] page, page_cached = fetch_nvd_cves(nvd, cache_dir, pub_start, pub_end, start_index) if page is None: print(f"WARNING: NVD fetch failed for window {pub_start}-{pub_end} @ {start_index}, skipping") window_idx += 1 start_index = 0 continue if not page_cached: time.sleep(nvd_delay_s) # only throttle on an actual network call, never on a cache hit vulns = page.get("vulnerabilities", []) if not vulns: window_idx += 1 start_index = 0 continue for item in vulns: cve = item["cve"] cve_id = cve["id"] if cve_id in already_processed: continue n_scanned += 1 refs = [r["url"] for r in cve.get("references", [])] commit_refs = extract_github_commit_refs(refs) if not commit_refs: log(cve_id, "rejected", reason="no_github_commit_ref") continue accepted_for_this_cve = False for ref in commit_refs: detail, detail_cached = get_commit_detail(gh, cache_dir, ref.owner, ref.repo, ref.sha) if not detail_cached: time.sleep(0.05) if detail is None: log(cve_id, "rejected", reason="commit_fetch_failed", repo=f"{ref.owner}/{ref.repo}", sha=ref.sha) continue parents = detail.get("parents", []) if not parents: log(cve_id, "rejected", reason="no_parent_commit", repo=f"{ref.owner}/{ref.repo}", sha=ref.sha) continue parent_sha = parents[0]["sha"] c_cpp_files = [f for f in detail.get("files", []) if is_c_cpp_file(f.get("filename", ""))] if not c_cpp_files: log(cve_id, "rejected", reason="no_c_cpp_files", repo=f"{ref.owner}/{ref.repo}", sha=ref.sha) continue if len(c_cpp_files) > 1: log(cve_id, "rejected", reason="multi_file_change", repo=f"{ref.owner}/{ref.repo}", sha=ref.sha, n_files=len(c_cpp_files)) continue path = c_cpp_files[0]["filename"] before, before_cached = get_file_content(gh, cache_dir, ref.owner, ref.repo, path, parent_sha) after, after_cached = get_file_content(gh, cache_dir, ref.owner, ref.repo, path, ref.sha) if not (before_cached and after_cached): time.sleep(0.05) if before is None or after is None: log(cve_id, "rejected", reason="file_content_fetch_failed", repo=f"{ref.owner}/{ref.repo}", sha=ref.sha, path=path) continue result = extract_single_function_change(before, after) if not result.accepted: log(cve_id, "rejected", reason=result.reason, repo=f"{ref.owner}/{ref.repo}", sha=ref.sha, path=path) continue row = { "sample_id": f"ood:{cve_id}:{ref.sha}:{result.func_name}", "source": "ood", "project": ref.repo, "cve": cve_id, "cwe": next( (d["description"][0]["value"] for d in cve.get("weaknesses", []) if d.get("description")), None ), "commit_id": ref.sha, "func_vuln": result.func_before, "func_patch": result.func_after, "commit_date": cve.get("published"), "pair_id": f"ood:{cve_id}:{ref.sha}", } if args.run_contamination_screen and contamination_model is not None: row["contamination_screen_response"] = run_contamination_screen(contamination_model, cve_id) out_f.write(json.dumps(row) + "\n") out_f.flush() n_accepted += 1 log(cve_id, "accepted", repo=f"{ref.owner}/{ref.repo}", sha=ref.sha) accepted_for_this_cve = True break # one accepted pair per CVE is enough if not accepted_for_this_cve: pass # already logged a rejection per attempted ref above already_processed.add(cve_id) if n_accepted >= args.target_count: break time.sleep(0.05) start_index += len(vulns) if page.get("totalResults", 0) <= start_index: window_idx += 1 start_index = 0 if n_scanned % 500 == 0 and n_scanned > 0: print(f"scanned {n_scanned} new CVEs this run, {n_accepted} pairs accepted total") finally: out_f.close() log_f.close() print(f"done: scanned {n_scanned} new CVEs this run, {n_accepted} pairs accepted total") if n_accepted < args.target_count: print(f"WARNING: target was {args.target_count} pairs, got {n_accepted}. " f"Re-run this script to continue (it resumes automatically); see {log_path} for rejection reasons.") if __name__ == "__main__": main()