Download scripts/build_ood_set.py from Dhruv1000/tracegrpo: direct link, hf CLI and curl.
- Browser
- Download file 13.7 kB
-
https://huggingface.co/Dhruv1000/tracegrpo/resolve/main/scripts/build_ood_set.py
- Command line
-
hf download hf://Dhruv1000/tracegrpo/scripts/build_ood_set.py
-
curl -L -o build_ood_set.py https://huggingface.co/Dhruv1000/tracegrpo/resolve/main/scripts/build_ood_set.py
13.7 kB
| """Phase 1 §3.1.4 CLI: mines the post-cutoff OOD CVE set. | |
| Queries NVD for CVEs published on/after --since, extracts GitHub commit | |
| references, fetches each commit's changed C/C++ files via the GitHub API, | |
| isolates single-function vuln/patch pairs, and (optionally) runs the | |
| contamination screen (prompts the backbone model with the CVE id and logs — | |
| never auto-drops — responses that look memorized). | |
| **Deviation from the literal spec text, documented here and in DECISIONS.md**: | |
| spec §3.1.4 says "clone repo at parent commit -> extract changed function(s)". | |
| We instead fetch just the two file versions (at the commit's parent SHA and | |
| at the commit SHA) via the GitHub Contents API. Same end result — isolated | |
| before/after function text — without a full repo clone, which matters given | |
| this machine's tight disk budget (tracegrpo/ENVIRONMENT.md) and because nothing | |
| downstream in this script needs the rest of the repository. | |
| Resume-safe and cache-backed: every NVD/GitHub response is disk-cached | |
| (src/data/http_cache.py) keyed by its request, and every CVE already present | |
| in --log is skipped on restart — killing and re-running this script costs at | |
| most the one CVE in flight, never re-does finished work, and a killed run's | |
| partial progress (out/log files) is exactly what the next run resumes from. | |
| Requires GITHUB_TOKEN (same one used by fetch_commit_dates.py) and shares its | |
| 5000/hour authenticated quota — the two should not run with large concurrent | |
| volume without accounting for that; this script backs off automatically when | |
| GitHub reports near-zero remaining quota (src/data/http_cache.py). | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import base64 | |
| import json | |
| import os | |
| import time | |
| from datetime import datetime, timedelta, timezone | |
| from pathlib import Path | |
| import requests | |
| from src.data.http_cache import cached_get_ex | |
| from src.data.ood_mining import extract_github_commit_refs, extract_single_function_change, is_c_cpp_file | |
| NVD_API = "https://services.nvd.nist.gov/rest/json/cves/2.0" | |
| CACHE_ROOT = Path("data/raw/ood/api_cache") | |
| def _nvd_session() -> requests.Session: | |
| s = requests.Session() | |
| api_key = os.environ.get("NVD_API_KEY") | |
| if api_key: | |
| s.headers.update({"apiKey": api_key}) | |
| return s | |
| def _github_session() -> requests.Session: | |
| token = os.environ.get("GITHUB_TOKEN") | |
| if not token: | |
| raise SystemExit("GITHUB_TOKEN env var required") | |
| s = requests.Session() | |
| s.headers.update({"Authorization": f"token {token}", "Accept": "application/vnd.github+json"}) | |
| return s | |
| def nvd_date_windows(since: str, window_days: int = 120): | |
| """NVD's CVE API 2.0 rejects any pubStartDate/pubEndDate range over 120 | |
| days (confirmed empirically: a 365-day range 404s with "Date range | |
| cannot exceed 120 days"), so we must page through time in <=120-day | |
| windows, not just via startIndex within one query.""" | |
| start = datetime.strptime(since, "%Y-%m-%d").replace(tzinfo=timezone.utc) | |
| now = datetime.now(timezone.utc) | |
| while start < now: | |
| end = min(start + timedelta(days=window_days - 1), now) | |
| yield start, end | |
| start = end + timedelta(seconds=1) | |
| def fetch_nvd_cves( | |
| session: requests.Session, cache_dir: Path, pub_start: datetime, pub_end: datetime, | |
| start_index: int, results_per_page: int = 200, | |
| ) -> tuple[dict | None, bool]: | |
| params = { | |
| "pubStartDate": pub_start.strftime("%Y-%m-%dT%H:%M:%S.000"), | |
| "pubEndDate": pub_end.strftime("%Y-%m-%dT%H:%M:%S.999"), | |
| "startIndex": start_index, | |
| "resultsPerPage": results_per_page, | |
| } | |
| cache_key = f"nvd:{params['pubStartDate']}:{params['pubEndDate']}:{start_index}:{results_per_page}" | |
| return cached_get_ex(session, NVD_API, cache_dir / "nvd", cache_key, params=params) | |
| def get_commit_detail(gh: requests.Session, cache_dir: Path, owner: str, repo: str, sha: str) -> tuple[dict | None, bool]: | |
| url = f"https://api.github.com/repos/{owner}/{repo}/commits/{sha}" | |
| return cached_get_ex(gh, url, cache_dir / "gh_commit", f"commit:{owner}/{repo}:{sha}") | |
| def get_file_content( | |
| gh: requests.Session, cache_dir: Path, owner: str, repo: str, path: str, ref: str | |
| ) -> tuple[str | None, bool]: | |
| url = f"https://api.github.com/repos/{owner}/{repo}/contents/{path}" | |
| data, cached = cached_get_ex(gh, url, cache_dir / "gh_content", f"content:{owner}/{repo}:{path}:{ref}", params={"ref": ref}) | |
| if data is None or data.get("encoding") != "base64": | |
| return None, cached | |
| try: | |
| return base64.b64decode(data["content"]).decode("utf-8", errors="replace"), cached | |
| except Exception: | |
| return None, cached | |
| def run_contamination_screen(model, cve_id: str) -> str: | |
| """spec §3.1.4: "prompt the backbone model with the CVE id and ask it to | |
| describe the bug; log responses; flag (do not auto-drop)". Caller passes | |
| an already-loaded vLLM `LLM` instance.""" | |
| from vllm import SamplingParams | |
| prompt = f"Describe the security vulnerability {cve_id} in detail, including the root cause and affected code." | |
| outputs = model.generate([prompt], SamplingParams(temperature=0.0, max_tokens=256)) | |
| return outputs[0].outputs[0].text | |
| def _load_processed_cve_ids(log_path: Path) -> set[str]: | |
| processed: set[str] = set() | |
| if log_path.exists(): | |
| with open(log_path) as f: | |
| for line in f: | |
| line = line.strip() | |
| if line: | |
| processed.add(json.loads(line)["cve_id"]) | |
| return processed | |
| def main(): | |
| parser = argparse.ArgumentParser() | |
| parser.add_argument("--since", default="2025-01-01") | |
| parser.add_argument("--target-count", type=int, default=200) | |
| parser.add_argument( | |
| "--max-cves-to-scan", type=int, default=20_000, | |
| help="Direct github.com/.../commit/<sha> references are rare in NVD's reference " | |
| "list (empirically ~0-a few per 50 CVEs scanned) — reaching --target-count " | |
| "pairs requires scanning many thousand CVEs.", | |
| ) | |
| parser.add_argument("--out", default="data/raw/ood/ood_pairs.jsonl") | |
| parser.add_argument("--log", default="data/raw/ood/collection_log.jsonl") | |
| parser.add_argument("--cache-dir", default=str(CACHE_ROOT)) | |
| parser.add_argument("--run-contamination-screen", action="store_true", default=False) | |
| parser.add_argument("--contamination-model", default="Qwen/Qwen3-4B") | |
| args = parser.parse_args() | |
| gh = _github_session() | |
| nvd = _nvd_session() | |
| cache_dir = Path(args.cache_dir) | |
| out_path = Path(args.out) | |
| log_path = Path(args.log) | |
| out_path.parent.mkdir(parents=True, exist_ok=True) | |
| already_processed = _load_processed_cve_ids(log_path) | |
| n_already_accepted = sum( | |
| 1 for line in (out_path.read_text().splitlines() if out_path.exists() else []) | |
| ) | |
| print(f"resuming: {len(already_processed)} CVEs already processed, " | |
| f"{n_already_accepted} pairs already accepted") | |
| contamination_model = None | |
| if args.run_contamination_screen: | |
| from vllm import LLM | |
| contamination_model = LLM(model=args.contamination_model, dtype="bfloat16", gpu_memory_utilization=0.85) | |
| nvd_has_key = bool(os.environ.get("NVD_API_KEY")) | |
| nvd_delay_s = 0.6 if nvd_has_key else 6.5 # unauthenticated: 5 req/30s; authenticated: 50 req/30s | |
| windows = list(nvd_date_windows(args.since)) | |
| window_idx = 0 | |
| start_index = 0 | |
| n_scanned = 0 | |
| n_accepted = n_already_accepted | |
| out_f = open(out_path, "a") | |
| log_f = open(log_path, "a") | |
| def log(cve_id: str, status: str, **extra): | |
| row = {"cve_id": cve_id, "status": status, **extra} | |
| log_f.write(json.dumps(row) + "\n") | |
| log_f.flush() | |
| try: | |
| while n_accepted < args.target_count and n_scanned < args.max_cves_to_scan and window_idx < len(windows): | |
| pub_start, pub_end = windows[window_idx] | |
| page, page_cached = fetch_nvd_cves(nvd, cache_dir, pub_start, pub_end, start_index) | |
| if page is None: | |
| print(f"WARNING: NVD fetch failed for window {pub_start}-{pub_end} @ {start_index}, skipping") | |
| window_idx += 1 | |
| start_index = 0 | |
| continue | |
| if not page_cached: | |
| time.sleep(nvd_delay_s) # only throttle on an actual network call, never on a cache hit | |
| vulns = page.get("vulnerabilities", []) | |
| if not vulns: | |
| window_idx += 1 | |
| start_index = 0 | |
| continue | |
| for item in vulns: | |
| cve = item["cve"] | |
| cve_id = cve["id"] | |
| if cve_id in already_processed: | |
| continue | |
| n_scanned += 1 | |
| refs = [r["url"] for r in cve.get("references", [])] | |
| commit_refs = extract_github_commit_refs(refs) | |
| if not commit_refs: | |
| log(cve_id, "rejected", reason="no_github_commit_ref") | |
| continue | |
| accepted_for_this_cve = False | |
| for ref in commit_refs: | |
| detail, detail_cached = get_commit_detail(gh, cache_dir, ref.owner, ref.repo, ref.sha) | |
| if not detail_cached: | |
| time.sleep(0.05) | |
| if detail is None: | |
| log(cve_id, "rejected", reason="commit_fetch_failed", | |
| repo=f"{ref.owner}/{ref.repo}", sha=ref.sha) | |
| continue | |
| parents = detail.get("parents", []) | |
| if not parents: | |
| log(cve_id, "rejected", reason="no_parent_commit", | |
| repo=f"{ref.owner}/{ref.repo}", sha=ref.sha) | |
| continue | |
| parent_sha = parents[0]["sha"] | |
| c_cpp_files = [f for f in detail.get("files", []) if is_c_cpp_file(f.get("filename", ""))] | |
| if not c_cpp_files: | |
| log(cve_id, "rejected", reason="no_c_cpp_files", | |
| repo=f"{ref.owner}/{ref.repo}", sha=ref.sha) | |
| continue | |
| if len(c_cpp_files) > 1: | |
| log(cve_id, "rejected", reason="multi_file_change", | |
| repo=f"{ref.owner}/{ref.repo}", sha=ref.sha, n_files=len(c_cpp_files)) | |
| continue | |
| path = c_cpp_files[0]["filename"] | |
| before, before_cached = get_file_content(gh, cache_dir, ref.owner, ref.repo, path, parent_sha) | |
| after, after_cached = get_file_content(gh, cache_dir, ref.owner, ref.repo, path, ref.sha) | |
| if not (before_cached and after_cached): | |
| time.sleep(0.05) | |
| if before is None or after is None: | |
| log(cve_id, "rejected", reason="file_content_fetch_failed", | |
| repo=f"{ref.owner}/{ref.repo}", sha=ref.sha, path=path) | |
| continue | |
| result = extract_single_function_change(before, after) | |
| if not result.accepted: | |
| log(cve_id, "rejected", reason=result.reason, | |
| repo=f"{ref.owner}/{ref.repo}", sha=ref.sha, path=path) | |
| continue | |
| row = { | |
| "sample_id": f"ood:{cve_id}:{ref.sha}:{result.func_name}", | |
| "source": "ood", | |
| "project": ref.repo, | |
| "cve": cve_id, | |
| "cwe": next( | |
| (d["description"][0]["value"] for d in cve.get("weaknesses", []) | |
| if d.get("description")), None | |
| ), | |
| "commit_id": ref.sha, | |
| "func_vuln": result.func_before, | |
| "func_patch": result.func_after, | |
| "commit_date": cve.get("published"), | |
| "pair_id": f"ood:{cve_id}:{ref.sha}", | |
| } | |
| if args.run_contamination_screen and contamination_model is not None: | |
| row["contamination_screen_response"] = run_contamination_screen(contamination_model, cve_id) | |
| out_f.write(json.dumps(row) + "\n") | |
| out_f.flush() | |
| n_accepted += 1 | |
| log(cve_id, "accepted", repo=f"{ref.owner}/{ref.repo}", sha=ref.sha) | |
| accepted_for_this_cve = True | |
| break # one accepted pair per CVE is enough | |
| if not accepted_for_this_cve: | |
| pass # already logged a rejection per attempted ref above | |
| already_processed.add(cve_id) | |
| if n_accepted >= args.target_count: | |
| break | |
| time.sleep(0.05) | |
| start_index += len(vulns) | |
| if page.get("totalResults", 0) <= start_index: | |
| window_idx += 1 | |
| start_index = 0 | |
| if n_scanned % 500 == 0 and n_scanned > 0: | |
| print(f"scanned {n_scanned} new CVEs this run, {n_accepted} pairs accepted total") | |
| finally: | |
| out_f.close() | |
| log_f.close() | |
| print(f"done: scanned {n_scanned} new CVEs this run, {n_accepted} pairs accepted total") | |
| if n_accepted < args.target_count: | |
| print(f"WARNING: target was {args.target_count} pairs, got {n_accepted}. " | |
| f"Re-run this script to continue (it resumes automatically); see {log_path} for rejection reasons.") | |
| if __name__ == "__main__": | |
| main() | |