File size: 3,766 Bytes
4554903 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 | """Verification runners — targeted tests and typecheck for the campus repo.
Used by the test_runner chip to verify generated code instead of
asserting it works. Both runners degrade honestly: when the repo (or
tsc/npm) isn't reachable from where Rivet runs, they return
`available=False` and the discipline gate caps confidence accordingly.
"""
import re
import tempfile
from dataclasses import dataclass
from pathlib import Path
from tools.guard import run_checked
@dataclass
class VerifyResult:
available: bool
passed: bool = False
command: str = ""
output: str = ""
class TestTools:
def __init__(self, repo_root: str = ""):
self.repo_root = repo_root
def repo_available(self) -> bool:
return bool(self.repo_root) and Path(self.repo_root).is_dir()
def typecheck(self, workspace: str = "server") -> VerifyResult:
"""tsc --noEmit over a workspace of the monorepo."""
if not self.repo_available():
return VerifyResult(available=False,
output="campus repo not on this machine")
cwd = str(Path(self.repo_root) / workspace)
result = run_checked(["npx", "tsc", "--noEmit"], cwd=cwd, timeout=300)
return VerifyResult(
available=True, passed=result.ok,
command=f"npx tsc --noEmit (in {workspace}/)",
output=(result.stdout + result.stderr)[-5000:],
)
def run_tests(self, test_file: str = "", workspace: str = "server") -> VerifyResult:
"""npm test, optionally targeted at one file."""
if not self.repo_available():
return VerifyResult(available=False,
output="campus repo not on this machine")
cwd = str(Path(self.repo_root) / workspace)
argv = ["npm", "test"]
if test_file:
argv += ["--", test_file]
result = run_checked(argv, cwd=cwd, timeout=600)
return VerifyResult(
available=True, passed=result.ok,
command=" ".join(argv) + f" (in {workspace}/)",
output=(result.stdout + result.stderr)[-5000:],
)
def syntax_check_snippet(self, code: str, lang: str = "ts") -> VerifyResult:
"""Standalone syntax check of a generated snippet via tsc.
Weaker than a repo typecheck (no project types) but catches
outright syntax errors even when the repo isn't present.
"""
tsc_probe = run_checked(["npx", "tsc", "--version"], timeout=30)
if not tsc_probe.ok:
return VerifyResult(available=False,
output="tsc not available on this machine")
suffix = ".ts" if lang in ("ts", "typescript") else ".js"
with tempfile.NamedTemporaryFile(
"w", suffix=suffix, delete=False) as tmp:
tmp.write(code)
tmp_path = tmp.name
try:
result = run_checked(
["npx", "tsc", "--noEmit", "--skipLibCheck", "--target",
"es2022", "--moduleResolution", "bundler", "--module",
"esnext", tmp_path],
timeout=60,
)
return VerifyResult(
available=True, passed=result.ok,
command="npx tsc --noEmit <snippet>",
output=(result.stdout + result.stderr)[-3000:],
)
finally:
Path(tmp_path).unlink(missing_ok=True)
def extract_code_blocks(markdown: str) -> list:
"""Pull fenced code blocks out of a draft answer for verification."""
blocks = []
for m in re.finditer(r"```(\w*)\n(.*?)```", markdown, re.DOTALL):
lang = (m.group(1) or "").lower()
blocks.append({"lang": lang, "code": m.group(2)})
return blocks
|