File size: 11,554 Bytes
b01bf09 0ae18df b01bf09 0ae18df b01bf09 b1e674a 0ae18df b1e674a b01bf09 0ae18df b01bf09 0ae18df 617604d 0ae18df b01bf09 b1e674a 0ae18df b1e674a 617604d b01bf09 617604d b01bf09 0ae18df | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 | using System.Text;
using System.Text.Json;
using FindAJev.Bench;
// findajev list | run <id> [--threads N] [--warmup N] [--repeats N] [--limit N] | rank | graph
var root = Environment.GetEnvironmentVariable("FINDAJEV_ROOT") ?? Directory.GetCurrentDirectory();
var registry = JsonSerializer.Deserialize<List<ModelSpec>>(File.ReadAllText(Path.Combine(root, "models.json")))!;
var argv = args.ToList();
string Opt(string name, string def) { var i = argv.IndexOf(name); return i >= 0 && i + 1 < argv.Count ? argv[i + 1] : def; }
try
{
return Dispatch();
}
catch (ArgumentException e) { Console.Error.WriteLine("error: " + e.Message); return 2; }
catch (FormatException e) { Console.Error.WriteLine("error: bad number in arguments (" + e.Message + ")"); return 2; }
int Dispatch()
{
switch (argv.FirstOrDefault())
{
case "list":
foreach (var m in registry)
Console.WriteLine($"{m.Id,-24} {m.Family,-7} {m.Precision,-5} {(File.Exists(Path.Combine(root, m.Onnx)) ? "fetched" : "missing")}");
return 0;
case "run":
{
var id = argv.ElementAtOrDefault(1) ?? throw new ArgumentException("run <model-id>");
var spec = registry.FirstOrDefault(m => m.Id == id) ?? throw new ArgumentException($"unknown model {id}");
var threads = int.Parse(Opt("--threads", Environment.ProcessorCount.ToString()));
var warmup = int.Parse(Opt("--warmup", "20"));
var repeats = int.Parse(Opt("--repeats", "1"));
var limit = int.Parse(Opt("--limit", "0"));
var suites = Opt("--suites", "") is { Length: > 0 } sv ? sv.Split(',').ToHashSet() : null; // suite ids from suites.json
var items = Data.Load(root, spec.Family, suites, limit);
var overrides = argv.Select((a, i) => (a, i)).Where(x => x.a == "--param" && x.i + 1 < argv.Count)
.Select(x => argv[x.i + 1].Split('=', 2)).ToDictionary(kv => kv[0], kv => long.Parse(kv[1]));
var cpus = int.Parse(Opt("--cpus", Environment.ProcessorCount.ToString()));
var policyDir = Path.Combine(root, "policies");
PolicyEngine? pe = argv.Contains("--no-policy") || !Directory.Exists(policyDir) ? null : new PolicyEngine(policyDir, overrides);
if (pe is not null && items.Any(i => i.Labels is null)) pe = null; // encoded data predates label strings: no policies possible
var machine = new RunMachine(spec, root, items, threads, warmup, repeats, pe, cpus);
var r = machine.Run();
// A run with a non-default suite selection or parameter override gets its own result file and is ranked separately.
var variant = string.Join(";", new[] { suites is null ? "" : "suites=" + string.Join(",", suites.OrderBy(x => x)) }
.Concat(overrides.OrderBy(kv => kv.Key).Select(kv => $"{kv.Key}={kv.Value}")).Where(x => x.Length > 0));
r.Variant = variant;
r.SuitesRun = (suites ?? Data.Registry(root).Select(x => x.Id).ToHashSet()).OrderBy(x => x).ToArray();
if (pe is not null) r.PolicyParams = pe.Params.ToDictionary(kv => kv.Key, kv => kv.Value);
var suffix = variant.Length == 0 ? "" : "." + Convert.ToHexString(System.Security.Cryptography.SHA1.HashData(System.Text.Encoding.UTF8.GetBytes(variant)))[..8].ToLowerInvariant();
var outDir = Path.Combine(root, "results");
Directory.CreateDirectory(outDir);
var resultPath = Path.Combine(outDir, $"{spec.Id}.t{threads}{suffix}.json");
File.WriteAllText(resultPath, JsonSerializer.Serialize(r, Json.Opts));
if (r.State == "Scored") File.WriteAllLines(resultPath[..^5] + ".preds.jsonl", machine.PredictionLines());
Console.WriteLine("RESULT_FILE " + Path.GetRelativePath(root, resultPath));
Console.WriteLine(r.State == "Scored"
? $"{r.Id} [{r.State}] acc={r.Accuracy:P1} p50={r.P50Ms:F1}ms p95={r.P95Ms:F1}ms {r.ItemsPerSec:F1}/s rss={r.PeakRssMb:F0}MB load={r.LoadSeconds:F1}s"
: $"{r.Id} [{r.State}] {r.Error}");
return r.State == "Scored" ? 0 : 1;
}
case "check": // pre-flight for the server, before any download: may this model be fetched and run here?
{
var id = argv.ElementAtOrDefault(1) ?? throw new ArgumentException("check <model-id>");
var spec = registry.FirstOrDefault(m => m.Id == id) ?? throw new ArgumentException($"unknown model {id}");
var pe = new PolicyEngine(Path.Combine(root, "policies"));
var threads = int.Parse(Opt("--threads", Environment.ProcessorCount.ToString()));
var cpus = int.Parse(Opt("--cpus", Environment.ProcessorCount.ToString()));
var ds = new[] { "FetchModel", "RunModel" }.Select(a => pe.AuthorizeRun(a, spec, threads, cpus)).ToList();
Console.WriteLine(JsonSerializer.Serialize(new { allow = ds.All(d => d.Allow), decisions = ds.Select(d => new { action = d.Action, allow = d.Allow, by = d.Reasons, error = d.Error }) }));
return ds.All(d => d.Allow) ? 0 : 3;
}
case "probe": // developer aid: run one raw native call from a JSON file: probe <authorize|partial|validate> <file>
{
var body = File.ReadAllText(argv[2]);
Console.WriteLine(argv[1] switch { "authorize" => CedarNative.Authorize(body), "partial" => CedarNative.AuthorizePartial(body), "validate" => CedarNative.ValidatePolicies(body), _ => "unknown call" });
return 0;
}
case "cedar-suite": // the Cedar checks: cedar-suite [--list] [--checks a,b] [--json]
{
if (argv.Contains("--list"))
{
Console.WriteLine(JsonSerializer.Serialize(CedarChecks.All, Json.Opts));
return 0;
}
var ids = Opt("--checks", "") is { Length: > 0 } cv ? cv.Split(',') : CedarChecks.All.Select(c => c.Id).ToArray();
var outcomes = ids.Select(id => CedarChecks.Run(id, root)).ToList();
if (argv.Contains("--json"))
Console.WriteLine(JsonSerializer.Serialize(new { cedar = CedarNative.SdkVersionString(), language = CedarNative.LangVersionString(), checks = outcomes }, Json.Opts));
else
foreach (var c in outcomes)
{
Console.WriteLine($"{(c.Passed ? "PASS" : "FAIL")} {c.Id,-16} {c.Summary} ({c.Seconds}s)");
foreach (var f in c.Findings.Take(8)) Console.WriteLine(" " + (f.Length > 220 ? f[..220] + "…" : f));
}
return outcomes.All(c => c.Passed || !c.Hard) ? 0 : 1;
}
case "cedar-test": // Cedar language conformance cases (policies/conformance/*.json)
{
var cats = Opt("--categories", "") is { Length: > 0 } cv ? cv.Split(',').ToHashSet() : null;
var res = Conformance.Run(root, cats);
foreach (var g in res.GroupBy(r => r.Category))
Console.WriteLine($"{g.Key,-14} {g.Count(r => r.Pass),3}/{g.Count()}");
foreach (var r in res.Where(r => !r.Pass)) Console.WriteLine($" FAIL [{r.Category}] {r.Name}: {r.Detail}");
Console.WriteLine($"cedar {CedarNative.SdkVersionString()} (language {CedarNative.LangVersionString()}): {res.Count(r => r.Pass)}/{res.Count} conformance cases passed");
return res.All(r => r.Pass) ? 0 : 1;
}
case "policy-test":
return PolicyCommands.Test(root);
case "policy-coverage":
return PolicyCommands.Coverage(root, argv.ElementAtOrDefault(1) ?? "julia");
case "promote-check": // promote-check <candidate-id> <champion-id> [--threads N] [--approved] [--principal Human|Learner]: may the candidate replace the champion? (logged to the ledger)
{
var cand = argv.ElementAtOrDefault(1) ?? throw new ArgumentException("promote-check <candidate-id> <champion-id>");
var champ = argv.ElementAtOrDefault(2) ?? throw new ArgumentException("promote-check <candidate-id> <champion-id>");
var th = Opt("--threads", "");
RunResult Load(string id)
{
var files = Directory.GetFiles(Path.Combine(root, "results"), $"{id}.t{(th.Length > 0 ? th : "*")}.json");
return files.Length > 0 ? JsonSerializer.Deserialize<RunResult>(File.ReadAllText(files.OrderBy(f => f).First()), Json.Opts)! : throw new ArgumentException($"no default-variant result for {id}");
}
var rc = Load(cand); var rh = Load(champ);
var held = Data.Registry(root).Where(x => x.HeldOut).Select(x => x.Id).ToHashSet();
var approved = argv.Contains("--approved");
var principal = Opt("--principal", "Human");
var ctx = Promotion.Context(rc, rh, approved, held);
var d = new PolicyEngine(Path.Combine(root, "policies")).AuthorizePromote(principal, cand, ctx);
var shown = ctx.ToDictionary(kv => kv.Key, kv => kv.Value.ToString());
Ledger.Append(root, "promotion", new { candidate = cand, champion = champ, principal, approved, allow = d.Allow, by = d.Reasons, error = d.Error, context = shown });
Console.WriteLine(JsonSerializer.Serialize(new { candidate = cand, champion = champ, principal, allow = d.Allow, by = d.Reasons, error = d.Error, context = shown }, Json.Opts));
return d.Allow ? 0 : 3;
}
case "curation-sim": // curation-sim [--budget PCT] [--annotator ACC]: replay recorded predictions through the admission policy (logged to the ledger)
{
var budget = double.Parse(Opt("--budget", "10"), System.Globalization.CultureInfo.InvariantCulture);
var acc = double.Parse(Opt("--annotator", "0.98"), System.Globalization.CultureInfo.InvariantCulture);
var (ex, skipped) = Curation.BuildExamples(Curation.LoadPreds(root));
if (ex.Count == 0) { Console.Error.WriteLine("no recorded predictions (results/<model>.t<N>.preds.jsonl); run a benchmark first"); return 2; }
Console.WriteLine($"{ex.Count} recorded tests ({skipped} skipped: models saw different options)");
Console.WriteLine($"{"scenario",-42} {"admit",6} {"human",6} {"self",6} {"noise%",7} {"yield%",7} {"caught%",8} leaks");
var summary = new List<object>(); var leaked = false;
foreach (var sc in LearningChecks.Scenarios(budget, acc))
{
var r = Curation.Simulate(root, ex, sc, out _);
var lk = r.LeakNoConsent + r.LeakNotOpen + r.LeakPii + r.LeakAttackSelf; if (sc.UseCedar && lk > 0) leaked = true;
Console.WriteLine($"{sc.Name,-42} {r.Admitted,6} {r.AdmittedHuman,6} {r.AdmittedSelf,6} {(r.Admitted == 0 ? 0 : 100.0 * r.WrongAdmitted / r.Admitted),7:F2} {100.0 * r.Admitted / r.Examples,7:F1} {100 * r.WrongErrorsCaught,8:F1} {lk}");
summary.Add(r);
}
Ledger.Append(root, "curation-sim", new { examples = ex.Count, budgetPct = budget, annotatorAccuracy = acc, scenarios = summary });
return leaked ? 1 : 0;
}
case "ledger-verify":
{
var (ok, n, problem) = Ledger.Verify(root);
Console.WriteLine(ok ? $"ledger OK: {n} entries, chain intact" : $"ledger BROKEN after {n} entries: {problem}");
return ok ? 0 : 1;
}
case "rank":
var md = Ranking.Render(Path.Combine(root, "results"));
File.WriteAllText(Path.Combine(root, "RANKING.md"), md);
Console.WriteLine(md);
return 0;
case "graph": // Graphviz DOT of the run lifecycle
var dummy = new RunMachine(registry[0], root, new(), 1, 0, 1);
Console.WriteLine(dummy.Dot());
return 0;
default:
Console.Error.WriteLine("usage: findajev list | run <id> [--threads N --warmup N --repeats N --limit N] | rank | graph | promote-check | curation-sim | ledger-verify");
return 2;
}
}
|