using System.Runtime.InteropServices; using System.Text.Json; using System.Text.Json.Nodes; using System.Text.RegularExpressions; using CedarDotNet; using CedarDotNet.Models; using CedarDotNet.Values; namespace FindAJev.Bench; public sealed record Pack(string[] Actions, Dictionary Heads, string? Suite = null, string[]? OptionAttrs = null, Dictionary? Refs = null) { public string SuiteName => Suite ?? "automation"; } public sealed record PolicyDecision(string Action, bool Allow, string[] Reasons, string? Error = null); /// /// Cedar policy engine for the harness: loads policies/*.cedar + schema, validates them against the schema with the native /// validator, and answers per-test and run-lifecycle authorization requests. Policy ids come from the @id annotation. /// public sealed class PolicyEngine { static readonly Regex IdRx = new("@id\\(\"([^\"]+)\"\\)", RegexOptions.Compiled); readonly Schema _schema; readonly string _schemaText; readonly Dictionary _test = new(); // id -> text, per-test policies readonly Dictionary _run = new(); // id -> text, run-lifecycle policies readonly Dictionary _oracle = new(); // id -> text, label-free oracle rules (Action::"Audit") readonly Dictionary _oracleByDomain = new(); readonly Dictionary _train = new(); // learning loop: training-data admission (Admit / Review) readonly Dictionary _promote = new(); // learning loop: candidate-model promotion (Promote) readonly List _ontology = new(); readonly Dictionary _byDomain = new(); public PolicyEngine Reordered() { var r = WithTestPolicies(_test.Reverse().ToDictionary(kv => kv.Key, kv => kv.Value)); return r; } public Dictionary Packs { get; } public IReadOnlyDictionary TestPolicies => _test; public int PolicyCountFor(string domain) => _byDomain[domain].StaticPolicies.Count; public IReadOnlyDictionary RunPolicies => _run; public IReadOnlyDictionary OraclePolicies => _oracle; public IReadOnlyDictionary TrainPolicies => _train; public IReadOnlyDictionary PromotePolicies => _promote; /// Effective policy parameters (params.json + overrides). public IReadOnlyDictionary Params { get; } static readonly Regex ParamRx = new("\\{\\{(\\w+)\\}\\}", RegexOptions.Compiled); public PolicyEngine(string dir, IReadOnlyDictionary? overrides = null) { var pp = new Dictionary(); var pfile = Path.Combine(dir, "params.json"); if (File.Exists(pfile)) foreach (var kv in JsonNode.Parse(File.ReadAllText(pfile))!.AsObject().Where(kv => !kv.Key.StartsWith("_"))) pp[kv.Key] = kv.Value!.GetValue(); foreach (var (k, v) in overrides ?? new Dictionary()) { if (!pp.ContainsKey(k)) throw new ArgumentException($"unknown policy parameter '{k}' (known: {string.Join(", ", pp.Keys)})"); pp[k] = v; } Params = pp; _schemaText = File.ReadAllText(Path.Combine(dir, "schema.cedarschema")); _schema = Schema.FromText(_schemaText); Packs = JsonSerializer.Deserialize(File.ReadAllText(Path.Combine(dir, "packs.json")))!["packs"]! .Deserialize>(new JsonSerializerOptions { PropertyNameCaseInsensitive = true })!; var ofile = Path.Combine(dir, "ontology.json"); if (File.Exists(ofile)) foreach (var e in JsonNode.Parse(File.ReadAllText(ofile))!["entities"]!.AsArray()) _ontology.Add(new Entity { Uid = EntityUid.Create(e!["type"]!.GetValue(), e["id"]!.GetValue()), Attrs = e["attrs"]!.AsObject().ToDictionary(kv => kv.Key, kv => kv.Value!.GetValueKind() == JsonValueKind.True ? (Value)true : kv.Value.GetValueKind() == JsonValueKind.False ? (Value)false : kv.Value.GetValueKind() == JsonValueKind.Number ? (Value)kv.Value.GetValue() : (Value)kv.Value.GetValue()), }); foreach (var file in Directory.GetFiles(dir, "*.cedar").OrderBy(f => f)) { var fname = Path.GetFileName(file); var target = fname == "run.cedar" ? _run : fname.StartsWith("oracle") ? _oracle : fname.StartsWith("training") ? _train : fname.StartsWith("promote") ? _promote : _test; var source = ParamRx.Replace(File.ReadAllText(file), m => pp.TryGetValue(m.Groups[1].Value, out var val) ? val.ToString() : throw new InvalidDataException($"{file}: unknown parameter {{{{{m.Groups[1].Value}}}}}")); foreach (var text in CedarUtilities.LoadPolicySet(source)) { var m = IdRx.Match(text); if (!m.Success) throw new InvalidDataException($"{file}: every policy needs an @id annotation: {text[..Math.Min(80, text.Length)]}"); if (!target.TryAdd(m.Groups[1].Value, text)) throw new InvalidDataException($"duplicate policy id {m.Groups[1].Value}"); } } foreach (var d in Packs.Keys) _byDomain[d] = new PolicySet { StaticPolicies = ForDomain(d) }; } /// A copy of this engine with different per-test policy texts (id -> text); used by mutation testing. public PolicyEngine WithTestPolicies(Dictionary replacement) { var copy = (PolicyEngine)MemberwiseClone(); copy._test.Clear(); foreach (var kv in replacement) copy._test[kv.Key] = kv.Value; copy._byDomain.Clear(); foreach (var d in Packs.Keys) copy._byDomain[d] = new PolicySet { StaticPolicies = copy.ForDomain(d) }; return copy; } /// Actions declared `in [AutoAct]` in the schema: the autonomous ones that guardrails target. public HashSet AutoActions() => Regex.Matches(_schemaText, @"action\s+(\w+)\s+in\s+\[AutoAct\]").Select(m => m.Groups[1].Value).ToHashSet(); /// Policies relevant to a domain: those naming it, plus those naming no domain at all (cross-domain guardrails). Dictionary ForDomain(string domain) => _test.Where(kv => kv.Value.Contains($"\"{domain}\"") || !Packs.Keys.Any(d => kv.Value.Contains($"\"{d}\""))) .ToDictionary(kv => kv.Key, kv => kv.Value); // ----------------------------------------------------------------------------------------------- validation [DllImport("cedar_dotnet_ffi", EntryPoint = "validate")] static extern IntPtr NativeValidate([MarshalAs(UnmanagedType.LPUTF8Str)] string call); [DllImport("cedar_dotnet_ffi", EntryPoint = "free_string")] static extern void NativeFree(IntPtr p); /// Validate every policy against the schema (strict mode). Returns human-readable problems; empty = valid. public List Validate() { var problems = new List(); foreach (var (name, set) in new[] { ("test", _test), ("run", _run), ("oracle", _oracle), ("training", _train), ("promote", _promote) }) { var call = new JsonObject { ["validationSettings"] = new JsonObject { ["mode"] = "strict" }, ["schema"] = _schemaText, ["policies"] = new JsonObject { ["staticPolicies"] = JsonSerializer.SerializeToNode(set) }, }; var ptr = NativeValidate(call.ToJsonString()); try { var res = JsonNode.Parse(Marshal.PtrToStringUTF8(ptr)!)!.AsObject(); if (res["type"]?.GetValue() != "success") problems.Add($"[{name}] {res["errors"]}"); else { foreach (var e in res["validationErrors"]!.AsArray()) problems.Add($"[{name}] {e}"); foreach (var w in res["validationWarnings"]!.AsArray()) problems.Add($"[{name}] warning: {w}"); } } finally { NativeFree(ptr); } } return problems; } // ----------------------------------------------------------------------------------------------- authorization static readonly EntityUid Session = EntityUid.Create("Session", "test"); static readonly EntityUid Runner = EntityUid.Create("Runner", "harness"); PolicyDecision Call(string action, EntityUid principal, Entity resource, Entity? principalEntity, Dictionary ctx, PolicySet policies) { var entities = new List { resource, principalEntity ?? new Entity { Uid = principal } }; var ans = CedarFunctions.IsAuthorized(new AuthorizationCall { Principal = principal, Action = EntityUid.Create("Action", action), Resource = resource.Uid, Context = ctx, Schema = _schema, ValidateRequest = true, Policies = policies, Entities = entities, }); return ans switch { AuthorizationAnswerSuccess ok => new PolicyDecision(action, ok.Response.Decision == Decision.Allow, ok.Response.Diagnostics.Reason.ToArray(), ok.Response.Diagnostics.Errors.Count > 0 ? string.Join("; ", ok.Response.Diagnostics.Errors.Select(e => $"{e.PolicyId}: {e.Error.Message}")) : null), AuthorizationAnswerFailure bad => new PolicyDecision(action, false, Array.Empty(), string.Join("; ", bad.Errors.Select(e => e.Message))), _ => new PolicyDecision(action, false, Array.Empty(), "unknown answer"), }; } /// Authorize one action for a test. ctx keys: "domain", "minConfidence" (long) plus the pack's head attributes. public PolicyDecision AuthorizeTest(string domain, string testId, string action, Dictionary ctx) { var item = new Entity { Uid = EntityUid.Create("Item", testId), Attrs = new Dictionary { ["domain"] = domain } }; return Call(action, Session, item, null, ctx, _byDomain[domain]); } /// /// Run the label-free oracle rules over a prediction's context. Allow = flagged as suspicious; Reasons = the rule ids that fired. /// Works for any domain: domains without a pack only get the generic rules. /// public PolicyDecision AuditTest(string domain, string testId, Dictionary ctx) { if (!_oracleByDomain.TryGetValue(domain, out var set)) { var known = Packs.Keys; set = _oracleByDomain[domain] = new PolicySet { StaticPolicies = _oracle.Where(kv => kv.Value.Contains($"\"{domain}\"") || !known.Any(d => kv.Value.Contains($"\"{d}\""))) .ToDictionary(kv => kv.Key, kv => kv.Value), }; } var item = new Entity { Uid = EntityUid.Create("Item", testId), Attrs = new Dictionary { ["domain"] = domain } }; var principal = new Entity { Uid = Session }; var ans = CedarFunctions.IsAuthorized(new AuthorizationCall { Principal = Session, Action = EntityUid.Create("Action", "Audit"), Resource = item.Uid, Context = ctx, Schema = _schema, ValidateRequest = true, Policies = set, Entities = new List { item, principal }.Concat(_ontology).ToList(), }); return ans switch { AuthorizationAnswerSuccess ok => new PolicyDecision("Audit", ok.Response.Decision == Decision.Allow, ok.Response.Diagnostics.Reason.ToArray(), ok.Response.Diagnostics.Errors.Count > 0 ? string.Join("; ", ok.Response.Diagnostics.Errors.Select(e => $"{e.PolicyId}: {e.Error.Message}")) : null), AuthorizationAnswerFailure bad => new PolicyDecision("Audit", false, Array.Empty(), string.Join("; ", bad.Errors.Select(e => e.Message))), _ => new PolicyDecision("Audit", false, Array.Empty(), "unknown answer"), }; } static readonly EntityUid Curator = EntityUid.Create("Curator", "nightly"); /// Learning loop: may this logged interaction be admitted to training (Admit) or shown to a human (Review)? ctx: see TrainCtx in the schema. public PolicyDecision AuthorizeTrain(string action, string domain, Dictionary ctx) { var example = new Entity { Uid = EntityUid.Create("Example", "e"), Attrs = new Dictionary { ["domain"] = domain } }; return Call(action, Curator, example, null, ctx, new PolicySet { StaticPolicies = _train }); } /// Learning loop: may this principal (Human or Learner) promote a candidate model? ctx: see PromoteCtx in the schema. public PolicyDecision AuthorizePromote(string principalType, string candidateId, Dictionary ctx) { var model = new Entity { Uid = EntityUid.Create("ModelVersion", candidateId), Attrs = new Dictionary { ["id"] = candidateId } }; return Call("Promote", EntityUid.Create(principalType, "p"), model, null, ctx, new PolicySet { StaticPolicies = _promote }); } /// The domain an oracle rule is written for (its text names the domain), or null for a generic rule. public string? OracleDomain(string ruleId) => _oracle.TryGetValue(ruleId, out var text) ? Packs.Keys.FirstOrDefault(d => text.Contains($"\"{d}\"")) : null; /// Authorize FetchModel / RunModel for the run lifecycle. public PolicyDecision AuthorizeRun(string action, ModelSpec m, int threads, int cpus) { var model = new Entity { Uid = EntityUid.Create("Model", m.Id), Attrs = new Dictionary { ["license"] = m.License, ["sizeMb"] = (long)m.SizeMb, ["publisher"] = m.Repo.Split('/')[0] }, }; var ctx = new Dictionary { ["threads"] = (long)threads, ["cpus"] = (long)cpus }; return Call(action, Runner, model, null, ctx, new PolicySet { StaticPolicies = _run }); } public static Dictionary Ctx(string domain, long minConfidence, long options, IEnumerable> attrs, long minMargin = 100) { var d = new Dictionary { ["domain"] = domain, ["minConfidence"] = minConfidence, ["options"] = options, ["minMargin"] = minMargin }; foreach (var (k, v) in attrs) d[k] = v; return d; } }