dev/constraints/engine.properties.test.js

/**
 * Property-based tests for the constraint engine.
 * =============================================================================
 * WHY PROPERTIES RATHER THAN EXAMPLES
 *
 * Our example tests check that the engine behaves correctly on the situations
 * we thought of. That is exactly the set of situations least likely to contain
 * a bug, because we thought of them.
 *
 * These tests instead state properties that must hold for EVERY rulebook and
 * EVERY context, and let fast-check search for a counterexample. When it finds
 * one it shrinks it to the smallest failing case and prints a seed, so the
 * failure is reproducible rather than a one-off.
 *
 * The important one is SAFETY: a COMPLETE signal must imply that every rule
 * was actually evaluated. That is a statement about the whole space of
 * rulebooks an operator could write, not about the handful we wrote.
 *
 *   node --test dev/constraints/engine.properties.test.js
 * =============================================================================
 */

'use strict';

const test = require('node:test');
const assert = require('node:assert');
const fc = require('fast-check');
const { evaluate, rollup, STATE, SIGNAL, R } = require('./engine.js');

const RUNS = Number(process.env.PROPERTY_RUNS || 500);

// ---------------------------------------------------------------------------
// Generators — arbitrary rulebooks, including ones we would never write.
// ---------------------------------------------------------------------------

/** A rule whose outcome we dictate, so the property can reason about it. */
const outcomeArb = fc.constantFrom('pass', 'fail', 'unknown', 'na', 'throw');

const ruleArb = fc.record({
  outcome: outcomeArb,
  waivable: fc.boolean(),
  applies: fc.constantFrom(true, false, null),   // null = cannot tell
});

function buildRulebook(specs) {
  return {
    id: 'generated',
    domain: 'property-test',
    rules: specs.map((s, i) => ({
      id: `P-${String(i).padStart(2, '0')}`,
      title: `generated rule ${i}`,
      waivable: s.waivable,
      applies: () => s.applies,
      evaluate: () => {
        switch (s.outcome) {
          case 'pass': return R.pass('generated pass');
          case 'fail': return R.fail('generated failure');
          case 'unknown': return R.unknown('generated unknown');
          case 'na': return R.na('generated not applicable');
          // A rule that throws is the case real rulebooks hit in production
          // and the case authors never write a test for.
          case 'throw': throw new Error('rule blew up');
          default: return R.unknown('unreachable');
        }
      },
    })),
  };
}

const rulebookArb = fc.array(ruleArb, { minLength: 1, maxLength: 8 }).map(buildRulebook);

// ---------------------------------------------------------------------------
// THE SAFETY PROPERTY — the one the whole project rests on.
// ---------------------------------------------------------------------------

test('SAFETY: COMPLETE implies every applicable rule was actually evaluated', () => {
  fc.assert(
    fc.property(rulebookArb, (rulebook) => {
      const rep = evaluate({ rulebook, context: {} });
      if (rep.signal !== SIGNAL.COMPLETE) return true;
      // Not one rule may be UNEVALUATED. "We didn't check" can never
      // contribute to "everything is fine".
      return rep.rules.every((r) => r.state !== STATE.UNEVALUATED);
    }),
    { numRuns: RUNS },
  );
});

test('SAFETY: a rule that throws never becomes SATISFIED', () => {
  fc.assert(
    fc.property(fc.array(ruleArb, { minLength: 1, maxLength: 6 }), (specs) => {
      const rep = evaluate({ rulebook: buildRulebook(specs), context: {} });
      return rep.rules.every((r, i) =>
        specs[i].outcome !== 'throw' || r.state !== STATE.SATISFIED);
    }),
    { numRuns: RUNS },
  );
});

test('SAFETY: any UNEVALUATED rule forces UNRESOLVED or BLOCKED', () => {
  fc.assert(
    fc.property(rulebookArb, (rulebook) => {
      const rep = evaluate({ rulebook, context: {} });
      const anyUnknown = rep.rules.some((r) => r.state === STATE.UNEVALUATED);
      if (!anyUnknown) return true;
      return rep.signal === SIGNAL.UNRESOLVED || rep.signal === SIGNAL.BLOCKED;
    }),
    { numRuns: RUNS },
  );
});

// ---------------------------------------------------------------------------
// WAIVER SOUNDNESS — you cannot waive what you never measured.
// ---------------------------------------------------------------------------

test('WAIVERS: waiving an unmeasured or non-negotiable rule never authorises', () => {
  fc.assert(
    fc.property(rulebookArb, (rulebook) => {
      // Waive absolutely everything — the most hostile possible operator.
      const waivers = rulebook.rules.map((r) => ({
        rule_id: r.id, by: 'property-test', reason: 'blanket waiver',
      }));
      const rep = evaluate({ rulebook, context: {}, waivers });
      for (const r of rep.rules) {
        if (r.state !== STATE.WAIVED) continue;
        // Only a genuinely VIOLATED, waivable rule may end up WAIVED.
        const spec = rulebook.rules.find((x) => x.id === r.id);
        if (spec.waivable === false) return false;
      }
      // And a blanket waiver can never turn an unknown into an authorisation.
      const stillUnknown = rep.rules.some((r) => r.state === STATE.UNEVALUATED);
      return !(stillUnknown && rep.authorised);
    }),
    { numRuns: RUNS },
  );
});

// ---------------------------------------------------------------------------
// DETERMINISM & ORDER INDEPENDENCE — the lattice properties.
// ---------------------------------------------------------------------------

test('DETERMINISM: the same inputs always produce the same signal', () => {
  fc.assert(
    fc.property(rulebookArb, (rulebook) => {
      const a = evaluate({ rulebook, context: {}, now: 0 });
      const b = evaluate({ rulebook, context: {}, now: 0 });
      return a.signal === b.signal && a.progress === b.progress;
    }),
    { numRuns: RUNS },
  );
});

test('LATTICE: rule order does not change the signal', () => {
  fc.assert(
    fc.property(
      fc.array(ruleArb, { minLength: 2, maxLength: 7 }),
      fc.integer({ min: 0, max: 1000 }),
      (specs, seed) => {
        const a = evaluate({ rulebook: buildRulebook(specs), context: {}, now: 0 });
        // deterministic shuffle driven by the generated seed
        const shuffled = specs.map((s, i) => [((i * 31 + seed) % specs.length), s])
          .sort((x, y) => x[0] - y[0]).map((p) => p[1]);
        const b = evaluate({ rulebook: buildRulebook(shuffled), context: {}, now: 0 });
        return a.signal === b.signal;
      },
    ),
    { numRuns: RUNS },
  );
});

test('LATTICE: rollup is order-independent and takes the worst state', () => {
  fc.assert(
    fc.property(fc.array(rulebookArb, { minLength: 1, maxLength: 5 }), (books) => {
      const reports = books.map((rulebook) => evaluate({ rulebook, context: {}, now: 0 }));
      const a = rollup(reports);
      const b = rollup([...reports].reverse());
      if (a.signal !== b.signal) return false;
      // the aggregate can never be better than the worst member
      const RANK = { COMPLETE: 0, PARTIAL: 1, UNRESOLVED: 2, BLOCKED: 3 };
      const worst = Math.max(...reports.map((r) => RANK[r.signal]));
      return RANK[a.signal] >= worst;
    }),
    { numRuns: Math.max(100, RUNS / 2) },
  );
});

// ---------------------------------------------------------------------------
// MONOTONICITY — evidence may only ever help, never hurt.
// ---------------------------------------------------------------------------

test('MONOTONICITY: turning an unknown into a pass never worsens the signal', () => {
  fc.assert(
    fc.property(
      fc.array(ruleArb, { minLength: 1, maxLength: 6 }),
      fc.nat(),
      (specs, pick) => {
        const RANK = { COMPLETE: 0, PARTIAL: 1, UNRESOLVED: 2, BLOCKED: 3 };
        const before = evaluate({ rulebook: buildRulebook(specs), context: {}, now: 0 });
        const i = specs.findIndex((s, n) => n >= pick % specs.length && s.outcome === 'unknown');
        if (i < 0) return true;
        const after = specs.map((s, n) => (n === i ? { ...s, outcome: 'pass' } : s));
        const rep = evaluate({ rulebook: buildRulebook(after), context: {}, now: 0 });
        return RANK[rep.signal] <= RANK[before.signal];
      },
    ),
    { numRuns: RUNS },
  );
});