{ "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "metrics.schema.json", "x-contract-version": 1, "$defs": { "DeceptionBaseline": { "additionalProperties": false, "properties": { "heard": { "type": "integer" }, "voters": { "type": "integer" }, "followed": { "type": "integer" }, "vote_follow_rate": { "anyOf": [ { "type": "number" }, { "type": "null" } ] }, "vote_follow_ci": { "anyOf": [ { "maxItems": 2, "minItems": 2, "prefixItems": [ { "type": "number" }, { "type": "number" } ], "type": "array" }, { "type": "null" } ] } }, "required": [ "heard", "voters", "followed", "vote_follow_rate", "vote_follow_ci" ], "title": "DeceptionBaseline", "type": "object" }, "DeceptionBySource": { "additionalProperties": false, "properties": { "self_tag": { "additionalProperties": { "additionalProperties": { "$ref": "#/$defs/DeceptionCell" }, "type": "object" }, "type": "object" }, "jev": { "additionalProperties": { "additionalProperties": { "$ref": "#/$defs/DeceptionCell" }, "type": "object" }, "type": "object" } }, "required": [ "self_tag", "jev" ], "title": "DeceptionBySource", "type": "object" }, "DeceptionCell": { "additionalProperties": false, "properties": { "heard": { "type": "integer" }, "voters": { "type": "integer" }, "followed": { "type": "integer" }, "vote_follow_rate": { "anyOf": [ { "type": "number" }, { "type": "null" } ] }, "vote_follow_ci": { "anyOf": [ { "maxItems": 2, "minItems": 2, "prefixItems": [ { "type": "number" }, { "type": "number" } ], "type": "array" }, { "type": "null" } ] }, "comparisons": { "type": "integer" }, "moved": { "type": "integer" }, "belief_shift_rate": { "anyOf": [ { "type": "number" }, { "type": "null" } ] }, "belief_shift_ci": { "anyOf": [ { "maxItems": 2, "minItems": 2, "prefixItems": [ { "type": "number" }, { "type": "number" } ], "type": "array" }, { "type": "null" } ] }, "baseline_rate": { "anyOf": [ { "type": "number" }, { "type": "null" } ] }, "baseline_voters": { "type": "integer" } }, "required": [ "heard", "voters", "followed", "vote_follow_rate", "vote_follow_ci", "comparisons", "moved", "belief_shift_rate", "belief_shift_ci", "baseline_rate", "baseline_voters" ], "title": "DeceptionCell", "type": "object" }, "DeceptionMatrix": { "additionalProperties": false, "properties": { "exploratory": { "type": "boolean" }, "pairs": { "$ref": "#/$defs/DeceptionBySource" }, "baseline": { "additionalProperties": { "additionalProperties": { "$ref": "#/$defs/DeceptionBaseline" }, "type": "object" }, "propertyNames": { "enum": [ "role_claim", "seer_wolf", "seer_clear", "accusation" ] }, "type": "object" } }, "required": [ "exploratory", "pairs", "baseline" ], "title": "DeceptionMatrix", "type": "object" }, "Detection": { "additionalProperties": false, "properties": { "n": { "type": "integer" }, "brier": { "anyOf": [ { "type": "number" }, { "type": "null" } ] }, "auc": { "anyOf": [ { "type": "number" }, { "type": "null" } ] }, "by_day": { "patternProperties": { "^\\d+$": { "$ref": "#/$defs/DetectionDay" } }, "type": "object", "additionalProperties": false } }, "required": [ "n", "brier", "auc", "by_day" ], "title": "Detection", "type": "object" }, "DetectionDay": { "additionalProperties": false, "properties": { "n": { "type": "integer" }, "brier": { "anyOf": [ { "type": "number" }, { "type": "null" } ] }, "auc": { "anyOf": [ { "type": "number" }, { "type": "null" } ] } }, "required": [ "n", "brier", "auc" ], "title": "DetectionDay", "type": "object" }, "DoctorSkill": { "additionalProperties": false, "properties": { "models": { "additionalProperties": { "$ref": "#/$defs/DoctorStats" }, "type": "object" }, "all": { "$ref": "#/$defs/DoctorStats" } }, "required": [ "models", "all" ], "title": "DoctorSkill", "type": "object" }, "DoctorStats": { "additionalProperties": false, "properties": { "nights": { "type": "integer" }, "saves": { "type": "integer" }, "self_protects": { "type": "integer" }, "fallbacks": { "type": "integer" }, "save_rate": { "anyOf": [ { "type": "number" }, { "type": "null" } ] }, "save_ci": { "anyOf": [ { "maxItems": 2, "minItems": 2, "prefixItems": [ { "type": "number" }, { "type": "number" } ], "type": "array" }, { "type": "null" } ] }, "chance_rate": { "anyOf": [ { "type": "number" }, { "type": "null" } ] } }, "required": [ "nights", "saves", "self_protects", "fallbacks", "save_rate", "save_ci", "chance_rate" ], "title": "DoctorStats", "type": "object" }, "GroupStats": { "additionalProperties": false, "properties": { "games": { "type": "integer" }, "wins": { "type": "integer" }, "draws": { "type": "integer" }, "losses": { "type": "integer" }, "win_rate": { "anyOf": [ { "type": "number" }, { "type": "null" } ] }, "win_ci": { "anyOf": [ { "maxItems": 2, "minItems": 2, "prefixItems": [ { "type": "number" }, { "type": "number" } ], "type": "array" }, { "type": "null" } ] }, "statements": { "type": "integer" }, "lies": { "$ref": "#/$defs/LiesBySource" }, "detection": { "$ref": "#/$defs/Detection" }, "persuasion": { "$ref": "#/$defs/Persuasion" }, "survival_days": { "anyOf": [ { "type": "number" }, { "type": "null" } ] }, "night_targeted": { "anyOf": [ { "type": "number" }, { "type": "null" } ] }, "voted_out": { "anyOf": [ { "type": "number" }, { "type": "null" } ] }, "invalid_output_rate": { "anyOf": [ { "type": "number" }, { "type": "null" } ] }, "invalid_attempt_rate": { "anyOf": [ { "type": "number" }, { "type": "null" } ] }, "fallback_rate": { "anyOf": [ { "type": "number" }, { "type": "null" } ] }, "retry_success_rate": { "anyOf": [ { "type": "number" }, { "type": "null" } ] }, "provider_errors": { "type": "integer" }, "provider_error_rate": { "anyOf": [ { "type": "number" }, { "type": "null" } ] }, "tokens_per_game": { "anyOf": [ { "type": "number" }, { "type": "null" } ] }, "cost_usd_per_game": { "anyOf": [ { "type": "number" }, { "type": "null" } ] } }, "required": [ "games", "wins", "draws", "losses", "win_rate", "win_ci", "statements", "lies", "detection", "persuasion", "survival_days", "night_targeted", "voted_out", "invalid_output_rate", "invalid_attempt_rate", "fallback_rate", "retry_success_rate", "provider_errors", "provider_error_rate", "tokens_per_game", "cost_usd_per_game" ], "title": "GroupStats", "type": "object" }, "HumanFlow": { "additionalProperties": false, "description": "deception_matrix counts summed by direction, plus `caught`: next vote for the liar.", "properties": { "heard": { "type": "integer" }, "voters": { "type": "integer" }, "followed": { "type": "integer" }, "vote_follow_rate": { "anyOf": [ { "type": "number" }, { "type": "null" } ] }, "vote_follow_ci": { "anyOf": [ { "maxItems": 2, "minItems": 2, "prefixItems": [ { "type": "number" }, { "type": "number" } ], "type": "array" }, { "type": "null" } ] }, "caught": { "type": "integer" }, "comparisons": { "type": "integer" }, "moved": { "type": "integer" }, "belief_shift_rate": { "anyOf": [ { "type": "number" }, { "type": "null" } ] }, "belief_shift_ci": { "anyOf": [ { "maxItems": 2, "minItems": 2, "prefixItems": [ { "type": "number" }, { "type": "number" } ], "type": "array" }, { "type": "null" } ] }, "catch_rate": { "anyOf": [ { "type": "number" }, { "type": "null" } ] }, "catch_ci": { "anyOf": [ { "maxItems": 2, "minItems": 2, "prefixItems": [ { "type": "number" }, { "type": "number" } ], "type": "array" }, { "type": "null" } ] } }, "required": [ "heard", "voters", "followed", "vote_follow_rate", "vote_follow_ci", "caught", "comparisons", "moved", "belief_shift_rate", "belief_shift_ci", "catch_rate", "catch_ci" ], "title": "HumanFlow", "type": "object" }, "HumanFlows": { "additionalProperties": false, "properties": { "human_to_model": { "$ref": "#/$defs/HumanFlow" }, "model_to_human": { "$ref": "#/$defs/HumanFlow" }, "model_to_model": { "$ref": "#/$defs/HumanFlow" } }, "required": [ "human_to_model", "model_to_human", "model_to_model" ], "title": "HumanFlows", "type": "object" }, "HumanFlowsBySource": { "additionalProperties": false, "properties": { "self_tag": { "$ref": "#/$defs/HumanFlows" }, "jev": { "$ref": "#/$defs/HumanFlows" } }, "required": [ "self_tag", "jev" ], "title": "HumanFlowsBySource", "type": "object" }, "HumanSeats": { "additionalProperties": false, "properties": { "seats": { "type": "integer" }, "wins": { "type": "integer" }, "draws": { "type": "integer" }, "losses": { "type": "integer" }, "win_rate": { "anyOf": [ { "type": "number" }, { "type": "null" } ] }, "win_ci": { "anyOf": [ { "maxItems": 2, "minItems": 2, "prefixItems": [ { "type": "number" }, { "type": "number" } ], "type": "array" }, { "type": "null" } ] } }, "required": [ "seats", "wins", "draws", "losses", "win_rate", "win_ci" ], "title": "HumanSeats", "type": "object" }, "Humans": { "additionalProperties": false, "description": "Humans vs models (#305): visitor seats against the AI seats. Counts only, no handles.", "properties": { "exploratory": { "type": "boolean" }, "games": { "type": "integer" }, "seats": { "type": "integer" }, "by_role": { "additionalProperties": { "$ref": "#/$defs/HumanSeats" }, "propertyNames": { "enum": [ "all", "wolf", "village", "seer", "doctor", "villager" ] }, "type": "object" }, "models_by_role": { "additionalProperties": { "$ref": "#/$defs/HumanSeats" }, "propertyNames": { "enum": [ "all", "wolf", "village", "seer", "doctor", "villager" ] }, "type": "object" }, "lies": { "$ref": "#/$defs/HumanFlowsBySource" } }, "required": [ "exploratory", "games", "seats", "by_role", "models_by_role", "lies" ], "title": "Humans", "type": "object" }, "LieRate": { "additionalProperties": false, "properties": { "self_tag": { "anyOf": [ { "type": "number" }, { "type": "null" } ] }, "jev": { "anyOf": [ { "type": "number" }, { "type": "null" } ] } }, "required": [ "self_tag", "jev" ], "title": "LieRate", "type": "object" }, "LieSuccess": { "additionalProperties": false, "properties": { "exploratory": { "type": "boolean" }, "comparisons": { "type": "integer" }, "moved": { "type": "integer" }, "voters": { "type": "integer" }, "followed": { "type": "integer" }, "base_voters": { "type": "integer" }, "base_followed": { "type": "integer" }, "base_abstained": { "type": "integer" }, "share_moved": { "anyOf": [ { "type": "number" }, { "type": "null" } ] }, "share_followed": { "anyOf": [ { "type": "number" }, { "type": "null" } ] }, "base_rate": { "anyOf": [ { "type": "number" }, { "type": "null" } ] } }, "required": [ "exploratory", "comparisons", "moved", "voters", "followed", "base_voters", "base_followed", "base_abstained", "share_moved", "share_followed", "base_rate" ], "title": "LieSuccess", "type": "object" }, "Lies": { "additionalProperties": false, "properties": { "lies": { "type": "integer" }, "by_type": { "additionalProperties": { "type": "integer" }, "propertyNames": { "enum": [ "false_role_claim", "false_report", "bussing", "hidden_knowledge" ] }, "type": "object" }, "lie_rate": { "anyOf": [ { "type": "number" }, { "type": "null" } ] }, "lie_success": { "$ref": "#/$defs/LieSuccess" } }, "required": [ "lies", "by_type", "lie_rate", "lie_success" ], "title": "Lies", "type": "object" }, "LiesBySource": { "additionalProperties": false, "properties": { "self_tag": { "$ref": "#/$defs/Lies" }, "jev": { "$ref": "#/$defs/Lies" } }, "required": [ "self_tag", "jev" ], "title": "LiesBySource", "type": "object" }, "ModelProfile": { "additionalProperties": false, "description": "One model's profile page: leaderboard numbers, detection, who it fools, speed, voice.", "properties": { "model": { "type": "string" }, "games": { "anyOf": [ { "type": "integer" }, { "type": "null" } ] }, "seats": { "type": "integer" }, "win_rate": { "anyOf": [ { "type": "number" }, { "type": "null" } ] }, "win_ci": { "anyOf": [ { "maxItems": 2, "minItems": 2, "prefixItems": [ { "type": "number" }, { "type": "number" } ], "type": "array" }, { "type": "null" } ] }, "wolf_win_rate": { "anyOf": [ { "type": "number" }, { "type": "null" } ] }, "village_win_rate": { "anyOf": [ { "type": "number" }, { "type": "null" } ] }, "openskill": { "anyOf": [ { "$ref": "#/$defs/Rating" }, { "type": "null" } ] }, "statements": { "type": "integer" }, "lie_rate": { "$ref": "#/$defs/LieRate" }, "detection": { "$ref": "#/$defs/ProfileDetection" }, "median_latency_ms": { "anyOf": [ { "type": "number" }, { "type": "null" } ] }, "voice": { "$ref": "#/$defs/Voice" }, "fools": { "$ref": "#/$defs/PairsBySource" }, "fooled_by": { "$ref": "#/$defs/PairsBySource" } }, "required": [ "model", "games", "seats", "win_rate", "win_ci", "wolf_win_rate", "village_win_rate", "openskill", "statements", "lie_rate", "detection", "median_latency_ms", "voice", "fools", "fooled_by" ], "title": "ModelProfile", "type": "object" }, "OpenSkill": { "additionalProperties": false, "properties": { "overall": { "additionalProperties": { "$ref": "#/$defs/Rating" }, "type": "object" }, "wolf": { "additionalProperties": { "$ref": "#/$defs/Rating" }, "type": "object" }, "village": { "additionalProperties": { "$ref": "#/$defs/Rating" }, "type": "object" }, "wolf_side": { "$ref": "#/$defs/SideAdvantage" } }, "required": [ "overall", "wolf", "village", "wolf_side" ], "title": "OpenSkill", "type": "object" }, "PairRow": { "additionalProperties": false, "description": "One deception_matrix cell seen from one side: `model` is the other model.", "properties": { "heard": { "type": "integer" }, "voters": { "type": "integer" }, "followed": { "type": "integer" }, "vote_follow_rate": { "anyOf": [ { "type": "number" }, { "type": "null" } ] }, "vote_follow_ci": { "anyOf": [ { "maxItems": 2, "minItems": 2, "prefixItems": [ { "type": "number" }, { "type": "number" } ], "type": "array" }, { "type": "null" } ] }, "comparisons": { "type": "integer" }, "moved": { "type": "integer" }, "belief_shift_rate": { "anyOf": [ { "type": "number" }, { "type": "null" } ] }, "belief_shift_ci": { "anyOf": [ { "maxItems": 2, "minItems": 2, "prefixItems": [ { "type": "number" }, { "type": "number" } ], "type": "array" }, { "type": "null" } ] }, "baseline_rate": { "anyOf": [ { "type": "number" }, { "type": "null" } ] }, "baseline_voters": { "type": "integer" }, "model": { "type": "string" } }, "required": [ "heard", "voters", "followed", "vote_follow_rate", "vote_follow_ci", "comparisons", "moved", "belief_shift_rate", "belief_shift_ci", "baseline_rate", "baseline_voters", "model" ], "title": "PairRow", "type": "object" }, "PairsBySource": { "additionalProperties": false, "properties": { "self_tag": { "items": { "$ref": "#/$defs/PairRow" }, "type": "array" }, "jev": { "items": { "$ref": "#/$defs/PairRow" }, "type": "array" } }, "required": [ "self_tag", "jev" ], "title": "PairsBySource", "type": "object" }, "Persuasion": { "additionalProperties": false, "properties": { "exploratory": { "type": "boolean" }, "accusations": { "type": "integer" }, "voters": { "type": "integer" }, "hits": { "type": "integer" }, "share": { "anyOf": [ { "type": "number" }, { "type": "null" } ] } }, "required": [ "exploratory", "accusations", "voters", "hits", "share" ], "title": "Persuasion", "type": "object" }, "ProfileDetection": { "additionalProperties": false, "properties": { "n": { "type": "integer" }, "auc": { "anyOf": [ { "type": "number" }, { "type": "null" } ] }, "brier": { "anyOf": [ { "type": "number" }, { "type": "null" } ] } }, "required": [ "n", "auc", "brier" ], "title": "ProfileDetection", "type": "object" }, "Rating": { "additionalProperties": false, "properties": { "mu": { "type": "number" }, "sigma": { "type": "number" }, "ordinal": { "type": "number" } }, "required": [ "mu", "sigma", "ordinal" ], "title": "Rating", "type": "object" }, "RoleSkill": { "additionalProperties": false, "description": "Doctor saves and seer hits per model against a chance rate. Human seats are left out.", "properties": { "doctor": { "$ref": "#/$defs/DoctorSkill" }, "seer": { "$ref": "#/$defs/SeerSkill" } }, "required": [ "doctor", "seer" ], "title": "RoleSkill", "type": "object" }, "SeerSkill": { "additionalProperties": false, "properties": { "models": { "additionalProperties": { "$ref": "#/$defs/SeerStats" }, "type": "object" }, "all": { "$ref": "#/$defs/SeerStats" } }, "required": [ "models", "all" ], "title": "SeerSkill", "type": "object" }, "SeerStats": { "additionalProperties": false, "properties": { "checks": { "type": "integer" }, "wolves_found": { "type": "integer" }, "fallbacks": { "type": "integer" }, "hit_rate": { "anyOf": [ { "type": "number" }, { "type": "null" } ] }, "hit_ci": { "anyOf": [ { "maxItems": 2, "minItems": 2, "prefixItems": [ { "type": "number" }, { "type": "number" } ], "type": "array" }, { "type": "null" } ] }, "chance_rate": { "anyOf": [ { "type": "number" }, { "type": "null" } ] } }, "required": [ "checks", "wolves_found", "fallbacks", "hit_rate", "hit_ci", "chance_rate" ], "title": "SeerStats", "type": "object" }, "SideAdvantage": { "additionalProperties": false, "description": "The wolf-side dummy player's rating per pool; {} when no game had a winner.", "properties": { "overall": { "$ref": "#/$defs/Rating" }, "sided": { "$ref": "#/$defs/Rating" } }, "title": "SideAdvantage", "type": "object" }, "Speed": { "additionalProperties": false, "properties": { "calls": { "type": "integer" }, "timed_calls": { "type": "integer" }, "median_latency_ms": { "anyOf": [ { "type": "number" }, { "type": "null" } ] }, "p90_latency_ms": { "anyOf": [ { "type": "integer" }, { "type": "null" } ] }, "output_tokens_per_s": { "anyOf": [ { "type": "number" }, { "type": "null" } ] }, "tokens_per_game": { "anyOf": [ { "type": "number" }, { "type": "null" } ] }, "cost_usd_per_game": { "anyOf": [ { "type": "number" }, { "type": "null" } ] }, "self_hosted": { "type": "boolean" } }, "required": [ "calls", "timed_calls", "median_latency_ms", "p90_latency_ms", "output_tokens_per_s", "tokens_per_game", "cost_usd_per_game", "self_hosted" ], "title": "Speed", "type": "object" }, "Summary": { "additionalProperties": false, "properties": { "games": { "type": "integer" }, "unfinished_games": { "type": "integer" }, "cost_usd": { "type": "number" }, "input_tokens": { "type": "integer" }, "output_tokens": { "type": "integer" }, "wolf_wins": { "type": "integer" }, "village_wins": { "type": "integer" }, "draws": { "type": "integer" }, "wolf_win_share": { "anyOf": [ { "type": "number" }, { "type": "null" } ] }, "village_win_share": { "anyOf": [ { "type": "number" }, { "type": "null" } ] }, "lies": { "additionalProperties": { "$ref": "#/$defs/SummaryLies" }, "propertyNames": { "enum": [ "self_tag", "jev" ] }, "type": "object" }, "top_openskill": { "anyOf": [ { "$ref": "#/$defs/TopOpenSkill" }, { "type": "null" } ] }, "top_wolf_win": { "anyOf": [ { "$ref": "#/$defs/TopWolfWin" }, { "type": "null" } ] }, "top_share_moved": { "additionalProperties": { "anyOf": [ { "$ref": "#/$defs/TopShareMoved" }, { "type": "null" } ] }, "propertyNames": { "enum": [ "self_tag", "jev" ] }, "type": "object" }, "min_comparisons": { "type": "integer" } }, "required": [ "games", "unfinished_games", "cost_usd", "input_tokens", "output_tokens", "wolf_wins", "village_wins", "draws", "wolf_win_share", "village_win_share", "lies", "top_openskill", "top_wolf_win", "top_share_moved", "min_comparisons" ], "title": "Summary", "type": "object" }, "SummaryLies": { "additionalProperties": false, "properties": { "lies": { "type": "integer" }, "statements": { "type": "integer" }, "lie_rate": { "anyOf": [ { "type": "number" }, { "type": "null" } ] }, "lie_success": { "$ref": "#/$defs/LieSuccess" } }, "required": [ "lies", "statements", "lie_rate", "lie_success" ], "title": "SummaryLies", "type": "object" }, "TopOpenSkill": { "additionalProperties": false, "properties": { "model": { "type": "string" }, "ordinal": { "type": "number" } }, "required": [ "model", "ordinal" ], "title": "TopOpenSkill", "type": "object" }, "TopShareMoved": { "additionalProperties": false, "properties": { "model": { "type": "string" }, "share_moved": { "type": "number" }, "comparisons": { "type": "integer" } }, "required": [ "model", "share_moved", "comparisons" ], "title": "TopShareMoved", "type": "object" }, "TopWolfWin": { "additionalProperties": false, "properties": { "model": { "type": "string" }, "win_rate": { "type": "number" }, "seats": { "type": "integer" } }, "required": [ "model", "win_rate", "seats" ], "title": "TopWolfWin", "type": "object" }, "Voice": { "additionalProperties": false, "properties": { "voice_id": { "type": "string" }, "fallback": { "type": "boolean" } }, "required": [ "voice_id", "fallback" ], "title": "Voice", "type": "object" } }, "additionalProperties": false, "description": "report/metrics.json: analysis.metrics.compute_metrics plus summary, speed and profiles.", "properties": { "games": { "type": "integer" }, "unfinished_games": { "type": "integer" }, "label_sources": { "items": { "enum": [ "self_tag", "jev" ], "type": "string" }, "type": "array" }, "exploratory": { "items": { "enum": [ "lie_success", "persuasion" ], "type": "string" }, "type": "array" }, "models": { "additionalProperties": { "additionalProperties": { "$ref": "#/$defs/GroupStats" }, "propertyNames": { "enum": [ "all", "wolf", "village", "seer", "doctor", "villager" ] }, "type": "object" }, "type": "object" }, "by_group": { "additionalProperties": { "$ref": "#/$defs/GroupStats" }, "propertyNames": { "enum": [ "all", "wolf", "village", "seer", "doctor", "villager" ] }, "type": "object" }, "openskill": { "$ref": "#/$defs/OpenSkill" }, "deception_matrix": { "$ref": "#/$defs/DeceptionMatrix" }, "humans": { "$ref": "#/$defs/Humans" }, "role_skill": { "$ref": "#/$defs/RoleSkill" }, "summary": { "$ref": "#/$defs/Summary" }, "speed": { "additionalProperties": { "$ref": "#/$defs/Speed" }, "type": "object" }, "profiles": { "additionalProperties": { "$ref": "#/$defs/ModelProfile" }, "type": "object" } }, "required": [ "games", "unfinished_games", "label_sources", "exploratory", "models", "by_group", "openskill", "deception_matrix", "humans", "role_skill", "summary", "speed", "profiles" ], "title": "Metrics", "type": "object" }