Spaces:
Running
Running
| { | |
| "$schema": "https://json-schema.org/draft/2020-12/schema", | |
| "$id": "metrics.schema.json", | |
| "x-contract-version": 1, | |
| "$defs": { | |
| "DeceptionBaseline": { | |
| "additionalProperties": false, | |
| "properties": { | |
| "heard": { | |
| "type": "integer" | |
| }, | |
| "voters": { | |
| "type": "integer" | |
| }, | |
| "followed": { | |
| "type": "integer" | |
| }, | |
| "vote_follow_rate": { | |
| "anyOf": [ | |
| { | |
| "type": "number" | |
| }, | |
| { | |
| "type": "null" | |
| } | |
| ] | |
| }, | |
| "vote_follow_ci": { | |
| "anyOf": [ | |
| { | |
| "maxItems": 2, | |
| "minItems": 2, | |
| "prefixItems": [ | |
| { | |
| "type": "number" | |
| }, | |
| { | |
| "type": "number" | |
| } | |
| ], | |
| "type": "array" | |
| }, | |
| { | |
| "type": "null" | |
| } | |
| ] | |
| } | |
| }, | |
| "required": [ | |
| "heard", | |
| "voters", | |
| "followed", | |
| "vote_follow_rate", | |
| "vote_follow_ci" | |
| ], | |
| "title": "DeceptionBaseline", | |
| "type": "object" | |
| }, | |
| "DeceptionBySource": { | |
| "additionalProperties": false, | |
| "properties": { | |
| "self_tag": { | |
| "additionalProperties": { | |
| "additionalProperties": { | |
| "$ref": "#/$defs/DeceptionCell" | |
| }, | |
| "type": "object" | |
| }, | |
| "type": "object" | |
| }, | |
| "jev": { | |
| "additionalProperties": { | |
| "additionalProperties": { | |
| "$ref": "#/$defs/DeceptionCell" | |
| }, | |
| "type": "object" | |
| }, | |
| "type": "object" | |
| } | |
| }, | |
| "required": [ | |
| "self_tag", | |
| "jev" | |
| ], | |
| "title": "DeceptionBySource", | |
| "type": "object" | |
| }, | |
| "DeceptionCell": { | |
| "additionalProperties": false, | |
| "properties": { | |
| "heard": { | |
| "type": "integer" | |
| }, | |
| "voters": { | |
| "type": "integer" | |
| }, | |
| "followed": { | |
| "type": "integer" | |
| }, | |
| "vote_follow_rate": { | |
| "anyOf": [ | |
| { | |
| "type": "number" | |
| }, | |
| { | |
| "type": "null" | |
| } | |
| ] | |
| }, | |
| "vote_follow_ci": { | |
| "anyOf": [ | |
| { | |
| "maxItems": 2, | |
| "minItems": 2, | |
| "prefixItems": [ | |
| { | |
| "type": "number" | |
| }, | |
| { | |
| "type": "number" | |
| } | |
| ], | |
| "type": "array" | |
| }, | |
| { | |
| "type": "null" | |
| } | |
| ] | |
| }, | |
| "comparisons": { | |
| "type": "integer" | |
| }, | |
| "moved": { | |
| "type": "integer" | |
| }, | |
| "belief_shift_rate": { | |
| "anyOf": [ | |
| { | |
| "type": "number" | |
| }, | |
| { | |
| "type": "null" | |
| } | |
| ] | |
| }, | |
| "belief_shift_ci": { | |
| "anyOf": [ | |
| { | |
| "maxItems": 2, | |
| "minItems": 2, | |
| "prefixItems": [ | |
| { | |
| "type": "number" | |
| }, | |
| { | |
| "type": "number" | |
| } | |
| ], | |
| "type": "array" | |
| }, | |
| { | |
| "type": "null" | |
| } | |
| ] | |
| }, | |
| "baseline_rate": { | |
| "anyOf": [ | |
| { | |
| "type": "number" | |
| }, | |
| { | |
| "type": "null" | |
| } | |
| ] | |
| }, | |
| "baseline_voters": { | |
| "type": "integer" | |
| } | |
| }, | |
| "required": [ | |
| "heard", | |
| "voters", | |
| "followed", | |
| "vote_follow_rate", | |
| "vote_follow_ci", | |
| "comparisons", | |
| "moved", | |
| "belief_shift_rate", | |
| "belief_shift_ci", | |
| "baseline_rate", | |
| "baseline_voters" | |
| ], | |
| "title": "DeceptionCell", | |
| "type": "object" | |
| }, | |
| "DeceptionMatrix": { | |
| "additionalProperties": false, | |
| "properties": { | |
| "exploratory": { | |
| "type": "boolean" | |
| }, | |
| "pairs": { | |
| "$ref": "#/$defs/DeceptionBySource" | |
| }, | |
| "baseline": { | |
| "additionalProperties": { | |
| "additionalProperties": { | |
| "$ref": "#/$defs/DeceptionBaseline" | |
| }, | |
| "type": "object" | |
| }, | |
| "propertyNames": { | |
| "enum": [ | |
| "role_claim", | |
| "seer_wolf", | |
| "seer_clear", | |
| "accusation" | |
| ] | |
| }, | |
| "type": "object" | |
| } | |
| }, | |
| "required": [ | |
| "exploratory", | |
| "pairs", | |
| "baseline" | |
| ], | |
| "title": "DeceptionMatrix", | |
| "type": "object" | |
| }, | |
| "Detection": { | |
| "additionalProperties": false, | |
| "properties": { | |
| "n": { | |
| "type": "integer" | |
| }, | |
| "brier": { | |
| "anyOf": [ | |
| { | |
| "type": "number" | |
| }, | |
| { | |
| "type": "null" | |
| } | |
| ] | |
| }, | |
| "auc": { | |
| "anyOf": [ | |
| { | |
| "type": "number" | |
| }, | |
| { | |
| "type": "null" | |
| } | |
| ] | |
| }, | |
| "by_day": { | |
| "patternProperties": { | |
| "^\\d+$": { | |
| "$ref": "#/$defs/DetectionDay" | |
| } | |
| }, | |
| "type": "object", | |
| "additionalProperties": false | |
| } | |
| }, | |
| "required": [ | |
| "n", | |
| "brier", | |
| "auc", | |
| "by_day" | |
| ], | |
| "title": "Detection", | |
| "type": "object" | |
| }, | |
| "DetectionDay": { | |
| "additionalProperties": false, | |
| "properties": { | |
| "n": { | |
| "type": "integer" | |
| }, | |
| "brier": { | |
| "anyOf": [ | |
| { | |
| "type": "number" | |
| }, | |
| { | |
| "type": "null" | |
| } | |
| ] | |
| }, | |
| "auc": { | |
| "anyOf": [ | |
| { | |
| "type": "number" | |
| }, | |
| { | |
| "type": "null" | |
| } | |
| ] | |
| } | |
| }, | |
| "required": [ | |
| "n", | |
| "brier", | |
| "auc" | |
| ], | |
| "title": "DetectionDay", | |
| "type": "object" | |
| }, | |
| "DoctorSkill": { | |
| "additionalProperties": false, | |
| "properties": { | |
| "models": { | |
| "additionalProperties": { | |
| "$ref": "#/$defs/DoctorStats" | |
| }, | |
| "type": "object" | |
| }, | |
| "all": { | |
| "$ref": "#/$defs/DoctorStats" | |
| } | |
| }, | |
| "required": [ | |
| "models", | |
| "all" | |
| ], | |
| "title": "DoctorSkill", | |
| "type": "object" | |
| }, | |
| "DoctorStats": { | |
| "additionalProperties": false, | |
| "properties": { | |
| "nights": { | |
| "type": "integer" | |
| }, | |
| "saves": { | |
| "type": "integer" | |
| }, | |
| "self_protects": { | |
| "type": "integer" | |
| }, | |
| "fallbacks": { | |
| "type": "integer" | |
| }, | |
| "save_rate": { | |
| "anyOf": [ | |
| { | |
| "type": "number" | |
| }, | |
| { | |
| "type": "null" | |
| } | |
| ] | |
| }, | |
| "save_ci": { | |
| "anyOf": [ | |
| { | |
| "maxItems": 2, | |
| "minItems": 2, | |
| "prefixItems": [ | |
| { | |
| "type": "number" | |
| }, | |
| { | |
| "type": "number" | |
| } | |
| ], | |
| "type": "array" | |
| }, | |
| { | |
| "type": "null" | |
| } | |
| ] | |
| }, | |
| "chance_rate": { | |
| "anyOf": [ | |
| { | |
| "type": "number" | |
| }, | |
| { | |
| "type": "null" | |
| } | |
| ] | |
| } | |
| }, | |
| "required": [ | |
| "nights", | |
| "saves", | |
| "self_protects", | |
| "fallbacks", | |
| "save_rate", | |
| "save_ci", | |
| "chance_rate" | |
| ], | |
| "title": "DoctorStats", | |
| "type": "object" | |
| }, | |
| "GroupStats": { | |
| "additionalProperties": false, | |
| "properties": { | |
| "games": { | |
| "type": "integer" | |
| }, | |
| "wins": { | |
| "type": "integer" | |
| }, | |
| "draws": { | |
| "type": "integer" | |
| }, | |
| "losses": { | |
| "type": "integer" | |
| }, | |
| "win_rate": { | |
| "anyOf": [ | |
| { | |
| "type": "number" | |
| }, | |
| { | |
| "type": "null" | |
| } | |
| ] | |
| }, | |
| "win_ci": { | |
| "anyOf": [ | |
| { | |
| "maxItems": 2, | |
| "minItems": 2, | |
| "prefixItems": [ | |
| { | |
| "type": "number" | |
| }, | |
| { | |
| "type": "number" | |
| } | |
| ], | |
| "type": "array" | |
| }, | |
| { | |
| "type": "null" | |
| } | |
| ] | |
| }, | |
| "statements": { | |
| "type": "integer" | |
| }, | |
| "lies": { | |
| "$ref": "#/$defs/LiesBySource" | |
| }, | |
| "detection": { | |
| "$ref": "#/$defs/Detection" | |
| }, | |
| "persuasion": { | |
| "$ref": "#/$defs/Persuasion" | |
| }, | |
| "survival_days": { | |
| "anyOf": [ | |
| { | |
| "type": "number" | |
| }, | |
| { | |
| "type": "null" | |
| } | |
| ] | |
| }, | |
| "night_targeted": { | |
| "anyOf": [ | |
| { | |
| "type": "number" | |
| }, | |
| { | |
| "type": "null" | |
| } | |
| ] | |
| }, | |
| "voted_out": { | |
| "anyOf": [ | |
| { | |
| "type": "number" | |
| }, | |
| { | |
| "type": "null" | |
| } | |
| ] | |
| }, | |
| "invalid_output_rate": { | |
| "anyOf": [ | |
| { | |
| "type": "number" | |
| }, | |
| { | |
| "type": "null" | |
| } | |
| ] | |
| }, | |
| "invalid_attempt_rate": { | |
| "anyOf": [ | |
| { | |
| "type": "number" | |
| }, | |
| { | |
| "type": "null" | |
| } | |
| ] | |
| }, | |
| "fallback_rate": { | |
| "anyOf": [ | |
| { | |
| "type": "number" | |
| }, | |
| { | |
| "type": "null" | |
| } | |
| ] | |
| }, | |
| "retry_success_rate": { | |
| "anyOf": [ | |
| { | |
| "type": "number" | |
| }, | |
| { | |
| "type": "null" | |
| } | |
| ] | |
| }, | |
| "provider_errors": { | |
| "type": "integer" | |
| }, | |
| "provider_error_rate": { | |
| "anyOf": [ | |
| { | |
| "type": "number" | |
| }, | |
| { | |
| "type": "null" | |
| } | |
| ] | |
| }, | |
| "tokens_per_game": { | |
| "anyOf": [ | |
| { | |
| "type": "number" | |
| }, | |
| { | |
| "type": "null" | |
| } | |
| ] | |
| }, | |
| "cost_usd_per_game": { | |
| "anyOf": [ | |
| { | |
| "type": "number" | |
| }, | |
| { | |
| "type": "null" | |
| } | |
| ] | |
| } | |
| }, | |
| "required": [ | |
| "games", | |
| "wins", | |
| "draws", | |
| "losses", | |
| "win_rate", | |
| "win_ci", | |
| "statements", | |
| "lies", | |
| "detection", | |
| "persuasion", | |
| "survival_days", | |
| "night_targeted", | |
| "voted_out", | |
| "invalid_output_rate", | |
| "invalid_attempt_rate", | |
| "fallback_rate", | |
| "retry_success_rate", | |
| "provider_errors", | |
| "provider_error_rate", | |
| "tokens_per_game", | |
| "cost_usd_per_game" | |
| ], | |
| "title": "GroupStats", | |
| "type": "object" | |
| }, | |
| "HumanFlow": { | |
| "additionalProperties": false, | |
| "description": "deception_matrix counts summed by direction, plus `caught`: next vote for the liar.", | |
| "properties": { | |
| "heard": { | |
| "type": "integer" | |
| }, | |
| "voters": { | |
| "type": "integer" | |
| }, | |
| "followed": { | |
| "type": "integer" | |
| }, | |
| "vote_follow_rate": { | |
| "anyOf": [ | |
| { | |
| "type": "number" | |
| }, | |
| { | |
| "type": "null" | |
| } | |
| ] | |
| }, | |
| "vote_follow_ci": { | |
| "anyOf": [ | |
| { | |
| "maxItems": 2, | |
| "minItems": 2, | |
| "prefixItems": [ | |
| { | |
| "type": "number" | |
| }, | |
| { | |
| "type": "number" | |
| } | |
| ], | |
| "type": "array" | |
| }, | |
| { | |
| "type": "null" | |
| } | |
| ] | |
| }, | |
| "caught": { | |
| "type": "integer" | |
| }, | |
| "comparisons": { | |
| "type": "integer" | |
| }, | |
| "moved": { | |
| "type": "integer" | |
| }, | |
| "belief_shift_rate": { | |
| "anyOf": [ | |
| { | |
| "type": "number" | |
| }, | |
| { | |
| "type": "null" | |
| } | |
| ] | |
| }, | |
| "belief_shift_ci": { | |
| "anyOf": [ | |
| { | |
| "maxItems": 2, | |
| "minItems": 2, | |
| "prefixItems": [ | |
| { | |
| "type": "number" | |
| }, | |
| { | |
| "type": "number" | |
| } | |
| ], | |
| "type": "array" | |
| }, | |
| { | |
| "type": "null" | |
| } | |
| ] | |
| }, | |
| "catch_rate": { | |
| "anyOf": [ | |
| { | |
| "type": "number" | |
| }, | |
| { | |
| "type": "null" | |
| } | |
| ] | |
| }, | |
| "catch_ci": { | |
| "anyOf": [ | |
| { | |
| "maxItems": 2, | |
| "minItems": 2, | |
| "prefixItems": [ | |
| { | |
| "type": "number" | |
| }, | |
| { | |
| "type": "number" | |
| } | |
| ], | |
| "type": "array" | |
| }, | |
| { | |
| "type": "null" | |
| } | |
| ] | |
| } | |
| }, | |
| "required": [ | |
| "heard", | |
| "voters", | |
| "followed", | |
| "vote_follow_rate", | |
| "vote_follow_ci", | |
| "caught", | |
| "comparisons", | |
| "moved", | |
| "belief_shift_rate", | |
| "belief_shift_ci", | |
| "catch_rate", | |
| "catch_ci" | |
| ], | |
| "title": "HumanFlow", | |
| "type": "object" | |
| }, | |
| "HumanFlows": { | |
| "additionalProperties": false, | |
| "properties": { | |
| "human_to_model": { | |
| "$ref": "#/$defs/HumanFlow" | |
| }, | |
| "model_to_human": { | |
| "$ref": "#/$defs/HumanFlow" | |
| }, | |
| "model_to_model": { | |
| "$ref": "#/$defs/HumanFlow" | |
| } | |
| }, | |
| "required": [ | |
| "human_to_model", | |
| "model_to_human", | |
| "model_to_model" | |
| ], | |
| "title": "HumanFlows", | |
| "type": "object" | |
| }, | |
| "HumanFlowsBySource": { | |
| "additionalProperties": false, | |
| "properties": { | |
| "self_tag": { | |
| "$ref": "#/$defs/HumanFlows" | |
| }, | |
| "jev": { | |
| "$ref": "#/$defs/HumanFlows" | |
| } | |
| }, | |
| "required": [ | |
| "self_tag", | |
| "jev" | |
| ], | |
| "title": "HumanFlowsBySource", | |
| "type": "object" | |
| }, | |
| "HumanSeats": { | |
| "additionalProperties": false, | |
| "properties": { | |
| "seats": { | |
| "type": "integer" | |
| }, | |
| "wins": { | |
| "type": "integer" | |
| }, | |
| "draws": { | |
| "type": "integer" | |
| }, | |
| "losses": { | |
| "type": "integer" | |
| }, | |
| "win_rate": { | |
| "anyOf": [ | |
| { | |
| "type": "number" | |
| }, | |
| { | |
| "type": "null" | |
| } | |
| ] | |
| }, | |
| "win_ci": { | |
| "anyOf": [ | |
| { | |
| "maxItems": 2, | |
| "minItems": 2, | |
| "prefixItems": [ | |
| { | |
| "type": "number" | |
| }, | |
| { | |
| "type": "number" | |
| } | |
| ], | |
| "type": "array" | |
| }, | |
| { | |
| "type": "null" | |
| } | |
| ] | |
| } | |
| }, | |
| "required": [ | |
| "seats", | |
| "wins", | |
| "draws", | |
| "losses", | |
| "win_rate", | |
| "win_ci" | |
| ], | |
| "title": "HumanSeats", | |
| "type": "object" | |
| }, | |
| "Humans": { | |
| "additionalProperties": false, | |
| "description": "Humans vs models (#305): visitor seats against the AI seats. Counts only, no handles.", | |
| "properties": { | |
| "exploratory": { | |
| "type": "boolean" | |
| }, | |
| "games": { | |
| "type": "integer" | |
| }, | |
| "seats": { | |
| "type": "integer" | |
| }, | |
| "by_role": { | |
| "additionalProperties": { | |
| "$ref": "#/$defs/HumanSeats" | |
| }, | |
| "propertyNames": { | |
| "enum": [ | |
| "all", | |
| "wolf", | |
| "village", | |
| "seer", | |
| "doctor", | |
| "villager" | |
| ] | |
| }, | |
| "type": "object" | |
| }, | |
| "models_by_role": { | |
| "additionalProperties": { | |
| "$ref": "#/$defs/HumanSeats" | |
| }, | |
| "propertyNames": { | |
| "enum": [ | |
| "all", | |
| "wolf", | |
| "village", | |
| "seer", | |
| "doctor", | |
| "villager" | |
| ] | |
| }, | |
| "type": "object" | |
| }, | |
| "lies": { | |
| "$ref": "#/$defs/HumanFlowsBySource" | |
| } | |
| }, | |
| "required": [ | |
| "exploratory", | |
| "games", | |
| "seats", | |
| "by_role", | |
| "models_by_role", | |
| "lies" | |
| ], | |
| "title": "Humans", | |
| "type": "object" | |
| }, | |
| "LieRate": { | |
| "additionalProperties": false, | |
| "properties": { | |
| "self_tag": { | |
| "anyOf": [ | |
| { | |
| "type": "number" | |
| }, | |
| { | |
| "type": "null" | |
| } | |
| ] | |
| }, | |
| "jev": { | |
| "anyOf": [ | |
| { | |
| "type": "number" | |
| }, | |
| { | |
| "type": "null" | |
| } | |
| ] | |
| } | |
| }, | |
| "required": [ | |
| "self_tag", | |
| "jev" | |
| ], | |
| "title": "LieRate", | |
| "type": "object" | |
| }, | |
| "LieSuccess": { | |
| "additionalProperties": false, | |
| "properties": { | |
| "exploratory": { | |
| "type": "boolean" | |
| }, | |
| "comparisons": { | |
| "type": "integer" | |
| }, | |
| "moved": { | |
| "type": "integer" | |
| }, | |
| "voters": { | |
| "type": "integer" | |
| }, | |
| "followed": { | |
| "type": "integer" | |
| }, | |
| "base_voters": { | |
| "type": "integer" | |
| }, | |
| "base_followed": { | |
| "type": "integer" | |
| }, | |
| "base_abstained": { | |
| "type": "integer" | |
| }, | |
| "share_moved": { | |
| "anyOf": [ | |
| { | |
| "type": "number" | |
| }, | |
| { | |
| "type": "null" | |
| } | |
| ] | |
| }, | |
| "share_followed": { | |
| "anyOf": [ | |
| { | |
| "type": "number" | |
| }, | |
| { | |
| "type": "null" | |
| } | |
| ] | |
| }, | |
| "base_rate": { | |
| "anyOf": [ | |
| { | |
| "type": "number" | |
| }, | |
| { | |
| "type": "null" | |
| } | |
| ] | |
| } | |
| }, | |
| "required": [ | |
| "exploratory", | |
| "comparisons", | |
| "moved", | |
| "voters", | |
| "followed", | |
| "base_voters", | |
| "base_followed", | |
| "base_abstained", | |
| "share_moved", | |
| "share_followed", | |
| "base_rate" | |
| ], | |
| "title": "LieSuccess", | |
| "type": "object" | |
| }, | |
| "Lies": { | |
| "additionalProperties": false, | |
| "properties": { | |
| "lies": { | |
| "type": "integer" | |
| }, | |
| "by_type": { | |
| "additionalProperties": { | |
| "type": "integer" | |
| }, | |
| "propertyNames": { | |
| "enum": [ | |
| "false_role_claim", | |
| "false_report", | |
| "bussing", | |
| "hidden_knowledge" | |
| ] | |
| }, | |
| "type": "object" | |
| }, | |
| "lie_rate": { | |
| "anyOf": [ | |
| { | |
| "type": "number" | |
| }, | |
| { | |
| "type": "null" | |
| } | |
| ] | |
| }, | |
| "lie_success": { | |
| "$ref": "#/$defs/LieSuccess" | |
| } | |
| }, | |
| "required": [ | |
| "lies", | |
| "by_type", | |
| "lie_rate", | |
| "lie_success" | |
| ], | |
| "title": "Lies", | |
| "type": "object" | |
| }, | |
| "LiesBySource": { | |
| "additionalProperties": false, | |
| "properties": { | |
| "self_tag": { | |
| "$ref": "#/$defs/Lies" | |
| }, | |
| "jev": { | |
| "$ref": "#/$defs/Lies" | |
| } | |
| }, | |
| "required": [ | |
| "self_tag", | |
| "jev" | |
| ], | |
| "title": "LiesBySource", | |
| "type": "object" | |
| }, | |
| "ModelProfile": { | |
| "additionalProperties": false, | |
| "description": "One model's profile page: leaderboard numbers, detection, who it fools, speed, voice.", | |
| "properties": { | |
| "model": { | |
| "type": "string" | |
| }, | |
| "games": { | |
| "anyOf": [ | |
| { | |
| "type": "integer" | |
| }, | |
| { | |
| "type": "null" | |
| } | |
| ] | |
| }, | |
| "seats": { | |
| "type": "integer" | |
| }, | |
| "win_rate": { | |
| "anyOf": [ | |
| { | |
| "type": "number" | |
| }, | |
| { | |
| "type": "null" | |
| } | |
| ] | |
| }, | |
| "win_ci": { | |
| "anyOf": [ | |
| { | |
| "maxItems": 2, | |
| "minItems": 2, | |
| "prefixItems": [ | |
| { | |
| "type": "number" | |
| }, | |
| { | |
| "type": "number" | |
| } | |
| ], | |
| "type": "array" | |
| }, | |
| { | |
| "type": "null" | |
| } | |
| ] | |
| }, | |
| "wolf_win_rate": { | |
| "anyOf": [ | |
| { | |
| "type": "number" | |
| }, | |
| { | |
| "type": "null" | |
| } | |
| ] | |
| }, | |
| "village_win_rate": { | |
| "anyOf": [ | |
| { | |
| "type": "number" | |
| }, | |
| { | |
| "type": "null" | |
| } | |
| ] | |
| }, | |
| "openskill": { | |
| "anyOf": [ | |
| { | |
| "$ref": "#/$defs/Rating" | |
| }, | |
| { | |
| "type": "null" | |
| } | |
| ] | |
| }, | |
| "statements": { | |
| "type": "integer" | |
| }, | |
| "lie_rate": { | |
| "$ref": "#/$defs/LieRate" | |
| }, | |
| "detection": { | |
| "$ref": "#/$defs/ProfileDetection" | |
| }, | |
| "median_latency_ms": { | |
| "anyOf": [ | |
| { | |
| "type": "number" | |
| }, | |
| { | |
| "type": "null" | |
| } | |
| ] | |
| }, | |
| "voice": { | |
| "$ref": "#/$defs/Voice" | |
| }, | |
| "fools": { | |
| "$ref": "#/$defs/PairsBySource" | |
| }, | |
| "fooled_by": { | |
| "$ref": "#/$defs/PairsBySource" | |
| } | |
| }, | |
| "required": [ | |
| "model", | |
| "games", | |
| "seats", | |
| "win_rate", | |
| "win_ci", | |
| "wolf_win_rate", | |
| "village_win_rate", | |
| "openskill", | |
| "statements", | |
| "lie_rate", | |
| "detection", | |
| "median_latency_ms", | |
| "voice", | |
| "fools", | |
| "fooled_by" | |
| ], | |
| "title": "ModelProfile", | |
| "type": "object" | |
| }, | |
| "OpenSkill": { | |
| "additionalProperties": false, | |
| "properties": { | |
| "overall": { | |
| "additionalProperties": { | |
| "$ref": "#/$defs/Rating" | |
| }, | |
| "type": "object" | |
| }, | |
| "wolf": { | |
| "additionalProperties": { | |
| "$ref": "#/$defs/Rating" | |
| }, | |
| "type": "object" | |
| }, | |
| "village": { | |
| "additionalProperties": { | |
| "$ref": "#/$defs/Rating" | |
| }, | |
| "type": "object" | |
| }, | |
| "wolf_side": { | |
| "$ref": "#/$defs/SideAdvantage" | |
| } | |
| }, | |
| "required": [ | |
| "overall", | |
| "wolf", | |
| "village", | |
| "wolf_side" | |
| ], | |
| "title": "OpenSkill", | |
| "type": "object" | |
| }, | |
| "PairRow": { | |
| "additionalProperties": false, | |
| "description": "One deception_matrix cell seen from one side: `model` is the other model.", | |
| "properties": { | |
| "heard": { | |
| "type": "integer" | |
| }, | |
| "voters": { | |
| "type": "integer" | |
| }, | |
| "followed": { | |
| "type": "integer" | |
| }, | |
| "vote_follow_rate": { | |
| "anyOf": [ | |
| { | |
| "type": "number" | |
| }, | |
| { | |
| "type": "null" | |
| } | |
| ] | |
| }, | |
| "vote_follow_ci": { | |
| "anyOf": [ | |
| { | |
| "maxItems": 2, | |
| "minItems": 2, | |
| "prefixItems": [ | |
| { | |
| "type": "number" | |
| }, | |
| { | |
| "type": "number" | |
| } | |
| ], | |
| "type": "array" | |
| }, | |
| { | |
| "type": "null" | |
| } | |
| ] | |
| }, | |
| "comparisons": { | |
| "type": "integer" | |
| }, | |
| "moved": { | |
| "type": "integer" | |
| }, | |
| "belief_shift_rate": { | |
| "anyOf": [ | |
| { | |
| "type": "number" | |
| }, | |
| { | |
| "type": "null" | |
| } | |
| ] | |
| }, | |
| "belief_shift_ci": { | |
| "anyOf": [ | |
| { | |
| "maxItems": 2, | |
| "minItems": 2, | |
| "prefixItems": [ | |
| { | |
| "type": "number" | |
| }, | |
| { | |
| "type": "number" | |
| } | |
| ], | |
| "type": "array" | |
| }, | |
| { | |
| "type": "null" | |
| } | |
| ] | |
| }, | |
| "baseline_rate": { | |
| "anyOf": [ | |
| { | |
| "type": "number" | |
| }, | |
| { | |
| "type": "null" | |
| } | |
| ] | |
| }, | |
| "baseline_voters": { | |
| "type": "integer" | |
| }, | |
| "model": { | |
| "type": "string" | |
| } | |
| }, | |
| "required": [ | |
| "heard", | |
| "voters", | |
| "followed", | |
| "vote_follow_rate", | |
| "vote_follow_ci", | |
| "comparisons", | |
| "moved", | |
| "belief_shift_rate", | |
| "belief_shift_ci", | |
| "baseline_rate", | |
| "baseline_voters", | |
| "model" | |
| ], | |
| "title": "PairRow", | |
| "type": "object" | |
| }, | |
| "PairsBySource": { | |
| "additionalProperties": false, | |
| "properties": { | |
| "self_tag": { | |
| "items": { | |
| "$ref": "#/$defs/PairRow" | |
| }, | |
| "type": "array" | |
| }, | |
| "jev": { | |
| "items": { | |
| "$ref": "#/$defs/PairRow" | |
| }, | |
| "type": "array" | |
| } | |
| }, | |
| "required": [ | |
| "self_tag", | |
| "jev" | |
| ], | |
| "title": "PairsBySource", | |
| "type": "object" | |
| }, | |
| "Persuasion": { | |
| "additionalProperties": false, | |
| "properties": { | |
| "exploratory": { | |
| "type": "boolean" | |
| }, | |
| "accusations": { | |
| "type": "integer" | |
| }, | |
| "voters": { | |
| "type": "integer" | |
| }, | |
| "hits": { | |
| "type": "integer" | |
| }, | |
| "share": { | |
| "anyOf": [ | |
| { | |
| "type": "number" | |
| }, | |
| { | |
| "type": "null" | |
| } | |
| ] | |
| } | |
| }, | |
| "required": [ | |
| "exploratory", | |
| "accusations", | |
| "voters", | |
| "hits", | |
| "share" | |
| ], | |
| "title": "Persuasion", | |
| "type": "object" | |
| }, | |
| "ProfileDetection": { | |
| "additionalProperties": false, | |
| "properties": { | |
| "n": { | |
| "type": "integer" | |
| }, | |
| "auc": { | |
| "anyOf": [ | |
| { | |
| "type": "number" | |
| }, | |
| { | |
| "type": "null" | |
| } | |
| ] | |
| }, | |
| "brier": { | |
| "anyOf": [ | |
| { | |
| "type": "number" | |
| }, | |
| { | |
| "type": "null" | |
| } | |
| ] | |
| } | |
| }, | |
| "required": [ | |
| "n", | |
| "auc", | |
| "brier" | |
| ], | |
| "title": "ProfileDetection", | |
| "type": "object" | |
| }, | |
| "Rating": { | |
| "additionalProperties": false, | |
| "properties": { | |
| "mu": { | |
| "type": "number" | |
| }, | |
| "sigma": { | |
| "type": "number" | |
| }, | |
| "ordinal": { | |
| "type": "number" | |
| } | |
| }, | |
| "required": [ | |
| "mu", | |
| "sigma", | |
| "ordinal" | |
| ], | |
| "title": "Rating", | |
| "type": "object" | |
| }, | |
| "RoleSkill": { | |
| "additionalProperties": false, | |
| "description": "Doctor saves and seer hits per model against a chance rate. Human seats are left out.", | |
| "properties": { | |
| "doctor": { | |
| "$ref": "#/$defs/DoctorSkill" | |
| }, | |
| "seer": { | |
| "$ref": "#/$defs/SeerSkill" | |
| } | |
| }, | |
| "required": [ | |
| "doctor", | |
| "seer" | |
| ], | |
| "title": "RoleSkill", | |
| "type": "object" | |
| }, | |
| "SeerSkill": { | |
| "additionalProperties": false, | |
| "properties": { | |
| "models": { | |
| "additionalProperties": { | |
| "$ref": "#/$defs/SeerStats" | |
| }, | |
| "type": "object" | |
| }, | |
| "all": { | |
| "$ref": "#/$defs/SeerStats" | |
| } | |
| }, | |
| "required": [ | |
| "models", | |
| "all" | |
| ], | |
| "title": "SeerSkill", | |
| "type": "object" | |
| }, | |
| "SeerStats": { | |
| "additionalProperties": false, | |
| "properties": { | |
| "checks": { | |
| "type": "integer" | |
| }, | |
| "wolves_found": { | |
| "type": "integer" | |
| }, | |
| "fallbacks": { | |
| "type": "integer" | |
| }, | |
| "hit_rate": { | |
| "anyOf": [ | |
| { | |
| "type": "number" | |
| }, | |
| { | |
| "type": "null" | |
| } | |
| ] | |
| }, | |
| "hit_ci": { | |
| "anyOf": [ | |
| { | |
| "maxItems": 2, | |
| "minItems": 2, | |
| "prefixItems": [ | |
| { | |
| "type": "number" | |
| }, | |
| { | |
| "type": "number" | |
| } | |
| ], | |
| "type": "array" | |
| }, | |
| { | |
| "type": "null" | |
| } | |
| ] | |
| }, | |
| "chance_rate": { | |
| "anyOf": [ | |
| { | |
| "type": "number" | |
| }, | |
| { | |
| "type": "null" | |
| } | |
| ] | |
| } | |
| }, | |
| "required": [ | |
| "checks", | |
| "wolves_found", | |
| "fallbacks", | |
| "hit_rate", | |
| "hit_ci", | |
| "chance_rate" | |
| ], | |
| "title": "SeerStats", | |
| "type": "object" | |
| }, | |
| "SideAdvantage": { | |
| "additionalProperties": false, | |
| "description": "The wolf-side dummy player's rating per pool; {} when no game had a winner.", | |
| "properties": { | |
| "overall": { | |
| "$ref": "#/$defs/Rating" | |
| }, | |
| "sided": { | |
| "$ref": "#/$defs/Rating" | |
| } | |
| }, | |
| "title": "SideAdvantage", | |
| "type": "object" | |
| }, | |
| "Speed": { | |
| "additionalProperties": false, | |
| "properties": { | |
| "calls": { | |
| "type": "integer" | |
| }, | |
| "timed_calls": { | |
| "type": "integer" | |
| }, | |
| "median_latency_ms": { | |
| "anyOf": [ | |
| { | |
| "type": "number" | |
| }, | |
| { | |
| "type": "null" | |
| } | |
| ] | |
| }, | |
| "p90_latency_ms": { | |
| "anyOf": [ | |
| { | |
| "type": "integer" | |
| }, | |
| { | |
| "type": "null" | |
| } | |
| ] | |
| }, | |
| "output_tokens_per_s": { | |
| "anyOf": [ | |
| { | |
| "type": "number" | |
| }, | |
| { | |
| "type": "null" | |
| } | |
| ] | |
| }, | |
| "tokens_per_game": { | |
| "anyOf": [ | |
| { | |
| "type": "number" | |
| }, | |
| { | |
| "type": "null" | |
| } | |
| ] | |
| }, | |
| "cost_usd_per_game": { | |
| "anyOf": [ | |
| { | |
| "type": "number" | |
| }, | |
| { | |
| "type": "null" | |
| } | |
| ] | |
| }, | |
| "self_hosted": { | |
| "type": "boolean" | |
| } | |
| }, | |
| "required": [ | |
| "calls", | |
| "timed_calls", | |
| "median_latency_ms", | |
| "p90_latency_ms", | |
| "output_tokens_per_s", | |
| "tokens_per_game", | |
| "cost_usd_per_game", | |
| "self_hosted" | |
| ], | |
| "title": "Speed", | |
| "type": "object" | |
| }, | |
| "Summary": { | |
| "additionalProperties": false, | |
| "properties": { | |
| "games": { | |
| "type": "integer" | |
| }, | |
| "unfinished_games": { | |
| "type": "integer" | |
| }, | |
| "cost_usd": { | |
| "type": "number" | |
| }, | |
| "input_tokens": { | |
| "type": "integer" | |
| }, | |
| "output_tokens": { | |
| "type": "integer" | |
| }, | |
| "wolf_wins": { | |
| "type": "integer" | |
| }, | |
| "village_wins": { | |
| "type": "integer" | |
| }, | |
| "draws": { | |
| "type": "integer" | |
| }, | |
| "wolf_win_share": { | |
| "anyOf": [ | |
| { | |
| "type": "number" | |
| }, | |
| { | |
| "type": "null" | |
| } | |
| ] | |
| }, | |
| "village_win_share": { | |
| "anyOf": [ | |
| { | |
| "type": "number" | |
| }, | |
| { | |
| "type": "null" | |
| } | |
| ] | |
| }, | |
| "lies": { | |
| "additionalProperties": { | |
| "$ref": "#/$defs/SummaryLies" | |
| }, | |
| "propertyNames": { | |
| "enum": [ | |
| "self_tag", | |
| "jev" | |
| ] | |
| }, | |
| "type": "object" | |
| }, | |
| "top_openskill": { | |
| "anyOf": [ | |
| { | |
| "$ref": "#/$defs/TopOpenSkill" | |
| }, | |
| { | |
| "type": "null" | |
| } | |
| ] | |
| }, | |
| "top_wolf_win": { | |
| "anyOf": [ | |
| { | |
| "$ref": "#/$defs/TopWolfWin" | |
| }, | |
| { | |
| "type": "null" | |
| } | |
| ] | |
| }, | |
| "top_share_moved": { | |
| "additionalProperties": { | |
| "anyOf": [ | |
| { | |
| "$ref": "#/$defs/TopShareMoved" | |
| }, | |
| { | |
| "type": "null" | |
| } | |
| ] | |
| }, | |
| "propertyNames": { | |
| "enum": [ | |
| "self_tag", | |
| "jev" | |
| ] | |
| }, | |
| "type": "object" | |
| }, | |
| "min_comparisons": { | |
| "type": "integer" | |
| } | |
| }, | |
| "required": [ | |
| "games", | |
| "unfinished_games", | |
| "cost_usd", | |
| "input_tokens", | |
| "output_tokens", | |
| "wolf_wins", | |
| "village_wins", | |
| "draws", | |
| "wolf_win_share", | |
| "village_win_share", | |
| "lies", | |
| "top_openskill", | |
| "top_wolf_win", | |
| "top_share_moved", | |
| "min_comparisons" | |
| ], | |
| "title": "Summary", | |
| "type": "object" | |
| }, | |
| "SummaryLies": { | |
| "additionalProperties": false, | |
| "properties": { | |
| "lies": { | |
| "type": "integer" | |
| }, | |
| "statements": { | |
| "type": "integer" | |
| }, | |
| "lie_rate": { | |
| "anyOf": [ | |
| { | |
| "type": "number" | |
| }, | |
| { | |
| "type": "null" | |
| } | |
| ] | |
| }, | |
| "lie_success": { | |
| "$ref": "#/$defs/LieSuccess" | |
| } | |
| }, | |
| "required": [ | |
| "lies", | |
| "statements", | |
| "lie_rate", | |
| "lie_success" | |
| ], | |
| "title": "SummaryLies", | |
| "type": "object" | |
| }, | |
| "TopOpenSkill": { | |
| "additionalProperties": false, | |
| "properties": { | |
| "model": { | |
| "type": "string" | |
| }, | |
| "ordinal": { | |
| "type": "number" | |
| } | |
| }, | |
| "required": [ | |
| "model", | |
| "ordinal" | |
| ], | |
| "title": "TopOpenSkill", | |
| "type": "object" | |
| }, | |
| "TopShareMoved": { | |
| "additionalProperties": false, | |
| "properties": { | |
| "model": { | |
| "type": "string" | |
| }, | |
| "share_moved": { | |
| "type": "number" | |
| }, | |
| "comparisons": { | |
| "type": "integer" | |
| } | |
| }, | |
| "required": [ | |
| "model", | |
| "share_moved", | |
| "comparisons" | |
| ], | |
| "title": "TopShareMoved", | |
| "type": "object" | |
| }, | |
| "TopWolfWin": { | |
| "additionalProperties": false, | |
| "properties": { | |
| "model": { | |
| "type": "string" | |
| }, | |
| "win_rate": { | |
| "type": "number" | |
| }, | |
| "seats": { | |
| "type": "integer" | |
| } | |
| }, | |
| "required": [ | |
| "model", | |
| "win_rate", | |
| "seats" | |
| ], | |
| "title": "TopWolfWin", | |
| "type": "object" | |
| }, | |
| "Voice": { | |
| "additionalProperties": false, | |
| "properties": { | |
| "voice_id": { | |
| "type": "string" | |
| }, | |
| "fallback": { | |
| "type": "boolean" | |
| } | |
| }, | |
| "required": [ | |
| "voice_id", | |
| "fallback" | |
| ], | |
| "title": "Voice", | |
| "type": "object" | |
| } | |
| }, | |
| "additionalProperties": false, | |
| "description": "report/metrics.json: analysis.metrics.compute_metrics plus summary, speed and profiles.", | |
| "properties": { | |
| "games": { | |
| "type": "integer" | |
| }, | |
| "unfinished_games": { | |
| "type": "integer" | |
| }, | |
| "label_sources": { | |
| "items": { | |
| "enum": [ | |
| "self_tag", | |
| "jev" | |
| ], | |
| "type": "string" | |
| }, | |
| "type": "array" | |
| }, | |
| "exploratory": { | |
| "items": { | |
| "enum": [ | |
| "lie_success", | |
| "persuasion" | |
| ], | |
| "type": "string" | |
| }, | |
| "type": "array" | |
| }, | |
| "models": { | |
| "additionalProperties": { | |
| "additionalProperties": { | |
| "$ref": "#/$defs/GroupStats" | |
| }, | |
| "propertyNames": { | |
| "enum": [ | |
| "all", | |
| "wolf", | |
| "village", | |
| "seer", | |
| "doctor", | |
| "villager" | |
| ] | |
| }, | |
| "type": "object" | |
| }, | |
| "type": "object" | |
| }, | |
| "by_group": { | |
| "additionalProperties": { | |
| "$ref": "#/$defs/GroupStats" | |
| }, | |
| "propertyNames": { | |
| "enum": [ | |
| "all", | |
| "wolf", | |
| "village", | |
| "seer", | |
| "doctor", | |
| "villager" | |
| ] | |
| }, | |
| "type": "object" | |
| }, | |
| "openskill": { | |
| "$ref": "#/$defs/OpenSkill" | |
| }, | |
| "deception_matrix": { | |
| "$ref": "#/$defs/DeceptionMatrix" | |
| }, | |
| "humans": { | |
| "$ref": "#/$defs/Humans" | |
| }, | |
| "role_skill": { | |
| "$ref": "#/$defs/RoleSkill" | |
| }, | |
| "summary": { | |
| "$ref": "#/$defs/Summary" | |
| }, | |
| "speed": { | |
| "additionalProperties": { | |
| "$ref": "#/$defs/Speed" | |
| }, | |
| "type": "object" | |
| }, | |
| "profiles": { | |
| "additionalProperties": { | |
| "$ref": "#/$defs/ModelProfile" | |
| }, | |
| "type": "object" | |
| } | |
| }, | |
| "required": [ | |
| "games", | |
| "unfinished_games", | |
| "label_sources", | |
| "exploratory", | |
| "models", | |
| "by_group", | |
| "openskill", | |
| "deception_matrix", | |
| "humans", | |
| "role_skill", | |
| "summary", | |
| "speed", | |
| "profiles" | |
| ], | |
| "title": "Metrics", | |
| "type": "object" | |
| } | |