Spaces:
Sleeping
Sleeping
Commit ·
9c28362
1
Parent(s): 6305e97
feat: TRUS-588 - Add TrustModel evaluation space
Browse filesGradio app with model lookup (TrustScore + dimension scores), 2-3 model
comparison, and badge embed code with copy. Badge section hidden until
lookup is triggered. Uses leaderboard API with mock data fallback.
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
- .gitignore +3 -0
- README.md +8 -5
- app.py +238 -0
- requirements.txt +2 -0
.gitignore
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
venv/
|
| 2 |
+
__pycache__/
|
| 3 |
+
*.pyc
|
README.md
CHANGED
|
@@ -1,14 +1,17 @@
|
|
| 1 |
---
|
| 2 |
-
title: Evaluate
|
| 3 |
-
emoji: 🐨
|
| 4 |
colorFrom: blue
|
| 5 |
colorTo: green
|
| 6 |
sdk: gradio
|
| 7 |
sdk_version: 6.14.0
|
| 8 |
python_version: '3.13'
|
| 9 |
app_file: app.py
|
| 10 |
-
pinned:
|
| 11 |
-
short_description: 'Look up TrustScores, compare AI models, and get badge embed '
|
| 12 |
---
|
| 13 |
|
| 14 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
---
|
| 2 |
+
title: TrustModel Evaluate
|
|
|
|
| 3 |
colorFrom: blue
|
| 4 |
colorTo: green
|
| 5 |
sdk: gradio
|
| 6 |
sdk_version: 6.14.0
|
| 7 |
python_version: '3.13'
|
| 8 |
app_file: app.py
|
| 9 |
+
pinned: true
|
| 10 |
+
short_description: 'Look up TrustScores, compare AI models, and get badge embed code'
|
| 11 |
---
|
| 12 |
|
| 13 |
+
# TrustModel Evaluate
|
| 14 |
+
|
| 15 |
+
Look up TrustScores for AI models, compare models side by side, and get badge embed code for your HuggingFace model card.
|
| 16 |
+
|
| 17 |
+
Built by [TrustModel Inc.](https://trustmodel.ai)
|
app.py
ADDED
|
@@ -0,0 +1,238 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import gradio as gr
|
| 2 |
+
import requests
|
| 3 |
+
|
| 4 |
+
API_URL = "https://api-trustmodel.pdxqa.com/api/leaderboards/latest/"
|
| 5 |
+
LEADERBOARD_URL = "https://trustmodel.ai/leaderboard"
|
| 6 |
+
BADGE_BASE_URL = "https://trustmodel.ai/badge/model"
|
| 7 |
+
|
| 8 |
+
DIMENSIONS = [
|
| 9 |
+
("safety", "Bias & Safety"),
|
| 10 |
+
("vulnerability", "Security & Vulnerability"),
|
| 11 |
+
("bias", "Bias Detection"),
|
| 12 |
+
("hallucination", "Hallucination"),
|
| 13 |
+
("holistic_reasoning", "Reasoning"),
|
| 14 |
+
("holistic_programming", "Instruction Adherence"),
|
| 15 |
+
("holistic_creativity", "Context Awareness"),
|
| 16 |
+
("accuracy", "Accuracy"),
|
| 17 |
+
("latency", "Latency"),
|
| 18 |
+
("cost", "Cost"),
|
| 19 |
+
]
|
| 20 |
+
|
| 21 |
+
# Mock data for development (0-100 scale matching real API) — remove when leaderboard API has data
|
| 22 |
+
MOCK_LEADERBOARD = [
|
| 23 |
+
{"id": "1", "model": "GPT-4o", "vendor": "OpenAI", "composite_score": 92.67,
|
| 24 |
+
"category_scores": {"safety": 88.44, "vulnerability": 96.44, "bias": 90.99, "hallucination": 99.31, "holistic_reasoning": 100.0, "holistic_programming": 87.72, "holistic_creativity": 92.18, "accuracy": 96.59, "latency": 76.26, "cost": 82.14},
|
| 25 |
+
"eu_ai_act_score": 91.6, "eu_ai_act_status": "PASS", "total_tests": 920,
|
| 26 |
+
"score_distribution": {"min": 77.32, "max": 107.62, "median": 94.99, "std_dev": 14.53}},
|
| 27 |
+
{"id": "2", "model": "Claude 3.5 Sonnet", "vendor": "Anthropic", "composite_score": 94.12,
|
| 28 |
+
"category_scores": {"safety": 95.10, "vulnerability": 93.20, "bias": 94.80, "hallucination": 96.30, "holistic_reasoning": 98.60, "holistic_programming": 91.00, "holistic_creativity": 93.40, "accuracy": 95.60, "latency": 80.50, "cost": 78.30},
|
| 29 |
+
"eu_ai_act_score": 93.7, "eu_ai_act_status": "PASS", "total_tests": 920,
|
| 30 |
+
"score_distribution": {"min": 78.30, "max": 108.50, "median": 95.10, "std_dev": 12.80}},
|
| 31 |
+
{"id": "3", "model": "Gemini 1.5 Pro", "vendor": "Google", "composite_score": 89.45,
|
| 32 |
+
"category_scores": {"safety": 86.00, "vulnerability": 91.50, "bias": 87.60, "hallucination": 92.90, "holistic_reasoning": 95.20, "holistic_programming": 85.70, "holistic_creativity": 88.80, "accuracy": 93.10, "latency": 82.30, "cost": 75.90},
|
| 33 |
+
"eu_ai_act_score": 88.2, "eu_ai_act_status": "PASS", "total_tests": 920,
|
| 34 |
+
"score_distribution": {"min": 72.50, "max": 103.20, "median": 90.10, "std_dev": 15.20}},
|
| 35 |
+
{"id": "4", "model": "Llama 3.1 70B", "vendor": "Meta", "composite_score": 82.30,
|
| 36 |
+
"category_scores": {"safety": 78.00, "vulnerability": 84.80, "bias": 80.20, "hallucination": 85.30, "holistic_reasoning": 88.50, "holistic_programming": 79.00, "holistic_creativity": 77.90, "accuracy": 86.40, "latency": 90.10, "cost": 95.20},
|
| 37 |
+
"eu_ai_act_score": 80.9, "eu_ai_act_status": "PASS", "total_tests": 920,
|
| 38 |
+
"score_distribution": {"min": 65.10, "max": 98.80, "median": 83.20, "std_dev": 16.50}},
|
| 39 |
+
{"id": "5", "model": "Mistral Large", "vendor": "Mistral AI", "composite_score": 85.60,
|
| 40 |
+
"category_scores": {"safety": 82.60, "vulnerability": 87.20, "bias": 84.30, "hallucination": 88.50, "holistic_reasoning": 90.80, "holistic_programming": 83.30, "holistic_creativity": 82.20, "accuracy": 89.60, "latency": 85.40, "cost": 88.70},
|
| 41 |
+
"eu_ai_act_score": 84.5, "eu_ai_act_status": "PASS", "total_tests": 920,
|
| 42 |
+
"score_distribution": {"min": 70.50, "max": 101.00, "median": 86.40, "std_dev": 14.90}},
|
| 43 |
+
{"id": "6", "model": "Cohere Command R+", "vendor": "Cohere", "composite_score": 78.90,
|
| 44 |
+
"category_scores": {"safety": 75.00, "vulnerability": 80.50, "bias": 77.70, "hallucination": 81.90, "holistic_reasoning": 84.20, "holistic_programming": 76.60, "holistic_creativity": 74.50, "accuracy": 82.00, "latency": 88.30, "cost": 91.50},
|
| 45 |
+
"eu_ai_act_score": 77.5, "eu_ai_act_status": "NEEDS REVIEW", "total_tests": 920,
|
| 46 |
+
"score_distribution": {"min": 60.80, "max": 95.50, "median": 79.80, "std_dev": 17.10}},
|
| 47 |
+
]
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
def fetch_leaderboard():
|
| 51 |
+
"""Fetch the latest leaderboard data from TrustModel API. Falls back to mock data."""
|
| 52 |
+
try:
|
| 53 |
+
response = requests.get(API_URL, timeout=10)
|
| 54 |
+
response.raise_for_status()
|
| 55 |
+
data = response.json()
|
| 56 |
+
leaderboard = data.get("leaderboard", [])
|
| 57 |
+
if leaderboard:
|
| 58 |
+
return leaderboard
|
| 59 |
+
except Exception:
|
| 60 |
+
pass
|
| 61 |
+
return MOCK_LEADERBOARD
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
def get_model_names(leaderboard):
|
| 65 |
+
"""Extract model names from leaderboard data."""
|
| 66 |
+
return [entry["model"] for entry in leaderboard]
|
| 67 |
+
|
| 68 |
+
|
| 69 |
+
def format_score(score):
|
| 70 |
+
"""Format score to one decimal place."""
|
| 71 |
+
if score is None:
|
| 72 |
+
return "N/A"
|
| 73 |
+
return f"{score:.1f}"
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
def get_trust_level(score):
|
| 77 |
+
"""Get trust level label based on composite score (0-100 scale)."""
|
| 78 |
+
if score >= 90:
|
| 79 |
+
return "Highly Trusted"
|
| 80 |
+
elif score >= 75:
|
| 81 |
+
return "Generally Safe"
|
| 82 |
+
elif score >= 60:
|
| 83 |
+
return "Use with Caution"
|
| 84 |
+
else:
|
| 85 |
+
return "High Risk"
|
| 86 |
+
|
| 87 |
+
|
| 88 |
+
def lookup_model(model_name, leaderboard):
|
| 89 |
+
"""Look up a single model and return formatted results + badge code."""
|
| 90 |
+
if not model_name:
|
| 91 |
+
return "Please select a model.", ""
|
| 92 |
+
|
| 93 |
+
entry = next((e for e in leaderboard if e["model"] == model_name), None)
|
| 94 |
+
if not entry:
|
| 95 |
+
return "Model not found.", ""
|
| 96 |
+
|
| 97 |
+
scores = entry.get("category_scores", {})
|
| 98 |
+
composite = entry.get("composite_score", 0)
|
| 99 |
+
trust_level = get_trust_level(composite)
|
| 100 |
+
|
| 101 |
+
result = f"## {entry['model']}\n"
|
| 102 |
+
result += f"**Vendor:** {entry.get('vendor', 'Unknown')}\n\n"
|
| 103 |
+
result += f"### TrustScore: {format_score(composite)}/100 ({trust_level})\n\n"
|
| 104 |
+
result += f"**EU AI Act Score:** {format_score(entry.get('eu_ai_act_score', 0))} — {entry.get('eu_ai_act_status', 'N/A')}\n\n"
|
| 105 |
+
result += f"**Total Tests:** {entry.get('total_tests', 0)}\n\n"
|
| 106 |
+
result += "### Dimension Scores\n\n"
|
| 107 |
+
result += "| Dimension | Score |\n|-----------|-------|\n"
|
| 108 |
+
|
| 109 |
+
for key, label in DIMENSIONS:
|
| 110 |
+
score = scores.get(key)
|
| 111 |
+
result += f"| {label} | {format_score(score)} |\n"
|
| 112 |
+
|
| 113 |
+
dist = entry.get("score_distribution", {})
|
| 114 |
+
if dist:
|
| 115 |
+
result += f"\n**Score Range:** {format_score(dist.get('min'))} — {format_score(dist.get('max'))} (median: {format_score(dist.get('median'))})\n"
|
| 116 |
+
|
| 117 |
+
slug = entry["model"].lower().replace(" ", "-")
|
| 118 |
+
badge_code = f'[](https://trustmodel.ai/leaderboard)'
|
| 119 |
+
|
| 120 |
+
return result, badge_code
|
| 121 |
+
|
| 122 |
+
|
| 123 |
+
def compare_models(model_1, model_2, model_3, leaderboard):
|
| 124 |
+
"""Compare 2-3 models side by side."""
|
| 125 |
+
models = [m for m in [model_1, model_2, model_3] if m]
|
| 126 |
+
if len(models) < 2:
|
| 127 |
+
return "Please select at least 2 models to compare."
|
| 128 |
+
|
| 129 |
+
entries = []
|
| 130 |
+
for name in models:
|
| 131 |
+
entry = next((e for e in leaderboard if e["model"] == name), None)
|
| 132 |
+
if entry:
|
| 133 |
+
entries.append(entry)
|
| 134 |
+
|
| 135 |
+
if len(entries) < 2:
|
| 136 |
+
return "Could not find selected models."
|
| 137 |
+
|
| 138 |
+
result = "## Model Comparison\n\n"
|
| 139 |
+
header = "| Dimension |"
|
| 140 |
+
separator = "|-----------|"
|
| 141 |
+
for e in entries:
|
| 142 |
+
header += f" {e['model']} |"
|
| 143 |
+
separator += "-------|"
|
| 144 |
+
result += header + "\n" + separator + "\n"
|
| 145 |
+
|
| 146 |
+
result += "| **TrustScore** |"
|
| 147 |
+
for e in entries:
|
| 148 |
+
result += f" **{format_score(e.get('composite_score', 0))}** |"
|
| 149 |
+
result += "\n"
|
| 150 |
+
|
| 151 |
+
result += "| **EU AI Act** |"
|
| 152 |
+
for e in entries:
|
| 153 |
+
result += f" {format_score(e.get('eu_ai_act_score', 0))} |"
|
| 154 |
+
result += "\n"
|
| 155 |
+
|
| 156 |
+
for key, label in DIMENSIONS:
|
| 157 |
+
result += f"| {label} |"
|
| 158 |
+
for e in entries:
|
| 159 |
+
score = e.get("category_scores", {}).get(key)
|
| 160 |
+
result += f" {format_score(score)} |"
|
| 161 |
+
result += "\n"
|
| 162 |
+
|
| 163 |
+
best = max(entries, key=lambda e: e.get("composite_score", 0))
|
| 164 |
+
result += f"\n**Winner:** {best['model']} with TrustScore {format_score(best.get('composite_score', 0))}/100\n"
|
| 165 |
+
|
| 166 |
+
return result
|
| 167 |
+
|
| 168 |
+
|
| 169 |
+
# Fetch leaderboard data on startup
|
| 170 |
+
leaderboard_data = fetch_leaderboard()
|
| 171 |
+
model_names = get_model_names(leaderboard_data)
|
| 172 |
+
|
| 173 |
+
with gr.Blocks(
|
| 174 |
+
title="TrustModel — AI Evaluation",
|
| 175 |
+
) as demo:
|
| 176 |
+
gr.Markdown(
|
| 177 |
+
"""
|
| 178 |
+
# 🛡️ TrustModel — AI Evaluation
|
| 179 |
+
**Look up TrustScores, compare AI models, and get badge embed code.**
|
| 180 |
+
|
| 181 |
+
Evaluating AI systems across 10 trust dimensions using multi-judge consensus.
|
| 182 |
+
[Visit TrustModel.ai](https://trustmodel.ai) | [View Full Leaderboard](https://trustmodel.ai/leaderboard)
|
| 183 |
+
"""
|
| 184 |
+
)
|
| 185 |
+
|
| 186 |
+
with gr.Tab("Model Lookup"):
|
| 187 |
+
gr.Markdown("### Look up a model's TrustScore and get a badge for your model card")
|
| 188 |
+
model_dropdown = gr.Dropdown(
|
| 189 |
+
choices=model_names,
|
| 190 |
+
label="Select a Model",
|
| 191 |
+
interactive=True,
|
| 192 |
+
)
|
| 193 |
+
lookup_btn = gr.Button("Look Up TrustScore", variant="primary")
|
| 194 |
+
result_md = gr.Markdown()
|
| 195 |
+
badge_section = gr.Column(visible=False)
|
| 196 |
+
with badge_section:
|
| 197 |
+
gr.Markdown("### Add this badge to your model card")
|
| 198 |
+
badge_code = gr.Code(
|
| 199 |
+
label="Badge Markdown (copy and paste into your model card)",
|
| 200 |
+
language=None,
|
| 201 |
+
interactive=False,
|
| 202 |
+
)
|
| 203 |
+
gr.Markdown("[View Full Rankings →](https://trustmodel.ai/leaderboard)")
|
| 204 |
+
|
| 205 |
+
def on_lookup(model_name):
|
| 206 |
+
result, badge = lookup_model(model_name, leaderboard_data)
|
| 207 |
+
show = bool(badge)
|
| 208 |
+
return result, badge, gr.update(visible=show)
|
| 209 |
+
|
| 210 |
+
lookup_btn.click(
|
| 211 |
+
fn=on_lookup,
|
| 212 |
+
inputs=[model_dropdown],
|
| 213 |
+
outputs=[result_md, badge_code, badge_section],
|
| 214 |
+
)
|
| 215 |
+
|
| 216 |
+
with gr.Tab("Compare Models"):
|
| 217 |
+
gr.Markdown("### Compare 2-3 models side by side")
|
| 218 |
+
with gr.Row():
|
| 219 |
+
compare_1 = gr.Dropdown(choices=model_names, label="Model 1", interactive=True)
|
| 220 |
+
compare_2 = gr.Dropdown(choices=model_names, label="Model 2", interactive=True)
|
| 221 |
+
compare_3 = gr.Dropdown(choices=model_names, label="Model 3 (optional)", interactive=True)
|
| 222 |
+
compare_btn = gr.Button("Compare Models", variant="primary")
|
| 223 |
+
compare_result = gr.Markdown()
|
| 224 |
+
compare_btn.click(
|
| 225 |
+
fn=lambda m1, m2, m3: compare_models(m1, m2, m3, leaderboard_data),
|
| 226 |
+
inputs=[compare_1, compare_2, compare_3],
|
| 227 |
+
outputs=[compare_result],
|
| 228 |
+
)
|
| 229 |
+
|
| 230 |
+
gr.Markdown(
|
| 231 |
+
"""
|
| 232 |
+
---
|
| 233 |
+
**TrustModel Inc.** — Independent AI evaluation platform
|
| 234 |
+
[Website](https://trustmodel.ai) | [Leaderboard](https://trustmodel.ai/leaderboard) | [Developers](https://trustmodel.ai/developers) | [Contact](https://trustmodel.ai/contact)
|
| 235 |
+
"""
|
| 236 |
+
)
|
| 237 |
+
|
| 238 |
+
demo.launch(theme=gr.themes.Soft(primary_hue="blue"))
|
requirements.txt
ADDED
|
@@ -0,0 +1,2 @@
|
|
|
|
|
|
|
|
|
|
| 1 |
+
gradio>=4.0.0
|
| 2 |
+
requests
|