| import os |
| import sys |
| import warnings |
|
|
| |
| warnings.filterwarnings("ignore", category=DeprecationWarning) |
| warnings.filterwarnings("ignore", category=FutureWarning) |
| warnings.filterwarnings("ignore", category=UserWarning) |
|
|
| os.environ["GRADIO_SSR"] = "0" |
| os.environ["GRADIO_SSR_MODE"] = "0" |
| os.environ["GRADIO_ANALYTICS_ENABLED"] = "False" |
|
|
| import json |
| import gradio as gr |
| import pandas as pd |
| from gradio_leaderboard import Leaderboard, SelectColumns |
|
|
| CURRENT_DIR = os.path.abspath(os.path.dirname(__file__)) |
| sys.path.insert(0, CURRENT_DIR) |
| for sub in ["src", "src/display", "src/submission", "src/leaderboard", "src/audit"]: |
| p = os.path.join(CURRENT_DIR, sub) |
| if os.path.exists(p): |
| sys.path.insert(0, p) |
|
|
| |
| try: |
| from src.db import mongo_get_all_certificates, mongo_save_certificate |
| except ImportError: |
| from db import mongo_get_all_certificates, mongo_save_certificate |
|
|
| |
| try: |
| from src.about import CITATION_BUTTON_LABEL, CITATION_BUTTON_TEXT, INTRODUCTION_TEXT, TITLE |
| except ImportError: |
| from about import CITATION_BUTTON_LABEL, CITATION_BUTTON_TEXT, INTRODUCTION_TEXT, TITLE |
|
|
| |
| try: |
| from src.display.css_html_js import custom_css |
| except ImportError: |
| try: |
| from src.css_html_js import custom_css |
| except ImportError: |
| try: |
| from display.css_html_js import custom_css |
| except ImportError: |
| from css_html_js import custom_css |
|
|
| |
| try: |
| from src.display.formatting import build_top_3_cards_html, render_audit_details_panel |
| except ImportError: |
| try: |
| from src.formatting import build_top_3_cards_html, render_audit_details_panel |
| except ImportError: |
| try: |
| from display.formatting import build_top_3_cards_html, render_audit_details_panel |
| except ImportError: |
| from formatting import build_top_3_cards_html, render_audit_details_panel |
|
|
| |
| try: |
| from src.display.utils import BENCHMARK_COLS, COLS, AutoEvalColumn, fields |
| except ImportError: |
| try: |
| from src.utils import BENCHMARK_COLS, COLS, AutoEvalColumn, fields |
| except ImportError: |
| try: |
| from display.utils import BENCHMARK_COLS, COLS, AutoEvalColumn, fields |
| except ImportError: |
| from utils import BENCHMARK_COLS, COLS, AutoEvalColumn, fields |
|
|
| |
| try: |
| from src.envs import EVAL_RESULTS_PATH |
| except ImportError: |
| from envs import EVAL_RESULTS_PATH |
|
|
| |
| try: |
| from src.populate import get_leaderboard_df, get_top_3_eval_cards |
| except ImportError: |
| from populate import get_leaderboard_df, get_top_3_eval_cards |
|
|
| |
| try: |
| from src.submission.submit import audit_or_search_model, clean_model_name, get_certificate_by_model_name |
| except ImportError: |
| try: |
| from src.submit import audit_or_search_model, clean_model_name, get_certificate_by_model_name |
| except ImportError: |
| try: |
| from submission.submit import audit_or_search_model, clean_model_name, get_certificate_by_model_name |
| except ImportError: |
| from submit import audit_or_search_model, clean_model_name, get_certificate_by_model_name |
|
|
|
|
| def seed_demo_results(): |
| if not os.path.exists(EVAL_RESULTS_PATH): |
| os.makedirs(EVAL_RESULTS_PATH, exist_ok=True) |
|
|
| existing_docs = mongo_get_all_certificates() |
| if not existing_docs and not os.listdir(EVAL_RESULTS_PATH): |
| mock_cert = { |
| "status": "ok", |
| "config": { |
| "model_name": "Qwen/Qwen2.5-0.5B-Instruct", |
| "model_sha": "main", |
| "architecture": "Qwen2ForCausalLM", |
| "params": 0.5, |
| "precision": "bfloat16", |
| "license": "apache-2.0", |
| }, |
| "audited_at": "2026-08-16 12:00:00", |
| "spectral": { |
| "stable_rank_mean": 86.74, |
| "effective_rank_mean": 430.55, |
| "condition_number_mean": 41650.41, |
| "matrices_sampled": 48, |
| }, |
| "observer": { |
| "tau": 0.95, |
| "D": 896, |
| "d": 428, |
| "blind_fraction": 0.7718, |
| "calibration_tokens": 10500, |
| "calibration_texts": 200, |
| "token_to_dim_ratio": 11.7, |
| "sample_adequate": True, |
| "min_ratio_floor": 5.0, |
| }, |
| "behavioral": { |
| "n_probes": 18, |
| "correct_count": 11, |
| "factual_accuracy": 0.6111, |
| "paraphrase_fidelity": 0.9058, |
| "per_item": [], |
| }, |
| "composite": { |
| "structural_risk": 43.3, |
| "behavioral_risk": 38.9, |
| "unvalidated_composite_score": 40.7, |
| "verdict": "MODERATE RISK", |
| "disclaimer": "Diagnostic signal only.", |
| }, |
| } |
| with open(os.path.join(EVAL_RESULTS_PATH, "Qwen__Qwen2.5-0.5B-Instruct_main.json"), "w", encoding="utf-8") as f: |
| json.dump(mock_cert, f, indent=2) |
| mongo_save_certificate(mock_cert) |
|
|
|
|
| seed_demo_results() |
|
|
| LEADERBOARD_DF = get_leaderboard_df(EVAL_RESULTS_PATH, "", COLS, BENCHMARK_COLS) |
| top_3_records = get_top_3_eval_cards(EVAL_RESULTS_PATH) |
| initial_cert = get_certificate_by_model_name("Qwen/Qwen2.5-0.5B-Instruct") if top_3_records else {} |
|
|
|
|
| def on_select_model(evt: gr.SelectData, df: pd.DataFrame): |
| """Triggered whenever any cell or row is clicked in the Leaderboard.""" |
| try: |
| if evt.value: |
| cleaned = clean_model_name(str(evt.value)) |
| if cleaned: |
| cert = get_certificate_by_model_name(cleaned) |
| if cert and cert.get("status") == "ok": |
| return render_audit_details_panel(cert) |
|
|
| if isinstance(df, pd.DataFrame) and evt.index is not None: |
| row_idx = evt.index[0] if isinstance(evt.index, (list, tuple)) else evt.index |
| if row_idx < len(df): |
| model_col = AutoEvalColumn.model.name |
| if model_col in df.columns: |
| raw_model = str(df.iloc[row_idx][model_col]) |
| cleaned = clean_model_name(raw_model) |
| cert = get_certificate_by_model_name(cleaned) |
| if cert: |
| return render_audit_details_panel(cert) |
|
|
| return render_audit_details_panel({}) |
| except Exception as e: |
| print(f"Selection error: {e}") |
| return render_audit_details_panel({}) |
|
|
|
|
| def on_top_card_click(model_name: str): |
| """Triggered whenever one of the Top 3 Cards is clicked.""" |
| cleaned = clean_model_name(model_name) |
| cert = get_certificate_by_model_name(cleaned) |
| if cert: |
| return render_audit_details_panel(cert) |
| return render_audit_details_panel({}) |
|
|
|
|
| def init_leaderboard(dataframe): |
| return Leaderboard( |
| value=dataframe, |
| datatype=[c.type for c in fields(AutoEvalColumn)], |
| select_columns=SelectColumns( |
| default_selection=[c.name for c in fields(AutoEvalColumn) if c.displayed_by_default], |
| cant_deselect=[c.name for c in fields(AutoEvalColumn) if c.never_hidden], |
| label="Select Columns to Display:", |
| ), |
| search_columns=[AutoEvalColumn.model.name, AutoEvalColumn.license.name], |
| hide_columns=[c.name for c in fields(AutoEvalColumn) if c.hidden], |
| interactive=False, |
| ) |
|
|
|
|
| def refresh_dashboard(): |
| df = get_leaderboard_df(EVAL_RESULTS_PATH, "", COLS, BENCHMARK_COLS) |
| top3 = get_top_3_eval_cards(EVAL_RESULTS_PATH) |
| return df, build_top_3_cards_html(top3) |
|
|
|
|
| custom_head_js = """ |
| <script> |
| window.selectTopModel = function(modelName) { |
| if (!modelName) return; |
| var el = document.querySelector('#hidden_card_model_input textarea') || document.querySelector('#hidden_card_model_input input'); |
| var btn = document.querySelector('#hidden_card_trigger_btn'); |
| if (el && btn) { |
| el.value = modelName; |
| el.dispatchEvent(new Event('input', { bubbles: true })); |
| el.dispatchEvent(new Event('change', { bubbles: true })); |
| btn.click(); |
| } |
| }; |
| </script> |
| """ |
|
|
| combined_css = custom_css + """ |
| .hidden-bridge { |
| position: absolute !important; |
| opacity: 0 !important; |
| pointer-events: none !important; |
| height: 0px !important; |
| width: 0px !important; |
| overflow: hidden !important; |
| margin: 0 !important; |
| padding: 0 !important; |
| } |
| """ |
|
|
| demo = gr.Blocks( |
| theme=gr.themes.Soft(primary_hue="sky", secondary_hue="slate"), |
| css=combined_css, |
| head=custom_head_js, |
| title="LLM-X-RAY Observatory", |
| ) |
|
|
| with demo: |
| gr.HTML(TITLE) |
| gr.Markdown(INTRODUCTION_TEXT) |
|
|
| hidden_card_input = gr.Textbox(elem_id="hidden_card_model_input", elem_classes=["hidden-bridge"]) |
| hidden_card_btn = gr.Button(elem_id="hidden_card_trigger_btn", elem_classes=["hidden-bridge"]) |
|
|
| top_cards_display = gr.HTML(build_top_3_cards_html(top_3_records)) |
|
|
| with gr.Row(): |
| with gr.Column(scale=3): |
| search_or_audit_input = gr.Textbox( |
| label="🔍 Search Registry or Enter Hugging Face Model URL to Audit", |
| placeholder="e.g. Qwen/Qwen2.5-0.5B-Instruct or MiniMaxAI/MiniMax-Music3", |
| lines=1, |
| max_lines=1, |
| show_label=True, |
| ) |
| trust_remote_code_checkbox = gr.Checkbox( |
| label="Trust Remote Code (enabled for custom architectures like MiniCPM / Supra / DeepSeek)", |
| value=True, |
| ) |
| audit_button = gr.Button("🔬 SCAN & AUDIT MODEL", variant="primary") |
| status_box = gr.HTML() |
|
|
| with gr.Column(scale=4): |
| results_display_panel = gr.HTML(render_audit_details_panel(initial_cert)) |
|
|
| leaderboard_table = init_leaderboard(LEADERBOARD_DF) |
|
|
| audit_button.click( |
| fn=audit_or_search_model, |
| inputs=[search_or_audit_input, trust_remote_code_checkbox], |
| outputs=[status_box, leaderboard_table, top_cards_display, results_display_panel], |
| ) |
| search_or_audit_input.submit( |
| fn=audit_or_search_model, |
| inputs=[search_or_audit_input, trust_remote_code_checkbox], |
| outputs=[status_box, leaderboard_table, top_cards_display, results_display_panel], |
| ) |
|
|
| leaderboard_table.select( |
| fn=on_select_model, |
| inputs=[leaderboard_table], |
| outputs=[results_display_panel], |
| ) |
|
|
| hidden_card_btn.click( |
| fn=on_top_card_click, |
| inputs=[hidden_card_input], |
| outputs=[results_display_panel], |
| ) |
|
|
| demo.load(fn=refresh_dashboard, outputs=[leaderboard_table, top_cards_display]) |
|
|
| with gr.Row(): |
| with gr.Accordion("📙 Methodology & Limitations", open=False): |
| gr.Textbox(value=CITATION_BUTTON_TEXT, label=CITATION_BUTTON_LABEL, lines=8, show_copy_button=True) |
|
|
| if __name__ == "__main__": |
| demo.queue(default_concurrency_limit=5).launch() |
