import streamlit as st st.set_page_config(layout="wide") header = st.container() margin1, center, margin2 = header.columns([1,9,1]) margin1, c1, c2, c3, c4, margin2 = header.columns([1,2.25,2.25,2.25,2.25,1]) header.write("""
""", unsafe_allow_html=True) st.markdown( """ """, unsafe_allow_html=True ) with center: st.title("AI Model Fairness Auditing") # -------------------------------------------------- # Strategy Definitions # -------------------------------------------------- CHOICES = { "owner": { "Provide to the Auditor Access to...": [ ("The Actual Deployed Model", "same_model"), ("A Fake Model Fabricated for the Audit", "fair_model"), ("Fabricated Inference Outputs (API only)", "fabricated_outputs"), ], "Train the Audit Model...": [ ("By Ensuring Generalizability", "generalize"), ("By Overfitting to One Audit Dataset", "overfit"), ("By Overfitting to Audit Dataset-like Distributions", "overfitdist"), ], }, "auditor": { "Audit with...": [ ("One Dataset", "auditone"), ("Multiple Datasets", "auditmany"), ], "Collect the Dataset(s) From...": [ ("Public Source(s)", "public"), ("Private Source(s)", "private"), ], "Infer the Model...": [ ("Through an API", "blackbox"), ("Through a Local Copy", "internals"), ], } } FORBIDDEN_COMBINATIONS = [ ("fabricated_outputs", "internals"), ("internals", "fabricated_outputs"), ("fabricated_outputs", "generalize"), ("fabricated_outputs", "overfit"), ("fabricated_outputs", "overfitdist"), ] def get_ui_choices(role): return { label: [opt[0] for opt in options] for label, options in CHOICES[role].items() } def get_action_map(): return { label: code for role in CHOICES.values() for options in role.values() for (label, code) in options } def get_active_actions(selection_dict): action_map = get_action_map() return [action_map[v] for v in selection_dict.values()] def is_forbidden(candidate_code, other_selected_codes): for a, b in FORBIDDEN_COMBINATIONS: if candidate_code == b and a in other_selected_codes: return True return False owner_choices = get_ui_choices("owner") auditor_choices = get_ui_choices("auditor") # -------------------------------------------------- # Session State # -------------------------------------------------- if "owner_selected" not in st.session_state: st.session_state.owner_selected = { "Provide to the Auditor Access to...": "The Actual Deployed Model", "Train the Audit Model...": "By Ensuring Generalizability" } if "auditor_selected" not in st.session_state: st.session_state.auditor_selected = { "Audit with...": "One Dataset", "Collect the Dataset(s) From...": "Public Source(s)", "Infer the Model...": "Through an API" } owner_active_list = get_active_actions(st.session_state.owner_selected) auditor_active_list = get_active_actions(st.session_state.auditor_selected) # -------------------------------------------------- # Toggle Functions # -------------------------------------------------- def toggle_owner(action): if action in st.session_state.owner_selected: st.session_state.owner_selected.remove(action) else: st.session_state.owner_selected.append(action) def toggle_auditor(action): if action in st.session_state.auditor_selected: st.session_state.auditor_selected.remove(action) else: st.session_state.auditor_selected.append(action) # -------------------------------------------------- # Layout # -------------------------------------------------- # left, right = st.columns(2) margin1, left, gap, right, margin2 = st.columns([1,8,1,8,1]) def select_choice(group, label, choice): st.session_state[group][label] = choice def choice_card(group_name, label, options, action_map, other_active): selected = st.session_state[group_name].get(label, None) # If the currently selected option becomes forbidden, deselect it if selected is not None and is_forbidden(action_map[selected], other_active): st.session_state[group_name].pop(label) selected = None # update active lists global owner_active_list, auditor_active_list owner_active_list = get_active_actions(st.session_state.owner_selected) auditor_active_list = get_active_actions(st.session_state.auditor_selected) with st.container(border=True): st.markdown(f"**{label}**") cols = st.columns(len(options)) for i, option in enumerate(options): code = action_map[option] disabled = (is_forbidden(code, other_active)) with cols[i]: st.button( option, key=f"{group_name}_{label}_{i}", on_click=select_choice, args=(group_name, label, option), use_container_width=True, type="primary" if selected == option else "tertiary", disabled=disabled ) # -------------------------------------------------- # Model Owner # -------------------------------------------------- with right: if "same_model" in owner_active_list and "generalize" in owner_active_list: st.subheader("Model Owner (Honest)") else: st.subheader("Model Owner (Dishonest)") for label, options in owner_choices.items(): choice_card( "owner_selected", label, options, get_action_map(), auditor_active_list + owner_active_list ) # -------------------------------------------------- # Auditor # -------------------------------------------------- with left: st.subheader("Auditor") for label, options in auditor_choices.items(): choice_card( "auditor_selected", label, options, get_action_map(), auditor_active_list + owner_active_list ) # -------------------------------------------------- # Score Calculation # -------------------------------------------------- nb_auditor_cost = 0 if "auditmany" in auditor_active_list: nb_auditor_cost += 1 if "private" in auditor_active_list: nb_auditor_cost += 1 if "internals" in auditor_active_list: nb_auditor_cost += 1 auditor_cost = "🟢 Low" if nb_auditor_cost == 1: auditor_cost = "🟡 Medium" if nb_auditor_cost == 2: auditor_cost = "🔴 High" if nb_auditor_cost == 3: auditor_cost = "🔴 Very High" nb_owner_cost = 0 if "overfitdist" in owner_active_list: nb_owner_cost += 1 if "generalize" in owner_active_list: nb_owner_cost += 2 if "fair_model" in owner_active_list: nb_owner_cost += 1 if "fabricated_outputs" in owner_active_list: nb_owner_cost += 1 owner_cost = "🟢 Low" if nb_owner_cost == 1: owner_cost = "🟡 Medium" if nb_owner_cost == 2: owner_cost = "🔴 High" if nb_owner_cost == 3: owner_cost = "🔴 Very High" reliability = "❌ Low" if "same_model" in owner_active_list and "generalize" in owner_active_list: reliability = "✅ High" if ("same_model" in owner_active_list or "fair_model" in owner_active_list) and "overfit" in owner_active_list and "auditmany" in auditor_active_list: reliability = "✅ High" if ("same_model" in owner_active_list or "fair_model" in owner_active_list) and "overfit" in owner_active_list and "auditone" in auditor_active_list and "private" in auditor_active_list: reliability = "⚠️ Medium" if ("same_model" in owner_active_list or "fair_model" in owner_active_list) and "overfitdist" in owner_active_list and "auditmany" in auditor_active_list and "private" in auditor_active_list: reliability = "⚠️ Medium" openness = "None" if "internals" in auditor_active_list: openness = "🔍 High (Model Weight Sharing)" if len(auditor_active_list)==0 and len(owner_active_list)==0: auditor_cost, owner_cost, reliability, openness = "N/A", "N/A", "N/A", "N/A" # -------------------------------------------------- # Dashboard # -------------------------------------------------- with c1: st.write("**Cost for The Auditor**") st.write(auditor_cost) with c2: st.write("**Cost for The Owner**") st.write(owner_cost) with c3: st.write("**Audit Reliability**") st.write(reliability) with c4: st.write("**Openness Requirements**") st.write(openness)