Download app.py from verifiability-tutorial/verifiability: direct link, hf CLI and curl.
- Browser
- Download file 8.94 kB
-
https://huggingface.co/spaces/verifiability-tutorial/verifiability/resolve/main/app.py
- Command line
-
hf download hf://spaces/verifiability-tutorial/verifiability/app.py
-
curl -L -o app.py https://huggingface.co/spaces/verifiability-tutorial/verifiability/resolve/main/app.py
8.94 kB
| import streamlit as st | |
| st.set_page_config(layout="wide") | |
| header = st.container() | |
| margin1, center, margin2 = header.columns([1,9,1]) | |
| margin1, c1, c2, c3, c4, margin2 = header.columns([1,2.25,2.25,2.25,2.25,1]) | |
| header.write("""<div class='fixed-header'/>""", unsafe_allow_html=True) | |
| st.markdown( | |
| """ | |
| <style> | |
| div[data-testid="stVerticalBlock"] div:has(div.fixed-header) { | |
| position: sticky; | |
| top: 2.875rem; | |
| z-index: 999; | |
| } | |
| .fixed-header { | |
| border-bottom: 1px solid black; | |
| } | |
| </style> | |
| """, | |
| unsafe_allow_html=True | |
| ) | |
| with center: | |
| st.title("AI Model Fairness Auditing") | |
| # -------------------------------------------------- | |
| # Strategy Definitions | |
| # -------------------------------------------------- | |
| CHOICES = { | |
| "owner": { | |
| "Provide to the Auditor Access to...": [ | |
| ("The Actual Deployed Model", "same_model"), | |
| ("A Fake Model Fabricated for the Audit", "fair_model"), | |
| ("Fabricated Inference Outputs (API only)", "fabricated_outputs"), | |
| ], | |
| "Train the Audit Model...": [ | |
| ("By Ensuring Generalizability", "generalize"), | |
| ("By Overfitting to One Audit Dataset", "overfit"), | |
| ("By Overfitting to Audit Dataset-like Distributions", "overfitdist"), | |
| ], | |
| }, | |
| "auditor": { | |
| "Audit with...": [ | |
| ("One Dataset", "auditone"), | |
| ("Multiple Datasets", "auditmany"), | |
| ], | |
| "Collect the Dataset(s) From...": [ | |
| ("Public Source(s)", "public"), | |
| ("Private Source(s)", "private"), | |
| ], | |
| "Infer the Model...": [ | |
| ("Through an API", "blackbox"), | |
| ("Through a Local Copy", "internals"), | |
| ], | |
| } | |
| } | |
| FORBIDDEN_COMBINATIONS = [ | |
| ("fabricated_outputs", "internals"), | |
| ("internals", "fabricated_outputs"), | |
| ("fabricated_outputs", "generalize"), | |
| ("fabricated_outputs", "overfit"), | |
| ("fabricated_outputs", "overfitdist"), | |
| ] | |
| def get_ui_choices(role): | |
| return { | |
| label: [opt[0] for opt in options] | |
| for label, options in CHOICES[role].items() | |
| } | |
| def get_action_map(): | |
| return { | |
| label: code | |
| for role in CHOICES.values() | |
| for options in role.values() | |
| for (label, code) in options | |
| } | |
| def get_active_actions(selection_dict): | |
| action_map = get_action_map() | |
| return [action_map[v] for v in selection_dict.values()] | |
| def is_forbidden(candidate_code, other_selected_codes): | |
| for a, b in FORBIDDEN_COMBINATIONS: | |
| if candidate_code == b and a in other_selected_codes: | |
| return True | |
| return False | |
| owner_choices = get_ui_choices("owner") | |
| auditor_choices = get_ui_choices("auditor") | |
| # -------------------------------------------------- | |
| # Session State | |
| # -------------------------------------------------- | |
| if "owner_selected" not in st.session_state: | |
| st.session_state.owner_selected = { | |
| "Provide to the Auditor Access to...": "The Actual Deployed Model", | |
| "Train the Audit Model...": "By Ensuring Generalizability" | |
| } | |
| if "auditor_selected" not in st.session_state: | |
| st.session_state.auditor_selected = { | |
| "Audit with...": "One Dataset", | |
| "Collect the Dataset(s) From...": "Public Source(s)", | |
| "Infer the Model...": "Through an API" | |
| } | |
| owner_active_list = get_active_actions(st.session_state.owner_selected) | |
| auditor_active_list = get_active_actions(st.session_state.auditor_selected) | |
| # -------------------------------------------------- | |
| # Toggle Functions | |
| # -------------------------------------------------- | |
| def toggle_owner(action): | |
| if action in st.session_state.owner_selected: | |
| st.session_state.owner_selected.remove(action) | |
| else: | |
| st.session_state.owner_selected.append(action) | |
| def toggle_auditor(action): | |
| if action in st.session_state.auditor_selected: | |
| st.session_state.auditor_selected.remove(action) | |
| else: | |
| st.session_state.auditor_selected.append(action) | |
| # -------------------------------------------------- | |
| # Layout | |
| # -------------------------------------------------- | |
| # left, right = st.columns(2) | |
| margin1, left, gap, right, margin2 = st.columns([1,8,1,8,1]) | |
| def select_choice(group, label, choice): | |
| st.session_state[group][label] = choice | |
| def choice_card(group_name, label, options, action_map, other_active): | |
| selected = st.session_state[group_name].get(label, None) | |
| # If the currently selected option becomes forbidden, deselect it | |
| if selected is not None and is_forbidden(action_map[selected], other_active): | |
| st.session_state[group_name].pop(label) | |
| selected = None | |
| # update active lists | |
| global owner_active_list, auditor_active_list | |
| owner_active_list = get_active_actions(st.session_state.owner_selected) | |
| auditor_active_list = get_active_actions(st.session_state.auditor_selected) | |
| with st.container(border=True): | |
| st.markdown(f"**{label}**") | |
| cols = st.columns(len(options)) | |
| for i, option in enumerate(options): | |
| code = action_map[option] | |
| disabled = (is_forbidden(code, other_active)) | |
| with cols[i]: | |
| st.button( | |
| option, | |
| key=f"{group_name}_{label}_{i}", | |
| on_click=select_choice, | |
| args=(group_name, label, option), | |
| use_container_width=True, | |
| type="primary" if selected == option else "tertiary", | |
| disabled=disabled | |
| ) | |
| # -------------------------------------------------- | |
| # Model Owner | |
| # -------------------------------------------------- | |
| with right: | |
| if "same_model" in owner_active_list and "generalize" in owner_active_list: | |
| st.subheader("Model Owner (Honest)") | |
| else: | |
| st.subheader("Model Owner (Dishonest)") | |
| for label, options in owner_choices.items(): | |
| choice_card( | |
| "owner_selected", | |
| label, | |
| options, | |
| get_action_map(), | |
| auditor_active_list + owner_active_list | |
| ) | |
| # -------------------------------------------------- | |
| # Auditor | |
| # -------------------------------------------------- | |
| with left: | |
| st.subheader("Auditor") | |
| for label, options in auditor_choices.items(): | |
| choice_card( | |
| "auditor_selected", | |
| label, | |
| options, | |
| get_action_map(), | |
| auditor_active_list + owner_active_list | |
| ) | |
| # -------------------------------------------------- | |
| # Score Calculation | |
| # -------------------------------------------------- | |
| nb_auditor_cost = 0 | |
| if "auditmany" in auditor_active_list: | |
| nb_auditor_cost += 1 | |
| if "private" in auditor_active_list: | |
| nb_auditor_cost += 1 | |
| if "internals" in auditor_active_list: | |
| nb_auditor_cost += 1 | |
| auditor_cost = "π’ Low" | |
| if nb_auditor_cost == 1: | |
| auditor_cost = "π‘ Medium" | |
| if nb_auditor_cost == 2: | |
| auditor_cost = "π΄ High" | |
| if nb_auditor_cost == 3: | |
| auditor_cost = "π΄ Very High" | |
| nb_owner_cost = 0 | |
| if "overfitdist" in owner_active_list: | |
| nb_owner_cost += 1 | |
| if "generalize" in owner_active_list: | |
| nb_owner_cost += 2 | |
| if "fair_model" in owner_active_list: | |
| nb_owner_cost += 1 | |
| if "fabricated_outputs" in owner_active_list: | |
| nb_owner_cost += 1 | |
| owner_cost = "π’ Low" | |
| if nb_owner_cost == 1: | |
| owner_cost = "π‘ Medium" | |
| if nb_owner_cost == 2: | |
| owner_cost = "π΄ High" | |
| if nb_owner_cost == 3: | |
| owner_cost = "π΄ Very High" | |
| reliability = "β Low" | |
| if "same_model" in owner_active_list and "generalize" in owner_active_list: | |
| reliability = "β High" | |
| if ("same_model" in owner_active_list or "fair_model" in owner_active_list) and "overfit" in owner_active_list and "auditmany" in auditor_active_list: | |
| reliability = "β High" | |
| if ("same_model" in owner_active_list or "fair_model" in owner_active_list) and "overfit" in owner_active_list and "auditone" in auditor_active_list and "private" in auditor_active_list: | |
| reliability = "β οΈ Medium" | |
| if ("same_model" in owner_active_list or "fair_model" in owner_active_list) and "overfitdist" in owner_active_list and "auditmany" in auditor_active_list and "private" in auditor_active_list: | |
| reliability = "β οΈ Medium" | |
| openness = "None" | |
| if "internals" in auditor_active_list: | |
| openness = "π High (Model Weight Sharing)" | |
| if len(auditor_active_list)==0 and len(owner_active_list)==0: | |
| auditor_cost, owner_cost, reliability, openness = "N/A", "N/A", "N/A", "N/A" | |
| # -------------------------------------------------- | |
| # Dashboard | |
| # -------------------------------------------------- | |
| with c1: | |
| st.write("**Cost for The Auditor**") | |
| st.write(auditor_cost) | |
| with c2: | |
| st.write("**Cost for The Owner**") | |
| st.write(owner_cost) | |
| with c3: | |
| st.write("**Audit Reliability**") | |
| st.write(reliability) | |
| with c4: | |
| st.write("**Openness Requirements**") | |
| st.write(openness) |