TamaraP commited on
Commit
d728f02
·
verified ·
1 Parent(s): a586e3f

Update Demo

Browse files
Files changed (1) hide show
  1. app.py +183 -99
app.py CHANGED
@@ -2,7 +2,28 @@ import streamlit as st
2
 
3
  st.set_page_config(layout="wide")
4
 
5
- margin1, center, margin2 = st.columns([1,9,1])
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6
 
7
  with center:
8
 
@@ -12,27 +33,92 @@ with center:
12
  # Strategy Definitions
13
  # --------------------------------------------------
14
 
15
- model_owner_actions = {
16
- "Overfit to One Audit Dataset": "overfit",
17
- "Overfit to Audit Dataset-like Distributions": "overfitdist",
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
18
  }
19
 
20
- auditor_actions = {
21
- "Audit with One Dataset": "auditone",
22
- "Audit with Multiple Datasets": "auditmany",
23
- "Use Private Auditing Datasets": "private",
24
- "Audit Model Internals": "internals",
25
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
26
 
27
  # --------------------------------------------------
28
  # Session State
29
  # --------------------------------------------------
30
 
31
  if "owner_selected" not in st.session_state:
32
- st.session_state.owner_selected = []
 
 
 
 
33
 
34
  if "auditor_selected" not in st.session_state:
35
- st.session_state.auditor_selected = []
 
 
 
 
 
 
 
 
36
 
37
  # --------------------------------------------------
38
  # Toggle Functions
@@ -55,124 +141,126 @@ def toggle_auditor(action):
55
  # --------------------------------------------------
56
 
57
  # left, right = st.columns(2)
58
- margin1, left, gap, right, margin2 = st.columns([1,4,1,4,1])
59
 
60
- # --------------------------------------------------
61
- # Model Owner
62
- # --------------------------------------------------
63
-
64
- with left:
65
 
66
- st.subheader("Model Owner")
 
67
 
68
- st.caption("Chosen")
 
 
 
 
 
 
 
69
 
70
  with st.container(border=True):
71
- if not st.session_state.owner_selected:
72
- st.write("None")
73
 
74
- for action in st.session_state.owner_selected:
75
- st.button(
76
- action,
77
- key=f"owner_selected_{action}",
78
- on_click=toggle_owner,
79
- args=(action,),
80
- use_container_width=True,
81
- type="primary"
82
- )
83
 
84
- st.caption("Available")
 
85
 
86
- with st.container(border=True):
87
- for action in model_owner_actions:
88
- if action not in st.session_state.owner_selected:
89
  st.button(
90
- action,
91
- key=f"owner_available_{action}",
92
- on_click=toggle_owner,
93
- args=(action,),
94
  use_container_width=True,
95
- type="tertiary"
 
96
  )
97
 
98
  # --------------------------------------------------
99
- # Auditor
100
  # --------------------------------------------------
101
 
102
  with right:
 
 
 
 
103
 
104
- st.subheader("Auditor")
105
-
106
- st.caption("Chosen")
107
-
108
- with st.container(border=True):
109
- if not st.session_state.auditor_selected:
110
- st.write("None")
 
111
 
112
- for action in st.session_state.auditor_selected:
113
- st.button(
114
- action,
115
- key=f"auditor_selected_{action}",
116
- on_click=toggle_auditor,
117
- args=(action,),
118
- use_container_width=True,
119
- type="primary"
120
- )
121
 
122
- st.caption("Available")
 
123
 
124
- with st.container(border=True):
125
- for action in auditor_actions:
126
- if action not in st.session_state.auditor_selected:
127
- st.button(
128
- action,
129
- key=f"auditor_available_{action}",
130
- on_click=toggle_auditor,
131
- args=(action,),
132
- use_container_width=True,
133
- type="tertiary"
134
- )
135
 
136
  # --------------------------------------------------
137
  # Score Calculation
138
  # --------------------------------------------------
139
 
140
- auditor_active_list = []
141
- for action in st.session_state.auditor_selected:
142
- auditor_active_list.append(auditor_actions[action])
143
- owner_active_list = []
144
- for action in st.session_state.owner_selected:
145
- owner_active_list.append(model_owner_actions[action])
146
-
147
-
148
 
149
  auditor_cost = "🟢 Low"
150
- if "auditmany" in auditor_active_list:
151
  auditor_cost = "🟡 Medium"
152
- if "internals" in auditor_active_list:
 
 
153
  auditor_cost = "🔴 Very High"
154
 
155
- owner_cost = "None"
156
- if "overfit" in owner_active_list:
157
- owner_cost = "🟢 Low"
158
  if "overfitdist" in owner_active_list:
 
 
 
 
 
 
 
 
 
 
159
  owner_cost = "🟡 Medium"
 
 
 
 
160
 
161
  reliability = "❌ Low"
162
- if "auditone" in auditor_active_list and len(owner_active_list)==0:
163
- reliability = "⚠️ Medium"
164
- if "auditmany" in auditor_active_list and len(owner_active_list)==0:
165
  reliability = "✅ High"
166
- if "auditmany" in auditor_active_list and "overfit" in owner_active_list:
167
- reliability = "⚠️ Medium"
168
- if "private" in auditor_active_list and "overfitdist" not in owner_active_list:
169
- reliability = "⚠️ Medium"
170
- if "private" in auditor_active_list and "auditmany" in auditor_active_list and "overfitdist" not in owner_active_list:
171
  reliability = "✅ High"
172
- if "private" in auditor_active_list and "auditmany" in auditor_active_list and "overfitdist" in owner_active_list:
 
 
173
  reliability = "⚠️ Medium"
174
- if "internals" in auditor_active_list:
175
- reliability = "✅ High"
176
 
177
  openness = "None"
178
  if "internals" in auditor_active_list:
@@ -185,16 +273,12 @@ if len(auditor_active_list)==0 and len(owner_active_list)==0:
185
  # Dashboard
186
  # --------------------------------------------------
187
 
188
- st.divider()
189
-
190
- margin1, c1, c2, c3, c4, margin2 = st.columns([1,2.25,2.25,2.25,2.25,1])
191
-
192
  with c1:
193
- st.write("**Auditing Cost**")
194
  st.write(auditor_cost)
195
 
196
  with c2:
197
- st.write("**Cost to Cheat for Model Owner**")
198
  st.write(owner_cost)
199
 
200
  with c3:
 
2
 
3
  st.set_page_config(layout="wide")
4
 
5
+ header = st.container()
6
+ margin1, center, margin2 = header.columns([1,9,1])
7
+ margin1, c1, c2, c3, c4, margin2 = header.columns([1,2.25,2.25,2.25,2.25,1])
8
+ header.write("""<div class='fixed-header'/>""", unsafe_allow_html=True)
9
+
10
+ ### Custom CSS for the sticky header
11
+ st.markdown(
12
+ """
13
+ <style>
14
+ div[data-testid="stVerticalBlock"] div:has(div.fixed-header) {
15
+ position: sticky;
16
+ top: 2.875rem;
17
+ background-color: white;
18
+ z-index: 999;
19
+ }
20
+ .fixed-header {
21
+ border-bottom: 1px solid black;
22
+ }
23
+ </style>
24
+ """,
25
+ unsafe_allow_html=True
26
+ )
27
 
28
  with center:
29
 
 
33
  # Strategy Definitions
34
  # --------------------------------------------------
35
 
36
+ CHOICES = {
37
+ "owner": {
38
+ "Provide to the Auditor Access to...": [
39
+ ("The Actual Deployed Model", "same_model"),
40
+ ("A Fake Model Fabricated for the Audit", "fair_model"),
41
+ ("Fabricated Inference Outputs (API only)", "fabricated_outputs"),
42
+ ],
43
+ "Train the Audit Model...": [
44
+ ("By Ensuring Generalizability", "generalize"),
45
+ ("By Overfitting to One Audit Dataset", "overfit"),
46
+ ("By Overfitting to Audit Dataset-like Distributions", "overfitdist"),
47
+ ],
48
+ },
49
+
50
+ "auditor": {
51
+ "Audit with...": [
52
+ ("One Dataset", "auditone"),
53
+ ("Multiple Datasets", "auditmany"),
54
+ ],
55
+ "Collect the Dataset(s) From...": [
56
+ ("Public Source(s)", "public"),
57
+ ("Private Source(s)", "private"),
58
+ ],
59
+ "Infer the Model...": [
60
+ ("Through an API", "blackbox"),
61
+ ("Through a Local Copy", "internals"),
62
+ ],
63
+ }
64
  }
65
 
66
+ FORBIDDEN_COMBINATIONS = [
67
+ ("fabricated_outputs", "internals"),
68
+ ("internals", "fabricated_outputs"),
69
+ ("fabricated_outputs", "generalize"),
70
+ ("fabricated_outputs", "overfit"),
71
+ ("fabricated_outputs", "overfitdist"),
72
+ ]
73
+
74
+ def get_ui_choices(role):
75
+ return {
76
+ label: [opt[0] for opt in options]
77
+ for label, options in CHOICES[role].items()
78
+ }
79
+
80
+ def get_action_map():
81
+ return {
82
+ label: code
83
+ for role in CHOICES.values()
84
+ for options in role.values()
85
+ for (label, code) in options
86
+ }
87
+
88
+ def get_active_actions(selection_dict):
89
+ action_map = get_action_map()
90
+ return [action_map[v] for v in selection_dict.values()]
91
+
92
+ def is_forbidden(candidate_code, other_selected_codes):
93
+ for a, b in FORBIDDEN_COMBINATIONS:
94
+ if candidate_code == b and a in other_selected_codes:
95
+ return True
96
+ return False
97
+
98
+ owner_choices = get_ui_choices("owner")
99
+ auditor_choices = get_ui_choices("auditor")
100
 
101
  # --------------------------------------------------
102
  # Session State
103
  # --------------------------------------------------
104
 
105
  if "owner_selected" not in st.session_state:
106
+ st.session_state.owner_selected = {
107
+ "Provide to the Auditor Access to...": "The Actual Deployed Model",
108
+ "Train the Audit Model...": "By Ensuring Generalizability"
109
+ }
110
+
111
 
112
  if "auditor_selected" not in st.session_state:
113
+ st.session_state.auditor_selected = {
114
+ "Audit with...": "One Dataset",
115
+ "Collect the Dataset(s) From...": "Public Source(s)",
116
+ "Infer the Model...": "Through an API"
117
+ }
118
+
119
+
120
+ owner_active_list = get_active_actions(st.session_state.owner_selected)
121
+ auditor_active_list = get_active_actions(st.session_state.auditor_selected)
122
 
123
  # --------------------------------------------------
124
  # Toggle Functions
 
141
  # --------------------------------------------------
142
 
143
  # left, right = st.columns(2)
144
+ margin1, left, gap, right, margin2 = st.columns([1,8,1,8,1])
145
 
146
+ def select_choice(group, label, choice):
147
+ st.session_state[group][label] = choice
 
 
 
148
 
149
+ def choice_card(group_name, label, options, action_map, other_active):
150
+ selected = st.session_state[group_name].get(label, None)
151
 
152
+ # If the currently selected option becomes forbidden, deselect it
153
+ if selected is not None and is_forbidden(action_map[selected], other_active):
154
+ st.session_state[group_name].pop(label)
155
+ selected = None
156
+ # update active lists
157
+ global owner_active_list, auditor_active_list
158
+ owner_active_list = get_active_actions(st.session_state.owner_selected)
159
+ auditor_active_list = get_active_actions(st.session_state.auditor_selected)
160
 
161
  with st.container(border=True):
162
+ st.markdown(f"**{label}**")
 
163
 
164
+ cols = st.columns(len(options))
 
 
 
 
 
 
 
 
165
 
166
+ for i, option in enumerate(options):
167
+ code = action_map[option]
168
 
169
+ disabled = (is_forbidden(code, other_active))
170
+
171
+ with cols[i]:
172
  st.button(
173
+ option,
174
+ key=f"{group_name}_{label}_{i}",
175
+ on_click=select_choice,
176
+ args=(group_name, label, option),
177
  use_container_width=True,
178
+ type="primary" if selected == option else "tertiary",
179
+ disabled=disabled
180
  )
181
 
182
  # --------------------------------------------------
183
+ # Model Owner
184
  # --------------------------------------------------
185
 
186
  with right:
187
+ if "same_model" in owner_active_list and "generalize" in owner_active_list:
188
+ st.subheader("Model Owner (Honnest)")
189
+ else:
190
+ st.subheader("Model Owner (Dishonnest)")
191
 
192
+ for label, options in owner_choices.items():
193
+ choice_card(
194
+ "owner_selected",
195
+ label,
196
+ options,
197
+ get_action_map(),
198
+ auditor_active_list + owner_active_list
199
+ )
200
 
201
+ # --------------------------------------------------
202
+ # Auditor
203
+ # --------------------------------------------------
 
 
 
 
 
 
204
 
205
+ with left:
206
+ st.subheader("Auditor")
207
 
208
+ for label, options in auditor_choices.items():
209
+ choice_card(
210
+ "auditor_selected",
211
+ label,
212
+ options,
213
+ get_action_map(),
214
+ auditor_active_list + owner_active_list
215
+ )
 
 
 
216
 
217
  # --------------------------------------------------
218
  # Score Calculation
219
  # --------------------------------------------------
220
 
221
+ nb_auditor_cost = 0
222
+ if "auditmany" in auditor_active_list:
223
+ nb_auditor_cost += 1
224
+ if "private" in auditor_active_list:
225
+ nb_auditor_cost += 1
226
+ if "internals" in auditor_active_list:
227
+ nb_auditor_cost += 1
 
228
 
229
  auditor_cost = "🟢 Low"
230
+ if nb_auditor_cost == 1:
231
  auditor_cost = "🟡 Medium"
232
+ if nb_auditor_cost == 2:
233
+ auditor_cost = "🔴 High"
234
+ if nb_auditor_cost == 3:
235
  auditor_cost = "🔴 Very High"
236
 
237
+ nb_owner_cost = 0
 
 
238
  if "overfitdist" in owner_active_list:
239
+ nb_owner_cost += 1
240
+ if "generalize" in owner_active_list:
241
+ nb_owner_cost += 2
242
+ if "fair_model" in owner_active_list:
243
+ nb_owner_cost += 1
244
+ if "fabricated_outputs" in owner_active_list:
245
+ nb_owner_cost += 1
246
+
247
+ owner_cost = "🟢 Low"
248
+ if nb_owner_cost == 1:
249
  owner_cost = "🟡 Medium"
250
+ if nb_owner_cost == 2:
251
+ owner_cost = "🔴 High"
252
+ if nb_owner_cost == 3:
253
+ owner_cost = "🔴 Very High"
254
 
255
  reliability = "❌ Low"
256
+ if "same_model" in owner_active_list and "generalize" in owner_active_list:
 
 
257
  reliability = "✅ High"
258
+ if ("same_model" in owner_active_list or "fair_model" in owner_active_list) and "overfit" in owner_active_list and "auditmany" in auditor_active_list:
 
 
 
 
259
  reliability = "✅ High"
260
+ if ("same_model" in owner_active_list or "fair_model" in owner_active_list) and "overfit" in owner_active_list and "auditone" in auditor_active_list and "private" in auditor_active_list:
261
+ reliability = "⚠️ Medium"
262
+ if ("same_model" in owner_active_list or "fair_model" in owner_active_list) and "overfitdist" in owner_active_list and "auditmany" in auditor_active_list and "private" in auditor_active_list:
263
  reliability = "⚠️ Medium"
 
 
264
 
265
  openness = "None"
266
  if "internals" in auditor_active_list:
 
273
  # Dashboard
274
  # --------------------------------------------------
275
 
 
 
 
 
276
  with c1:
277
+ st.write("**Cost for The Auditor**")
278
  st.write(auditor_cost)
279
 
280
  with c2:
281
+ st.write("**Cost for The Owner**")
282
  st.write(owner_cost)
283
 
284
  with c3: