jaydeepb commited on
Commit
261dfe0
·
verified ·
1 Parent(s): ec2d7ab

Intro references tabs by name, Score tab notes Pythia-1.4B, remove all bold

Browse files
Files changed (1) hide show
  1. app.py +12 -11
app.py CHANGED
@@ -124,8 +124,8 @@ def draw(dataset, which):
124
  feats = {k: float(r[k]) for k in FEAT_ORDER}
125
  X = np.nan_to_num(np.array([[feats[k] for k in b["feats"]]]), posinf=1e10, neginf=-1e10)
126
  risk = float(b["clf"].predict_proba(b["scaler"].transform(X))[0, 1])
127
- verdict += (f"\n\n### Classifier risk score: **{risk:.3f}**\n"
128
- f"Computed by the memorization classifier from base-model features measured *before* "
129
  f"fine-tuning. The plots below show where "
130
  f"this example's feature values fall relative to memorized and not-memorized examples.")
131
  return r.prefix, gen, verdict, dist_figure(feats, dataset)
@@ -146,18 +146,19 @@ def predict(text, clf_name):
146
  X = np.nan_to_num(np.array([[feats[k] for k in b["feats"]]]), posinf=1e10, neginf=-1e10)
147
  risk = float(b["clf"].predict_proba(b["scaler"].transform(X))[0, 1])
148
  flag = "high" if risk >= 0.5 else "low"
149
- md = (f"### Predicted memorization risk: **{risk:.3f}** ({flag})\n"
150
- f"Forecast from base-model features using the classifier trained on **{clf_name}**. "
151
  f"There is no ground truth for arbitrary text, so treat it as a relative risk score. "
152
  f"The plots below show where your text's features fall relative to memorized and not-memorized "
153
- f"examples in **{clf_name}**.")
154
  return md, dist_figure(feats, clf_name)
155
 
156
 
157
  INTRO = """# Predicting Memorization Before Fine-Tuning
158
- A classifier trained only on **pre-fine-tuning** base-model features predicts which sequences a
159
- fine-tuned model will memorize. **Explore** verifies this against ground truth on held-out examples from
160
- a fresh fine-tuning run; **Score your own text** applies the same classifier to a passage you provide.
 
161
  """
162
 
163
  with gr.Blocks(title="Predicting Memorization Before Fine-Tuning") as demo:
@@ -177,9 +178,9 @@ with gr.Blocks(title="Predicting Memorization Before Fine-Tuning") as demo:
177
  feat = gr.Plot(label="Where this example's features fall (memorized vs not-memorized)")
178
  go.click(draw, [d_in, f_in], [pref, gen, verdict, feat])
179
  with gr.Tab("Score your own text"):
180
- gr.Markdown("Paste a passage (at least ~100 tokens). We compute its base-model features and "
181
- "return the classifier's **predicted** memorization risk, and show where your text's "
182
- "features fall relative to memorized and not-memorized examples.")
183
  txt = gr.Textbox(label="Your text", lines=6)
184
  clf_in = gr.Dropdown(sorted(CLFS.keys()), value=sorted(CLFS.keys())[0],
185
  label="Compare against dataset")
 
124
  feats = {k: float(r[k]) for k in FEAT_ORDER}
125
  X = np.nan_to_num(np.array([[feats[k] for k in b["feats"]]]), posinf=1e10, neginf=-1e10)
126
  risk = float(b["clf"].predict_proba(b["scaler"].transform(X))[0, 1])
127
+ verdict += (f"\n\n### Classifier risk score: {risk:.3f}\n"
128
+ f"Computed by the memorization classifier from base-model features measured before "
129
  f"fine-tuning. The plots below show where "
130
  f"this example's feature values fall relative to memorized and not-memorized examples.")
131
  return r.prefix, gen, verdict, dist_figure(feats, dataset)
 
146
  X = np.nan_to_num(np.array([[feats[k] for k in b["feats"]]]), posinf=1e10, neginf=-1e10)
147
  risk = float(b["clf"].predict_proba(b["scaler"].transform(X))[0, 1])
148
  flag = "high" if risk >= 0.5 else "low"
149
+ md = (f"### Predicted memorization risk: {risk:.3f} ({flag})\n"
150
+ f"Forecast from base-model features using the classifier trained on {clf_name}. "
151
  f"There is no ground truth for arbitrary text, so treat it as a relative risk score. "
152
  f"The plots below show where your text's features fall relative to memorized and not-memorized "
153
+ f"examples in {clf_name}.")
154
  return md, dist_figure(feats, clf_name)
155
 
156
 
157
  INTRO = """# Predicting Memorization Before Fine-Tuning
158
+ A classifier trained only on pre-fine-tuning base-model features predicts which sequences a
159
+ fine-tuned model will memorize. The "Explore held-out examples" tab verifies this against ground truth on
160
+ held-out examples from a fresh fine-tuning run; the "Score your own text" tab applies the same classifier
161
+ to a passage you provide.
162
  """
163
 
164
  with gr.Blocks(title="Predicting Memorization Before Fine-Tuning") as demo:
 
178
  feat = gr.Plot(label="Where this example's features fall (memorized vs not-memorized)")
179
  go.click(draw, [d_in, f_in], [pref, gen, verdict, feat])
180
  with gr.Tab("Score your own text"):
181
+ gr.Markdown("Paste a passage (at least ~100 tokens). We compute its base-model features using "
182
+ "Pythia-1.4B and return the classifier's predicted memorization risk, and show where "
183
+ "your text's features fall relative to memorized and not-memorized examples.")
184
  txt = gr.Textbox(label="Your text", lines=6)
185
  clf_in = gr.Dropdown(sorted(CLFS.keys()), value=sorted(CLFS.keys())[0],
186
  label="Compare against dataset")