Spaces:
Sleeping
Sleeping
Intro references tabs by name, Score tab notes Pythia-1.4B, remove all bold
Browse files
app.py
CHANGED
|
@@ -124,8 +124,8 @@ def draw(dataset, which):
|
|
| 124 |
feats = {k: float(r[k]) for k in FEAT_ORDER}
|
| 125 |
X = np.nan_to_num(np.array([[feats[k] for k in b["feats"]]]), posinf=1e10, neginf=-1e10)
|
| 126 |
risk = float(b["clf"].predict_proba(b["scaler"].transform(X))[0, 1])
|
| 127 |
-
verdict += (f"\n\n### Classifier risk score:
|
| 128 |
-
f"Computed by the memorization classifier from base-model features measured
|
| 129 |
f"fine-tuning. The plots below show where "
|
| 130 |
f"this example's feature values fall relative to memorized and not-memorized examples.")
|
| 131 |
return r.prefix, gen, verdict, dist_figure(feats, dataset)
|
|
@@ -146,18 +146,19 @@ def predict(text, clf_name):
|
|
| 146 |
X = np.nan_to_num(np.array([[feats[k] for k in b["feats"]]]), posinf=1e10, neginf=-1e10)
|
| 147 |
risk = float(b["clf"].predict_proba(b["scaler"].transform(X))[0, 1])
|
| 148 |
flag = "high" if risk >= 0.5 else "low"
|
| 149 |
-
md = (f"### Predicted memorization risk:
|
| 150 |
-
f"Forecast from base-model features using the classifier trained on
|
| 151 |
f"There is no ground truth for arbitrary text, so treat it as a relative risk score. "
|
| 152 |
f"The plots below show where your text's features fall relative to memorized and not-memorized "
|
| 153 |
-
f"examples in
|
| 154 |
return md, dist_figure(feats, clf_name)
|
| 155 |
|
| 156 |
|
| 157 |
INTRO = """# Predicting Memorization Before Fine-Tuning
|
| 158 |
-
A classifier trained only on
|
| 159 |
-
fine-tuned model will memorize.
|
| 160 |
-
a fresh fine-tuning run;
|
|
|
|
| 161 |
"""
|
| 162 |
|
| 163 |
with gr.Blocks(title="Predicting Memorization Before Fine-Tuning") as demo:
|
|
@@ -177,9 +178,9 @@ with gr.Blocks(title="Predicting Memorization Before Fine-Tuning") as demo:
|
|
| 177 |
feat = gr.Plot(label="Where this example's features fall (memorized vs not-memorized)")
|
| 178 |
go.click(draw, [d_in, f_in], [pref, gen, verdict, feat])
|
| 179 |
with gr.Tab("Score your own text"):
|
| 180 |
-
gr.Markdown("Paste a passage (at least ~100 tokens). We compute its base-model features
|
| 181 |
-
"return the classifier's
|
| 182 |
-
"features fall relative to memorized and not-memorized examples.")
|
| 183 |
txt = gr.Textbox(label="Your text", lines=6)
|
| 184 |
clf_in = gr.Dropdown(sorted(CLFS.keys()), value=sorted(CLFS.keys())[0],
|
| 185 |
label="Compare against dataset")
|
|
|
|
| 124 |
feats = {k: float(r[k]) for k in FEAT_ORDER}
|
| 125 |
X = np.nan_to_num(np.array([[feats[k] for k in b["feats"]]]), posinf=1e10, neginf=-1e10)
|
| 126 |
risk = float(b["clf"].predict_proba(b["scaler"].transform(X))[0, 1])
|
| 127 |
+
verdict += (f"\n\n### Classifier risk score: {risk:.3f}\n"
|
| 128 |
+
f"Computed by the memorization classifier from base-model features measured before "
|
| 129 |
f"fine-tuning. The plots below show where "
|
| 130 |
f"this example's feature values fall relative to memorized and not-memorized examples.")
|
| 131 |
return r.prefix, gen, verdict, dist_figure(feats, dataset)
|
|
|
|
| 146 |
X = np.nan_to_num(np.array([[feats[k] for k in b["feats"]]]), posinf=1e10, neginf=-1e10)
|
| 147 |
risk = float(b["clf"].predict_proba(b["scaler"].transform(X))[0, 1])
|
| 148 |
flag = "high" if risk >= 0.5 else "low"
|
| 149 |
+
md = (f"### Predicted memorization risk: {risk:.3f} ({flag})\n"
|
| 150 |
+
f"Forecast from base-model features using the classifier trained on {clf_name}. "
|
| 151 |
f"There is no ground truth for arbitrary text, so treat it as a relative risk score. "
|
| 152 |
f"The plots below show where your text's features fall relative to memorized and not-memorized "
|
| 153 |
+
f"examples in {clf_name}.")
|
| 154 |
return md, dist_figure(feats, clf_name)
|
| 155 |
|
| 156 |
|
| 157 |
INTRO = """# Predicting Memorization Before Fine-Tuning
|
| 158 |
+
A classifier trained only on pre-fine-tuning base-model features predicts which sequences a
|
| 159 |
+
fine-tuned model will memorize. The "Explore held-out examples" tab verifies this against ground truth on
|
| 160 |
+
held-out examples from a fresh fine-tuning run; the "Score your own text" tab applies the same classifier
|
| 161 |
+
to a passage you provide.
|
| 162 |
"""
|
| 163 |
|
| 164 |
with gr.Blocks(title="Predicting Memorization Before Fine-Tuning") as demo:
|
|
|
|
| 178 |
feat = gr.Plot(label="Where this example's features fall (memorized vs not-memorized)")
|
| 179 |
go.click(draw, [d_in, f_in], [pref, gen, verdict, feat])
|
| 180 |
with gr.Tab("Score your own text"):
|
| 181 |
+
gr.Markdown("Paste a passage (at least ~100 tokens). We compute its base-model features using "
|
| 182 |
+
"Pythia-1.4B and return the classifier's predicted memorization risk, and show where "
|
| 183 |
+
"your text's features fall relative to memorized and not-memorized examples.")
|
| 184 |
txt = gr.Textbox(label="Your text", lines=6)
|
| 185 |
clf_in = gr.Dropdown(sorted(CLFS.keys()), value=sorted(CLFS.keys())[0],
|
| 186 |
label="Compare against dataset")
|