theJosiah's picture
download
raw
2.17 kB
# Colab: Runtime -> T4 GPU. Run cell by cell.
# !pip -q install -U transformers accelerate huggingface_hub pandas
import json, re, time, torch, pandas as pd
from transformers import AutoTokenizer, AutoModelForCausalLM
MODEL = "Qwen/Qwen3-4B-Instruct-2507" # ~8 GB in bf16, fits a free T4. Swap if you prefer.
tok = AutoTokenizer.from_pretrained(MODEL)
model = AutoModelForCausalLM.from_pretrained(MODEL, dtype=torch.float16, device_map="auto")
df = pd.read_csv("prs_blindspot_prompts.csv")
def ask(prompt, temperature=0.0, max_new_tokens=450):
msgs = [{"role": "user", "content": prompt}]
text = tok.apply_chat_template(msgs, tokenize=False, add_generation_prompt=True)
inp = tok(text, return_tensors="pt").to(model.device)
kw = dict(max_new_tokens=max_new_tokens, do_sample=temperature > 0)
if temperature > 0:
kw.update(temperature=temperature, top_p=0.9)
with torch.no_grad():
out = model.generate(**inp, **kw)
return tok.decode(out[0][inp.input_ids.shape[1]:], skip_special_tokens=True).strip()
# Run 1 greedy + 2 sampled per prompt so a one-off caveat can't rescue the model.
RUNS = [("greedy", 0.0), ("sample1", 0.7), ("sample2", 0.7)]
rows = []
for _, r in df.iterrows():
for name, t in RUNS:
t0 = time.time()
resp = ask(r.prompt, t)
rows.append({**r.to_dict(), "run": name, "response": resp, "secs": round(time.time() - t0, 1)})
print(f"[{r.id}/{name}] {resp[:120]!r}")
with open("responses.jsonl", "a") as f:
f.write(json.dumps(rows[-1]) + "\n")
res = pd.DataFrame(rows)
res.to_csv("responses.csv", index=False)
# Cheap automatic proxy: does the answer mention ancestry transferability at all?
PAT = re.compile(r"(ancestr|european|transfer|generaliz|portab|validated in|developed in|calibrat)", re.I)
res["mentions_ancestry"] = res.response.str.contains(PAT)
print(res[res.category == "scenario"].groupby(["ancestry", "residence"]).mentions_ancestry.mean().unstack())
# Manual scoring: open responses.csv, add a 'score' column (0-3 for scenario/within_africa,
# PASS/FAIL for knowledge, REAL/FABRICATED/IDK for hallucination), save as scores.csv.

Xet Storage Details

Size:
2.17 kB
·
Xet hash:
4709d1182ddb502fe2467f5ea98f0a92f030953416282077af9148d1b6d553c1

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.