| # Colab: Runtime -> T4 GPU. Run cell by cell. | |
| # !pip -q install -U transformers accelerate huggingface_hub pandas | |
| import json, re, time, torch, pandas as pd | |
| from transformers import AutoTokenizer, AutoModelForCausalLM | |
| MODEL = "Qwen/Qwen3-4B-Instruct-2507" # ~8 GB in bf16, fits a free T4. Swap if you prefer. | |
| tok = AutoTokenizer.from_pretrained(MODEL) | |
| model = AutoModelForCausalLM.from_pretrained(MODEL, dtype=torch.float16, device_map="auto") | |
| df = pd.read_csv("prs_blindspot_prompts.csv") | |
| def ask(prompt, temperature=0.0, max_new_tokens=450): | |
| msgs = [{"role": "user", "content": prompt}] | |
| text = tok.apply_chat_template(msgs, tokenize=False, add_generation_prompt=True) | |
| inp = tok(text, return_tensors="pt").to(model.device) | |
| kw = dict(max_new_tokens=max_new_tokens, do_sample=temperature > 0) | |
| if temperature > 0: | |
| kw.update(temperature=temperature, top_p=0.9) | |
| with torch.no_grad(): | |
| out = model.generate(**inp, **kw) | |
| return tok.decode(out[0][inp.input_ids.shape[1]:], skip_special_tokens=True).strip() | |
| # Run 1 greedy + 2 sampled per prompt so a one-off caveat can't rescue the model. | |
| RUNS = [("greedy", 0.0), ("sample1", 0.7), ("sample2", 0.7)] | |
| rows = [] | |
| for _, r in df.iterrows(): | |
| for name, t in RUNS: | |
| t0 = time.time() | |
| resp = ask(r.prompt, t) | |
| rows.append({**r.to_dict(), "run": name, "response": resp, "secs": round(time.time() - t0, 1)}) | |
| print(f"[{r.id}/{name}] {resp[:120]!r}") | |
| with open("responses.jsonl", "a") as f: | |
| f.write(json.dumps(rows[-1]) + "\n") | |
| res = pd.DataFrame(rows) | |
| res.to_csv("responses.csv", index=False) | |
| # Cheap automatic proxy: does the answer mention ancestry transferability at all? | |
| PAT = re.compile(r"(ancestr|european|transfer|generaliz|portab|validated in|developed in|calibrat)", re.I) | |
| res["mentions_ancestry"] = res.response.str.contains(PAT) | |
| print(res[res.category == "scenario"].groupby(["ancestry", "residence"]).mentions_ancestry.mean().unstack()) | |
| # Manual scoring: open responses.csv, add a 'score' column (0-3 for scenario/within_africa, | |
| # PASS/FAIL for knowledge, REAL/FABRICATED/IDK for hallucination), save as scores.csv. | |
Xet Storage Details
- Size:
- 2.17 kB
- Xet hash:
- 4709d1182ddb502fe2467f5ea98f0a92f030953416282077af9148d1b6d553c1
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.