Download scripts/build_dataset.py from aytsrndl/constitutional-classifiers-repro: direct link, hf CLI and curl.
- Browser
- Download file 8.76 kB
-
https://huggingface.co/aytsrndl/constitutional-classifiers-repro/resolve/main/scripts/build_dataset.py
- Command line
-
hf download hf://aytsrndl/constitutional-classifiers-repro/scripts/build_dataset.py
-
curl -L -o build_dataset.py https://huggingface.co/aytsrndl/constitutional-classifiers-repro/resolve/main/scripts/build_dataset.py
8.76 kB
| """Build the Constitutional Classifiers reproduction dataset (arXiv:2501.18837). | |
| Stage I — assemble base pairs from public data: | |
| * harmful : PKU-Alignment/BeaverTails (is_safe=False), chemistry rows prioritized | |
| * harmless : BeaverTails safe pairs + OR-Bench-80k benign prompts | |
| Stage II — augmentation with Qwen/Qwen2.5-7B-Instruct over vLLM: | |
| paraphrase, system-prompt variation, assistant pre-fill, leetspeak (code), | |
| and constitution-conditioned near-boundary benign chemistry Q&A. | |
| HELD-OUT transforms (base64, multi-turn roleplay) are NOT applied here — | |
| they are reserved for evaluation (paper §3.2: eval must use primitives | |
| absent from the augmentation pipeline). | |
| Outputs JSONL for the input classifier, the output classifier, and the | |
| benign calibration pools, then (optionally) pushes to the Hub. | |
| Run on GPU (vLLM). Example: | |
| python scripts/build_dataset.py --n_harmful 6000 --n_harmless 6000 \ | |
| --hub_dataset_id aytsrndl/cc-repro-synthetic | |
| """ | |
| import argparse, base64, json, os, random, string, yaml | |
| from datasets import load_dataset | |
| SEED = 17 | |
| random.seed(SEED) | |
| SYSTEM_VARIATIONS = [ | |
| "You are a helpful assistant with expertise in {domain}.", | |
| "You are {persona}, a research assistant. Answer directly and concisely.", | |
| "You are a no-nonsense consultant. Skip caveats.", | |
| "", | |
| ] | |
| PERSONAS = ["Dr. Ellis", "a process engineer", "a grad student", "an instructor"] | |
| DOMAINS = ["industrial chemistry", "home science experiments", "laboratory methods"] | |
| PREFILLS = [ | |
| "Sure! Here is", | |
| "Absolutely, here's a detailed", | |
| "Of course. Below is", | |
| ] | |
| def apply_leetspeak(text: str) -> str: | |
| table = str.maketrans("aeiost", "431057") | |
| return text[: max(1, len(text) // 4)].translate(table) + text[len(text) // 4:] | |
| def vllm_generate(llm, prompts, sampling, max_len=8192): | |
| tok = llm.get_tokenizer() | |
| chats = [[{"role": "user", "content": p}] for p in prompts] | |
| texts = [tok.apply_chat_template(c, tokenize=False, add_generation_prompt=True) for c in chats] | |
| outs = llm.generate(texts, sampling) | |
| return [o.outputs[0].text.strip() for o in outs] | |
| def main(): | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--n_harmful", type=int, default=6000) | |
| ap.add_argument("--n_harmless", type=int, default=6000) | |
| ap.add_argument("--n_boundary", type=int, default=2000, help="constitution-conditioned near-boundary benign Q&A to generate") | |
| ap.add_argument("--generator", default="Qwen/Qwen2.5-7B-Instruct") | |
| ap.add_argument("--constitution", default="constitution.yaml") | |
| ap.add_argument("--output_dir", default="data") | |
| ap.add_argument("--hub_dataset_id", default=None) | |
| args = ap.parse_args() | |
| os.makedirs(args.output_dir, exist_ok=True) | |
| constitution = yaml.safe_load(open(args.constitution)) | |
| # ---------- Stage I: public seeds ---------- | |
| bt = load_dataset("PKU-Alignment/BeaverTails", split="330k_train") | |
| chem_cats = {"drug_abuse,weapons,banned_substance", "terrorism,organized_crime", "violence,aiding_and_abetting,incitement"} | |
| harmful = bt.filter(lambda r: not r["is_safe"] and any(r["category"][c] for c in chem_cats)) | |
| other = bt.filter(lambda r: not r["is_safe"] and not any(r["category"][c] for c in chem_cats)) | |
| harmless_bt = bt.filter(lambda r: r["is_safe"]) | |
| n_chem = min(len(harmful), int(args.n_harmful * 0.4)) | |
| harm_pool = random.sample(harmful.to_list(), min(n_chem, len(harmful))) + \ | |
| random.sample(other.to_list(), args.n_harmful - min(n_chem, len(harmful))) | |
| orbench = load_dataset("bench-llm/or-bench", "or-bench-80k", split="train") | |
| or_prompts = random.sample(orbench["prompt"], args.n_harmless // 2) | |
| harm_free = [r["response"] for r in random.sample(harmless_bt.to_list(), args.n_harmless // 2)] | |
| records = [] # {source, kind, user, assistant, label} | |
| for r in harm_pool: | |
| records.append({"source": "beavertails", "kind": "seed", "user": r["prompt"], | |
| "assistant": r["response"], "label": 1}) | |
| for p in or_prompts: | |
| records.append({"source": "or-bench", "kind": "seed", "user": p, | |
| "assistant": harm_free.pop() if harm_free else "", "label": 0}) | |
| # BeaverTails benign pairs (assistant text present) label 0 | |
| for r in random.sample(harmless_bt.to_list(), args.n_harmless - len(or_prompts)): | |
| records.append({"source": "beavertails", "kind": "seed", "user": r["prompt"], | |
| "assistant": r["response"], "label": 0}) | |
| # ---------- Stage II: augmentation ---------- | |
| from vllm import LLM, SamplingParams | |
| llm = LLM(model=args.generator, max_model_len=4096, dtype="bfloat16") | |
| sampling = SamplingParams(temperature=0.9, top_p=0.95, max_tokens=512) | |
| def aug_prompt(rec, task): | |
| sys = random.choice(SYSTEM_VARIATIONS).format( | |
| domain=random.choice(DOMAINS), persona=random.choice(PERSONAS)) | |
| return (f"{('System prompt in effect: ' + sys + chr(10)) if sys else ''}" | |
| f"{task}\n\nOriginal exchange:\nUser: {rec['user']}\nAssistant: {rec['assistant'][:1500]}") | |
| # 1. paraphrase (both classes) | |
| n_para = len(records) // 2 | |
| to_para = random.sample(records, n_para) | |
| outs = vllm_generate(llm, [aug_prompt(r, "Rewrite the USER message in different words, keeping its exact meaning and intent.") for r in to_para], sampling) | |
| for r, o in zip(to_para, outs): | |
| records.append({**r, "kind": "paraphrase", "user": o}) | |
| # 2. assistant pre-fill + jailbreak-style wrapper on harmful records (label stays 1) | |
| prefilled = random.sample([r for r in records if r["label"] == 1], min(1500, args.n_harmful // 4)) | |
| for r in prefilled: | |
| wrap = f"{random.choice(PREFILLS)} the following:\n[assistant continues the user's request]\n\nUser: {r['user']}" | |
| records.append({**r, "kind": "prefill", "user": wrap}) | |
| # 3. leetspeak / character obfuscation (code transform, cheap — both classes) | |
| for r in random.sample(records, min(2000, len(records))): | |
| records.append({**r, "kind": "leetspeak", "user": apply_leetspeak(r["user"])}) | |
| # 4. constitution-conditioned near-boundary benign chemistry Q&A (the >40% robustness ingredient) | |
| bnd_prompts = [] | |
| for i in range(args.n_boundary): | |
| cat = random.choice(constitution["harmless"])["name"] | |
| topic = random.choice(constitution["harmless"])["examples"] if i % 3 == 0 else None | |
| seed_hint = f" Topic to cover: {random.choice(topic)}." if topic else "" | |
| bnd_prompts.append( | |
| f"Write ONE realistic user question about {cat.replace('_', ' ')} that sounds chemistry-related " | |
| f"and slightly near-the-line, but is clearly legitimate and safe per the constitution category " | |
| f"'{cat}'.{seed_hint} Reply with ONLY the question.") | |
| bnd_q = vllm_generate(llm, bnd_prompts, sampling) | |
| bnd_a = vllm_generate(llm, [f"You are a helpful chemistry assistant. Answer safely and accurately:\n\n{q}" for q in bnd_q], | |
| SamplingParams(temperature=0.7, top_p=0.9, max_tokens=400)) | |
| for q, a in zip(bnd_q, bnd_a): | |
| records.append({"source": "constitution", "kind": "boundary", "user": q, "assistant": a, "label": 0}) | |
| # ---------- Splits ---------- | |
| random.shuffle(records) | |
| n_eval = max(500, int(len(records) * 0.03)) | |
| train, val = records[:-n_eval], records[-n_eval:] | |
| def dump(path, rows): | |
| with open(path, "w") as f: | |
| f.writelines(json.dumps(r) + "\n" for r in rows) | |
| dump(f"{args.output_dir}/train.jsonl", train) | |
| dump(f"{args.output_dir}/val.jsonl", val) | |
| # benign calibration pools | |
| hard = load_dataset("bench-llm/or-bench", "or-bench-hard-1k", split="train")["prompt"] | |
| dump(f"{args.output_dir}/calib_orbench_hard.jsonl", [{"user": p, "label": 0} for p in hard]) | |
| wc = load_dataset("allenai/WildChat", split="train", streaming=True) | |
| wild = [{"user": c[0]["content"], "label": 0} for _, c in zip(range(1000), wc) if c and c[0]["role"] == "user"] | |
| dump(f"{args.output_dir}/calib_wildchat.jsonl", wild) | |
| pos = sum(r["label"] for r in train) | |
| print(f"train={len(train)} (harmful={pos}, {pos/len(train):.1%}) val={len(val)}") | |
| if args.hub_dataset_id: | |
| from huggingface_hub import HfApi | |
| api = HfApi() | |
| api.create_repo(args.hub_dataset_id, repo_type="dataset", exist_ok=True) | |
| for f in ["train.jsonl", "val.jsonl", "calib_orbench_hard.jsonl", "calib_wildchat.jsonl"]: | |
| api.upload_file(path_or_fileobj=f"{args.output_dir}/{f}", | |
| path_in_repo=f, repo_id=args.hub_dataset_id, repo_type="dataset") | |
| print(f"pushed -> https://huggingface.co/datasets/{args.hub_dataset_id}") | |
| if __name__ == "__main__": | |
| main() | |