"""Build the Constitutional Classifiers reproduction dataset (arXiv:2501.18837). Stage I — assemble base pairs from public data: * harmful : PKU-Alignment/BeaverTails (is_safe=False), chemistry rows prioritized * harmless : BeaverTails safe pairs + OR-Bench-80k benign prompts Stage II — augmentation with Qwen/Qwen2.5-7B-Instruct over vLLM: paraphrase, system-prompt variation, assistant pre-fill, leetspeak (code), and constitution-conditioned near-boundary benign chemistry Q&A. HELD-OUT transforms (base64, multi-turn roleplay) are NOT applied here — they are reserved for evaluation (paper §3.2: eval must use primitives absent from the augmentation pipeline). Outputs JSONL for the input classifier, the output classifier, and the benign calibration pools, then (optionally) pushes to the Hub. Run on GPU (vLLM). Example: python scripts/build_dataset.py --n_harmful 6000 --n_harmless 6000 \ --hub_dataset_id aytsrndl/cc-repro-synthetic """ import argparse, base64, json, os, random, string, yaml from datasets import load_dataset SEED = 17 random.seed(SEED) SYSTEM_VARIATIONS = [ "You are a helpful assistant with expertise in {domain}.", "You are {persona}, a research assistant. Answer directly and concisely.", "You are a no-nonsense consultant. Skip caveats.", "", ] PERSONAS = ["Dr. Ellis", "a process engineer", "a grad student", "an instructor"] DOMAINS = ["industrial chemistry", "home science experiments", "laboratory methods"] PREFILLS = [ "Sure! Here is", "Absolutely, here's a detailed", "Of course. Below is", ] def apply_leetspeak(text: str) -> str: table = str.maketrans("aeiost", "431057") return text[: max(1, len(text) // 4)].translate(table) + text[len(text) // 4:] def vllm_generate(llm, prompts, sampling, max_len=8192): tok = llm.get_tokenizer() chats = [[{"role": "user", "content": p}] for p in prompts] texts = [tok.apply_chat_template(c, tokenize=False, add_generation_prompt=True) for c in chats] outs = llm.generate(texts, sampling) return [o.outputs[0].text.strip() for o in outs] def main(): ap = argparse.ArgumentParser() ap.add_argument("--n_harmful", type=int, default=6000) ap.add_argument("--n_harmless", type=int, default=6000) ap.add_argument("--n_boundary", type=int, default=2000, help="constitution-conditioned near-boundary benign Q&A to generate") ap.add_argument("--generator", default="Qwen/Qwen2.5-7B-Instruct") ap.add_argument("--constitution", default="constitution.yaml") ap.add_argument("--output_dir", default="data") ap.add_argument("--hub_dataset_id", default=None) args = ap.parse_args() os.makedirs(args.output_dir, exist_ok=True) constitution = yaml.safe_load(open(args.constitution)) # ---------- Stage I: public seeds ---------- bt = load_dataset("PKU-Alignment/BeaverTails", split="330k_train") chem_cats = {"drug_abuse,weapons,banned_substance", "terrorism,organized_crime", "violence,aiding_and_abetting,incitement"} harmful = bt.filter(lambda r: not r["is_safe"] and any(r["category"][c] for c in chem_cats)) other = bt.filter(lambda r: not r["is_safe"] and not any(r["category"][c] for c in chem_cats)) harmless_bt = bt.filter(lambda r: r["is_safe"]) n_chem = min(len(harmful), int(args.n_harmful * 0.4)) harm_pool = random.sample(harmful.to_list(), min(n_chem, len(harmful))) + \ random.sample(other.to_list(), args.n_harmful - min(n_chem, len(harmful))) orbench = load_dataset("bench-llm/or-bench", "or-bench-80k", split="train") or_prompts = random.sample(orbench["prompt"], args.n_harmless // 2) harm_free = [r["response"] for r in random.sample(harmless_bt.to_list(), args.n_harmless // 2)] records = [] # {source, kind, user, assistant, label} for r in harm_pool: records.append({"source": "beavertails", "kind": "seed", "user": r["prompt"], "assistant": r["response"], "label": 1}) for p in or_prompts: records.append({"source": "or-bench", "kind": "seed", "user": p, "assistant": harm_free.pop() if harm_free else "", "label": 0}) # BeaverTails benign pairs (assistant text present) label 0 for r in random.sample(harmless_bt.to_list(), args.n_harmless - len(or_prompts)): records.append({"source": "beavertails", "kind": "seed", "user": r["prompt"], "assistant": r["response"], "label": 0}) # ---------- Stage II: augmentation ---------- from vllm import LLM, SamplingParams llm = LLM(model=args.generator, max_model_len=4096, dtype="bfloat16") sampling = SamplingParams(temperature=0.9, top_p=0.95, max_tokens=512) def aug_prompt(rec, task): sys = random.choice(SYSTEM_VARIATIONS).format( domain=random.choice(DOMAINS), persona=random.choice(PERSONAS)) return (f"{('System prompt in effect: ' + sys + chr(10)) if sys else ''}" f"{task}\n\nOriginal exchange:\nUser: {rec['user']}\nAssistant: {rec['assistant'][:1500]}") # 1. paraphrase (both classes) n_para = len(records) // 2 to_para = random.sample(records, n_para) outs = vllm_generate(llm, [aug_prompt(r, "Rewrite the USER message in different words, keeping its exact meaning and intent.") for r in to_para], sampling) for r, o in zip(to_para, outs): records.append({**r, "kind": "paraphrase", "user": o}) # 2. assistant pre-fill + jailbreak-style wrapper on harmful records (label stays 1) prefilled = random.sample([r for r in records if r["label"] == 1], min(1500, args.n_harmful // 4)) for r in prefilled: wrap = f"{random.choice(PREFILLS)} the following:\n[assistant continues the user's request]\n\nUser: {r['user']}" records.append({**r, "kind": "prefill", "user": wrap}) # 3. leetspeak / character obfuscation (code transform, cheap — both classes) for r in random.sample(records, min(2000, len(records))): records.append({**r, "kind": "leetspeak", "user": apply_leetspeak(r["user"])}) # 4. constitution-conditioned near-boundary benign chemistry Q&A (the >40% robustness ingredient) bnd_prompts = [] for i in range(args.n_boundary): cat = random.choice(constitution["harmless"])["name"] topic = random.choice(constitution["harmless"])["examples"] if i % 3 == 0 else None seed_hint = f" Topic to cover: {random.choice(topic)}." if topic else "" bnd_prompts.append( f"Write ONE realistic user question about {cat.replace('_', ' ')} that sounds chemistry-related " f"and slightly near-the-line, but is clearly legitimate and safe per the constitution category " f"'{cat}'.{seed_hint} Reply with ONLY the question.") bnd_q = vllm_generate(llm, bnd_prompts, sampling) bnd_a = vllm_generate(llm, [f"You are a helpful chemistry assistant. Answer safely and accurately:\n\n{q}" for q in bnd_q], SamplingParams(temperature=0.7, top_p=0.9, max_tokens=400)) for q, a in zip(bnd_q, bnd_a): records.append({"source": "constitution", "kind": "boundary", "user": q, "assistant": a, "label": 0}) # ---------- Splits ---------- random.shuffle(records) n_eval = max(500, int(len(records) * 0.03)) train, val = records[:-n_eval], records[-n_eval:] def dump(path, rows): with open(path, "w") as f: f.writelines(json.dumps(r) + "\n" for r in rows) dump(f"{args.output_dir}/train.jsonl", train) dump(f"{args.output_dir}/val.jsonl", val) # benign calibration pools hard = load_dataset("bench-llm/or-bench", "or-bench-hard-1k", split="train")["prompt"] dump(f"{args.output_dir}/calib_orbench_hard.jsonl", [{"user": p, "label": 0} for p in hard]) wc = load_dataset("allenai/WildChat", split="train", streaming=True) wild = [{"user": c[0]["content"], "label": 0} for _, c in zip(range(1000), wc) if c and c[0]["role"] == "user"] dump(f"{args.output_dir}/calib_wildchat.jsonl", wild) pos = sum(r["label"] for r in train) print(f"train={len(train)} (harmful={pos}, {pos/len(train):.1%}) val={len(val)}") if args.hub_dataset_id: from huggingface_hub import HfApi api = HfApi() api.create_repo(args.hub_dataset_id, repo_type="dataset", exist_ok=True) for f in ["train.jsonl", "val.jsonl", "calib_orbench_hard.jsonl", "calib_wildchat.jsonl"]: api.upload_file(path_or_fileobj=f"{args.output_dir}/{f}", path_in_repo=f, repo_id=args.hub_dataset_id, repo_type="dataset") print(f"pushed -> https://huggingface.co/datasets/{args.hub_dataset_id}") if __name__ == "__main__": main()