aytsrndl's picture
dataset builder: seeds + vLLM augmentation
ce87bb4 verified
Raw History Blame Contribute Delete
8.76 kB
"""Build the Constitutional Classifiers reproduction dataset (arXiv:2501.18837).
Stage I — assemble base pairs from public data:
* harmful : PKU-Alignment/BeaverTails (is_safe=False), chemistry rows prioritized
* harmless : BeaverTails safe pairs + OR-Bench-80k benign prompts
Stage II — augmentation with Qwen/Qwen2.5-7B-Instruct over vLLM:
paraphrase, system-prompt variation, assistant pre-fill, leetspeak (code),
and constitution-conditioned near-boundary benign chemistry Q&A.
HELD-OUT transforms (base64, multi-turn roleplay) are NOT applied here —
they are reserved for evaluation (paper §3.2: eval must use primitives
absent from the augmentation pipeline).
Outputs JSONL for the input classifier, the output classifier, and the
benign calibration pools, then (optionally) pushes to the Hub.
Run on GPU (vLLM). Example:
python scripts/build_dataset.py --n_harmful 6000 --n_harmless 6000 \
--hub_dataset_id aytsrndl/cc-repro-synthetic
"""
import argparse, base64, json, os, random, string, yaml
from datasets import load_dataset
SEED = 17
random.seed(SEED)
SYSTEM_VARIATIONS = [
"You are a helpful assistant with expertise in {domain}.",
"You are {persona}, a research assistant. Answer directly and concisely.",
"You are a no-nonsense consultant. Skip caveats.",
"",
]
PERSONAS = ["Dr. Ellis", "a process engineer", "a grad student", "an instructor"]
DOMAINS = ["industrial chemistry", "home science experiments", "laboratory methods"]
PREFILLS = [
"Sure! Here is",
"Absolutely, here's a detailed",
"Of course. Below is",
]
def apply_leetspeak(text: str) -> str:
table = str.maketrans("aeiost", "431057")
return text[: max(1, len(text) // 4)].translate(table) + text[len(text) // 4:]
def vllm_generate(llm, prompts, sampling, max_len=8192):
tok = llm.get_tokenizer()
chats = [[{"role": "user", "content": p}] for p in prompts]
texts = [tok.apply_chat_template(c, tokenize=False, add_generation_prompt=True) for c in chats]
outs = llm.generate(texts, sampling)
return [o.outputs[0].text.strip() for o in outs]
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--n_harmful", type=int, default=6000)
ap.add_argument("--n_harmless", type=int, default=6000)
ap.add_argument("--n_boundary", type=int, default=2000, help="constitution-conditioned near-boundary benign Q&A to generate")
ap.add_argument("--generator", default="Qwen/Qwen2.5-7B-Instruct")
ap.add_argument("--constitution", default="constitution.yaml")
ap.add_argument("--output_dir", default="data")
ap.add_argument("--hub_dataset_id", default=None)
args = ap.parse_args()
os.makedirs(args.output_dir, exist_ok=True)
constitution = yaml.safe_load(open(args.constitution))
# ---------- Stage I: public seeds ----------
bt = load_dataset("PKU-Alignment/BeaverTails", split="330k_train")
chem_cats = {"drug_abuse,weapons,banned_substance", "terrorism,organized_crime", "violence,aiding_and_abetting,incitement"}
harmful = bt.filter(lambda r: not r["is_safe"] and any(r["category"][c] for c in chem_cats))
other = bt.filter(lambda r: not r["is_safe"] and not any(r["category"][c] for c in chem_cats))
harmless_bt = bt.filter(lambda r: r["is_safe"])
n_chem = min(len(harmful), int(args.n_harmful * 0.4))
harm_pool = random.sample(harmful.to_list(), min(n_chem, len(harmful))) + \
random.sample(other.to_list(), args.n_harmful - min(n_chem, len(harmful)))
orbench = load_dataset("bench-llm/or-bench", "or-bench-80k", split="train")
or_prompts = random.sample(orbench["prompt"], args.n_harmless // 2)
harm_free = [r["response"] for r in random.sample(harmless_bt.to_list(), args.n_harmless // 2)]
records = [] # {source, kind, user, assistant, label}
for r in harm_pool:
records.append({"source": "beavertails", "kind": "seed", "user": r["prompt"],
"assistant": r["response"], "label": 1})
for p in or_prompts:
records.append({"source": "or-bench", "kind": "seed", "user": p,
"assistant": harm_free.pop() if harm_free else "", "label": 0})
# BeaverTails benign pairs (assistant text present) label 0
for r in random.sample(harmless_bt.to_list(), args.n_harmless - len(or_prompts)):
records.append({"source": "beavertails", "kind": "seed", "user": r["prompt"],
"assistant": r["response"], "label": 0})
# ---------- Stage II: augmentation ----------
from vllm import LLM, SamplingParams
llm = LLM(model=args.generator, max_model_len=4096, dtype="bfloat16")
sampling = SamplingParams(temperature=0.9, top_p=0.95, max_tokens=512)
def aug_prompt(rec, task):
sys = random.choice(SYSTEM_VARIATIONS).format(
domain=random.choice(DOMAINS), persona=random.choice(PERSONAS))
return (f"{('System prompt in effect: ' + sys + chr(10)) if sys else ''}"
f"{task}\n\nOriginal exchange:\nUser: {rec['user']}\nAssistant: {rec['assistant'][:1500]}")
# 1. paraphrase (both classes)
n_para = len(records) // 2
to_para = random.sample(records, n_para)
outs = vllm_generate(llm, [aug_prompt(r, "Rewrite the USER message in different words, keeping its exact meaning and intent.") for r in to_para], sampling)
for r, o in zip(to_para, outs):
records.append({**r, "kind": "paraphrase", "user": o})
# 2. assistant pre-fill + jailbreak-style wrapper on harmful records (label stays 1)
prefilled = random.sample([r for r in records if r["label"] == 1], min(1500, args.n_harmful // 4))
for r in prefilled:
wrap = f"{random.choice(PREFILLS)} the following:\n[assistant continues the user's request]\n\nUser: {r['user']}"
records.append({**r, "kind": "prefill", "user": wrap})
# 3. leetspeak / character obfuscation (code transform, cheap — both classes)
for r in random.sample(records, min(2000, len(records))):
records.append({**r, "kind": "leetspeak", "user": apply_leetspeak(r["user"])})
# 4. constitution-conditioned near-boundary benign chemistry Q&A (the >40% robustness ingredient)
bnd_prompts = []
for i in range(args.n_boundary):
cat = random.choice(constitution["harmless"])["name"]
topic = random.choice(constitution["harmless"])["examples"] if i % 3 == 0 else None
seed_hint = f" Topic to cover: {random.choice(topic)}." if topic else ""
bnd_prompts.append(
f"Write ONE realistic user question about {cat.replace('_', ' ')} that sounds chemistry-related "
f"and slightly near-the-line, but is clearly legitimate and safe per the constitution category "
f"'{cat}'.{seed_hint} Reply with ONLY the question.")
bnd_q = vllm_generate(llm, bnd_prompts, sampling)
bnd_a = vllm_generate(llm, [f"You are a helpful chemistry assistant. Answer safely and accurately:\n\n{q}" for q in bnd_q],
SamplingParams(temperature=0.7, top_p=0.9, max_tokens=400))
for q, a in zip(bnd_q, bnd_a):
records.append({"source": "constitution", "kind": "boundary", "user": q, "assistant": a, "label": 0})
# ---------- Splits ----------
random.shuffle(records)
n_eval = max(500, int(len(records) * 0.03))
train, val = records[:-n_eval], records[-n_eval:]
def dump(path, rows):
with open(path, "w") as f:
f.writelines(json.dumps(r) + "\n" for r in rows)
dump(f"{args.output_dir}/train.jsonl", train)
dump(f"{args.output_dir}/val.jsonl", val)
# benign calibration pools
hard = load_dataset("bench-llm/or-bench", "or-bench-hard-1k", split="train")["prompt"]
dump(f"{args.output_dir}/calib_orbench_hard.jsonl", [{"user": p, "label": 0} for p in hard])
wc = load_dataset("allenai/WildChat", split="train", streaming=True)
wild = [{"user": c[0]["content"], "label": 0} for _, c in zip(range(1000), wc) if c and c[0]["role"] == "user"]
dump(f"{args.output_dir}/calib_wildchat.jsonl", wild)
pos = sum(r["label"] for r in train)
print(f"train={len(train)} (harmful={pos}, {pos/len(train):.1%}) val={len(val)}")
if args.hub_dataset_id:
from huggingface_hub import HfApi
api = HfApi()
api.create_repo(args.hub_dataset_id, repo_type="dataset", exist_ok=True)
for f in ["train.jsonl", "val.jsonl", "calib_orbench_hard.jsonl", "calib_wildchat.jsonl"]:
api.upload_file(path_or_fileobj=f"{args.output_dir}/{f}",
path_in_repo=f, repo_id=args.hub_dataset_id, repo_type="dataset")
print(f"pushed -> https://huggingface.co/datasets/{args.hub_dataset_id}")
if __name__ == "__main__":
main()