Blame for v2/prepare_data.py

"""Build the v2 curriculum on a remote CPU; no model inference."""
import json,random,re,hashlib,collections,urllib.request,urllib.parse,time,itertools
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
from huggingface_hub import HfApi
ROOT=Path(__file__).resolve().parent
rng=random.Random(20261113)
def viewer_rows(repo,config,max_rows=8000):
    def fetch(offset):
        url="https://datasets-server.huggingface.co/rows?"+urllib.parse.urlencode({"dataset":repo,"config":config,"split":"train","offset":offset,"length":100})
        for attempt in range(4):
            try:
                data=json.load(urllib.request.urlopen(url,timeout=60))
                return [r["row"] for r in data["rows"]]
            except Exception:
                if attempt==3:raise
                time.sleep(2**attempt)
    with ThreadPoolExecutor(max_workers=4) as pool:
        for start in range(1000,1000+max_rows,400):
            pages=list(pool.map(fetch,range(start,start+400,100)))
            records=[r for p in pages for r in p]
            random.Random(5517+start).shuffle(records)
            print("DATA_FETCH",config,start,len(records),flush=True)
            yield from records
SYSTEM="You are Astheral, an AI assistant developed by SEE7 using DeepHermes 3. Give useful, direct answers. Follow requested formats exactly. Solve the whole problem, account for every relevant quantity, and check the result. Distinguish known facts from missing information. Engage naturally with fictional roleplay and legitimate technical work."
THINK="You are a deep thinking AI. Solve the problem carefully. Enclose a concise, complete derivation in <think>...</think>, check all relevant quantities, then give the final answer. Avoid repeating calculations."
rows=[]
def add(user,answer,mode="instruct",source="astheral-curriculum",extra=""):
    rows.append({"mode":mode,"source":source,"messages":[{"role":"system","content":SYSTEM+("\n"+extra if extra else "")+("\n"+THINK if mode=="thinking" else "")},{"role":"user","content":user},{"role":"assistant","content":answer}]})
# Preserve authored style, identity, roleplay, uncertainty and code examples.
original=[json.loads(s) for s in (ROOT/"original.jsonl").read_text().splitlines()]
for r in original:
    if r.get("category") in ["general","coding"]:
        add(r["messages"][1]["content"],r["messages"][2]["content"],r["mode"],"astheral-authored-v1")
nouns=[("amber beads","violet beads"),("oak saplings","pine saplings"),("fiction books","history books"),("brass keys","silver keys"),("green tiles","white tiles"),("large boxes","small boxes")]
for i in range(180):
    a=rng.randint(12,95);b=rng.randint(8,70);removed=rng.randint(1,a-1);x,y=nouns[i%len(nouns)]
    mode="thinking" if i%2 else "instruct"
    u=f"A collection contains {a} {x} and {b} {y}. Someone takes away {removed} {x}. How many items remain altogether? Explain briefly."
    remain=a-removed;total=remain+b
    steps=f"There are {remain} {x} left because {a} - {removed} = {remain}. The {b} {y} are unchanged, so {remain} + {b} = {total} items remain."
    add(u,f"<think>{steps}</think>\n{total} items remain." if mode=="thinking" else steps,mode)
for i in range(120):
    a=rng.randint(3,40);b=rng.randint(2,18);c=rng.randint(1,35)
    typ=i%4;mode="thinking" if i%3 else "instruct"
    if typ==0:
        u=f"A shop has {a} trays with {b} pastries on each. It sells {c} pastries. How many remain?";ans=a*b-c
        if ans<0:continue
        steps=f"The trays contain {a} × {b} = {a*b} pastries. Subtract the {c} sold: {a*b} - {c} = {ans}."
    elif typ==1:
        u=f"A journey consists of {a} kilometres in the morning and {b} kilometres in the afternoon, for {c} days. What is the total distance?";ans=(a+b)*c;steps=f"Daily distance is {a} + {b} = {a+b} km. Across {c} days: {a+b} × {c} = {ans} km."
    elif typ==2:
        u=f"A group has {a} adults and {b} children. Each person receives {c} tokens. How many tokens are needed?";ans=(a+b)*c;steps=f"Count everyone: {a} + {b} = {a+b} people. Multiply by {c} tokens each: {a+b} × {c} = {ans}."
    else:
        u=f"A worker earns {a} pounds per hour for {b} hours and spends {c} pounds on travel. What is the amount left after travel?";ans=a*b-c;steps=f"Earnings are {a} × {b} = {a*b} pounds. After travel: {a*b} - {c} = {ans} pounds."
    add(u+ " Show the calculation.",f"<think>{steps}</think>\nAnswer: {ans}." if mode=="thinking" else steps,mode)
words=["copper","maple","saffron","harbor","cobalt","sunrise","meadow","cedar","silver","comet","orchid","willow"]
for i in range(160):
    selected=rng.sample(words,rng.randint(2,5));typ=i%4
    if typ==0:u="Write the following words in the same order, each on its own line. Preserve spelling and case; no bullets, blank lines or introduction: "+", ".join(selected);ans="\n".join(selected)
    elif typ==1:u="Reply with exactly this text, preserving capitalization and punctuation: "+" ".join(selected)+"!";ans=" ".join(selected)+"!"
    elif typ==2:u="Return only a JSON array of these words in their original order: "+", ".join(selected);ans=json.dumps(selected)
    else:u="List these words using one '- ' bullet per line and no other text: "+", ".join(selected);ans="\n".join("- "+w for w in selected)
    add(u,ans)
for i in range(100):
    a=rng.randint(-70,90);b=rng.randint(-70,90);c=rng.randint(-70,90)
    if i%2:
        add(f'Return only a JSON object containing "minimum", "maximum" and "count" for this list: [{a}, {b}, {c}].',json.dumps({"minimum":min(a,b,c),"maximum":max(a,b,c),"count":3}))
    else:
        add(f"Return exactly one CSV row, without a header: name=item{i}, quantity={a}, price={abs(b)}.",f"item{i},{a},{abs(b)}")
# Short multi-turn memory and correction.
for i in range(40):
    name=words[i%len(words)]+str(i);value=rng.randint(100,900)
    rows.append({"mode":"instruct","source":"astheral-curriculum","messages":[{"role":"system","content":SYSTEM},{"role":"user","content":f"My project is {name} and its code is {value}."},{"role":"assistant","content":"Got it."},{"role":"user","content":"Return only the project code."},{"role":"assistant","content":str(value)}]})
# General instruction retention; these four SmolTalk components use Apache 2.0.
api=HfApi();sources=[]
for cfg,count in [("smol-magpie-ultra",650),("smol-constraints",350),("smol-rewrite",100),("smol-summarize",100)]:
    repo="HuggingFaceTB/smoltalk";rev=api.dataset_info(repo).sha
    ds=viewer_rows(repo,cfg)
    accepted=0
    for record in ds:
        ms=record["messages"]
        user=next((m["content"] for m in ms if m["role"]=="user"),None)
        answer=next((m["content"] for m in ms if m["role"]=="assistant"),None)
        if not user or not answer or len(user)+len(answer)>4200 or len(answer)<30:continue
        extra="\n".join(m["content"] for m in ms if m["role"]=="system")
        add(user,answer,"instruct",repo+"/"+cfg,extra)
        accepted+=1
        if accepted>=count:break
    if accepted<count:raise RuntimeError("Too few records: "+cfg)
    assert api.dataset_info(repo).sha==rev,"Dataset changed during extraction"
    sources.append({"repo":repo,"config":cfg,"revision":rev,"count":accepted,"license":"Apache-2.0"})
    print("DATA_COMPONENT",cfg,accepted,flush=True)
# Real multi-step math explanations from the training split only.
repo="openai/gsm8k";rev=api.dataset_info(repo).sha
ds=viewer_rows(repo,"main",max_rows=800)
for i,r in enumerate(itertools.islice(ds,400)):
    steps,answer=r["answer"].rsplit("####",1);steps=re.sub(r"<<.*?>>","",steps).strip();answer=answer.strip()
    mode="thinking" if i%4 else "instruct"
    add(r["question"],f"<think>{steps}</think>\nAnswer: {answer}" if mode=="thinking" else steps+"\nAnswer: "+answer,mode,repo)
sources.append({"repo":repo,"revision":rev,"split":"train","count":400,"license":"MIT"})
# Executable-filtered general Python examples.
repo="bigcode/self-oss-instruct-sc2-exec-filter-50k";rev=api.dataset_info(repo).sha
ds=viewer_rows(repo,"default")
accepted=0
for r in ds:
    u=r["instruction"];a=r["response"]
    if "python" not in u.lower() or len(u)+len(a)>4200 or len(a)<40:continue
    if any(w in u.lower() for w in ["exploit","malware","payload","password crack","sql injection"]):continue
    add(u,a,"instruct",repo)
    accepted+=1
    if accepted>=250:break
sources.append({"repo":repo,"revision":rev,"count":accepted,"license":"ODC-BY-1.0"})
# Deduplicate complete conversations, without repeatedly weighting identical answers.
seen=set();dedup=[]
for r in rows:
    k=json.dumps(r["messages"],sort_keys=True)
    if k not in seen:seen.add(k);dedup.append(r)
rng.shuffle(dedup)
(ROOT/"curriculum.jsonl").write_text("\n".join(json.dumps(r,ensure_ascii=False) for r in dedup)+"\n")
(ROOT/"settings.json").write_text(json.dumps({"system":SYSTEM,"thinking":THINK},indent=2))
(ROOT/"data_sources.json").write_text(json.dumps({"sources":sources,"counts":dict(collections.Counter(r["source"] for r in dedup)),"total":len(dedup),"seed":20261113},indent=2))
api.upload_folder(repo_id="SEE7/Astheral",folder_path=ROOT,path_in_repo="v2",ignore_patterns=["__pycache__/*","original.jsonl"],commit_message="Stage v2 balanced curriculum and reproducible training pipeline")
print("CURRICULUM_READY",len(dedup),flush=True)