| """Build the v2 curriculum on a remote CPU; no model inference.""" |
| import json,random,re,hashlib,collections,urllib.request,urllib.parse,time,itertools |
| from concurrent.futures import ThreadPoolExecutor |
| from pathlib import Path |
| from huggingface_hub import HfApi |
| ROOT=Path(__file__).resolve().parent |
| rng=random.Random(20261113) |
| def viewer_rows(repo,config,max_rows=8000): |
| def fetch(offset): |
| url="https://datasets-server.huggingface.co/rows?"+urllib.parse.urlencode({"dataset":repo,"config":config,"split":"train","offset":offset,"length":100}) |
| for attempt in range(4): |
| try: |
| data=json.load(urllib.request.urlopen(url,timeout=60)) |
| return [r["row"] for r in data["rows"]] |
| except Exception: |
| if attempt==3:raise |
| time.sleep(2**attempt) |
| with ThreadPoolExecutor(max_workers=4) as pool: |
| for start in range(1000,1000+max_rows,400): |
| pages=list(pool.map(fetch,range(start,start+400,100))) |
| records=[r for p in pages for r in p] |
| random.Random(5517+start).shuffle(records) |
| print("DATA_FETCH",config,start,len(records),flush=True) |
| yield from records |
| SYSTEM="You are Astheral, an AI assistant developed by SEE7 using DeepHermes 3. Give useful, direct answers. Follow requested formats exactly. Solve the whole problem, account for every relevant quantity, and check the result. Distinguish known facts from missing information. Engage naturally with fictional roleplay and legitimate technical work." |
| THINK="You are a deep thinking AI. Solve the problem carefully. Enclose a concise, complete derivation in <think>...</think>, check all relevant quantities, then give the final answer. Avoid repeating calculations." |
| rows=[] |
| def add(user,answer,mode="instruct",source="astheral-curriculum",extra=""): |
| rows.append({"mode":mode,"source":source,"messages":[{"role":"system","content":SYSTEM+("\n"+extra if extra else "")+("\n"+THINK if mode=="thinking" else "")},{"role":"user","content":user},{"role":"assistant","content":answer}]}) |
| |
| original=[json.loads(s) for s in (ROOT/"original.jsonl").read_text().splitlines()] |
| for r in original: |
| if r.get("category") in ["general","coding"]: |
| add(r["messages"][1]["content"],r["messages"][2]["content"],r["mode"],"astheral-authored-v1") |
| nouns=[("amber beads","violet beads"),("oak saplings","pine saplings"),("fiction books","history books"),("brass keys","silver keys"),("green tiles","white tiles"),("large boxes","small boxes")] |
| for i in range(180): |
| a=rng.randint(12,95);b=rng.randint(8,70);removed=rng.randint(1,a-1);x,y=nouns[i%len(nouns)] |
| mode="thinking" if i%2 else "instruct" |
| u=f"A collection contains {a} {x} and {b} {y}. Someone takes away {removed} {x}. How many items remain altogether? Explain briefly." |
| remain=a-removed;total=remain+b |
| steps=f"There are {remain} {x} left because {a} - {removed} = {remain}. The {b} {y} are unchanged, so {remain} + {b} = {total} items remain." |
| add(u,f"<think>{steps}</think>\n{total} items remain." if mode=="thinking" else steps,mode) |
| for i in range(120): |
| a=rng.randint(3,40);b=rng.randint(2,18);c=rng.randint(1,35) |
| typ=i%4;mode="thinking" if i%3 else "instruct" |
| if typ==0: |
| u=f"A shop has {a} trays with {b} pastries on each. It sells {c} pastries. How many remain?";ans=a*b-c |
| if ans<0:continue |
| steps=f"The trays contain {a} × {b} = {a*b} pastries. Subtract the {c} sold: {a*b} - {c} = {ans}." |
| elif typ==1: |
| u=f"A journey consists of {a} kilometres in the morning and {b} kilometres in the afternoon, for {c} days. What is the total distance?";ans=(a+b)*c;steps=f"Daily distance is {a} + {b} = {a+b} km. Across {c} days: {a+b} × {c} = {ans} km." |
| elif typ==2: |
| u=f"A group has {a} adults and {b} children. Each person receives {c} tokens. How many tokens are needed?";ans=(a+b)*c;steps=f"Count everyone: {a} + {b} = {a+b} people. Multiply by {c} tokens each: {a+b} × {c} = {ans}." |
| else: |
| u=f"A worker earns {a} pounds per hour for {b} hours and spends {c} pounds on travel. What is the amount left after travel?";ans=a*b-c;steps=f"Earnings are {a} × {b} = {a*b} pounds. After travel: {a*b} - {c} = {ans} pounds." |
| add(u+ " Show the calculation.",f"<think>{steps}</think>\nAnswer: {ans}." if mode=="thinking" else steps,mode) |
| words=["copper","maple","saffron","harbor","cobalt","sunrise","meadow","cedar","silver","comet","orchid","willow"] |
| for i in range(160): |
| selected=rng.sample(words,rng.randint(2,5));typ=i%4 |
| if typ==0:u="Write the following words in the same order, each on its own line. Preserve spelling and case; no bullets, blank lines or introduction: "+", ".join(selected);ans="\n".join(selected) |
| elif typ==1:u="Reply with exactly this text, preserving capitalization and punctuation: "+" ".join(selected)+"!";ans=" ".join(selected)+"!" |
| elif typ==2:u="Return only a JSON array of these words in their original order: "+", ".join(selected);ans=json.dumps(selected) |
| else:u="List these words using one '- ' bullet per line and no other text: "+", ".join(selected);ans="\n".join("- "+w for w in selected) |
| add(u,ans) |
| for i in range(100): |
| a=rng.randint(-70,90);b=rng.randint(-70,90);c=rng.randint(-70,90) |
| if i%2: |
| add(f'Return only a JSON object containing "minimum", "maximum" and "count" for this list: [{a}, {b}, {c}].',json.dumps({"minimum":min(a,b,c),"maximum":max(a,b,c),"count":3})) |
| else: |
| add(f"Return exactly one CSV row, without a header: name=item{i}, quantity={a}, price={abs(b)}.",f"item{i},{a},{abs(b)}") |
| |
| for i in range(40): |
| name=words[i%len(words)]+str(i);value=rng.randint(100,900) |
| rows.append({"mode":"instruct","source":"astheral-curriculum","messages":[{"role":"system","content":SYSTEM},{"role":"user","content":f"My project is {name} and its code is {value}."},{"role":"assistant","content":"Got it."},{"role":"user","content":"Return only the project code."},{"role":"assistant","content":str(value)}]}) |
| |
| api=HfApi();sources=[] |
| for cfg,count in [("smol-magpie-ultra",650),("smol-constraints",350),("smol-rewrite",100),("smol-summarize",100)]: |
| repo="HuggingFaceTB/smoltalk";rev=api.dataset_info(repo).sha |
| ds=viewer_rows(repo,cfg) |
| accepted=0 |
| for record in ds: |
| ms=record["messages"] |
| user=next((m["content"] for m in ms if m["role"]=="user"),None) |
| answer=next((m["content"] for m in ms if m["role"]=="assistant"),None) |
| if not user or not answer or len(user)+len(answer)>4200 or len(answer)<30:continue |
| extra="\n".join(m["content"] for m in ms if m["role"]=="system") |
| add(user,answer,"instruct",repo+"/"+cfg,extra) |
| accepted+=1 |
| if accepted>=count:break |
| if accepted<count:raise RuntimeError("Too few records: "+cfg) |
| assert api.dataset_info(repo).sha==rev,"Dataset changed during extraction" |
| sources.append({"repo":repo,"config":cfg,"revision":rev,"count":accepted,"license":"Apache-2.0"}) |
| print("DATA_COMPONENT",cfg,accepted,flush=True) |
| |
| repo="openai/gsm8k";rev=api.dataset_info(repo).sha |
| ds=viewer_rows(repo,"main",max_rows=800) |
| for i,r in enumerate(itertools.islice(ds,400)): |
| steps,answer=r["answer"].rsplit("####",1);steps=re.sub(r"<<.*?>>","",steps).strip();answer=answer.strip() |
| mode="thinking" if i%4 else "instruct" |
| add(r["question"],f"<think>{steps}</think>\nAnswer: {answer}" if mode=="thinking" else steps+"\nAnswer: "+answer,mode,repo) |
| sources.append({"repo":repo,"revision":rev,"split":"train","count":400,"license":"MIT"}) |
| |
| repo="bigcode/self-oss-instruct-sc2-exec-filter-50k";rev=api.dataset_info(repo).sha |
| ds=viewer_rows(repo,"default") |
| accepted=0 |
| for r in ds: |
| u=r["instruction"];a=r["response"] |
| if "python" not in u.lower() or len(u)+len(a)>4200 or len(a)<40:continue |
| if any(w in u.lower() for w in ["exploit","malware","payload","password crack","sql injection"]):continue |
| add(u,a,"instruct",repo) |
| accepted+=1 |
| if accepted>=250:break |
| sources.append({"repo":repo,"revision":rev,"count":accepted,"license":"ODC-BY-1.0"}) |
| |
| seen=set();dedup=[] |
| for r in rows: |
| k=json.dumps(r["messages"],sort_keys=True) |
| if k not in seen:seen.add(k);dedup.append(r) |
| rng.shuffle(dedup) |
| (ROOT/"curriculum.jsonl").write_text("\n".join(json.dumps(r,ensure_ascii=False) for r in dedup)+"\n") |
| (ROOT/"settings.json").write_text(json.dumps({"system":SYSTEM,"thinking":THINK},indent=2)) |
| (ROOT/"data_sources.json").write_text(json.dumps({"sources":sources,"counts":dict(collections.Counter(r["source"] for r in dedup)),"total":len(dedup),"seed":20261113},indent=2)) |
| api.upload_folder(repo_id="SEE7/Astheral",folder_path=ROOT,path_in_repo="v2",ignore_patterns=["__pycache__/*","original.jsonl"],commit_message="Stage v2 balanced curriculum and reproducible training pipeline") |
| print("CURRICULUM_READY",len(dedup),flush=True) |
|
|
