Text Generation
Transformers
Safetensors
English
llama
astheral
instruct
qlora
conversational
text-generation-inference
Instructions to use SEE7/Astheral with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use SEE7/Astheral with Transformers:
# Use a pipeline as a high-level helper from transformers import pipeline pipe = pipeline("text-generation", model="SEE7/Astheral") messages = [ {"role": "user", "content": "Who are you?"}, ] pipe(messages)# pip install -U transformers accelerate # Load model directly from transformers import AutoTokenizer, AutoModelForCausalLM tokenizer = AutoTokenizer.from_pretrained("SEE7/Astheral") model = AutoModelForCausalLM.from_pretrained("SEE7/Astheral", device_map="auto") messages = [ {"role": "user", "content": "Who are you?"}, ] inputs = tokenizer.apply_chat_template( messages, add_generation_prompt=True, tokenize=True, return_dict=True, return_tensors="pt", ).to(model.device) outputs = model.generate(**inputs, max_new_tokens=256) print(tokenizer.decode(outputs[0][inputs["input_ids"].shape[-1]:])) - Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- vLLM
How to use SEE7/Astheral with vLLM:
Install from pip and serve model
# Install vLLM from pip: pip install vllm # Start the vLLM server: vllm serve "SEE7/Astheral" # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:8000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "SEE7/Astheral", "messages": [ { "role": "user", "content": "What is the capital of France?" } ] }'Use Docker
docker model run hf.co/SEE7/Astheral
- SGLang
How to use SEE7/Astheral with SGLang:
Install from pip and serve model
# Install SGLang from pip: pip install sglang # Start the SGLang server: python3 -m sglang.launch_server \ --model-path "SEE7/Astheral" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "SEE7/Astheral", "messages": [ { "role": "user", "content": "What is the capital of France?" } ] }'Use Docker images
docker run --gpus all \ --shm-size 32g \ -p 30000:30000 \ -v ~/.cache/huggingface:/root/.cache/huggingface \ --env "HF_TOKEN=<secret>" \ --ipc=host \ lmsysorg/sglang:latest \ python3 -m sglang.launch_server \ --model-path "SEE7/Astheral" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "SEE7/Astheral", "messages": [ { "role": "user", "content": "What is the capital of France?" } ] }' - Docker Model Runner
How to use SEE7/Astheral with Docker Model Runner:
docker model run hf.co/SEE7/Astheral
Download holdout_eval.py from SEE7/Astheral: direct link, hf CLI and curl.
- Browser
- Download file 8.58 kB
-
https://huggingface.co/SEE7/Astheral/resolve/main/holdout_eval.py
- Command line
-
hf download hf://SEE7/Astheral/holdout_eval.py
-
curl -L -o holdout_eval.py https://huggingface.co/SEE7/Astheral/resolve/main/holdout_eval.py
8.58 kB
| # /// script | |
| # dependencies = ["torch==2.8.0", "transformers==4.56.2", "huggingface_hub==0.34.4", "accelerate==1.10.1", "safetensors==0.6.2"] | |
| # /// | |
| """Independent GSM8K and HumanEval subset comparison on remote GPUs.""" | |
| import json,re,sys,subprocess,tempfile,ast,urllib.request,urllib.parse,gc,hashlib | |
| from pathlib import Path | |
| import torch | |
| from transformers import AutoModelForCausalLM,AutoTokenizer | |
| from huggingface_hub import HfApi,hf_hub_download,CommitOperationAdd | |
| def task_check(case,text): | |
| final=text.split("</think>")[-1] | |
| numbers=re.findall(r"-?\d+(?:\.\d+)?",final.replace(",","")) | |
| return bool(numbers) and abs(float(numbers[-1])-float(case["expected"]))<1e-6 | |
| api=HfApi();REPO="SEE7/Astheral";BASE="NousResearch/DeepHermes-3-Llama-3-8B-Preview";REV="53d902c7dcfa4c749a3b455149064f82af2e2549" | |
| ROOT=Path(__file__).resolve().parent | |
| BRANCH="v3-candidate-20261113" | |
| revision=api.model_info(REPO,revision=BRANCH).sha | |
| settings=json.loads(Path(hf_hub_download(REPO,"settings.json",revision=revision)).read_text()) | |
| def rows(repo,config,split,offset,length): | |
| q=urllib.parse.urlencode(dict(dataset=repo,config=config,split=split,offset=offset,length=length)) | |
| return [r["row"] for r in json.load(urllib.request.urlopen("https://datasets-server.huggingface.co/rows?"+q,timeout=60))["rows"]] | |
| math_revision=api.dataset_info("openai/gsm8k").sha | |
| code_revision=api.dataset_info("openai/openai_humaneval").sha | |
| cases=[] | |
| for i,r in enumerate(rows("openai/gsm8k","main","test",231,8)): | |
| expected=float(r["answer"].rsplit("####",1)[1].strip().replace(",","")) | |
| cases.append({"id":"gsm8k-test-"+str(231+i),"mode":"thinking","prompt":r["question"]+"\nGive the final numerical answer after your explanation.","kind":"number","expected":expected,"source":"openai/gsm8k","source_revision":math_revision}) | |
| chosen={"HumanEval/"+str(i) for i in [0,3,7,12,18,25,36,45]} | |
| for r in rows("openai/openai_humaneval","openai_humaneval","test",0,50): | |
| if r["task_id"] not in chosen:continue | |
| cases.append({"id":r["task_id"],"mode":"instruct","prompt":"Return only the complete Python implementation, including necessary imports, for the following function specification:\n\n"+r["prompt"],"kind":"code","entry_point":r["entry_point"],"tests":r["test"]+"\ncheck("+r["entry_point"]+")","source":"openai/openai_humaneval","source_revision":code_revision}) | |
| assert len(cases)==16 | |
| def check_code(c,text): | |
| blocks=re.findall(r"\x60\x60\x60(?:python)?\s*\n(.*?)\x60\x60\x60",text,re.S) | |
| code=blocks[0] if blocks else text.strip() | |
| try: | |
| tree=ast.parse(code) | |
| for n in ast.walk(tree): | |
| if isinstance(n,(ast.Import,ast.ImportFrom)): | |
| names=[a.name.split(".")[0] for a in n.names] if isinstance(n,ast.Import) else [(n.module or "").split(".")[0]] | |
| if any(x not in ["math","collections","itertools","functools","re","typing","statistics","heapq","bisect","string"] for x in names):return False | |
| if isinstance(n,ast.Name) and n.id in ["open","exec","eval","compile","__import__","__builtins__","globals","locals","getattr","setattr","input"]:return False | |
| if isinstance(n,ast.Attribute) and n.attr.startswith("__"):return False | |
| with tempfile.TemporaryDirectory() as d: | |
| p=Path(d)/"check.py" | |
| p.write_text("import resource\nresource.setrlimit(resource.RLIMIT_CPU,(3,3))\nresource.setrlimit(resource.RLIMIT_AS,(536870912,536870912))\n"+code+"\n"+c["tests"]) | |
| r=subprocess.run([sys.executable,"-I",str(p)],cwd=d,timeout=5,capture_output=True,env={"PATH":"/usr/bin:/bin"}) | |
| return r.returncode==0 | |
| except Exception:return False | |
| all_results={};scores={};paired_results={} | |
| source=Path(hf_hub_download(REPO,'v3/train_v3.py',revision=revision)).read_text() | |
| grade_node=next(n for n in ast.parse(source).body if isinstance(n,ast.FunctionDef) and n.name=='grade') | |
| grade_scope={'json':json,'re':re};exec(compile(ast.Module(body=[grade_node],type_ignores=[]),'grader','exec'),grade_scope) | |
| paired_cases=json.loads(Path(hf_hub_download(REPO,'v3-results/heldout_cases.json',revision=revision)).read_text()) | |
| for tag,repo,rev in [("base",REPO,"d9aa899722be4788f57107c2951c72d0ec6614ee"),("astheral",REPO,revision)]: | |
| tok=AutoTokenizer.from_pretrained(repo,revision=rev);tok.pad_token_id=128004;tok.padding_side="left" | |
| model=AutoModelForCausalLM.from_pretrained(repo,revision=rev,torch_dtype=torch.bfloat16,device_map={"":"cuda:0"},attn_implementation="sdpa").eval() | |
| paired=[] | |
| with torch.inference_mode(): | |
| for c in paired_cases: | |
| inp=tok.apply_chat_template([{"role":"system","content":settings["system"]},{"role":"user","content":c["prompt"]}],tokenize=True,add_generation_prompt=True,return_dict=True,return_tensors="pt").to("cuda") | |
| output=model.generate(**inp,max_new_tokens=96,do_sample=False,pad_token_id=128004,eos_token_id=128009) | |
| text=tok.decode(output[0,inp["input_ids"].shape[1]:],skip_special_tokens=True) | |
| paired.append({**c,"output":text,"pass":grade_scope["grade"](c,text)}) | |
| print("BF16_PAIRED",tag,c["id"],paired[-1]["pass"],flush=True) | |
| paired_results[tag]=paired | |
| result=[] | |
| with torch.inference_mode(): | |
| for mode in ["instruct","thinking"]: | |
| selected=[c for c in cases if c["mode"]==mode] | |
| for i in range(0,len(selected),2): | |
| group=selected[i:i+2] | |
| system=settings["system"]+("\n"+settings["thinking"] if mode=="thinking" else "") | |
| texts=[tok.apply_chat_template([{"role":"system","content":system},{"role":"user","content":c["prompt"]}],tokenize=False,add_generation_prompt=True) for c in group] | |
| inputs=tok(texts,padding=True,return_tensors="pt").to("cuda") | |
| out=model.generate(**inputs,max_new_tokens=1536 if mode=="thinking" else 768,do_sample=False,pad_token_id=128004,eos_token_id=128009) | |
| for c,o in zip(group,out): | |
| ids=o[inputs["input_ids"].shape[1]:] | |
| text=tok.decode(ids,skip_special_tokens=True) | |
| passed=check_code(c,text) if mode=="instruct" else task_check(c,text) | |
| result.append({**c,"output":text,"pass":passed,"hit_token_limit":len(ids)>=(1536 if mode=="thinking" else 768) and 128009 not in ids.tolist()}) | |
| print("HOLDOUT",tag,c["id"],passed,flush=True) | |
| all_results[tag]=result | |
| scores[tag]={} | |
| for kind in ["code","number"]: | |
| chosen_rows=[r for r in result if r["kind"]==kind] | |
| scores[tag][kind]={"passed":sum(r["pass"] for r in chosen_rows),"total":len(chosen_rows),"token_limited":sum(r["hit_token_limit"] for r in chosen_rows)} | |
| del model;gc.collect();torch.cuda.empty_cache() | |
| passed=sum(v["passed"] for v in scores["astheral"].values())>=sum(v["passed"] for v in scores["base"].values()) and all(scores["astheral"][k]["passed"]>=scores["base"][k]["passed"] for k in ["code","number"]) | |
| passed=passed and sum(r["pass"] for r in paired_results["astheral"])>=sum(r["pass"] for r in paired_results["base"]) | |
| report={"comparison":"V2 versus V3, both BF16","v2_revision":"d9aa899722be4788f57107c2951c72d0ec6614ee","paired_bf16_scores":{k:sum(r["pass"] for r in v) for k,v in paired_results.items()},"suite":"GSM8K test rows 231-238; HumanEval selected tasks 0,3,7,12,18,25,36,45","scores":scores,"no_material_regression_gate":passed,"astheral_revision":revision,"settings":{"sampling":False,"code_max_new_tokens":768,"math_max_new_tokens":1536},"hardware":torch.cuda.get_device_name(0),"source_revisions":{"gsm8k":math_revision,"humaneval":code_revision}} | |
| changed={"paired_bf16_outputs.json":json.dumps(paired_results,indent=2),"holdout_evaluation.json":json.dumps(report,indent=2),"holdout_outputs.json":json.dumps(all_results,indent=2),"holdout_eval.py":Path(__file__).read_text(),"HUMANEVAL_LICENSE.txt":urllib.request.urlopen("https://raw.githubusercontent.com/openai/human-eval/master/LICENSE",timeout=60).read().decode()} | |
| manifest=json.loads(Path(hf_hub_download(REPO,"artifact_manifest.json")).read_text()) | |
| for n,b in changed.items():manifest[n]={"bytes":len(b.encode()),"sha256":hashlib.sha256(b.encode()).hexdigest()} | |
| changed["artifact_manifest.json"]=json.dumps(manifest,indent=2) | |
| api.create_commit(repo_id=REPO,revision=BRANCH,operations=[CommitOperationAdd(path_in_repo=n,path_or_fileobj=b.encode()) for n,b in changed.items()],commit_message="Record measured V2 versus V3 GSM8K and HumanEval comparison") | |
| print("HOLDOUT_REPORT",json.dumps(report),flush=True) | |
| if not passed:raise SystemExit(5) | |