Text Generation
Transformers
Safetensors
GGUF
English
qwen3_5
image-text-to-text
decision-model
typed-decisions
calibration
calibrated-probabilities
classification
tool-selection
agent-routing
decision-index
jevbench
jev-compatible
systemone
wald
wald-q4b
qwen3.5
4b
vllm
reasoning
llama.cpp
conversational
Eval Results (legacy)
Instructions to use org2ai/Wald-4B with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use org2ai/Wald-4B with Transformers:
# Use a pipeline as a high-level helper from transformers import pipeline pipe = pipeline("text-generation", model="org2ai/Wald-4B") messages = [ { "role": "user", "content": [ {"type": "image", "url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/p-blog/candy.JPG"}, {"type": "text", "text": "What animal is on the candy?"} ] }, ] pipe(text=messages)# pip install -U transformers accelerate # Load model directly from transformers import AutoProcessor, AutoModelForMultimodalLM processor = AutoProcessor.from_pretrained("org2ai/Wald-4B") model = AutoModelForMultimodalLM.from_pretrained("org2ai/Wald-4B", device_map="auto") messages = [ { "role": "user", "content": [ {"type": "image", "url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/p-blog/candy.JPG"}, {"type": "text", "text": "What animal is on the candy?"} ] }, ] inputs = processor.apply_chat_template( messages, add_generation_prompt=True, tokenize=True, return_dict=True, return_tensors="pt", ).to(model.device) outputs = model.generate(**inputs, max_new_tokens=256) print(processor.decode(outputs[0][inputs["input_ids"].shape[-1]:])) - Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- vLLM
How to use org2ai/Wald-4B with vLLM:
Install from pip and serve model
# Install vLLM from pip: pip install vllm # Start the vLLM server: vllm serve "org2ai/Wald-4B" # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:8000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "org2ai/Wald-4B", "messages": [ { "role": "user", "content": "What is the capital of France?" } ] }'Use Docker
docker model run hf.co/org2ai/Wald-4B
- SGLang
How to use org2ai/Wald-4B with SGLang:
Install from pip and serve model
# Install SGLang from pip: pip install sglang # Start the SGLang server: python3 -m sglang.launch_server \ --model-path "org2ai/Wald-4B" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "org2ai/Wald-4B", "messages": [ { "role": "user", "content": "What is the capital of France?" } ] }'Use Docker images
docker run --gpus all \ --shm-size 32g \ -p 30000:30000 \ -v ~/.cache/huggingface:/root/.cache/huggingface \ --env "HF_TOKEN=<secret>" \ --ipc=host \ lmsysorg/sglang:latest \ python3 -m sglang.launch_server \ --model-path "org2ai/Wald-4B" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "org2ai/Wald-4B", "messages": [ { "role": "user", "content": "What is the capital of France?" } ] }' - Docker Model Runner
How to use org2ai/Wald-4B with Docker Model Runner:
docker model run hf.co/org2ai/Wald-4B
org2ai
Wald-Q4B v2 (04701-c22 + Qwen3.5-4B vision tower): main = v2; v1.x at tags v1.2 / v1.2-main / v1.1 / v1.0
ef7481d Download evaluation/v2/code/di03_parallel.py from org2ai/Wald-4B: direct link, hf CLI and curl.
- Browser
- Download file 5.7 kB
-
https://huggingface.co/org2ai/Wald-4B/resolve/main/evaluation/v2/code/di03_parallel.py
- Command line
-
hf download hf://org2ai/Wald-4B/evaluation/v2/code/di03_parallel.py
-
curl -L -o di03_parallel.py https://huggingface.co/org2ai/Wald-4B/resolve/main/evaluation/v2/code/di03_parallel.py
5.7 kB
| """Bounded public DI inference, compatible with the official results schema. | |
| One immutable request per run_id. No power actions or model loading. Declared | |
| concurrent throughput measurements are not leaderboard serial latency. Score | |
| with the unmodified, pinned official 0.3 kit after inference has ended. | |
| """ | |
| import argparse,collections,concurrent.futures as cf,hashlib,json,os,socket,sys,threading,time | |
| from pathlib import Path | |
| from decision_index.engines.base import Unsupported, validate | |
| from decision_index.runner import stamp | |
| from decision_index.suite.io import Suite,dumps | |
| from di03_native_auto_engine import WaldNativeAuto | |
| def sha(p): | |
| with Path(p).open('rb') as f:return hashlib.file_digest(f,'sha256').hexdigest() | |
| def write_json(p,v): | |
| tmp=Path(str(p)+'.tmp') | |
| with tmp.open('w') as f:json.dump(v,f,indent=2);f.flush();os.fsync(f.fileno()) | |
| tmp.replace(p) | |
| def run(c,engine_factory=WaldNativeAuto): | |
| assert c['status'] in ('APPROVED_DI03_C16B_AUTO_FRESH_GPU_4H','APPROVED_DI03_C16B_AUTO_UNTIL_COMPLETE') | |
| deadline=c.get('deadline_epoch') | |
| if c['status']=='APPROVED_DI03_C16B_AUTO_UNTIL_COMPLETE':assert deadline is None and c['shutdown_on_completion'] is True | |
| else:assert deadline==c['GPU_boot_epoch']+14400 | |
| assert c['model_id']=='04400-c18' and c['policy']=='gate0.7' and c['budget']==512 | |
| assert c['concurrency'] in (64,128) | |
| assert socket.gethostname()==c['hostname'] | |
| assert sha(__file__)==c['runner_sha256'] | |
| assert deadline is None or time.time()<deadline-180 | |
| for p,h in c['source_pins'].items():assert sha(p)==h | |
| replicas=c.get('replicas',1);rank=c.get('replica_rank',0) | |
| assert replicas in (1,4) and 0<=rank<replicas | |
| def assigned(row):return int(hashlib.sha256(row['_evaluation']['run_id'].encode()).hexdigest(),16)%replicas==rank | |
| suite=Suite(c['suite_dir'],'0.3');verified=suite.verify(strict=True) | |
| expected=sum(1 for row in suite.rows() if assigned(row)) if replicas>1 else 140620 | |
| assert verified['edition']=='0.3' | |
| out=Path(c['out_dir']);out.mkdir(exist_ok=False) | |
| opts=dict(native_path=c['native_path'],temperature_path=c['temperature_path'],temperature_sha256=c['temperature_sha256'],deadline_epoch=deadline,endpoint=f'http://127.0.0.1:{8371+rank}',model=c['model_id'],max_model_len=131072,reference_root=c.get('reference_root')) | |
| first=engine_factory(**opts) | |
| environment={'engine':'wald-v2-native-auto0.7','model_source':first.provenance,'engine_options':opts,'concurrency':c['concurrency'],'replicas':replicas,'replica_rank':rank,'partition':'SHA256(run_id) modulo replicas','expected_requests':expected,'frozen_corpus':verified,'latency':first.latency,'runner_sha256':sha(__file__),'root_contract_sha256':c['root_contract_sha256'],'boot_epoch':c['GPU_boot_epoch'],'deadline_epoch':deadline,'no_model_startup_or_power_operations':True} | |
| write_json(out/'environment.json',environment) | |
| # The first client runs only benchmark-free synthetic warmup. | |
| first.warmup();local=threading.local();seen=set();counts=collections.Counter();start=time.time();stop=False | |
| def one(row): | |
| e=row['_evaluation'];t=time.perf_counter();r={**e,'engine':'wald-v2-native-auto0.7','started_utc':stamp()} | |
| try: | |
| if deadline is not None and time.time()>=deadline-180:return None | |
| if not hasattr(local,'engine'):local.engine=engine_factory(**opts) | |
| response,_=local.engine(row['state'],row['questions']);validate(row['questions'],response) | |
| r.update(status='ok',response=response) | |
| except Unsupported as exc:r.update(status='unsupported',error=str(exc)) | |
| except Exception as exc:r.update(status='error',exception=type(exc).__name__,error=str(exc)) | |
| ms=(time.perf_counter()-t)*1000;r.update(completed_utc=stamp(),total_wall_ms=ms,model_request_wall_ms=ms);return r | |
| rows=iter(row for row in suite.rows() if assigned(row));f=(out/'results.jsonl').open('x') | |
| try: | |
| with cf.ThreadPoolExecutor(max_workers=c['concurrency']) as pool: | |
| pending={} | |
| def submit(): | |
| row=next(rows,None) | |
| if row is None:return False | |
| rid=row['_evaluation']['run_id'];assert rid not in seen;seen.add(rid) | |
| pending[pool.submit(one,row)]=rid;return True | |
| for _ in range(c['concurrency']): | |
| if not submit():break | |
| while pending: | |
| if deadline is not None and time.time()>=deadline-180:stop=True | |
| done,_=cf.wait(pending,timeout=1,return_when=cf.FIRST_COMPLETED) | |
| for future in done: | |
| pending.pop(future);r=future.result() | |
| if r is not None: | |
| f.write(dumps(r)+'\n');f.flush();counts[r['status']]+=1 | |
| if r.get('exception') in ('OutOfMemoryError','AcceleratorError') or 'device-side assert' in r.get('error',''):stop=True | |
| if not stop:submit() | |
| if counts['error']>=5 and not counts['ok']:stop=True | |
| write_json(out/'status.json',{'status':'DRAINING_AT_BOUND_OR_FAILURE' if stop else 'EVALUATING','completed':sum(counts.values()),'expected':expected,'counts':dict(counts),'epoch':time.time(),'elapsed_seconds':time.time()-start,'deadline_epoch':deadline,'official_serial_latency_measured':False}) | |
| finally:f.flush();os.fsync(f.fileno());f.close() | |
| record={'status':'ALL_PUBLIC_REQUESTS_ATTEMPTED' if sum(counts.values())==expected else 'BOUND_OR_FAILURE_PARTIAL','counts':dict(counts),'expected':expected,'completed':sum(counts.values()),'epoch':time.time(),'results_sha256':sha(out/'results.jsonl'),'score_pending':True,'actual_provider_OFF_pending':True,'no_policy_selection_from_TEST':True} | |
| write_json(out/'terminal.json',record) | |
| return record | |
| def main(): | |
| p=argparse.ArgumentParser();p.add_argument('--contract',required=True);p.add_argument('--sha256',required=True);a=p.parse_args() | |
| assert sha(a.contract)==a.sha256;c=json.loads(Path(a.contract).read_text());c['root_contract_sha256']=a.sha256 | |
| print(json.dumps(run(c))) | |
| if __name__=='__main__':main() | |