Text Generation
Transformers
Safetensors
English
qwen3_5_text
qwen3.5
tinyllama
riscv
int4
in-weights
compiled-weights
exact-arithmetic
qwenti
conversational
Instructions to use OpceanAI/qwenti-inference with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use OpceanAI/qwenti-inference with Transformers:
# Use a pipeline as a high-level helper from transformers import pipeline pipe = pipeline("text-generation", model="OpceanAI/qwenti-inference") messages = [ {"role": "user", "content": "Who are you?"}, ] pipe(messages)# pip install -U transformers accelerate # Load model directly from transformers import AutoTokenizer, AutoModelForCausalLM tokenizer = AutoTokenizer.from_pretrained("OpceanAI/qwenti-inference") model = AutoModelForCausalLM.from_pretrained("OpceanAI/qwenti-inference", device_map="auto") messages = [ {"role": "user", "content": "Who are you?"}, ] inputs = tokenizer.apply_chat_template( messages, add_generation_prompt=True, tokenize=True, return_dict=True, return_tensors="pt", ).to(model.device) outputs = model.generate(**inputs, max_new_tokens=256) print(tokenizer.decode(outputs[0][inputs["input_ids"].shape[-1]:])) - Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- vLLM
How to use OpceanAI/qwenti-inference with vLLM:
Install from pip and serve model
# Install vLLM from pip: pip install vllm # Start the vLLM server: vllm serve "OpceanAI/qwenti-inference" # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:8000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "OpceanAI/qwenti-inference", "messages": [ { "role": "user", "content": "What is the capital of France?" } ] }'Use Docker
docker model run hf.co/OpceanAI/qwenti-inference
- SGLang
How to use OpceanAI/qwenti-inference with SGLang:
Install from pip and serve model
# Install SGLang from pip: pip install sglang # Start the SGLang server: python3 -m sglang.launch_server \ --model-path "OpceanAI/qwenti-inference" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "OpceanAI/qwenti-inference", "messages": [ { "role": "user", "content": "What is the capital of France?" } ] }'Use Docker images
docker run --gpus all \ --shm-size 32g \ -p 30000:30000 \ -v ~/.cache/huggingface:/root/.cache/huggingface \ --env "HF_TOKEN=<secret>" \ --ipc=host \ lmsysorg/sglang:latest \ python3 -m sglang.launch_server \ --model-path "OpceanAI/qwenti-inference" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "OpceanAI/qwenti-inference", "messages": [ { "role": "user", "content": "What is the capital of France?" } ] }' - Docker Model Runner
How to use OpceanAI/qwenti-inference with Docker Model Runner:
docker model run hf.co/OpceanAI/qwenti-inference
Download results.json from OpceanAI/qwenti-inference: direct link, hf CLI and curl.
- Browser
- Download file 3.61 kB
-
https://huggingface.co/OpceanAI/qwenti-inference/resolve/main/results.json
- Command line
-
hf download hf://OpceanAI/qwenti-inference/results.json
-
curl -L -o results.json https://huggingface.co/OpceanAI/qwenti-inference/resolve/main/results.json
3.61 kB
| { | |
| "sections": { | |
| "hardware": { | |
| "gpu": "NVIDIA RTX PRO 6000 Blackwell Server Edition", | |
| "vram_gb": 95.6, | |
| "driver": "580.82.07", | |
| "torch": "2.14.1+cu130", | |
| "cuda": "13.0", | |
| "capability": "12.0", | |
| "cpus": 48, | |
| "ram_gb": 176.9, | |
| "python": "3.13.16" | |
| }, | |
| "setup": { | |
| "export": "bash: line 1: cd: /content/tl: No such file or directory", | |
| "rc": [ | |
| 0, | |
| 1, | |
| 1, | |
| 1 | |
| ] | |
| }, | |
| "model": { | |
| "error": "CalledProcessError: Command '['cp', '/root/.cache/huggingface/hub/models--Maykeye--TinyLLama-v0/snapshots/298338802ab94432b917bcce11382aa151aee50f/model.safetensors', '/content/tl/model.safetensors']' returned non-zero exit status 1.", | |
| "tb": "Traceback (most recent call last):\n File \"/tmp/ipykernel_1496/361909880.py\", line 23, in deco\n R[\"sections\"][name] = fn()\n ~~^^\n File \"/tmp/ipykernel_1496/361909880.py\", line 73, in model\n subprocess.run([\"cp\", f\"{p}/{f}\", f\"/content/tl/{f}\"], check=True)\n ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/lib/python3.13/subprocess.py\", line 578, in run\n raise CalledProcessError(retcode, process.args,\n output=stdout, stderr=stderr)\nsubprocess.CalledProcessError: Command '['cp', '/root/.cache/huggingface/hub/models--Maykeye--TinyLLama-v0/snapshots/298338802ab94432b917bcce11382aa151aee50f/model.safetensors', '/content/tl/model.safetensors']' returned non-zero exit status 1.\n" | |
| }, | |
| "trace": { | |
| "error": "ModuleNotFoundError: No module named 'chipformer'", | |
| "tb": "Traceback (most recent call last):\n File \"/tmp/ipykernel_1496/361909880.py\", line 23, in deco\n R[\"sections\"][name] = fn()\n ~~^^\n File \"/tmp/ipykernel_1496/361909880.py\", line 89, in trace\n from chipformer.rv import elf, ref\nModuleNotFoundError: No module named 'chipformer'\n" | |
| }, | |
| "executor": { | |
| "error": "ModuleNotFoundError: No module named 'chipformer'", | |
| "tb": "Traceback (most recent call last):\n File \"/tmp/ipykernel_1496/361909880.py\", line 23, in deco\n R[\"sections\"][name] = fn()\n ~~^^\n File \"/tmp/ipykernel_1496/361909880.py\", line 112, in executor\n from chipformer.rv import core, elf, fastexec, ref\nModuleNotFoundError: No module named 'chipformer'\n" | |
| }, | |
| "merge": { | |
| "rc": 0, | |
| "size_mb": 27408, | |
| "circuit_layers": null, | |
| "tail": "use `flash-linear-attention` is not installed. This is correct but much slower; install `flash-linear-attention` for the optimized kernel.\n{\"hidden\": 4096, \"chip_layers\": 36, \"slots\": 1457, \"circuit\": {\"layers\": 36, \"dims\": 3259, \"heads\": 457, \"ffn_units\": 46685, \"max_units\": 8648, \"max_heads\": 16}}" | |
| }, | |
| "degradation": { | |
| "per_text": [ | |
| { | |
| "kw": 0.000528, | |
| "top1": 1.0, | |
| "positions": 18 | |
| }, | |
| { | |
| "kw": 0.000782, | |
| "top1": 1.0, | |
| "positions": 21 | |
| }, | |
| { | |
| "kw": 0.000657, | |
| "top1": 1.0, | |
| "positions": 14 | |
| }, | |
| { | |
| "kw": 0.000616, | |
| "top1": 1.0, | |
| "positions": 18 | |
| }, | |
| { | |
| "kw": 0.000701, | |
| "top1": 1.0, | |
| "positions": 13 | |
| }, | |
| { | |
| "kw": 0.000621, | |
| "top1": 1.0, | |
| "positions": 13 | |
| }, | |
| { | |
| "kw": 0.000808, | |
| "top1": 1.0, | |
| "positions": 21 | |
| }, | |
| { | |
| "kw": 0.000566, | |
| "top1": 1.0, | |
| "positions": 17 | |
| } | |
| ], | |
| "kl_mean": 0.00066, | |
| "positions": 135, | |
| "sample_continuation": "!\nA. London\nB. Paris\nC. Berlin\nD. Madrid\n\n<think>\n", | |
| "decode_tok_s": 23.86, | |
| "peak_vram_gb": 34.7, | |
| "base_layers": 32, | |
| "merged_layers": 68 | |
| } | |
| } | |
| } |