Instructions to use WaveCut/YuE2-3B-OrbitQuant-W4A4 with libraries, inference providers, notebooks, and local apps. Follow these links to get started.

Libraries

How to use WaveCut/YuE2-3B-OrbitQuant-W4A4 with Transformers:

# Use a pipeline as a high-level helper
from transformers import pipeline

pipe = pipeline("text-generation", model="WaveCut/YuE2-3B-OrbitQuant-W4A4")

# pip install -U transformers accelerate
# Load model directly
from transformers import AutoModelForCausalLM
model = AutoModelForCausalLM.from_pretrained("WaveCut/YuE2-3B-OrbitQuant-W4A4", device_map="auto")

Notebooks
Google Colab
Kaggle
Local Apps Settings

vLLM

How to use WaveCut/YuE2-3B-OrbitQuant-W4A4 with vLLM:

Install from pip and serve model

# Install vLLM from pip:
pip install vllm
# Start the vLLM server:
vllm serve "WaveCut/YuE2-3B-OrbitQuant-W4A4"
# Call the server using curl (OpenAI-compatible API):
curl -X POST "http://localhost:8000/v1/completions" \
	-H "Content-Type: application/json" \
	--data '{
		"model": "WaveCut/YuE2-3B-OrbitQuant-W4A4",
		"prompt": "Once upon a time,",
		"max_tokens": 512,
		"temperature": 0.5
	}'

Use Docker

docker model run hf.co/WaveCut/YuE2-3B-OrbitQuant-W4A4

SGLang

How to use WaveCut/YuE2-3B-OrbitQuant-W4A4 with SGLang:

Install from pip and serve model

# Install SGLang from pip:
pip install sglang
# Start the SGLang server:
python3 -m sglang.launch_server \
    --model-path "WaveCut/YuE2-3B-OrbitQuant-W4A4" \
    --host 0.0.0.0 \
    --port 30000
# Call the server using curl (OpenAI-compatible API):
curl -X POST "http://localhost:30000/v1/completions" \
	-H "Content-Type: application/json" \
	--data '{
		"model": "WaveCut/YuE2-3B-OrbitQuant-W4A4",
		"prompt": "Once upon a time,",
		"max_tokens": 512,
		"temperature": 0.5
	}'

Use Docker images

docker run --gpus all \
    --shm-size 32g \
    -p 30000:30000 \
    -v ~/.cache/huggingface:/root/.cache/huggingface \
    --env "HF_TOKEN=<secret>" \
    --ipc=host \
    lmsysorg/sglang:latest \
    python3 -m sglang.launch_server \
        --model-path "WaveCut/YuE2-3B-OrbitQuant-W4A4" \
        --host 0.0.0.0 \
        --port 30000
# Call the server using curl (OpenAI-compatible API):
curl -X POST "http://localhost:30000/v1/completions" \
	-H "Content-Type: application/json" \
	--data '{
		"model": "WaveCut/YuE2-3B-OrbitQuant-W4A4",
		"prompt": "Once upon a time,",
		"max_tokens": 512,
		"temperature": 0.5
	}'

Docker Model Runner
How to use WaveCut/YuE2-3B-OrbitQuant-W4A4 with Docker Model Runner:
```
docker model run hf.co/WaveCut/YuE2-3B-OrbitQuant-W4A4
```

YuE2-3B-OrbitQuant-W4A4 / src /runtime /yue2 /device_ring_sampling.py

WaveCut

Combine decode and sampling with request-owned device history

19e9ccf verified about 2 months ago

Raw History Blame Contribute Delete

1.9 kB

"""Request-owned device history for captured sampling."""
import torch
from .captured_sampling import CapturedSampler
from .protocol import MUSIC_END, CODEC_OFFSET, CODEC_SIZE, ABC_END, EOD

class DeviceRingSampler(CapturedSampler):
    @torch.inference_mode()
    def __init__(self, logits, sampling, generator, *, phase='semantic'):
        if sampling.penalty_window < 1:
            raise ValueError('Device history requires a positive window')
        self.counter = torch.zeros((),device=logits.device,dtype=torch.long)
        self.expected_step = 0
        super().__init__(logits,sampling,generator,phase=phase)
        # Constructor warmup executes updates; capture itself only records them.
        self.counter.zero_()
        self.recent.fill_(-1)

    def _compute(self):
        self.allow_eos.copy_(self.counter >= self.sampling.min_tokens)
        selected = super()._compute()
        index = (self.counter % self.sampling.penalty_window).reshape(1,1)
        local = torch.where(selected == ABC_END, EOD, selected) if self.phase == 'abc' else selected-MUSIC_END
        self.recent.scatter_(-1,index,local)
        self.counter.add_(1)
        return selected

    @torch.inference_mode()
    def __call__(self,logits,history,step):
        if self.closed:
            raise RuntimeError('DeviceRingSampler is closed')
        if step != self.expected_step or len(history) != step:
            raise ValueError('Device history requires consecutive steps from an empty history')
        if self.phase == 'abc':
            self.logits[...,:EOD].copy_(logits[...,:EOD])
            self.logits[...,EOD].copy_(logits[...,ABC_END])
        else:
            self.logits.copy_(logits[...,MUSIC_END:CODEC_OFFSET+CODEC_SIZE])
        self.graph.replay()
        self.expected_step += 1
        return self.output

    def close(self):
        super().close()
        self.counter = None

	"""Request-owned device history for captured sampling."""
	import torch
	from .captured_sampling import CapturedSampler
	from .protocol import MUSIC_END, CODEC_OFFSET, CODEC_SIZE, ABC_END, EOD

	class DeviceRingSampler(CapturedSampler):
	@torch.inference_mode()
	def __init__(self, logits, sampling, generator, *, phase='semantic'):
	if sampling.penalty_window < 1:
	raise ValueError('Device history requires a positive window')
	self.counter = torch.zeros((),device=logits.device,dtype=torch.long)
	self.expected_step = 0
	super().__init__(logits,sampling,generator,phase=phase)
	# Constructor warmup executes updates; capture itself only records them.
	self.counter.zero_()
	self.recent.fill_(-1)

	def _compute(self):
	self.allow_eos.copy_(self.counter >= self.sampling.min_tokens)
	selected = super()._compute()
	index = (self.counter % self.sampling.penalty_window).reshape(1,1)
	local = torch.where(selected == ABC_END, EOD, selected) if self.phase == 'abc' else selected-MUSIC_END
	self.recent.scatter_(-1,index,local)
	self.counter.add_(1)
	return selected

	@torch.inference_mode()
	def __call__(self,logits,history,step):
	if self.closed:
	raise RuntimeError('DeviceRingSampler is closed')
	if step != self.expected_step or len(history) != step:
	raise ValueError('Device history requires consecutive steps from an empty history')
	if self.phase == 'abc':
	self.logits[...,:EOD].copy_(logits[...,:EOD])
	self.logits[...,EOD].copy_(logits[...,ABC_END])
	else:
	self.logits.copy_(logits[...,MUSIC_END:CODEC_OFFSET+CODEC_SIZE])
	self.graph.replay()
	self.expected_step += 1
	return self.output

	def close(self):
	super().close()
	self.counter = None