Instructions to use WaveCut/YuE2-3B-OrbitQuant-W4A4 with libraries, inference providers, notebooks, and local apps. Follow these links to get started.

Libraries

How to use WaveCut/YuE2-3B-OrbitQuant-W4A4 with Transformers:

# Use a pipeline as a high-level helper
from transformers import pipeline

pipe = pipeline("text-generation", model="WaveCut/YuE2-3B-OrbitQuant-W4A4")

# pip install -U transformers accelerate
# Load model directly
from transformers import AutoModelForCausalLM
model = AutoModelForCausalLM.from_pretrained("WaveCut/YuE2-3B-OrbitQuant-W4A4", device_map="auto")

Notebooks
Google Colab
Kaggle
Local Apps Settings

vLLM

How to use WaveCut/YuE2-3B-OrbitQuant-W4A4 with vLLM:

Install from pip and serve model

# Install vLLM from pip:
pip install vllm
# Start the vLLM server:
vllm serve "WaveCut/YuE2-3B-OrbitQuant-W4A4"
# Call the server using curl (OpenAI-compatible API):
curl -X POST "http://localhost:8000/v1/completions" \
	-H "Content-Type: application/json" \
	--data '{
		"model": "WaveCut/YuE2-3B-OrbitQuant-W4A4",
		"prompt": "Once upon a time,",
		"max_tokens": 512,
		"temperature": 0.5
	}'

Use Docker

docker model run hf.co/WaveCut/YuE2-3B-OrbitQuant-W4A4

SGLang

How to use WaveCut/YuE2-3B-OrbitQuant-W4A4 with SGLang:

Install from pip and serve model

# Install SGLang from pip:
pip install sglang
# Start the SGLang server:
python3 -m sglang.launch_server \
    --model-path "WaveCut/YuE2-3B-OrbitQuant-W4A4" \
    --host 0.0.0.0 \
    --port 30000
# Call the server using curl (OpenAI-compatible API):
curl -X POST "http://localhost:30000/v1/completions" \
	-H "Content-Type: application/json" \
	--data '{
		"model": "WaveCut/YuE2-3B-OrbitQuant-W4A4",
		"prompt": "Once upon a time,",
		"max_tokens": 512,
		"temperature": 0.5
	}'

Use Docker images

docker run --gpus all \
    --shm-size 32g \
    -p 30000:30000 \
    -v ~/.cache/huggingface:/root/.cache/huggingface \
    --env "HF_TOKEN=<secret>" \
    --ipc=host \
    lmsysorg/sglang:latest \
    python3 -m sglang.launch_server \
        --model-path "WaveCut/YuE2-3B-OrbitQuant-W4A4" \
        --host 0.0.0.0 \
        --port 30000
# Call the server using curl (OpenAI-compatible API):
curl -X POST "http://localhost:30000/v1/completions" \
	-H "Content-Type: application/json" \
	--data '{
		"model": "WaveCut/YuE2-3B-OrbitQuant-W4A4",
		"prompt": "Once upon a time,",
		"max_tokens": 512,
		"temperature": 0.5
	}'

Docker Model Runner
How to use WaveCut/YuE2-3B-OrbitQuant-W4A4 with Docker Model Runner:
```
docker model run hf.co/WaveCut/YuE2-3B-OrbitQuant-W4A4
```

YuE2-3B-OrbitQuant-W4A4 / evaluation /profiles /single-process-cli.json

WaveCut

Runtime update: released native 1.0.5, memory flags (VAE tile 256, lazy KV handoff), verified-digest cache, background VAE load, chunked INT8 head quantization, compile paths only with --warm-compile; measurements refreshed

cb66b66 verified about 2 months ago

Raw History Blame Contribute Delete

2.95 kB

{
 "exact-1": {
  "seconds_including_load": 17.64824853464961,
  "e2e_seconds": 17.331095344386995,
  "audio_seconds": 168.91866666666667,
  "load": {
   "resolve_and_integrity_seconds": 0.20458987448364496,
   "mot_load_seconds": 0.6507547041401267
  },
  "vae_seconds": 0.9485859358683228,
  "nar_seconds": 2.7367109656333923,
  "mode": "exact",
  "memory": {
   "t0": {
    "peak_gib": 2.845,
    "alloc_gib": 2.831,
    "reserved_gib": 3.068,
    "nvml_used_gib": 3.928
   },
   "plan_done": {
    "peak_gib": 3.108,
    "alloc_gib": 2.855,
    "reserved_gib": 3.371,
    "nvml_used_gib": 4.328
   },
   "semantic_done": {
    "peak_gib": 3.566,
    "alloc_gib": 2.863,
    "reserved_gib": 3.719,
    "nvml_used_gib": 4.678
   },
   "nar_done": {
    "peak_gib": 4.048,
    "alloc_gib": 2.864,
    "reserved_gib": 4.357,
    "nvml_used_gib": 5.317
   },
   "vae_done": {
    "peak_gib": 3.729,
    "alloc_gib": 3.111,
    "reserved_gib": 4.357,
    "nvml_used_gib": 5.326
   }
  }
 },
 "exact-2": {
  "seconds_including_load": 17.66493225377053,
  "e2e_seconds": 17.360788581892848,
  "audio_seconds": 168.91866666666667,
  "load": {
   "resolve_and_integrity_seconds": 0.19648857228457928,
   "mot_load_seconds": 0.6907868329435587
  },
  "vae_seconds": 0.95797960460186,
  "nar_seconds": 2.7402642825618386,
  "mode": "exact",
  "memory": {
   "t0": {
    "peak_gib": 2.845,
    "alloc_gib": 2.831,
    "reserved_gib": 3.068,
    "nvml_used_gib": 3.928
   },
   "plan_done": {
    "peak_gib": 3.108,
    "alloc_gib": 2.855,
    "reserved_gib": 3.371,
    "nvml_used_gib": 4.328
   },
   "semantic_done": {
    "peak_gib": 3.566,
    "alloc_gib": 2.863,
    "reserved_gib": 3.719,
    "nvml_used_gib": 4.678
   },
   "nar_done": {
    "peak_gib": 4.048,
    "alloc_gib": 2.864,
    "reserved_gib": 4.357,
    "nvml_used_gib": 5.317
   },
   "vae_done": {
    "peak_gib": 3.729,
    "alloc_gib": 3.111,
    "reserved_gib": 4.357,
    "nvml_used_gib": 5.326
   }
  }
 },
 "fast-1": {
  "seconds_including_load": 13.86062375176698,
  "e2e_seconds": 13.60636106133461,
  "audio_seconds": 140.67866666666666,
  "load": {
   "resolve_and_integrity_seconds": 0.20314890518784523,
   "mot_load_seconds": 0.6868843296542764
  },
  "vae_seconds": 0.8286430640146136,
  "nar_seconds": 2.0424561528488994,
  "mode": "fast",
  "memory": {
   "t0": {
    "peak_gib": 5.724,
    "alloc_gib": 2.479,
    "reserved_gib": 2.756,
    "nvml_used_gib": 3.674
   },
   "plan_done": {
    "peak_gib": 2.754,
    "alloc_gib": 2.479,
    "reserved_gib": 2.98,
    "nvml_used_gib": 3.922
   },
   "semantic_done": {
    "peak_gib": 3.17,
    "alloc_gib": 3.067,
    "reserved_gib": 3.309,
    "nvml_used_gib": 4.252
   },
   "nar_done": {
    "peak_gib": 3.12,
    "alloc_gib": 2.488,
    "reserved_gib": 3.4,
    "nvml_used_gib": 4.36
   },
   "vae_done": {
    "peak_gib": 3.353,
    "alloc_gib": 2.735,
    "reserved_gib": 3.615,
    "nvml_used_gib": 4.584
   }
  }
 }
}

	{
	"exact-1": {
	"seconds_including_load": 17.64824853464961,
	"e2e_seconds": 17.331095344386995,
	"audio_seconds": 168.91866666666667,
	"load": {
	"resolve_and_integrity_seconds": 0.20458987448364496,
	"mot_load_seconds": 0.6507547041401267
	},
	"vae_seconds": 0.9485859358683228,
	"nar_seconds": 2.7367109656333923,
	"mode": "exact",
	"memory": {
	"t0": {
	"peak_gib": 2.845,
	"alloc_gib": 2.831,
	"reserved_gib": 3.068,
	"nvml_used_gib": 3.928
	},
	"plan_done": {
	"peak_gib": 3.108,
	"alloc_gib": 2.855,
	"reserved_gib": 3.371,
	"nvml_used_gib": 4.328
	},
	"semantic_done": {
	"peak_gib": 3.566,
	"alloc_gib": 2.863,
	"reserved_gib": 3.719,
	"nvml_used_gib": 4.678
	},
	"nar_done": {
	"peak_gib": 4.048,
	"alloc_gib": 2.864,
	"reserved_gib": 4.357,
	"nvml_used_gib": 5.317
	},
	"vae_done": {
	"peak_gib": 3.729,
	"alloc_gib": 3.111,
	"reserved_gib": 4.357,
	"nvml_used_gib": 5.326
	}
	}
	},
	"exact-2": {
	"seconds_including_load": 17.66493225377053,
	"e2e_seconds": 17.360788581892848,
	"audio_seconds": 168.91866666666667,
	"load": {
	"resolve_and_integrity_seconds": 0.19648857228457928,
	"mot_load_seconds": 0.6907868329435587
	},
	"vae_seconds": 0.95797960460186,
	"nar_seconds": 2.7402642825618386,
	"mode": "exact",
	"memory": {
	"t0": {
	"peak_gib": 2.845,
	"alloc_gib": 2.831,
	"reserved_gib": 3.068,
	"nvml_used_gib": 3.928
	},
	"plan_done": {
	"peak_gib": 3.108,
	"alloc_gib": 2.855,
	"reserved_gib": 3.371,
	"nvml_used_gib": 4.328
	},
	"semantic_done": {
	"peak_gib": 3.566,
	"alloc_gib": 2.863,
	"reserved_gib": 3.719,
	"nvml_used_gib": 4.678
	},
	"nar_done": {
	"peak_gib": 4.048,
	"alloc_gib": 2.864,
	"reserved_gib": 4.357,
	"nvml_used_gib": 5.317
	},
	"vae_done": {
	"peak_gib": 3.729,
	"alloc_gib": 3.111,
	"reserved_gib": 4.357,
	"nvml_used_gib": 5.326
	}
	}
	},
	"fast-1": {
	"seconds_including_load": 13.86062375176698,
	"e2e_seconds": 13.60636106133461,
	"audio_seconds": 140.67866666666666,
	"load": {
	"resolve_and_integrity_seconds": 0.20314890518784523,
	"mot_load_seconds": 0.6868843296542764
	},
	"vae_seconds": 0.8286430640146136,
	"nar_seconds": 2.0424561528488994,
	"mode": "fast",
	"memory": {
	"t0": {
	"peak_gib": 5.724,
	"alloc_gib": 2.479,
	"reserved_gib": 2.756,
	"nvml_used_gib": 3.674
	},
	"plan_done": {
	"peak_gib": 2.754,
	"alloc_gib": 2.479,
	"reserved_gib": 2.98,
	"nvml_used_gib": 3.922
	},
	"semantic_done": {
	"peak_gib": 3.17,
	"alloc_gib": 3.067,
	"reserved_gib": 3.309,
	"nvml_used_gib": 4.252
	},
	"nar_done": {
	"peak_gib": 3.12,
	"alloc_gib": 2.488,
	"reserved_gib": 3.4,
	"nvml_used_gib": 4.36
	},
	"vae_done": {
	"peak_gib": 3.353,
	"alloc_gib": 2.735,
	"reserved_gib": 3.615,
	"nvml_used_gib": 4.584
	}
	}
	}
	}