Instructions to use ddwang2000/EchoChat with libraries, inference providers, notebooks, and local apps. Follow these links to get started.

Libraries

How to use ddwang2000/EchoChat with Transformers:

# Use a pipeline as a high-level helper
from transformers import pipeline

pipe = pipeline("text-generation", model="ddwang2000/EchoChat", trust_remote_code=True)

# pip install -U transformers accelerate
# Load model directly
from transformers import AutoTokenizer, AutoModelForCausalLM

tokenizer = AutoTokenizer.from_pretrained("ddwang2000/EchoChat", trust_remote_code=True)
model = AutoModelForCausalLM.from_pretrained("ddwang2000/EchoChat", trust_remote_code=True, device_map="auto")

Notebooks
Google Colab
Kaggle
Local Apps Settings

vLLM

How to use ddwang2000/EchoChat with vLLM:

Install from pip and serve model

# Install vLLM from pip:
pip install vllm
# Start the vLLM server:
vllm serve "ddwang2000/EchoChat"
# Call the server using curl (OpenAI-compatible API):
curl -X POST "http://localhost:8000/v1/chat/completions" \
	-H "Content-Type: application/json" \
	--data '{
		"model": "ddwang2000/EchoChat",
		"messages": [
			{
				"role": "user",
				"content": "What is the capital of France?"
			}
		]
	}'

Use Docker

docker model run hf.co/ddwang2000/EchoChat

SGLang

How to use ddwang2000/EchoChat with SGLang:

Install from pip and serve model

# Install SGLang from pip:
pip install sglang
# Start the SGLang server:
python3 -m sglang.launch_server \
    --model-path "ddwang2000/EchoChat" \
    --host 0.0.0.0 \
    --port 30000
# Call the server using curl (OpenAI-compatible API):
curl -X POST "http://localhost:30000/v1/chat/completions" \
	-H "Content-Type: application/json" \
	--data '{
		"model": "ddwang2000/EchoChat",
		"messages": [
			{
				"role": "user",
				"content": "What is the capital of France?"
			}
		]
	}'

Use Docker images

docker run --gpus all \
    --shm-size 32g \
    -p 30000:30000 \
    -v ~/.cache/huggingface:/root/.cache/huggingface \
    --env "HF_TOKEN=<secret>" \
    --ipc=host \
    lmsysorg/sglang:latest \
    python3 -m sglang.launch_server \
        --model-path "ddwang2000/EchoChat" \
        --host 0.0.0.0 \
        --port 30000
# Call the server using curl (OpenAI-compatible API):
curl -X POST "http://localhost:30000/v1/chat/completions" \
	-H "Content-Type: application/json" \
	--data '{
		"model": "ddwang2000/EchoChat",
		"messages": [
			{
				"role": "user",
				"content": "What is the capital of France?"
			}
		]
	}'

Docker Model Runner
How to use ddwang2000/EchoChat with Docker Model Runner:
```
docker model run hf.co/ddwang2000/EchoChat
```

EchoChat / configuration_echochat.py

ddwang2000

Upload EchoChat model components

b6d2bc8 verified about 1 month ago

Raw History Blame Contribute Delete

2.57 kB

# Modified for EchoChat distribution: renamed public entry points; computation unchanged.
from transformers.models.qwen2.configuration_qwen2 import Qwen2Config


class EchoChatConfig(Qwen2Config):
    def __init__(
        self,
        vocab_size=163840,
        hidden_size=4096,
        intermediate_size=11008,
        num_hidden_layers=32,
        num_attention_heads=32,
        num_key_value_heads=None,
        hidden_act="silu",
        initializer_range=0.02,
        rms_norm_eps=1e-6,
        use_cache=True,
        rope_theta=10000.0,
        rope_scaling=None,
        tie_word_embeddings=False,
        kimia_mimo_layers: int = 6,
        kimia_mimo_audiodelaytokens: int = 5,
        kimia_mimo_transformer_from_layer_index: int = 21,
        kimia_audio_output_vocab: int = 16896,
        kimia_text_output_vocab: int = 152064,
        num_audio_special_tokens: int = 512,
        num_base_tokens: int = 151643,
        kimia_token_offset: int = 152064,
        use_whisper_feature: bool = True,
        kimia_adaptor_input_dim: int = 5120,
        kimia_media_begin: int = 151661,
        kimia_media_end: int = 151663,
        **kwargs,
    ):
        super().__init__(
            vocab_size=vocab_size,
            hidden_size=hidden_size,
            intermediate_size=intermediate_size,
            num_hidden_layers=num_hidden_layers,
            num_attention_heads=num_attention_heads,
            num_key_value_heads=num_key_value_heads,
            hidden_act=hidden_act,
            initializer_range=initializer_range,
            rms_norm_eps=rms_norm_eps,
            use_cache=use_cache,
            tie_word_embeddings=tie_word_embeddings,
            rope_theta=rope_theta,
            rope_scaling=rope_scaling,
            **kwargs,
        )

        self.kimia_mimo_layers = kimia_mimo_layers
        self.kimia_mimo_audiodelaytokens = kimia_mimo_audiodelaytokens
        # vocab
        self.kimia_mimo_transformer_from_layer_index = (
            kimia_mimo_transformer_from_layer_index
        )
        self.kimia_audio_output_vocab = kimia_audio_output_vocab
        self.kimia_text_output_vocab = kimia_text_output_vocab
        self.num_audio_special_tokens = num_audio_special_tokens
        self.num_base_tokens = num_base_tokens
        self.kimia_token_offset = kimia_token_offset
        self.use_whisper_feature = use_whisper_feature
        self.kimia_adaptor_input_dim = kimia_adaptor_input_dim
        # special tokens
        self.kimia_media_begin = kimia_media_begin
        self.kimia_media_end = kimia_media_end

	# Modified for EchoChat distribution: renamed public entry points; computation unchanged.
	from transformers.models.qwen2.configuration_qwen2 import Qwen2Config


	class EchoChatConfig(Qwen2Config):
	def __init__(
	self,
	vocab_size=163840,
	hidden_size=4096,
	intermediate_size=11008,
	num_hidden_layers=32,
	num_attention_heads=32,
	num_key_value_heads=None,
	hidden_act="silu",
	initializer_range=0.02,
	rms_norm_eps=1e-6,
	use_cache=True,
	rope_theta=10000.0,
	rope_scaling=None,
	tie_word_embeddings=False,
	kimia_mimo_layers: int = 6,
	kimia_mimo_audiodelaytokens: int = 5,
	kimia_mimo_transformer_from_layer_index: int = 21,
	kimia_audio_output_vocab: int = 16896,
	kimia_text_output_vocab: int = 152064,
	num_audio_special_tokens: int = 512,
	num_base_tokens: int = 151643,
	kimia_token_offset: int = 152064,
	use_whisper_feature: bool = True,
	kimia_adaptor_input_dim: int = 5120,
	kimia_media_begin: int = 151661,
	kimia_media_end: int = 151663,
	**kwargs,
	):
	super().__init__(
	vocab_size=vocab_size,
	hidden_size=hidden_size,
	intermediate_size=intermediate_size,
	num_hidden_layers=num_hidden_layers,
	num_attention_heads=num_attention_heads,
	num_key_value_heads=num_key_value_heads,
	hidden_act=hidden_act,
	initializer_range=initializer_range,
	rms_norm_eps=rms_norm_eps,
	use_cache=use_cache,
	tie_word_embeddings=tie_word_embeddings,
	rope_theta=rope_theta,
	rope_scaling=rope_scaling,
	**kwargs,
	)

	self.kimia_mimo_layers = kimia_mimo_layers
	self.kimia_mimo_audiodelaytokens = kimia_mimo_audiodelaytokens
	# vocab
	self.kimia_mimo_transformer_from_layer_index = (
	kimia_mimo_transformer_from_layer_index
	)
	self.kimia_audio_output_vocab = kimia_audio_output_vocab
	self.kimia_text_output_vocab = kimia_text_output_vocab
	self.num_audio_special_tokens = num_audio_special_tokens
	self.num_base_tokens = num_base_tokens
	self.kimia_token_offset = kimia_token_offset
	self.use_whisper_feature = use_whisper_feature
	self.kimia_adaptor_input_dim = kimia_adaptor_input_dim
	# special tokens
	self.kimia_media_begin = kimia_media_begin
	self.kimia_media_end = kimia_media_end