Instructions to use grim3000/cogvlm-chat-hf with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use grim3000/cogvlm-chat-hf with Transformers:
# Use a pipeline as a high-level helper from transformers import pipeline pipe = pipeline("text-generation", model="grim3000/cogvlm-chat-hf", trust_remote_code=True)# pip install -U transformers accelerate # Load model directly from transformers import AutoModelForCausalLM model = AutoModelForCausalLM.from_pretrained("grim3000/cogvlm-chat-hf", trust_remote_code=True, device_map="auto") - Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- vLLM
How to use grim3000/cogvlm-chat-hf with vLLM:
Install from pip and serve model
# Install vLLM from pip: pip install vllm # Start the vLLM server: vllm serve "grim3000/cogvlm-chat-hf" # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:8000/v1/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "grim3000/cogvlm-chat-hf", "prompt": "Once upon a time,", "max_tokens": 512, "temperature": 0.5 }'Use Docker
docker model run hf.co/grim3000/cogvlm-chat-hf
- SGLang
How to use grim3000/cogvlm-chat-hf with SGLang:
Install from pip and serve model
# Install SGLang from pip: pip install sglang # Start the SGLang server: python3 -m sglang.launch_server \ --model-path "grim3000/cogvlm-chat-hf" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "grim3000/cogvlm-chat-hf", "prompt": "Once upon a time,", "max_tokens": 512, "temperature": 0.5 }'Use Docker images
docker run --gpus all \ --shm-size 32g \ -p 30000:30000 \ -v ~/.cache/huggingface:/root/.cache/huggingface \ --env "HF_TOKEN=<secret>" \ --ipc=host \ lmsysorg/sglang:latest \ python3 -m sglang.launch_server \ --model-path "grim3000/cogvlm-chat-hf" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "grim3000/cogvlm-chat-hf", "prompt": "Once upon a time,", "max_tokens": 512, "temperature": 0.5 }' - Docker Model Runner
How to use grim3000/cogvlm-chat-hf with Docker Model Runner:
docker model run hf.co/grim3000/cogvlm-chat-hf
Download handler.py from grim3000/cogvlm-chat-hf: direct link, hf CLI and curl.
- Browser
- Download file 5.1 kB
-
https://huggingface.co/grim3000/cogvlm-chat-hf/resolve/main/handler.py
- Command line
-
hf download hf://grim3000/cogvlm-chat-hf/handler.py
-
curl -L -o handler.py https://huggingface.co/grim3000/cogvlm-chat-hf/resolve/main/handler.py
5.1 kB
| from typing import Dict, List, Any | |
| from transformers import pipeline | |
| from PIL import Image | |
| import requests | |
| from transformers import AutoModelForCausalLM, LlamaTokenizer | |
| import torch | |
| from accelerate import ( | |
| init_empty_weights, | |
| infer_auto_device_map, | |
| load_checkpoint_and_dispatch, | |
| ) | |
| import os | |
| import logging | |
| from transformers import logging as hf_logging | |
| logging.basicConfig(level=logging.INFO) | |
| hf_logging.set_verbosity_debug() | |
| def list_files(directory, depth, max_depth=5): | |
| # Lists all files and directories in the given directory | |
| for filename in os.listdir(directory): | |
| print(os.path.join(directory, filename)) | |
| if not os.path.isfile(filename) and depth < max_depth: | |
| list_files(os.path.join(directory, filename), depth + 1, max_depth) | |
| class EndpointHandler: | |
| def __init__(self, path=""): | |
| # Preload all the elements you are going to need at inference. | |
| # self.pipeline = pipeline( | |
| # "text-generation", model="THUDM/cogvlm-chat-hf", trust_remote_code=True | |
| # ) | |
| # self.model = AutoModelForCausalLM.from_pretrained( | |
| # "THUDM/cogvlm-chat-hf", trust_remote_code=True | |
| # ) | |
| self.tokenizer = LlamaTokenizer.from_pretrained("lmsys/vicuna-7b-v1.5") | |
| self.model = ( | |
| AutoModelForCausalLM.from_pretrained( | |
| "THUDM/cogvlm-chat-hf", | |
| torch_dtype=torch.bfloat16, | |
| low_cpu_mem_usage=True, | |
| trust_remote_code=True, | |
| ) | |
| .to("cuda") | |
| .eval() | |
| ) | |
| # DISTRIBUTED GPUS | |
| # with init_empty_weights(): | |
| # self.model = AutoModelForCausalLM.from_pretrained( | |
| # "THUDM/cogvlm-chat-hf", | |
| # torch_dtype=torch.bfloat16, | |
| # low_cpu_mem_usage=True, | |
| # trust_remote_code=True, | |
| # ) | |
| # # print("LISTING FILES IN ", "/root/.cache/huggingface") | |
| # # list_files("/root/.cache/huggingface", 0, 5) | |
| # device_map = infer_auto_device_map( | |
| # self.model, | |
| # max_memory={ | |
| # 0: "12GiB", | |
| # 1: "12GiB", | |
| # 2: "12GiB", | |
| # 3: "12GiB", | |
| # "cpu": "180GiB", | |
| # }, | |
| # no_split_module_classes=["CogVLMDecoderLayer"], | |
| # ) | |
| # self.model = load_checkpoint_and_dispatch( | |
| # self.model, | |
| # "/root/.cache/huggingface/hub/models--THUDM--cogvlm-chat-hf/snapshots/8abca878c4257412c4c38eeafaed3fe27a036730", | |
| # device_map=device_map, | |
| # no_split_module_classes=["CogVLMDecoderLayer"], | |
| # ) | |
| # self.model = self.model.eval() | |
| ## DISTRIBUTED GPUS | |
| def __call__(self, data: Dict[str, Any]) -> List[Dict[str, Any]]: | |
| """ | |
| data args: | |
| inputs (:obj: `str` | `PIL.Image` | `np.array`) | |
| kwargs | |
| Return: | |
| A :obj:`list` | `dict`: will be serialized and returned | |
| """ | |
| query = data["query"] | |
| img_uri = data["img_uri"] | |
| image = Image.open( | |
| requests.get( | |
| img_uri, | |
| stream=True, | |
| ).raw | |
| ).convert("RGB") | |
| inputs = self.model.build_conversation_input_ids( | |
| self.tokenizer, | |
| query=query, | |
| history=[], | |
| images=[image], | |
| template_version="vqa", | |
| ) # vqa mode | |
| inputs = { | |
| "input_ids": inputs["input_ids"].unsqueeze(0).to("cuda"), | |
| "token_type_ids": inputs["token_type_ids"].unsqueeze(0).to("cuda"), | |
| "attention_mask": inputs["attention_mask"].unsqueeze(0).to("cuda"), | |
| "images": [[inputs["images"][0].to("cuda").to(torch.bfloat16)]], | |
| } | |
| gen_kwargs = {"max_length": 2048, "do_sample": False} | |
| with torch.no_grad(): | |
| outputs = self.model.generate(**inputs, **gen_kwargs) | |
| print("outputs 1: ", outputs) | |
| outputs = outputs[:, inputs["input_ids"].shape[1] :] | |
| print("outputs 2: ", outputs) | |
| response = self.tokenizer.decode(outputs[0]) | |
| return response | |
| # query = "How many houses are there in this cartoon?" | |
| # image = Image.open( | |
| # requests.get( | |
| # "https://github.com/THUDM/CogVLM/blob/main/examples/3.jpg?raw=true", stream=True | |
| # ).raw | |
| # ).convert("RGB") | |
| # inputs = model.build_conversation_input_ids( | |
| # tokenizer, query=query, history=[], images=[image], template_version="vqa" | |
| # ) # vqa mode | |
| # inputs = { | |
| # "input_ids": inputs["input_ids"].unsqueeze(0).to("cuda"), | |
| # "token_type_ids": inputs["token_type_ids"].unsqueeze(0).to("cuda"), | |
| # "attention_mask": inputs["attention_mask"].unsqueeze(0).to("cuda"), | |
| # "images": [[inputs["images"][0].to("cuda").to(torch.bfloat16)]], | |
| # } | |
| # gen_kwargs = {"max_length": 2048, "do_sample": False} | |
| # with torch.no_grad(): | |
| # outputs = model.generate(**inputs, **gen_kwargs) | |
| # outputs = outputs[:, inputs["input_ids"].shape[1] :] | |
| # print(tokenizer.decode(outputs[0])) | |