Instructions to use injazsmart/thoth_test with libraries, inference providers, notebooks, and local apps. Follow these links to get started.

Libraries

How to use injazsmart/thoth_test with Transformers:

# Use a pipeline as a high-level helper
from transformers import pipeline

pipe = pipeline("image-text-to-text", model="injazsmart/thoth_test")
messages = [
    {
        "role": "user",
        "content": [
            {"type": "image", "url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/p-blog/candy.JPG"},
            {"type": "text", "text": "What animal is on the candy?"}
        ]
    },
]
pipe(text=messages)

# Load model directly
from transformers import AutoProcessor, AutoModelForMultimodalLM

processor = AutoProcessor.from_pretrained("injazsmart/thoth_test")
model = AutoModelForMultimodalLM.from_pretrained("injazsmart/thoth_test", device_map="auto")
messages = [
    {
        "role": "user",
        "content": [
            {"type": "image", "url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/p-blog/candy.JPG"},
            {"type": "text", "text": "What animal is on the candy?"}
        ]
    },
]
inputs = processor.apply_chat_template(
	messages,
	add_generation_prompt=True,
	tokenize=True,
	return_dict=True,
	return_tensors="pt",
).to(model.device)

outputs = model.generate(**inputs, max_new_tokens=40)
print(processor.decode(outputs[0][inputs["input_ids"].shape[-1]:]))

PEFT
How to use injazsmart/thoth_test with PEFT:
```
Task type is invalid.
```
Notebooks
Google Colab
Kaggle
Local Apps Settings

vLLM

How to use injazsmart/thoth_test with vLLM:

Install from pip and serve model

# Install vLLM from pip:
pip install vllm
# Start the vLLM server:
vllm serve "injazsmart/thoth_test"
# Call the server using curl (OpenAI-compatible API):
curl -X POST "http://localhost:8000/v1/chat/completions" \
	-H "Content-Type: application/json" \
	--data '{
		"model": "injazsmart/thoth_test",
		"messages": [
			{
				"role": "user",
				"content": [
					{
						"type": "text",
						"text": "Describe this image in one sentence."
					},
					{
						"type": "image_url",
						"image_url": {
							"url": "https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg"
						}
					}
				]
			}
		]
	}'

Use Docker

docker model run hf.co/injazsmart/thoth_test

SGLang

How to use injazsmart/thoth_test with SGLang:

Install from pip and serve model

# Install SGLang from pip:
pip install sglang
# Start the SGLang server:
python3 -m sglang.launch_server \
    --model-path "injazsmart/thoth_test" \
    --host 0.0.0.0 \
    --port 30000
# Call the server using curl (OpenAI-compatible API):
curl -X POST "http://localhost:30000/v1/chat/completions" \
	-H "Content-Type: application/json" \
	--data '{
		"model": "injazsmart/thoth_test",
		"messages": [
			{
				"role": "user",
				"content": [
					{
						"type": "text",
						"text": "Describe this image in one sentence."
					},
					{
						"type": "image_url",
						"image_url": {
							"url": "https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg"
						}
					}
				]
			}
		]
	}'

Use Docker images

docker run --gpus all \
    --shm-size 32g \
    -p 30000:30000 \
    -v ~/.cache/huggingface:/root/.cache/huggingface \
    --env "HF_TOKEN=<secret>" \
    --ipc=host \
    lmsysorg/sglang:latest \
    python3 -m sglang.launch_server \
        --model-path "injazsmart/thoth_test" \
        --host 0.0.0.0 \
        --port 30000
# Call the server using curl (OpenAI-compatible API):
curl -X POST "http://localhost:30000/v1/chat/completions" \
	-H "Content-Type: application/json" \
	--data '{
		"model": "injazsmart/thoth_test",
		"messages": [
			{
				"role": "user",
				"content": [
					{
						"type": "text",
						"text": "Describe this image in one sentence."
					},
					{
						"type": "image_url",
						"image_url": {
							"url": "https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg"
						}
					}
				]
			}
		]
	}'

Docker Model Runner
How to use injazsmart/thoth_test with Docker Model Runner:
```
docker model run hf.co/injazsmart/thoth_test
```

thoth_test / debug.log

motamad

Upload folder using huggingface_hub

e24b7c9 verified 9 months ago

Raw

History Blame Contribute Delete

8.67 kB

	[2025-10-22 10:50:22,584] [DEBUG] [axolotl.utils.config.log_gpu_memory_usage:127] [PID:2355] baseline 0.000GB ()
	[2025-10-22 10:50:22,584] [INFO] [axolotl.cli.config.load_cfg:248] [PID:2355] config:
	{
	"activation_offloading": false,
	"adapter": "qlora",
	"axolotl_config_path": "config.yaml",
	"base_model": "Qwen/Qwen2.5-VL-7B-Instruct",
	"base_model_config": "Qwen/Qwen2.5-VL-7B-Instruct",
	"batch_size": 8,
	"bf16": true,
	"capabilities": {
	"bf16": true,
	"compute_capability": "sm_90",
	"fp8": false,
	"n_gpu": 1,
	"n_node": 1
	},
	"context_parallel_size": 1,
	"dataloader_num_workers": 1,
	"dataloader_pin_memory": true,
	"dataloader_prefetch_factor": 256,
	"dataset_processes": 20,
	"datasets": [
	{
	"message_property_mappings": {
	"content": "content",
	"role": "role"
	},
	"path": "/workspace/fine-tuning/data/data.json",
	"trust_remote_code": false,
	"type": "vl"
	}
	],
	"ddp": false,
	"device": "cuda:0",
	"dion_rank_fraction": 1.0,
	"dion_rank_multiple_of": 1,
	"env_capabilities": {
	"torch_version": "2.7.1"
	},
	"eval_batch_size": 1,
	"eval_causal_lm_metrics": [
	"sacrebleu",
	"comet",
	"ter",
	"chrf"
	],
	"eval_max_new_tokens": 128,
	"eval_table_size": 0,
	"experimental_skip_move_to_device": true,
	"flash_attention": true,
	"fp16": false,
	"gradient_accumulation_steps": 8,
	"gradient_checkpointing": true,
	"gradient_checkpointing_kwargs": {
	"use_reentrant": true
	},
	"include_tkps": true,
	"is_multimodal": true,
	"learning_rate": 0.0002,
	"lisa_layers_attribute": "model.layers",
	"load_best_model_at_end": false,
	"load_in_4bit": true,
	"load_in_8bit": false,
	"local_rank": 0,
	"lora_alpha": 16,
	"lora_dropout": 0.05,
	"lora_r": 64,
	"lora_target_modules": [
	"q_proj",
	"v_proj",
	"k_proj",
	"o_proj",
	"gate_proj",
	"down_proj",
	"up_proj",
	"lm_head",
	"vision_tower"
	],
	"loraplus_lr_embedding": 1e-06,
	"lr_scheduler": "cosine",
	"mean_resizing_embeddings": false,
	"micro_batch_size": 1,
	"model_config_type": "qwen2_5_vl",
	"num_epochs": 3.0,
	"optimizer": "paged_adamw_32bit",
	"output_dir": "/workspace/fine-tuning/output/mymodel",
	"pretrain_multipack_attn": true,
	"processor_config": "Qwen/Qwen2.5-VL-7B-Instruct",
	"profiler_steps_start": 0,
	"qlora_sharded_model_loading": false,
	"ray_num_workers": 1,
	"resources_per_worker": {
	"GPU": 1
	},
	"sample_packing_bin_size": 200,
	"sample_packing_group_size": 100000,
	"save_only_model": false,
	"save_safetensors": true,
	"save_steps": 200,
	"save_total_limit": 3,
	"sequence_len": 2048,
	"shuffle_before_merging_datasets": false,
	"shuffle_merged_datasets": true,
	"skip_prepare_dataset": false,
	"streaming_multipack_buffer_size": 10000,
	"strict": false,
	"tensor_parallel_size": 1,
	"tiled_mlp_use_original_mlp": true,
	"tokenizer_config": "Qwen/Qwen2.5-VL-7B-Instruct",
	"tokenizer_save_jinja_files": true,
	"torch_compile": false,
	"torch_dtype": "torch.bfloat16",
	"train_on_inputs": false,
	"trl": {
	"log_completions": false,
	"mask_truncated_completions": false,
	"ref_model_mixup_alpha": 0.9,
	"ref_model_sync_steps": 64,
	"scale_rewards": true,
	"sync_ref_model": false,
	"use_vllm": false,
	"vllm_server_host": "0.0.0.0",
	"vllm_server_port": 8000
	},
	"trust_remote_code": true,
	"use_ray": false,
	"val_set_size": 0.0,
	"vllm": {
	"device": "auto",
	"dtype": "auto",
	"gpu_memory_utilization": 0.9,
	"host": "0.0.0.0",
	"port": 8000
	},
	"weight_decay": 0.0,
	"world_size": 1
	}
	[2025-10-22 10:50:23,548] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:278] [PID:2355] EOS: 151645 / <\|im_end\|>
	[2025-10-22 10:50:23,548] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:279] [PID:2355] BOS: None / None
	[2025-10-22 10:50:23,548] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:280] [PID:2355] PAD: 151643 / <\|endoftext\|>
	[2025-10-22 10:50:23,548] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:281] [PID:2355] UNK: None / None
	[2025-10-22 10:50:23,549] [INFO] [axolotl.utils.data.shared.load_preprocessed_dataset:476] [PID:2355] Unable to find prepared dataset in last_run_prepared/b27c6ba83f346393518eaef14d5ae408
	[2025-10-22 10:50:23,549] [INFO] [axolotl.utils.data.sft._load_raw_datasets:320] [PID:2355] Loading raw datasets...
	[2025-10-22 10:50:23,549] [WARNING] [axolotl.utils.data.sft._load_raw_datasets:322] [PID:2355] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`.
	[2025-10-22 10:50:23,988] [INFO] [axolotl.utils.data.wrappers.get_dataset_wrapper:87] [PID:2355] Loading dataset: /workspace/fine-tuning/data/data.json with base_type: vl and prompt_style: None
	[2025-10-22 10:50:23,989] [ERROR] [axolotl.utils.data.wrappers.handle_unknown_dataset_strategy:53] [PID:2355] unhandled prompt tokenization strategy: vl.
	Traceback (most recent call last):
	File "<frozen runpy>", line 198, in _run_module_as_main
	File "<frozen runpy>", line 88, in _run_code
	File "/workspace/axolotl/src/axolotl/cli/train.py", line 121, in <module>
	fire.Fire(do_cli)
	File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/fire/core.py", line 135, in Fire
	component_trace = _Fire(component, args, parsed_flag_args, context, name)
	^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
	File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/fire/core.py", line 468, in _Fire
	component, remaining_args = _CallAndUpdateTrace(
	^^^^^^^^^^^^^^^^^^^^
	File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/fire/core.py", line 684, in _CallAndUpdateTrace
	component = fn(varargs, *kwargs)
	^^^^^^^^^^^^^^^^^^^^^^
	File "/workspace/axolotl/src/axolotl/cli/train.py", line 88, in do_cli
	return do_train(parsed_cfg, parsed_cli_args)
	^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
	File "/workspace/axolotl/src/axolotl/cli/train.py", line 43, in do_train
	dataset_meta = load_datasets(cfg=cfg, cli_args=cli_args)
	^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
	File "/workspace/axolotl/src/axolotl/common/datasets.py", line 59, in load_datasets
	train_dataset, eval_dataset, total_num_steps, prompters = prepare_datasets(
	^^^^^^^^^^^^^^^^^
	File "/workspace/axolotl/src/axolotl/utils/data/utils.py", line 50, in wrapper
	return func(args, *kwargs)
	^^^^^^^^^^^^^^^^^^^^^
	File "/workspace/axolotl/src/axolotl/utils/data/sft.py", line 65, in prepare_datasets
	return _prepare_standard_dataset(cfg, tokenizer, processor)
	^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
	File "/workspace/axolotl/src/axolotl/utils/data/sft.py", line 98, in _prepare_standard_dataset
	train_dataset, eval_dataset, prompters = loader.load(_load_datasets)
	^^^^^^^^^^^^^^^^^^^^^^^^^^^
	File "/workspace/axolotl/src/axolotl/utils/data/lock.py", line 38, in load
	result = load_fn()
	^^^^^^^^^
	File "/workspace/axolotl/src/axolotl/utils/data/sft.py", line 77, in _load_datasets
	train_dataset, eval_dataset, prompters = _load_and_prepare_datasets(
	^^^^^^^^^^^^^^^^^^^^^^^^^^^
	File "/workspace/axolotl/src/axolotl/utils/data/sft.py", line 503, in _load_and_prepare_datasets
	dataset, prompters = _load_tokenized_prepared_datasets(
	^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
	File "/workspace/axolotl/src/axolotl/utils/data/sft.py", line 299, in _load_tokenized_prepared_datasets
	dataset, prompters = _load_raw_datasets(
	^^^^^^^^^^^^^^^^^^^
	File "/workspace/axolotl/src/axolotl/utils/data/sft.py", line 331, in _load_raw_datasets
	dataset_wrapper, dataset_prompter = _load_and_process_single_dataset(
	^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
	File "/workspace/axolotl/src/axolotl/utils/data/sft.py", line 401, in _load_and_process_single_dataset
	dataset_wrapper, dataset_prompter = get_dataset_wrapper(
	^^^^^^^^^^^^^^^^^^^^
	File "/workspace/axolotl/src/axolotl/utils/data/wrappers.py", line 131, in get_dataset_wrapper
	handle_unknown_dataset_strategy(dataset_config)
	File "/workspace/axolotl/src/axolotl/utils/data/wrappers.py", line 54, in handle_unknown_dataset_strategy
	raise ValueError(error_message)
	ValueError: unhandled prompt tokenization strategy: vl.