Instructions to use anilsuthar2004/qwen-bot with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- llama.cpp
How to use anilsuthar2004/qwen-bot with llama.cpp:
Install (macOS, Linux)
curl -LsSf https://llama.app/install.sh | sh # Start a local OpenAI-compatible server with a web UI: llama serve -hf anilsuthar2004/qwen-bot # Run inference directly in the terminal: llama cli -hf anilsuthar2004/qwen-bot
Install from WinGet (Windows)
winget install llama.cpp # Start a local OpenAI-compatible server with a web UI: llama serve -hf anilsuthar2004/qwen-bot # Run inference directly in the terminal: llama cli -hf anilsuthar2004/qwen-bot
Build from source code
git clone https://github.com/ggml-org/llama.cpp.git cd llama.cpp cmake -B build cmake --build build -j --target llama-app # Start a local OpenAI-compatible server with a web UI: ./build/bin/llama serve -hf anilsuthar2004/qwen-bot # Run inference directly in the terminal: ./build/bin/llama cli -hf anilsuthar2004/qwen-bot
Use Docker
docker model run hf.co/anilsuthar2004/qwen-bot
- Llama new
- LM Studio
- Jan
- Ollama
How to use anilsuthar2004/qwen-bot with Ollama:
ollama run hf.co/anilsuthar2004/qwen-bot
- Unsloth Desktop
- Docker Model Runner
How to use anilsuthar2004/qwen-bot with Docker Model Runner:
docker model run hf.co/anilsuthar2004/qwen-bot
- Lemonade
How to use anilsuthar2004/qwen-bot with Lemonade:
Pull the model
# Download Lemonade from https://lemonade-server.ai/ lemonade pull anilsuthar2004/qwen-bot
Run and chat with the model
lemonade run user.qwen-bot-{{QUANT_TAG}}List all available models
lemonade list
- Atomic Chat
File size: 4,100 Bytes
645e264 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 | #include "models.h"
// DeepSeek-V4-Flash-Vision encoder (deepseek4v)
//
// native-resolution ViT (RMSNorm, SwiGLU, 2D RoPE, no CLS / learned pos-embd)
// then the "aligner": 3x3 patch merge (torch.nn.functional.unfold) + 2-layer GELU MLP
//
// the graph outputs the complete LLM token block, built from the aligner output and 4 learned sentinel embeddings:
//
// [PAD]*lead_pad [START] <interleaved rows> [PAD]*pad_last [END]
//
// each aligner row ends with a NEWLINE, an odd row count is padded with a full row of PADs
// pairs of adjacent rows are interleaved column-wise ("N-layout")
// the mapping is precomputed on CPU as the "layout_idx" input (see set_input in clip.cpp)
//
// ref: inference/vision.py and inference/image_processor.py in the HF repo
ggml_cgraph * clip_graph_deepseek4v::build() {
const int n_merge = hparams.n_merge;
// 2D input positions
ggml_tensor * positions = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_patches * 4);
ggml_set_name(positions, "positions");
ggml_set_input(positions);
int sections[4] = {d_head/4, d_head/4, 0, 0};
auto add_pos = [&](ggml_tensor * cur, const clip_layer &) {
return ggml_rope_multi(ctx0, cur, positions, nullptr,
d_head/2, sections, GGML_ROPE_TYPE_VISION,
0, hparams.rope_theta, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f);
};
ggml_tensor * inp = build_inp();
ggml_tensor * cur = build_vit(
inp, n_patches,
NORM_TYPE_RMS,
hparams.ffn_op,
nullptr, // no learned pos embd
add_pos);
cb(cur, "vit_out", -1);
// aligner patch merge: zero-pad the patch grid to a multiple of n_merge
// then F.unfold == im2col with a dummy kernel (same trick as pixtral)
{
cur = ggml_reshape_3d(ctx0, cur, n_embd, n_patches_x, n_patches_y);
cur = ggml_permute(ctx0, cur, 2, 0, 1, 3); // [x, y, n_embd]
cur = ggml_cont(ctx0, cur);
const int pad_x = (n_merge - n_patches_x % n_merge) % n_merge;
const int pad_y = (n_merge - n_patches_y % n_merge) % n_merge;
if (pad_x || pad_y) {
cur = ggml_pad(ctx0, cur, pad_x, pad_y, 0, 0);
}
ggml_tensor * kernel = ggml_view_3d(ctx0, cur, n_merge, n_merge, cur->ne[2], 0, 0, 0);
cur = ggml_im2col(ctx0, kernel, cur, n_merge, n_merge, 0, 0, 1, 1, true, inp->type);
cur = ggml_reshape_2d(ctx0, cur, cur->ne[0], cur->ne[1] * cur->ne[2]);
// aligner MLP (F.gelu in the reference == erf-based gelu)
cur = build_ffn(cur,
model.mm_1_w, model.mm_1_b,
nullptr, nullptr,
model.mm_2_w, model.mm_2_b,
FFN_GELU_ERF,
-1);
cb(cur, "aligner_out", -1);
}
// assemble the token block: append the sentinel embeddings as extra rows
// then reorder everything with the precomputed layout index
{
const int64_t n_embd_out = cur->ne[0];
const int64_t n_grid = cur->ne[1]; // n_llm_w * n_llm_h
// rows n_grid + 0..3, keep in sync with the index computation in set_input
ggml_tensor * sentinels[] = {
model.token_embd_img_start,
model.token_embd_img_end,
model.image_newline,
model.token_embd_img_pad,
};
for (ggml_tensor * tok : sentinels) {
cur = ggml_concat(ctx0, cur, ggml_reshape_2d(ctx0, tok, n_embd_out, 1), 1);
}
const int n_llm_w = CLIP_ALIGN(n_patches_x, n_merge) / n_merge;
const int n_llm_h = CLIP_ALIGN(n_patches_y, n_merge) / n_merge;
const int n_out = dsv4_get_block_layout(n_llm_w, n_llm_h, img.lead_pad).n_out;
GGML_ASSERT(n_grid == n_llm_w * n_llm_h);
ggml_tensor * layout_idx = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_out);
ggml_set_name(layout_idx, "layout_idx");
ggml_set_input(layout_idx);
cur = ggml_get_rows(ctx0, cur, layout_idx);
}
// build the graph
ggml_build_forward_expand(gf, cur);
return gf;
}
|