Instructions to use ChapAF/intent-0.1-0.5b with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- llama.cpp
How to use ChapAF/intent-0.1-0.5b with llama.cpp:
Install (macOS, Linux)
curl -LsSf https://llama.app/install.sh | sh # Start a local OpenAI-compatible server with a web UI: llama serve -hf ChapAF/intent-0.1-0.5b:F16 # Run inference directly in the terminal: llama cli -hf ChapAF/intent-0.1-0.5b:F16
Install from WinGet (Windows)
winget install llama.cpp # Start a local OpenAI-compatible server with a web UI: llama serve -hf ChapAF/intent-0.1-0.5b:F16 # Run inference directly in the terminal: llama cli -hf ChapAF/intent-0.1-0.5b:F16
Build from source code
git clone https://github.com/ggml-org/llama.cpp.git cd llama.cpp cmake -B build cmake --build build -j --target llama-app # Start a local OpenAI-compatible server with a web UI: ./build/bin/llama serve -hf ChapAF/intent-0.1-0.5b:F16 # Run inference directly in the terminal: ./build/bin/llama cli -hf ChapAF/intent-0.1-0.5b:F16
Use Docker
docker model run hf.co/ChapAF/intent-0.1-0.5b:F16
- Llama new
- LM Studio
- Jan
- Ollama
How to use ChapAF/intent-0.1-0.5b with Ollama:
ollama run hf.co/ChapAF/intent-0.1-0.5b:F16
- Unsloth Desktop
- Docker Model Runner
How to use ChapAF/intent-0.1-0.5b with Docker Model Runner:
docker model run hf.co/ChapAF/intent-0.1-0.5b:F16
- Lemonade
How to use ChapAF/intent-0.1-0.5b with Lemonade:
Pull the model
# Download Lemonade from https://lemonade-server.ai/ lemonade pull ChapAF/intent-0.1-0.5b:F16
Run and chat with the model
lemonade run user.intent-0.1-0.5b-F16
List all available models
lemonade list
- Atomic Chat
Download pretraining/model.py from ChapAF/intent-0.1-0.5b: direct link, hf CLI and curl.
- Browser
- Download file 10.9 kB
-
https://huggingface.co/ChapAF/intent-0.1-0.5b/resolve/main/pretraining/model.py
- Command line
-
hf download hf://ChapAF/intent-0.1-0.5b/pretraining/model.py
-
curl -L -o model.py https://huggingface.co/ChapAF/intent-0.1-0.5b/resolve/main/pretraining/model.py
10.9 kB
| from __future__ import annotations | |
| import math | |
| from dataclasses import dataclass | |
| import torch | |
| import torch.nn as nn | |
| import torch.nn.functional as F | |
| import torch.utils.checkpoint | |
| try: | |
| from flash_attn import flash_attn_func as _flash_attn_func | |
| HAS_FLASH_ATTN = True | |
| except Exception: | |
| # A package can be discoverable while its CUDA extension is ABI-incompatible | |
| # with the installed Torch. Treat that as unavailable so startup falls | |
| # back safely to PyTorch SDPA instead of crashing on the first batch. | |
| _flash_attn_func = None | |
| HAS_FLASH_ATTN = False | |
| try: | |
| import liger_kernel # noqa: F401 | |
| HAS_LIGER = True | |
| except Exception: | |
| HAS_LIGER = False | |
| class ModelConfig: | |
| vocab_size: int = 32_768 | |
| sequence_len: int = 2_048 | |
| n_layer: int = 21 | |
| n_head: int = 10 | |
| n_kv_head: int = 10 | |
| n_embd: int = 1_280 | |
| mlp_ratio: float = 4.0 | |
| rope_base: float = 10_000.0 | |
| dropout: float = 0.0 | |
| use_qk_norm: bool = True | |
| use_liger: bool = True | |
| attention_backend: str = "auto" # auto, flash_attn, sdpa | |
| compute_dtype: str = "bfloat16" | |
| def head_dim(self) -> int: | |
| if self.n_embd % self.n_head: | |
| raise ValueError("n_embd must be divisible by n_head") | |
| return self.n_embd // self.n_head | |
| def intermediate_size(self) -> int: | |
| return int(self.n_embd * self.mlp_ratio) | |
| def _rmsnorm_cls(use_liger: bool): | |
| if use_liger and HAS_LIGER: | |
| try: | |
| from liger_kernel.transformers import LigerRMSNorm | |
| return LigerRMSNorm | |
| except (ImportError, AttributeError): | |
| pass | |
| return None | |
| class RMSNorm(nn.Module): | |
| """Parameterless RMSNorm executed in the activation dtype.""" | |
| def __init__(self, dim: int, eps: float = 1e-6, use_liger: bool = False): | |
| super().__init__() | |
| self.dim = dim | |
| self.eps = eps | |
| liger = _rmsnorm_cls(use_liger) | |
| self._impl = liger(dim, eps=eps) if liger is not None else None | |
| def forward(self, x: torch.Tensor) -> torch.Tensor: | |
| if self._impl is not None: | |
| return self._impl(x) | |
| return F.rms_norm(x, (self.dim,), eps=self.eps) | |
| class Linear(nn.Linear): | |
| """Keep fp32 master weights while running matmuls in activation dtype.""" | |
| def forward(self, x: torch.Tensor) -> torch.Tensor: | |
| weight = self.weight.to(dtype=x.dtype) | |
| bias = self.bias.to(dtype=x.dtype) if self.bias is not None else None | |
| return F.linear(x, weight, bias) | |
| def _rotate_half(x: torch.Tensor) -> torch.Tensor: | |
| x1, x2 = x[..., : x.shape[-1] // 2], x[..., x.shape[-1] // 2 :] | |
| return torch.cat((-x2, x1), dim=-1) | |
| def _apply_rope(x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor) -> torch.Tensor: | |
| return (x * cos) + (_rotate_half(x) * sin) | |
| class CausalSelfAttention(nn.Module): | |
| def __init__(self, config: ModelConfig): | |
| super().__init__() | |
| if config.n_head % config.n_kv_head: | |
| raise ValueError("n_head must be divisible by n_kv_head") | |
| self.n_head = config.n_head | |
| self.n_kv_head = config.n_kv_head | |
| self.head_dim = config.head_dim | |
| self.dropout = config.dropout | |
| self.backend = config.attention_backend | |
| if self.backend == "auto": | |
| self.backend = "flash_attn" if HAS_FLASH_ATTN else "sdpa" | |
| self.qkv = Linear(config.n_embd, (config.n_head + 2 * config.n_kv_head) * self.head_dim, bias=False) | |
| self.proj = Linear(config.n_embd, config.n_embd, bias=False) | |
| self.q_norm = RMSNorm(self.head_dim, use_liger=config.use_liger) if config.use_qk_norm else nn.Identity() | |
| self.k_norm = RMSNorm(self.head_dim, use_liger=config.use_liger) if config.use_qk_norm else nn.Identity() | |
| def forward(self, x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor) -> torch.Tensor: | |
| batch, length, _ = x.shape | |
| qkv = self.qkv(x).view(batch, length, self.n_head + 2 * self.n_kv_head, self.head_dim) | |
| q = qkv[:, :, : self.n_head] | |
| k = qkv[:, :, self.n_head : self.n_head + self.n_kv_head] | |
| v = qkv[:, :, self.n_head + self.n_kv_head :] | |
| q = _apply_rope(self.q_norm(q), cos[:, :length], sin[:, :length]) | |
| k = _apply_rope(self.k_norm(k), cos[:, :length], sin[:, :length]) | |
| if self.n_kv_head != self.n_head: | |
| repeats = self.n_head // self.n_kv_head | |
| k = k.repeat_interleave(repeats, dim=2) | |
| v = v.repeat_interleave(repeats, dim=2) | |
| if self.backend == "flash_attn" and HAS_FLASH_ATTN and x.is_cuda and x.dtype in (torch.float16, torch.bfloat16): | |
| y = _flash_attn_func(q, k, v, dropout_p=self.dropout if self.training else 0.0, causal=True) | |
| else: | |
| q_t = q.transpose(1, 2) | |
| k_t = k.transpose(1, 2) | |
| v_t = v.transpose(1, 2) | |
| y = F.scaled_dot_product_attention( | |
| q_t, | |
| k_t, | |
| v_t, | |
| dropout_p=self.dropout if self.training else 0.0, | |
| is_causal=True, | |
| ).transpose(1, 2) | |
| return self.proj(y.reshape(batch, length, -1)) | |
| class MLP(nn.Module): | |
| def __init__(self, config: ModelConfig): | |
| super().__init__() | |
| hidden = config.intermediate_size | |
| self.fc = Linear(config.n_embd, hidden, bias=False) | |
| self.proj = Linear(hidden, config.n_embd, bias=False) | |
| def forward(self, x: torch.Tensor) -> torch.Tensor: | |
| # ReLU-squared is the dense FNN used by nanochat and avoids a gated | |
| # third matrix multiply while retaining a useful smooth positive gate. | |
| return self.proj(F.relu(self.fc(x)).square()) | |
| class Block(nn.Module): | |
| def __init__(self, config: ModelConfig): | |
| super().__init__() | |
| self.attn_norm = RMSNorm(config.n_embd, use_liger=config.use_liger) | |
| self.attn = CausalSelfAttention(config) | |
| self.mlp_norm = RMSNorm(config.n_embd, use_liger=config.use_liger) | |
| self.mlp = MLP(config) | |
| def forward(self, x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor) -> torch.Tensor: | |
| x = x + self.attn(self.attn_norm(x), cos, sin) | |
| x = x + self.mlp(self.mlp_norm(x)) | |
| return x | |
| class GPT(nn.Module): | |
| def __init__(self, config: ModelConfig): | |
| super().__init__() | |
| self.config = config | |
| self.activation_checkpointing = False | |
| self.token_embedding = nn.Embedding(config.vocab_size, config.n_embd) | |
| self.blocks = nn.ModuleList([Block(config) for _ in range(config.n_layer)]) | |
| self.final_norm = RMSNorm(config.n_embd, use_liger=config.use_liger) | |
| self.lm_head = Linear(config.n_embd, config.vocab_size, bias=False) | |
| inv_freq = 1.0 / (config.rope_base ** (torch.arange(0, config.head_dim, 2).float() / config.head_dim)) | |
| positions = torch.arange(config.sequence_len).float() | |
| freqs = torch.outer(positions, inv_freq) | |
| emb = torch.cat((freqs, freqs), dim=-1) | |
| self.register_buffer("rope_cos", emb.cos()[None, :, None, :], persistent=False) | |
| self.register_buffer("rope_sin", emb.sin()[None, :, None, :], persistent=False) | |
| self.apply(self._init_weights) | |
| def _init_weights(self, module: nn.Module) -> None: | |
| if isinstance(module, nn.Linear): | |
| std = 0.02 | |
| if module is self.lm_head: | |
| std = 0.02 | |
| nn.init.normal_(module.weight, mean=0.0, std=std) | |
| elif isinstance(module, nn.Embedding): | |
| nn.init.normal_(module.weight, mean=0.0, std=0.02) | |
| def forward( | |
| self, | |
| input_ids: torch.Tensor, | |
| targets: torch.Tensor | None = None, | |
| loss_mask: torch.Tensor | None = None, | |
| return_logits: bool = True, | |
| ): | |
| if input_ids.shape[1] > self.config.sequence_len: | |
| raise ValueError(f"sequence length {input_ids.shape[1]} exceeds {self.config.sequence_len}") | |
| x = self.token_embedding(input_ids) | |
| compute_dtype = getattr(torch, self.config.compute_dtype) | |
| if x.dtype != compute_dtype: | |
| x = x.to(compute_dtype) | |
| cos = self.rope_cos.to(device=x.device, dtype=x.dtype) | |
| sin = self.rope_sin.to(device=x.device, dtype=x.dtype) | |
| for block in self.blocks: | |
| if self.training and self.activation_checkpointing: | |
| x = torch.utils.checkpoint.checkpoint(block, x, cos, sin, use_reentrant=False) | |
| else: | |
| x = block(x, cos, sin) | |
| logits = self.lm_head(self.final_norm(x)) | |
| if targets is None: | |
| return logits | |
| if loss_mask is None: | |
| loss = F.cross_entropy(logits.reshape(-1, logits.size(-1)).float(), targets.reshape(-1)) | |
| else: | |
| if loss_mask.shape != targets.shape: | |
| raise ValueError("loss_mask must have the same shape as targets") | |
| # The SFT mask is binary. Convert masked positions to the native | |
| # cross-entropy ignore index so PyTorch can use its fused mean | |
| # reduction instead of materializing a loss value for every | |
| # vocabulary row and then multiplying by the mask. This keeps the | |
| # assistant-only objective identical while avoiding a substantial | |
| # SFT throughput penalty. | |
| masked_targets = targets.masked_fill(~loss_mask, -100) | |
| loss = F.cross_entropy( | |
| logits.reshape(-1, logits.size(-1)).float(), | |
| masked_targets.reshape(-1), | |
| ignore_index=-100, | |
| reduction="sum", | |
| ) | |
| loss = loss / loss_mask.sum().to(dtype=loss.dtype).clamp_min(1.0) | |
| return (logits, loss) if return_logits else loss | |
| def num_parameters(self, non_embedding: bool = False) -> int: | |
| total = sum(p.numel() for p in self.parameters()) | |
| if non_embedding: | |
| total -= self.token_embedding.weight.numel() + self.lm_head.weight.numel() | |
| return total | |
| def architecture_summary(config: ModelConfig) -> dict: | |
| matrix_per_layer = ( | |
| (config.n_head + 2 * config.n_kv_head) * config.head_dim * config.n_embd | |
| + config.n_embd * config.n_embd | |
| + config.n_embd * config.intermediate_size | |
| + config.intermediate_size * config.n_embd | |
| ) | |
| norm_parameters = (2 * config.n_layer + 1) * config.n_embd if config.use_liger else 0 | |
| parameter_count = 2 * config.vocab_size * config.n_embd + config.n_layer * matrix_per_layer + norm_parameters | |
| return { | |
| "parameters": parameter_count, | |
| "non_embedding_parameters": parameter_count - 2 * config.vocab_size * config.n_embd, | |
| "attention_backend": config.attention_backend, | |
| "selected_attention_backend": ( | |
| "flash_attn" if config.attention_backend == "auto" and HAS_FLASH_ATTN else "sdpa" | |
| ), | |
| "flash_attn_available": HAS_FLASH_ATTN, | |
| "liger_available": HAS_LIGER, | |
| "config": config.__dict__, | |
| } | |