Text Generation
Transformers
Safetensors
English
Chinese
qwen3
ksa
ksa-v1
custom_code
long-context
conversational
text-generation-inference
Instructions to use OpenOneRec/KSA-1.9B-base with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use OpenOneRec/KSA-1.9B-base with Transformers:
# Use a pipeline as a high-level helper from transformers import pipeline pipe = pipeline("text-generation", model="OpenOneRec/KSA-1.9B-base", trust_remote_code=True) messages = [ {"role": "user", "content": "Who are you?"}, ] pipe(messages)# pip install -U transformers accelerate # Load model directly from transformers import AutoTokenizer, AutoModelForCausalLM tokenizer = AutoTokenizer.from_pretrained("OpenOneRec/KSA-1.9B-base", trust_remote_code=True) model = AutoModelForCausalLM.from_pretrained("OpenOneRec/KSA-1.9B-base", trust_remote_code=True, device_map="auto") messages = [ {"role": "user", "content": "Who are you?"}, ] inputs = tokenizer.apply_chat_template( messages, add_generation_prompt=True, tokenize=True, return_dict=True, return_tensors="pt", ).to(model.device) outputs = model.generate(**inputs, max_new_tokens=256) print(tokenizer.decode(outputs[0][inputs["input_ids"].shape[-1]:])) - Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- vLLM
How to use OpenOneRec/KSA-1.9B-base with vLLM:
Install from pip and serve model
# Install vLLM from pip: pip install vllm # Start the vLLM server: vllm serve "OpenOneRec/KSA-1.9B-base" # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:8000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "OpenOneRec/KSA-1.9B-base", "messages": [ { "role": "user", "content": "What is the capital of France?" } ] }'Use Docker
docker model run hf.co/OpenOneRec/KSA-1.9B-base
- SGLang
How to use OpenOneRec/KSA-1.9B-base with SGLang:
Install from pip and serve model
# Install SGLang from pip: pip install sglang # Start the SGLang server: python3 -m sglang.launch_server \ --model-path "OpenOneRec/KSA-1.9B-base" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "OpenOneRec/KSA-1.9B-base", "messages": [ { "role": "user", "content": "What is the capital of France?" } ] }'Use Docker images
docker run --gpus all \ --shm-size 32g \ -p 30000:30000 \ -v ~/.cache/huggingface:/root/.cache/huggingface \ --env "HF_TOKEN=<secret>" \ --ipc=host \ lmsysorg/sglang:latest \ python3 -m sglang.launch_server \ --model-path "OpenOneRec/KSA-1.9B-base" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "OpenOneRec/KSA-1.9B-base", "messages": [ { "role": "user", "content": "What is the capital of France?" } ] }' - Docker Model Runner
How to use OpenOneRec/KSA-1.9B-base with Docker Model Runner:
docker model run hf.co/OpenOneRec/KSA-1.9B-base
Download kernel/ksa_attn/mask.py from OpenOneRec/KSA-1.9B-base: direct link, hf CLI and curl.
- Browser
- Download file 10.5 kB
-
https://huggingface.co/OpenOneRec/KSA-1.9B-base/resolve/main/kernel/ksa_attn/mask.py
- Command line
-
hf download hf://OpenOneRec/KSA-1.9B-base/kernel/ksa_attn/mask.py
-
curl -L -o mask.py https://huggingface.co/OpenOneRec/KSA-1.9B-base/resolve/main/kernel/ksa_attn/mask.py
10.5 kB
| import cutlass | |
| from cutlass import cute | |
| from flash_attn.cute.seqlen_info import SeqlenInfoQK | |
| from flash_attn.cute.utils import scalar_to_ssa, ssa_to_scalar | |
| def summary_block_sparse_attn_mask_func( | |
| batch_idx_ssa: cute.TensorSSA, | |
| head_idx_ssa: cute.TensorSSA, | |
| q_idx_ssa: cute.TensorSSA, | |
| kv_idx_ssa: cute.TensorSSA, | |
| seqlen_info: SeqlenInfoQK, | |
| aux_tensors, | |
| block_size_tokens: cutlass.Constexpr[int], | |
| local_window_tokens: cutlass.Constexpr[int], | |
| local_alignment_tokens: cutlass.Constexpr[int] = 0, | |
| ): | |
| """Per-query raw-token mask selected through summary-block indices. | |
| ``aux_tensors[0]`` is a packed uint8 tensor with layout | |
| ``[batch, query, ceil(raw_block_count / 8)]``. Bit ``j`` is set exactly | |
| when the query selected summary/raw block ``j``. Packing the dynamic | |
| ``[B,Q,K]`` indices before launching attention avoids materialising the | |
| quadratic ``[B,Q,raw_block_count]`` boolean route. | |
| The CuTe scheduler still works in 128-token tiles. A scheduled tile is | |
| only a conservative union; this function is evaluated for every score and | |
| preserves exact per-query, per-``block_size_tokens`` semantics. | |
| """ | |
| del head_idx_ssa, seqlen_info | |
| raw_block_idx_ssa = kv_idx_ssa // block_size_tokens | |
| route_byte_idx_ssa = raw_block_idx_ssa // 8 | |
| route_bit_idx_ssa = raw_block_idx_ssa % 8 | |
| packed_route = aux_tensors[0] | |
| route_byte = packed_route[ | |
| ssa_to_scalar(batch_idx_ssa), | |
| ssa_to_scalar(q_idx_ssa), | |
| ssa_to_scalar(route_byte_idx_ssa), | |
| ] | |
| route_bit = 1 << ssa_to_scalar(route_bit_idx_ssa) | |
| selected = (route_byte & route_bit) != 0 | |
| selected_ssa = scalar_to_ssa(selected, cutlass.Boolean) | |
| causal = kv_idx_ssa <= q_idx_ssa | |
| raw_local_start = q_idx_ssa - local_window_tokens + 1 | |
| clamped_local_start = cute.where( | |
| raw_local_start > 0, | |
| raw_local_start, | |
| cute.full_like(raw_local_start, 0), | |
| ) | |
| # Preserve the original summary-block API: callers that do not provide a | |
| # separate local alignment retain ``block_size_tokens`` alignment. Raw | |
| # token routing explicitly supplies eight here while its route span is one. | |
| aligned_local_start = ( | |
| clamped_local_start // block_size_tokens | |
| ) * block_size_tokens | |
| if local_alignment_tokens > 0: | |
| aligned_local_start = ( | |
| clamped_local_start // local_alignment_tokens | |
| ) * local_alignment_tokens | |
| local = kv_idx_ssa >= aligned_local_start | |
| return causal & (local | selected_ssa) | |
| def hierarchical_summary_attn_score_mod( | |
| score_ssa: cute.TensorSSA, | |
| batch_idx_ssa: cute.TensorSSA, | |
| head_idx_ssa: cute.TensorSSA, | |
| q_idx: cute.TensorSSA, | |
| kv_idx: cute.TensorSSA, | |
| seqlen_info: SeqlenInfoQK, | |
| aux_tensors, | |
| meta_length: cutlass.Constexpr[int], | |
| meta_logit_bias: cutlass.Constexpr[float], | |
| ): | |
| """Add log(G) to each physical meta entry before softmax.""" | |
| return cute.where( | |
| kv_idx < meta_length, | |
| score_ssa + meta_logit_bias, | |
| score_ssa, | |
| ) | |
| def hierarchical_summary_attn_mask_func( | |
| batch_idx_ssa: cute.TensorSSA, | |
| head_idx_ssa: cute.TensorSSA, | |
| q_idx_ssa: cute.TensorSSA, | |
| kv_idx_ssa: cute.TensorSSA, | |
| seqlen_info: SeqlenInfoQK, | |
| aux_tensors, | |
| chunk_size: cutlass.Constexpr[int], | |
| sliding_chunk_num: cutlass.Constexpr[int], | |
| group_size: cutlass.Constexpr[int], | |
| meta_length: cutlass.Constexpr[int], | |
| swap_indices: cutlass.Constexpr[bool], | |
| ): | |
| """Compressed-KSA mask for K/V laid out as ``[meta, base]``.""" | |
| if swap_indices: | |
| q_idx_ssa, kv_idx_ssa = kv_idx_ssa, q_idx_ssa | |
| block_size = chunk_size + 1 | |
| q_chunk_id = q_idx_ssa // block_size | |
| q_offset = q_idx_ssa % block_size | |
| is_q_text = q_offset < chunk_size | |
| raw_window_start = q_chunk_id - sliding_chunk_num | |
| window_start = cute.where( | |
| raw_window_start > 0, | |
| raw_window_start, | |
| cute.full_like(raw_window_start, 0), | |
| ) | |
| complete_group_count = window_start // group_size | |
| complete_end = complete_group_count * group_size | |
| is_meta = kv_idx_ssa < meta_length | |
| base_idx = kv_idx_ssa - meta_length | |
| kv_chunk_id = base_idx // block_size | |
| kv_offset = base_idx % block_size | |
| is_kv_text = kv_offset < chunk_size | |
| is_kv_summary = kv_offset == chunk_size | |
| meta_group_id = kv_idx_ssa | |
| valid_meta = is_q_text & is_meta & (meta_group_id < complete_group_count) | |
| valid_text = ( | |
| is_q_text & (is_meta ^ 1) & is_kv_text | |
| & (((kv_chunk_id >= window_start) & (kv_chunk_id < q_chunk_id)) | |
| | ((kv_chunk_id == q_chunk_id) & (base_idx <= q_idx_ssa))) | |
| ) | |
| valid_residual = ( | |
| is_q_text & (is_meta ^ 1) & is_kv_summary | |
| & (kv_chunk_id >= complete_end) & (kv_chunk_id < window_start) | |
| ) | |
| valid_summary = ( | |
| (is_q_text ^ 1) & (is_meta ^ 1) & (kv_chunk_id == q_chunk_id) | |
| & (is_kv_text | (is_kv_summary & (base_idx <= q_idx_ssa))) | |
| ) | |
| return valid_meta | valid_text | valid_residual | valid_summary | |
| def summary_attn_mask_func( | |
| batch_idx_ssa: cute.TensorSSA, | |
| head_idx_ssa: cute.TensorSSA, | |
| q_idx_ssa: cute.TensorSSA, | |
| kv_idx_ssa: cute.TensorSSA, | |
| seqlen_info: SeqlenInfoQK, | |
| aux_tensors, | |
| chunk_size: cutlass.Constexpr[int], | |
| summary_num: cutlass.Constexpr[int], | |
| sliding_chunk_num: cutlass.Constexpr[int], | |
| prefix_summary_length: cutlass.Constexpr[int], | |
| swap_indices: cutlass.Constexpr[bool], | |
| ): | |
| if swap_indices: | |
| q_idx_ssa, kv_idx_ssa = kv_idx_ssa, q_idx_ssa | |
| q_chunk_id = q_idx_ssa // (chunk_size + summary_num) | |
| kv_chunk_id = kv_idx_ssa // (chunk_size + summary_num) | |
| is_sliding_chunk = (q_chunk_id - kv_chunk_id) <= sliding_chunk_num | |
| is_q_summary = (q_idx_ssa % (chunk_size + summary_num)) >= chunk_size | |
| is_kv_summary = (kv_idx_ssa % (chunk_size + summary_num)) >= chunk_size | |
| is_causal = q_idx_ssa >= kv_idx_ssa | |
| valid_kv_q_summary = is_q_summary & (kv_chunk_id == q_chunk_id) & is_causal | |
| valid_sliding_q_text = is_sliding_chunk & (is_kv_summary ^ 1) | |
| valid_summary_q_text = is_kv_summary & ((q_chunk_id - kv_chunk_id) > sliding_chunk_num) | |
| valid_kv_q_text = (is_q_summary ^ 1) & is_causal & (valid_sliding_q_text | valid_summary_q_text) | |
| return valid_kv_q_summary | valid_kv_q_text | |
| def summary_attn_contiguous_mask_func( | |
| batch_idx_ssa: cute.TensorSSA, | |
| head_idx_ssa: cute.TensorSSA, | |
| q_idx_ssa: cute.TensorSSA, | |
| kv_idx_ssa: cute.TensorSSA, | |
| seqlen_info: SeqlenInfoQK, | |
| aux_tensors, | |
| chunk_size: cutlass.Constexpr[int], | |
| summary_num: cutlass.Constexpr[int], | |
| sliding_chunk_num: cutlass.Constexpr[int], | |
| prefix_summary_length: cutlass.Constexpr[int], | |
| swap_indices: cutlass.Constexpr[bool], | |
| ): | |
| if swap_indices: | |
| q_idx_ssa, kv_idx_ssa = kv_idx_ssa, q_idx_ssa | |
| q_chunk_id = q_idx_ssa // (chunk_size + summary_num) | |
| kv_chunk_id = (kv_idx_ssa - prefix_summary_length) // (chunk_size + summary_num) | |
| is_sliding_chunk = (q_chunk_id - kv_chunk_id) <= sliding_chunk_num | |
| is_q_summary = (q_idx_ssa % (chunk_size + summary_num)) >= chunk_size | |
| is_kv_summary = ((kv_idx_ssa - prefix_summary_length) % (chunk_size + summary_num)) >= chunk_size | |
| is_causal = q_idx_ssa >= (kv_idx_ssa - prefix_summary_length) | |
| valid_kv_q_summary = is_q_summary & (kv_chunk_id == q_chunk_id) & is_causal | |
| valid_sliding_q_text = is_sliding_chunk & (is_kv_summary ^ 1) & (kv_idx_ssa >= prefix_summary_length) | |
| valid_summary_q_text = ((kv_idx_ssa // summary_num) + sliding_chunk_num) < q_chunk_id | |
| valid_kv_q_text = (is_q_summary ^ 1) & ((is_causal & valid_sliding_q_text) | valid_summary_q_text) | |
| return valid_kv_q_summary | valid_kv_q_text | |
| # --------------------------------------------------------------------------- | |
| # Sliding-chunk-only variants: text tokens never attend to old summary tokens | |
| # --------------------------------------------------------------------------- | |
| def sliding_chunk_mask_func( | |
| batch_idx_ssa: cute.TensorSSA, | |
| head_idx_ssa: cute.TensorSSA, | |
| q_idx_ssa: cute.TensorSSA, | |
| kv_idx_ssa: cute.TensorSSA, | |
| seqlen_info: SeqlenInfoQK, | |
| aux_tensors, | |
| chunk_size: cutlass.Constexpr[int], | |
| summary_num: cutlass.Constexpr[int], | |
| sliding_chunk_num: cutlass.Constexpr[int], | |
| prefix_summary_length: cutlass.Constexpr[int], | |
| swap_indices: cutlass.Constexpr[bool], | |
| ): | |
| if swap_indices: | |
| q_idx_ssa, kv_idx_ssa = kv_idx_ssa, q_idx_ssa | |
| q_chunk_id = q_idx_ssa // (chunk_size + summary_num) | |
| kv_chunk_id = kv_idx_ssa // (chunk_size + summary_num) | |
| is_sliding_chunk = (q_chunk_id - kv_chunk_id) <= sliding_chunk_num | |
| is_q_summary = (q_idx_ssa % (chunk_size + summary_num)) >= chunk_size | |
| is_kv_summary = (kv_idx_ssa % (chunk_size + summary_num)) >= chunk_size | |
| is_causal = q_idx_ssa >= kv_idx_ssa | |
| valid_kv_q_summary = is_q_summary & (kv_chunk_id == q_chunk_id) & is_causal | |
| valid_sliding_q_text = is_sliding_chunk & (is_kv_summary ^ 1) | |
| valid_kv_q_text = (is_q_summary ^ 1) & is_causal & valid_sliding_q_text | |
| return valid_kv_q_summary | valid_kv_q_text | |
| def sliding_chunk_contiguous_mask_func( | |
| batch_idx_ssa: cute.TensorSSA, | |
| head_idx_ssa: cute.TensorSSA, | |
| q_idx_ssa: cute.TensorSSA, | |
| kv_idx_ssa: cute.TensorSSA, | |
| seqlen_info: SeqlenInfoQK, | |
| aux_tensors, | |
| chunk_size: cutlass.Constexpr[int], | |
| summary_num: cutlass.Constexpr[int], | |
| sliding_chunk_num: cutlass.Constexpr[int], | |
| prefix_summary_length: cutlass.Constexpr[int], | |
| swap_indices: cutlass.Constexpr[bool], | |
| ): | |
| if swap_indices: | |
| q_idx_ssa, kv_idx_ssa = kv_idx_ssa, q_idx_ssa | |
| q_chunk_id = q_idx_ssa // (chunk_size + summary_num) | |
| kv_chunk_id = (kv_idx_ssa - prefix_summary_length) // (chunk_size + summary_num) | |
| is_sliding_chunk = (q_chunk_id - kv_chunk_id) <= sliding_chunk_num | |
| is_q_summary = (q_idx_ssa % (chunk_size + summary_num)) >= chunk_size | |
| is_kv_summary = ((kv_idx_ssa - prefix_summary_length) % (chunk_size + summary_num)) >= chunk_size | |
| is_causal = q_idx_ssa >= (kv_idx_ssa - prefix_summary_length) | |
| valid_kv_q_summary = is_q_summary & (kv_chunk_id == q_chunk_id) & is_causal | |
| valid_sliding_q_text = is_sliding_chunk & (is_kv_summary ^ 1) & (kv_idx_ssa >= prefix_summary_length) | |
| valid_kv_q_text = (is_q_summary ^ 1) & is_causal & valid_sliding_q_text | |
| return valid_kv_q_summary | valid_kv_q_text | |