Text Classification
Transformers
qwen3_5_text
typed-decisions
decision-model
system-one
jev-compatible
custom-code
Instructions to use GyrTechnology/saif-4b with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use GyrTechnology/saif-4b with Transformers:
# Use a pipeline as a high-level helper from transformers import pipeline pipe = pipeline("text-classification", model="GyrTechnology/saif-4b")# pip install -U transformers accelerate # Load model directly from transformers import AutoModel model = AutoModel.from_pretrained("GyrTechnology/saif-4b", device_map="auto") - Notebooks
- Google Colab
- Kaggle
File size: 1,651 Bytes
cc61c41 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 | """Owner-selected bf16 segmented attention and shared-prefix inference."""
import os
import torch
import saif_v2_core as core
from transformers.models.qwen3_5 import modeling_qwen3_5 as modeling
MASK=core.attention_mask
ATTENTION=modeling.ALL_ATTENTION_FUNCTIONS['sdpa']
LAYOUT=None
def mask(layout, attention, device, dtype, boolean=False):
global LAYOUT
assert attention=='bidir'
LAYOUT=layout
return {'full_attention':None,'linear_attention':None}
def segmented(module,query,key,value,attention_mask,dropout=0.,scaling=None,**kwargs):
assert dropout==0 and LAYOUT is not None
prefix=LAYOUT.prefix
assert query.shape[-2]==len(LAYOUT.ids)
if module.num_key_value_groups>1:
key=key.repeat_interleave(module.num_key_value_groups,dim=1)
value=value.repeat_interleave(module.num_key_value_groups,dim=1)
def attend(q,k,v):
return torch.nn.functional.scaled_dot_product_attention(q,k,v,dropout_p=0.,is_causal=False,scale=scaling)
output=[attend(query[:,:,:prefix],key[:,:,:prefix],value[:,:,:prefix])]
for start,end in LAYOUT.blocks:
output.append(attend(query[:,:,start:end],torch.cat((key[:,:,:prefix],key[:,:,start:end]),dim=2),torch.cat((value[:,:,:prefix],value[:,:,start:end]),dim=2)))
return torch.cat(output,dim=2).transpose(1,2).contiguous(),None
def install():
core.attention_mask=mask
modeling.ALL_ATTENTION_FUNCTIONS['sdpa']=segmented
os.environ['SAIF_EXPERIMENTAL_SHARED_DELTA']='1'
def uninstall():
core.attention_mask=MASK
modeling.ALL_ATTENTION_FUNCTIONS['sdpa']=ATTENTION
os.environ.pop('SAIF_EXPERIMENTAL_SHARED_DELTA',None)
|