"""Pin/convert SmolLM2 for pure Luau inference. Never runs in the shipped game.""" from pathlib import Path import json, base64, hashlib, unicodedata, argparse, xml.etree.ElementTree as ET import numpy as np from safetensors.torch import load_file import torch ROOT=Path(__file__).resolve().parents[1] parser=argparse.ArgumentParser();parser.add_argument('--model-dir',type=Path,default=ROOT/'model-source');args=parser.parse_args() SOURCE=args.model_dir OUT=ROOT/'assets/model'; OUT.mkdir(parents=True,exist_ok=True) weights=load_file(str(SOURCE/'model.safetensors')) xml=ET.Element('roblox',version='4'); root=ET.SubElement(xml,'Item',{'class':'Folder','referent':'ModelData'}) def props(item,name): p=ET.SubElement(item,'Properties');ET.SubElement(p,'string',name='Name').text=name;return p props(root,'ModelData') def value(parent,name,data): item=ET.SubElement(parent,'Item',{'class':'StringValue'});p=props(item,name);ET.SubElement(p,'string',name='Value').text=data manifest={'revision':json.loads((SOURCE/'revision.json').read_text()),'tensors':{},'format':'group64-symmetric-int4-v1','weightBytes':0} qweights={} for name,t in weights.items(): a=t.float().numpy(); shape=list(a.shape) if a.ndim==2: grouped=a.reshape(a.shape[0],-1,64) scale=np.maximum(np.max(np.abs(grouped),axis=2)/7,1e-12).astype('=17 else base64.b64encode(s.encode()).decode() merges=[] for rank,m in enumerate(tok['model']['merges']): a,b=m.split(' ') if isinstance(m,str) else m merges.append([vocab[a],vocab[b],vocab[a+b]]) ranges=[] start=last=kind=None for cp in range(0x110000): ch=chr(cp);cat=unicodedata.category(ch);k=1 if cat.startswith('L') else 2 if cat.startswith('N') else 3 if ch.isspace() else 0 if k==kind and cp==last+1:last=cp else: if kind:ranges.append([start,last,kind]) start=last=cp;kind=k if kind:ranges.append([start,last,kind]) data={'tokens':byid,'merges':merges,'classes':ranges,'specials':{x['content']:x['id'] for x in tok['added_tokens']}} # JSON in StringValue avoids enormous Luau literal compilation. r=ET.Element('roblox',version='4');item=ET.SubElement(r,'Item',{'class':'StringValue'});p=props(item,'TokenizerData');ET.SubElement(p,'string',name='Value').text=json.dumps(data,separators=(',',':')) ET.ElementTree(r).write(OUT/'TokenizerData.rbxmx',encoding='utf-8',xml_declaration=True) from transformers import AutoModelForCausalLM,AutoTokenizer model=AutoModelForCausalLM.from_pretrained(str(SOURCE),dtype=torch.float32,local_files_only=True).eval() model.load_state_dict(qweights,strict=False);model.tie_weights() ht=AutoTokenizer.from_pretrained(str(SOURCE),local_files_only=True) texts=['Hello','Hello, world! 123 45.6','café — 日本語\n\n hello ','I\'m here. We\'ll test it!\tOK',' 42 apples\n','<|im_start|>user\nHi!<|im_end|>\n<|im_start|>assistant\n'] fixtures={'tokenizer':[{'text':s,'ids':ht.encode(s,add_special_tokens=False)} for s in texts],'forward':[]} with torch.no_grad(): for s in ['Hello','The capital of France is','<|im_start|>user\nHi!<|im_end|>\n<|im_start|>assistant\n']: ids=ht.encode(s,add_special_tokens=False);o=model(torch.tensor([ids]));vals,ix=torch.topk(o.logits[0,-1],10) fixtures['forward'].append({'text':s,'ids':ids,'topIds':ix.tolist(),'topLogits':vals.tolist()}) messages=[{'role':'system','content':'You are a helpful AI assistant named SmolLM, trained by Hugging Face. Keep replies short.'},{'role':'user','content':'Say hello in one short sentence.'}] ids=ht.apply_chat_template(messages,add_generation_prompt=True,return_tensors='pt') if not isinstance(ids,torch.Tensor):ids=ids['input_ids'] out=model.generate(ids,max_new_tokens=32,do_sample=False) fixtures['referenceChat']=ht.decode(out[0,ids.shape[-1]:]); print('REFERENCE:',fixtures['referenceChat']) (ROOT/'tests/fixtures.json').write_text(json.dumps(fixtures,ensure_ascii=False,indent=2)) print('EXPORTED',manifest['weightBytes'],flush=True) project=json.loads((ROOT/'test.project.json').read_text());project['tree']['ServerStorage']['Fixtures']['$properties']['Value']=json.dumps(fixtures,ensure_ascii=False);(ROOT/'test.project.json').write_text(json.dumps(project,indent=2))