Download tools/export_model.py from maddiedreese/lamplight: direct link, hf CLI and curl.
- Browser
- Download file 5.67 kB
-
https://huggingface.co/maddiedreese/lamplight/resolve/main/tools/export_model.py
- Command line
-
hf download hf://maddiedreese/lamplight/tools/export_model.py
-
curl -L -o export_model.py https://huggingface.co/maddiedreese/lamplight/resolve/main/tools/export_model.py
5.67 kB
| """Pin/convert SmolLM2 for pure Luau inference. Never runs in the shipped game.""" | |
| from pathlib import Path | |
| import json, base64, hashlib, unicodedata, argparse, xml.etree.ElementTree as ET | |
| import numpy as np | |
| from safetensors.torch import load_file | |
| import torch | |
| ROOT=Path(__file__).resolve().parents[1] | |
| parser=argparse.ArgumentParser();parser.add_argument('--model-dir',type=Path,default=ROOT/'model-source');args=parser.parse_args() | |
| SOURCE=args.model_dir | |
| OUT=ROOT/'assets/model'; OUT.mkdir(parents=True,exist_ok=True) | |
| weights=load_file(str(SOURCE/'model.safetensors')) | |
| xml=ET.Element('roblox',version='4'); root=ET.SubElement(xml,'Item',{'class':'Folder','referent':'ModelData'}) | |
| def props(item,name): | |
| p=ET.SubElement(item,'Properties');ET.SubElement(p,'string',name='Name').text=name;return p | |
| props(root,'ModelData') | |
| def value(parent,name,data): | |
| item=ET.SubElement(parent,'Item',{'class':'StringValue'});p=props(item,name);ET.SubElement(p,'string',name='Value').text=data | |
| manifest={'revision':json.loads((SOURCE/'revision.json').read_text()),'tensors':{},'format':'group64-symmetric-int4-v1','weightBytes':0} | |
| qweights={} | |
| for name,t in weights.items(): | |
| a=t.float().numpy(); shape=list(a.shape) | |
| if a.ndim==2: | |
| grouped=a.reshape(a.shape[0],-1,64) | |
| scale=np.maximum(np.max(np.abs(grouped),axis=2)/7,1e-12).astype('<f4') | |
| q=np.clip(np.rint(grouped/scale[:,:,None]),-7,7).astype('i1').reshape(a.shape) | |
| flat=(q.reshape(-1).astype('i2')+8).astype('u1') | |
| packed=flat[::2]|(flat[1::2]<<4) | |
| data=packed.tobytes()+scale.tobytes();kind='q4g64' | |
| qweights[name]=torch.from_numpy((q.reshape(grouped.shape).astype('f4')*scale[:,:,None]).reshape(a.shape)) | |
| else: | |
| data=a.astype('<f4').tobytes();kind='f32';qweights[name]=t.float() | |
| key=name.replace('.','_'); node=ET.SubElement(root,'Item',{'class':'Folder'});props(node,key) | |
| for i,off in enumerate(range(0,len(data),196608)): | |
| value(node,f'{i:04}',base64.b64encode(data[off:off+196608]).decode()) | |
| manifest['tensors'][name]={'key':key,'shape':shape,'kind':kind,'bytes':len(data),'sha256':hashlib.sha256(data).hexdigest()} | |
| manifest['weightBytes']+=len(data) | |
| print(name,shape,len(data),flush=True) | |
| value(root,'Manifest',json.dumps(manifest,separators=(',',':'))) | |
| ET.ElementTree(xml).write(OUT/'ModelData.rbxmx',encoding='utf-8',xml_declaration=True) | |
| (OUT/'manifest.json').write_text(json.dumps(manifest,indent=2)) | |
| # Byte-level BPE token strings converted to raw bytes; pair merge ranks remain exact. | |
| tok=json.loads((SOURCE/'tokenizer.json').read_text()) | |
| bs=list(range(33,127))+list(range(161,173))+list(range(174,256));cs=bs[:];n=0 | |
| for b in range(256): | |
| if b not in bs:bs.append(b);cs.append(256+n);n+=1 | |
| rev={chr(c):b for b,c in zip(bs,cs)} | |
| def raw(s):return bytes(rev[c] for c in s) | |
| vocab=tok['model']['vocab']; byid=['']*len(vocab) | |
| for s,i in vocab.items():byid[i]=base64.b64encode(raw(s)).decode() if i>=17 else base64.b64encode(s.encode()).decode() | |
| merges=[] | |
| for rank,m in enumerate(tok['model']['merges']): | |
| a,b=m.split(' ') if isinstance(m,str) else m | |
| merges.append([vocab[a],vocab[b],vocab[a+b]]) | |
| ranges=[] | |
| start=last=kind=None | |
| for cp in range(0x110000): | |
| ch=chr(cp);cat=unicodedata.category(ch);k=1 if cat.startswith('L') else 2 if cat.startswith('N') else 3 if ch.isspace() else 0 | |
| if k==kind and cp==last+1:last=cp | |
| else: | |
| if kind:ranges.append([start,last,kind]) | |
| start=last=cp;kind=k | |
| if kind:ranges.append([start,last,kind]) | |
| data={'tokens':byid,'merges':merges,'classes':ranges,'specials':{x['content']:x['id'] for x in tok['added_tokens']}} | |
| # JSON in StringValue avoids enormous Luau literal compilation. | |
| r=ET.Element('roblox',version='4');item=ET.SubElement(r,'Item',{'class':'StringValue'});p=props(item,'TokenizerData');ET.SubElement(p,'string',name='Value').text=json.dumps(data,separators=(',',':')) | |
| ET.ElementTree(r).write(OUT/'TokenizerData.rbxmx',encoding='utf-8',xml_declaration=True) | |
| from transformers import AutoModelForCausalLM,AutoTokenizer | |
| model=AutoModelForCausalLM.from_pretrained(str(SOURCE),dtype=torch.float32,local_files_only=True).eval() | |
| model.load_state_dict(qweights,strict=False);model.tie_weights() | |
| ht=AutoTokenizer.from_pretrained(str(SOURCE),local_files_only=True) | |
| texts=['Hello','Hello, world! 123 45.6','café — 日本語\n\n hello ','I\'m here. We\'ll test it!\tOK',' 42 apples\n','<|im_start|>user\nHi!<|im_end|>\n<|im_start|>assistant\n'] | |
| fixtures={'tokenizer':[{'text':s,'ids':ht.encode(s,add_special_tokens=False)} for s in texts],'forward':[]} | |
| with torch.no_grad(): | |
| for s in ['Hello','The capital of France is','<|im_start|>user\nHi!<|im_end|>\n<|im_start|>assistant\n']: | |
| ids=ht.encode(s,add_special_tokens=False);o=model(torch.tensor([ids]));vals,ix=torch.topk(o.logits[0,-1],10) | |
| fixtures['forward'].append({'text':s,'ids':ids,'topIds':ix.tolist(),'topLogits':vals.tolist()}) | |
| messages=[{'role':'system','content':'You are a helpful AI assistant named SmolLM, trained by Hugging Face. Keep replies short.'},{'role':'user','content':'Say hello in one short sentence.'}] | |
| ids=ht.apply_chat_template(messages,add_generation_prompt=True,return_tensors='pt') | |
| if not isinstance(ids,torch.Tensor):ids=ids['input_ids'] | |
| out=model.generate(ids,max_new_tokens=32,do_sample=False) | |
| fixtures['referenceChat']=ht.decode(out[0,ids.shape[-1]:]); print('REFERENCE:',fixtures['referenceChat']) | |
| (ROOT/'tests/fixtures.json').write_text(json.dumps(fixtures,ensure_ascii=False,indent=2)) | |
| print('EXPORTED',manifest['weightBytes'],flush=True) | |
| project=json.loads((ROOT/'test.project.json').read_text());project['tree']['ServerStorage']['Fixtures']['$properties']['Value']=json.dumps(fixtures,ensure_ascii=False);(ROOT/'test.project.json').write_text(json.dumps(project,indent=2)) | |