Download llm2asm/tokenizer.lisp from Snapkitty/nekomata: direct link, hf CLI and curl.
- Browser
- Download file 2.18 kB
-
https://huggingface.co/Snapkitty/nekomata/resolve/main/llm2asm/tokenizer.lisp
- Command line
-
hf download hf://Snapkitty/nekomata/llm2asm/tokenizer.lisp
-
curl -L -o tokenizer.lisp https://huggingface.co/Snapkitty/nekomata/resolve/main/llm2asm/tokenizer.lisp
2.18 kB
| (defpackage #:nekod.llm2asm | |
| (:use #:cl) | |
| (:export | |
| #:tokenize-intent | |
| #:intent-token | |
| #:token-id | |
| #:token-class | |
| #:intent-token-text | |
| #:intent-token-class | |
| #:intent-token-bytes | |
| #:lookup-kernel | |
| #:compile-intent | |
| #:safe-compile-intent | |
| #:infra-ast | |
| #:infra-ast-type | |
| #:infra-ast-target | |
| #:infra-ast-constraint | |
| #:infra-ast-priority)) | |
| (in-package #:nekod.llm2asm) | |
| ;; Byte-pair encoding for x86-64 / ARM64 opcodes | |
| ;; The kernel map from sovereign-shadow-compiler feeds this | |
| ;; Known x86-64 primitive opcodes (from sovereign-shadow-compiler KERNEL_MAP) | |
| (defvar *x86-kernel-map* | |
| '(("ADD" . #(#x48 #x89 #xf8 #x48 #x01 #xf0 #xc3)) | |
| ("MUL" . #(#x48 #x89 #xf8 #x48 #x0f #xaf #xc6 #xc3)) | |
| ("XOR" . #(#x48 #x89 #xf8 #x48 #x31 #xf0 #xc3)) | |
| ("LOOP" . #(#x48 #x31 #xc0 #x48 #xff #xc0 #x48 #x39 #xf8 #x7c #xf8 #xc3)) | |
| ("MEMCPY" . #(#x48 #x89 #xd1 #xf3 #xa4 #xc3)) | |
| ("MEMSET" . #(#x48 #x89 #xd1 #x40 #x88 #xf0 #xf3 #xaa #xc3)) | |
| ("STRCMP" . #(#x48 #x31 #xc0 #x8a #x0f #x8a #x16 #x38 #xd1 #x75 #x0c | |
| #x84 #xc9 #x74 #x0a #x48 #xff #xc7 #x48 #xff #xc6 #xeb | |
| #xee #x48 #x19 #xc0 #x48 #x83 #xc8 #x01 #xc3)))) | |
| (defstruct intent-token | |
| id | |
| text | |
| class ; :op :operand :constraint :intent | |
| bytes ; raw bytes if this is a known kernel op | |
| ) | |
| (defun tokenize-intent (intent-string) | |
| "Tokenize an intent string into intent-token list." | |
| (let* ((words (uiop:split-string (string-upcase intent-string))) | |
| (tokens (loop for word in words | |
| for id from 0 | |
| collect (let ((kernel-bytes (cdr (assoc word *x86-kernel-map* :test #'string=)))) | |
| (make-intent-token | |
| :id id | |
| :text word | |
| :class (if kernel-bytes :op :intent) | |
| :bytes kernel-bytes))))) | |
| tokens)) | |
| (defun lookup-kernel (op-name) | |
| "Look up raw bytes for a known kernel operation." | |
| (cdr (assoc (string-upcase op-name) *x86-kernel-map* :test #'string=))) | |