nekomata / llm2asm /tokenizer.lisp
SNAPKITTYWEST's picture
push from SNAPKITTYWEST/nekomata
3b70664 verified
Raw History Blame Contribute Delete
2.18 kB
(defpackage #:nekod.llm2asm
(:use #:cl)
(:export
#:tokenize-intent
#:intent-token
#:token-id
#:token-class
#:intent-token-text
#:intent-token-class
#:intent-token-bytes
#:lookup-kernel
#:compile-intent
#:safe-compile-intent
#:infra-ast
#:infra-ast-type
#:infra-ast-target
#:infra-ast-constraint
#:infra-ast-priority))
(in-package #:nekod.llm2asm)
;; Byte-pair encoding for x86-64 / ARM64 opcodes
;; The kernel map from sovereign-shadow-compiler feeds this
;; Known x86-64 primitive opcodes (from sovereign-shadow-compiler KERNEL_MAP)
(defvar *x86-kernel-map*
'(("ADD" . #(#x48 #x89 #xf8 #x48 #x01 #xf0 #xc3))
("MUL" . #(#x48 #x89 #xf8 #x48 #x0f #xaf #xc6 #xc3))
("XOR" . #(#x48 #x89 #xf8 #x48 #x31 #xf0 #xc3))
("LOOP" . #(#x48 #x31 #xc0 #x48 #xff #xc0 #x48 #x39 #xf8 #x7c #xf8 #xc3))
("MEMCPY" . #(#x48 #x89 #xd1 #xf3 #xa4 #xc3))
("MEMSET" . #(#x48 #x89 #xd1 #x40 #x88 #xf0 #xf3 #xaa #xc3))
("STRCMP" . #(#x48 #x31 #xc0 #x8a #x0f #x8a #x16 #x38 #xd1 #x75 #x0c
#x84 #xc9 #x74 #x0a #x48 #xff #xc7 #x48 #xff #xc6 #xeb
#xee #x48 #x19 #xc0 #x48 #x83 #xc8 #x01 #xc3))))
(defstruct intent-token
id
text
class ; :op :operand :constraint :intent
bytes ; raw bytes if this is a known kernel op
)
(defun tokenize-intent (intent-string)
"Tokenize an intent string into intent-token list."
(let* ((words (uiop:split-string (string-upcase intent-string)))
(tokens (loop for word in words
for id from 0
collect (let ((kernel-bytes (cdr (assoc word *x86-kernel-map* :test #'string=))))
(make-intent-token
:id id
:text word
:class (if kernel-bytes :op :intent)
:bytes kernel-bytes)))))
tokens))
(defun lookup-kernel (op-name)
"Look up raw bytes for a known kernel operation."
(cdr (assoc (string-upcase op-name) *x86-kernel-map* :test #'string=)))