Update Sol architecture implementation
Browse files- modeling_sol_lite.py +4 -4
modeling_sol_lite.py
CHANGED
|
@@ -199,7 +199,7 @@ class ReleasedSolLiteControl(nn.Module):
|
|
| 199 |
return F.linear(self.norm(x), self.embedding.weight)
|
| 200 |
|
| 201 |
|
| 202 |
-
class
|
| 203 |
"""XSA + selective recurrence + loop conditioning + EngramLite."""
|
| 204 |
|
| 205 |
def __init__(self, cfg: SolLiteConfig = SolLiteConfig()):
|
|
@@ -257,7 +257,7 @@ def variant_config(name: str) -> SolLiteConfig:
|
|
| 257 |
def build_variant(name: str) -> nn.Module:
|
| 258 |
if name == "released_control":
|
| 259 |
return ReleasedSolLiteControl()
|
| 260 |
-
return
|
| 261 |
|
| 262 |
|
| 263 |
def architecture_audit() -> dict[str, int | float | bool]:
|
|
@@ -290,7 +290,7 @@ def load_model(model_dir: str | Path, device: str | torch.device = "cpu"):
|
|
| 290 |
raw = json.loads((model_dir / "config.json").read_text())
|
| 291 |
fields = SolLiteConfig.__dataclass_fields__
|
| 292 |
cfg = SolLiteConfig(**{key: value for key, value in raw.items() if key in fields})
|
| 293 |
-
model =
|
| 294 |
model.load_state_dict(load_file(model_dir / "model.safetensors"), strict=True)
|
| 295 |
model.to(device).eval()
|
| 296 |
tokenizer = AutoTokenizer.from_pretrained(model_dir)
|
|
@@ -299,7 +299,7 @@ def load_model(model_dir: str | Path, device: str | torch.device = "cpu"):
|
|
| 299 |
|
| 300 |
@torch.inference_mode()
|
| 301 |
def generate(
|
| 302 |
-
model:
|
| 303 |
tokenizer,
|
| 304 |
prompt: str,
|
| 305 |
max_new_tokens: int = 64,
|
|
|
|
| 199 |
return F.linear(self.norm(x), self.embedding.weight)
|
| 200 |
|
| 201 |
|
| 202 |
+
class SolForCausalLM(nn.Module):
|
| 203 |
"""XSA + selective recurrence + loop conditioning + EngramLite."""
|
| 204 |
|
| 205 |
def __init__(self, cfg: SolLiteConfig = SolLiteConfig()):
|
|
|
|
| 257 |
def build_variant(name: str) -> nn.Module:
|
| 258 |
if name == "released_control":
|
| 259 |
return ReleasedSolLiteControl()
|
| 260 |
+
return SolForCausalLM(variant_config(name))
|
| 261 |
|
| 262 |
|
| 263 |
def architecture_audit() -> dict[str, int | float | bool]:
|
|
|
|
| 290 |
raw = json.loads((model_dir / "config.json").read_text())
|
| 291 |
fields = SolLiteConfig.__dataclass_fields__
|
| 292 |
cfg = SolLiteConfig(**{key: value for key, value in raw.items() if key in fields})
|
| 293 |
+
model = SolForCausalLM(cfg)
|
| 294 |
model.load_state_dict(load_file(model_dir / "model.safetensors"), strict=True)
|
| 295 |
model.to(device).eval()
|
| 296 |
tokenizer = AutoTokenizer.from_pretrained(model_dir)
|
|
|
|
| 299 |
|
| 300 |
@torch.inference_mode()
|
| 301 |
def generate(
|
| 302 |
+
model: SolForCausalLM,
|
| 303 |
tokenizer,
|
| 304 |
prompt: str,
|
| 305 |
max_new_tokens: int = 64,
|