j0no12 commited on
Commit
cba079d
·
verified ·
1 Parent(s): ecf4cf9

Update Sol architecture implementation

Browse files
Files changed (1) hide show
  1. modeling_sol_lite.py +4 -4
modeling_sol_lite.py CHANGED
@@ -199,7 +199,7 @@ class ReleasedSolLiteControl(nn.Module):
199
  return F.linear(self.norm(x), self.embedding.weight)
200
 
201
 
202
- class SolLiteForCausalLM(nn.Module):
203
  """XSA + selective recurrence + loop conditioning + EngramLite."""
204
 
205
  def __init__(self, cfg: SolLiteConfig = SolLiteConfig()):
@@ -257,7 +257,7 @@ def variant_config(name: str) -> SolLiteConfig:
257
  def build_variant(name: str) -> nn.Module:
258
  if name == "released_control":
259
  return ReleasedSolLiteControl()
260
- return SolLiteForCausalLM(variant_config(name))
261
 
262
 
263
  def architecture_audit() -> dict[str, int | float | bool]:
@@ -290,7 +290,7 @@ def load_model(model_dir: str | Path, device: str | torch.device = "cpu"):
290
  raw = json.loads((model_dir / "config.json").read_text())
291
  fields = SolLiteConfig.__dataclass_fields__
292
  cfg = SolLiteConfig(**{key: value for key, value in raw.items() if key in fields})
293
- model = SolLiteForCausalLM(cfg)
294
  model.load_state_dict(load_file(model_dir / "model.safetensors"), strict=True)
295
  model.to(device).eval()
296
  tokenizer = AutoTokenizer.from_pretrained(model_dir)
@@ -299,7 +299,7 @@ def load_model(model_dir: str | Path, device: str | torch.device = "cpu"):
299
 
300
  @torch.inference_mode()
301
  def generate(
302
- model: SolLiteForCausalLM,
303
  tokenizer,
304
  prompt: str,
305
  max_new_tokens: int = 64,
 
199
  return F.linear(self.norm(x), self.embedding.weight)
200
 
201
 
202
+ class SolForCausalLM(nn.Module):
203
  """XSA + selective recurrence + loop conditioning + EngramLite."""
204
 
205
  def __init__(self, cfg: SolLiteConfig = SolLiteConfig()):
 
257
  def build_variant(name: str) -> nn.Module:
258
  if name == "released_control":
259
  return ReleasedSolLiteControl()
260
+ return SolForCausalLM(variant_config(name))
261
 
262
 
263
  def architecture_audit() -> dict[str, int | float | bool]:
 
290
  raw = json.loads((model_dir / "config.json").read_text())
291
  fields = SolLiteConfig.__dataclass_fields__
292
  cfg = SolLiteConfig(**{key: value for key, value in raw.items() if key in fields})
293
+ model = SolForCausalLM(cfg)
294
  model.load_state_dict(load_file(model_dir / "model.safetensors"), strict=True)
295
  model.to(device).eval()
296
  tokenizer = AutoTokenizer.from_pretrained(model_dir)
 
299
 
300
  @torch.inference_mode()
301
  def generate(
302
+ model: SolForCausalLM,
303
  tokenizer,
304
  prompt: str,
305
  max_new_tokens: int = 64,