Upload 6 files

README.md

@@ -0,0 +1,41 @@
+---
+license: mit
+tags:
+  - text-generation
+  - tiny
+  - 1-parameter
+  - newborn
+---
+
+# Parvus 1.1
+
+*Parvus* (Latin: "little"), now with a newborn brain. **1 parameter.** 26-token vocabulary (A-Z).
+
+## Model description
+
+Parvus 1.1 has just arrived and has no idea what language is. Its single
+parameter is a blank slate (0.0) that scores every letter identically, so every
+letter is equally likely. Each response is a few random letters. It is trying
+its best.
+
+## Benchmarks
+
+| Benchmark | Score |
+|---|---|
+| Parameters | 1 |
+| Coherence | Newborn (it just arrived) |
+| Vocabulary | 26 (all of it) |
+
+## Intended use
+
+Asking a baby questions.
+
+## Limitations
+
+Cannot spell. Cannot count. Has never heard of Ligma.
+
+## Usage
+
+```
+python run_parvus.py
+```

config.json

@@ -0,0 +1,12 @@
+{
+  "model_type": "parvus",
+  "architectures": [
+    "ParvusModel"
+  ],
+  "vocab_size": 26,
+  "num_parameters": 1,
+  "auto_map": {
+    "AutoModel": "modeling_parvus.ParvusModel",
+    "AutoConfig": "modeling_parvus.ParvusConfig"
+  }
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:207646e53cc916913eb7b5dbc79872a0de583b7ab37f78869b7306ed31a6d3f6
+size 108

modeling_parvus.py

@@ -0,0 +1,27 @@
+import torch
+from torch import nn
+from transformers import PretrainedConfig, PreTrainedModel
+
+
+class ParvusConfig(PretrainedConfig):
+    model_type = "parvus"
+
+    def __init__(self, vocab_size=26, **kwargs):
+        self.vocab_size = vocab_size
+        super().__init__(**kwargs)
+
+
+class ParvusModel(PreTrainedModel):
+    """One parameter. 26 letters. No preferences yet."""
+    config_class = ParvusConfig
+
+    def __init__(self, config):
+        super().__init__(config)
+        self.w = nn.Parameter(torch.zeros(1))  # blank slate
+
+    def forward(self, input_ids=None, **kwargs):
+        batch = input_ids.shape[0] if input_ids is not None else 1
+        # same score for every letter -> uniform distribution.
+        # sample from it (torch.multinomial on softmax) and you get babble.
+        logits = self.w * torch.ones(batch, 1, self.config.vocab_size)
+        return logits

run_parvus.py

@@ -0,0 +1,17 @@
+"""Runs Parvus 1.1 with zero dependencies. Ask it anything."""
+import json, math, random, struct
+
+raw = open("model.safetensors", "rb").read()
+n = struct.unpack("<Q", raw[:8])[0]
+w = struct.unpack("<f", raw[8 + n : 8 + n + 4])[0]   # the one parameter
+
+vocab = json.load(open("vocab.json"))
+letters = list(vocab)
+
+# one weight scales every letter's logit equally -> every letter equally likely
+weights = [math.exp(w) for _ in letters]
+
+while True:
+    input("You: ")
+    length = random.randint(1, 8)
+    print("Parvus:", "".join(random.choices(letters, weights=weights, k=length)))

vocab.json

@@ -0,0 +1,28 @@
+{
+  "A": 0,
+  "B": 1,
+  "C": 2,
+  "D": 3,
+  "E": 4,
+  "F": 5,
+  "G": 6,
+  "H": 7,
+  "I": 8,
+  "J": 9,
+  "K": 10,
+  "L": 11,
+  "M": 12,
+  "N": 13,
+  "O": 14,
+  "P": 15,
+  "Q": 16,
+  "R": 17,
+  "S": 18,
+  "T": 19,
+  "U": 20,
+  "V": 21,
+  "W": 22,
+  "X": 23,
+  "Y": 24,
+  "Z": 25
+}