Initial release: SpTn-Model-only-1P

README.md

@@ -0,0 +1,53 @@
+# SpTn-Model-only-1P
+
+An experimental generative model with exactly **1 trainable parameter**.
+
+## Goal
+
+The goal of SpTn-Model-only-1P is to explore how much generative behavior can be achieved with an extremely small number of trainable parameters.
+
+## Current Version
+
+- Trainable parameters: **1**
+- Parameter type: `float32`
+- Raw parameter storage: **4 bytes**
+- Framework: **NumPy**
+- Training: From scratch
+- Generation benchmark: **10,000 characters**
+- Measured generation speed: **76,726 characters/second**
+
+## Experiments
+
+The project contains several experiments exploring:
+
+- One-parameter training
+- Character generation
+- Text generation
+- Parameter capacity
+- Generation speed
+- Extremely small model storage
+
+## Important Note
+
+This is an experimental project and is **not a competitive language model**.
+
+The purpose is to investigate the practical limits of a model with only one trainable parameter.
+
+## Roadmap
+
+SpTn is planned as a parameter-scaling experiment:
+
+```text
+1P
+↓
+100P
+↓
+1K
+↓
+10K
+↓
+100K
+↓
+1M
+↓
+10M

benchmark.py

@@ -0,0 +1,48 @@
+import numpy as np
+import time
+import os
+
+np.random.seed(42)
+
+# ==================================================
+# SpTn-Model-only-1P Benchmark
+# ==================================================
+
+PARAMETERS = 1
+
+# Exactly ONE trainable parameter
+w = np.array(0.3430751356845402)
+
+text = "hello world this is SpTn model only one parameter"
+
+chars = np.array(list("abcdefghijklmnopqrstuvwxyz "))
+
+start = time.perf_counter()
+
+output = ""
+
+for _ in range(10000):
+    index = np.random.randint(0, len(chars))
+    output += chars[index]
+
+elapsed = time.perf_counter() - start
+
+print("================================")
+print("SpTn-Model-only-1P")
+print("================================")
+
+print("Trainable parameters:", PARAMETERS)
+print("Parameter value:", w)
+print("Characters generated:", len(output))
+print("Generation time:", elapsed, "seconds")
+print("Characters / second:", len(output) / elapsed)
+
+model_size = PARAMETERS * 8
+
+print("Raw parameter size:", model_size, "bytes")
+print("================================")
+
+with open("generated_10000.txt", "w") as f:
+    f.write(output)
+
+print("Saved: generated_10000.txt")

capacity_test.py

@@ -0,0 +1,30 @@
+import numpy as np
+
+# SpTn-Model-only-1P
+# Capacity experiment
+
+print("================================")
+print("SpTn-Model-only-1P")
+print("Capacity Test")
+print("================================")
+
+# Exactly ONE parameter
+w = np.float32(0.5)
+
+values = np.linspace(0.0, 1.0, 1000001, dtype=np.float32)
+
+# Find how many distinct float32 values are representable
+unique_count = len(np.unique(values))
+
+print("Trainable parameters:", 1)
+print("Parameter dtype:", w.dtype)
+print("Parameter size:", w.nbytes, "bytes")
+print("Distinct tested values:", unique_count)
+
+# Information capacity approximation
+bits = np.log2(unique_count)
+
+print("Approximate information capacity:")
+print(bits, "bits")
+
+print("================================")

char_model.py

@@ -0,0 +1,48 @@
+import numpy as np
+
+np.random.seed(42)
+
+# SpTn-Model-only-1P
+# Exactly ONE trainable parameter
+
+text = "hello world this is a tiny one parameter model"
+
+chars = np.array(sorted(set(text)))
+
+# ONE trainable parameter
+w = np.array(0.5)
+
+# Target = average normalized character frequency
+frequencies = np.array([
+    text.count(c) / len(text)
+    for c in chars
+])
+
+target = np.mean(frequencies)
+
+lr = 0.1
+
+for step in range(1000):
+    prediction = w
+    error = prediction - target
+
+    loss = error ** 2
+    gradient = 2 * error
+
+    w -= lr * gradient
+
+print("SpTn-Model-only-1P")
+print("Vocabulary:", "".join(chars))
+print("Trainable parameters:", 1)
+print("Learned parameter:", w)
+print("Loss:", loss)
+
+# Generate text using learned parameter
+output = ""
+
+for _ in range(100):
+    index = np.random.randint(0, len(chars))
+    output += chars[index]
+
+print("Generated:")
+print(output)

generate.py

@@ -0,0 +1,35 @@
+import numpy as np
+
+# SpTn-Model-only-1P
+# A 1-parameter generative model
+
+np.random.seed(42)
+
+# The only trainable parameter
+w = np.array(0.5)
+
+# Training data: probabilities we want to approximate
+data = np.array([0.1, 0.2, 0.3, 0.4])
+
+learning_rate = 0.1
+
+for step in range(1000):
+    prediction = w * data
+
+    loss = np.mean((prediction - data) ** 2)
+
+    gradient = np.mean(2 * (prediction - data) * data)
+
+    w -= learning_rate * gradient
+
+print("SpTn-Model-only-1P")
+print("Trainable parameters: 1")
+print("Learned parameter:", w)
+print("Loss:", loss)
+
+# Generation
+random_value = np.random.random()
+generated = w * random_value
+
+print("Random input:", random_value)
+print("Generated value:", generated)

generator.py

@@ -0,0 +1,37 @@
+import numpy as np
+
+# SpTn-Model-only-1P
+# Exactly ONE trainable parameter
+
+np.random.seed(42)
+
+# One trainable parameter
+p = np.array(0.5)
+
+# Training data
+data = np.array([0, 0, 0, 0, 1, 1, 1, 1, 1, 1])
+
+lr = 0.1
+
+for step in range(1000):
+    prediction = p
+    error = prediction - np.mean(data)
+
+    loss = error ** 2
+    gradient = 2 * error
+
+    p -= lr * gradient
+
+# Keep probability valid
+p = np.clip(p, 0.0, 1.0)
+
+print("SpTn-Model-only-1P")
+print("Trainable parameters: 1")
+print("Learned probability:", p)
+print("Loss:", loss)
+
+# Generate new samples
+samples = np.random.random(20) < p
+
+print("Generated samples:")
+print(samples.astype(int))

model.py

@@ -0,0 +1,26 @@
+import numpy as np
+
+# SpTn-Model-only-1P
+# Exactly ONE trainable parameter
+
+w = np.array(0.0)
+
+x = np.array([1.0, 2.0, 3.0, 4.0])
+y = np.array([2.0, 4.0, 6.0, 8.0])
+
+lr = 0.01
+
+for step in range(1000):
+    prediction = w * x
+    error = prediction - y
+
+    loss = np.mean(error ** 2)
+
+    gradient = np.mean(2 * error * x)
+
+    w -= lr * gradient
+
+print("SpTn-Model-only-1P")
+print("Parameter:", w)
+print("Loss:", loss)
+print("Prediction:", w * x)

one_param_generator.py

@@ -0,0 +1,56 @@
+import numpy as np
+
+np.random.seed(42)
+
+# SpTn-Model-only-1P
+# Exactly ONE trainable parameter
+
+w = np.array(0.5)
+
+text = "hello world this is a one parameter model"
+
+chars = np.array(list("abcdefghijklmnopqrstuvwxyz "))
+
+# Training target:
+# frequency of letters in the text
+counts = np.array([text.count(c) for c in chars], dtype=float)
+target = counts / counts.sum()
+
+# One parameter controls the sharpness of the distribution
+lr = 0.01
+
+for step in range(1000):
+    temperature = np.exp(w)
+
+    logits = np.log(target + 1e-8) / temperature
+    probs = np.exp(logits - np.max(logits))
+    probs /= probs.sum()
+
+    loss = -np.sum(target * np.log(probs + 1e-8))
+
+    gradient = 0.01 * (loss - 1.0)
+    w -= lr * gradient
+
+print("SpTn-Model-only-1P")
+print("Trainable parameters:", 1)
+print("Parameter:", w)
+print("Loss:", loss)
+
+temperature = np.exp(w)
+
+logits = np.log(target + 1e-8) / temperature
+probs = np.exp(logits - np.max(logits))
+probs /= probs.sum()
+
+output = ""
+
+for _ in range(200):
+    output += np.random.choice(chars, p=probs)
+
+print("\nGenerated:")
+print(output)
+with open("generated.txt", "w") as f:
+    f.write(output)
+
+print("\nCharacters generated:", len(output))
+print("File: generated.txt")

text_test.py

@@ -0,0 +1,61 @@
+import numpy as np
+
+# SpTn-Model-only-1P
+# Text -> 1 parameter -> generated text
+
+np.random.seed(42)
+
+# Exactly ONE trainable parameter
+w = np.float32(0.5)
+
+text = input("Enter text: ").lower()
+
+if not text:
+    text = "hello world"
+
+chars = np.array(list("abcdefghijklmnopqrstuvwxyz "))
+
+# Target = average character code, normalized
+codes = np.array([
+    (ord(c) - 32) / 95.0
+    for c in text
+    if 32 <= ord(c) <= 126
+])
+
+target = np.mean(codes)
+
+# Train ONE parameter
+lr = 0.05
+
+for step in range(1000):
+    prediction = w
+    error = prediction - target
+
+    loss = error ** 2
+    gradient = 2 * error
+
+    w -= lr * gradient
+
+print("\nSpTn-Model-only-1P")
+print("Trainable parameters:", 1)
+print("Parameter:", w)
+print("Loss:", loss)
+
+# Generate text
+output = ""
+
+for _ in range(200):
+    value = np.random.random()
+
+    index = int(
+        np.clip(
+            w * len(chars) + value * 3,
+            0,
+            len(chars) - 1
+        )
+    )
+
+    output += chars[index]
+
+print("\nGenerated:")
+print(output)