File size: 2,241 Bytes
28f4123 a03c99b 28f4123 a03c99b 28f4123 a03c99b 28f4123 a03c99b 28f4123 a03c99b 28f4123 a03c99b 28f4123 a03c99b 28f4123 a03c99b | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 | import numpy as np
# SpTn-Model-only-1P
# Exactly ONE trainable parameter.
MODEL_FILE = "sp_tn_1p.npz"
# Fixed training corpus.
# These are data, not trainable parameters.
text = """
hello world
this is a tiny experimental model
sp tn one parameter
research experiment
hello model
"""
# Fixed vocabulary.
# These values are NOT trainable parameters.
vocab = sorted(set(text))
char_to_id = {c: i for i, c in enumerate(vocab)}
# Training examples: previous character -> next character.
pairs = []
for i in range(len(text) - 1):
x = text[i]
y = text[i + 1]
pairs.append((char_to_id[x], char_to_id[y]))
# Exactly ONE trainable parameter.
w = np.array(0.5, dtype=np.float32)
learning_rate = 0.05
steps = 5000
def logits(parameter, input_id):
"""
Fixed feature function.
The only trainable value is `parameter`.
"""
n = len(vocab)
ids = np.arange(n, dtype=np.float32)
# Fixed features.
# No trainable weights here.
feature = np.sin((ids + 1.0) * (input_id + 1.0))
return parameter * feature
def softmax(values):
values = values - np.max(values)
exp_values = np.exp(values)
return exp_values / np.sum(exp_values)
for step in range(steps):
total_loss = 0.0
total_gradient = 0.0
for input_id, target_id in pairs:
z = logits(w, input_id)
probabilities = softmax(z)
loss = -np.log(probabilities[target_id] + 1e-12)
total_loss += loss
ids = np.arange(len(vocab), dtype=np.float32)
feature = np.sin((ids + 1.0) * (input_id + 1.0))
expected_feature = np.sum(probabilities * feature)
gradient = expected_feature - feature[target_id]
total_gradient += gradient
total_gradient /= len(pairs)
w -= learning_rate * total_gradient
if step % 500 == 0:
average_loss = total_loss / len(pairs)
print(
f"step={step} "
f"loss={average_loss:.6f} "
f"w={float(w):.6f}"
)
np.savez(
MODEL_FILE,
w=w,
vocab=np.array(vocab),
)
print()
print("SpTn-Model-only-1P")
print("Model saved:", MODEL_FILE)
print("Trainable parameters:", 1)
print("Parameter:", float(w))
print("Final loss:", float(total_loss / len(pairs)))
|