File size: 1,481 Bytes
9b51f42 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 | """FlyGPT configuration for Hugging Face `transformers` (loaded with trust_remote_code=True)."""
from transformers import PretrainedConfig
class FlyGPTConfig(PretrainedConfig):
model_type = "flygpt"
def __init__(
self,
vocab_size: int = 65,
num_neurons: int = 0,
num_edges: int = 0,
embedding_dim: int = 32,
num_input_neurons: int = 256,
num_output_neurons: int = 512,
microsteps: int = 2,
activation: str = "tanh",
learned_leak: bool = True,
leak_init: float = 0.5,
degree_normalization: bool = True,
init_scale: float = 0.1,
graph: dict | None = None,
training_state: dict | None = None,
**kwargs,
):
self.vocab_size = vocab_size
self.num_neurons = num_neurons
self.num_edges = num_edges
self.embedding_dim = embedding_dim
self.num_input_neurons = num_input_neurons
self.num_output_neurons = num_output_neurons
self.microsteps = microsteps
self.activation = activation
self.learned_leak = learned_leak
self.leak_init = leak_init
self.degree_normalization = degree_normalization
self.init_scale = init_scale
self.graph = graph or {} # source, hash, region filter, stats, provenance
self.training_state = training_state or {} # "init" or the run that produced the values
super().__init__(**kwargs)
|