Upload folder using huggingface_hub

app.py

@@ -12,7 +12,7 @@ from src.task.inference.classify import TextClassifier
 
 # Paths
 # DATA_PATH = Path("data/tiny_shakespear.txt")
-GENERATOR_MODEL_PATH = Path("checkpoints/mh_tiny_text_generator.pt")
+GENERATOR_MODEL_PATH = Path("checkpoints/tiny_poet.pt")
 
 CLASSIFIER_MODEL_PATH = Path(
     "checkpoints/mh_tiny_text_classifier.pt"
@@ -22,31 +22,22 @@ CLASSIFIER_TOKENIZER_PATH = Path(
     "checkpoints/ag_news_tokenizer.json"
     )
 
-# Model config
-config = ModelConfig()
-config.number_of_heads = 2
-
-# Tokenizer: in future use a reusable vocab instead of loading the entire corpus
-# text = DATA_PATH.read_text(encoding="utf-8",)
-# tokenizer = CharacterTokenizer(text)
 
 TEXT_TOKENIZER_PATH = Path("checkpoints/shakespeare_tokenizer.json")
 tokenizer = CharacterTokenizer.load(TEXT_TOKENIZER_PATH)
 
-# Recreate the generator model
-config.number_of_heads = 2
+generator_config = ModelConfig(context_length=128, d_model=128, d_ff=512, num_layers=4, number_of_heads=4)
+
 model = load_model(
     checkpoint_path=GENERATOR_MODEL_PATH,
-    config=config,
+    config=generator_config,
     vocab_size=tokenizer.vocab_size,
     )
 
-
-# A. Generator
 text_generator = TextGenerator(
     model=model,
     tokenizer=tokenizer,
-    context_length=config.context_length,
+    context_length=generator_config.context_length,
     )
 
 # Text Generation
@@ -68,18 +59,18 @@ def generate_text(
 # B. Classifier
 classifier_tokenizer = CharacterTokenizer.load(CLASSIFIER_TOKENIZER_PATH)
 
-config.context_length = 384
-config.number_of_heads = 2
+classifier_config = ModelConfig(context_length=384, d_model=64, d_ff=256, num_layers=2)
+
 classifier_model = load_classifier_model(
     checkpoint_path=CLASSIFIER_MODEL_PATH,
-    config=config,
+    config=classifier_config,
     vocab_size=classifier_tokenizer.vocab_size,
     )
 
 text_classifier = TextClassifier(
     model=classifier_model,
     tokenizer=classifier_tokenizer,
-    context_length=config.context_length,
+    context_length=classifier_config.context_length,
     )
 
 CLASS_NAMES = [

checkpoints/tiny_poet.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ae86a4a481806e36be89c8fb79ec7d879a822673f07926eb6f80cf40b211b597
+size 3389619

notebooks/.ipynb_checkpoints/Untitled-checkpoint.ipynb

@@ -0,0 +1,6 @@
+{
+ "cells": [],
+ "metadata": {},
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

notebooks/checkpoints/poet_take1.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a46f5ae3aa75742aa57608d71df5756e9b129d2940e650c45092f1447df5fd7a
+size 470389

pyproject.toml

@@ -3,8 +3,16 @@ name = "tiny-text-transformer"
 version = "0.1.0"
 requires-python = ">=3.14"
 dependencies = [
+    "datasets>=5.0.1",
     "gradio>=6.26.0",
+    "matplotlib>=3.11.2",
     "numpy>=2.5.2",
     "spaces>=0.51.2",
     "torch>=2.13.0",
 ]
+
+[dependency-groups]
+dev = [
+    "ipykernel>=7.3.0",
+    "jupyterlab>=4.6.3",
+]

scripts/.ipynb_checkpoints/train_classifier-checkpoint.py

@@ -0,0 +1,140 @@
+from pathlib import Path
+
+import torch
+
+from src.infra.dataset.classification import TextClassificationDataset
+from src.infra.dataset.util import split_tokens, get_batch, build_label, build_text
+from src.infra.tokenizer import CharacterTokenizer
+from src.infra.dataset.autoregressive import AutoRegressiveDataset
+
+from src.task.config import ModelConfig,TrainingConfig
+
+from src.task.model import TinyTransformerLM, TinyClassifier
+from src.task.training.classification import train
+import pandas as pd
+import numpy as np
+from src.task.training.util import TrainingLogger, count_parameters, separator
+
+
+
+# Custom Logger
+logger = TrainingLogger(
+    "logs/text_classifier.log"
+    )
+
+
+# Configs
+separator("Configuration")
+
+torch.manual_seed(42)
+# Configs
+model_config = ModelConfig()
+training_config = TrainingConfig()
+
+# Overrides
+model_config.context_length = 384
+model_config.number_of_heads = 2
+
+training_config.num_steps = 10 
+training_config.eval_interval = 1
+training_config.num_train_samples = None
+
+logger.log(model_config)
+logger.log(training_config)
+
+# Load the corpus
+separator("Corpus")
+
+TRAIN_PATH = Path("data/ag-news/train.csv")
+TEST_PATH = Path("data/ag-news/test.csv")
+checkpoint = "mh_tiny_text_classifier.pt"
+
+if not TRAIN_PATH.exists():
+    raise FileNotFoundError(f"Training file not found: {TRAIN_PATH}")
+
+if not TEST_PATH.exists():
+    raise FileNotFoundError(f"Test file not found: {TEST_PATH}")
+
+separator("Load corpus")
+
+train_corpus = pd.read_csv(TRAIN_PATH)
+if training_config.num_train_samples is not None:
+    train_corpus = train_corpus.head(training_config.num_train_samples)
+
+test_corpus = pd.read_csv(TEST_PATH)
+
+train_text = build_text(train_corpus)
+test_text = build_text(test_corpus)
+
+train_labels = build_label(train_corpus)
+test_labels = build_label(test_corpus)
+
+
+# Tokens
+separator("Tokenization")
+
+tokenizer = CharacterTokenizer("".join(train_text))
+
+CLASS_TOKENIZER_PATH = Path("checkpoints/ag_news_tokenizer.json")
+tokenizer.save(CLASS_TOKENIZER_PATH)
+
+vocab_size = tokenizer.vocab_size
+print(f"Vocab size: {vocab_size}")
+
+# Datasets
+separator("Datasets")
+
+train_dataset = TextClassificationDataset(
+    texts=train_text,
+    labels=train_labels,
+    tokenizer=tokenizer,
+    max_length=model_config.context_length
+    )
+
+val_dataset = TextClassificationDataset(
+    texts=test_text,
+    labels=test_labels,
+    tokenizer=tokenizer,
+    max_length=model_config.context_length
+    )
+
+print(f"Train examples: {len(train_dataset):,}")
+
+# Models
+separator("Transformer")
+
+model = TinyClassifier(
+    config=model_config,
+    vocab_size=tokenizer.vocab_size,
+    num_classes=4,
+    )
+
+print(model)
+
+
+# Train........
+separator("Training")
+
+train(
+    model=model,
+    train_dataset=train_dataset,
+    val_dataset=val_dataset,
+    training_config=training_config,
+    )
+
+
+separator("Saving")
+
+checkpoint_dir = Path("checkpoints")
+checkpoint_dir.mkdir(parents=True, exist_ok=True)
+
+checkpoint_path = (
+    checkpoint_dir / checkpoint
+)
+
+torch.save(
+    model.state_dict(),
+    checkpoint_path,
+    )
+
+print(f"Checkpoint saved to: {checkpoint_path}")

scripts/generate_text.py

@@ -8,7 +8,7 @@ from src.task.inference.inference import TextGenerator
 
 # Runtime configs
 
-checkpoint = "checkpoints/mh_tiny_text_generator.pt"
+checkpoint = "checkpoints/tiny_poet.pt"
 
 TEXT_TOKENIZER_PATH = Path("checkpoints/shakespeare_tokenizer.json")
 tokenizer = CharacterTokenizer.load(
@@ -17,14 +17,13 @@ tokenizer = CharacterTokenizer.load(
 
 # Conf
 
-config = ModelConfig()
-config.number_of_heads = 2
+generator_config = ModelConfig(context_length=128, d_model=128, d_ff=512, num_layers=4, number_of_heads=4)
 
 
 # Load model/checkpoint from file 
 model = load_model(
     checkpoint_path=checkpoint,
-    config=config,
+    config=generator_config,
     vocab_size=tokenizer.vocab_size,
 )
 
@@ -33,7 +32,7 @@ model = load_model(
 text_generator = TextGenerator(
     model=model,
     tokenizer=tokenizer,
-    context_length=config.context_length,
+    context_length=generator_config.context_length,
 )
 
 

scripts/train_generator.py

@@ -29,10 +29,10 @@ separator("Configuration")
 torch.manual_seed(42)
 
 # Configs
-model_config = ModelConfig()
+model_config = ModelConfig(context_length=128, d_model=128, d_ff=512, num_layers=4, number_of_heads=4)
+
 training_config = TrainingConfig()
 # Overrides
-model_config.number_of_heads = 2
 training_config.num_steps = 10000 
 training_config.eval_interval = 1000
 
@@ -119,14 +119,20 @@ logger.log(model)
 # Train........
 separator("Training")
 
-train(
+# train(
+#     model=model,
+#     train_dataset=train_dataset,
+#     val_dataset=val_dataset,
+#     training_config=training_config,
+#     )
+
+model, steps, train_history, val_history = train(
     model=model,
     train_dataset=train_dataset,
     val_dataset=val_dataset,
     training_config=training_config,
     )
 
-
 # Final test evaluation
 separator("Test Evaluation")
 

src/infra/dataset/util.py

@@ -10,7 +10,7 @@ def split_tokens(
     tokens: torch.Tensor,
     train_ratio: float = 0.90,
     val_ratio: float = 0.05,
-) -> tuple[
+    ) -> tuple[
     torch.Tensor,
     torch.Tensor,
     torch.Tensor,

src/infra/tokenizer.py

@@ -50,7 +50,15 @@ class CharacterTokenizer:
             for token_id in token_ids
             if token_id != self.pad_token_id
         )
-
+    
+    #do not assume tokenids are python integerser
+    # def decode(self, token_ids):
+    #     return "".join(
+    #         self.itos[int(token_id)]
+    #         for token_id in token_ids
+    #         if int(token_id) != self.pad_token_id
+    #     )
+    
     def pad(
             self,
             sequences,

src/task/config.py

@@ -8,7 +8,7 @@ class ModelConfig:
     d_model: int = 64       #model dim
     d_ff: int = 256         #feed forward dim
     num_layers: int = 2     #transformer blocks
-    number_of_heads: int = 1
+    number_of_heads: int = 2
     dropout: float = 0.1    #percent. avoid overfitting, break emerging biases
 
 

src/task/training/autoregressive.py

@@ -21,6 +21,10 @@ def train(
         lr=training_config.learning_rate,
         )
 
+    train_history = []
+    val_history = []
+    steps = []
+
     model.train()
 
     for step in range(training_config.num_steps):
@@ -39,6 +43,10 @@ def train(
                 training_config=training_config,
                 )
 
+            steps.append(step)
+            train_history.append(train_loss)
+            val_history.append(val_loss)
+
             print(
                 f"step {step:4d} | "
                 f"train {train_loss:.4f} | "
@@ -61,7 +69,8 @@ def train(
         loss.backward()
         optimizer.step()
 
-    return model
+    #return model
+    return model, steps, train_history, val_history
 
 
 @torch.no_grad()    #do not calculate gradients