rsumit1234 commited on
Commit
ea87153
·
verified ·
1 Parent(s): 5b817d6

Describe training by time, not cost

Browse files
Files changed (1) hide show
  1. app.py +3 -3
app.py CHANGED
@@ -97,8 +97,8 @@ def chat_fn(message, history, max_tokens, temperature):
97
 
98
  ABOUT = """
99
  **A 20-layer chat model (~390M transformer + ~500M embedding parameters) trained completely from scratch** — tokenizer, pretraining and chat fine-tuning,
100
- no pretrained weights anywhere — using [karpathy/nanochat](https://github.com/karpathy/nanochat) on a single rented
101
- H100 for about **$13** (3.5B tokens, ~6 h).
102
 
103
  It writes fluent English, holds a conversation, writes poems and explains things simply, but it **confidently makes up
104
  facts** (it is ~200x smaller than GPT-3.5). This free CPU Space is slow: replies stream at a few words per second.
@@ -108,7 +108,7 @@ Weights: [sumitkClasses/nanochat-d20-chat](https://huggingface.co/sumitkClasses/
108
  demo = gr.ChatInterface(
109
  chat_fn,
110
  type="messages",
111
- title="nanochat-d20 · a chat model trained from scratch for $13",
112
  description=ABOUT,
113
  additional_inputs=[
114
  gr.Slider(32, 1024, value=min(DEFAULT_MAX_TOKENS, 1024), step=32, label="Max new tokens"),
 
97
 
98
  ABOUT = """
99
  **A 20-layer chat model (~390M transformer + ~500M embedding parameters) trained completely from scratch** — tokenizer, pretraining and chat fine-tuning,
100
+ no pretrained weights anywhere — using [karpathy/nanochat](https://github.com/karpathy/nanochat) on a single H100:
101
+ **~5 hours of pretraining** on 3.5B tokens plus **47 minutes of chat fine-tuning**.
102
 
103
  It writes fluent English, holds a conversation, writes poems and explains things simply, but it **confidently makes up
104
  facts** (it is ~200x smaller than GPT-3.5). This free CPU Space is slow: replies stream at a few words per second.
 
108
  demo = gr.ChatInterface(
109
  chat_fn,
110
  type="messages",
111
+ title="nanochat-d20 · a chat model trained from scratch",
112
  description=ABOUT,
113
  additional_inputs=[
114
  gr.Slider(32, 1024, value=min(DEFAULT_MAX_TOKENS, 1024), step=32, label="Max new tokens"),