Upload 4 files
Browse files- CLAUDE.md +26 -0
- app.py +61 -0
- inflections_funcs.py +133 -0
- requirements.txt +4 -0
CLAUDE.md
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
## Project Overview
|
| 2 |
+
This repo contains code generating text using GEMMA 4 via the transformers library
|
| 3 |
+
|
| 4 |
+
### Inflections
|
| 5 |
+
The goal of this work is to use GEMMA 4 to generate 5 beams in response to a prompt. Each beam is parsed and can be displayed along with it's total 'score.' Each beam can also be displayed with a score-per-token.
|
| 6 |
+
|
| 7 |
+
#### What needs work
|
| 8 |
+
- create a second version of the inflections.py file with all class methods as separate functions. they should each return needed variables needed to call the next function,
|
| 9 |
+
- modify the app.py so that the input elemnet takes the prompt and the output displays the first beam, with any token with score < 0.6 is highlighted in red.
|
| 10 |
+
|
| 11 |
+
## Tech Stack
|
| 12 |
+
- Language: Python
|
| 13 |
+
- packages:
|
| 14 |
+
* transformers
|
| 15 |
+
* torch
|
| 16 |
+
|
| 17 |
+
## Project Structure
|
| 18 |
+
```
|
| 19 |
+
llm_inflections/
|
| 20 |
+
inflections.py # inflection class
|
| 21 |
+
app.py # gradio gui
|
| 22 |
+
|
| 23 |
+
```
|
| 24 |
+
|
| 25 |
+
## Other information
|
| 26 |
+
- assume any API keys can be obtained as environment variables.
|
app.py
ADDED
|
@@ -0,0 +1,61 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import gradio as gr
|
| 2 |
+
import spaces
|
| 3 |
+
from inflections_funcs import start_model, make_beams, get_beam_tokens, calculate_score_vectors
|
| 4 |
+
|
| 5 |
+
# Global model initialization
|
| 6 |
+
print("Loading model...")
|
| 7 |
+
model, processor = start_model()
|
| 8 |
+
print("Model loaded.")
|
| 9 |
+
|
| 10 |
+
@spaces.GPU
|
| 11 |
+
def predict(prompt):
|
| 12 |
+
"""
|
| 13 |
+
Generates a response and returns an HTML string with tokens highlighted by probability.
|
| 14 |
+
"""
|
| 15 |
+
# Generate beams
|
| 16 |
+
generated_dicts, transcription = make_beams(model, processor, prompt)
|
| 17 |
+
|
| 18 |
+
# Get tokens and scores for the first beam
|
| 19 |
+
beam_tokens = get_beam_tokens(generated_dicts, processor)
|
| 20 |
+
score_vectors = calculate_score_vectors(model, generated_dicts)
|
| 21 |
+
|
| 22 |
+
if not beam_tokens or not score_vectors:
|
| 23 |
+
return "<span style='color: grey'>No tokens generated.</span>"
|
| 24 |
+
|
| 25 |
+
first_beam_tokens = beam_tokens[0]
|
| 26 |
+
first_beam_scores = score_vectors[0]
|
| 27 |
+
|
| 28 |
+
# Construct HTML output
|
| 29 |
+
html_output = '<div style="font-family: monospace; white-space: pre-wrap; line-height: 1.5;">'
|
| 30 |
+
|
| 31 |
+
for token, score in zip(first_beam_tokens, first_beam_scores):
|
| 32 |
+
# Replace BPE space marker with actual space for readability, but keep token boundaries
|
| 33 |
+
display_token = token.replace(' ', ' ')
|
| 34 |
+
|
| 35 |
+
if score < 0.6:
|
| 36 |
+
color = "red"
|
| 37 |
+
else:
|
| 38 |
+
color = "black"
|
| 39 |
+
|
| 40 |
+
html_output += f'<span style="color: {color}; border: 1px solid #eee; padding: 0 2px; margin: 0 1px;" title="Score: {score:.4f}">{display_token}</span>'
|
| 41 |
+
|
| 42 |
+
html_output += '</div>'
|
| 43 |
+
return html_output
|
| 44 |
+
|
| 45 |
+
# Gradio Interface
|
| 46 |
+
with gr.Blocks() as demo:
|
| 47 |
+
gr.Markdown("# LLM Token Probability Visualizer")
|
| 48 |
+
gr.Markdown("Input a prompt and see the first beam's tokens. Tokens with a probability score < 0.6 are highlighted in red.")
|
| 49 |
+
|
| 50 |
+
with gr.Row():
|
| 51 |
+
with gr.Column():
|
| 52 |
+
prompt_input = gr.Textbox(label="Prompt", placeholder="Enter your prompt here...", lines=3)
|
| 53 |
+
submit_btn = gr.Button("Generate")
|
| 54 |
+
|
| 55 |
+
with gr.Column():
|
| 56 |
+
output_html = gr.HTML(label="Highlighted Beam")
|
| 57 |
+
|
| 58 |
+
submit_btn.click(fn=predict, inputs=prompt_input, outputs=output_html)
|
| 59 |
+
|
| 60 |
+
if __name__ == "__main__":
|
| 61 |
+
demo.launch()
|
inflections_funcs.py
ADDED
|
@@ -0,0 +1,133 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from transformers import AutoProcessor, AutoModelForCausalLM
|
| 2 |
+
import torch
|
| 3 |
+
import torch.nn.functional as F
|
| 4 |
+
from torch import exp
|
| 5 |
+
from typing import Any, Tuple, List
|
| 6 |
+
|
| 7 |
+
def start_model(model_id: str = "google/gemma-4-31B-it"):
|
| 8 |
+
'''
|
| 9 |
+
Initializes and returns the processor and model.
|
| 10 |
+
'''
|
| 11 |
+
processor = AutoProcessor.from_pretrained(model_id)
|
| 12 |
+
model = AutoModelForCausalLM.from_pretrained(
|
| 13 |
+
model_id,
|
| 14 |
+
dtype="auto",
|
| 15 |
+
device_map="auto"
|
| 16 |
+
)
|
| 17 |
+
print(f'Model {model_id} has been installed.')
|
| 18 |
+
return model, processor
|
| 19 |
+
|
| 20 |
+
def make_beams(model: AutoModelForCausalLM, processor: AutoProcessor, initial_prompt: str) -> Tuple[Any, List[str]]:
|
| 21 |
+
'''
|
| 22 |
+
Generates 5 beams in response to a prompt.
|
| 23 |
+
'''
|
| 24 |
+
messages = [
|
| 25 |
+
{"role": "system", "content": "You are a helpful assistant."},
|
| 26 |
+
{"role": "user", "content": initial_prompt},
|
| 27 |
+
]
|
| 28 |
+
|
| 29 |
+
# Process input
|
| 30 |
+
text = processor.apply_chat_template(
|
| 31 |
+
messages,
|
| 32 |
+
tokenize=False,
|
| 33 |
+
add_generation_prompt=True,
|
| 34 |
+
enable_thinking=False
|
| 35 |
+
)
|
| 36 |
+
inputs = processor(text=text, return_tensors="pt").to(model.device)
|
| 37 |
+
|
| 38 |
+
# Generate output
|
| 39 |
+
generated_dicts = model.generate(**inputs,
|
| 40 |
+
max_new_tokens=1024,
|
| 41 |
+
num_beams=5, # Enables beam search with 5 active beams
|
| 42 |
+
num_return_sequences=5, # Returns the top result
|
| 43 |
+
return_dict_in_generate=True,
|
| 44 |
+
output_scores=True)
|
| 45 |
+
|
| 46 |
+
transcription = processor.batch_decode(generated_dicts.sequences, skip_special_tokens=True)
|
| 47 |
+
|
| 48 |
+
print('Keys in model output -------------------')
|
| 49 |
+
for key in generated_dicts:
|
| 50 |
+
print(key)
|
| 51 |
+
print('----------------------------------------')
|
| 52 |
+
|
| 53 |
+
print('Beam scores ----------------------------')
|
| 54 |
+
for score in generated_dicts.sequences_scores:
|
| 55 |
+
print(exp(score).item())
|
| 56 |
+
print('----------------------------------------')
|
| 57 |
+
|
| 58 |
+
return generated_dicts, transcription
|
| 59 |
+
|
| 60 |
+
def parse_beams(transcription: List[str]) -> List[str]:
|
| 61 |
+
'''
|
| 62 |
+
Parses beams to extract only the response after 'model\nthought'.
|
| 63 |
+
'''
|
| 64 |
+
beam_text = []
|
| 65 |
+
for beam in transcription:
|
| 66 |
+
parts = beam.split('''model\nthought''')
|
| 67 |
+
if len(parts) > 1:
|
| 68 |
+
response = parts[1].strip('\n')
|
| 69 |
+
else:
|
| 70 |
+
response = beam
|
| 71 |
+
beam_text.append(response)
|
| 72 |
+
|
| 73 |
+
print('Beams have been parsed. --------------')
|
| 74 |
+
return beam_text
|
| 75 |
+
|
| 76 |
+
def get_beam_tokens(generated_dicts: Any, processor: AutoProcessor) -> List[List[str]]:
|
| 77 |
+
'''
|
| 78 |
+
Decodes the generated sequences into individual tokens for each beam.
|
| 79 |
+
'''
|
| 80 |
+
beam_tokens = []
|
| 81 |
+
# The number of generated tokens is the length of the scores list
|
| 82 |
+
gen_len = len(generated_dicts.scores)
|
| 83 |
+
|
| 84 |
+
for sequence in generated_dicts.sequences:
|
| 85 |
+
# Calculate input length for this specific sequence
|
| 86 |
+
total_len = sequence.shape[0]
|
| 87 |
+
input_len = total_len - gen_len
|
| 88 |
+
|
| 89 |
+
# Extract only the generated token IDs
|
| 90 |
+
generated_ids = sequence[input_len:].tolist()
|
| 91 |
+
|
| 92 |
+
# Convert IDs to tokens (e.g., ' Hello', ' world')
|
| 93 |
+
tokens = processor.tokenizer.convert_ids_to_tokens(generated_ids)
|
| 94 |
+
beam_tokens.append(tokens)
|
| 95 |
+
|
| 96 |
+
return beam_tokens
|
| 97 |
+
|
| 98 |
+
def calculate_score_vectors(model: AutoModelForCausalLM, generated_dicts: Any) -> List[List[float]]:
|
| 99 |
+
'''
|
| 100 |
+
Creates a score vector for each beam containing the probability of each
|
| 101 |
+
token that was chosen.
|
| 102 |
+
'''
|
| 103 |
+
# Number of sequences generated
|
| 104 |
+
num_sequences = generated_dicts.sequences.shape[0]
|
| 105 |
+
# Number of generated tokens (excluding prompt)
|
| 106 |
+
gen_len = len(generated_dicts.scores)
|
| 107 |
+
# Total length of sequences (including prompt)
|
| 108 |
+
total_len = generated_dicts.sequences.shape[1]
|
| 109 |
+
# Input length (prompt length)
|
| 110 |
+
input_len = total_len - gen_len
|
| 111 |
+
|
| 112 |
+
# To get the accurate probabilities for the final beams,
|
| 113 |
+
# we pass the final sequences back through the model.
|
| 114 |
+
with torch.no_grad():
|
| 115 |
+
outputs = model(generated_dicts.sequences)
|
| 116 |
+
logits = outputs.logits # shape: (num_sequences, total_len, vocab_size)
|
| 117 |
+
|
| 118 |
+
# Convert logits to probabilities
|
| 119 |
+
probs = F.softmax(logits, dim=-1) # shape: (num_sequences, total_len, vocab_size)
|
| 120 |
+
|
| 121 |
+
score_vectors = []
|
| 122 |
+
for i in range(num_sequences):
|
| 123 |
+
beam_probs = []
|
| 124 |
+
# The generated tokens are at positions [input_len, total_len)
|
| 125 |
+
# The probability for the token at position t is found at logits index t-1
|
| 126 |
+
for t in range(input_len, total_len):
|
| 127 |
+
token_id = generated_dicts.sequences[i, t]
|
| 128 |
+
# Probability for token at position t is at index t-1 in the logits
|
| 129 |
+
prob = probs[i, t - 1, token_id].item()
|
| 130 |
+
beam_probs.append(prob)
|
| 131 |
+
score_vectors.append(beam_probs)
|
| 132 |
+
|
| 133 |
+
return score_vectors
|
requirements.txt
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Transformers==5.10.2
|
| 2 |
+
Torch==2.11.0+cu128
|
| 3 |
+
Accelerate==1.13.0
|
| 4 |
+
Torchvision==0.26.0+cu128
|