cafierom commited on
Commit
8788c39
·
verified ·
1 Parent(s): d5db586

Upload 4 files

Browse files
Files changed (4) hide show
  1. CLAUDE.md +26 -0
  2. app.py +61 -0
  3. inflections_funcs.py +133 -0
  4. requirements.txt +4 -0
CLAUDE.md ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ## Project Overview
2
+ This repo contains code generating text using GEMMA 4 via the transformers library
3
+
4
+ ### Inflections
5
+ The goal of this work is to use GEMMA 4 to generate 5 beams in response to a prompt. Each beam is parsed and can be displayed along with it's total 'score.' Each beam can also be displayed with a score-per-token.
6
+
7
+ #### What needs work
8
+ - create a second version of the inflections.py file with all class methods as separate functions. they should each return needed variables needed to call the next function,
9
+ - modify the app.py so that the input elemnet takes the prompt and the output displays the first beam, with any token with score < 0.6 is highlighted in red.
10
+
11
+ ## Tech Stack
12
+ - Language: Python
13
+ - packages:
14
+ * transformers
15
+ * torch
16
+
17
+ ## Project Structure
18
+ ```
19
+ llm_inflections/
20
+ inflections.py # inflection class
21
+ app.py # gradio gui
22
+
23
+ ```
24
+
25
+ ## Other information
26
+ - assume any API keys can be obtained as environment variables.
app.py ADDED
@@ -0,0 +1,61 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import gradio as gr
2
+ import spaces
3
+ from inflections_funcs import start_model, make_beams, get_beam_tokens, calculate_score_vectors
4
+
5
+ # Global model initialization
6
+ print("Loading model...")
7
+ model, processor = start_model()
8
+ print("Model loaded.")
9
+
10
+ @spaces.GPU
11
+ def predict(prompt):
12
+ """
13
+ Generates a response and returns an HTML string with tokens highlighted by probability.
14
+ """
15
+ # Generate beams
16
+ generated_dicts, transcription = make_beams(model, processor, prompt)
17
+
18
+ # Get tokens and scores for the first beam
19
+ beam_tokens = get_beam_tokens(generated_dicts, processor)
20
+ score_vectors = calculate_score_vectors(model, generated_dicts)
21
+
22
+ if not beam_tokens or not score_vectors:
23
+ return "<span style='color: grey'>No tokens generated.</span>"
24
+
25
+ first_beam_tokens = beam_tokens[0]
26
+ first_beam_scores = score_vectors[0]
27
+
28
+ # Construct HTML output
29
+ html_output = '<div style="font-family: monospace; white-space: pre-wrap; line-height: 1.5;">'
30
+
31
+ for token, score in zip(first_beam_tokens, first_beam_scores):
32
+ # Replace BPE space marker with actual space for readability, but keep token boundaries
33
+ display_token = token.replace(' ', ' ')
34
+
35
+ if score < 0.6:
36
+ color = "red"
37
+ else:
38
+ color = "black"
39
+
40
+ html_output += f'<span style="color: {color}; border: 1px solid #eee; padding: 0 2px; margin: 0 1px;" title="Score: {score:.4f}">{display_token}</span>'
41
+
42
+ html_output += '</div>'
43
+ return html_output
44
+
45
+ # Gradio Interface
46
+ with gr.Blocks() as demo:
47
+ gr.Markdown("# LLM Token Probability Visualizer")
48
+ gr.Markdown("Input a prompt and see the first beam's tokens. Tokens with a probability score < 0.6 are highlighted in red.")
49
+
50
+ with gr.Row():
51
+ with gr.Column():
52
+ prompt_input = gr.Textbox(label="Prompt", placeholder="Enter your prompt here...", lines=3)
53
+ submit_btn = gr.Button("Generate")
54
+
55
+ with gr.Column():
56
+ output_html = gr.HTML(label="Highlighted Beam")
57
+
58
+ submit_btn.click(fn=predict, inputs=prompt_input, outputs=output_html)
59
+
60
+ if __name__ == "__main__":
61
+ demo.launch()
inflections_funcs.py ADDED
@@ -0,0 +1,133 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from transformers import AutoProcessor, AutoModelForCausalLM
2
+ import torch
3
+ import torch.nn.functional as F
4
+ from torch import exp
5
+ from typing import Any, Tuple, List
6
+
7
+ def start_model(model_id: str = "google/gemma-4-31B-it"):
8
+ '''
9
+ Initializes and returns the processor and model.
10
+ '''
11
+ processor = AutoProcessor.from_pretrained(model_id)
12
+ model = AutoModelForCausalLM.from_pretrained(
13
+ model_id,
14
+ dtype="auto",
15
+ device_map="auto"
16
+ )
17
+ print(f'Model {model_id} has been installed.')
18
+ return model, processor
19
+
20
+ def make_beams(model: AutoModelForCausalLM, processor: AutoProcessor, initial_prompt: str) -> Tuple[Any, List[str]]:
21
+ '''
22
+ Generates 5 beams in response to a prompt.
23
+ '''
24
+ messages = [
25
+ {"role": "system", "content": "You are a helpful assistant."},
26
+ {"role": "user", "content": initial_prompt},
27
+ ]
28
+
29
+ # Process input
30
+ text = processor.apply_chat_template(
31
+ messages,
32
+ tokenize=False,
33
+ add_generation_prompt=True,
34
+ enable_thinking=False
35
+ )
36
+ inputs = processor(text=text, return_tensors="pt").to(model.device)
37
+
38
+ # Generate output
39
+ generated_dicts = model.generate(**inputs,
40
+ max_new_tokens=1024,
41
+ num_beams=5, # Enables beam search with 5 active beams
42
+ num_return_sequences=5, # Returns the top result
43
+ return_dict_in_generate=True,
44
+ output_scores=True)
45
+
46
+ transcription = processor.batch_decode(generated_dicts.sequences, skip_special_tokens=True)
47
+
48
+ print('Keys in model output -------------------')
49
+ for key in generated_dicts:
50
+ print(key)
51
+ print('----------------------------------------')
52
+
53
+ print('Beam scores ----------------------------')
54
+ for score in generated_dicts.sequences_scores:
55
+ print(exp(score).item())
56
+ print('----------------------------------------')
57
+
58
+ return generated_dicts, transcription
59
+
60
+ def parse_beams(transcription: List[str]) -> List[str]:
61
+ '''
62
+ Parses beams to extract only the response after 'model\nthought'.
63
+ '''
64
+ beam_text = []
65
+ for beam in transcription:
66
+ parts = beam.split('''model\nthought''')
67
+ if len(parts) > 1:
68
+ response = parts[1].strip('\n')
69
+ else:
70
+ response = beam
71
+ beam_text.append(response)
72
+
73
+ print('Beams have been parsed. --------------')
74
+ return beam_text
75
+
76
+ def get_beam_tokens(generated_dicts: Any, processor: AutoProcessor) -> List[List[str]]:
77
+ '''
78
+ Decodes the generated sequences into individual tokens for each beam.
79
+ '''
80
+ beam_tokens = []
81
+ # The number of generated tokens is the length of the scores list
82
+ gen_len = len(generated_dicts.scores)
83
+
84
+ for sequence in generated_dicts.sequences:
85
+ # Calculate input length for this specific sequence
86
+ total_len = sequence.shape[0]
87
+ input_len = total_len - gen_len
88
+
89
+ # Extract only the generated token IDs
90
+ generated_ids = sequence[input_len:].tolist()
91
+
92
+ # Convert IDs to tokens (e.g., ' Hello', ' world')
93
+ tokens = processor.tokenizer.convert_ids_to_tokens(generated_ids)
94
+ beam_tokens.append(tokens)
95
+
96
+ return beam_tokens
97
+
98
+ def calculate_score_vectors(model: AutoModelForCausalLM, generated_dicts: Any) -> List[List[float]]:
99
+ '''
100
+ Creates a score vector for each beam containing the probability of each
101
+ token that was chosen.
102
+ '''
103
+ # Number of sequences generated
104
+ num_sequences = generated_dicts.sequences.shape[0]
105
+ # Number of generated tokens (excluding prompt)
106
+ gen_len = len(generated_dicts.scores)
107
+ # Total length of sequences (including prompt)
108
+ total_len = generated_dicts.sequences.shape[1]
109
+ # Input length (prompt length)
110
+ input_len = total_len - gen_len
111
+
112
+ # To get the accurate probabilities for the final beams,
113
+ # we pass the final sequences back through the model.
114
+ with torch.no_grad():
115
+ outputs = model(generated_dicts.sequences)
116
+ logits = outputs.logits # shape: (num_sequences, total_len, vocab_size)
117
+
118
+ # Convert logits to probabilities
119
+ probs = F.softmax(logits, dim=-1) # shape: (num_sequences, total_len, vocab_size)
120
+
121
+ score_vectors = []
122
+ for i in range(num_sequences):
123
+ beam_probs = []
124
+ # The generated tokens are at positions [input_len, total_len)
125
+ # The probability for the token at position t is found at logits index t-1
126
+ for t in range(input_len, total_len):
127
+ token_id = generated_dicts.sequences[i, t]
128
+ # Probability for token at position t is at index t-1 in the logits
129
+ prob = probs[i, t - 1, token_id].item()
130
+ beam_probs.append(prob)
131
+ score_vectors.append(beam_probs)
132
+
133
+ return score_vectors
requirements.txt ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ Transformers==5.10.2
2
+ Torch==2.11.0+cu128
3
+ Accelerate==1.13.0
4
+ Torchvision==0.26.0+cu128