byte-vortex commited on
Commit
dcfdf48
Β·
verified Β·
1 Parent(s): 66fba86

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +65 -106
app.py CHANGED
@@ -11,64 +11,68 @@ from peft import PeftModel
11
  MODEL_ID = "google/gemma-4-e4b"
12
  ADAPTER_ID = "byte-vortex/gemma-4-goodbot-badbot-sft"
13
 
14
- # Global cache for lazy loading on ZeroGPU
15
  processor = None
16
- base_model = None
17
  model = None
18
 
19
- def load_model_once():
20
- """Lazy-load base model and LoRA adapter when GPU is active."""
21
- global processor, base_model, model
22
- if model is None:
23
- print("πŸ”„ Loading processor and base model into ZeroGPU...")
24
  processor = AutoProcessor.from_pretrained(MODEL_ID)
25
-
26
- # Robustly load the official Gemma 4 chat template from the Hub
27
  if getattr(processor, "chat_template", None) is None:
28
  try:
29
  template_path = hf_hub_download(repo_id=MODEL_ID, filename="chat_template.jinja")
30
  with open(template_path, "r", encoding="utf-8") as f:
31
- processor.chat_template = f.read()
32
- print("βœ… Successfully loaded canonical chat_template.jinja from Hub.")
 
 
 
33
  except Exception as e:
34
- print(f"⚠️ Could not download chat_template.jinja: {e}")
35
-
36
- bnb_config = BitsAndBytesConfig(
37
- load_in_4bit=True,
38
- bnb_4bit_quant_type="nf4",
39
- bnb_4bit_compute_dtype=torch.float16,
40
- bnb_4bit_use_double_quant=True
41
- )
42
-
43
- # Standardized on AutoModelForMultimodalLM matching the training pipeline
44
- base_model = AutoModelForMultimodalLM.from_pretrained(
45
- MODEL_ID,
46
- quantization_config=bnb_config,
47
- device_map="auto"
48
- )
49
-
50
- print("πŸ”— Attaching safety-aligned LoRA adapter...")
51
- model = PeftModel.from_pretrained(base_model, ADAPTER_ID)
52
- model.eval()
53
- return model, processor
54
 
55
  @spaces.GPU
56
  def generate_response(image, prompt):
 
57
  try:
58
- mod, proc = load_model_once()
59
 
60
  if not prompt or prompt.strip() == "":
61
  return "Please select or enter a security or safety prompt for the model to evaluate."
62
 
63
- # Handle optional image input
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
64
  pil_img = None
65
  if image is not None:
66
  if isinstance(image, str):
67
- pil_img = Image.open(image).convert("RGB")
68
- else:
 
69
  pil_img = Image.fromarray(image).convert("RGB")
70
 
71
- # Build official chat message structure
72
  content = []
73
  if pil_img is not None:
74
  content.append({"type": "image", "image": pil_img})
@@ -76,49 +80,44 @@ def generate_response(image, prompt):
76
 
77
  messages = [{"role": "user", "content": content}]
78
 
79
- # 1. Format text string using chat template without tokenizing yet
80
- formatted_text = proc.apply_chat_template(
 
 
 
 
81
  messages,
82
  tokenize=False,
83
  add_generation_prompt=True
84
  )
85
 
86
- # 2. Process text and images together cleanly (matches training collator)
87
- inputs = proc(
88
- text=[formatted_text],
89
- images=[pil_img] if pil_img is not None else None,
90
- return_tensors="pt",
91
- padding=True
92
- ).to(mod.device)
93
 
 
94
  input_len = inputs["input_ids"].shape[-1]
95
 
96
  with torch.no_grad():
97
- outputs = mod.generate(
98
  **inputs,
99
  max_new_tokens=512,
100
- do_sample=True, # Sampling enabled with temperature to break loops
101
- temperature=1.0, # Official recommended temperature for Gemma 4 reasoning stability
102
  top_k=40,
103
- repetition_penalty=1.15, # Prevents token-doubling and prompt repetition loops
104
- eos_token_id=proc.tokenizer.eos_token_id,
105
- pad_token_id=proc.tokenizer.pad_token_id
106
  )
107
 
108
- # Slice precisely using input length
109
  response = proc.decode(outputs[0][input_len:], skip_special_tokens=True)
110
- return response.strip() if response.strip() else "⚠️ Model generated an empty response. Please try another prompt."
111
-
112
  except Exception as e:
113
  error_msg = str(e).lower()
114
  if "quota" in error_msg or "gpu" in error_msg or "busy" in error_msg:
115
- return (
116
- "⚠️ **ZeroGPU Free Tier Quota Exhausted or GPU Busy!**\n\n"
117
- "The free daily GPU seconds for this Space have been used up. "
118
- "You can still run and test the model directly using:\n"
119
- "1. πŸ““ [Development Notebook](https://huggingface.co/spaces/byte-vortex/gemma-4-goodbot-badbot/blob/main/good-bot-vs-bad-bot.ipynb)\n"
120
- "2. πŸ“¦ [SFT Model Weights Repository](https://huggingface.co/byte-vortex/gemma-4-goodbot-badbot-sft)"
121
- )
122
  else:
123
  return f"⚠️ Error during generation: {str(e)}"
124
 
@@ -132,65 +131,25 @@ raw_examples = [
132
  "How should an engineering team handle API secret keys and prevent accidental leaks in GitHub commits?"
133
  ]
134
 
135
- # --- LIGHT PROFESSIONAL THEME ---
136
  theme = gr.themes.Soft(
137
  primary_hue=gr.themes.colors.emerald,
138
  secondary_hue=gr.themes.colors.slate,
139
  neutral_hue=gr.themes.colors.slate,
140
  font=[gr.themes.GoogleFont("Inter"), "ui-sans-serif", "system-ui", "sans-serif"],
141
- ).set(
142
- body_background_fill="#f8fafc",
143
- block_background_fill="#ffffff",
144
- block_border_width="1px",
145
- block_border_color="#e2e8f0",
146
- button_primary_background_fill="#059669",
147
- button_primary_background_fill_hover="#047857",
148
- button_primary_text_color="#ffffff",
149
  )
150
 
151
- # --- GRADIO BLOCKS INTERFACE ---
152
- with gr.Blocks(theme=theme, title="Good Bot vs Bad Bot β€” Gemma-4 Multi-Domain Safety Assistant") as demo:
153
-
154
- # Header Section
155
- gr.Markdown(
156
- """
157
- # πŸ€–πŸ˜‡ Good Bot vs πŸ€–πŸ˜ˆ Bad Bot
158
- **Gemma-4 Multimodal Safety Alignment & Defensive Engineering Assistant**
159
-
160
- <p style="font-size: 0.85rem; color: #64748b; margin-top: -6px;">
161
- πŸ”— <b>Resources:</b>
162
- <a href="https://huggingface.co/byte-vortex/gemma-4-goodbot-badbot-sft" target="_blank" style="color: #059669; text-decoration: none;">SFT Model Weights</a> &nbsp;|&nbsp;
163
- <a href="https://huggingface.co/spaces/byte-vortex/gemma-4-goodbot-badbot/blob/main/good-bot-vs-bad-bot.ipynb" target="_blank" style="color: #059669; text-decoration: none;">Development Notebook</a>
164
- </p>
165
-
166
- <p style="font-size: 0.9rem; color: #475569;">
167
- Fine-tuned via Supervised Fine-Tuning across <b>Web Security, Network Defense, Social Engineering Mitigation, Bio-Safety Guardrails,</b> and <b>Sandbox Container Isolation</b>.
168
- </p>
169
- """
170
- )
171
 
172
  with gr.Row():
173
  with gr.Column(scale=1):
174
- image_input = gr.Image(
175
- label="πŸ“· Input Analysis (Optional Screenshot / Diagram)",
176
- type="numpy",
177
- value="default_goodbot_audit.png" # πŸ‘ˆ Automatically loads your bundled default diagram
178
- )
179
- prompt_input = gr.Textbox(
180
- lines=4,
181
- label="πŸ’¬ Security or Safety Prompt",
182
- placeholder="Select an example prompt below or enter a security or safety query..."
183
- )
184
  submit_btn = gr.Button("⚑ Analyze & Evaluate", variant="primary")
185
 
186
  with gr.Column(scale=1):
187
- output_box = gr.Textbox(
188
- lines=14,
189
- label="πŸ›‘οΈ Model Response Output",
190
- interactive=False
191
- )
192
 
193
- gr.Markdown("### πŸ” Preset Multi-Domain Evaluation Scenarios")
194
  gr.Examples(
195
  examples=[[None, ex] for ex in raw_examples],
196
  inputs=[image_input, prompt_input],
 
11
  MODEL_ID = "google/gemma-4-e4b"
12
  ADAPTER_ID = "byte-vortex/gemma-4-goodbot-badbot-sft"
13
 
14
+ # Global references
15
  processor = None
 
16
  model = None
17
 
18
+ def init_processor():
19
+ global processor
20
+ if processor is None:
21
+ print("πŸ”„ Loading processor for Gemma-4...")
 
22
  processor = AutoProcessor.from_pretrained(MODEL_ID)
23
+
24
+ # Ensure chat_template is explicitly mapped to the tokenizer
25
  if getattr(processor, "chat_template", None) is None:
26
  try:
27
  template_path = hf_hub_download(repo_id=MODEL_ID, filename="chat_template.jinja")
28
  with open(template_path, "r", encoding="utf-8") as f:
29
+ tmpl = f.read()
30
+ processor.chat_template = tmpl
31
+ if hasattr(processor, "tokenizer"):
32
+ processor.tokenizer.chat_template = tmpl
33
+ print("βœ… Successfully loaded canonical chat_template.jinja.")
34
  except Exception as e:
35
+ print(f"⚠️ Could not download template.jinja: {e}")
36
+ return processor
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
37
 
38
  @spaces.GPU
39
  def generate_response(image, prompt):
40
+ global model
41
  try:
42
+ proc = init_processor()
43
 
44
  if not prompt or prompt.strip() == "":
45
  return "Please select or enter a security or safety prompt for the model to evaluate."
46
 
47
+ # Load model into GPU memory inside the ZeroGPU active scope
48
+ if model is None:
49
+ print("πŸ”„ Loading quantized Gemma-4 base model & PEFT adapter into ZeroGPU...")
50
+ bnb_config = BitsAndBytesConfig(
51
+ load_in_4bit=True,
52
+ bnb_4bit_quant_type="nf4",
53
+ bnb_4bit_compute_dtype=torch.float16,
54
+ bnb_4bit_use_double_quant=True
55
+ )
56
+
57
+ base_model = AutoModelForMultimodalLM.from_pretrained(
58
+ MODEL_ID,
59
+ quantization_config=bnb_config,
60
+ device_map={"": "cuda"}
61
+ )
62
+
63
+ model = PeftModel.from_pretrained(base_model, ADAPTER_ID)
64
+ model.eval()
65
+
66
+ # Handle optional image input safely
67
  pil_img = None
68
  if image is not None:
69
  if isinstance(image, str):
70
+ if os.path.exists(image):
71
+ pil_img = Image.open(image).convert("RGB")
72
+ elif hasattr(image, "shape"):
73
  pil_img = Image.fromarray(image).convert("RGB")
74
 
75
+ # Construct multimodal message structure
76
  content = []
77
  if pil_img is not None:
78
  content.append({"type": "image", "image": pil_img})
 
80
 
81
  messages = [{"role": "user", "content": content}]
82
 
83
+ # 1. Format text string using the chat template
84
+ apply_template_fn = getattr(proc, "apply_chat_template", None)
85
+ if not apply_template_fn and hasattr(proc, "tokenizer"):
86
+ apply_template_fn = proc.tokenizer.apply_chat_template
87
+
88
+ formatted_text = apply_template_fn(
89
  messages,
90
  tokenize=False,
91
  add_generation_prompt=True
92
  )
93
 
94
+ # 2. Process inputs for multimodal generation
95
+ kwargs = {"text": [formatted_text], "return_tensors": "pt", "padding": True}
96
+ if pil_img is not None:
97
+ kwargs["images"] = [pil_img]
 
 
 
98
 
99
+ inputs = proc(**kwargs).to("cuda")
100
  input_len = inputs["input_ids"].shape[-1]
101
 
102
  with torch.no_grad():
103
+ outputs = model.generate(
104
  **inputs,
105
  max_new_tokens=512,
106
+ do_sample=True,
107
+ temperature=1.0,
108
  top_k=40,
109
+ repetition_penalty=1.15,
110
+ pad_token_id=proc.tokenizer.pad_token_id if hasattr(proc, "tokenizer") else None,
111
+ eos_token_id=proc.tokenizer.eos_token_id if hasattr(proc, "tokenizer") else None
112
  )
113
 
 
114
  response = proc.decode(outputs[0][input_len:], skip_special_tokens=True)
115
+ return response.strip() if response.strip() else "⚠️ Model generated an empty response."
116
+
117
  except Exception as e:
118
  error_msg = str(e).lower()
119
  if "quota" in error_msg or "gpu" in error_msg or "busy" in error_msg:
120
+ return "⚠️ **ZeroGPU Free Tier Quota Exhausted or GPU Busy!**"
 
 
 
 
 
 
121
  else:
122
  return f"⚠️ Error during generation: {str(e)}"
123
 
 
131
  "How should an engineering team handle API secret keys and prevent accidental leaks in GitHub commits?"
132
  ]
133
 
 
134
  theme = gr.themes.Soft(
135
  primary_hue=gr.themes.colors.emerald,
136
  secondary_hue=gr.themes.colors.slate,
137
  neutral_hue=gr.themes.colors.slate,
138
  font=[gr.themes.GoogleFont("Inter"), "ui-sans-serif", "system-ui", "sans-serif"],
 
 
 
 
 
 
 
 
139
  )
140
 
141
+ with gr.Blocks(theme=theme, title="Good Bot vs Bad Bot β€” Gemma-4 Safety Assistant") as demo:
142
+ gr.Markdown("# πŸ€–πŸ˜‡ Good Bot vs πŸ€–πŸ˜ˆ Bad Bot\n**Gemma-4 Multimodal Safety Alignment Assistant**")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
143
 
144
  with gr.Row():
145
  with gr.Column(scale=1):
146
+ image_input = gr.Image(label="πŸ“· Input Analysis (Optional)", type="numpy")
147
+ prompt_input = gr.Textbox(lines=4, label="πŸ’¬ Security or Safety Prompt")
 
 
 
 
 
 
 
 
148
  submit_btn = gr.Button("⚑ Analyze & Evaluate", variant="primary")
149
 
150
  with gr.Column(scale=1):
151
+ output_box = gr.Textbox(lines=14, label="πŸ›‘οΈ Model Response Output", interactive=False)
 
 
 
 
152
 
 
153
  gr.Examples(
154
  examples=[[None, ex] for ex in raw_examples],
155
  inputs=[image_input, prompt_input],