Workpam commited on
Commit
b02a4db
·
verified ·
1 Parent(s): 60dcb20

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +201 -34
app.py CHANGED
@@ -1,41 +1,208 @@
1
- import os
2
  import gradio as gr
3
- import requests
4
- import base64
5
- import io
6
-
7
- HF_MODEL = "llava-hf/llava-v1.6-mistral-7b-hf"
8
- HF_TOKEN = os.getenv("HF_API_TOKEN")
9
- API_URL = f"https://api-inference.huggingface.co/models/{HF_MODEL}"
10
- HEADERS = {"Authorization": f"Bearer {HF_TOKEN}"}
11
-
12
- def generate_alt_text(image):
13
- buffered = io.BytesIO()
14
- image.save(buffered, format="JPEG")
15
- b64 = base64.b64encode(buffered.getvalue()).decode()
16
-
17
- payload = {
18
- "inputs": [
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
19
  {
20
  "role": "user",
21
  "content": [
22
- {"type": "image", "image": b64},
23
- {"type": "text", "text": "Describe this image in detail"}
24
  ],
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
25
  }
26
- ],
27
- "options": {"wait_for_model": True},
28
- }
29
-
30
- resp = requests.post(API_URL, headers=HEADERS, json=payload)
31
- if resp.status_code != 200:
32
- return f"Error: {resp.status_code}"
33
- output = resp.json()
34
- return output[0]["generated_text"].strip()
35
-
36
- gr.Interface(
37
- fn=generate_alt_text,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
38
  inputs=gr.Image(type="pil"),
39
- outputs=gr.Textbox(label="Alt Text from LLaVA"),
40
- title="Inference‑based Rich Alt Text"
41
- ).launch()
 
 
 
 
 
 
 
 
 
 
1
  import gradio as gr
2
+ from transformers import AutoProcessor, LlavaForConditionalGeneration
3
+ from PIL import Image
4
+ import torch
5
+ import json
6
+
7
+ # --- Device / dtype selection ---
8
+ use_cuda = torch.cuda.is_available()
9
+ dtype = torch.float16 if use_cuda else torch.float32 # float16 only on GPU
10
+ device = "cuda" if use_cuda else "cpu"
11
+
12
+ model_name = "llava-hf/llava-v1.6-mistral-7b-hf"
13
+
14
+ # Load processor and model. Let HF handle device placement if using device_map.
15
+ processor = AutoProcessor.from_pretrained(model_name)
16
+
17
+ model = LlavaForConditionalGeneration.from_pretrained(
18
+ model_name,
19
+ torch_dtype=dtype,
20
+ device_map="auto" if use_cuda else None,
21
+ )
22
+ model.eval()
23
+
24
+ # System instruction: explicit schema for structured JSON output
25
+ SYSTEM_INSTRUCTION = (
26
+ "You are an assistant that generates detailed alt text for images. "
27
+ "Produce a JSON object with these keys:\n"
28
+ "1. main_objects: list of the primary objects/things in the image.\n"
29
+ "2. visible_text: any readable text (quote it); null if none.\n"
30
+ "3. background: description of the background (excluding main objects).\n"
31
+ "4. colors_materials: dominant colors, textures, materials visible.\n"
32
+ "5. spatial_relationships: how objects relate/are positioned relative to each other.\n"
33
+ "6. uncertainties: list of things you are unsure about (use qualifiers like 'appears to be', 'possibly').\n"
34
+ "7. summary_alt_text: one concise sentence suitable as alt text for accessibility.\n"
35
+ "After the JSON, also provide a short human-readable paragraph summarizing the scene."
36
+ )
37
+
38
+ def generate_structured_alt_text(image: Image.Image) -> dict:
39
+ """
40
+ Returns a dict with structured fields plus a fallback message if on CPU.
41
+ """
42
+ try:
43
+ image = image.convert("RGB")
44
+
45
+ # CPU fallback / performance adjustments
46
+ if not use_cuda:
47
+ # aggressively resize to keep inference lightweight
48
+ max_side = 800
49
+ if max(image.size) > max_side:
50
+ scale = max_side / max(image.size)
51
+ image = image.resize((int(image.width * scale), int(image.height * scale)), Image.BICUBIC)
52
+ else:
53
+ # on GPU, cap size but allow higher fidelity
54
+ max_side = 1024
55
+ if max(image.size) > max_side:
56
+ scale = max_side / max(image.size)
57
+ image = image.resize((int(image.width * scale), int(image.height * scale)), Image.BICUBIC)
58
+
59
+ # Build conversation with schema request
60
+ user_prompt = (
61
+ "Please analyze the image and output the JSON as per the system schema. "
62
+ "Focus on objects, text, background, colors, spatial layout, and uncertainties. "
63
+ "Then give a concise alt-text sentence."
64
+ )
65
+ conversation = [
66
+ {"role": "system", "content": SYSTEM_INSTRUCTION},
67
  {
68
  "role": "user",
69
  "content": [
70
+ {"type": "image", "image": image},
71
+ {"type": "text", "text": user_prompt},
72
  ],
73
+ },
74
+ ]
75
+
76
+ with torch.inference_mode():
77
+ inputs = processor.apply_chat_template(
78
+ conversation,
79
+ add_generation_prompt=True,
80
+ tokenize=True,
81
+ return_tensors="pt",
82
+ )
83
+ # Move to model device
84
+ for k, v in inputs.items():
85
+ inputs[k] = v.to(model.device)
86
+
87
+ # Adjust generation params depending on device
88
+ if use_cuda:
89
+ gen_kwargs = dict(
90
+ **inputs,
91
+ max_new_tokens=150,
92
+ do_sample=True,
93
+ temperature=0.7,
94
+ top_p=0.9,
95
+ repetition_penalty=1.1,
96
+ early_stopping=True,
97
+ )
98
+ else:
99
+ # CPU: deterministic, shorter, less sampling for speed/stability
100
+ gen_kwargs = dict(
101
+ **inputs,
102
+ max_new_tokens=100,
103
+ do_sample=False, # greedy
104
+ early_stopping=True,
105
+ )
106
+
107
+ output_ids = model.generate(**gen_kwargs)
108
+ raw_output = processor.batch_decode(output_ids, skip_special_tokens=True)[0].strip()
109
+
110
+ # Attempt to split JSON and narrative
111
+ structured = {}
112
+ narrative = ""
113
+ try:
114
+ # Heuristic: locate first '{' and matching '}' to extract JSON
115
+ first_brace = raw_output.find("{")
116
+ if first_brace != -1:
117
+ # naive brace matching
118
+ depth = 0
119
+ for idx in range(first_brace, len(raw_output)):
120
+ if raw_output[idx] == "{":
121
+ depth += 1
122
+ elif raw_output[idx] == "}":
123
+ depth -= 1
124
+ if depth == 0:
125
+ json_text = raw_output[first_brace: idx + 1]
126
+ # Parse JSON (allow trailing commas / minor fixes)
127
+ structured = json.loads(json_text.replace("\n", " ").strip())
128
+ narrative = raw_output[idx + 1 :].strip()
129
+ break
130
+ if not structured:
131
+ # Fallback: try to parse entire output as JSON
132
+ structured = json.loads(raw_output)
133
+ else:
134
+ # No JSON detected; fallback to putting entire caption into summary_alt_text
135
+ narrative = raw_output
136
+ except Exception:
137
+ # Parsing failed: fall back to minimal structure
138
+ structured = {
139
+ "main_objects": None,
140
+ "visible_text": None,
141
+ "background": None,
142
+ "colors_materials": None,
143
+ "spatial_relationships": None,
144
+ "uncertainties": None,
145
+ "summary_alt_text": None,
146
  }
147
+ narrative = raw_output
148
+
149
+ # Ensure summary_alt_text exists
150
+ if not structured.get("summary_alt_text"):
151
+ # Create a one-sentence alt text from narrative or main_objects
152
+ if narrative:
153
+ # naive truncation
154
+ structured["summary_alt_text"] = narrative.split(".")[0].strip() + "."
155
+ else:
156
+ structured["summary_alt_text"] = "Image description unavailable."
157
+
158
+ result = {
159
+ "structured": structured,
160
+ "narrative": narrative,
161
+ "device": device,
162
+ "note": (
163
+ "Running on CPU: output is shorter and deterministic." if not use_cuda else "Running on GPU: full-quality generation."
164
+ ),
165
+ }
166
+ return result
167
+
168
+ except RuntimeError as e:
169
+ if "out of memory" in str(e).lower():
170
+ if use_cuda:
171
+ torch.cuda.empty_cache()
172
+ return {"error": "Out of memory during generation. Try smaller image or reduce tokens."}
173
+ return {"error": f"Runtime error during generation: {e}"}
174
+ except Exception as e:
175
+ return {"error": f"Unexpected error: {e}"}
176
+
177
+
178
+ def gradio_wrapper(image: Image.Image):
179
+ out = generate_structured_alt_text(image)
180
+ # Format output: pretty JSON plus narrative
181
+ if "error" in out:
182
+ return out["error"]
183
+ structured = out["structured"]
184
+ human_readable = out["narrative"]
185
+ # Combine into a display string
186
+ display = "Structured JSON:\n"
187
+ try:
188
+ display += json.dumps(structured, indent=2, ensure_ascii=False)
189
+ except Exception:
190
+ display += str(structured)
191
+ display += f"\n\nNarrative summary:\n{human_readable}\n\n(Note: {out['note']})"
192
+ return display
193
+
194
+
195
+ iface = gr.Interface(
196
+ fn=gradio_wrapper,
197
  inputs=gr.Image(type="pil"),
198
+ outputs=gr.Textbox(label="Structured Alt Text + Summary", lines=15),
199
+ title="Structured Alt Text Generator (LLaVA) with CPU Fallback",
200
+ description=(
201
+ "Generates a JSON breakdown of the image (objects, text, background, etc.) plus a concise alt-text sentence. "
202
+ "If running on CPU, it automatically falls back to a lighter generation mode."
203
+ ),
204
+ allow_flagging="never",
205
+ )
206
+
207
+ if __name__ == "__main__":
208
+ iface.launch()