Workpam commited on
Commit
bca956f
·
verified ·
1 Parent(s): 17722fa

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +44 -12
app.py CHANGED
@@ -32,6 +32,23 @@ if device.type == "cuda":
32
  ocr_reader = easyocr.Reader(["en"], gpu=(device.type == "cuda"))
33
 
34
  # === Default label sets (extendable) ===
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
35
  DEFAULT_PRODUCT_LABELS = [
36
  "trucker hat", "baseball cap", "cap", "board game box", "product box", "doll",
37
  "brush marker", "paint marker", "marker", "card game box", "bottle", "package",
@@ -141,21 +158,36 @@ def extract_and_clean_ocr(image: Image.Image):
141
  # === Captioning with BLIP (ensemble of prompts) ===
142
 
143
  def get_blip_caption(image: Image.Image, prompt: str = None):
144
- query = prompt or "Describe the product in the image including its type, visible text/logo, brand if any, and key visual features."
145
- inputs = blip_processor(image, text=query, return_tensors="pt").to(device)
146
- with torch.inference_mode():
147
- out = blip_model.generate(
148
- **inputs,
149
- max_new_tokens=60 if prompt is None else 40,
150
- num_beams=4,
151
- do_sample=False,
152
- early_stopping=True,
153
- no_repeat_ngram_size=2
154
- )
155
- caption = blip_processor.decode(out[0], skip_special_tokens=True)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
156
  caption = re.sub(r'^(the|a|an)\s+', '', caption, flags=re.IGNORECASE).strip()
157
  return caption.capitalize()
158
 
 
159
  def get_background_caption(image: Image.Image):
160
  prompt = "Describe only the background of this image, excluding the main object."
161
  caption = get_blip_caption(image, prompt=prompt)
 
32
  ocr_reader = easyocr.Reader(["en"], gpu=(device.type == "cuda"))
33
 
34
  # === Default label sets (extendable) ===
35
+
36
+ from difflib import get_close_matches
37
+
38
+ # a small reservoir of likely words to help clean OCR; expand as needed
39
+ OCR_DICTIONARY = [
40
+ "the", "world", "of", "blame", "man", "watch", "pink", "Mattel", "logo", "edition",
41
+ "limited", "series", "heroism", "teamwork", "flash", "point"
42
+ ]
43
+
44
+ def normalize_ocr_text(raw: str):
45
+ tokens = re.findall(r"\w+", raw.lower())
46
+ cleaned_tokens = []
47
+ for t in tokens:
48
+ match = get_close_matches(t, OCR_DICTIONARY, n=1, cutoff=0.8)
49
+ cleaned_tokens.append(match[0] if match else t)
50
+ return " ".join(cleaned_tokens).capitalize()
51
+
52
  DEFAULT_PRODUCT_LABELS = [
53
  "trucker hat", "baseball cap", "cap", "board game box", "product box", "doll",
54
  "brush marker", "paint marker", "marker", "card game box", "bottle", "package",
 
158
  # === Captioning with BLIP (ensemble of prompts) ===
159
 
160
  def get_blip_caption(image: Image.Image, prompt: str = None):
161
+ def generate_with(p):
162
+ inputs = blip_processor(image, text=p, return_tensors="pt").to(device)
163
+ with torch.inference_mode():
164
+ out = blip_model.generate(
165
+ **inputs,
166
+ max_new_tokens=60 if prompt is None else 40,
167
+ num_beams=4,
168
+ do_sample=False,
169
+ early_stopping=True,
170
+ no_repeat_ngram_size=2
171
+ )
172
+ caption = blip_processor.decode(out[0], skip_special_tokens=True).strip()
173
+ return caption
174
+
175
+ base_prompt = prompt or "Describe the product in the image including its type, visible text/logo, brand if any, and key visual features."
176
+ caption = generate_with(base_prompt)
177
+
178
+ # If caption is too short or just repeats the instruction, retry with a different phrasing
179
+ if (
180
+ len(caption.split()) < 3
181
+ or base_prompt.lower() in caption.lower()
182
+ or re.search(r"describe the product", caption, flags=re.IGNORECASE)
183
+ ):
184
+ alt_prompt = "Provide a concise descriptive caption of the product, its brand, and distinguishing visual attributes."
185
+ caption = generate_with(alt_prompt)
186
+
187
  caption = re.sub(r'^(the|a|an)\s+', '', caption, flags=re.IGNORECASE).strip()
188
  return caption.capitalize()
189
 
190
+
191
  def get_background_caption(image: Image.Image):
192
  prompt = "Describe only the background of this image, excluding the main object."
193
  caption = get_blip_caption(image, prompt=prompt)