Update app.py
Browse files
app.py
CHANGED
|
@@ -32,6 +32,23 @@ if device.type == "cuda":
|
|
| 32 |
ocr_reader = easyocr.Reader(["en"], gpu=(device.type == "cuda"))
|
| 33 |
|
| 34 |
# === Default label sets (extendable) ===
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 35 |
DEFAULT_PRODUCT_LABELS = [
|
| 36 |
"trucker hat", "baseball cap", "cap", "board game box", "product box", "doll",
|
| 37 |
"brush marker", "paint marker", "marker", "card game box", "bottle", "package",
|
|
@@ -141,21 +158,36 @@ def extract_and_clean_ocr(image: Image.Image):
|
|
| 141 |
# === Captioning with BLIP (ensemble of prompts) ===
|
| 142 |
|
| 143 |
def get_blip_caption(image: Image.Image, prompt: str = None):
|
| 144 |
-
|
| 145 |
-
|
| 146 |
-
|
| 147 |
-
|
| 148 |
-
|
| 149 |
-
|
| 150 |
-
|
| 151 |
-
|
| 152 |
-
|
| 153 |
-
|
| 154 |
-
|
| 155 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 156 |
caption = re.sub(r'^(the|a|an)\s+', '', caption, flags=re.IGNORECASE).strip()
|
| 157 |
return caption.capitalize()
|
| 158 |
|
|
|
|
| 159 |
def get_background_caption(image: Image.Image):
|
| 160 |
prompt = "Describe only the background of this image, excluding the main object."
|
| 161 |
caption = get_blip_caption(image, prompt=prompt)
|
|
|
|
| 32 |
ocr_reader = easyocr.Reader(["en"], gpu=(device.type == "cuda"))
|
| 33 |
|
| 34 |
# === Default label sets (extendable) ===
|
| 35 |
+
|
| 36 |
+
from difflib import get_close_matches
|
| 37 |
+
|
| 38 |
+
# a small reservoir of likely words to help clean OCR; expand as needed
|
| 39 |
+
OCR_DICTIONARY = [
|
| 40 |
+
"the", "world", "of", "blame", "man", "watch", "pink", "Mattel", "logo", "edition",
|
| 41 |
+
"limited", "series", "heroism", "teamwork", "flash", "point"
|
| 42 |
+
]
|
| 43 |
+
|
| 44 |
+
def normalize_ocr_text(raw: str):
|
| 45 |
+
tokens = re.findall(r"\w+", raw.lower())
|
| 46 |
+
cleaned_tokens = []
|
| 47 |
+
for t in tokens:
|
| 48 |
+
match = get_close_matches(t, OCR_DICTIONARY, n=1, cutoff=0.8)
|
| 49 |
+
cleaned_tokens.append(match[0] if match else t)
|
| 50 |
+
return " ".join(cleaned_tokens).capitalize()
|
| 51 |
+
|
| 52 |
DEFAULT_PRODUCT_LABELS = [
|
| 53 |
"trucker hat", "baseball cap", "cap", "board game box", "product box", "doll",
|
| 54 |
"brush marker", "paint marker", "marker", "card game box", "bottle", "package",
|
|
|
|
| 158 |
# === Captioning with BLIP (ensemble of prompts) ===
|
| 159 |
|
| 160 |
def get_blip_caption(image: Image.Image, prompt: str = None):
|
| 161 |
+
def generate_with(p):
|
| 162 |
+
inputs = blip_processor(image, text=p, return_tensors="pt").to(device)
|
| 163 |
+
with torch.inference_mode():
|
| 164 |
+
out = blip_model.generate(
|
| 165 |
+
**inputs,
|
| 166 |
+
max_new_tokens=60 if prompt is None else 40,
|
| 167 |
+
num_beams=4,
|
| 168 |
+
do_sample=False,
|
| 169 |
+
early_stopping=True,
|
| 170 |
+
no_repeat_ngram_size=2
|
| 171 |
+
)
|
| 172 |
+
caption = blip_processor.decode(out[0], skip_special_tokens=True).strip()
|
| 173 |
+
return caption
|
| 174 |
+
|
| 175 |
+
base_prompt = prompt or "Describe the product in the image including its type, visible text/logo, brand if any, and key visual features."
|
| 176 |
+
caption = generate_with(base_prompt)
|
| 177 |
+
|
| 178 |
+
# If caption is too short or just repeats the instruction, retry with a different phrasing
|
| 179 |
+
if (
|
| 180 |
+
len(caption.split()) < 3
|
| 181 |
+
or base_prompt.lower() in caption.lower()
|
| 182 |
+
or re.search(r"describe the product", caption, flags=re.IGNORECASE)
|
| 183 |
+
):
|
| 184 |
+
alt_prompt = "Provide a concise descriptive caption of the product, its brand, and distinguishing visual attributes."
|
| 185 |
+
caption = generate_with(alt_prompt)
|
| 186 |
+
|
| 187 |
caption = re.sub(r'^(the|a|an)\s+', '', caption, flags=re.IGNORECASE).strip()
|
| 188 |
return caption.capitalize()
|
| 189 |
|
| 190 |
+
|
| 191 |
def get_background_caption(image: Image.Image):
|
| 192 |
prompt = "Describe only the background of this image, excluding the main object."
|
| 193 |
caption = get_blip_caption(image, prompt=prompt)
|