works with 1 page pdf canadian tire :)

app.py

@@ -12,7 +12,7 @@ import gradio as gr
 from ultralytics import YOLO
 
 
-search_google = True
+search_google = False
 # GOOGLE_API_KEY = "AIzaSyCD5eIu3N2uD6Vf5GE0VLBpkPsJ3mEZgA0"
 # GOOGLE_CSE_ID = "b536303d151b94476"
 # GOOGLE_API_KEY = "AIzaSyBUvPKAZa2TddKYDJI9t0ac2A_JJqdgWHo"

app_file.py

@@ -12,7 +12,7 @@ import gradio as gr
 from ultralytics import YOLO
 import pypdfium2 as pdfium
 
-search_google = True
+search_google = False
 # GOOGLE_API_KEY = "AIzaSyCD5eIu3N2uD6Vf5GE0VLBpkPsJ3mEZgA0"
 # GOOGLE_CSE_ID = "b536303d151b94476"
 # GOOGLE_API_KEY = "AIzaSyBUvPKAZa2TddKYDJI9t0ac2A_JJqdgWHo"
@@ -76,14 +76,20 @@ def pdf_to_image(pdf):
     assert n_pages > 0, "The PDF document is empty"
     assert n_pages == 1, "The PDF document has more than one page"
     page = pdf.get_page(0)
-    pil_image = page.render().to_pil()
+    bitmap = page.render(
+        scale=5,  # 72dpi resolution
+        rotation=0,  # no additional rotation
+        # ... further rendering options
+    )
+    pil_image = bitmap.to_pil()
+    pil_image.save('test.png')
     width_img, height_img = pil_image.size
-    print("width", width_img, "height", height_img)
-    width, height = page.get_size()
-    print("width", width, "height", height)
-    assert int(width) == width_img, "The width of the image is not the same as the width of the page"
-    assert int(height) == height_img, "The height of the image is not the same as the height of the page"
-    return pil_image, width, height, page
+    print("width image", width_img, "height image", height_img)
+    width_pdf, height_pdf = page.get_size()
+    print("width pdf ", width_pdf, "height pdf ", height_pdf)
+    #assert int(width_pdf) == width_img, "The width of the image is not the same as the width of the page"
+    #assert int(height_pdf) == height_img, "The height of the image is not the same as the height of the page"
+    return pil_image, width_pdf, height_pdf, page
 
 def app(pdf, selected_shop):
     if pdf is None:
@@ -165,13 +171,16 @@ with gr.Blocks() as demo:
         # texts = []
         sku = []
 
-        image, width, height, first_page = pdf_to_image(pdf)
+        image, width_pdf, height_pdf, first_page = pdf_to_image(pdf)
         image_name ="test.jpg"
         image.save(image_name)
 
         # # Load a text page helper
         textpage = first_page.get_textpage()
 
+        text_all = textpage.get_text_range()
+        print("text_all", text_all)
+
         #print("selected_shop:", selected_shop)
         search_url = shops[selected_shop]['url']
         ref_number_regex = shops[selected_shop]['regex']
@@ -181,34 +190,60 @@ with gr.Blocks() as demo:
             brands = [brand.upper() for brand in brands]
 
         if mode == 'rbf':
-            predictions = get_yolo_predictions(img)
+            predictions = get_yolo_predictions(image)
         else:
             # local inference
             results = get_yolo_predictions_local(image, confidence_threshold)
             for result in results:
-                coordinates = result.boxes.xyxy
+                coordinates = result.boxes.xyxyn
                 coordinates = coordinates.to("cpu")
                 coordinates = coordinates.numpy()
                 print("FOUND {} objects".format(len(coordinates)))
+                # print("coordinates", coordinates)
                 for idx, coord in enumerate(coordinates):
                     # crop the image to the bounding box
                     # expansion_factor = 0.01
-                    x = int(coord[0])
-                    y = int(coord[1])
-                    w = int(coord[2] - coord[0])
-                    h = int(coord[3] - coord[1])
+                    x = coord[0]
+                    y = coord[1]
+                    w = coord[2] - coord[0]
+                    h = coord[3] - coord[1]
                     # sub_image = image[y:y + h, x:x + w]
+                    print("x, y, w, h", x, y, w, h)
+
+                    print('width_pdf', width_pdf)
+                    print('height_pdf', height_pdf)
+
+                    enlarge_factor = 0.01
+                    pdf_x = x * width_pdf
+                    pdf_y = y * height_pdf + enlarge_factor * height_pdf
+                    pdf_w = w * width_pdf + enlarge_factor * width_pdf
+                    pdf_h = h * height_pdf
+                    print("PDF", pdf_x, pdf_y, pdf_w, pdf_h)
+
+                    img_x = int(x * image.width)
+                    img_y = int(y * image.height)
+                    img_w = int(w * image.width)
+                    img_h = int(h * image.height)
+                    print("IMG", img_x, img_y, img_w, img_h)
 
 
                     if shops[selected_shop]['url'] is None or shops[selected_shop]['url'] == "":
                         # just append the hotspot to hostpots
-                        hostpots.append(((x, y, x + w, y + h), str(idx)))
+                        hostpots.append(((img_x, img_y, img_x + img_w, img_y + img_h), str(idx)))
                         continue
 
                     print('---------------------')
                     # Extract text
-                    text = textpage.get_text_bounded(left=x+10, bottom=height-y-h+10, right=x+w-10, top=height-y-10)
-                    print("Coordinates", x, y, w, h)
+                    # text = textpage.get_text_bounded(left=pdf_x - enlarge_factor * width_pdf,
+                    #                                  bottom=height_pdf - pdf_y - pdf_h - (enlarge_factor*height_pdf),
+                    #                                  right=pdf_x+pdf_w + (enlarge_factor*width_pdf),
+                    #                                  top=height_pdf-pdf_y + (enlarge_factor*height_pdf))
+                    text = textpage.get_text_bounded(left=pdf_x + 50,
+                                                     bottom=height_pdf - pdf_y - pdf_h,
+                                                     right=pdf_x + pdf_w + 50,
+                                                     top=height_pdf - pdf_y)
+
+
                     print("TEXT", text)
 
                     # for object_number, text in enumerate(texts, start=1):
@@ -290,6 +325,7 @@ with gr.Blocks() as demo:
 
                     #print("PRODUCT NAME: ", product_name)
 
+                    print("FOUND {} ref numbers".format(len(ref_numbers)))
                     for ref in ref_numbers:
                         if search_google:
                             # Use Google Search API to find items with the SKU
@@ -305,8 +341,13 @@ with gr.Blocks() as demo:
                         if ref not in sku:
                             sku.append(ref)
                             # if search_query not in hotspots_sku_links:
-                        hotspots_sku_links.append([{'sku': ref, 'url': search_query, 'prices': prices, 'product_name': product_name}])
-                        hostpots.append(((int(coord[0]), int(coord[1]), int(coord[2]), int(coord[3])), str(ref)))
+                            hotspots_sku_links.append([{'sku': ref, 'url': search_query, 'prices': prices, 'product_name': product_name}])
+                            hostpots.append(((img_x, img_y, (img_x + img_w), (img_h+img_y)), str(ref)))
+                            print("----- ADDED ------")
+                    if len(ref_numbers) == 0:
+                        hostpots.append(((img_x, img_y, (img_x + img_w), (img_h+img_y)), str(idx)))
+                        hotspots_sku_links.append([{'sku': None, 'url': None, 'prices': None, 'product_name': None}])
+                        print("++++++ ADDED +++++")
 
             print("hotspots_sku_links: ", hotspots_sku_links)
             print("skus: ", sku)