File size: 16,483 Bytes
7544e55
 
 
 
 
 
59fb6b1
 
 
7544e55
 
 
 
c79fcf6
 
 
59fb6b1
 
 
7544e55
 
 
 
 
 
 
 
5cc6315
7544e55
 
 
 
cb9aed1
 
 
 
 
 
 
 
 
 
ef45551
 
a8126f9
 
 
 
4c55c98
a8126f9
 
 
 
 
 
 
 
61bc4d8
040af7c
a8126f9
 
4fe4062
a8126f9
 
 
 
cb9aed1
7544e55
 
 
 
 
 
 
 
 
dc4d307
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
43be9bf
 
 
dc4d307
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1921322
dc4d307
1921322
a8126f9
 
2f81e5b
220d180
a8126f9
cfe09e9
52316b5
a8126f9
 
 
 
dc4d307
 
 
 
 
 
 
 
 
 
 
 
 
7544e55
dc4d307
 
 
 
 
 
a8126f9
 
 
 
 
dc4d307
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
7544e55
dc4d307
 
0d70c72
dc4d307
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1921322
dc4d307
 
 
 
 
 
 
 
 
27328ef
dc4d307
 
 
 
 
 
27328ef
 
 
dc4d307
27328ef
dc4d307
 
 
 
 
 
 
27328ef
dc4d307
 
 
 
 
 
 
 
 
 
 
 
 
 
 
21ff58d
 
 
 
 
 
 
 
 
 
 
 
 
 
dc4d307
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
be8e1ef
dc4d307
 
 
 
0d70c72
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
from datetime import datetime
from uuid import uuid4
from fastapi import FastAPI, HTTPException, UploadFile, File, Request, BackgroundTasks, Form
from fastapi.middleware.cors import CORSMiddleware
from pydantic import BaseModel
from httpx import Timeout
import numpy as np
import cv2
import base64
import httpx
import logging
import boto3
import base64
import firebase_admin
from firebase_admin import credentials
from firebase_admin import firestore

app = FastAPI()

app.add_middleware(
    CORSMiddleware,
    allow_origins=["*"],  # Allows all origins
    allow_credentials=True,  # Allows credentials (such as cookies) to be sent with requests
    allow_methods=["*"],  # Allows all methods
    allow_headers=["*"],  # Allows all headers
)

timeout = Timeout(1000.0, connect=600.0)

logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
logging.info("Application starting up")

def get_secret(secret_name):
    secret_path = f"/app/secrets/{secret_name}.txt"
    try:
        with open(secret_path, 'r') as secret_file:
            return secret_file.read().strip()
    except FileNotFoundError:
        raise Exception(f"Secret file for {secret_name} not found")

# Use the function to get your DEZGO_API_KEY
dezgo_api_key = get_secret('dezgo_api_key')
appsScriptsURL = get_secret('appsscript_url')

firebase_config = {
  "type": "service_account",
  "project_id": "resheet-k6x6vh",
  "private_key_id": get_secret('fb_priv_key_id'),
  "private_key": get_secret('fb_priv_key').replace('\\n', '\n'),
  "client_email": get_secret('client_email'),
  "client_id": get_secret('client_id'),
  "auth_uri": "https://accounts.google.com/o/oauth2/auth",
  "token_uri": "https://oauth2.googleapis.com/token",
  "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
  "client_x509_cert_url": get_secret('client_x509_cert_url'),
  "universe_domain": "googleapis.com"
}

print(firebase_config)
cred = credentials.Certificate(firebase_config)
firebase_admin.initialize_app(cred, {
    'projectId': firebase_config['project_id'],
})

db = firestore.client()
users_ref = db.collection('users')

class ImageData(BaseModel):
    sheetID: str
    base64Image: str
    fileName: str
    mimeType: str

class UnprocessedImage(BaseModel):
    sheetID: str
    s3Url: str

@app.post("/redirect")
async def redirect_to_google_apps_script(request: Request):
    # Get the body of the incoming request
    body = await request.body()

    async with httpx.AsyncClient(timeout=300.0, follow_redirects=True) as client:  # Follow redirects
        response = await client.post(
            'https://script.google.com/macros/s/AKfycbw9_31pMwQVhJPezJ9qlMRCAamQKRREnun-tfT_TnS5TOwOVcfdgU1Y9xTpAv6bZuiKOA/exec',
            data=body
        )
        if response.status_code != 200:
            logging.error(f"Unexpected response from Google Apps Script API: {response.status_code} - {response.text}")
            raise HTTPException(status_code=500, detail="Error from external API.")
        return response.json()

dezgo_headers = {
    'X-Dezgo-Key': dezgo_api_key,
}

# update the google sheet with failure
async def record_failure(s3_original_url, failed_task, sheetID):
    body = {
        "s3Url": s3_original_url,
        "mode": "failed",
        "message": failed_task,
        "sheetID": sheetID,
    }
    async with httpx.AsyncClient(follow_redirects=True) as client:
        response = await client.post(appsScriptsURL, json=body)
        if response.status_code != 200:
            logging.error(f"Unexpected response from Google Apps Script API: {response.status_code} - {response.text}")
            raise HTTPException(status_code=500, detail="Error from external API.")

async def post_to_google_apps_script(appsScriptsURL, body, s3_orginal_url):
    async with httpx.AsyncClient(follow_redirects=True) as client:
        response = await client.post(appsScriptsURL, json=body)
        if response.status_code != 200:
            logging.error(f"Unexpected response from Google Apps Script API: {response.status_code} - {response.text}")
            await record_failure(s3_orginal_url, 'Failed to queue processed image.', body.sheetID)

async def background_worker(sheeturl, image_contents, filename, content_type, appsScriptsURL, user_ref):
    sheetID = sheeturl.split('/')[-2]
    user_doc = users_ref.document(user_ref)
    # get the 'total_uploaded' field from the user's document
    user_data = user_doc.get().to_dict()
    logging.info(user_data)
    
    if user_data is None:
        logging.info('user not found')
        await record_failure('', 'User not found.', sheetID)
        return {"message": "User not found."}
    total_uploaded = user_data.get('total_uploaded', 0)

    
    image_data = {
        'sheetID': sheetID,
        'fileName': filename,
        'mimeType': content_type
    }

    s3_client = boto3.client('s3')
    bucket_name = 're-sheet'
    object_name = sheetID + '/' + image_data['fileName']
    cut_images = []
    s3_orginal_url = f"https://{bucket_name}.s3.amazonaws.com/{object_name}"
    masked_image = await detect_receipts(image_contents, s3_orginal_url, sheetID)
    await cutting_image(masked_image, image_contents, cut_images, s3_orginal_url, sheetID)
    
    queued = []
    # save the processed image(s) to s3
    if not cut_images:
        await record_failure(s3_orginal_url, 'No valid receipts found.', sheetID)
        return {"message": "No valid receipts found."}
    else:
        # get the number of total cut photos
        receipt_count = cut_images.__len__()
        # set the user's total_uploaded field to the current total_uploaded + receipt_count
        user_doc.update({'total_uploaded': total_uploaded + receipt_count})
        
        for i, base64_str in enumerate(cut_images, start=1):
            image_bytes = base64.b64decode(base64_str)
            object_name = f"{object_name}_processed_{i}.png"
            response = s3_client.put_object(
                Bucket=bucket_name,
                Key=object_name,
                Body=image_bytes,
                ContentType='image/png'
            )
            status = response.get("ResponseMetadata", {}).get("HTTPStatusCode")
            if status != 200:
                await record_failure(s3_orginal_url, 'Failed to upload processed image to S3.', sheetID)
                return {"message": "Failed to upload processed image to S3."}
            # otherwise put it as queued
            object_s3_url = f"https://{bucket_name}.s3.amazonaws.com/{object_name}"
            queued.append(object_s3_url)
            # queue the image to google apps script web app
        body ={
            "queued": queued,
            "mode": "mobile",
            "sheetID": sheetID,
        }
        
        print(body)
        logging.info(body)
        await post_to_google_apps_script(appsScriptsURL, body, s3_orginal_url)
        return {"message": "Image processing completed."}
    
# receive a sheetID and an image as multipart form data
@app.post('/mobile-process-receipts')
async def mobile_process_receipts(background_tasks: BackgroundTasks, sheeturl: str = Form(...), user_ref: str = Form(...), image: UploadFile = File(...)):
    image_contents = await image.read()
    uuid = str(uuid4())
    timestamp = str(datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
    sheetID = sheeturl.split('/')[-2]
    
    message = "์˜์ˆ˜๊ธฐ AI๊ฐ€ ๋ฐ์ดํ„ฐ ๋ถ„์„ ์ค‘์ž…๋‹ˆ๋‹ค. ์ž ์‹œ๋งŒ ๊ธฐ๋‹ค๋ ค์ฃผ์„ธ์š”."
    data = [timestamp, uuid, message]
    body = {
        "sheetID": sheetID,
        "data": data,
        "mode": "alert-user"
    }
    async with httpx.AsyncClient(follow_redirects=True) as client:
        response = await client.post(appsScriptsURL, json=body)
        if response.status_code != 200:
            logging.error(f"Unexpected response from Google Apps Script API: {response.status_code} - {response.text}")
            raise HTTPException(status_code=500, detail="Error from external API.")
        
    background_tasks.add_task(background_worker, sheeturl, image_contents, uuid, image.content_type, appsScriptsURL, user_ref)    

    return {"message": "Image processing completed."}


@app.post('/process-receipts')
async def mobile_process_receipts(background_tasks: BackgroundTasks, request: Request):
    data = await request.json()
    base64_str = data['base64Image']
    prefix, base64_str = base64_str.split(',', 1)  # Split on the first comma

    content_type = data['mimeType']
    sheeturl = data['sheetID']
    
    checkPhoto = {
        'sheetID': sheeturl.split('/')[-2],
        'mode': "check",
        'photo': base64_str,
        'mimeType': content_type,
        'fileName': 'test.png'
    }
    async with httpx.AsyncClient(timeout=300, follow_redirects=True) as client:
        response = await client.post(appsScriptsURL, json=checkPhoto)
        if response.status_code != 200:
            logging.error(f"Unexpected response from Google Apps Script API: {response.status_code} - {response.text}")
            raise HTTPException(status_code=500, detail="Error from external API.")
        if response.json()['status'] == 'invalid':
            return {"message": "Invalid photo."}

    image_contents = base64.b64decode(base64_str)
    uuid = str(uuid4())
    timestamp = str(datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
    message = "์˜์ˆ˜๊ธฐ AI๊ฐ€ ๋ฐ์ดํ„ฐ ๋ถ„์„ ์ค‘์ž…๋‹ˆ๋‹ค. ์ž ์‹œ๋งŒ ๊ธฐ๋‹ค๋ ค์ฃผ์„ธ์š”."
    data = [timestamp, uuid, message]
    body = {
        "sheetID": sheeturl.split('/')[-2],
        "data": data,
        "mode": "alert-user"
    }
    async with httpx.AsyncClient(follow_redirects=True) as client:
        response = await client.post(appsScriptsURL, json=body)
        if response.status_code != 200:
            logging.error(f"Unexpected response from Google Apps Script API: {response.status_code} - {response.text}")
            raise HTTPException(status_code=500, detail="Error from external API.")
        
    # before continuing on, search db for user via sheetID
    sheetID = sheeturl.split('/')[-2]
    # Query the database for documents where 'sheetID' field is equal to sheetID
    query_ref = users_ref.where('sheetID', '==', sheetID)
    # Get the documents that match the query
    user_docs = query_ref.stream()

    # user_docs is an iterator of DocumentSnapshot
    # You can convert it to a list if you want to work with it directly
    user_docs = list(user_docs)

    user_ref = user_docs[0].id
    
    background_tasks.add_task(background_worker, sheeturl, image_contents, uuid, content_type, appsScriptsURL,user_ref)    

    return {"message": "Image processing completed."}
    
# send the image to dezgo
async def detect_receipts(image_contents, s3_orginal_url, sheetID):
    files = {
        'image': ('filename', image_contents),
        'mode': 'mask',
    }
    logging.info("Sending image to Dezgo API")
    async with httpx.AsyncClient(timeout=timeout) as client:
        response = await client.post('https://api.dezgo.com/remove-background', headers=dezgo_headers, files=files)
        # if successful, pass the masked image to cutting_image function
        if response.status_code == 200:
            masked_image = response.content
            return masked_image
        else:
            logging.error(f"Unexpected response from Dezgo API: {response.status_code} - {response.text}")
            await record_failure(s3_orginal_url, 'Dezgo API failed.', sheetID)
            raise HTTPException(status_code=500, detail="Error from external API.")

# receive the masked image and process it
async def cutting_image(masked_image, image_contents, cut_images, s3_orginal_url, sheetID):
    # Convert the masked_image (which is in bytes format) to a numpy array. 
    # The dtype of the array is set to np.uint8 because images are usually 8-bit per channel.
    nparr_masked = np.frombuffer(masked_image, np.uint8)

    # Check if the size of the numpy array is 0, which means that the conversion failed.
    if nparr_masked.size == 0:
        # Log an error message indicating that the conversion failed.
        logging.error("Failed to convert API response to numpy array.")
        # Record the reason for the failure.
        record_failure(s3_orginal_url, 'failed to convert API response to numpy array.', sheetID)
        # Raise an HTTPException with a 500 status code (Internal Server Error) and a detail message.
        raise HTTPException(status_code=500, detail="Error processing API response data.")

    # Decode the numpy array to an image using OpenCV. The flag cv2.IMREAD_COLOR loads the image in color.
    masked_image = cv2.imdecode(nparr_masked, cv2.IMREAD_COLOR)

    # Check if the decoding failed, which would result in masked_image being None.
    if masked_image is None:
        # Log an error message indicating that the decoding failed.
        logging.error("Failed to decode image data.")
        # Record the reason for the failure.
        record_failure(s3_orginal_url, 'failed to decode image data.', sheetID)
        # Raise an HTTPException with a 500 status code (Internal Server Error) and a detail message.
        raise HTTPException(status_code=500, detail="Error decoding image data.")
    
    nparr_original = np.frombuffer(image_contents, np.uint8)
    original_image = cv2.imdecode(nparr_original, cv2.IMREAD_COLOR)
    
    original_aspect_ratio = original_image.shape[1] / original_image.shape[0]
    masked_aspect_ratio = masked_image.shape[1] / masked_image.shape[0]
    
    # if the image has been rotated by dezgo, rotate it back
    if abs(original_aspect_ratio - masked_aspect_ratio) > 0.1: 
        # Rotate the masked image 90 degrees clockwise
        masked_image = cv2.rotate(masked_image, cv2.ROTATE_90_CLOCKWISE)
    
    original_image_shape = original_image.shape[:2]
    
    resized_masked_image = cv2.resize(masked_image, (original_image_shape[1], original_image_shape[0]))
    


    blurred_masked_image = cv2.GaussianBlur(resized_masked_image, (5, 5), 0)
    _, thresh_masked_image = cv2.threshold(blurred_masked_image, 175, 255, cv2.THRESH_BINARY)
    edges = cv2.Canny(thresh_masked_image, 75, 150)
    contours, hierarchy = cv2.findContours(edges.copy(), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)

    padding = 20
    for contour in contours:
        perimeter = cv2.arcLength(contour, True)
        approximation = cv2.approxPolyDP(contour, 0.02 * perimeter, True)
        x, y, w, h = cv2.boundingRect(approximation)

        # Add padding to the bounding box coordinates while ensuring they are within the image boundaries
        x_pad = max(x - padding, 0)
        y_pad = max(y - padding, 0)
        w_pad = min(w + 2 * padding, original_image.shape[1] - x_pad)
        h_pad = min(h + 2 * padding, original_image.shape[0] - y_pad)
        
        # logging.info(f"Bounding box: x={x_pad}, y={y_pad}, w={w_pad}, h={h_pad}")

        # Check if the area of the bounding rectangle is greater than 50px x 50px
        if w_pad > 150 or h_pad > 150:
            cropped_image = original_image[y_pad:y_pad + h_pad, x_pad:x_pad + w_pad]
            _, img_encoded = cv2.imencode('.jpg', cropped_image, [int(cv2.IMWRITE_JPEG_QUALITY), 90])
            base64_str = base64.b64encode(img_encoded).decode('utf-8')
            cut_images.append(base64_str)


# @app.post('/test')
# async def mobile_process_receipts(background_tasks: BackgroundTasks, sheeturl: str = Form(...), user_ref: str = Form(...), image: UploadFile = File(...)):
#     image_contents = await image.read()
#     uuid = str(uuid4())
#     timestamp = str(datetime.now().strftime("%Y-%m-%d-%H-%M-%S"))
#     message = "์˜์ˆ˜๊ธฐ๊ฐ€ ๋ฐ์ดํ„ฐ ๋ถ„์„ ์ค‘์ž…๋‹ˆ๋‹ค. ์ž ์‹œ๋งŒ ๊ธฐ๋‹ค๋ ค์ฃผ์„ธ์š”."
#     data = [timestamp, uuid, message]
#     body = {
#         "sheetID": sheeturl.split('/')[-2],
#         "data": data,
#         "mode": "alert-user"
#     }
#     async with httpx.AsyncClient(follow_redirects=True) as client:
#         response = await client.post(appsScriptsURL, json=body)
#         if response.status_code != 200:
#             logging.error(f"Unexpected response from Google Apps Script API: {response.status_code} - {response.text}")
#             raise HTTPException(status_code=500, detail="Error from external API.")
#     background_tasks.add_task(background_worker, sheeturl, image_contents, image.filename, image.content_type, appsScriptsURL)    
#     return {"message": "Image processing completed."}