m8chaa commited on
Commit
02fba21
·
verified ·
1 Parent(s): 07ba5c4

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +431 -420
app.py CHANGED
@@ -1,23 +1,23 @@
1
- from datetime import datetime
2
- from uuid import uuid4
3
  from fastapi import FastAPI, HTTPException, UploadFile, File, Request, BackgroundTasks, Form
4
  from fastapi.middleware.cors import CORSMiddleware
5
  from pydantic import BaseModel
6
  from httpx import Timeout
7
- import numpy as np
8
- import cv2
9
- import base64
10
  import httpx
11
  import logging
12
- import boto3
13
  import base64
14
  import firebase_admin
15
  from firebase_admin import credentials
16
  from firebase_admin import firestore
17
  import json
18
- from typing import List
 
 
 
 
 
19
  # start application
20
  app = FastAPI()
 
21
  # allow all origins
22
  app.add_middleware(
23
  CORSMiddleware,
@@ -26,453 +26,464 @@ app.add_middleware(
26
  allow_methods=["*"], # Allows all methods
27
  allow_headers=["*"], # Allows all headers
28
  )
 
29
  # set up logging
30
  logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
31
  logging.info("Application starting up")
32
 
33
- # get secret from file
34
- def get_secret(secret_name):
35
- secret_path = f"/app/secrets/{secret_name}.txt"
36
- try:
37
- with open(secret_path, 'r') as secret_file:
38
- return secret_file.read().strip()
39
- except FileNotFoundError:
40
- raise Exception(f"Secret file for {secret_name} not found")
41
-
42
- # get appropriate API keys and URLs
43
- dezgo_api_key = get_secret('dezgo_api_key')
44
- appsScriptsURL = get_secret('appsscript_url')
45
- timeout = Timeout(1000.0, connect=600.0)
46
 
47
  # set up firebase
48
- firebase_config = {
49
- "type": "service_account",
50
- "project_id": "resheet-k6x6vh",
51
- "private_key_id": get_secret('fb_priv_key_id'),
52
- "private_key": get_secret('fb_priv_key').replace('\\n', '\n'),
53
- "client_email": get_secret('client_email'),
54
- "client_id": get_secret('client_id'),
55
- "auth_uri": "https://accounts.google.com/o/oauth2/auth",
56
- "token_uri": "https://oauth2.googleapis.com/token",
57
- "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
58
- "client_x509_cert_url": get_secret('client_x509_cert_url'),
59
- "universe_domain": "googleapis.com"
60
- }
61
- cred = credentials.Certificate(firebase_config)
62
- firebase_admin.initialize_app(cred, {
63
- 'projectId': firebase_config['project_id'],
64
- })
65
- db = firestore.client()
66
- # get db
67
- users_ref = db.collection('users')
68
 
69
 
70
- class ImageData(BaseModel):
71
- sheetID: str
72
- base64Image: str
73
- fileName: str
74
- mimeType: str
75
 
76
- class UnprocessedImage(BaseModel):
77
- sheetID: str
78
- s3Url: str
 
 
 
 
79
 
 
80
 
81
- # functions
82
- # update the google sheet with failure
83
- async def record_failure(s3_original_url, failed_task, sheetID):
84
- body = {
85
- "s3Url": s3_original_url,
86
- "mode": "failed",
87
- "message": failed_task,
88
- "sheetID": sheetID,
89
- }
90
- async with httpx.AsyncClient(timeout=300.0, follow_redirects=True) as client:
91
- response = await client.post(appsScriptsURL, json=body)
92
- if response.status_code != 200:
93
- logging.error(f"Unexpected response from Google Apps Script API: {response.status_code} - {response.text}")
94
- raise HTTPException(status_code=500, detail="Error from external API.")
95
-
96
- # post the processed image to google apps script
97
- async def post_to_google_apps_script(appsScriptsURL, body, s3_orginal_url):
98
- async with httpx.AsyncClient(timeout=300.0, follow_redirects=True) as client:
99
- response = await client.post(appsScriptsURL, json=body)
100
- if response.status_code != 200:
101
- logging.error(f"Unexpected response from Google Apps Script API: {response.status_code} - {response.text}")
102
- await record_failure(s3_orginal_url, 'Failed to queue processed image.', body.sheetID)
103
- raise HTTPException(status_code=500, detail="Error from external API.")
104
- return response.json()
105
-
106
- # background worker
107
- async def background_worker(sheeturl, image_contents, filename, content_type, appsScriptsURL, user_ref):
108
- sheetID = sheeturl.split('/')[-2]
109
- user_doc = users_ref.document(user_ref)
110
- # get the 'total_uploaded' field from the user's document
111
- user_data = user_doc.get().to_dict()
112
- logging.info(user_data)
113
-
114
- if user_data is None:
115
- logging.info('user not found')
116
- await record_failure('', 'User not found.', sheetID)
117
- return {"message": "User not found."}
118
-
119
- image_data = {
120
- 'sheetID': sheetID,
121
- 'fileName': filename,
122
- 'mimeType': content_type
123
- }
124
 
125
- s3_client = boto3.client('s3')
126
- bucket_name = 're-sheet'
127
- object_name = sheetID + '/' + image_data['fileName']
128
- cut_images = []
129
- s3_orginal_url = f"https://{bucket_name}.s3.amazonaws.com/{object_name}"
130
- masked_image = await detect_receipts(image_contents, s3_orginal_url, sheetID)
131
- await cutting_image(masked_image, image_contents, cut_images, s3_orginal_url, sheetID)
132
-
133
- queued = []
134
- # save the processed image(s) to s3
135
- if not cut_images:
136
- await record_failure(s3_orginal_url, 'No valid receipts found.', sheetID)
137
- return {"message": "No valid receipts found."}
138
- else:
139
- for i, base64_str in enumerate(cut_images, start=1):
140
- image_bytes = base64.b64decode(base64_str)
141
- object_name = f"{object_name}_processed_{i}.jpg"
142
- response = s3_client.put_object(
143
- Bucket=bucket_name,
144
- Key=object_name,
145
- Body=image_bytes,
146
- ContentType='image/jpeg'
147
- )
148
- status = response.get("ResponseMetadata", {}).get("HTTPStatusCode")
149
- if status != 200:
150
- await record_failure(s3_orginal_url, 'Failed to upload processed image to S3.', sheetID)
151
- return {"message": "Failed to upload processed image to S3."}
152
- # otherwise put it as queued
153
- object_s3_url = f"https://{bucket_name}.s3.amazonaws.com/{object_name}"
154
- queued.append(object_s3_url)
155
- # queue the image to google apps script web app
156
- body ={
157
- "queued": queued,
158
- "mode": "mobile",
159
- "sheetID": sheetID,
160
- }
161
- logging.info(body)
162
- countedDoc = await post_to_google_apps_script(appsScriptsURL, body, s3_orginal_url)
163
-
164
- updates = {}
165
- if countedDoc['num_receipts'] > 0:
166
- total_uploaded = user_data.get('receipts_uploaded', 0)
167
- current_upload = 'receipts_uploaded'
168
- updates[current_upload] = user_data.get(current_upload, 0) + countedDoc['num_receipts']
169
-
170
- if countedDoc['num_business_cards'] > 0:
171
- total_uploaded = user_data.get('business_cards_uploaded', 0)
172
- current_upload = 'business_cards_uploaded'
173
- updates[current_upload] = user_data.get(current_upload, 0) + countedDoc['num_business_cards']
174
-
175
- if updates:
176
- user_doc.update(updates)
177
-
178
- return {"message": "Image processing completed."}
179
-
180
- # send the image to dezgo
181
- async def detect_receipts(image_contents, s3_orginal_url, sheetID):
182
- files = {
183
- 'image': ('filename', image_contents),
184
- 'mode': 'mask',
185
- }
186
- logging.info("Sending image to Dezgo API")
187
- dezgo_headers = {
188
- 'X-Dezgo-Key': dezgo_api_key,
189
- }
190
- async with httpx.AsyncClient(timeout=timeout) as client:
191
- response = await client.post('https://api.dezgo.com/remove-background', headers=dezgo_headers, files=files)
192
- # if successful, pass the masked image to cutting_image function
193
- if response.status_code == 200:
194
- masked_image = response.content
195
- return masked_image
196
- else:
197
- logging.error(f"Unexpected response from Dezgo API: {response.status_code} - {response.text}")
198
- await record_failure(s3_orginal_url, 'Dezgo API failed.', sheetID)
199
- raise HTTPException(status_code=500, detail="Error from external API.")
200
-
201
- # receive the masked image and process it
202
- async def cutting_image(masked_image, image_contents, cut_images, s3_orginal_url, sheetID):
203
- # Convert the masked_image (which is in bytes format) to a numpy array.
204
- # The dtype of the array is set to np.uint8 because images are usually 8-bit per channel.
205
- nparr_masked = np.frombuffer(masked_image, np.uint8)
206
-
207
- # Check if the size of the numpy array is 0, which means that the conversion failed.
208
- if nparr_masked.size == 0:
209
- # Log an error message indicating that the conversion failed.
210
- logging.error("Failed to convert API response to numpy array.")
211
- # Record the reason for the failure.
212
- record_failure(s3_orginal_url, 'failed to convert API response to numpy array.', sheetID)
213
- # Raise an HTTPException with a 500 status code (Internal Server Error) and a detail message.
214
- raise HTTPException(status_code=500, detail="Error processing API response data.")
215
-
216
- # Decode the numpy array to an image using OpenCV. The flag cv2.IMREAD_COLOR loads the image in color.
217
- masked_image = cv2.imdecode(nparr_masked, cv2.IMREAD_COLOR)
218
-
219
- # Check if the decoding failed, which would result in masked_image being None.
220
- if masked_image is None:
221
- # Log an error message indicating that the decoding failed.
222
- logging.error("Failed to decode image data.")
223
- # Record the reason for the failure.
224
- record_failure(s3_orginal_url, 'failed to decode image data.', sheetID)
225
- # Raise an HTTPException with a 500 status code (Internal Server Error) and a detail message.
226
- raise HTTPException(status_code=500, detail="Error decoding image data.")
227
-
228
- nparr_original = np.frombuffer(image_contents, np.uint8)
229
- original_image = cv2.imdecode(nparr_original, cv2.IMREAD_COLOR)
230
-
231
- original_aspect_ratio = original_image.shape[1] / original_image.shape[0]
232
- masked_aspect_ratio = masked_image.shape[1] / masked_image.shape[0]
233
-
234
- # if the image has been rotated by dezgo, rotate it back
235
- if abs(original_aspect_ratio - masked_aspect_ratio) > 0.1:
236
- # Rotate the masked image 90 degrees clockwise
237
- masked_image = cv2.rotate(masked_image, cv2.ROTATE_90_CLOCKWISE)
238
-
239
- original_image_shape = original_image.shape[:2]
240
-
241
- resized_masked_image = cv2.resize(masked_image, (original_image_shape[1], original_image_shape[0]))
242
-
243
 
 
 
244
 
245
- blurred_masked_image = cv2.GaussianBlur(resized_masked_image, (5, 5), 0)
246
- _, thresh_masked_image = cv2.threshold(blurred_masked_image, 175, 255, cv2.THRESH_BINARY)
247
- edges = cv2.Canny(thresh_masked_image, 75, 150)
248
- contours, hierarchy = cv2.findContours(edges.copy(), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
 
 
249
 
250
- padding = 20
251
- for contour in contours:
252
- perimeter = cv2.arcLength(contour, True)
253
- approximation = cv2.approxPolyDP(contour, 0.02 * perimeter, True)
254
- x, y, w, h = cv2.boundingRect(approximation)
255
 
256
- # Add padding to the bounding box coordinates while ensuring they are within the image boundaries
257
- x_pad = max(x - padding, 0)
258
- y_pad = max(y - padding, 0)
259
- w_pad = min(w + 2 * padding, original_image.shape[1] - x_pad)
260
- h_pad = min(h + 2 * padding, original_image.shape[0] - y_pad)
 
 
 
 
 
 
 
 
 
 
 
 
261
 
262
- # logging.info(f"Bounding box: x={x_pad}, y={y_pad}, w={w_pad}, h={h_pad}")
263
-
264
- # Check if the area of the bounding rectangle is greater than 50px x 50px
265
- if w_pad > 150 or h_pad > 150:
266
- cropped_image = original_image[y_pad:y_pad + h_pad, x_pad:x_pad + w_pad]
267
- _, img_encoded = cv2.imencode('.jpg', cropped_image, [int(cv2.IMWRITE_JPEG_QUALITY), 90])
268
- base64_str = base64.b64encode(img_encoded).decode('utf-8')
269
- cut_images.append(base64_str)
270
 
271
- # redirect to google apps script
272
- @app.post("/redirect")
273
- async def redirect_to_google_apps_script(request: Request):
274
- # Get the body of the incoming request
275
- body = await request.body()
276
- # Check if the body is empty
277
- if not body:
278
- # If the body is empty, raise an HTTPException with a 400 status code (Bad Request) and a detail message.
279
- raise HTTPException(status_code=400, detail="Empty request body.")
280
-
281
- async with httpx.AsyncClient(timeout=300.0, follow_redirects=True) as client: # Follow redirects
282
- response = await client.post(
283
- 'https://script.google.com/macros/s/AKfycbw9_31pMwQVhJPezJ9qlMRCAamQKRREnun-tfT_TnS5TOwOVcfdgU1Y9xTpAv6bZuiKOA/exec',
284
- data=body
285
- )
286
- if response.status_code != 200:
287
- logging.error(f"Unexpected response from Google Apps Script API: {response.status_code} - {response.text}")
288
- raise HTTPException(status_code=500, detail="Error from external API.")
289
- return response.json()
290
 
291
- # receive a sheetID and an image as multipart form data
292
- @app.post('/mobile-process')
293
- async def mobile_process_receipts(background_tasks: BackgroundTasks, sheeturl: str = Form(...), user_ref: str = Form(...), image: UploadFile = File(...)):
294
- image_contents = await image.read()
295
- uuid = str(uuid4())
296
- sheetID = sheeturl.split('/')[-2]
297
- base_64_image = base64.b64encode(image_contents).decode('utf-8')
298
  body = {
299
- "sheetID": sheetID,
300
- "base64Image": base_64_image,
301
- "fileName": uuid,
302
- "mode": "alert-user",
303
- "mimeType": image.content_type,
304
- }
305
- async with httpx.AsyncClient(follow_redirects=True, timeout=timeout) as client:
306
- response = await client.post(appsScriptsURL, json=body)
307
- print(response)
308
- if response.content:
309
- print(response.json())
310
- else:
311
- print("Empty response")
312
- return {"message": "No documents were found."}
313
- if response.status_code != 200:
314
- logging.error(f"Unexpected response from Google Apps Script API: {response.status_code} - {response.text}")
315
- raise HTTPException(status_code=500, detail="Error in trying to connect to your sheet. Please try again.")
316
-
317
- foundList = response.json().get('foundList')
318
- if not foundList:
319
- # alert user that no documents were found
320
- return {"message": "No documents were found."}
321
- # if foundList contains "business_card" then check if business_cards_uploaded is equal to or over 500
322
- user_doc = users_ref.document(user_ref).get().to_dict()
323
- if not user_doc.get('paid', False):
324
- if 'business_card' in foundList:
325
- if user_doc.get('business_cards_uploaded', 0) >= 500:
326
- return {"message": "You have reached the maximum number of business cards uploaded."}
327
- # if foundList contains "receipt" then check if receipts_uploaded is equal to or over 500
328
- if 'receipt' in foundList:
329
- if user_doc.get('receipts_uploaded', 0) >= 500:
330
- return {"message": "You have reached the maximum number of receipts uploaded."}
331
-
332
- background_tasks.add_task(background_worker, sheeturl, image_contents, uuid, image.content_type, appsScriptsURL, user_ref)
333
-
334
- return {"message": "Your image has been queued. It may take up to 5 minutes."}
335
 
 
 
 
336
 
337
- @app.post('/process')
338
- async def mobile_process_receipts(background_tasks: BackgroundTasks, request: Request):
339
- data = await request.json()
340
- base64_str = data['base64Image']
341
- prefix, base64_str = base64_str.split(',', 1) # Split on the first comma
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
342
 
343
- content_type = data['mimeType']
344
- sheeturl = data['sheetID']
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
345
 
346
- image_contents = base64.b64decode(base64_str)
347
- uuid = str(uuid4())
348
- timestamp = str(datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
349
- message = "영수기 AI가 데이터 분석 중입니다. 잠시만 기다려주세요."
350
- data = [timestamp, uuid, message]
351
- body = {
352
- "sheetID": sheeturl.split('/')[-2],
353
- "data": data,
354
- "mode": "alert-user"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
355
  }
356
- async with httpx.AsyncClient(timeout=300.0, follow_redirects=True) as client:
357
- response = await client.post(appsScriptsURL, json=body)
358
- if response.status_code != 200:
359
- logging.error(f"Unexpected response from Google Apps Script API: {response.status_code} - {response.text}")
360
- raise HTTPException(status_code=500, detail="Error from external API.")
361
 
362
- # before continuing on, search db for user via sheetID
363
- sheetID = sheeturl.split('/')[-2]
364
- # Query the database for documents where 'sheetID' field is equal to sheetID
365
- query_ref = users_ref.where('sheetID', '==', sheetID)
366
- # Get the documents that match the query
367
- user_docs = query_ref.stream()
368
-
369
- # user_docs is an iterator of DocumentSnapshot
370
- # You can convert it to a list if you want to work with it directly
371
- user_docs = list(user_docs)
372
-
373
- user_ref = user_docs[0].id
 
 
 
 
374
 
375
- background_tasks.add_task(background_worker, sheeturl, image_contents, uuid, content_type, appsScriptsURL, user_ref)
376
-
377
- return {"message": "Image processing completed."}
378
-
379
-
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
380
 
381
- @app.post('/mobile-sync')
382
- async def sync_contacts(background_tasks: BackgroundTasks, sheeturl: str = Form(...), syncType: str = Form(...), data: str = Form(...), user_ref: str = Form(...),):
383
- data = json.loads(data)
384
- sheetID = sheeturl.split('/')[-2]
385
- body = {
386
- "sheetID": sheetID,
387
- "data": data,
388
- "mode": "sync",
389
- "syncType": syncType,
390
- "userRef": user_ref,
391
- }
392
 
393
- print(body)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
394
 
395
- background_tasks.add_task(post_to_url, background_tasks, appsScriptsURL, body)
396
- return {"message": "Contacts processing completed."}
397
-
398
- async def post_to_url(background_tasks: BackgroundTasks, url: str, json: dict):
399
- async with httpx.AsyncClient(timeout=300.0, follow_redirects=True) as client:
400
- response = await client.post(url, json=json)
401
- if response.status_code != 200:
402
- logging.error(f"Unexpected response from Google Apps Script API: {response.status_code} - {response.text}")
403
- raise HTTPException(status_code=500, detail="Error from external API.")
404
-
405
- @app.post('/update-userdoc')
406
- async def update_user_doc(request:Request, background_tasks: BackgroundTasks):
407
- data = await request.json()
408
- print(data)
409
- user_doc = users_ref.document(data['userRef'])
410
- # get the 'total_uploaded' field from the user's document
411
- user_data = user_doc.get().to_dict()
412
 
413
- if data['syncType'] == 'sms':
414
- sms_uploaded = user_data.get('sms_uploaded', 0)
415
- user_doc.update({'sms_uploaded': sms_uploaded + data['status']})
416
- elif data['syncType'] == 'contacts':
417
- contact_uploaded = user_data.get('contact_uploaded', 0)
418
- user_doc.update({'contact_uploaded': contact_uploaded + data['status']})
419
- return {"message": "User document updated."}
420
-
421
- from fastapi import FastAPI, Request, HTTPException
422
- from fastapi.responses import JSONResponse
423
-
424
- @app.exception_handler(HTTPException)
425
- async def http_exception_handler(request: Request, exc: HTTPException):
426
- return JSONResponse(
427
- status_code=exc.status_code,
428
- content={"message": f"Oops! {exc.detail}. You made a request to {request.url}"}
429
- )
430
 
431
- @app.post('/check-userdoc')
432
- async def check_user_doc(currentUserUid: str = Form(...)):
433
- user_doc = users_ref.document(currentUserUid)
434
- return {"dbUserExists": user_doc.get().exists}
435
-
436
- @app.post('/detect-photo')
437
- async def detect_photo(background_tasks: BackgroundTasks, sheetID: str = Form(...), image: UploadFile = File(...)):
438
- image_contents = await image.read()
439
- image_pil = Image.open(io.BytesIO(image_contents))
440
- image_jpeg = io.BytesIO()
441
- image_pil.save(image_jpeg, format='JPEG')
442
- image_jpeg.seek(0)
443
- image_contents_jpeg = image_jpeg.read()
444
-
445
- uuid = str(uuid4())
446
- # save to s3
447
- s3_client = boto3.client('s3')
448
- bucket_name = 're-sheet'
449
- object_name = sheetID + '/' + uuid
450
- s3_orginal_url = f"https://{bucket_name}.s3.amazonaws.com/{object_name}"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
451
 
452
- response = s3_client.put_object(
453
- Bucket=bucket_name,
454
- Key=object_name,
455
- Body=image_contents_jpeg,
456
- ContentType='image/jpeg'
457
- )
458
- status = response.get("ResponseMetadata", {}).get("HTTPStatusCode")
459
- if status != 200:
460
- await record_failure(s3_orginal_url, 'Failed to upload processed image to S3.', sheetID)
461
- return {"message": "Failed to upload processed image to S3."}
462
- # otherwise put it as queued
463
- object_s3_url = f"https://{bucket_name}.s3.amazonaws.com/{object_name}"
464
 
465
- body = {
466
- "s3Url": object_s3_url,
467
- "mode": "process-photo",
468
- "sheetID": sheetID,
469
- }
470
- async with httpx.AsyncClient(timeout=300.0, follow_redirects=True) as client:
471
- response = await client.post(appsScriptsURL, json=body)
472
- if response.status_code != 200:
473
- logging.error(f"Unexpected response from Google Apps Script API: {response.status_code} - {response.text}")
474
- raise HTTPException(status_code=500, detail="Error from external API.")
475
 
476
- return {"message": "Image processing queued."}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
477
 
 
 
 
 
 
 
478
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
  from fastapi import FastAPI, HTTPException, UploadFile, File, Request, BackgroundTasks, Form
2
  from fastapi.middleware.cors import CORSMiddleware
3
  from pydantic import BaseModel
4
  from httpx import Timeout
 
 
 
5
  import httpx
6
  import logging
 
7
  import base64
8
  import firebase_admin
9
  from firebase_admin import credentials
10
  from firebase_admin import firestore
11
  import json
12
+ from datetime import datetime
13
+
14
+ from typing import List, Optional, Dict
15
+ from google.oauth2 import service_account
16
+ from googleapiclient.discovery import build
17
+
18
  # start application
19
  app = FastAPI()
20
+
21
  # allow all origins
22
  app.add_middleware(
23
  CORSMiddleware,
 
26
  allow_methods=["*"], # Allows all methods
27
  allow_headers=["*"], # Allows all headers
28
  )
29
+
30
  # set up logging
31
  logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
32
  logging.info("Application starting up")
33
 
34
+ request_timeout = Timeout(1000.0, connect=600.0)
 
 
 
 
 
 
 
 
 
 
 
 
35
 
36
  # set up firebase
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
37
 
38
 
39
+ # Load Firebase configuration from a file
40
+ with open('/app/secrets/fb_secret.json', 'r') as f:
41
+ firebase_configuration = json.load(f)
 
 
42
 
43
+ firebase_credentials = credentials.Certificate(firebase_configuration)
44
+ firebase_admin.initialize_app(firebase_credentials, {
45
+ 'projectId': firebase_configuration['project_id'],
46
+ })
47
+ firestore_db = firestore.client()
48
+ users_collection_ref = firestore_db.collection('users')
49
+ transactions_collection_ref = firestore_db.collection('transactions')
50
 
51
+ SCOPES = ['https://www.googleapis.com/auth/spreadsheets','https://www.googleapis.com/auth/drive']
52
 
53
+ with open('/app/secrets/openai_key.txt', 'r') as f:
54
+ OPENAI_KEY = f.read()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
55
 
56
+ APICredential = service_account.Credentials.from_service_account_file(
57
+ '/app/secrets/googleapi_secret.json', scopes=SCOPES)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
58
 
59
+ sheet_service = build('sheets', 'v4', credentials=APICredential)
60
+ drive_service = build('drive', 'v3', credentials=APICredential)
61
 
62
+ # first three are "등록일시", "썸네일", "바로가기",
63
+ receipt_sheet_headers = ['내용', '영수증 발행일시', '발행점 상호', '사업자 등록번호', '카드번호', '지불방식', '결제수단', '금액(원)', '내역']
64
+ # at index 1 '등록 일시', at last index '명함보기'
65
+ business_card_sheet_headers = ['명함', '이름', '휴대전화', '이메일', '유선전화', '팩스번호', '회사명', '부서명', '주소']
66
+ sms_sheet_headers = ['날짜', '계좌', '입금', '출금', '잔액', '문자']
67
+ contact_sheet_headers = ['명함', '등록 일시', '이름', '휴대전화', '이메일', '유선전화', '팩스번호', '회사명', '부서명', '주소', '명함보기']
68
 
 
 
 
 
 
69
 
70
+ receipts_ss = 0
71
+ business_cards_ss = 344204272
72
+ contacts_ss = 1943060316
73
+ sms_ss = 113871696
74
+
75
+ class SheetUpdateRequest(BaseModel):
76
+ target_sheet_id: int
77
+ data: list
78
+
79
+ def convert_dicts_to_list(dicts: List[Dict], add_link: bool = False,
80
+ image_id: str = '', link_text: str = '',
81
+ link_position: int = -1) -> List[List[str]]:
82
+ rows = []
83
+ timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
84
+
85
+ for dict_item in dicts:
86
+ row = list(dict_item.values())
87
 
88
+ # Always add timestamp to the first index
89
+ row.insert(0, timestamp)
 
 
 
 
 
 
90
 
91
+ if add_link:
92
+ # Ensure the list length is at least 9
93
+ while len(row) < 9:
94
+ row.append('')
95
+ if link_position == -1: # Append link at the end
96
+ row.append(f'=HYPERLINK("https://drive.google.com/file/d/{image_id}/view?usp=sharing", "{link_text}")')
97
+ else:
98
+ row.insert(link_position, f'=HYPERLINK("https://drive.google.com/file/d/{image_id}/view?usp=sharing", "{link_text}")')
99
+
100
+ rows.append(row)
101
+ print(rows)
102
+ return rows
 
 
 
 
 
 
 
103
 
104
+ async def request_gpt4o_completion(image_id: str, user_credit: int):
105
+ openai_url = "https://api.openai.com/v1/chat/completions"
106
+ headers = {"Authorization": f"Bearer {OPENAI_KEY}", "Content-Type": "application/json"}
107
+ image_url = f"https://drive.usercontent.google.com/u/0/uc?id={image_id}&export=download"
 
 
 
108
  body = {
109
+ "model": "gpt-4o",
110
+ "response_format": { "type": "json_object" },
111
+ "messages": [
112
+ {
113
+ "role": "user",
114
+ "content": [
115
+ {
116
+ "type": "text",
117
+ "text": f'''
118
+ 아래의 순서대로 실행해주세요:
119
+ 1. {image_url} 이 링크에서 파일 다운로드가 완료될 때까지 기다리세요.
120
+ 2. 이미지의 모든 영수증과 명함을 식별하고, 각각의 정보를 추출하세요.
121
+ 3. 작성할 때 영수증과 명함 개수의 합이 {user_credit}을 넘지 않도록 하세요.
122
+ 4. 추출한 정보를 {receipt_sheet_headers}란이 있는 영수증 시트 또는 {business_card_sheet_headers}란이 있는 명함 시트에 넣을 수 있도록 JSON으로 정리하세요.
123
+ json.receipts.forEach() 또는 json.busi_cards.forEach()
124
+ (item) => insert row where header field is item.[fieldName]
125
+ '''
126
+ },
127
+ {
128
+ "type": "image_url",
129
+ "image_url": {
130
+ "url": image_url,
131
+ "detail": "high"
132
+ }
133
+ }
134
+ ]
135
+ }
136
+ ],
137
+ "max_tokens": 4096
138
+ }
 
 
 
 
 
 
139
 
140
+ logging.info(f"Request URL: {openai_url}")
141
+ logging.info(f"Request Headers: {headers}")
142
+ logging.info(f"Request Body: {json.dumps(body, indent=2)}")
143
 
144
+ try:
145
+ async with httpx.AsyncClient(follow_redirects=True, timeout=300.0) as client:
146
+ response = await client.post(openai_url, headers=headers, json=body)
147
+ response.raise_for_status() # Raise an error for bad status codes
148
+ except httpx.RequestError as exc:
149
+ logging.error(f"An error occurred while requesting {exc.request.url!r}.")
150
+ logging.error(f"Exception: {str(exc)}")
151
+ return None
152
+ except httpx.HTTPStatusError as exc:
153
+ logging.error(f"HTTP error occurred: {str(exc)}")
154
+ logging.error(f"Response: {exc.response.text}")
155
+ return None
156
+
157
+ logging.info(f"Response Status Code: {response.status_code}")
158
+ logging.info(f"Response Body: {response.text}")
159
+
160
+ completion = response.json()
161
+ if 'choices' in completion and len(completion['choices']) > 0:
162
+ message_content = completion['choices'][0]['message']['content']
163
+ print(message_content)
164
+ return message_content
165
+ else:
166
+ print("No response received")
167
+ return None
168
+
169
+ async def move_file_to_folder(file_id, current_parents, new_parents):
170
+ try:
171
 
172
+ # Ensure current_parents is a list of parent IDs
173
+ if not isinstance(current_parents, list):
174
+ current_parents = [current_parents]
175
+ if not isinstance(new_parents, list):
176
+ new_parents = [new_parents]
177
+
178
+ # Retrieve the existing parents to remove
179
+ file = drive_service.files().get(fileId=file_id, fields='parents').execute()
180
+ existing_parents = file.get('parents')
181
+ logging.info(f"Existing parents of the file {file_id}: {existing_parents}")
182
+
183
+ # Remove only the current parents
184
+ parents_to_remove = [parent for parent in current_parents if parent in existing_parents]
185
+ parents_to_add = [parent for parent in new_parents if parent not in existing_parents]
186
+
187
+ previous_parents = ",".join(parents_to_remove)
188
+ logging.info(f"Parents to remove: {parents_to_remove}")
189
+ logging.info(f"Parents to add: {parents_to_add}")
190
+
191
+ # Move the file to the new folders while keeping it in yungsoogi folder
192
+ if parents_to_add:
193
+ file = drive_service.files().update(
194
+ fileId=file_id,
195
+ addParents=",".join(parents_to_add),
196
+ removeParents=previous_parents,
197
+ fields='id, parents'
198
+ ).execute()
199
+ logging.info(f"File {file_id} moved to new folders {new_parents}.")
200
+ else:
201
+ logging.info(f"File {file_id} already in the target folders {new_parents}.")
202
+ except Exception as e:
203
+ logging.error(f"Failed to move file {file_id} to new folders: {e}")
204
 
205
+
206
+ async def update_user_sheet(spreadsheet_id:str, sheet_id:int, sheet_name, data: List[List[str]], is_reset: Optional[bool] = False):
207
+ print(f"Spreadsheet ID: {spreadsheet_id}, Sheet ID: {sheet_id}, sheet name: {sheet_name}, Data: {data}")
208
+ # first identify the area to insert the data
209
+ no_of_rows = len(data)
210
+ no_of_cols = len(data[0])
211
+
212
+ if is_reset:
213
+ sheet_name = f"{sheet_name} {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}"
214
+ duplicate_requests = [{
215
+ "duplicateSheet": {
216
+ "sourceSheetId": contacts_ss,
217
+ "newSheetName": sheet_name
218
+ }
219
+ }]
220
+ duplicate_response = sheet_service.spreadsheets().batchUpdate(
221
+ spreadsheetId=spreadsheet_id,
222
+ body={
223
+ 'requests': duplicate_requests
224
+ }
225
+ ).execute()
226
+ sheet_id = duplicate_response['replies'][0]['duplicateSheet']['properties']['sheetId']
227
+
228
+ # Insert new rows
229
+ requests = [{
230
+ "insertDimension": {
231
+ "range": {
232
+ "sheetId": sheet_id,
233
+ "dimension": "ROWS",
234
+ "startIndex": 1,
235
+ "endIndex": no_of_rows + 1
236
+ },
237
+ "inheritFromBefore": False,
238
+ }
239
+ }]
240
+
241
+
242
+ response = sheet_service.spreadsheets().batchUpdate(
243
+ spreadsheetId=spreadsheet_id,
244
+ body={
245
+ 'requests': requests
246
+ }
247
+ ).execute()
248
+
249
+ update_body = {
250
+ 'values': data
251
  }
252
+ start_column = 'A'
253
+ end_column = chr(ord(start_column) + no_of_cols - 1)
254
+ range_to_update = f'{sheet_name}!{start_column}2:{end_column}{no_of_rows + 1}'
 
 
255
 
256
+ result = sheet_service.spreadsheets().values().update(
257
+ spreadsheetId=spreadsheet_id,
258
+ range=range_to_update,
259
+ valueInputOption='USER_ENTERED',
260
+ body=update_body
261
+ ).execute()
262
+
263
+ async def copy_sms_sheet(phone_no: str, spreadsheet_id: str):
264
+ requests = [
265
+ {
266
+ "duplicateSheet": {
267
+ "sourceSheetId": sms_ss,
268
+ "newSheetName": phone_no
269
+ }
270
+ }
271
+ ]
272
 
273
+ response = sheet_service.spreadsheets().batchUpdate(
274
+ spreadsheetId=spreadsheet_id,
275
+ body={
276
+ 'requests': requests
277
+ }
278
+ ).execute()
279
+ print(response)
280
+ new_sheet_id = response['replies'][0]['duplicateSheet']['properties']['sheetId']
281
+ return new_sheet_id
282
+
283
+ async def update_fb_transaction(transaction_ref: firestore.DocumentReference, no_sms: Optional[int] = 0, no_contacts: Optional[int] = 0, no_receipts: Optional[int] = 0, no_business_cards: Optional[int] = 0):
284
+ try:
285
+ # Update the transaction
286
+ transaction_ref.update({
287
+ "no_sms": firestore.Increment(no_sms),
288
+ "no_contacts": firestore.Increment(no_contacts),
289
+ "no_receipts": firestore.Increment(no_receipts),
290
+ "no_business_cards": firestore.Increment(no_business_cards)
291
+ })
292
+
293
+ return {"status": "Success"}
294
+ except Exception as e:
295
+ logging.error(f"Error updating transaction: {e}")
296
+ return {"status": f"Error: {str(e)}"}
297
 
 
 
 
 
 
 
 
 
 
 
 
298
 
299
+ @app.post("/check-userdoc")
300
+ async def check_user_doc(user_id: str = Form(...)):
301
+ user_ref = users_collection_ref.document(user_id)
302
+ user = user_ref.get()
303
+ if user.exists:
304
+ return {"dbUserExists": True}
305
+ new_transaction_ref = transactions_collection_ref.document(user_id)
306
+ new_transaction_ref.set({
307
+ "no_sms": 0,
308
+ "no_contacts": 0,
309
+ "no_receipts": 0,
310
+ "no_business_cards": 0,
311
+ "purchased_credit": {
312
+ "image_detection": 100,
313
+ "sync_data": 500,
314
+ },
315
+ "uid": user_id,
316
+ })
317
+ return {"dbUserExists": False}
318
+
319
+
320
+
321
+ # image upload
322
+ # receive image id and user_id
323
+ # check if they have enough credits
324
+ # post to gpt4o with the user credit info
325
+ # identify the sheet_id to update by checking in user_id
326
+ # update user sheet
327
+ # update firebase transaction
328
+ # privatize the image and move the image to appropriate folder(s)
329
+ @app.post("/process-image")
330
+ async def process_photo(image_id: str = Form(...), user_id: str = Form(...)):
331
+ transaction_ref = transactions_collection_ref.document(user_id)
332
+ transaction = transaction_ref.get()
333
+ # If transaction does not exist, create a new one
334
+ if not transaction.exists:
335
+ transaction_ref.set({
336
+ "no_sms": 0,
337
+ "no_contacts": 0,
338
+ "no_receipts": 0,
339
+ "no_business_cards": 0,
340
+ "purchased_credit": {
341
+ "image_detection": 100,
342
+ "sync_data": 500,
343
+ }
344
+ })
345
+ transaction = transaction_ref.get()
346
+
347
+ transaction_data = transaction.to_dict()
348
 
349
+ # check if any of the limits have been reached
350
+ if transaction_data['no_receipts'] + transaction_data['no_business_cards'] >= transaction_data['purchased_credit']['image_detection']:
351
+ return {"error": "You have reached the limit of the number of items you can process. Please upgrade your plan."}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
352
 
353
+ user_ref = users_collection_ref.document(user_id)
354
+ user = user_ref.get()
355
+ user_data = user.to_dict()
356
+
 
 
 
 
 
 
 
 
 
 
 
 
 
357
 
358
+ try:
359
+ parent_folders = [user_data['gDrive_metadata']['yungsoogi'], user_data['gDrive_metadata']['uploaded']]
360
+ dataJson = await request_gpt4o_completion(image_id, transaction_data['purchased_credit']['image_detection'])
361
+ if dataJson is None:
362
+ return {"error": "An error occurred while processing the image"}
363
+
364
+ data = json.loads(dataJson)
365
+ logging.info(f"user_data: {user_data}")
366
+ # Check if 'receipts' key exists and the length
367
+ found_receipt_no = len(data['receipts']) if 'receipts' in data else 0
368
+ found_business_cards_no = len(data['busi_cards']) if 'busi_cards' in data else 0
369
+ new_folders = []
370
+ timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
371
+ # https://drive.google.com/file/d/1cFIA09PBqFjM8YvIz1D60p7HMGk1uBvo/view?usp=sharing
372
+ download_image_url = f"https://drive.google.com/uc?id={image_id}&export=download"
373
+ image_url = f"https://drive.google.com/file/d/{image_id}/view?usp=sharing"
374
+ # 이미지를 보여주어야 할때 =image({image_url})를 사용하고 갈 수 있는 경로를 만들때는 =HYPERLINK({image_url}, '바로가기')를 사용하세요.
375
+ if found_receipt_no > 0:
376
+ new_folders.append(user_data['gDrive_metadata']['receipts'])
377
+ receipts_data = convert_dicts_to_list(
378
+ data['receipts'],
379
+ add_link=True,
380
+ image_id=image_id,
381
+ link_text="바로가기",
382
+ link_position=1 # Link third
383
+ )
384
+ print(receipts_data)
385
+ await update_user_sheet(user_data['gDrive_metadata']['spreadsheet'], receipts_ss, "영수증", receipts_data)
386
+ if found_business_cards_no > 0:
387
+ new_folders.append(user_data['gDrive_metadata']['business_cards'])
388
+ business_cards_data = convert_dicts_to_list(
389
+ data['busi_cards'],
390
+ add_link=True,
391
+ image_id=image_id,
392
+ link_text="명함보기",
393
+ link_position=-1 # Link at the end
394
+ )
395
+ await update_user_sheet(user_data['gDrive_metadata']['spreadsheet'], business_cards_ss, "명함", business_cards_data)
396
 
397
+ print(f"{found_receipt_no}, {found_business_cards_no}")
 
 
 
 
 
 
 
 
 
 
 
398
 
399
+ status = await update_fb_transaction(
400
+ transaction_ref,
401
+ no_receipts=found_receipt_no,
402
+ no_business_cards=found_business_cards_no
403
+ )
 
 
 
 
 
404
 
405
+
406
+ if status['status'] == "Success":
407
+ await move_file_to_folder(image_id, parent_folders, new_folders)
408
+ return {"success": True}
409
+ else:
410
+ logging.error(f"Transaction update failed with status: {status['status']}")
411
+ return {"error": "An error has occurred while updating the transaction"}
412
+ except Exception as e:
413
+ logging.error(f"An error occurred: {e}")
414
+ # Move the image file to the error folder
415
+ await move_file_to_folder(image_id, parent_folders, user_data['gDrive_metadata']['error'])
416
+ return {"error": str(e)}
417
+
418
+ @app.post("/sync-data")
419
+ async def process_photo(data_json: str = Form(...), user_id: str = Form(...)):
420
+ print(data_json)
421
+ data: Dict[str, Any] = json.loads(data_json)
422
+ transaction_ref = transactions_collection_ref.document(user_id)
423
+ transaction = transaction_ref.get()
424
+
425
+ if not transaction.exists:
426
+ transaction_ref.set({
427
+ "no_sms": 0,
428
+ "no_contacts": 0,
429
+ "no_receipts": 0,
430
+ "no_business_cards": 0,
431
+ "purchased_credit": {
432
+ "image_detection": 100,
433
+ "sync_data": 500,
434
+ }
435
+ })
436
+ transaction = transaction_ref.get()
437
+
438
+ transaction_data = transaction.to_dict()
439
+
440
+ if transaction_data['no_contacts'] + transaction_data['no_sms'] >= transaction_data['purchased_credit']['sync_data']:
441
+ return {"error": "You have reached the limit of the number of items you can process. Please upgrade your plan."}
442
+
443
+ user_ref = users_collection_ref.document(user_id)
444
+ user = user_ref.get()
445
+ user_data = user.to_dict()
446
 
447
+ try:
448
+ if data['isContacts']:
449
+ await update_user_sheet(user_data['gDrive_metadata']['spreadsheet'], contacts_ss, "연락처", data['data'], True)
450
+ else:
451
+ if 'sms_ids' not in user_data:
452
+ user_data['sms_ids'] = {}
453
 
454
+ existing_phones = set(user_data.get('sms_ids', {}).keys())
455
+ new_phones = [phone['address'] for phone in data['data'] if phone['address'] not in existing_phones]
456
+
457
+ print(f"Existing phones: {existing_phones}")
458
+
459
+ for phone_address in new_phones:
460
+ new_sheet_id = await copy_sms_sheet(phone_address, user_data['gDrive_metadata']['spreadsheet'])
461
+ print(f"New sheet id: {new_sheet_id}")
462
+ user_data['sms_ids'][phone_address] = {'sheet_id': new_sheet_id, 'last_updated': "1970-01-01T00:00:00.000"}
463
+ print(user_data['sms_ids'])
464
+
465
+ for phone in data['data']:
466
+ last_updated = user_data['sms_ids'][phone['address']]['last_updated']
467
+ print(f"Last updated: {last_updated}")
468
+
469
+ # Convert last_updated to datetime for comparison
470
+ last_updated_dt = datetime.fromisoformat(last_updated)
471
+ update_message = [sms for sms in phone['sms'] if datetime.fromisoformat(sms[0]) > last_updated_dt]
472
+ print(f"Update message: {update_message}")
473
+
474
+ if not update_message:
475
+ continue
476
+
477
+ sheet_id = user_data['sms_ids'][phone['address']]['sheet_id']
478
+ print(f"Sheet ID: {sheet_id}")
479
+
480
+ await update_user_sheet(user_data['gDrive_metadata']['spreadsheet'], sheet_id, phone['address'], update_message, False)
481
+
482
+ user_data['sms_ids'][phone['address']]['last_updated'] = phone['sms'][0][0]
483
+
484
+ user_ref.update({'sms_ids': user_data['sms_ids']})
485
+
486
+ return {"success": True}
487
+ except Exception as e:
488
+ logging.error(f"An error occurred: {e}")
489
+ return {"error": str(e)}