add check for file availability before opening

data_sources/upload_file.py

@@ -2,9 +2,20 @@ import pandas as pd
 import sqlite3
 import csv
 import json
-import os
 import time
 
+def is_file_done_saving(file_path):
+    try:
+        with open(file_path, 'r') as f:
+            contents = f
+
+        if contents:
+            return True
+        else:
+            return False
+    except PermissionError:
+        return False
+
 def get_delimiter(file_path, bytes = 4096):
     sniffer = csv.Sniffer()
     data = open(file_path, "r").read(bytes)
@@ -19,7 +30,6 @@ def read_file(file):
             contents = json.load(f)
         df = pd.json_normalize(contents)
     elif file.endswith('.ndjson'):
-        time.sleep(5)
         with open(file, 'r') as f:
             contents = f.read()
         data = [json.loads(str(item)) for item in contents.strip().split('\n')]
@@ -33,6 +43,13 @@ def read_file(file):
     return df
 
 def process_data_upload(data_file, session_hash):
+    total_time = 0
+    while not is_file_done_saving(data_file):
+        total_time += .5
+        time.sleep(.5)
+        if total_time > 10:
+            break
+        
     df = read_file(data_file)
 
     # Read each sheet and store data in a DataFrame