Spaces:
Sleeping
Sleeping
File size: 6,807 Bytes
9e7b45f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 | import cloudinary
import cloudinary.uploader
import cloudinary.api
import pickle
import io
import os
import logging
import urllib.request
from flask import current_app
logger = logging.getLogger(__name__)
def init_cloudinary():
"""Initialize cloudinary configuration if not already configured"""
cloud_name = os.environ.get('CLOUDINARY_CLOUD_NAME')
if not cloud_name:
return False
if not cloudinary.config().cloud_name:
cloudinary.config(
cloud_name = cloud_name,
api_key = os.environ.get('CLOUDINARY_API_KEY'),
api_secret = os.environ.get('CLOUDINARY_API_SECRET'),
secure = True
)
return True
def upload_pickle_to_cloudinary(local_path, cloud_name):
"""
Upload a local pickle file to Cloudinary as a raw file
Args:
local_path: Path to the local .pickle file
cloud_name: Public ID to use in Cloudinary (e.g. 'CE_3')
"""
try:
if not init_cloudinary():
logger.error("Cloudinary not configured. Skipping upload.")
return None
print(f"DEBUG: Syncing {cloud_name}.pickle to Cloudinary...")
# Use resource_type='raw' for non-image files like .pickle
response = cloudinary.uploader.upload(
local_path,
public_id = cloud_name,
folder = "facemarkpro/encodings",
resource_type = "raw",
overwrite = True,
unique_filename = False
)
print(f"DEBUG: Successfully uploaded {cloud_name}.pickle")
logger.info(f"Successfully uploaded {cloud_name}.pickle to Cloudinary: {response.get('secure_url')}")
return response.get('secure_url')
except Exception as e:
print(f"DEBUG: Cloudinary upload failed: {e}")
logger.error(f"Error uploading pickle to Cloudinary: {e}")
return None
def upload_pickle_to_cloudinary_from_memory(data_dict, cloud_name):
"""
Upload a dictionary to Cloudinary as a .pickle file directly from memory
"""
try:
if not init_cloudinary():
return None
print(f"DEBUG: Syncing {cloud_name}.pickle to Cloudinary from memory...")
# Convert dict to pickle bytes
pickle_bytes = pickle.dumps(data_dict)
buffer = io.BytesIO(pickle_bytes)
# Upload to Cloudinary
response = cloudinary.uploader.upload(
buffer,
public_id = cloud_name,
folder = "facemarkpro/encodings",
resource_type = "raw",
overwrite = True
)
print(f"DEBUG: Successfully uploaded {cloud_name}.pickle (memory)")
return response.get('secure_url')
except Exception as e:
print(f"DEBUG: Cloudinary memory upload failed: {e}")
logger.error(f"Error uploading pickle from memory: {e}")
return None
def list_encodings_from_cloudinary():
"""
List all pickle filenames in the facemarkpro/encodings folder
"""
try:
if not init_cloudinary():
return []
print("DEBUG: Listing encodings from Cloudinary...")
# Use resources with prefix
response = cloudinary.api.resources(
type="upload",
resource_type="raw",
prefix="facemarkpro/encodings/"
)
resources = response.get('resources', [])
filenames = []
for res in resources:
# res['public_id'] will be 'facemarkpro/encodings/BRANCH_SEM'
public_id = res['public_id']
filename = public_id.split('/')[-1]
filenames.append(f"{filename}.pickle")
return filenames
except Exception as e:
print(f"DEBUG: Cloudinary list failed: {e}")
return []
def download_pickle_from_cloudinary(cloud_name, local_path):
"""
Download a pickle file from Cloudinary if it exists
Args:
cloud_name: Public ID in Cloudinary (e.g. 'CE_3')
local_path: Where to save the file locally
"""
try:
if not init_cloudinary():
return False
# Get the URL for the raw file
print(f"DEBUG: Checking for {cloud_name}.pickle in Cloudinary...")
try:
# Use a faster check or direct URL if possible, or keep resource call
resource = cloudinary.api.resource(f"facemarkpro/encodings/{cloud_name}", resource_type="raw")
url = resource.get('secure_url')
except Exception:
print(f"DEBUG: {cloud_name}.pickle not found on cloud.")
logger.info(f"Pickle {cloud_name} not found on Cloudinary")
return False
if not url:
return False
try:
print(f"DEBUG: Downloading {cloud_name}.pickle...")
with urllib.request.urlopen(url, timeout=5) as response:
if response.status == 200:
os.makedirs(os.path.dirname(local_path), exist_ok=True)
with open(local_path, 'wb') as f:
f.write(response.read())
print(f"DEBUG: Successfully downloaded {cloud_name}.pickle")
logger.info(f"Successfully downloaded {cloud_name}.pickle from Cloudinary to {local_path}")
return True
else:
logger.error(f"Failed to download pickle from Cloudinary, status code: {response.status}")
return False
except Exception as e:
logger.error(f"Network error downloading pickle: {e}")
return False
except Exception as e:
logger.error(f"Error downloading pickle from Cloudinary: {e}")
return False
def get_pickle_from_cloudinary(cloud_name):
"""
Fetch pickle data directly from Cloudinary into memory
Args:
cloud_name: Public ID in Cloudinary
Returns:
The deserialized pickle object or None
"""
try:
if not init_cloudinary():
return None
print(f"DEBUG: Fetching {cloud_name} from cloud to memory...")
try:
resource = cloudinary.api.resource(f"facemarkpro/encodings/{cloud_name}", resource_type="raw")
url = resource.get('secure_url')
except Exception:
return None
if not url:
return None
with urllib.request.urlopen(url, timeout=10) as response:
if response.status == 200:
data = pickle.loads(response.read())
print(f"DEBUG: Successfully loaded {cloud_name} into memory")
return data
except Exception as e:
logger.error(f"Error loading pickle from cloud to memory: {e}")
return None
return None
|