dn-scribe commited on
Commit
7df56c5
·
1 Parent(s): 6ab6d23

fix: s3 with valint (content type = multipart)

Browse files
Files changed (4) hide show
  1. .vscode/launch.json +7 -0
  2. config.py +2 -1
  3. scripts/test_s3.py +417 -0
  4. storage.py +23 -85
.vscode/launch.json CHANGED
@@ -11,6 +11,13 @@
11
  "request": "launch",
12
  "program": "main.py",
13
  "console": "internalConsole"
 
 
 
 
 
 
 
14
  }
15
  ]
16
  }
 
11
  "request": "launch",
12
  "program": "main.py",
13
  "console": "internalConsole"
14
+ },
15
+ {
16
+ "name": "test s3",
17
+ "type": "debugpy",
18
+ "request": "launch",
19
+ "program": "tmp/test_s3.py",
20
+ "console": "internalConsole"
21
  }
22
  ]
23
  }
config.py CHANGED
@@ -18,7 +18,8 @@ class Config:
18
 
19
  AWS_ACCESS_KEY_ID = os.environ.get('AWS_ACCESS_KEY_ID')
20
  AWS_SECRET_ACCESS_KEY = os.environ.get('AWS_SECRET_ACCESS_KEY')
21
- AWS_REGION = os.environ.get('AWS_REGION', 'us-east-1')
 
22
  S3_BUCKET = os.environ.get('S3_BUCKET')
23
 
24
  # MinIO settings
 
18
 
19
  AWS_ACCESS_KEY_ID = os.environ.get('AWS_ACCESS_KEY_ID')
20
  AWS_SECRET_ACCESS_KEY = os.environ.get('AWS_SECRET_ACCESS_KEY')
21
+ AWS_SESSION_TOKEN = os.environ.get('AWS_SESSION_TOKEN')
22
+ AWS_REGION = os.environ.get('AWS_DEFAULT_REGION', 'il-central-1')
23
  S3_BUCKET = os.environ.get('S3_BUCKET')
24
 
25
  # MinIO settings
scripts/test_s3.py ADDED
@@ -0,0 +1,417 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ """
3
+ S3 Standalone Test Script for Attestation Store
4
+
5
+ This script tests S3 operations with presigned URLs:
6
+ 1. Generates a presigned URL for uploading a file
7
+ 2. Uploads a test file to the S3 bucket using the presigned URL
8
+ 3. Generates a presigned URL for downloading the file
9
+ 4. Downloads the file using the presigned URL
10
+ 5. Compares the original and downloaded files
11
+
12
+ The script supports both AWS S3 and MinIO storage backends.
13
+ """
14
+
15
+ import os
16
+ import sys
17
+ import uuid
18
+ import json
19
+ import boto3
20
+ import hashlib
21
+ import logging
22
+ import requests
23
+ import tempfile
24
+ import argparse
25
+ from datetime import timedelta
26
+ from botocore.config import Config as BotoConfig
27
+ from urllib.parse import urlparse
28
+
29
+ # Configure logging
30
+ logging.basicConfig(
31
+ level=logging.INFO,
32
+ format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
33
+ )
34
+ logger = logging.getLogger("s3_test")
35
+
36
+ # Default settings
37
+ DEFAULT_EXPIRATION = 3600 # 1 hour
38
+ DEFAULT_REGION = "il-central-1"
39
+ TEST_CONTENT = json.dumps({"test": "data", "timestamp": str(uuid.uuid4())})
40
+
41
+
42
+ class S3StorageBackend:
43
+ """
44
+ AWS S3 storage backend (compatible with the actual implementation in the project)
45
+ """
46
+
47
+ def __init__(self, access_key, secret_key, region, bucket, session_token=None):
48
+ self.bucket = bucket
49
+
50
+ # Configure boto3 client
51
+ cfg = BotoConfig(signature_version="s3v4", s3={"addressing_style": "virtual"})
52
+
53
+ # If keys are empty, allow the default provider chain (env vars, IAM role, etc.)
54
+ client_kwargs = dict(service_name="s3", region_name=region, config=cfg)
55
+ if access_key and secret_key:
56
+ client_kwargs.update(
57
+ aws_access_key_id=access_key,
58
+ aws_secret_access_key=secret_key,
59
+ )
60
+ if session_token: # required for ASIA... keys
61
+ client_kwargs.update(aws_session_token=session_token)
62
+
63
+ self.s3_client = boto3.client(**client_kwargs)
64
+ logger.info(f"Initialized S3 client for region {region}, bucket {bucket}")
65
+
66
+ def _normalize_content_type(self, content_type):
67
+ """Normalize content type"""
68
+ if not content_type:
69
+ return "application/json"
70
+
71
+ if content_type == "cyclonedx-json":
72
+ return "application/cyclonedx+json"
73
+
74
+ return "application/json"
75
+
76
+ def generate_upload_url(self, file_name, content_type=None, session_id=None, expiration=DEFAULT_EXPIRATION):
77
+ """Generate presigned URL for S3 upload"""
78
+ file_id = session_id or str(uuid.uuid4())
79
+ storage_key = f"attestations/{file_id}/{file_name}"
80
+ ct = self._normalize_content_type(content_type)
81
+
82
+ params = {
83
+ "Bucket": self.bucket,
84
+ "Key": storage_key,
85
+ "ContentType": ct,
86
+ }
87
+
88
+ url = self.s3_client.generate_presigned_url(
89
+ "put_object",
90
+ Params=params,
91
+ ExpiresIn=expiration,
92
+ HttpMethod="PUT",
93
+ )
94
+
95
+ logger.info(f"S3 presigned PUT: {url}")
96
+ logger.info(f"S3 region: {self.s3_client.meta.region_name}, bucket: {self.bucket}, key: {storage_key}")
97
+ return {"upload_url": url, "storage_path": storage_key}
98
+
99
+ def generate_download_url(self, storage_path, expiration=DEFAULT_EXPIRATION):
100
+ """Generate presigned URL for S3 download"""
101
+ url = self.s3_client.generate_presigned_url(
102
+ "get_object",
103
+ Params={"Bucket": self.bucket, "Key": storage_path},
104
+ ExpiresIn=expiration,
105
+ )
106
+ logger.info(f"S3 presigned GET: {url}")
107
+ return url
108
+
109
+ def delete_file(self, storage_path):
110
+ """Delete S3 object"""
111
+ try:
112
+ self.s3_client.delete_object(Bucket=self.bucket, Key=storage_path)
113
+ logger.info(f"Successfully deleted S3 object: {storage_path}")
114
+ return True
115
+ except Exception as e:
116
+ logger.error(f"Failed to delete S3 object {storage_path}: {e}")
117
+ return False
118
+
119
+
120
+ class MinIOStorageBackend:
121
+ """
122
+ MinIO storage backend (compatible with the actual implementation in the project)
123
+ """
124
+
125
+ def __init__(self, endpoint, access_key, secret_key, bucket, secure=True):
126
+ try:
127
+ from minio import Minio
128
+
129
+ self.bucket = bucket
130
+ self.client = Minio(
131
+ endpoint,
132
+ access_key=access_key,
133
+ secret_key=secret_key,
134
+ secure=secure
135
+ )
136
+
137
+ # Create bucket if it doesn't exist
138
+ if not self.client.bucket_exists(bucket):
139
+ self.client.make_bucket(bucket)
140
+ logger.info(f"Created MinIO bucket: {bucket}")
141
+ else:
142
+ logger.info(f"Using existing MinIO bucket: {bucket}")
143
+
144
+ except ImportError:
145
+ logger.error("MinIO Python client not found. Please install with: pip install minio")
146
+ sys.exit(1)
147
+ except Exception as e:
148
+ logger.error(f"Failed to initialize MinIO client: {e}")
149
+ sys.exit(1)
150
+
151
+ def generate_upload_url(self, file_name, content_type=None, session_id=None, expiration=DEFAULT_EXPIRATION):
152
+ """Generate presigned URL for MinIO upload"""
153
+ file_id = session_id or str(uuid.uuid4())
154
+ storage_key = f"attestations/{file_id}/{file_name}"
155
+
156
+ storage_key = storage_key.replace('//', '/')
157
+
158
+ upload_url = self.client.get_presigned_url(
159
+ "PUT",
160
+ self.bucket,
161
+ storage_key,
162
+ expires=timedelta(seconds=expiration)
163
+ )
164
+
165
+ logger.info(f"MinIO presigned PUT: {upload_url}")
166
+ return {
167
+ 'upload_url': upload_url,
168
+ 'storage_path': storage_key
169
+ }
170
+
171
+ def generate_download_url(self, storage_path, expiration=DEFAULT_EXPIRATION):
172
+ """Generate presigned URL for MinIO download"""
173
+ url = self.client.get_presigned_url(
174
+ "GET",
175
+ self.bucket,
176
+ storage_path,
177
+ expires=timedelta(seconds=expiration)
178
+ )
179
+ logger.info(f"MinIO presigned GET: {url}")
180
+ return url
181
+
182
+ def delete_file(self, storage_path):
183
+ """Delete MinIO object"""
184
+ try:
185
+ self.client.remove_object(self.bucket, storage_path)
186
+ logger.info(f"Successfully deleted MinIO object: {storage_path}")
187
+ return True
188
+ except Exception as e:
189
+ logger.error(f"Failed to delete MinIO object {storage_path}: {e}")
190
+ return False
191
+
192
+
193
+ def create_test_file(content=None):
194
+ """Create a temporary test file"""
195
+ if content is None:
196
+ content = TEST_CONTENT
197
+
198
+ fd, path = tempfile.mkstemp(suffix=".json")
199
+ try:
200
+ with os.fdopen(fd, 'w') as f:
201
+ f.write(content)
202
+ return path
203
+ except Exception as e:
204
+ logger.error(f"Failed to create test file: {e}")
205
+ os.unlink(path)
206
+ sys.exit(1)
207
+
208
+
209
+ def calculate_file_hash(file_path):
210
+ """Calculate SHA-256 hash of a file"""
211
+ sha256 = hashlib.sha256()
212
+ with open(file_path, "rb") as f:
213
+ for chunk in iter(lambda: f.read(4096), b""):
214
+ sha256.update(chunk)
215
+ return sha256.hexdigest()
216
+
217
+
218
+ def upload_to_presigned_url(file_path, upload_url, content_type="application/json"):
219
+ """Upload a file using a presigned URL"""
220
+ try:
221
+ with open(file_path, "rb") as f:
222
+ headers = {"Content-Type": content_type}
223
+ response = requests.put(upload_url, data=f, headers=headers)
224
+ response.raise_for_status()
225
+ logger.info(f"Successfully uploaded file to {upload_url}, status: {response.status_code}")
226
+ return True
227
+ except Exception as e:
228
+ logger.error(f"Failed to upload file using presigned URL: {e}")
229
+ return False
230
+
231
+
232
+ def download_from_presigned_url(download_url, output_path):
233
+ """Download a file using a presigned URL"""
234
+ try:
235
+ response = requests.get(download_url)
236
+ response.raise_for_status()
237
+ with open(output_path, "wb") as f:
238
+ f.write(response.content)
239
+ logger.info(f"Successfully downloaded file from {download_url} to {output_path}")
240
+ return True
241
+ except Exception as e:
242
+ logger.error(f"Failed to download file using presigned URL: {e}")
243
+ return False
244
+
245
+
246
+ def verify_files_match(original_path, downloaded_path):
247
+ """Verify that the original and downloaded files match"""
248
+ original_hash = calculate_file_hash(original_path)
249
+ downloaded_hash = calculate_file_hash(downloaded_path)
250
+
251
+ if original_hash == downloaded_hash:
252
+ logger.info(f"Files match! Hash: {original_hash}")
253
+ return True
254
+ else:
255
+ logger.error(f"Files do not match! Original hash: {original_hash}, Downloaded hash: {downloaded_hash}")
256
+ return False
257
+
258
+
259
+ def parse_args():
260
+ """Parse command line arguments"""
261
+ parser = argparse.ArgumentParser(description="Test S3 operations with presigned URLs")
262
+ parser.add_argument("--provider", choices=["aws", "minio"], default="aws",
263
+ help="Storage provider (aws or minio)")
264
+
265
+ # AWS S3 options
266
+ parser.add_argument("--aws-access-key", help="AWS Access Key ID")
267
+ parser.add_argument("--aws-secret-key", help="AWS Secret Access Key")
268
+ parser.add_argument("--aws-region", default=DEFAULT_REGION, help="AWS Region")
269
+ parser.add_argument("--s3-bucket", help="S3 Bucket name")
270
+
271
+ # MinIO options
272
+ parser.add_argument("--minio-endpoint", help="MinIO endpoint (e.g., 'minio.example.com:9000')")
273
+ parser.add_argument("--minio-access-key", help="MinIO Access Key")
274
+ parser.add_argument("--minio-secret-key", help="MinIO Secret Key")
275
+ parser.add_argument("--minio-bucket", help="MinIO Bucket name")
276
+ parser.add_argument("--minio-secure", default=True, type=bool, help="Use secure connection (HTTPS)")
277
+
278
+ # Common options
279
+ parser.add_argument("--file", help="Path to file to upload (creates a test file if not provided)")
280
+ parser.add_argument("--expiration", type=int, default=DEFAULT_EXPIRATION,
281
+ help="Presigned URL expiration time in seconds")
282
+ parser.add_argument("--no-cleanup", action="store_true", help="Do not delete the uploaded file")
283
+
284
+ return parser.parse_args()
285
+
286
+
287
+ def read_env_vars():
288
+ """Read S3 configuration from environment variables"""
289
+ env_vars = {
290
+ "aws_access_key": os.environ.get("AWS_ACCESS_KEY_ID"),
291
+ "aws_secret_key": os.environ.get("AWS_SECRET_ACCESS_KEY"),
292
+ "aws_region": os.environ.get("AWS_DEFAULT_REGION", DEFAULT_REGION),
293
+ "s3_bucket": os.environ.get("S3_BUCKET"),
294
+ "minio_endpoint": os.environ.get("MINIO_ENDPOINT"),
295
+ "minio_access_key": os.environ.get("MINIO_ACCESS_KEY"),
296
+ "minio_secret_key": os.environ.get("MINIO_SECRET_KEY"),
297
+ "minio_bucket": os.environ.get("MINIO_BUCKET"),
298
+ "minio_secure": os.environ.get("MINIO_SECURE", "true").lower() == "true",
299
+ }
300
+ return env_vars
301
+
302
+
303
+ def extract_hostname_from_url(url):
304
+ """Extract hostname from URL for logging purposes"""
305
+ parsed = urlparse(url)
306
+ return parsed.netloc
307
+
308
+
309
+ def main():
310
+ """Main function"""
311
+ args = parse_args()
312
+ env_vars = read_env_vars()
313
+
314
+ # Combine command line args with env vars (cmd line takes precedence)
315
+ config = {k: getattr(args, k) if getattr(args, k) is not None else env_vars.get(k) for k in env_vars}
316
+
317
+ # Create storage backend
318
+ if args.provider == "aws":
319
+ required = ["aws_access_key", "aws_secret_key", "aws_region", "s3_bucket"]
320
+ if any(config.get(k) is None for k in required):
321
+ logger.error("Missing required AWS S3 configuration. Please provide AWS credentials and bucket name.")
322
+ sys.exit(1)
323
+
324
+ storage = S3StorageBackend(
325
+ access_key=config["aws_access_key"],
326
+ secret_key=config["aws_secret_key"],
327
+ region=config["aws_region"],
328
+ bucket=config["s3_bucket"]
329
+ )
330
+ else: # minio
331
+ required = ["minio_endpoint", "minio_access_key", "minio_secret_key", "minio_bucket"]
332
+ if any(config.get(k) is None for k in required):
333
+ logger.error("Missing required MinIO configuration. Please provide MinIO credentials and bucket name.")
334
+ sys.exit(1)
335
+
336
+ storage = MinIOStorageBackend(
337
+ endpoint=config["minio_endpoint"],
338
+ access_key=config["minio_access_key"],
339
+ secret_key=config["minio_secret_key"],
340
+ bucket=config["minio_bucket"],
341
+ secure=config["minio_secure"]
342
+ )
343
+
344
+ # Create a test file if not provided
345
+ input_file = args.file
346
+ temp_file_created = False
347
+ if not input_file:
348
+ input_file = create_test_file()
349
+ temp_file_created = True
350
+
351
+ logger.info(f"Using test file: {input_file}")
352
+ file_name = os.path.basename(input_file)
353
+
354
+ try:
355
+ # Step 1: Generate presigned upload URL
356
+ logger.info("Step 1: Generating presigned upload URL...")
357
+ upload_result = storage.generate_upload_url(
358
+ file_name=file_name,
359
+ content_type="application/json",
360
+ expiration=args.expiration
361
+ )
362
+ upload_url = upload_result["upload_url"]
363
+ storage_path = upload_result["storage_path"]
364
+
365
+ logger.info(f"Storage path: {storage_path}")
366
+ logger.info(f"Upload URL hostname: {extract_hostname_from_url(upload_url)}")
367
+
368
+ # Step 2: Upload file using presigned URL
369
+ logger.info("\nStep 2: Uploading file using presigned URL...")
370
+ if not upload_to_presigned_url(input_file, upload_url):
371
+ logger.error("Upload failed. Exiting.")
372
+ # sys.exit(1)
373
+
374
+ # Step 3: Generate presigned download URL
375
+ logger.info("\nStep 3: Generating presigned download URL...")
376
+ download_url = storage.generate_download_url(storage_path, args.expiration)
377
+ logger.info(f"Download URL hostname: {extract_hostname_from_url(download_url)}")
378
+
379
+ # Step 4: Download file using presigned URL
380
+ logger.info("\nStep 4: Downloading file using presigned URL...")
381
+ download_file = tempfile.mktemp(suffix=".json")
382
+ if not download_from_presigned_url(download_url, download_file):
383
+ logger.error("Download failed. Exiting.")
384
+ # sys.exit(1)
385
+
386
+ # Step 5: Verify files match
387
+ logger.info("\nStep 5: Verifying file integrity...")
388
+ if verify_files_match(input_file, download_file):
389
+ logger.info("✓ TEST PASSED: Files match perfectly!")
390
+ else:
391
+ logger.error("✗ TEST FAILED: Files do not match!")
392
+ sys.exit(1)
393
+
394
+ # Step 6: Clean up (delete the uploaded file)
395
+ if not args.no_cleanup:
396
+ logger.info("\nStep 6: Cleaning up...")
397
+ if storage.delete_file(storage_path):
398
+ logger.info("Successfully deleted the uploaded file.")
399
+ else:
400
+ logger.warning("Failed to delete the uploaded file.")
401
+
402
+ except Exception as e:
403
+ logger.error(f"An error occurred: {e}")
404
+ # sys.exit(1)
405
+ finally:
406
+ # Clean up temporary files
407
+ if temp_file_created and os.path.exists(input_file):
408
+ os.unlink(input_file)
409
+ logger.debug(f"Deleted temporary input file: {input_file}")
410
+
411
+ if 'download_file' in locals() and os.path.exists(download_file):
412
+ os.unlink(download_file)
413
+ logger.debug(f"Deleted temporary download file: {download_file}")
414
+
415
+
416
+ if __name__ == "__main__":
417
+ main()
storage.py CHANGED
@@ -102,17 +102,19 @@ class S3StorageBackend(StorageBackend):
102
 
103
  self.s3_client = boto3.client(**client_kwargs)
104
 
105
- def _normalize_content_type(self, content_type):
106
- if not content_type:
107
- return "application/json"
108
- if content_type == "cyclonedx-json":
109
- return "application/cyclonedx+json"
110
- return content_type
111
 
112
  def generate_upload_url(self, file_name, content_type=None, session_id=None):
113
  file_id = session_id or str(uuid.uuid4())
114
  storage_key = f"attestations/{file_id}/{file_name}"
115
- ct = self._normalize_content_type(content_type)
 
 
 
 
 
 
 
 
116
 
117
  params = {
118
  "Bucket": self.bucket,
@@ -132,92 +134,27 @@ class S3StorageBackend(StorageBackend):
132
  return {"upload_url": url, "storage_path": storage_key}
133
 
134
  def generate_download_url(self, storage_path):
135
- return self.s3_client.generate_presigned_url(
136
- "get_object",
137
- Params={"Bucket": self.bucket, "Key": storage_path},
138
- ExpiresIn=current_app.config["STORAGE_CONFIG"].PRESIGNED_URL_EXPIRATION,
139
- )
 
 
 
 
 
 
140
 
141
  def delete_file(self, storage_path):
142
  try:
143
  self.s3_client.delete_object(Bucket=self.bucket, Key=storage_path)
 
144
  return True
145
  except Exception as e:
146
  logger.error(f"Failed to delete S3 object {storage_path}: {e}")
147
  return False
148
 
149
- # class S3StorageBackend(StorageBackend):
150
- # """AWS S3 storage backend"""
151
-
152
- # def __init__(self, access_key, secret_key, region, bucket):
153
- # self.bucket = bucket
154
- # import botocore
155
- # from botocore.config import Config as BotoConfig
156
- # self.s3_client = boto3.client(
157
- # 's3',
158
- # aws_access_key_id=access_key,
159
- # aws_secret_access_key=secret_key,
160
- # region_name=region,
161
- # endpoint_url=f"https://s3.{region}.amazonaws.com",
162
- # config=BotoConfig(
163
- # # signature_version="s3v4",
164
- # s3={'addressing_style': 'virtual'})
165
- # )
166
-
167
- # def generate_upload_url(self, file_name, content_type=None, session_id=None):
168
- # """Generate presigned URL for S3 upload"""
169
- # file_id = session_id or str(uuid.uuid4())
170
- # storage_key = f"attestations/{file_id}/{file_name}"
171
-
172
- # params = {
173
- # 'Bucket': self.bucket,
174
- # 'Key': storage_key,
175
- # }
176
-
177
-
178
- # if content_type:
179
- # if content_type == 'cyclonedx-json':
180
- # content_type = 'application/cyclonedx+json'
181
- # else: content_type = 'application/json'
182
- # else:
183
- # content_type = 'application/json'
184
-
185
- # params['ContentType'] = content_type
186
-
187
- # upload_url = self.s3_client.generate_presigned_url(
188
- # 'put_object',
189
- # Params=params,
190
- # ExpiresIn=current_app.config['STORAGE_CONFIG'].PRESIGNED_URL_EXPIRATION,
191
- # HttpMethod='PUT'
192
- # )
193
-
194
- # logger.info(f"Generated S3 presigned upload URL: {upload_url}")
195
- # logger.info(f"S3 region: {self.s3_client.meta.region_name}, bucket: {self.bucket}, key: {storage_key}")
196
- # return {
197
- # 'upload_url': upload_url,
198
- # 'storage_path': storage_key
199
- # }
200
-
201
- # def generate_download_url(self, storage_path):
202
- # """Generate presigned URL for S3 download"""
203
- # return self.s3_client.generate_presigned_url(
204
- # 'get_object',
205
- # Params={
206
- # 'Bucket': self.bucket,
207
- # 'Key': storage_path
208
- # },
209
- # ExpiresIn=current_app.config['STORAGE_CONFIG'].PRESIGNED_URL_EXPIRATION
210
- # )
211
-
212
- # def delete_file(self, storage_path):
213
- # """Delete S3 object"""
214
- # try:
215
- # self.s3_client.delete_object(Bucket=self.bucket, Key=storage_path)
216
- # return True
217
- # except Exception as e:
218
- # logger.error(f"Failed to delete S3 object {storage_path}: {e}")
219
- # return False
220
-
221
  class MinIOStorageBackend(StorageBackend):
222
  """MinIO storage backend"""
223
 
@@ -307,7 +244,8 @@ def get_storage_backend():
307
  config.AWS_ACCESS_KEY_ID,
308
  config.AWS_SECRET_ACCESS_KEY,
309
  config.AWS_REGION,
310
- config.S3_BUCKET
 
311
  )
312
  else:
313
  raise ValueError("Cloud storage configured but no valid backend settings found")
 
102
 
103
  self.s3_client = boto3.client(**client_kwargs)
104
 
 
 
 
 
 
 
105
 
106
  def generate_upload_url(self, file_name, content_type=None, session_id=None):
107
  file_id = session_id or str(uuid.uuid4())
108
  storage_key = f"attestations/{file_id}/{file_name}"
109
+
110
+ # Normalize storage path (replace double slashes with single)
111
+ storage_key = storage_key.replace('//', '/')
112
+
113
+ # Handle content type carefully - this is crucial for signature generation
114
+ # If your client is consistently sending multipart/form-data, force that content type here
115
+ ct = "multipart/form-data" # Force this content type for now
116
+
117
+ logger.info(f"Using content type {ct} for file {file_name}")
118
 
119
  params = {
120
  "Bucket": self.bucket,
 
134
  return {"upload_url": url, "storage_path": storage_key}
135
 
136
  def generate_download_url(self, storage_path):
137
+ try:
138
+ url = self.s3_client.generate_presigned_url(
139
+ "get_object",
140
+ Params={"Bucket": self.bucket, "Key": storage_path},
141
+ ExpiresIn=current_app.config["STORAGE_CONFIG"].PRESIGNED_URL_EXPIRATION,
142
+ )
143
+ logger.info(f"S3 presigned GET: {url}")
144
+ return url
145
+ except Exception as e:
146
+ logger.error(f"Failed to generate S3 download URL for {storage_path}: {e}")
147
+ return None
148
 
149
  def delete_file(self, storage_path):
150
  try:
151
  self.s3_client.delete_object(Bucket=self.bucket, Key=storage_path)
152
+ logger.info(f"Successfully deleted S3 object: {storage_path}")
153
  return True
154
  except Exception as e:
155
  logger.error(f"Failed to delete S3 object {storage_path}: {e}")
156
  return False
157
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
158
  class MinIOStorageBackend(StorageBackend):
159
  """MinIO storage backend"""
160
 
 
244
  config.AWS_ACCESS_KEY_ID,
245
  config.AWS_SECRET_ACCESS_KEY,
246
  config.AWS_REGION,
247
+ config.S3_BUCKET,
248
+ session_token=config.AWS_SESSION_TOKEN if hasattr(config, 'AWS_SESSION_TOKEN') else None
249
  )
250
  else:
251
  raise ValueError("Cloud storage configured but no valid backend settings found")