Buckets:
AnonymousSpottedToad/human_verified_bucket / codebase /python /SINTEF-9012_SINDIT20 /test_s3_uploads.py
| #!/usr/bin/env python3 | |
| """ | |
| Test script for MinIO/S3 object storage. | |
| This script tests S3 connectivity and operations directly with MinIO, | |
| including listing buckets, listing objects, uploading and downloading objects. | |
| """ | |
| import argparse | |
| import sys | |
| from datetime import timedelta | |
| from pathlib import Path | |
| from minio import Minio | |
| from minio.error import S3Error | |
| class MinIOTester: | |
| """Helper class for testing MinIO/S3 operations.""" | |
| def __init__( | |
| self, | |
| endpoint: str, | |
| access_key: str, | |
| secret_key: str, | |
| secure: bool = True, | |
| ): | |
| """ | |
| Initialize the MinIOTester. | |
| Args: | |
| endpoint: MinIO endpoint (e.g., minio.sintef.cloud or localhost:9000) | |
| access_key: Access key for MinIO | |
| secret_key: Secret key for MinIO | |
| secure: Use HTTPS if True, HTTP if False | |
| """ | |
| self.endpoint = endpoint | |
| self.access_key = access_key | |
| self.secure = secure | |
| try: | |
| self.client = Minio( | |
| endpoint, | |
| access_key=access_key, | |
| secret_key=secret_key, | |
| secure=secure, | |
| ) | |
| print(f"✓ MinIO client initialized") | |
| print(f" Endpoint: {endpoint}") | |
| print(f" Secure: {secure}") | |
| except Exception as e: | |
| print(f"✗ Failed to initialize MinIO client: {e}") | |
| raise | |
| def list_buckets(self) -> list: | |
| """List all buckets.""" | |
| try: | |
| buckets = self.client.list_buckets() | |
| return [{"name": b.name, "creation_date": b.creation_date} for b in buckets] | |
| except S3Error as e: | |
| print(f"✗ Failed to list buckets: {e}") | |
| return [] | |
| def bucket_exists(self, bucket: str) -> bool: | |
| """Check if a bucket exists.""" | |
| try: | |
| return self.client.bucket_exists(bucket) | |
| except S3Error as e: | |
| print(f"✗ Error checking bucket: {e}") | |
| return False | |
| def create_bucket(self, bucket: str) -> bool: | |
| """Create a new bucket.""" | |
| try: | |
| if self.client.bucket_exists(bucket): | |
| print(f"⚠ Bucket '{bucket}' already exists") | |
| return False | |
| self.client.make_bucket(bucket) | |
| print(f"✓ Created bucket: {bucket}") | |
| return True | |
| except S3Error as e: | |
| print(f"✗ Failed to create bucket: {e}") | |
| return False | |
| def list_objects( | |
| self, bucket: str, prefix: str = "", recursive: bool = True | |
| ) -> list: | |
| """ | |
| List objects in a bucket. | |
| Args: | |
| bucket: Bucket name | |
| prefix: Optional prefix to filter objects | |
| recursive: List recursively if True | |
| Returns: | |
| List of object information dictionaries | |
| """ | |
| try: | |
| objects = self.client.list_objects( | |
| bucket, prefix=prefix, recursive=recursive | |
| ) | |
| result = [] | |
| for obj in objects: | |
| result.append( | |
| { | |
| "key": obj.object_name, | |
| "size": obj.size, | |
| "etag": obj.etag, | |
| "last_modified": obj.last_modified, | |
| "content_type": obj.content_type, | |
| } | |
| ) | |
| return result | |
| except S3Error as e: | |
| print(f"✗ Failed to list objects: {e}") | |
| return [] | |
| def get_object_info(self, bucket: str, key: str) -> dict: | |
| """ | |
| Get information about a specific object. | |
| Args: | |
| bucket: Bucket name | |
| key: Object key | |
| Returns: | |
| Object information dictionary | |
| """ | |
| try: | |
| stat = self.client.stat_object(bucket, key) | |
| return { | |
| "key": key, | |
| "size": stat.size, | |
| "etag": stat.etag, | |
| "last_modified": stat.last_modified, | |
| "content_type": stat.content_type, | |
| "metadata": stat.metadata, | |
| } | |
| except S3Error as e: | |
| print(f"✗ Failed to get object info: {e}") | |
| return {} | |
| def download_object(self, bucket: str, key: str, output_path: str = None) -> bool: | |
| """ | |
| Download an object from MinIO. | |
| Args: | |
| bucket: Bucket name | |
| key: Object key | |
| output_path: Optional path to save the file | |
| Returns: | |
| True if successful | |
| """ | |
| try: | |
| if output_path: | |
| self.client.fget_object(bucket, key, output_path) | |
| print(f"✓ Downloaded to: {output_path}") | |
| else: | |
| response = self.client.get_object(bucket, key) | |
| data = response.read() | |
| response.close() | |
| response.release_conn() | |
| print(f"✓ Download successful ({len(data)} bytes)") | |
| print(f" Preview: {data[:100]}") | |
| return True | |
| except S3Error as e: | |
| print(f"✗ Failed to download object: {e}") | |
| return False | |
| def upload_object(self, bucket: str, key: str, file_path: str) -> bool: | |
| """ | |
| Upload a file to MinIO. | |
| Args: | |
| bucket: Bucket name | |
| key: Object key (destination path) | |
| file_path: Local file path to upload | |
| Returns: | |
| True if successful | |
| """ | |
| try: | |
| path = Path(file_path) | |
| if not path.exists(): | |
| print(f"✗ File not found: {file_path}") | |
| return False | |
| self.client.fput_object(bucket, key, str(path)) | |
| print(f"✓ Uploaded: {file_path} -> {bucket}/{key}") | |
| return True | |
| except S3Error as e: | |
| print(f"✗ Failed to upload object: {e}") | |
| return False | |
| def delete_object(self, bucket: str, key: str) -> bool: | |
| """ | |
| Delete an object from MinIO. | |
| Args: | |
| bucket: Bucket name | |
| key: Object key | |
| Returns: | |
| True if successful | |
| """ | |
| try: | |
| self.client.remove_object(bucket, key) | |
| print(f"✓ Deleted: {bucket}/{key}") | |
| return True | |
| except S3Error as e: | |
| print(f"✗ Failed to delete object: {e}") | |
| return False | |
| def generate_presigned_url(self, bucket: str, key: str, expires: int = 3600) -> str: | |
| """ | |
| Generate a presigned URL for an object. | |
| Args: | |
| bucket: Bucket name | |
| key: Object key | |
| expires: Expiration time in seconds | |
| Returns: | |
| Presigned URL string | |
| """ | |
| try: | |
| url = self.client.presigned_get_object( | |
| bucket, key, expires=timedelta(seconds=expires) | |
| ) | |
| return url | |
| except S3Error as e: | |
| print(f"✗ Failed to generate presigned URL: {e}") | |
| return "" | |
| def main(): | |
| """Main entry point for the script.""" | |
| parser = argparse.ArgumentParser( | |
| description="Test MinIO/S3 operations directly", | |
| formatter_class=argparse.RawDescriptionHelpFormatter, | |
| epilog=""" | |
| Examples: | |
| # List all buckets | |
| python test_s3_uploads.py \\ | |
| --endpoint minio.sintef.cloud \\ | |
| --access-key YOUR_ACCESS_KEY \\ | |
| --secret-key YOUR_SECRET_KEY \\ | |
| --secure \\ # use --secure for HTTPS (443) | |
| --command buckets | |
| # List objects in a bucket | |
| python test_s3_uploads.py \\ | |
| --endpoint minio.sintef.cloud \\ | |
| --access-key YOUR_ACCESS_KEY \\ | |
| --secret-key YOUR_SECRET_KEY \\ | |
| --secure \\ # omit this flag for HTTP endpoint | |
| --command list --bucket my-bucket | |
| # List objects with prefix | |
| python test_s3_uploads.py \\ | |
| --endpoint minio.sintef.cloud \\ | |
| --access-key YOUR_ACCESS_KEY \\ | |
| --secret-key YOUR_SECRET_KEY \\ | |
| --secure \\ # omit this flag for HTTP endpoint | |
| --command list --bucket my-bucket --prefix uploads/ | |
| # Get object info | |
| python test_s3_uploads.py \\ | |
| --endpoint minio.sintef.cloud \\ | |
| --access-key YOUR_ACCESS_KEY \\ | |
| --secret-key YOUR_SECRET_KEY \\ | |
| --secure \\ # omit this flag for HTTP endpoint | |
| --command get --bucket my-bucket --key uploads/test.jpg | |
| # Download object | |
| python test_s3_uploads.py \\ | |
| --endpoint minio.sintef.cloud \\ | |
| --access-key YOUR_ACCESS_KEY \\ | |
| --secret-key YOUR_SECRET_KEY \\ | |
| --secure \\ # omit this flag for HTTP endpoint | |
| --command download --bucket my-bucket | |
| --key uploads/test.jpg \\ | |
| --output downloaded.jpg | |
| # Upload object | |
| python test_s3_uploads.py \\ | |
| --endpoint minio.sintef.cloud \\ | |
| --access-key YOUR_ACCESS_KEY \\ | |
| --secret-key YOUR_SECRET_KEY \\ | |
| --secure \\ # omit this flag for HTTP endpoint | |
| --command upload --bucket my-bucket | |
| --key uploads/new-file.jpg \\ | |
| --file local-file.jpg | |
| # Create bucket | |
| python test_s3_uploads.py \\ | |
| --endpoint minio.sintef.cloud \\ | |
| --access-key YOUR_ACCESS_KEY \\ | |
| --secret-key YOUR_SECRET_KEY \\ | |
| --secure \\ # omit this flag for HTTP endpoint | |
| --command create-bucket --bucket new-bucket | |
| # Generate presigned URL | |
| python test_s3_uploads.py \\ | |
| --endpoint minio.sintef.cloud \\ | |
| --access-key YOUR_ACCESS_KEY \\ | |
| --secret-key YOUR_SECRET_KEY \\ | |
| --secure \\ | |
| --command presigned --bucket my-bucket | |
| --key uploads/test.jpg | |
| """, | |
| ) | |
| parser.add_argument( | |
| "--endpoint", | |
| required=True, | |
| help="MinIO endpoint (e.g., minio.sintef.cloud or localhost:9000)", | |
| ) | |
| parser.add_argument( | |
| "--access-key", | |
| required=True, | |
| help="MinIO access key", | |
| ) | |
| parser.add_argument( | |
| "--secret-key", | |
| required=True, | |
| help="MinIO secret key", | |
| ) | |
| parser.add_argument( | |
| "--secure", | |
| action="store_true", | |
| help="Use HTTPS (default: HTTP)", | |
| ) | |
| parser.add_argument( | |
| "--command", | |
| choices=[ | |
| "buckets", | |
| "create-bucket", | |
| "list", | |
| "get", | |
| "download", | |
| "upload", | |
| "delete", | |
| "presigned", | |
| ], | |
| default="buckets", | |
| help="Command to execute", | |
| ) | |
| parser.add_argument("--bucket", help="Bucket name") | |
| parser.add_argument("--key", help="Object key") | |
| parser.add_argument("--prefix", default="", help="Prefix filter for listing") | |
| parser.add_argument("--file", help="Local file path (for upload)") | |
| parser.add_argument("--output", help="Output file path (for download)") | |
| parser.add_argument( | |
| "--expires", type=int, default=3600, help="Presigned URL expiration in seconds" | |
| ) | |
| args = parser.parse_args() | |
| print("=" * 80) | |
| print("MinIO/S3 Test Script") | |
| print("=" * 80) | |
| print(f"Endpoint: {args.endpoint}") | |
| print(f"Secure: {args.secure}") | |
| print(f"Command: {args.command}") | |
| print() | |
| # Initialize tester | |
| try: | |
| tester = MinIOTester( | |
| endpoint=args.endpoint, | |
| access_key=args.access_key, | |
| secret_key=args.secret_key, | |
| secure=args.secure, | |
| ) | |
| except Exception as e: | |
| print(f"✗ Initialization failed: {e}") | |
| sys.exit(1) | |
| # Execute command | |
| try: | |
| if args.command == "buckets": | |
| print("Listing all buckets...") | |
| buckets = tester.list_buckets() | |
| if buckets: | |
| print(f"\n✓ Found {len(buckets)} bucket(s):") | |
| for bucket in buckets: | |
| print(f" - {bucket['name']}") | |
| print(f" Created: {bucket['creation_date']}") | |
| else: | |
| print(" No buckets found") | |
| elif args.command == "create-bucket": | |
| if not args.bucket: | |
| print("✗ --bucket is required for create-bucket command") | |
| sys.exit(1) | |
| print(f"Creating bucket '{args.bucket}'...") | |
| tester.create_bucket(args.bucket) | |
| elif args.command == "list": | |
| if not args.bucket: | |
| print("✗ --bucket is required for list command") | |
| sys.exit(1) | |
| # Check if bucket exists first | |
| if not tester.bucket_exists(args.bucket): | |
| print(f"✗ Bucket '{args.bucket}' does not exist") | |
| sys.exit(1) | |
| print(f"Listing objects in bucket '{args.bucket}'...") | |
| if args.prefix: | |
| print(f" Prefix filter: '{args.prefix}'") | |
| objects = tester.list_objects(args.bucket, args.prefix) | |
| if objects: | |
| print(f"\n✓ Found {len(objects)} object(s):") | |
| for obj in objects: | |
| print(f"\n Key: {obj['key']}") | |
| print(f" Size: {obj['size']} bytes") | |
| print(f" Last Modified: {obj['last_modified']}") | |
| print(f" Content Type: {obj['content_type']}") | |
| print(f" ETag: {obj['etag']}") | |
| else: | |
| print(" No objects found") | |
| elif args.command == "get": | |
| if not args.bucket or not args.key: | |
| print("✗ --bucket and --key are required for get command") | |
| sys.exit(1) | |
| print(f"Getting object info: {args.bucket}/{args.key}") | |
| obj_info = tester.get_object_info(args.bucket, args.key) | |
| if obj_info: | |
| print("\n✓ Object Information:") | |
| print(f" Key: {obj_info['key']}") | |
| print(f" Size: {obj_info['size']} bytes") | |
| print(f" Last Modified: {obj_info['last_modified']}") | |
| print(f" Content Type: {obj_info['content_type']}") | |
| print(f" ETag: {obj_info['etag']}") | |
| if obj_info.get("metadata"): | |
| print(f" Metadata: {obj_info['metadata']}") | |
| elif args.command == "download": | |
| if not args.bucket or not args.key: | |
| print("✗ --bucket and --key are required for download command") | |
| sys.exit(1) | |
| print(f"Downloading object: {args.bucket}/{args.key}") | |
| success = tester.download_object(args.bucket, args.key, args.output) | |
| if not success: | |
| sys.exit(1) | |
| elif args.command == "upload": | |
| if not args.bucket or not args.key or not args.file: | |
| print("✗ --bucket, --key, and --file are required for upload command") | |
| sys.exit(1) | |
| print(f"Uploading file: {args.file} -> {args.bucket}/{args.key}") | |
| success = tester.upload_object(args.bucket, args.key, args.file) | |
| if not success: | |
| sys.exit(1) | |
| elif args.command == "delete": | |
| if not args.bucket or not args.key: | |
| print("✗ --bucket and --key are required for delete command") | |
| sys.exit(1) | |
| print(f"Deleting object: {args.bucket}/{args.key}") | |
| success = tester.delete_object(args.bucket, args.key) | |
| if not success: | |
| sys.exit(1) | |
| elif args.command == "presigned": | |
| if not args.bucket or not args.key: | |
| print("✗ --bucket and --key are required for presigned command") | |
| sys.exit(1) | |
| print(f"Generating presigned URL for: {args.bucket}/{args.key}") | |
| print(f" Expires in: {args.expires} seconds") | |
| url = tester.generate_presigned_url(args.bucket, args.key, args.expires) | |
| if url: | |
| print(f"\n✓ Presigned URL:") | |
| print(f" {url}") | |
| else: | |
| sys.exit(1) | |
| except S3Error as e: | |
| print(f"\n✗ MinIO Error: {e}") | |
| sys.exit(1) | |
| except Exception as e: | |
| print(f"\n✗ Error: {e}") | |
| import traceback | |
| traceback.print_exc() | |
| sys.exit(1) | |
| print("\n" + "=" * 80) | |
| print("✓ Operation completed successfully") | |
| print("=" * 80) | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 16.2 kB
- Xet hash:
- 068d74b26363f6b14390080a8a658cf04ec184605806445ff2c58e1dfdc42a87
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.