Buckets:
| #!/usr/bin/env python3 | |
| """ | |
| Script to upload a file to S3 using the SINDIT API. | |
| This script: | |
| 1. Creates an S3 object node in the knowledge graph | |
| 2. Retrieves a presigned upload URL | |
| 3. Uploads the file using the presigned URL | |
| 4. Verifies the upload was successful | |
| Usage: | |
| python upload_to_s3.py --help | |
| Example: | |
| python upload_to_s3.py | |
| --file test.jpg --bucket my-bucket --key uploads/test.jpg | |
| --connection-uri http://sindit.sinef.cloud/quasar/connection | |
| --api-url http://localhost:8000 | |
| """ | |
| import argparse | |
| import json | |
| import sys | |
| import time | |
| from pathlib import Path | |
| from typing import Optional | |
| import requests | |
| class S3Uploader: | |
| """Helper class for uploading files to S3 via SINDIT API.""" | |
| def __init__( | |
| self, | |
| api_base_url: str = "http://localhost:8000", | |
| username: str = None, | |
| password: str = None, | |
| token: str = None, | |
| ): | |
| """ | |
| Initialize the S3Uploader. | |
| Args: | |
| api_base_url: Base URL of the SINDIT API | |
| username: Username for authentication (if using username/password) | |
| password: Password for authentication (if using username/password) | |
| token: Bearer token for authentication (if already authenticated) | |
| """ | |
| self.api_base_url = api_base_url.rstrip("/") | |
| self.session = requests.Session() | |
| self.token = token | |
| if token: | |
| self.session.headers["Authorization"] = f"Bearer {token}" | |
| elif username and password: | |
| self._authenticate(username, password) | |
| def _authenticate(self, username: str, password: str): | |
| """Authenticate with the API and get a token.""" | |
| auth_url = f"{self.api_base_url}/token" | |
| response = self.session.post( | |
| auth_url, | |
| data={"username": username, "password": password}, | |
| headers={"Content-Type": "application/x-www-form-urlencoded"}, | |
| ) | |
| response.raise_for_status() | |
| token_data = response.json() | |
| self.token = token_data["access_token"] | |
| self.session.headers["Authorization"] = f"Bearer {self.token}" | |
| print(f"✓ Authenticated as {username}") | |
| def create_s3_object_node( | |
| self, | |
| bucket: str, | |
| key: str, | |
| connection_uri: str, | |
| asset_uri: Optional[str] = None, | |
| label: Optional[str] = None, | |
| description: Optional[str] = None, | |
| ) -> dict: | |
| """ | |
| Create an S3 object node in the knowledge graph. | |
| Args: | |
| bucket: S3 bucket name | |
| key: S3 object key (path within bucket) | |
| connection_uri: URI of the S3 connection in the knowledge graph | |
| asset_uri: Optional URI to link this property to an asset | |
| label: Optional label for the node | |
| description: Optional description for the node | |
| Returns: | |
| Response from the API containing the node URI and upload information | |
| """ | |
| endpoint = f"{self.api_base_url}/kg/s3_object" | |
| # Generate a URI for the S3 object node if not provided | |
| if asset_uri is None: | |
| asset_uri = f"http://sindit.sintef.no/s3/{bucket}/{key.replace('/', '-')}" | |
| payload = { | |
| "uri": asset_uri, | |
| "bucket": bucket, | |
| "key": key, | |
| "propertyConnection": {"uri": connection_uri}, | |
| } | |
| if label: | |
| payload["label"] = label | |
| if description: | |
| payload["description"] = description | |
| print("\nCreating S3 object node...") | |
| print(f" Bucket: {bucket}") | |
| print(f" Key: {key}") | |
| print(f" Connection: {connection_uri}") | |
| response = self.session.post(endpoint, json=payload) | |
| # Try to get error details before raising | |
| if not response.ok: | |
| try: | |
| error_detail = response.json() | |
| print(f"✗ API Error: {response.status_code}") | |
| print(f" Detail: {json.dumps(error_detail, indent=2)}") | |
| except Exception as e: | |
| print(f"✗ API Error: {response.status_code}") | |
| print(f" Response: {response.text}") | |
| print(f" Exception: {e}") | |
| response.raise_for_status() | |
| result = response.json() | |
| print("✓ S3 object node created") | |
| return result | |
| def get_node_data(self, node_uri: str) -> dict: | |
| """ | |
| Get the full node data from the knowledge graph. | |
| Args: | |
| node_uri: URI of the node | |
| Returns: | |
| Node data including upload/download URLs | |
| """ | |
| endpoint = f"{self.api_base_url}/kg/node" | |
| params = {"node_uri": node_uri} | |
| response = self.session.get(endpoint, params=params) | |
| if not response.ok: | |
| try: | |
| error_detail = response.json() | |
| print(f"✗ Failed to retrieve node (status {response.status_code})") | |
| print(f" Error: {json.dumps(error_detail, indent=2)}") | |
| except Exception as e: | |
| print(f"✗ Failed to retrieve node (status {response.status_code})") | |
| print(f" Response: {response.text}") | |
| print(f" Exception: {e}") | |
| response.raise_for_status() | |
| return response.json() | |
| def upload_file_with_presigned_post( | |
| self, upload_data: dict, file_path: str | |
| ) -> bool: | |
| """ | |
| Upload a file using presigned POST data. | |
| Args: | |
| upload_data: Dictionary containing 'url' and 'fields' from presigned POST | |
| file_path: Path to the file to upload | |
| Returns: | |
| True if upload was successful | |
| """ | |
| url = upload_data.get("url") | |
| fields = upload_data.get("fields", {}) | |
| if not url: | |
| print("✗ No upload URL found in the data") | |
| return False | |
| print(f"\nUploading file: {file_path}") | |
| print(f" Upload URL: {url}") | |
| print(f" Form fields: {list(fields.keys())}") | |
| with open(file_path, "rb") as f: | |
| # Don't use 'files' parameter - use the presigned POST format | |
| # The 'file' field must be last | |
| files = {"file": (Path(file_path).name, f)} | |
| # Use requests without letting it set Content-Type header | |
| response = requests.post(url, data=fields, files=files) | |
| if response.status_code in (200, 204): | |
| print("✓ File uploaded successfully") | |
| return True | |
| else: | |
| print(f"✗ Upload failed with status {response.status_code}") | |
| print(f" Response: {response.text[:500]}") | |
| return False | |
| def upload_file_with_presigned_put(self, upload_url: str, file_path: str) -> bool: | |
| """ | |
| Upload a file using presigned PUT URL. | |
| Args: | |
| upload_url: Presigned PUT URL | |
| file_path: Path to the file to upload | |
| Returns: | |
| True if upload was successful | |
| """ | |
| print(f"\nUploading file: {file_path}") | |
| print(f" Upload URL: {upload_url[:80]}...") | |
| with open(file_path, "rb") as f: | |
| response = requests.put(upload_url, data=f) | |
| if response.status_code in (200, 204): | |
| print("✓ File uploaded successfully") | |
| return True | |
| else: | |
| print(f"✗ Upload failed with status {response.status_code}") | |
| print(f" Response: {response.text}") | |
| return False | |
| def wait_for_upload_url( | |
| self, | |
| node_uri: str, | |
| max_attempts: int = 6, | |
| delay: int = 2, | |
| initial_delay: int = 3, | |
| ) -> Optional[dict]: | |
| """ | |
| Poll the node until upload URL is available. | |
| Args: | |
| node_uri: URI of the S3 object node | |
| max_attempts: Maximum number of polling attempts (default: 6) | |
| delay: Delay in seconds between attempts (default: 2) | |
| initial_delay: Initial delay before first attempt | |
| to allow connector to start (default: 3) | |
| Returns: | |
| Upload data if found, None otherwise | |
| """ | |
| print(f"\nWaiting for upload URL to be generated...") | |
| # Wait initially to allow async connector startup | |
| if initial_delay > 0: | |
| print(f" Waiting {initial_delay}s for connector startup...") | |
| time.sleep(initial_delay) | |
| for attempt in range(max_attempts): | |
| node_data = self.get_node_data(node_uri) | |
| # Debug: show what fields are present in the node | |
| if attempt == 0: | |
| available_fields = list(node_data.keys()) | |
| print(f" Node fields: {', '.join(available_fields)}") | |
| if "propertyValue" in node_data: | |
| print(f" propertyValue type: {type(node_data['propertyValue'])}") | |
| # Check for upload URL in various possible locations | |
| # Try 'propertyValue' first (standard field name in KG) | |
| if "propertyValue" in node_data: | |
| prop_value = node_data["propertyValue"] | |
| # Handle string URLs (PUT method) | |
| if isinstance(prop_value, str): | |
| print("✓ Upload URL received (from propertyValue)") | |
| return {"url": prop_value, "method": "PUT"} | |
| # Handle dict with url/fields (POST method) | |
| if isinstance(prop_value, dict) and "url" in prop_value: | |
| print("✓ Upload URL received (from propertyValue)") | |
| value = prop_value | |
| # Handle case where 'fields' might be stored as a string | |
| if "fields" in value and isinstance(value["fields"], str): | |
| try: | |
| import ast | |
| value["fields"] = ast.literal_eval(value["fields"]) | |
| print(" (Parsed fields from string format)") | |
| except Exception as e: | |
| print(f" Warning: Could not parse fields string: {e}") | |
| value["method"] = "POST" | |
| return value | |
| # Also check 'value' field | |
| if "value" in node_data: | |
| val = node_data["value"] | |
| # Handle string URLs | |
| if isinstance(val, str): | |
| print("✓ Upload URL received (from value)") | |
| return {"url": val, "method": "PUT"} | |
| # Handle dict | |
| if isinstance(val, dict) and "url" in val: | |
| print("✓ Upload URL received (from value)") | |
| value = val | |
| # Handle case where 'fields' might be stored as a string | |
| if "fields" in value and isinstance(value["fields"], str): | |
| try: | |
| import ast | |
| value["fields"] = ast.literal_eval(value["fields"]) | |
| print(" (Parsed fields from string format)") | |
| except Exception as e: | |
| print(f" Warning: Could not parse fields string: {e}") | |
| value["method"] = "POST" | |
| return value | |
| if "uploadUrl" in node_data: | |
| print("✓ Upload URL received") | |
| return {"url": node_data["uploadUrl"], "method": "PUT"} | |
| if attempt < max_attempts - 1: | |
| print( | |
| f" Attempt {attempt + 1}/{max_attempts}, retrying in {delay}s..." | |
| ) | |
| time.sleep(delay) | |
| print(f"\n✗ Upload URL not available after {max_attempts} attempts") | |
| # Provide diagnostics if still failing | |
| if "propertyConnection" in node_data: | |
| conn = node_data["propertyConnection"] | |
| is_connected = conn.get("isConnected", False) | |
| if not is_connected: | |
| print(f"\n Troubleshooting:") | |
| print(f" - Connection is not active (isConnected: {is_connected})") | |
| print(f" - Connection URI: {conn.get('uri', 'unknown')}") | |
| print(f" - Verify S3 credentials in the vault") | |
| print(f" - Check SINDIT API logs for connection errors") | |
| print( | |
| f" - Try restarting connections via /connection/refresh endpoint" | |
| ) | |
| return None | |
| def main(): | |
| """Main entry point for the script.""" | |
| parser = argparse.ArgumentParser( | |
| description="Upload a file to S3 using SINDIT API", | |
| formatter_class=argparse.RawDescriptionHelpFormatter, | |
| epilog=""" | |
| Examples: | |
| # Upload with specific connection URI | |
| python upload_to_s3.py --file data.json \\ | |
| --bucket my-bucket --key data/file.json \\ | |
| --api-url https://sindit.example.com \\ | |
| --connection-uri http://sindit.sintef.no/quasar/connection | |
| # Upload with authentication | |
| python upload_to_s3.py --f test.jpg | |
| --b my-bucket --k uploads/test.jpg \\ | |
| --api-url http://localhost:8000 \\ | |
| --connection-uri http://sindit.sintef.no/quasar/connection | |
| --username admin --password secret | |
| """, | |
| ) | |
| parser.add_argument( | |
| "--file", "-f", required=True, help="Path to the file to upload" | |
| ) | |
| parser.add_argument("--bucket", "-b", required=True, help="S3 bucket name") | |
| parser.add_argument( | |
| "--key", "-k", required=True, help="S3 object key (path within bucket)" | |
| ) | |
| parser.add_argument( | |
| "--api-url", | |
| required=True, | |
| help="""Base URL of the SINDIT API | |
| (e.g., http://localhost:8000 or https://api.example.com)""", | |
| ) | |
| parser.add_argument( | |
| "--connection-uri", | |
| default="http://sindit.sintef.no/connection/s3", | |
| help="""URI of the S3 connection in the knowledge graph | |
| (e.g., http://sindit.sintef.no/connection/minio)""", | |
| ) | |
| parser.add_argument("--username", help="Username for API authentication") | |
| parser.add_argument("--password", help="Password for API authentication") | |
| parser.add_argument("--token", help="Bearer token for API authentication") | |
| parser.add_argument( | |
| "--asset-uri", help="Optional URI to link this property to an asset" | |
| ) | |
| parser.add_argument("--label", help="Optional label for the S3 object node") | |
| parser.add_argument( | |
| "--description", help="Optional description for the S3 object node" | |
| ) | |
| args = parser.parse_args() | |
| # Validate file exists | |
| file_path = Path(args.file) | |
| if not file_path.exists(): | |
| print(f"✗ Error: File not found: {file_path}") | |
| sys.exit(1) | |
| print("=" * 80) | |
| print("SINDIT S3 File Upload") | |
| print("=" * 80) | |
| # Initialize uploader | |
| try: | |
| uploader = S3Uploader( | |
| api_base_url=args.api_url, | |
| username=args.username, | |
| password=args.password, | |
| token=args.token, | |
| ) | |
| except requests.HTTPError as e: | |
| print(f"✗ Authentication failed: {e}") | |
| sys.exit(1) | |
| # Create S3 object node | |
| try: | |
| result = uploader.create_s3_object_node( | |
| bucket=args.bucket, | |
| key=args.key, | |
| connection_uri=args.connection_uri, | |
| asset_uri=args.asset_uri, | |
| label=args.label, | |
| description=args.description, | |
| ) | |
| except requests.HTTPError as e: | |
| print(f"✗ Failed to create S3 object node: {e}") | |
| if e.response is not None: | |
| try: | |
| error_detail = e.response.json() | |
| print(f" Error details: {json.dumps(error_detail, indent=2)}") | |
| except Exception as e: | |
| print(f" Response: {e.response.text}") | |
| print(f" Exception: {e}") | |
| sys.exit(1) | |
| # Extract node URI from result | |
| # The API returns {'result': True} on success, so we use the URI we sent | |
| if args.asset_uri: | |
| node_uri = args.asset_uri | |
| elif "result" in result and isinstance(result["result"], str): | |
| # If API returns a URI string, use that | |
| node_uri = result["result"] | |
| else: | |
| # Otherwise, use the generated URI from the request | |
| node_uri = ( | |
| f"http://sindit.sintef.no/s3/{args.bucket}/{args.key.replace('/', '-')}" | |
| ) | |
| print(f" Node URI: {node_uri}") | |
| # Wait for upload URL to be generated | |
| upload_data = uploader.wait_for_upload_url(node_uri) | |
| if not upload_data: | |
| print("\n✗ Failed to get upload URL") | |
| sys.exit(1) | |
| # Upload the file | |
| success = False | |
| method = upload_data.get("method", "PUT") | |
| if method == "POST" or "fields" in upload_data: | |
| # Presigned POST | |
| success = uploader.upload_file_with_presigned_post(upload_data, str(file_path)) | |
| elif "url" in upload_data: | |
| # Presigned PUT (default) | |
| success = uploader.upload_file_with_presigned_put( | |
| upload_data["url"], str(file_path) | |
| ) | |
| if success: | |
| print("\n" + "=" * 80) | |
| print("✓ Upload completed successfully!") | |
| print("=" * 80) | |
| print(f"\nFile: {file_path.name}") | |
| print(f"Bucket: {args.bucket}") | |
| print(f"Key: {args.key}") | |
| print(f"Node URI: {node_uri}") | |
| else: | |
| print("\n✗ Upload failed") | |
| sys.exit(1) | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 17.4 kB
- Xet hash:
- f7aa20380dbbe6bbd7c2663d87ded918d3f08052629bd135d1f12ee8929fa086
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.