Spaces:
Sleeping
Sleeping
Download phase3_native_reference_generator.py from RadixLaboratory/radixlab-anime: direct link, hf CLI and curl.
- Browser
- Download file 22.8 kB
-
https://huggingface.co/spaces/RadixLaboratory/radixlab-anime/resolve/main/phase3_native_reference_generator.py
- Command line
-
hf download hf://spaces/RadixLaboratory/radixlab-anime/phase3_native_reference_generator.py
-
curl -L -o phase3_native_reference_generator.py https://huggingface.co/spaces/RadixLaboratory/radixlab-anime/resolve/main/phase3_native_reference_generator.py
22.8 kB
| """ | |
| Phase 3: Native Reference Image Generator | |
| ========================================== | |
| This module implements Vertex AI Native Reference Images using the capability model. | |
| It provides true subject locking for character and location consistency across scenes. | |
| Key Features: | |
| - Service Account Authentication (required for native references) | |
| - Vertex AI imagen-3.0-capability-001 model | |
| - Native reference images with REFERENCE_TYPE_SUBJECT | |
| - Subject type inference (PERSON, ANIMAL, PRODUCT) | |
| - Fallback to Phase 2 when service account unavailable | |
| WHY CAPABILITY MODEL IS USED: | |
| ----------------------------- | |
| The imagen-3.0-capability-001 model supports native reference images, which allows | |
| passing actual image data as reference subjects. This provides much stronger identity | |
| consistency compared to prompt-based descriptions (Phase 2). | |
| WHY API KEY IS NOT ALLOWED HERE: | |
| -------------------------------- | |
| Native reference image features require the Vertex AI SDK with service account | |
| authentication. The REST API with API key (used in Phase 2) does not support | |
| the referenceImages parameter required for subject locking. | |
| Authentication Requirements: | |
| - Service account with roles/aiplatform.user | |
| - GOOGLE_APPLICATION_CREDENTIALS environment variable pointing to service account JSON | |
| """ | |
| import os | |
| import base64 | |
| import uuid | |
| from typing import Optional, Dict, List, Any, Tuple | |
| from PIL import Image | |
| # ============================================================================= | |
| # CONFIGURATION | |
| # ============================================================================= | |
| # Service account credentials path | |
| GOOGLE_APPLICATION_CREDENTIALS = os.getenv("GOOGLE_APPLICATION_CREDENTIALS") | |
| # GCP Project configuration | |
| GCP_PROJECT_ID = os.getenv("GCP_PROJECT_ID") | |
| GCP_LOCATION = os.getenv("GCP_LOCATION", "us-central1") | |
| # Model configuration | |
| CAPABILITY_MODEL_ID = "imagen-3.0-capability-001" | |
| # Output directory (inherited from main app) | |
| BASE_DIR = os.path.dirname(os.path.abspath(__file__)) | |
| OUTPUT_DIR_IMG = os.path.join(BASE_DIR, "outputs", "images") | |
| # Ensure output directory exists | |
| os.makedirs(OUTPUT_DIR_IMG, exist_ok=True) | |
| # ============================================================================= | |
| # SERVICE ACCOUNT INITIALIZATION | |
| # ============================================================================= | |
| # | |
| # Supports two authentication methods: | |
| # | |
| # 1. FILE-BASED (Local development): | |
| # Set GOOGLE_APPLICATION_CREDENTIALS=/path/to/service-account.json | |
| # | |
| # 2. SECRET-BASED (Hugging Face Spaces / Cloud deployment): | |
| # Set GCP_SERVICE_ACCOUNT_JSON=<entire JSON content as string> | |
| # The module will automatically write this to a temp file | |
| # | |
| # ============================================================================= | |
| PHASE3_AVAILABLE = False | |
| _vertex_client = None | |
| _imagen_model = None | |
| _temp_credentials_file = None # Track temp file for cleanup | |
| def setup_credentials_from_secret() -> Optional[str]: | |
| """ | |
| Set up credentials from HF Spaces secret (GCP_SERVICE_ACCOUNT_JSON). | |
| This allows storing the entire service account JSON as an environment | |
| variable/secret, which is required for Hugging Face Spaces deployment. | |
| Returns: | |
| Path to temporary credentials file, or None if not configured | |
| """ | |
| global _temp_credentials_file | |
| import json | |
| import tempfile | |
| # Check for JSON content in environment variable | |
| sa_json = os.getenv("GCP_SERVICE_ACCOUNT_JSON") | |
| if not sa_json: | |
| return None | |
| try: | |
| # Validate JSON structure | |
| creds = json.loads(sa_json) | |
| required_fields = ["type", "project_id", "private_key", "client_email"] | |
| missing = [f for f in required_fields if f not in creds] | |
| if missing: | |
| print(f"β οΈ GCP_SERVICE_ACCOUNT_JSON missing fields: {', '.join(missing)}") | |
| return None | |
| if creds.get("type") != "service_account": | |
| print(f"β οΈ Invalid credential type in GCP_SERVICE_ACCOUNT_JSON") | |
| return None | |
| # Write to temporary file | |
| fd, temp_path = tempfile.mkstemp(suffix=".json", prefix="gcp_sa_") | |
| with os.fdopen(fd, 'w') as f: | |
| json.dump(creds, f) | |
| _temp_credentials_file = temp_path | |
| # Set environment variable for Google libraries | |
| os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = temp_path | |
| print(f"π Phase 3: Credentials loaded from HF secret (GCP_SERVICE_ACCOUNT_JSON)") | |
| print(f" Service account: {creds.get('client_email')}") | |
| return temp_path | |
| except json.JSONDecodeError as e: | |
| print(f"β οΈ Invalid JSON in GCP_SERVICE_ACCOUNT_JSON: {e}") | |
| return None | |
| except Exception as e: | |
| print(f"β οΈ Error setting up credentials from secret: {e}") | |
| return None | |
| def check_service_account_credentials() -> Tuple[bool, str]: | |
| """ | |
| Validate that service account credentials are properly configured. | |
| Supports both: | |
| - File-based: GOOGLE_APPLICATION_CREDENTIALS=/path/to/file.json | |
| - Secret-based: GCP_SERVICE_ACCOUNT_JSON=<json content> (for HF Spaces) | |
| Returns: | |
| Tuple of (is_valid, message) | |
| """ | |
| # First, try to set up from HF Spaces secret if available | |
| if os.getenv("GCP_SERVICE_ACCOUNT_JSON"): | |
| temp_path = setup_credentials_from_secret() | |
| if temp_path: | |
| return True, f"Credentials loaded from HF secret" | |
| # Then check file-based credentials | |
| creds_path = os.getenv("GOOGLE_APPLICATION_CREDENTIALS") | |
| if not creds_path: | |
| return False, "No credentials configured (set GOOGLE_APPLICATION_CREDENTIALS or GCP_SERVICE_ACCOUNT_JSON)" | |
| if not os.path.isfile(creds_path): | |
| return False, f"Service account file not found: {creds_path}" | |
| # Validate JSON structure | |
| try: | |
| import json | |
| with open(creds_path, 'r') as f: | |
| creds = json.load(f) | |
| required_fields = ["type", "project_id", "private_key", "client_email"] | |
| missing = [f for f in required_fields if f not in creds] | |
| if missing: | |
| return False, f"Service account JSON missing fields: {', '.join(missing)}" | |
| if creds.get("type") != "service_account": | |
| return False, f"Invalid credential type: {creds.get('type')} (expected 'service_account')" | |
| return True, f"Service account validated: {creds.get('client_email')}" | |
| except json.JSONDecodeError as e: | |
| return False, f"Invalid JSON in service account file: {e}" | |
| except Exception as e: | |
| return False, f"Error reading service account file: {e}" | |
| def init_phase3_vertex_ai() -> bool: | |
| """ | |
| Initialize Vertex AI with service account authentication for Phase 3. | |
| This initialization is separate from Phase 2 (API key) and uses the | |
| full Vertex AI SDK with service account credentials. | |
| Returns: | |
| True if initialization successful, False otherwise | |
| """ | |
| global PHASE3_AVAILABLE, _vertex_client, _imagen_model | |
| # Check credentials first | |
| is_valid, message = check_service_account_credentials() | |
| if not is_valid: | |
| print(f"β οΈ Phase 3 (Native References): {message}") | |
| print(" β Falling back to Phase 2 (prompt-based consistency)") | |
| return False | |
| print(f"π Phase 3: {message}") | |
| # Check project ID | |
| project_id = os.getenv("GCP_PROJECT_ID") | |
| if not project_id: | |
| print("β οΈ Phase 3: GCP_PROJECT_ID not set") | |
| print(" β Falling back to Phase 2 (prompt-based consistency)") | |
| return False | |
| try: | |
| # Import Vertex AI SDK | |
| import vertexai | |
| from vertexai.preview.vision_models import ImageGenerationModel | |
| # Initialize Vertex AI with service account | |
| location = os.getenv("GCP_LOCATION", "us-central1") | |
| vertexai.init(project=project_id, location=location) | |
| # Load the capability model | |
| print(f"π Loading Vertex AI model: {CAPABILITY_MODEL_ID}...") | |
| _imagen_model = ImageGenerationModel.from_pretrained(CAPABILITY_MODEL_ID) | |
| # Verify model loaded successfully | |
| if _imagen_model is None: | |
| raise RuntimeError("Model failed to load") | |
| PHASE3_AVAILABLE = True | |
| print(f"β Phase 3 (Native References) initialized successfully") | |
| print(f" Model: {CAPABILITY_MODEL_ID}") | |
| print(f" Project: {project_id}") | |
| print(f" Location: {location}") | |
| return True | |
| except ImportError as e: | |
| print(f"β οΈ Phase 3: Vertex AI SDK not installed - {e}") | |
| print(" Install with: pip install google-cloud-aiplatform vertexai") | |
| print(" β Falling back to Phase 2 (prompt-based consistency)") | |
| return False | |
| except Exception as e: | |
| print(f"β οΈ Phase 3: Initialization failed - {e}") | |
| print(" β Falling back to Phase 2 (prompt-based consistency)") | |
| return False | |
| def is_phase3_available() -> bool: | |
| """Check if Phase 3 native reference generation is available.""" | |
| return PHASE3_AVAILABLE and _imagen_model is not None | |
| # ============================================================================= | |
| # SUBJECT TYPE INFERENCE | |
| # ============================================================================= | |
| def infer_subject_type(ref_key: str, image_path: Optional[str] = None) -> str: | |
| """ | |
| Infer the subject type from the reference key or image content. | |
| Subject types supported by Vertex AI: | |
| - PERSON: Human characters | |
| - ANIMAL: Animals, creatures | |
| - PRODUCT: Objects, items, products | |
| Args: | |
| ref_key: Reference key name (e.g., "hero", "cat", "sword") | |
| image_path: Optional path to image for content-based inference | |
| Returns: | |
| One of: "PERSON", "ANIMAL", "PRODUCT" | |
| """ | |
| ref_lower = ref_key.lower() if ref_key else "" | |
| # Person indicators | |
| person_keywords = [ | |
| "person", "human", "man", "woman", "boy", "girl", "child", "kid", | |
| "hero", "villain", "character", "protagonist", "antagonist", | |
| "father", "mother", "daughter", "son", "parent", "adult", | |
| "doctor", "nurse", "teacher", "student", "scientist", "professor", | |
| "warrior", "knight", "mage", "wizard", "witch", | |
| "narrator", "speaker", "presenter", | |
| "pediatrician", "medical" | |
| ] | |
| # Animal indicators | |
| animal_keywords = [ | |
| "animal", "creature", "beast", "pet", | |
| "cat", "dog", "bird", "fish", "rabbit", "hamster", | |
| "lion", "tiger", "bear", "wolf", "fox", "deer", | |
| "dragon", "unicorn", "phoenix", "monster", | |
| "horse", "cow", "pig", "sheep", "goat", | |
| "elephant", "giraffe", "zebra", "monkey", | |
| "snake", "lizard", "frog", "turtle" | |
| ] | |
| # Product indicators (check first since some might conflict with defaults) | |
| product_keywords = [ | |
| "object", "item", "product", "thing", "prop", | |
| "sword", "weapon", "shield", "armor", "helmet", | |
| "car", "vehicle", "ship", "boat", "plane", "aircraft", | |
| "building", "house", "tower", "castle", | |
| "book", "scroll", "map", "letter", | |
| "potion", "bottle", "cup", "glass", "bowl", | |
| "ring", "necklace", "crown", "jewel", "gem", | |
| "wand", "staff", "orb", "crystal", | |
| "food", "fruit", "vegetable", "bread", "cake", | |
| "furniture", "chair", "table", "bed", "desk", | |
| "tool", "hammer", "wrench", "screwdriver", | |
| "machine", "robot", "device", "gadget", "computer" | |
| ] | |
| # Check for product keywords first | |
| for keyword in product_keywords: | |
| if keyword in ref_lower: | |
| return "PRODUCT" | |
| # Check for person keywords | |
| for keyword in person_keywords: | |
| if keyword in ref_lower: | |
| return "PERSON" | |
| # Check for animal keywords | |
| for keyword in animal_keywords: | |
| if keyword in ref_lower: | |
| return "ANIMAL" | |
| # Default to PERSON for character references | |
| # Most character references in animation are people | |
| character_indicators = ["char", "actor", "role", "cast"] | |
| for indicator in character_indicators: | |
| if indicator in ref_lower: | |
| return "PERSON" | |
| # Default to PERSON for unrecognized references | |
| # (most common case in animation production) | |
| return "PERSON" | |
| # ============================================================================= | |
| # REFERENCE IMAGE PREPARATION | |
| # ============================================================================= | |
| def prepare_reference_image(image_path: str) -> Optional[bytes]: | |
| """ | |
| Prepare a reference image for the Vertex AI API. | |
| Ensures image meets API requirements: | |
| - Supported formats (PNG, JPEG) | |
| - Reasonable file size | |
| Args: | |
| image_path: Path to the reference image | |
| Returns: | |
| Image bytes, or None on failure | |
| """ | |
| if not image_path or not os.path.isfile(image_path): | |
| return None | |
| try: | |
| # Read and validate image | |
| with Image.open(image_path) as img: | |
| # Convert to RGB if necessary (remove alpha channel) | |
| if img.mode in ('RGBA', 'LA', 'P'): | |
| background = Image.new('RGB', img.size, (255, 255, 255)) | |
| if img.mode == 'P': | |
| img = img.convert('RGBA') | |
| background.paste(img, mask=img.split()[-1] if img.mode == 'RGBA' else None) | |
| img = background | |
| elif img.mode != 'RGB': | |
| img = img.convert('RGB') | |
| # Resize if too large (max 4096px on longest side) | |
| max_size = 4096 | |
| if max(img.size) > max_size: | |
| ratio = max_size / max(img.size) | |
| new_size = (int(img.size[0] * ratio), int(img.size[1] * ratio)) | |
| img = img.resize(new_size, Image.Resampling.LANCZOS) | |
| # Save to bytes | |
| import io | |
| buffer = io.BytesIO() | |
| img.save(buffer, format='PNG') | |
| return buffer.getvalue() | |
| except Exception as e: | |
| print(f"β οΈ Error preparing reference image {image_path}: {e}") | |
| return None | |
| def load_reference_image_as_vertex_image(image_path: str): | |
| """ | |
| Load a reference image as a Vertex AI Image object. | |
| Args: | |
| image_path: Path to the reference image | |
| Returns: | |
| Vertex AI Image object, or None on failure | |
| """ | |
| if not image_path or not os.path.isfile(image_path): | |
| return None | |
| try: | |
| from vertexai.preview.vision_models import Image as VertexImage | |
| # Load image using Vertex AI's Image class | |
| return VertexImage.load_from_file(image_path) | |
| except Exception as e: | |
| print(f"β οΈ Error loading reference image as Vertex Image: {e}") | |
| return None | |
| # ============================================================================= | |
| # NATIVE REFERENCE IMAGE GENERATION | |
| # ============================================================================= | |
| def generate_with_native_references( | |
| prompt: str, | |
| scene_id: str, | |
| variant: str, | |
| ref_images: Dict[str, Any] = None, | |
| aspect_ratio: str = "9:16" | |
| ) -> Optional[str]: | |
| """ | |
| Generate an image using Vertex AI native reference images. | |
| This function uses the imagen-3.0-capability-001 model with true subject | |
| locking via the referenceImages parameter. | |
| Args: | |
| prompt: The text prompt for image generation | |
| scene_id: Scene identifier for naming | |
| variant: "A" or "B" for the variant | |
| ref_images: Dict with: | |
| - 'character': Single character path (backward compat) | |
| - 'characters': List of (key, path) tuples | |
| - 'location': Location path | |
| aspect_ratio: Aspect ratio (default "9:16" for vertical) | |
| Returns: | |
| Path to generated image, or None on failure | |
| """ | |
| global _imagen_model | |
| if not is_phase3_available(): | |
| print("β οΈ Phase 3 not available, cannot generate with native references") | |
| return None | |
| try: | |
| from vertexai.preview.vision_models import Image as VertexImage | |
| print(f"π¨ Phase 3: Generating with native references for {scene_id}_{variant}...") | |
| # Build reference images list | |
| reference_images = [] | |
| reference_id = 1 | |
| # Process character references | |
| if ref_images: | |
| characters_list = ref_images.get("characters", []) | |
| # Add single character if no list provided | |
| if not characters_list and ref_images.get("character"): | |
| char_path = ref_images.get("character") | |
| if char_path and os.path.isfile(char_path): | |
| characters_list = [("character", char_path)] | |
| # Process each character reference | |
| for key, path in characters_list: | |
| if path and os.path.isfile(path): | |
| try: | |
| ref_image = VertexImage.load_from_file(path) | |
| subject_type = infer_subject_type(key, path) | |
| reference_images.append({ | |
| "referenceImage": ref_image, | |
| "referenceType": "REFERENCE_TYPE_SUBJECT", | |
| "referenceId": reference_id, | |
| "subjectType": subject_type | |
| }) | |
| print(f" π Reference {reference_id}: {key} ({subject_type}) β {os.path.basename(path)}") | |
| reference_id += 1 | |
| except Exception as e: | |
| print(f" β οΈ Failed to load reference {key}: {e}") | |
| # Process location reference (as style/background reference) | |
| location_path = ref_images.get("location") | |
| if location_path and os.path.isfile(location_path): | |
| try: | |
| loc_image = VertexImage.load_from_file(location_path) | |
| # Locations are treated as style references, not subjects | |
| reference_images.append({ | |
| "referenceImage": loc_image, | |
| "referenceType": "REFERENCE_TYPE_STYLE", | |
| "referenceId": reference_id | |
| }) | |
| print(f" π Reference {reference_id}: location (STYLE) β {os.path.basename(location_path)}") | |
| reference_id += 1 | |
| except Exception as e: | |
| print(f" β οΈ Failed to load location reference: {e}") | |
| # Build generation config | |
| generation_kwargs = { | |
| "prompt": f"Kurzgesagt style illustration, 2D flat vector animation, vibrant colors, minimalist design, {prompt}", | |
| "number_of_images": 1, | |
| "aspect_ratio": aspect_ratio, | |
| } | |
| # NOTE: The Vertex AI Python SDK's generate_images() does NOT support | |
| # reference_images parameter directly. Native reference images require | |
| # using the REST API with the imagen-3.0-capability-001 model. | |
| # For now, we skip Phase 3 and use Phase 2 fallback. | |
| if reference_images: | |
| print(f" β οΈ SDK limitation: reference_images not supported in generate_images()") | |
| print(f" β Falling back to Phase 2 (prompt-based consistency)") | |
| return None | |
| # Generate image without references | |
| response = _imagen_model.generate_images(**generation_kwargs) | |
| if response.images and len(response.images) > 0: | |
| # Save generated image | |
| output_path = os.path.join( | |
| OUTPUT_DIR_IMG, | |
| f"{scene_id}_{variant}_{uuid.uuid4().hex[:8]}.png" | |
| ) | |
| response.images[0].save(output_path) | |
| print(f"β Phase 3: Image generated: {output_path}") | |
| return output_path | |
| else: | |
| print(f"β οΈ Phase 3: No images in response") | |
| return None | |
| except Exception as e: | |
| print(f"β Phase 3 generation failed: {e}") | |
| import traceback | |
| traceback.print_exc() | |
| return None | |
| # ============================================================================= | |
| # FALLBACK CHECK & HYBRID GENERATION | |
| # ============================================================================= | |
| def generate_image_with_fallback( | |
| prompt: str, | |
| scene_id: str, | |
| variant: str, | |
| ref_images: Dict[str, Any] = None, | |
| phase2_fallback_fn = None, | |
| aspect_ratio: str = "9:16" | |
| ) -> Optional[str]: | |
| """ | |
| Generate an image using Phase 3 (native references) if available, | |
| otherwise fall back to Phase 2 (prompt-based). | |
| This is the main entry point for hybrid Phase 2/3 generation. | |
| Args: | |
| prompt: The text prompt for image generation | |
| scene_id: Scene identifier for naming | |
| variant: "A" or "B" for the variant | |
| ref_images: Reference images dictionary | |
| phase2_fallback_fn: Fallback function for Phase 2 generation | |
| aspect_ratio: Aspect ratio for generation | |
| Returns: | |
| Path to generated image, or None on failure | |
| """ | |
| # Try Phase 3 first if available | |
| if is_phase3_available(): | |
| print(f"π· Using Phase 3 (Native Reference Images) for {scene_id}_{variant}") | |
| result = generate_with_native_references( | |
| prompt=prompt, | |
| scene_id=scene_id, | |
| variant=variant, | |
| ref_images=ref_images, | |
| aspect_ratio=aspect_ratio | |
| ) | |
| if result: | |
| return result | |
| else: | |
| print(f"β οΈ Phase 3 generation failed, falling back to Phase 2") | |
| # Fall back to Phase 2 | |
| if phase2_fallback_fn: | |
| print(f"πΆ Using Phase 2 (Prompt-based) for {scene_id}_{variant}") | |
| return phase2_fallback_fn(prompt, scene_id, variant, ref_images) | |
| return None | |
| # ============================================================================= | |
| # INITIALIZATION ON MODULE LOAD | |
| # ============================================================================= | |
| # Attempt to initialize Phase 3 on module import | |
| # This is safe - if it fails, Phase 2 will be used as fallback | |
| print("\n" + "=" * 60) | |
| print("π Initializing Phase 3 (Native Reference Images)...") | |
| print("=" * 60) | |
| init_phase3_vertex_ai() | |
| print("=" * 60 + "\n") | |