"""Shared Modal/HF engine. All model assets come from one self-contained snapshot.""" import base64 import time from io import BytesIO from pathlib import Path from threading import Lock from pydantic import BaseModel, ConfigDict, Field, field_validator from model_config import GGUF_FILENAME, Settings class GenerationRequest(BaseModel): model_config = ConfigDict(extra='forbid', str_strip_whitespace=True) inputs: str = Field(min_length=1, max_length=10000) width: int = Field(default=1024, ge=256, le=2048) height: int = Field(default=1024, ge=256, le=2048) num_inference_steps: int = Field(default=16, ge=4, le=16) seed: int = Field(default=42, ge=0, le=2**63-1) @field_validator('width', 'height') @classmethod def multiple_of_32(cls, value): if value % 32: raise ValueError('Dimensions must be multiples of 32') return value class LegacyRequest(GenerationRequest): num_inference_steps: int = Field(default=40, ge=1, le=1000) guidance_scale: float = Field(default=6, ge=0, le=100) use_refiner: bool = False negative_prompt: str | None = None high_noise_frac: float = Field(default=.8, ge=0, le=1) def native(self): return GenerationRequest(inputs=self.inputs, width=self.width, height=self.height, seed=self.seed, num_inference_steps=16) def download_weights(settings): from huggingface_hub import snapshot_download patterns = ['modular_model_index.json', 'checkpoint_manifest.json', 'transformer/*.json', 'vae/*', 'text_encoder/*'] if settings.quantization == 'bf16': patterns.append('transformer/*.safetensors') else: patterns.append(GGUF_FILENAME) return snapshot_download(settings.model_id, revision=settings.revision, allow_patterns=patterns, max_workers=4) class Engine: def __init__(self, settings=None, model_path=None): self.settings = settings or Settings.from_env() self.model_path = model_path self.pipe = None self.lock = Lock() def load(self): import torch from diffusers import (AutoencoderKLFlux2, Flux3Transformer2DModel, GGUFQuantizationConfig, ModularPipeline) from transformers import AutoProcessor, BitsAndBytesConfig, Qwen3VLForConditionalGeneration started = time.perf_counter() if not torch.cuda.is_available() or not torch.cuda.is_bf16_supported(): raise RuntimeError('A BF16-capable CUDA GPU is required') path = Path(self.model_path or download_weights(self.settings)) checkpoint = path / GGUF_FILENAME if self.settings.quantization == 'q8_0' and not checkpoint.is_file(): raise FileNotFoundError(f'Missing {GGUF_FILENAME} in {path}; use the complete Socialtrait checkpoint.') self.resolved_path = str(path) blocks = ModularPipeline.from_pretrained(str(path), local_files_only=True).blocks blocks.sub_blocks.pop('prompt_upsampler') # init_pipeline accepts a local directory but no Hub keyword arguments. pipe = blocks.init_pipeline(str(path)) # Resolve every component from this snapshot, including the Q8 GGUF. if self.settings.quantization == 'bf16': transformer = Flux3Transformer2DModel.from_pretrained( path, subfolder='transformer', dtype=torch.bfloat16, device_map={'': 'cuda:0'}, low_cpu_mem_usage=True, local_files_only=True) encoder_quant = {} else: transformer = Flux3Transformer2DModel.from_single_file( str(checkpoint), config=str(path), subfolder='transformer', quantization_config=GGUFQuantizationConfig(compute_dtype=torch.bfloat16), dtype=torch.bfloat16, device_map={'': 'cuda:0'}, local_files_only=True) if not hasattr(Flux3Transformer2DModel, '_serving_original_dtype'): original_dtype = Flux3Transformer2DModel.dtype Flux3Transformer2DModel._serving_original_dtype = original_dtype Flux3Transformer2DModel.dtype = property(lambda model: model._serving_compute_dtype if hasattr(model, '_serving_compute_dtype') else original_dtype.fget(model)) transformer._serving_compute_dtype = torch.bfloat16 encoder_quant = {'quantization_config': BitsAndBytesConfig(load_in_4bit=True, bnb_4bit_quant_type='nf4', bnb_4bit_use_double_quant=True, bnb_4bit_compute_dtype=torch.bfloat16)} encoder = Qwen3VLForConditionalGeneration.from_pretrained( Path(path)/'text_encoder', dtype=torch.bfloat16, device_map={'': 'cuda:0'}, low_cpu_mem_usage=True, local_files_only=True, **encoder_quant) vae = AutoencoderKLFlux2.from_pretrained(path, subfolder='vae', dtype=torch.float32, device_map={'': 'cuda:0'}, low_cpu_mem_usage=True, local_files_only=True) pipe.load_components(names=['image_processor', 'scheduler'], local_files_only=True) pipe.update_components(transformer=transformer, text_encoder=encoder, vae=vae, tokenizer=AutoProcessor.from_pretrained(Path(path)/'text_encoder', local_files_only=True)) pipe.vae.enable_tiling() pipe.to('cuda') self.component_devices = {name: sorted({str(p.device) for p in getattr(pipe, name).parameters()}) for name in ('transformer', 'text_encoder', 'vae')} assert all(d.startswith('cuda') for devices in self.component_devices.values() for d in devices) self.pipe = pipe self.load_seconds = time.perf_counter()-started def health(self): return {'ready': self.pipe is not None, 'model': self.settings.model_id, 'revision': self.settings.revision, 'quantization': self.settings.quantization, 'default_steps': 16, 'offload': 'none', 'prompt_upsampling': False} def generate(self, request, image_format='PNG'): import torch if self.pipe is None: raise RuntimeError('Model is not loaded') with self.lock: torch.cuda.synchronize() torch.cuda.reset_peak_memory_stats() started = time.perf_counter() try: with torch.inference_mode(): image = self.pipe(prompt=request.inputs, width=request.width, height=request.height, num_inference_steps=request.num_inference_steps, generator=torch.Generator(device='cuda').manual_seed(request.seed), output='images')[0] torch.cuda.synchronize() except torch.cuda.OutOfMemoryError: torch.cuda.empty_cache() raise metadata = {**self.health(), 'request': request.model_dump(), 'text_encoder_dtype': 'bfloat16' if self.settings.quantization == 'bf16' else 'nf4', 'vae_dtype': 'float32', 'component_devices': self.component_devices, 'device': torch.cuda.get_device_name(), 'device_total_gib': torch.cuda.get_device_properties(0).total_memory/2**30, 'generation_seconds': time.perf_counter()-started, 'load_seconds': self.load_seconds, 'peak_allocated_gib': torch.cuda.max_memory_allocated()/2**30, 'peak_reserved_gib': torch.cuda.max_memory_reserved()/2**30} output = BytesIO() image.convert('RGB').save(output, format=image_format) return {'image': base64.b64encode(output.getvalue()).decode(), 'mime_type': 'image/jpeg' if image_format == 'JPEG' else 'image/png', 'metadata': metadata} def legacy(self, payload): request = LegacyRequest.model_validate(payload) result = self.generate(request.native(), 'JPEG') result['metadata']['compatibility'] = {'num_inference_steps': 16, 'ignored_sdxl_fields': ['guidance_scale', 'use_refiner', 'negative_prompt', 'high_noise_frac']} return result