stsachit's picture
Publish self-contained A10 deployment and offline validation
52a6059 verified
Raw History Blame Contribute Delete
8.17 kB
"""Shared Modal/HF engine. All model assets come from one self-contained snapshot."""
import base64
import time
from io import BytesIO
from pathlib import Path
from threading import Lock
from pydantic import BaseModel, ConfigDict, Field, field_validator
from model_config import GGUF_FILENAME, Settings
class GenerationRequest(BaseModel):
model_config = ConfigDict(extra='forbid', str_strip_whitespace=True)
inputs: str = Field(min_length=1, max_length=10000)
width: int = Field(default=1024, ge=256, le=2048)
height: int = Field(default=1024, ge=256, le=2048)
num_inference_steps: int = Field(default=16, ge=4, le=16)
seed: int = Field(default=42, ge=0, le=2**63-1)
@field_validator('width', 'height')
@classmethod
def multiple_of_32(cls, value):
if value % 32:
raise ValueError('Dimensions must be multiples of 32')
return value
class LegacyRequest(GenerationRequest):
num_inference_steps: int = Field(default=40, ge=1, le=1000)
guidance_scale: float = Field(default=6, ge=0, le=100)
use_refiner: bool = False
negative_prompt: str | None = None
high_noise_frac: float = Field(default=.8, ge=0, le=1)
def native(self):
return GenerationRequest(inputs=self.inputs, width=self.width, height=self.height,
seed=self.seed, num_inference_steps=16)
def download_weights(settings):
from huggingface_hub import snapshot_download
patterns = ['modular_model_index.json', 'checkpoint_manifest.json',
'transformer/*.json', 'vae/*', 'text_encoder/*']
if settings.quantization == 'bf16':
patterns.append('transformer/*.safetensors')
else:
patterns.append(GGUF_FILENAME)
return snapshot_download(settings.model_id, revision=settings.revision,
allow_patterns=patterns, max_workers=4)
class Engine:
def __init__(self, settings=None, model_path=None):
self.settings = settings or Settings.from_env()
self.model_path = model_path
self.pipe = None
self.lock = Lock()
def load(self):
import torch
from diffusers import (AutoencoderKLFlux2, Flux3Transformer2DModel,
GGUFQuantizationConfig, ModularPipeline)
from transformers import AutoProcessor, BitsAndBytesConfig, Qwen3VLForConditionalGeneration
started = time.perf_counter()
if not torch.cuda.is_available() or not torch.cuda.is_bf16_supported():
raise RuntimeError('A BF16-capable CUDA GPU is required')
path = Path(self.model_path or download_weights(self.settings))
checkpoint = path / GGUF_FILENAME
if self.settings.quantization == 'q8_0' and not checkpoint.is_file():
raise FileNotFoundError(f'Missing {GGUF_FILENAME} in {path}; use the complete Socialtrait checkpoint.')
self.resolved_path = str(path)
blocks = ModularPipeline.from_pretrained(str(path), local_files_only=True).blocks
blocks.sub_blocks.pop('prompt_upsampler')
# init_pipeline accepts a local directory but no Hub keyword arguments.
pipe = blocks.init_pipeline(str(path))
# Resolve every component from this snapshot, including the Q8 GGUF.
if self.settings.quantization == 'bf16':
transformer = Flux3Transformer2DModel.from_pretrained(
path, subfolder='transformer', dtype=torch.bfloat16,
device_map={'': 'cuda:0'}, low_cpu_mem_usage=True, local_files_only=True)
encoder_quant = {}
else:
transformer = Flux3Transformer2DModel.from_single_file(
str(checkpoint), config=str(path), subfolder='transformer',
quantization_config=GGUFQuantizationConfig(compute_dtype=torch.bfloat16),
dtype=torch.bfloat16, device_map={'': 'cuda:0'}, local_files_only=True)
if not hasattr(Flux3Transformer2DModel, '_serving_original_dtype'):
original_dtype = Flux3Transformer2DModel.dtype
Flux3Transformer2DModel._serving_original_dtype = original_dtype
Flux3Transformer2DModel.dtype = property(lambda model:
model._serving_compute_dtype if hasattr(model, '_serving_compute_dtype')
else original_dtype.fget(model))
transformer._serving_compute_dtype = torch.bfloat16
encoder_quant = {'quantization_config': BitsAndBytesConfig(load_in_4bit=True,
bnb_4bit_quant_type='nf4', bnb_4bit_use_double_quant=True,
bnb_4bit_compute_dtype=torch.bfloat16)}
encoder = Qwen3VLForConditionalGeneration.from_pretrained(
Path(path)/'text_encoder', dtype=torch.bfloat16, device_map={'': 'cuda:0'},
low_cpu_mem_usage=True, local_files_only=True, **encoder_quant)
vae = AutoencoderKLFlux2.from_pretrained(path, subfolder='vae', dtype=torch.float32,
device_map={'': 'cuda:0'}, low_cpu_mem_usage=True, local_files_only=True)
pipe.load_components(names=['image_processor', 'scheduler'], local_files_only=True)
pipe.update_components(transformer=transformer, text_encoder=encoder, vae=vae,
tokenizer=AutoProcessor.from_pretrained(Path(path)/'text_encoder', local_files_only=True))
pipe.vae.enable_tiling()
pipe.to('cuda')
self.component_devices = {name: sorted({str(p.device) for p in getattr(pipe, name).parameters()})
for name in ('transformer', 'text_encoder', 'vae')}
assert all(d.startswith('cuda') for devices in self.component_devices.values() for d in devices)
self.pipe = pipe
self.load_seconds = time.perf_counter()-started
def health(self):
return {'ready': self.pipe is not None, 'model': self.settings.model_id,
'revision': self.settings.revision, 'quantization': self.settings.quantization,
'default_steps': 16, 'offload': 'none', 'prompt_upsampling': False}
def generate(self, request, image_format='PNG'):
import torch
if self.pipe is None:
raise RuntimeError('Model is not loaded')
with self.lock:
torch.cuda.synchronize()
torch.cuda.reset_peak_memory_stats()
started = time.perf_counter()
try:
with torch.inference_mode():
image = self.pipe(prompt=request.inputs, width=request.width, height=request.height,
num_inference_steps=request.num_inference_steps,
generator=torch.Generator(device='cuda').manual_seed(request.seed), output='images')[0]
torch.cuda.synchronize()
except torch.cuda.OutOfMemoryError:
torch.cuda.empty_cache()
raise
metadata = {**self.health(), 'request': request.model_dump(),
'text_encoder_dtype': 'bfloat16' if self.settings.quantization == 'bf16' else 'nf4',
'vae_dtype': 'float32', 'component_devices': self.component_devices,
'device': torch.cuda.get_device_name(),
'device_total_gib': torch.cuda.get_device_properties(0).total_memory/2**30,
'generation_seconds': time.perf_counter()-started, 'load_seconds': self.load_seconds,
'peak_allocated_gib': torch.cuda.max_memory_allocated()/2**30,
'peak_reserved_gib': torch.cuda.max_memory_reserved()/2**30}
output = BytesIO()
image.convert('RGB').save(output, format=image_format)
return {'image': base64.b64encode(output.getvalue()).decode(),
'mime_type': 'image/jpeg' if image_format == 'JPEG' else 'image/png', 'metadata': metadata}
def legacy(self, payload):
request = LegacyRequest.model_validate(payload)
result = self.generate(request.native(), 'JPEG')
result['metadata']['compatibility'] = {'num_inference_steps': 16,
'ignored_sdxl_fields': ['guidance_scale', 'use_refiner', 'negative_prompt', 'high_noise_frac']}
return result