Instructions to use socialtrait/flux-3-image-klein-infendpoint with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Diffusers
How to use socialtrait/flux-3-image-klein-infendpoint with Diffusers:
pip install -U diffusers transformers accelerate
import torch from diffusers import DiffusionPipeline # switch to "mps" for apple devices pipe = DiffusionPipeline.from_pretrained("socialtrait/flux-3-image-klein-infendpoint", dtype=torch.bfloat16, device_map="cuda") prompt = "Astronaut in a jungle, cold color palette, muted colors, detailed, 8k" image = pipe(prompt).images[0] - Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- Draw Things
- DiffusionBee
Download flux3_engine.py from socialtrait/flux-3-image-klein-infendpoint: direct link, hf CLI and curl.
- Browser
- Download file 8.17 kB
-
https://huggingface.co/socialtrait/flux-3-image-klein-infendpoint/resolve/main/flux3_engine.py
- Command line
-
hf download hf://socialtrait/flux-3-image-klein-infendpoint/flux3_engine.py
-
curl -L -o flux3_engine.py https://huggingface.co/socialtrait/flux-3-image-klein-infendpoint/resolve/main/flux3_engine.py
8.17 kB
| """Shared Modal/HF engine. All model assets come from one self-contained snapshot.""" | |
| import base64 | |
| import time | |
| from io import BytesIO | |
| from pathlib import Path | |
| from threading import Lock | |
| from pydantic import BaseModel, ConfigDict, Field, field_validator | |
| from model_config import GGUF_FILENAME, Settings | |
| class GenerationRequest(BaseModel): | |
| model_config = ConfigDict(extra='forbid', str_strip_whitespace=True) | |
| inputs: str = Field(min_length=1, max_length=10000) | |
| width: int = Field(default=1024, ge=256, le=2048) | |
| height: int = Field(default=1024, ge=256, le=2048) | |
| num_inference_steps: int = Field(default=16, ge=4, le=16) | |
| seed: int = Field(default=42, ge=0, le=2**63-1) | |
| def multiple_of_32(cls, value): | |
| if value % 32: | |
| raise ValueError('Dimensions must be multiples of 32') | |
| return value | |
| class LegacyRequest(GenerationRequest): | |
| num_inference_steps: int = Field(default=40, ge=1, le=1000) | |
| guidance_scale: float = Field(default=6, ge=0, le=100) | |
| use_refiner: bool = False | |
| negative_prompt: str | None = None | |
| high_noise_frac: float = Field(default=.8, ge=0, le=1) | |
| def native(self): | |
| return GenerationRequest(inputs=self.inputs, width=self.width, height=self.height, | |
| seed=self.seed, num_inference_steps=16) | |
| def download_weights(settings): | |
| from huggingface_hub import snapshot_download | |
| patterns = ['modular_model_index.json', 'checkpoint_manifest.json', | |
| 'transformer/*.json', 'vae/*', 'text_encoder/*'] | |
| if settings.quantization == 'bf16': | |
| patterns.append('transformer/*.safetensors') | |
| else: | |
| patterns.append(GGUF_FILENAME) | |
| return snapshot_download(settings.model_id, revision=settings.revision, | |
| allow_patterns=patterns, max_workers=4) | |
| class Engine: | |
| def __init__(self, settings=None, model_path=None): | |
| self.settings = settings or Settings.from_env() | |
| self.model_path = model_path | |
| self.pipe = None | |
| self.lock = Lock() | |
| def load(self): | |
| import torch | |
| from diffusers import (AutoencoderKLFlux2, Flux3Transformer2DModel, | |
| GGUFQuantizationConfig, ModularPipeline) | |
| from transformers import AutoProcessor, BitsAndBytesConfig, Qwen3VLForConditionalGeneration | |
| started = time.perf_counter() | |
| if not torch.cuda.is_available() or not torch.cuda.is_bf16_supported(): | |
| raise RuntimeError('A BF16-capable CUDA GPU is required') | |
| path = Path(self.model_path or download_weights(self.settings)) | |
| checkpoint = path / GGUF_FILENAME | |
| if self.settings.quantization == 'q8_0' and not checkpoint.is_file(): | |
| raise FileNotFoundError(f'Missing {GGUF_FILENAME} in {path}; use the complete Socialtrait checkpoint.') | |
| self.resolved_path = str(path) | |
| blocks = ModularPipeline.from_pretrained(str(path), local_files_only=True).blocks | |
| blocks.sub_blocks.pop('prompt_upsampler') | |
| # init_pipeline accepts a local directory but no Hub keyword arguments. | |
| pipe = blocks.init_pipeline(str(path)) | |
| # Resolve every component from this snapshot, including the Q8 GGUF. | |
| if self.settings.quantization == 'bf16': | |
| transformer = Flux3Transformer2DModel.from_pretrained( | |
| path, subfolder='transformer', dtype=torch.bfloat16, | |
| device_map={'': 'cuda:0'}, low_cpu_mem_usage=True, local_files_only=True) | |
| encoder_quant = {} | |
| else: | |
| transformer = Flux3Transformer2DModel.from_single_file( | |
| str(checkpoint), config=str(path), subfolder='transformer', | |
| quantization_config=GGUFQuantizationConfig(compute_dtype=torch.bfloat16), | |
| dtype=torch.bfloat16, device_map={'': 'cuda:0'}, local_files_only=True) | |
| if not hasattr(Flux3Transformer2DModel, '_serving_original_dtype'): | |
| original_dtype = Flux3Transformer2DModel.dtype | |
| Flux3Transformer2DModel._serving_original_dtype = original_dtype | |
| Flux3Transformer2DModel.dtype = property(lambda model: | |
| model._serving_compute_dtype if hasattr(model, '_serving_compute_dtype') | |
| else original_dtype.fget(model)) | |
| transformer._serving_compute_dtype = torch.bfloat16 | |
| encoder_quant = {'quantization_config': BitsAndBytesConfig(load_in_4bit=True, | |
| bnb_4bit_quant_type='nf4', bnb_4bit_use_double_quant=True, | |
| bnb_4bit_compute_dtype=torch.bfloat16)} | |
| encoder = Qwen3VLForConditionalGeneration.from_pretrained( | |
| Path(path)/'text_encoder', dtype=torch.bfloat16, device_map={'': 'cuda:0'}, | |
| low_cpu_mem_usage=True, local_files_only=True, **encoder_quant) | |
| vae = AutoencoderKLFlux2.from_pretrained(path, subfolder='vae', dtype=torch.float32, | |
| device_map={'': 'cuda:0'}, low_cpu_mem_usage=True, local_files_only=True) | |
| pipe.load_components(names=['image_processor', 'scheduler'], local_files_only=True) | |
| pipe.update_components(transformer=transformer, text_encoder=encoder, vae=vae, | |
| tokenizer=AutoProcessor.from_pretrained(Path(path)/'text_encoder', local_files_only=True)) | |
| pipe.vae.enable_tiling() | |
| pipe.to('cuda') | |
| self.component_devices = {name: sorted({str(p.device) for p in getattr(pipe, name).parameters()}) | |
| for name in ('transformer', 'text_encoder', 'vae')} | |
| assert all(d.startswith('cuda') for devices in self.component_devices.values() for d in devices) | |
| self.pipe = pipe | |
| self.load_seconds = time.perf_counter()-started | |
| def health(self): | |
| return {'ready': self.pipe is not None, 'model': self.settings.model_id, | |
| 'revision': self.settings.revision, 'quantization': self.settings.quantization, | |
| 'default_steps': 16, 'offload': 'none', 'prompt_upsampling': False} | |
| def generate(self, request, image_format='PNG'): | |
| import torch | |
| if self.pipe is None: | |
| raise RuntimeError('Model is not loaded') | |
| with self.lock: | |
| torch.cuda.synchronize() | |
| torch.cuda.reset_peak_memory_stats() | |
| started = time.perf_counter() | |
| try: | |
| with torch.inference_mode(): | |
| image = self.pipe(prompt=request.inputs, width=request.width, height=request.height, | |
| num_inference_steps=request.num_inference_steps, | |
| generator=torch.Generator(device='cuda').manual_seed(request.seed), output='images')[0] | |
| torch.cuda.synchronize() | |
| except torch.cuda.OutOfMemoryError: | |
| torch.cuda.empty_cache() | |
| raise | |
| metadata = {**self.health(), 'request': request.model_dump(), | |
| 'text_encoder_dtype': 'bfloat16' if self.settings.quantization == 'bf16' else 'nf4', | |
| 'vae_dtype': 'float32', 'component_devices': self.component_devices, | |
| 'device': torch.cuda.get_device_name(), | |
| 'device_total_gib': torch.cuda.get_device_properties(0).total_memory/2**30, | |
| 'generation_seconds': time.perf_counter()-started, 'load_seconds': self.load_seconds, | |
| 'peak_allocated_gib': torch.cuda.max_memory_allocated()/2**30, | |
| 'peak_reserved_gib': torch.cuda.max_memory_reserved()/2**30} | |
| output = BytesIO() | |
| image.convert('RGB').save(output, format=image_format) | |
| return {'image': base64.b64encode(output.getvalue()).decode(), | |
| 'mime_type': 'image/jpeg' if image_format == 'JPEG' else 'image/png', 'metadata': metadata} | |
| def legacy(self, payload): | |
| request = LegacyRequest.model_validate(payload) | |
| result = self.generate(request.native(), 'JPEG') | |
| result['metadata']['compatibility'] = {'num_inference_steps': 16, | |
| 'ignored_sdxl_fields': ['guidance_scale', 'use_refiner', 'negative_prompt', 'high_noise_frac']} | |
| return result | |