Turn the prefix KV cache off when it would not fit next to the resident weights
Browse files
app.py
CHANGED
|
@@ -58,6 +58,16 @@ ASPECT_RATIOS = {
|
|
| 58 |
}
|
| 59 |
RESOLUTION_CHOICES = [("1K — fastest", 1024), ("1.5K", 1536), ("2K — model default", 2048)]
|
| 60 |
AOTI_STEP_FACTOR = 0.78
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 61 |
|
| 62 |
TRANSPARENCY_INFO = (
|
| 63 |
"Describe what to generate, or how to edit the input images (refer to them as "
|
|
@@ -228,8 +238,11 @@ def estimate_duration(
|
|
| 228 |
steps, res, cfg = 40, 1024, 1.0
|
| 229 |
n_images = len(input_images or [])
|
| 230 |
tokens = (res / 16.0) ** 2
|
|
|
|
|
|
|
|
|
|
| 231 |
per_step = 2.05e-5 * tokens**1.21 * (2.0 if cfg > 1.0 else 1.0)
|
| 232 |
-
if AOTI_BLOCKS:
|
| 233 |
per_step *= AOTI_STEP_FACTOR
|
| 234 |
fixed = 3.0 + 2.0 * n_images + 1.5e-6 * res * res
|
| 235 |
return int(min(340, math.ceil((fixed + steps * per_step) * 1.12)))
|
|
@@ -257,7 +270,8 @@ def _diffuse(
|
|
| 257 |
width, height = resolve_size(resolution, aspect_ratio)
|
| 258 |
|
| 259 |
negative_prompt = (negative_prompt or "").strip()
|
| 260 |
-
|
|
|
|
| 261 |
if negative_prompt and float(true_cfg_scale) > 1.0:
|
| 262 |
call_kwargs["negative_prompt"] = negative_prompt
|
| 263 |
call_kwargs["true_cfg_scale"] = float(true_cfg_scale)
|
|
@@ -275,7 +289,8 @@ def _diffuse(
|
|
| 275 |
).images[0]
|
| 276 |
print(
|
| 277 |
f"[generate] images={len(image_paths)} steps={num_inference_steps} "
|
| 278 |
-
f"res={resolution} ratio={aspect_ratio} size={image.size} "
|
|
|
|
| 279 |
f"elapsed={time.perf_counter() - started:.1f}s",
|
| 280 |
flush=True,
|
| 281 |
)
|
|
|
|
| 58 |
}
|
| 59 |
RESOLUTION_CHOICES = [("1K — fastest", 1024), ("1.5K", 1536), ("2K — model default", 2048)]
|
| 60 |
AOTI_STEP_FACTOR = 0.78
|
| 61 |
+
KV_CACHE_BYTES_PER_TOKEN = 32 * 2 * 4096 * 2
|
| 62 |
+
KV_CACHE_BUDGET_GB = 10.0
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
def prefix_tokens(n_images: int, resolution: int) -> int:
|
| 66 |
+
return n_images * (int(resolution) // 16) ** 2
|
| 67 |
+
|
| 68 |
+
|
| 69 |
+
def kv_cache_fits(n_images: int, resolution: int) -> bool:
|
| 70 |
+
return prefix_tokens(n_images, resolution) * KV_CACHE_BYTES_PER_TOKEN <= KV_CACHE_BUDGET_GB * 1e9
|
| 71 |
|
| 72 |
TRANSPARENCY_INFO = (
|
| 73 |
"Describe what to generate, or how to edit the input images (refer to them as "
|
|
|
|
| 238 |
steps, res, cfg = 40, 1024, 1.0
|
| 239 |
n_images = len(input_images or [])
|
| 240 |
tokens = (res / 16.0) ** 2
|
| 241 |
+
prefix = prefix_tokens(n_images, res)
|
| 242 |
+
cached = kv_cache_fits(n_images, res)
|
| 243 |
+
tokens += 0.35 * prefix if cached else prefix
|
| 244 |
per_step = 2.05e-5 * tokens**1.21 * (2.0 if cfg > 1.0 else 1.0)
|
| 245 |
+
if AOTI_BLOCKS and cached:
|
| 246 |
per_step *= AOTI_STEP_FACTOR
|
| 247 |
fixed = 3.0 + 2.0 * n_images + 1.5e-6 * res * res
|
| 248 |
return int(min(340, math.ceil((fixed + steps * per_step) * 1.12)))
|
|
|
|
| 270 |
width, height = resolve_size(resolution, aspect_ratio)
|
| 271 |
|
| 272 |
negative_prompt = (negative_prompt or "").strip()
|
| 273 |
+
use_kv_cache = kv_cache_fits(len(image_paths), int(resolution))
|
| 274 |
+
call_kwargs = {"use_kv_cache": use_kv_cache}
|
| 275 |
if negative_prompt and float(true_cfg_scale) > 1.0:
|
| 276 |
call_kwargs["negative_prompt"] = negative_prompt
|
| 277 |
call_kwargs["true_cfg_scale"] = float(true_cfg_scale)
|
|
|
|
| 289 |
).images[0]
|
| 290 |
print(
|
| 291 |
f"[generate] images={len(image_paths)} steps={num_inference_steps} "
|
| 292 |
+
f"res={resolution} ratio={aspect_ratio} size={image.size} kv_cache={use_kv_cache} "
|
| 293 |
+
f"aoti={AOTI_LOADED} "
|
| 294 |
f"elapsed={time.perf_counter() - started:.1f}s",
|
| 295 |
flush=True,
|
| 296 |
)
|