multimodalart HF Staff commited on
Commit
2d21e30
·
verified ·
1 Parent(s): 98e0470

Turn the prefix KV cache off when it would not fit next to the resident weights

Browse files
Files changed (1) hide show
  1. app.py +18 -3
app.py CHANGED
@@ -58,6 +58,16 @@ ASPECT_RATIOS = {
58
  }
59
  RESOLUTION_CHOICES = [("1K — fastest", 1024), ("1.5K", 1536), ("2K — model default", 2048)]
60
  AOTI_STEP_FACTOR = 0.78
 
 
 
 
 
 
 
 
 
 
61
 
62
  TRANSPARENCY_INFO = (
63
  "Describe what to generate, or how to edit the input images (refer to them as "
@@ -228,8 +238,11 @@ def estimate_duration(
228
  steps, res, cfg = 40, 1024, 1.0
229
  n_images = len(input_images or [])
230
  tokens = (res / 16.0) ** 2
 
 
 
231
  per_step = 2.05e-5 * tokens**1.21 * (2.0 if cfg > 1.0 else 1.0)
232
- if AOTI_BLOCKS:
233
  per_step *= AOTI_STEP_FACTOR
234
  fixed = 3.0 + 2.0 * n_images + 1.5e-6 * res * res
235
  return int(min(340, math.ceil((fixed + steps * per_step) * 1.12)))
@@ -257,7 +270,8 @@ def _diffuse(
257
  width, height = resolve_size(resolution, aspect_ratio)
258
 
259
  negative_prompt = (negative_prompt or "").strip()
260
- call_kwargs = {}
 
261
  if negative_prompt and float(true_cfg_scale) > 1.0:
262
  call_kwargs["negative_prompt"] = negative_prompt
263
  call_kwargs["true_cfg_scale"] = float(true_cfg_scale)
@@ -275,7 +289,8 @@ def _diffuse(
275
  ).images[0]
276
  print(
277
  f"[generate] images={len(image_paths)} steps={num_inference_steps} "
278
- f"res={resolution} ratio={aspect_ratio} size={image.size} "
 
279
  f"elapsed={time.perf_counter() - started:.1f}s",
280
  flush=True,
281
  )
 
58
  }
59
  RESOLUTION_CHOICES = [("1K — fastest", 1024), ("1.5K", 1536), ("2K — model default", 2048)]
60
  AOTI_STEP_FACTOR = 0.78
61
+ KV_CACHE_BYTES_PER_TOKEN = 32 * 2 * 4096 * 2
62
+ KV_CACHE_BUDGET_GB = 10.0
63
+
64
+
65
+ def prefix_tokens(n_images: int, resolution: int) -> int:
66
+ return n_images * (int(resolution) // 16) ** 2
67
+
68
+
69
+ def kv_cache_fits(n_images: int, resolution: int) -> bool:
70
+ return prefix_tokens(n_images, resolution) * KV_CACHE_BYTES_PER_TOKEN <= KV_CACHE_BUDGET_GB * 1e9
71
 
72
  TRANSPARENCY_INFO = (
73
  "Describe what to generate, or how to edit the input images (refer to them as "
 
238
  steps, res, cfg = 40, 1024, 1.0
239
  n_images = len(input_images or [])
240
  tokens = (res / 16.0) ** 2
241
+ prefix = prefix_tokens(n_images, res)
242
+ cached = kv_cache_fits(n_images, res)
243
+ tokens += 0.35 * prefix if cached else prefix
244
  per_step = 2.05e-5 * tokens**1.21 * (2.0 if cfg > 1.0 else 1.0)
245
+ if AOTI_BLOCKS and cached:
246
  per_step *= AOTI_STEP_FACTOR
247
  fixed = 3.0 + 2.0 * n_images + 1.5e-6 * res * res
248
  return int(min(340, math.ceil((fixed + steps * per_step) * 1.12)))
 
270
  width, height = resolve_size(resolution, aspect_ratio)
271
 
272
  negative_prompt = (negative_prompt or "").strip()
273
+ use_kv_cache = kv_cache_fits(len(image_paths), int(resolution))
274
+ call_kwargs = {"use_kv_cache": use_kv_cache}
275
  if negative_prompt and float(true_cfg_scale) > 1.0:
276
  call_kwargs["negative_prompt"] = negative_prompt
277
  call_kwargs["true_cfg_scale"] = float(true_cfg_scale)
 
289
  ).images[0]
290
  print(
291
  f"[generate] images={len(image_paths)} steps={num_inference_steps} "
292
+ f"res={resolution} ratio={aspect_ratio} size={image.size} kv_cache={use_kv_cache} "
293
+ f"aoti={AOTI_LOADED} "
294
  f"elapsed={time.perf_counter() - started:.1f}s",
295
  flush=True,
296
  )