| """Keep the fal FLUX inpaint container WARM so customers never hit a cold start. |
| |
| fal serverless scales the FLUX container to zero when idle; the next request then |
| pays ~2-3 min to spin it back up (cold). During salon hours this loop sends a small |
| inpaint request every `interval` seconds so the container stays hot and real |
| customer requests run warm (seconds-of-spin, not minutes). |
| |
| COST WARNING (learned the hard way): a ping is NOT free, and NOT cheap-per-pixel. |
| fal flux-general bills $0.075 per MEGAPIXEL with an effective ~1 MP minimum, so even |
| a 256px ping costs ~$0.075. At a 45s interval that is ~800 pings/10h ≈ ~$60/day just |
| to keep warm — far more than paying ~$0.075 per ACTUAL customer photo. Ping keep-warm |
| is therefore NOT economical for this endpoint; prefer fal dedicated/keep_alive (if its |
| fixed price beats your volume) or simply accept the occasional cold start + a queue/UX. |
| This script is now mainly a diagnostic. |
| |
| CRITICAL TUNING: the interval must be SHORTER than fal's idle timeout, or the |
| container scales down between pings and customers (and the pings themselves) still |
| hit cold starts. We saw this live at 150s: pings alternated ~4s (warm) and |
| 100-170s (cold) — i.e. fal's idle window is under 150s. This script now CLASSIFIES |
| each ping (warm vs COLD) and warns when it's leaking, so you can dial the interval |
| down empirically (try 90s, then 60s) until COLD stops appearing. |
| |
| Gated on FAL_KEY (this shell only; never logged/committed). Each ping bills ~$0.075 |
| (see COST WARNING above) — shorter interval = more pings = much more cost. |
| |
| Stop with Ctrl+C. Pilot Ready: NOT CONFIRMED. |
| """ |
| from __future__ import annotations |
|
|
| import argparse |
| import sys |
| import time |
| from io import BytesIO |
| from pathlib import Path |
|
|
| sys.path.insert(0, str(Path(__file__).resolve().parents[1])) |
|
|
| from app.services.fal_client import FalUnavailable, fal_inpaint_model, fal_real_enabled, run_fal_inpaint |
|
|
|
|
| def classify(ok: bool, duration: float, warm_max: float) -> str: |
| """Label a ping: 'warm' (fast = container was hot), 'COLD' (slow = it had to |
| spin up, i.e. keep-warm is leaking), or 'FAIL' (errored/timed out).""" |
| if not ok: |
| return "FAIL" |
| return "warm" if duration <= warm_max else "COLD" |
|
|
|
|
| def should_warn_leak(n: int, label: str) -> bool: |
| """True only when a ping signals a real between-ping leak. Ping #1 is the |
| expected startup spin-up (container is down at launch) — never a leak; only |
| a non-warm ping from #2 on means the container scaled down between pings.""" |
| return n >= 2 and label != "warm" |
|
|
|
|
| def sleep_seconds(interval: float, elapsed: float) -> float: |
| """Fixed-cadence sleep: hold a CONSTANT start-to-start interval. Sleeping a flat |
| `interval` AFTER each ping let a slow (cold ~28s) ping push the next gap out |
| (60+28=88s), which kept tripping fal's idle window and cascaded more cold starts. |
| Subtracting the elapsed ping time keeps the real gap == interval.""" |
| return max(1.0, interval - elapsed) |
|
|
|
|
| def is_fatal_error(msg: str) -> bool: |
| """A NON-retryable fal failure (billing/auth) — keep-warm must STOP, not keep |
| hammering a locked account every cadence with a misleading 'lower the interval' |
| hint. Seen live: HTTP 403 'User is locked. Reason: Exhausted balance.'""" |
| m = (msg or "").lower() |
| return any(s in m for s in ( |
| "exhausted balance", "user is locked", "http 403", "http 401", |
| "unauthorized", "forbidden", "payment", "billing", |
| )) |
|
|
|
|
| def summary_line(counts: dict) -> str: |
| """Rolling tally string. warm-rate = warm / total pings so far.""" |
| total = sum(counts.values()) |
| warm = counts.get("warm", 0) |
| rate = (100.0 * warm / total) if total else 0.0 |
| return (f"tally: {warm} warm / {counts.get('COLD', 0)} COLD / " |
| f"{counts.get('FAIL', 0)} FAIL (warm-rate {rate:.0f}%)") |
|
|
|
|
| def _tiny_png(size=256): |
| from PIL import Image, ImageDraw |
|
|
| img = Image.new("RGB", (size, size), (128, 128, 128)) |
| mask = Image.new("L", (size, size), 0) |
| ImageDraw.Draw(mask).ellipse([size * 0.3, size * 0.3, size * 0.7, size * 0.7], fill=255) |
| a = BytesIO(); img.save(a, "PNG") |
| b = BytesIO(); mask.save(b, "PNG") |
| return a.getvalue(), b.getvalue() |
|
|
|
|
| def main(argv: list[str] | None = None) -> int: |
| ap = argparse.ArgumentParser(description="Keep the fal FLUX container warm") |
| ap.add_argument("--interval", type=int, default=90, |
| help="target seconds START-to-START (fixed cadence); MUST be under " |
| "fal's idle timeout. 60s measured ~55%% warm -> try 45, then 30") |
| ap.add_argument("--steps", type=int, default=4, help="cheap ping inference steps") |
| ap.add_argument("--size", type=int, default=256, help="tiny ping image size") |
| ap.add_argument("--warm-max", type=float, default=20.0, |
| help="ping under this many seconds = warm; above = COLD (spin-up)") |
| ap.add_argument("--timeout", type=int, default=240, |
| help="per-ping timeout; must exceed a cold spin-up (~2-3 min)") |
| args = ap.parse_args(argv) |
|
|
| if not fal_real_enabled(): |
| print("REFUSED: FAL_KEY not set (no pings sent).") |
| return 2 |
|
|
| img, mask = _tiny_png(args.size) |
| print(f"keep-warm: model={fal_inpaint_model()} every {args.interval}s " |
| f"(tiny {args.size}px, {args.steps} steps, warm<= {args.warm_max}s). Ctrl+C to stop.") |
| counts = {"warm": 0, "COLD": 0, "FAIL": 0} |
| n = 0 |
| while True: |
| n += 1 |
| t = time.time() |
| err_msg = "" |
| try: |
| run_fal_inpaint(img, mask, "warm", strength=0.2, num_inference_steps=args.steps, |
| timeout_s=args.timeout) |
| dur = time.time() - t |
| label = classify(True, dur, args.warm_max) |
| print(f" ping #{n}: {label} ({dur:.1f}s)") |
| except FalUnavailable as exc: |
| label = "FAIL"; err_msg = str(exc) |
| print(f" ping #{n}: FAIL ({err_msg})") |
| except Exception as exc: |
| label = "FAIL"; err_msg = type(exc).__name__ |
| print(f" ping #{n}: FAIL ({err_msg})") |
| counts[label] += 1 |
|
|
| |
| |
| if label == "FAIL" and is_fatal_error(err_msg): |
| print(f" {summary_line(counts)}") |
| print(" STOP: fatal fal error (billing/auth) — this is NOT a keep-warm " |
| "issue. Fix the account (top up at fal.ai/dashboard/billing) or the " |
| "key, then restart. Halting pings.") |
| return 1 |
|
|
| |
| |
| |
| |
| |
| if should_warn_leak(n, label): |
| print(f" {summary_line(counts)}") |
| if label == "COLD": |
| print(f" hint: COLD on ping #{n} (between-ping) -> container scaled " |
| f"down; idle timeout < {args.interval}s; lower --interval " |
| f"(try {max(30, args.interval - 15)}).") |
| else: |
| print(f" note: ping #{n} FAILED (transient/network) — retrying next cadence.") |
| elif n == 1 and label != "warm": |
| print(" (ping #1 = startup spin-up, expected cold; not a leak. " |
| "Let it run and watch pings #2+ to judge the interval.)") |
| elif label == "warm" and n % 10 == 0: |
| print(f" {summary_line(counts)}") |
| time.sleep(sleep_seconds(args.interval, time.time() - t)) |
|
|
|
|
| if __name__ == "__main__": |
| try: |
| raise SystemExit(main()) |
| except KeyboardInterrupt: |
| print("\nkeep-warm: stopped.") |
|
|
