replyflow-rag / tests /load_test_throttle.py
Qurrah's picture
ADD: implement load testing script for Shopify API to evaluate throttling behavior
e5dc3cb
Raw History Blame Contribute Delete
4.44 kB
"""
Real load test against your actual Shopify store β€” fires enough concurrent
GraphQL requests to trigger Shopify's real cost-based throttling, so you can
watch your retry logic kick in live in the logs.
⚠️ WARNING before running this:
- This burns real API rate-limit budget on your store. Only run against
a DEV/TEST store, never production, unless you know what you're doing.
- Shopify's throttle bucket is shared across your whole app for that
store β€” this WILL temporarily slow down/throttle any other requests
hitting the same store while the test runs.
- Start with CONCURRENT_REQUESTS = 5 and increase gradually. Going
straight to 50 is how you find out the hard way what your bucket size is.
Usage:
export SHOP_DOMAIN="your-dev-store.myshopify.com"
export SHOPIFY_ACCESS_TOKEN="shpat_xxxxx"
python load_test_throttle.py
"""
import os
import time
import logging
from concurrent.futures import ThreadPoolExecutor, as_completed
from dotenv import load_dotenv
load_dotenv() # reads .env from the current directory into os.environ
logging.basicConfig(level=logging.INFO, format="%(asctime)s | %(levelname)s | %(message)s")
logger = logging.getLogger(__name__)
# --- ADJUST THIS IMPORT to match your project structure ---
from clients.shopify.product_search_api import search_products
SHOP_DOMAIN = os.environ.get("SHOPIFY_SHOP_DOMAIN")
ACCESS_TOKEN = os.environ.get("SHOPIFY_ACCESS_TOKEN")
CONCURRENT_REQUESTS = 15 # start low, raise gradually
QUERY = "" # empty = match everything, cheapest possible query
LIMIT_PER_REQUEST = 50 # higher limit = more query "cost" per call = throttles faster
def fire_one(request_num: int) -> tuple[int, bool, str]:
"""Returns (request_num, succeeded, message)."""
start = time.time()
try:
products = search_products(SHOP_DOMAIN, ACCESS_TOKEN, QUERY, limit=LIMIT_PER_REQUEST)
elapsed = time.time() - start
return request_num, True, f"OK β€” {len(products)} products in {elapsed:.2f}s"
except Exception as e:
elapsed = time.time() - start
return request_num, False, f"FAILED after {elapsed:.2f}s β€” {type(e).__name__}: {e}"
def main():
if not SHOP_DOMAIN or not ACCESS_TOKEN:
raise SystemExit(
"Set SHOP_DOMAIN and SHOPIFY_ACCESS_TOKEN environment variables first.\n"
" export SHOP_DOMAIN='your-dev-store.myshopify.com'\n"
" export SHOPIFY_ACCESS_TOKEN='shpat_xxxxx'"
)
logger.info(
"Firing %d concurrent requests at %s (limit=%d per request)...",
CONCURRENT_REQUESTS, SHOP_DOMAIN, LIMIT_PER_REQUEST,
)
logger.info("Watch for 'THROTTLED' or '429' log lines from product_search_api itself.")
start = time.time()
results = []
with ThreadPoolExecutor(max_workers=CONCURRENT_REQUESTS) as executor:
futures = [executor.submit(fire_one, i) for i in range(CONCURRENT_REQUESTS)]
for future in as_completed(futures):
request_num, ok, message = future.result()
status = "βœ…" if ok else "❌"
logger.info("%s Request #%02d: %s", status, request_num, message)
results.append(ok)
total_time = time.time() - start
succeeded = sum(results)
failed = len(results) - succeeded
print("\n" + "=" * 60)
print(f"Total time: {total_time:.2f}s")
print(f"Succeeded: {succeeded}/{len(results)}")
print(f"Failed: {failed}/{len(results)}")
print("=" * 60)
if failed == 0 and total_time > 3:
print(
"No hard failures, but if total time is noticeably higher than a\n"
"single request would take, some of these likely got throttled and\n"
"silently succeeded after your retry logic kicked in β€” check the\n"
"logs above for 'THROTTLED' or '429' lines to confirm."
)
elif failed == 0:
print(
"Everything succeeded fast β€” you didn't hit the throttle threshold.\n"
"Try raising CONCURRENT_REQUESTS or LIMIT_PER_REQUEST and re-run."
)
else:
print(
"Some requests failed even after retries β€” you exceeded your\n"
"MAX_RATE_LIMIT_RETRIES budget under sustained load. This is\n"
"expected once you push hard enough; it tells you your retry\n"
"ceiling under this load level."
)
if __name__ == "__main__":
main()