| """Retry wrapper for reads off the network-mounted bucket (hfmount): these |
| fail transiently with plain I/O errors sometimes -- confirmed in production |
| logs (DuckDB "Input/output error" reading a parquet file, torch.load |
| hitting [Errno 5]) on files that read fine moments later with no code |
| change. Not a size or speed issue -- a flaky mount, so the fix is retrying, |
| not optimizing.""" |
| import time |
|
|
|
|
| def retry(fn, attempts=4, base_delay=2.0, label=""): |
| for attempt in range(1, attempts + 1): |
| try: |
| return fn() |
| except Exception as e: |
| if attempt == attempts: |
| raise |
| delay = base_delay * (2 ** (attempt - 1)) |
| print(f"[retry] {label} attempt {attempt}/{attempts} failed ({e}); retrying in {delay:.0f}s") |
| time.sleep(delay) |
|
|
