| |
| import os |
| from datetime import datetime |
| from time import sleep |
|
|
| from loguru import logger |
|
|
| while True: |
| try: |
| with open("progress.txt", "r") as f: |
| progress = f.read() |
| last_mod_time = datetime.fromtimestamp(os.path.getmtime("progress.txt")) |
| if (datetime.now() - last_mod_time).seconds > 60 * 7: |
| |
| logger.info("restarting server to fix cuda issues (device side asserts)") |
| os.system("/usr/bin/bash kill -SIGHUP `pgrep gunicorn`") |
| os.system("/usr/bin/bash kill -SIGHUP `pgrep uvicorn`") |
| os.system("kill -9 `pgrep gunicorn`") |
| os.system("kill -9 `pgrep uvicorn`") |
| os.system("killall -9 uvicorn") |
| os.system("ps | grep uvicorn | awk '{print $1}' | xargs kill -9") |
|
|
| if progress == "done": |
| break |
| except Exception as e: |
| print(e) |
| pass |
| sleep(60*5) |
|
|
