| #!/usr/bin/env python3 | |
| import argparse,json,re | |
| from pathlib import Path | |
| ap=argparse.ArgumentParser();ap.add_argument("log");a=ap.parse_args() | |
| t=Path(a.log).read_text(errors="replace") if Path(a.log).is_file() else "" | |
| def has(p): return re.search(p,t,re.I|re.S) is not None | |
| root="UNCLASSIFIED" | |
| if has(r"on startup is less than desired GPU memory utilization"): | |
| root="STARTUP_UTIL_GATE" | |
| elif has(r"Invalid address format:"): | |
| root="RAY_ADDRESS_FORMAT" | |
| elif has(r"CUDA out of memory|torch\.OutOfMemoryError"): | |
| oom=min([m.start() for m in re.finditer(r"CUDA out of memory|torch\.OutOfMemoryError",t,re.I)] or [10**18]) | |
| loaded=max([m.start() for m in re.finditer(r"Loading model weights took|Model loading took",t,re.I)] or [-1]) | |
| root="CUDA_OOM_POSTLOAD" if loaded>=0 and loaded<oom else "CUDA_OOM_DURING_MODEL_LOAD" | |
| elif has(r"Out of memory:\s*Killed process|oom-kill|\bSIGKILL\b|exit code -9"): | |
| root="HOST_OOM_KILL" | |
| elif has(r"nccl(?:System|Internal|Remote)Error|NCCL.*(?:ERROR|unhandled)"): | |
| root="NCCL_ERROR" | |
| elif has(r"Gloo.*(?:connectFullMesh|failed|error)"): | |
| root="GLOO_ERROR" | |
| elif has(r"LoRA.*(?:error|failed|unsupported|incompatible)|(?:error|failed).*LoRA"): | |
| root="LORA_ERROR" | |
| print(json.dumps({"root_cause":root},indent=2)) | |