| """Bin a pytest trace into a coarse failure category. |
| |
| The goal is a single tag per failure that aggregates well in a report. We do |
| this with cheap regex on the raw trace string. Categories, checked in order: |
| |
| OOM — torch OutOfMemoryError, "CUDA out of memory" |
| load_error — failure raised before forward (from_pretrained, safetensors, |
| HFValidationError, missing weight, gated/auth) |
| cuda_runtime — runtime CUDA/cuBLAS/cuDNN/nvrtc/Triton compile errors |
| output_mismatch — text / tensor comparison failures (assertEqual, Expectations, |
| Tensor-likes are not close, etc.) |
| import_or_config — Python-level errors before the test body runs |
| (ImportError, AttributeError on config, TypeError on |
| __init__ signatures). |
| other — fallback |
| """ |
|
|
| from __future__ import annotations |
|
|
| import re |
|
|
|
|
| _OOM_PAT = re.compile(r"OutOfMemoryError|CUDA out of memory|MallocFailure|HIP out of memory", re.I) |
| _LOAD_PAT = re.compile( |
| r"from_pretrained|safetensors\.|HFValidationError|Repository Not Found|gated|" |
| r"Cannot read|UnboundLocalError.*loading|FileNotFoundError|access requested|" |
| r"401 Client Error|403 Client Error", |
| re.I, |
| ) |
| _CUDA_RUNTIME_PAT = re.compile( |
| r"CUDA error|CUBLAS_STATUS|CUDNN_STATUS|cudnn[_ ]frontend|nvrtc|" |
| r"triton\.compiler|RuntimeError: Triton|c10::Error|NCCL.*error", |
| re.I, |
| ) |
| _OUTPUT_MISMATCH_PAT = re.compile( |
| r"Tensor-likes are not close|" |
| r"assertEqual|assertSequenceEqual|self\.assertListEqual|" |
| r"assertAlmostEqual|assertGreater|expected_text|" |
| r"AssertionError", |
| re.I | re.DOTALL, |
| ) |
| _IMPORT_CFG_PAT = re.compile( |
| r"^.*ImportError|ModuleNotFoundError|" |
| r"AttributeError:.*(config|object has no attribute)|" |
| r"TypeError:.*(__init__|got an unexpected keyword argument)|" |
| r"ValueError:.*Unrecognized configuration", |
| re.I | re.M, |
| ) |
|
|
|
|
| def classify(trace: str) -> str: |
| if not trace: |
| return "other" |
| for tag, pat in ( |
| ("OOM", _OOM_PAT), |
| ("load_error", _LOAD_PAT), |
| ("cuda_runtime", _CUDA_RUNTIME_PAT), |
| ("import_or_config", _IMPORT_CFG_PAT), |
| ("output_mismatch", _OUTPUT_MISMATCH_PAT), |
| ): |
| if pat.search(trace): |
| return tag |
| return "other" |
|
|
|
|
| def short_excerpt(trace: str, max_chars: int = 240) -> str: |
| """Take the LAST non-empty line of the trace (the actual exception line). |
| Then trim to `max_chars`. |
| """ |
| if not trace: |
| return "" |
| for line in reversed(trace.splitlines()): |
| line = line.strip() |
| if line: |
| return (line[: max_chars - 1] + "…") if len(line) > max_chars else line |
| return "" |
|
|
|
|
| if __name__ == "__main__": |
| samples = [ |
| "tests/...py:42: torch.OutOfMemoryError: CUDA out of memory.", |
| "RuntimeError: CUDA error: CUBLAS_STATUS_EXECUTION_FAILED", |
| "AssertionError: 'Paris' != 'capital of France'", |
| "ImportError: cannot import name 'foo'", |
| "HFValidationError: Repository Not Found for url: ...", |
| "Tensor-likes are not close (...) max_abs_diff=0.5", |
| ] |
| for s in samples: |
| print(f"{classify(s):<18} {s}") |
|
|
