File size: 13,221 Bytes
c0bce59 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 | """Generic worker loop shared by every model adapter.
The loop is model-agnostic: it reads the requests file, resumes from the results file,
calls ``Adapter.run_batch`` in fixed-size batches, and appends one fsynced result line per
request. A failing batch is retried item by item so a single bad clip cannot take its
neighbours down; every failure is recorded with its exception type and message (never
dropped) and counted in the summary. Model loading failures are fatal: they propagate and
the process exits non-zero.
"""
from __future__ import annotations
import abc
import argparse
import hashlib
import importlib
import importlib.metadata
import json
import logging
import sys
import time
import traceback
from collections.abc import Mapping, Sequence
from dataclasses import dataclass, replace
from datetime import UTC, datetime
from pathlib import Path
from typing import ClassVar
from nida.bench.protocol import (
ProtocolError,
WorkerSummary,
WorkRequest,
WorkResult,
append_result,
drop_results,
load_requests,
load_results,
pending_requests,
read_summary,
write_summary,
)
__all__ = [
"MAX_CONSECUTIVE_FAILURES",
"Adapter",
"ModelRef",
"TooManyFailuresError",
"WorkerOptions",
"main_for",
"run_worker",
]
log = logging.getLogger(__name__)
#: Abort when this many items in a row fail: the model/device is broken, not the clips.
MAX_CONSECUTIVE_FAILURES = 10
#: Progress is logged when this many more items are done (and at the end).
PROGRESS_EVERY = 200
#: When the *same* failure run (same clip ids) aborts more than this many times in a row,
#: it is evidently repeatable rather than transient and is kept as recorded failures so a
#: rerun can move past it. Unrelated aborts never accumulate (see ``_abort_count``).
MAX_ABORTS_BEFORE_KEEPING = 2
class TooManyFailuresError(RuntimeError):
"""Consecutive item failures suggest a broken model or device (e.g. CUDA context)."""
@dataclass(frozen=True)
class ModelRef:
"""Which model to load.
Attributes:
model_id: Hub id, model card name or local path, interpreted by the adapter.
revision: Pinned revision (commit SHA) or ``""`` when the source has none.
"""
model_id: str
revision: str
class Adapter(abc.ABC):
"""One model family. Subclasses keep heavy imports inside their methods."""
name: ClassVar[str]
#: Distribution names whose installed versions are recorded in the summary.
distributions: ClassVar[tuple[str, ...]] = ()
@abc.abstractmethod
def load(self, model: ModelRef, device: str, batch_size: int) -> None:
"""Load the model onto ``device``. Exceptions here abort the worker."""
@abc.abstractmethod
def run_batch(self, batch: Sequence[WorkRequest]) -> list[Mapping[str, object]]:
"""Return one JSON-compatible output mapping per request, in the same order."""
def peak_vram_gb(self) -> float | None:
"""Peak CUDA memory allocated by this process, if torch with CUDA is present."""
try:
torch = importlib.import_module("torch")
except ImportError:
return None
if not torch.cuda.is_available():
return None
return round(float(torch.cuda.max_memory_allocated()) / 1024**3, 2)
@dataclass(frozen=True)
class WorkerOptions:
requests: Path
results: Path
summary: Path
model: ModelRef
device: str
batch_size: int
retry_failed: bool
def _now() -> str:
return datetime.now(UTC).isoformat(timespec="seconds")
def _describe(err: BaseException) -> str:
return f"{type(err).__name__}: {err}"
def _package_versions(names: Sequence[str]) -> dict[str, str]:
versions: dict[str, str] = {}
for name in names:
try:
versions[name] = importlib.metadata.version(name)
except importlib.metadata.PackageNotFoundError:
versions[name] = "not-installed"
return versions
def _abort_counter_path(results: Path) -> Path:
return results.with_name(results.name + ".aborts.json")
def _abort_count(results: Path, failure_ids: Sequence[str]) -> int:
"""Record an abort and return how often *this* failure run has aborted in a row.
The counter stores a signature of the failing clip ids: a different failure run
restarts at 1, and :func:`_clear_abort_count` removes it after any run that finishes
without aborting, so transient incidents in different sessions never add up.
"""
signature = hashlib.sha1("\n".join(sorted(failure_ids)).encode("utf-8")).hexdigest()
path = _abort_counter_path(results)
count = 1
if path.exists():
try:
previous = json.loads(path.read_text(encoding="utf-8"))
if previous.get("signature") == signature:
count = int(previous.get("count", 0)) + 1
except (ValueError, AttributeError, TypeError) as err:
# A damaged counter must not wedge the worker; recount from this abort.
log.warning("ignoring unreadable abort counter %s (%s); restarting at 1", path, err)
tmp = path.with_name(f".{path.name}.tmp")
tmp.write_text(json.dumps({"signature": signature, "count": count}), encoding="utf-8")
tmp.replace(path)
return count
def _clear_abort_count(results: Path) -> None:
_abort_counter_path(results).unlink(missing_ok=True)
def _drop_failed(path: Path) -> None:
"""Rewrite the results file without failed entries (for ``--retry-failed``)."""
results = load_results(path, repair=True)
kept = [r.to_json() + "\n" for r in results.values() if r.ok]
tmp = path.with_name(f".{path.name}.tmp")
tmp.write_text("".join(kept), encoding="utf-8")
tmp.replace(path)
def _validated_outputs(batch: Sequence[WorkRequest], outputs: object) -> list[Mapping[str, object]]:
if not isinstance(outputs, list) or len(outputs) != len(batch):
got = len(outputs) if isinstance(outputs, list) else type(outputs).__name__
raise ProtocolError(f"adapter returned {got} outputs for a batch of {len(batch)}")
for output in outputs:
if not isinstance(output, Mapping):
raise ProtocolError(f"adapter output must be a mapping, got {type(output).__name__}")
return outputs
def _run_one_batch(adapter: Adapter, batch: Sequence[WorkRequest]) -> list[WorkResult]:
started = time.perf_counter()
try:
outputs = _validated_outputs(batch, adapter.run_batch(batch))
except Exception as err: # noqa: BLE001 - recorded per item below, never swallowed
if len(batch) == 1:
log.warning("item %s failed: %s", batch[0].clip_id, _describe(err))
return [
WorkResult(
clip_id=batch[0].clip_id,
output=None,
error=_describe(err),
infer_seconds=time.perf_counter() - started,
)
]
log.warning("batch of %d failed (%s); retrying items one by one", len(batch), err)
results: list[WorkResult] = []
for request in batch:
results.extend(_run_one_batch(adapter, [request]))
return results
per_item = (time.perf_counter() - started) / len(batch)
return [
WorkResult(clip_id=r.clip_id, output=dict(o), error=None, infer_seconds=per_item)
for r, o in zip(batch, outputs, strict=True)
]
def run_worker(adapter: Adapter, options: WorkerOptions) -> WorkerSummary:
"""Process every pending request and write the summary. Returns the summary."""
if options.batch_size < 1:
raise ValueError(f"batch_size must be >= 1, got {options.batch_size}")
started_at = _now()
requests = load_requests(options.requests)
if options.retry_failed and options.results.exists():
_drop_failed(options.results)
done = load_results(options.results, repair=True)
pending = pending_requests(requests, done)
log.info(
"%s: %d requests, %d already done, %d pending",
adapter.name,
len(requests),
len(done),
len(pending),
)
previous = read_summary(options.summary) if options.summary.exists() else None
load_seconds = 0.0
if pending:
load_start = time.perf_counter()
adapter.load(options.model, options.device, options.batch_size)
load_seconds = time.perf_counter() - load_start
options.results.parent.mkdir(parents=True, exist_ok=True)
failure_run: list[WorkResult] = []
aborted = False
with options.results.open("a", encoding="utf-8") as handle:
for start in range(0, len(pending), options.batch_size):
batch = pending[start : start + options.batch_size]
for result in _run_one_batch(adapter, batch):
append_result(handle, result)
failure_run = [] if result.ok else [*failure_run, result]
if len(failure_run) >= MAX_CONSECUTIVE_FAILURES:
aborted = True
break
if aborted:
break
finished = start + len(batch)
if (
finished == len(pending)
or finished // PROGRESS_EVERY != start // PROGRESS_EVERY
):
log.info("%s: %d/%d", adapter.name, finished, len(pending))
if aborted:
aborts = _abort_count(options.results, [r.clip_id for r in failure_run])
if aborts <= MAX_ABORTS_BEFORE_KEEPING:
# Most likely infrastructure (e.g. a broken CUDA context), not bad clips: make
# them pending again so a plain rerun retries them instead of scoring them as
# empty hypotheses.
drop_results(options.results, [r.clip_id for r in failure_run])
outcome = "results removed, they will be retried"
else:
outcome = (
f"same failure run aborted {aborts} times: kept as recorded failures so "
"the next run moves on; use --retry-failed to try them again"
)
raise TooManyFailuresError(
f"{adapter.name}: {len(failure_run)} consecutive failures ({outcome}), "
f"last: {failure_run[-1].error}"
)
_clear_abort_count(options.results) # finished without aborting
final = load_results(options.results)
n_ok = sum(1 for r in final.values() if r.ok)
if not pending and previous is not None:
# No-op resume: keep the measurements of the invocation that did the work.
summary = replace(previous, n_ok=n_ok, n_error=len(final) - n_ok)
else:
peaks = [v for v in (adapter.peak_vram_gb(), previous and previous.peak_vram_gb) if v]
invocations = int(previous.extra.get("invocations", "1")) + 1 if previous else 1
summary = WorkerSummary(
worker=adapter.name,
model_id=options.model.model_id,
revision=options.model.revision,
load_seconds=round(load_seconds, 3),
peak_vram_gb=max(peaks) if peaks else None,
n_ok=n_ok,
n_error=len(final) - n_ok,
started_at=previous.started_at if previous else started_at,
ended_at=_now(),
packages=_package_versions(adapter.distributions),
extra={
"python": sys.version.split()[0],
"device": options.device,
"invocations": str(invocations),
},
)
write_summary(options.summary, summary)
return summary
def _parse(argv: Sequence[str] | None) -> WorkerOptions:
parser = argparse.ArgumentParser(description="Run a Nida benchmark worker.")
parser.add_argument("--requests", type=Path, required=True)
parser.add_argument("--results", type=Path, required=True)
parser.add_argument("--summary", type=Path, default=None)
parser.add_argument("--model-id", required=True)
parser.add_argument("--revision", default="")
parser.add_argument("--device", default="cuda")
parser.add_argument("--batch-size", type=int, default=8)
parser.add_argument("--retry-failed", action="store_true")
args = parser.parse_args(argv)
summary = args.summary or args.results.with_name(args.results.stem + ".summary.json")
return WorkerOptions(
requests=args.requests,
results=args.results,
summary=summary,
model=ModelRef(model_id=args.model_id, revision=args.revision),
device=args.device,
batch_size=args.batch_size,
retry_failed=args.retry_failed,
)
def main_for(adapter: Adapter, argv: Sequence[str] | None = None) -> int:
"""CLI entry point used by each worker module's ``__main__`` block."""
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
options = _parse(argv)
try:
summary = run_worker(adapter, options)
except (ProtocolError, OSError, ValueError, TooManyFailuresError):
traceback.print_exc()
return 2
print(f"{adapter.name}: ok={summary.n_ok} error={summary.n_error}")
return 0
|