File size: 1,901 Bytes
7cffee1 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 | import argparse
import csv
import re
from pathlib import Path
METRIC_PATTERN = re.compile(
r"^\s*step\s+(?P<step>[\d,]+)\s+"
r"tokens\s+(?P<tokens>[\d,]+)\s+"
r"loss\s+(?P<loss>[-+]?\d+(?:\.\d+)?)\s+"
r".*?\bwall\s+(?P<wall>[-+]?\d+(?:\.\d+)?)s\b"
)
def parse_log(path: Path) -> list[dict[str, object]]:
rows = []
with path.open(encoding="utf-8", errors="replace") as log_file:
for line in log_file:
match = METRIC_PATTERN.search(line)
if match is None:
continue
rows.append(
{
"run": path.parent.name,
"step": int(match.group("step").replace(",", "")),
"tokens": int(match.group("tokens").replace(",", "")),
"loss": float(match.group("loss")),
"wall_time_seconds": float(match.group("wall")),
}
)
if not rows:
raise ValueError(f"No metric rows found in {path}")
return rows
def main() -> None:
argument_parser = argparse.ArgumentParser()
argument_parser.add_argument("logs", nargs="+", type=Path)
argument_parser.add_argument("--output", "-o", required=True, type=Path)
args = argument_parser.parse_args()
rows = []
for log_path in args.logs:
rows.extend(parse_log(log_path))
fieldnames = ["run", "step", "tokens", "loss", "wall_time_seconds"]
with args.output.open("w", newline="", encoding="utf-8") as output_file:
writer = csv.DictWriter(output_file, fieldnames=fieldnames)
writer.writeheader()
writer.writerows(rows)
for log_path in args.logs:
log_rows = [row for row in rows if row["run"] == log_path.parent.name]
print(f"{log_path}: {len(log_rows)} metric rows")
print(f"Wrote {len(rows)} rows to {args.output}")
if __name__ == "__main__":
main()
|