File size: 3,471 Bytes
0ca529a
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
import logging
import sys

_FORMAT = "[%(asctime)s] [%(levelname)s] [%(filename)s:%(funcName)s:%(lineno)s] %(message)s"
_DATE_FORMAT = "%Y-%m-%d %H:%M:%S"

_LEVEL_COLORS = {"DEBUG": "36", "INFO": "32", "WARNING": "33", "ERROR": "31", "CRITICAL": "1;31"}
_DIM = "\033[2m"
_RESET = "\033[0m"


def define_wandb_metrics(wandb) -> None:
    """Configure the shared step metric for all trainer metric namespaces."""
    wandb.define_metric("train/global_step")
    for namespace in ("train", "distill", "rollout", "timing"):
        wandb.define_metric(
            f"{namespace}/*",
            step_metric="train/global_step",
            step_sync=True,
        )
    wandb.define_metric("eval/global_step")
    wandb.define_metric("eval/*", step_metric="eval/global_step", step_sync=True)


def normalize_eval_metrics(metrics):
    """Convert trainer metric names and scalar values to the eval namespace."""
    normalized = {}
    for key, value in metrics.items():
        key = key if key.startswith("eval/") else f"eval/{key.split('/', 1)[-1]}"
        normalized[key] = value.item() if hasattr(value, "item") else float(value)
    return normalized


def log_eval_metrics(strategy, wandb, metrics, global_step):
    """Log evaluation metrics to the console and wandb."""
    metrics = normalize_eval_metrics(metrics)
    if not metrics:
        return metrics

    values = ", ".join(f"{key}: {value:.6f}" for key, value in sorted(metrics.items()))
    strategy.log(f"Evaluation at global step {global_step}, {values}")
    if wandb is not None:
        wandb.log({"eval/global_step": global_step, **metrics})
    return metrics


class ColoredNewLineFormatter(logging.Formatter):
    """Colored formatter that also aligns multi-line messages."""

    def format(self, record):
        c = _LEVEL_COLORS.get(record.levelname)
        if c:
            saved = record.levelname, record.filename, record.funcName, record.lineno
            bold = "1;" if record.levelname == "CRITICAL" else ""
            record.levelname = f"\033[{bold}{c}m{record.levelname}{_RESET}"
            record.filename = f"{_DIM}{record.filename}"
            record.funcName = f"{record.funcName}"
            record.lineno = f"{saved[3]}{_RESET}"
            msg = super().format(record)
            record.levelname, record.filename, record.funcName, record.lineno = saved
        else:
            msg = super().format(record)

        if record.message != "":
            parts = msg.split(record.message)
            msg = msg.replace("\n", "\r\n" + parts[0])
        return msg


_root_logger = logging.getLogger("kdflow")
_default_handler = None


def _setup_logger():
    _root_logger.setLevel(logging.DEBUG)
    global _default_handler
    if _default_handler is None:
        _default_handler = logging.StreamHandler(sys.stdout)
        _default_handler.flush = sys.stdout.flush  # type: ignore
        _default_handler.setLevel(logging.INFO)
        _root_logger.addHandler(_default_handler)
    fmt = ColoredNewLineFormatter(_FORMAT, datefmt=_DATE_FORMAT)
    _default_handler.setFormatter(fmt)
    # Setting this will avoid the message
    # being propagated to the parent logger.
    _root_logger.propagate = False


_setup_logger()


def init_logger(name: str):
    # Use the same settings as above for root logger
    logger = logging.getLogger(name)
    logger.setLevel(logging.DEBUG)
    logger.addHandler(_default_handler)
    logger.propagate = False
    return logger