File size: 9,324 Bytes
30072b2
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
#!/usr/bin/env python3
"""logdiff - collapse repeated log lines and whole stack traces into counted templates.

Usage:
  logdiff [FILE ...] [--max N] [--top K] [--since-file STATE] [--only-new] [--no-redact-check]
  cmd 2>&1 | logdiff

  --max N           max output lines (default 60). Error/warn templates and traces are printed first-class:
                    they are never dropped in favour of normal lines.
  --top K           keep only the K most frequent normal (non-error) templates
  --since-file F    remember template hashes across runs (JSON, mode 600); new ones are tagged NEW
  --only-new        with --since-file: print only templates never seen before (great for cron/agents)
How: each line is normalised (timestamps, UUIDs, hex ids, IPs, numbers, quoted strings, paths' numeric parts
-> <TS> <UUID> <HEX> <IP> <N> <STR>), identical templates are merged into 'xCOUNT  first example'.
Python tracebacks ('Traceback (most recent call last):' ... 'XError: msg') and Java/JS traces
(an exception line followed by indented 'at ...' frames, incl. 'Caused by:') are one unit each;
identical traces collapse to 'xCOUNT trace: Type: msg (N frames, last: ...)'.
ANSI colours and progress-bar carriage returns are stripped. Secret-looking strings are redacted.
Footer: 'logdiff: IN lines/bytes -> OUT lines/bytes (-P%) templates=T traces=X'.
No AI, no network. Exit 0 (1 if a file can't be read, 2 usage).

Examples:
  logdiff /var/log/app.log --max 40
  journalctl -u myjob --since today | logdiff --since-file ~/.cache/logdiff/myjob.json --only-new
"""
import signal as _sig; _sig.signal(_sig.SIGPIPE, _sig.SIG_DFL)  # quiet on `| head`
import sys, os, re, json, hashlib, collections
SECRET = re.compile(r"(nsec1[0-9a-z]{20,}|sk-[A-Za-z0-9_-]{16,}|gh[pousr]_[A-Za-z0-9]{20,}|xox[abp]-[A-Za-z0-9-]{10,}|AKIA[0-9A-Z]{16}|(?i:(?:api[_-]?key|token|secret|password|passwd|authorization|bearer)(?:\s*[:=]\s*|\s+)[A-Za-z0-9._~+/-]{8,}=*)|\b[A-Fa-f0-9]{40,}\b|(?<![/\w.-])(?=[A-Za-z0-9_+-]*\d)(?=[A-Za-z0-9_+-]*[A-Za-z])[A-Za-z0-9_+-]{40,}={0,2}(?![/\w.]))")
def redact(s): return SECRET.sub("[REDACTED]", s)
ANSI = re.compile(r"\x1b\[[0-9;?]*[A-Za-z]|\x1b\][^\x07]*\x07")
NORM = [(re.compile(r"\d{4}-\d\d-\d\d[T ]\d\d:\d\d(:\d\d)?([.,]\d+)?(Z|[+-]\d\d:?\d\d)?"), "<TS>"),
        (re.compile(r"\b(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) +\d{1,2} \d\d:\d\d:\d\d"), "<TS>"),
        (re.compile(r"\b\d\d:\d\d:\d\d([.,]\d+)?\b"), "<TS>"),
        (re.compile(r"\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\b"), "<UUID>"),
        (re.compile(r"\b(?:\d{1,3}\.){3}\d{1,3}(?::\d+)?\b"), "<IP>"),
        (re.compile(r"\b(?=[0-9a-fA-F]*\d)[0-9a-fA-F]{6,}\b|0x[0-9a-fA-F]+"), "<HEX>"),
        (re.compile(r"\"[^\"]{0,200}\"|'[^']{0,200}'"), "<STR>"),
        (re.compile(r"\d+(\.\d+)?"), "<N>")]
ERR = re.compile(r"(error|fail|fatal|exception|traceback|warn|denied|refused|timeout|timed out|panic|abort|killed|\b5\d\d\b|\b429\b)", re.I)
PY_START = re.compile(r"^\s*Traceback \(most recent call last\):")
EXC = re.compile(r"^\s*(?:Caused by: |Exception in thread \"[^\"]*\" )?([A-Za-z_$][\w$.]*(?:Error|Exception|Exit|Interrupt|Warning|Throwable)|[A-Z]\w*Error)\b:?\s*(.*)$")
FRAME = re.compile(r"^\s+(at\s+\S|\.\.\. \d+ more)")
def norm(l):
    for rx, rep in NORM: l = rx.sub(rep, l)
    return l
def units(lines):
    """yield ('line', text) or ('trace', [lines])"""
    i, n = 0, len(lines)
    while i < n:
        l = lines[i]
        if PY_START.match(l):
            j = i + 1
            while j < n and (lines[j].startswith((" ", "\t")) or lines[j].strip() == "" or
                             lines[j].startswith(("During handling", "The above exception"))
                             or PY_START.match(lines[j])):
                j += 1
            if j < n and EXC.match(lines[j]): j += 1      # the final 'XError: msg' line
            yield ("trace", lines[i:j]); i = j; continue
        if EXC.match(l) and i + 1 < n and FRAME.match(lines[i + 1]):
            j = i + 1
            while j < n and (FRAME.match(lines[j]) or lines[j].lstrip().startswith("Caused by:")): j += 1
            yield ("trace", lines[i:j]); i = j; continue
        yield ("line", l); i += 1
def trace_info(tl):
    exc = next((m for m in (EXC.match(x) for x in reversed(tl)) if m and not PY_START.match(m.string)), None)
    etype, emsg = (exc.group(1), exc.group(2)) if exc else ("Trace", tl[0].strip())
    frames = [x.strip() for x in tl if re.match(r"^\s+(File \"|at\s)", x)]
    key_parts = [norm(f) for f in frames] + [etype]
    return etype, emsg, frames, hashlib.sha1("\n".join(key_parts).encode()).hexdigest()[:16]
def main(a):
    files, mx, top, sf, only_new = [], 60, None, None, False
    while a:
        x = a.pop(0)
        if x in ("-h", "--help"): print(__doc__); return 0
        elif x == "--max": mx = int(a.pop(0))
        elif x == "--top": top = int(a.pop(0))
        elif x == "--since-file": sf = os.path.expanduser(a.pop(0))
        elif x == "--only-new": only_new = True
        elif x.startswith("--"): print(f"logdiff: unknown arg {x}", file=sys.stderr); return 2
        else: files.append(x)
    if only_new and not sf: print("logdiff: --only-new needs --since-file", file=sys.stderr); return 2
    raw = []
    try:
        if files:
            for f in files:
                with open(f, encoding="utf-8", errors="replace") as fh: raw.append(fh.read())
        else: raw.append(sys.stdin.buffer.read().decode("utf-8", "replace"))
    except OSError as e: print(f"logdiff: {e}", file=sys.stderr); return 1
    text = "".join(raw); in_bytes = len(text.encode())
    lines = []
    for l in ANSI.sub("", text).split("\n"):
        if "\r" in l: l = l.rstrip("\r").split("\r")[-1]
        lines.append(l.rstrip())
    in_lines = sum(1 for l in lines if l)
    groups = collections.OrderedDict()     # key -> dict
    ntr = 0
    for kind, u in units(lines):
        if kind == "line":
            if not u.strip(): continue
            k = "L" + hashlib.sha1(norm(u).encode()).hexdigest()[:16]
            g = groups.setdefault(k, {"kind": "line", "ex": u, "last": u, "n": 0, "err": bool(ERR.search(u))})
            g["n"] += 1; g["last"] = u
        else:
            ntr += 1
            etype, emsg, frames, h = trace_info(u)
            k = "T" + h
            g = groups.setdefault(k, {"kind": "trace", "etype": etype, "emsg": emsg, "frames": frames, "n": 0, "err": True, "msgs": set()})
            g["n"] += 1; g["msgs"].add(norm(emsg))
    seen = set()
    if sf and os.path.exists(sf):
        try: seen = set(json.load(open(sf)).get("templates", []))
        except (OSError, ValueError): seen = set()
    new = {k for k in groups if k not in seen}
    items = list(groups.items())
    if only_new: items = [(k, g) for k, g in items if k in new]
    errs = [(k, g) for k, g in items if g["err"]]
    normal = [(k, g) for k, g in items if not g["err"]]
    if top is not None:
        keep = {k for k, _ in sorted(normal, key=lambda kg: -kg[1]["n"])[:top]}
        normal = [(k, g) for k, g in normal if k in keep]
    budget_norm = max(0, mx - len(errs))
    if len(normal) > budget_norm:                     # keep the most frequent normal templates
        keep = {k for k, _ in sorted(normal, key=lambda kg: -kg[1]["n"])[:budget_norm]}
        dropped = len(normal) - len(keep); normal = [(k, g) for k, g in normal if k in keep]
    else: dropped = 0
    order = {k: i for i, k in enumerate(groups)}
    chosen = sorted(errs[:mx] + normal, key=lambda kg: order[kg[0]])
    out = []
    for k, g in chosen:
        tag = "NEW " if sf and k in new else ""
        if g["kind"] == "line":
            s = f"{tag}x{g['n']:<4} {g['ex'][:300]}"
            if g["n"] > 1 and g["last"] != g["ex"]: s += f"   [last: {g['last'][:120]}]"
        else:
            last = g["frames"][-1][:140] if g["frames"] else "-"
            var = f", {len(g['msgs'])} msg variants" if len(g["msgs"]) > 1 else ""
            s = f"{tag}x{g['n']:<4} trace: {g['etype']}: {g['emsg'][:160]} ({len(g['frames'])} frames, last: {last}{var})"
        out.append(redact(s))
    if dropped: out.append(f"... {dropped} rarer normal templates omitted (raise --max)")
    if len(errs) > mx: out.append(f"... {len(errs) - mx} more error/trace templates omitted")
    body = "\n".join(out)
    if body: print(body)
    ob = len(body.encode()) + (1 if body else 0)
    pct = f"{100 - 100 * ob / max(in_bytes, 1):.1f}"
    print(f"logdiff: {in_lines} lines/{in_bytes} B -> {len(out)} lines/{ob} B (-{pct}%) templates={sum(1 for g in groups.values() if g['kind']=='line')} traces={ntr} ({sum(1 for g in groups.values() if g['kind']=='trace')} unique)"
          + (f" new={len(new)}" if sf else ""), file=sys.stderr if os.environ.get("LOGDIFF_FOOTER_STDERR") else sys.stdout)
    if sf:
        d = os.path.dirname(sf)
        if d: os.makedirs(d, exist_ok=True)
        tmp = sf + ".tmp"; fd = os.open(tmp, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
        with os.fdopen(fd, "w") as fh: json.dump({"templates": sorted(seen | set(groups))[-50000:]}, fh)
        os.replace(tmp, sf)
    return 0
if __name__ == "__main__":
    try: sys.exit(main(sys.argv[1:]))
    except BrokenPipeError: sys.exit(0)
    except ValueError as e: print(f"logdiff: {e}", file=sys.stderr); sys.exit(2)