#!/usr/bin/env python3 """logdiff - collapse repeated log lines and whole stack traces into counted templates. Usage: logdiff [FILE ...] [--max N] [--top K] [--since-file STATE] [--only-new] [--no-redact-check] cmd 2>&1 | logdiff --max N max output lines (default 60). Error/warn templates and traces are printed first-class: they are never dropped in favour of normal lines. --top K keep only the K most frequent normal (non-error) templates --since-file F remember template hashes across runs (JSON, mode 600); new ones are tagged NEW --only-new with --since-file: print only templates never seen before (great for cron/agents) How: each line is normalised (timestamps, UUIDs, hex ids, IPs, numbers, quoted strings, paths' numeric parts -> ), identical templates are merged into 'xCOUNT first example'. Python tracebacks ('Traceback (most recent call last):' ... 'XError: msg') and Java/JS traces (an exception line followed by indented 'at ...' frames, incl. 'Caused by:') are one unit each; identical traces collapse to 'xCOUNT trace: Type: msg (N frames, last: ...)'. ANSI colours and progress-bar carriage returns are stripped. Secret-looking strings are redacted. Footer: 'logdiff: IN lines/bytes -> OUT lines/bytes (-P%) templates=T traces=X'. No AI, no network. Exit 0 (1 if a file can't be read, 2 usage). Examples: logdiff /var/log/app.log --max 40 journalctl -u myjob --since today | logdiff --since-file ~/.cache/logdiff/myjob.json --only-new """ import signal as _sig; _sig.signal(_sig.SIGPIPE, _sig.SIG_DFL) # quiet on `| head` import sys, os, re, json, hashlib, collections SECRET = re.compile(r"(nsec1[0-9a-z]{20,}|sk-[A-Za-z0-9_-]{16,}|gh[pousr]_[A-Za-z0-9]{20,}|xox[abp]-[A-Za-z0-9-]{10,}|AKIA[0-9A-Z]{16}|(?i:(?:api[_-]?key|token|secret|password|passwd|authorization|bearer)(?:\s*[:=]\s*|\s+)[A-Za-z0-9._~+/-]{8,}=*)|\b[A-Fa-f0-9]{40,}\b|(?"), (re.compile(r"\b(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) +\d{1,2} \d\d:\d\d:\d\d"), ""), (re.compile(r"\b\d\d:\d\d:\d\d([.,]\d+)?\b"), ""), (re.compile(r"\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\b"), ""), (re.compile(r"\b(?:\d{1,3}\.){3}\d{1,3}(?::\d+)?\b"), ""), (re.compile(r"\b(?=[0-9a-fA-F]*\d)[0-9a-fA-F]{6,}\b|0x[0-9a-fA-F]+"), ""), (re.compile(r"\"[^\"]{0,200}\"|'[^']{0,200}'"), ""), (re.compile(r"\d+(\.\d+)?"), "")] ERR = re.compile(r"(error|fail|fatal|exception|traceback|warn|denied|refused|timeout|timed out|panic|abort|killed|\b5\d\d\b|\b429\b)", re.I) PY_START = re.compile(r"^\s*Traceback \(most recent call last\):") EXC = re.compile(r"^\s*(?:Caused by: |Exception in thread \"[^\"]*\" )?([A-Za-z_$][\w$.]*(?:Error|Exception|Exit|Interrupt|Warning|Throwable)|[A-Z]\w*Error)\b:?\s*(.*)$") FRAME = re.compile(r"^\s+(at\s+\S|\.\.\. \d+ more)") def norm(l): for rx, rep in NORM: l = rx.sub(rep, l) return l def units(lines): """yield ('line', text) or ('trace', [lines])""" i, n = 0, len(lines) while i < n: l = lines[i] if PY_START.match(l): j = i + 1 while j < n and (lines[j].startswith((" ", "\t")) or lines[j].strip() == "" or lines[j].startswith(("During handling", "The above exception")) or PY_START.match(lines[j])): j += 1 if j < n and EXC.match(lines[j]): j += 1 # the final 'XError: msg' line yield ("trace", lines[i:j]); i = j; continue if EXC.match(l) and i + 1 < n and FRAME.match(lines[i + 1]): j = i + 1 while j < n and (FRAME.match(lines[j]) or lines[j].lstrip().startswith("Caused by:")): j += 1 yield ("trace", lines[i:j]); i = j; continue yield ("line", l); i += 1 def trace_info(tl): exc = next((m for m in (EXC.match(x) for x in reversed(tl)) if m and not PY_START.match(m.string)), None) etype, emsg = (exc.group(1), exc.group(2)) if exc else ("Trace", tl[0].strip()) frames = [x.strip() for x in tl if re.match(r"^\s+(File \"|at\s)", x)] key_parts = [norm(f) for f in frames] + [etype] return etype, emsg, frames, hashlib.sha1("\n".join(key_parts).encode()).hexdigest()[:16] def main(a): files, mx, top, sf, only_new = [], 60, None, None, False while a: x = a.pop(0) if x in ("-h", "--help"): print(__doc__); return 0 elif x == "--max": mx = int(a.pop(0)) elif x == "--top": top = int(a.pop(0)) elif x == "--since-file": sf = os.path.expanduser(a.pop(0)) elif x == "--only-new": only_new = True elif x.startswith("--"): print(f"logdiff: unknown arg {x}", file=sys.stderr); return 2 else: files.append(x) if only_new and not sf: print("logdiff: --only-new needs --since-file", file=sys.stderr); return 2 raw = [] try: if files: for f in files: with open(f, encoding="utf-8", errors="replace") as fh: raw.append(fh.read()) else: raw.append(sys.stdin.buffer.read().decode("utf-8", "replace")) except OSError as e: print(f"logdiff: {e}", file=sys.stderr); return 1 text = "".join(raw); in_bytes = len(text.encode()) lines = [] for l in ANSI.sub("", text).split("\n"): if "\r" in l: l = l.rstrip("\r").split("\r")[-1] lines.append(l.rstrip()) in_lines = sum(1 for l in lines if l) groups = collections.OrderedDict() # key -> dict ntr = 0 for kind, u in units(lines): if kind == "line": if not u.strip(): continue k = "L" + hashlib.sha1(norm(u).encode()).hexdigest()[:16] g = groups.setdefault(k, {"kind": "line", "ex": u, "last": u, "n": 0, "err": bool(ERR.search(u))}) g["n"] += 1; g["last"] = u else: ntr += 1 etype, emsg, frames, h = trace_info(u) k = "T" + h g = groups.setdefault(k, {"kind": "trace", "etype": etype, "emsg": emsg, "frames": frames, "n": 0, "err": True, "msgs": set()}) g["n"] += 1; g["msgs"].add(norm(emsg)) seen = set() if sf and os.path.exists(sf): try: seen = set(json.load(open(sf)).get("templates", [])) except (OSError, ValueError): seen = set() new = {k for k in groups if k not in seen} items = list(groups.items()) if only_new: items = [(k, g) for k, g in items if k in new] errs = [(k, g) for k, g in items if g["err"]] normal = [(k, g) for k, g in items if not g["err"]] if top is not None: keep = {k for k, _ in sorted(normal, key=lambda kg: -kg[1]["n"])[:top]} normal = [(k, g) for k, g in normal if k in keep] budget_norm = max(0, mx - len(errs)) if len(normal) > budget_norm: # keep the most frequent normal templates keep = {k for k, _ in sorted(normal, key=lambda kg: -kg[1]["n"])[:budget_norm]} dropped = len(normal) - len(keep); normal = [(k, g) for k, g in normal if k in keep] else: dropped = 0 order = {k: i for i, k in enumerate(groups)} chosen = sorted(errs[:mx] + normal, key=lambda kg: order[kg[0]]) out = [] for k, g in chosen: tag = "NEW " if sf and k in new else "" if g["kind"] == "line": s = f"{tag}x{g['n']:<4} {g['ex'][:300]}" if g["n"] > 1 and g["last"] != g["ex"]: s += f" [last: {g['last'][:120]}]" else: last = g["frames"][-1][:140] if g["frames"] else "-" var = f", {len(g['msgs'])} msg variants" if len(g["msgs"]) > 1 else "" s = f"{tag}x{g['n']:<4} trace: {g['etype']}: {g['emsg'][:160]} ({len(g['frames'])} frames, last: {last}{var})" out.append(redact(s)) if dropped: out.append(f"... {dropped} rarer normal templates omitted (raise --max)") if len(errs) > mx: out.append(f"... {len(errs) - mx} more error/trace templates omitted") body = "\n".join(out) if body: print(body) ob = len(body.encode()) + (1 if body else 0) pct = f"{100 - 100 * ob / max(in_bytes, 1):.1f}" print(f"logdiff: {in_lines} lines/{in_bytes} B -> {len(out)} lines/{ob} B (-{pct}%) templates={sum(1 for g in groups.values() if g['kind']=='line')} traces={ntr} ({sum(1 for g in groups.values() if g['kind']=='trace')} unique)" + (f" new={len(new)}" if sf else ""), file=sys.stderr if os.environ.get("LOGDIFF_FOOTER_STDERR") else sys.stdout) if sf: d = os.path.dirname(sf) if d: os.makedirs(d, exist_ok=True) tmp = sf + ".tmp"; fd = os.open(tmp, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) with os.fdopen(fd, "w") as fh: json.dump({"templates": sorted(seen | set(groups))[-50000:]}, fh) os.replace(tmp, sf) return 0 if __name__ == "__main__": try: sys.exit(main(sys.argv[1:])) except BrokenPipeError: sys.exit(0) except ValueError as e: print(f"logdiff: {e}", file=sys.stderr); sys.exit(2)