File size: 9,324 Bytes
30072b2 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 | #!/usr/bin/env python3
"""logdiff - collapse repeated log lines and whole stack traces into counted templates.
Usage:
logdiff [FILE ...] [--max N] [--top K] [--since-file STATE] [--only-new] [--no-redact-check]
cmd 2>&1 | logdiff
--max N max output lines (default 60). Error/warn templates and traces are printed first-class:
they are never dropped in favour of normal lines.
--top K keep only the K most frequent normal (non-error) templates
--since-file F remember template hashes across runs (JSON, mode 600); new ones are tagged NEW
--only-new with --since-file: print only templates never seen before (great for cron/agents)
How: each line is normalised (timestamps, UUIDs, hex ids, IPs, numbers, quoted strings, paths' numeric parts
-> <TS> <UUID> <HEX> <IP> <N> <STR>), identical templates are merged into 'xCOUNT first example'.
Python tracebacks ('Traceback (most recent call last):' ... 'XError: msg') and Java/JS traces
(an exception line followed by indented 'at ...' frames, incl. 'Caused by:') are one unit each;
identical traces collapse to 'xCOUNT trace: Type: msg (N frames, last: ...)'.
ANSI colours and progress-bar carriage returns are stripped. Secret-looking strings are redacted.
Footer: 'logdiff: IN lines/bytes -> OUT lines/bytes (-P%) templates=T traces=X'.
No AI, no network. Exit 0 (1 if a file can't be read, 2 usage).
Examples:
logdiff /var/log/app.log --max 40
journalctl -u myjob --since today | logdiff --since-file ~/.cache/logdiff/myjob.json --only-new
"""
import signal as _sig; _sig.signal(_sig.SIGPIPE, _sig.SIG_DFL) # quiet on `| head`
import sys, os, re, json, hashlib, collections
SECRET = re.compile(r"(nsec1[0-9a-z]{20,}|sk-[A-Za-z0-9_-]{16,}|gh[pousr]_[A-Za-z0-9]{20,}|xox[abp]-[A-Za-z0-9-]{10,}|AKIA[0-9A-Z]{16}|(?i:(?:api[_-]?key|token|secret|password|passwd|authorization|bearer)(?:\s*[:=]\s*|\s+)[A-Za-z0-9._~+/-]{8,}=*)|\b[A-Fa-f0-9]{40,}\b|(?<![/\w.-])(?=[A-Za-z0-9_+-]*\d)(?=[A-Za-z0-9_+-]*[A-Za-z])[A-Za-z0-9_+-]{40,}={0,2}(?![/\w.]))")
def redact(s): return SECRET.sub("[REDACTED]", s)
ANSI = re.compile(r"\x1b\[[0-9;?]*[A-Za-z]|\x1b\][^\x07]*\x07")
NORM = [(re.compile(r"\d{4}-\d\d-\d\d[T ]\d\d:\d\d(:\d\d)?([.,]\d+)?(Z|[+-]\d\d:?\d\d)?"), "<TS>"),
(re.compile(r"\b(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) +\d{1,2} \d\d:\d\d:\d\d"), "<TS>"),
(re.compile(r"\b\d\d:\d\d:\d\d([.,]\d+)?\b"), "<TS>"),
(re.compile(r"\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\b"), "<UUID>"),
(re.compile(r"\b(?:\d{1,3}\.){3}\d{1,3}(?::\d+)?\b"), "<IP>"),
(re.compile(r"\b(?=[0-9a-fA-F]*\d)[0-9a-fA-F]{6,}\b|0x[0-9a-fA-F]+"), "<HEX>"),
(re.compile(r"\"[^\"]{0,200}\"|'[^']{0,200}'"), "<STR>"),
(re.compile(r"\d+(\.\d+)?"), "<N>")]
ERR = re.compile(r"(error|fail|fatal|exception|traceback|warn|denied|refused|timeout|timed out|panic|abort|killed|\b5\d\d\b|\b429\b)", re.I)
PY_START = re.compile(r"^\s*Traceback \(most recent call last\):")
EXC = re.compile(r"^\s*(?:Caused by: |Exception in thread \"[^\"]*\" )?([A-Za-z_$][\w$.]*(?:Error|Exception|Exit|Interrupt|Warning|Throwable)|[A-Z]\w*Error)\b:?\s*(.*)$")
FRAME = re.compile(r"^\s+(at\s+\S|\.\.\. \d+ more)")
def norm(l):
for rx, rep in NORM: l = rx.sub(rep, l)
return l
def units(lines):
"""yield ('line', text) or ('trace', [lines])"""
i, n = 0, len(lines)
while i < n:
l = lines[i]
if PY_START.match(l):
j = i + 1
while j < n and (lines[j].startswith((" ", "\t")) or lines[j].strip() == "" or
lines[j].startswith(("During handling", "The above exception"))
or PY_START.match(lines[j])):
j += 1
if j < n and EXC.match(lines[j]): j += 1 # the final 'XError: msg' line
yield ("trace", lines[i:j]); i = j; continue
if EXC.match(l) and i + 1 < n and FRAME.match(lines[i + 1]):
j = i + 1
while j < n and (FRAME.match(lines[j]) or lines[j].lstrip().startswith("Caused by:")): j += 1
yield ("trace", lines[i:j]); i = j; continue
yield ("line", l); i += 1
def trace_info(tl):
exc = next((m for m in (EXC.match(x) for x in reversed(tl)) if m and not PY_START.match(m.string)), None)
etype, emsg = (exc.group(1), exc.group(2)) if exc else ("Trace", tl[0].strip())
frames = [x.strip() for x in tl if re.match(r"^\s+(File \"|at\s)", x)]
key_parts = [norm(f) for f in frames] + [etype]
return etype, emsg, frames, hashlib.sha1("\n".join(key_parts).encode()).hexdigest()[:16]
def main(a):
files, mx, top, sf, only_new = [], 60, None, None, False
while a:
x = a.pop(0)
if x in ("-h", "--help"): print(__doc__); return 0
elif x == "--max": mx = int(a.pop(0))
elif x == "--top": top = int(a.pop(0))
elif x == "--since-file": sf = os.path.expanduser(a.pop(0))
elif x == "--only-new": only_new = True
elif x.startswith("--"): print(f"logdiff: unknown arg {x}", file=sys.stderr); return 2
else: files.append(x)
if only_new and not sf: print("logdiff: --only-new needs --since-file", file=sys.stderr); return 2
raw = []
try:
if files:
for f in files:
with open(f, encoding="utf-8", errors="replace") as fh: raw.append(fh.read())
else: raw.append(sys.stdin.buffer.read().decode("utf-8", "replace"))
except OSError as e: print(f"logdiff: {e}", file=sys.stderr); return 1
text = "".join(raw); in_bytes = len(text.encode())
lines = []
for l in ANSI.sub("", text).split("\n"):
if "\r" in l: l = l.rstrip("\r").split("\r")[-1]
lines.append(l.rstrip())
in_lines = sum(1 for l in lines if l)
groups = collections.OrderedDict() # key -> dict
ntr = 0
for kind, u in units(lines):
if kind == "line":
if not u.strip(): continue
k = "L" + hashlib.sha1(norm(u).encode()).hexdigest()[:16]
g = groups.setdefault(k, {"kind": "line", "ex": u, "last": u, "n": 0, "err": bool(ERR.search(u))})
g["n"] += 1; g["last"] = u
else:
ntr += 1
etype, emsg, frames, h = trace_info(u)
k = "T" + h
g = groups.setdefault(k, {"kind": "trace", "etype": etype, "emsg": emsg, "frames": frames, "n": 0, "err": True, "msgs": set()})
g["n"] += 1; g["msgs"].add(norm(emsg))
seen = set()
if sf and os.path.exists(sf):
try: seen = set(json.load(open(sf)).get("templates", []))
except (OSError, ValueError): seen = set()
new = {k for k in groups if k not in seen}
items = list(groups.items())
if only_new: items = [(k, g) for k, g in items if k in new]
errs = [(k, g) for k, g in items if g["err"]]
normal = [(k, g) for k, g in items if not g["err"]]
if top is not None:
keep = {k for k, _ in sorted(normal, key=lambda kg: -kg[1]["n"])[:top]}
normal = [(k, g) for k, g in normal if k in keep]
budget_norm = max(0, mx - len(errs))
if len(normal) > budget_norm: # keep the most frequent normal templates
keep = {k for k, _ in sorted(normal, key=lambda kg: -kg[1]["n"])[:budget_norm]}
dropped = len(normal) - len(keep); normal = [(k, g) for k, g in normal if k in keep]
else: dropped = 0
order = {k: i for i, k in enumerate(groups)}
chosen = sorted(errs[:mx] + normal, key=lambda kg: order[kg[0]])
out = []
for k, g in chosen:
tag = "NEW " if sf and k in new else ""
if g["kind"] == "line":
s = f"{tag}x{g['n']:<4} {g['ex'][:300]}"
if g["n"] > 1 and g["last"] != g["ex"]: s += f" [last: {g['last'][:120]}]"
else:
last = g["frames"][-1][:140] if g["frames"] else "-"
var = f", {len(g['msgs'])} msg variants" if len(g["msgs"]) > 1 else ""
s = f"{tag}x{g['n']:<4} trace: {g['etype']}: {g['emsg'][:160]} ({len(g['frames'])} frames, last: {last}{var})"
out.append(redact(s))
if dropped: out.append(f"... {dropped} rarer normal templates omitted (raise --max)")
if len(errs) > mx: out.append(f"... {len(errs) - mx} more error/trace templates omitted")
body = "\n".join(out)
if body: print(body)
ob = len(body.encode()) + (1 if body else 0)
pct = f"{100 - 100 * ob / max(in_bytes, 1):.1f}"
print(f"logdiff: {in_lines} lines/{in_bytes} B -> {len(out)} lines/{ob} B (-{pct}%) templates={sum(1 for g in groups.values() if g['kind']=='line')} traces={ntr} ({sum(1 for g in groups.values() if g['kind']=='trace')} unique)"
+ (f" new={len(new)}" if sf else ""), file=sys.stderr if os.environ.get("LOGDIFF_FOOTER_STDERR") else sys.stdout)
if sf:
d = os.path.dirname(sf)
if d: os.makedirs(d, exist_ok=True)
tmp = sf + ".tmp"; fd = os.open(tmp, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
with os.fdopen(fd, "w") as fh: json.dump({"templates": sorted(seen | set(groups))[-50000:]}, fh)
os.replace(tmp, sf)
return 0
if __name__ == "__main__":
try: sys.exit(main(sys.argv[1:]))
except BrokenPipeError: sys.exit(0)
except ValueError as e: print(f"logdiff: {e}", file=sys.stderr); sys.exit(2)
|