Upload detect_anomalies.py with huggingface_hub

detect_anomalies.py

@@ -1,11 +1,9 @@
-# detect_anomalies.py
-# Standalone anomaly predictor: loads the trained LogSentry-LM and flags log
-# lines the model finds "surprising" (high next-token loss => likely anomalous).
-#
-# Usage:
+# file: detect_anomalies.py
+# Standalone predictor: load the trained model + saved threshold and flag lines
+# the model finds surprising. Run after train_and_generate.py has saved a checkpoint.
 #   python detect_anomalies.py "Failed password for root from 1.2.3.4 port 22 ssh2"
 #   python detect_anomalies.py --file some_logs.txt
-#   python detect_anomalies.py            # runs a built-in demo set
+#   python detect_anomalies.py                     # built-in demo lines
 import os
 import sys
 import argparse
@@ -29,11 +27,11 @@ def pick_device():
 
 def load_model(device):
     if not os.path.exists(MODEL_FILE):
-        sys.exit(f"Model checkpoint '{MODEL_FILE}' not found. "
+        sys.exit(f"[ERROR] Model checkpoint '{MODEL_FILE}' not found. "
                  f"Run `python train_and_generate.py` first to train and save it.")
     ckpt = torch.load(MODEL_FILE, map_location=device)
-    model = LogSentryLM(vocab_size=ckpt["vocab_size"],
-                        max_seq_len=ckpt["max_seq_len"]).to(device)
+    # Rebuild the exact architecture from the saved config, then load weights
+    model = LogSentryLM(vocab_size=ckpt["vocab_size"], max_seq_len=ckpt["max_seq_len"]).to(device)
     model.load_state_dict(ckpt["model_state"])
     model.eval()
     tokenizer = build_or_load_tokenizer(LOG_FILE, TOKENIZER_FILE)
@@ -43,7 +41,7 @@ def load_model(device):
 def predict(model, tokenizer, lines, ckpt, device):
     threshold = ckpt["threshold"]
     max_seq_len = ckpt["max_seq_len"]
-    print(f"Anomaly threshold (loss > {threshold:.3f} => ANOMALY)\n")
+    print(f"[DETECT] Anomaly threshold (loss > {threshold:.3f} => ANOMALY)\n")
     for line in lines:
         result = score_line(model, tokenizer, line, device, max_seq_len=max_seq_len)
         if result is None:
@@ -69,7 +67,7 @@ if __name__ == "__main__":
     elif args.lines:
         lines = args.lines
     else:
-        # Built-in demo: two genuine log lines + one clearly off-distribution line.
+        # demo: two genuine lines + one clearly off-distribution line
         lines = [
             "Failed password for root from 112.95.230.3 port 49204 ssh2",
             "[Sun Dec 04 04:47:44 2005] [notice] workerEnv.init() ok /etc/httpd/conf/workers2.properties",