Spaces:
Sleeping
Sleeping
| """ | |
| One-off profiler for the two strictly-cleaned CSVs to inform IsolationForest | |
| tuning. Reports schema, per-subscriber row counts, feature distributions, | |
| hour-of-day histograms, and the actual flag rate the current model produces | |
| on each file at the current defaults. | |
| Run from `teleprocessinggh/`: | |
| .venv\\Scripts\\python.exe profile_iforest.py | |
| """ | |
| import os | |
| import sys | |
| from collections import Counter, defaultdict | |
| from pathlib import Path | |
| import numpy as np | |
| import pandas as pd | |
| # Make `telecom_anomaly.*` imports resolve when run from this folder. | |
| sys.path.insert(0, str(Path(__file__).parent)) | |
| from telecom_anomaly.core.models import TelecomConfig # noqa: E402 | |
| from telecom_anomaly.detection.isolation_forest import ( # noqa: E402 | |
| IsolationForestDetector, | |
| ) | |
| from telecom_anomaly.validation.validator import DataValidator # noqa: E402 | |
| CSV_PATHS = [ | |
| r"C:\my_project\work\GH_Police\main project\backend\telecom_anomaly_detection" | |
| r"\data\strictly_cleaned_data\246060920_calls_strictly_cleaned.csv", | |
| r"C:\my_project\work\GH_Police\main project\backend\telecom_anomaly_detection" | |
| r"\data\strictly_cleaned_data\591296805 calls (1)_strictly_cleaned.csv", | |
| ] | |
| def hr(c="-", n=72): | |
| print(c * n) | |
| def profile(path): | |
| name = os.path.basename(path) | |
| print() | |
| hr("=") | |
| print(f"FILE: {name}") | |
| hr("=") | |
| df = pd.read_csv(path, dtype=str, keep_default_na=False, low_memory=False) | |
| print(f"Rows: {len(df):,} Columns: {len(df.columns)}") | |
| print(f"Headers: {list(df.columns)}") | |
| v = DataValidator() | |
| # Derive msisdn (matches the reader's logic). | |
| def _msisdn(r): | |
| explicit = r.get("msisdn", "") | |
| if v.validate_msisdn(explicit): | |
| return explicit | |
| return v.derive_msisdn( | |
| r.get("call_direction", ""), | |
| r.get("calling_no", ""), | |
| r.get("called_no", ""), | |
| ) | |
| df["_msisdn"] = df.apply(_msisdn, axis=1) | |
| valid = df[df["_msisdn"] != ""].copy() | |
| print(f"Rows with derivable msisdn: {len(valid):,} / {len(df):,}") | |
| # Per-subscriber row counts | |
| sub_counts = valid["_msisdn"].value_counts() | |
| print(f"Distinct subscribers: {len(sub_counts):,}") | |
| print(f" rows-per-subscriber min={sub_counts.min()} " | |
| f"p50={int(sub_counts.median())} p90={int(sub_counts.quantile(0.9))} " | |
| f"max={sub_counts.max()}") | |
| eligible = sub_counts[sub_counts >= 50] | |
| print(f"Subscribers with >=50 rows (eligible for IForest): " | |
| f"{len(eligible)} ({eligible.sum():,} rows)") | |
| print(f"Top 5 subscribers by row count:") | |
| for m, c in sub_counts.head(5).items(): | |
| print(f" {m} -> {c:,} rows") | |
| # Categorical distributions | |
| print() | |
| print("usage_type:") | |
| for k, c in valid["usage_type"].value_counts().head(10).items(): | |
| print(f" {k or '<empty>':20s} {c:6,}") | |
| print("call_direction:") | |
| for k, c in valid["call_direction"].value_counts().head(10).items(): | |
| # call_direction often contains a newline; collapse for readability | |
| kk = " ".join(str(k).split()) | |
| print(f" {kk or '<empty>':20s} {c:6,}") | |
| # Duration distribution | |
| dur = pd.to_numeric(valid["duration"], errors="coerce").fillna(0) | |
| print() | |
| print(f"duration (sec) min={dur.min():.0f} " | |
| f"p50={dur.quantile(0.5):.0f} p90={dur.quantile(0.9):.0f} " | |
| f"p99={dur.quantile(0.99):.0f} max={dur.max():.0f}") | |
| print(f" duration < 3s : {(dur < 3).sum():,} " | |
| f"(flash-call territory)") | |
| print(f" duration > 1hr : {(dur > 3600).sum():,} " | |
| f"(extreme-call territory)") | |
| # Hour distribution | |
| def _hour(t): | |
| try: | |
| return int(str(t).split(":")[0]) | |
| except Exception: | |
| return -1 | |
| valid["_hour"] = valid["event_time"].apply(_hour) | |
| hist = valid[valid["_hour"] >= 0]["_hour"].value_counts().sort_index() | |
| print() | |
| print("Hour-of-day histogram (counts):") | |
| for h in range(24): | |
| count = int(hist.get(h, 0)) | |
| bar = "#" * min(60, count // max(1, sub_counts.sum() // 800)) | |
| marker = " *night*" if (h >= 23 or h < 5) else "" | |
| print(f" {h:02d}:00 {count:6,} {bar}{marker}") | |
| night = valid[(valid["_hour"] >= 23) | ((valid["_hour"] >= 0) & (valid["_hour"] < 5))] | |
| print(f"Night-hour rows (23:00-04:59): {len(night):,} " | |
| f"({100*len(night)/max(1,len(valid)):.1f}%)") | |
| # Coordinate coverage | |
| lat = pd.to_numeric(valid["latitude"], errors="coerce") | |
| lon = pd.to_numeric(valid["longitude"], errors="coerce") | |
| print() | |
| print(f"latitude present: {lat.notna().sum():,} " | |
| f"({100*lat.notna().sum()/len(valid):.1f}%)") | |
| print(f"longitude present: {lon.notna().sum():,} " | |
| f"({100*lon.notna().sum()/len(valid):.1f}%)") | |
| # Region / district cardinality | |
| print(f"distinct regions : {valid['region'].nunique()}") | |
| print(f"distinct districts: {valid['district'].nunique()}") | |
| if 'imei' in valid.columns: | |
| print(f"distinct IMEIs : {valid['imei'].nunique()}") | |
| # ----- Run IForest at multiple contamination levels ----- | |
| print() | |
| print("IsolationForest flag rates per contamination (current min_samples=50):") | |
| # Build records list in the shape the detector expects. | |
| records = [] | |
| for _, r in valid.iterrows(): | |
| records.append({ | |
| "msisdn": r["_msisdn"], | |
| "duration": pd.to_numeric(r.get("duration"), errors="coerce") or 0.0, | |
| "event_time": r.get("event_time", ""), | |
| "is_night": (_hour(r.get("event_time")) >= 23 | |
| or 0 <= _hour(r.get("event_time")) < 5), | |
| "usage_type": r.get("usage_type", ""), | |
| "call_direction": r.get("call_direction", ""), | |
| "latitude": float(r["latitude"]) | |
| if r.get("latitude") not in ("", None) and pd.notna( | |
| pd.to_numeric(r["latitude"], errors="coerce")) else None, | |
| "longitude": float(r["longitude"]) | |
| if r.get("longitude") not in ("", None) and pd.notna( | |
| pd.to_numeric(r["longitude"], errors="coerce")) else None, | |
| "region": r.get("region", ""), | |
| "district": r.get("district", ""), | |
| }) | |
| for contamination in ["auto", 0.10, 0.05, 0.02, 0.01]: | |
| cfg = TelecomConfig() | |
| cfg.iforest_contamination = contamination | |
| cfg.iforest_min_samples = 50 | |
| det = IsolationForestDetector(cfg) | |
| flagged = det.detect(records) | |
| scored_subs = sum( | |
| 1 for s, idxs in | |
| _group_subscribers(records).items() if len(idxs) >= 50 | |
| ) | |
| scored_rows = sum( | |
| len(idxs) for s, idxs in | |
| _group_subscribers(records).items() if len(idxs) >= 50 | |
| ) | |
| rate = 100 * len(flagged) / max(1, scored_rows) | |
| print(f" contamination={str(contamination):>5} " | |
| f"flagged={len(flagged):4} " | |
| f"of scored_rows={scored_rows:5} " | |
| f"({rate:5.2f}%) " | |
| f"across {scored_subs} subscribers") | |
| def _group_subscribers(records): | |
| g = defaultdict(list) | |
| for i, r in enumerate(records): | |
| m = r.get("msisdn", "") | |
| if m: | |
| g[m].append(i) | |
| return g | |
| if __name__ == "__main__": | |
| for path in CSV_PATHS: | |
| if not os.path.exists(path): | |
| print(f"Missing: {path}") | |
| continue | |
| profile(path) | |