genee123 commited on
Commit
dfd0230
·
verified ·
1 Parent(s): 1e137a6

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +197 -67
app.py CHANGED
@@ -1,104 +1,237 @@
1
- def predict_url(url):
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2
  try:
3
- # Initialize result dictionary
4
- result = {
5
- "display": "",
6
- "status": "",
7
- "source": "",
8
- "legit_prob": 0.0,
9
- "phish_prob": 0.0,
10
- "decision": "UNKNOWN" # Will be set to either "PHISHING" or "LEGITIMATE"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
11
  }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
12
 
13
- # Fix incomplete URLs
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
14
  if not url.startswith(("http://", "https://")):
15
  url = "https://" + url
16
 
17
  # Step 1: Check with PhishTank (Highest Priority)
18
  phishtank_result = check_url_phishtank(url)
19
  if phishtank_result and phishtank_result.get('valid', False) and phishtank_result.get('verified', False):
20
- result["status"] = "⚠️ CONFIRMED PHISHING (PhishTank)"
21
- result["source"] = "Verified and valid entry in PhishTank database"
22
- result["legit_prob"] = 0.0
23
- result["phish_prob"] = 1.0
24
- result["decision"] = "PHISHING"
25
-
26
- result["display"] = f"""
27
- ## {result['status']}
28
- **Source**: {result['source']}
29
  **Details**:
30
  - Verified at: {phishtank_result.get('verified_at', 'N/A')}
31
  - [View Report]({phishtank_result.get('phish_detail_page', '#')})
32
- **Legitimate Probability**: {result['legit_prob']:.4f}
33
- **Phishing Probability**: {result['phish_prob']:.4f}
34
  """
35
- return result
36
 
37
  # Step 2: Check with Google Safe Browsing
38
  gsb_result = check_url_safebrowsing(url)
39
  if gsb_result and "matches" in gsb_result:
40
  threat_type = gsb_result["matches"][0]["threatType"]
41
- result["status"] = "⚠️ CONFIRMED PHISHING (Google Safe Browsing)"
42
- result["source"] = f"Detected as {threat_type}"
43
- result["legit_prob"] = 0.0
44
- result["phish_prob"] = 1.0
45
- result["decision"] = "PHISHING"
46
-
47
- result["display"] = f"""
48
- ## {result['status']}
49
- **Source**: {result['source']}
50
  **Details**:
51
  - Google Safe Browsing has identified this URL as malicious
52
- **Legitimate Probability**: {result['legit_prob']:.4f}
53
- **Phishing Probability**: {result['phish_prob']:.4f}
54
  """
55
- return result
56
 
57
  # Step 3: Run Machine Learning Model
58
  legit_prob, phish_prob = predict_with_model(url)
59
  if legit_prob is None or phish_prob is None:
60
- result["display"] = "## ❌ Error\nModel prediction failed"
61
- result["decision"] = "ERROR"
62
- return result
63
 
64
  # Apply 0.85 threshold for phishing
65
  if phish_prob >= 0.85:
66
- result["status"] = "⚠️ SUSPECTED PHISHING"
67
- result["source"] = "XGBoost Model (Probability ≥ 0.85)"
68
- result["decision"] = "PHISHING"
69
  else:
70
- result["status"] = "✅ LIKELY LEGITIMATE"
71
- result["source"] = "XGBoost Model"
72
- result["decision"] = "LEGITIMATE"
73
-
74
- result["legit_prob"] = legit_prob
75
- result["phish_prob"] = phish_prob
76
-
77
- result["display"] = f"""
78
- ## {result['status']}
79
- **Source**: {result['source']}
80
  **Details**:
81
  - Not found in PhishTank or Google Safe Browsing databases
82
- **Legitimate Probability**: {result['legit_prob']:.4f}
83
- **Phishing Probability**: {result['phish_prob']:.4f}
84
  """
85
-
86
- return result
87
 
88
  except Exception as e:
89
- return {
90
- "display": f"## ❌ Error\n{str(e)}",
91
- "decision": "ERROR"
92
- }
93
 
94
- # Gradio Interface
95
  interface = gr.Interface(
96
  fn=predict_url,
97
  inputs=gr.Textbox(label="Enter URL", placeholder="https://example.com"),
98
- outputs=[
99
- gr.Markdown(label="Security Assessment"),
100
- gr.Textbox(label="Result", visible=True) # Add a textbox for the decision
101
- ],
102
  title="Phishing URL Detector",
103
  description=(
104
  "Three-tier detection system: "
@@ -110,11 +243,8 @@ interface = gr.Interface(
110
  ["https://www.apple.com"],
111
  ["https://login-facebook-secure.xyz/login.php"],
112
  ["https://bit.ly/suspicious-download"],
113
- ["https://telkomsa-admin-online-net.weebly.com/"]
114
  ]
115
  )
116
 
117
- def display_result(res):
118
- return res["display"], res["decision"]
119
-
120
  interface.launch(server_name="0.0.0.0", server_port=7860)
 
1
+ import numpy as np
2
+ import pandas as pd
3
+ import re
4
+ import gradio as gr
5
+ import requests
6
+ import json
7
+ import xgboost as xgb
8
+ import joblib
9
+ from urllib.parse import urlparse
10
+
11
+ # Load artifacts
12
+ feature_names = joblib.load('feature_names.joblib')
13
+ non_numeric_columns = joblib.load('non_numeric_columns.joblib')
14
+ scaler = joblib.load('scaler.joblib')
15
+
16
+ # Load label encoders
17
+ label_encoders = {}
18
+ for col in non_numeric_columns:
19
  try:
20
+ le = joblib.load(f'le_{col}.joblib')
21
+ label_encoders[col] = le
22
+ except:
23
+ pass
24
+
25
+ # Apparently, Kailangan nya sa Native sya, Not the Joblib Version
26
+ model = xgb.Booster()
27
+ model.load_model('xgboost_best_model.json')
28
+
29
+ # API Configuration
30
+ GOOGLE_API_KEY = "AIzaSyCmXigpl6t7MZyvq8Rv37XW_lZVCi0oMSM"
31
+ PHISHTANK_USER_AGENT = "phishtank/PHISH"
32
+
33
+ def check_url_phishtank(url):
34
+ # THIS WOULD BE THE PHISHTANK
35
+ endpoint = "https://checkurl.phishtank.com/checkurl/"
36
+ payload = {"url": url, "format": "json"}
37
+ headers = {"User-Agent": PHISHTANK_USER_AGENT}
38
+
39
+ try:
40
+ response = requests.post(endpoint, data=payload, headers=headers, timeout=5)
41
+ response.raise_for_status()
42
+ data = response.json()
43
+
44
+ # Check if URL is verified phishing in PhishTank
45
+ if data.get('results', {}).get('in_database', False):
46
+ return data['results']
47
+ return None
48
+ except Exception as e:
49
+ print(f"PhishTank API error: {str(e)}")
50
+ return None
51
+
52
+ def check_url_safebrowsing(url):
53
+ # This would be The Safe Browsing (GOOGLE)
54
+ endpoint = f"https://safebrowsing.googleapis.com/v4/threatMatches:find?key={GOOGLE_API_KEY}"
55
+ payload = {
56
+ "client": {"clientId": "PhishShield", "clientVersion": "1.0"},
57
+ "threatInfo": {
58
+ "threatTypes": ["MALWARE", "SOCIAL_ENGINEERING", "UNWANTED_SOFTWARE"],
59
+ "platformTypes": ["ANY_PLATFORM"],
60
+ "threatEntryTypes": ["URL"],
61
+ "threatEntries": [{"url": url}]
62
  }
63
+ }
64
+ try:
65
+ response = requests.post(endpoint, json=payload, timeout=5)
66
+ response.raise_for_status()
67
+ return response.json()
68
+ except Exception as e:
69
+ print(f"Google Safe Browsing API error: {str(e)}")
70
+ return None
71
+
72
+ def extract_features(url):
73
+ # FOR THE MODEL (OUR MODEL) FEATURE EXTRACTION
74
+ parsed = urlparse(url)
75
+ hostname = parsed.hostname if parsed.hostname else ''
76
+ path = parsed.path if parsed.path else ''
77
+
78
+ entropy = 0.0
79
+ if len(url) > 0:
80
+ for c in set(url):
81
+ p = url.count(c) / len(url)
82
+ entropy -= p * np.log2(p) if p > 0 else 0
83
+
84
+ features = {
85
+ "length_url": len(url),
86
+ "length_hostname": len(hostname),
87
+ "ip": 1 if any(char.isdigit() for char in hostname) else 0,
88
+ "nb_dots": url.count('.'),
89
+ "nb_hyphens": url.count('-'),
90
+ "nb_at": url.count('@'),
91
+ "nb_qm": url.count('?'),
92
+ "nb_and": url.count('&'),
93
+ "nb_or": url.count('|'),
94
+ "nb_eq": url.count('='),
95
+ "nb_underscore": url.count('_'),
96
+ "nb_tilde": url.count('~'),
97
+ "nb_percent": url.count('%'),
98
+ "nb_slash": url.count('/'),
99
+ "nb_star": url.count('*'),
100
+ "nb_colon": url.count(':'),
101
+ "nb_comma": url.count(','),
102
+ "nb_semicolumn": url.count(';'),
103
+ "nb_dollar": url.count('$'),
104
+ "nb_space": url.count(' '),
105
+ "nb_www": 1 if "www" in url else 0,
106
+ "nb_com": 1 if ".com" in url else 0,
107
+ "nb_dslash": url.count('//'),
108
+ "http_in_path": 1 if "http" in path else 0,
109
+ "https_token": 1 if "https" in url else 0,
110
+ "ratio_digits_url": sum(c.isdigit() for c in url) / len(url) if len(url) > 0 else 0,
111
+ "ratio_digits_host": sum(c.isdigit() for c in hostname) / len(hostname) if hostname else 0,
112
+ "punycode": 1 if re.search(r'xn--', url, re.IGNORECASE) else 0,
113
+ "port": parsed.port if parsed.port else 0,
114
+ "tld_in_path": 1 if any(tld in path for tld in ['.com', '.net', '.org', '.gov', '.edu']) else 0,
115
+ "tld_in_subdomain": 1 if any(tld in hostname for tld in ['.com', '.net', '.org', '.gov', '.edu']) else 0,
116
+ "abnormal_subdomain": 1 if len(hostname.split('.')) > 3 else 0,
117
+ "nb_subdomains": len(hostname.split('.')) - 1,
118
+ "prefix_suffix": 1 if url.startswith("www") else 0,
119
+ "shortening_service": 1 if any(short in url for short in ['bit.ly', 'goo.gl', 'tinyurl.com']) else 0,
120
+ "path_extension": 1 if any(ext in path for ext in ['.exe', '.zip', '.rar', '.tar', '.pdf']) else 0,
121
+ "length_words_raw": len(url.split()),
122
+ "char_repeat": len(set(url)),
123
+ "shortest_words_raw": min(len(word) for word in url.split()) if url.split() else 0,
124
+ "longest_words_raw": max(len(word) for word in url.split()) if url.split() else 0,
125
+ "shortest_word_host": min(len(word) for word in hostname.split('.')) if hostname else 0,
126
+ "longest_word_host": max(len(word) for word in hostname.split('.')) if hostname else 0,
127
+ "shortest_word_path": min(len(word) for word in path.split('/')) if path else 0,
128
+ "longest_word_path": max(len(word) for word in path.split('/')) if path else 0,
129
+ "avg_words_raw": np.mean([len(word) for word in url.split()]) if url.split() else 0,
130
+ "avg_word_host": np.mean([len(word) for word in hostname.split('.')]) if hostname else 0,
131
+ "avg_word_path": np.mean([len(word) for word in path.split('/')]) if path else 0,
132
+ "phish_hints": 1 if any(kw in url.lower() for kw in ['login', 'secure', 'verify', 'account']) else 0,
133
+ "domain_in_brand": 1 if 'apple' in hostname.lower() else 0,
134
+ "brand_in_subdomain": 1 if 'apple' in (hostname.split('.')[0] if hostname else '') else 0,
135
+ "brand_in_path": 1 if 'apple' in path.lower() else 0,
136
+ "suspicious_tld": 1 if hostname.endswith(('.xyz', '.top', '.club', '.gq', '.cf', '.tk')) else 0,
137
+ "entropy": entropy
138
+ }
139
+ return features
140
+
141
+ def predict_with_model(url):
142
+ # Prediction with XGBoost First
143
+ try:
144
+ # Extract features
145
+ features = extract_features(url)
146
 
147
+ # Create input DataFrame
148
+ input_data = pd.DataFrame([features], columns=feature_names)
149
+
150
+ # Encode non-numeric features
151
+ for col in non_numeric_columns:
152
+ if col in input_data.columns and col in label_encoders:
153
+ try:
154
+ input_data[col] = label_encoders[col].transform(input_data[col].astype(str))
155
+ except ValueError:
156
+ input_data[col] = len(label_encoders[col].classes_)
157
+
158
+ # Scale features
159
+ input_scaled = scaler.transform(input_data)
160
+
161
+ # Convert to DMatrix format
162
+ dmatrix = xgb.DMatrix(input_scaled)
163
+
164
+ # Predict
165
+ phish_prob = model.predict(dmatrix)[0]
166
+ legit_prob = 1 - phish_prob
167
+
168
+ return legit_prob, phish_prob
169
+ except Exception as e:
170
+ print(f"Model prediction error: {str(e)}")
171
+ return None, None
172
+
173
+ def predict_url(url):
174
+ try:
175
+ # Fix incomplete URLs (Probably Redundant, but Imma keep it here)
176
  if not url.startswith(("http://", "https://")):
177
  url = "https://" + url
178
 
179
  # Step 1: Check with PhishTank (Highest Priority)
180
  phishtank_result = check_url_phishtank(url)
181
  if phishtank_result and phishtank_result.get('valid', False) and phishtank_result.get('verified', False):
182
+ return f"""
183
+ ## ⚠️ CONFIRMED PHISHING (PhishTank)
184
+ **Source**: Verified and valid entry in PhishTank database
 
 
 
 
 
 
185
  **Details**:
186
  - Verified at: {phishtank_result.get('verified_at', 'N/A')}
187
  - [View Report]({phishtank_result.get('phish_detail_page', '#')})
188
+ **Legitimate Probability**: 0.0
189
+ **Phishing Probability**: 1.0
190
  """
 
191
 
192
  # Step 2: Check with Google Safe Browsing
193
  gsb_result = check_url_safebrowsing(url)
194
  if gsb_result and "matches" in gsb_result:
195
  threat_type = gsb_result["matches"][0]["threatType"]
196
+ return f"""
197
+ ## ⚠️ CONFIRMED PHISHING (Google Safe Browsing)
198
+ **Source**: Detected as {threat_type}
 
 
 
 
 
 
199
  **Details**:
200
  - Google Safe Browsing has identified this URL as malicious
201
+ **Legitimate Probability**: 0.0
202
+ **Phishing Probability**: 1.0
203
  """
 
204
 
205
  # Step 3: Run Machine Learning Model
206
  legit_prob, phish_prob = predict_with_model(url)
207
  if legit_prob is None or phish_prob is None:
208
+ return "## ❌ Error\nModel prediction failed"
 
 
209
 
210
  # Apply 0.85 threshold for phishing
211
  if phish_prob >= 0.85:
212
+ status = "⚠️ SUSPECTED PHISHING"
213
+ source = "XGBoost Model (Probability ≥ 0.85)"
 
214
  else:
215
+ status = "✅ LIKELY LEGITIMATE"
216
+ source = "XGBoost Model"
217
+
218
+ return f"""
219
+ ## {status}
220
+ **Source**: {source}
 
 
 
 
221
  **Details**:
222
  - Not found in PhishTank or Google Safe Browsing databases
223
+ **Legitimate Probability**: {legit_prob:.4f}
224
+ **Phishing Probability**: {phish_prob:.4f}
225
  """
 
 
226
 
227
  except Exception as e:
228
+ return f"## ❌ Error\n{str(e)}"
 
 
 
229
 
230
+ # Gradio Interface (HUHUHUUHUH)
231
  interface = gr.Interface(
232
  fn=predict_url,
233
  inputs=gr.Textbox(label="Enter URL", placeholder="https://example.com"),
234
+ outputs=gr.Markdown(label="Security Assessment"),
 
 
 
235
  title="Phishing URL Detector",
236
  description=(
237
  "Three-tier detection system: "
 
243
  ["https://www.apple.com"],
244
  ["https://login-facebook-secure.xyz/login.php"],
245
  ["https://bit.ly/suspicious-download"],
246
+ ["https://telkomsa-admin-online-net.weebly.com/"] # Test with PhishTank example
247
  ]
248
  )
249
 
 
 
 
250
  interface.launch(server_name="0.0.0.0", server_port=7860)