mktgtech commited on
Commit
91b921b
·
verified ·
1 Parent(s): ff7534f

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +345 -159
app.py CHANGED
@@ -1,180 +1,366 @@
1
- import time
2
  import os
3
- import requests
4
- import pandas as pd
5
- import gradio as gr
 
6
  from urllib.parse import urlparse
7
 
8
- # =====================================================
9
- # CONFIGURATION
10
- # =====================================================
11
-
12
- # 🔐 Hugging Face Secret
13
- HASDATA_API_KEY = os.getenv("HASDATA_API_KEY")
14
-
15
- HASDATA_ENDPOINT = "https://api.hasdata.com/scrape/google/serp"
16
-
17
- TARGET_DOMAIN = "rishabhsoft.com"
18
- COUNTRY = "us" # change to "in" for India
19
- LANGUAGE = "en"
20
- LOCATION = "United States" if COUNTRY == "us" else "India"
21
-
22
- MAX_PAGES = 10 # Top 100
23
- RESULTS_PER_PAGE = 10
24
- SLEEP_TIME = 1
25
-
26
- if not HASDATA_API_KEY:
27
- raise RuntimeError("HASDATA_API_KEY not found. Add it in Hugging Face Secrets.")
28
-
29
- # =====================================================
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
30
  # HELPERS
31
- # =====================================================
32
- def normalize_domain(url: str) -> str:
33
- try:
34
- return urlparse(url).netloc.replace("www.", "").lower()
35
- except Exception:
36
- return ""
37
-
38
- # =====================================================
39
- # CORE RANK CHECK
40
- # =====================================================
41
- def check_domain_rank(keyword: str) -> pd.DataFrame:
42
- keyword = keyword.strip()
43
- results = []
44
-
45
- headers = {
46
- "x-api-key": HASDATA_API_KEY
47
- }
48
-
49
- for page in range(MAX_PAGES):
50
- start = page * RESULTS_PER_PAGE
51
-
52
- params = {
53
- "q": keyword,
54
- "gl": COUNTRY,
55
- "hl": LANGUAGE,
56
- "domain": "google.com",
57
- "location": LOCATION,
58
- "start": start,
59
- "num": RESULTS_PER_PAGE,
60
- "deviceType": "desktop"
61
- }
62
-
63
- response = requests.get(
64
- HASDATA_ENDPOINT,
65
- headers=headers,
66
- params=params,
67
- timeout=30
68
- )
69
-
70
- if response.status_code != 200:
71
- print(f"API Error [{response.status_code}]: {response.text}")
72
- break
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
73
 
74
- data = response.json()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
75
 
76
- organic_results = (
77
- data.get("organic_results")
78
- or data.get("organicResults")
79
- or []
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
80
  )
81
 
82
- for idx, item in enumerate(organic_results, start=1):
83
- url = item.get("link") or item.get("url") or ""
84
- if not url:
85
- continue
86
-
87
- domain = normalize_domain(url)
88
-
89
- if domain.endswith(TARGET_DOMAIN):
90
- results.append({
91
- "Keyword": keyword,
92
- "Domain": f"https://www.{TARGET_DOMAIN}/",
93
- "Page": page + 1,
94
- "Position on Page": idx,
95
- "Absolute Rank": start + idx,
96
- "URL": url,
97
- "Title": item.get("title")
 
 
 
 
 
 
 
 
 
98
  })
99
 
100
- if results:
101
- break
102
-
103
- time.sleep(SLEEP_TIME)
104
-
105
- if results:
106
- return pd.DataFrame(results)
107
-
108
- return pd.DataFrame([{
109
- "Keyword": keyword,
110
- "Domain": f"https://www.{TARGET_DOMAIN}/",
111
- "Page": "Not Found",
112
- "Position on Page": "Not Found",
113
- "Absolute Rank": "Not Found",
114
- "URL": None,
115
- "Title": "Not ranking in top 100"
116
- }])
117
-
118
- # =====================================================
119
- # BULK EXCEL HANDLER
120
- # =====================================================
121
- def run_bulk_excel(file):
122
- if file is None:
123
- return None, None
124
-
125
- df = pd.read_excel(file)
126
-
127
- if "keyword" not in df.columns:
128
- raise ValueError("Excel must contain a 'keyword' column")
129
-
130
- all_results = []
131
-
132
- for kw in df["keyword"].dropna().unique():
133
- all_results.append(check_domain_rank(str(kw)))
134
-
135
- final_df = pd.concat(all_results, ignore_index=True)
 
136
 
137
- output_file = "hasdata_keyword_ranking.xlsx"
138
- final_df.to_excel(output_file, index=False)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
139
 
140
- return final_df, output_file
 
141
 
142
- # =====================================================
143
- # GRADIO UI
144
- # =====================================================
145
- with gr.Blocks(title="Hasdata Google Rank Checker") as demo:
146
- gr.Markdown("## 🔍 Google Keyword Rank Checker (Hasdata)")
147
- gr.Markdown(f"**Target Domain:** https://www.{TARGET_DOMAIN}/")
148
- gr.Markdown(f"**Country:** {COUNTRY.upper()} | **Device:** Desktop")
149
 
150
- excel_input = gr.File(
151
- label="Upload Excel (.xlsx) with 'keyword' column",
152
- file_types=[".xlsx"]
 
153
  )
154
-
155
- run_btn = gr.Button("Check Rankings")
156
-
157
- output_table = gr.Dataframe(
158
- headers=[
159
- "Keyword",
160
- "Domain",
161
- "Page",
162
- "Position on Page",
163
- "Absolute Rank",
164
- "URL",
165
- "Title"
166
- ],
167
- wrap=True
 
 
 
 
 
 
 
 
 
 
 
 
 
 
168
  )
169
 
170
- download_file = gr.File(label="Download Result Excel")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
171
 
172
  run_btn.click(
173
- fn=run_bulk_excel,
174
- inputs=excel_input,
175
- outputs=[output_table, download_file]
176
  )
177
 
178
- # 🔒 REQUIRED for Hugging Face + Python 3.12
179
- demo.queue()
180
- demo.launch()
 
 
1
  import os
2
+ import json
3
+ import warnings
4
+ from functools import lru_cache
5
+ from datetime import date, timedelta
6
  from urllib.parse import urlparse
7
 
8
+ import pandas as pd
9
+ import gradio as gr
10
+ import plotly.graph_objects as go
11
+
12
+ from google.oauth2 import service_account
13
+ from google.analytics.data_v1beta import BetaAnalyticsDataClient
14
+ from google.analytics.data_v1beta.types import (
15
+ RunReportRequest,
16
+ FilterExpression,
17
+ FilterExpressionList,
18
+ Filter
19
+ )
20
+
21
+ from openai import OpenAI
22
+
23
+ # ===============================
24
+ # RUNTIME SAFETY
25
+ # ===============================
26
+ warnings.filterwarnings("ignore", category=FutureWarning)
27
+ warnings.filterwarnings("ignore", message=".*Invalid file descriptor.*")
28
+
29
+ # ===============================
30
+ # CONFIG
31
+ # ===============================
32
+ GA4_PROPERTY_ID = os.environ["GA4_PROPERTY_ID"]
33
+ SERVICE_ACCOUNT_JSON = json.loads(os.environ["SERVICE_ACCOUNT_JSON"])
34
+ OPENAI_API_KEY = os.environ["OPENAI_API_KEY"]
35
+ SITE_DOMAIN = "rishabhsoft.com"
36
+
37
+ llm = OpenAI(api_key=OPENAI_API_KEY)
38
+
39
+ # ===============================
40
+ # GA CLIENT
41
+ # ===============================
42
+ with open("service_account.json", "w") as f:
43
+ json.dump(SERVICE_ACCOUNT_JSON, f)
44
+
45
+ creds = service_account.Credentials.from_service_account_file("service_account.json")
46
+ ga = BetaAnalyticsDataClient(credentials=creds)
47
+
48
+ # ===============================
49
  # HELPERS
50
+ # ===============================
51
+ def clean_path(v):
52
+ if not v:
53
+ return "Unattributed (Page Unknown)"
54
+ p = urlparse(v).path or v
55
+ return "/ (Home)" if p == "/" else p
56
+
57
+ def classify_source(src):
58
+ if not src or src == "(direct)":
59
+ return "External"
60
+ if src.startswith("/"):
61
+ return "Internal"
62
+ if src.startswith("http"):
63
+ return "Internal" if SITE_DOMAIN in urlparse(src).netloc else "External"
64
+ return "External"
65
+
66
+ def prev_period(start, end):
67
+ s = pd.to_datetime(start)
68
+ e = pd.to_datetime(end)
69
+ delta = (e - s).days + 1
70
+ prev_end = s - timedelta(days=1)
71
+ prev_start = prev_end - timedelta(days=delta - 1)
72
+ return str(prev_start.date()), str(prev_end.date())
73
+
74
+ # ===============================
75
+ # FAST PAGE LOAD
76
+ # ===============================
77
+ @lru_cache(maxsize=3)
78
+ def fetch_top_pages(start, end):
79
+ req = RunReportRequest(
80
+ property=f"properties/{GA4_PROPERTY_ID}",
81
+ dimensions=[{"name": "pagePath"}],
82
+ metrics=[{"name": "screenPageViews"}],
83
+ date_ranges=[{"start_date": start, "end_date": end}],
84
+ order_bys=[{"metric": {"metric_name": "screenPageViews"}, "desc": True}],
85
+ limit=300
86
+ )
87
+ res = ga.run_report(req)
88
+ return sorted({r.dimension_values[0].value for r in res.rows})
89
+
90
+ # ===============================
91
+ # JOURNEY DATA
92
+ # ===============================
93
+ def fetch_journey(start, end, landing_page):
94
+ req = RunReportRequest(
95
+ property=f"properties/{GA4_PROPERTY_ID}",
96
+ dimensions=[{"name": "pageReferrer"}, {"name": "pagePath"}],
97
+ metrics=[{"name": "screenPageViews"}],
98
+ date_ranges=[{"start_date": start, "end_date": end}],
99
+ dimension_filter=FilterExpression(
100
+ filter=Filter(
101
+ field_name="landingPage",
102
+ string_filter={"match_type": 4, "value": landing_page}
103
+ )
104
+ ),
105
+ limit=3000
106
+ )
107
 
108
+ rows = []
109
+ for r in ga.run_report(req).rows:
110
+ rows.append({
111
+ "From": r.dimension_values[0].value or "(direct)",
112
+ "To": r.dimension_values[1].value,
113
+ "Views": int(r.metric_values[0].value)
114
+ })
115
+
116
+ return pd.DataFrame(rows)
117
+
118
+ # ===============================
119
+ # ACTION ATTRIBUTION (FORMS + EMAIL)
120
+ # ===============================
121
+ def fetch_action_attribution(start, end, landing_page):
122
+ """
123
+ Returns dataframe with enforced schema:
124
+ Final Action Page | Previous Page / Source | Country | Source Type | Action | Count
125
+ """
126
+
127
+ ACTION_EVENTS = {
128
+ "Form Submit": [
129
+ "generate_lead",
130
+ "form_submit",
131
+ "submit",
132
+ "contact_form_submit"
133
+ ],
134
+ "Email Click": [
135
+ "email_click",
136
+ "mailto_click",
137
+ "click_email"
138
+ ]
139
+ }
140
 
141
+ rows = []
142
+
143
+ for action, events in ACTION_EVENTS.items():
144
+ req = RunReportRequest(
145
+ property=f"properties/{GA4_PROPERTY_ID}",
146
+ dimensions=[
147
+ {"name": "pagePath"},
148
+ {"name": "pageReferrer"},
149
+ {"name": "country"},
150
+ {"name": "eventName"}
151
+ ],
152
+ metrics=[{"name": "eventCount"}],
153
+ date_ranges=[{"start_date": start, "end_date": end}],
154
+ dimension_filter=FilterExpression(
155
+ and_group=FilterExpressionList(expressions=[
156
+ FilterExpression(
157
+ filter=Filter(
158
+ field_name="eventName",
159
+ in_list_filter={"values": events}
160
+ )
161
+ ),
162
+ FilterExpression(
163
+ filter=Filter(
164
+ field_name="landingPage",
165
+ string_filter={"match_type": 4, "value": landing_page}
166
+ )
167
+ )
168
+ ])
169
+ ),
170
+ limit=3000
171
  )
172
 
173
+ res = ga.run_report(req)
174
+
175
+ for r in res.rows:
176
+ final_page = clean_path(r.dimension_values[0].value)
177
+ prev_page = clean_path(r.dimension_values[1].value)
178
+ country = r.dimension_values[2].value or "Unknown"
179
+ count = int(r.metric_values[0].value)
180
+
181
+ if final_page == "Unattributed (Page Unknown)":
182
+ rows.append({
183
+ "Final Action Page": "Unattributed (Page Unknown)",
184
+ "Previous Page / Source": "Unknown",
185
+ "Country": country,
186
+ "Source Type": "Unknown",
187
+ "Action": action,
188
+ "Count": count
189
+ })
190
+ else:
191
+ rows.append({
192
+ "Final Action Page": final_page,
193
+ "Previous Page / Source": prev_page,
194
+ "Country": country,
195
+ "Source Type": classify_source(prev_page),
196
+ "Action": action,
197
+ "Count": count
198
  })
199
 
200
+ EXPECTED_COLUMNS = [
201
+ "Final Action Page",
202
+ "Previous Page / Source",
203
+ "Country",
204
+ "Source Type",
205
+ "Action",
206
+ "Count"
207
+ ]
208
+
209
+ df = pd.DataFrame(rows)
210
+
211
+ if df.empty:
212
+ return pd.DataFrame(columns=EXPECTED_COLUMNS)
213
+
214
+ return df
215
+
216
+ # ===============================
217
+ # AI INTENT CLASSIFIER
218
+ # ===============================
219
+ @lru_cache(maxsize=100)
220
+ def classify_intent(start_page, pages_tuple, converted):
221
+ prompt = f"""
222
+ Classify user intent into ONE:
223
+ Conversion, Evaluation, Research, Navigation, Low Intent
224
+ Journey:
225
+ Start Page: {start_page}
226
+ Pages Visited: {", ".join(pages_tuple[:5])}
227
+ Converted: {"Yes" if converted else "No"}
228
+ Return:
229
+ Intent: <Intent>
230
+ Reason: <Short explanation>
231
+ """
232
+ resp = llm.chat.completions.create(
233
+ model="gpt-4o-mini",
234
+ messages=[{"role": "user", "content": prompt}],
235
+ temperature=0
236
+ )
237
 
238
+ intent, reason = "Unknown", ""
239
+ for line in resp.choices[0].message.content.splitlines():
240
+ if line.lower().startswith("intent"):
241
+ intent = line.split(":", 1)[1].strip()
242
+ if line.lower().startswith("reason"):
243
+ reason = line.split(":", 1)[1].strip()
244
+
245
+ return intent, reason
246
+
247
+ def intent_confidence(form_count, page_depth):
248
+ if form_count > 0:
249
+ return "High"
250
+ if page_depth >= 3:
251
+ return "Medium"
252
+ return "Low"
253
+
254
+ # ===============================
255
+ # MAIN ANALYSIS
256
+ # ===============================
257
+ def run_journey(start, end, landing_page):
258
+ df = fetch_journey(start, end, landing_page)
259
+
260
+ if df.empty:
261
+ return None, pd.DataFrame(), "### No data for this selection."
262
+
263
+ action_df = fetch_action_attribution(start, end, landing_page)
264
+
265
+ if action_df.empty:
266
+ form_count = 0
267
+ email_count = 0
268
+ else:
269
+ form_count = int(
270
+ action_df.loc[action_df["Action"] == "Form Submit", "Count"].sum()
271
+ )
272
+ email_count = int(
273
+ action_df.loc[action_df["Action"] == "Email Click", "Count"].sum()
274
+ )
275
 
276
+ total_users = int(df["Views"].sum())
277
+ conv_rate = round((form_count / total_users) * 100, 2) if total_users else 0.0
278
 
279
+ top_pages = df.sort_values("Views", ascending=False)["To"].unique().tolist()
 
 
 
 
 
 
280
 
281
+ intent, reason = classify_intent(
282
+ clean_path(landing_page),
283
+ tuple(clean_path(p) for p in top_pages),
284
+ form_count > 0
285
  )
286
+ confidence = intent_confidence(form_count, len(top_pages))
287
+
288
+ labels = ["Start", "Exit", "Form Submit", "Email Click"]
289
+ sources, targets, values = [], [], []
290
+
291
+ if total_users:
292
+ sources.append(0)
293
+ targets.append(1)
294
+ values.append(total_users - form_count - email_count)
295
+
296
+ if form_count:
297
+ sources.append(0)
298
+ targets.append(2)
299
+ values.append(form_count)
300
+
301
+ if email_count:
302
+ sources.append(0)
303
+ targets.append(3)
304
+ values.append(email_count)
305
+
306
+ fig = go.Figure(go.Sankey(
307
+ node=dict(label=labels, pad=24, thickness=18),
308
+ link=dict(source=sources, target=targets, value=values)
309
+ ))
310
+
311
+ fig.update_layout(
312
+ title=f"Journey → Final Actions ({clean_path(landing_page)})",
313
+ height=600
314
  )
315
 
316
+ summary = f"""
317
+ ### 🧭 Journey Summary
318
+ **Start Page:** `{clean_path(landing_page)}`
319
+ **Date Range:** {start} → {end}
320
+ **Conversions**
321
+ - Form Submits: **{form_count}**
322
+ - Email Clicks: **{email_count}**
323
+ - Conversion Rate: **{conv_rate:.2f}%**
324
+ **Intent**
325
+ - Primary Intent: **{intent}**
326
+ - Confidence: **{confidence}**
327
+ - Reason: {reason}
328
+ ℹ️ *Unattributed actions are clearly labeled when GA4 does not provide page context.*
329
+ """
330
+
331
+ return fig, action_df, summary
332
+
333
+ # ===============================
334
+ # UI
335
+ # ===============================
336
+ with gr.Blocks() as app:
337
+ gr.Markdown("## 🚀 Page-Aware Customer Journey Attribution (GA4 + AI)")
338
+
339
+ with gr.Row():
340
+ start_date = gr.Textbox("2025-12-01", label="Start Date")
341
+ end_date = gr.Textbox(str(date.today()), label="End Date")
342
+
343
+ page_path = gr.Dropdown(label="Starting Page", choices=[], allow_custom_value=True)
344
+
345
+ with gr.Row():
346
+ load_btn = gr.Button("⚡ Load Pages")
347
+ run_btn = gr.Button("Analyze Journey", variant="primary")
348
+
349
+ journey_chart = gr.Plot()
350
+ action_table = gr.Dataframe(label="Final Action Attribution (Page-Scoped)")
351
+ summary_md = gr.Markdown()
352
+
353
+ load_btn.click(
354
+ lambda s, e: gr.update(choices=fetch_top_pages(s, e)),
355
+ inputs=[start_date, end_date],
356
+ outputs=page_path
357
+ )
358
 
359
  run_btn.click(
360
+ run_journey,
361
+ inputs=[start_date, end_date, page_path],
362
+ outputs=[journey_chart, action_table, summary_md]
363
  )
364
 
365
+ if __name__ == "__main__":
366
+ app.launch()