Spaces:
Sleeping
Sleeping
Update app.py
Browse files
app.py
CHANGED
|
@@ -13,83 +13,189 @@ from datetime import date, timedelta
|
|
| 13 |
API_TOKEN = os.getenv("LEADFEEDER_API_TOKEN")
|
| 14 |
APP_PASSWORD = os.getenv("APP_PASSWORD")
|
| 15 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 16 |
if not API_TOKEN or not APP_PASSWORD:
|
| 17 |
-
|
|
|
|
| 18 |
|
| 19 |
-
ACCOUNT_ID = "255333"
|
| 20 |
BASE_URL = "https://api.leadfeeder.com"
|
| 21 |
PAGE_SIZE = 100
|
| 22 |
-
|
| 23 |
HEADERS = {
|
| 24 |
"Authorization": f"Token token={API_TOKEN}",
|
| 25 |
"Accept": "application/json"
|
| 26 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 27 |
|
| 28 |
# =====================================================
|
| 29 |
-
# PRESETS
|
| 30 |
# =====================================================
|
| 31 |
DASHBOARD_PRESETS = {
|
|
|
|
| 32 |
"Total Visits by Industry": ("primary_industry", "total_visits", "sum"),
|
|
|
|
|
|
|
| 33 |
"Total Visits by Country": ("country", "total_visits", "sum"),
|
| 34 |
"Avg Quality by Industry": ("primary_industry", "lead_quality_score", "mean"),
|
| 35 |
"Top Accounts by Visits": ("company_name", "total_visits", "sum"),
|
| 36 |
-
"Companies by Country": ("country", "company_name", "count"),
|
| 37 |
-
"Visits by City": ("city", "total_visits", "sum"),
|
| 38 |
-
"Accounts by Assignee": ("assignee", "company_name", "count"),
|
| 39 |
-
"Visits by Assignee": ("assignee", "total_visits", "sum"),
|
| 40 |
}
|
| 41 |
|
| 42 |
TREND_PRESETS = {
|
| 43 |
"Total Visits Trend": ("total_visits", "sum", None),
|
|
|
|
|
|
|
| 44 |
"Active Accounts Trend": ("company_name", "count", None),
|
| 45 |
"Avg Lead Quality Trend": ("lead_quality_score", "mean", None),
|
| 46 |
"Visits Trend by Industry": ("total_visits", "sum", "primary_industry"),
|
| 47 |
-
"Visits Trend by Country": ("total_visits", "sum", "country"),
|
| 48 |
-
"Visits Trend by Assignee": ("total_visits", "sum", "assignee"),
|
| 49 |
}
|
| 50 |
|
| 51 |
# =====================================================
|
| 52 |
-
#
|
| 53 |
# =====================================================
|
| 54 |
def make_request(url, params=None):
|
| 55 |
retries = 3
|
| 56 |
while retries > 0:
|
| 57 |
-
|
| 58 |
-
|
| 59 |
-
|
| 60 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 61 |
retries -= 1
|
| 62 |
-
|
| 63 |
-
|
| 64 |
-
return r.json()
|
| 65 |
-
raise Exception("Max retries exceeded")
|
| 66 |
|
| 67 |
-
# =====================================================
|
| 68 |
-
# DATA FETCHING
|
| 69 |
-
# =====================================================
|
| 70 |
def fetch_basic_leads(start_date, end_date):
|
| 71 |
-
""" Step 1: Fetch ALL company details (fast fetch). """
|
| 72 |
page = 1
|
| 73 |
rows = []
|
| 74 |
-
|
| 75 |
-
print(f"π Fetching full company list for {start_date} to {end_date}...")
|
| 76 |
|
| 77 |
while True:
|
| 78 |
try:
|
| 79 |
js = make_request(
|
| 80 |
f"{BASE_URL}/accounts/{ACCOUNT_ID}/leads",
|
| 81 |
params={
|
| 82 |
-
"start_date": start_date,
|
| 83 |
-
"
|
| 84 |
-
"
|
| 85 |
-
"page[size]": PAGE_SIZE,
|
| 86 |
-
"include": "location"
|
| 87 |
}
|
| 88 |
)
|
| 89 |
data = js.get("data", [])
|
| 90 |
-
if not data:
|
| 91 |
-
|
| 92 |
-
|
| 93 |
included = js.get("included", [])
|
| 94 |
loc_map = {str(i["id"]): i["attributes"] for i in included if i["type"] == "locations"}
|
| 95 |
|
|
@@ -102,285 +208,211 @@ def fetch_basic_leads(start_date, end_date):
|
|
| 102 |
"lead_id": lead.get("id"),
|
| 103 |
"company_name": a.get("name"),
|
| 104 |
"website_url": a.get("website_url"),
|
| 105 |
-
"phone": a.get("phone"),
|
| 106 |
-
"business_id": a.get("business_id"),
|
| 107 |
"primary_industry": a.get("industry"),
|
| 108 |
-
"all_industries": ", ".join([i.get("name") for i in a.get("industries", [])]) if a.get("industries") else None,
|
| 109 |
-
"first_visit_date": a.get("first_visit_date"),
|
| 110 |
"last_visit_date": a.get("last_visit_date"),
|
| 111 |
"total_visits": a.get("visits"),
|
| 112 |
"lead_quality_score": a.get("quality"),
|
| 113 |
-
"revenue": a.get("revenue"),
|
| 114 |
-
"employee_count": a.get("employee_count"),
|
| 115 |
-
"employees_min": a.get("employees_range", {}).get("min") if a.get("employees_range") else None,
|
| 116 |
-
"employees_max": a.get("employees_range", {}).get("max") if a.get("employees_range") else None,
|
| 117 |
-
"assignee": a.get("assignee"),
|
| 118 |
-
"emailed_to": a.get("emailed_to"),
|
| 119 |
-
"crm_lead_id": a.get("crm_lead_id"),
|
| 120 |
-
"crm_organization_id": a.get("crm_organization_id"),
|
| 121 |
-
"tags": ", ".join(a.get("tags", [])) if a.get("tags") else None,
|
| 122 |
-
"linkedin_url": a.get("linkedin_url"),
|
| 123 |
-
"twitter_handle": a.get("twitter_handle"),
|
| 124 |
-
"facebook_url": a.get("facebook_url"),
|
| 125 |
"country": loc.get("country"),
|
| 126 |
-
"region": loc.get("region"),
|
| 127 |
"city": loc.get("city"),
|
| 128 |
-
|
| 129 |
"landing_page_path": None,
|
| 130 |
"exit_page_path": None
|
| 131 |
})
|
| 132 |
-
|
| 133 |
-
print(f"β
Page {page} loaded. Rows: {len(rows)}")
|
| 134 |
page += 1
|
| 135 |
-
|
| 136 |
except Exception as e:
|
| 137 |
-
print(f"Error
|
| 138 |
break
|
| 139 |
-
|
| 140 |
df = pd.DataFrame(rows)
|
| 141 |
if not df.empty:
|
| 142 |
df["last_visit_date"] = pd.to_datetime(df["last_visit_date"], errors="coerce")
|
| 143 |
return df
|
| 144 |
|
| 145 |
def enrich_leads_with_visits(df, start_date, end_date, max_rows=None, progress=gr.Progress()):
|
| 146 |
-
|
| 147 |
-
Step 2: Deep enrich with visit_route parsing for Exit Page.
|
| 148 |
-
"""
|
| 149 |
-
if df.empty:
|
| 150 |
-
return df
|
| 151 |
-
|
| 152 |
target_df = df.head(max_rows) if max_rows else df
|
| 153 |
total = len(target_df)
|
| 154 |
|
| 155 |
-
print(f"π΅οΈ
|
| 156 |
|
| 157 |
for index, row in target_df.iterrows():
|
| 158 |
-
if
|
| 159 |
-
|
| 160 |
-
|
| 161 |
lead_id = row["lead_id"]
|
| 162 |
try:
|
|
|
|
| 163 |
visit_data = make_request(
|
| 164 |
f"{BASE_URL}/accounts/{ACCOUNT_ID}/leads/{lead_id}/visits",
|
| 165 |
-
params={
|
| 166 |
-
"start_date": start_date,
|
| 167 |
-
"end_date": end_date,
|
| 168 |
-
"page[size]": 1,
|
| 169 |
-
"include": "page_views"
|
| 170 |
-
}
|
| 171 |
)
|
| 172 |
-
|
| 173 |
visits = visit_data.get("data", [])
|
| 174 |
-
included = visit_data.get("included", [])
|
| 175 |
-
landing = None
|
| 176 |
-
exit_p = None
|
| 177 |
-
|
| 178 |
if visits:
|
| 179 |
-
|
| 180 |
-
|
| 181 |
|
| 182 |
-
#
|
| 183 |
-
|
|
|
|
|
|
|
|
|
|
| 184 |
|
| 185 |
-
# B. Parse 'visit_route'
|
| 186 |
-
visit_route = v_attrs.get("visit_route", [])
|
| 187 |
-
if visit_route and isinstance(visit_route, list):
|
| 188 |
-
last_step = visit_route[-1]
|
| 189 |
-
if not exit_p: exit_p = last_step.get("page_path") or last_step.get("page_url")
|
| 190 |
-
|
| 191 |
-
if not landing:
|
| 192 |
-
first_step = visit_route[0]
|
| 193 |
-
landing = first_step.get("page_path") or first_step.get("page_url")
|
| 194 |
-
|
| 195 |
-
# C. Deep Link via 'included' Page Views
|
| 196 |
-
if not landing or not exit_p:
|
| 197 |
-
pv_map = {p["id"]: p["attributes"] for p in included if p["type"] == "page_views"}
|
| 198 |
-
pv_ids = [r["id"] for r in visit.get("relationships", {}).get("page_views", {}).get("data", [])]
|
| 199 |
-
|
| 200 |
-
if pv_ids:
|
| 201 |
-
if not landing:
|
| 202 |
-
first_pv = pv_map.get(pv_ids[0])
|
| 203 |
-
if first_pv: landing = first_pv.get("url") or first_pv.get("path")
|
| 204 |
-
if not exit_p:
|
| 205 |
-
last_pv = pv_map.get(pv_ids[-1])
|
| 206 |
-
if last_pv: exit_p = last_pv.get("url") or last_pv.get("path")
|
| 207 |
-
|
| 208 |
df.at[index, "landing_page_path"] = landing
|
| 209 |
-
|
| 210 |
-
|
| 211 |
-
print(f"Failed to enrich lead {lead_id}: {e}")
|
| 212 |
|
| 213 |
if max_rows and index % 5 == 0:
|
| 214 |
-
progress(index / total, desc="Enriching...")
|
| 215 |
|
| 216 |
return df
|
| 217 |
|
| 218 |
# =====================================================
|
| 219 |
-
#
|
| 220 |
# =====================================================
|
| 221 |
-
def
|
| 222 |
-
|
| 223 |
-
try:
|
| 224 |
-
url = f"{BASE_URL}/accounts/{ACCOUNT_ID}/leads/{lead_id}/visits"
|
| 225 |
-
params = {
|
| 226 |
-
"start_date": start_date,
|
| 227 |
-
"end_date": end_date,
|
| 228 |
-
"page[size]": 1,
|
| 229 |
-
"include": "page_views"
|
| 230 |
-
}
|
| 231 |
-
r = requests.get(url, headers=HEADERS, params=params)
|
| 232 |
-
return json.dumps(r.json(), indent=2)
|
| 233 |
-
except Exception as e:
|
| 234 |
-
return str(e)
|
| 235 |
-
|
| 236 |
-
# =====================================================
|
| 237 |
-
# DATA WRAPPERS
|
| 238 |
-
# =====================================================
|
| 239 |
-
def load_preview(start, end):
|
| 240 |
df = fetch_basic_leads(start, end)
|
| 241 |
if df.empty: return df, pd.DataFrame(), "β οΈ No data found."
|
| 242 |
|
|
|
|
|
|
|
| 243 |
df_preview = df.copy()
|
| 244 |
-
df_preview = enrich_leads_with_visits(df_preview, start, end, max_rows=
|
|
|
|
|
|
|
|
|
|
| 245 |
|
| 246 |
-
|
| 247 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 248 |
|
| 249 |
def download_full_excel(df, start, end):
|
| 250 |
if df is None or df.empty: return None
|
| 251 |
-
print("β³
|
| 252 |
-
|
| 253 |
-
|
| 254 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 255 |
return path
|
| 256 |
|
| 257 |
# =====================================================
|
| 258 |
-
#
|
| 259 |
# =====================================================
|
| 260 |
-
def build_kpis(df):
|
| 261 |
-
if df is None or df.empty: return 0, 0, 0, 0, 0, 0
|
| 262 |
-
return (
|
| 263 |
-
len(df),
|
| 264 |
-
df["total_visits"].gt(0).sum(),
|
| 265 |
-
int(df["total_visits"].sum()),
|
| 266 |
-
round(df["lead_quality_score"].mean(), 2),
|
| 267 |
-
round(df["crm_organization_id"].notna().mean() * 100, 1),
|
| 268 |
-
round(df["linkedin_url"].notna().mean() * 100, 1),
|
| 269 |
-
)
|
| 270 |
-
|
| 271 |
def build_dashboard(df, preset, top_n):
|
| 272 |
if df is None or df.empty: return px.bar(title="No Data")
|
|
|
|
| 273 |
dim, metric, agg = DASHBOARD_PRESETS[preset]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 274 |
if agg == "count":
|
| 275 |
grouped = df.groupby(dim, dropna=False).size().reset_index(name="value")
|
| 276 |
else:
|
| 277 |
grouped = df.groupby(dim, dropna=False)[metric].agg(agg).reset_index(name="value")
|
| 278 |
-
|
| 279 |
-
|
| 280 |
-
|
| 281 |
-
if df is None or df.empty:
|
| 282 |
-
return gr.update(choices=[], value=None, visible=False)
|
| 283 |
-
metric, agg, segment = TREND_PRESETS[preset]
|
| 284 |
-
if not segment:
|
| 285 |
-
return gr.update(choices=[], value=None, visible=False)
|
| 286 |
-
options = sorted(df[segment].dropna().unique().tolist())
|
| 287 |
-
top_5 = df[segment].value_counts().head(5).index.tolist()
|
| 288 |
-
return gr.update(choices=options, value=top_5, visible=True, label=f"Filter {segment}")
|
| 289 |
|
| 290 |
-
def build_trend(df, preset, grain
|
| 291 |
-
if df is None or df.empty: return px.line(title="No Data
|
|
|
|
| 292 |
metric, agg, segment = TREND_PRESETS[preset]
|
| 293 |
-
|
| 294 |
|
| 295 |
-
if segment and filter_values:
|
| 296 |
-
df = df[df[segment].isin(filter_values)]
|
| 297 |
-
if df.empty: return px.line(title="No data for selected filters")
|
| 298 |
-
|
| 299 |
if grain == "Weekly":
|
| 300 |
-
|
| 301 |
-
elif grain == "Monthly":
|
| 302 |
-
df["period"] = df["last_visit_date"].dt.to_period("M").astype(str)
|
| 303 |
else:
|
| 304 |
-
|
| 305 |
|
| 306 |
val = metric
|
| 307 |
if agg == "count":
|
| 308 |
-
|
| 309 |
val = "_v"
|
| 310 |
|
| 311 |
-
if segment:
|
| 312 |
-
ts =
|
| 313 |
-
|
| 314 |
-
return px.line(ts, x="period", y=val, color=segment, template="plotly_white", title=title)
|
| 315 |
|
| 316 |
-
ts =
|
| 317 |
-
return px.line(ts, x="period", y=val, markers=True,
|
| 318 |
|
| 319 |
# =====================================================
|
| 320 |
-
# UI LAYOUT
|
| 321 |
# =====================================================
|
| 322 |
-
with gr.Blocks(title="Leadfeeder
|
| 323 |
-
gr.Markdown("## π Leadfeeder Analytics")
|
| 324 |
|
|
|
|
| 325 |
with gr.Row():
|
| 326 |
-
|
| 327 |
-
|
| 328 |
|
|
|
|
| 329 |
df_state = gr.State()
|
| 330 |
-
status = gr.Markdown()
|
| 331 |
|
| 332 |
with gr.Tabs():
|
| 333 |
# --- TAB 1: DATA ---
|
| 334 |
with gr.Tab("π Data & Report"):
|
| 335 |
with gr.Row():
|
| 336 |
-
|
| 337 |
-
|
| 338 |
|
| 339 |
with gr.Row():
|
| 340 |
-
btn_load = gr.Button("1. Load Data
|
| 341 |
-
btn_dl = gr.Button("2.
|
| 342 |
-
|
| 343 |
|
| 344 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 345 |
|
| 346 |
-
btn_load.click(load_preview, [start, end], [df_state, table, status])
|
| 347 |
-
btn_dl.click(download_full_excel, [df_state, start, end], file_dl)
|
| 348 |
-
|
| 349 |
# --- TAB 2: DASHBOARD ---
|
| 350 |
with gr.Tab("π Dashboard"):
|
| 351 |
-
kpis = [gr.Number(label=l) for l in ["Total Companies", "Active Companies", "Total Visits", "Avg Quality", "CRM %", "LinkedIn %"]]
|
| 352 |
-
gr.Button("Refresh KPIs").click(build_kpis, df_state, kpis)
|
| 353 |
-
|
| 354 |
with gr.Row():
|
| 355 |
-
|
| 356 |
-
top_n = gr.Slider(5, 50,
|
|
|
|
|
|
|
| 357 |
|
| 358 |
-
chart = gr.Plot()
|
| 359 |
-
gr.Button("Build View").click(build_dashboard, [df_state, preset, top_n], chart)
|
| 360 |
-
|
| 361 |
# --- TAB 3: TRENDS ---
|
| 362 |
with gr.Tab("π Trends"):
|
| 363 |
with gr.Row():
|
| 364 |
-
|
| 365 |
-
|
|
|
|
|
|
|
| 366 |
|
| 367 |
-
|
| 368 |
-
|
|
|
|
|
|
|
| 369 |
|
| 370 |
-
|
| 371 |
-
|
| 372 |
-
|
| 373 |
-
|
| 374 |
-
|
| 375 |
-
)
|
| 376 |
-
|
| 377 |
-
|
| 378 |
-
with gr.Tab("π οΈ Debugger"):
|
| 379 |
-
gr.Markdown("Inspect raw API data for a specific lead.")
|
| 380 |
-
dbg_id = gr.Textbox(label="Lead ID")
|
| 381 |
-
dbg_btn = gr.Button("Inspect Raw JSON")
|
| 382 |
-
dbg_out = gr.Code(language="json")
|
| 383 |
-
dbg_btn.click(inspect_raw_json, [dbg_id, start, end], dbg_out)
|
| 384 |
|
| 385 |
if __name__ == "__main__":
|
| 386 |
demo.launch(server_name="0.0.0.0", server_port=7860)
|
|
|
|
| 13 |
API_TOKEN = os.getenv("LEADFEEDER_API_TOKEN")
|
| 14 |
APP_PASSWORD = os.getenv("APP_PASSWORD")
|
| 15 |
|
| 16 |
+
# If testing locally without env vars, uncomment and set manually:
|
| 17 |
+
# API_TOKEN = "your_token_here"
|
| 18 |
+
# APP_PASSWORD = "password"
|
| 19 |
+
|
| 20 |
if not API_TOKEN or not APP_PASSWORD:
|
| 21 |
+
# Fallback for safe run if secrets missing (prevents crash on immediate launch)
|
| 22 |
+
print("β οΈ WARNING: Secrets missing. App will launch but API calls will fail.")
|
| 23 |
|
| 24 |
+
ACCOUNT_ID = "255333"
|
| 25 |
BASE_URL = "https://api.leadfeeder.com"
|
| 26 |
PAGE_SIZE = 100
|
|
|
|
| 27 |
HEADERS = {
|
| 28 |
"Authorization": f"Token token={API_TOKEN}",
|
| 29 |
"Accept": "application/json"
|
| 30 |
}
|
| 31 |
+
CONFIG_FILE = "campaign_rules.json"
|
| 32 |
+
|
| 33 |
+
# =====================================================
|
| 34 |
+
# 1. CAMPAIGN & CONFIG MANAGEMENT
|
| 35 |
+
# =====================================================
|
| 36 |
+
DEFAULT_CAMPAIGNS = {
|
| 37 |
+
"MS Stack": [
|
| 38 |
+
"microsoft", "power", "dynamics", "crm", "365", "xamarin", "sharepoint",
|
| 39 |
+
"dot", ".net", "asp.net", "azure", "copilot", "real-estate-website-development",
|
| 40 |
+
"manufacturing-production", "legacy-hrms", "healthcare-appointment", "eld-fleet",
|
| 41 |
+
"legacy-erp", "clinical-trial", "digital-e-learning", "bi-analytics",
|
| 42 |
+
"application-modernization", "data-warehouse", "workforce", "contingent"
|
| 43 |
+
],
|
| 44 |
+
"Fintech": [
|
| 45 |
+
"fintech", "finance", "money", "credit", "debit", "card", "invest", "bank",
|
| 46 |
+
"wallet", "account", "bookkeeping", "payment", "loan", "nfc", "lend", "debt",
|
| 47 |
+
"tax", "invoice", "quot", "borrow", "bill", "coin", "currency", "aml", "kyc",
|
| 48 |
+
"blockchain", "cryptocurrency", "budget", "expense", "wealth", "sox", "sarbanes"
|
| 49 |
+
],
|
| 50 |
+
"Adtech": [
|
| 51 |
+
"programmatic", "publisher", "media", "entertainment", "trading", "ott",
|
| 52 |
+
"email", "news", "social", "marketing", "video", "audio", "radio", "livestream",
|
| 53 |
+
"bid", "anime", "dooh", "vod", "rtb", "supply-side-platform", "demand-side-platform",
|
| 54 |
+
"data-management-platform"
|
| 55 |
+
]
|
| 56 |
+
}
|
| 57 |
+
|
| 58 |
+
def load_campaign_rules():
|
| 59 |
+
"""Load rules from JSON or fallback to defaults."""
|
| 60 |
+
if os.path.exists(CONFIG_FILE):
|
| 61 |
+
try:
|
| 62 |
+
with open(CONFIG_FILE, "r") as f:
|
| 63 |
+
return json.load(f)
|
| 64 |
+
except:
|
| 65 |
+
return DEFAULT_CAMPAIGNS
|
| 66 |
+
return DEFAULT_CAMPAIGNS
|
| 67 |
+
|
| 68 |
+
def save_campaign_rules(rules):
|
| 69 |
+
"""Save rules to JSON."""
|
| 70 |
+
with open(CONFIG_FILE, "w") as f:
|
| 71 |
+
json.dump(rules, f, indent=4)
|
| 72 |
+
return "β
Settings Saved!"
|
| 73 |
+
|
| 74 |
+
def get_rules_text(rules):
|
| 75 |
+
"""Convert dict to readable text for UI."""
|
| 76 |
+
return json.dumps(rules, indent=2)
|
| 77 |
+
|
| 78 |
+
def update_rules_from_text(text):
|
| 79 |
+
"""Parse text back to dict and save."""
|
| 80 |
+
try:
|
| 81 |
+
new_rules = json.loads(text)
|
| 82 |
+
save_campaign_rules(new_rules)
|
| 83 |
+
return "β
Configuration updated successfully!", new_rules
|
| 84 |
+
except json.JSONDecodeError:
|
| 85 |
+
return "β Error: Invalid JSON format. Please check syntax.", None
|
| 86 |
+
|
| 87 |
+
# =====================================================
|
| 88 |
+
# 2. DATA PROCESSING LOGIC
|
| 89 |
+
# =====================================================
|
| 90 |
+
def categorize_quality(score):
|
| 91 |
+
if pd.isna(score): return "Unknown"
|
| 92 |
+
s = int(score)
|
| 93 |
+
if 8 <= s <= 10: return "High Quality (8-10)"
|
| 94 |
+
if 5 <= s <= 7: return "Mid Quality (5-7)"
|
| 95 |
+
if 1 <= s <= 4: return "Low Quality (1-4)"
|
| 96 |
+
return "Low Quality (0)"
|
| 97 |
+
|
| 98 |
+
def assign_campaign(url, rules):
|
| 99 |
+
if not url or not isinstance(url, str):
|
| 100 |
+
return "Uncategorized"
|
| 101 |
+
|
| 102 |
+
url_lower = url.lower()
|
| 103 |
+
# Check each campaign group
|
| 104 |
+
for campaign_name, keywords in rules.items():
|
| 105 |
+
for kw in keywords:
|
| 106 |
+
if kw.lower() in url_lower:
|
| 107 |
+
return campaign_name
|
| 108 |
+
return "Other"
|
| 109 |
+
|
| 110 |
+
def process_business_logic(df):
|
| 111 |
+
"""Apply Quality Grouping and Campaign Logic to the DataFrame."""
|
| 112 |
+
if df.empty: return df
|
| 113 |
+
|
| 114 |
+
# 1. Load Rules
|
| 115 |
+
rules = load_campaign_rules()
|
| 116 |
+
|
| 117 |
+
# 2. Apply Quality Logic
|
| 118 |
+
df["Quality Group"] = df["lead_quality_score"].apply(categorize_quality)
|
| 119 |
+
|
| 120 |
+
# 3. Apply Campaign Logic (Check Landing Page mostly)
|
| 121 |
+
# We prioritize landing_page_path, fallback to first_visit_url if available?
|
| 122 |
+
# For now, strictly landing_page_path or blank.
|
| 123 |
+
def get_campaign(row):
|
| 124 |
+
# Check landing page
|
| 125 |
+
path = row.get("landing_page_path")
|
| 126 |
+
res = assign_campaign(path, rules)
|
| 127 |
+
if res != "Other": return res
|
| 128 |
+
|
| 129 |
+
# Optional: Check all industries if landing page fails?
|
| 130 |
+
# ind = row.get("all_industries", "")
|
| 131 |
+
# if ind: return assign_campaign(ind, rules)
|
| 132 |
+
|
| 133 |
+
return "Other"
|
| 134 |
+
|
| 135 |
+
df["Campaign"] = df.apply(get_campaign, axis=1)
|
| 136 |
+
return df
|
| 137 |
|
| 138 |
# =====================================================
|
| 139 |
+
# 3. PRESETS (UPDATED)
|
| 140 |
# =====================================================
|
| 141 |
DASHBOARD_PRESETS = {
|
| 142 |
+
"Total Visits by Campaign": ("Campaign", "total_visits", "sum"),
|
| 143 |
"Total Visits by Industry": ("primary_industry", "total_visits", "sum"),
|
| 144 |
+
"Lead Quality Breakdown": ("Quality Group", "company_name", "count"),
|
| 145 |
+
"Campaign Performance (Leads)": ("Campaign", "company_name", "count"),
|
| 146 |
"Total Visits by Country": ("country", "total_visits", "sum"),
|
| 147 |
"Avg Quality by Industry": ("primary_industry", "lead_quality_score", "mean"),
|
| 148 |
"Top Accounts by Visits": ("company_name", "total_visits", "sum"),
|
|
|
|
|
|
|
|
|
|
|
|
|
| 149 |
}
|
| 150 |
|
| 151 |
TREND_PRESETS = {
|
| 152 |
"Total Visits Trend": ("total_visits", "sum", None),
|
| 153 |
+
"Visits Trend by Campaign": ("total_visits", "sum", "Campaign"),
|
| 154 |
+
"Visits Trend by Quality": ("total_visits", "sum", "Quality Group"),
|
| 155 |
"Active Accounts Trend": ("company_name", "count", None),
|
| 156 |
"Avg Lead Quality Trend": ("lead_quality_score", "mean", None),
|
| 157 |
"Visits Trend by Industry": ("total_visits", "sum", "primary_industry"),
|
|
|
|
|
|
|
| 158 |
}
|
| 159 |
|
| 160 |
# =====================================================
|
| 161 |
+
# 4. API & FETCHER
|
| 162 |
# =====================================================
|
| 163 |
def make_request(url, params=None):
|
| 164 |
retries = 3
|
| 165 |
while retries > 0:
|
| 166 |
+
try:
|
| 167 |
+
r = requests.get(url, headers=HEADERS, params=params, timeout=45)
|
| 168 |
+
if r.status_code == 429:
|
| 169 |
+
print("β³ Rate limit hit. Sleeping for 60 seconds...")
|
| 170 |
+
time.sleep(61)
|
| 171 |
+
retries -= 1
|
| 172 |
+
continue
|
| 173 |
+
r.raise_for_status()
|
| 174 |
+
return r.json()
|
| 175 |
+
except Exception as e:
|
| 176 |
+
print(f"Request failed: {e}")
|
| 177 |
retries -= 1
|
| 178 |
+
time.sleep(2)
|
| 179 |
+
raise Exception("Max retries exceeded or API Error")
|
|
|
|
|
|
|
| 180 |
|
|
|
|
|
|
|
|
|
|
| 181 |
def fetch_basic_leads(start_date, end_date):
|
|
|
|
| 182 |
page = 1
|
| 183 |
rows = []
|
| 184 |
+
print(f"π Fetching leads {start_date} to {end_date}...")
|
|
|
|
| 185 |
|
| 186 |
while True:
|
| 187 |
try:
|
| 188 |
js = make_request(
|
| 189 |
f"{BASE_URL}/accounts/{ACCOUNT_ID}/leads",
|
| 190 |
params={
|
| 191 |
+
"start_date": start_date, "end_date": end_date,
|
| 192 |
+
"page[number]": page, "page[size]": PAGE_SIZE,
|
| 193 |
+
"include": "location"
|
|
|
|
|
|
|
| 194 |
}
|
| 195 |
)
|
| 196 |
data = js.get("data", [])
|
| 197 |
+
if not data: break
|
| 198 |
+
|
|
|
|
| 199 |
included = js.get("included", [])
|
| 200 |
loc_map = {str(i["id"]): i["attributes"] for i in included if i["type"] == "locations"}
|
| 201 |
|
|
|
|
| 208 |
"lead_id": lead.get("id"),
|
| 209 |
"company_name": a.get("name"),
|
| 210 |
"website_url": a.get("website_url"),
|
|
|
|
|
|
|
| 211 |
"primary_industry": a.get("industry"),
|
|
|
|
|
|
|
| 212 |
"last_visit_date": a.get("last_visit_date"),
|
| 213 |
"total_visits": a.get("visits"),
|
| 214 |
"lead_quality_score": a.get("quality"),
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 215 |
"country": loc.get("country"),
|
|
|
|
| 216 |
"city": loc.get("city"),
|
| 217 |
+
# Init fields for enrichment
|
| 218 |
"landing_page_path": None,
|
| 219 |
"exit_page_path": None
|
| 220 |
})
|
|
|
|
|
|
|
| 221 |
page += 1
|
|
|
|
| 222 |
except Exception as e:
|
| 223 |
+
print(f"Error page {page}: {e}")
|
| 224 |
break
|
| 225 |
+
|
| 226 |
df = pd.DataFrame(rows)
|
| 227 |
if not df.empty:
|
| 228 |
df["last_visit_date"] = pd.to_datetime(df["last_visit_date"], errors="coerce")
|
| 229 |
return df
|
| 230 |
|
| 231 |
def enrich_leads_with_visits(df, start_date, end_date, max_rows=None, progress=gr.Progress()):
|
| 232 |
+
if df.empty: return df
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 233 |
target_df = df.head(max_rows) if max_rows else df
|
| 234 |
total = len(target_df)
|
| 235 |
|
| 236 |
+
print(f"π΅οΈ Enriching {total} rows...")
|
| 237 |
|
| 238 |
for index, row in target_df.iterrows():
|
| 239 |
+
# Skip if already has landing page
|
| 240 |
+
if row.get("landing_page_path"): continue
|
| 241 |
+
|
| 242 |
lead_id = row["lead_id"]
|
| 243 |
try:
|
| 244 |
+
# We fetch 1 visit to get landing page
|
| 245 |
visit_data = make_request(
|
| 246 |
f"{BASE_URL}/accounts/{ACCOUNT_ID}/leads/{lead_id}/visits",
|
| 247 |
+
params={"start_date": start_date, "end_date": end_date, "page[size]": 1}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 248 |
)
|
|
|
|
| 249 |
visits = visit_data.get("data", [])
|
|
|
|
|
|
|
|
|
|
|
|
|
| 250 |
if visits:
|
| 251 |
+
attrs = visits[0].get("attributes", {})
|
| 252 |
+
landing = attrs.get("landing_page_path") or attrs.get("landing_page_url")
|
| 253 |
|
| 254 |
+
# Fallback to visit route if top level attr is missing
|
| 255 |
+
if not landing:
|
| 256 |
+
route = attrs.get("visit_route", [])
|
| 257 |
+
if route:
|
| 258 |
+
landing = route[0].get("page_path")
|
| 259 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 260 |
df.at[index, "landing_page_path"] = landing
|
| 261 |
+
except:
|
| 262 |
+
pass
|
|
|
|
| 263 |
|
| 264 |
if max_rows and index % 5 == 0:
|
| 265 |
+
progress(index / total, desc="Enriching Visits...")
|
| 266 |
|
| 267 |
return df
|
| 268 |
|
| 269 |
# =====================================================
|
| 270 |
+
# 5. WRAPPERS
|
| 271 |
# =====================================================
|
| 272 |
+
def load_and_process(start, end):
|
| 273 |
+
# 1. Fetch Basic
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 274 |
df = fetch_basic_leads(start, end)
|
| 275 |
if df.empty: return df, pd.DataFrame(), "β οΈ No data found."
|
| 276 |
|
| 277 |
+
# 2. Enrich (Preview 20 rows for speed in UI)
|
| 278 |
+
# We MUST enrich landing pages to get Campaigns working
|
| 279 |
df_preview = df.copy()
|
| 280 |
+
df_preview = enrich_leads_with_visits(df_preview, start, end, max_rows=20)
|
| 281 |
+
|
| 282 |
+
# 3. Apply Business Logic (Campaigns + Quality)
|
| 283 |
+
df_preview = process_business_logic(df_preview)
|
| 284 |
|
| 285 |
+
# Note: Main DF is NOT enriched yet (too slow), so Campaign column will be empty/Other
|
| 286 |
+
# until user clicks "Download" or we force enrichment.
|
| 287 |
+
# For better UI, we can try to enrich ALL if size < 200, else warn.
|
| 288 |
+
if len(df) < 300:
|
| 289 |
+
df = enrich_leads_with_visits(df, start, end) # Enrich all if small
|
| 290 |
+
df = process_business_logic(df)
|
| 291 |
+
status = f"β
Loaded & Enriched {len(df)} companies."
|
| 292 |
+
else:
|
| 293 |
+
# Just process logic on what we have (mostly empty landing pages)
|
| 294 |
+
df = process_business_logic(df)
|
| 295 |
+
status = f"β
Loaded {len(df)} companies. (Preview enriched only. Download for full data)."
|
| 296 |
+
|
| 297 |
+
return df, df_preview.head(20), status
|
| 298 |
|
| 299 |
def download_full_excel(df, start, end):
|
| 300 |
if df is None or df.empty: return None
|
| 301 |
+
print("β³ Full Enrichment for Export...")
|
| 302 |
+
|
| 303 |
+
# Ensure all rows have visits
|
| 304 |
+
enriched = enrich_leads_with_visits(df.copy(), start, end)
|
| 305 |
+
# Re-apply logic now that we have landing pages
|
| 306 |
+
final_df = process_business_logic(enriched)
|
| 307 |
+
|
| 308 |
+
path = "/tmp/leadfeeder_campaign_data.xlsx"
|
| 309 |
+
final_df.to_excel(path, index=False)
|
| 310 |
return path
|
| 311 |
|
| 312 |
# =====================================================
|
| 313 |
+
# 6. CHARTING
|
| 314 |
# =====================================================
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 315 |
def build_dashboard(df, preset, top_n):
|
| 316 |
if df is None or df.empty: return px.bar(title="No Data")
|
| 317 |
+
|
| 318 |
dim, metric, agg = DASHBOARD_PRESETS[preset]
|
| 319 |
+
|
| 320 |
+
# Check if column exists (Campaign might be missing if not enriched)
|
| 321 |
+
if dim not in df.columns:
|
| 322 |
+
return px.bar(title=f"Column '{dim}' not available. Try Downloading Full Data first.")
|
| 323 |
+
|
| 324 |
if agg == "count":
|
| 325 |
grouped = df.groupby(dim, dropna=False).size().reset_index(name="value")
|
| 326 |
else:
|
| 327 |
grouped = df.groupby(dim, dropna=False)[metric].agg(agg).reset_index(name="value")
|
| 328 |
+
|
| 329 |
+
fig = px.bar(grouped.sort_values("value", ascending=False).head(top_n), x=dim, y="value", title=preset, color=dim)
|
| 330 |
+
return fig
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 331 |
|
| 332 |
+
def build_trend(df, preset, grain):
|
| 333 |
+
if df is None or df.empty: return px.line(title="No Data")
|
| 334 |
+
|
| 335 |
metric, agg, segment = TREND_PRESETS[preset]
|
| 336 |
+
df_t = df.dropna(subset=["last_visit_date"]).copy()
|
| 337 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 338 |
if grain == "Weekly":
|
| 339 |
+
df_t["period"] = df_t["last_visit_date"].dt.to_period("W").astype(str)
|
|
|
|
|
|
|
| 340 |
else:
|
| 341 |
+
df_t["period"] = df_t["last_visit_date"].dt.date
|
| 342 |
|
| 343 |
val = metric
|
| 344 |
if agg == "count":
|
| 345 |
+
df_t["_v"] = 1
|
| 346 |
val = "_v"
|
| 347 |
|
| 348 |
+
if segment and segment in df_t.columns:
|
| 349 |
+
ts = df_t.groupby(["period", segment])[val].agg(agg).reset_index()
|
| 350 |
+
return px.line(ts, x="period", y=val, color=segment, title=preset)
|
|
|
|
| 351 |
|
| 352 |
+
ts = df_t.groupby("period")[val].agg(agg).reset_index()
|
| 353 |
+
return px.line(ts, x="period", y=val, markers=True, title=preset)
|
| 354 |
|
| 355 |
# =====================================================
|
| 356 |
+
# 7. UI LAYOUT
|
| 357 |
# =====================================================
|
| 358 |
+
with gr.Blocks(title="Leadfeeder Campaign Analytics") as demo:
|
| 359 |
+
gr.Markdown("## π Leadfeeder Analytics + Campaign Manager")
|
| 360 |
|
| 361 |
+
# Auth
|
| 362 |
with gr.Row():
|
| 363 |
+
pwd_input = gr.Textbox(type="password", label="App Password")
|
| 364 |
+
auth_btn = gr.Button("Login")
|
| 365 |
|
| 366 |
+
# State
|
| 367 |
df_state = gr.State()
|
|
|
|
| 368 |
|
| 369 |
with gr.Tabs():
|
| 370 |
# --- TAB 1: DATA ---
|
| 371 |
with gr.Tab("π Data & Report"):
|
| 372 |
with gr.Row():
|
| 373 |
+
start_dt = gr.Textbox(label="Start Date", value=(date.today()-timedelta(days=30)).isoformat())
|
| 374 |
+
end_dt = gr.Textbox(label="End Date", value=date.today().isoformat())
|
| 375 |
|
| 376 |
with gr.Row():
|
| 377 |
+
btn_load = gr.Button("1. Load Data", variant="primary")
|
| 378 |
+
btn_dl = gr.Button("2. Download Excel (Full Analysis)")
|
| 379 |
+
file_out = gr.File(label="Exported File")
|
| 380 |
|
| 381 |
+
status_txt = gr.Markdown("Ready.")
|
| 382 |
+
table = gr.Dataframe(label="Preview Data", interactive=True)
|
| 383 |
+
|
| 384 |
+
btn_load.click(load_and_process, [start_dt, end_dt], [df_state, table, status_txt])
|
| 385 |
+
btn_dl.click(download_full_excel, [df_state, start_dt, end_dt], file_out)
|
| 386 |
|
|
|
|
|
|
|
|
|
|
| 387 |
# --- TAB 2: DASHBOARD ---
|
| 388 |
with gr.Tab("π Dashboard"):
|
|
|
|
|
|
|
|
|
|
| 389 |
with gr.Row():
|
| 390 |
+
dash_preset = gr.Dropdown(list(DASHBOARD_PRESETS.keys()), label="Chart Type", value="Total Visits by Campaign")
|
| 391 |
+
top_n = gr.Slider(5, 50, 10, label="Top N Items")
|
| 392 |
+
dash_plot = gr.Plot()
|
| 393 |
+
gr.Button("Refresh Chart").click(build_dashboard, [df_state, dash_preset, top_n], dash_plot)
|
| 394 |
|
|
|
|
|
|
|
|
|
|
| 395 |
# --- TAB 3: TRENDS ---
|
| 396 |
with gr.Tab("π Trends"):
|
| 397 |
with gr.Row():
|
| 398 |
+
trend_preset = gr.Dropdown(list(TREND_PRESETS.keys()), label="Trend Type", value="Visits Trend by Campaign")
|
| 399 |
+
grain_opt = gr.Radio(["Daily", "Weekly"], value="Daily", label="Granularity")
|
| 400 |
+
trend_plot = gr.Plot()
|
| 401 |
+
gr.Button("Show Trend").click(build_trend, [df_state, trend_preset, grain_opt], trend_plot)
|
| 402 |
|
| 403 |
+
# --- TAB 4: SETTINGS (CAMPAIGN MANAGER) ---
|
| 404 |
+
with gr.Tab("βοΈ Campaign Settings"):
|
| 405 |
+
gr.Markdown("### Manage Campaign Keywords")
|
| 406 |
+
gr.Markdown("Edit the JSON structure below to add/remove campaigns or keywords. Changes apply on next 'Load Data'.")
|
| 407 |
|
| 408 |
+
# Load initial settings
|
| 409 |
+
init_rules = get_rules_text(load_campaign_rules())
|
| 410 |
+
|
| 411 |
+
json_editor = gr.Code(value=init_rules, language="json", label="Campaign Configuration (JSON)")
|
| 412 |
+
save_btn = gr.Button("πΎ Save Configuration", variant="primary")
|
| 413 |
+
config_status = gr.Markdown()
|
| 414 |
+
|
| 415 |
+
save_btn.click(update_rules_from_text, json_editor, [config_status, gr.State()])
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 416 |
|
| 417 |
if __name__ == "__main__":
|
| 418 |
demo.launch(server_name="0.0.0.0", server_port=7860)
|