Spaces:
Running
Running
Add IND Admissions & KYC Master DB (/api/ind) with phone & email range lookups and update SBI master
Browse files- engine.py +66 -0
- ind_partitions.json +16 -0
- main.py +24 -1
- static/index.html +21 -0
engine.py
CHANGED
|
@@ -72,6 +72,7 @@ class LookupEngine:
|
|
| 72 |
self.twitter_partitions = _load_index("twitter_index.json", [])
|
| 73 |
self.ig_phone_partitions = _load_index("instagram_phone_index.json", [])
|
| 74 |
self.ig_user_partitions = _load_index("instagram_username_index.json", [])
|
|
|
|
| 75 |
|
| 76 |
logger.info(
|
| 77 |
f"Loaded Master Index Routers: "
|
|
@@ -85,6 +86,7 @@ class LookupEngine:
|
|
| 85 |
f"Twitter({len(self.twitter_partitions)}), "
|
| 86 |
f"InstagramPhone({len(self.ig_phone_partitions)}), "
|
| 87 |
f"InstagramUser({len(self.ig_user_partitions)}), "
|
|
|
|
| 88 |
f"TelecomPhone({len(self.partitions.get('phone', []))})"
|
| 89 |
)
|
| 90 |
|
|
@@ -130,6 +132,8 @@ class LookupEngine:
|
|
| 130 |
warm_targets.append((self.hdfc_partitions[0]["url"], "mobile"))
|
| 131 |
if self.tg_partitions:
|
| 132 |
warm_targets.append((self.tg_partitions[0]["url"], "phone"))
|
|
|
|
|
|
|
| 133 |
|
| 134 |
for url, col in warm_targets:
|
| 135 |
try:
|
|
@@ -930,6 +934,60 @@ class LookupEngine:
|
|
| 930 |
self.cache[cache_key] = res
|
| 931 |
return res
|
| 932 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 933 |
# ------------------------------------------------------------------
|
| 934 |
# 🌟 11. UNIFIED COMBINED SEARCH ACROSS ALL 10 MASTER DATABASES
|
| 935 |
# ------------------------------------------------------------------
|
|
@@ -964,6 +1022,7 @@ class LookupEngine:
|
|
| 964 |
"airtel": self.executor.submit(self.query_airtel, mobile),
|
| 965 |
"hdfc": self.executor.submit(self.query_hdfc, mobile),
|
| 966 |
"sbi": self.executor.submit(self.query_sbi, str(mobile)),
|
|
|
|
| 967 |
"bsnl": self.executor.submit(self.query_bsnl, mobile),
|
| 968 |
"paytm": self.executor.submit(self.query_paytm, mobile),
|
| 969 |
"flipkart": self.executor.submit(self.query_flipkart, mobile),
|
|
@@ -1059,6 +1118,13 @@ class LookupEngine:
|
|
| 1059 |
loc = ", ".join(filter(None, [r.get("address"), r.get("city"), r.get("state"), r.get("pincode")]))
|
| 1060 |
_add_unique(locations, loc)
|
| 1061 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1062 |
# Calculate hits count
|
| 1063 |
hits = 0
|
| 1064 |
for src, data in results.items():
|
|
|
|
| 72 |
self.twitter_partitions = _load_index("twitter_index.json", [])
|
| 73 |
self.ig_phone_partitions = _load_index("instagram_phone_index.json", [])
|
| 74 |
self.ig_user_partitions = _load_index("instagram_username_index.json", [])
|
| 75 |
+
self.ind_partitions = _load_index("ind_partitions.json", [])
|
| 76 |
|
| 77 |
logger.info(
|
| 78 |
f"Loaded Master Index Routers: "
|
|
|
|
| 86 |
f"Twitter({len(self.twitter_partitions)}), "
|
| 87 |
f"InstagramPhone({len(self.ig_phone_partitions)}), "
|
| 88 |
f"InstagramUser({len(self.ig_user_partitions)}), "
|
| 89 |
+
f"IND({len(self.ind_partitions)}), "
|
| 90 |
f"TelecomPhone({len(self.partitions.get('phone', []))})"
|
| 91 |
)
|
| 92 |
|
|
|
|
| 132 |
warm_targets.append((self.hdfc_partitions[0]["url"], "mobile"))
|
| 133 |
if self.tg_partitions:
|
| 134 |
warm_targets.append((self.tg_partitions[0]["url"], "phone"))
|
| 135 |
+
if self.ind_partitions:
|
| 136 |
+
warm_targets.append((self.ind_partitions[0]["url"], "phone"))
|
| 137 |
|
| 138 |
for url, col in warm_targets:
|
| 139 |
try:
|
|
|
|
| 934 |
self.cache[cache_key] = res
|
| 935 |
return res
|
| 936 |
|
| 937 |
+
# ------------------------------------------------------------------
|
| 938 |
+
# 10d. IND Education, Admissions & KYC Master Database
|
| 939 |
+
# ------------------------------------------------------------------
|
| 940 |
+
def query_ind(self, query_input: Any) -> Dict[str, Any]:
|
| 941 |
+
t0 = time.time()
|
| 942 |
+
raw_val = str(query_input).strip()
|
| 943 |
+
clean_phone = self.sanitize_phone(raw_val)
|
| 944 |
+
|
| 945 |
+
cache_key = f"ind:{clean_phone or raw_val.lower()}"
|
| 946 |
+
with self.cache_lock:
|
| 947 |
+
if cache_key in self.cache:
|
| 948 |
+
return {**self.cache[cache_key], "cached": True, "latency_ms": round((time.time() - t0) * 1000, 2)}
|
| 949 |
+
|
| 950 |
+
cols = [
|
| 951 |
+
"phone", "email", "name", "father_name", "mother_name", "dob",
|
| 952 |
+
"aadhaar", "pan", "address", "city", "state", "pincode",
|
| 953 |
+
"college_name", "bank_name", "account_number", "ifsc_code",
|
| 954 |
+
"role", "source"
|
| 955 |
+
]
|
| 956 |
+
|
| 957 |
+
records = []
|
| 958 |
+
if clean_phone:
|
| 959 |
+
phone_int = int(clean_phone)
|
| 960 |
+
partition = self._route_range(self.ind_partitions, phone_int, "min_id", "max_id")
|
| 961 |
+
if partition:
|
| 962 |
+
q = f"SELECT {', '.join(cols)} FROM '{partition['url']}' WHERE phone = {phone_int} LIMIT 10"
|
| 963 |
+
records = self._execute_query(q, cols)
|
| 964 |
+
else:
|
| 965 |
+
urls = [p["url"] for p in self.ind_partitions] if self.ind_partitions else []
|
| 966 |
+
if urls:
|
| 967 |
+
url_list_str = "[" + ", ".join([f"'{u}'" for u in urls]) + "]"
|
| 968 |
+
if len(raw_val) == 12 and raw_val.isdigit():
|
| 969 |
+
q = f"SELECT {', '.join(cols)} FROM read_parquet({url_list_str}) WHERE aadhaar = '{raw_val}' LIMIT 10"
|
| 970 |
+
elif len(raw_val) == 10 and raw_val.isalnum() and not raw_val.isdigit():
|
| 971 |
+
q = f"SELECT {', '.join(cols)} FROM read_parquet({url_list_str}) WHERE UPPER(pan) = '{raw_val.upper()}' LIMIT 10"
|
| 972 |
+
elif "@" in raw_val:
|
| 973 |
+
q = f"SELECT {', '.join(cols)} FROM read_parquet({url_list_str}) WHERE LOWER(email) = '{raw_val.lower()}' LIMIT 10"
|
| 974 |
+
else:
|
| 975 |
+
q = f"SELECT {', '.join(cols)} FROM read_parquet({url_list_str}) WHERE LOWER(name) LIKE '%{raw_val.lower()}%' LIMIT 10"
|
| 976 |
+
records = self._execute_query(q, cols)
|
| 977 |
+
|
| 978 |
+
res = {
|
| 979 |
+
"status": "success" if records else "not_found",
|
| 980 |
+
"database": "IND Admissions & KYC Master",
|
| 981 |
+
"query": raw_val,
|
| 982 |
+
"count": len(records),
|
| 983 |
+
"records": records,
|
| 984 |
+
"latency_ms": round((time.time() - t0) * 1000, 2),
|
| 985 |
+
"cached": False
|
| 986 |
+
}
|
| 987 |
+
with self.cache_lock:
|
| 988 |
+
self.cache[cache_key] = res
|
| 989 |
+
return res
|
| 990 |
+
|
| 991 |
# ------------------------------------------------------------------
|
| 992 |
# 🌟 11. UNIFIED COMBINED SEARCH ACROSS ALL 10 MASTER DATABASES
|
| 993 |
# ------------------------------------------------------------------
|
|
|
|
| 1022 |
"airtel": self.executor.submit(self.query_airtel, mobile),
|
| 1023 |
"hdfc": self.executor.submit(self.query_hdfc, mobile),
|
| 1024 |
"sbi": self.executor.submit(self.query_sbi, str(mobile)),
|
| 1025 |
+
"ind": self.executor.submit(self.query_ind, str(mobile)),
|
| 1026 |
"bsnl": self.executor.submit(self.query_bsnl, mobile),
|
| 1027 |
"paytm": self.executor.submit(self.query_paytm, mobile),
|
| 1028 |
"flipkart": self.executor.submit(self.query_flipkart, mobile),
|
|
|
|
| 1118 |
loc = ", ".join(filter(None, [r.get("address"), r.get("city"), r.get("state"), r.get("pincode")]))
|
| 1119 |
_add_unique(locations, loc)
|
| 1120 |
|
| 1121 |
+
# 10. Parse IND Admissions & Education Master DB
|
| 1122 |
+
for r in results.get("ind", {}).get("records", []):
|
| 1123 |
+
_add_unique(names, r.get("name"))
|
| 1124 |
+
_add_unique(emails, r.get("email"))
|
| 1125 |
+
loc = ", ".join(filter(None, [r.get("address"), r.get("city"), r.get("state"), r.get("pincode")]))
|
| 1126 |
+
_add_unique(locations, loc)
|
| 1127 |
+
|
| 1128 |
# Calculate hits count
|
| 1129 |
hits = 0
|
| 1130 |
for src, data in results.items():
|
ind_partitions.json
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[
|
| 2 |
+
{
|
| 3 |
+
"part": 0,
|
| 4 |
+
"min_id": 6000000000,
|
| 5 |
+
"max_id": 9699907928,
|
| 6 |
+
"file": "part_0000_6000000000_9699907928.parquet",
|
| 7 |
+
"url": "hf://datasets/ItsXyron/IND-Master-DB/data/part_0000_6000000000_9699907928.parquet"
|
| 8 |
+
},
|
| 9 |
+
{
|
| 10 |
+
"part": 1,
|
| 11 |
+
"min_id": 9699907942,
|
| 12 |
+
"max_id": 9999999999,
|
| 13 |
+
"file": "part_0001_9699907942_9999999999.parquet",
|
| 14 |
+
"url": "hf://datasets/ItsXyron/IND-Master-DB/data/part_0001_9699907942_9999999999.parquet"
|
| 15 |
+
}
|
| 16 |
+
]
|
main.py
CHANGED
|
@@ -320,6 +320,26 @@ async def lookup_sbi(
|
|
| 320 |
return res
|
| 321 |
|
| 322 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 323 |
@app.get("/api/phone/{phone_number}")
|
| 324 |
async def lookup_by_phone(phone_number: str, auth_meta: Dict[str, Any] = Depends(verify_api_key)):
|
| 325 |
"""Search Telecom 2.5B Database by phone number."""
|
|
@@ -403,6 +423,7 @@ async def unified_lookup(
|
|
| 403 |
family: Optional[str] = Query(None, description="Direct Aadhaar family query"),
|
| 404 |
user_id: Optional[str] = Query(None, description="Direct Telegram User ID"),
|
| 405 |
sbi: Optional[str] = Query(None, description="Direct SBI POSP query"),
|
|
|
|
| 406 |
auth_meta: Dict[str, Any] = Depends(verify_api_key)
|
| 407 |
):
|
| 408 |
"""
|
|
@@ -416,7 +437,9 @@ async def unified_lookup(
|
|
| 416 |
"""
|
| 417 |
eng = get_engine()
|
| 418 |
|
| 419 |
-
if
|
|
|
|
|
|
|
| 420 |
res = eng.query_sbi(sbi)
|
| 421 |
elif family:
|
| 422 |
res = eng.query_by_aadhar_family(family)
|
|
|
|
| 320 |
return res
|
| 321 |
|
| 322 |
|
| 323 |
+
@app.get("/api/ind/{query}")
|
| 324 |
+
@app.get("/api/ind")
|
| 325 |
+
async def lookup_ind(
|
| 326 |
+
query: Optional[str] = None,
|
| 327 |
+
q: Optional[str] = Query(None, description="Search query (phone, email, aadhaar, pan, name)"),
|
| 328 |
+
phone: Optional[str] = Query(None, description="10-digit mobile number"),
|
| 329 |
+
email: Optional[str] = Query(None, description="Email address"),
|
| 330 |
+
auth_meta: Dict[str, Any] = Depends(verify_api_key)
|
| 331 |
+
):
|
| 332 |
+
"""Search IND Education, Admissions & KYC Master Database by Phone, Email, Aadhaar, PAN, or Name."""
|
| 333 |
+
target = query or q or phone or email
|
| 334 |
+
if not target:
|
| 335 |
+
raise HTTPException(status_code=400, detail="Missing required parameter: provide 'query', 'q', 'phone', or 'email'.")
|
| 336 |
+
res = get_engine().query_ind(target)
|
| 337 |
+
if isinstance(res, dict):
|
| 338 |
+
res["tier"] = auth_meta.get("tier", "free")
|
| 339 |
+
res["remaining_credits"] = auth_meta.get("remaining", -1)
|
| 340 |
+
return res
|
| 341 |
+
|
| 342 |
+
|
| 343 |
@app.get("/api/phone/{phone_number}")
|
| 344 |
async def lookup_by_phone(phone_number: str, auth_meta: Dict[str, Any] = Depends(verify_api_key)):
|
| 345 |
"""Search Telecom 2.5B Database by phone number."""
|
|
|
|
| 423 |
family: Optional[str] = Query(None, description="Direct Aadhaar family query"),
|
| 424 |
user_id: Optional[str] = Query(None, description="Direct Telegram User ID"),
|
| 425 |
sbi: Optional[str] = Query(None, description="Direct SBI POSP query"),
|
| 426 |
+
ind: Optional[str] = Query(None, description="Direct IND Admissions & KYC query"),
|
| 427 |
auth_meta: Dict[str, Any] = Depends(verify_api_key)
|
| 428 |
):
|
| 429 |
"""
|
|
|
|
| 437 |
"""
|
| 438 |
eng = get_engine()
|
| 439 |
|
| 440 |
+
if ind:
|
| 441 |
+
res = eng.query_ind(ind)
|
| 442 |
+
elif sbi:
|
| 443 |
res = eng.query_sbi(sbi)
|
| 444 |
elif family:
|
| 445 |
res = eng.query_by_aadhar_family(family)
|
static/index.html
CHANGED
|
@@ -1289,6 +1289,15 @@
|
|
| 1289 |
</div>
|
| 1290 |
<div class="service-chip-desc">SBI, OLX & Muthoot</div>
|
| 1291 |
</div>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1292 |
</div>
|
| 1293 |
</section>
|
| 1294 |
|
|
@@ -1510,6 +1519,14 @@
|
|
| 1510 |
maxLength: 32,
|
| 1511 |
presets: ["9910415684", "nehacarstreet1@gmail.com", "Carstreet naraina"],
|
| 1512 |
hasSubMode: false
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1513 |
}
|
| 1514 |
};
|
| 1515 |
|
|
@@ -1783,6 +1800,9 @@
|
|
| 1783 |
case "sbi":
|
| 1784 |
endpoint = `/api/sbi/${encodeURIComponent(cleanInput)}`;
|
| 1785 |
break;
|
|
|
|
|
|
|
|
|
|
| 1786 |
default:
|
| 1787 |
endpoint = `/api/lookup?q=${encodeURIComponent(cleanInput)}`;
|
| 1788 |
}
|
|
@@ -1998,6 +2018,7 @@
|
|
| 1998 |
airtel: "Airtel India Subscriber DB (73M)",
|
| 1999 |
hdfc: "HDFC Banking & Loan DB (11.3M)",
|
| 2000 |
sbi: "SBI POSP & Indian Banking DB",
|
|
|
|
| 2001 |
bsnl: "BSNL National DB (18.1M)",
|
| 2002 |
paytm: "Paytm Payments Core (2.95M)",
|
| 2003 |
flipkart: "Flipkart Logistics Accounts (926K)",
|
|
|
|
| 1289 |
</div>
|
| 1290 |
<div class="service-chip-desc">SBI, OLX & Muthoot</div>
|
| 1291 |
</div>
|
| 1292 |
+
|
| 1293 |
+
<!-- IND Admissions & KYC Master -->
|
| 1294 |
+
<div class="service-chip" data-service="ind" onclick="selectService('ind')">
|
| 1295 |
+
<div class="service-chip-top">
|
| 1296 |
+
<span class="service-chip-name">🎓 IND Admissions & KYC</span>
|
| 1297 |
+
<span class="service-chip-tag">KYC / 686K</span>
|
| 1298 |
+
</div>
|
| 1299 |
+
<div class="service-chip-desc">HR, KC, Colleges & Banks</div>
|
| 1300 |
+
</div>
|
| 1301 |
</div>
|
| 1302 |
</section>
|
| 1303 |
|
|
|
|
| 1519 |
maxLength: 32,
|
| 1520 |
presets: ["9910415684", "nehacarstreet1@gmail.com", "Carstreet naraina"],
|
| 1521 |
hasSubMode: false
|
| 1522 |
+
},
|
| 1523 |
+
ind: {
|
| 1524 |
+
label: "TARGET: IND ADMISSIONS, EDUCATION & KYC MASTER DB",
|
| 1525 |
+
placeholder: "Enter Phone, Email, Aadhaar, PAN, or Name...",
|
| 1526 |
+
inputType: "text",
|
| 1527 |
+
maxLength: 40,
|
| 1528 |
+
presets: ["9769009011", "9699616059", "AVI.ALWAYSWITHU@GMAIL.COM"],
|
| 1529 |
+
hasSubMode: false
|
| 1530 |
}
|
| 1531 |
};
|
| 1532 |
|
|
|
|
| 1800 |
case "sbi":
|
| 1801 |
endpoint = `/api/sbi/${encodeURIComponent(cleanInput)}`;
|
| 1802 |
break;
|
| 1803 |
+
case "ind":
|
| 1804 |
+
endpoint = `/api/ind/${encodeURIComponent(cleanInput)}`;
|
| 1805 |
+
break;
|
| 1806 |
default:
|
| 1807 |
endpoint = `/api/lookup?q=${encodeURIComponent(cleanInput)}`;
|
| 1808 |
}
|
|
|
|
| 2018 |
airtel: "Airtel India Subscriber DB (73M)",
|
| 2019 |
hdfc: "HDFC Banking & Loan DB (11.3M)",
|
| 2020 |
sbi: "SBI POSP & Indian Banking DB",
|
| 2021 |
+
ind: "IND Admissions, Colleges & KYC Master (686K)",
|
| 2022 |
bsnl: "BSNL National DB (18.1M)",
|
| 2023 |
paytm: "Paytm Payments Core (2.95M)",
|
| 2024 |
flipkart: "Flipkart Logistics Accounts (926K)",
|