Deploy v3.0 key tester with multi-source scraping + per-provider testing

key_tester.py

@@ -1,188 +1,288 @@
 #!/usr/bin/env python3
-import json, os, re, subprocess, time, urllib.request, urllib.error, ssl, random
+"""key_tester.py – yt-keys HF Space (v3.0 – premium model version)"""
+import json, os, re, time, math, urllib.request, urllib.error, ssl
 from pathlib import Path
 
 HF_TOKEN      = os.environ.get("HF_TOKEN", "")
-DATASET_REPO  = os.environ.get("OPENCLAW_DATASET_REPO", "arshitmalik/yt-pipeline-data")
+DATASET       = os.environ.get("OPENCLAW_DATASET_REPO", "arshitmalik/yt-pipeline-data")
 POOL_KEY      = "verified_keys.json"
 LOG_FILE      = Path("/app/tester.log")
-KEYSCAN_DIR   = Path("/app/keyscan")
-PROXY_URLS    = [
-    "https://aiapiv2.pekpik.com/v1",
-    "https://api.chatanywhere.tech/v1",
-    "https://api.chatanywhere.org/v1",
-    "https://free.gpt.ge/v1",
-]
-TEST_ENDPOINT = "/models"
+TESTER_SPACE  = "arshitmalik/yt-keys"
+PRIMARY_PROXY = "https://aiapiv2.pekpik.com/v1"
+PROVIDER_PRIORITY   = ["anthropic", "openai", "google", "mistral", "xai", "others"]
+MAX_PER_PROVIDER    = 20
+MAX_TESTS_PER_CYCLE = 60
+INTER_TEST_DELAY    = 0.8
+INTER_FETCH_DELAY   = 1.0
+SCRAPE_INTERVAL     = 600
+RATE_LIMIT_STRIKES  = 2
 
 ctx = ssl.create_default_context()
 ctx.check_hostname = False
-ctx.verify_mode = ssl.CERT_NONE
+ctx.verify_mode = False
 
 def log(msg):
     line = f"[{time.strftime('%H:%M:%S')}] {msg}"
-    print(line)
+    print(line, flush=True)
     with open(LOG_FILE, "a") as f: f.write(line + "\n")
 
-def fetch_url(url, timeout=15):
+def fetch(url, timeout=20, extra_headers=None):
+    headers = {
+        "User-Agent": "Mozilla/5.0 (compatible; FreeKeyBot/3.0; +https://github.com)",
+        "Accept": "text/html,application/json,text/plain,*/*",
+    }
+    if extra_headers: headers.update(extra_headers)
     try:
-        req = urllib.request.Request(url, headers={"User-Agent": "yt-keys/2.0"})
+        req = urllib.request.Request(url, headers=headers)
         with urllib.request.urlopen(req, timeout=timeout, context=ctx) as r:
             return r.read().decode("utf-8", errors="replace")
-    except Exception:
-        return None
+    except: return None
+
+_PLACEHOLDER = re.compile(
+    r"(x{4,}|0{6,}|1{6,}|a{4,}|qwerty|asdfgh|placeholder|your[\-_]?key|"
+    r"insert|replace|changeme|sk-test|sk-demo|sk-fake|12345|abcde|"
+    r"sk-proj-[X]{4,}|YOUR|EXAMPLE|SAMPLE|REDACTED)", re.I)
+_KEY_PATTERNS = [
+    re.compile(r"(sk-[A-Za-z0-9]{48,})"),
+    re.compile(r"(sk-ant-api\d{2}-[A-Za-z0-9\-_]{64,})"),
+    re.compile(r"(AIza[0-9A-Za-z\-_]{35})"),
+    re.compile(r"(xai-[A-Za-z0-9]{20,})"),
+    re.compile(r"\b(fk[A-Za-z0-9]{40,})\b"),
+    re.compile(r"(sk-or-v1-[A-Za-z0-9]{48,})"),
+    re.compile(r"\b(gsk_[A-Za-z0-9]{40,})\b"),
+    re.compile(r"\b([A-Za-z0-9]{32})\b"),
+]
+
+def shannon_entropy(s):
+    if not s: return 0.0
+    counts = [s.count(c)/len(s) for c in set(s)]
+    return -sum(p*math.log2(p) for p in counts)
+
+def is_plausible_key(key):
+    if len(key) < 32 or len(key) > 512: return False
+    if _PLACEHOLDER.search(key): return False
+    if shannon_entropy(key) < 3.5: return False
+    if max(key.count(c) for c in set(key)) > len(key)*0.65: return False
+    return True
+
+def extract_keys(text):
+    found = set()
+    for pat in _KEY_PATTERNS:
+        for m in pat.finditer(text):
+            k = m.group(1)
+            if is_plausible_key(k): found.add(k)
+    return found
 
 def guess_provider(key):
-    if key.startswith("sk-ant-api"): return "anthropic"
-    if key.startswith("AIza"):       return "google"
-    if key.startswith("xai-"):       return "xai"
-    if key.startswith("sk-"):        return "openai"
+    k = key.lower()
+    if k.startswith("sk-ant-api"): return "anthropic"
+    if k.startswith("aiza"): return "google"
+    if k.startswith("xai-"): return "xai"
+    if k.startswith("sk-or-v1"): return "others"
+    if k.startswith("gsk_"): return "others"
+    if k.startswith("mistral"): return "mistral"
+    if k.startswith("sk-") or k.startswith("fk"): return "openai"
     return "others"
 
-def test_key(key, proxy_url):
+def _fetch_raw_github(repo, filename):
+    return fetch(f"https://raw.githubusercontent.com/{repo}/main/{filename}", timeout=15)
+
+def scrape_alistaitsacle():
+    keys = set()
+    for fname in ["README.md","README_CN.md","README_JA.md","README_KO.md","README_ES.md","README_PT.md"]:
+        text = _fetch_raw_github("alistaitsacle/free-llm-api-keys", fname)
+        if text:
+            found = extract_keys(text)
+            if found: log(f"    alistaitsacle/{fname}: {len(found)} candidates")
+            keys.update(found)
+        time.sleep(INTER_FETCH_DELAY)
+    return keys
+
+_GITHUB_REPOS = [
+    ("chatanywhere/GPT_API_free",["README.md","README_EN.md"]),
+    ("creazycar/GPT_API_free",["README.md"]),
+    ("ReimagineTruth/gpt_api_free",["README.md"]),
+    ("OrrieZ/gpt_api_free",["README.md"]),
+    ("FairyWorld/free_ai_GPT_API_free",["README.md"]),
+    ("taurusduan/GPT_API_free-lvyou",["README.md"]),
+    ("Poghappy/gpt_api_free",["README.md"]),
+    ("popjane/free_chatgpt_api",["README.md"]),
+    ("onlywyl/free_chatgpt_api",["README.md"]),
+    ("dan1471/FREE-openai-api-keys",["README.md","keys.txt","free_keys.md"]),
+    ("racalado/free-openai-api-keys",["README.md","keys.md"]),
+    ("CyYxl2024/freeai",["README.md"]),
+    ("qwq202/One-FAS-API",["README.md"]),
+    ("vibheksoni/free-ai",["README.md","keys.txt"]),
+    ("Mr-Abood/GPTFree",["README.md"]),
+    ("BuluBulugege/Free-BAI",["README.md"]),
+    ("thesecondchance/freeflow-llm",["README.md","keys.md"]),
+    ("tashfeenahmed/freellmapi",["README.md"]),
+    ("mufeng510/Free-ChatGPT-API",["README.md"]),
+    ("lizhenmiao/gpt4free-ts",["README.md"]),
+    ("ling-drag0n/api-pool",["README.md","pool.json","keys.json"]),
+    ("fengqiaozhu/openrouter-free-pool",["README.md","keys.json","pool.json"]),
+    ("mahdi-marjani/free-llm-api",["README.md","keys.txt"]),
+    ("realasfngl/ChatGPT",["README.md"]),
+    ("Butakov-Andrey/freegpt",["README.md"]),
+    ("elphen-wang/FreeAI",["README.md","keys.md"]),
+    ("devashish2024/freegpt",["README.md"]),
+    ("msmarkgu/RelayFreeLLM",["README.md"]),
+    ("piyush-tyagi-13/llm-keypool",["README.md","keys.json"]),
+    ("Mirrowel/LLM-API-Key-Proxy",["README.md"]),
+    ("p32929/rotato",["README.md"]),
+    ("SreejanPersonal/Free-Unoffical-OpenAI-API",["README.md","keys.txt"]),
+    ("suyashb734/cliagents",["README.md"]),
+]
+
+def scrape_github_repos():
+    keys = set()
+    for repo, files in _GITHUB_REPOS:
+        for fname in files:
+            text = _fetch_raw_github(repo, fname)
+            if text:
+                found = extract_keys(text)
+                if found: log(f"    {repo}/{fname}: {len(found)} candidates")
+                keys.update(found)
+            time.sleep(INTER_FETCH_DELAY)
+    return keys
+
+_WEB_ENDPOINTS = [
+    "https://alistaitsacle.github.io/free-llm-api-keys/",
+    "https://api.zaochuanqiu.online/",
+    "https://api.zaochuanqiu.online/v1/keys",
+    "https://api.zaochuanqiu.online/keys",
+    "https://api.algion.dev/",
+    "https://api.algion.dev/keys",
+    "https://api.algion.dev/v1/models",
+    "https://freeapi.geekgpt.site/",
+    "https://freeapi.geekgpt.site/keys",
+    "https://freeapi.geekgpt.site/api/keys",
+    "https://free.gpt.ge/",
+    "https://aipipe.org/",
+    "https://freetheai.xyz/",
+]
+
+def scrape_web_endpoints():
+    keys = set()
+    for url in _WEB_ENDPOINTS:
+        text = fetch(url, timeout=12)
+        if text:
+            found = extract_keys(text)
+            if found: log(f"    {url}: {len(found)} candidates")
+            keys.update(found)
+        time.sleep(0.5)
+    return keys
+
+def test_key(key, provider):
+    if provider == "anthropic":
+        url = "https://api.anthropic.com/v1/models"
+        headers = {"x-api-key": key, "anthropic-version": "2023-06-01"}
+    elif provider == "google":
+        url = f"https://generativelanguage.googleapis.com/v1beta/models?key={key}"
+        headers = {}
+    elif provider == "mistral":
+        url = "https://api.mistral.ai/v1/models"
+        headers = {"Authorization": f"Bearer {key}"}
+    elif provider == "xai":
+        url = "https://api.x.ai/v1/models"
+        headers = {"Authorization": f"Bearer {key}"}
+    else:
+        url = f"{PRIMARY_PROXY}/models"
+        headers = {"Authorization": f"Bearer {key}"}
     try:
-        req = urllib.request.Request(f"{proxy_url}{TEST_ENDPOINT}",
-                                     headers={"Authorization": f"Bearer {key}"})
-        with urllib.request.urlopen(req, timeout=8, context=ctx) as r:
-            return 'valid' if r.status == 200 else 'invalid'
+        req = urllib.request.Request(url, headers=headers)
+        with urllib.request.urlopen(req, timeout=10, context=ctx) as r:
+            return r.status == 200
     except urllib.error.HTTPError as e:
-        return 'rate_limited' if e.code == 429 else 'invalid'
-    except Exception:
-        return 'invalid'
+        return None if e.code == 429 else False
+    except: return None
 
-def upload_verified(pool):
+def upload_pool(verified):
+    ordered = {p: verified.get(p, []) for p in PROVIDER_PRIORITY}
+    total = sum(len(v) for v in ordered.values())
+    summary = ", ".join(f"{p}:{len(ordered[p])}" for p in PROVIDER_PRIORITY if ordered[p])
     try:
         from huggingface_hub import HfApi
-        api = HfApi(token=HF_TOKEN)
-        api.upload_file(
-            path_or_fileobj=json.dumps(pool, indent=2).encode(),
-            path_in_repo=POOL_KEY, repo_id=DATASET_REPO, repo_type="dataset")
-        log(f"Uploaded {sum(len(v) for v in pool.values())} verified keys")
+        HfApi(token=HF_TOKEN).upload_file(
+            path_or_fileobj=json.dumps(ordered, indent=2).encode(),
+            path_in_repo=POOL_KEY, repo_id=DATASET, repo_type="dataset",
+            commit_message=f"keypool: {total} verified keys ({summary})")
+        log(f"Uploaded {total} keys to dataset [{summary}]")
     except Exception as e:
         log(f"Upload failed: {e}")
 
-def run_keyscan():
-    if not KEYSCAN_DIR.exists():
-        log("keyscan directory missing, skipping")
-        return []
+def factory_rebuild_self():
     try:
-        result = subprocess.run(
-            ["node", str(KEYSCAN_DIR / "index.js"), "--json"],
-            capture_output=True, text=True, timeout=120, cwd=str(KEYSCAN_DIR)
-        )
-        if result.returncode != 0:
-            log(f"keyscan error: {result.stderr[:200]}")
-            return []
-        data = json.loads(result.stdout)
-        keys = [item["key"] for item in data if item.get("key")]
-        log(f"keyscan found {len(keys)} keys")
-        return keys
+        from huggingface_hub import HfApi
+        HfApi(token=HF_TOKEN).restart_space(TESTER_SPACE, factory_reboot=True)
+        log("Factory rebuild triggered – space will restart")
     except Exception as e:
-        log(f"keyscan failed: {e}")
-        return []
-
-SOURCES = [
-    "https://raw.githubusercontent.com/alistaitsacle/free-llm-api-keys/main/README.md",
-    "https://raw.githubusercontent.com/chatanywhere/GPT_API_free/main/README.md",
-    "https://raw.githubusercontent.com/creazycar/GPT_API_free/main/README.md",
-    "https://raw.githubusercontent.com/ReimagineTruth/gpt_api_free/main/README.md",
-    "https://raw.githubusercontent.com/OrrieZ/gpt_api_free/main/README.md",
-    "https://raw.githubusercontent.com/FairyWorld/free_ai_GPT_API_free/main/README.md",
-    "https://raw.githubusercontent.com/taurusduan/GPT_API_free-lvyou/main/README.md",
-    "https://raw.githubusercontent.com/Poghappy/gpt_api_free/main/README.md",
-    "https://raw.githubusercontent.com/popjane/free_chatgpt_api/main/README.md",
-    "https://raw.githubusercontent.com/onlywyl/free_chatgpt_api/main/README.md",
-    "https://raw.githubusercontent.com/dan1471/FREE-openai-api-keys/main/README.md",
-    "https://raw.githubusercontent.com/racalado/free-openai-api-keys/main/README.md",
-    "https://raw.githubusercontent.com/CyYxl2024/freeai/main/README.md",
-    "https://raw.githubusercontent.com/qwq202/One-FAS-API/main/README.md",
-    "https://raw.githubusercontent.com/vibheksoni/free-ai/main/README.md",
-    "https://raw.githubusercontent.com/Mr-Abood/GPTFree/main/README.md",
-    "https://raw.githubusercontent.com/BuluBulugege/Free-BAI/main/README.md",
-    "https://raw.githubusercontent.com/thesecondchance/freeflow-llm/main/README.md",
-    "https://raw.githubusercontent.com/tashfeenahmed/freellmapi/main/README.md",
-    "https://raw.githubusercontent.com/mufeng510/Free-ChatGPT-API/main/README.md",
-    "https://raw.githubusercontent.com/lizhenmiao/gpt4free-ts/main/README.md",
-    "https://raw.githubusercontent.com/ling-drag0n/api-pool/main/README.md",
-    "https://raw.githubusercontent.com/fengqiaozhu/openrouter-free-pool/main/README.md",
-    "https://raw.githubusercontent.com/mahdi-marjani/free-llm-api/main/README.md",
-    "https://raw.githubusercontent.com/realasfngl/ChatGPT/main/README.md",
-    "https://raw.githubusercontent.com/Butakov-Andrey/freegpt/main/README.md",
-    "https://raw.githubusercontent.com/elphen-wang/FreeAI/main/README.md",
-    "https://raw.githubusercontent.com/devashish2024/freegpt/main/README.md",
-    "https://alistaitsacle.github.io/free-llm-api-keys/",
-    "https://free.v36.cm",
-    "https://api.zaochuanqiu.online",
-    "https://aipipe.org/",
-    "https://freetheai.xyz",
-    "https://api.algion.dev",
-    "https://freeapi.geekgpt.site",
-]
+        log(f"Factory rebuild failed: {e}")
 
-def manual_scrape():
-    all_keys = set()
-    for url in SOURCES:
-        log(f"Scraping {url}")
-        text = fetch_url(url)
-        if not text: continue
-        matches = re.findall(r'(?:sk-[A-Za-z0-9]{48,}|AIza[0-9A-Za-z\-_]{35}|xai-[A-Za-z0-9]{20,}|sk-ant-api[A-Za-z0-9\-_]{40,})', text)
-        for m in matches:
-            if re.search(r'(x{5,}|qwerty|placeholder|example|0{10,})', m, re.I): continue
-            all_keys.add(m)
-        log(f"  -> {len(matches)} candidate keys")
-    return list(all_keys)
+def run_cycle():
+    log("="*60)
+    log("SCRAPE CYCLE STARTING")
+    log("="*60)
+    log("[1/4] Scraping alistaitsacle (primary source)...")
+    candidates = scrape_alistaitsacle()
+    log(f"      -> {len(candidates)} candidates so far")
+    log("[2/4] Scraping secondary GitHub repos...")
+    candidates.update(scrape_github_repos())
+    log(f"      -> {len(candidates)} candidates total")
+    log("[3/4] Probing web endpoints...")
+    candidates.update(scrape_web_endpoints())
+    log(f"      -> {len(candidates)} candidates total")
+    if not candidates:
+        log("No candidates found in any source. Sleeping until next cycle.")
+        return
+    log(f"[4/4] Testing up to {MAX_TESTS_PER_CYCLE} keys (delay: {INTER_TEST_DELAY}s each)...")
+    verified = {p: [] for p in PROVIDER_PRIORITY}
+    rate_limit_strikes = {p: 0 for p in PROVIDER_PRIORITY}
+    tested = 0
+    sorted_candidates = sorted(candidates, key=lambda k: PROVIDER_PRIORITY.index(guess_provider(k)) if guess_provider(k) in PROVIDER_PRIORITY else 99)
+    for key in sorted_candidates:
+        prov = guess_provider(key)
+        if prov not in verified: prov = "others"
+        if len(verified[prov]) >= MAX_PER_PROVIDER: continue
+        if rate_limit_strikes.get(prov, 0) >= RATE_LIMIT_STRIKES: continue
+        if tested >= MAX_TESTS_PER_CYCLE:
+            log(f"Hit test cap ({MAX_TESTS_PER_CYCLE}). Saving what we have.")
+            break
+        result = test_key(key, prov)
+        tested += 1
+        if result is True:
+            verified[prov].append(key)
+            log(f"  OK [{prov}] {key[:18]}...")
+        elif result is None:
+            rate_limit_strikes[prov] = rate_limit_strikes.get(prov, 0) + 1
+            if rate_limit_strikes[prov] >= RATE_LIMIT_STRIKES:
+                log(f"  RATE [{prov}] x{rate_limit_strikes[prov]} — skipping provider")
+        time.sleep(INTER_TEST_DELAY)
+    total = sum(len(v) for v in verified.values())
+    log(f"Cycle complete: {tested} keys tested, {total} verified")
+    if total > 0:
+        upload_pool(verified)
+    else:
+        log("No verified keys this cycle.")
+        providers_seen = set(guess_provider(k) for k in candidates)
+        providers_rate_limited = {p for p, strikes in rate_limit_strikes.items() if strikes >= RATE_LIMIT_STRIKES}
+        if providers_seen and providers_seen <= providers_rate_limited:
+            log("All provider endpoints rate-limited. Triggering factory rebuild...")
+            factory_rebuild_self()
+            return
 
 def main():
-    log("Key tester 2.0 started (keyscan priority)")
+    log("yt-keys Key Tester v3.0 — starting")
+    log(f"Primary proxy: {PRIMARY_PROXY}")
+    log(f"Cycle interval: {SCRAPE_INTERVAL}s | Max tests/cycle: {MAX_TESTS_PER_CYCLE}")
     while True:
-        log("Running keyscan...")
-        candidates = run_keyscan()
-        if not candidates:
-            log("keyscan returned nothing, falling back to manual scraping")
-            candidates = manual_scrape()
-        if not candidates:
-            log("No candidates at all. Sleeping 600s")
-            time.sleep(600)
-            continue
-
-        random.shuffle(candidates)
-        test_queue = candidates[:50]
-        verified = {p: [] for p in ["anthropic", "openai", "google", "mistral", "xai", "others"]}
-        proxy_idx = 0
-        global_rate_limited = False
-
-        for key in test_queue:
-            prov = guess_provider(key)
-            if global_rate_limited: break
-            for _ in range(len(PROXY_URLS)):
-                p_url = PROXY_URLS[proxy_idx % len(PROXY_URLS)]
-                res = test_key(key, p_url)
-                if res == 'valid':
-                    verified[prov].append(key)
-                    log(f"  OK [{prov}] {key[:14]}...")
-                    proxy_idx += 1
-                    break
-                elif res == 'rate_limited':
-                    proxy_idx += 1
-                    if proxy_idx % len(PROXY_URLS) == 0:
-                        global_rate_limited = True
-                        log("All proxies rate-limited")
-                    continue
-                else:
-                    proxy_idx += 1
-            else:
-                pass
-            if global_rate_limited: break
-
-        if global_rate_limited:
-            log("Rate-limited on all proxies, will retry after sleep")
-        else:
-            if sum(len(v) for v in verified.values()) > 0:
-                upload_verified(verified)
-            else:
-                log("No verified keys this cycle")
-        log("Sleeping 600s...")
-        time.sleep(600)
+        try:
+            run_cycle()
+        except Exception as e:
+            import traceback
+            log(f"ERROR in cycle: {e}\n{traceback.format_exc()}")
+        log(f"Sleeping {SCRAPE_INTERVAL}s until next cycle...")
+        time.sleep(SCRAPE_INTERVAL)
 
 if __name__ == "__main__":
     main()