| """
|
| trigger.py — push & run the scraper kernel on Kaggle from the dashboard.
|
|
|
| Fire-and-forget: `kaggle kernels push` uploads a generated runner script and
|
| Kaggle executes it immediately on their servers. The kernel pulls the project
|
| code from the attached Kaggle Dataset (gmaps-scraper-code), scrapes, scores,
|
| and pushes leads.csv to the HF Dataset repo. The dashboard never waits for it.
|
|
|
| Credentials (Space settings):
|
| KAGGLE_USERNAME variable — kaggle.com username
|
| KAGGLE_KEY secret — API key from kaggle.com/settings -> Create New Token
|
| HF_TOKEN secret — must be a WRITE token (the kernel uses it to upload)
|
|
|
| Security note: the HF token is embedded in the generated kernel source, so the
|
| kernel MUST stay private (is_private is forced to true below).
|
| """
|
|
|
| import json
|
| import os
|
| import re
|
| import subprocess
|
| import tempfile
|
| from pathlib import Path
|
|
|
| KERNEL_SLUG = "gmaps-scraper-auto"
|
| CODE_DATASET = "gmaps-scraper-code"
|
|
|
| RUNNER_TEMPLATE = '''"""Auto-generated by the dashboard trigger. Do not edit by hand."""
|
| import glob
|
| import os
|
| import shutil
|
| import subprocess
|
| import sys
|
|
|
| os.environ["HF_TOKEN"] = "__HF_TOKEN__"
|
| os.environ["HF_DATASET_REPO"] = "__HF_REPO__"
|
|
|
| KEYWORD = "__KEYWORD__"
|
| CITY = "__CITY__"
|
| MAX_RESULTS = __MAX_RESULTS__
|
| WITH_DETAILS = __WITH_DETAILS__
|
| MIN_REVIEWS = __MIN_REVIEWS__
|
| MIN_RATING = __MIN_RATING__
|
| ONLY_NEW = __ONLY_NEW__
|
| MAX_REVIEWS = __MAX_REVIEWS__
|
| MAX_PHOTOS = __MAX_PHOTOS__
|
|
|
|
|
| def sh(cmd, check=True):
|
| print(f"$ {cmd}")
|
| rc = subprocess.run(cmd, shell=True).returncode
|
| if check and rc != 0:
|
| sys.exit(rc)
|
| return rc
|
|
|
|
|
| # 1. Copy the project code from the attached Kaggle Dataset (any mount layout)
|
| REQUIRED = ("scraper.py", "process.py", "enrich.py", "push_to_hf.py")
|
| for path in glob.glob("/kaggle/input/**/*.py", recursive=True):
|
| if os.path.basename(path) in REQUIRED:
|
| shutil.copy(path, ".")
|
| print(f"Copied {path}")
|
|
|
| # 2. Install dependencies
|
| sh("pip install -q playwright pandas requests beautifulsoup4 huggingface_hub")
|
| sh("playwright install --with-deps chromium")
|
|
|
| # 3. Scrape — never fatal: captcha/blocks leave a partial CSV behind
|
| details = "--details" if WITH_DETAILS else ""
|
| extra = ""
|
| if WITH_DETAILS and MAX_REVIEWS:
|
| extra += f" --max-reviews {MAX_REVIEWS}"
|
| if WITH_DETAILS and MAX_PHOTOS:
|
| extra += f" --max-photos {MAX_PHOTOS}"
|
| sh(f'python scraper.py --keyword "{KEYWORD}" --city "{CITY}" '
|
| f"--max-results {MAX_RESULTS} --min-rating {MIN_RATING} "
|
| f"--min-reviews {MIN_REVIEWS} {details}{extra}", check=False)
|
|
|
| # A killed run may only have the checkpoint file — promote it
|
| if not os.path.exists("data/leads.csv") and os.path.exists("data/leads_partial.csv"):
|
| shutil.copy("data/leads_partial.csv", "data/leads.csv")
|
| print("Recovered leads from the partial checkpoint.")
|
| if not os.path.exists("data/leads.csv"):
|
| print("No CSV produced (blocked before anything was scraped) — nothing to push.")
|
| sys.exit(0)
|
|
|
| # 4. Score (+ optionally drop leads already in the HF dataset)
|
| dedupe = "--dedupe-hf" if ONLY_NEW else ""
|
| sh(f"python process.py --min-reviews {MIN_REVIEWS} --min-rating {MIN_RATING} {dedupe}")
|
|
|
| # 5. Push whatever we got (an empty run is skipped, not an error)
|
| sh(f'python push_to_hf.py --file data/leads_scored.csv --label "{KEYWORD} {CITY}"',
|
| check=False)
|
| print("DONE — check the run list in the dashboard.")
|
| '''
|
|
|
|
|
| def _env(username: str, key: str) -> dict:
|
| """Provide Kaggle credentials via env vars AND ~/.kaggle/kaggle.json.
|
|
|
| Different kaggle CLI versions look in different places, so cover both.
|
| """
|
| env = os.environ.copy()
|
| env["KAGGLE_USERNAME"] = username
|
| env["KAGGLE_KEY"] = key
|
|
|
| cfg_dir = Path.home() / ".kaggle"
|
| cfg_file = cfg_dir / "kaggle.json"
|
| try:
|
| cfg_dir.mkdir(parents=True, exist_ok=True)
|
| cfg_file.write_text(json.dumps({"username": username, "key": key}),
|
| encoding="utf-8")
|
| cfg_file.chmod(0o600)
|
| except OSError:
|
| pass
|
|
|
| return env
|
|
|
|
|
| def _sanitize(text: str) -> str:
|
| """Keep kernel source injection-proof: strip quotes/backslashes/newlines."""
|
| return re.sub(r'["\'\\\n\r]', " ", text).strip()
|
|
|
|
|
| def trigger_scrape(*, keyword: str, city: str, max_results: int,
|
| with_details: bool, min_reviews: int,
|
| hf_repo: str, hf_token: str,
|
| kaggle_username: str, kaggle_key: str,
|
| min_rating: float = 0.0, only_new: bool = False,
|
| max_reviews: int = 0, max_photos: int = 0) -> str:
|
| """Generate the kernel folder and push it to Kaggle. Returns CLI output."""
|
| if not hf_token:
|
| raise ValueError("HF_TOKEN is missing — it must be a WRITE token.")
|
|
|
| runner = (RUNNER_TEMPLATE
|
| .replace("__HF_TOKEN__", hf_token)
|
| .replace("__HF_REPO__", _sanitize(hf_repo))
|
| .replace("__KEYWORD__", _sanitize(keyword))
|
| .replace("__CITY__", _sanitize(city))
|
| .replace("__MAX_RESULTS__", str(int(max_results)))
|
| .replace("__WITH_DETAILS__", str(bool(with_details)))
|
| .replace("__MIN_REVIEWS__", str(int(min_reviews)))
|
| .replace("__MIN_RATING__", str(float(min_rating)))
|
| .replace("__ONLY_NEW__", str(bool(only_new)))
|
| .replace("__MAX_REVIEWS__", str(int(max_reviews)))
|
| .replace("__MAX_PHOTOS__", str(int(max_photos))))
|
|
|
| metadata = {
|
| "id": f"{kaggle_username}/{KERNEL_SLUG}",
|
| "title": KERNEL_SLUG,
|
| "code_file": "runner.py",
|
| "language": "python",
|
| "kernel_type": "script",
|
| "is_private": True,
|
| "enable_internet": True,
|
| "enable_gpu": False,
|
| "dataset_sources": [f"{kaggle_username}/{CODE_DATASET}"],
|
| "competition_sources": [],
|
| "kernel_sources": [],
|
| }
|
|
|
| with tempfile.TemporaryDirectory() as d:
|
| Path(d, "runner.py").write_text(runner, encoding="utf-8")
|
| Path(d, "kernel-metadata.json").write_text(
|
| json.dumps(metadata, indent=1), encoding="utf-8")
|
|
|
| result = subprocess.run(
|
| ["kaggle", "kernels", "push", "-p", d],
|
| capture_output=True, text=True, timeout=120,
|
| env=_env(kaggle_username, kaggle_key),
|
| )
|
|
|
| if result.returncode != 0:
|
| raise RuntimeError(result.stderr.strip() or result.stdout.strip())
|
| return result.stdout.strip()
|
|
|
|
|
| def kernel_status(kaggle_username: str, kaggle_key: str) -> str:
|
| """Ask Kaggle for the current status of the triggered kernel."""
|
| result = subprocess.run(
|
| ["kaggle", "kernels", "status", f"{kaggle_username}/{KERNEL_SLUG}"],
|
| capture_output=True, text=True, timeout=60,
|
| env=_env(kaggle_username, kaggle_key),
|
| )
|
| return (result.stdout + result.stderr).strip()
|
|
|
