reddit_client.py

import os
import time
import random
import requests
import logging
from typing import List, Dict, Optional

# Set up logging
logger = logging.getLogger(__name__)

# Simple User-Agent header – change to your own
HEADERS = {"User-Agent": "AltomicAi/3.0 (by /u/yourusername)"}


class RateLimiter:
    """Simple rate limiter to enforce a given number of requests per second."""
    def __init__(self, requests_per_second: float):
        self.interval = 1.0 / requests_per_second if requests_per_second > 0 else 0
        self.last_request = 0.0

    def wait(self):
        if self.interval <= 0:
            return
        now = time.time()
        elapsed = now - self.last_request
        if elapsed < self.interval:
            time.sleep(self.interval - elapsed)
        self.last_request = time.time()


class RedditClient:
    BASE_URL = "https://www.reddit.com"

    def __init__(self, requests_per_second: float = 0.3):
        self.rate_limiter = RateLimiter(requests_per_second)

        # Load proxies from environment variable (Hugging Face secret)
        proxy_list_str = os.environ.get("PROXY_LIST")
        print(f"[DEBUG] PROXY_LIST env var: {'SET (length ' + str(len(proxy_list_str)) + ')' if proxy_list_str else 'NOT SET'}", flush=True)

        self.proxy_list = self._parse_proxies(proxy_list_str)
        print(f"[DEBUG] Loaded {len(self.proxy_list)} proxies", flush=True)

        if self.proxy_list:
            print(f"[DEBUG] First proxy: {self.proxy_list[0] if self.proxy_list else 'None'}", flush=True)
        else:
            print("[WARNING] No proxies loaded. Requests will be made directly (likely blocked).", flush=True)

    def _parse_proxies(self, content: Optional[str]) -> List[Dict[str, str]]:
        """Convert proxy lines (http://user:pass@ip:port) to requests proxy dicts."""
        proxies = []
        if not content:
            return proxies
        for line in content.strip().splitlines():
            line = line.strip()
            if line and not line.startswith('#'):
                # Ensure the line starts with http:// or https://
                if not line.startswith('http'):
                    line = f"http://{line}"
                proxies.append({"http": line, "https": line})
        return proxies

    def _get_random_proxy(self) -> Optional[Dict[str, str]]:
        if not self.proxy_list:
            return None
        return random.choice(self.proxy_list)

    def _request(self, url: str, params: dict = None) -> dict:
        """Make a request with rate limiting and proxy rotation."""
        self.rate_limiter.wait()
        proxy = self._get_random_proxy()
        print(f"[DEBUG] Requesting {url} with proxy: {proxy if proxy else 'DIRECT'}", flush=True)

        try:
            resp = requests.get(
                url,
                headers=HEADERS,
                params=params,
                proxies=proxy,
                timeout=15
            )
            print(f"[DEBUG] Response status: {resp.status_code}", flush=True)
            resp.raise_for_status()
            return resp.json()
        except requests.exceptions.RequestException as e:
            print(f"[ERROR] Reddit request failed: {e}", flush=True)
            if hasattr(e, 'response') and e.response is not None:
                print(f"[ERROR] Response text: {e.response.text[:500]}", flush=True)
            return {}

    def get_top_posts(self, subreddit: str, limit: int = 30, time_filter: str = "day") -> List[dict]:
        """
        Fetch top posts from a subreddit, handling pagination up to `limit` posts.
        Returns a list of post dicts.
        """
        print(f"[DEBUG] Fetching top posts from r/{subreddit}, limit={limit}, time_filter={time_filter}", flush=True)
        all_posts = []
        after = None
        per_page = min(limit, 100)
        remaining = limit

        while remaining > 0:
            params = {
                "limit": min(remaining, per_page),
                "t": time_filter,
            }
            if after:
                params["after"] = after

            url = f"{self.BASE_URL}/r/{subreddit}/top.json"
            data = self._request(url, params)
            if not data:
                print(f"[WARNING] No data returned from Reddit for {url}", flush=True)
                break

            children = data.get("data", {}).get("children", [])
            if not children:
                print(f"[INFO] No children in response for {subreddit}", flush=True)
                break

            for child in children:
                all_posts.append(child["data"])
                remaining -= 1
                if remaining == 0:
                    break

            after = data.get("data", {}).get("after")
            if not after:
                print(f"[INFO] No 'after' token, pagination finished", flush=True)
                break

        print(f"[DEBUG] Fetched {len(all_posts)} posts from r/{subreddit}", flush=True)
        return all_posts

    def get_post_comments(self, post_id: str, limit: int = 20) -> List[dict]:
        """Fetch top-level comments for a post."""
        url = f"{self.BASE_URL}/comments/{post_id}.json"
        data = self._request(url)
        if not data or not isinstance(data, list) or len(data) < 2:
            return []
        comments_data = data[1].get("data", {}).get("children", [])
        comments = [c["data"] for c in comments_data if c["kind"] == "t1"]
        return comments[:limit]

    def get_subreddit_info(self, subreddit: str) -> dict:
        """Fetch subreddit metadata."""
        url = f"{self.BASE_URL}/r/{subreddit}/about.json"
        data = self._request(url)
        return data.get("data", {}) if data else {}