Blame for core/address_processor.py

"""
Address Processor - Smart address handling for location.apply and state.apply.

Handles:
- HQ address extraction with priority rules
- US address selection when multiple locations exist
- State abbreviation ↔ full name mapping
- Hiring concentration analysis
- Non-US location filtering
"""

import re
from typing import Optional, Tuple, Dict


# US State abbreviations and full names mapping
US_STATES = {
    "alabama": "Alabama", "alaska": "Alaska", "arizona": "Arizona",
    "arkansas": "Arkansas", "california": "California", "colorado": "Colorado",
    "connecticut": "Connecticut", "delaware": "Delaware", "florida": "Florida",
    "georgia": "Georgia", "hawaii": "Hawaii", "idaho": "Idaho",
    "illinois": "Illinois", "indiana": "Indiana", "iowa": "Iowa",
    "kansas": "Kansas", "kentucky": "Kentucky", "louisiana": "Louisiana",
    "maine": "Maine", "maryland": "Maryland", "massachusetts": "Massachusetts",
    "michigan": "Michigan", "minnesota": "Minnesota", "mississippi": "Mississippi",
    "missouri": "Missouri", "montana": "Montana", "nebraska": "Nebraska",
    "nevada": "Nevada", "new hampshire": "New Hampshire", "new jersey": "New Jersey",
    "new mexico": "New Mexico", "new york": "New York",
    "north carolina": "North Carolina", "north dakota": "North Dakota",
    "ohio": "Ohio", "oklahoma": "Oklahoma", "oregon": "Oregon",
    "pennsylvania": "Pennsylvania", "rhode island": "Rhode Island",
    "south carolina": "South Carolina", "south dakota": "South Dakota",
    "tennessee": "Tennessee", "texas": "Texas", "utah": "Utah",
    "vermont": "Vermont", "virginia": "Virginia", "washington": "Washington",
    "west virginia": "West Virginia", "wisconsin": "Wisconsin", "wyoming": "Wyoming",
    "district of columbia": "District of Columbia", "washington dc": "District of Columbia",
    "dc": "District of Columbia",
}

STATE_ABBREV = {
    "AL": "Alabama", "AK": "Alaska", "AZ": "Arizona", "AR": "Arkansas",
    "CA": "California", "CO": "Colorado", "CT": "Connecticut", "DE": "Delaware",
    "FL": "Florida", "GA": "Georgia", "HI": "Hawaii", "ID": "Idaho",
    "IL": "Illinois", "IN": "Indiana", "IA": "Iowa", "KS": "Kansas",
    "KY": "Kentucky", "LA": "Louisiana", "ME": "Maine", "MD": "Maryland",
    "MA": "Massachusetts", "MI": "Michigan", "MN": "Minnesota", "MS": "Mississippi",
    "MO": "Missouri", "MT": "Montana", "NE": "Nebraska", "NV": "Nevada",
    "NH": "New Hampshire", "NJ": "New Jersey", "NM": "New Mexico", "NY": "New York",
    "NC": "North Carolina", "ND": "North Dakota", "OH": "Ohio", "OK": "Oklahoma",
    "OR": "Oregon", "PA": "Pennsylvania", "RI": "Rhode Island",
    "SC": "South Carolina", "SD": "South Dakota", "TN": "Tennessee",
    "TX": "Texas", "UT": "Utah", "VT": "Vermont", "VA": "Virginia",
    "WA": "Washington", "WV": "West Virginia", "WI": "Wisconsin", "WY": "Wyoming",
    "DC": "District of Columbia",
}

# Reverse mapping: full name → abbreviation
STATE_TO_ABBREV = {v: k for k, v in STATE_ABBREV.items()}
# Add lowercase versions
STATE_TO_ABBREV.update({v.lower(): k for k, v in STATE_ABBREV.items()})


class AddressProcessor:
    """Smart address processing for location.apply and state.apply columns."""

    # Known non-US locations (country/city names)
    NON_US_LOCATIONS = {
        "india", "china", "japan", "korea", "singapore", "hong kong",
        "canada", "uk", "united kingdom", "england", "germany", "france",
        "australia", "brazil", "mexico", "spain", "italy", "netherlands",
        "sweden", "switzerland", "ireland", "dubai", "uae", "bangalore",
        "bengaluru", "mumbai", "delhi", "toronto", "vancouver", "london",
        "berlin", "paris", "sydney", "tokyo", "seoul", "dublin", "amsterdam",
        "stockholm", "zurich", "hyderabad", "pune", "chennai", "gurgaon",
        "noida", "manila", "jakarta", "bangkok", "ho chi minh", "saigon",
        "kuala lumpur", "taipei", "shanghai", "beijing", "shenzhen",
        "warsaw", "lisbon", "madrid", "barcelona", "milan", "rome",
        "vienna", "brussels", "oslo", "helsinki", "copenhagen",
        "auckland", "wellington", "tel aviv", "istanbul", "moscow",
        "cape town", "johannesburg", "nairobi", "lagos", "cairo",
        "buenos aires", "sao paulo", "santiago", "bogota", "lima",
    }

    def determine_location_apply(
        self,
        wellfound_location: str,
        ai_contact_data: Dict,
        scraped_addresses: list,
    ) -> Tuple[Optional[str], Optional[str]]:
        """
        Determine location.apply (city) and state.apply (state).

        Priority:
        1. US HQ address from AI contact extraction
        2. US office from AI contact extraction
        3. Hiring focus location from AI extraction
        4. Wellfound location (if US)
        5. First US address found on website
        """
        location_apply = None
        state_apply = None

        # Priority 1: HQ address if it's in the US
        hq_city = ai_contact_data.get("headquarters_city")
        hq_state = ai_contact_data.get("headquarters_state")
        hq_country = ai_contact_data.get("headquarters_country", "").lower()

        if hq_city and self._is_us_location(hq_state, hq_country):
            location_apply = self._normalize_city(hq_city)
            state_apply = self._normalize_state(hq_state)

        # Priority 2: US office
        if not location_apply:
            us_city = ai_contact_data.get("us_office_city")
            us_state = ai_contact_data.get("us_office_state")
            if us_city:
                state_apply = self._normalize_state(us_state)
                location_apply = self._normalize_city(us_city)

        # Priority 3: Hiring focus location (if US)
        if not location_apply:
            hiring_loc = ai_contact_data.get("hiring_focus_location", "")
            city, state = self._parse_location_string(hiring_loc)
            if city and state and self._is_us_state(state):
                location_apply = self._normalize_city(city)
                state_apply = self._normalize_state(state)

        # Priority 4: Parse wellfound location for US parts
        if not location_apply and wellfound_location:
            city, state = self._extract_us_from_wellfound_location(wellfound_location)
            if city:
                location_apply = self._normalize_city(city)
                if state:
                    state_apply = self._normalize_state(state)

        # Priority 5: First US address from scraping
        if not location_apply and scraped_addresses:
            for addr in scraped_addresses:
                city, state = self._parse_address_string(addr)
                if city and state and self._is_us_state(state):
                    location_apply = self._normalize_city(city)
                    state_apply = self._normalize_state(state)
                    break

        return location_apply, state_apply

    def _is_us_location(self, state: Optional[str], country: Optional[str]) -> bool:
        """Check if a location is in the US."""
        if country and country.lower() in ("us", "usa", "united states", "united states of america"):
            return True
        if state and self._is_us_state(state):
            return True
        return False

    def _is_us_state(self, state: str) -> bool:
        """Check if a string is a valid US state."""
        s = state.strip().lower()
        if s in US_STATES:
            return True
        if s.upper() in STATE_ABBREV:
            return True
        return False

    def _normalize_state(self, state: str) -> Optional[str]:
        """Convert state to full name. Returns None if input is empty or unrecognized."""
        if not state:
            return None
        s = state.strip()
        if s.upper() in STATE_ABBREV:
            return STATE_ABBREV[s.upper()]
        s_lower = s.lower()
        if s_lower in US_STATES:
            return US_STATES[s_lower]
        return s

    def _normalize_city(self, city: str) -> str:
        """Normalize city name — strip trailing punctuation and split on comma."""
        if not city:
            return ""
        city = city.strip().strip(",.;")
        # Handle "City, State" format — keep only the city part
        if "," in city:
            city = city.split(",")[0].strip()
        return city

    def _parse_location_string(self, location: str) -> Tuple[Optional[str], Optional[str]]:
        """Parse 'City, State' or 'City, State, Country' format."""
        if not location:
            return None, None
        parts = [p.strip() for p in location.split(",")]
        if len(parts) >= 2:
            city = parts[0]
            for part in parts[1:]:
                if self._is_us_state(part):
                    return city, part
            return city, None
        return None, None

    def _extract_us_from_wellfound_location(self, location: str) -> Tuple[Optional[str], Optional[str]]:
        """Extract US city/state from wellfound location string like 'San Francisco , New York'."""
        if not location:
            return None, None

        parts = [p.strip() for p in location.split(",")]

        # Try to find a US state abbreviation among the parts
        for part in parts:
            part_stripped = part.strip()
            if self._is_us_state(part_stripped):
                # This is a state — look for preceding city
                idx = parts.index(part_stripped)
                if idx > 0:
                    city = parts[idx - 1].strip()
                    # Verify city is not a non-US location
                    if city.lower() not in self.NON_US_LOCATIONS:
                        return city, part_stripped
                # State name as city? (e.g., "New York")
                if part_stripped.lower() in ("new york", "washington"):
                    return part_stripped, part_stripped

        # Check if any part is a known US city
        us_cities = {
            "san francisco": "California", "new york": "New York",
            "new york city": "New York", "los angeles": "California",
            "chicago": "Illinois", "boston": "Massachusetts",
            "seattle": "Washington", "austin": "Texas",
            "denver": "Colorado", "portland": "Oregon",
            "san diego": "California", "san jose": "California",
            "palo alto": "California", "mountain view": "California",
            "menlo park": "California", "redwood city": "California",
            "sunnyvale": "California", "santa clara": "California",
            "oakland": "California", "berkeley": "California",
            "miami": "Florida", "atlanta": "Georgia",
            "dallas": "Texas", "houston": "Texas",
            "philadelphia": "Pennsylvania", "phoenix": "Arizona",
            "minneapolis": "Minnesota", "detroit": "Michigan",
            "salt lake city": "Utah", "raleigh": "North Carolina",
            "charlotte": "North Carolina", "nashville": "Tennessee",
            "boulder": "Colorado", "cambridge": "Massachusetts",
            "pittsburgh": "Pennsylvania", "ann arbor": "Michigan",
            "madison": "Wisconsin", "irvine": "California",
            "santa monica": "California", "cupertino": "California",
            "bellevue": "Washington", "arlington": "Virginia",
            "reston": "Virginia", "mclean": "Virginia",
            "bethesda": "Maryland", "columbia": "Maryland",
            "st louis": "Missouri", "kansas city": "Missouri",
            "indianapolis": "Indiana", "columbus": "Ohio",
            "cleveland": "Ohio", "cincinnati": "Ohio",
            "tampa": "Florida", "orlando": "Florida",
            "durham": "North Carolina", "chapel hill": "North Carolina",
            "remote": None,
        }

        for part in parts:
            part_lower = part.strip().lower()
            if part_lower in us_cities and us_cities[part_lower]:
                return part.strip(), us_cities[part_lower]

        return None, None

    def _parse_address_string(self, addr: str) -> Tuple[Optional[str], Optional[str]]:
        """Parse a US address string into city and state.

        Handles:
        - "123 St, City, State ZIP" (e.g. "123 Main St, San Francisco, CA 94105")
        - "City, State ZIP" (e.g. "San Francisco, CA 94105")
        - "City, State ZIP+4" (e.g. "New York, NY 10001-1234")
        - "City, ST" (e.g. "Austin, TX")
        - "City, StateName" (e.g. "Austin, Texas")
        """
        if not addr:
            return None, None

        # Strategy: Find a state abbreviation or state name in the string,
        # then look backwards for the preceding city name.

        # Scan for 2-letter state abbreviation
        state_abbr_match = re.search(
            r',\s*([A-Z]{2})\b(?!\s*[a-z])', addr
        )
        if state_abbr_match:
            state = state_abbr_match.group(1)
            if self._is_us_state(state):
                # Everything before the comma is "prefix" — extract city from it
                prefix = addr[:state_abbr_match.start()].rstrip(', ')
                city = self._extract_city_from_prefix(prefix)
                if city:
                    return city, state

        # Scan for full state name after a comma
        # Sort by length descending to match longer names first (e.g. "North Carolina" before "North")
        sorted_states = sorted(
            ((name, abbr) for name, abbr in STATE_TO_ABBREV.items() if abbr),
            key=lambda x: len(x[0]), reverse=True
        )
        for state_name, abbr in sorted_states:
            # Find ALL matches and use the LAST one (closest to end of string)
            pattern = r',\s*\b' + re.escape(state_name) + r'\b'
            matches = list(re.finditer(pattern, addr, re.IGNORECASE))
            if matches:
                last_match = matches[-1]
                prefix = addr[:last_match.start()].rstrip(', ')
                city = self._extract_city_from_prefix(prefix)
                if city:
                    return city, abbr

        return None, None

    def _extract_city_from_prefix(self, prefix: str) -> Optional[str]:
        """Extract city name from the part of address before the state.

        Examples:
        - "123 Main St, San Francisco" → "San Francisco"
        - "San Francisco" → "San Francisco"
        - "100 Tech Blvd, Austin" → "Austin"
        """
        if not prefix:
            return None

        # Split by comma and take the last meaningful segment
        parts = [p.strip().strip('"\'') for p in prefix.split(',')]
        parts = [p for p in parts if p and not p.isdigit() and len(p) > 1]

        if not parts:
            return None

        # Last segment is usually the city
        city_candidate = parts[-1]

        # Remove leading street numbers if present (e.g. "100 Tech Blvd, Austin" → take "Austin")
        # But if the last segment starts with a digit, try the previous segment
        if city_candidate and city_candidate[0].isdigit():
            if len(parts) >= 2:
                city_candidate = parts[-2]

        # Clean up
        city_candidate = city_candidate.strip().strip(',.;')

        # Validate: should be at least 2 chars, not a number
        if len(city_candidate) >= 2 and not city_candidate.isdigit():
            return city_candidate

        return None