chat-backend/scraping/proxy_manager.py

"""
Proxy rotation and anti-detection utilities
"""
import random
import time
from typing import List, Dict, Optional
import logging

logger = logging.getLogger(__name__)


class ProxyRotator:
    """Manage proxy rotation for web scraping"""

    def __init__(self, proxies: List[str] = None):
        """
        Initialize proxy rotator

        Args:
            proxies: List of proxy URLs in format 'http://user:pass@host:port'
        """
        self.proxies = proxies or []
        self.current_index = 0
        self.failed_proxies = set()

    def get_next_proxy(self) -> Optional[Dict[str, str]]:
        """Get next working proxy"""
        if not self.proxies:
            return None

        available_proxies = [p for i, p in enumerate(self.proxies)
                           if i not in self.failed_proxies]

        if not available_proxies:
            # Reset failed proxies if all failed
            self.failed_proxies.clear()
            available_proxies = self.proxies

        proxy_url = random.choice(available_proxies)

        return {
            'http': proxy_url,
            'https': proxy_url
        }

    def mark_proxy_failed(self, proxy_url: str):
        """Mark a proxy as failed"""
        try:
            index = self.proxies.index(proxy_url)
            self.failed_proxies.add(index)
        except ValueError:
            pass


class UserAgentRotator:
    """Rotate user agents to avoid detection"""

    def __init__(self):
        self.user_agents = [
            'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
            'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
            'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:89.0) Gecko/20100101 Firefox/89.0',
            'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.1 Safari/605.1.15',
            'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Edge/91.0.864.59',
            'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
        ]

    def get_random_user_agent(self) -> str:
        """Get a random user agent"""
        return random.choice(self.user_agents)


class AntiDetectionManager:
    """Manage anti-detection techniques"""

    def __init__(self, proxies: List[str] = None, enable_delays: bool = True):
        self.proxy_rotator = ProxyRotator(proxies)
        self.user_agent_rotator = UserAgentRotator()
        self.enable_delays = enable_delays

    def get_session_config(self) -> Dict:
        """Get configuration for a scraping session"""
        config = {
            'headers': {
                'User-Agent': self.user_agent_rotator.get_random_user_agent(),
                'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
                'Accept-Language': 'en-US,en;q=0.5',
                'Accept-Encoding': 'gzip, deflate',
                'Connection': 'keep-alive',
                'Upgrade-Insecure-Requests': '1',
            }
        }

        # Add proxy if available
        proxy = self.proxy_rotator.get_next_proxy()
        if proxy:
            config['proxies'] = proxy

        return config

    def add_random_delay(self, min_delay: float = 1.0, max_delay: float = 3.0):
        """Add random delay between requests"""
        if self.enable_delays:
            delay = random.uniform(min_delay, max_delay)
            logger.debug(f"Adding {delay:.2f}s delay")
            time.sleep(delay)

    def get_playwright_config(self) -> Dict:
        """Get Playwright-specific configuration"""
        return {
            'user_agent': self.user_agent_rotator.get_random_user_agent(),
            'viewport': random.choice([
                {'width': 1920, 'height': 1080},
                {'width': 1366, 'height': 768},
                {'width': 1440, 'height': 900},
                {'width': 1536, 'height': 864},
            ]),
            'extra_http_headers': {
                'Accept-Language': 'en-US,en;q=0.9',
                'Accept-Encoding': 'gzip, deflate, br',
            }
        }


# Example usage configurations
FREE_PROXY_LIST = [
    # Add free proxies here (not recommended for production)
    # 'http://proxy1:port',
    # 'http://proxy2:port',
]

PREMIUM_PROXY_SERVICES = {
    'brightdata': {
        'endpoint': 'http://brd-customer-{customer_id}-zone-{zone}:{password}@zproxy.lum-superproxy.io:22225',
        'description': 'Bright Data residential proxies'
    },
    'oxylabs': {
        'endpoint': 'http://customer-{username}:{password}@pr.oxylabs.io:7777',
        'description': 'Oxylabs residential proxies'
    },
    'smartproxy': {
        'endpoint': 'http://{username}:{password}@gate.smartproxy.com:7000',
        'description': 'Smartproxy residential proxies'
    }
}


def create_anti_detection_manager(proxy_config: Optional[Dict] = None) -> AntiDetectionManager:
    """
    Factory function to create an anti-detection manager

    Args:
        proxy_config: Configuration for proxy service
                     {'service': 'brightdata', 'username': 'user', 'password': 'pass', 'customer_id': 'id'}
    """
    proxies = []

    if proxy_config:
        service = proxy_config.get('service')
        if service in PREMIUM_PROXY_SERVICES:
            # Configure premium proxy service
            endpoint = PREMIUM_PROXY_SERVICES[service]['endpoint']

            # Format the endpoint with user credentials
            formatted_endpoint = endpoint.format(
                username=proxy_config.get('username', ''),
                password=proxy_config.get('password', ''),
                customer_id=proxy_config.get('customer_id', ''),
                zone=proxy_config.get('zone', 'residential')
            )
            proxies = [formatted_endpoint]

    return AntiDetectionManager(proxies=proxies)