mirror of
https://github.com/thecyberlearn/chat-backend.git
synced 2026-08-18 14:52:53 +00:00
a0fce0b484
Features: - Django REST API backend - Multi-strategy web scraping system (Beautiful Soup, Playwright, Firecrawl) - Anti-detection features (proxy rotation, user-agent rotation) - Data export functionality (JSON, CSV, TXT) - Business and CrawledPage models - Enhanced crawling service with fallback mechanisms 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude <noreply@anthropic.com>
44 lines
1.1 KiB
Python
44 lines
1.1 KiB
Python
from __future__ import annotations
|
|
|
|
import typing
|
|
|
|
from .url import Url
|
|
|
|
if typing.TYPE_CHECKING:
|
|
from ..connection import ProxyConfig
|
|
|
|
|
|
def connection_requires_http_tunnel(
|
|
proxy_url: Url | None = None,
|
|
proxy_config: ProxyConfig | None = None,
|
|
destination_scheme: str | None = None,
|
|
) -> bool:
|
|
"""
|
|
Returns True if the connection requires an HTTP CONNECT through the proxy.
|
|
|
|
:param URL proxy_url:
|
|
URL of the proxy.
|
|
:param ProxyConfig proxy_config:
|
|
Proxy configuration from poolmanager.py
|
|
:param str destination_scheme:
|
|
The scheme of the destination. (i.e https, http, etc)
|
|
"""
|
|
# If we're not using a proxy, no way to use a tunnel.
|
|
if proxy_url is None:
|
|
return False
|
|
|
|
# HTTP destinations never require tunneling, we always forward.
|
|
if destination_scheme == "http":
|
|
return False
|
|
|
|
# Support for forwarding with HTTPS proxies and HTTPS destinations.
|
|
if (
|
|
proxy_url.scheme == "https"
|
|
and proxy_config
|
|
and proxy_config.use_forwarding_for_https
|
|
):
|
|
return False
|
|
|
|
# Otherwise always use a tunnel.
|
|
return True
|