mirror of
https://github.com/thecyberlearn/chat-backend.git
synced 2026-08-18 21:52:53 +00:00
a0fce0b484
Features: - Django REST API backend - Multi-strategy web scraping system (Beautiful Soup, Playwright, Firecrawl) - Anti-detection features (proxy rotation, user-agent rotation) - Data export functionality (JSON, CSV, TXT) - Business and CrawledPage models - Enhanced crawling service with fallback mechanisms 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude <noreply@anthropic.com>
29 lines
1.0 KiB
Python
29 lines
1.0 KiB
Python
import urllib.parse
|
|
|
|
|
|
class PackageIndex:
|
|
"""Represents a Package Index and provides easier access to endpoints"""
|
|
|
|
__slots__ = ["url", "netloc", "simple_url", "pypi_url", "file_storage_domain"]
|
|
|
|
def __init__(self, url: str, file_storage_domain: str) -> None:
|
|
super().__init__()
|
|
self.url = url
|
|
self.netloc = urllib.parse.urlsplit(url).netloc
|
|
self.simple_url = self._url_for_path("simple")
|
|
self.pypi_url = self._url_for_path("pypi")
|
|
|
|
# This is part of a temporary hack used to block installs of PyPI
|
|
# packages which depend on external urls only necessary until PyPI can
|
|
# block such packages themselves
|
|
self.file_storage_domain = file_storage_domain
|
|
|
|
def _url_for_path(self, path: str) -> str:
|
|
return urllib.parse.urljoin(self.url, path)
|
|
|
|
|
|
PyPI = PackageIndex("https://pypi.org/", file_storage_domain="files.pythonhosted.org")
|
|
TestPyPI = PackageIndex(
|
|
"https://test.pypi.org/", file_storage_domain="test-files.pythonhosted.org"
|
|
)
|