mirror of
https://github.com/thecyberlearn/chat-backend.git
synced 2026-08-18 23:12:53 +00:00
a0fce0b484
Features: - Django REST API backend - Multi-strategy web scraping system (Beautiful Soup, Playwright, Firecrawl) - Anti-detection features (proxy rotation, user-agent rotation) - Data export functionality (JSON, CSV, TXT) - Business and CrawledPage models - Enhanced crawling service with fallback mechanisms 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude <noreply@anthropic.com>
64 lines
1.9 KiB
Python
64 lines
1.9 KiB
Python
"""
|
|
Scraping functionality for Firecrawl v2 API.
|
|
"""
|
|
|
|
from typing import Optional, Dict, Any
|
|
from ..types import ScrapeOptions, Document
|
|
from ..utils.normalize import normalize_document_input
|
|
from ..utils import HttpClient, handle_response_error, prepare_scrape_options, validate_scrape_options
|
|
|
|
|
|
def _prepare_scrape_request(url: str, options: Optional[ScrapeOptions] = None) -> Dict[str, Any]:
|
|
"""
|
|
Prepare a scrape request payload for v2 API.
|
|
|
|
Args:
|
|
url: URL to scrape
|
|
options: ScrapeOptions (snake_case) to convert and include
|
|
|
|
Returns:
|
|
Request payload dictionary with camelCase fields
|
|
"""
|
|
if not url or not url.strip():
|
|
raise ValueError("URL cannot be empty")
|
|
|
|
request_data: Dict[str, Any] = {"url": url.strip()}
|
|
|
|
if options is not None:
|
|
validated = validate_scrape_options(options)
|
|
if validated is not None:
|
|
opts = prepare_scrape_options(validated)
|
|
if opts:
|
|
request_data.update(opts)
|
|
|
|
return request_data
|
|
|
|
def scrape(client: HttpClient, url: str, options: Optional[ScrapeOptions] = None) -> Document:
|
|
"""
|
|
Scrape a single URL and return the document.
|
|
|
|
The v2 API returns: { success: boolean, data: Document }
|
|
We surface just the Document to callers.
|
|
|
|
Args:
|
|
client: HTTP client instance
|
|
url: URL to scrape
|
|
options: Scraping options (snake_case)
|
|
|
|
Returns:
|
|
Document
|
|
"""
|
|
payload = _prepare_scrape_request(url, options)
|
|
|
|
response = client.post("/v2/scrape", payload)
|
|
|
|
if not response.ok:
|
|
handle_response_error(response, "scrape")
|
|
|
|
body = response.json()
|
|
if not body.get("success"):
|
|
raise Exception(body.get("error", "Unknown error occurred"))
|
|
|
|
document_data = body.get("data", {})
|
|
normalized = normalize_document_input(document_data)
|
|
return Document(**normalized) |