mirror of
https://github.com/thecyberlearn/chat-backend.git
synced 2026-08-18 15:32:53 +00:00
a0fce0b484
Features: - Django REST API backend - Multi-strategy web scraping system (Beautiful Soup, Playwright, Firecrawl) - Anti-detection features (proxy rotation, user-agent rotation) - Data export functionality (JSON, CSV, TXT) - Business and CrawledPage models - Enhanced crawling service with fallback mechanisms 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude <noreply@anthropic.com>
42 lines
1.3 KiB
Python
42 lines
1.3 KiB
Python
"""Utilities for working with paths."""
|
|
|
|
from collections.abc import Sequence
|
|
from contextlib import suppress
|
|
|
|
|
|
def normalize_path_segments(segments: Sequence[str]) -> list[str]:
|
|
"""Drop '.' and '..' from a sequence of str segments"""
|
|
|
|
resolved_path: list[str] = []
|
|
|
|
for seg in segments:
|
|
if seg == "..":
|
|
# ignore any .. segments that would otherwise cause an
|
|
# IndexError when popped from resolved_path if
|
|
# resolving for rfc3986
|
|
with suppress(IndexError):
|
|
resolved_path.pop()
|
|
elif seg != ".":
|
|
resolved_path.append(seg)
|
|
|
|
if segments and segments[-1] in (".", ".."):
|
|
# do some post-processing here.
|
|
# if the last segment was a relative dir,
|
|
# then we need to append the trailing '/'
|
|
resolved_path.append("")
|
|
|
|
return resolved_path
|
|
|
|
|
|
def normalize_path(path: str) -> str:
|
|
# Drop '.' and '..' from str path
|
|
prefix = ""
|
|
if path and path[0] == "/":
|
|
# preserve the "/" root element of absolute paths, copying it to the
|
|
# normalised output as per sections 5.2.4 and 6.2.2.3 of rfc3986.
|
|
prefix = "/"
|
|
path = path[1:]
|
|
|
|
segments = path.split("/")
|
|
return prefix + "/".join(normalize_path_segments(segments))
|