mirror of
https://github.com/thecyberlearn/chat-backend.git
synced 2026-08-18 18:32:54 +00:00
a0fce0b484
Features: - Django REST API backend - Multi-strategy web scraping system (Beautiful Soup, Playwright, Firecrawl) - Anti-detection features (proxy rotation, user-agent rotation) - Data export functionality (JSON, CSV, TXT) - Business and CrawledPage models - Enhanced crawling service with fallback mechanisms 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude <noreply@anthropic.com>
55 lines
2.0 KiB
Python
55 lines
2.0 KiB
Python
from django.db.models.sql.compiler import (
|
|
SQLAggregateCompiler,
|
|
SQLCompiler,
|
|
SQLDeleteCompiler,
|
|
)
|
|
from django.db.models.sql.compiler import SQLInsertCompiler as BaseSQLInsertCompiler
|
|
from django.db.models.sql.compiler import SQLUpdateCompiler
|
|
|
|
__all__ = [
|
|
"SQLAggregateCompiler",
|
|
"SQLCompiler",
|
|
"SQLDeleteCompiler",
|
|
"SQLInsertCompiler",
|
|
"SQLUpdateCompiler",
|
|
]
|
|
|
|
|
|
class InsertUnnest(list):
|
|
"""
|
|
Sentinel value to signal DatabaseOperations.bulk_insert_sql() that the
|
|
UNNEST strategy should be used for the bulk insert.
|
|
"""
|
|
|
|
def __str__(self):
|
|
return "UNNEST(%s)" % ", ".join(self)
|
|
|
|
|
|
class SQLInsertCompiler(BaseSQLInsertCompiler):
|
|
def assemble_as_sql(self, fields, value_rows):
|
|
# Specialize bulk-insertion of literal values through UNNEST to
|
|
# reduce the time spent planning the query.
|
|
if (
|
|
# The optimization is not worth doing if there is a single
|
|
# row as it will result in the same number of placeholders.
|
|
len(value_rows) <= 1
|
|
# Lack of fields denote the usage of the DEFAULT keyword
|
|
# for the insertion of empty rows.
|
|
or any(field is None for field in fields)
|
|
# Fields that don't use standard internal types might not be
|
|
# unnest'able (e.g. array and geometry types are known to be
|
|
# problematic).
|
|
or any(
|
|
(field.target_field if field.is_relation else field).get_internal_type()
|
|
not in self.connection.data_types
|
|
for field in fields
|
|
)
|
|
# Compilable cannot be combined in an array of literal values.
|
|
or any(any(hasattr(value, "as_sql") for value in row) for row in value_rows)
|
|
):
|
|
return super().assemble_as_sql(fields, value_rows)
|
|
db_types = [field.db_type(self.connection) for field in fields]
|
|
return InsertUnnest(["(%%s)::%s[]" % db_type for db_type in db_types]), [
|
|
list(map(list, zip(*value_rows)))
|
|
]
|