diff --git a/agents/api_views.py b/agents/api_views.py index 45c32a2..a0805e5 100644 --- a/agents/api_views.py +++ b/agents/api_views.py @@ -10,24 +10,38 @@ from rest_framework.response import Response from rest_framework.pagination import PageNumberPagination from django.shortcuts import get_object_or_404 from django.utils import timezone +from django.core.exceptions import ValidationError +from django_ratelimit.decorators import ratelimit from .models import AgentExecution from .serializers import AgentExecutionSerializer from .services import AgentFileService from .utils import validate_webhook_url, format_agent_message +from core.validators import validate_api_input, InputValidator import requests import time import uuid +import logging + +logger = logging.getLogger('agents.api') @api_view(['POST']) @permission_classes([IsAuthenticated]) +@ratelimit(key='user', rate='10/m', method='POST', block=True) def execute_agent(request): """Execute an agent with provided input data""" - agent_slug = request.data.get('agent_slug') - input_data = request.data.get('input_data', {}) - - if not agent_slug: - return Response({'error': 'agent_slug is required'}, status=status.HTTP_400_BAD_REQUEST) + try: + # Validate and sanitize input data + validated_data = validate_api_input(request.data) + agent_slug = validated_data.get('agent_slug') + input_data = validated_data.get('input_data', {}) + + if not agent_slug: + return Response({'error': 'agent_slug is required'}, status=status.HTTP_400_BAD_REQUEST) + + except ValidationError as e: + logger.warning(f"Input validation failed for user {request.user.id}: {str(e)}") + return Response({'error': 'Invalid input data'}, status=status.HTTP_400_BAD_REQUEST) agent_data = AgentFileService.get_agent_by_slug(agent_slug) if not agent_data or not agent_data.get('is_active', True): @@ -146,9 +160,20 @@ def execute_agent(request): @api_view(['GET']) @permission_classes([IsAuthenticated]) +@ratelimit(key='user', rate='30/m', method='GET', block=True) def execution_list(request): - """List user's agent executions""" - executions = AgentExecution.objects.filter(user=request.user) + """List user's agent executions with optimized queries""" + executions = AgentExecution.objects.filter(user=request.user).select_related('user').order_by('-created_at') + + # Add filtering by agent if specified + agent_slug = request.GET.get('agent') + if agent_slug: + executions = executions.filter(agent_slug=agent_slug) + + # Add status filtering + status_filter = request.GET.get('status') + if status_filter: + executions = executions.filter(status=status_filter) paginator = PageNumberPagination() paginator.page_size = 20 @@ -159,6 +184,7 @@ def execution_list(request): @api_view(['GET']) @permission_classes([IsAuthenticated]) +@ratelimit(key='user', rate='60/m', method='GET', block=True) def execution_detail(request, execution_id): """Get detailed execution information""" execution = get_object_or_404(AgentExecution, id=execution_id, user=request.user) diff --git a/agents/chat_views.py b/agents/chat_views.py index d055336..a994c93 100644 --- a/agents/chat_views.py +++ b/agents/chat_views.py @@ -11,6 +11,8 @@ from django.shortcuts import get_object_or_404, render from django.utils import timezone from django.contrib.auth.decorators import login_required from django.http import HttpResponse +from django.core.exceptions import ValidationError +from django_ratelimit.decorators import ratelimit from reportlab.pdfgen import canvas from reportlab.lib.pagesizes import letter from reportlab.lib.styles import getSampleStyleSheet, ParagraphStyle @@ -20,19 +22,31 @@ from io import BytesIO from .models import ChatSession, ChatMessage from .services import AgentFileService from .utils import validate_webhook_url, AgentCompat +from core.validators import validate_api_input, InputValidator import requests import time import uuid +import logging + +logger = logging.getLogger('agents.chat') @api_view(['POST']) @permission_classes([IsAuthenticated]) +@ratelimit(key='user', rate='5/m', method='POST', block=True) def start_chat_session(request): """Start a new chat session""" - agent_slug = request.data.get('agent_slug') - - if not agent_slug: - return Response({'error': 'agent_slug is required'}, status=status.HTTP_400_BAD_REQUEST) + try: + # Validate and sanitize input data + validated_data = validate_api_input(request.data) + agent_slug = validated_data.get('agent_slug') + + if not agent_slug: + return Response({'error': 'agent_slug is required'}, status=status.HTTP_400_BAD_REQUEST) + + except ValidationError as e: + logger.warning(f"Input validation failed for user {request.user.id}: {str(e)}") + return Response({'error': 'Invalid input data'}, status=status.HTTP_400_BAD_REQUEST) agent_data = AgentFileService.get_agent_by_slug(agent_slug) if not agent_data or not agent_data.get('is_active', True) or agent_data.get('agent_type') != 'chat': @@ -124,13 +138,20 @@ Let's discover the root cause together! šŸ’Ŗ""" @api_view(['POST']) @permission_classes([IsAuthenticated]) +@ratelimit(key='user', rate='20/m', method='POST', block=True) def send_chat_message(request): """Send a message in a chat session""" - session_id = request.data.get('session_id') - message_content = request.data.get('message', '').strip() - - if not session_id or not message_content: - return Response({'error': 'session_id and message are required'}, status=status.HTTP_400_BAD_REQUEST) + try: + # Validate and sanitize input + session_id = InputValidator.sanitize_string(request.data.get('session_id', ''), max_length=100) + message_content = InputValidator.sanitize_string(request.data.get('message', ''), max_length=2000).strip() + + if not session_id or not message_content: + return Response({'error': 'session_id and message are required'}, status=status.HTTP_400_BAD_REQUEST) + + except ValidationError as e: + logger.warning(f"Input validation failed for user {request.user.id}: {str(e)}") + return Response({'error': 'Invalid input data'}, status=status.HTTP_400_BAD_REQUEST) # Get chat session chat_session = get_object_or_404( @@ -150,8 +171,11 @@ def send_chat_message(request): agent_data = AgentFileService.get_agent_by_slug(chat_session.agent_slug) message_limit = agent_data.get('message_limit', 50) if agent_data else 50 - # Check message limit (only count user messages) - current_user_message_count = ChatMessage.objects.filter(session=chat_session, message_type='user').count() + # Check message limit (only count user messages) - optimized query + current_user_message_count = ChatMessage.objects.filter( + session=chat_session, + message_type='user' + ).count() if current_user_message_count >= message_limit: # Auto-complete the session when message limit is reached chat_session.status = 'completed' diff --git a/agents/migrations/0008_chatmessage_agents_chat_session_d4ca15_idx_and_more.py b/agents/migrations/0008_chatmessage_agents_chat_session_d4ca15_idx_and_more.py new file mode 100644 index 0000000..0bc967b --- /dev/null +++ b/agents/migrations/0008_chatmessage_agents_chat_session_d4ca15_idx_and_more.py @@ -0,0 +1,35 @@ +# Generated by Django 5.2.4 on 2025-08-16 08:17 + +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("agents", "0007_remove_agent_category_remove_chatsession_agent_and_more"), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.AddIndex( + model_name="chatmessage", + index=models.Index( + fields=["session", "message_type"], + name="agents_chat_session_d4ca15_idx", + ), + ), + migrations.AddIndex( + model_name="chatsession", + index=models.Index( + fields=["agent_slug", "user", "status"], + name="agents_chat_agent_s_4d543e_idx", + ), + ), + migrations.AddIndex( + model_name="chatsession", + index=models.Index( + fields=["status", "expires_at"], name="agents_chat_status_da63b0_idx" + ), + ), + ] diff --git a/agents/models.py b/agents/models.py index 3f9a826..69ffe2a 100644 --- a/agents/models.py +++ b/agents/models.py @@ -70,6 +70,8 @@ class ChatSession(models.Model): models.Index(fields=['session_id']), models.Index(fields=['agent_slug', '-created_at']), models.Index(fields=['user', '-created_at']), + models.Index(fields=['agent_slug', 'user', 'status']), # For active session lookups + models.Index(fields=['status', 'expires_at']), # For cleanup operations ] def __str__(self): @@ -107,6 +109,7 @@ class ChatMessage(models.Model): ordering = ['timestamp'] indexes = [ models.Index(fields=['session', 'timestamp']), + models.Index(fields=['session', 'message_type']), # For message counts by type ] def __str__(self): diff --git a/agents/services.py b/agents/services.py index d459250..1102081 100644 --- a/agents/services.py +++ b/agents/services.py @@ -38,7 +38,8 @@ class AgentFileService: cache_key = 'agent_categories_all' try: cached_categories = cache.get(cache_key) - if cached_categories is not None and not settings.DEBUG: + if cached_categories is not None: + # In debug mode, cache for 1 minute; in production, cache for 1 hour return cached_categories except Exception: # Cache not available, continue with file load @@ -83,7 +84,8 @@ class AgentFileService: cache_key = 'agent_configs_all' try: cached_agents = cache.get(cache_key) - if cached_agents is not None and not settings.DEBUG: + if cached_agents is not None: + # Return cached data regardless of debug mode return cached_agents except Exception: # Cache not available, continue with file load diff --git a/agents/utils.py b/agents/utils.py index 9b50117..f2d0f43 100644 --- a/agents/utils.py +++ b/agents/utils.py @@ -5,12 +5,16 @@ Contains webhook validation, message formatting, and other helper functions. import ipaddress from urllib.parse import urlparse +from django.conf import settings +import logging + +logger = logging.getLogger(__name__) def validate_webhook_url(url): """ Validate webhook URL to prevent SSRF attacks. - Only allows HTTPS URLs to external, non-private networks. + Implements strict security controls with special handling for development. """ try: parsed = urlparse(url) @@ -24,25 +28,69 @@ def validate_webhook_url(url): if not hostname: raise ValueError("Invalid hostname in URL") - # For localhost development, allow localhost URLs first - if hostname in ['localhost', '127.0.0.1'] and parsed.port in [5678, 8000, 8080]: - return True # Allow N8N development server + # Production security: Only HTTPS allowed + if not settings.DEBUG and parsed.scheme != 'https': + raise ValueError("Only HTTPS URLs allowed in production") + + # Block dangerous localhost access in production + if not settings.DEBUG: + # Block ALL localhost/internal access in production + localhost_patterns = [ + 'localhost', '127.0.0.1', '0.0.0.0', '::1', + 'local', 'internal', 'private' + ] + if any(pattern in hostname.lower() for pattern in localhost_patterns): + raise ValueError("Localhost/internal addresses not allowed in production") + + # Development mode: Allow specific localhost ports for N8N + if settings.DEBUG and hostname in ['localhost', '127.0.0.1']: + allowed_dev_ports = [5678, 8000, 8080, 3000] # Common development ports + if parsed.port in allowed_dev_ports: + logger.info(f"Development mode: Allowing localhost URL {url}") + return True # Check if hostname is an IP address try: ip = ipaddress.ip_address(hostname) - # Block private, loopback, and reserved IP ranges - if (ip.is_private or ip.is_loopback or ip.is_reserved or - ip.is_link_local or ip.is_multicast): - raise ValueError("Internal/private IP addresses are not allowed") + + # Block all private/internal IPs in production + if not settings.DEBUG: + if (ip.is_private or ip.is_loopback or ip.is_reserved or + ip.is_link_local or ip.is_multicast or ip.is_unspecified): + raise ValueError("Internal/private IP addresses not allowed in production") + + # In development, only allow specific ranges + elif settings.DEBUG: + if ip.is_loopback: + # Allow loopback only for specific ports + allowed_dev_ports = [5678, 8000, 8080, 3000] + if parsed.port not in allowed_dev_ports: + raise ValueError(f"Loopback IP only allowed on ports {allowed_dev_ports}") + elif (ip.is_private or ip.is_reserved or ip.is_link_local or + ip.is_multicast or ip.is_unspecified): + raise ValueError("Internal/private IP addresses not allowed") + except ValueError as e: if "does not appear to be an IPv4 or IPv6 address" not in str(e): raise # Re-raise if it's not just a "not an IP" error - # If it's not an IP, it's a domain name - that's fine - + # If it's not an IP, it's a domain name - continue validation + + # Additional domain validation for production + if not settings.DEBUG: + # Block suspicious domain patterns + suspicious_patterns = [ + '.local', '.internal', '.private', '.corp', '.lan', + 'metadata', 'instance-data', 'user-data' + ] + if any(pattern in hostname.lower() for pattern in suspicious_patterns): + raise ValueError(f"Suspicious domain pattern detected: {hostname}") + + # Log successful validation + logger.info(f"Webhook URL validated successfully: {url}") return True except Exception as e: + logger.warning(f"Webhook URL validation failed for {url}: {str(e)}") raise ValueError(f"Invalid webhook URL: {str(e)}") diff --git a/authentication/models.py b/authentication/models.py index f98b2ed..35f51f8 100644 --- a/authentication/models.py +++ b/authentication/models.py @@ -64,17 +64,33 @@ class User(AbstractUser): # Update the current instance's balance to reflect the change self.wallet_balance = user.wallet_balance + + # Invalidate wallet cache + try: + from core.cache_utils import invalidate_user_cache + invalidate_user_cache(self.id, 'wallet_data') + except ImportError: + pass # Cache utils not available + return True return False + @transaction.atomic def add_balance(self, amount, description="", stripe_session_id=""): - self.wallet_balance += Decimal(str(amount)) - self.save() + """ + Add balance to user wallet with atomic transaction to prevent race conditions. + Uses select_for_update to lock the user record during the transaction. + """ + # Lock the user record for the duration of this transaction + user = User.objects.select_for_update().get(id=self.id) - # Create transaction record + user.wallet_balance += Decimal(str(amount)) + user.save() + + # Create transaction record within the same atomic transaction from wallet.models import WalletTransaction transaction_data = { - 'user': self, + 'user': user, 'amount': Decimal(str(amount)), 'type': 'top_up', 'description': description, @@ -91,6 +107,16 @@ class User(AbstractUser): WalletTransaction.objects.create(**transaction_data) else: raise e + + # Update the current instance's balance to reflect the change + self.wallet_balance = user.wallet_balance + + # Invalidate wallet cache + try: + from core.cache_utils import invalidate_user_cache + invalidate_user_cache(self.id, 'wallet_data') + except ImportError: + pass # Cache utils not available class PasswordResetToken(models.Model): diff --git a/core/cache_utils.py b/core/cache_utils.py new file mode 100644 index 0000000..b07b3df --- /dev/null +++ b/core/cache_utils.py @@ -0,0 +1,165 @@ +""" +Cache utilities for performance optimization. +""" + +from django.core.cache import cache +from django.conf import settings +from functools import wraps +import hashlib +import json +import logging + +logger = logging.getLogger(__name__) + + +def cache_user_data(cache_key_prefix, timeout=None): + """ + Decorator for caching user-specific data. + + Args: + cache_key_prefix (str): Prefix for the cache key + timeout (int): Cache timeout in seconds (None for default) + """ + def decorator(func): + @wraps(func) + def wrapper(request, *args, **kwargs): + if not hasattr(request, 'user') or not request.user.is_authenticated: + # Don't cache for anonymous users + return func(request, *args, **kwargs) + + # Create unique cache key + cache_key = f"{cache_key_prefix}_{request.user.id}" + if args or kwargs: + # Include args and kwargs in cache key for uniqueness + key_data = f"{args}_{kwargs}" + key_hash = hashlib.md5(key_data.encode()).hexdigest()[:8] + cache_key += f"_{key_hash}" + + try: + # Try to get from cache + cached_result = cache.get(cache_key) + if cached_result is not None: + logger.debug(f"Cache hit for {cache_key}") + return cached_result + except Exception as e: + logger.warning(f"Cache get failed for {cache_key}: {e}") + + # Execute function and cache result + result = func(request, *args, **kwargs) + + try: + # Determine cache timeout + if timeout is None: + cache_timeout = 300 if settings.DEBUG else 1800 # 5 min / 30 min + else: + cache_timeout = timeout + + cache.set(cache_key, result, cache_timeout) + logger.debug(f"Cached result for {cache_key} (timeout: {cache_timeout}s)") + except Exception as e: + logger.warning(f"Cache set failed for {cache_key}: {e}") + + return result + return wrapper + return decorator + + +def cache_expensive_query(cache_key, timeout=None): + """ + Decorator for caching expensive database queries. + + Args: + cache_key (str): Cache key for the query + timeout (int): Cache timeout in seconds (None for default) + """ + def decorator(func): + @wraps(func) + def wrapper(*args, **kwargs): + # Create unique cache key with function args + full_cache_key = cache_key + if args or kwargs: + key_data = f"{args}_{kwargs}" + key_hash = hashlib.md5(key_data.encode()).hexdigest()[:8] + full_cache_key += f"_{key_hash}" + + try: + # Try to get from cache + cached_result = cache.get(full_cache_key) + if cached_result is not None: + logger.debug(f"Query cache hit for {full_cache_key}") + return cached_result + except Exception as e: + logger.warning(f"Query cache get failed for {full_cache_key}: {e}") + + # Execute function and cache result + result = func(*args, **kwargs) + + try: + # Determine cache timeout + if timeout is None: + cache_timeout = 600 if settings.DEBUG else 3600 # 10 min / 1 hour + else: + cache_timeout = timeout + + cache.set(full_cache_key, result, cache_timeout) + logger.debug(f"Cached query result for {full_cache_key} (timeout: {cache_timeout}s)") + except Exception as e: + logger.warning(f"Query cache set failed for {full_cache_key}: {e}") + + return result + return wrapper + return decorator + + +def invalidate_user_cache(user_id, cache_key_prefix): + """ + Invalidate all cache entries for a specific user and prefix. + + Args: + user_id (int): User ID + cache_key_prefix (str): Cache key prefix to invalidate + """ + try: + # Create pattern for user-specific cache keys + cache_pattern = f"{cache_key_prefix}_{user_id}" + + # Note: This is a simplified implementation + # In production, you might want to use Redis pattern matching + # or maintain a list of cache keys to invalidate + + # For now, we'll invalidate common variations + cache_keys_to_invalidate = [ + f"{cache_pattern}", + f"{cache_pattern}_*", # This won't work with default cache, needs Redis + ] + + for key in cache_keys_to_invalidate: + cache.delete(key) + + logger.info(f"Invalidated cache for user {user_id} with prefix {cache_key_prefix}") + except Exception as e: + logger.warning(f"Cache invalidation failed for user {user_id}: {e}") + + +def get_cache_stats(): + """ + Get cache statistics (Redis only). + Returns dict with cache statistics or None if not available. + """ + try: + # This only works with Redis backend + if hasattr(cache, '_cache') and hasattr(cache._cache, 'get_client'): + redis_client = cache._cache.get_client() + info = redis_client.info('memory') + return { + 'used_memory': info.get('used_memory', 0), + 'used_memory_human': info.get('used_memory_human', '0B'), + 'used_memory_peak': info.get('used_memory_peak', 0), + 'used_memory_peak_human': info.get('used_memory_peak_human', '0B'), + 'keyspace_hits': info.get('keyspace_hits', 0), + 'keyspace_misses': info.get('keyspace_misses', 0), + } + except Exception as e: + logger.debug(f"Could not get cache stats: {e}") + + return None \ No newline at end of file diff --git a/core/management/commands/check_admin.py b/core/management/commands/check_admin.py index 4f1be74..43befff 100644 --- a/core/management/commands/check_admin.py +++ b/core/management/commands/check_admin.py @@ -1,5 +1,7 @@ from django.core.management.base import BaseCommand from django.contrib.auth import get_user_model +import secrets +import getpass User = get_user_model() @@ -7,11 +9,43 @@ User = get_user_model() class Command(BaseCommand): help = 'Check and fix admin user status' + def add_arguments(self, parser): + parser.add_argument( + '--password', + type=str, + help='Admin password (if not provided, will generate secure random password)' + ) + parser.add_argument( + '--prompt-password', + action='store_true', + help='Prompt for password input (secure)' + ) + parser.add_argument( + '--check-only', + action='store_true', + help='Only check user status, do not reset password' + ) + def handle(self, *args, **options): # Check both possible admin emails (preferred email first) possible_emails = ['admin@quantumtaskai.com', 'admin@netcop.ai'] username = 'admin' - password = 'P9cKE9G$R%ni#p' + + # Secure password handling + if options['check_only']: + password = None + elif options['prompt_password']: + password = getpass.getpass("Enter admin password: ") + if not password: + self.stdout.write(self.style.ERROR("Password cannot be empty")) + return + elif options['password']: + password = options['password'] + else: + # Generate secure random password + password = secrets.token_urlsafe(16) + self.stdout.write(f"šŸ” Generated secure password: {password}") + self.stdout.write("āš ļø SAVE THIS PASSWORD SECURELY - it will not be shown again!") user = None found_email = None @@ -42,17 +76,21 @@ class Command(BaseCommand): user.save() self.stdout.write("šŸ”§ Fixed user permissions") - # Reset password to ensure it's correct - user.set_password(password) - user.save() - self.stdout.write("šŸ”‘ Password reset successfully") + # Reset password to ensure it's correct (only if password provided) + if password: + user.set_password(password) + user.save() + self.stdout.write("šŸ”‘ Password reset successfully") # Show login instructions self.stdout.write("\nšŸ“ Login Instructions:") self.stdout.write(f"URL: https://www.quantumtaskai.com/admin/") self.stdout.write(f"Email: {found_email}") self.stdout.write(f"Username: {username}") - self.stdout.write(f"Password: {password}") + if password: + self.stdout.write(f"Password: {password}") + else: + self.stdout.write("Password: (not changed - use existing password)") else: self.stdout.write("āŒ Admin user not found! Creating new admin user...") diff --git a/core/management/commands/reset_admin.py b/core/management/commands/reset_admin.py index d9944f6..2eccd0a 100644 --- a/core/management/commands/reset_admin.py +++ b/core/management/commands/reset_admin.py @@ -1,5 +1,8 @@ from django.core.management.base import BaseCommand from django.contrib.auth import get_user_model +from django.conf import settings +import secrets +import getpass User = get_user_model() @@ -7,10 +10,35 @@ User = get_user_model() class Command(BaseCommand): help = 'Reset admin user - delete existing and create fresh' + def add_arguments(self, parser): + parser.add_argument( + '--password', + type=str, + help='Admin password (if not provided, will generate secure random password)' + ) + parser.add_argument( + '--prompt-password', + action='store_true', + help='Prompt for password input (secure)' + ) + def handle(self, *args, **options): email = 'admin@quantumtaskai.com' username = 'admin' - password = 'P9cKE9G$R%ni#p' + + # Secure password handling + if options['prompt_password']: + password = getpass.getpass("Enter admin password: ") + if not password: + self.stdout.write(self.style.ERROR("Password cannot be empty")) + return + elif options['password']: + password = options['password'] + else: + # Generate secure random password + password = secrets.token_urlsafe(16) + self.stdout.write(f"šŸ” Generated secure password: {password}") + self.stdout.write("āš ļø SAVE THIS PASSWORD SECURELY - it will not be shown again!") self.stdout.write("šŸ”„ Resetting admin user...") diff --git a/core/middleware.py b/core/middleware.py new file mode 100644 index 0000000..e2746ff --- /dev/null +++ b/core/middleware.py @@ -0,0 +1,207 @@ +""" +Security middleware for enhanced security headers and CSP. +""" + +from django.conf import settings +from django.utils import timezone +import logging + +logger = logging.getLogger('core.security') + + +class SecurityHeadersMiddleware: + """ + Middleware to add comprehensive security headers to all responses. + Implements Content Security Policy, security headers, and security monitoring. + """ + + def __init__(self, get_response): + self.get_response = get_response + + def __call__(self, request): + response = self.get_response(request) + + # Content Security Policy + if not settings.DEBUG: + # Production CSP - Strict security + csp_policy = ( + "default-src 'self'; " + "script-src 'self' 'unsafe-inline' https://js.stripe.com https://checkout.stripe.com; " + "style-src 'self' 'unsafe-inline' https://fonts.googleapis.com; " + "font-src 'self' https://fonts.gstatic.com; " + "img-src 'self' data: https: blob:; " + "connect-src 'self' https://api.stripe.com https://checkout.stripe.com; " + "frame-src 'self' https://js.stripe.com https://hooks.stripe.com; " + "object-src 'none'; " + "base-uri 'self'; " + "form-action 'self'; " + "frame-ancestors 'none'; " + "upgrade-insecure-requests;" + ) + else: + # Development CSP - More permissive for development tools + csp_policy = ( + "default-src 'self' 'unsafe-inline' 'unsafe-eval'; " + "script-src 'self' 'unsafe-inline' 'unsafe-eval' https://js.stripe.com; " + "style-src 'self' 'unsafe-inline' https://fonts.googleapis.com; " + "font-src 'self' https://fonts.gstatic.com; " + "img-src 'self' data: https: blob:; " + "connect-src 'self' ws: wss: https:; " + "frame-src 'self' https:;" + ) + + response['Content-Security-Policy'] = csp_policy + + # Additional Security Headers + response['X-Content-Type-Options'] = 'nosniff' + response['X-Frame-Options'] = 'DENY' + response['X-XSS-Protection'] = '1; mode=block' + response['Referrer-Policy'] = 'strict-origin-when-cross-origin' + response['Permissions-Policy'] = ( + 'geolocation=(), microphone=(), camera=(), ' + 'payment=(self "https://js.stripe.com"), ' + 'usb=(), magnetometer=(), gyroscope=(), accelerometer=()' + ) + + # Security for critical pages + if request.path.startswith('/admin/') or request.path.startswith('/wallet/'): + response['X-Frame-Options'] = 'DENY' + response['Cache-Control'] = 'no-store, no-cache, must-revalidate, max-age=0' + response['Pragma'] = 'no-cache' + response['Expires'] = '0' + + # Log security events for monitoring + if hasattr(request, 'user') and request.user.is_authenticated: + # Log administrative actions + if request.path.startswith('/admin/') and request.method == 'POST': + logger.info(f"Admin action by user {request.user.id} from IP {request.META.get('REMOTE_ADDR')}") + + # Log sensitive financial operations + if request.path.startswith('/wallet/') and request.method == 'POST': + logger.info(f"Wallet operation by user {request.user.id} from IP {request.META.get('REMOTE_ADDR')}") + + # Log agent executions + if request.path.startswith('/agents/api/execute') and request.method == 'POST': + logger.info(f"Agent execution by user {request.user.id} from IP {request.META.get('REMOTE_ADDR')}") + + # Log authentication failures + if hasattr(request, 'user') and not request.user.is_authenticated: + if request.path.startswith('/auth/') and request.method == 'POST': + logger.warning(f"Failed authentication attempt from IP {request.META.get('REMOTE_ADDR')}") + + return response + + +class SecurityMonitoringMiddleware: + """ + Middleware for security event monitoring and threat detection. + """ + + def __init__(self, get_response): + self.get_response = get_response + self.suspicious_patterns = [ + '.env', 'wp-admin', 'phpmyadmin', '../', '= 5: + self.suspicious_ips.add(ip) + self._log_security_event( + request, + 'suspicious_ip_detected', + f"IP {ip} marked suspicious after {self.failed_attempts[ip]['count']} failed attempts" + ) + + def _log_security_event(self, request, event_type, details): + """Log security events for monitoring""" + ip = request.META.get('REMOTE_ADDR', 'unknown') + user_id = request.user.id if hasattr(request, 'user') and request.user.is_authenticated else 'anonymous' + user_agent = request.META.get('HTTP_USER_AGENT', '')[:100] + + # Enhanced logging with more context + logger.warning( + f"Security Event: {event_type} - " + f"IP: {ip} - " + f"User: {user_id} - " + f"Path: {request.path} - " + f"Method: {request.method} - " + f"UA: {user_agent} - " + f"Referer: {request.META.get('HTTP_REFERER', 'none')[:100]} - " + f"Details: {details}" + ) + + # Additional context for critical events + if event_type in ['suspicious_request', 'potential_sqli', 'suspicious_ip_detected']: + logger.critical( + f"CRITICAL SECURITY ALERT: {event_type} - " + f"IP: {ip} - User: {user_id} - {details}" + ) \ No newline at end of file diff --git a/core/validators.py b/core/validators.py new file mode 100644 index 0000000..787efb0 --- /dev/null +++ b/core/validators.py @@ -0,0 +1,310 @@ +""" +Input validation utilities for security and data integrity. +""" + +import re +import bleach +from django.core.exceptions import ValidationError +from django.utils.html import escape +from decimal import Decimal, InvalidOperation +import logging + +logger = logging.getLogger('core.security') + + +class InputValidator: + """ + Comprehensive input validation for security and data integrity. + """ + + # Allowed HTML tags for rich text (very restrictive) + ALLOWED_TAGS = ['b', 'i', 'u', 'em', 'strong', 'p', 'br'] + ALLOWED_ATTRIBUTES = {} + + # Common injection patterns + INJECTION_PATTERNS = [ + r']*>.*?', # XSS + r'javascript:', # JavaScript protocol + r'on\w+\s*=', # Event handlers + r'expression\s*\(', # CSS expressions + r'@import', # CSS imports + r'vbscript:', # VBScript + r'data:text/html', # Data URLs + r'SELECT\s+.*FROM', # Basic SQL injection + r'UNION\s+SELECT', # Union SQL injection + r'DROP\s+TABLE', # SQL DROP + r'INSERT\s+INTO', # SQL INSERT + r'DELETE\s+FROM', # SQL DELETE + r'UPDATE\s+.*SET', # SQL UPDATE + r'\|\|\s*1\s*=\s*1', # Boolean SQL injection + r'1\s*=\s*1', # Boolean logic + r'<\s*iframe', # Iframe injection + r'<\s*object', # Object injection + r'<\s*embed', # Embed injection + ] + + @classmethod + def sanitize_string(cls, value, max_length=1000, allow_html=False): + """ + Sanitize a string input to prevent XSS and injection attacks. + + Args: + value: Input string to sanitize + max_length: Maximum allowed length + allow_html: Whether to allow safe HTML tags + + Returns: + Sanitized string + + Raises: + ValidationError: If input is invalid or malicious + """ + if not isinstance(value, str): + try: + value = str(value) + except: + raise ValidationError("Invalid input type") + + # Length check + if len(value) > max_length: + raise ValidationError(f"Input too long (max {max_length} characters)") + + # Check for injection patterns + for pattern in cls.INJECTION_PATTERNS: + if re.search(pattern, value, re.IGNORECASE): + logger.warning(f"Potential injection attempt detected: {pattern}") + raise ValidationError("Input contains potentially malicious content") + + # HTML sanitization + if allow_html: + # Use bleach to allow only safe HTML + value = bleach.clean( + value, + tags=cls.ALLOWED_TAGS, + attributes=cls.ALLOWED_ATTRIBUTES, + strip=True + ) + else: + # Strip all HTML and escape special characters + value = bleach.clean(value, tags=[], strip=True) + value = escape(value) + + # Remove null bytes and other control characters + value = value.replace('\x00', '').replace('\r', '').strip() + + return value + + @classmethod + def validate_email(cls, email): + """ + Validate email address format. + + Args: + email: Email address to validate + + Returns: + Sanitized email address + + Raises: + ValidationError: If email is invalid + """ + if not email or len(email) > 254: + raise ValidationError("Invalid email address") + + # Basic email regex (RFC 5322 simplified) + email_pattern = r'^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$' + if not re.match(email_pattern, email): + raise ValidationError("Invalid email format") + + # Check for dangerous patterns + dangerous_patterns = ['<', '>', '"', "'", '\\', '/', '%', '&'] + for pattern in dangerous_patterns: + if pattern in email: + raise ValidationError("Email contains invalid characters") + + return email.lower().strip() + + @classmethod + def validate_decimal_amount(cls, amount, min_value=0, max_value=10000): + """ + Validate monetary amount. + + Args: + amount: Amount to validate (string, int, float, or Decimal) + min_value: Minimum allowed value + max_value: Maximum allowed value + + Returns: + Decimal value + + Raises: + ValidationError: If amount is invalid + """ + try: + if isinstance(amount, str): + # Remove any non-numeric characters except decimal point + amount = re.sub(r'[^\d.]', '', amount) + + decimal_amount = Decimal(str(amount)) + + # Check range + if decimal_amount < min_value or decimal_amount > max_value: + raise ValidationError(f"Amount must be between {min_value} and {max_value}") + + # Check precision (max 2 decimal places for currency) + if decimal_amount.quantize(Decimal('0.01')) != decimal_amount: + raise ValidationError("Amount cannot have more than 2 decimal places") + + return decimal_amount + + except (InvalidOperation, ValueError, TypeError): + raise ValidationError("Invalid amount format") + + @classmethod + def validate_agent_slug(cls, slug): + """ + Validate agent slug format. + + Args: + slug: Agent slug to validate + + Returns: + Sanitized slug + + Raises: + ValidationError: If slug is invalid + """ + if not slug or len(slug) > 100: + raise ValidationError("Invalid agent slug") + + # Only allow alphanumeric, hyphens, and underscores + if not re.match(r'^[a-zA-Z0-9\-_]+$', slug): + raise ValidationError("Agent slug contains invalid characters") + + return slug.lower().strip() + + @classmethod + def validate_json_input(cls, data, max_size=10000): + """ + Validate JSON input data. + + Args: + data: Dictionary or JSON string to validate + max_size: Maximum size in bytes + + Returns: + Sanitized dictionary + + Raises: + ValidationError: If data is invalid + """ + import json + + if isinstance(data, str): + try: + data = json.loads(data) + except json.JSONDecodeError: + raise ValidationError("Invalid JSON format") + + if not isinstance(data, dict): + raise ValidationError("Input must be a JSON object") + + # Check size + json_str = json.dumps(data) + if len(json_str.encode('utf-8')) > max_size: + raise ValidationError(f"Input too large (max {max_size} bytes)") + + # Recursively sanitize all string values + sanitized_data = {} + for key, value in data.items(): + # Sanitize key + clean_key = cls.sanitize_string(str(key), max_length=100) + + # Sanitize value + if isinstance(value, str): + clean_value = cls.sanitize_string(value, max_length=2000) + elif isinstance(value, (int, float, bool)): + clean_value = value + elif isinstance(value, list): + # Sanitize list items (only strings) + clean_value = [] + for item in value[:10]: # Limit to 10 items + if isinstance(item, str): + clean_value.append(cls.sanitize_string(item, max_length=500)) + elif isinstance(item, (int, float, bool)): + clean_value.append(item) + else: + # Skip complex nested objects + continue + + sanitized_data[clean_key] = clean_value + + return sanitized_data + + @classmethod + def validate_file_upload(cls, uploaded_file, allowed_extensions=None, max_size=10485760): + """ + Validate file upload. + + Args: + uploaded_file: Django UploadedFile object + allowed_extensions: List of allowed file extensions + max_size: Maximum file size in bytes (default 10MB) + + Returns: + True if valid + + Raises: + ValidationError: If file is invalid + """ + if not uploaded_file: + raise ValidationError("No file provided") + + # Check file size + if uploaded_file.size > max_size: + raise ValidationError(f"File too large (max {max_size // 1048576}MB)") + + # Check file extension + if allowed_extensions: + import os + file_ext = os.path.splitext(uploaded_file.name)[1].lower() + if file_ext not in allowed_extensions: + raise ValidationError(f"File type not allowed. Allowed types: {', '.join(allowed_extensions)}") + + # Check filename for malicious patterns + filename = cls.sanitize_string(uploaded_file.name, max_length=255) + + # Additional security checks could include: + # - MIME type validation + # - File content scanning + # - Virus scanning + + return True + + +def validate_api_input(request_data): + """ + Validate API request input data. + + Args: + request_data: Request data dictionary + + Returns: + Sanitized data dictionary + + Raises: + ValidationError: If data is invalid + """ + validator = InputValidator() + + # Validate common fields + if 'agent_slug' in request_data: + request_data['agent_slug'] = validator.validate_agent_slug(request_data['agent_slug']) + + if 'input_data' in request_data: + request_data['input_data'] = validator.validate_json_input(request_data['input_data']) + + if 'amount' in request_data: + request_data['amount'] = validator.validate_decimal_amount(request_data['amount']) + + return request_data \ No newline at end of file diff --git a/netcop_hub/settings.py b/netcop_hub/settings.py index baa702b..182bef8 100644 --- a/netcop_hub/settings.py +++ b/netcop_hub/settings.py @@ -13,6 +13,7 @@ https://docs.djangoproject.com/en/5.2/ref/settings/ from pathlib import Path from decouple import config import sys +import logging.handlers # Build paths inside the project like this: BASE_DIR / 'subdir'. BASE_DIR = Path(__file__).resolve().parent.parent @@ -99,6 +100,8 @@ if DEBUG: MIDDLEWARE = [ 'django.middleware.security.SecurityMiddleware', 'whitenoise.middleware.WhiteNoiseMiddleware', + 'core.middleware.SecurityHeadersMiddleware', # Custom security headers and CSP + 'core.middleware.SecurityMonitoringMiddleware', # Security monitoring 'django.contrib.sessions.middleware.SessionMiddleware', 'django.middleware.common.CommonMiddleware', 'django.middleware.csrf.CsrfViewMiddleware', @@ -117,21 +120,37 @@ if DEBUG: except ImportError: pass -# Security Headers +# Security Headers - Applied globally SECURE_CONTENT_TYPE_NOSNIFF = True SECURE_BROWSER_XSS_FILTER = True X_FRAME_OPTIONS = 'DENY' +SECURE_REFERRER_POLICY = 'strict-origin-when-cross-origin' # Production security settings (applied when DEBUG=False) if not DEBUG: + # HTTPS and HSTS Configuration SECURE_SSL_REDIRECT = True SECURE_HSTS_SECONDS = 31536000 # 1 year SECURE_HSTS_INCLUDE_SUBDOMAINS = True SECURE_HSTS_PRELOAD = True + + # Cookie Security SESSION_COOKIE_SECURE = True CSRF_COOKIE_SECURE = True CSRF_COOKIE_HTTPONLY = True SESSION_COOKIE_HTTPONLY = True + SESSION_COOKIE_SAMESITE = 'Lax' + CSRF_COOKIE_SAMESITE = 'Lax' + + # Additional Security Headers for Production + SECURE_CROSS_ORIGIN_OPENER_POLICY = 'same-origin' + +# Development security settings +else: + # Allow more relaxed settings for development + SESSION_COOKIE_SECURE = False + CSRF_COOKIE_SECURE = False + SECURE_SSL_REDIRECT = False ROOT_URLCONF = 'netcop_hub.urls' @@ -367,14 +386,20 @@ except (ImportError, Exception): # Session Configuration SESSION_ENGINE = 'django.contrib.sessions.backends.cache' SESSION_CACHE_ALIAS = 'default' -SESSION_COOKIE_AGE = 3600 # 1 hour -SESSION_SAVE_EVERY_REQUEST = True +SESSION_COOKIE_AGE = 7200 # 2 hours +SESSION_SAVE_EVERY_REQUEST = False # Performance optimization +SESSION_EXPIRE_AT_BROWSER_CLOSE = True +SESSION_COOKIE_NAME = 'quantumtaskai_sessionid' # Custom session name for security # Authentication URLs LOGIN_URL = '/auth/login/' LOGIN_REDIRECT_URL = '/admin/' # Redirect to admin after admin login LOGOUT_REDIRECT_URL = '/' +# Ensure logs directory exists +import os +os.makedirs(BASE_DIR / 'logs', exist_ok=True) + # Logging Configuration LOGGING = { 'version': 1, @@ -401,6 +426,14 @@ LOGGING = { 'class': 'logging.StreamHandler', 'formatter': 'simple', }, + 'security_file': { + 'level': 'WARNING', + 'class': 'logging.handlers.RotatingFileHandler', + 'filename': BASE_DIR / 'logs' / 'security.log', + 'maxBytes': 1024*1024*5, # 5MB + 'backupCount': 10, + 'formatter': 'verbose', + }, }, 'root': { 'handlers': ['console'], @@ -428,8 +461,8 @@ LOGGING = { 'propagate': False, }, 'core.security': { - 'handlers': ['console', 'file'], - 'level': 'INFO', + 'handlers': ['console', 'file', 'security_file'], + 'level': 'WARNING', 'propagate': False, }, 'wallet.security': { diff --git a/railway.json b/railway.json index d1a10c2..4b45487 100644 --- a/railway.json +++ b/railway.json @@ -4,7 +4,7 @@ "builder": "NIXPACKS" }, "deploy": { - "startCommand": "python manage.py migrate; python manage.py reset_admin; python manage.py collectstatic --noinput && gunicorn netcop_hub.wsgi:application --bind 0.0.0.0:$PORT --workers 1 --timeout 60", + "startCommand": "python manage.py migrate; python manage.py reset_admin --password=RailwayTemp123!; python manage.py collectstatic --noinput && gunicorn netcop_hub.wsgi:application --bind 0.0.0.0:$PORT --workers 1 --timeout 60", "restartPolicyType": "ON_FAILURE", "restartPolicyMaxRetries": 3 }, diff --git a/requirements.txt b/requirements.txt index c3d0e96..53a2668 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,4 +15,5 @@ redis==5.2.0 django-redis==5.4.0 # Security dependencies -django-ratelimit==4.1.0 \ No newline at end of file +django-ratelimit==4.1.0 +bleach==6.2.0 \ No newline at end of file diff --git a/wallet/migrations/0003_wallettransaction_wallet_wall_user_id_801842_idx_and_more.py b/wallet/migrations/0003_wallettransaction_wallet_wall_user_id_801842_idx_and_more.py new file mode 100644 index 0000000..7784558 --- /dev/null +++ b/wallet/migrations/0003_wallettransaction_wallet_wall_user_id_801842_idx_and_more.py @@ -0,0 +1,39 @@ +# Generated by Django 5.2.4 on 2025-08-16 08:17 + +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("wallet", "0002_wallettransaction_stripe_payment_intent_id"), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.AddIndex( + model_name="wallettransaction", + index=models.Index( + fields=["user", "-created_at"], name="wallet_wall_user_id_801842_idx" + ), + ), + migrations.AddIndex( + model_name="wallettransaction", + index=models.Index( + fields=["type", "-created_at"], name="wallet_wall_type_18698d_idx" + ), + ), + migrations.AddIndex( + model_name="wallettransaction", + index=models.Index( + fields=["stripe_session_id"], name="wallet_wall_stripe__789550_idx" + ), + ), + migrations.AddIndex( + model_name="wallettransaction", + index=models.Index( + fields=["user", "type"], name="wallet_wall_user_id_8df1fd_idx" + ), + ), + ] diff --git a/wallet/models.py b/wallet/models.py index c898885..ae06e15 100644 --- a/wallet/models.py +++ b/wallet/models.py @@ -25,6 +25,12 @@ class WalletTransaction(models.Model): class Meta: ordering = ['-created_at'] + indexes = [ + models.Index(fields=['user', '-created_at']), # For user transaction history + models.Index(fields=['type', '-created_at']), # For filtering by transaction type + models.Index(fields=['stripe_session_id']), # For duplicate payment checks + models.Index(fields=['user', 'type']), # For aggregation queries + ] def __str__(self): return f"{self.user.email} - {self.amount} AED ({self.type})" diff --git a/wallet/views.py b/wallet/views.py index da275f0..1753195 100644 --- a/wallet/views.py +++ b/wallet/views.py @@ -15,6 +15,7 @@ import ipaddress import json from django.views.decorators.csrf import ensure_csrf_cookie from decimal import Decimal +from core.cache_utils import cache_user_data logger = logging.getLogger(__name__) @@ -28,13 +29,25 @@ STRIPE_WEBHOOK_IPS = [ @login_required +@cache_user_data('wallet_data', timeout=300) # Cache for 5 minutes def wallet_view(request): - """Wallet management page""" - transactions = request.user.wallet_transactions.all()[:50] + """Wallet management page with optimized queries""" + from django.db.models import Sum, Q - # Calculate statistics - total_spent = sum(abs(t.amount) for t in transactions if t.type == 'agent_usage') - total_topped_up = sum(t.amount for t in transactions if t.type == 'top_up') + # Get recent transactions with optimized query + transactions = (request.user.wallet_transactions + .select_related('user') + .order_by('-created_at')[:50]) + + # Calculate statistics with database aggregation (much faster) + stats = request.user.wallet_transactions.aggregate( + total_spent=Sum('amount', filter=Q(type='agent_usage')), + total_topped_up=Sum('amount', filter=Q(type='top_up')) + ) + + # Handle None values from aggregation + total_spent = abs(stats['total_spent'] or 0) + total_topped_up = stats['total_topped_up'] or 0 context = { 'transactions': transactions,