From 97acea62f56b533b7cfac1be0f189990ba2de2c6 Mon Sep 17 00:00:00 2001 From: thecyberlearn Date: Thu, 4 Sep 2025 11:00:56 +0530 Subject: [PATCH] Major simplification: Remove over-engineered features for basic CapRover deployment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit BREAKING CHANGES: - Remove complex security middleware, validators, and cache utils - Replace 430-line CLAUDE.md with 156-line simplified version - Remove 5 complex CapRover optimization docs (1,000+ lines) - Add simplified alternatives: * simple_settings.py - Basic Django config * simple_services.py - No-cache agent management * requirements-simple.txt - 10 vs 24 dependencies * Dockerfile.simple - Streamlined container * README-SIMPLE.md - Focused deployment guide RATIONALE: - Original was enterprise-grade (Redis, CSP, security monitoring) - New version focuses on core CapRover deployment needs - 75% reduction in documentation complexity - Removed: Redis, rate limiting, security scanning, rollback systems - Kept: Basic Django, agents, auth, payments, static files 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude --- CAPROVER_MONITORING_SETUP.md | 152 ---------- CAPROVER_OPTIMIZATION_MASTER.md | 288 ------------------ CAPROVER_REDIS_SETUP.md | 38 --- CAPROVER_SCALING_OPTIMIZATION.md | 287 ------------------ CAPROVER_SECURITY_BACKUP.md | 241 --------------- CLAUDE-SIMPLE.md | 157 ++++++++++ CLAUDE.md | 505 +++++++------------------------ Dockerfile.simple | 26 ++ README-SIMPLE.md | 61 ++++ README.md | 190 +++--------- ROLLBACK.md | 103 ------- agents/simple_services.py | 73 +++++ captain-definition-simple | 4 + core/cache_utils.py | 165 ---------- core/middleware.py | 192 ------------ core/validators.py | 310 ------------------- docs/AGENT_CREATION.md | 284 ++--------------- docs/SIMPLE_AGENT_GUIDE.md | 46 +++ netcop_hub/simple_settings.py | 139 +++++++++ requirements-simple.txt | 18 ++ 20 files changed, 710 insertions(+), 2569 deletions(-) delete mode 100644 CAPROVER_MONITORING_SETUP.md delete mode 100644 CAPROVER_OPTIMIZATION_MASTER.md delete mode 100644 CAPROVER_REDIS_SETUP.md delete mode 100644 CAPROVER_SCALING_OPTIMIZATION.md delete mode 100644 CAPROVER_SECURITY_BACKUP.md create mode 100644 CLAUDE-SIMPLE.md create mode 100644 Dockerfile.simple create mode 100644 README-SIMPLE.md delete mode 100644 ROLLBACK.md create mode 100644 agents/simple_services.py create mode 100644 captain-definition-simple delete mode 100644 core/cache_utils.py delete mode 100644 core/middleware.py delete mode 100644 core/validators.py create mode 100644 docs/SIMPLE_AGENT_GUIDE.md create mode 100644 netcop_hub/simple_settings.py create mode 100644 requirements-simple.txt diff --git a/CAPROVER_MONITORING_SETUP.md b/CAPROVER_MONITORING_SETUP.md deleted file mode 100644 index df1d3ae..0000000 --- a/CAPROVER_MONITORING_SETUP.md +++ /dev/null @@ -1,152 +0,0 @@ -# CapRover Monitoring and Logging Optimization - -## 1. Enhanced Logging Configuration - -### Django Logging (Already Optimized) -Your current logging setup in settings.py is good. Additional optimizations: - -```python -# Add to settings.py -LOGGING_LEVEL = config('LOGGING_LEVEL', default='INFO') - -# Performance monitoring -PERFORMANCE_MONITORING = { - 'SLOW_QUERY_THRESHOLD': 1.0, # Log queries slower than 1 second - 'MEMORY_THRESHOLD': 100, # MB -} -``` - -## 2. Application Performance Monitoring (APM) - -### Option A: Django Debug Toolbar (Development) -Already configured for DEBUG=True environments. - -### Option B: Simple Performance Middleware -Add custom middleware for production monitoring: - -```python -# In core/middleware.py -class PerformanceMonitoringMiddleware: - def __init__(self, get_response): - self.get_response = get_response - - def __call__(self, request): - import time - start_time = time.time() - - response = self.get_response(request) - - duration = time.time() - start_time - if duration > 2.0: # Log slow requests - logger.warning(f"Slow request: {request.path} took {duration:.2f}s") - - response['X-Response-Time'] = f"{duration:.3f}" - return response -``` - -## 3. CapRover Native Monitoring - -### Enable App Monitoring -1. **CapRover Dashboard** → **Apps** → **quantumtaskai** → **App Configs** -2. **Enable**: "Log Rotation" -3. **Set**: "Max Log Size" to 100MB -4. **Set**: "Max Files" to 5 - -### Resource Limits -```yaml -# In captain-definition (advanced) -{ - "schemaVersion": 2, - "dockerfilePath": "./Dockerfile.captain", - "containerHttpPort": 80, - "resources": { - "memory": "512m", - "cpu": "0.5" - } -} -``` - -## 4. External Monitoring Options - -### Option A: Netdata (Lightweight) -1. **One-Click Apps** → Search "Netdata" -2. **Deploy** with default settings -3. **Access**: Monitor system resources in real-time - -### Option B: Grafana + Prometheus (Advanced) -1. **Deploy Prometheus** from One-Click Apps -2. **Deploy Grafana** from One-Click Apps -3. **Configure** Django metrics export - -## 5. Health Checks and Uptime Monitoring - -### Application Health Endpoint -Create a health check endpoint in Django: - -```python -# In core/views.py -from django.http import JsonResponse -from django.core.cache import cache -from django.db import connection - -def health_check(request): - """Application health check endpoint""" - try: - # Test database - with connection.cursor() as cursor: - cursor.execute("SELECT 1") - - # Test Redis - cache.set('health_check', 'ok', 10) - cache_status = cache.get('health_check') == 'ok' - - return JsonResponse({ - 'status': 'healthy', - 'database': 'ok', - 'cache': 'ok' if cache_status else 'error', - 'timestamp': timezone.now().isoformat() - }) - except Exception as e: - return JsonResponse({ - 'status': 'unhealthy', - 'error': str(e) - }, status=500) -``` - -### External Uptime Monitoring -- **UptimeRobot** (Free tier available) -- **Pingdom** -- **StatusCake** - -Monitor: `https://quantumtaskai.captain.your-domain.com/health/` - -## 6. Log Analysis - -### Centralized Logging (Optional) -1. **Deploy ELK Stack** (Elasticsearch, Logstash, Kibana) -2. **Configure** Django to send logs to Logstash -3. **Analyze** logs in Kibana dashboard - -### Simple Log Analysis -```bash -# Monitor application logs -docker logs -f [container-id] - -# Search for errors -docker logs [container-id] 2>&1 | grep -i error - -# Monitor performance -docker logs [container-id] 2>&1 | grep "Slow request" -``` - -## 7. Alerts and Notifications - -### Webhook Notifications -Set up webhooks for critical alerts: -- **High memory usage** -- **Database connection failures** -- **Application errors** -- **Long response times** - -### Email Notifications -Configure Django to send email alerts for critical issues using your existing email setup. \ No newline at end of file diff --git a/CAPROVER_OPTIMIZATION_MASTER.md b/CAPROVER_OPTIMIZATION_MASTER.md deleted file mode 100644 index f8d9dfd..0000000 --- a/CAPROVER_OPTIMIZATION_MASTER.md +++ /dev/null @@ -1,288 +0,0 @@ -# CapRover Optimization Master Guide - Quantum Tasks AI - -## 🚀 Complete Optimization Implementation - -This master guide consolidates all optimization strategies for your Quantum Tasks AI CapRover deployment. - ---- - -## ✅ **Optimization Checklist** - -### **Phase 1: Core Performance (Immediate Impact)** -- [x] **Docker Image Optimization** - Multi-stage build, layer optimization -- [x] **Gunicorn Configuration** - Workers, threads, timeout optimization -- [x] **Database Connection Pooling** - PostgreSQL optimization -- [x] **Redis Caching** - Multi-layer caching strategy -- [ ] **Deploy Redis** - Set up dedicated Redis instance -- [ ] **Update Environment Variables** - Add Redis and performance settings - -### **Phase 2: Monitoring & Reliability (High Impact)** -- [x] **Health Checks** - Docker HEALTHCHECK implementation -- [x] **Logging Optimization** - Structured logging configuration -- [ ] **Deploy Monitoring Stack** - Netdata or custom monitoring -- [ ] **Set up Alerts** - Performance and error notifications -- [ ] **Implement Backup Strategy** - Automated database backups - -### **Phase 3: Security & Compliance (Critical)** -- [x] **Security Headers** - Comprehensive security implementation -- [x] **Database Security** - User permissions and access control -- [ ] **SSL/TLS Optimization** - HTTPS enforcement and HSTS -- [ ] **Security Monitoring** - Intrusion detection and logging -- [ ] **Regular Security Updates** - Automated update strategy - -### **Phase 4: Scaling & Advanced Features (Growth)** -- [ ] **Resource Limits** - Container resource management -- [ ] **Auto-scaling Setup** - CPU/Memory based scaling -- [ ] **Load Testing** - Performance benchmarking -- [ ] **CDN Integration** - Static file optimization (optional) - ---- - -## 🎯 **Quick Implementation Plan** - -### **Step 1: Deploy Optimized Code (5 minutes)** -```bash -# Commit and push optimizations -git add . -git commit -m "Implement CapRover performance optimizations" -git push - -# Redeploy in CapRover -# Dashboard → Apps → quantumtaskai → Deployment → Force Build -``` - -### **Step 2: Deploy Redis (10 minutes)** -1. **CapRover Dashboard** → **One-Click Apps** → **Redis** -2. **Configure**: - - App Name: `quantum-tasks-redis` - - Password: `your-secure-redis-password` -3. **Add Environment Variable**: - ```env - REDIS_URL=redis://:your-secure-redis-password@srv-captain--quantum-tasks-redis:6379/1 - ``` - -### **Step 3: Update Resource Settings (5 minutes)** -1. **App Configs** → **Resources**: - - Memory Limit: 512MB - - Memory Reservation: 256MB - - CPU Limit: 0.5 -2. **Enable Health Checks** (already in optimized Dockerfile) - -### **Step 4: Configure Monitoring (15 minutes)** -1. **Deploy Netdata**: One-Click Apps → Netdata -2. **Set up Log Rotation**: App Configs → Enable log rotation -3. **Configure Alerts**: Email notifications for critical issues - ---- - -## 📊 **Performance Improvements Expected** - -### **Before Optimization** -- **Response Time**: 500-1000ms -- **Memory Usage**: 200-400MB per request spike -- **Database Queries**: Unoptimized, no connection pooling -- **Caching**: Basic Django cache only -- **Scaling**: Manual intervention required - -### **After Optimization** -- **Response Time**: 100-300ms (50-70% improvement) -- **Memory Usage**: Consistent 256-400MB with better efficiency -- **Database Queries**: Connection pooling, 50% faster queries -- **Caching**: Multi-layer Redis caching, 80% cache hit rate -- **Scaling**: Automated scaling based on metrics - -### **Capacity Improvements** -- **Concurrent Users**: 10x increase (10 → 100+ users) -- **Agent Executions**: 5x faster processing -- **Database Load**: 60% reduction in connection overhead -- **Static Files**: Near-instant delivery with compression - ---- - -## 🔧 **Environment Variables Update** - -Add these to your CapRover environment variables: - -```env -# Performance Optimization -REDIS_URL=redis://:your-secure-redis-password@srv-captain--quantum-tasks-redis:6379/1 -CACHE_TTL=300 -SESSION_COOKIE_AGE=7200 -PYTHONUNBUFFERED=1 - -# Database Optimization -CONN_MAX_AGE=600 -DATABASE_CONN_HEALTH_CHECKS=true - -# Security Enhancement -SECURE_SSL_REDIRECT=true -SESSION_COOKIE_SECURE=true -CSRF_COOKIE_SECURE=true - -# Monitoring -LOGGING_LEVEL=INFO -PERFORMANCE_MONITORING=true - -# Resource Limits -GUNICORN_WORKERS=2 -GUNICORN_THREADS=4 -``` - ---- - -## 📈 **Monitoring Dashboard Setup** - -### **Key Metrics to Monitor** -1. **Application Performance**: - - Response time (< 300ms target) - - Error rate (< 1% target) - - Throughput (requests/minute) - -2. **Resource Usage**: - - CPU utilization (< 60% average) - - Memory usage (< 400MB per instance) - - Database connections (< 15 active) - -3. **Business Metrics**: - - Agent execution success rate - - User registration rate - - Payment processing success - -### **Alert Thresholds** -```yaml -Critical Alerts: - - CPU > 85% for 5 minutes - - Memory > 90% for 2 minutes - - Error rate > 5% for 3 minutes - - Database connections > 18 - -Warning Alerts: - - Response time > 500ms average - - CPU > 70% for 10 minutes - - Memory > 80% for 5 minutes - - Disk space > 85% -``` - ---- - -## 🔐 **Security Hardening Checklist** - -### **Immediate Actions** -- [ ] Enable HTTPS enforcement in CapRover -- [ ] Update all default passwords -- [ ] Create dedicated database user for quantum_render -- [ ] Enable CapRover firewall rules -- [ ] Configure security headers (already implemented) - -### **Regular Maintenance** -- [ ] Weekly security updates on host server -- [ ] Monthly password rotation -- [ ] Quarterly security audit -- [ ] Semi-annual disaster recovery test - ---- - -## 💾 **Backup Strategy Implementation** - -### **Automated Backup Schedule** -```bash -# Database backups (daily at 2 AM) -0 2 * * * /scripts/backup-database.sh - -# Application data backups (weekly, Sunday 3 AM) -0 3 * * 0 /scripts/backup-application.sh - -# Configuration backups (monthly) -0 4 1 * * /scripts/backup-configuration.sh -``` - -### **Backup Verification** -- [ ] Test restore procedure monthly -- [ ] Verify backup integrity weekly -- [ ] Document recovery procedures -- [ ] Train team on restore process - ---- - -## 🚀 **Scaling Implementation** - -### **Horizontal Scaling Setup** -1. **Configure Load Balancing**: App Configs → Enable Load Balancer -2. **Set Instance Count**: Start with 2 instances -3. **Session Management**: Redis sessions (already configured) -4. **Health Check Endpoint**: `/health/` (implement in Django) - -### **Auto-scaling Triggers** -```bash -# Scale up conditions -CPU > 70% for 5 minutes AND instances < 5 - -# Scale down conditions -CPU < 30% for 10 minutes AND instances > 1 -``` - ---- - -## 📋 **Implementation Timeline** - -### **Week 1: Core Optimizations** -- [x] Docker and Django optimizations (completed) -- [ ] Deploy Redis -- [ ] Update environment variables -- [ ] Test performance improvements - -### **Week 2: Monitoring & Security** -- [ ] Deploy monitoring stack -- [ ] Implement security hardening -- [ ] Set up automated backups -- [ ] Configure alerts - -### **Week 3: Scaling & Advanced Features** -- [ ] Implement auto-scaling -- [ ] Load testing and optimization -- [ ] Documentation updates -- [ ] Team training - -### **Week 4: Validation & Maintenance** -- [ ] Performance validation -- [ ] Disaster recovery testing -- [ ] Process documentation -- [ ] Monitoring fine-tuning - ---- - -## 🎉 **Success Metrics** - -### **Performance KPIs** -- **Page Load Time**: < 2 seconds -- **API Response Time**: < 200ms -- **Database Query Time**: < 50ms -- **Cache Hit Rate**: > 80% -- **Uptime**: > 99.9% - -### **Business KPIs** -- **User Experience**: Faster agent executions -- **Cost Efficiency**: 30% reduction in server costs -- **Scalability**: Handle 10x more concurrent users -- **Reliability**: Zero unplanned downtime -- **Security**: No security incidents - ---- - -## 📞 **Support & Maintenance** - -### **Regular Health Checks** -- Daily: Monitor dashboards and alerts -- Weekly: Review performance metrics -- Monthly: Backup testing and security updates -- Quarterly: Capacity planning and optimization review - -### **Troubleshooting Resources** -- **Logs**: CapRover app logs and Netdata metrics -- **Database**: pgAdmin monitoring and query analysis -- **Cache**: Redis CLI for cache inspection -- **Application**: Django debug tools and health checks - ---- - -This master guide provides a complete roadmap for optimizing your CapRover deployment. Start with Phase 1 for immediate impact, then progressively implement additional phases based on your needs and growth requirements. \ No newline at end of file diff --git a/CAPROVER_REDIS_SETUP.md b/CAPROVER_REDIS_SETUP.md deleted file mode 100644 index 9aa5f41..0000000 --- a/CAPROVER_REDIS_SETUP.md +++ /dev/null @@ -1,38 +0,0 @@ -# Redis Setup for CapRover Optimization - -## Deploy Redis in CapRover - -### Step 1: Deploy Redis -1. **CapRover Dashboard** → **Apps** → **One-Click Apps/Databases** -2. **Search**: `Redis` -3. **Configure**: - - App Name: `quantum-tasks-redis` - - Version: `7-alpine` (recommended) - - Password: `your-secure-redis-password` -4. **Deploy** - -### Step 2: Update Environment Variables -Add to your quantum_render app environment variables: - -```env -REDIS_URL=redis://:your-secure-redis-password@srv-captain--quantum-tasks-redis:6379/1 -CACHE_TTL=300 -SESSION_COOKIE_AGE=7200 -``` - -### Step 3: Redis Configuration Benefits -- **Session Storage**: Store user sessions in Redis instead of database -- **Database Query Caching**: Cache expensive database queries -- **Agent Execution Caching**: Cache agent results temporarily -- **User Balance Caching**: Cache wallet balances for faster access - -### Step 4: Monitor Redis Usage -Access Redis via CapRover logs or connect with Redis CLI: -```bash -# Via container -docker exec -it [redis-container] redis-cli -# Check memory usage -INFO memory -# Check key statistics -INFO keyspace -``` \ No newline at end of file diff --git a/CAPROVER_SCALING_OPTIMIZATION.md b/CAPROVER_SCALING_OPTIMIZATION.md deleted file mode 100644 index b29eb1a..0000000 --- a/CAPROVER_SCALING_OPTIMIZATION.md +++ /dev/null @@ -1,287 +0,0 @@ -# CapRover Auto-scaling and Resource Optimization - -## 1. Container Resource Limits - -### Update captain-definition for Resource Management -```json -{ - "schemaVersion": 2, - "dockerfilePath": "./Dockerfile.captain", - "containerHttpPort": 80, - "resources": { - "memory": "512m", - "memoryReservation": "256m", - "cpu": 0.5, - "cpuReservation": 0.25 - }, - "healthcheck": { - "test": ["CMD", "python", "manage.py", "check"], - "interval": "30s", - "timeout": "10s", - "retries": 3, - "startPeriod": "40s" - } -} -``` - -### CapRover App Configuration -1. **CapRover Dashboard** → **Apps** → **quantumtaskai** → **App Configs** -2. **Resources** section: - - **Memory Limit**: 512MB - - **Memory Reservation**: 256MB - - **CPU Limit**: 0.5 (50% of one CPU core) - - **CPU Reservation**: 0.25 (25% guaranteed) - -## 2. Horizontal Scaling (Multiple Instances) - -### Load Balancing Setup -1. **App Configs** → **Enable** "Load Balancer" -2. **Set** "Instance Count" to 2-3 instances -3. **Configure** "Health Check Path" to `/health/` - -### Session Affinity (Important for Django) -Since you're using Redis for sessions, sticky sessions aren't needed: -- **Disable** session affinity -- **Enable** Redis session storage (already configured) -- **Sessions persist** across all instances - -### Database Connection Pooling -Update database settings for multiple instances: -```python -# In settings.py -if config('CAPROVER_GIT_COMMIT_SHA', default=''): - # CapRover environment - optimize for multiple instances - DATABASES['default']['CONN_MAX_AGE'] = 300 # Shorter connection lifetime - DATABASES['default']['OPTIONS']['MAX_CONNS'] = 10 # Fewer connections per instance -``` - -## 3. Vertical Scaling (Resource Monitoring) - -### Memory Optimization -```python -# Add to Django settings -if not DEBUG: - # Production memory optimizations - MIDDLEWARE = [ - 'django.middleware.gzip.GZipMiddleware', # Compress responses - ] + MIDDLEWARE - - # Enable template caching - TEMPLATES[0]['OPTIONS']['loaders'] = [ - ('django.template.loaders.cached.Loader', [ - 'django.template.loaders.filesystem.Loader', - 'django.template.loaders.app_directories.Loader', - ]), - ] -``` - -### Database Query Optimization -```python -# Add to apps/core/middleware.py -class DatabaseOptimizationMiddleware: - def __init__(self, get_response): - self.get_response = get_response - - def __call__(self, request): - from django.db import connection, reset_queries - - # Reset queries for this request - reset_queries() - - response = self.get_response(request) - - # Log slow or numerous queries in production - if not settings.DEBUG and len(connection.queries) > 10: - logger.warning(f"High query count: {len(connection.queries)} queries for {request.path}") - - return response -``` - -## 4. Auto-scaling Triggers - -### CPU-based Scaling -```bash -# Monitor CPU usage -docker stats [container-id] - -# Scale up when CPU > 70% for 5 minutes -# Scale down when CPU < 30% for 10 minutes -``` - -### Memory-based Scaling -```bash -# Monitor memory usage -docker exec [container-id] free -m - -# Scale up when memory > 80% for 5 minutes -# Scale down when memory < 40% for 10 minutes -``` - -### Custom Metrics Scaling -Monitor application-specific metrics: -- **Active user sessions** (Redis keys) -- **Agent execution queue length** -- **Database connection pool usage** -- **Response time averages** - -## 5. Performance Monitoring Scripts - -### Create Monitoring Script -```bash -#!/bin/bash -# monitoring.sh - -CONTAINER_ID=$(docker ps | grep quantumtaskai | awk '{print $1}') - -# Get resource usage -CPU_USAGE=$(docker stats --no-stream $CONTAINER_ID | tail -1 | awk '{print $3}' | sed 's/%//') -MEM_USAGE=$(docker stats --no-stream $CONTAINER_ID | tail -1 | awk '{print $4}' | sed 's/%//') - -echo "CPU Usage: $CPU_USAGE%" -echo "Memory Usage: $MEM_USAGE%" - -# Alert if high usage -if (( $(echo "$CPU_USAGE > 80" | bc -l) )); then - echo "HIGH CPU ALERT: $CPU_USAGE%" - # Send notification (email, webhook, etc.) -fi - -if (( $(echo "$MEM_USAGE > 85" | bc -l) )); then - echo "HIGH MEMORY ALERT: $MEM_USAGE%" - # Send notification (email, webhook, etc.) -fi - -# Check application health -HEALTH_CHECK=$(curl -s -o /dev/null -w "%{http_code}" https://quantumtaskai.captain.your-domain.com/health/) -if [ "$HEALTH_CHECK" != "200" ]; then - echo "APPLICATION HEALTH ALERT: HTTP $HEALTH_CHECK" - # Send notification -fi -``` - -### Cron Job for Monitoring -```bash -# Add to crontab -*/5 * * * * /path/to/monitoring.sh >> /var/log/quantum-monitor.log 2>&1 -``` - -## 6. Caching Strategy for Scale - -### Multi-layer Caching -```python -# In views.py - example caching strategy -from django.core.cache import cache -from django.views.decorators.cache import cache_page -from django.utils.decorators import method_decorator - -@method_decorator(cache_page(60 * 5), name='dispatch') # 5 minutes -class AgentListView(ListView): - model = Agent - - def get_queryset(self): - cache_key = f"agents_list_{self.request.user.id}" - queryset = cache.get(cache_key) - - if queryset is None: - queryset = Agent.objects.select_related().prefetch_related('category') - cache.set(cache_key, queryset, 60 * 10) # 10 minutes - - return queryset -``` - -### Redis Cluster for High Availability (Advanced) -For very high load, consider Redis Cluster: -1. **Deploy multiple Redis instances** -2. **Configure Redis Cluster** -3. **Update Django Redis settings** for cluster mode - -## 7. Load Testing and Optimization - -### Load Testing Tools -```bash -# Install Apache Bench -sudo apt-get install apache2-utils - -# Test with concurrent users -ab -n 1000 -c 10 https://quantumtaskai.captain.your-domain.com/ - -# Test specific endpoints -ab -n 500 -c 5 https://quantumtaskai.captain.your-domain.com/agents/ - -# Load test with POST data -ab -n 100 -c 5 -p post_data.json -T application/json https://quantumtaskai.captain.your-domain.com/agents/api/execute/ -``` - -### Performance Benchmarks -Target performance metrics: -- **Response time**: < 200ms for cached pages -- **Database queries**: < 50ms per query -- **Memory usage**: < 400MB per instance -- **CPU usage**: < 60% average -- **Concurrent users**: 100+ simultaneous users - -## 8. Auto-scaling Scripts - -### Simple Auto-scaler Script -```bash -#!/bin/bash -# auto-scaler.sh - -APP_NAME="quantumtaskai" -MIN_INSTANCES=1 -MAX_INSTANCES=5 -CPU_THRESHOLD_UP=70 -CPU_THRESHOLD_DOWN=30 - -# Get current instance count -CURRENT_INSTANCES=$(docker ps | grep $APP_NAME | wc -l) - -# Get average CPU usage -AVG_CPU=$(docker stats --no-stream $(docker ps -q --filter name=$APP_NAME) | awk 'NR>1 {sum += $3; count++} END {print sum/count}' | sed 's/%//') - -echo "Current instances: $CURRENT_INSTANCES" -echo "Average CPU: $AVG_CPU%" - -# Scale up logic -if (( $(echo "$AVG_CPU > $CPU_THRESHOLD_UP" | bc -l) )) && [ $CURRENT_INSTANCES -lt $MAX_INSTANCES ]; then - echo "Scaling UP: CPU at $AVG_CPU%" - # Implement scaling up logic (CapRover API call) - curl -X POST https://captain.your-domain.com/api/v2/user/apps/appData/quantumtaskai \ - -H "x-captain-auth: $CAPTAIN_TOKEN" \ - -d '{"instanceCount": '$((CURRENT_INSTANCES + 1))'}' -fi - -# Scale down logic -if (( $(echo "$AVG_CPU < $CPU_THRESHOLD_DOWN" | bc -l) )) && [ $CURRENT_INSTANCES -gt $MIN_INSTANCES ]; then - echo "Scaling DOWN: CPU at $AVG_CPU%" - # Implement scaling down logic (CapRover API call) - curl -X POST https://captain.your-domain.com/api/v2/user/apps/appData/quantumtaskai \ - -H "x-captain-auth: $CAPTAIN_TOKEN" \ - -d '{"instanceCount": '$((CURRENT_INSTANCES - 1))'}' -fi -``` - -## 9. Cost Optimization - -### Resource Right-sizing -- **Start small**: 512MB RAM, 0.5 CPU -- **Monitor usage**: Scale up only when needed -- **Regular reviews**: Monthly resource usage analysis - -### Efficient Resource Usage -- **Shared services**: Use shared PostgreSQL and Redis -- **Image optimization**: Multi-stage Docker builds -- **Caching**: Reduce database load with strategic caching -- **Compression**: Enable gzip compression - -### Schedule-based Scaling -```bash -# Scale up during peak hours (9 AM - 6 PM) -0 9 * * 1-5 /scripts/scale-up.sh - -# Scale down during off-hours -0 18 * * 1-5 /scripts/scale-down.sh -0 0 * * 6-7 /scripts/scale-down.sh -``` - -This comprehensive scaling strategy ensures your Quantum Tasks AI application can handle varying loads efficiently while maintaining cost-effectiveness. \ No newline at end of file diff --git a/CAPROVER_SECURITY_BACKUP.md b/CAPROVER_SECURITY_BACKUP.md deleted file mode 100644 index 7f3ddde..0000000 --- a/CAPROVER_SECURITY_BACKUP.md +++ /dev/null @@ -1,241 +0,0 @@ -# CapRover Security and Backup Optimization - -## 1. Enhanced Security Configuration - -### SSL/TLS Optimization -1. **CapRover Dashboard** → **Apps** → **quantumtaskai** → **HTTP Settings** -2. **Enable**: Force HTTPS -3. **Enable**: HTTP Strict Transport Security (HSTS) -4. **Set**: HSTS Max Age to 31536000 (1 year) - -### Security Headers (Already Implemented) -Your Django settings already include excellent security headers: -- Content Security Policy (CSP) -- X-Frame-Options -- X-Content-Type-Options -- Referrer-Policy - -### Additional Security Environment Variables -Add these to your CapRover environment variables: - -```env -# Security Settings -SECURE_PROXY_SSL_HEADER=HTTP_X_FORWARDED_PROTO,https -SECURE_SSL_REDIRECT=true -SESSION_COOKIE_SECURE=true -CSRF_COOKIE_SECURE=true - -# Rate Limiting -RATELIMIT_ENABLE=true -RATELIMIT_USE_CACHE=default - -# Additional Security -ALLOWED_HOSTS=quantumtaskai.captain.your-domain.com,your-custom-domain.com -CSRF_TRUSTED_ORIGINS=https://quantumtaskai.captain.your-domain.com,https://your-custom-domain.com -``` - -## 2. Database Security - -### PostgreSQL Security Hardening -1. **Access pgAdmin** → **quantum-digital-db** -2. **Create specific user** for quantum_render: -```sql --- Create dedicated user for quantum_render -CREATE USER quantum_render_user WITH PASSWORD 'secure-unique-password'; -GRANT CONNECT ON DATABASE quantum-tasks-db TO quantum_render_user; -GRANT USAGE ON SCHEMA public TO quantum_render_user; -GRANT CREATE ON SCHEMA public TO quantum_render_user; - --- Grant necessary permissions -GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA public TO quantum_render_user; -GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA public TO quantum_render_user; -``` - -3. **Update DATABASE_URL**: -```env -DATABASE_URL=postgres://quantum_render_user:secure-unique-password@srv-captain--quantum-digital-db:5432/quantum-tasks-db -``` - -### Database Connection Security -- Use connection pooling (already configured) -- Enable SSL connections if supported -- Regular password rotation - -## 3. Backup Strategy - -### Automated Database Backups - -#### Option A: CapRover Cron Jobs -Create a backup container that runs scheduled backups: - -**Dockerfile.backup:** -```dockerfile -FROM postgres:15-alpine - -RUN apk add --no-cache aws-cli - -COPY backup-script.sh /backup-script.sh -RUN chmod +x /backup-script.sh - -ENTRYPOINT ["/backup-script.sh"] -``` - -**backup-script.sh:** -```bash -#!/bin/bash -DATE=$(date +%Y%m%d_%H%M%S) -BACKUP_FILE="quantum-tasks-db-backup-$DATE.sql" - -# Create backup -pg_dump -h srv-captain--quantum-digital-db -U quantum_user -d quantum-tasks-db > /tmp/$BACKUP_FILE - -# Compress backup -gzip /tmp/$BACKUP_FILE - -# Upload to cloud storage (optional) -# aws s3 cp /tmp/$BACKUP_FILE.gz s3://your-backup-bucket/ - -# Keep local copy for quick restore -cp /tmp/$BACKUP_FILE.gz /backups/ - -# Clean old backups (keep last 7 days) -find /backups -name "*.gz" -mtime +7 -delete - -echo "Backup completed: $BACKUP_FILE.gz" -``` - -#### Option B: Manual Backup Commands -```bash -# Create manual backup -docker exec [postgres-container] pg_dump -U quantum_user quantum-tasks-db > backup_$(date +%Y%m%d).sql - -# Restore from backup -docker exec -i [postgres-container] psql -U quantum_user quantum-tasks-db < backup_file.sql -``` - -### Application Data Backup - -#### Media Files Backup -```bash -# Backup media files -docker exec [app-container] tar -czf /tmp/media-backup-$(date +%Y%m%d).tar.gz /app/media/ - -# Copy to host -docker cp [app-container]:/tmp/media-backup-$(date +%Y%m%d).tar.gz ./backups/ -``` - -#### Configuration Backup -```bash -# Backup CapRover configuration -# From CapRover server -cp -r /captain/data ./caprover-config-backup-$(date +%Y%m%d) -``` - -### Cloud Backup Integration - -#### AWS S3 Integration -```env -# Add to environment variables -AWS_ACCESS_KEY_ID=your-access-key -AWS_SECRET_ACCESS_KEY=your-secret-key -AWS_DEFAULT_REGION=us-east-1 -BACKUP_BUCKET=quantum-tasks-backups -``` - -#### Google Cloud Storage -```env -GOOGLE_CLOUD_PROJECT=your-project-id -GOOGLE_APPLICATION_CREDENTIALS=/path/to/service-account.json -``` - -## 4. Security Monitoring - -### Log Monitoring for Security Events -```python -# Add to Django settings -SECURITY_EVENTS_TO_LOG = [ - 'authentication_failed', - 'permission_denied', - 'suspicious_operation', - 'rate_limit_exceeded' -] - -# Custom logging for security events -LOGGING['loggers']['security'] = { - 'handlers': ['security_file'], - 'level': 'WARNING', - 'propagate': False, -} -``` - -### Intrusion Detection -- Monitor failed login attempts -- Track unusual API usage patterns -- Alert on multiple failed authentication attempts -- Log all admin actions - -### Automated Security Updates -```bash -# Regular security updates (run on host) -apt update && apt upgrade -y - -# Docker image updates (rebuild regularly) -# CapRover → Apps → quantumtaskai → Deployment → Force Build -``` - -## 5. Disaster Recovery Plan - -### Recovery Time Objectives (RTO) -- **Database**: < 1 hour -- **Application**: < 30 minutes -- **Full System**: < 2 hours - -### Recovery Steps -1. **Database Recovery**: - ```bash - # Restore database from latest backup - docker exec -i [postgres-container] psql -U quantum_user quantum-tasks-db < latest_backup.sql - ``` - -2. **Application Recovery**: - ```bash - # Redeploy application - # CapRover → Apps → quantumtaskai → Force Build - # Run migrations if needed - docker exec [app-container] python manage.py migrate - ``` - -3. **Configuration Recovery**: - - Restore environment variables from backup - - Verify DNS and domain settings - - Test all integrations (Stripe, email, APIs) - -### Testing Recovery Procedures -- Monthly backup restoration tests -- Quarterly disaster recovery drills -- Document all procedures and update regularly - -## 6. Security Best Practices - -### Environment Variables Security -- Use strong, unique passwords -- Rotate secrets regularly (every 90 days) -- Never commit secrets to Git -- Use different keys for staging/production - -### Network Security -- Restrict database access to application containers only -- Use CapRover's internal network for inter-container communication -- Configure firewall rules on the host server - -### Application Security -- Keep Django and dependencies updated -- Regular security audits with `python -m pip audit` -- Monitor security advisories for used packages -- Implement proper input validation and sanitization - -### Access Control -- Use strong passwords for CapRover admin -- Enable two-factor authentication where possible -- Regular access reviews and permission audits -- Separate staging and production environments \ No newline at end of file diff --git a/CLAUDE-SIMPLE.md b/CLAUDE-SIMPLE.md new file mode 100644 index 0000000..e7d376a --- /dev/null +++ b/CLAUDE-SIMPLE.md @@ -0,0 +1,157 @@ +# CLAUDE-SIMPLE.md + +This file provides simplified guidance for CapRover deployment of Quantum Tasks AI. + +## Simplified Project Overview + +Quantum Tasks AI is a basic Django AI agent marketplace. Users can browse agents and execute them through simple web forms. + +**Core Architecture:** +- **Django Framework**: Basic Django 5.2.4 setup +- **Agent System**: File-based JSON agent configs with simple execution +- **Authentication**: Basic Django user authentication +- **Payments**: Simple Stripe integration +- **Database**: SQLite for development, PostgreSQL for production + +## Quick Development Setup + +```bash +# Use virtual environment +source venv/bin/activate + +# Install minimal dependencies +pip install -r requirements-simple.txt + +# Run migrations +python manage.py migrate + +# Create superuser +python manage.py createsuperuser + +# Start server +python manage.py runserver +``` + +## Simple CapRover Deployment + +### 1. Use Simplified Configuration + +Update your Django settings to use the simplified version: + +```python +# In manage.py, wsgi.py, etc., change: +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'netcop_hub.simple_settings') +``` + +### 2. Environment Variables (Minimal) + +```env +SECRET_KEY=your-secret-key-here +DEBUG=false +ALLOWED_HOSTS=yourdomain.com +DATABASE_URL=postgres://user:pass@host:5432/dbname + +# Optional - for email +EMAIL_HOST_USER=your-email@gmail.com +EMAIL_HOST_PASSWORD=your-app-password + +# Optional - for payments +STRIPE_SECRET_KEY=sk_test_your_stripe_key +``` + +### 3. Captain Definition + +```json +{ + "schemaVersion": 2, + "dockerfilePath": "./Dockerfile.simple" +} +``` + +### 4. Simple Dockerfile + +Create `Dockerfile.simple`: + +```dockerfile +FROM python:3.11-slim + +WORKDIR /app +COPY requirements-simple.txt . +RUN pip install -r requirements-simple.txt + +COPY . . +RUN python manage.py collectstatic --noinput + +EXPOSE 3000 +CMD ["gunicorn", "netcop_hub.wsgi:application", "--bind", "0.0.0.0:3000"] +``` + +## Core Features + +### Agent Management +- JSON file-based agents in `agents/configs/agents/` +- Simple categories in `agents/configs/categories/categories.json` +- Basic execution through web forms + +### Apps Structure +- **core/**: Homepage and basic views +- **authentication/**: User registration/login +- **agents/**: Agent marketplace and execution +- **wallet/**: Basic Stripe payments + +### Simple Agent System + +Use `agents.simple_services.SimpleAgentService` instead of the complex caching system: + +```python +from agents.simple_services import SimpleAgentService + +# Get all agents +agents = SimpleAgentService.get_all_agents() + +# Get specific agent +agent = SimpleAgentService.get_agent('agent-slug') + +# Get categories +categories = SimpleAgentService.get_categories() +``` + +## Deployment Commands + +```bash +# Collect static files +python manage.py collectstatic --noinput + +# Run migrations +python manage.py migrate + +# Start with gunicorn +gunicorn netcop_hub.wsgi:application --bind 0.0.0.0:3000 +``` + +## What Was Removed + +- Complex security middleware and CSP +- Advanced caching systems +- Performance optimizations +- Complex validation systems +- Advanced logging and monitoring +- Redis dependencies +- Rate limiting +- Security scanning +- Emergency rollback systems + +## Simple Architecture Status + +- ✅ **Basic Django Setup** - Standard Django configuration +- ✅ **File-based Agents** - Simple JSON agent configs +- ✅ **Basic Authentication** - Django's built-in auth +- ✅ **Simple Payments** - Basic Stripe integration +- ✅ **Static Files** - WhiteNoise for production +- ✅ **Database** - SQLite/PostgreSQL support +- ✅ **Minimal Dependencies** - Only essential packages + +This version focuses on core functionality for CapRover deployment without enterprise-grade optimizations. + +--- +Last updated: 2025-09-04 (Simplified for CapRover) \ No newline at end of file diff --git a/CLAUDE.md b/CLAUDE.md index 499e5d5..e7d376a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,430 +1,157 @@ -# CLAUDE.md +# CLAUDE-SIMPLE.md -This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. +This file provides simplified guidance for CapRover deployment of Quantum Tasks AI. -## Project Overview +## Simplified Project Overview -Quantum Tasks AI is a Django-based AI agent marketplace platform. Users can access AI agent services through a web interface, with execution handled via two distinct systems: N8N webhook integrations and direct form access integrations. +Quantum Tasks AI is a basic Django AI agent marketplace. Users can browse agents and execute them through simple web forms. -**Key Architecture:** -- **Django Framework**: Main web application using Django 5.2.4 -- **Agent System**: Database-driven agents app with dual integration systems: - - **Webhook Agents**: N8N integrations for complex processing - - **Direct Access Agents**: Form-based integrations (JotForm, etc.) -- **Authentication**: Custom user model with email verification -- **Payments**: Stripe integration with wallet system (supports free agents) -- **Database**: SQLite for development, PostgreSQL for production (Railway) -- **Static Files**: WhiteNoise for production static file serving +**Core Architecture:** +- **Django Framework**: Basic Django 5.2.4 setup +- **Agent System**: File-based JSON agent configs with simple execution +- **Authentication**: Basic Django user authentication +- **Payments**: Simple Stripe integration +- **Database**: SQLite for development, PostgreSQL for production -## Development Commands +## Quick Development Setup -### Environment Setup ```bash # Use virtual environment source venv/bin/activate -# Install dependencies -pip install -r requirements.txt # Production -pip install -r requirements-dev.txt # Development +# Install minimal dependencies +pip install -r requirements-simple.txt -# Start development server -./run_dev.sh # Recommended - includes migration checks -# OR -python manage.py runserver # Direct Django server -``` - -### Database Operations -```bash -# Make migrations -python manage.py makemigrations - -# Apply migrations +# Run migrations python manage.py migrate # Create superuser python manage.py createsuperuser -# Database shell -python manage.py dbshell - -# Check database configuration -python manage.py check_db +# Start server +python manage.py runserver ``` -### Agent Management (File-Based System) -```bash -# Agents are managed via JSON files - no commands needed! -# Simply add/edit JSON files in agents/configs/agents/ +## Simple CapRover Deployment -# View agent statistics -python -c " -from agents.services import AgentFileService -stats = AgentFileService.get_agent_stats() -print('Agent Stats:', stats) -" +### 1. Use Simplified Configuration + +Update your Django settings to use the simplified version: + +```python +# In manage.py, wsgi.py, etc., change: +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'netcop_hub.simple_settings') ``` -### Testing -```bash -# Run Django tests -python manage.py test +### 2. Environment Variables (Minimal) -# Run pytest (if configured) -pytest +```env +SECRET_KEY=your-secret-key-here +DEBUG=false +ALLOWED_HOSTS=yourdomain.com +DATABASE_URL=postgres://user:pass@host:5432/dbname -# Run specific app tests -python manage.py test authentication -python manage.py test agents -python manage.py test wallet +# Optional - for email +EMAIL_HOST_USER=your-email@gmail.com +EMAIL_HOST_PASSWORD=your-app-password -# Custom test scripts -python tests/simple_test.py -python tests/check_agents.py +# Optional - for payments +STRIPE_SECRET_KEY=sk_test_your_stripe_key ``` -### Code Quality (Development Dependencies) -```bash -# Format code -black . +### 3. Captain Definition -# Sort imports -isort . - -# Lint code -flake8 - -# Type checking (if available) -mypy . +```json +{ + "schemaVersion": 2, + "dockerfilePath": "./Dockerfile.simple" +} ``` -### Production Commands +### 4. Simple Dockerfile + +Create `Dockerfile.simple`: + +```dockerfile +FROM python:3.11-slim + +WORKDIR /app +COPY requirements-simple.txt . +RUN pip install -r requirements-simple.txt + +COPY . . +RUN python manage.py collectstatic --noinput + +EXPOSE 3000 +CMD ["gunicorn", "netcop_hub.wsgi:application", "--bind", "0.0.0.0:3000"] +``` + +## Core Features + +### Agent Management +- JSON file-based agents in `agents/configs/agents/` +- Simple categories in `agents/configs/categories/categories.json` +- Basic execution through web forms + +### Apps Structure +- **core/**: Homepage and basic views +- **authentication/**: User registration/login +- **agents/**: Agent marketplace and execution +- **wallet/**: Basic Stripe payments + +### Simple Agent System + +Use `agents.simple_services.SimpleAgentService` instead of the complex caching system: + +```python +from agents.simple_services import SimpleAgentService + +# Get all agents +agents = SimpleAgentService.get_all_agents() + +# Get specific agent +agent = SimpleAgentService.get_agent('agent-slug') + +# Get categories +categories = SimpleAgentService.get_categories() +``` + +## Deployment Commands + ```bash # Collect static files python manage.py collectstatic --noinput -# Production server (via Gunicorn) -gunicorn netcop_hub.wsgi:application +# Run migrations +python manage.py migrate + +# Start with gunicorn +gunicorn netcop_hub.wsgi:application --bind 0.0.0.0:3000 ``` -## Core Architecture +## What Was Removed -### Apps Structure -- **authentication/**: Custom user model, email verification, password reset -- **core/**: Homepage, error handlers, utility functions -- **agents/**: File-based agent system (marketplace, execution history, REST API for executions) -- **wallet/**: Stripe payments, wallet management, transactions +- Complex security middleware and CSP +- Advanced caching systems +- Performance optimizations +- Complex validation systems +- Advanced logging and monitoring +- Redis dependencies +- Rate limiting +- Security scanning +- Emergency rollback systems -### Agent System (agents app) -**Key Files:** -- `agents/services.py`: AgentFileService - file-based agent management -- `agents/configs/agents/`: JSON agent configuration files -- `agents/configs/categories/`: JSON category configuration files -- `agents/models.py`: AgentExecution, ChatSession models (execution history) -- `agents/views.py`: Main imports for backwards compatibility -- `agents/api_views.py`: REST API endpoints (execute_agent, execution_list/detail) -- `agents/chat_views.py`: Chat session management and message handling -- `agents/web_views.py`: Web interface views (marketplace, agent detail pages) -- `agents/direct_access_views.py`: External form integration handlers -- `agents/utils.py`: Utility functions (webhook validation, message formatting) -- `agents/templates/agents/`: Dynamic agent templates and marketplace -- `templates/career_navigator.html`: Direct access form template +## Simple Architecture Status -**Dual Integration Systems:** +- ✅ **Basic Django Setup** - Standard Django configuration +- ✅ **File-based Agents** - Simple JSON agent configs +- ✅ **Basic Authentication** - Django's built-in auth +- ✅ **Simple Payments** - Basic Stripe integration +- ✅ **Static Files** - WhiteNoise for production +- ✅ **Database** - SQLite/PostgreSQL support +- ✅ **Minimal Dependencies** - Only essential packages -**System 1: Webhook Agents (N8N Integration)** -1. User browses marketplace (`/agents/`) -2. Clicks "Try Now" → Agent detail page (`/agents/{slug}/`) -3. Fills dynamic form → Form submission calls `/agents/api/execute/` -4. N8N webhook processes request and returns response -5. Results displayed with file upload support - -**System 2: Direct Access Agents (Form Integration)** -1. User browses marketplace (`/agents/`) -2. Clicks special "Try Now" button → Direct access (`/agents/{slug}/access/`) -3. Payment processed → Redirect to form page (`/agents/{slug}/`) -4. Form displays embedded interface (JotForm, etc.) -5. User interacts directly with external form system - -### Database Models -**User Management:** -- `authentication.User`: Custom user model with email verification -- `authentication.PasswordResetToken`: Password reset tokens -- `authentication.EmailVerificationToken`: Email verification tokens - -**Agents:** -- `agents.Agent`: Agent definitions with JSON form schemas and pricing -- `agents.AgentCategory`: Agent categories with icons and descriptions -- `agents.AgentExecution`: Execution history and results tracking - -**Payments:** -- `wallet.Wallet`: User wallet with balance tracking -- `wallet.WalletTransaction`: Transaction history and Stripe integration - -### Settings Configuration -**Environment Variables (Required for Production):** -- `SECRET_KEY`: Django secret key -- `ALLOWED_HOSTS`: Comma-separated list of allowed hosts -- `EMAIL_HOST_USER`, `EMAIL_HOST_PASSWORD`: SMTP credentials -- `STRIPE_SECRET_KEY`, `STRIPE_WEBHOOK_SECRET`: Stripe API keys -- `DATABASE_URL`: PostgreSQL connection string (Railway) - -**Current System:** -The platform supports **8 total agents** across **6 categories**: -- **4 Webhook Agents** (N8N integration): Social Ads Generator, Job Posting Generator, PDF Summarizer, 5 Whys Analyzer -- **4 Direct Access Agents** (External forms): CyberSec Career Navigator, AI Brand Strategist, Lean Six Sigma Expert, SWOT Analysis Expert - -For detailed agent information and creation instructions, see `docs/AGENT_CREATION.md`. - -### URL Structure -``` -/ # Homepage (core app) -/digital-branding/ # Digital branding services page -/auth/ # Authentication (login, register, etc.) -/agents/ # Agent marketplace (agents app) -/agents/{slug}/ # Individual agent pages (webhook agents) -/agents/{slug}/access/ # Direct access agent payment processing -/wallet/ # Wallet management -/admin/ # Django admin -``` - -### Key Components -**Agent Configuration (File-driven):** -- All agent metadata stored in JSON files (pricing, descriptions, webhooks) -- JSON form schemas for dynamic form generation -- Instant agent creation by adding JSON files (no commands needed) -- Automatic database sync for foreign key compatibility - -**Templates:** -- `templates/base.html`: Main layout with navigation -- `templates/components/`: Reusable UI components -- `agents/templates/agents/`: Dynamic agent forms and marketplace pages - -## Adding New Agents - -For comprehensive agent creation instructions, see **`docs/AGENT_CREATION.md`**. - -**Quick Summary:** -1. Create JSON config in `agents/configs/agents/your-agent-name.json` -2. Git push (or restart server locally) -3. Agent appears in marketplace automatically - no commands needed! - -The platform supports 2 agent types: -- **Webhook Agents** - N8N integration with dynamic forms -- **Direct Access Agents** - External forms (JotForm, etc.) with embedded interfaces - -## External Service Wrappers - -**Advanced template-based system for external forms, events, and integrations with automatic CSP support:** - -**Configuration:** Edit `EXTERNAL_PAGES` dict in `core/views.py`: -```python -EXTERNAL_PAGES = { - 'event': { - 'title': 'Event Registration', - 'description': 'Register for our upcoming event', - 'external_url': 'https://form.jotform.com/252214924850455', - 'template': 'iframe', # iframe, landing, or redirect - }, - 'cea': { - 'title': 'CEA Registration', - 'description': 'Access CEA registration form', - 'external_url': 'https://agent.jotform.com/0198a8860b46796895f2a40367a6cea4df0c', - 'template': 'iframe', - }, - 'cea1': { - 'title': 'CEA1 Registration', - 'description': 'Access CEA1 registration form', - 'external_url': 'https://agent.jotform.com/0198b221344f78088bfc6fc6598d649db6e5', - 'template': 'iframe', - }, -} -``` - -**Templates Available:** -- `templates/wrapper/iframe.html` - Full-screen iframe embed (auto CSP support) -- `templates/wrapper/landing.html` - Branded landing page with embed (auto CSP support) -- `templates/wrapper/redirect.html` - Auto-redirect with countdown - -**Access:** `/{page-name}/` (e.g., `/event/`, `/cea/`, `/cea1/`) - -**Features:** -- **🚀 Automatic CSP Support** - No content blocking for external iframes -- **🛡️ Smart Security** - Relaxed CSP only for iframe/landing pages -- **📱 Mobile Responsive** - Works on all devices -- **⚡ Zero Configuration** - Add to EXTERNAL_PAGES and it works immediately -- **🔒 Rate Limited** - IP-based protection (30 requests/minute) -- **🎨 Consistent Branding** - Inherits site design system - -**Supported External Services (Auto-Whitelisted):** -- JotForm (form.jotform.com, agent.jotform.com, cdn.jotfor.ms) -- Calendly (calendly.com, assets.calendly.com) -- Typeform (typeform.com, *.typeform.com) -- Airtable (airtable.com, *.airtable.com) -- HubSpot (hubspot.com, *.hubspot.com) -- Zapier (zapier.com, *.zapier.com) -- Google Analytics/GTM - -**Adding New External Services:** -1. Add entry to `EXTERNAL_PAGES` in `core/views.py` -2. Choose template: `iframe`, `landing`, or `redirect` -3. Access immediately at `/{page-name}/` - no other configuration needed! - -## Social Media Integration - -**Rich social media previews implemented in `templates/base.html`:** - -**Open Graph Tags:** -- `og:title` - Page title for social sharing -- `og:description` - Page description -- `og:image` - Preview image (`static/img/og-image.png`) -- `og:url` - Canonical page URL -- `og:site_name` - "Quantum Tasks AI" - -**Twitter Card Tags:** -- `twitter:card` - Large image format -- `twitter:title/description/image` - Twitter-specific metadata - -**Custom Per-Page:** Override blocks in templates: -```django -{% block og_title %}Custom Page Title{% endblock %} -{% block meta_description %}Custom description{% endblock %} -``` - -**Result:** Rich previews on WhatsApp, Discord, Twitter, LinkedIn with branded image and professional descriptions. - -## Security & Performance Optimizations - -**🛡️ Comprehensive Security System:** - -**Security Middleware (`core/middleware.py`):** -- **Smart Content Security Policy (CSP)** - Automatic detection of pages needing external iframe support -- **Security Headers** - X-Content-Type-Options, X-XSS-Protection, Referrer-Policy, Permissions-Policy -- **X-Frame-Options** - Dynamic handling (SAMEORIGIN for iframe pages, DENY for others) -- **Security Monitoring** - Logs suspicious activity, failed auth attempts, SQL injection attempts -- **Threat Detection** - Pattern matching for common attack vectors - -**Input Validation (`core/validators.py`):** -- **XSS Prevention** - HTML sanitization with bleach -- **SQL Injection Protection** - Pattern detection and input cleaning -- **File Upload Security** - Extension validation, size limits, filename sanitization -- **Decimal/Amount Validation** - Secure monetary value handling -- **Email Validation** - RFC-compliant with security checks - -**Cache System (`core/cache_utils.py`):** -- **Smart Cache Keys** - User-specific, agent-specific caching -- **Cache Invalidation** - Automatic cleanup on data changes -- **Performance Optimization** - Reduces database queries - -**Database Security:** -- **Atomic Transactions** - ACID compliance for wallet operations -- **Index Optimization** - Performance indexes on frequently queried fields -- **Migration Safety** - Foreign key constraint handling - -**Rate Limiting:** -- **IP-based Protection** - 30 requests/minute for external pages -- **Agent Execution Limits** - Prevents abuse of AI services -- **Authentication Throttling** - Failed login attempt tracking - -**🚀 Performance Features:** -- **Database Optimization** - select_related, prefetch_related for efficient queries -- **Static File Optimization** - WhiteNoise compression and caching -- **Smart Caching** - User balance, agent data, and execution history caching -- **Logging Optimization** - Structured logging with rotation - -## Production Deployment - -**Railway Configuration:** -- Automatic deployment from git repository -- PostgreSQL database provided by Railway -- Environment variables configured in Railway dashboard -- Static files served via WhiteNoise -- **Secure Admin Creation** - `reset_admin` command with foreign key safety - -**Security Features:** -- **Production CSP** - Strict policy for non-iframe pages -- **CSRF Protection** - Django CSRF middleware enabled -- **Rate Limiting** - django-ratelimit on sensitive endpoints -- **Secure Headers** - Complete security header suite -- **HTTPS Enforcement** - Secure cookies and HSTS -- **Session Security** - Secure session configuration -- **Input Sanitization** - All user input validated and cleaned - -**Emergency Rollback System:** -- **Complete rollback documentation** in `ROLLBACK.md` -- **30-second emergency recovery** - Simple git commands -- **Zero data loss** - All changes committed safely -- **Selective rollback** - Can revert specific components - -## Development Notes - -- **Database**: Uses SQLite by default for development reliability -- **Cache**: Redis preferred, falls back to local memory cache -- **Email**: Console backend in development, SMTP in production -- **Debug Tools**: Debug toolbar and Django extensions available in development -- **Static Files**: Collected to `staticfiles/` directory for production -- **Media Files**: User uploads stored in `media/` directory - -## Common Development Tasks - -**Adding new environment variables:** -1. Add to `settings.py` with `config()` call -2. Add to required_env_vars list if production-required -3. Document in this file - -**Database changes:** -1. Make model changes -2. Run `python manage.py makemigrations` -3. Review migration file -4. Run `python manage.py migrate` - -**Testing agent webhooks locally:** -1. Use ngrok or similar to expose local server -2. Update webhook URLs in agent database records -3. Test agent execution flow -4. Check AgentExecution records and results display - -## System Status - -**Current Status: ✅ STABLE COMPREHENSIVE SYSTEM** -- **8 agents** confirmed working and tested (4 webhook + 4 direct access) -- **6 categories** with clean, logical organization -- **Dual integration architecture** with clear separation and documentation -- **Streamlined agent creation** via JSON configs (instant file-based loading) -- **Scalable architecture** ready for 100+ agents - -**Current Agents:** -- **Webhook Agents (4)**: Social Ads Generator, Job Posting Generator, PDF Summarizer, 5 Whys Analyzer -- **Direct Access Agents (4)**: CyberSec Career Navigator, AI Brand Strategist, Lean Six Sigma Expert, SWOT Analysis Expert - -**Latest Changes (2025-08-16):** -- **🛡️ Comprehensive Security Optimization** - Complete security overhaul with CSP, input validation, and threat detection -- **🚀 Smart External Iframe System** - Future-proof CSP handling for external services (JotForm, Calendly, etc.) -- **🔧 Railway Deployment Fixes** - Fixed admin command foreign key constraints and deployment blockers -- **⚡ Performance Enhancements** - Database optimization, caching, and query improvements -- **📝 Emergency Rollback System** - Complete rollback documentation with 30-second recovery -- **🔒 Input Validation** - XSS prevention, SQL injection protection, file upload security -- **📊 Security Monitoring** - Comprehensive logging and threat detection -- **🎯 External Service Pages** - Added /event/, /cea/, /cea1/ with automatic CSP support - -**Architecture Status:** -- **🛡️ Production-Ready Security** - Enterprise-grade security implementation -- **🚀 Future-Proof External Integration** - Automatic CSP support for new external services -- **⚡ High Performance** - Optimized database queries and smart caching -- **🔧 Railway Deployment Ready** - All deployment issues resolved -- **📝 Complete Documentation** - Security, rollback, and development guides -- **🎯 Zero-Config External Pages** - Add to EXTERNAL_PAGES and it works immediately -- **🔒 Comprehensive Input Validation** - All user input sanitized and validated - -**Future Development:** -- **New agents** should follow patterns in `docs/AGENT_CREATION.md` -- **Use existing categories first** to avoid unnecessary proliferation -- **JSON file-based approach** is the only supported creation method -- **New views** should be added to appropriate focused modules (api_views, chat_views, web_views, direct_access_views) +This version focuses on core functionality for CapRover deployment without enterprise-grade optimizations. --- -Last updated: 2025-08-16 (Security & Performance Optimization Complete) - -## Documentation -- **Quick Agent Requests**: See `docs/AGENT_REQUEST_TEMPLATE.md` for simple agent request template -- **Agent Creation**: See `docs/AGENT_CREATION.md` for comprehensive agent creation guide -- **Project Overview**: This file (CLAUDE.md) for Django development and architecture +Last updated: 2025-09-04 (Simplified for CapRover) \ No newline at end of file diff --git a/Dockerfile.simple b/Dockerfile.simple new file mode 100644 index 0000000..a0b4125 --- /dev/null +++ b/Dockerfile.simple @@ -0,0 +1,26 @@ +FROM python:3.11-slim + +# Set working directory +WORKDIR /app + +# Install system dependencies +RUN apt-get update && apt-get install -y \ + gcc \ + libpq-dev \ + && rm -rf /var/lib/apt/lists/* + +# Copy and install Python dependencies +COPY requirements-simple.txt . +RUN pip install --no-cache-dir -r requirements-simple.txt + +# Copy application code +COPY . . + +# Collect static files +RUN python manage.py collectstatic --noinput --settings=netcop_hub.simple_settings + +# Expose port +EXPOSE 3000 + +# Start gunicorn +CMD ["gunicorn", "netcop_hub.wsgi:application", "--bind", "0.0.0.0:3000", "--settings", "netcop_hub.simple_settings"] \ No newline at end of file diff --git a/README-SIMPLE.md b/README-SIMPLE.md new file mode 100644 index 0000000..ac96f43 --- /dev/null +++ b/README-SIMPLE.md @@ -0,0 +1,61 @@ +# Quantum Tasks AI - Simple CapRover Deployment + +A basic Django AI agent marketplace for CapRover deployment. + +## Quick Start + +```bash +# Install dependencies +pip install -r requirements-simple.txt + +# Run migrations +python manage.py migrate + +# Create admin user +python manage.py createsuperuser + +# Start development server +python manage.py runserver +``` + +## CapRover Deployment + +1. **Upload to CapRover**: Use `captain-definition-simple` +2. **Set Environment Variables**: + ``` + SECRET_KEY=your-secret-key + DEBUG=false + ALLOWED_HOSTS=yourdomain.com + DATABASE_URL=postgres://user:pass@host/db + ``` +3. **Deploy**: Force build in CapRover dashboard + +## Project Structure + +- `agents/` - AI agent marketplace +- `authentication/` - User registration/login +- `wallet/` - Basic Stripe payments +- `core/` - Homepage and utilities + +## Adding Agents + +Add JSON files to `agents/configs/agents/`: + +```json +{ + "name": "My Agent", + "category": "productivity", + "price": 5, + "description": "Simple agent description", + "webhook_url": "https://your-webhook.com", + "form_schema": { + "fields": [{"name": "input", "type": "text", "label": "Input"}] + } +} +``` + +## Documentation + +- `CLAUDE-SIMPLE.md` - Detailed setup guide +- `CAPROVER_DEPLOYMENT_GUIDE.md` - Basic deployment steps +- `CAPROVER_DEPLOYMENT_COMPLETE_GUIDE.md` - Complete setup \ No newline at end of file diff --git a/README.md b/README.md index 83b0013..ac96f43 100644 --- a/README.md +++ b/README.md @@ -1,165 +1,61 @@ -# Quantum Tasks AI - CapRover Deployment +# Quantum Tasks AI - Simple CapRover Deployment -🚀 **CapRover-optimized deployment of the Quantum Tasks AI platform** +A basic Django AI agent marketplace for CapRover deployment. -## Overview - -This repository contains the CapRover deployment version of Quantum Tasks AI, a Django-based AI agent marketplace platform with comprehensive performance optimizations for production deployment. - -## 🎯 Key Features - -- **AI Agent Marketplace**: File-based agent system with dual integration types -- **Stripe Payment Integration**: Wallet system with transaction tracking -- **N8N Webhook Processing**: External AI service integrations -- **Email Verification System**: User authentication and notifications -- **Advanced Caching**: Redis-based multi-layer caching -- **Security Hardened**: Comprehensive security middleware and headers - -## 🛠 CapRover Deployment - -### Quick Deploy -1. **Create CapRover App**: `quantumtaskai` -2. **Configure Git Deployment**: Point to this repository -3. **Set Environment Variables**: See complete guide below -4. **Deploy Redis**: From CapRover One-Click Apps -5. **Force Build**: Deploy the application - -### Complete Deployment Guide -📖 **[CAPROVER_DEPLOYMENT_COMPLETE_GUIDE.md](./CAPROVER_DEPLOYMENT_COMPLETE_GUIDE.md)** - Step-by-step deployment instructions - -### Optimization Guides -- 🚀 **[CAPROVER_OPTIMIZATION_MASTER.md](./CAPROVER_OPTIMIZATION_MASTER.md)** - Complete optimization suite -- 🔴 **[CAPROVER_REDIS_SETUP.md](./CAPROVER_REDIS_SETUP.md)** - Redis caching configuration -- 📊 **[CAPROVER_MONITORING_SETUP.md](./CAPROVER_MONITORING_SETUP.md)** - Monitoring and logging -- 🔐 **[CAPROVER_SECURITY_BACKUP.md](./CAPROVER_SECURITY_BACKUP.md)** - Security and disaster recovery -- 📈 **[CAPROVER_SCALING_OPTIMIZATION.md](./CAPROVER_SCALING_OPTIMIZATION.md)** - Auto-scaling strategies - -## 🔧 Environment Variables - -### Required Variables -```env -# Database -DATABASE_URL=postgres://username:password@srv-captain--your-postgres-db:5432/your-database - -# Django Core -SECRET_KEY=your-generated-secret-key -DEBUG=false -ALLOWED_HOSTS=your-app.captain.your-domain.com - -# Redis Caching -REDIS_URL=redis://:your-redis-password@srv-captain--your-redis:6379/1 - -# Email Configuration -EMAIL_HOST_USER=your-email@example.com -EMAIL_HOST_PASSWORD=your-app-specific-password - -# Stripe Integration -STRIPE_SECRET_KEY=sk_live_your_stripe_secret_key -STRIPE_WEBHOOK_SECRET=whsec_your_webhook_secret - -# AI Services -OPENAI_API_KEY=sk-proj-your_openai_api_key -GROQ_API_KEY=gsk_your_groq_api_key -SERPAPI_API_KEY=your_serpapi_key -``` - -## 📊 Performance Optimizations - -### Implemented Optimizations -- ✅ **Docker Multi-layer Caching** - 50% faster builds -- ✅ **Database Connection Pooling** - 60% reduction in DB load -- ✅ **Redis Multi-layer Caching** - 80% cache hit rate -- ✅ **Gunicorn Tuning** - Optimal worker/thread configuration -- ✅ **Static File Compression** - WhiteNoise with compression -- ✅ **Health Checks** - Container health monitoring -- ✅ **Security Headers** - Comprehensive security implementation - -### Expected Performance -- **Response Time**: 100-300ms (70% improvement) -- **Concurrent Users**: 100+ simultaneous users -- **Memory Efficiency**: Stable 256-400MB usage -- **Auto-scaling**: CPU/Memory based scaling - -## 🏗 Architecture - -### Core Components -- **Django 5.2.4**: Main web framework -- **PostgreSQL**: Production database with connection pooling -- **Redis**: Multi-layer caching and session storage -- **Gunicorn**: WSGI server with optimized configuration -- **WhiteNoise**: Static file serving with compression - -### Agent System -- **File-based Configuration**: JSON-driven agent definitions -- **Dual Integration Types**: N8N webhooks + Direct access forms -- **Dynamic Form Generation**: Runtime form creation from JSON schemas -- **Execution Tracking**: Complete audit trail of agent usage - -## 🚀 Scaling Features - -- **Horizontal Scaling**: Multi-instance deployment with load balancing -- **Auto-scaling**: CPU/Memory threshold-based scaling -- **Resource Management**: Container resource limits and reservations -- **Health Monitoring**: Application and infrastructure health checks - -## 🔐 Security Features - -- **Content Security Policy**: Strict CSP with iframe support for external forms -- **Input Validation**: XSS and SQL injection protection -- **Rate Limiting**: IP-based request throttling -- **Security Headers**: Complete security header implementation -- **Database Security**: Dedicated users and permission management - -## 📈 Monitoring & Observability - -- **Application Metrics**: Response times, error rates, throughput -- **Infrastructure Metrics**: CPU, memory, disk, network usage -- **Business Metrics**: Agent executions, user registrations, payments -- **Alerting**: Email and webhook notifications for critical issues - -## 🛠 Development Commands +## Quick Start ```bash -# Apply migrations +# Install dependencies +pip install -r requirements-simple.txt + +# Run migrations python manage.py migrate -# Create superuser +# Create admin user python manage.py createsuperuser -# Collect static files -python manage.py collectstatic --noinput - -# Check application health -python manage.py check - -# Test agent system -python manage.py shell -c "from agents.services import AgentFileService; print(AgentFileService.get_agent_stats())" +# Start development server +python manage.py runserver ``` -## 📚 Documentation +## CapRover Deployment -- **[CLAUDE.md](./CLAUDE.md)** - Complete project documentation -- **[ROLLBACK.md](./ROLLBACK.md)** - Emergency rollback procedures -- **Agent Creation Guide** - `docs/AGENT_CREATION.md` +1. **Upload to CapRover**: Use `captain-definition-simple` +2. **Set Environment Variables**: + ``` + SECRET_KEY=your-secret-key + DEBUG=false + ALLOWED_HOSTS=yourdomain.com + DATABASE_URL=postgres://user:pass@host/db + ``` +3. **Deploy**: Force build in CapRover dashboard -## 🤝 Contributing +## Project Structure -This is the production CapRover deployment repository. For development: +- `agents/` - AI agent marketplace +- `authentication/` - User registration/login +- `wallet/` - Basic Stripe payments +- `core/` - Homepage and utilities -1. **Clone this repository** -2. **Follow CapRover deployment guide** -3. **Use environment-specific settings** -4. **Test thoroughly before production deployment** +## Adding Agents -## 📞 Support +Add JSON files to `agents/configs/agents/`: -- **Deployment Issues**: Check CapRover deployment guides -- **Performance Issues**: Review optimization documentation -- **Security Concerns**: Follow security hardening guide -- **Scaling Questions**: Consult auto-scaling documentation +```json +{ + "name": "My Agent", + "category": "productivity", + "price": 5, + "description": "Simple agent description", + "webhook_url": "https://your-webhook.com", + "form_schema": { + "fields": [{"name": "input", "type": "text", "label": "Input"}] + } +} +``` ---- +## Documentation -**🎉 Optimized for CapRover Production Deployment** - -This repository contains the complete, production-ready CapRover deployment with enterprise-grade optimizations for performance, security, and scalability. \ No newline at end of file +- `CLAUDE-SIMPLE.md` - Detailed setup guide +- `CAPROVER_DEPLOYMENT_GUIDE.md` - Basic deployment steps +- `CAPROVER_DEPLOYMENT_COMPLETE_GUIDE.md` - Complete setup \ No newline at end of file diff --git a/ROLLBACK.md b/ROLLBACK.md deleted file mode 100644 index 5ae1375..0000000 --- a/ROLLBACK.md +++ /dev/null @@ -1,103 +0,0 @@ -# 🔄 Emergency Rollback Guide - -## ⚡ QUICK ROLLBACK (30 seconds) - -If **anything fails** after deployment, run these commands immediately: - -```bash -# Navigate to project -cd /home/amit/projects/quantum_ai_v3 - -# Emergency: Revert ALL changes to last working state -git reset --hard 1cfdac2 # Last clean commit before optimizations -git clean -fd # Remove any untracked files - -# Verify clean state -git status # Should show "working tree clean" -``` - -## 🎯 SELECTIVE ROLLBACK - -### If Railway deployment fails: -```bash -# Revert just deployment configs -git checkout HEAD~1 -- railway.json -git checkout HEAD~1 -- requirements.txt -git checkout HEAD~1 -- netcop_hub/settings.py -``` - -### If database issues: -```bash -# Rollback migrations -python manage.py migrate agents 0007 -python manage.py migrate wallet 0002 -``` - -### If import errors: -```bash -# Remove new security files -rm -f core/middleware.py -rm -f core/validators.py -rm -f core/cache_utils.py -git checkout HEAD~1 -- netcop_hub/settings.py -``` - -## 📍 COMMIT REFERENCES - -- **Current (optimized)**: `87ec7cc` - Security & Performance Optimization -- **Last safe state**: `1cfdac2` - Remove Demo Business Calculator agent -- **Clean baseline**: `2583d83` - Add Demo Business Calculator agent - -## 🚨 EMERGENCY CONTACTS - -**If you need to rollback:** - -1. **Stop Railway deployment** (if in progress) -2. **Run quick rollback commands above** -3. **Verify application works locally**: `python manage.py runserver` -4. **Redeploy clean state** to Railway -5. **Test deployment works** - -## 🔍 TROUBLESHOOTING - -**Common failure patterns:** - -| Error | Quick Fix | -|-------|-----------| -| `ModuleNotFoundError: bleach` | `git checkout HEAD~1 -- requirements.txt` | -| `NameError: logging` | `git checkout HEAD~1 -- netcop_hub/settings.py` | -| Migration failure | `python manage.py migrate --fake` | -| Railway hanging | `git checkout HEAD~1 -- railway.json` | -| Admin command error | `git checkout HEAD~1 -- core/management/commands/` | - -## ✅ RECOVERY VERIFICATION - -After rollback, verify these work: - -```bash -# Test Django startup -python manage.py check - -# Test migrations -python manage.py showmigrations - -# Test admin creation -python manage.py check_admin - -# Test server startup -python manage.py runserver -``` - -## 🔒 SAFETY NOTES - -- ✅ **All changes are committed** - no data loss possible -- ✅ **Rollback is instant** - under 30 seconds -- ✅ **Can re-apply later** - commit `87ec7cc` preserves all work -- ✅ **Database safe** - migrations can be rolled back -- ✅ **Railway safe** - original configs preserved - ---- - -**Emergency Rollback Time: < 30 seconds** -**Data Loss Risk: ZERO** -**Recovery Success Rate: 100%** \ No newline at end of file diff --git a/agents/simple_services.py b/agents/simple_services.py new file mode 100644 index 0000000..a2ff667 --- /dev/null +++ b/agents/simple_services.py @@ -0,0 +1,73 @@ +""" +Simplified agent service for basic CapRover deployment +""" +import json +import os +from pathlib import Path +from typing import Dict, List, Optional + +class SimpleAgentService: + """Simple agent management without caching or complex features""" + + BASE_DIR = Path(__file__).resolve().parent.parent + AGENTS_CONFIG_DIR = BASE_DIR / 'agents' / 'configs' / 'agents' + CATEGORIES_CONFIG_FILE = BASE_DIR / 'agents' / 'configs' / 'categories' / 'categories.json' + + @classmethod + def get_all_agents(cls) -> List[Dict]: + """Load all agent configurations from JSON files""" + agents = [] + + if not cls.AGENTS_CONFIG_DIR.exists(): + return agents + + for file_path in cls.AGENTS_CONFIG_DIR.glob('*.json'): + try: + with open(file_path, 'r', encoding='utf-8') as f: + agent_data = json.load(f) + agent_data['slug'] = file_path.stem + agents.append(agent_data) + except (json.JSONDecodeError, FileNotFoundError) as e: + print(f"Error loading agent {file_path}: {e}") + continue + + return agents + + @classmethod + def get_agent(cls, slug: str) -> Optional[Dict]: + """Get a specific agent by slug""" + file_path = cls.AGENTS_CONFIG_DIR / f'{slug}.json' + + if not file_path.exists(): + return None + + try: + with open(file_path, 'r', encoding='utf-8') as f: + agent_data = json.load(f) + agent_data['slug'] = slug + return agent_data + except (json.JSONDecodeError, FileNotFoundError): + return None + + @classmethod + def get_categories(cls) -> Dict[str, Dict]: + """Load category configurations""" + try: + with open(cls.CATEGORIES_CONFIG_FILE, 'r', encoding='utf-8') as f: + return json.load(f) + except (json.JSONDecodeError, FileNotFoundError): + return {} + + @classmethod + def get_agents_by_category(cls) -> Dict[str, List[Dict]]: + """Group agents by category""" + agents = cls.get_all_agents() + categories = {} + + for agent in agents: + category = agent.get('category', 'other') + if category not in categories: + categories[category] = [] + categories[category].append(agent) + + return categories \ No newline at end of file diff --git a/captain-definition-simple b/captain-definition-simple new file mode 100644 index 0000000..0d8633d --- /dev/null +++ b/captain-definition-simple @@ -0,0 +1,4 @@ +{ + "schemaVersion": 2, + "dockerfilePath": "./Dockerfile.simple" +} \ No newline at end of file diff --git a/core/cache_utils.py b/core/cache_utils.py deleted file mode 100644 index b07b3df..0000000 --- a/core/cache_utils.py +++ /dev/null @@ -1,165 +0,0 @@ -""" -Cache utilities for performance optimization. -""" - -from django.core.cache import cache -from django.conf import settings -from functools import wraps -import hashlib -import json -import logging - -logger = logging.getLogger(__name__) - - -def cache_user_data(cache_key_prefix, timeout=None): - """ - Decorator for caching user-specific data. - - Args: - cache_key_prefix (str): Prefix for the cache key - timeout (int): Cache timeout in seconds (None for default) - """ - def decorator(func): - @wraps(func) - def wrapper(request, *args, **kwargs): - if not hasattr(request, 'user') or not request.user.is_authenticated: - # Don't cache for anonymous users - return func(request, *args, **kwargs) - - # Create unique cache key - cache_key = f"{cache_key_prefix}_{request.user.id}" - if args or kwargs: - # Include args and kwargs in cache key for uniqueness - key_data = f"{args}_{kwargs}" - key_hash = hashlib.md5(key_data.encode()).hexdigest()[:8] - cache_key += f"_{key_hash}" - - try: - # Try to get from cache - cached_result = cache.get(cache_key) - if cached_result is not None: - logger.debug(f"Cache hit for {cache_key}") - return cached_result - except Exception as e: - logger.warning(f"Cache get failed for {cache_key}: {e}") - - # Execute function and cache result - result = func(request, *args, **kwargs) - - try: - # Determine cache timeout - if timeout is None: - cache_timeout = 300 if settings.DEBUG else 1800 # 5 min / 30 min - else: - cache_timeout = timeout - - cache.set(cache_key, result, cache_timeout) - logger.debug(f"Cached result for {cache_key} (timeout: {cache_timeout}s)") - except Exception as e: - logger.warning(f"Cache set failed for {cache_key}: {e}") - - return result - return wrapper - return decorator - - -def cache_expensive_query(cache_key, timeout=None): - """ - Decorator for caching expensive database queries. - - Args: - cache_key (str): Cache key for the query - timeout (int): Cache timeout in seconds (None for default) - """ - def decorator(func): - @wraps(func) - def wrapper(*args, **kwargs): - # Create unique cache key with function args - full_cache_key = cache_key - if args or kwargs: - key_data = f"{args}_{kwargs}" - key_hash = hashlib.md5(key_data.encode()).hexdigest()[:8] - full_cache_key += f"_{key_hash}" - - try: - # Try to get from cache - cached_result = cache.get(full_cache_key) - if cached_result is not None: - logger.debug(f"Query cache hit for {full_cache_key}") - return cached_result - except Exception as e: - logger.warning(f"Query cache get failed for {full_cache_key}: {e}") - - # Execute function and cache result - result = func(*args, **kwargs) - - try: - # Determine cache timeout - if timeout is None: - cache_timeout = 600 if settings.DEBUG else 3600 # 10 min / 1 hour - else: - cache_timeout = timeout - - cache.set(full_cache_key, result, cache_timeout) - logger.debug(f"Cached query result for {full_cache_key} (timeout: {cache_timeout}s)") - except Exception as e: - logger.warning(f"Query cache set failed for {full_cache_key}: {e}") - - return result - return wrapper - return decorator - - -def invalidate_user_cache(user_id, cache_key_prefix): - """ - Invalidate all cache entries for a specific user and prefix. - - Args: - user_id (int): User ID - cache_key_prefix (str): Cache key prefix to invalidate - """ - try: - # Create pattern for user-specific cache keys - cache_pattern = f"{cache_key_prefix}_{user_id}" - - # Note: This is a simplified implementation - # In production, you might want to use Redis pattern matching - # or maintain a list of cache keys to invalidate - - # For now, we'll invalidate common variations - cache_keys_to_invalidate = [ - f"{cache_pattern}", - f"{cache_pattern}_*", # This won't work with default cache, needs Redis - ] - - for key in cache_keys_to_invalidate: - cache.delete(key) - - logger.info(f"Invalidated cache for user {user_id} with prefix {cache_key_prefix}") - except Exception as e: - logger.warning(f"Cache invalidation failed for user {user_id}: {e}") - - -def get_cache_stats(): - """ - Get cache statistics (Redis only). - Returns dict with cache statistics or None if not available. - """ - try: - # This only works with Redis backend - if hasattr(cache, '_cache') and hasattr(cache._cache, 'get_client'): - redis_client = cache._cache.get_client() - info = redis_client.info('memory') - return { - 'used_memory': info.get('used_memory', 0), - 'used_memory_human': info.get('used_memory_human', '0B'), - 'used_memory_peak': info.get('used_memory_peak', 0), - 'used_memory_peak_human': info.get('used_memory_peak_human', '0B'), - 'keyspace_hits': info.get('keyspace_hits', 0), - 'keyspace_misses': info.get('keyspace_misses', 0), - } - except Exception as e: - logger.debug(f"Could not get cache stats: {e}") - - return None \ No newline at end of file diff --git a/core/middleware.py b/core/middleware.py deleted file mode 100644 index a52ed4d..0000000 --- a/core/middleware.py +++ /dev/null @@ -1,192 +0,0 @@ -""" -Security middleware for enhanced security headers and CSP. -""" - -from django.conf import settings -from django.utils import timezone -import logging - -logger = logging.getLogger('core.security') - - -class SecurityHeadersMiddleware: - """ - Middleware to add comprehensive security headers to all responses. - Implements Content Security Policy, security headers, and security monitoring. - """ - - def __init__(self, get_response): - self.get_response = get_response - - - def __call__(self, request): - response = self.get_response(request) - - # Note: CSP removed - Django's built-in security + input validation provides better protection - # Complex CSP was causing more issues than security benefits - - # Additional Security Headers - response['X-Content-Type-Options'] = 'nosniff' - response['X-XSS-Protection'] = '1; mode=block' - response['Referrer-Policy'] = 'strict-origin-when-cross-origin' - response['Permissions-Policy'] = ( - 'geolocation=(), microphone=(), camera=(), ' - 'payment=(self "https://js.stripe.com"), ' - 'usb=(), magnetometer=(), gyroscope=(), accelerometer=()' - ) - - # X-Frame-Options handling - Simple and effective - if (request.path.endswith('/display/') and '/agents/' in request.path) or \ - (request.path.count('/') == 2 and not request.path.startswith(('/admin/', '/auth/', '/wallet/', '/agents/'))): - # Allow iframe embedding for agent display pages and external wrapper pages - response['X-Frame-Options'] = 'SAMEORIGIN' - else: - # Deny framing for all other pages - response['X-Frame-Options'] = 'DENY' - - # Optimize static asset serving - if request.path.startswith('/static/'): - response['Cache-Control'] = 'public, max-age=31536000' # 1 year cache for static assets - - # Security for critical pages - elif request.path.startswith('/admin/') or request.path.startswith('/wallet/'): - response['X-Frame-Options'] = 'DENY' - response['Cache-Control'] = 'no-store, no-cache, must-revalidate, max-age=0' - response['Pragma'] = 'no-cache' - response['Expires'] = '0' - - # Log security events for monitoring - if hasattr(request, 'user') and request.user.is_authenticated: - # Log administrative actions - if request.path.startswith('/admin/') and request.method == 'POST': - logger.info(f"Admin action by user {request.user.id} from IP {request.META.get('REMOTE_ADDR')}") - - # Log sensitive financial operations - if request.path.startswith('/wallet/') and request.method == 'POST': - logger.info(f"Wallet operation by user {request.user.id} from IP {request.META.get('REMOTE_ADDR')}") - - # Log agent executions - if request.path.startswith('/agents/api/execute') and request.method == 'POST': - logger.info(f"Agent execution by user {request.user.id} from IP {request.META.get('REMOTE_ADDR')}") - - # Log authentication failures - if hasattr(request, 'user') and not request.user.is_authenticated: - if request.path.startswith('/auth/') and request.method == 'POST': - logger.warning(f"Failed authentication attempt from IP {request.META.get('REMOTE_ADDR')}") - - return response - - -class SecurityMonitoringMiddleware: - """ - Middleware for security event monitoring and threat detection. - """ - - def __init__(self, get_response): - self.get_response = get_response - self.suspicious_patterns = [ - '.env', 'wp-admin', 'phpmyadmin', '../', '= 5: - self.suspicious_ips.add(ip) - self._log_security_event( - request, - 'suspicious_ip_detected', - f"IP {ip} marked suspicious after {self.failed_attempts[ip]['count']} failed attempts" - ) - - def _log_security_event(self, request, event_type, details): - """Log security events for monitoring""" - ip = request.META.get('REMOTE_ADDR', 'unknown') - user_id = request.user.id if hasattr(request, 'user') and request.user.is_authenticated else 'anonymous' - user_agent = request.META.get('HTTP_USER_AGENT', '')[:100] - - # Enhanced logging with more context - logger.warning( - f"Security Event: {event_type} - " - f"IP: {ip} - " - f"User: {user_id} - " - f"Path: {request.path} - " - f"Method: {request.method} - " - f"UA: {user_agent} - " - f"Referer: {request.META.get('HTTP_REFERER', 'none')[:100]} - " - f"Details: {details}" - ) - - # Additional context for critical events - if event_type in ['suspicious_request', 'potential_sqli', 'suspicious_ip_detected']: - logger.critical( - f"CRITICAL SECURITY ALERT: {event_type} - " - f"IP: {ip} - User: {user_id} - {details}" - ) \ No newline at end of file diff --git a/core/validators.py b/core/validators.py deleted file mode 100644 index 787efb0..0000000 --- a/core/validators.py +++ /dev/null @@ -1,310 +0,0 @@ -""" -Input validation utilities for security and data integrity. -""" - -import re -import bleach -from django.core.exceptions import ValidationError -from django.utils.html import escape -from decimal import Decimal, InvalidOperation -import logging - -logger = logging.getLogger('core.security') - - -class InputValidator: - """ - Comprehensive input validation for security and data integrity. - """ - - # Allowed HTML tags for rich text (very restrictive) - ALLOWED_TAGS = ['b', 'i', 'u', 'em', 'strong', 'p', 'br'] - ALLOWED_ATTRIBUTES = {} - - # Common injection patterns - INJECTION_PATTERNS = [ - r']*>.*?', # XSS - r'javascript:', # JavaScript protocol - r'on\w+\s*=', # Event handlers - r'expression\s*\(', # CSS expressions - r'@import', # CSS imports - r'vbscript:', # VBScript - r'data:text/html', # Data URLs - r'SELECT\s+.*FROM', # Basic SQL injection - r'UNION\s+SELECT', # Union SQL injection - r'DROP\s+TABLE', # SQL DROP - r'INSERT\s+INTO', # SQL INSERT - r'DELETE\s+FROM', # SQL DELETE - r'UPDATE\s+.*SET', # SQL UPDATE - r'\|\|\s*1\s*=\s*1', # Boolean SQL injection - r'1\s*=\s*1', # Boolean logic - r'<\s*iframe', # Iframe injection - r'<\s*object', # Object injection - r'<\s*embed', # Embed injection - ] - - @classmethod - def sanitize_string(cls, value, max_length=1000, allow_html=False): - """ - Sanitize a string input to prevent XSS and injection attacks. - - Args: - value: Input string to sanitize - max_length: Maximum allowed length - allow_html: Whether to allow safe HTML tags - - Returns: - Sanitized string - - Raises: - ValidationError: If input is invalid or malicious - """ - if not isinstance(value, str): - try: - value = str(value) - except: - raise ValidationError("Invalid input type") - - # Length check - if len(value) > max_length: - raise ValidationError(f"Input too long (max {max_length} characters)") - - # Check for injection patterns - for pattern in cls.INJECTION_PATTERNS: - if re.search(pattern, value, re.IGNORECASE): - logger.warning(f"Potential injection attempt detected: {pattern}") - raise ValidationError("Input contains potentially malicious content") - - # HTML sanitization - if allow_html: - # Use bleach to allow only safe HTML - value = bleach.clean( - value, - tags=cls.ALLOWED_TAGS, - attributes=cls.ALLOWED_ATTRIBUTES, - strip=True - ) - else: - # Strip all HTML and escape special characters - value = bleach.clean(value, tags=[], strip=True) - value = escape(value) - - # Remove null bytes and other control characters - value = value.replace('\x00', '').replace('\r', '').strip() - - return value - - @classmethod - def validate_email(cls, email): - """ - Validate email address format. - - Args: - email: Email address to validate - - Returns: - Sanitized email address - - Raises: - ValidationError: If email is invalid - """ - if not email or len(email) > 254: - raise ValidationError("Invalid email address") - - # Basic email regex (RFC 5322 simplified) - email_pattern = r'^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$' - if not re.match(email_pattern, email): - raise ValidationError("Invalid email format") - - # Check for dangerous patterns - dangerous_patterns = ['<', '>', '"', "'", '\\', '/', '%', '&'] - for pattern in dangerous_patterns: - if pattern in email: - raise ValidationError("Email contains invalid characters") - - return email.lower().strip() - - @classmethod - def validate_decimal_amount(cls, amount, min_value=0, max_value=10000): - """ - Validate monetary amount. - - Args: - amount: Amount to validate (string, int, float, or Decimal) - min_value: Minimum allowed value - max_value: Maximum allowed value - - Returns: - Decimal value - - Raises: - ValidationError: If amount is invalid - """ - try: - if isinstance(amount, str): - # Remove any non-numeric characters except decimal point - amount = re.sub(r'[^\d.]', '', amount) - - decimal_amount = Decimal(str(amount)) - - # Check range - if decimal_amount < min_value or decimal_amount > max_value: - raise ValidationError(f"Amount must be between {min_value} and {max_value}") - - # Check precision (max 2 decimal places for currency) - if decimal_amount.quantize(Decimal('0.01')) != decimal_amount: - raise ValidationError("Amount cannot have more than 2 decimal places") - - return decimal_amount - - except (InvalidOperation, ValueError, TypeError): - raise ValidationError("Invalid amount format") - - @classmethod - def validate_agent_slug(cls, slug): - """ - Validate agent slug format. - - Args: - slug: Agent slug to validate - - Returns: - Sanitized slug - - Raises: - ValidationError: If slug is invalid - """ - if not slug or len(slug) > 100: - raise ValidationError("Invalid agent slug") - - # Only allow alphanumeric, hyphens, and underscores - if not re.match(r'^[a-zA-Z0-9\-_]+$', slug): - raise ValidationError("Agent slug contains invalid characters") - - return slug.lower().strip() - - @classmethod - def validate_json_input(cls, data, max_size=10000): - """ - Validate JSON input data. - - Args: - data: Dictionary or JSON string to validate - max_size: Maximum size in bytes - - Returns: - Sanitized dictionary - - Raises: - ValidationError: If data is invalid - """ - import json - - if isinstance(data, str): - try: - data = json.loads(data) - except json.JSONDecodeError: - raise ValidationError("Invalid JSON format") - - if not isinstance(data, dict): - raise ValidationError("Input must be a JSON object") - - # Check size - json_str = json.dumps(data) - if len(json_str.encode('utf-8')) > max_size: - raise ValidationError(f"Input too large (max {max_size} bytes)") - - # Recursively sanitize all string values - sanitized_data = {} - for key, value in data.items(): - # Sanitize key - clean_key = cls.sanitize_string(str(key), max_length=100) - - # Sanitize value - if isinstance(value, str): - clean_value = cls.sanitize_string(value, max_length=2000) - elif isinstance(value, (int, float, bool)): - clean_value = value - elif isinstance(value, list): - # Sanitize list items (only strings) - clean_value = [] - for item in value[:10]: # Limit to 10 items - if isinstance(item, str): - clean_value.append(cls.sanitize_string(item, max_length=500)) - elif isinstance(item, (int, float, bool)): - clean_value.append(item) - else: - # Skip complex nested objects - continue - - sanitized_data[clean_key] = clean_value - - return sanitized_data - - @classmethod - def validate_file_upload(cls, uploaded_file, allowed_extensions=None, max_size=10485760): - """ - Validate file upload. - - Args: - uploaded_file: Django UploadedFile object - allowed_extensions: List of allowed file extensions - max_size: Maximum file size in bytes (default 10MB) - - Returns: - True if valid - - Raises: - ValidationError: If file is invalid - """ - if not uploaded_file: - raise ValidationError("No file provided") - - # Check file size - if uploaded_file.size > max_size: - raise ValidationError(f"File too large (max {max_size // 1048576}MB)") - - # Check file extension - if allowed_extensions: - import os - file_ext = os.path.splitext(uploaded_file.name)[1].lower() - if file_ext not in allowed_extensions: - raise ValidationError(f"File type not allowed. Allowed types: {', '.join(allowed_extensions)}") - - # Check filename for malicious patterns - filename = cls.sanitize_string(uploaded_file.name, max_length=255) - - # Additional security checks could include: - # - MIME type validation - # - File content scanning - # - Virus scanning - - return True - - -def validate_api_input(request_data): - """ - Validate API request input data. - - Args: - request_data: Request data dictionary - - Returns: - Sanitized data dictionary - - Raises: - ValidationError: If data is invalid - """ - validator = InputValidator() - - # Validate common fields - if 'agent_slug' in request_data: - request_data['agent_slug'] = validator.validate_agent_slug(request_data['agent_slug']) - - if 'input_data' in request_data: - request_data['input_data'] = validator.validate_json_input(request_data['input_data']) - - if 'amount' in request_data: - request_data['amount'] = validator.validate_decimal_amount(request_data['amount']) - - return request_data \ No newline at end of file diff --git a/docs/AGENT_CREATION.md b/docs/AGENT_CREATION.md index ceb28c0..db2bb26 100644 --- a/docs/AGENT_CREATION.md +++ b/docs/AGENT_CREATION.md @@ -1,276 +1,46 @@ -# Agent Creation Guide +# Simple Agent Creation Guide -Modern guide for adding new agents to the Quantum Tasks AI platform using the streamlined file-based system. +## Quick Agent Creation -## Overview +1. **Create JSON file** in `agents/configs/agents/your-agent-name.json` +2. **Add basic config**: -Quantum Tasks AI uses a **simple file-based agent system**: -1. Create JSON configuration file -2. Commit to git -3. Agent appears in marketplace automatically - -**Two Agent Types:** -- **Webhook Agents** - N8N integrations with dynamic forms -- **Direct Access Agents** - External forms (JotForm, etc.) with payment processing - -## Current Agents (8 Total) - -### Webhook Agents (4) -- **Social Ads Generator** - 6.00 AED - Social media ad creation -- **Job Posting Generator** - 10.00 AED - Professional job postings -- **PDF Summarizer** - 8.00 AED - Document analysis and summarization -- **5 Whys Analyzer** - 15.00 AED - Interactive root cause analysis - -### Direct Access Agents (4) -- **CyberSec Career Navigator** - FREE - Career guidance -- **AI Brand Strategist** - FREE - Brand strategy consultation -- **Lean Six Sigma Expert** - FREE - Process improvement -- **SWOT Analysis Expert** - FREE - Strategic analysis - -## Categories (6 Available) -Use existing categories first to avoid proliferation: - -- **`analysis`** 🧠 - Problem-solving, strategic analysis -- **`career-education`** 🎓 - Career guidance, professional development -- **`document-processing`** 📄 - PDF analysis, file processing -- **`human-resources`** 💼 - Job postings, HR automation -- **`marketing`** 📢 - Social ads, content marketing -- **`consulting`** 💼 - Business consultation, expert advice - -## Creating New Agents - -### Only Step: Create JSON Configuration ⚡ - -Add new file in `agents/configs/agents/your-agent-name.json`: - -**That's it! No other files needed.** - -#### Webhook Agent Example: ```json { - "slug": "content-optimizer", - "name": "Content Optimizer", - "short_description": "AI-powered content optimization and enhancement", - "description": "Enhance your content with AI-powered optimization suggestions, tone analysis, and improvement recommendations.", - "category": "marketing", - "price": 5.0, - "agent_type": "form", - "system_type": "webhook", + "name": "Your Agent Name", + "description": "What your agent does", + "category": "productivity", + "price": 5, + "webhook_url": "https://your-webhook-endpoint.com", "form_schema": { "fields": [ { - "name": "content", - "type": "textarea", - "label": "Content to Optimize", + "name": "input", + "type": "text", + "label": "Your Input", "required": true - }, - { - "name": "content_type", - "type": "select", - "label": "Content Type", - "required": true, - "options": [ - {"value": "blog", "label": "Blog Post"}, - {"value": "social", "label": "Social Media"}, - {"value": "email", "label": "Email Marketing"} - ] } ] - }, - "webhook_url": "http://localhost:5678/webhook/content-optimizer", - "access_url_name": "", - "display_url_name": "" + } } ``` -#### Direct Access Agent Example: -```json -{ - "slug": "business-coach", - "name": "Business Coach", - "short_description": "Expert business coaching consultation", - "description": "Get professional business coaching insights and strategic guidance from experienced consultants.", - "category": "consulting", - "price": 0.0, - "agent_type": "form", - "system_type": "direct_access", - "form_schema": {"fields": []}, - "webhook_url": "https://form.jotform.com/your-form-id", - "access_url_name": "agents:direct_access_handler", - "display_url_name": "agents:direct_access_display" -} -``` +3. **Restart server** - Agent appears automatically -### Step 2: Commit to Git (Optional for Production) -```bash -git add agents/configs/agents/your-agent-name.json -git commit -m "Add new agent: Your Agent Name 🤖 Generated with Claude Code" -git push -``` +## Available Categories -### Step 3: Done! ✅ -- **Development:** Agent loads automatically (or restart: `python manage.py runserver`) -- **Production:** Railway auto-deploys and agent appears in marketplace +Edit `agents/configs/categories/categories.json`: +- `productivity` - Work tools +- `content` - Content creation +- `analysis` - Data analysis +- `communication` - Communication tools -**No Python code, no URL routes, no templates needed!** 🎉 +## Field Types -## JSON Configuration Reference +- `text` - Single line text +- `textarea` - Multi-line text +- `number` - Numeric input +- `email` - Email input +- `file` - File upload -### Required Fields: -- `slug` - URL identifier (kebab-case) -- `name` - Display name -- `short_description` - Brief marketplace description -- `description` - Full description -- `category` - Must match existing category -- `price` - Price in AED (0.0 for free) -- `agent_type` - Always "form" -- `system_type` - "webhook" or "direct_access" - -### System-Specific Fields: - -**Webhook Agents:** -- `form_schema` - Form field definitions -- `webhook_url` - N8N webhook endpoint -- `access_url_name` - Empty "" -- `display_url_name` - Empty "" - -**Direct Access Agents:** -- `form_schema` - Usually `{"fields": []}` -- `webhook_url` - External form URL -- `access_url_name` - "agents:direct_access_handler" -- `display_url_name` - "agents:direct_access_display" - -## Form Field Types (Webhook Agents) - -- `text` - Single-line text input -- `textarea` - Multi-line text input -- `select` - Dropdown with options array -- `file` - File upload with drag-and-drop -- `url` - URL input with validation -- `checkbox` - Boolean checkbox - -## System Architecture Benefits 🚀 - -### True File-Based System -- **Zero Manual Coding**: Generic handlers for both agent types automatically handle everything -- **Zero URL Configuration**: Dynamic routing based on JSON config properties -- **Zero Templates**: Single generic template works for all direct access agents -- **Zero Database Setup**: Pure file-based loading with intelligent caching -- **Zero Error Handling**: Automatic webhook error detection and user feedback - -### Agent Type Handling -- **Webhook Agents**: Automatically generate dynamic forms from `form_schema` -- **Direct Access Agents**: Automatically handle payment processing + external redirect -- **Both Types**: Work with only JSON configuration, no additional code -- **Error Handling**: Automatically detects webhook failures (OpenAI quota, N8N issues, timeouts) and shows user-friendly messages - -### Development Workflow Comparison -```bash -# ❌ Old Complex Way (6+ steps) -1. Create JSON config -2. Write Python view functions -3. Add URL routes -4. Create HTML templates -5. Update view imports -6. Write error handling code -7. Test and debug - -# ✅ New Simple Way (1 step) -1. Create JSON config -# Done! Everything else is automatic 🎉 -# - Forms, routing, error handling, user feedback all automated -``` - -### How Both Agent Types Work Now - -**Webhook Agents:** -- JSON config → Dynamic form via `agent_detail_view` -- Form submission → N8N webhook → Results display -- Automatic error detection and user feedback -- No individual Python code needed - -**Direct Access Agents:** -- JSON config → Generic payment handler -- Payment → Generic iframe display -- No individual Python code needed - -## Automatic Error Handling 🛡️ - -The platform now includes **completely automated error handling** for all agents: - -### What's Automatically Handled -- **N8N Webhook Failures**: Service down, timeouts, configuration errors -- **AI Service Limits**: OpenAI quota exceeded, rate limits, API errors -- **Network Issues**: Connection failures, DNS problems, timeouts -- **Invalid Responses**: Malformed data, unexpected formats - -### User Experience -- **Clear Error Messages**: "Agent is temporarily unavailable. Please try again later." -- **Persistent Display**: Error shown in results area (won't disappear like notifications) -- **No Technical Jargon**: Simple, friendly language instead of HTTP status codes - -### Developer Benefits -- **Zero Configuration**: No error handling code needed in JSON configs -- **Automatic Detection**: System distinguishes between success and various failure types -- **Consistent UX**: All agents have identical error handling behavior -- **Debug Friendly**: Technical errors still logged to console for troubleshooting - -### Examples of Handled Errors -```json -// N8N Response (OpenAI quota exceeded) -{ - "errorMessage": "You exceeded your current quota", - "errorDetails": {"httpCode": "429"} -} -// → User sees: "Agent is temporarily unavailable. Please try again later." - -// HTTP 500 from webhook -// → User sees: "Agent is temporarily unavailable. Please try again later." - -// Connection timeout -// → User sees: "Agent is temporarily unavailable. Please try again later." -``` - -All technical details are logged for debugging, but users always see the same friendly message. - -## Custom Integration (Advanced) - -For agents needing custom behavior, add views to appropriate modules: - -- **API endpoints:** `agents/api_views.py` -- **Chat functionality:** `agents/chat_views.py` -- **Web interfaces:** `agents/web_views.py` -- **Direct access handlers:** `agents/direct_access_views.py` -- **Utilities:** `agents/utils.py` - -Then add URL routes in `agents/urls.py` and update marketplace template if needed. - -## External Services - -### For Webhook Agents: -- Create N8N workflow at webhook URL -- Configure to accept JSON payload with `sessionId`, `message`, etc. - -### For Direct Access Agents: -- Create external form (JotForm, Google Forms, etc.) -- Ensure form URL is publicly accessible - -## Railway Deployment - -**Automatic Process:** -1. ✅ Git push triggers Railway deployment -2. ✅ Agent files are processed automatically -3. ✅ New agents appear in production marketplace -4. ✅ No manual database work required - -## Quick Tips - -- **Use existing categories** - Avoid creating unnecessary new categories -- **Keep descriptions clear** - Users should understand what the agent does -- **Test webhook URLs** - Ensure N8N endpoints are accessible -- **Free vs Paid** - Set price to 0.0 for free agents -- **Consistent naming** - Use kebab-case for slugs, Title Case for names - ---- - -*Last updated: 2025-01-15* \ No newline at end of file +That's it! Your agent will appear in the marketplace. \ No newline at end of file diff --git a/docs/SIMPLE_AGENT_GUIDE.md b/docs/SIMPLE_AGENT_GUIDE.md new file mode 100644 index 0000000..db2bb26 --- /dev/null +++ b/docs/SIMPLE_AGENT_GUIDE.md @@ -0,0 +1,46 @@ +# Simple Agent Creation Guide + +## Quick Agent Creation + +1. **Create JSON file** in `agents/configs/agents/your-agent-name.json` +2. **Add basic config**: + +```json +{ + "name": "Your Agent Name", + "description": "What your agent does", + "category": "productivity", + "price": 5, + "webhook_url": "https://your-webhook-endpoint.com", + "form_schema": { + "fields": [ + { + "name": "input", + "type": "text", + "label": "Your Input", + "required": true + } + ] + } +} +``` + +3. **Restart server** - Agent appears automatically + +## Available Categories + +Edit `agents/configs/categories/categories.json`: +- `productivity` - Work tools +- `content` - Content creation +- `analysis` - Data analysis +- `communication` - Communication tools + +## Field Types + +- `text` - Single line text +- `textarea` - Multi-line text +- `number` - Numeric input +- `email` - Email input +- `file` - File upload + +That's it! Your agent will appear in the marketplace. \ No newline at end of file diff --git a/netcop_hub/simple_settings.py b/netcop_hub/simple_settings.py new file mode 100644 index 0000000..0c7044c --- /dev/null +++ b/netcop_hub/simple_settings.py @@ -0,0 +1,139 @@ +""" +Simplified Django settings for CapRover deployment +""" + +from pathlib import Path +from decouple import config +import os + +BASE_DIR = Path(__file__).resolve().parent.parent + +# Security +SECRET_KEY = config('SECRET_KEY', default='your-secret-key-here') +DEBUG = config('DEBUG', default=False, cast=bool) +ALLOWED_HOSTS = config('ALLOWED_HOSTS', default='*').split(',') + +# Application definition +INSTALLED_APPS = [ + 'django.contrib.admin', + 'django.contrib.auth', + 'django.contrib.contenttypes', + 'django.contrib.sessions', + 'django.contrib.messages', + 'django.contrib.staticfiles', + 'rest_framework', + + # Local apps + 'core', + 'authentication', + 'agents', + 'wallet', +] + +MIDDLEWARE = [ + 'django.middleware.security.SecurityMiddleware', + 'whitenoise.middleware.WhiteNoiseMiddleware', + 'django.contrib.sessions.middleware.SessionMiddleware', + 'django.middleware.common.CommonMiddleware', + 'django.middleware.csrf.CsrfViewMiddleware', + 'django.contrib.auth.middleware.AuthenticationMiddleware', + 'django.contrib.messages.middleware.MessageMiddleware', + 'django.middleware.clickjacking.XFrameOptionsMiddleware', +] + +ROOT_URLCONF = 'netcop_hub.urls' + +TEMPLATES = [ + { + 'BACKEND': 'django.template.backends.django.DjangoTemplates', + 'DIRS': [BASE_DIR / 'templates'], + 'APP_DIRS': True, + 'OPTIONS': { + 'context_processors': [ + 'django.template.context_processors.debug', + 'django.template.context_processors.request', + 'django.contrib.auth.context_processors.auth', + 'django.contrib.messages.context_processors.messages', + ], + }, + }, +] + +WSGI_APPLICATION = 'netcop_hub.wsgi.application' + +# Database +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.sqlite3', + 'NAME': BASE_DIR / 'db.sqlite3', + } +} + +# Use PostgreSQL if DATABASE_URL is provided (production) +database_url = config('DATABASE_URL', default='') +if database_url: + import dj_database_url + DATABASES['default'] = dj_database_url.parse(database_url) + +# Custom user model +AUTH_USER_MODEL = 'authentication.User' + +# Internationalization +LANGUAGE_CODE = 'en-us' +TIME_ZONE = 'UTC' +USE_I18N = True +USE_TZ = True + +# Static files +STATIC_URL = '/static/' +STATICFILES_DIRS = [BASE_DIR / 'static'] +STATIC_ROOT = BASE_DIR / 'staticfiles' + +# WhiteNoise for static files in production +STATICFILES_STORAGE = 'whitenoise.storage.CompressedManifestStaticFilesStorage' + +# Media files +MEDIA_URL = '/media/' +MEDIA_ROOT = BASE_DIR / 'media' + +# Default primary key field type +DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField' + +# Email configuration +EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend' +if not DEBUG: + EMAIL_BACKEND = 'django.core.mail.backends.smtp.EmailBackend' + EMAIL_HOST = 'smtp.gmail.com' + EMAIL_PORT = 587 + EMAIL_USE_TLS = True + EMAIL_HOST_USER = config('EMAIL_HOST_USER', default='') + EMAIL_HOST_PASSWORD = config('EMAIL_HOST_PASSWORD', default='') + +# Stripe configuration +STRIPE_SECRET_KEY = config('STRIPE_SECRET_KEY', default='') +STRIPE_WEBHOOK_SECRET = config('STRIPE_WEBHOOK_SECRET', default='') + +# REST Framework +REST_FRAMEWORK = { + 'DEFAULT_AUTHENTICATION_CLASSES': [ + 'rest_framework.authentication.SessionAuthentication', + ], + 'DEFAULT_PERMISSION_CLASSES': [ + 'rest_framework.permissions.IsAuthenticated', + ], +} + +# Logging +LOGGING = { + 'version': 1, + 'disable_existing_loggers': False, + 'handlers': { + 'console': { + 'class': 'logging.StreamHandler', + }, + }, + 'root': { + 'handlers': ['console'], + 'level': 'INFO', + }, +} \ No newline at end of file diff --git a/requirements-simple.txt b/requirements-simple.txt new file mode 100644 index 0000000..043877a --- /dev/null +++ b/requirements-simple.txt @@ -0,0 +1,18 @@ +# Core Django dependencies for CapRover deployment +Django==5.2.4 +djangorestframework==3.15.2 +python-decouple==3.8 + +# Database +dj-database-url==2.1.0 +psycopg2-binary==2.9.10 + +# Static files +whitenoise==6.8.2 + +# Production server +gunicorn==21.2.0 + +# Essential integrations +stripe==12.3.0 +requests==2.32.4 \ No newline at end of file