project initialization

backend/core/management/commands/optimize_database.py

@@ -0,0 +1,554 @@
+"""
+Database Optimization Management Command
+
+This management command provides comprehensive database optimization utilities
+for the multi-tenant SaaS platform, including index management, query optimization,
+performance analysis, and maintenance operations specifically designed for
+Malaysian deployment scenarios.
+"""
+
+import argparse
+import json
+import logging
+import sys
+from typing import List, Dict, Any, Optional
+from django.core.management.base import BaseCommand, CommandError
+from django.db import connection
+from django.core.cache import cache
+from django.conf import settings
+from django.utils import timezone
+from django_tenants.utils import get_tenant_model, schema_context
+
+from core.optimization.query_optimization import (
+    DatabaseOptimizer,
+    QueryOptimizer,
+    CacheManager,
+    DatabaseMaintenance
+)
+from core.optimization.index_manager import (
+    IndexManager,
+    IndexType,
+    IndexStatus
+)
+from core.optimization.config import (
+    get_config,
+    DatabaseConfig,
+    validate_environment_config
+)
+
+
+logger = logging.getLogger(__name__)
+
+
+class Command(BaseCommand):
+    """
+    Database optimization management command.
+
+    Usage:
+        python manage.py optimize_database <action> [options]
+
+    Actions:
+        analyze - Analyze database performance
+        indexes - Manage database indexes
+        queries - Optimize database queries
+        cache - Manage database cache
+        maintenance - Perform database maintenance
+        config - Show configuration
+        malaysian - Malaysian-specific optimizations
+        report - Generate comprehensive report
+    """
+
+    help = 'Optimize database performance for the multi-tenant SaaS platform'
+
+    def add_arguments(self, parser):
+        """Add command arguments."""
+        parser.add_argument(
+            'action',
+            choices=[
+                'analyze', 'indexes', 'queries', 'cache',
+                'maintenance', 'config', 'malaysian', 'report'
+            ],
+            help='Optimization action to perform'
+        )
+
+        parser.add_argument(
+            '--tenant',
+            help='Specific tenant schema to optimize'
+        )
+
+        parser.add_argument(
+            '--environment',
+            choices=['production', 'staging', 'development'],
+            default='production',
+            help='Environment configuration to use'
+        )
+
+        parser.add_argument(
+            '--dry-run',
+            action='store_true',
+            help='Show what would be done without executing'
+        )
+
+        parser.add_argument(
+            '--verbose',
+            action='store_true',
+            help='Enable verbose output'
+        )
+
+        parser.add_argument(
+            '--output',
+            choices=['json', 'table', 'summary'],
+            default='table',
+            help='Output format'
+        )
+
+        parser.add_argument(
+            '--hours',
+            type=int,
+            default=24,
+            help='Number of hours to analyze (default: 24)'
+        )
+
+        parser.add_argument(
+            '--index-action',
+            choices=['create', 'drop', 'rebuild', 'analyze'],
+            help='Specific index action to perform'
+        )
+
+        parser.add_argument(
+            '--cache-action',
+            choices=['clear', 'stats', 'warmup'],
+            help='Cache management action'
+        )
+
+    def handle(self, *args, **options):
+        """Handle the command."""
+        self.setup_logging(options.get('verbose'))
+
+        action = options['action']
+        tenant_schema = options.get('tenant')
+        environment = options.get('environment')
+        dry_run = options.get('dry_run')
+        output_format = options.get('output')
+
+        # Validate configuration
+        if not validate_environment_config(environment):
+            raise CommandError(f"Invalid configuration for environment: {environment}")
+
+        # Get configuration
+        config = get_config(environment)
+
+        if dry_run:
+            self.stdout.write(
+                self.style.WARNING(f"DRY RUN MODE - No changes will be made")
+            )
+
+        try:
+            if action == 'analyze':
+                self.analyze_database(config, tenant_schema, options, output_format)
+            elif action == 'indexes':
+                self.manage_indexes(config, tenant_schema, options, output_format)
+            elif action == 'queries':
+                self.optimize_queries(config, tenant_schema, options, output_format)
+            elif action == 'cache':
+                self.manage_cache(config, tenant_schema, options, output_format)
+            elif action == 'maintenance':
+                self.perform_maintenance(config, tenant_schema, options, output_format)
+            elif action == 'config':
+                self.show_configuration(config, output_format)
+            elif action == 'malaysian':
+                self.optimize_malaysian(config, tenant_schema, options, output_format)
+            elif action == 'report':
+                self.generate_report(config, tenant_schema, options, output_format)
+            else:
+                raise CommandError(f"Unknown action: {action}")
+
+        except Exception as e:
+            logger.error(f"Error during optimization: {e}")
+            raise CommandError(f"Optimization failed: {e}")
+
+    def setup_logging(self, verbose: bool):
+        """Setup logging configuration."""
+        level = logging.DEBUG if verbose else logging.INFO
+        logging.basicConfig(
+            level=level,
+            format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+        )
+
+    def analyze_database(self, config: DatabaseConfig, tenant_schema: Optional[str],
+                       options: Dict[str, Any], output_format: str):
+        """Analyze database performance."""
+        self.stdout.write("Analyzing database performance...")
+
+        optimizer = DatabaseOptimizer(tenant_schema)
+
+        # Analyze query performance
+        hours = options.get('hours', 24)
+        performance_analysis = optimizer.analyze_query_performance(hours)
+
+        # Analyze indexes
+        index_manager = IndexManager(tenant_schema)
+        index_performance = index_manager.analyze_index_performance()
+
+        # Get table statistics
+        table_stats = DatabaseMaintenance.get_table_sizes()
+
+        # Combine results
+        analysis_results = {
+            'performance_analysis': performance_analysis,
+            'index_analysis': index_performance,
+            'table_statistics': table_stats,
+            'optimization_recommendations': optimizer.get_optimization_report()
+        }
+
+        self.output_results(analysis_results, output_format)
+
+    def manage_indexes(self, config: DatabaseConfig, tenant_schema: Optional[str],
+                      options: Dict[str, Any], output_format: str):
+        """Manage database indexes."""
+        index_action = options.get('index_action')
+        dry_run = options.get('dry_run')
+
+        index_manager = IndexManager(tenant_schema)
+
+        if index_action == 'analyze':
+            self.stdout.write("Analyzing indexes...")
+            results = index_manager.analyze_index_performance()
+            self.output_results(results, output_format)
+
+        elif index_action == 'create':
+            self.stdout.write("Creating Malaysian-specific indexes...")
+            created = index_manager.create_malaysian_indexes()
+            created.extend(index_manager.create_multi_tenant_indexes())
+
+            if dry_run:
+                self.stdout.write(f"Would create {len(created)} indexes")
+            else:
+                self.stdout.write(
+                    self.style.SUCCESS(f"Created {len(created)} indexes")
+                )
+
+        elif index_action == 'drop':
+            self.stdout.write("Analyzing unused indexes...")
+            performance_analysis = index_manager.analyze_index_performance()
+            unused_recommendations = [
+                r for r in performance_analysis['recommendations']
+                if r.action == 'drop'
+            ]
+
+            if dry_run:
+                self.stdout.write(f"Would drop {len(unused_recommendations)} unused indexes")
+            else:
+                results = index_manager.execute_recommendations(
+                    unused_recommendations, dry_run
+                )
+                self.stdout.write(
+                    self.style.SUCCESS(f"Dropped {results['executed']} indexes")
+                )
+
+        elif index_action == 'rebuild':
+            self.stdout.write("Rebuilding fragmented indexes...")
+            performance_analysis = index_manager.analyze_index_performance()
+            rebuild_recommendations = [
+                r for r in performance_analysis['recommendations']
+                if r.action == 'rebuild'
+            ]
+
+            if dry_run:
+                self.stdout.write(f"Would rebuild {len(rebuild_recommendations)} indexes")
+            else:
+                results = index_manager.execute_recommendations(
+                    rebuild_recommendations, dry_run
+                )
+                self.stdout.write(
+                    self.style.SUCCESS(f"Rebuilt {results['executed']} indexes")
+                )
+
+        else:
+            # Show index statistics
+            stats = index_manager.get_index_statistics()
+            self.output_results(stats, output_format)
+
+    def optimize_queries(self, config: DatabaseConfig, tenant_schema: Optional[str],
+                        options: Dict[str, Any], output_format: str):
+        """Optimize database queries."""
+        self.stdout.write("Optimizing database queries...")
+
+        optimizer = DatabaseOptimizer(tenant_schema)
+
+        # Get optimization report
+        report = optimizer.get_optimization_report()
+
+        # Optimize Malaysian queries
+        malaysian_opts = optimizer.optimize_malaysian_queries()
+
+        # Add to report
+        report['malaysian_optimizations'] = malaysian_opts
+
+        self.output_results(report, output_format)
+
+    def manage_cache(self, config: DatabaseConfig, tenant_schema: Optional[str],
+                    options: Dict[str, Any], output_format: str):
+        """Manage database cache."""
+        cache_action = options.get('cache_action')
+
+        cache_manager = CacheManager()
+
+        if cache_action == 'clear':
+            self.stdout.write("Clearing cache...")
+            if options.get('dry_run'):
+                self.stdout.write("Would clear all cache")
+            else:
+                cache.clear()
+                self.stdout.write(
+                    self.style.SUCCESS("Cache cleared successfully")
+                )
+
+        elif cache_action == 'stats':
+            self.stdout.write("Getting cache statistics...")
+            try:
+                # Get Redis stats if using Redis
+                if 'redis' in str(config.cache.backend):
+                    import redis
+                    r = redis.from_url(config.cache.location)
+                    stats = r.info()
+                    self.output_results(stats, output_format)
+                else:
+                    self.stdout.write("Cache statistics not available for current backend")
+            except Exception as e:
+                self.stdout.write(
+                    self.style.ERROR(f"Error getting cache stats: {e}")
+                )
+
+        elif cache_action == 'warmup':
+            self.stdout.write("Warming up cache...")
+            # Implement cache warmup logic here
+            self.stdout.write("Cache warmup completed")
+
+        else:
+            # Show cache configuration
+            cache_config = {
+                'backend': config.cache.backend.value,
+                'location': config.cache.location,
+                'timeout': config.cache.timeout,
+                'key_prefix': config.cache.key_prefix,
+                'enabled': config.performance.enable_caching
+            }
+            self.output_results(cache_config, output_format)
+
+    def perform_maintenance(self, config: DatabaseConfig, tenant_schema: Optional[str],
+                          options: Dict[str, Any], output_format: str):
+        """Perform database maintenance."""
+        self.stdout.write("Performing database maintenance...")
+
+        maintenance = DatabaseMaintenance()
+
+        # Run maintenance tasks
+        with connection.cursor() as cursor:
+            # Analyze tables
+            cursor.execute("ANALYZE VERBOSE")
+            self.stdout.write("Analyzed database tables")
+
+            # Update statistics
+            cursor.execute("VACUUM ANALYZE")
+            self.stdout.write("Vacuumed and analyzed database")
+
+        # Get maintenance results
+        results = {
+            'tables_analyzed': len(DatabaseMaintenance.get_table_sizes()),
+            'maintenance_completed': timezone.now(),
+            'next_recommended': timezone.now() + timezone.timedelta(days=7)
+        }
+
+        self.output_results(results, output_format)
+
+    def show_configuration(self, config: DatabaseConfig, output_format: str):
+        """Show current database configuration."""
+        self.stdout.write("Database Configuration:")
+
+        # Get all configuration settings
+        db_config = config.get_database_optimization_settings()
+
+        # Add Django settings
+        db_config['django_database'] = config.get_django_database_config()
+        db_config['django_cache'] = config.get_django_cache_config()
+
+        # Add validation warnings
+        warnings = config.validate_configuration()
+        if warnings:
+            db_config['warnings'] = warnings
+
+        # Add recommendations
+        recommendations = config.get_performance_recommendations()
+        if recommendations:
+            db_config['recommendations'] = recommendations
+
+        self.output_results(db_config, output_format)
+
+    def optimize_malaysian(self, config: DatabaseConfig, tenant_schema: Optional[str],
+                         options: Dict[str, Any], output_format: str):
+        """Perform Malaysian-specific optimizations."""
+        self.stdout.write("Performing Malaysian-specific optimizations...")
+
+        optimizer = DatabaseOptimizer(tenant_schema)
+        index_manager = IndexManager(tenant_schema)
+
+        # Create Malaysian indexes
+        created_indexes = index_manager.create_malaysian_indexes()
+
+        # Optimize Malaysian queries
+        malaysian_opts = optimizer.optimize_malaysian_queries()
+
+        # Get Malaysian-specific configuration
+        malaysian_config = {
+            'indexes_created': len(created_indexes),
+            'index_names': created_indexes,
+            'sst_queries_optimized': malaysian_opts['sst_queries_optimized'],
+            'ic_validation_optimized': malaysian_opts['ic_validation_optimized'],
+            'address_queries_optimized': malaysian_opts['address_queries_optimized'],
+            'localization_improvements': malaysian_opts['localization_improvements'],
+            'malaysian_config': {
+                'timezone': config.malaysian.timezone,
+                'locale': config.malaysian.locale,
+                'currency': config.malaysian.currency,
+                'local_caching_enabled': config.malaysian.enable_local_caching
+            }
+        }
+
+        self.output_results(malaysian_config, output_format)
+
+    def generate_report(self, config: DatabaseConfig, tenant_schema: Optional[str],
+                       options: Dict[str, Any], output_format: str):
+        """Generate comprehensive optimization report."""
+        self.stdout.write("Generating comprehensive optimization report...")
+
+        optimizer = DatabaseOptimizer(tenant_schema)
+        index_manager = IndexManager(tenant_schema)
+
+        # Collect all data for report
+        report_data = {
+            'report_generated': timezone.now(),
+            'environment': config.environment,
+            'tenant_schema': tenant_schema,
+            'configuration': config.get_database_optimization_settings(),
+            'performance_analysis': optimizer.analyze_query_performance(),
+            'index_analysis': index_manager.analyze_index_performance(),
+            'index_statistics': index_manager.get_index_statistics(),
+            'optimization_report': optimizer.get_optimization_report(),
+            'table_statistics': DatabaseMaintenance.get_table_sizes(),
+            'malaysian_optimizations': optimizer.optimize_malaysian_queries(),
+            'configuration_validation': config.validate_configuration(),
+            'recommendations': config.get_performance_recommendations()
+        }
+
+        self.output_results(report_data, output_format)
+
+    def output_results(self, results: Dict[str, Any], output_format: str):
+        """Output results in specified format."""
+        if output_format == 'json':
+            self.output_json(results)
+        elif output_format == 'table':
+            self.output_table(results)
+        elif output_format == 'summary':
+            self.output_summary(results)
+        else:
+            self.output_table(results)
+
+    def output_json(self, results: Dict[str, Any]):
+        """Output results as JSON."""
+        # Convert datetime objects to strings
+        def json_serializer(obj):
+            if hasattr(obj, 'isoformat'):
+                return obj.isoformat()
+            elif hasattr(obj, 'value'):
+                return obj.value
+            elif hasattr(obj, '__dict__'):
+                return obj.__dict__
+            return str(obj)
+
+        json_output = json.dumps(results, indent=2, default=json_serializer)
+        self.stdout.write(json_output)
+
+    def output_table(self, results: Dict[str, Any]):
+        """Output results as formatted tables."""
+        for key, value in results.items():
+            self.stdout.write(f"\n{self.style.SUCCESS(key.upper()}:}")
+            if isinstance(value, dict):
+                for sub_key, sub_value in value.items():
+                    self.stdout.write(f"  {sub_key}: {sub_value}")
+            elif isinstance(value, list):
+                for i, item in enumerate(value):
+                    self.stdout.write(f"  {i+1}. {item}")
+            else:
+                self.stdout.write(f"  {value}")
+
+    def output_summary(self, results: Dict[str, Any]):
+        """Output results as summary."""
+        self.stdout.write(self.style.SUCCESS("OPTIMIZATION SUMMARY:"))
+
+        # Extract key metrics
+        total_queries = results.get('performance_analysis', {}).get('total_queries', 0)
+        slow_queries = results.get('performance_analysis', {}).get('slow_queries', 0)
+        total_indexes = results.get('index_analysis', {}).get('total_indexes', 0)
+        unused_indexes = results.get('index_analysis', {}).get('unused_indexes', 0)
+        recommendations = results.get('index_analysis', {}).get('recommendations', [])
+
+        self.stdout.write(f"• Total queries analyzed: {total_queries}")
+        self.stdout.write(f"• Slow queries found: {slow_queries}")
+        self.stdout.write(f"• Total indexes: {total_indexes}")
+        self.stdout.write(f"• Unused indexes: {unused_indexes}")
+        self.stdout.write(f"• Recommendations: {len(recommendations)}")
+
+        if recommendations:
+            self.stdout.write("\nTOP RECOMMENDATIONS:")
+            for i, rec in enumerate(recommendations[:5]):
+                priority = rec.get('priority', 'medium')
+                action = rec.get('action', 'unknown')
+                reason = rec.get('reason', 'No reason provided')
+                self.stdout.write(f"  {i+1}. [{priority.upper()}] {action}: {reason}")
+
+        # Malaysian-specific summary
+        malaysian_opts = results.get('malaysian_optimizations', {})
+        if malaysian_opts:
+            self.stdout.write(f"\nMALAYSIAN OPTIMIZATIONS:")
+            self.stdout.write(f"• SST queries optimized: {malaysian_opts.get('sst_queries_optimized', 0)}")
+            self.stdout.write(f"• IC validation optimized: {malaysian_opts.get('ic_validation_optimized', False)}")
+            self.stdout.write(f"• Address queries optimized: {malaysian_opts.get('address_queries_optimized', 0)}")
+
+    def create_progress_bar(self, total: int, description: str):
+        """Create a simple progress bar."""
+        return ProgressBar(total, description)
+
+
+class ProgressBar:
+    """Simple progress bar for command line output."""
+
+    def __init__(self, total: int, description: str):
+        self.total = total
+        self.current = 0
+        self.description = description
+
+    def update(self, increment: int = 1):
+        """Update progress."""
+        self.current += increment
+        self._draw()
+
+    def _draw(self):
+        """Draw progress bar."""
+        if self.total == 0:
+            return
+
+        progress = self.current / self.total
+        bar_length = 50
+        filled = int(bar_length * progress)
+        bar = '█' * filled + '-' * (bar_length - filled)
+
+        percent = progress * 100
+        self.stdout.write(f"\r{self.description}: |{bar}| {percent:.1f}% ({self.current}/{self.total})")
+        self.stdout.flush()
+
+    def finish(self):
+        """Finish progress bar."""
+        self._draw()
+        self.stdout.write("\n")
+        self.stdout.flush()