multitenetsaas/backend/core/management/commands/optimize_database.py

"""
Database Optimization Management Command

This management command provides comprehensive database optimization utilities
for the multi-tenant SaaS platform, including index management, query optimization,
performance analysis, and maintenance operations specifically designed for
Malaysian deployment scenarios.
"""

import argparse
import json
import logging
import sys
from typing import List, Dict, Any, Optional
from django.core.management.base import BaseCommand, CommandError
from django.db import connection
from django.core.cache import cache
from django.conf import settings
from django.utils import timezone
from django_tenants.utils import get_tenant_model, schema_context

from core.optimization.query_optimization import (
    DatabaseOptimizer,
    QueryOptimizer,
    CacheManager,
    DatabaseMaintenance
)
from core.optimization.index_manager import (
    IndexManager,
    IndexType,
    IndexStatus
)
from core.optimization.config import (
    get_config,
    DatabaseConfig,
    validate_environment_config
)


logger = logging.getLogger(__name__)


class Command(BaseCommand):
    """
    Database optimization management command.

    Usage:
        python manage.py optimize_database <action> [options]

    Actions:
        analyze - Analyze database performance
        indexes - Manage database indexes
        queries - Optimize database queries
        cache - Manage database cache
        maintenance - Perform database maintenance
        config - Show configuration
        malaysian - Malaysian-specific optimizations
        report - Generate comprehensive report
    """

    help = 'Optimize database performance for the multi-tenant SaaS platform'

    def add_arguments(self, parser):
        """Add command arguments."""
        parser.add_argument(
            'action',
            choices=[
                'analyze', 'indexes', 'queries', 'cache',
                'maintenance', 'config', 'malaysian', 'report'
            ],
            help='Optimization action to perform'
        )

        parser.add_argument(
            '--tenant',
            help='Specific tenant schema to optimize'
        )

        parser.add_argument(
            '--environment',
            choices=['production', 'staging', 'development'],
            default='production',
            help='Environment configuration to use'
        )

        parser.add_argument(
            '--dry-run',
            action='store_true',
            help='Show what would be done without executing'
        )

        parser.add_argument(
            '--verbose',
            action='store_true',
            help='Enable verbose output'
        )

        parser.add_argument(
            '--output',
            choices=['json', 'table', 'summary'],
            default='table',
            help='Output format'
        )

        parser.add_argument(
            '--hours',
            type=int,
            default=24,
            help='Number of hours to analyze (default: 24)'
        )

        parser.add_argument(
            '--index-action',
            choices=['create', 'drop', 'rebuild', 'analyze'],
            help='Specific index action to perform'
        )

        parser.add_argument(
            '--cache-action',
            choices=['clear', 'stats', 'warmup'],
            help='Cache management action'
        )

    def handle(self, *args, **options):
        """Handle the command."""
        self.setup_logging(options.get('verbose'))

        action = options['action']
        tenant_schema = options.get('tenant')
        environment = options.get('environment')
        dry_run = options.get('dry_run')
        output_format = options.get('output')

        # Validate configuration
        if not validate_environment_config(environment):
            raise CommandError(f"Invalid configuration for environment: {environment}")

        # Get configuration
        config = get_config(environment)

        if dry_run:
            self.stdout.write(
                self.style.WARNING(f"DRY RUN MODE - No changes will be made")
            )

        try:
            if action == 'analyze':
                self.analyze_database(config, tenant_schema, options, output_format)
            elif action == 'indexes':
                self.manage_indexes(config, tenant_schema, options, output_format)
            elif action == 'queries':
                self.optimize_queries(config, tenant_schema, options, output_format)
            elif action == 'cache':
                self.manage_cache(config, tenant_schema, options, output_format)
            elif action == 'maintenance':
                self.perform_maintenance(config, tenant_schema, options, output_format)
            elif action == 'config':
                self.show_configuration(config, output_format)
            elif action == 'malaysian':
                self.optimize_malaysian(config, tenant_schema, options, output_format)
            elif action == 'report':
                self.generate_report(config, tenant_schema, options, output_format)
            else:
                raise CommandError(f"Unknown action: {action}")

        except Exception as e:
            logger.error(f"Error during optimization: {e}")
            raise CommandError(f"Optimization failed: {e}")

    def setup_logging(self, verbose: bool):
        """Setup logging configuration."""
        level = logging.DEBUG if verbose else logging.INFO
        logging.basicConfig(
            level=level,
            format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
        )

    def analyze_database(self, config: DatabaseConfig, tenant_schema: Optional[str],
                       options: Dict[str, Any], output_format: str):
        """Analyze database performance."""
        self.stdout.write("Analyzing database performance...")

        optimizer = DatabaseOptimizer(tenant_schema)

        # Analyze query performance
        hours = options.get('hours', 24)
        performance_analysis = optimizer.analyze_query_performance(hours)

        # Analyze indexes
        index_manager = IndexManager(tenant_schema)
        index_performance = index_manager.analyze_index_performance()

        # Get table statistics
        table_stats = DatabaseMaintenance.get_table_sizes()

        # Combine results
        analysis_results = {
            'performance_analysis': performance_analysis,
            'index_analysis': index_performance,
            'table_statistics': table_stats,
            'optimization_recommendations': optimizer.get_optimization_report()
        }

        self.output_results(analysis_results, output_format)

    def manage_indexes(self, config: DatabaseConfig, tenant_schema: Optional[str],
                      options: Dict[str, Any], output_format: str):
        """Manage database indexes."""
        index_action = options.get('index_action')
        dry_run = options.get('dry_run')

        index_manager = IndexManager(tenant_schema)

        if index_action == 'analyze':
            self.stdout.write("Analyzing indexes...")
            results = index_manager.analyze_index_performance()
            self.output_results(results, output_format)

        elif index_action == 'create':
            self.stdout.write("Creating Malaysian-specific indexes...")
            created = index_manager.create_malaysian_indexes()
            created.extend(index_manager.create_multi_tenant_indexes())

            if dry_run:
                self.stdout.write(f"Would create {len(created)} indexes")
            else:
                self.stdout.write(
                    self.style.SUCCESS(f"Created {len(created)} indexes")
                )

        elif index_action == 'drop':
            self.stdout.write("Analyzing unused indexes...")
            performance_analysis = index_manager.analyze_index_performance()
            unused_recommendations = [
                r for r in performance_analysis['recommendations']
                if r.action == 'drop'
            ]

            if dry_run:
                self.stdout.write(f"Would drop {len(unused_recommendations)} unused indexes")
            else:
                results = index_manager.execute_recommendations(
                    unused_recommendations, dry_run
                )
                self.stdout.write(
                    self.style.SUCCESS(f"Dropped {results['executed']} indexes")
                )

        elif index_action == 'rebuild':
            self.stdout.write("Rebuilding fragmented indexes...")
            performance_analysis = index_manager.analyze_index_performance()
            rebuild_recommendations = [
                r for r in performance_analysis['recommendations']
                if r.action == 'rebuild'
            ]

            if dry_run:
                self.stdout.write(f"Would rebuild {len(rebuild_recommendations)} indexes")
            else:
                results = index_manager.execute_recommendations(
                    rebuild_recommendations, dry_run
                )
                self.stdout.write(
                    self.style.SUCCESS(f"Rebuilt {results['executed']} indexes")
                )

        else:
            # Show index statistics
            stats = index_manager.get_index_statistics()
            self.output_results(stats, output_format)

    def optimize_queries(self, config: DatabaseConfig, tenant_schema: Optional[str],
                        options: Dict[str, Any], output_format: str):
        """Optimize database queries."""
        self.stdout.write("Optimizing database queries...")

        optimizer = DatabaseOptimizer(tenant_schema)

        # Get optimization report
        report = optimizer.get_optimization_report()

        # Optimize Malaysian queries
        malaysian_opts = optimizer.optimize_malaysian_queries()

        # Add to report
        report['malaysian_optimizations'] = malaysian_opts

        self.output_results(report, output_format)

    def manage_cache(self, config: DatabaseConfig, tenant_schema: Optional[str],
                    options: Dict[str, Any], output_format: str):
        """Manage database cache."""
        cache_action = options.get('cache_action')

        cache_manager = CacheManager()

        if cache_action == 'clear':
            self.stdout.write("Clearing cache...")
            if options.get('dry_run'):
                self.stdout.write("Would clear all cache")
            else:
                cache.clear()
                self.stdout.write(
                    self.style.SUCCESS("Cache cleared successfully")
                )

        elif cache_action == 'stats':
            self.stdout.write("Getting cache statistics...")
            try:
                # Get Redis stats if using Redis
                if 'redis' in str(config.cache.backend):
                    import redis
                    r = redis.from_url(config.cache.location)
                    stats = r.info()
                    self.output_results(stats, output_format)
                else:
                    self.stdout.write("Cache statistics not available for current backend")
            except Exception as e:
                self.stdout.write(
                    self.style.ERROR(f"Error getting cache stats: {e}")
                )

        elif cache_action == 'warmup':
            self.stdout.write("Warming up cache...")
            # Implement cache warmup logic here
            self.stdout.write("Cache warmup completed")

        else:
            # Show cache configuration
            cache_config = {
                'backend': config.cache.backend.value,
                'location': config.cache.location,
                'timeout': config.cache.timeout,
                'key_prefix': config.cache.key_prefix,
                'enabled': config.performance.enable_caching
            }
            self.output_results(cache_config, output_format)

    def perform_maintenance(self, config: DatabaseConfig, tenant_schema: Optional[str],
                          options: Dict[str, Any], output_format: str):
        """Perform database maintenance."""
        self.stdout.write("Performing database maintenance...")

        maintenance = DatabaseMaintenance()

        # Run maintenance tasks
        with connection.cursor() as cursor:
            # Analyze tables
            cursor.execute("ANALYZE VERBOSE")
            self.stdout.write("Analyzed database tables")

            # Update statistics
            cursor.execute("VACUUM ANALYZE")
            self.stdout.write("Vacuumed and analyzed database")

        # Get maintenance results
        results = {
            'tables_analyzed': len(DatabaseMaintenance.get_table_sizes()),
            'maintenance_completed': timezone.now(),
            'next_recommended': timezone.now() + timezone.timedelta(days=7)
        }

        self.output_results(results, output_format)

    def show_configuration(self, config: DatabaseConfig, output_format: str):
        """Show current database configuration."""
        self.stdout.write("Database Configuration:")

        # Get all configuration settings
        db_config = config.get_database_optimization_settings()

        # Add Django settings
        db_config['django_database'] = config.get_django_database_config()
        db_config['django_cache'] = config.get_django_cache_config()

        # Add validation warnings
        warnings = config.validate_configuration()
        if warnings:
            db_config['warnings'] = warnings

        # Add recommendations
        recommendations = config.get_performance_recommendations()
        if recommendations:
            db_config['recommendations'] = recommendations

        self.output_results(db_config, output_format)

    def optimize_malaysian(self, config: DatabaseConfig, tenant_schema: Optional[str],
                         options: Dict[str, Any], output_format: str):
        """Perform Malaysian-specific optimizations."""
        self.stdout.write("Performing Malaysian-specific optimizations...")

        optimizer = DatabaseOptimizer(tenant_schema)
        index_manager = IndexManager(tenant_schema)

        # Create Malaysian indexes
        created_indexes = index_manager.create_malaysian_indexes()

        # Optimize Malaysian queries
        malaysian_opts = optimizer.optimize_malaysian_queries()

        # Get Malaysian-specific configuration
        malaysian_config = {
            'indexes_created': len(created_indexes),
            'index_names': created_indexes,
            'sst_queries_optimized': malaysian_opts['sst_queries_optimized'],
            'ic_validation_optimized': malaysian_opts['ic_validation_optimized'],
            'address_queries_optimized': malaysian_opts['address_queries_optimized'],
            'localization_improvements': malaysian_opts['localization_improvements'],
            'malaysian_config': {
                'timezone': config.malaysian.timezone,
                'locale': config.malaysian.locale,
                'currency': config.malaysian.currency,
                'local_caching_enabled': config.malaysian.enable_local_caching
            }
        }

        self.output_results(malaysian_config, output_format)

    def generate_report(self, config: DatabaseConfig, tenant_schema: Optional[str],
                       options: Dict[str, Any], output_format: str):
        """Generate comprehensive optimization report."""
        self.stdout.write("Generating comprehensive optimization report...")

        optimizer = DatabaseOptimizer(tenant_schema)
        index_manager = IndexManager(tenant_schema)

        # Collect all data for report
        report_data = {
            'report_generated': timezone.now(),
            'environment': config.environment,
            'tenant_schema': tenant_schema,
            'configuration': config.get_database_optimization_settings(),
            'performance_analysis': optimizer.analyze_query_performance(),
            'index_analysis': index_manager.analyze_index_performance(),
            'index_statistics': index_manager.get_index_statistics(),
            'optimization_report': optimizer.get_optimization_report(),
            'table_statistics': DatabaseMaintenance.get_table_sizes(),
            'malaysian_optimizations': optimizer.optimize_malaysian_queries(),
            'configuration_validation': config.validate_configuration(),
            'recommendations': config.get_performance_recommendations()
        }

        self.output_results(report_data, output_format)

    def output_results(self, results: Dict[str, Any], output_format: str):
        """Output results in specified format."""
        if output_format == 'json':
            self.output_json(results)
        elif output_format == 'table':
            self.output_table(results)
        elif output_format == 'summary':
            self.output_summary(results)
        else:
            self.output_table(results)

    def output_json(self, results: Dict[str, Any]):
        """Output results as JSON."""
        # Convert datetime objects to strings
        def json_serializer(obj):
            if hasattr(obj, 'isoformat'):
                return obj.isoformat()
            elif hasattr(obj, 'value'):
                return obj.value
            elif hasattr(obj, '__dict__'):
                return obj.__dict__
            return str(obj)

        json_output = json.dumps(results, indent=2, default=json_serializer)
        self.stdout.write(json_output)

    def output_table(self, results: Dict[str, Any]):
        """Output results as formatted tables."""
        for key, value in results.items():
            self.stdout.write(f"\n{self.style.SUCCESS(key.upper()}:}")
            if isinstance(value, dict):
                for sub_key, sub_value in value.items():
                    self.stdout.write(f"  {sub_key}: {sub_value}")
            elif isinstance(value, list):
                for i, item in enumerate(value):
                    self.stdout.write(f"  {i+1}. {item}")
            else:
                self.stdout.write(f"  {value}")

    def output_summary(self, results: Dict[str, Any]):
        """Output results as summary."""
        self.stdout.write(self.style.SUCCESS("OPTIMIZATION SUMMARY:"))

        # Extract key metrics
        total_queries = results.get('performance_analysis', {}).get('total_queries', 0)
        slow_queries = results.get('performance_analysis', {}).get('slow_queries', 0)
        total_indexes = results.get('index_analysis', {}).get('total_indexes', 0)
        unused_indexes = results.get('index_analysis', {}).get('unused_indexes', 0)
        recommendations = results.get('index_analysis', {}).get('recommendations', [])

        self.stdout.write(f"• Total queries analyzed: {total_queries}")
        self.stdout.write(f"• Slow queries found: {slow_queries}")
        self.stdout.write(f"• Total indexes: {total_indexes}")
        self.stdout.write(f"• Unused indexes: {unused_indexes}")
        self.stdout.write(f"• Recommendations: {len(recommendations)}")

        if recommendations:
            self.stdout.write("\nTOP RECOMMENDATIONS:")
            for i, rec in enumerate(recommendations[:5]):
                priority = rec.get('priority', 'medium')
                action = rec.get('action', 'unknown')
                reason = rec.get('reason', 'No reason provided')
                self.stdout.write(f"  {i+1}. [{priority.upper()}] {action}: {reason}")

        # Malaysian-specific summary
        malaysian_opts = results.get('malaysian_optimizations', {})
        if malaysian_opts:
            self.stdout.write(f"\nMALAYSIAN OPTIMIZATIONS:")
            self.stdout.write(f"• SST queries optimized: {malaysian_opts.get('sst_queries_optimized', 0)}")
            self.stdout.write(f"• IC validation optimized: {malaysian_opts.get('ic_validation_optimized', False)}")
            self.stdout.write(f"• Address queries optimized: {malaysian_opts.get('address_queries_optimized', 0)}")

    def create_progress_bar(self, total: int, description: str):
        """Create a simple progress bar."""
        return ProgressBar(total, description)


class ProgressBar:
    """Simple progress bar for command line output."""

    def __init__(self, total: int, description: str):
        self.total = total
        self.current = 0
        self.description = description

    def update(self, increment: int = 1):
        """Update progress."""
        self.current += increment
        self._draw()

    def _draw(self):
        """Draw progress bar."""
        if self.total == 0:
            return

        progress = self.current / self.total
        bar_length = 50
        filled = int(bar_length * progress)
        bar = '█' * filled + '-' * (bar_length - filled)

        percent = progress * 100
        self.stdout.write(f"\r{self.description}: |{bar}| {percent:.1f}% ({self.current}/{self.total})")
        self.stdout.flush()

    def finish(self):
        """Finish progress bar."""
        self._draw()
        self.stdout.write("\n")
        self.stdout.flush()