""" Database Optimization Management Command This management command provides comprehensive database optimization utilities for the multi-tenant SaaS platform, including index management, query optimization, performance analysis, and maintenance operations specifically designed for Malaysian deployment scenarios. """ import argparse import json import logging import sys from typing import List, Dict, Any, Optional from django.core.management.base import BaseCommand, CommandError from django.db import connection from django.core.cache import cache from django.conf import settings from django.utils import timezone from django_tenants.utils import get_tenant_model, schema_context from core.optimization.query_optimization import ( DatabaseOptimizer, QueryOptimizer, CacheManager, DatabaseMaintenance ) from core.optimization.index_manager import ( IndexManager, IndexType, IndexStatus ) from core.optimization.config import ( get_config, DatabaseConfig, validate_environment_config ) logger = logging.getLogger(__name__) class Command(BaseCommand): """ Database optimization management command. Usage: python manage.py optimize_database [options] Actions: analyze - Analyze database performance indexes - Manage database indexes queries - Optimize database queries cache - Manage database cache maintenance - Perform database maintenance config - Show configuration malaysian - Malaysian-specific optimizations report - Generate comprehensive report """ help = 'Optimize database performance for the multi-tenant SaaS platform' def add_arguments(self, parser): """Add command arguments.""" parser.add_argument( 'action', choices=[ 'analyze', 'indexes', 'queries', 'cache', 'maintenance', 'config', 'malaysian', 'report' ], help='Optimization action to perform' ) parser.add_argument( '--tenant', help='Specific tenant schema to optimize' ) parser.add_argument( '--environment', choices=['production', 'staging', 'development'], default='production', help='Environment configuration to use' ) parser.add_argument( '--dry-run', action='store_true', help='Show what would be done without executing' ) parser.add_argument( '--verbose', action='store_true', help='Enable verbose output' ) parser.add_argument( '--output', choices=['json', 'table', 'summary'], default='table', help='Output format' ) parser.add_argument( '--hours', type=int, default=24, help='Number of hours to analyze (default: 24)' ) parser.add_argument( '--index-action', choices=['create', 'drop', 'rebuild', 'analyze'], help='Specific index action to perform' ) parser.add_argument( '--cache-action', choices=['clear', 'stats', 'warmup'], help='Cache management action' ) def handle(self, *args, **options): """Handle the command.""" self.setup_logging(options.get('verbose')) action = options['action'] tenant_schema = options.get('tenant') environment = options.get('environment') dry_run = options.get('dry_run') output_format = options.get('output') # Validate configuration if not validate_environment_config(environment): raise CommandError(f"Invalid configuration for environment: {environment}") # Get configuration config = get_config(environment) if dry_run: self.stdout.write( self.style.WARNING(f"DRY RUN MODE - No changes will be made") ) try: if action == 'analyze': self.analyze_database(config, tenant_schema, options, output_format) elif action == 'indexes': self.manage_indexes(config, tenant_schema, options, output_format) elif action == 'queries': self.optimize_queries(config, tenant_schema, options, output_format) elif action == 'cache': self.manage_cache(config, tenant_schema, options, output_format) elif action == 'maintenance': self.perform_maintenance(config, tenant_schema, options, output_format) elif action == 'config': self.show_configuration(config, output_format) elif action == 'malaysian': self.optimize_malaysian(config, tenant_schema, options, output_format) elif action == 'report': self.generate_report(config, tenant_schema, options, output_format) else: raise CommandError(f"Unknown action: {action}") except Exception as e: logger.error(f"Error during optimization: {e}") raise CommandError(f"Optimization failed: {e}") def setup_logging(self, verbose: bool): """Setup logging configuration.""" level = logging.DEBUG if verbose else logging.INFO logging.basicConfig( level=level, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s' ) def analyze_database(self, config: DatabaseConfig, tenant_schema: Optional[str], options: Dict[str, Any], output_format: str): """Analyze database performance.""" self.stdout.write("Analyzing database performance...") optimizer = DatabaseOptimizer(tenant_schema) # Analyze query performance hours = options.get('hours', 24) performance_analysis = optimizer.analyze_query_performance(hours) # Analyze indexes index_manager = IndexManager(tenant_schema) index_performance = index_manager.analyze_index_performance() # Get table statistics table_stats = DatabaseMaintenance.get_table_sizes() # Combine results analysis_results = { 'performance_analysis': performance_analysis, 'index_analysis': index_performance, 'table_statistics': table_stats, 'optimization_recommendations': optimizer.get_optimization_report() } self.output_results(analysis_results, output_format) def manage_indexes(self, config: DatabaseConfig, tenant_schema: Optional[str], options: Dict[str, Any], output_format: str): """Manage database indexes.""" index_action = options.get('index_action') dry_run = options.get('dry_run') index_manager = IndexManager(tenant_schema) if index_action == 'analyze': self.stdout.write("Analyzing indexes...") results = index_manager.analyze_index_performance() self.output_results(results, output_format) elif index_action == 'create': self.stdout.write("Creating Malaysian-specific indexes...") created = index_manager.create_malaysian_indexes() created.extend(index_manager.create_multi_tenant_indexes()) if dry_run: self.stdout.write(f"Would create {len(created)} indexes") else: self.stdout.write( self.style.SUCCESS(f"Created {len(created)} indexes") ) elif index_action == 'drop': self.stdout.write("Analyzing unused indexes...") performance_analysis = index_manager.analyze_index_performance() unused_recommendations = [ r for r in performance_analysis['recommendations'] if r.action == 'drop' ] if dry_run: self.stdout.write(f"Would drop {len(unused_recommendations)} unused indexes") else: results = index_manager.execute_recommendations( unused_recommendations, dry_run ) self.stdout.write( self.style.SUCCESS(f"Dropped {results['executed']} indexes") ) elif index_action == 'rebuild': self.stdout.write("Rebuilding fragmented indexes...") performance_analysis = index_manager.analyze_index_performance() rebuild_recommendations = [ r for r in performance_analysis['recommendations'] if r.action == 'rebuild' ] if dry_run: self.stdout.write(f"Would rebuild {len(rebuild_recommendations)} indexes") else: results = index_manager.execute_recommendations( rebuild_recommendations, dry_run ) self.stdout.write( self.style.SUCCESS(f"Rebuilt {results['executed']} indexes") ) else: # Show index statistics stats = index_manager.get_index_statistics() self.output_results(stats, output_format) def optimize_queries(self, config: DatabaseConfig, tenant_schema: Optional[str], options: Dict[str, Any], output_format: str): """Optimize database queries.""" self.stdout.write("Optimizing database queries...") optimizer = DatabaseOptimizer(tenant_schema) # Get optimization report report = optimizer.get_optimization_report() # Optimize Malaysian queries malaysian_opts = optimizer.optimize_malaysian_queries() # Add to report report['malaysian_optimizations'] = malaysian_opts self.output_results(report, output_format) def manage_cache(self, config: DatabaseConfig, tenant_schema: Optional[str], options: Dict[str, Any], output_format: str): """Manage database cache.""" cache_action = options.get('cache_action') cache_manager = CacheManager() if cache_action == 'clear': self.stdout.write("Clearing cache...") if options.get('dry_run'): self.stdout.write("Would clear all cache") else: cache.clear() self.stdout.write( self.style.SUCCESS("Cache cleared successfully") ) elif cache_action == 'stats': self.stdout.write("Getting cache statistics...") try: # Get Redis stats if using Redis if 'redis' in str(config.cache.backend): import redis r = redis.from_url(config.cache.location) stats = r.info() self.output_results(stats, output_format) else: self.stdout.write("Cache statistics not available for current backend") except Exception as e: self.stdout.write( self.style.ERROR(f"Error getting cache stats: {e}") ) elif cache_action == 'warmup': self.stdout.write("Warming up cache...") # Implement cache warmup logic here self.stdout.write("Cache warmup completed") else: # Show cache configuration cache_config = { 'backend': config.cache.backend.value, 'location': config.cache.location, 'timeout': config.cache.timeout, 'key_prefix': config.cache.key_prefix, 'enabled': config.performance.enable_caching } self.output_results(cache_config, output_format) def perform_maintenance(self, config: DatabaseConfig, tenant_schema: Optional[str], options: Dict[str, Any], output_format: str): """Perform database maintenance.""" self.stdout.write("Performing database maintenance...") maintenance = DatabaseMaintenance() # Run maintenance tasks with connection.cursor() as cursor: # Analyze tables cursor.execute("ANALYZE VERBOSE") self.stdout.write("Analyzed database tables") # Update statistics cursor.execute("VACUUM ANALYZE") self.stdout.write("Vacuumed and analyzed database") # Get maintenance results results = { 'tables_analyzed': len(DatabaseMaintenance.get_table_sizes()), 'maintenance_completed': timezone.now(), 'next_recommended': timezone.now() + timezone.timedelta(days=7) } self.output_results(results, output_format) def show_configuration(self, config: DatabaseConfig, output_format: str): """Show current database configuration.""" self.stdout.write("Database Configuration:") # Get all configuration settings db_config = config.get_database_optimization_settings() # Add Django settings db_config['django_database'] = config.get_django_database_config() db_config['django_cache'] = config.get_django_cache_config() # Add validation warnings warnings = config.validate_configuration() if warnings: db_config['warnings'] = warnings # Add recommendations recommendations = config.get_performance_recommendations() if recommendations: db_config['recommendations'] = recommendations self.output_results(db_config, output_format) def optimize_malaysian(self, config: DatabaseConfig, tenant_schema: Optional[str], options: Dict[str, Any], output_format: str): """Perform Malaysian-specific optimizations.""" self.stdout.write("Performing Malaysian-specific optimizations...") optimizer = DatabaseOptimizer(tenant_schema) index_manager = IndexManager(tenant_schema) # Create Malaysian indexes created_indexes = index_manager.create_malaysian_indexes() # Optimize Malaysian queries malaysian_opts = optimizer.optimize_malaysian_queries() # Get Malaysian-specific configuration malaysian_config = { 'indexes_created': len(created_indexes), 'index_names': created_indexes, 'sst_queries_optimized': malaysian_opts['sst_queries_optimized'], 'ic_validation_optimized': malaysian_opts['ic_validation_optimized'], 'address_queries_optimized': malaysian_opts['address_queries_optimized'], 'localization_improvements': malaysian_opts['localization_improvements'], 'malaysian_config': { 'timezone': config.malaysian.timezone, 'locale': config.malaysian.locale, 'currency': config.malaysian.currency, 'local_caching_enabled': config.malaysian.enable_local_caching } } self.output_results(malaysian_config, output_format) def generate_report(self, config: DatabaseConfig, tenant_schema: Optional[str], options: Dict[str, Any], output_format: str): """Generate comprehensive optimization report.""" self.stdout.write("Generating comprehensive optimization report...") optimizer = DatabaseOptimizer(tenant_schema) index_manager = IndexManager(tenant_schema) # Collect all data for report report_data = { 'report_generated': timezone.now(), 'environment': config.environment, 'tenant_schema': tenant_schema, 'configuration': config.get_database_optimization_settings(), 'performance_analysis': optimizer.analyze_query_performance(), 'index_analysis': index_manager.analyze_index_performance(), 'index_statistics': index_manager.get_index_statistics(), 'optimization_report': optimizer.get_optimization_report(), 'table_statistics': DatabaseMaintenance.get_table_sizes(), 'malaysian_optimizations': optimizer.optimize_malaysian_queries(), 'configuration_validation': config.validate_configuration(), 'recommendations': config.get_performance_recommendations() } self.output_results(report_data, output_format) def output_results(self, results: Dict[str, Any], output_format: str): """Output results in specified format.""" if output_format == 'json': self.output_json(results) elif output_format == 'table': self.output_table(results) elif output_format == 'summary': self.output_summary(results) else: self.output_table(results) def output_json(self, results: Dict[str, Any]): """Output results as JSON.""" # Convert datetime objects to strings def json_serializer(obj): if hasattr(obj, 'isoformat'): return obj.isoformat() elif hasattr(obj, 'value'): return obj.value elif hasattr(obj, '__dict__'): return obj.__dict__ return str(obj) json_output = json.dumps(results, indent=2, default=json_serializer) self.stdout.write(json_output) def output_table(self, results: Dict[str, Any]): """Output results as formatted tables.""" for key, value in results.items(): self.stdout.write(f"\n{self.style.SUCCESS(key.upper()}:}") if isinstance(value, dict): for sub_key, sub_value in value.items(): self.stdout.write(f" {sub_key}: {sub_value}") elif isinstance(value, list): for i, item in enumerate(value): self.stdout.write(f" {i+1}. {item}") else: self.stdout.write(f" {value}") def output_summary(self, results: Dict[str, Any]): """Output results as summary.""" self.stdout.write(self.style.SUCCESS("OPTIMIZATION SUMMARY:")) # Extract key metrics total_queries = results.get('performance_analysis', {}).get('total_queries', 0) slow_queries = results.get('performance_analysis', {}).get('slow_queries', 0) total_indexes = results.get('index_analysis', {}).get('total_indexes', 0) unused_indexes = results.get('index_analysis', {}).get('unused_indexes', 0) recommendations = results.get('index_analysis', {}).get('recommendations', []) self.stdout.write(f"• Total queries analyzed: {total_queries}") self.stdout.write(f"• Slow queries found: {slow_queries}") self.stdout.write(f"• Total indexes: {total_indexes}") self.stdout.write(f"• Unused indexes: {unused_indexes}") self.stdout.write(f"• Recommendations: {len(recommendations)}") if recommendations: self.stdout.write("\nTOP RECOMMENDATIONS:") for i, rec in enumerate(recommendations[:5]): priority = rec.get('priority', 'medium') action = rec.get('action', 'unknown') reason = rec.get('reason', 'No reason provided') self.stdout.write(f" {i+1}. [{priority.upper()}] {action}: {reason}") # Malaysian-specific summary malaysian_opts = results.get('malaysian_optimizations', {}) if malaysian_opts: self.stdout.write(f"\nMALAYSIAN OPTIMIZATIONS:") self.stdout.write(f"• SST queries optimized: {malaysian_opts.get('sst_queries_optimized', 0)}") self.stdout.write(f"• IC validation optimized: {malaysian_opts.get('ic_validation_optimized', False)}") self.stdout.write(f"• Address queries optimized: {malaysian_opts.get('address_queries_optimized', 0)}") def create_progress_bar(self, total: int, description: str): """Create a simple progress bar.""" return ProgressBar(total, description) class ProgressBar: """Simple progress bar for command line output.""" def __init__(self, total: int, description: str): self.total = total self.current = 0 self.description = description def update(self, increment: int = 1): """Update progress.""" self.current += increment self._draw() def _draw(self): """Draw progress bar.""" if self.total == 0: return progress = self.current / self.total bar_length = 50 filled = int(bar_length * progress) bar = '█' * filled + '-' * (bar_length - filled) percent = progress * 100 self.stdout.write(f"\r{self.description}: |{bar}| {percent:.1f}% ({self.current}/{self.total})") self.stdout.flush() def finish(self): """Finish progress bar.""" self._draw() self.stdout.write("\n") self.stdout.flush()