project initialization

backend/monitoring/alerts.py

@@ -0,0 +1,584 @@
+"""
+Alert management system for the Malaysian SME SaaS platform.
+Provides comprehensive alerting with Malaysian context.
+"""
+
+import json
+import logging
+import smtplib
+import requests
+from datetime import datetime, timedelta
+from typing import Dict, List, Any, Optional, Callable
+from email.mime.text import MimeText
+from email.mime.multipart import MimeMultipart
+from django.conf import settings
+from django.core.mail import send_mail
+from django.utils import timezone
+from django.db import connection
+from prometheus_client import Counter, Gauge
+import redis
+
+logger = logging.getLogger(__name__)
+
+# Alert metrics
+ALERTS_TOTAL = Counter('alerts_total', 'Total alerts generated', ['severity', 'category'])
+ALERTS_RESOLVED = Counter('alerts_resolved_total', 'Total alerts resolved', ['severity', 'category'])
+ALERTS_ACTIVE = Gauge('alerts_active', 'Currently active alerts', ['severity', 'category'])
+
+class AlertSeverity:
+    """Alert severity levels."""
+    INFO = 'info'
+    WARNING = 'warning'
+    ERROR = 'error'
+    CRITICAL = 'critical'
+
+class AlertCategory:
+    """Alert categories."""
+    SYSTEM = 'system'
+    APPLICATION = 'application'
+    DATABASE = 'database'
+    CACHE = 'cache'
+    SECURITY = 'security'
+    BUSINESS = 'business'
+    MALAYSIAN = 'malaysian'
+
+class Alert:
+    """Single alert instance."""
+
+    def __init__(
+        self,
+        title: str,
+        description: str,
+        severity: str,
+        category: str,
+        metadata: Optional[Dict[str, Any]] = None
+    ):
+        self.id = f"{int(timezone.now().timestamp())}-{hash(title)}"
+        self.title = title
+        self.description = description
+        self.severity = severity
+        self.category = category
+        self.metadata = metadata or {}
+        self.created_at = timezone.now()
+        self.resolved_at = None
+        self.acknowledged_at = None
+        self.acknowledged_by = None
+        self.tags = self.metadata.get('tags', [])
+        self.source = self.metadata.get('source', 'system')
+        self.tenant = self.metadata.get('tenant', 'all')
+
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert alert to dictionary."""
+        return {
+            'id': self.id,
+            'title': self.title,
+            'description': self.description,
+            'severity': self.severity,
+            'category': self.category,
+            'metadata': self.metadata,
+            'created_at': self.created_at.isoformat(),
+            'resolved_at': self.resolved_at.isoformat() if self.resolved_at else None,
+            'acknowledged_at': self.acknowledged_at.isoformat() if self.acknowledged_at else None,
+            'acknowledged_by': self.acknowledged_by,
+            'tags': self.tags,
+            'source': self.source,
+            'tenant': self.tenant,
+            'status': self.get_status(),
+        }
+
+    def get_status(self) -> str:
+        """Get alert status."""
+        if self.resolved_at:
+            return 'resolved'
+        elif self.acknowledged_at:
+            return 'acknowledged'
+        else:
+            return 'active'
+
+    def acknowledge(self, user: str):
+        """Acknowledge alert."""
+        self.acknowledged_at = timezone.now()
+        self.acknowledged_by = user
+        logger.info(f"Alert {self.id} acknowledged by {user}")
+
+    def resolve(self, user: Optional[str] = None):
+        """Resolve alert."""
+        self.resolved_at = timezone.now()
+        if user:
+            self.acknowledged_by = user
+        logger.info(f"Alert {self.id} resolved by {user or 'system'}")
+
+        # Update metrics
+        ALERTS_RESOLVED.labels(
+            severity=self.severity,
+            category=self.category
+        ).inc()
+
+class AlertRule:
+    """Alert rule definition."""
+
+    def __init__(
+        self,
+        name: str,
+        condition: Callable[[], bool],
+        title_template: str,
+        description_template: str,
+        severity: str,
+        category: str,
+        cooldown_minutes: int = 15,
+        enabled: bool = True
+    ):
+        self.name = name
+        self.condition = condition
+        self.title_template = title_template
+        self.description_template = description_template
+        self.severity = severity
+        self.category = category
+        self.cooldown_minutes = cooldown_minutes
+        self.enabled = enabled
+        self.last_triggered = None
+        self.metadata = {}
+
+    def should_trigger(self) -> bool:
+        """Check if rule should trigger alert."""
+        if not self.enabled:
+            return False
+
+        # Check cooldown
+        if self.last_triggered:
+            cooldown_until = self.last_triggered + timedelta(minutes=self.cooldown_minutes)
+            if timezone.now() < cooldown_until:
+                return False
+
+        # Check condition
+        try:
+            return self.condition()
+        except Exception as e:
+            logger.error(f"Error checking alert rule {self.name}: {e}")
+            return False
+
+    def trigger(self, metadata: Optional[Dict[str, Any]] = None) -> Alert:
+        """Trigger alert from rule."""
+        self.last_triggered = timezone.now()
+        self.metadata = metadata or {}
+
+        # Format title and description
+        title = self.title_template.format(**self.metadata)
+        description = self.description_template.format(**self.metadata)
+
+        return Alert(
+            title=title,
+            description=description,
+            severity=self.severity,
+            category=self.category,
+            metadata={**self.metadata, 'rule_name': self.name}
+        )
+
+class AlertManager:
+    """Main alert management system."""
+
+    def __init__(self):
+        self.rules: List[AlertRule] = []
+        self.active_alerts: Dict[str, Alert] = {}
+        self.alert_history: List[Alert] = []
+        self.notifiers = []
+        self.redis_client = None
+        self.initialize_redis()
+        self.setup_default_rules()
+        self.setup_notifiers()
+
+    def initialize_redis(self):
+        """Initialize Redis connection for alert persistence."""
+        try:
+            self.redis_client = redis.from_url(settings.REDIS_URL)
+        except Exception as e:
+            logger.warning(f"Failed to initialize Redis for alerts: {e}")
+
+    def setup_default_rules(self):
+        """Setup default alert rules."""
+        # System alerts
+        self.add_rule(AlertRule(
+            name='high_cpu_usage',
+            condition=self._check_high_cpu_usage,
+            title_template='High CPU Usage Detected',
+            description_template='CPU usage is {cpu_usage}% on server {server}',
+            severity=AlertSeverity.WARNING,
+            category=AlertCategory.SYSTEM,
+            cooldown_minutes=10
+        ))
+
+        self.add_rule(AlertRule(
+            name='critical_cpu_usage',
+            condition=self._check_critical_cpu_usage,
+            title_template='Critical CPU Usage',
+            description_template='CPU usage is {cpu_usage}% on server {server} - immediate attention required',
+            severity=AlertSeverity.CRITICAL,
+            category=AlertCategory.SYSTEM,
+            cooldown_minutes=5
+        ))
+
+        # Database alerts
+        self.add_rule(AlertRule(
+            name='database_connection_errors',
+            condition=self._check_database_connection_errors,
+            title_template='Database Connection Errors',
+            description_template='Database connection errors detected: {error_count} errors in the last 5 minutes',
+            severity=AlertSeverity.ERROR,
+            category=AlertCategory.DATABASE,
+            cooldown_minutes=5
+        ))
+
+        # Application alerts
+        self.add_rule(AlertRule(
+            name='high_error_rate',
+            condition=self._check_high_error_rate,
+            title_template='High Application Error Rate',
+            description_template='Application error rate is {error_rate}% (threshold: 5%)',
+            severity=AlertSeverity.WARNING,
+            category=AlertCategory.APPLICATION,
+            cooldown_minutes=15
+        ))
+
+        # Business alerts
+        self.add_rule(AlertRule(
+            name='low_active_users',
+            condition=self._check_low_active_users,
+            title_template='Low Active Users',
+            description_template='Only {active_users} active users detected (threshold: {threshold})',
+            severity=AlertSeverity.INFO,
+            category=AlertCategory.BUSINESS,
+            cooldown_minutes=60
+        ))
+
+        # Malaysian-specific alerts
+        self.add_rule(AlertRule(
+            name='malaysian_service_degradation',
+            condition=self._check_malaysian_service_degradation,
+            title_template='Malaysian Service Degradation',
+            description_template='Malaysian service availability is {availability}% (threshold: 99%)',
+            severity=AlertSeverity.WARNING,
+            category=AlertCategory.MALAYSIAN,
+            cooldown_minutes=10
+        ))
+
+        # Security alerts
+        self.add_rule(AlertRule(
+            name='suspicious_login_activity',
+            condition=self._check_suspicious_login_activity,
+            title_template='Suspicious Login Activity',
+            description_template='Detected {failed_logins} failed login attempts from IP {ip_address}',
+            severity=AlertSeverity.WARNING,
+            category=AlertCategory.SECURITY,
+            cooldown_minutes=15
+        ))
+
+    def setup_notifiers(self):
+        """Setup notification channels."""
+        # Email notifier
+        if settings.EMAIL_HOST:
+            self.add_notifier(EmailNotifier())
+
+        # Slack notifier
+        if hasattr(settings, 'SLACK_WEBHOOK_URL'):
+            self.add_notifier(SlackNotifier())
+
+        # SMS notifier for critical alerts (Malaysian numbers)
+        if hasattr(settings, 'SMS_API_KEY'):
+            self.add_notifier(SMSNotifier())
+
+    def add_rule(self, rule: AlertRule):
+        """Add alert rule."""
+        self.rules.append(rule)
+        logger.info(f"Added alert rule: {rule.name}")
+
+    def add_notifier(self, notifier):
+        """Add notification channel."""
+        self.notifiers.append(notifier)
+        logger.info(f"Added notifier: {notifier.__class__.__name__}")
+
+    def check_rules(self):
+        """Check all alert rules and trigger if needed."""
+        for rule in self.rules:
+            try:
+                if rule.should_trigger():
+                    alert = rule.trigger()
+                    self.trigger_alert(alert)
+            except Exception as e:
+                logger.error(f"Error checking rule {rule.name}: {e}")
+
+    def trigger_alert(self, alert: Alert):
+        """Trigger new alert."""
+        # Check if similar active alert exists
+        for existing_alert in self.active_alerts.values():
+            if (existing_alert.title == alert.title and
+                existing_alert.severity == alert.severity and
+                existing_alert.get_status() == 'active'):
+                logger.debug(f"Similar alert already active: {existing_alert.id}")
+                return
+
+        # Add alert
+        self.active_alerts[alert.id] = alert
+        self.alert_history.append(alert)
+
+        # Update metrics
+        ALERTS_TOTAL.labels(
+            severity=alert.severity,
+            category=alert.category
+        ).inc()
+
+        # Keep only recent history
+        if len(self.alert_history) > 1000:
+            self.alert_history = self.alert_history[-1000:]
+
+        # Store in Redis
+        if self.redis_client:
+            try:
+                self.redis_client.setex(
+                    f"alert:{alert.id}",
+                    86400,  # 24 hours
+                    json.dumps(alert.to_dict())
+                )
+            except Exception as e:
+                logger.error(f"Failed to store alert in Redis: {e}")
+
+        # Send notifications
+        self.send_notifications(alert)
+
+        logger.warning(f"Alert triggered: {alert.title} ({alert.severity})")
+
+    def resolve_alert(self, alert_id: str, user: Optional[str] = None):
+        """Resolve alert."""
+        if alert_id in self.active_alerts:
+            alert = self.active_alerts[alert_id]
+            alert.resolve(user)
+            del self.active_alerts[alert_id]
+
+            # Update Redis
+            if self.redis_client:
+                try:
+                    self.redis_client.delete(f"alert:{alert_id}")
+                except Exception as e:
+                    logger.error(f"Failed to delete alert from Redis: {e}")
+
+            logger.info(f"Alert resolved: {alert.title}")
+
+    def acknowledge_alert(self, alert_id: str, user: str):
+        """Acknowledge alert."""
+        if alert_id in self.active_alerts:
+            alert = self.active_alerts[alert_id]
+            alert.acknowledge(user)
+            logger.info(f"Alert acknowledged: {alert.title} by {user}")
+
+    def get_active_alerts(self, severity: Optional[str] = None, category: Optional[str] = None) -> List[Alert]:
+        """Get active alerts with optional filtering."""
+        alerts = list(self.active_alerts.values())
+
+        if severity:
+            alerts = [a for a in alerts if a.severity == severity]
+
+        if category:
+            alerts = [a for a in alerts if a.category == category]
+
+        return alerts
+
+    def get_alert_history(self, hours: int = 24) -> List[Alert]:
+        """Get alert history for specified hours."""
+        since = timezone.now() - timedelta(hours=hours)
+        return [a for a in self.alert_history if a.created_at >= since]
+
+    def send_notifications(self, alert: Alert):
+        """Send alert notifications."""
+        for notifier in self.notifiers:
+            try:
+                if notifier.should_notify(alert):
+                    notifier.send(alert)
+            except Exception as e:
+                logger.error(f"Error sending notification via {notifier.__class__.__name__}: {e}")
+
+    # Alert condition methods
+    def _check_high_cpu_usage(self) -> bool:
+        """Check for high CPU usage."""
+        try:
+            import psutil
+            cpu_usage = psutil.cpu_percent(interval=1)
+            return cpu_usage > 80
+        except Exception:
+            return False
+
+    def _check_critical_cpu_usage(self) -> bool:
+        """Check for critical CPU usage."""
+        try:
+            import psutil
+            cpu_usage = psutil.cpu_percent(interval=1)
+            return cpu_usage > 90
+        except Exception:
+            return False
+
+    def _check_database_connection_errors(self) -> bool:
+        """Check for database connection errors."""
+        try:
+            # This would integrate with your error tracking system
+            # For now, return False as placeholder
+            return False
+        except Exception:
+            return False
+
+    def _check_high_error_rate(self) -> bool:
+        """Check for high application error rate."""
+        try:
+            # This would check application error rates
+            # For now, return False as placeholder
+            return False
+        except Exception:
+            return False
+
+    def _check_low_active_users(self) -> bool:
+        """Check for low active users."""
+        try:
+            from django.contrib.auth import get_user_model
+            User = get_user_model()
+            five_minutes_ago = timezone.now() - timedelta(minutes=5)
+            active_count = User.objects.filter(
+                last_login__gte=five_minutes_ago,
+                is_active=True
+            ).count()
+            return active_count < 10
+        except Exception:
+            return False
+
+    def _check_malaysian_service_degradation(self) -> bool:
+        """Check for Malaysian service degradation."""
+        try:
+            # This would check Malaysian-specific service health
+            # For now, return False as placeholder
+            return False
+        except Exception:
+            return False
+
+    def _check_suspicious_login_activity(self) -> bool:
+        """Check for suspicious login activity."""
+        try:
+            # This would check for suspicious login patterns
+            # For now, return False as placeholder
+            return False
+        except Exception:
+            return False
+
+class EmailNotifier:
+    """Email notification system."""
+
+    def should_notify(self, alert: Alert) -> bool:
+        """Check if should send email notification."""
+        # Send emails for warnings and above
+        return alert.severity in [AlertSeverity.WARNING, AlertSeverity.ERROR, AlertSeverity.CRITICAL]
+
+    def send(self, alert: Alert):
+        """Send email notification."""
+        try:
+            subject = f"[{alert.severity.upper()}] {alert.title}"
+            message = f"""
+Alert Details:
+- Title: {alert.title}
+- Severity: {alert.severity}
+- Category: {alert.category}
+- Description: {alert.description}
+- Time: {alert.created_at}
+- Source: {alert.source}
+- Tenant: {alert.tenant}
+
+Additional Information:
+{json.dumps(alert.metadata, indent=2)}
+            """
+
+            send_mail(
+                subject,
+                message,
+                settings.DEFAULT_FROM_EMAIL,
+                settings.ALERT_EMAIL_RECIPIENTS,
+                fail_silently=False
+            )
+
+            logger.info(f"Email notification sent for alert: {alert.id}")
+        except Exception as e:
+            logger.error(f"Failed to send email notification: {e}")
+
+class SlackNotifier:
+    """Slack notification system."""
+
+    def should_notify(self, alert: Alert) -> bool:
+        """Check if should send Slack notification."""
+        # Send Slack for all alerts
+        return True
+
+    def send(self, alert: Alert):
+        """Send Slack notification."""
+        try:
+            webhook_url = settings.SLACK_WEBHOOK_URL
+
+            # Color based on severity
+            colors = {
+                AlertSeverity.INFO: '#36a64f',
+                AlertSeverity.WARNING: '#ff9500',
+                AlertSeverity.ERROR: '#ff0000',
+                AlertSeverity.CRITICAL: '#990000'
+            }
+
+            payload = {
+                'text': f'{alert.severity.upper()}: {alert.title}',
+                'attachments': [{
+                    'color': colors.get(alert.severity, '#36a64f'),
+                    'title': alert.title,
+                    'text': alert.description,
+                    'fields': [
+                        {'title': 'Severity', 'value': alert.severity, 'short': True},
+                        {'title': 'Category', 'value': alert.category, 'short': True},
+                        {'title': 'Time', 'value': alert.created_at.strftime('%Y-%m-%d %H:%M:%S'), 'short': True},
+                        {'title': 'Tenant', 'value': alert.tenant, 'short': True},
+                    ],
+                    'footer': 'Malaysian SME Platform Alert System',
+                    'ts': int(alert.created_at.timestamp())
+                }]
+            }
+
+            response = requests.post(webhook_url, json=payload, timeout=10)
+            response.raise_for_status()
+
+            logger.info(f"Slack notification sent for alert: {alert.id}")
+        except Exception as e:
+            logger.error(f"Failed to send Slack notification: {e}")
+
+class SMSNotifier:
+    """SMS notification system for critical alerts."""
+
+    def should_notify(self, alert: Alert) -> bool:
+        """Check if should send SMS notification."""
+        # Only send SMS for critical alerts
+        return alert.severity == AlertSeverity.CRITICAL
+
+    def send(self, alert: Alert):
+        """Send SMS notification."""
+        try:
+            # This would integrate with Malaysian SMS service
+            # For now, just log the attempt
+            logger.info(f"SMS notification would be sent for critical alert: {alert.id}")
+
+            # Example integration with Malaysian SMS service
+            # sms_api_url = settings.SMS_API_URL
+            # api_key = settings.SMS_API_KEY
+            # recipients = settings.CRITICAL_ALERT_SMS_RECIPIENTS
+
+            # message = f"CRITICAL: {alert.title}. {alert.description[:100]}"
+            # payload = {
+            #     'api_key': api_key,
+            #     'recipients': recipients,
+            #     'message': message
+            # }
+
+            # response = requests.post(sms_api_url, json=payload, timeout=10)
+            # response.raise_for_status()
+
+        except Exception as e:
+            logger.error(f"Failed to send SMS notification: {e}")
+
+# Global alert manager instance
+alert_manager = AlertManager()