project initialization

backend/core/caching/cache_manager.py

@@ -0,0 +1,429 @@
+"""
+Multi-tenant caching strategies for Malaysian SME SaaS platform.
+Provides advanced caching with Malaysian-specific optimizations.
+"""
+
+import json
+import logging
+import hashlib
+import pickle
+from datetime import datetime, timedelta
+from typing import Any, Dict, List, Optional, Union, Tuple
+from django.core.cache import cache
+from django.conf import settings
+from django.db import connection, models
+from django_redis import get_redis_connection
+from django.contrib.auth import get_user_model
+from django.core.serializers.json import DjangoJSONEncoder
+from django.utils import timezone
+from django_tenants.utils import get_tenant_model, get_public_schema_name, get_tenant_schema_name
+from rest_framework.response import Response
+
+from .config import DatabaseConfig, CacheConfig
+
+logger = logging.getLogger(__name__)
+User = get_user_model()
+TenantModel = get_tenant_model()
+
+
+class CacheKeyGenerator:
+    """Generates cache keys with multi-tenant support and Malaysian context."""
+
+    def __init__(self):
+        self.tenant_prefix = self._get_tenant_prefix()
+        self.malaysia_prefix = "my_sme"
+
+    def _get_tenant_prefix(self) -> str:
+        """Get current tenant prefix for cache keys."""
+        try:
+            tenant = TenantModel.objects.get(schema_name=connection.schema_name)
+            return f"tenant_{tenant.id}"
+        except Exception:
+            return "public"
+
+    def generate_key(
+        self,
+        key_type: str,
+        identifier: str,
+        subkey: Optional[str] = None,
+        context: Optional[Dict[str, Any]] = None
+    ) -> str:
+        """Generate standardized cache key."""
+        components = [
+            self.malaysia_prefix,
+            self.tenant_prefix,
+            key_type,
+            identifier
+        ]
+
+        if subkey:
+            components.append(subkey)
+
+        if context:
+            context_hash = hashlib.md5(
+                json.dumps(context, sort_keys=True).encode()
+            ).hexdigest()[:8]
+            components.append(context_hash)
+
+        return ":".join(components)
+
+    def generate_malaysian_key(
+        self,
+        entity_type: str,
+        identifier: Union[str, int],
+        malaysian_context: Optional[Dict[str, Any]] = None
+    ) -> str:
+        """Generate Malaysian-specific cache key."""
+        return self.generate_key(
+            "my",
+            f"{entity_type}_{identifier}",
+            context=malaysian_context
+        )
+
+
+class CacheManager:
+    """Advanced cache management with multi-tenant support."""
+
+    def __init__(self, config: Optional[CacheConfig] = None):
+        self.config = config or CacheConfig()
+        self.key_generator = CacheKeyGenerator()
+        self.redis_client = None
+
+        if self.config.use_redis:
+            try:
+                self.redis_client = get_redis_connection("default")
+            except Exception as e:
+                logger.warning(f"Redis connection failed: {e}")
+
+    def get(
+        self,
+        key: str,
+        default: Any = None,
+        version: Optional[int] = None
+    ) -> Any:
+        """Get value from cache with error handling."""
+        try:
+            return cache.get(key, default=default, version=version)
+        except Exception as e:
+            logger.error(f"Cache get error for key {key}: {e}")
+            return default
+
+    def set(
+        self,
+        key: str,
+        value: Any,
+        timeout: Optional[int] = None,
+        version: Optional[int] = None
+    ) -> bool:
+        """Set value in cache with error handling."""
+        try:
+            timeout = timeout or self.config.default_timeout
+            return cache.set(key, value, timeout=timeout, version=version)
+        except Exception as e:
+            logger.error(f"Cache set error for key {key}: {e}")
+            return False
+
+    def delete(self, key: str, version: Optional[int] = None) -> bool:
+        """Delete key from cache."""
+        try:
+            return cache.delete(key, version=version)
+        except Exception as e:
+            logger.error(f"Cache delete error for key {key}: {e}")
+            return False
+
+    def clear_tenant_cache(self, tenant_id: Optional[int] = None) -> bool:
+        """Clear all cache for a specific tenant."""
+        try:
+            if tenant_id:
+                pattern = f"*:tenant_{tenant_id}:*"
+            else:
+                pattern = f"*:{self.key_generator.tenant_prefix}:*"
+
+            if self.redis_client:
+                keys = self.redis_client.keys(pattern)
+                if keys:
+                    self.redis_client.delete(*keys)
+
+            return True
+        except Exception as e:
+            logger.error(f"Error clearing tenant cache: {e}")
+            return False
+
+    def get_cache_stats(self) -> Dict[str, Any]:
+        """Get cache statistics."""
+        stats = {
+            "tenant": self.key_generator.tenant_prefix,
+            "redis_available": self.redis_client is not None,
+            "default_timeout": self.config.default_timeout,
+        }
+
+        if self.redis_client:
+            try:
+                info = self.redis_client.info()
+                stats.update({
+                    "used_memory": info.get("used_memory_human", "N/A"),
+                    "connected_clients": info.get("connected_clients", 0),
+                    "total_commands_processed": info.get("total_commands_processed", 0),
+                })
+            except Exception as e:
+                logger.error(f"Error getting Redis stats: {e}")
+
+        return stats
+
+
+class MalaysianDataCache:
+    """Specialized caching for Malaysian data and validations."""
+
+    def __init__(self, cache_manager: CacheManager):
+        self.cache = cache_manager
+
+    def get_cached_ic_validation(self, ic_number: str) -> Optional[Dict[str, Any]]:
+        """Get cached IC validation result."""
+        key = self.cache.key_generator.generate_malaysian_key(
+            "ic_validation",
+            ic_number
+        )
+        return self.cache.get(key)
+
+    def set_cached_ic_validation(
+        self,
+        ic_number: str,
+        validation_result: Dict[str, Any]
+    ) -> bool:
+        """Cache IC validation result."""
+        key = self.cache.key_generator.generate_malaysian_key(
+            "ic_validation",
+            ic_number
+        )
+        return self.cache.set(key, validation_result, timeout=86400)  # 24 hours
+
+    def get_cached_sst_rate(self, state: str, category: str) -> Optional[float]:
+        """Get cached SST rate."""
+        key = self.cache.key_generator.generate_malaysian_key(
+            "sst_rate",
+            f"{state}_{category}"
+        )
+        return self.cache.get(key)
+
+    def set_cached_sst_rate(
+        self,
+        state: str,
+        category: str,
+        rate: float
+    ) -> bool:
+        """Cache SST rate."""
+        key = self.cache.key_generator.generate_malaysian_key(
+            "sst_rate",
+            f"{state}_{category}"
+        )
+        return self.cache.set(key, rate, timeout=604800)  # 7 days
+
+    def get_cached_postcode_data(self, postcode: str) -> Optional[Dict[str, Any]]:
+        """Get cached postcode data."""
+        key = self.cache.key_generator.generate_malaysian_key(
+            "postcode",
+            postcode
+        )
+        return self.cache.get(key)
+
+    def set_cached_postcode_data(
+        self,
+        postcode: str,
+        postcode_data: Dict[str, Any]
+    ) -> bool:
+        """Cache postcode data."""
+        key = self.cache.key_generator.generate_malaysian_key(
+            "postcode",
+            postcode
+        )
+        return self.cache.set(key, postcode_data, timeout=2592000)  # 30 days
+
+
+class QueryCache:
+    """Intelligent query caching with automatic invalidation."""
+
+    def __init__(self, cache_manager: CacheManager):
+        self.cache = cache_manager
+        self.query_hashes = set()
+
+    def generate_query_hash(self, query: str, params: Optional[tuple] = None) -> str:
+        """Generate hash for query identification."""
+        query_string = query.strip().lower()
+        if params:
+            query_string += str(params)
+        return hashlib.md5(query_string.encode()).hexdigest()
+
+    def cache_query_result(
+        self,
+        query: str,
+        result: Any,
+        params: Optional[tuple] = None,
+        timeout: Optional[int] = None
+    ) -> bool:
+        """Cache query result."""
+        query_hash = self.generate_query_hash(query, params)
+        key = self.cache.key_generator.generate_key("query", query_hash)
+
+        success = self.cache.set(key, result, timeout=timeout)
+        if success:
+            self.query_hashes.add(query_hash)
+
+        return success
+
+    def get_cached_query_result(
+        self,
+        query: str,
+        params: Optional[tuple] = None
+    ) -> Optional[Any]:
+        """Get cached query result."""
+        query_hash = self.generate_query_hash(query, params)
+        key = self.cache.key_generator.generate_key("query", query_hash)
+        return self.cache.get(key)
+
+    def invalidate_model_cache(self, model_name: str) -> int:
+        """Invalidate cache for a specific model."""
+        invalidated = 0
+        for query_hash in list(self.query_hashes):
+            if model_name.lower() in query_hash:
+                key = self.cache.key_generator.generate_key("query", query_hash)
+                if self.cache.delete(key):
+                    invalidated += 1
+                    self.query_hashes.discard(query_hash)
+
+        return invalidated
+
+
+class TenantCacheManager:
+    """Multi-tenant cache management with isolation."""
+
+    def __init__(self):
+        self.cache_managers = {}
+
+    def get_cache_manager(self, tenant_id: Optional[int] = None) -> CacheManager:
+        """Get cache manager for specific tenant."""
+        if not tenant_id:
+            tenant_id = self._get_current_tenant_id()
+
+        if tenant_id not in self.cache_managers:
+            config = CacheConfig()
+            config.tenant_isolation = True
+            config.tenant_prefix = f"tenant_{tenant_id}"
+            self.cache_managers[tenant_id] = CacheManager(config)
+
+        return self.cache_managers[tenant_id]
+
+    def _get_current_tenant_id(self) -> int:
+        """Get current tenant ID."""
+        try:
+            tenant = TenantModel.objects.get(schema_name=connection.schema_name)
+            return tenant.id
+        except Exception:
+            return 0  # Public schema
+
+    def clear_all_tenant_cache(self) -> Dict[str, Any]:
+        """Clear cache for all tenants."""
+        results = {"cleared_tenants": 0, "errors": []}
+
+        for tenant_id, cache_manager in self.cache_managers.items():
+            try:
+                if cache_manager.clear_tenant_cache(tenant_id):
+                    results["cleared_tenants"] += 1
+            except Exception as e:
+                results["errors"].append(f"Tenant {tenant_id}: {e}")
+
+        return results
+
+    def get_tenant_cache_stats(self) -> Dict[str, Any]:
+        """Get cache statistics for all tenants."""
+        stats = {"tenants": {}, "total_tenants": len(self.cache_managers)}
+
+        for tenant_id, cache_manager in self.cache_managers.items():
+            stats["tenants"][str(tenant_id)] = cache_manager.get_cache_stats()
+
+        return stats
+
+
+class CacheWarmer:
+    """Proactive cache warming for critical data."""
+
+    def __init__(self, cache_manager: CacheManager):
+        self.cache = cache_manager
+        self.malaysian_cache = MalaysianDataCache(cache_manager)
+
+    def warm_malaysian_data(self) -> Dict[str, int]:
+        """Warm cache with Malaysian reference data."""
+        warmed = {"ic_validations": 0, "sst_rates": 0, "postcodes": 0}
+
+        # Warm SST rates
+        sst_rates = self._get_sst_rates_to_warm()
+        for state, category, rate in sst_rates:
+            if self.malaysian_cache.set_cached_sst_rate(state, category, rate):
+                warmed["sst_rates"] += 1
+
+        # Warm postcode data
+        postcodes = self._get_postcodes_to_warm()
+        for postcode, data in postcodes:
+            if self.malaysian_cache.set_cached_postcode_data(postcode, data):
+                warmed["postcodes"] += 1
+
+        return warmed
+
+    def warm_user_data(self, user_ids: List[int]) -> int:
+        """Warm cache with user data."""
+        warmed = 0
+
+        for user_id in user_ids:
+            try:
+                user = User.objects.get(id=user_id)
+                key = self.cache.key_generator.generate_key("user", str(user_id))
+                user_data = {
+                    "id": user.id,
+                    "username": user.username,
+                    "email": user.email,
+                    "is_active": user.is_active,
+                    "last_login": user.last_login,
+                }
+                if self.cache.set(key, user_data):
+                    warmed += 1
+            except User.DoesNotExist:
+                continue
+
+        return warmed
+
+    def _get_sst_rates_to_warm(self) -> List[Tuple[str, str, float]]:
+        """Get SST rates to warm in cache."""
+        # Common Malaysian states and categories
+        states = ["Johor", "Kedah", "Kelantan", "Melaka", "Negeri Sembilan",
+                 "Pahang", "Perak", "Perlis", "Pulau Pinang", "Sabah",
+                 "Sarawak", "Selangor", "Terengganu", "WP Kuala Lumpur",
+                 "WP Labuan", "WP Putrajaya"]
+        categories = ["standard", "food", "medical", "education"]
+
+        rates = []
+        for state in states:
+            for category in categories:
+                rate = 0.06 if category == "standard" else 0.0
+                rates.append((state, category, rate))
+
+        return rates
+
+    def _get_postcodes_to_warm(self) -> List[Tuple[str, Dict[str, Any]]]:
+        """Get postcode data to warm in cache."""
+        # Common Malaysian postcodes
+        postcodes = [
+            ("50000", {"city": "Kuala Lumpur", "state": "WP Kuala Lumpur"}),
+            ("50480", {"city": "Kuala Lumpur", "state": "WP Kuala Lumpur"}),
+            ("80000", {"city": "Johor Bahru", "state": "Johor"}),
+            ("93000", {"city": "Kuching", "state": "Sarawak"}),
+            ("88300", {"city": "Kota Kinabalu", "state": "Sabah"}),
+        ]
+
+        return postcodes
+
+
+# Global instances
+tenant_cache_manager = TenantCacheManager()
+cache_manager = CacheManager()
+malaysian_cache = MalaysianDataCache(cache_manager)
+query_cache = QueryCache(cache_manager)
+cache_warmer = CacheWarmer(cache_manager)