project initialization
Some checks failed
System Monitoring / Health Checks (push) Has been cancelled
System Monitoring / Performance Monitoring (push) Has been cancelled
System Monitoring / Database Monitoring (push) Has been cancelled
System Monitoring / Cache Monitoring (push) Has been cancelled
System Monitoring / Log Monitoring (push) Has been cancelled
System Monitoring / Resource Monitoring (push) Has been cancelled
System Monitoring / Uptime Monitoring (push) Has been cancelled
System Monitoring / Backup Monitoring (push) Has been cancelled
System Monitoring / Security Monitoring (push) Has been cancelled
System Monitoring / Monitoring Dashboard (push) Has been cancelled
System Monitoring / Alerting (push) Has been cancelled
Security Scanning / Dependency Scanning (push) Has been cancelled
Security Scanning / Code Security Scanning (push) Has been cancelled
Security Scanning / Secrets Scanning (push) Has been cancelled
Security Scanning / Container Security Scanning (push) Has been cancelled
Security Scanning / Compliance Checking (push) Has been cancelled
Security Scanning / Security Dashboard (push) Has been cancelled
Security Scanning / Security Remediation (push) Has been cancelled
Some checks failed
System Monitoring / Health Checks (push) Has been cancelled
System Monitoring / Performance Monitoring (push) Has been cancelled
System Monitoring / Database Monitoring (push) Has been cancelled
System Monitoring / Cache Monitoring (push) Has been cancelled
System Monitoring / Log Monitoring (push) Has been cancelled
System Monitoring / Resource Monitoring (push) Has been cancelled
System Monitoring / Uptime Monitoring (push) Has been cancelled
System Monitoring / Backup Monitoring (push) Has been cancelled
System Monitoring / Security Monitoring (push) Has been cancelled
System Monitoring / Monitoring Dashboard (push) Has been cancelled
System Monitoring / Alerting (push) Has been cancelled
Security Scanning / Dependency Scanning (push) Has been cancelled
Security Scanning / Code Security Scanning (push) Has been cancelled
Security Scanning / Secrets Scanning (push) Has been cancelled
Security Scanning / Container Security Scanning (push) Has been cancelled
Security Scanning / Compliance Checking (push) Has been cancelled
Security Scanning / Security Dashboard (push) Has been cancelled
Security Scanning / Security Remediation (push) Has been cancelled
This commit is contained in:
393
.github/workflows/ci-cd.yml
vendored
Normal file
393
.github/workflows/ci-cd.yml
vendored
Normal file
@@ -0,0 +1,393 @@
|
||||
name: CI/CD Pipeline
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ main, develop ]
|
||||
pull_request:
|
||||
branches: [ main, develop ]
|
||||
workflow_dispatch:
|
||||
|
||||
env:
|
||||
REGISTRY: ghcr.io
|
||||
IMAGE_NAME: ${{ github.repository }}
|
||||
|
||||
jobs:
|
||||
test:
|
||||
name: Run Tests
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:15
|
||||
env:
|
||||
POSTGRES_PASSWORD: postgres
|
||||
POSTGRES_DB: test_db
|
||||
options: >-
|
||||
--health-cmd pg_isready
|
||||
--health-interval 10s
|
||||
--health-timeout 5s
|
||||
--health-retries 5
|
||||
ports:
|
||||
- 5432:5432
|
||||
redis:
|
||||
image: redis:7-alpine
|
||||
options: >-
|
||||
--health-cmd "redis-cli ping"
|
||||
--health-interval 10s
|
||||
--health-timeout 5s
|
||||
--health-retries 5
|
||||
ports:
|
||||
- 6379:6379
|
||||
|
||||
strategy:
|
||||
matrix:
|
||||
python-version: [3.9, 3.10, 3.11]
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Python ${{ matrix.python-version }}
|
||||
uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
|
||||
- name: Cache pip packages
|
||||
uses: actions/cache@v3
|
||||
with:
|
||||
path: ~/.cache/pip
|
||||
key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-pip-
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install -r requirements.txt
|
||||
pip install -r requirements-dev.txt
|
||||
pip install coveralls
|
||||
|
||||
- name: Set up environment
|
||||
run: |
|
||||
cp backend/.env.example backend/.env
|
||||
cp frontend/.env.example frontend/.env
|
||||
chmod +x backend/manage.py
|
||||
|
||||
- name: Run database migrations
|
||||
run: |
|
||||
cd backend
|
||||
python manage.py migrate
|
||||
env:
|
||||
DATABASE_URL: postgres://postgres:postgres@localhost:5432/test_db
|
||||
REDIS_URL: redis://localhost:6379/0
|
||||
|
||||
- name: Run backend tests
|
||||
run: |
|
||||
cd backend
|
||||
python manage.py test --verbosity=2 --cov=. --cov-report=xml --cov-report=term-missing
|
||||
env:
|
||||
DATABASE_URL: postgres://postgres:postgres@localhost:5432/test_db
|
||||
REDIS_URL: redis://localhost:6379/0
|
||||
SECRET_KEY: test-secret-key-for-ci
|
||||
|
||||
- name: Run frontend tests
|
||||
run: |
|
||||
cd frontend
|
||||
npm install
|
||||
npm run test
|
||||
npm run build
|
||||
|
||||
- name: Run integration tests
|
||||
run: |
|
||||
cd backend
|
||||
python manage.py test tests.integration --verbosity=2
|
||||
env:
|
||||
DATABASE_URL: postgres://postgres:postgres@localhost:5432/test_db
|
||||
REDIS_URL: redis://localhost:6379/0
|
||||
|
||||
- name: Upload coverage to Codecov
|
||||
uses: codecov/codecov-action@v3
|
||||
with:
|
||||
file: ./coverage.xml
|
||||
flags: unittests
|
||||
name: codecov-umbrella
|
||||
|
||||
security:
|
||||
name: Security Scan
|
||||
runs-on: ubuntu-latest
|
||||
needs: test
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Run Bandit Security Scan
|
||||
uses: PyCQA/bandit-action@v1
|
||||
with:
|
||||
path: backend
|
||||
config: .bandit
|
||||
|
||||
- name: Run Safety Check
|
||||
run: |
|
||||
pip install safety
|
||||
safety check -r requirements.txt
|
||||
|
||||
- name: Run Semgrep Security Scan
|
||||
uses: returntocorp/semgrep-action@v1
|
||||
with:
|
||||
config: p/security-audit
|
||||
paths: backend
|
||||
|
||||
code-quality:
|
||||
name: Code Quality
|
||||
runs-on: ubuntu-latest
|
||||
needs: test
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: 3.10
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install -r requirements-dev.txt
|
||||
|
||||
- name: Run Black formatting check
|
||||
run: |
|
||||
black --check backend/
|
||||
|
||||
- name: Run Flake8 linting
|
||||
run: |
|
||||
flake8 backend/
|
||||
|
||||
- name: Run isort import sorting check
|
||||
run: |
|
||||
isort --check-only backend/
|
||||
|
||||
- name: Run MyPy type checking
|
||||
run: |
|
||||
mypy backend/ --ignore-missing-imports
|
||||
|
||||
- name: Run ESLint for frontend
|
||||
run: |
|
||||
cd frontend
|
||||
npm install
|
||||
npm run lint
|
||||
|
||||
build-and-push:
|
||||
name: Build and Push Images
|
||||
runs-on: ubuntu-latest
|
||||
needs: [test, security, code-quality]
|
||||
if: github.ref == 'refs/heads/main'
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Log in to Container Registry
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ${{ env.REGISTRY }}
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Extract metadata
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
|
||||
tags: |
|
||||
type=ref,event=branch
|
||||
type=ref,event=pr
|
||||
type=sha,prefix={{branch}}-
|
||||
type=raw,value=latest,enable={{is_default_branch}}
|
||||
|
||||
- name: Build backend image
|
||||
uses: docker/build-push-action@v5
|
||||
with:
|
||||
context: backend
|
||||
platforms: linux/amd64,linux/arm64
|
||||
push: true
|
||||
tags: ${{ steps.meta.outputs.tags }}-backend
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
|
||||
- name: Build frontend image
|
||||
uses: docker/build-push-action@v5
|
||||
with:
|
||||
context: frontend
|
||||
platforms: linux/amd64,linux/arm64
|
||||
push: true
|
||||
tags: ${{ steps.meta.outputs.tags }}-frontend
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
|
||||
staging-deploy:
|
||||
name: Deploy to Staging
|
||||
runs-on: ubuntu-latest
|
||||
needs: build-and-push
|
||||
if: github.ref == 'refs/heads/develop'
|
||||
|
||||
environment:
|
||||
name: staging
|
||||
url: https://staging.malaysian-sme-platform.com
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Deploy to staging
|
||||
uses: appleboy/ssh-action@v1.0.0
|
||||
with:
|
||||
host: ${{ secrets.STAGING_HOST }}
|
||||
username: ${{ secrets.STAGING_USER }}
|
||||
key: ${{ secrets.STAGING_SSH_KEY }}
|
||||
script: |
|
||||
cd /opt/malaysian-sme-platform
|
||||
docker-compose -f docker-compose.staging.yml pull
|
||||
docker-compose -f docker-compose.staging.yml up -d
|
||||
docker system prune -f
|
||||
|
||||
- name: Run health checks
|
||||
run: |
|
||||
curl -f https://staging.malaysian-sme-platform.com/health/ || exit 1
|
||||
curl -f https://staging.malaysian-sme-platform.com/api/health/ || exit 1
|
||||
|
||||
production-deploy:
|
||||
name: Deploy to Production
|
||||
runs-on: ubuntu-latest
|
||||
needs: build-and-push
|
||||
if: github.ref == 'refs/heads/main'
|
||||
|
||||
environment:
|
||||
name: production
|
||||
url: https://api.malaysian-sme-platform.com
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Create GitHub deployment
|
||||
uses: actions/create-deployment@v1
|
||||
id: deployment
|
||||
with:
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
environment: production
|
||||
|
||||
- name: Deploy to production
|
||||
uses: appleboy/ssh-action@v1.0.0
|
||||
with:
|
||||
host: ${{ secrets.PRODUCTION_HOST }}
|
||||
username: ${{ secrets.PRODUCTION_USER }}
|
||||
key: ${{ secrets.PRODUCTION_SSH_KEY }}
|
||||
script: |
|
||||
# Create backup before deployment
|
||||
./scripts/backup-database.sh
|
||||
|
||||
# Deploy with zero downtime
|
||||
cd /opt/malaysian-sme-platform
|
||||
|
||||
# Pull new images
|
||||
docker-compose -f docker-compose.prod.yml pull
|
||||
|
||||
# Perform rolling update
|
||||
docker-compose -f docker-compose.prod.yml up -d --no-deps backend
|
||||
sleep 30
|
||||
docker-compose -f docker-compose.prod.yml up -d --no-deps frontend
|
||||
|
||||
# Run database migrations
|
||||
docker-compose -f docker-compose.prod.yml exec -T backend python manage.py migrate
|
||||
|
||||
# Collect static files
|
||||
docker-compose -f docker-compose.prod.yml exec -T backend python manage.py collectstatic --noinput
|
||||
|
||||
# Clean up
|
||||
docker system prune -f
|
||||
|
||||
- name: Run production health checks
|
||||
run: |
|
||||
curl -f https://api.malaysian-sme-platform.com/health/ || exit 1
|
||||
curl -f https://api.malaysian-sme-platform.com/api/health/ || exit 1
|
||||
curl -f https://app.malaysian-sme-platform.com/ || exit 1
|
||||
|
||||
- name: Update deployment status
|
||||
uses: actions/update-deployment@v1
|
||||
if: always()
|
||||
with:
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
deployment_id: ${{ steps.deployment.outputs.deployment_id }}
|
||||
state: ${{ job.status }}
|
||||
|
||||
performance-test:
|
||||
name: Performance Testing
|
||||
runs-on: ubuntu-latest
|
||||
needs: staging-deploy
|
||||
if: github.ref == 'refs/heads/develop'
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up k6
|
||||
uses: grafana/k6-action@v0.3.0
|
||||
with:
|
||||
filename: tests/performance/load-test.js
|
||||
|
||||
- name: Run performance tests
|
||||
run: |
|
||||
cd tests/performance
|
||||
k6 run load-test.js --env STAGING_URL=https://staging.malaysian-sme-platform.com
|
||||
|
||||
- name: Upload performance results
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: performance-results
|
||||
path: tests/performance/results/
|
||||
|
||||
notify:
|
||||
name: Notify Team
|
||||
runs-on: ubuntu-latest
|
||||
needs: [production-deploy, performance-test]
|
||||
if: always() && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/develop')
|
||||
|
||||
steps:
|
||||
- name: Send Slack notification
|
||||
uses: 8398a7/action-slack@v3
|
||||
with:
|
||||
status: ${{ job.status }}
|
||||
channel: '#deployments'
|
||||
webhook_url: ${{ secrets.SLACK_WEBHOOK }}
|
||||
env:
|
||||
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK }}
|
||||
|
||||
- name: Send email notification
|
||||
uses: dawidd6/action-send-mail@v3
|
||||
with:
|
||||
server_address: smtp.gmail.com
|
||||
server_port: 465
|
||||
username: ${{ secrets.EMAIL_USERNAME }}
|
||||
password: ${{ secrets.EMAIL_PASSWORD }}
|
||||
subject: "Deployment ${{ job.status }} - ${{ github.repository }}"
|
||||
body: |
|
||||
Deployment to ${{ github.ref }} completed with status: ${{ job.status }}
|
||||
|
||||
Commit: ${{ github.sha }}
|
||||
Author: ${{ github.actor }}
|
||||
|
||||
View logs: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
||||
to: devops@malaysian-sme-platform.com
|
||||
from: ci-cd@malaysian-sme-platform.com
|
||||
421
.github/workflows/deployment.yml
vendored
Normal file
421
.github/workflows/deployment.yml
vendored
Normal file
@@ -0,0 +1,421 @@
|
||||
name: Deployment Pipeline
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ main, develop ]
|
||||
tags: [ 'v*' ]
|
||||
pull_request:
|
||||
types: [opened, synchronize, reopened]
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
environment:
|
||||
description: 'Target environment'
|
||||
required: true
|
||||
default: 'staging'
|
||||
type: choice
|
||||
options:
|
||||
- staging
|
||||
- production
|
||||
version:
|
||||
description: 'Version to deploy'
|
||||
required: false
|
||||
type: string
|
||||
|
||||
env:
|
||||
REGISTRY: ghcr.io
|
||||
IMAGE_NAME: ${{ github.repository }}
|
||||
|
||||
jobs:
|
||||
prepare-deployment:
|
||||
name: Prepare Deployment
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
environment: ${{ steps.env.outputs.environment }}
|
||||
version: ${{ steps.version.outputs.version }}
|
||||
should_deploy: ${{ steps.deployment-check.outputs.should_deploy }}
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Determine environment
|
||||
id: env
|
||||
run: |
|
||||
if [ "${{ github.ref }}" = "refs/heads/main" ]; then
|
||||
echo "environment=production" >> $GITHUB_OUTPUT
|
||||
elif [ "${{ github.ref }}" = "refs/heads/develop" ]; then
|
||||
echo "environment=staging" >> $GITHUB_OUTPUT
|
||||
elif [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
|
||||
echo "environment=${{ github.event.inputs.environment }}" >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "environment=none" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
|
||||
- name: Determine version
|
||||
id: version
|
||||
run: |
|
||||
if [[ "${{ github.ref }}" =~ ^refs/tags/v ]]; then
|
||||
echo "version=${GITHUB_REF#refs/tags/v}" >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "version=${{ github.sha }}" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
|
||||
- name: Check deployment readiness
|
||||
id: deployment-check
|
||||
run: |
|
||||
if [ "${{ steps.env.outputs.environment }}" = "none" ]; then
|
||||
echo "should_deploy=false" >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "should_deploy=true" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
|
||||
build-and-test:
|
||||
name: Build and Test
|
||||
runs-on: ubuntu-latest
|
||||
needs: prepare-deployment
|
||||
if: needs.prepare-deployment.outputs.should_deploy == 'true'
|
||||
|
||||
strategy:
|
||||
matrix:
|
||||
service: [backend, frontend]
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Log in to Container Registry
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ${{ env.REGISTRY }}
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Extract metadata
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
|
||||
tags: |
|
||||
type=ref,event=branch
|
||||
type=ref,event=pr
|
||||
type=sha,prefix={{branch}}-
|
||||
type=raw,value=latest,enable={{is_default_branch}}
|
||||
type=raw,value=${{ needs.prepare-deployment.outputs.version }}
|
||||
|
||||
- name: Build ${{ matrix.service }} image
|
||||
uses: docker/build-push-action@v5
|
||||
with:
|
||||
context: ${{ matrix.service }}
|
||||
platforms: linux/amd64,linux/arm64
|
||||
push: true
|
||||
tags: ${{ steps.meta.outputs.tags }}-${{ matrix.service }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
build-args: |
|
||||
VERSION=${{ needs.prepare-deployment.outputs.version }}
|
||||
ENVIRONMENT=${{ needs.prepare-deployment.outputs.environment }}
|
||||
|
||||
- name: Run security scan on ${{ matrix.service }}
|
||||
uses: aquasecurity/trivy-action@master
|
||||
with:
|
||||
image-ref: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ needs.prepare-deployment.outputs.version }}-${{ matrix.service }}
|
||||
format: 'sarif'
|
||||
output: 'trivy-${{ matrix.service }}.sarif'
|
||||
|
||||
- name: Upload Trivy scan results
|
||||
uses: github/codeql-action/upload-sarif@v2
|
||||
with:
|
||||
sarif_file: 'trivy-${{ matrix.service }}.sarif'
|
||||
|
||||
pre-deployment-checks:
|
||||
name: Pre-deployment Checks
|
||||
runs-on: ubuntu-latest
|
||||
needs: [prepare-deployment, build-and-test]
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Run database migration check
|
||||
env:
|
||||
DATABASE_URL: ${{ secrets[format('{0}_DATABASE_URL', upper(needs.prepare-deployment.outputs.environment))] }}
|
||||
run: |
|
||||
python scripts/check-migrations.py
|
||||
|
||||
- name: Validate configuration
|
||||
run: |
|
||||
python scripts/validate-config.py --environment ${{ needs.prepare-deployment.outputs.environment }}
|
||||
|
||||
- name: Check deployment prerequisites
|
||||
run: |
|
||||
python scripts/deployment-prerequisites.py --environment ${{ needs.prepare-deployment.outputs.environment }}
|
||||
|
||||
- name: Generate deployment manifest
|
||||
run: |
|
||||
python scripts/generate-deployment-manifest.py \
|
||||
--environment ${{ needs.prepare-deployment.outputs.environment }} \
|
||||
--version ${{ needs.prepare-deployment.outputs.version }} \
|
||||
--output deployment-manifest.json
|
||||
|
||||
- name: Upload deployment manifest
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: deployment-manifest
|
||||
path: deployment-manifest.json
|
||||
|
||||
staging-deployment:
|
||||
name: Deploy to Staging
|
||||
runs-on: ubuntu-latest
|
||||
needs: [prepare-deployment, pre-deployment-checks]
|
||||
if: needs.prepare-deployment.outputs.environment == 'staging'
|
||||
|
||||
environment:
|
||||
name: staging
|
||||
url: https://staging.malaysian-sme-platform.com
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Download deployment manifest
|
||||
uses: actions/download-artifact@v3
|
||||
with:
|
||||
name: deployment-manifest
|
||||
|
||||
- name: Deploy to staging
|
||||
uses: appleboy/ssh-action@v1.0.0
|
||||
with:
|
||||
host: ${{ secrets.STAGING_HOST }}
|
||||
username: ${{ secrets.STAGING_USER }}
|
||||
key: ${{ secrets.STAGING_SSH_KEY }}
|
||||
script: |
|
||||
cd /opt/malaysian-sme-platform
|
||||
|
||||
# Backup current deployment
|
||||
./scripts/backup-deployment.sh staging
|
||||
|
||||
# Pull new images
|
||||
docker-compose -f docker-compose.staging.yml pull
|
||||
|
||||
# Deploy with blue-green strategy
|
||||
docker-compose -f docker-compose.staging-bluegreen.yml up -d
|
||||
|
||||
# Wait for health checks
|
||||
./scripts/wait-for-health.sh staging
|
||||
|
||||
# Switch traffic
|
||||
docker-compose -f docker-compose.staging.yml down
|
||||
docker-compose -f docker-compose.staging.yml up -d
|
||||
|
||||
# Run database migrations
|
||||
docker-compose -f docker-compose.staging.yml exec -T backend python manage.py migrate
|
||||
|
||||
# Collect static files
|
||||
docker-compose -f docker-compose.staging.yml exec -T backend python manage.py collectstatic --noinput
|
||||
|
||||
# Clean up
|
||||
docker system prune -f
|
||||
|
||||
- name: Run post-deployment tests
|
||||
run: |
|
||||
python scripts/post-deployment-tests.py --environment staging
|
||||
|
||||
- name: Verify deployment
|
||||
run: |
|
||||
curl -f https://staging.malaysian-sme-platform.com/health/ || exit 1
|
||||
curl -f https://staging.malaysian-sme-platform.com/api/health/ || exit 1
|
||||
|
||||
- name: Send deployment notification
|
||||
uses: 8398a7/action-slack@v3
|
||||
with:
|
||||
status: success
|
||||
channel: '#deployments'
|
||||
webhook_url: ${{ secrets.SLACK_WEBHOOK }}
|
||||
env:
|
||||
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK }}
|
||||
|
||||
production-deployment:
|
||||
name: Deploy to Production
|
||||
runs-on: ubuntu-latest
|
||||
needs: [prepare-deployment, pre-deployment-checks]
|
||||
if: needs.prepare-deployment.outputs.environment == 'production'
|
||||
|
||||
environment:
|
||||
name: production
|
||||
url: https://api.malaysian-sme-platform.com
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Create production deployment
|
||||
uses: actions/create-deployment@v1
|
||||
id: deployment
|
||||
with:
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
environment: production
|
||||
ref: ${{ github.sha }}
|
||||
|
||||
- name: Download deployment manifest
|
||||
uses: actions/download-artifact@v3
|
||||
with:
|
||||
name: deployment-manifest
|
||||
|
||||
- name: Create backup before deployment
|
||||
run: |
|
||||
ssh -i ${{ secrets.PRODUCTION_SSH_KEY }} ${{ secrets.PRODUCTION_USER }}@${{ secrets.PRODUCTION_HOST }} \
|
||||
'cd /opt/malaysian-sme-platform && ./scripts/backup-database.sh'
|
||||
|
||||
- name: Deploy to production
|
||||
uses: appleboy/ssh-action@v1.0.0
|
||||
with:
|
||||
host: ${{ secrets.PRODUCTION_HOST }}
|
||||
username: ${{ secrets.PRODUCTION_USER }}
|
||||
key: ${{ secrets.PRODUCTION_SSH_KEY }}
|
||||
script: |
|
||||
cd /opt/malaysian-sme-platform
|
||||
|
||||
# Pre-deployment checks
|
||||
./scripts/pre-deployment-checks.sh production
|
||||
|
||||
# Rolling deployment
|
||||
docker-compose -f docker-compose.prod.yml pull
|
||||
|
||||
# Deploy backend first
|
||||
docker-compose -f docker-compose.prod.yml up -d --no-deps backend
|
||||
sleep 30
|
||||
|
||||
# Verify backend health
|
||||
./scripts/wait-for-backend-health.sh
|
||||
|
||||
# Deploy frontend
|
||||
docker-compose -f docker-compose.prod.yml up -d --no-deps frontend
|
||||
sleep 15
|
||||
|
||||
# Run database migrations
|
||||
docker-compose -f docker-compose.prod.yml exec -T backend python manage.py migrate --noinput
|
||||
|
||||
# Collect static files
|
||||
docker-compose -f docker-compose.prod.yml exec -T backend python manage.py collectstatic --noinput
|
||||
|
||||
# Update remaining services
|
||||
docker-compose -f docker-compose.prod.yml up -d
|
||||
|
||||
# Post-deployment verification
|
||||
./scripts/post-deployment-verification.sh
|
||||
|
||||
- name: Run production smoke tests
|
||||
run: |
|
||||
python scripts/smoke-tests.py --environment production
|
||||
|
||||
- name: Verify deployment
|
||||
run: |
|
||||
curl -f https://api.malaysian-sme-platform.com/health/ || exit 1
|
||||
curl -f https://app.malaysian-sme-platform.com/ || exit 1
|
||||
curl -f https://admin.malaysian-sme-platform.com/ || exit 1
|
||||
|
||||
- name: Update deployment status
|
||||
uses: actions/update-deployment@v1
|
||||
if: always()
|
||||
with:
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
deployment_id: ${{ steps.deployment.outputs.deployment_id }}
|
||||
state: ${{ job.status }}
|
||||
|
||||
- name: Send deployment notification
|
||||
if: always()
|
||||
uses: 8398a7/action-slack@v3
|
||||
with:
|
||||
status: ${{ job.status }}
|
||||
channel: '#deployments'
|
||||
webhook_url: ${{ secrets.SLACK_WEBHOOK }}
|
||||
env:
|
||||
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK }}
|
||||
|
||||
rollback-protection:
|
||||
name: Rollback Protection
|
||||
runs-on: ubuntu-latest
|
||||
needs: [staging-deployment, production-deployment]
|
||||
if: always() && (needs.staging-deployment.result == 'failure' || needs.production-deployment.result == 'failure')
|
||||
|
||||
steps:
|
||||
- name: Initiate rollback
|
||||
run: |
|
||||
python scripts/initiate-rollback.py \
|
||||
--environment ${{ needs.prepare-deployment.outputs.environment }} \
|
||||
--failure-reason "Deployment failed" \
|
||||
--rollback-to ${{ github.event.before }}
|
||||
|
||||
- name: Send rollback notification
|
||||
uses: 8398a7/action-slack@v3
|
||||
with:
|
||||
status: failure
|
||||
channel: '#emergency'
|
||||
webhook_url: ${{ secrets.EMERGENCY_SLACK_WEBHOOK }}
|
||||
env:
|
||||
SLACK_WEBHOOK_URL: ${{ secrets.EMERGENCY_SLACK_WEBHOOK }}
|
||||
|
||||
deployment-verification:
|
||||
name: Deployment Verification
|
||||
runs-on: ubuntu-latest
|
||||
needs: [staging-deployment, production-deployment]
|
||||
if: always() && (needs.staging-deployment.result == 'success' || needs.production-deployment.result == 'success')
|
||||
|
||||
steps:
|
||||
- name: Run end-to-end tests
|
||||
run: |
|
||||
python scripts/e2e-tests.py \
|
||||
--environment ${{ needs.prepare-deployment.outputs.environment }} \
|
||||
--timeout 300
|
||||
|
||||
- name: Performance validation
|
||||
run: |
|
||||
python scripts/performance-validation.py \
|
||||
--environment ${{ needs.prepare-deployment.outputs.environment }} \
|
||||
--threshold-percent 10
|
||||
|
||||
- name: Security validation
|
||||
run: |
|
||||
python scripts/security-validation.py \
|
||||
--environment ${{ needs.prepare-deployment.outputs.environment }}
|
||||
|
||||
- name: Generate deployment report
|
||||
run: |
|
||||
python scripts/generate-deployment-report.py \
|
||||
--environment ${{ needs.prepare-deployment.outputs.environment }} \
|
||||
--version ${{ needs.prepare-deployment.outputs.version }} \
|
||||
--output deployment-report.html
|
||||
|
||||
- name: Upload deployment report
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: deployment-report
|
||||
path: deployment-report.html
|
||||
|
||||
cleanup:
|
||||
name: Cleanup
|
||||
runs-on: ubuntu-latest
|
||||
needs: [deployment-verification, rollback-protection]
|
||||
if: always()
|
||||
|
||||
steps:
|
||||
- name: Cleanup old Docker images
|
||||
run: |
|
||||
ssh -i ${{ secrets.PRODUCTION_SSH_KEY }} ${{ secrets.PRODUCTION_USER }}@${{ secrets.PRODUCTION_HOST }} \
|
||||
'docker system prune -f --filter "until=72h"'
|
||||
|
||||
- name: Cleanup old backups
|
||||
run: |
|
||||
ssh -i ${{ secrets.PRODUCTION_SSH_KEY }} ${{ secrets.PRODUCTION_USER }}@${{ secrets.PRODUCTION_HOST }} \
|
||||
'find /backups -name "*.sql" -mtime +7 -delete'
|
||||
|
||||
- name: Update deployment metrics
|
||||
run: |
|
||||
python scripts/update-deployment-metrics.py \
|
||||
--environment ${{ needs.prepare-deployment.outputs.environment }} \
|
||||
--status ${{ job.status }} \
|
||||
--duration ${{ job.duration }}
|
||||
375
.github/workflows/monitoring.yml
vendored
Normal file
375
.github/workflows/monitoring.yml
vendored
Normal file
@@ -0,0 +1,375 @@
|
||||
name: System Monitoring
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: '0 */6 * * *' # Every 6 hours
|
||||
workflow_dispatch:
|
||||
push:
|
||||
branches: [ main, develop ]
|
||||
|
||||
jobs:
|
||||
health-checks:
|
||||
name: Health Checks
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Run production health checks
|
||||
run: |
|
||||
# API Health
|
||||
curl -f https://api.malaysian-sme-platform.com/health/ || exit 1
|
||||
curl -f https://api.malaysian-sme-platform.com/api/health/ || exit 1
|
||||
|
||||
# Application Health
|
||||
curl -f https://app.malaysian-sme-platform.com/ || exit 1
|
||||
|
||||
# Database Health
|
||||
curl -f https://api.malaysian-sme-platform.com/api/health/database/ || exit 1
|
||||
|
||||
# Cache Health
|
||||
curl -f https://api.malaysian-sme-platform.com/api/health/cache/ || exit 1
|
||||
|
||||
- name: Run staging health checks
|
||||
run: |
|
||||
curl -f https://staging.malaysian-sme-platform.com/health/ || exit 1
|
||||
curl -f https://staging.malaysian-sme-platform.com/api/health/ || exit 1
|
||||
|
||||
performance-monitoring:
|
||||
name: Performance Monitoring
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up k6
|
||||
uses: grafana/k6-action@v0.3.0
|
||||
with:
|
||||
filename: tests/performance/monitoring.js
|
||||
|
||||
- name: Run performance monitoring
|
||||
run: |
|
||||
cd tests/performance
|
||||
k6 run monitoring.js \
|
||||
--env PROD_URL=https://api.malaysian-sme-platform.com \
|
||||
--env STAGING_URL=https://staging.malaysian-sme-platform.com
|
||||
|
||||
- name: Upload performance results
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: performance-monitoring-results
|
||||
path: tests/performance/results/
|
||||
|
||||
database-monitoring:
|
||||
name: Database Monitoring
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: 3.10
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install psycopg2-binary pandas matplotlib
|
||||
|
||||
- name: Run database health checks
|
||||
env:
|
||||
DATABASE_URL: ${{ secrets.PRODUCTION_DATABASE_URL }}
|
||||
run: |
|
||||
python scripts/database-health-check.py
|
||||
|
||||
- name: Generate database metrics report
|
||||
env:
|
||||
DATABASE_URL: ${{ secrets.PRODUCTION_DATABASE_URL }}
|
||||
run: |
|
||||
python scripts/database-metrics.py --output database-metrics.json
|
||||
|
||||
- name: Upload database reports
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: database-monitoring-reports
|
||||
path: |
|
||||
database-metrics.json
|
||||
database-health-report.json
|
||||
|
||||
cache-monitoring:
|
||||
name: Cache Monitoring
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: 3.10
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install redis pandas
|
||||
|
||||
- name: Run Redis health checks
|
||||
env:
|
||||
REDIS_URL: ${{ secrets.PRODUCTION_REDIS_URL }}
|
||||
run: |
|
||||
python scripts/redis-health-check.py
|
||||
|
||||
- name: Generate cache metrics report
|
||||
env:
|
||||
REDIS_URL: ${{ secrets.PRODUCTION_REDIS_URL }}
|
||||
run: |
|
||||
python scripts/cache-metrics.py --output cache-metrics.json
|
||||
|
||||
- name: Upload cache reports
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: cache-monitoring-reports
|
||||
path: |
|
||||
cache-metrics.json
|
||||
redis-health-report.json
|
||||
|
||||
log-monitoring:
|
||||
name: Log Monitoring
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Analyze application logs
|
||||
env:
|
||||
LOGS_ACCESS_KEY: ${{ secrets.LOGS_ACCESS_KEY }}
|
||||
run: |
|
||||
python scripts/log-analysis.py \
|
||||
--hours 6 \
|
||||
--error-threshold 10 \
|
||||
--warning-threshold 50 \
|
||||
--output log-analysis-report.json
|
||||
|
||||
- name: Check for critical errors
|
||||
run: |
|
||||
python scripts/critical-error-check.py \
|
||||
--hours 1 \
|
||||
--notification-webhook ${{ secrets.SLACK_WEBHOOK }}
|
||||
|
||||
- name: Upload log reports
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: log-monitoring-reports
|
||||
path: |
|
||||
log-analysis-report.json
|
||||
error-summary.json
|
||||
|
||||
resource-monitoring:
|
||||
name: Resource Monitoring
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Monitor system resources
|
||||
env:
|
||||
MONITORING_API_KEY: ${{ secrets.MONITORING_API_KEY }}
|
||||
run: |
|
||||
python scripts/resource-monitoring.py \
|
||||
--output resource-metrics.json
|
||||
|
||||
- name: Check resource thresholds
|
||||
run: |
|
||||
python scripts/resource-threshold-check.py \
|
||||
--cpu-threshold 80 \
|
||||
--memory-threshold 85 \
|
||||
--disk-threshold 90 \
|
||||
--output threshold-report.json
|
||||
|
||||
- name: Upload resource reports
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: resource-monitoring-reports
|
||||
path: |
|
||||
resource-metrics.json
|
||||
threshold-report.json
|
||||
|
||||
uptime-monitoring:
|
||||
name: Uptime Monitoring
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Check service uptime
|
||||
run: |
|
||||
python scripts/uptime-check.py \
|
||||
--services api,app,admin,static \
|
||||
--timeout 30 \
|
||||
--output uptime-report.json
|
||||
|
||||
- name: Verify SSL certificates
|
||||
run: |
|
||||
python scripts/ssl-check.py \
|
||||
--domains api.malaysian-sme-platform.com,app.malaysian-sme-platform.com \
|
||||
--output ssl-report.json
|
||||
|
||||
- name: Upload uptime reports
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: uptime-monitoring-reports
|
||||
path: |
|
||||
uptime-report.json
|
||||
ssl-report.json
|
||||
|
||||
backup-monitoring:
|
||||
name: Backup Monitoring
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Check backup status
|
||||
env:
|
||||
BACKUP_ACCESS_KEY: ${{ secrets.BACKUP_ACCESS_KEY }}
|
||||
run: |
|
||||
python scripts/backup-check.py \
|
||||
--hours 24 \
|
||||
--output backup-report.json
|
||||
|
||||
- name: Verify backup integrity
|
||||
run: |
|
||||
python scripts/backup-integrity.py \
|
||||
--verify-latest 3 \
|
||||
--output integrity-report.json
|
||||
|
||||
- name: Upload backup reports
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: backup-monitoring-reports
|
||||
path: |
|
||||
backup-report.json
|
||||
integrity-report.json
|
||||
|
||||
security-monitoring:
|
||||
name: Security Monitoring
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Check for security events
|
||||
env:
|
||||
SECURITY_API_KEY: ${{ secrets.SECURITY_API_KEY }}
|
||||
run: |
|
||||
python scripts/security-monitoring.py \
|
||||
--hours 6 \
|
||||
--output security-events.json
|
||||
|
||||
- name: Analyze authentication patterns
|
||||
env:
|
||||
AUTH_LOGS_ACCESS_KEY: ${{ secrets.AUTH_LOGS_ACCESS_KEY }}
|
||||
run: |
|
||||
python scripts/auth-pattern-analysis.py \
|
||||
--hours 24 \
|
||||
--output auth-patterns.json
|
||||
|
||||
- name: Upload security reports
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: security-monitoring-reports
|
||||
path: |
|
||||
security-events.json
|
||||
auth-patterns.json
|
||||
|
||||
monitoring-dashboard:
|
||||
name: Monitoring Dashboard
|
||||
runs-on: ubuntu-latest
|
||||
needs: [health-checks, performance-monitoring, database-monitoring, cache-monitoring, log-monitoring, resource-monitoring, uptime-monitoring, backup-monitoring, security-monitoring]
|
||||
if: always()
|
||||
|
||||
steps:
|
||||
- name: Download all reports
|
||||
uses: actions/download-artifact@v3
|
||||
|
||||
- name: Generate monitoring dashboard
|
||||
run: |
|
||||
python scripts/generate-monitoring-dashboard.py
|
||||
|
||||
- name: Upload monitoring dashboard
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: monitoring-dashboard
|
||||
path: monitoring-dashboard.html
|
||||
|
||||
- name: Send monitoring summary to Slack
|
||||
uses: 8398a7/action-slack@v3
|
||||
with:
|
||||
status: ${{ job.status }}
|
||||
channel: '#monitoring'
|
||||
webhook_url: ${{ secrets.SLACK_WEBHOOK }}
|
||||
env:
|
||||
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK }}
|
||||
|
||||
alerting:
|
||||
name: Alerting
|
||||
runs-on: ubuntu-latest
|
||||
needs: monitoring-dashboard
|
||||
if: failure()
|
||||
|
||||
steps:
|
||||
- name: Create alert issue
|
||||
uses: actions/github-script@v6
|
||||
with:
|
||||
script: |
|
||||
github.rest.issues.create({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
title: `🚨 Monitoring Alert - ${{ github.run_number }}`,
|
||||
body: `Monitoring checks failed for run #${{ github.run_number }}.
|
||||
|
||||
**Time:** ${{ github.event_name }} at ${{ github.run_started_at }}
|
||||
**Repository:** ${{ github.repository }}
|
||||
|
||||
Please review the monitoring reports and investigate the issues.
|
||||
|
||||
📋 **Monitoring Reports:**
|
||||
- [Health Checks](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})
|
||||
- [Performance Monitoring](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})
|
||||
- [Database Monitoring](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})
|
||||
- [Cache Monitoring](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})
|
||||
- [Log Monitoring](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})
|
||||
- [Resource Monitoring](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})
|
||||
- [Uptime Monitoring](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})
|
||||
- [Backup Monitoring](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})
|
||||
- [Security Monitoring](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})
|
||||
|
||||
🎯 **Immediate Actions:**
|
||||
1. Review failed monitoring checks
|
||||
2. Investigate service availability
|
||||
3. Check system resources
|
||||
4. Verify backup integrity
|
||||
5. Address any security events
|
||||
|
||||
This issue was automatically created by the monitoring system.`,
|
||||
labels: ['monitoring', 'alert', 'priority-critical']
|
||||
});
|
||||
|
||||
- name: Send emergency notification
|
||||
uses: 8398a7/action-slack@v3
|
||||
with:
|
||||
status: failure
|
||||
channel: '#emergency'
|
||||
webhook_url: ${{ secrets.EMERGENCY_SLACK_WEBHOOK }}
|
||||
env:
|
||||
SLACK_WEBHOOK_URL: ${{ secrets.EMERGENCY_SLACK_WEBHOOK }}
|
||||
364
.github/workflows/quality.yml
vendored
Normal file
364
.github/workflows/quality.yml
vendored
Normal file
@@ -0,0 +1,364 @@
|
||||
name: Code Quality
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ main, develop ]
|
||||
pull_request:
|
||||
branches: [ main, develop ]
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
python-quality:
|
||||
name: Python Code Quality
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: 3.10
|
||||
|
||||
- name: Cache pip packages
|
||||
uses: actions/cache@v3
|
||||
with:
|
||||
path: ~/.cache/pip
|
||||
key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-pip-
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install -r requirements.txt
|
||||
pip install -r requirements-dev.txt
|
||||
|
||||
- name: Run Black formatting check
|
||||
run: |
|
||||
black --check --diff backend/
|
||||
|
||||
- name: Run isort import sorting check
|
||||
run: |
|
||||
isort --check-only --diff backend/
|
||||
|
||||
- name: Run Flake8 linting
|
||||
run: |
|
||||
flake8 backend/ --format=junit-xml --output-file=flake8-report.xml
|
||||
|
||||
- name: Run Pylint static analysis
|
||||
run: |
|
||||
pylint backend/ --exit-zero --output-format=pylint_junit.JunitReporter --output-file=pylint-report.xml
|
||||
|
||||
- name: Run MyPy type checking
|
||||
run: |
|
||||
mypy backend/ --ignore-missing-imports --junit-xml=mypy-report.xml
|
||||
|
||||
- name: Run Bandit security linting
|
||||
run: |
|
||||
bandit -r backend/ -f json -o bandit-report.json
|
||||
|
||||
- name: Run Radon code complexity analysis
|
||||
run: |
|
||||
pip install radon
|
||||
radon cc backend/ -a -nb -o json > radon-report.json
|
||||
|
||||
- name: Run vulture dead code detection
|
||||
run: |
|
||||
pip install vulture
|
||||
vulture backend/ --min-confidence 70 --format json > vulture-report.json
|
||||
|
||||
- name: Upload quality reports
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: python-quality-reports
|
||||
path: |
|
||||
flake8-report.xml
|
||||
pylint-report.xml
|
||||
mypy-report.xml
|
||||
bandit-report.json
|
||||
radon-report.json
|
||||
vulture-report.json
|
||||
|
||||
javascript-quality:
|
||||
name: JavaScript Code Quality
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Node.js
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: '18'
|
||||
cache: 'npm'
|
||||
cache-dependency-path: frontend/package-lock.json
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
cd frontend
|
||||
npm install
|
||||
|
||||
- name: Run ESLint
|
||||
run: |
|
||||
cd frontend
|
||||
npm run lint -- --format junit --output-file ../eslint-report.xml
|
||||
|
||||
- name: Run Prettier formatting check
|
||||
run: |
|
||||
cd frontend
|
||||
npm run format:check
|
||||
|
||||
- name: Run TypeScript type checking
|
||||
run: |
|
||||
cd frontend
|
||||
npm run type-check
|
||||
|
||||
- name: Run SonarQube scan
|
||||
uses: sonarqube-quality-gate-action@master
|
||||
env:
|
||||
SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
|
||||
|
||||
- name: Upload JavaScript quality reports
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: javascript-quality-reports
|
||||
path: |
|
||||
eslint-report.xml
|
||||
|
||||
test-coverage:
|
||||
name: Test Coverage Analysis
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:15
|
||||
env:
|
||||
POSTGRES_PASSWORD: postgres
|
||||
POSTGRES_DB: test_db
|
||||
options: >-
|
||||
--health-cmd pg_isready
|
||||
--health-interval 10s
|
||||
--health-timeout 5s
|
||||
--health-retries 5
|
||||
ports:
|
||||
- 5432:5432
|
||||
redis:
|
||||
image: redis:7-alpine
|
||||
options: >-
|
||||
--health-cmd "redis-cli ping"
|
||||
--health-interval 10s
|
||||
--health-timeout 5s
|
||||
--health-retries 5
|
||||
ports:
|
||||
- 6379:6379
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: 3.10
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install -r requirements.txt
|
||||
pip install -r requirements-dev.txt
|
||||
pip install coverage[toml] coveralls
|
||||
|
||||
- name: Run tests with coverage
|
||||
run: |
|
||||
cd backend
|
||||
coverage run --source=. manage.py test --verbosity=2
|
||||
coverage xml
|
||||
coverage report --show-missing
|
||||
coverage html
|
||||
env:
|
||||
DATABASE_URL: postgres://postgres:postgres@localhost:5432/test_db
|
||||
REDIS_URL: redis://localhost:6379/0
|
||||
SECRET_KEY: test-secret-key-for-ci
|
||||
|
||||
- name: Upload coverage to Codecov
|
||||
uses: codecov/codecov-action@v3
|
||||
with:
|
||||
file: ./coverage.xml
|
||||
flags: unittests
|
||||
name: codecov-umbrella
|
||||
|
||||
- name: Upload coverage reports
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: coverage-reports
|
||||
path: |
|
||||
coverage.xml
|
||||
htmlcov/
|
||||
|
||||
performance-analysis:
|
||||
name: Performance Analysis
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: 3.10
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install -r requirements.txt
|
||||
pip install line-profiler memory-profiler
|
||||
|
||||
- name: Run Python performance analysis
|
||||
run: |
|
||||
cd backend
|
||||
python -m cProfile -o profile_output.prof manage.py test --verbosity=0
|
||||
python -m memory_profiler scripts/memory-profile.py > memory-profile.txt
|
||||
|
||||
- name: Analyze performance results
|
||||
run: |
|
||||
pip install snakeviz
|
||||
snakeviz profile_output.prof --server
|
||||
|
||||
- name: Upload performance reports
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: performance-reports
|
||||
path: |
|
||||
profile_output.prof
|
||||
memory-profile.txt
|
||||
|
||||
documentation-quality:
|
||||
name: Documentation Quality
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: 3.10
|
||||
|
||||
- name: Install documentation tools
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install sphinx sphinx-rtd-theme pydoc-markdown
|
||||
pip install -r requirements.txt
|
||||
|
||||
- name: Check docstring coverage
|
||||
run: |
|
||||
pip install interrogate
|
||||
interrogate backend/ --verbose --ignore-init-method --ignore-module --ignore-private --fail-under=80
|
||||
|
||||
- name: Generate documentation
|
||||
run: |
|
||||
cd docs
|
||||
make html
|
||||
|
||||
- name: Check for broken links
|
||||
run: |
|
||||
pip install linkchecker
|
||||
linkchecker docs/_build/html/index.html
|
||||
|
||||
- name: Upload documentation
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: documentation-build
|
||||
path: docs/_build/html/
|
||||
|
||||
code-metrics:
|
||||
name: Code Metrics
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: 3.10
|
||||
|
||||
- name: Install analysis tools
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install lizard radon xenon
|
||||
|
||||
- name: Calculate code metrics
|
||||
run: |
|
||||
lizard backend/ --csv > lizard-metrics.csv
|
||||
radon raw backend/ --json > radon-metrics.json
|
||||
xenon --max-absolute A --max-modules A --max-average A backend/ > xenon-report.txt
|
||||
|
||||
- name: Generate quality dashboard
|
||||
run: |
|
||||
python scripts/generate-quality-dashboard.py
|
||||
|
||||
- name: Upload metrics reports
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: code-metrics
|
||||
path: |
|
||||
lizard-metrics.csv
|
||||
radon-metrics.json
|
||||
xenon-report.txt
|
||||
quality-dashboard.html
|
||||
|
||||
quality-gate:
|
||||
name: Quality Gate
|
||||
runs-on: ubuntu-latest
|
||||
needs: [python-quality, javascript-quality, test-coverage, documentation-quality, code-metrics]
|
||||
if: always()
|
||||
|
||||
steps:
|
||||
- name: Download all reports
|
||||
uses: actions/download-artifact@v3
|
||||
|
||||
- name: Evaluate quality gate
|
||||
run: |
|
||||
python scripts/evaluate-quality-gate.py
|
||||
|
||||
- name: Create quality issue if gate fails
|
||||
if: failure()
|
||||
uses: actions/github-script@v6
|
||||
with:
|
||||
script: |
|
||||
github.rest.issues.create({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
title: `🔍 Quality Gate Failed - ${{ github.sha }}`,
|
||||
body: `Code quality checks failed for commit ${{ github.sha }}.
|
||||
|
||||
**Branch:** ${{ github.ref }}
|
||||
**Commit:** ${{ github.sha }}
|
||||
**Author:** ${{ github.actor }}
|
||||
|
||||
Please review the quality reports and address the issues.
|
||||
|
||||
📋 **Quality Reports:**
|
||||
- [Python Quality](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})
|
||||
- [JavaScript Quality](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})
|
||||
- [Test Coverage](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})
|
||||
- [Documentation Quality](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})
|
||||
- [Code Metrics](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})
|
||||
|
||||
🎯 **Action Items:**
|
||||
1. Review and fix code style issues
|
||||
2. Address security vulnerabilities
|
||||
3. Improve test coverage where needed
|
||||
4. Update documentation
|
||||
5. Refactor complex code
|
||||
|
||||
This issue was automatically created by the CI/CD pipeline.`,
|
||||
labels: ['quality', 'bug', 'needs-attention']
|
||||
});
|
||||
308
.github/workflows/security.yml
vendored
Normal file
308
.github/workflows/security.yml
vendored
Normal file
@@ -0,0 +1,308 @@
|
||||
name: Security Scanning
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ main, develop ]
|
||||
pull_request:
|
||||
branches: [ main, develop ]
|
||||
schedule:
|
||||
- cron: '0 2 * * *' # Daily at 2 AM UTC
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
dependency-scanning:
|
||||
name: Dependency Scanning
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Run pip-audit
|
||||
run: |
|
||||
pip install pip-audit
|
||||
pip-audit -r requirements.txt --format json --output dependency-report.json
|
||||
pip-audit -r requirements-dev.txt --format json --output dev-dependency-report.json
|
||||
|
||||
- name: Run Safety
|
||||
run: |
|
||||
pip install safety
|
||||
safety check -r requirements.txt --json --output safety-report.json
|
||||
safety check -r requirements-dev.txt --json --output dev-safety-report.json
|
||||
|
||||
- name: Run npm audit
|
||||
run: |
|
||||
cd frontend
|
||||
npm install
|
||||
npm audit --json > npm-audit-report.json
|
||||
npm audit fix || true
|
||||
|
||||
- name: Generate Dependency License Report
|
||||
run: |
|
||||
pip install pip-licenses
|
||||
pip-licenses --format=json --output-path=license-report.json
|
||||
cd frontend
|
||||
npx license-report --output=json --only=prod > ../frontend-license-report.json
|
||||
|
||||
- name: Upload security reports
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: security-reports
|
||||
path: |
|
||||
dependency-report.json
|
||||
dev-dependency-report.json
|
||||
safety-report.json
|
||||
dev-safety-report.json
|
||||
npm-audit-report.json
|
||||
license-report.json
|
||||
frontend-license-report.json
|
||||
|
||||
code-scanning:
|
||||
name: Code Security Scanning
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Initialize CodeQL
|
||||
uses: github/codeql-action/init@v2
|
||||
with:
|
||||
languages: python, javascript
|
||||
|
||||
- name: Autobuild
|
||||
uses: github/codeql-action/autobuild@v2
|
||||
|
||||
- name: Perform CodeQL Analysis
|
||||
uses: github/codeql-action/analyze@v2
|
||||
|
||||
secrets-scanning:
|
||||
name: Secrets Scanning
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0 # Fetch all history for secret scanning
|
||||
|
||||
- name: Run Gitleaks
|
||||
uses: gitleaks/gitleaks-action@v2
|
||||
with:
|
||||
config-path: .gitleaks.toml
|
||||
report-path: gitleaks-report.json
|
||||
|
||||
- name: Run TruffleHog
|
||||
uses: trufflesecurity/trufflehog@v3
|
||||
with:
|
||||
path: .
|
||||
base: main
|
||||
head: HEAD
|
||||
extra_args: --json --only-verified
|
||||
|
||||
- name: Run detect-secrets
|
||||
run: |
|
||||
pip install detect-secrets
|
||||
detect-secrets scan . > detect-secrets-report.json
|
||||
|
||||
- name: Upload secrets reports
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: secrets-reports
|
||||
path: |
|
||||
gitleaks-report.json
|
||||
trufflehog-report.json
|
||||
detect-secrets-report.json
|
||||
|
||||
container-scanning:
|
||||
name: Container Security Scanning
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Build backend image
|
||||
run: |
|
||||
cd backend
|
||||
docker build -t malaysian-sme-backend:test .
|
||||
|
||||
- name: Build frontend image
|
||||
run: |
|
||||
cd frontend
|
||||
docker build -t malaysian-sme-frontend:test .
|
||||
|
||||
- name: Run Trivy vulnerability scanner
|
||||
uses: aquasecurity/trivy-action@master
|
||||
with:
|
||||
image-ref: 'malaysian-sme-backend:test'
|
||||
format: 'json'
|
||||
output: 'trivy-backend-report.json'
|
||||
|
||||
- name: Run Trivy on frontend
|
||||
uses: aquasecurity/trivy-action@master
|
||||
with:
|
||||
image-ref: 'malaysian-sme-frontend:test'
|
||||
format: 'json'
|
||||
output: 'trivy-frontend-report.json'
|
||||
|
||||
- name: Run Snyk container scan
|
||||
uses: snyk/actions/docker@master
|
||||
env:
|
||||
SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
|
||||
with:
|
||||
image: malaysian-sme-backend:test
|
||||
args: --file=backend/Dockerfile --json-file-output=snyk-backend-report.json
|
||||
|
||||
- name: Run Snyk on frontend
|
||||
uses: snyk/actions/docker@master
|
||||
env:
|
||||
SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
|
||||
with:
|
||||
image: malaysian-sme-frontend:test
|
||||
args: --file=frontend/Dockerfile --json-file-output=snyk-frontend-report.json
|
||||
|
||||
- name: Upload container security reports
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: container-security-reports
|
||||
path: |
|
||||
trivy-backend-report.json
|
||||
trivy-frontend-report.json
|
||||
snyk-backend-report.json
|
||||
snyk-frontend-report.json
|
||||
|
||||
compliance-checking:
|
||||
name: Compliance Checking
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Run Checkov infrastructure scanning
|
||||
uses: bridgecrewio/checkov-action@master
|
||||
with:
|
||||
directory: .
|
||||
framework: dockerfile
|
||||
output: cli
|
||||
soft_fail: true
|
||||
|
||||
- name: Run GDPR compliance checks
|
||||
run: |
|
||||
pip install gdpr-compliance-checker
|
||||
gdpr-compliance-check --config .gdpr-config.json --output gdpr-report.json
|
||||
|
||||
- name: Run PDPA compliance checks (Malaysia specific)
|
||||
run: |
|
||||
python scripts/pdpa-compliance-check.py --output pdpa-report.json
|
||||
|
||||
- name: Upload compliance reports
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: compliance-reports
|
||||
path: |
|
||||
gdpr-report.json
|
||||
pdpa-report.json
|
||||
|
||||
security-dashboard:
|
||||
name: Security Dashboard
|
||||
runs-on: ubuntu-latest
|
||||
needs: [dependency-scanning, code-scanning, secrets-scanning, container-scanning, compliance-checking]
|
||||
if: always()
|
||||
|
||||
steps:
|
||||
- name: Download all reports
|
||||
uses: actions/download-artifact@v3
|
||||
|
||||
- name: Generate security dashboard
|
||||
run: |
|
||||
pip install jinja2
|
||||
python scripts/generate-security-dashboard.py
|
||||
|
||||
- name: Upload security dashboard
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: security-dashboard
|
||||
path: security-dashboard.html
|
||||
|
||||
- name: Comment on PR with security results
|
||||
if: github.event_name == 'pull_request'
|
||||
uses: actions/github-script@v6
|
||||
with:
|
||||
script: |
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
// Read security reports
|
||||
const dependencyReport = JSON.parse(fs.readFileSync('security-reports/dependency-report.json', 'utf8'));
|
||||
const trivyReport = JSON.parse(fs.readFileSync('container-security-reports/trivy-backend-report.json', 'utf8'));
|
||||
|
||||
// Generate summary
|
||||
const dependencyVulnerabilities = dependencyReport.vulnerabilities || [];
|
||||
const containerVulnerabilities = trivyReport.Results || [];
|
||||
|
||||
let comment = `## 🔒 Security Scan Results\n\n`;
|
||||
comment += `### Dependency Vulnerabilities: ${dependencyVulnerabilities.length}\n`;
|
||||
comment += `### Container Vulnerabilities: ${containerVulnerabilities.length}\n\n`;
|
||||
|
||||
if (dependencyVulnerabilities.length > 0 || containerVulnerabilities.length > 0) {
|
||||
comment += `⚠️ **Security issues found. Please review the reports.**\n\n`;
|
||||
comment += `📋 **Detailed Reports:**\n`;
|
||||
comment += `- [Dependency Report](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})\n`;
|
||||
comment += `- [Container Security Report](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})\n`;
|
||||
} else {
|
||||
comment += `✅ **No security issues found!**\n`;
|
||||
}
|
||||
|
||||
// Post comment
|
||||
github.rest.issues.createComment({
|
||||
issue_number: context.issue.number,
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
body: comment
|
||||
});
|
||||
|
||||
remediation:
|
||||
name: Security Remediation
|
||||
runs-on: ubuntu-latest
|
||||
needs: security-dashboard
|
||||
if: failure() && github.event_name == 'push'
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Create remediation issue
|
||||
uses: actions/github-script@v6
|
||||
with:
|
||||
script: |
|
||||
github.rest.issues.create({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
title: `🔒 Security Issues Detected - ${{ github.sha }}`,
|
||||
body: `Security scanning detected vulnerabilities in commit ${{ github.sha }}.
|
||||
|
||||
**Branch:** ${{ github.ref }}
|
||||
**Commit:** ${{ github.sha }}
|
||||
**Author:** ${{ github.actor }}
|
||||
|
||||
Please review the security reports and take appropriate action.
|
||||
|
||||
📋 **Reports:**
|
||||
- [Security Dashboard](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})
|
||||
- [Dependency Report](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})
|
||||
- [Container Security Report](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})
|
||||
|
||||
🎯 **Action Items:**
|
||||
1. Review and assess critical vulnerabilities
|
||||
2. Update affected dependencies
|
||||
3. Rebuild and redeploy secure containers
|
||||
4. Verify fixes with security scans
|
||||
|
||||
This issue was automatically created by the CI/CD pipeline.`,
|
||||
labels: ['security', 'bug', 'priority-high']
|
||||
});
|
||||
Reference in New Issue
Block a user