The DevOps Delivery Pipeline: End-to-End Framework¶

# .pre-commit-config.yaml
repos:
  - repo: https://github.com/pre-commit/pre-commit-hooks
    rev: v4.5.0
    hooks:
      - id: trailing-whitespace
      - id: end-of-file-fixer
      - id: check-yaml
      - id: check-json
      - id: check-merge-conflict
      - id: detect-private-key        # Catch secrets early
      - id: check-added-large-files
        args: ['--maxkb=500']

  - repo: https://github.com/psf/black
    rev: 24.3.0
    hooks:
      - id: black

  - repo: https://github.com/astral-sh/ruff-pre-commit
    rev: v0.3.0
    hooks:
      - id: ruff
        args: [--fix]

  - repo: https://github.com/Yelp/detect-secrets
    rev: v1.4.0
    hooks:
      - id: detect-secrets
        args: ['--baseline', '.secrets.baseline']

  - repo: https://github.com/python-jsonschema/check-jsonschema
    rev: 0.28.0
    hooks:
      - id: check-github-workflows

# .releaserc.yml — semantic-release configuration
branches:
  - main
  - name: beta
    prerelease: true

plugins:
  - "@semantic-release/commit-analyzer"
  - "@semantic-release/release-notes-generator"
  - "@semantic-release/changelog"
  - "@semantic-release/git"
  - "@semantic-release/github"

verifyConditions:
  - "@semantic-release/github"

prepare:
  - "@semantic-release/changelog"
  - "@semantic-release/git"

publish:
  - "@semantic-release/github"

# rollout.yaml
apiVersion: argoproj.io/v1alpha1
kind: Rollout
metadata:
  name: payment-service
spec:
  replicas: 10
  strategy:
    blueGreen:
      activeService: payment-service-active
      previewService: payment-service-preview
      autoPromotionEnabled: false       # Manual promotion required
      scaleDownDelaySeconds: 600        # Keep blue alive 10 min
      prePromotionAnalysis:
        templates:
        - templateName: error-rate-check
        args:
        - name: service-name
          value: payment-service-preview

# Kubernetes Canary with Argo Rollouts
spec:
  strategy:
    canary:
      steps:
      - setWeight: 5          # 5% traffic to canary
      - pause: {duration: 5m} # Wait 5 minutes
      - analysis:             # Automated analysis
          templates:
          - templateName: canary-analysis
      - setWeight: 20         # 20% if analysis passes
      - pause: {duration: 10m}
      - setWeight: 50
      - pause: {duration: 10m}
      - setWeight: 100        # Full promotion
      canaryService: payment-service-canary
      stableService: payment-service-stable
      trafficRouting:
        istio:
          virtualService:
            name: payment-service-vsvc

# Falco rule: detect container trying to write to /etc
- rule: Write to /etc in container
  desc: Attempt to write to /etc directory in a container
  condition: >
    container and
    open_write and
    fd.name startswith /etc
  output: "File opened for writing under /etc (%user.name %proc.name %fd.name)"
  priority: ERROR

# SLO definition (YAML for Pyrra / Sloth)
apiVersion: pyrra.dev/v1alpha1
kind: ServiceLevelObjective
metadata:
  name: payment-service-availability
spec:
  target: "99.9"          # 99.9% success rate
  window: 30d             # Measured over 30 days
  indicator:
    ratio:
      errors:
        metric: http_requests_total{job="payment-service",status=~"5.."}
      total:
        metric: http_requests_total{job="payment-service"}

  # Alert when we're burning error budget too fast
  alerting:
    burnRateAlerts:
    - short: 5m
      long: 1h
      burnRate: 14.4      # 1h burn: page immediately
      severity: critical
    - short: 30m
      long: 6h
      burnRate: 6         # 6h burn: warn the team
      severity: warning

# .github/workflows/ci-cd.yml
name: CI/CD Pipeline

on:
  push:
    branches: [main]
  pull_request:
    branches: [main]

env:
  REGISTRY: harbor.mycompany.com
  IMAGE_NAME: payment-service
  PYTHON_VERSION: "3.12"

jobs:
  # ─── Stage 1: Code Quality ──────────────────────────────────────
  code-quality:
    name: Code Quality
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: ${{ env.PYTHON_VERSION }}

      - name: Install dependencies
        run: pip install poetry && poetry install

      - name: Lint and format check
        run: |
          poetry run black --check .
          poetry run ruff check .

      - name: SAST — Semgrep
        uses: semgrep/semgrep-action@v1
        with:
          config: p/python p/owasp-top-ten

      - name: SonarQube scan
        uses: SonarSource/sonarqube-scan-action@v2
        env:
          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
          SONAR_HOST_URL: ${{ secrets.SONAR_HOST_URL }}

  # ─── Stage 2: Unit Tests ────────────────────────────────────────
  unit-tests:
    name: Unit Tests
    runs-on: ubuntu-latest
    needs: code-quality
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: ${{ env.PYTHON_VERSION }}

      - name: Install and test
        run: |
          pip install poetry && poetry install
          poetry run pytest tests/unit \
            --cov=src \
            --cov-report=xml \
            --cov-fail-under=80 \
            -v

      - name: Upload coverage
        uses: codecov/codecov-action@v4
        with:
          token: ${{ secrets.CODECOV_TOKEN }}

  # ─── Stage 3: Integration Tests ─────────────────────────────────
  integration-tests:
    name: Integration Tests
    runs-on: ubuntu-latest
    needs: unit-tests
    services:
      postgres:
        image: postgres:16
        env:
          POSTGRES_PASSWORD: testpass
          POSTGRES_DB: testdb
        options: >-
          --health-cmd pg_isready
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5
      redis:
        image: redis:7
        options: >-
          --health-cmd "redis-cli ping"
          --health-interval 10s

    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: ${{ env.PYTHON_VERSION }}

      - name: Run integration tests
        env:
          DATABASE_URL: postgresql://postgres:testpass@localhost/testdb
          REDIS_URL: redis://localhost:6379
        run: |
          pip install poetry && poetry install
          poetry run pytest tests/integration -v

  # ─── Stage 4: Build and Scan Image ──────────────────────────────
  build-image:
    name: Build & Scan Image
    runs-on: ubuntu-latest
    needs: integration-tests
    if: github.event_name == 'push'
    outputs:
      image-digest: ${{ steps.build.outputs.digest }}
    steps:
      - uses: actions/checkout@v4

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3

      - name: Login to registry
        uses: docker/login-action@v3
        with:
          registry: ${{ env.REGISTRY }}
          username: ${{ secrets.REGISTRY_USER }}
          password: ${{ secrets.REGISTRY_PASSWORD }}

      - name: Build image
        id: build
        uses: docker/build-push-action@v5
        with:
          context: .
          push: false
          load: true
          tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ github.sha }}
          cache-from: type=gha
          cache-to: type=gha,mode=max

      - name: Generate SBOM
        uses: anchore/sbom-action@v0
        with:
          image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ github.sha }}
          format: cyclonedx-json
          output-file: sbom.json

      - name: Scan image — Trivy
        uses: aquasecurity/trivy-action@master
        with:
          image-ref: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ github.sha }}
          format: table
          exit-code: 1
          severity: HIGH,CRITICAL

      - name: Push image
        if: success()
        run: |
          docker push ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ github.sha }}

      - name: Sign image
        uses: sigstore/cosign-installer@v3
        run: |
          cosign sign --key env://COSIGN_PRIVATE_KEY \
            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ github.sha }}
        env:
          COSIGN_PRIVATE_KEY: ${{ secrets.COSIGN_PRIVATE_KEY }}

  # ─── Stage 5: Deploy to Staging ─────────────────────────────────
  deploy-staging:
    name: Deploy to Staging
    runs-on: ubuntu-latest
    needs: build-image
    environment: staging
    steps:
      - uses: actions/checkout@v4

      - name: Update image tag in GitOps repo
        run: |
          git clone https://x-token:${{ secrets.GITOPS_TOKEN }}@github.com/myorg/gitops-repo.git
          cd gitops-repo
          yq e '.spec.template.spec.containers[0].image = "${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ github.sha }}"' \
            -i apps/payment-service/staging/deployment.yaml
          git commit -am "chore: deploy payment-service ${{ github.sha }} to staging"
          git push

      - name: Wait for ArgoCD sync
        run: |
          argocd app wait payment-service-staging \
            --timeout 300 \
            --health

      - name: Run smoke tests
        run: |
          poetry run pytest tests/smoke \
            --base-url=https://payment-staging.mycompany.com

  # ─── Stage 6: Deploy to Production ──────────────────────────────
  deploy-production:
    name: Deploy to Production
    runs-on: ubuntu-latest
    needs: deploy-staging
    environment: production
    steps:
      - uses: actions/checkout@v4

      - name: Update image tag in GitOps repo (production)
        run: |
          git clone https://x-token:${{ secrets.GITOPS_TOKEN }}@github.com/myorg/gitops-repo.git
          cd gitops-repo
          yq e '.spec.template.spec.containers[0].image = "${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ github.sha }}"' \
            -i apps/payment-service/production/deployment.yaml
          git commit -am "chore: deploy payment-service ${{ github.sha }} to production"
          git push

      - name: Create GitHub release
        uses: semantic-release/semantic-release@v23

      - name: Notify team
        uses: slackapi/slack-github-action@v1
        with:
          payload: |
            {
              "text": "✅ payment-service deployed to production",
              "blocks": [
                {
                  "type": "section",
                  "text": {
                    "type": "mrkdwn",
                    "text": "*payment-service* deployed to production\nCommit: ${{ github.sha }}\nTriggered by: ${{ github.actor }}"
                  }
                }
              ]
            }
        env:
          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK }}