Engineering Standards for DevOps: The Complete Guide¶

# Feature flag config per environment
# development/.env
FF_NEW_CHECKOUT_FLOW=true
FF_AI_RECOMMENDATIONS=true

# staging/.env
FF_NEW_CHECKOUT_FLOW=true
FF_AI_RECOMMENDATIONS=false

# production/.env
FF_NEW_CHECKOUT_FLOW=false   # flip to true when ready to release
FF_AI_RECOMMENDATIONS=false

# .github/workflows/ci-cd.yml
name: CI/CD Pipeline

on:
  push:
    branches: [main]
  pull_request:
    branches: [main]

env:
  REGISTRY: ghcr.io
  IMAGE_NAME: ${{ github.repository }}
  PYTHON_VERSION: "3.12"

jobs:
  # ── STAGE 1: Code Quality ─────────────────────────────────────
  quality:
    name: Code Quality
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - uses: actions/setup-python@v5
        with:
          python-version: ${{ env.PYTHON_VERSION }}
          cache: pip

      - name: Install dependencies
        run: pip install -r requirements-dev.txt

      - name: Lint (ruff)
        run: ruff check . --output-format=github

      - name: Format check (black)
        run: black --check .

      - name: Type check (mypy)
        run: mypy src/

      - name: Detect secrets
        uses: reviewdog/action-detect-secrets@v0.21.0

  # ── STAGE 2: Tests ────────────────────────────────────────────
  test:
    name: Tests
    runs-on: ubuntu-latest
    needs: quality
    services:
      postgres:
        image: postgres:16-alpine
        env:
          POSTGRES_DB: test_db
          POSTGRES_USER: test
          POSTGRES_PASSWORD: test
        options: >-
          --health-cmd pg_isready
          --health-interval 10s
        ports:
          - 5432:5432
      redis:
        image: redis:7-alpine
        ports:
          - 6379:6379

    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: ${{ env.PYTHON_VERSION }}
          cache: pip

      - name: Install dependencies
        run: pip install -r requirements-dev.txt

      - name: Run unit tests with coverage
        run: |
          pytest tests/unit/ \
            --cov=src \
            --cov-report=xml \
            --cov-report=term-missing \
            --cov-fail-under=80 \
            -v
        env:
          DATABASE_URL: postgresql://test:test@localhost:5432/test_db
          REDIS_URL: redis://localhost:6379/0

      - name: Run integration tests
        run: pytest tests/integration/ -v
        env:
          DATABASE_URL: postgresql://test:test@localhost:5432/test_db

      - name: Upload coverage to Codecov
        uses: codecov/codecov-action@v4
        with:
          files: coverage.xml
          fail_ci_if_error: true

  # ── STAGE 3: Security Scan ────────────────────────────────────
  security:
    name: Security Scan
    runs-on: ubuntu-latest
    needs: quality
    steps:
      - uses: actions/checkout@v4

      - name: SAST — Semgrep
        uses: semgrep/semgrep-action@v1
        with:
          config: >-
            p/python
            p/owasp-top-ten
            p/secrets
        env:
          SEMGREP_APP_TOKEN: ${{ secrets.SEMGREP_APP_TOKEN }}

      - name: Dependency scan — pip-audit
        run: |
          pip install pip-audit
          pip-audit --requirement requirements.txt \
                    --format json \
                    --output pip-audit-report.json
          pip-audit --requirement requirements.txt  # fail on high CVEs

      - name: Container scan — Trivy
        uses: aquasecurity/trivy-action@master
        with:
          scan-type: fs
          security-checks: vuln,secret
          severity: HIGH,CRITICAL
          exit-code: 1

  # ── STAGE 4: Build & Push Image ──────────────────────────────
  build:
    name: Build Container Image
    runs-on: ubuntu-latest
    needs: [test, security]
    permissions:
      contents: read
      packages: write
    outputs:
      image_tag: ${{ steps.meta.outputs.tags }}
      image_digest: ${{ steps.push.outputs.digest }}

    steps:
      - uses: actions/checkout@v4

      - name: Log in to Container Registry
        uses: docker/login-action@v3
        with:
          registry: ${{ env.REGISTRY }}
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Extract Docker metadata
        id: meta
        uses: docker/metadata-action@v5
        with:
          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
          tags: |
            type=sha,prefix=,format=long
            type=ref,event=branch
            type=semver,pattern={{version}}

      - name: Build and push
        id: push
        uses: docker/build-push-action@v5
        with:
          context: .
          push: true
          tags: ${{ steps.meta.outputs.tags }}
          labels: ${{ steps.meta.outputs.labels }}
          cache-from: type=gha
          cache-to: type=gha,mode=max

  # ── STAGE 5: Deploy to Staging ────────────────────────────────
  deploy-staging:
    name: Deploy to Staging
    runs-on: ubuntu-latest
    needs: build
    if: github.ref == 'refs/heads/main'
    environment: staging

    steps:
      - name: Deploy to Kubernetes (staging)
        run: |
          kubectl set image deployment/order-service \
            order-service=${{ needs.build.outputs.image_tag }} \
            --namespace=staging
          kubectl rollout status deployment/order-service \
            --namespace=staging --timeout=5m

      - name: Run smoke tests against staging
        run: |
          ./scripts/smoke-test.sh ${{ vars.STAGING_URL }}

  # ── STAGE 6: Deploy to Production ────────────────────────────
  deploy-production:
    name: Deploy to Production
    runs-on: ubuntu-latest
    needs: deploy-staging
    environment: production   # requires manual approval

    steps:
      - name: Deploy canary (5%)
        run: kubectl apply -f k8s/canary-5pct.yaml

      - name: Monitor canary (5 minutes)
        run: ./scripts/monitor-canary.sh --duration=300 --error-threshold=1

      - name: Promote to full rollout
        run: |
          kubectl set image deployment/order-service \
            order-service=${{ needs.build.outputs.image_tag }} \
            --namespace=production
          kubectl rollout status deployment/order-service \
            --namespace=production --timeout=10m

      - name: Post-deploy smoke tests
        run: ./scripts/smoke-test.sh ${{ vars.PRODUCTION_URL }}

# Semgrep — SAST in CI (.github/workflows/security.yml)
- name: SAST with Semgrep
  uses: semgrep/semgrep-action@v1
  with:
    config: >-
      p/python            # Python-specific rules
      p/owasp-top-ten     # OWASP Top 10 checks
      p/secrets           # Hardcoded secrets
      p/sql-injection     # SQL injection patterns
      p/xss               # Cross-site scripting

# SonarQube — code quality + security in one
sonar.projectKey=order-service
sonar.sources=src
sonar.tests=tests
sonar.python.coverage.reportPaths=coverage.xml
sonar.qualitygate.wait=true

# Quality Gate blocks deployment if:
# - Coverage < 80%
# - Duplicated code > 3%
# - Security Rating < A
# - Reliability Rating < A
# - New bugs introduced > 0

# OWASP ZAP in CI pipeline (against staging environment)
- name: DAST with OWASP ZAP
  uses: zaproxy/action-full-scan@v0.10.0
  with:
    target: ${{ vars.STAGING_URL }}
    rules_file_name: .zap/rules.tsv
    cmd_options: '-a -j'      # ajax spider, JSON output
    fail_action: true          # fail CI on medium+ risk findings

# GitHub Dependabot — automatic dependency updates
# .github/dependabot.yml
version: 2
updates:
  - package-ecosystem: pip
    directory: /
    schedule:
      interval: weekly
    open-pull-requests-limit: 10
    labels:
      - dependencies
      - security
    ignore:
      - dependency-name: "*"
        update-types: ["version-update:semver-major"]   # manual for major bumps

  - package-ecosystem: docker
    directory: /
    schedule:
      interval: weekly

  - package-ecosystem: github-actions
    directory: /
    schedule:
      interval: weekly

# detect-secrets — prevent secrets before commit
# .pre-commit-config.yaml
- repo: https://github.com/Yelp/detect-secrets
  rev: v1.5.0
  hooks:
    - id: detect-secrets
      args: ['--baseline', '.secrets.baseline']

# GitHub Advanced Security — secret scanning on push
# (enabled in repo settings)
# Alerts on: AWS keys, Stripe keys, GitHub tokens, 200+ pattern types
# Can block push if secret detected (push protection)

# Trivy — container vulnerability scanning
- name: Scan container image
  uses: aquasecurity/trivy-action@master
  with:
    image-ref: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ github.sha }}
    format: sarif
    output: trivy-results.sarif
    severity: HIGH,CRITICAL
    exit-code: 1    # fail pipeline on HIGH/CRITICAL CVEs

# .github/workflows/release.yml
- name: Semantic Release
  uses: cycjimmy/semantic-release-action@v4
  env:
    GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
  # On main merge:
  # feat: → bump MINOR, tag v1.2.0
  # fix:  → bump PATCH, tag v1.1.1
  # BREAKING CHANGE → bump MAJOR, tag v2.0.0

# Grafana dashboard panels (minimum per service):
panels:
  - title: "Request Rate (req/s)"
    query: rate(http_requests_total[5m])

  - title: "Error Rate (%)"
    query: |
      sum(rate(http_requests_total{status_code=~"5.."}[5m]))
      / sum(rate(http_requests_total[5m])) * 100

  - title: "Latency p99 (ms)"
    query: |
      histogram_quantile(0.99,
        rate(http_request_duration_seconds_bucket[5m])) * 1000

  - title: "Active Orders"
    query: orders_active_total

  - title: "Error Types"
    query: rate(order_processing_errors_total[5m])

# SLO definitions per service
service: order-service
slos:
  - name: availability
    description: "Orders API is available"
    target: 99.9%             # 8.7 hours downtime/year
    measurement: |
      1 - (rate(http_requests_total{status_code=~"5.."}[30d])
           / rate(http_requests_total[30d]))

  - name: latency
    description: "Order placement completes in < 2 seconds"
    target: 95%               # 95% of requests under 2s
    measurement: |
      histogram_quantile(0.95, rate(http_request_duration_seconds_bucket[30d])) < 2

  - name: correctness
    description: "Orders are processed without errors"
    target: 99.5%
    measurement: |
      1 - (rate(order_processing_errors_total[30d])
           / rate(orders_placed_total[30d]))

# k8s/deployment.yaml — complete standard template
apiVersion: apps/v1
kind: Deployment
metadata:
  name: order-service
  namespace: production
  labels:
    app: order-service
    version: "1.2.0"
    team: platform
spec:
  replicas: 3                   # minimum 3 for HA in production
  strategy:
    type: RollingUpdate
    rollingUpdate:
      maxSurge: 1               # allow 1 extra pod during update
      maxUnavailable: 0         # never reduce below desired count
  selector:
    matchLabels:
      app: order-service
  template:
    metadata:
      labels:
        app: order-service
        version: "1.2.0"
      annotations:
        prometheus.io/scrape: "true"
        prometheus.io/port: "8000"
        prometheus.io/path: "/metrics"
    spec:
      # Security context — run as non-root
      securityContext:
        runAsNonRoot: true
        runAsUser: 1001
        fsGroup: 1001

      # Terminate gracefully
      terminationGracePeriodSeconds: 30

      containers:
      - name: order-service
        image: ghcr.io/mycompany/order-service:sha-abc1234  # exact SHA, not 'latest'
        ports:
        - containerPort: 8000

        # Resource limits — ALWAYS set these
        resources:
          requests:             # guaranteed allocation
            memory: "256Mi"
            cpu: "100m"
          limits:               # maximum allocation
            memory: "512Mi"
            cpu: "500m"

        # Environment from ConfigMap and Secrets
        envFrom:
        - configMapRef:
            name: order-service-config
        - secretRef:
            name: order-service-secrets
        env:
        - name: POD_NAME
          valueFrom:
            fieldRef:
              fieldPath: metadata.name
        - name: POD_NAMESPACE
          valueFrom:
            fieldRef:
              fieldPath: metadata.namespace

        # Health probes — all three, always
        startupProbe:
          httpGet:
            path: /health/live
            port: 8000
          failureThreshold: 30
          periodSeconds: 10

        livenessProbe:
          httpGet:
            path: /health/live
            port: 8000
          initialDelaySeconds: 0
          periodSeconds: 10
          failureThreshold: 3

        readinessProbe:
          httpGet:
            path: /health/ready
            port: 8000
          periodSeconds: 5
          failureThreshold: 2

        # Graceful shutdown
        lifecycle:
          preStop:
            exec:
              command: ["/bin/sh", "-c", "sleep 5"]

        # Read-only filesystem where possible
        securityContext:
          allowPrivilegeEscalation: false
          readOnlyRootFilesystem: true
          capabilities:
            drop: ["ALL"]

        volumeMounts:
        - name: tmp
          mountPath: /tmp       # writable tmp if needed

      volumes:
      - name: tmp
        emptyDir: {}

      # Pod anti-affinity — spread across nodes
      affinity:
        podAntiAffinity:
          preferredDuringSchedulingIgnoredDuringExecution:
          - weight: 100
            podAffinityTerm:
              labelSelector:
                matchExpressions:
                - key: app
                  operator: In
                  values:
                  - order-service
              topologyKey: kubernetes.io/hostname

# k8s/hpa.yaml — autoscaling
apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
  name: order-service-hpa
spec:
  scaleTargetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: order-service
  minReplicas: 3
  maxReplicas: 20
  metrics:
  - type: Resource
    resource:
      name: cpu
      target:
        type: Utilization
        averageUtilization: 70
  - type: Resource
    resource:
      name: memory
      target:
        type: Utilization
        averageUtilization: 80

# argocd/order-service-app.yaml
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
  name: order-service
  namespace: argocd
spec:
  project: production
  source:
    repoURL: https://github.com/mycompany/k8s-manifests
    targetRevision: main
    path: services/order-service/production
  destination:
    server: https://kubernetes.default.svc
    namespace: production
  syncPolicy:
    automated:
      prune: true         # remove resources deleted from git
      selfHeal: true      # revert manual changes in cluster
    syncOptions:
    - CreateNamespace=true