Chapter 40: CI/CD & Deployment Strategies

# Kubernetes blue-green deployment
apiVersion: v1
kind: Service
metadata:
  name: myapp-service
spec:
  selector:
    app: myapp
    version: blue  # Switch to "green" to cut over
  ports:
    - port: 80
      targetPort: 8080

---
# Blue deployment
apiVersion: apps/v1
kind: Deployment
metadata:
  name: myapp-blue
spec:
  replicas: 3
  selector:
    matchLabels:
      app: myapp
      version: blue
  template:
    metadata:
      labels:
        app: myapp
        version: blue
    spec:
      containers:
      - name: myapp
        image: myapp:1.0.0
        ports:
        - containerPort: 8080

---
# Green deployment
apiVersion: apps/v1
kind: Deployment
metadata:
  name: myapp-green
spec:
  replicas: 3
  selector:
    matchLabels:
      app: myapp
      version: green
  template:
    metadata:
      labels:
        app: myapp
        version: green
    spec:
      containers:
      - name: myapp
        image: myapp:1.1.0
        ports:
        - containerPort: 8080

#!/bin/bash
# Blue-green deployment automation

CURRENT_ENV=$(kubectl get service myapp-service -o jsonpath='{.spec.selector.version}')
NEW_ENV=$([ "$CURRENT_ENV" = "blue" ] && echo "green" || echo "blue")

echo "Current environment: $CURRENT_ENV"
echo "Deploying to: $NEW_ENV"

# Deploy new version to idle environment
kubectl set image deployment/myapp-${NEW_ENV} \
  myapp=myapp:${NEW_VERSION}

# Wait for rollout
kubectl rollout status deployment/myapp-${NEW_ENV}

# Run smoke tests against new environment
echo "Running smoke tests..."
NEW_POD_IP=$(kubectl get pods -l version=${NEW_ENV} -o jsonpath='{.items[0].status.podIP}')
curl -f http://${NEW_POD_IP}:8080/health || exit 1
curl -f http://${NEW_POD_IP}:8080/api/test || exit 1

# Switch traffic
echo "Switching traffic to $NEW_ENV"
kubectl patch service myapp-service -p '{"spec":{"selector":{"version":"'${NEW_ENV}'"}}}'

# Monitor for 5 minutes
echo "Monitoring new environment..."
sleep 300

# Check error rates
ERROR_RATE=$(curl -s http://prometheus:9090/api/v1/query?query=rate\(http_errors_total\[5m\]\) | jq '.data.result[0].value[1]')
if (( $(echo "$ERROR_RATE > 0.01" | bc -l) )); then
  echo "Error rate too high! Rolling back..."
  kubectl patch service myapp-service -p '{"spec":{"selector":{"version":"'${CURRENT_ENV}'"}}}'
  exit 1
fi

echo "Deployment successful! Old environment ($CURRENT_ENV) is now idle."

# Virtual Service for canary routing
apiVersion: networking.istio.io/v1beta1
kind: VirtualService
metadata:
  name: myapp
spec:
  hosts:
  - myapp.example.com
  http:
  - match:
    - headers:
        canary:
          exact: "true"
    route:
    - destination:
        host: myapp
        subset: v2
  - route:
    - destination:
        host: myapp
        subset: v1
      weight: 95
    - destination:
        host: myapp
        subset: v2
      weight: 5  # Start with 5% canary traffic

---
# Destination Rule defining subsets
apiVersion: networking.istio.io/v1beta1
kind: DestinationRule
metadata:
  name: myapp
spec:
  host: myapp
  subsets:
  - name: v1
    labels:
      version: v1
  - name: v2
    labels:
      version: v2

interface CanaryMetrics {
  errorRate: number;
  latencyP99: number;
  successRate: number;
}

interface CanaryConfig {
  stages: { weight: number; duration: number }[];
  thresholds: {
    maxErrorRate: number;
    maxLatency: number;
    minSuccessRate: number;
  };
}

class CanaryDeployment {
  async executeCanary(config: CanaryConfig): Promise {
    for (const stage of config.stages) {
      console.log(`Shifting ${stage.weight}% traffic to canary`);
      await this.updateTrafficWeight(stage.weight);

      // Wait for metrics to stabilize
      await this.sleep(stage.duration);

      // Collect metrics from both versions
      const baselineMetrics = await this.collectMetrics('v1');
      const canaryMetrics = await this.collectMetrics('v2');

      // Compare metrics
      const passed = this.analyzeMetrics(
        baselineMetrics,
        canaryMetrics,
        config.thresholds
      );

      if (!passed) {
        console.error('Canary analysis failed! Rolling back...');
        await this.rollback();
        return false;
      }

      console.log(`Stage ${stage.weight}% passed`);
    }

    console.log('Canary deployment successful!');
    return true;
  }

  private analyzeMetrics(
    baseline: CanaryMetrics,
    canary: CanaryMetrics,
    thresholds: CanaryConfig['thresholds']
  ): boolean {
    // Absolute thresholds
    if (canary.errorRate > thresholds.maxErrorRate) {
      console.error(`Error rate ${canary.errorRate} exceeds threshold`);
      return false;
    }

    if (canary.latencyP99 > thresholds.maxLatency) {
      console.error(`P99 latency ${canary.latencyP99}ms exceeds threshold`);
      return false;
    }

    if (canary.successRate < thresholds.minSuccessRate) {
      console.error(`Success rate ${canary.successRate} below threshold`);
      return false;
    }

    // Relative comparison (canary shouldn't be significantly worse)
    const errorRateIncrease =
      (canary.errorRate - baseline.errorRate) / baseline.errorRate;
    if (errorRateIncrease > 0.2) {  // 20% increase
      console.error('Error rate increased by more than 20%');
      return false;
    }

    const latencyIncrease =
      (canary.latencyP99 - baseline.latencyP99) / baseline.latencyP99;
    if (latencyIncrease > 0.3) {  // 30% increase
      console.error('Latency increased by more than 30%');
      return false;
    }

    return true;
  }

  private async collectMetrics(version: string): Promise {
    // Query Prometheus for metrics
    const errorRate = await this.queryPrometheus(
      `rate(http_requests_total{version="${version}",code=~"5.."}[5m])`
    );

    const latencyP99 = await this.queryPrometheus(
      `histogram_quantile(0.99, rate(http_request_duration_seconds_bucket{version="${version}"}[5m]))`
    );

    const successRate = await this.queryPrometheus(
      `rate(http_requests_total{version="${version}",code=~"2.."}[5m])`
    );

    return { errorRate, latencyP99, successRate };
  }

  private async updateTrafficWeight(weight: number): Promise {
    // Update Istio VirtualService weights
    const vs = await this.k8s.getVirtualService('myapp');
    vs.spec.http[0].route[0].weight = 100 - weight;  // v1
    vs.spec.http[0].route[1].weight = weight;        // v2
    await this.k8s.updateVirtualService(vs);
  }

  private async rollback(): Promise {
    await this.updateTrafficWeight(0);  // Route all traffic to v1
  }
}

import { LDClient, LDContext } from 'launchdarkly-node-server-sdk';

interface FeatureFlagContext {
  userId: string;
  userEmail: string;
  country: string;
  plan: 'free' | 'pro' | 'enterprise';
  custom?: Record;
}

class FeatureFlagService {
  private client: LDClient;

  constructor(sdkKey: string) {
    this.client = LDClient.init(sdkKey);
  }

  async isEnabled(
    flagKey: string,
    context: FeatureFlagContext,
    defaultValue = false
  ): Promise {
    const ldContext: LDContext = {
      kind: 'user',
      key: context.userId,
      email: context.userEmail,
      country: context.country,
      plan: context.plan,
      ...context.custom,
    };

    return await this.client.variation(flagKey, ldContext, defaultValue);
  }

  async getVariation(
    flagKey: string,
    context: FeatureFlagContext,
    defaultValue: T
  ): Promise {
    const ldContext: LDContext = {
      kind: 'user',
      key: context.userId,
      email: context.userEmail,
      country: context.country,
      plan: context.plan,
      ...context.custom,
    };

    return await this.client.variation(flagKey, ldContext, defaultValue);
  }

  async getAllFlags(context: FeatureFlagContext): Promise> {
    const ldContext: LDContext = {
      kind: 'user',
      key: context.userId,
      email: context.userEmail,
      country: context.country,
      plan: context.plan,
      ...context.custom,
    };

    return await this.client.allFlagsState(ldContext);
  }

  async close(): Promise {
    await this.client.close();
  }
}

// Usage in application
const flagService = new FeatureFlagService(process.env.LAUNCHDARKLY_SDK_KEY!);

app.get('/api/search', async (req, res) => {
  const user = req.user;

  // Simple boolean flag
  const useNewSearch = await flagService.isEnabled(
    'new-search-algorithm',
    {
      userId: user.id,
      userEmail: user.email,
      country: user.country,
      plan: user.plan,
    }
  );

  if (useNewSearch) {
    return res.json(await newSearchAlgorithm(req.query));
  }

  return res.json(await legacySearch(req.query));
});

app.get('/api/recommendations', async (req, res) => {
  const user = req.user;

  // Multi-variant flag for A/B testing
  const algorithm = await flagService.getVariation(
    'recommendation-algorithm',
    {
      userId: user.id,
      userEmail: user.email,
      country: user.country,
      plan: user.plan,
    },
    'control'
  );

  switch (algorithm) {
    case 'collaborative-filtering':
      return res.json(await collaborativeFiltering(user.id));
    case 'content-based':
      return res.json(await contentBased(user.id));
    case 'hybrid':
      return res.json(await hybridRecommendations(user.id));
    default:
      return res.json(await legacyRecommendations(user.id));
  }
});

{
  "flag": "new-checkout-flow",
  "targeting": [
    {
      "variation": "enabled",
      "values": ["internal-employee-segment"]
    },
    {
      "variation": "enabled",
      "values": ["beta-tester-segment"]
    },
    {
      "variation": "enabled",
      "clauses": [
        {
          "attribute": "email",
          "op": "endsWith",
          "values": ["@company.com"]
        }
      ]
    },
    {
      "variation": "enabled",
      "clauses": [
        {
          "attribute": "plan",
          "op": "in",
          "values": ["enterprise"]
        }
      ]
    }
  ],
  "rollout": {
    "variations": [
      { "variation": "disabled", "weight": 95000 },
      { "variation": "enabled", "weight": 5000 }
    ]
  },
  "offVariation": "disabled"
}

apiVersion: argoproj.io/v1alpha1
kind: Rollout
metadata:
  name: myapp
spec:
  replicas: 5
  strategy:
    canary:
      # Canary service (for analysis)
      canaryService: myapp-canary
      # Stable service (production traffic)
      stableService: myapp-stable
      # Traffic routing
      trafficRouting:
        istio:
          virtualService:
            name: myapp
            routes:
            - primary
      # Progressive stages
      steps:
      - setWeight: 5
      - pause: { duration: 5m }
      - analysis:
          templates:
          - templateName: success-rate
          - templateName: latency
      - setWeight: 10
      - pause: { duration: 5m }
      - analysis:
          templates:
          - templateName: success-rate
          - templateName: latency
      - setWeight: 25
      - pause: { duration: 10m }
      - analysis:
          templates:
          - templateName: success-rate
          - templateName: latency
      - setWeight: 50
      - pause: { duration: 10m }
      - analysis:
          templates:
          - templateName: success-rate
          - templateName: latency
      - setWeight: 75
      - pause: { duration: 10m }
      - analysis:
          templates:
          - templateName: success-rate
          - templateName: latency
  revisionHistoryLimit: 3
  selector:
    matchLabels:
      app: myapp
  template:
    metadata:
      labels:
        app: myapp
    spec:
      containers:
      - name: myapp
        image: myapp:v1.2.0
        ports:
        - containerPort: 8080

---
# Analysis Template: Success Rate
apiVersion: argoproj.io/v1alpha1
kind: AnalysisTemplate
metadata:
  name: success-rate
spec:
  metrics:
  - name: success-rate
    interval: 1m
    successCondition: result[0] >= 0.95
    failureLimit: 3
    provider:
      prometheus:
        address: http://prometheus:9090
        query: |
          sum(rate(http_requests_total{status=~"2.."}[5m]))
          /
          sum(rate(http_requests_total[5m]))

---
# Analysis Template: Latency
apiVersion: argoproj.io/v1alpha1
kind: AnalysisTemplate
metadata:
  name: latency
spec:
  metrics:
  - name: latency-p99
    interval: 1m
    successCondition: result[0] <= 500
    failureLimit: 3
    provider:
      prometheus:
        address: http://prometheus:9090
        query: |
          histogram_quantile(0.99,
            rate(http_request_duration_seconds_bucket[5m])
          ) * 1000

apiVersion: flagger.app/v1beta1
kind: Canary
metadata:
  name: myapp
spec:
  targetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: myapp
  # Service mesh provider
  provider: istio
  # Deployment strategy
  progressDeadlineSeconds: 600
  service:
    port: 8080
    targetPort: 8080
  # Canary analysis
  analysis:
    interval: 1m
    threshold: 10
    maxWeight: 50
    stepWeight: 5
    # Metrics
    metrics:
    - name: request-success-rate
      thresholdRange:
        min: 99
      interval: 1m
    - name: request-duration
      thresholdRange:
        max: 500
      interval: 1m
    # Webhooks for custom checks
    webhooks:
    - name: load-test
      url: http://loadtest-service/
      timeout: 5s
      metadata:
        type: cmd
        cmd: "hey -z 1m -q 10 -c 2 http://myapp-canary:8080/"
    - name: smoke-test
      url: http://smoke-test-service/
      timeout: 30s
  # Alerting
  alerts:
  - name: slack
    severity: error
    providerRef:
      name: slack
      namespace: flagger

apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
  name: myapp-production
  namespace: argocd
spec:
  # Source repository
  source:
    repoURL: https://github.com/company/k8s-manifests
    targetRevision: main
    path: apps/myapp/overlays/production
  # Destination cluster
  destination:
    server: https://kubernetes.default.svc
    namespace: production
  # Sync policy
  syncPolicy:
    automated:
      prune: true      # Delete resources not in Git
      selfHeal: true   # Revert manual changes
      allowEmpty: false
    syncOptions:
    - CreateNamespace=true
    retry:
      limit: 5
      backoff:
        duration: 5s
        factor: 2
        maxDuration: 3m
  # Health check
  ignoreDifferences:
  - group: apps
    kind: Deployment
    jsonPointers:
    - /spec/replicas  # Ignore HPA-managed replicas

#!/bin/bash
# GitOps deployment workflow

# 1. CI builds and pushes image
docker build -t myapp:${VERSION} .
docker push myapp:${VERSION}

# 2. Update manifest in Git
git clone https://github.com/company/k8s-manifests
cd k8s-manifests

# Update image tag in Kustomization
cd apps/myapp/overlays/production
kustomize edit set image myapp=myapp:${VERSION}

# Commit and push
git add .
git commit -m "chore: update myapp to ${VERSION}"
git push origin main

# 3. ArgoCD detects change and syncs automatically
# (or trigger manually)
argocd app sync myapp-production

# 4. Monitor rollout
argocd app wait myapp-production --health --timeout 600

# Istio VirtualService for traffic mirroring
apiVersion: networking.istio.io/v1beta1
kind: VirtualService
metadata:
  name: myapp
spec:
  hosts:
  - myapp.example.com
  http:
  - match:
    - uri:
        prefix: "/api"
    route:
    - destination:
        host: myapp
        subset: stable
      weight: 100
    mirror:
      host: myapp
      subset: canary
    mirrorPercentage:
      value: 100  # Mirror 100% of traffic

// Continuous synthetic tests against production
import { chromium } from 'playwright';

class SyntheticMonitor {
  async runUserJourney(): Promise {
    const browser = await chromium.launch();
    const context = await browser.newContext();
    const page = await context.newPage();

    try {
      // Critical user journey
      await page.goto('https://app.example.com');
      await page.click('button:has-text("Sign In")');
      await page.fill('input[name="email"]', 'synthetic@example.com');
      await page.fill('input[name="password"]', process.env.SYNTHETIC_PASSWORD!);
      await page.click('button[type="submit"]');

      // Verify dashboard loaded
      await page.waitForSelector('.dashboard', { timeout: 5000 });

      // Perform key action
      await page.click('button:has-text("Create")');
      await page.fill('input[name="title"]', 'Synthetic Test');
      await page.click('button:has-text("Save")');

      // Verify success
      await page.waitForSelector('.success-message', { timeout: 5000 });

      this.reportSuccess('user-journey-complete');
    } catch (error) {
      this.reportFailure('user-journey-failed', error);
      throw error;
    } finally {
      await browser.close();
    }
  }

  private reportSuccess(metric: string): void {
    // Send to monitoring system
    this.cloudwatch.putMetricData({
      Namespace: 'SyntheticMonitoring',
      MetricData: [{
        MetricName: metric,
        Value: 1,
        Unit: 'Count',
        Timestamp: new Date(),
      }],
    });
  }

  private reportFailure(metric: string, error: any): void {
    this.cloudwatch.putMetricData({
      Namespace: 'SyntheticMonitoring',
      MetricData: [{
        MetricName: metric,
        Value: 1,
        Unit: 'Count',
        Timestamp: new Date(),
      }],
    });

    // Alert on-call
    this.pagerduty.trigger({
      summary: `Synthetic monitor failed: ${metric}`,
      severity: 'critical',
      details: { error: error.message, stack: error.stack },
    });
  }
}

// Run every 5 minutes
setInterval(() => new SyntheticMonitor().runUserJourney(), 5 * 60 * 1000);

# Chaos Mesh experiment
apiVersion: chaos-mesh.org/v1alpha1
kind: PodChaos
metadata:
  name: pod-failure-experiment
spec:
  action: pod-failure
  mode: one
  duration: "30s"
  selector:
    namespaces:
    - production
    labelSelectors:
      app: myapp
  scheduler:
    cron: "0 2 * * *"  # Run at 2 AM daily

---
# Network delay experiment
apiVersion: chaos-mesh.org/v1alpha1
kind: NetworkChaos
metadata:
  name: network-delay-experiment
spec:
  action: delay
  mode: one
  selector:
    namespaces:
    - production
    labelSelectors:
      app: myapp
  delay:
    latency: "200ms"
    correlation: "100"
    jitter: "50ms"
  duration: "5m"

#!/bin/bash
# Automated rollback script

DEPLOYMENT=$1
NAMESPACE=${2:-production}

echo "Initiating rollback for $DEPLOYMENT in $NAMESPACE"

# Get current and previous revisions
CURRENT_REVISION=$(kubectl rollout history deployment/$DEPLOYMENT -n $NAMESPACE | tail -n 1 | awk '{print $1}')
PREVIOUS_REVISION=$((CURRENT_REVISION - 1))

echo "Rolling back from revision $CURRENT_REVISION to $PREVIOUS_REVISION"

# Execute rollback
kubectl rollout undo deployment/$DEPLOYMENT -n $NAMESPACE --to-revision=$PREVIOUS_REVISION

# Wait for rollback to complete
kubectl rollout status deployment/$DEPLOYMENT -n $NAMESPACE --timeout=300s

if [ $? -eq 0 ]; then
  echo "Rollback successful"

  # Verify health
  READY_REPLICAS=$(kubectl get deployment $DEPLOYMENT -n $NAMESPACE -o jsonpath='{.status.readyReplicas}')
  DESIRED_REPLICAS=$(kubectl get deployment $DEPLOYMENT -n $NAMESPACE -o jsonpath='{.spec.replicas}')

  if [ "$READY_REPLICAS" = "$DESIRED_REPLICAS" ]; then
    echo "All replicas healthy: $READY_REPLICAS/$DESIRED_REPLICAS"
  else
    echo "WARNING: Only $READY_REPLICAS/$DESIRED_REPLICAS replicas ready"
    exit 1
  fi

  # Notify team
  curl -X POST https://hooks.slack.com/services/XXX \
    -d "{\"text\":\"✅ Rollback successful: $DEPLOYMENT to revision $PREVIOUS_REVISION\"}"
else
  echo "Rollback failed!"
  curl -X POST https://hooks.slack.com/services/XXX \
    -d "{\"text\":\"❌ Rollback FAILED: $DEPLOYMENT in $NAMESPACE\"}"
  exit 1
fi

-- Phase 1: Add new column (backward compatible)
-- Old code ignores it, new code writes to it
ALTER TABLE users ADD COLUMN email_verified_at TIMESTAMP NULL;

-- Phase 2: Backfill data (background job)
-- UPDATE users SET email_verified_at = verified_at WHERE email_verified = true;

-- Phase 3: Deploy new code that reads from email_verified_at

-- Phase 4: After rollback window (e.g., 7 days), remove old column
-- ALTER TABLE users DROP COLUMN email_verified;