Cilium vs. Calico vs. Flannel: 2025 CNI Benchmark

# Cilium ConfigMap with advanced features
apiVersion: v1
kind: ConfigMap
metadata:
  name: cilium-config
  namespace: kube-system
data:
  # eBPF-based kube-proxy replacement
  kube-proxy-replacement: "true"
  enable-ipv4: "true"
  enable-ipv6: "false"
  
  # Advanced load balancing
  enable-session-affinity: "true"
  enable-host-legacy-routing: "false"
  enable-local-redirect-policy: "true"
  
  # Security and observability
  enable-hubble: "true"
  hubble-listen-address: ":4244"
  enable-policy: "default"
  policy-enforcement-mode: "default"
  
  # Performance optimizations
  enable-bandwidth-manager: "true"
  enable-bbr: "true"
  bpf-lb-acceleration: "native"
  
  # Cluster mesh for multi-cluster
  cluster-name: "production-us-east"
  cluster-id: "1"

# Calico Installation with Typha
apiVersion: operator.tigera.io/v1
kind: Installation
metadata:
  name: default
spec:
  calicoNetwork:
    ipPools:
    - blockSize: 26
      cidr: 192.168.0.0/16
      encapsulation: VXLANCrossSubnet
      natOutgoing: Enabled
      nodeSelector: all()
    nodeAddressAutodetectionV4:
      interface: "eth0"
    linuxDataplane: Iptables
    hostPorts: Enabled
  typhaMetricsPort: 9093
  nodeMetricsPort: 9091
---
# Calico configuration
apiVersion: v1
kind: ConfigMap
metadata:
  name: calico-config
  namespace: kube-system
data:
  # BGP configuration
  calico_backend: "bird"
  cluster_type: "k8s,bgp"
  
  # IP-in-IP and VXLAN
  calico_ipv4pool_ipip: "CrossSubnet"
  calico_ipv4pool_vxlan: "Never"
  
  # Typha for large clusters
  typha_service_name: "calico-typha"
  
  # Felix configuration
  felix_ipinipmtu: "1440"
  felix_vxlanmtu: "1410"
  felix_wireguardmtu: "1420"

# Flannel DaemonSet configuration
apiVersion: v1
kind: ConfigMap
metadata:
  name: kube-flannel-cfg
  namespace: kube-flannel
data:
  cni-conf.json: |
    {
      "name": "cbr0",
      "cniVersion": "0.3.1",
      "plugins": [
        {
          "type": "flannel",
          "delegate": {
            "hairpinMode": true,
            "isDefaultGateway": true
          }
        },
        {
          "type": "portmap",
          "capabilities": {
            "portMappings": true
          }
        }
      ]
    }
  net-conf.json: |
    {
      "Network": "10.244.0.0/16",
      "Backend": {
        "Type": "vxlan",
        "Port": 8472,
        "VNI": 1,
        "DirectRouting": true
      }
    }

# Cilium Layer 7 HTTP policy
apiVersion: cilium.io/v2
kind: CiliumNetworkPolicy
metadata:
  name: frontend-policy
spec:
  endpointSelector:
    matchLabels:
      app: frontend
  egress:
  - toEndpoints:
    - matchLabels:
        app: backend
    toPorts:
    - ports:
      - port: "8080"
        protocol: TCP
      rules:
        http:
        - method: "GET"
          path: "/api/v1/.*"
        - method: "POST"
          path: "/api/v1/orders"
          headers:
          - "Authorization: Bearer.*"
---
# Cilium DNS policy
apiVersion: cilium.io/v2
kind: CiliumNetworkPolicy
metadata:
  name: dns-policy
spec:
  endpointSelector:
    matchLabels:
      app: web-app
  egress:
  - toEndpoints:
    - matchLabels:
        k8s:io.kubernetes.pod.namespace: kube-system
        k8s:k8s-app: kube-dns
    toPorts:
    - ports:
      - port: "53"
        protocol: UDP
      rules:
        dns:
        - matchPattern: "*.company.com"
        - matchName: "api.external.com"

# Calico GlobalNetworkPolicy for cluster-wide rules
apiVersion: projectcalico.org/v3
kind: GlobalNetworkPolicy
metadata:
  name: deny-all-non-system
spec:
  order: 1000
  selector: projectcalico.org/namespace != "kube-system"
  types:
  - Ingress
  - Egress
  egress:
  # Allow DNS
  - action: Allow
    protocol: UDP
    destination:
      selector: k8s-app == "kube-dns"
      ports:
      - 53
  # Allow access to Kubernetes API
  - action: Allow
    protocol: TCP
    destination:
      nets:
      - 10.96.0.1/32
      ports:
      - 443
---
# Calico application-specific policy
apiVersion: projectcalico.org/v3
kind: NetworkPolicy
metadata:
  name: backend-policy
  namespace: production
spec:
  selector: app == "backend"
  types:
  - Ingress
  - Egress
  ingress:
  - action: Allow
    source:
      selector: app == "frontend"
    protocol: TCP
    destination:
      ports:
      - 8080
  egress:
  - action: Allow
    protocol: TCP
    destination:
      selector: app == "database"
      ports:
      - 5432

# Cilium service mesh configuration
apiVersion: v1
kind: ConfigMap
metadata:
  name: cilium-config
data:
  enable-envoy-config: "true"
  enable-l7-proxy: "true"
  
  # Ingress controller
  enable-ingress-controller: "true"
  ingress-lb-annotation-prefixes: "service.beta.kubernetes.io,service.kubernetes.io,cloud.google.com"
  
  # Gateway API
  enable-gateway-api: "true"
  gateway-api-hostnetwork-enabled: "false"
---
# Cilium Ingress
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: cilium-ingress
  annotations:
    ingress.cilium.io/loadbalancer-mode: "shared"
spec:
  ingressClassName: cilium
  rules:
  - host: myapp.company.com
    http:
      paths:
      - path: /
        pathType: Prefix
        backend:
          service:
            name: myapp-service
            port:
              number: 80

# Cilium cluster mesh configuration
apiVersion: v1
kind: ConfigMap
metadata:
  name: cilium-config
data:
  cluster-name: "us-east-1"
  cluster-id: "1"
  enable-clustermesh: "true"
  clustermesh-config: "/var/lib/cilium/clustermesh/"
  
  # Cross-cluster service discovery
  enable-external-ips: "true"
  enable-cross-cluster-service: "true"
---
# External service for cross-cluster access
apiVersion: v1
kind: Service
metadata:
  name: cross-cluster-service
  annotations:
    io.cilium/global-service: "true"
spec:
  type: ClusterIP
  ports:
  - port: 80
    targetPort: 8080
  selector:
    app: backend

# Calico BGP configuration for multi-cluster
apiVersion: projectcalico.org/v3
kind: BGPConfiguration
metadata:
  name: default
spec:
  logSeverityScreen: Info
  nodeToNodeMeshEnabled: true
  asNumber: 65001
---
# BGP Peer for external connectivity
apiVersion: projectcalico.org/v3
kind: BGPPeer
metadata:
  name: rack-tor-switch
spec:
  peerIP: 192.168.1.1
  asNumber: 65000
  nodeSelector: rack == "rack-1"

# Hubble configuration for network observability
apiVersion: v1
kind: ConfigMap
metadata:
  name: hubble-config
data:
  enable-hubble: "true"
  hubble-listen-address: ":4244"
  hubble-metrics-server: ":9965"
  hubble-metrics: |
    dns:query;ignoreAAAA
    drop
    tcp
    flow
    port-distribution
    icmp
    http
---
# Hubble UI deployment
apiVersion: apps/v1
kind: Deployment
metadata:
  name: hubble-ui
spec:
  replicas: 1
  selector:
    matchLabels:
      k8s-app: hubble-ui
  template:
    metadata:
      labels:
        k8s-app: hubble-ui
    spec:
      containers:
      - name: frontend
        image: quay.io/cilium/hubble-ui:v0.12.0
        ports:
        - containerPort: 8081
        env:
        - name: HUBBLE_SERVICE
          value: "hubble-relay:80"

# Calico Felix metrics configuration
apiVersion: v1
kind: ConfigMap
metadata:
  name: calico-config
data:
  felix_prometheusmetricsenabled: "true"
  felix_prometheusmetricsport: "9091"
  felix_prometheusgometricsenabled: "true"
  
  # Typha metrics
  typha_prometheusmetricsenabled: "true"
  typha_prometheusmetricsport: "9093"
---
# ServiceMonitor for Prometheus
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: calico-metrics
spec:
  selector:
    matchLabels:
      k8s-app: calico-node
  endpoints:
  - port: calico-metrics-port
    interval: 30s
    path: /metrics

# Cilium status and connectivity check
cilium status --verbose
cilium connectivity test

# eBPF program inspection
cilium bpf lb list
cilium bpf ct list global
cilium bpf policy get <endpoint-id>

# Hubble flow monitoring
hubble observe --type drop
hubble observe --from-pod frontend --to-pod backend
hubble observe --protocol tcp --port 443

# Calico node status
calicoctl node status
calicoctl get nodes -o wide

# BGP and routing information
calicoctl get bgppeers
calicoctl get ippools
calicoctl get workloadendpoints

# Policy verification
calicoctl get networkpolicies
calicoctl get globalnetworkpolicies

# Flannel subnet information
cat /run/flannel/subnet.env
ip route show | grep flannel

# VXLAN interface status
ip link show flannel.1
bridge fdb show dev flannel.1

# Cilium performance tuning
apiVersion: v1
kind: ConfigMap
metadata:
  name: cilium-config
data:
  # eBPF map sizing
  bpf-lb-map-max: "65536"
  bpf-policy-map-max: "16384"
  bpf-ct-global-tcp-max: "524288"
  bpf-ct-global-any-max: "262144"
  
  # Performance optimizations
  enable-bpf-tproxy: "true"
  enable-host-legacy-routing: "false"
  enable-bandwidth-manager: "true"
  
  # Operator scaling
  operator-replicas: "3"
  operator-prometheus-serve-addr: ":9963"

# Calico Typha for large clusters
apiVersion: apps/v1
kind: Deployment
metadata:
  name: calico-typha
spec:
  replicas: 3  # Scale based on cluster size
  template:
    spec:
      containers:
      - name: calico-typha
        image: calico/typha:v3.25.0
        env:
        - name: TYPHA_LOGSEVERITYSCREEN
          value: "info"
        - name: TYPHA_PROMETHEUSMETRICSENABLED
          value: "true"
        - name: TYPHA_CONNECTIONREBALANCINGMODE
          value: "kubernetes"
        resources:
          limits:
            cpu: 1000m
            memory: 1Gi
          requests:
            cpu: 500m
            memory: 512Mi

# Cilium Azure configuration
apiVersion: v1
kind: ConfigMap
metadata:
  name: cilium-config
data:
  azure-subscription-id: "12345678-1234-1234-1234-123456789012"
  azure-resource-group: "production-rg"
  azure-vm-scale-set: "aks-nodepool1-12345678-vmss"
  azure-user-assigned-identity-id: "/subscriptions/.../microsoft.managedidentity/userassignedidentities/cilium-identity"
  enable-azure-sources: "true"

# High-performance workload configuration
apiVersion: v1
kind: Pod
metadata:
  name: high-perf-app
  annotations:
    io.cilium.proxy-visibility: "<Egress/53/UDP/DNS>,<Egress/80/TCP/HTTP>"
spec:
  containers:
  - name: app
    image: high-perf-app:latest
    resources:
      limits:
        cpu: 4000m
        memory: 8Gi

# Complex policy environment
apiVersion: projectcalico.org/v3
kind: NetworkPolicy
metadata:
  name: microservices-policy
spec:
  selector: tier == "application"
  types:
  - Ingress
  - Egress
  ingress:
  - action: Allow
    source:
      selector: tier == "frontend"
    protocol: TCP
    destination:
      ports: [8080, 9090]

# Simple overlay configuration
apiVersion: v1
kind: ConfigMap
metadata:
  name: kube-flannel-cfg
data:
  net-conf.json: |
    {
      "Network": "10.244.0.0/16",
      "Backend": {
        "Type": "host-gw"
      }
    }

# Drain and migrate nodes
kubectl drain node-1 --ignore-daemonsets
# Replace CNI on node
kubectl uncordon node-1

# Verify connectivity
kubectl run test-pod --image=busybox --rm -it -- /bin/sh
# Test network connectivity