[nginx] Connect failed when updating pod

[Anson2048]

k8s configure

$ kubectl get node
NAME             STATUS    AGE       VERSION
192.168.161.62   Ready     5d        v1.6.4
192.168.161.63   Ready     5d        v1.6.4
192.168.161.64   Ready     5d        v1.6.4

$ kubectl get deployment -n kube-system ingress-nginx-controller -o yaml
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  annotations:
    deployment.kubernetes.io/revision: "2"
    kubectl.kubernetes.io/last-applied-configuration: |
      {"apiVersion":"extensions/v1beta1","kind":"Deployment","metadata":{"annotations":{},"labels":{"project":"ingress","service":"ingress-nginx-controller"},"name":"ingress-nginx-controller","namespace":"kube-system"},"spec":{"replicas":1,"revisionHistoryLimit":10,"strategy":{"rollingUpdate":{"maxSurge":1,"maxUnavailable":0}},"template":{"metadata":{"labels":{"project":"ingress","service":"ingress-nginx-controller"}},"spec":{"containers":[{"args":["/nginx-ingress-controller","--default-backend-service=$(POD_NAMESPACE)/ingress-default-http-backend","--configmap=$(POD_NAMESPACE)/ingress-nginx-controller"],"env":[{"name":"POD_NAME","valueFrom":{"fieldRef":{"fieldPath":"metadata.name"}}},{"name":"POD_NAMESPACE","valueFrom":{"fieldRef":{"fieldPath":"metadata.namespace"}}}],"image":"gcr.io/google_containers/nginx-ingress-controller:0.9.0-beta.11","livenessProbe":{"failureThreshold":3,"httpGet":{"path":"/healthz","port":10254,"scheme":"HTTP"},"initialDelaySeconds":120,"periodSeconds":2,"timeoutSeconds":30},"name":"ingress-nginx-controller","ports":[{"containerPort":80},{"containerPort":443},{"containerPort":18080}],"readinessProbe":{"failureThreshold":3,"httpGet":{"path":"/healthz","port":10254,"scheme":"HTTP"},"periodSeconds":2,"successThreshold":2,"timeoutSeconds":10},"resources":{"limits":{"memory":"500Mi"},"requests":{"cpu":0.4,"memory":"500Mi"}}}],"dnsPolicy":"ClusterFirstWithHostNet","hostNetwork":true,"nodeSelector":{"kube-system-ingress-nginx-controller":"true"},"terminationGracePeriodSeconds":120}}}}
  creationTimestamp: 2017-08-22T10:45:31Z
  generation: 3
  labels:
    project: ingress
    service: ingress-nginx-controller
  name: ingress-nginx-controller
  namespace: kube-system
  resourceVersion: "749798"
  selfLink: /apis/extensions/v1beta1/namespaces/kube-system/deployments/ingress-nginx-controller
  uid: 05186485-8727-11e7-adc8-001ec9386b01
spec:
  replicas: 1
  revisionHistoryLimit: 10
  selector:
    matchLabels:
      project: ingress
      service: ingress-nginx-controller
  strategy:
    rollingUpdate:
      maxSurge: 1
      maxUnavailable: 0
    type: RollingUpdate
  template:
    metadata:
      creationTimestamp: null
      labels:
        project: ingress
        service: ingress-nginx-controller
    spec:
      containers:
      - args:
        - /nginx-ingress-controller
        - --default-backend-service=$(POD_NAMESPACE)/ingress-default-http-backend
        - --configmap=$(POD_NAMESPACE)/ingress-nginx-controller
        env:
        - name: POD_NAME
          valueFrom:
            fieldRef:
              apiVersion: v1
              fieldPath: metadata.name
        - name: POD_NAMESPACE
          valueFrom:
            fieldRef:
              apiVersion: v1
              fieldPath: metadata.namespace
        image: gcr.io/google_containers/nginx-ingress-controller:0.9.0-beta.11
        imagePullPolicy: IfNotPresent
        livenessProbe:
          failureThreshold: 3
          httpGet:
            path: /healthz
            port: 10254
            scheme: HTTP
          initialDelaySeconds: 120
          periodSeconds: 2
          successThreshold: 1
          timeoutSeconds: 30
        name: ingress-nginx-controller
        ports:
        - containerPort: 80
          hostPort: 80
          protocol: TCP
        - containerPort: 443
          hostPort: 443
          protocol: TCP
        - containerPort: 18080
          hostPort: 18080
          protocol: TCP
        readinessProbe:
          failureThreshold: 3
          httpGet:
            path: /healthz
            port: 10254
            scheme: HTTP
          periodSeconds: 2
          successThreshold: 2
          timeoutSeconds: 10
        resources:
          limits:
            memory: 500Mi
          requests:
            cpu: 400m
            memory: 500Mi
        terminationMessagePath: /dev/termination-log
        terminationMessagePolicy: File
      dnsPolicy: ClusterFirstWithHostNet
      hostNetwork: true
      nodeSelector:
        kube-system-ingress-nginx-controller: "true"
      restartPolicy: Always
      schedulerName: default-scheduler
      securityContext: {}
      terminationGracePeriodSeconds: 120
status:
  availableReplicas: 1
  conditions:
  - lastTransitionTime: 2017-08-22T10:45:38Z
    lastUpdateTime: 2017-08-22T10:45:38Z
    message: Deployment has minimum availability.
    reason: MinimumReplicasAvailable
    status: "True"
    type: Available
  observedGeneration: 3
  readyReplicas: 1
  replicas: 1
  updatedReplicas: 1

project: https://github.com/Anson2048/test-ingress

deploy nginx project

$ ./init.sh
deployment "nginx" configured
ingress "nginx" configured
service "nginx" configured

run ab test

./ab-test.sh

update deployment

./update.sh

got ingress error log

2017/08/28 06:29:38 [error] 2907#2907: *2310084 connect() failed (111: Connection refused) while connecting to upstream, client: 192.168.225.161, server: anson2049.com, request: "GET / HTTP/1.0", upstream: "http://10.32.0.3:80/", host: "anson2049.com:32080"

[aledbf]

@Anson2048 this is normal. This is just how nginx indicates there was an error reaching an upstream.
The default configuration enable retries to the next available upstream server

Edit: running your test I have less than 10 errors in 300000 requests with 3 updates

[inetfuture]

@Anson2048 what's the ab test result, is there any failure?

[Anson2048]

@aledbf thanks for your help, I have just tested it, it really retries to the next available upstream server.

[Anson2048]

@inetfuture no failure

[aledbf]

@Anson2048 can we close this issue?

[Anson2048]

ok