POC - Test if NAP WAF v5 starting up with readOnlyRootFileSystem enabled in NIC container & waf_enforcer & waf_config_mgr

[shaun-nx]

Context

UPDATE: TL;DR

Users CAN use NIC v3.7.0 + WAF v5 with the readOnlyRootFilesystem
NOTE: Users that install NIC + WAF v5 via Helm needs to be aware about this fixed issue.

---

NOTE the code snippet below is not relevant for NIC + WAF v5.

Timebox: 2 days

The NAP team set the follow tmp directories in the http context

http {
        ....
        # Temporary directories for kubernetes "readonlyfilesystem"
        client_body_temp_path /tmp/nginx-client-body;
        proxy_temp_path       /tmp/nginx-proxy;
        fastcgi_temp_path     /tmp/nginx-fastcgi;
        uwsgi_temp_path       /tmp/nginx-uwsgi;
        scgi_temp_path        /tmp/nginx-scgi;
}



apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: local-path-pvc
spec:
  accessModes:
    - ReadWriteOnce
  storageClassName: local-path
  resources:
    requests:
      storage: 2Gi
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: policy-file
data:
  policy.json: |
    {
      "policy": {
        "name": "my_policy",
        "template": {
          "name": "POLICY_TEMPLATE_NGINX_BASE"
        },
        "applicationLanguage": "utf-8",
        "enforcementMode": "blocking"
      }
    }
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: waf-nginx-conf
data:
  test_nginx.conf: |
    user nginx;
    worker_processes  4;

    load_module modules/ngx_http_app_protect_module.so;

    error_log /var/log/nginx/error.log debug;
    pid        /tmp/nginx.pid;

    # working_directory /tmp/cores;
    worker_rlimit_core 1000M;

    events {
        worker_connections  65536;
    }

    http {
        include       /etc/nginx/mime.types;
        default_type  application/octet-stream;
        sendfile        on;
        keepalive_timeout  65;

        # WAF enforcer address
        app_protect_enforcer_address 127.0.0.1:50000;

        access_log  /var/log/nginx/access.log;

        # Temporary directories for kubernetes "readonlyfilesystem"
        client_body_temp_path /tmp/nginx-client-body;
        proxy_temp_path       /tmp/nginx-proxy;
        fastcgi_temp_path     /tmp/nginx-fastcgi;
        uwsgi_temp_path       /tmp/nginx-uwsgi;
        scgi_temp_path        /tmp/nginx-scgi;

        server {
            listen       80;
            server_name  localhost;
            proxy_http_version 1.1;
            app_protect_enable on;
            app_protect_policy_file "/etc/app_protect/bundles/compiled_policy.tgz";

            app_protect_security_log_enable on;
            app_protect_security_log log_all syslog:server=127.0.0.1:515;

            location / {
                client_max_body_size 0;
                default_type text/html;
                # Pass traffic to testing web server inside the pod
                proxy_pass http://127.0.0.1:8080/$request_uri;
            }
        }
    }
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: compiler-node
spec:
  replicas: 1
  selector:
    matchLabels:
      app: compiler-node
  template:
    metadata:
      labels:
        app: compiler-node
    spec:
      containers:
        ################################## Compiler ##################################
        - name: compiler-node
          image: NAPX_COMPILER_IMAGE:NAPX_COMPILER_TAG
          imagePullPolicy: Always
          command: ["/bin/bash"]
          args:
          - "-c"
          - |
            /opt/app_protect/bin/apcompile -p /src/policy.json -o /dst/compiled_policy.tgz
            tail -f /dev/null
          # args: ["-p" , "/src/policy.json" , "-o" , "/dst/compiled_policy.tgz"]
          volumeMounts:
            - mountPath: /dst # Saves the bundle to be used later in the bundles pvc
              name: nap5-bundles
            - name: policy-file
              mountPath: /src # Mounts the ConfigMap here to access policy.json
      volumes:
        - name: policy-file
          configMap:
            name: policy-file
        - name: nap5-bundles
          persistentVolumeClaim:
            claimName: local-path-pvc
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: nginx
spec:
  selector:
    matchLabels:
      app: nginx
  replicas: 1
  template:
    metadata:
      labels:
        app: nginx
    spec:
      containers:
        - name: waf-nginx
          image: NAPX_IMAGE:NAPX_TAG
          imagePullPolicy: Always
          securityContext:
            readOnlyRootFilesystem: true
          command: ["/bin/bash"]
          args:
          - "-c"
          - |
            mkdir -p /tmp/cores
            until [ -f /etc/app_protect/bundles/compiled_policy.tgz ]; do sleep 1; done
            nginx -c /tmp/policy/test_nginx.conf -g 'daemon off;' &
            ncat -vlkp 515 > /tmp/nclog 2>&1 &
            tail -f /dev/null
          volumeMounts:
            - name: app-protect-bd-config
              mountPath: /opt/app_protect/bd_config
            - name: app-protect-config
              mountPath: /opt/app_protect/config
            - name: tmp-volume
              mountPath: /tmp
            - name: nginx-log
              mountPath: /var/log/nginx
            - name: app-protect-bundles
              mountPath: /etc/app_protect/bundles
            - name: waf-nginx-conf    # Mounting the nginx.conf file, not required as part of the deployment
              mountPath: /tmp/policy
        - name: enforcer
          image: ENFORCER_IMAGE:ENFORCER_TAG
          imagePullPolicy: Always
          securityContext:
            readOnlyRootFilesystem: true
          env:
            - name: ENFORCER_PORT
              value: "50000"
          volumeMounts:
            - name: app-protect-bd-config
              mountPath: /opt/app_protect/bd_config
        - name: config-mgr
          image: CONFIG_MGR_IMAGE:CONFIG_MGR_TAG
          imagePullPolicy: Always
          securityContext:
            allowPrivilegeEscalation: false
            readOnlyRootFilesystem: true
            capabilities:
              drop:
                - all
          volumeMounts:
            - name: app-protect-bd-config
              mountPath: /opt/app_protect/bd_config
            - name: app-protect-config
              mountPath: /opt/app_protect/config
            - name: app-protect-bundles
              mountPath: /etc/app_protect/bundles
        ################################## Backend Server ##################################
        - name: testing-webserver
          image: artifactory.f5net.com/f5-wafqatools-docker/customwebserver
          imagePullPolicy: Always
          ports:
            - containerPort: 8080
      volumes:
        - name: app-protect-bd-config
          emptyDir: {}
        - name: app-protect-config
          emptyDir: {}
        - name: nginx-log
          emptyDir: {}
        - name: tmp-volume
          emptyDir: {}
        - name: app-protect-bundles
          persistentVolumeClaim:
            claimName: local-path-pvc
        - name: waf-nginx-conf
          configMap:
            name: waf-nginx-conf
---
apiVersion: v1
kind: Service
metadata:
  name: nginx
spec:
  type: NodePort
  ports:
    - port: 80
  selector:
    app: nginx

[github-actions]

Hi @shaun-nx thanks for reporting!

Be sure to check out the docs and the Contributing Guidelines while you wait for a human to take a look at this 🙂

Cheers!

[jjngx]

Scope:

1. Test NAP WAF v5 starting up with readOnlyRootFileSystem is enabled in NIC container & waf_enforcer and waf_config_mgr.

• deployment (NIC v3.7.0 + WAF v5) done in the minikube using Helm
• readOnlyRootFilesystem configured on NIC+WAFv5, enforcer and manager containers

Steps:

• Build NIC + NAP (3.7.0)
• Pull waf_enforcer and waf_config_mgr images
• Create a dummy, valid policy bundle (*.tgz)

➜  kubernetes-ingress git:(test/waf5-rootfs) ✗ docker run --rm -v $(pwd):$(pwd) waf-compiler:5.3.0 -p $(pwd)/wafv5.json -o $(pwd)/compiled_policy.tgz
WARNING: The requested image's platform (linux/amd64) does not match the detected host platform (linux/arm64/v8) and no specific platform was requested
{"warnings":["'open-api-files' does not contain 'link' or 'contents'."],"attack_signatures_package":{"version":"2024.08.27","revisionDatetime":"2024-08-27T16:07:51Z"},"filename":"/Users/j.jarosz/code/kubernetes-ingress/compiled_policy.tgz","bot_signatures_package":{"version":"2024.08.27","revisionDatetime":"2024-08-27T07:08:27Z"},"compiler_engine":"full","completed_successfully":true,"threat_campaigns_package":{"version":"2024.08.28","revisionDatetime":"2024-08-28T13:27:14Z"},"file_size":1794853}

➜  kubernetes-ingress git:(test/waf5-rootfs) ✗ ls -lt | grep tgz
-rw-r--r--   1 j.jarosz  staff   1794853  2 Oct 15:41 compiled_policy.tgz

• Place the bundle in the right path (/etc/app_protect/bundles - path defined in the NIC)
• Create PV and PVC
• Start NIC + WAF v5
• Deploy Policy object (referencing the bundle)
• Deploy VS
• Verify the system is working: 1) deploy and follow steps to verify app protect functionality app_protect_v5 example

Expected results: no erros, NIC logs show no problems, system is deployed,

• Create a policy - referencing the bundle
• Verify NIC + WAF do not log errors
• 3 containers in the pods are running (nic, enforcer and manager)
• Deployed VS with the WAF policy is running (status is valid)

kubectl describe of the pod

{
    "apiVersion": "v1",
    "kind": "Pod",
    "metadata": {
        "annotations": {
            "prometheus.io/port": "9113",
            "prometheus.io/scheme": "http",
            "prometheus.io/scrape": "true"
        },
        "creationTimestamp": "2024-10-03T10:48:51Z",
        "generateName": "my-release-nginx-ingress-controller-76bc574556-",
        "labels": {
            "app.kubernetes.io/instance": "my-release",
            "app.kubernetes.io/name": "nginx-ingress",
            "app.kubernetes.io/version": "3.7.0-SNAPSHOT",
            "app.nginx.org/version": "1.25.5-nginx-plus-r32-p1",
            "appprotect.f5.com/version": "5.3.0",
            "pod-template-hash": "76bc574556"
        },
        "name": "my-release-nginx-ingress-controller-76bc574556-q27nz",
        "namespace": "default",
        "ownerReferences": [
            {
                "apiVersion": "apps/v1",
                "blockOwnerDeletion": true,
                "controller": true,
                "kind": "ReplicaSet",
                "name": "my-release-nginx-ingress-controller-76bc574556",
                "uid": "85e5a4a9-816e-4350-9637-1093982a92bf"
            }
        ],
        "resourceVersion": "736",
        "uid": "fe212f4c-c8c6-4627-94f3-c585abfd4ea0"
    },
    "spec": {
        "automountServiceAccountToken": true,
        "containers": [
            {
                "args": [
                    "-nginx-plus=true",
                    "-nginx-reload-timeout=60000",
                    "-enable-app-protect=true",
                    "-app-protect-enforcer-address=\"127.0.0.1:50000\"",
                    "-enable-app-protect-dos=false",
                    "-nginx-configmaps=$(POD_NAMESPACE)/my-release-nginx-ingress",
                    "-ingress-class=nginx",
                    "-health-status=false",
                    "-health-status-uri=/nginx-health",
                    "-nginx-debug=false",
                    "-v=3",
                    "-nginx-status=true",
                    "-nginx-status-port=8080",
                    "-nginx-status-allow-cidrs=127.0.0.1",
                    "-report-ingress-status",
                    "-external-service=my-release-nginx-ingress-controller",
                    "-enable-leader-election=true",
                    "-leader-election-lock-name=my-release-nginx-ingress-leader-election",
                    "-enable-prometheus-metrics=true",
                    "-prometheus-metrics-listen-port=9113",
                    "-prometheus-tls-secret=",
                    "-enable-service-insight=false",
                    "-service-insight-listen-port=9114",
                    "-service-insight-tls-secret=",
                    "-enable-custom-resources=true",
                    "-enable-snippets=true",
                    "-disable-ipv6=false",
                    "-enable-tls-passthrough=false",
                    "-enable-cert-manager=false",
                    "-enable-oidc=false",
                    "-enable-external-dns=false",
                    "-default-http-listener-port=80",
                    "-default-https-listener-port=443",
                    "-ready-status=true",
                    "-ready-status-port=8081",
                    "-enable-latency-metrics=false",
                    "-ssl-dynamic-reload=true",
                    "-enable-telemetry-reporting=true",
                    "-weight-changes-dynamic-reload=false"
                ],
                "env": [
                    {
                        "name": "POD_NAMESPACE",
                        "valueFrom": {
                            "fieldRef": {
                                "apiVersion": "v1",
                                "fieldPath": "metadata.namespace"
                            }
                        }
                    },
                    {
                        "name": "POD_NAME",
                        "valueFrom": {
                            "fieldRef": {
                                "apiVersion": "v1",
                                "fieldPath": "metadata.name"
                            }
                        }
                    }
                ],
                "image": "nginx/nginx-ingress:local",
                "imagePullPolicy": "IfNotPresent",
                "name": "nginx-ingress",
                "ports": [
                    {
                        "containerPort": 80,
                        "name": "http",
                        "protocol": "TCP"
                    },
                    {
                        "containerPort": 443,
                        "name": "https",
                        "protocol": "TCP"
                    },
                    {
                        "containerPort": 9113,
                        "name": "prometheus",
                        "protocol": "TCP"
                    },
                    {
                        "containerPort": 8081,
                        "name": "readiness-port",
                        "protocol": "TCP"
                    }
                ],
                "readinessProbe": {
                    "failureThreshold": 3,
                    "httpGet": {
                        "path": "/nginx-ready",
                        "port": "readiness-port",
                        "scheme": "HTTP"
                    },
                    "periodSeconds": 1,
                    "successThreshold": 1,
                    "timeoutSeconds": 1
                },
                "resources": {
                    "requests": {
                        "cpu": "100m",
                        "memory": "128Mi"
                    }
                },
                "securityContext": {
                    "readOnlyRootFilesystem": true
                },
                "terminationMessagePath": "/dev/termination-log",
                "terminationMessagePolicy": "File",
                "volumeMounts": [
                    {
                        "mountPath": "/etc/nginx",
                        "name": "nginx-etc"
                    },
                    {
                        "mountPath": "/var/cache/nginx",
                        "name": "nginx-cache"
                    },
                    {
                        "mountPath": "/var/lib/nginx",
                        "name": "nginx-lib"
                    },
                    {
                        "mountPath": "/var/log/nginx",
                        "name": "nginx-log"
                    },
                    {
                        "mountPath": "/opt/app_protect/bd_config",
                        "name": "app-protect-bd-config"
                    },
                    {
                        "mountPath": "/opt/app_protect/config",
                        "name": "app-protect-config"
                    },
                    {
                        "mountPath": "/etc/app_protect/bundles",
                        "name": "app-protect-bundles"
                    },
                    {
                        "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount",
                        "name": "kube-api-access-qndh9",
                        "readOnly": true
                    }
                ]
            },
            {
                "env": [
                    {
                        "name": "ENFORCER_PORT",
                        "value": "50000"
                    }
                ],
                "image": "private-registry.nginx.com/nap/waf-enforcer:5.3.0",
                "imagePullPolicy": "IfNotPresent",
                "name": "waf-enforcer",
                "resources": {},
                "securityContext": {
                    "readOnlyRootFilesystem": true
                },
                "terminationMessagePath": "/dev/termination-log",
                "terminationMessagePolicy": "File",
                "volumeMounts": [
                    {
                        "mountPath": "/opt/app_protect/bd_config",
                        "name": "app-protect-bd-config"
                    },
                    {
                        "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount",
                        "name": "kube-api-access-qndh9",
                        "readOnly": true
                    }
                ]
            },
            {
                "image": "private-registry.nginx.com/nap/waf-config-mgr:5.3.0",
                "imagePullPolicy": "IfNotPresent",
                "name": "waf-config-mgr",
                "resources": {},
                "securityContext": {
                    "allowPrivilegeEscalation": false,
                    "capabilities": {
                        "drop": [
                            "all"
                        ]
                    },
                    "readOnlyRootFilesystem": true,
                    "runAsNonRoot": true,
                    "runAsUser": 101
                },
                "terminationMessagePath": "/dev/termination-log",
                "terminationMessagePolicy": "File",
                "volumeMounts": [
                    {
                        "mountPath": "/opt/app_protect/bd_config",
                        "name": "app-protect-bd-config"
                    },
                    {
                        "mountPath": "/opt/app_protect/config",
                        "name": "app-protect-config"
                    },
                    {
                        "mountPath": "/etc/app_protect/bundles",
                        "name": "app-protect-bundles"
                    },
                    {
                        "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount",
                        "name": "kube-api-access-qndh9",
                        "readOnly": true
                    }
                ]
            }
        ],
        "dnsPolicy": "ClusterFirst",
        "enableServiceLinks": true,
        "initContainers": [
            {
                "command": [
                    "cp",
                    "-vdR",
                    "/etc/nginx/.",
                    "/mnt/etc"
                ],
                "image": "nginx/nginx-ingress:local",
                "imagePullPolicy": "IfNotPresent",
                "name": "init-nginx-ingress",
                "resources": {
                    "requests": {
                        "cpu": "100m",
                        "memory": "128Mi"
                    }
                },
                "securityContext": {
                    "allowPrivilegeEscalation": false,
                    "capabilities": {
                        "drop": [
                            "ALL"
                        ]
                    },
                    "readOnlyRootFilesystem": true,
                    "runAsNonRoot": true,
                    "runAsUser": 101
                },
                "terminationMessagePath": "/dev/termination-log",
                "terminationMessagePolicy": "File",
                "volumeMounts": [
                    {
                        "mountPath": "/mnt/etc",
                        "name": "nginx-etc"
                    },
                    {
                        "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount",
                        "name": "kube-api-access-qndh9",
                        "readOnly": true
                    }
                ]
            }
        ],
        "nodeName": "minikube",
        "preemptionPolicy": "PreemptLowerPriority",
        "priority": 0,
        "restartPolicy": "Always",
        "schedulerName": "default-scheduler",
        "securityContext": {
            "seccompProfile": {
                "type": "RuntimeDefault"
            }
        },
        "serviceAccount": "my-release-nginx-ingress",
        "serviceAccountName": "my-release-nginx-ingress",
        "terminationGracePeriodSeconds": 30,
        "tolerations": [
            {
                "effect": "NoExecute",
                "key": "node.kubernetes.io/not-ready",
                "operator": "Exists",
                "tolerationSeconds": 300
            },
            {
                "effect": "NoExecute",
                "key": "node.kubernetes.io/unreachable",
                "operator": "Exists",
                "tolerationSeconds": 300
            }
        ],
        "volumes": [
            {
                "emptyDir": {},
                "name": "nginx-etc"
            },
            {
                "emptyDir": {},
                "name": "nginx-cache"
            },
            {
                "emptyDir": {},
                "name": "nginx-lib"
            },
            {
                "emptyDir": {},
                "name": "nginx-log"
            },
            {
                "emptyDir": {},
                "name": "app-protect-bd-config"
            },
            {
                "emptyDir": {},
                "name": "app-protect-config"
            },
            {
                "name": "app-protect-bundles",
                "persistentVolumeClaim": {
                    "claimName": "pvc-bundle"
                }
            },
            {
                "name": "kube-api-access-qndh9",
                "projected": {
                    "defaultMode": 420,
                    "sources": [
                        {
                            "serviceAccountToken": {
                                "expirationSeconds": 3607,
                                "path": "token"
                            }
                        },
                        {
                            "configMap": {
                                "items": [
                                    {
                                        "key": "ca.crt",
                                        "path": "ca.crt"
                                    }
                                ],
                                "name": "kube-root-ca.crt"
                            }
                        },
                        {
                            "downwardAPI": {
                                "items": [
                                    {
                                        "fieldRef": {
                                            "apiVersion": "v1",
                                            "fieldPath": "metadata.namespace"
                                        },
                                        "path": "namespace"
                                    }
                                ]
                            }
                        }
                    ]
                }
            }
        ]
    },
    "status": {
        "conditions": [
            {
                "lastProbeTime": null,
                "lastTransitionTime": "2024-10-03T10:49:22Z",
                "status": "True",
                "type": "PodReadyToStartContainers"
            },
            {
                "lastProbeTime": null,
                "lastTransitionTime": "2024-10-03T10:49:23Z",
                "status": "True",
                "type": "Initialized"
            },
            {
                "lastProbeTime": null,
                "lastTransitionTime": "2024-10-03T10:49:29Z",
                "status": "True",
                "type": "Ready"
            },
            {
                "lastProbeTime": null,
                "lastTransitionTime": "2024-10-03T10:49:29Z",
                "status": "True",
                "type": "ContainersReady"
            },
            {
                "lastProbeTime": null,
                "lastTransitionTime": "2024-10-03T10:49:22Z",
                "status": "True",
                "type": "PodScheduled"
            }
        ],
        "containerStatuses": [
            {
                "containerID": "docker://d8cdec2efd27905e3351c1be5b19c226247925e59c6d936db4a8d38344d5f3e3",
                "image": "nginx/nginx-ingress:local",
                "imageID": "docker://sha256:9d8f24fe0ac22f6826679097196236f0ebe405b6017146aee110eb4166a2f56e",
                "lastState": {},
                "name": "nginx-ingress",
                "ready": true,
                "restartCount": 0,
                "started": true,
                "state": {
                    "running": {
                        "startedAt": "2024-10-03T10:49:23Z"
                    }
                },
                "volumeMounts": [
                    {
                        "mountPath": "/etc/nginx",
                        "name": "nginx-etc"
                    },
                    {
                        "mountPath": "/var/cache/nginx",
                        "name": "nginx-cache"
                    },
                    {
                        "mountPath": "/var/lib/nginx",
                        "name": "nginx-lib"
                    },
                    {
                        "mountPath": "/var/log/nginx",
                        "name": "nginx-log"
                    },
                    {
                        "mountPath": "/opt/app_protect/bd_config",
                        "name": "app-protect-bd-config"
                    },
                    {
                        "mountPath": "/opt/app_protect/config",
                        "name": "app-protect-config"
                    },
                    {
                        "mountPath": "/etc/app_protect/bundles",
                        "name": "app-protect-bundles"
                    },
                    {
                        "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount",
                        "name": "kube-api-access-qndh9",
                        "readOnly": true,
                        "recursiveReadOnly": "Disabled"
                    }
                ]
            },
            {
                "containerID": "docker://f443f8c095656f73f9aa0b4010be038b1d435942508a7e7455c7c543f61c4063",
                "image": "private-registry.nginx.com/nap/waf-config-mgr:5.3.0",
                "imageID": "docker-pullable://private-registry.nginx.com/nap/waf-config-mgr@sha256:518c05da9c967f5fc1a39941f27c0006b4a6b28cb08e94ca2d85b88075fc1cf9",
                "lastState": {},
                "name": "waf-config-mgr",
                "ready": true,
                "restartCount": 0,
                "started": true,
                "state": {
                    "running": {
                        "startedAt": "2024-10-03T10:49:23Z"
                    }
                },
                "volumeMounts": [
                    {
                        "mountPath": "/opt/app_protect/bd_config",
                        "name": "app-protect-bd-config"
                    },
                    {
                        "mountPath": "/opt/app_protect/config",
                        "name": "app-protect-config"
                    },
                    {
                        "mountPath": "/etc/app_protect/bundles",
                        "name": "app-protect-bundles"
                    },
                    {
                        "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount",
                        "name": "kube-api-access-qndh9",
                        "readOnly": true,
                        "recursiveReadOnly": "Disabled"
                    }
                ]
            },
            {
                "containerID": "docker://ba5fa07dee8206a21df732fcfbb964ea00ad98283e94f647dd62ed693b6149cc",
                "image": "private-registry.nginx.com/nap/waf-enforcer:5.3.0",
                "imageID": "docker-pullable://private-registry.nginx.com/nap/waf-enforcer@sha256:0115b3e91ee5e0b7fef0470c5afeff101d6ebb7b8c726d81225e9fe8d835d9c9",
                "lastState": {},
                "name": "waf-enforcer",
                "ready": true,
                "restartCount": 0,
                "started": true,
                "state": {
                    "running": {
                        "startedAt": "2024-10-03T10:49:23Z"
                    }
                },
                "volumeMounts": [
                    {
                        "mountPath": "/opt/app_protect/bd_config",
                        "name": "app-protect-bd-config"
                    },
                    {
                        "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount",
                        "name": "kube-api-access-qndh9",
                        "readOnly": true,
                        "recursiveReadOnly": "Disabled"
                    }
                ]
            }
        ],
        "hostIP": "192.168.49.2",
        "hostIPs": [
            {
                "ip": "192.168.49.2"
            }
        ],
        "initContainerStatuses": [
            {
                "containerID": "docker://c13d9af4a94b1c3e8f014ae4eb0a5e338ae958b73eb49f24fc19452dbb1c36db",
                "image": "nginx/nginx-ingress:local",
                "imageID": "docker://sha256:9d8f24fe0ac22f6826679097196236f0ebe405b6017146aee110eb4166a2f56e",
                "lastState": {},
                "name": "init-nginx-ingress",
                "ready": true,
                "restartCount": 0,
                "started": false,
                "state": {
                    "terminated": {
                        "containerID": "docker://c13d9af4a94b1c3e8f014ae4eb0a5e338ae958b73eb49f24fc19452dbb1c36db",
                        "exitCode": 0,
                        "finishedAt": "2024-10-03T10:49:22Z",
                        "reason": "Completed",
                        "startedAt": "2024-10-03T10:49:22Z"
                    }
                },
                "volumeMounts": [
                    {
                        "mountPath": "/mnt/etc",
                        "name": "nginx-etc"
                    },
                    {
                        "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount",
                        "name": "kube-api-access-qndh9",
                        "readOnly": true,
                        "recursiveReadOnly": "Disabled"
                    }
                ]
            }
        ],
        "phase": "Running",
        "podIP": "10.244.0.4",
        "podIPs": [
            {
                "ip": "10.244.0.4"
            }
        ],
        "qosClass": "Burstable",
        "startTime": "2024-10-03T10:49:22Z"
    }
}