fix: add generated target for all node IPs

docs/reference/configuration/custom resources/packages-v1alpha1-cr.md

@@ -124,7 +124,7 @@ tableOfContents:
     </tr>
   </thead>
   <tbody>
-    <tr><td style="white-space: nowrap;">description</td><td style="white-space: nowrap;">string</td><td>A description of the policy, this will become part of the policy name</td></tr><tr><td style="white-space: nowrap;">direction</td><td style="white-space: nowrap;">string (enum):<ul><li><code>Ingress</code></li><li><code>Egress</code></li></ul></td><td>The direction of the traffic</td></tr><tr><td style="white-space: nowrap;">labels</td><td style="white-space: nowrap;"></td><td>The labels to apply to the policy</td></tr><tr><td style="white-space: nowrap;">podLabels</td><td style="white-space: nowrap;"></td><td>Deprecated: use selector</td></tr><tr><td style="white-space: nowrap;">port</td><td style="white-space: nowrap;">number</td><td>The port to allow (protocol is always TCP)</td></tr><tr><td style="white-space: nowrap;">ports</td><td style="white-space: nowrap;">number[]</td><td>A list of ports to allow (protocol is always TCP)</td></tr><tr><td style="white-space: nowrap;">remoteCidr</td><td style="white-space: nowrap;">string</td><td>Custom generated policy CIDR</td></tr><tr><td style="white-space: nowrap;">remoteGenerated</td><td style="white-space: nowrap;">string (enum):<ul><li><code>KubeAPI</code></li><li><code>IntraNamespace</code></li><li><code>CloudMetadata</code></li><li><code>Anywhere</code></li></ul></td><td>Custom generated remote selector for the policy</td></tr><tr><td style="white-space: nowrap;">remoteNamespace</td><td style="white-space: nowrap;">string</td><td>The remote namespace to allow traffic to/from. Use * or empty string to allow all namespaces</td></tr><tr><td style="white-space: nowrap;">remotePodLabels</td><td style="white-space: nowrap;"></td><td>Deprecated: use remoteSelector</td></tr><tr><td style="white-space: nowrap;">remoteSelector</td><td style="white-space: nowrap;"></td><td>The remote pod selector labels to allow traffic to/from</td></tr><tr><td style="white-space: nowrap;">selector</td><td style="white-space: nowrap;"></td><td>Labels to match pods in the namespace to apply the policy to. Leave empty to apply to all pods in the namespace</td></tr>
+    <tr><td style="white-space: nowrap;">description</td><td style="white-space: nowrap;">string</td><td>A description of the policy, this will become part of the policy name</td></tr><tr><td style="white-space: nowrap;">direction</td><td style="white-space: nowrap;">string (enum):<ul><li><code>Ingress</code></li><li><code>Egress</code></li></ul></td><td>The direction of the traffic</td></tr><tr><td style="white-space: nowrap;">labels</td><td style="white-space: nowrap;"></td><td>The labels to apply to the policy</td></tr><tr><td style="white-space: nowrap;">podLabels</td><td style="white-space: nowrap;"></td><td>Deprecated: use selector</td></tr><tr><td style="white-space: nowrap;">port</td><td style="white-space: nowrap;">number</td><td>The port to allow (protocol is always TCP)</td></tr><tr><td style="white-space: nowrap;">ports</td><td style="white-space: nowrap;">number[]</td><td>A list of ports to allow (protocol is always TCP)</td></tr><tr><td style="white-space: nowrap;">remoteCidr</td><td style="white-space: nowrap;">string</td><td>Custom generated policy CIDR</td></tr><tr><td style="white-space: nowrap;">remoteGenerated</td><td style="white-space: nowrap;">string (enum):<ul><li><code>KubeAPI</code></li><li><code>KubeNodes</code></li><li><code>IntraNamespace</code></li><li><code>CloudMetadata</code></li><li><code>Anywhere</code></li></ul></td><td>Custom generated remote selector for the policy</td></tr><tr><td style="white-space: nowrap;">remoteNamespace</td><td style="white-space: nowrap;">string</td><td>The remote namespace to allow traffic to/from. Use * or empty string to allow all namespaces</td></tr><tr><td style="white-space: nowrap;">remotePodLabels</td><td style="white-space: nowrap;"></td><td>Deprecated: use remoteSelector</td></tr><tr><td style="white-space: nowrap;">remoteSelector</td><td style="white-space: nowrap;"></td><td>The remote pod selector labels to allow traffic to/from</td></tr><tr><td style="white-space: nowrap;">selector</td><td style="white-space: nowrap;"></td><td>Labels to match pods in the namespace to apply the policy to. Leave empty to apply to all pods in the namespace</td></tr>
   </tbody>
 </table>
 </div>

src/pepr/config.ts

@@ -39,6 +39,9 @@ export const UDSConfig = {
   // Static CIDR range to use for KubeAPI instead of k8s watch
   kubeApiCidr: process.env.KUBEAPI_CIDR,
 
+  // Static CIDRs to use for KubeNodes instead of k8s watch. Comma separated list of CIDRs.
+  kubeNodeCidrs: process.env.KUBENODE_CIDRS,
+
   // Track if UDS Core identity-authorization layer is deployed
   isIdentityDeployed: false,
 };

src/pepr/operator/controllers/network/generate.ts

@@ -11,6 +11,7 @@ import { anywhere, anywhereInCluster } from "./generators/anywhere";
 import { cloudMetadata } from "./generators/cloudMetadata";
 import { intraNamespace } from "./generators/intraNamespace";
 import { kubeAPI } from "./generators/kubeAPI";
+import { nodeCIDRs } from "./generators/kubeNodes";
 import { remoteCidr } from "./generators/remoteCidr";
 
 function isWildcardNamespace(namespace: string) {
@@ -26,6 +27,10 @@ function getPeers(policy: Allow): V1NetworkPolicyPeer[] {
         peers = kubeAPI();
         break;
 
+      case RemoteGenerated.KubeNodes:
+        peers = nodeCIDRs();
+        break;
+
       case RemoteGenerated.CloudMetadata:
         peers = cloudMetadata;
         break;
@@ -93,7 +98,7 @@ export function generate(namespace: string, policy: Allow): kind.NetworkPolicy {
     };
   }
 
-  // Add the generated policy label (used to track KubeAPI policies)
+  // Add the generated policy label (used to track KubeAPI and KubeNodes policies)
   if (policy.remoteGenerated) {
     generated.metadata!.labels!["uds/generated"] = policy.remoteGenerated;
   }

src/pepr/operator/controllers/network/generators/kubeNodes.spec.ts

@@ -0,0 +1,198 @@
+/**
+ * Copyright 2024 Defense Unicorns
+ * SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial
+ */
+
+import { beforeEach, beforeAll, describe, expect, it, jest } from "@jest/globals";
+
+import {
+  initAllNodesTarget,
+  nodeCIDRs,
+  updateKubeNodesFromCreateUpdate,
+  updateKubeNodesFromDelete,
+} from "./kubeNodes";
+import { K8s, kind } from "pepr";
+import { V1NetworkPolicyList } from "@kubernetes/client-node";
+import { anywhere } from "./anywhere";
+
+type KubernetesList<T> = {
+  items: T[];
+};
+
+jest.mock("pepr", () => {
+  const originalModule = jest.requireActual("pepr") as object;
+  return {
+    ...originalModule,
+    K8s: jest.fn(),
+    kind: {
+      Node: "Node",
+      NetworkPolicy: "NetworkPolicy",
+    },
+  };
+});
+
+describe("kubeNodes module", () => {
+  const mockNodeList = {
+    items: [
+      {
+        metadata: { name: "node1" },
+        status: {
+          addresses: [{ type: "InternalIP", address: "10.0.0.1" }],
+          conditions: [{ type: "Ready", status: "True" }],
+        },
+      },
+      {
+        metadata: { name: "node2" },
+        status: {
+          addresses: [{ type: "InternalIP", address: "10.0.0.2" }],
+          conditions: [{ type: "Ready", status: "True" }],
+        },
+      },
+    ],
+  };
+
+  const mockNetworkPolicyList: V1NetworkPolicyList = {
+    apiVersion: "networking.k8s.io/v1",
+    kind: "NetworkPolicyList",
+    items: [
+      {
+        apiVersion: "networking.k8s.io/v1",
+        kind: "NetworkPolicy",
+        metadata: {
+          name: "example-policy",
+          namespace: "default",
+        },
+        spec: {
+          podSelector: {}, // required field
+          policyTypes: ["Egress"], // or ["Ingress"], or both
+          egress: [
+            {
+              to: [{ ipBlock: { cidr: "0.0.0.0/0" } }], // an IP we don't want
+            },
+          ],
+        },
+      },
+    ],
+  };
+
+  const mockK8sGetNodes = jest.fn<() => Promise<KubernetesList<kind.Node>>>();
+  const mockGetNetworkPolicies = jest.fn<() => Promise<KubernetesList<kind.NetworkPolicy>>>();
+  const mockApply = jest.fn();
+
+  beforeAll(() => {
+    (K8s as jest.Mock).mockImplementation(() => ({
+      Get: mockK8sGetNodes,
+      WithLabel: jest.fn(() => ({
+        Get: mockGetNetworkPolicies,
+      })),
+      Apply: mockApply,
+    }));
+  });
+
+  beforeEach(() => {
+    jest.clearAllMocks();
+  });
+
+  describe("initAllNodesTarget", () => {
+    it("should initialize nodeSet with internal IPs from nodes", async () => {
+      mockK8sGetNodes.mockResolvedValue(mockNodeList);
+      await initAllNodesTarget();
+      const cidrs = nodeCIDRs();
+      // Should have two IPs from mockNodeList
+      expect(cidrs).toHaveLength(2);
+      expect(cidrs).toEqual(
+        expect.arrayContaining([
+          { ipBlock: { cidr: "10.0.0.1/32" } },
+          { ipBlock: { cidr: "10.0.0.2/32" } },
+        ]),
+      );
+    });
+  });
+
+  describe("nodeCIDRs", () => {
+    it("should return anywhere if no nodes known", async () => {
+      mockK8sGetNodes.mockResolvedValue({ items: [] });
+      await initAllNodesTarget();
+      const cidrs = nodeCIDRs();
+      // expect it to match "anywhere"
+      expect(cidrs).toEqual([anywhere]);
+    });
+  });
+
+  describe("updateKubeNodesFromCreateUpdate", () => {
+    it("should add a node IP if node is ready", async () => {
+      mockK8sGetNodes.mockResolvedValueOnce({ items: [] });
+      mockGetNetworkPolicies.mockResolvedValue(mockNetworkPolicyList);
+      await initAllNodesTarget(); // start empty
+      await updateKubeNodesFromCreateUpdate(mockNodeList.items[0]);
+      let cidrs = nodeCIDRs();
+      expect(cidrs).toHaveLength(1);
+      expect(cidrs[0].ipBlock?.cidr).toBe("10.0.0.1/32");
+      expect(mockApply).toHaveBeenCalled();
+
+      await updateKubeNodesFromCreateUpdate(mockNodeList.items[1]);
+      cidrs = nodeCIDRs();
+      expect(cidrs).toHaveLength(2);
+      expect(cidrs[1].ipBlock?.cidr).toBe("10.0.0.2/32");
+      expect(mockApply).toHaveBeenCalled();
+    });
+
+    it("should not add a node IP if node not ready", async () => {
+      const notReadyNode = {
+        metadata: { name: "node3" },
+        status: {
+          addresses: [{ type: "InternalIP", address: "10.0.0.3" }],
+          conditions: [{ type: "Ready", status: "False" }],
+        },
+      };
+      mockK8sGetNodes.mockResolvedValueOnce({ items: [] });
+      await initAllNodesTarget(); // start empty
+      await updateKubeNodesFromCreateUpdate(notReadyNode);
+      const cidrs = nodeCIDRs();
+      expect(cidrs).toEqual([anywhere]);
+      expect(mockApply).toHaveBeenCalled(); // Still called to update polices even if empty
+    });
+
+    it("should remove a node that's no longer ready", async () => {
+      mockK8sGetNodes.mockResolvedValue(mockNodeList);
+      await initAllNodesTarget();
+      let cidrs = nodeCIDRs();
+      // Should have two IPs from mockNodeList
+      expect(cidrs).toHaveLength(2);
+      expect(cidrs).toEqual(
+        expect.arrayContaining([
+          { ipBlock: { cidr: "10.0.0.1/32" } },
+          { ipBlock: { cidr: "10.0.0.2/32" } },
+        ]),
+      );
+
+      const notReadyNode = {
+        metadata: { name: "node2" },
+        status: {
+          addresses: [{ type: "InternalIP", address: "10.0.0.1" }],
+          conditions: [{ type: "Ready", status: "False" }],
+        },
+      };
+      await updateKubeNodesFromCreateUpdate(notReadyNode);
+      cidrs = nodeCIDRs();
+      expect(cidrs).toHaveLength(1);
+      expect(cidrs).toEqual(expect.arrayContaining([{ ipBlock: { cidr: "10.0.0.2/32" } }]));
+      expect(mockApply).toHaveBeenCalled(); // Still called to update polices even if empty
+    });
+  });
+
+  describe("updateKubeNodesFromDelete", () => {
+    it("should remove the node IP from nodeSet", async () => {
+      mockK8sGetNodes.mockResolvedValueOnce(mockNodeList);
+      await initAllNodesTarget();
+      const cidrsBeforeDelete = nodeCIDRs();
+      expect(cidrsBeforeDelete).toHaveLength(2);
+
+      await updateKubeNodesFromDelete(mockNodeList.items[0]);
+      const cidrsAfterDelete = nodeCIDRs();
+      expect(cidrsAfterDelete).toHaveLength(1);
+      expect(cidrsAfterDelete[0].ipBlock?.cidr).toBe("10.0.0.2/32");
+      expect(mockApply).toHaveBeenCalled();
+    });
+  });
+});