diff --git a/src/cloud-api-adaptor/cmd/agent-protocol-forwarder/main.go b/src/cloud-api-adaptor/cmd/agent-protocol-forwarder/main.go index 47178c7b8..0d218566c 100644 --- a/src/cloud-api-adaptor/cmd/agent-protocol-forwarder/main.go +++ b/src/cloud-api-adaptor/cmd/agent-protocol-forwarder/main.go @@ -93,7 +93,9 @@ func (cfg *Config) Setup() (cmd.Starter, error) { return nil, err } - if secureComms { + if secureComms || cfg.daemonConfig.SecureComms { + var inbounds, outbounds []string + ppssh.Singleton() host, port, err := net.SplitHostPort(cfg.listenAddr) if err != nil { @@ -103,15 +105,32 @@ func (cfg *Config) Setup() (cmd.Starter, error) { logger.Printf("Address %s is changed to 127.0.0.1:%s since secure-comms is enabled.", cfg.listenAddr, port) cfg.listenAddr = "127.0.0.1:" + port } - inbounds := append([]string{"BOTH_PHASES:KBS:8080"}, strings.Split(secureCommsInbounds, ",")...) - outbounds := append([]string{"KUBERNETES_PHASE:KATAAGENT:" + port}, strings.Split(secureCommsOutbounds, ",")...) // Create a Client that will approach the api-server-rest service at the podns // To obtain secrets from KBS, we approach the api-server-rest service which then approaches the CDH asking for a secret resource // the CDH than contact the KBS (possibly after approaching Attestation Agent for a token) and the KBS serves the requested key // The communication between the CDH (and Attestation Agent) and the KBS is performed via an SSH tunnel named "KBS" apic := apic.NewApiClient(API_SERVER_REST_PORT, cfg.podNamespace) - services = append(services, ppssh.NewSshServer(inbounds, outbounds, ppssh.GetSecret(apic.GetKey), sshutil.SSHPORT)) + + ppSecrets := ppssh.NewPpSecrets(ppssh.GetSecret(apic.GetKey)) + + if secureComms { + // CoCo in production + ppSecrets.AddKey(ppssh.WN_PUBLIC_KEY) + ppSecrets.AddKey(ppssh.PP_PRIVATE_KEY) + inbounds = append([]string{"BOTH_PHASES:KBS:8080"}, strings.Split(secureCommsInbounds, ",")...) + outbounds = append([]string{"KUBERNETES_PHASE:KATAAGENT:" + port}, strings.Split(secureCommsOutbounds, ",")...) + + } else { + // Never here under CoCo in production + // Set secureComms using daemonConfig for testing + ppSecrets.SetKey(ppssh.WN_PUBLIC_KEY, cfg.daemonConfig.WnPublicKey) + ppSecrets.SetKey(ppssh.PP_PRIVATE_KEY, cfg.daemonConfig.PpPrivateKey) + inbounds = append([]string{"BOTH_PHASES:KBS:8080"}, strings.Split(cfg.daemonConfig.SecureCommsInbounds, ",")...) + outbounds = append([]string{"KUBERNETES_PHASE:KATAAGENT:" + port}, strings.Split(cfg.daemonConfig.SecureCommsOutbounds, ",")...) + } + + services = append(services, ppssh.NewSshServer(inbounds, outbounds, ppSecrets, sshutil.SSHPORT)) } else { if !disableTLS { cfg.tlsConfig = &tlsConfig diff --git a/src/cloud-api-adaptor/cmd/cloud-api-adaptor/main.go b/src/cloud-api-adaptor/cmd/cloud-api-adaptor/main.go index c6bd3db9a..d2c760265 100644 --- a/src/cloud-api-adaptor/cmd/cloud-api-adaptor/main.go +++ b/src/cloud-api-adaptor/cmd/cloud-api-adaptor/main.go @@ -9,9 +9,11 @@ import ( "fmt" "io" "os" + "strings" "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/cmd" "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/adaptor" + "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/adaptor/cloud" "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/adaptor/proxy" daemon "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/forwarder" "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/podnetwork/tunneler/vxlan" @@ -28,7 +30,7 @@ const ( ) type daemonConfig struct { - serverConfig adaptor.ServerConfig + serverConfig cloud.ServerConfig networkConfig } @@ -86,12 +88,14 @@ func (cfg *daemonConfig) Setup() (cmd.Starter, error) { } var ( - disableTLS bool - tlsConfig tlsutil.TLSConfig - secureComms bool - secureCommsInbounds string - secureCommsOutbounds string - secureCommsKbsAddr string + disableTLS bool + tlsConfig tlsutil.TLSConfig + secureComms bool + secureCommsInbounds string + secureCommsOutbounds string + secureCommsPpInbounds string + secureCommsPpOutbounds string + secureCommsKbsAddr string ) cmd.Parse(programName, os.Args[1:], func(flags *flag.FlagSet) { @@ -112,9 +116,11 @@ func (cfg *daemonConfig) Setup() (cmd.Starter, error) { flags.BoolVar(&tlsConfig.SkipVerify, "tls-skip-verify", false, "Skip TLS certificate verification - use it only for testing") flags.BoolVar(&disableTLS, "disable-tls", false, "Disable TLS encryption - use it only for testing") flags.BoolVar(&secureComms, "secure-comms", false, "Use SSH to secure communication between cluster and peer pods") - flags.StringVar(&secureCommsInbounds, "secure-comms-inbounds", "", "Inbound tags for secure communication tunnels") - flags.StringVar(&secureCommsOutbounds, "secure-comms-outbounds", "", "Outbound tags for secure communication tunnels") - flags.StringVar(&secureCommsKbsAddr, "secure-comms-kbs", "kbs-service.kbs-operator-system:8080", "Address of a KBS Service for Secure-Comms") + flags.StringVar(&secureCommsInbounds, "secure-comms-inbounds", "", "WN Inbound tags for secure communication tunnels") + flags.StringVar(&secureCommsOutbounds, "secure-comms-outbounds", "", "WN Outbound tags for secure communication tunnels") + flags.StringVar(&secureCommsPpInbounds, "secure-comms-pp-inbounds", "", "PP Inbound tags for secure communication tunnels") + flags.StringVar(&secureCommsPpOutbounds, "secure-comms-pp-outbounds", "", "PP Outbound tags for secure communication tunnels") + flags.StringVar(&secureCommsKbsAddr, "secure-comms-kbs", "kbs-service.trustee-operator-system:8080", "Address of a Trustee Service for Secure-Comms") flags.DurationVar(&cfg.serverConfig.ProxyTimeout, "proxy-timeout", proxy.DefaultProxyTimeout, "Maximum timeout in minutes for establishing agent proxy connection") flags.StringVar(&cfg.networkConfig.TunnelType, "tunnel-type", podnetwork.DefaultTunnelType, "Tunnel provider") @@ -137,10 +143,15 @@ func (cfg *daemonConfig) Setup() (cmd.Starter, error) { if err != nil { return nil, fmt.Errorf("secure comms failed to initialize KubeMgr: %w", err) } + if strings.EqualFold(secureCommsKbsAddr, "false") { + secureCommsKbsAddr = "" + } cfg.serverConfig.SecureComms = true cfg.serverConfig.SecureCommsInbounds = secureCommsInbounds cfg.serverConfig.SecureCommsOutbounds = secureCommsOutbounds + cfg.serverConfig.SecureCommsPpInbounds = secureCommsPpInbounds + cfg.serverConfig.SecureCommsPpOutbounds = secureCommsPpOutbounds cfg.serverConfig.SecureCommsKbsAddress = secureCommsKbsAddr } else { if !disableTLS { diff --git a/src/cloud-api-adaptor/docs/SecureComms.md b/src/cloud-api-adaptor/docs/SecureComms.md index 38ac81bf9..22a71a9ab 100644 --- a/src/cloud-api-adaptor/docs/SecureComms.md +++ b/src/cloud-api-adaptor/docs/SecureComms.md @@ -27,7 +27,35 @@ Once the "Kubernetes Phase" SSH channel is established, Secure Comms connects th See [Secure Comms Architecture Slides](./SecureComms.pdf) for more details. -## Setup +## Setup for for testing without Trustee (and for non-CoCo peerpods) + +### Deploy CAA +Use any of the option for installing CAA depending on the cloud driver used. + + +### Activate Secure-Comms feature from CAA side +Activate Secure-Comms from CAA side by changing the `SECURE_COMMS` parameter of the `peer-pods-cm` configMap in the `confidential-containers-system` namespace to `"true"`. + +```sh +kubectl -n confidential-containers-system get cm peer-pods-cm -o yaml | sed "s/SECURE_COMMS: \"false\"/SECURE_COMMS: \"true\"/"|kubectl apply -f - +``` + +You may also include additional Inbounds and Outbounds configurations to the Adaptor side using the `SECURE_COMMS_INBOUNDS` and `SECURE_COMMS_OUTBOUNDS` config points. +You may also add Inbounds and Outbounds configurations to the Forwarder side using the `SECURE_COMMS_PP_INBOUNDS` and `SECURE_COMMS_PP_OUTBOUNDS` config points. [See more details regarding Inbounds and Outbounds below.](#adding-named-tunnels-to-the-ssh-channel) + +Use `kubectl edit cm peer-pods-cm -n confidential-containers-system` to make such changes in the configMap, for example: +```sh +apiVersion: v1 +data: + ... + SECURE_COMMS: "true" + SECURE_COMMS_OUTBOUNDS: "KUBERNETES_PHASE:mytunnel:149.81.64.62:7777" + SECURE_COMMS_PP_INBOUNDS: "KUBERNETES_PHASE:mytunnel:6666" + SECURE_COMMS_KBS_ADDR: "false" + ... +``` + +## Setup for CoCo with Trustee ### Deploy CAA Use any of the option for installing CAA depending on the cloud driver used. @@ -37,19 +65,25 @@ Deploy Trustee-Operator by following instructions at [trustee Operator Getting S Make sure to uncomment the secret generation as recommended for both public and private key (`kbs-auth-public-key` and `kbs-client` secrets). -Copy the kbs-client secret from the `kbs-operator-system` namespace to the `confidential-containers-system` ns. This can be done using: +```sh +kubectl get secrets -n trustee-operator-system +NAME TYPE DATA AGE +kbs-auth-public-key Opaque 1 28h +kbs-client Opaque 1 28h +``` + +Copy the kbs-client secret from the `trustee-operator-system` namespace to the `confidential-containers-system` ns. This can be done using: ```sh -kubectl get secret kbs-client -n kbs-operator-system -o json|jq --arg ns "confidential-containers-system" 'del(.metadata["creationTimestamp","resourceVersion","selfLink","uid","annotations"]) | .metadata.namespace |= $ns' |kubectl apply -f - +kubectl get secret kbs-client -n trustee-operator-system -o json|jq --arg ns "confidential-containers-system" 'del(.metadata["creationTimestamp","resourceVersion","selfLink","uid","annotations"]) | .metadata.namespace |= $ns' |kubectl apply -f - ``` For a testing environment, you may need to change the policy of the KBS and AS using the KBS Client to allow all or fit your own policy. One way to do that is: ```sh -kubectl -n kbs-operator-system exec deployment/trustee-deployment --container as -it -- /bin/bash - sed -i.bak 's/^default allow = false/default allow = true/' /opt/confidential-containers/attestation-service/opa/default.rego +kubectl -n trustee-operator-system exec deployment/trustee-deployment --container as -it -- sed -i.bak 's/^default allow = false/default allow = true/' /opt/confidential-containers/attestation-service/opa/default.rego -kubectl -n kbs-operator-system get cm resource-policy -o yaml | sed "s/default allow = false/default allow = true/"|kubectl apply -f - +kubectl -n trustee-operator-system get cm resource-policy -o yaml | sed "s/default allow = false/default allow = true/"|kubectl apply -f - ``` ### Build a podvm that enforces Secure-Comms @@ -59,27 +93,68 @@ Change the `src/cloud-api-adaptor/podvm/files/etc/systemd/system/agent-protocol- ExecStart=/usr/local/bin/agent-protocol-forwarder -pod-namespace /run/netns/podns -secure-comms -kata-agent-socket /run/kata-containers/agent.sock $TLS_OPTIONS $OPTIONS ``` -You may also include additional Inbounds and Outbounds configurations to the Forwarder using the `-secure-comms-inbounds` and `-secure-comms-outbounds` flags. See more details regarding Inbounds and Outbounds below. +You may also include additional Inbounds and Outbounds configurations to the Forwarder using the `-secure-comms-inbounds` and `-secure-comms-outbounds` flags. [See more details regarding Inbounds and Outbounds below.](#adding-named-tunnels-to-the-ssh-channel) + +For example: +```sh +ExecStart=/usr/local/bin/agent-protocol-forwarder -kata-agent-namespace /run/netns/podns -secure-comms -secure-comms-inbounds KUBERNETES_PHASE:mytunnel:6666 -kata-agent-socket /run/kata-containers/agent.sock $TLS_OPTIONS $OPTIONS +``` Once you changed `podvm/files/etc/systemd/system/agent-protocol-forwarder.service`, you will need to [rebuild the podvm](./../podvm/README.md). ### Activate CAA Secure-Comms feature -Use `kubectl edit cm peer-pods-cm -n confidential-containers-system` to add to the `peer-pods-cm` config map at the `confidential-containers-system` namespace: +Activate Secure-Comms of CAA by changing the `SECURE_COMMS` parameter of the `peer-pods-cm` configMap in the `confidential-containers-system` namespace to `"true"`. + +```sh +kubectl -n confidential-containers-system get cm peer-pods-cm -o yaml | sed "s/SECURE_COMMS: \"false\"/SECURE_COMMS: \"true\"/"|kubectl apply -f - +``` + +Set InitData to point KBC services to IP address 127.0.0.1 +```sh +cat < /tmp/initdata.txt +algorithm = "sha384" +version = "0.1.0" + +[data] +"aa.toml" = ''' +[token_configs] +[token_configs.coco_as] +url = 'http://127.0.0.1:8080' + +[token_configs.kbs] +url = 'http://127.0.0.1:8080' +''' +"cdh.toml" = ''' +socket = 'unix:///run/confidential-containers/cdh.sock' +credentials = [] + +[kbc] +name = 'cc_kbc' +url = 'http://127.0.0.1:8080' +''' +EOF +export INITDATA=`base64 -w 0 /tmp/initdata.txt` +kubectl -n confidential-containers-system get cm peer-pods-cm -o yaml | sed 's/^INITDATA: .*/INITDATA: '$INITDATA'/'|kubectl apply -f - + +``` + +You may also include additional Inbounds and Outbounds configurations to the Adaptor using the `SECURE_COMMS_INBOUNDS` and `SECURE_COMMS_OUTBOUNDS` config points. [See more details regarding Inbounds and Outbounds below.](#adding-named-tunnels-to-the-ssh-channel) + +Use `kubectl edit cm peer-pods-cm -n confidential-containers-system` to make such changes in the configMap, for example: ```sh apiVersion: v1 data: ... SECURE_COMMS: "true" + SECURE_COMMS_OUTBOUNDS: "KUBERNETES_PHASE:mytunnel:149.81.64.62:7777" ... ``` -You may also include additional Inbounds and Outbounds configurations to the Adaptor using the `SECURE_COMMS_INBOUNDS` and `SECURE_COMMS_OUTBOUNDS` config points. See more details regarding Inbounds and Outbounds below. - You may also set the KBS address using the `SECURE_COMMS_KBS_ADDR` config point. -### Adding named tunnels to the SSH channel +## Adding named tunnels to the SSH channel Named tunnels can be added to the SSH channel. Adding a named tunnel requires adding an Inbound at one of the SSH channel peers and an Outbound at the other SSH channel peer. The Inbound and Outbound both carry the name of the tunnel being created. |---------Tunnel----------| @@ -117,5 +192,4 @@ Alternatively, the client and server can be separately executed in independent t - Add DeleteResource() support in KBS, KBC, api-server-rest, than cleanup resources added by Secure Comms to KBS whenever a Peer Pod fail to be created or when a Peer Pod is terminated. - Add support for running the vxlan tunnel traffic via a Secure Comms SSH tunnel -- Add support for non-confidential Peer Pods which do not go via an Attestation Phase. - Add support for KBS identities allowing a Peer Pod to register its own identity in KBS and replace the current Secure Comms mechanism which delivers a private key to the Peer Pod via the KBS diff --git a/src/cloud-api-adaptor/entrypoint.sh b/src/cloud-api-adaptor/entrypoint.sh index 3810e6bb9..eac62036e 100755 --- a/src/cloud-api-adaptor/entrypoint.sh +++ b/src/cloud-api-adaptor/entrypoint.sh @@ -24,6 +24,8 @@ optionals+="" [[ "${SECURE_COMMS}" == "true" ]] && optionals+="-secure-comms " [[ "${SECURE_COMMS_INBOUNDS}" ]] && optionals+="-secure-comms-inbounds ${SECURE_COMMS_INBOUNDS} " [[ "${SECURE_COMMS_OUTBOUNDS}" ]] && optionals+="-secure-comms-outbounds ${SECURE_COMMS_OUTBOUNDS} " +[[ "${SECURE_COMMS_PP_INBOUNDS}" ]] && optionals+="-secure-comms-pp-inbounds ${SECURE_COMMS_PP_INBOUNDS} " +[[ "${SECURE_COMMS_PP_OUTBOUNDS}" ]] && optionals+="-secure-comms-pp-outbounds ${SECURE_COMMS_PP_OUTBOUNDS} " [[ "${SECURE_COMMS_KBS_ADDR}" ]] && optionals+="-secure-comms-kbs ${SECURE_COMMS_KBS_ADDR} " [[ "${PEERPODS_LIMIT_PER_NODE}" ]] && optionals+="-peerpods-limit-per-node ${PEERPODS_LIMIT_PER_NODE} " diff --git a/src/cloud-api-adaptor/pkg/adaptor/cloud/cloud.go b/src/cloud-api-adaptor/pkg/adaptor/cloud/cloud.go index 7afadd439..f040eb744 100644 --- a/src/cloud-api-adaptor/pkg/adaptor/cloud/cloud.go +++ b/src/cloud-api-adaptor/pkg/adaptor/cloud/cloud.go @@ -16,6 +16,7 @@ import ( "path/filepath" "strings" "sync" + "time" "github.com/containerd/containerd/pkg/cri/annotations" pb "github.com/kata-containers/kata-containers/src/runtime/protocols/hypervisor" @@ -27,6 +28,7 @@ import ( "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/podnetwork" "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/securecomms/wnssh" "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/util" + "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/util/tlsutil" provider "github.com/confidential-containers/cloud-api-adaptor/src/cloud-providers" putil "github.com/confidential-containers/cloud-api-adaptor/src/cloud-providers/util" "github.com/confidential-containers/cloud-api-adaptor/src/cloud-providers/util/cloudinit" @@ -48,6 +50,24 @@ type InitData struct { Data map[string]string `toml:"data,omitempty"` } +type ServerConfig struct { + TLSConfig *tlsutil.TLSConfig + SocketPath string + PauseImage string + PodsDir string + ForwarderPort string + ProxyTimeout time.Duration + Initdata string + EnableCloudConfigVerify bool + SecureComms bool + SecureCommsInbounds string + SecureCommsOutbounds string + SecureCommsPpInbounds string + SecureCommsPpOutbounds string + SecureCommsKbsAddress string + PeerPodsLimitPerNode int +} + var logger = log.New(log.Writer(), "[adaptor/cloud] ", log.LstdFlags|log.Lmsgprefix) func (s *cloudService) addSandbox(sid sandboxID, sandbox *sandbox) error { @@ -87,15 +107,20 @@ func (s *cloudService) removeSandbox(id sandboxID) error { } func NewService(provider provider.Provider, proxyFactory proxy.Factory, workerNode podnetwork.WorkerNode, - secureComms bool, secureCommsInbounds, secureCommsOutbounds, kbsAddress, podsDir, - daemonPort, initdata, sshport string) Service { + serverConfig *ServerConfig, sshport string) Service { var err error var sshClient *wnssh.SshClient - if secureComms { - inbounds := append([]string{"KUBERNETES_PHASE:KATAAGENT:0"}, strings.Split(secureCommsInbounds, ",")...) - outbounds := append([]string{"BOTH_PHASES:KBS:" + kbsAddress}, strings.Split(secureCommsOutbounds, ",")...) - sshClient, err = wnssh.InitSshClient(inbounds, outbounds, kbsAddress, sshport) + if serverConfig.SecureComms { + inbounds := append([]string{"KUBERNETES_PHASE:KATAAGENT:0"}, strings.Split(serverConfig.SecureCommsInbounds, ",")...) + + var outbounds []string + outbounds = append(outbounds, strings.Split(serverConfig.SecureCommsOutbounds, ",")...) + if len(serverConfig.SecureCommsKbsAddress) > 0 { + outbounds = append(outbounds, "BOTH_PHASES:KBS:"+serverConfig.SecureCommsKbsAddress) + } + + sshClient, err = wnssh.InitSshClient(inbounds, outbounds, serverConfig.SecureCommsKbsAddress, sshport) if err != nil { log.Fatalf("InitSshClient %v", err) } @@ -105,9 +130,7 @@ func NewService(provider provider.Provider, proxyFactory proxy.Factory, workerNo provider: provider, proxyFactory: proxyFactory, sandboxes: map[sandboxID]*sandbox{}, - podsDir: podsDir, - daemonPort: daemonPort, - initdata: initdata, + serverConfig: serverConfig, workerNode: workerNode, sshClient: sshClient, } @@ -220,7 +243,7 @@ func (s *cloudService) CreateVM(ctx context.Context, req *pb.CreateVMRequest) (r return nil, fmt.Errorf("failed to inspect netns %s: %w", netNSPath, err) } - podDir := filepath.Join(s.podsDir, string(sid)) + podDir := filepath.Join(s.serverConfig.PodsDir, string(sid)) if err := os.MkdirAll(podDir, os.ModePerm); err != nil { return nil, fmt.Errorf("creating a pod directory: %s, %w", podDir, err) } @@ -264,6 +287,22 @@ func (s *cloudService) CreateVM(ctx context.Context, req *pb.CreateVMRequest) (r daemonConfig.TLSServerKey = string(keyPEM) } + var ci *wnssh.SshClientInstance + + if s.sshClient != nil { + var ppPrivateKey []byte + ci, ppPrivateKey = s.sshClient.InitPP(context.Background(), string(sid)) + if ci == nil { + return nil, fmt.Errorf("failed sshClient.InitPP") + } + + daemonConfig.WnPublicKey = s.sshClient.GetWnPublicKey() + daemonConfig.PpPrivateKey = ppPrivateKey + daemonConfig.SecureCommsOutbounds = s.serverConfig.SecureCommsPpOutbounds + daemonConfig.SecureCommsInbounds = s.serverConfig.SecureCommsPpInbounds + daemonConfig.SecureComms = true + } + daemonJSON, err := json.MarshalIndent(daemonConfig, "", " ") if err != nil { return nil, fmt.Errorf("generating JSON data: %w", err) @@ -304,19 +343,19 @@ func (s *cloudService) CreateVM(ctx context.Context, req *pb.CreateVMRequest) (r logger.Printf("initdata in Pod annotation: %s", initdataStr) if initdataStr == "" { - logger.Printf("initdata in pod annotation is empty, use global initdata: %s", s.initdata) - initdataStr = s.initdata + logger.Printf("initdata in pod annotation is empty, use global initdata: %s", s.serverConfig.Initdata) + initdataStr = s.serverConfig.Initdata } if initdataStr != "" { decodedBytes, err := base64.StdEncoding.DecodeString(initdataStr) if err != nil { - return nil, fmt.Errorf("Error base64 decode initdata: %w", err) + return nil, fmt.Errorf("error base64 decode initdata: %w", err) } initdata := InitData{} err = toml.Unmarshal(decodedBytes, &initdata) if err != nil { - return nil, fmt.Errorf("Error unmarshalling initdata: %w", err) + return nil, fmt.Errorf("error unmarshalling initdata: %w", err) } cloudConfig.WriteFiles = append(cloudConfig.WriteFiles, cloudinit.WriteFile{ @@ -326,14 +365,15 @@ func (s *cloudService) CreateVM(ctx context.Context, req *pb.CreateVMRequest) (r } sandbox := &sandbox{ - id: sid, - podName: pod, - podNamespace: namespace, - netNSPath: netNSPath, - agentProxy: agentProxy, - podNetwork: podNetworkConfig, - cloudConfig: cloudConfig, - spec: vmSpec, + id: sid, + podName: pod, + podNamespace: namespace, + netNSPath: netNSPath, + agentProxy: agentProxy, + podNetwork: podNetworkConfig, + cloudConfig: cloudConfig, + spec: vmSpec, + sshClientInst: ci, } if err := s.addSandbox(sid, sandbox); err != nil { @@ -377,24 +417,16 @@ func (s *cloudService) StartVM(ctx context.Context, req *pb.StartVMRequest) (res logger.Printf("created an instance %s for sandbox %s", instance.Name, sid) instanceIP := instance.IPs[0].String() - forwarderPort := s.daemonPort + forwarderPort := s.serverConfig.ForwarderPort if s.sshClient != nil { - ci := s.sshClient.InitPP(context.Background(), string(sid), instance.IPs) - if ci == nil { - return nil, fmt.Errorf("failed sshClient.InitPP") - } - - if err := ci.Start(); err != nil { + if err := sandbox.sshClientInst.Start(instance.IPs); err != nil { return nil, fmt.Errorf("failed SshClientInstance.Start: %w", err) } // Set agentProxy instanceIP = "127.0.0.1" - forwarderPort = ci.GetPort("KATAAGENT") - - // Set ci in sandbox - sandbox.sshClientInst = ci + forwarderPort = sandbox.sshClientInst.GetPort("KATAAGENT") } if err := s.workerNode.Setup(sandbox.netNSPath, instance.IPs, sandbox.podNetwork); err != nil { diff --git a/src/cloud-api-adaptor/pkg/adaptor/cloud/cloud_test.go b/src/cloud-api-adaptor/pkg/adaptor/cloud/cloud_test.go index a8030ef72..0b829a450 100644 --- a/src/cloud-api-adaptor/pkg/adaptor/cloud/cloud_test.go +++ b/src/cloud-api-adaptor/pkg/adaptor/cloud/cloud_test.go @@ -117,7 +117,14 @@ func TestCloudService(t *testing.T) { podsDir: dir, } - s := NewService(&mockProvider{}, proxyFactory, &mockWorkerNode{}, false, "", "", "", dir, forwarder.DefaultListenPort, "", "") + cfg := &ServerConfig{ + SecureComms: false, + PodsDir: dir, + ForwarderPort: forwarder.DefaultListenPort, + } + + // false, "", "", "", "", "", dir, forwarder.DefaultListenPort, "" + s := NewService(&mockProvider{}, proxyFactory, &mockWorkerNode{}, cfg, "") assert.NotNil(t, s) @@ -160,7 +167,12 @@ func TestCloudServiceWithSecureComms(t *testing.T) { // create a podvm gkc := test.NewGetKeyClient("9019") ctx2, cancel := context.WithCancel(context.Background()) - sshServer := ppssh.NewSshServer([]string{"BOTH_PHASES:KBS:9019"}, []string{"KUBERNETES_PHASE:KATAAGENT:127.0.0.1:7111"}, ppssh.GetSecret(gkc.GetKey), sshport) + + ppSecrets := ppssh.NewPpSecrets(ppssh.GetSecret(gkc.GetKey)) + ppSecrets.AddKey(ppssh.WN_PUBLIC_KEY) + ppSecrets.AddKey(ppssh.PP_PRIVATE_KEY) + + sshServer := ppssh.NewSshServer([]string{"BOTH_PHASES:KBS:9019"}, []string{"KUBERNETES_PHASE:KATAAGENT:127.0.0.1:7111"}, ppSecrets, sshport) _ = sshServer.Start(ctx2) defer func() { cancel() @@ -172,7 +184,14 @@ func TestCloudServiceWithSecureComms(t *testing.T) { podsDir: dir, } - s := NewService(&mockProvider{}, proxyFactory, &mockWorkerNode{}, true, "", "", "127.0.0.1:9009", dir, forwarder.DefaultListenPort, "", sshport) + cfg := &ServerConfig{ + SecureComms: true, + PodsDir: dir, + ForwarderPort: forwarder.DefaultListenPort, + SecureCommsKbsAddress: "127.0.0.1:9009", + } + + s := NewService(&mockProvider{}, proxyFactory, &mockWorkerNode{}, cfg, sshport) assert.NotNil(t, s) diff --git a/src/cloud-api-adaptor/pkg/adaptor/cloud/types.go b/src/cloud-api-adaptor/pkg/adaptor/cloud/types.go index 6d71b6933..e91fff077 100644 --- a/src/cloud-api-adaptor/pkg/adaptor/cloud/types.go +++ b/src/cloud-api-adaptor/pkg/adaptor/cloud/types.go @@ -31,12 +31,10 @@ type cloudService struct { workerNode podnetwork.WorkerNode sandboxes map[sandboxID]*sandbox cond *sync.Cond - podsDir string - daemonPort string mutex sync.Mutex ppService *k8sops.PeerPodService - initdata string sshClient *wnssh.SshClient + serverConfig *ServerConfig } type sandboxID string diff --git a/src/cloud-api-adaptor/pkg/adaptor/server.go b/src/cloud-api-adaptor/pkg/adaptor/server.go index 5539777a5..0eae22fef 100644 --- a/src/cloud-api-adaptor/pkg/adaptor/server.go +++ b/src/cloud-api-adaptor/pkg/adaptor/server.go @@ -10,7 +10,6 @@ import ( "os" "path/filepath" "sync" - "time" "github.com/containerd/ttrpc" pbHypervisor "github.com/kata-containers/kata-containers/src/runtime/protocols/hypervisor" @@ -21,7 +20,6 @@ import ( "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/adaptor/vminfo" "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/podnetwork" "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/securecomms/sshutil" - "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/util/tlsutil" pbPodVMInfo "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/proto/podvminfo" provider "github.com/confidential-containers/cloud-api-adaptor/src/cloud-providers" ) @@ -33,22 +31,6 @@ const ( DefaultPodsDir = "/run/peerpod/pods" ) -type ServerConfig struct { - TLSConfig *tlsutil.TLSConfig - SocketPath string - PauseImage string - PodsDir string - ForwarderPort string - ProxyTimeout time.Duration - Initdata string - EnableCloudConfigVerify bool - SecureComms bool - SecureCommsInbounds string - SecureCommsOutbounds string - SecureCommsKbsAddress string - PeerPodsLimitPerNode int -} - type Server interface { Start(ctx context.Context) error Shutdown() error @@ -68,14 +50,12 @@ type server struct { PeerPodsLimitPerNode int } -func NewServer(provider provider.Provider, cfg *ServerConfig, workerNode podnetwork.WorkerNode) Server { +func NewServer(provider provider.Provider, cfg *cloud.ServerConfig, workerNode podnetwork.WorkerNode) Server { logger.Printf("server config: %#v", cfg) agentFactory := proxy.NewFactory(cfg.PauseImage, cfg.TLSConfig, cfg.ProxyTimeout) - cloudService := cloud.NewService(provider, agentFactory, workerNode, - cfg.SecureComms, cfg.SecureCommsInbounds, cfg.SecureCommsOutbounds, - cfg.SecureCommsKbsAddress, cfg.PodsDir, cfg.ForwarderPort, cfg.Initdata, sshutil.SSHPORT) + cloudService := cloud.NewService(provider, agentFactory, workerNode, cfg, sshutil.SSHPORT) vmInfoService := vminfo.NewService(cloudService) return &server{ diff --git a/src/cloud-api-adaptor/pkg/adaptor/server_test.go b/src/cloud-api-adaptor/pkg/adaptor/server_test.go index baa9fc095..da51ae12b 100644 --- a/src/cloud-api-adaptor/pkg/adaptor/server_test.go +++ b/src/cloud-api-adaptor/pkg/adaptor/server_test.go @@ -13,6 +13,7 @@ import ( "testing" "time" + "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/adaptor/cloud" "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/adaptor/proxy" "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/podnetwork/tunneler" provider "github.com/confidential-containers/cloud-api-adaptor/src/cloud-providers" @@ -150,7 +151,7 @@ func newServer(t *testing.T, socketPath, podsDir string) Server { port := startAgentServer(t) provider := &mockProvider{} - serverConfig := &ServerConfig{ + serverConfig := &cloud.ServerConfig{ SocketPath: socketPath, PodsDir: podsDir, ForwarderPort: port, diff --git a/src/cloud-api-adaptor/pkg/adaptor/shim_test.go b/src/cloud-api-adaptor/pkg/adaptor/shim_test.go index 688973cc4..264ee587c 100644 --- a/src/cloud-api-adaptor/pkg/adaptor/shim_test.go +++ b/src/cloud-api-adaptor/pkg/adaptor/shim_test.go @@ -17,6 +17,7 @@ import ( "testing" "time" + "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/adaptor/cloud" daemon "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/forwarder" "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/forwarder/interceptor" "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/podnetwork" @@ -86,7 +87,7 @@ func TestShim(t *testing.T) { workerNode = podnetwork.NewWorkerNode(t, "", 0, 0) } - serverConfig := &ServerConfig{ + serverConfig := &cloud.ServerConfig{ SocketPath: helperSocketPath, PodsDir: podsDir, ForwarderPort: port, diff --git a/src/cloud-api-adaptor/pkg/forwarder/forwarder.go b/src/cloud-api-adaptor/pkg/forwarder/forwarder.go index fd8a694d0..d175cea85 100644 --- a/src/cloud-api-adaptor/pkg/forwarder/forwarder.go +++ b/src/cloud-api-adaptor/pkg/forwarder/forwarder.go @@ -42,6 +42,12 @@ type Config struct { TLSServerKey string `json:"tls-server-key,omitempty"` TLSServerCert string `json:"tls-server-cert,omitempty"` TLSClientCA string `json:"tls-client-ca,omitempty"` + + PpPrivateKey []byte `json:"sc-pp-prv,omitempty"` + WnPublicKey []byte `json:"sc-wn-pub,omitempty"` + SecureCommsInbounds string `json:"sc-inbounds,omitempty"` + SecureCommsOutbounds string `json:"sc-outbounds,omitempty"` + SecureComms bool `json:"sc,omitempty"` } type Daemon interface { diff --git a/src/cloud-api-adaptor/pkg/securecomms/ppssh/ppsecrets.go b/src/cloud-api-adaptor/pkg/securecomms/ppssh/ppsecrets.go index 1707f6473..48bc6221e 100644 --- a/src/cloud-api-adaptor/pkg/securecomms/ppssh/ppsecrets.go +++ b/src/cloud-api-adaptor/pkg/securecomms/ppssh/ppsecrets.go @@ -5,7 +5,6 @@ import ( ) type PpSecrets struct { - keys []string secrets map[string][]byte getSecret GetSecret } @@ -14,44 +13,54 @@ type GetSecret func(name string) ([]byte, error) func NewPpSecrets(getSecret GetSecret) *PpSecrets { return &PpSecrets{ - keys: []string{}, secrets: make(map[string][]byte), getSecret: getSecret, } } -func (fs *PpSecrets) AddKey(key string) { - fs.keys = append(fs.keys, key) +func (sec *PpSecrets) AddKey(key string) { + if _, ok := sec.secrets[key]; ok { + return + } + sec.secrets[key] = nil } -func (fs *PpSecrets) GetKey(key string) []byte { - return fs.secrets[key] +func (sec *PpSecrets) GetKey(key string) []byte { + return sec.secrets[key] } -func (fs *PpSecrets) Go() { - sleeptime := time.Duration(1) +func (sec *PpSecrets) SetKey(key string, keydata []byte) { + sec.secrets[key] = keydata +} - for len(fs.keys) > 0 { - key := fs.keys[0] - logger.Printf("PpSecrets obtaining key %s", key) +func (sec *PpSecrets) Go() { + sleeptime := time.Duration(1) - data, err := fs.getSecret(key) - if err == nil && len(data) > 0 { - logger.Printf("PpSecrets %s success", key) - fs.secrets[key] = data - fs.keys = fs.keys[1:] + for key, keydata := range sec.secrets { + if keydata != nil { continue } - if err != nil { - logger.Printf("PpSecrets %s getSecret err: %v", key, err) - } else { - logger.Printf("PpSecrets %s getSecret returned an empty secret", key) - } + logger.Printf("PpSecrets obtaining key %s", key) + + // loop until we get a valid key + for { + keydata, err := sec.getSecret(key) + if err == nil && len(keydata) > 0 { + logger.Printf("PpSecrets %s success", key) + sec.secrets[key] = keydata + break + } + if err != nil { + logger.Printf("PpSecrets %s getSecret err: %v", key, err) + } else { + logger.Printf("PpSecrets %s getSecret returned an empty secret", key) + } - time.Sleep(sleeptime * time.Second) - sleeptime *= 2 - if sleeptime > 30 { - sleeptime = 30 + time.Sleep(sleeptime * time.Second) + sleeptime *= 2 + if sleeptime > 30 { + sleeptime = 30 + } } } } diff --git a/src/cloud-api-adaptor/pkg/securecomms/ppssh/ppssh.go b/src/cloud-api-adaptor/pkg/securecomms/ppssh/ppssh.go index d13a24383..08feb291d 100644 --- a/src/cloud-api-adaptor/pkg/securecomms/ppssh/ppssh.go +++ b/src/cloud-api-adaptor/pkg/securecomms/ppssh/ppssh.go @@ -30,7 +30,7 @@ type SshServer struct { outbounds sshproxy.Outbounds wg sync.WaitGroup readyCh chan struct{} - getSecret GetSecret + ppSecrets *PpSecrets sshport string listener net.Listener ctx context.Context @@ -42,9 +42,9 @@ type SshServer struct { // Structure of an inbound tag: "::" // Structure of an outbound tag: ":::" // Phase may be "A" (Attestation), "K" (Kubernetes), or "B" (Both) -func NewSshServer(inbound_strings, outbounds_strings []string, getSecret GetSecret, sshport string) *SshServer { +func NewSshServer(inbound_strings, outbounds_strings []string, ppSecrets *PpSecrets, sshport string) *SshServer { s := &SshServer{ - getSecret: getSecret, + ppSecrets: ppSecrets, sshport: sshport, readyCh: make(chan struct{}), } @@ -119,11 +119,8 @@ func (s *SshServer) attestationPhase() *ssh.ServerConfig { for ctx.Err() == nil { logger.Printf("Attestation phase: getting keys from KBS\n") - ppSecrets := NewPpSecrets(s.getSecret) - ppSecrets.AddKey(WN_PUBLIC_KEY) - ppSecrets.AddKey(PP_PRIVATE_KEY) - ppSecrets.Go() // wait for the keys - config, err := initKubernetesPhaseSshConfig(ppSecrets) + s.ppSecrets.Go() // wait for the keys + config, err := initKubernetesPhaseSshConfig(s.ppSecrets) if err == nil { logger.Printf("Attestation phase: InitKubernetesPhaseSshConfig is ready\n") peer.Upgrade() diff --git a/src/cloud-api-adaptor/pkg/securecomms/ppssh/ppssh_test.go b/src/cloud-api-adaptor/pkg/securecomms/ppssh/ppssh_test.go index c6746a8e3..bee6f6b6d 100644 --- a/src/cloud-api-adaptor/pkg/securecomms/ppssh/ppssh_test.go +++ b/src/cloud-api-adaptor/pkg/securecomms/ppssh/ppssh_test.go @@ -183,7 +183,12 @@ func TestPpssh(t *testing.T) { // Forwarder Initialization ctx := context.Background() ctx, cancel := context.WithCancel(ctx) - sshServer := NewSshServer([]string{"BOTH_PHASES:KBS:9002"}, []string{"KUBERNETES_PHASE:ABC:127.0.0.1:7105"}, GetSecret(getKey), sshport) + + ppSecrets := NewPpSecrets(GetSecret(getKey)) + ppSecrets.AddKey(WN_PUBLIC_KEY) + ppSecrets.AddKey(PP_PRIVATE_KEY) + + sshServer := NewSshServer([]string{"BOTH_PHASES:KBS:9002"}, []string{"KUBERNETES_PHASE:ABC:127.0.0.1:7105"}, ppSecrets, sshport) _ = sshServer.Start(ctx) clientSshPeer, conn := getAttestationClient(t, sshport) clientSshPeer.AddTags(inbounds, outbounds) diff --git a/src/cloud-api-adaptor/pkg/securecomms/wnssh/wnssh.go b/src/cloud-api-adaptor/pkg/securecomms/wnssh/wnssh.go index d252d609f..87a962805 100644 --- a/src/cloud-api-adaptor/pkg/securecomms/wnssh/wnssh.go +++ b/src/cloud-api-adaptor/pkg/securecomms/wnssh/wnssh.go @@ -25,6 +25,7 @@ type SshClient struct { inboundStrings []string outboundStrings []string sshport string + wnPublicKey []byte } type SshClientInstance struct { @@ -72,22 +73,25 @@ func InitSshClient(inbound_strings, outbound_strings []string, kbsAddress string return nil, fmt.Errorf("unable to parse private key: %v", err) } - kbscPrivateKey, _, err := kubemgr.KubeMgr.ReadSecret(sshutil.KBS_CLIENT_SECRET) - if err != nil { - return nil, fmt.Errorf("failed to read KBS client secret: %w", err) - } + var kc *KbsClient + if len(kbsAddress) > 0 { + kbscPrivateKey, _, err := kubemgr.KubeMgr.ReadSecret(sshutil.KBS_CLIENT_SECRET) + if err != nil { + return nil, fmt.Errorf("failed to read KBS client secret: %w", err) + } - kc := InitKbsClient(kbsAddress) - err = kc.SetPemSecret(kbscPrivateKey) - if err != nil { - return nil, fmt.Errorf("KbsClient - %v", err) - } + kc = InitKbsClient(kbsAddress) + err = kc.SetPemSecret(kbscPrivateKey) + if err != nil { + return nil, fmt.Errorf("KbsClient - %v", err) + } - wnSecretPath := "default/sshclient/publicKey" - logger.Printf("Updating KBS with secret for: %s", wnSecretPath) - err = kc.PostResource(wnSecretPath, wnPublicKey) - if err != nil { - return nil, fmt.Errorf("failed to PostResource WN Secret: %v", err) + wnSecretPath := "default/sshclient/publicKey" + logger.Printf("Updating KBS with secret for: %s", wnSecretPath) + err = kc.PostResource(wnSecretPath, wnPublicKey) + if err != nil { + return nil, fmt.Errorf("failed to PostResource WN Secret: %v", err) + } } sshClient := &SshClient{ @@ -96,6 +100,7 @@ func InitSshClient(inbound_strings, outbound_strings []string, kbsAddress string inboundStrings: inbound_strings, outboundStrings: outbound_strings, sshport: sshport, + wnPublicKey: wnPublicKey, } return sshClient, nil @@ -123,9 +128,13 @@ func (ci *SshClientInstance) DisconnectPP(sid string) { kubemgr.KubeMgr.DeleteSecret(PpSecretName(sid)) } -func (c *SshClient) InitPP(ctx context.Context, sid string, ipAddr []netip.Addr) *SshClientInstance { +func (c *SshClient) GetWnPublicKey() []byte { + return c.wnPublicKey +} + +func (c *SshClient) InitPP(ctx context.Context, sid string) (ci *SshClientInstance, ppPrivateKey []byte) { // Create peerPod Secret named peerPodId - var ppPublicKey, ppPrivateKey []byte + var ppPublicKey []byte var err error var kubernetesPhase bool @@ -136,43 +145,39 @@ func (c *SshClient) InitPP(ctx context.Context, sid string, ipAddr []netip.Addr) ppPrivateKey, ppPublicKey, err = kubemgr.KubeMgr.CreateSecret(PpSecretName(sid)) if err != nil { logger.Printf("Failed to create PP secret: %v", err) - return nil + return } } else { // we already have a store secret for this PP kubernetesPhase = true } - // >>> Update the KBS about the SID's Secret !!! <<< - sidSecretPath := fmt.Sprintf("default/pp-%s/privateKey", sid) - logger.Printf("Updating KBS with secret for: %s", sidSecretPath) - err = c.kc.PostResource(sidSecretPath, ppPrivateKey) - if err != nil { - logger.Printf("Failed to PostResource PP Secret: %v", err) - return nil - } + if c.kc != nil { + // >>> Update the KBS about the SID's Secret !!! <<< + sidSecretPath := fmt.Sprintf("default/pp-%s/privateKey", sid) + logger.Printf("Updating KBS with secret for: %s", sidSecretPath) + err = c.kc.PostResource(sidSecretPath, ppPrivateKey) + if err != nil { + logger.Printf("Failed to PostResource PP Secret: %v", err) + return + } + } var ppSshPublicKeyBytes []byte if len(ppPublicKey) > 0 { ppSshPublicKey, _, _, _, err := ssh.ParseAuthorizedKey(ppPublicKey) if err != nil { logger.Printf("Unable to ParseAuthorizedKey serverPublicKey: %v", err) - return nil + return } ppSshPublicKeyBytes = ppSshPublicKey.Marshal() } - ppAddr := make([]string, len(ipAddr)) - for i, ip := range ipAddr { - ppAddr[i] = ip.String() + ":" + c.sshport - } - ctx, cancel := context.WithCancel(ctx) - ci := &SshClientInstance{ + ci = &SshClientInstance{ sid: sid, ppPublicKey: ppSshPublicKeyBytes, - ppAddr: ppAddr, sshClient: c, ctx: ctx, cancel: cancel, @@ -188,10 +193,17 @@ func (c *SshClient) InitPP(ctx context.Context, sid string, ipAddr []netip.Addr) logger.Fatalf("Failed to parse outbound tag %v: %v", c.outboundStrings, err) } - return ci + return } -func (ci *SshClientInstance) Start() error { +func (ci *SshClientInstance) Start(ipAddr []netip.Addr) error { + ppAddr := make([]string, len(ipAddr)) + for i, ip := range ipAddr { + ppAddr[i] = ip.String() + ":" + ci.sshClient.sshport + } + + ci.ppAddr = ppAddr + if !ci.kubernetesPhase { // Attestation phase logger.Println("Attestation phase: starting") diff --git a/src/cloud-api-adaptor/pkg/securecomms/wnssh/wnssh_test.go b/src/cloud-api-adaptor/pkg/securecomms/wnssh/wnssh_test.go index 64b6d3304..ecfb3f440 100644 --- a/src/cloud-api-adaptor/pkg/securecomms/wnssh/wnssh_test.go +++ b/src/cloud-api-adaptor/pkg/securecomms/wnssh/wnssh_test.go @@ -32,7 +32,7 @@ func TestSshProxyReverseKBS(t *testing.T) { ////////// CAA StartVM ipAddr, _ := netip.ParseAddr("127.0.0.1") // ipAddr of the VM ipAddrs := []netip.Addr{ipAddr} - ci := sshClient.InitPP(context.Background(), "sid", ipAddrs) + ci, _ := sshClient.InitPP(context.Background(), "sid") if ci == nil { log.Fatalf("failed InitiatePeerPodTunnel") } @@ -46,12 +46,17 @@ func TestSshProxyReverseKBS(t *testing.T) { // create a podvm gkc := test.NewGetKeyClient("7030") ctx2, cancel2 := context.WithCancel(context.Background()) - sshServer := ppssh.NewSshServer([]string{"BOTH_PHASES:KBS:7030", "KUBERNETES_PHASE:KUBEAPI:16443", "KUBERNETES_PHASE:DNS:9053"}, []string{"KUBERNETES_PHASE:KATAAGENT:127.0.0.1:7121"}, ppssh.GetSecret(gkc.GetKey), sshport) + + ppSecrets := ppssh.NewPpSecrets(ppssh.GetSecret(gkc.GetKey)) + ppSecrets.AddKey(ppssh.WN_PUBLIC_KEY) + ppSecrets.AddKey(ppssh.PP_PRIVATE_KEY) + + sshServer := ppssh.NewSshServer([]string{"BOTH_PHASES:KBS:7030", "KUBERNETES_PHASE:KUBEAPI:16443", "KUBERNETES_PHASE:DNS:9053"}, []string{"KUBERNETES_PHASE:KATAAGENT:127.0.0.1:7121"}, ppSecrets, sshport) _ = sshServer.Start(ctx2) // Forwarder Initialization - if err := ci.Start(); err != nil { + if err := ci.Start(ipAddrs); err != nil { log.Fatalf("failed ci.Start: %s", err) } diff --git a/src/cloud-api-adaptor/test/securecomms/ppssh.go b/src/cloud-api-adaptor/test/securecomms/ppssh.go index b9a15d82d..337f3f39e 100644 --- a/src/cloud-api-adaptor/test/securecomms/ppssh.go +++ b/src/cloud-api-adaptor/test/securecomms/ppssh.go @@ -20,7 +20,12 @@ func PP() { // Forwarder Initialization ctx := context.Background() ctx, cancel := context.WithCancel(ctx) - sshServer := ppssh.NewSshServer([]string{"BOTH_PHASES:KBS:7000", "KUBERNETES_PHASE:KUBEAPI:16443", "KUBERNETES_PHASE:DNS:9053"}, []string{"KUBERNETES_PHASE:KATAAGENT:127.0.0.1:7131"}, ppssh.GetSecret(getKey), sshutil.SSHPORT) + + ppSecrets := ppssh.NewPpSecrets(ppssh.GetSecret(getKey)) + ppSecrets.AddKey(ppssh.WN_PUBLIC_KEY) + ppSecrets.AddKey(ppssh.PP_PRIVATE_KEY) + + sshServer := ppssh.NewSshServer([]string{"BOTH_PHASES:KBS:7000", "KUBERNETES_PHASE:KUBEAPI:16443", "KUBERNETES_PHASE:DNS:9053"}, []string{"KUBERNETES_PHASE:KATAAGENT:127.0.0.1:7131"}, ppSecrets, sshutil.SSHPORT) _ = sshServer.Start(ctx) time.Sleep(1 * time.Minute) cancel() diff --git a/src/cloud-api-adaptor/test/securecomms/wnssh.go b/src/cloud-api-adaptor/test/securecomms/wnssh.go index 026125e03..406d78c48 100644 --- a/src/cloud-api-adaptor/test/securecomms/wnssh.go +++ b/src/cloud-api-adaptor/test/securecomms/wnssh.go @@ -30,7 +30,7 @@ func WN() bool { ipAddr, _ := netip.ParseAddr("127.0.0.1") // ipAddr of the VM ipAddrs := []netip.Addr{ipAddr} ctx := context.Background() - ci := sshClient.InitPP(ctx, "sid", ipAddrs) + ci, _ := sshClient.InitPP(ctx, "sid") if ci == nil { log.Fatalf("failed InitiatePeerPodTunnel") } @@ -41,7 +41,7 @@ func WN() bool { log.Fatalf("failed find port") } - if err := ci.Start(); err != nil { + if err := ci.Start(ipAddrs); err != nil { log.Fatalf("failed ci.Start: %s", err) }