[DO NOT MERGE] move openshift-sdn pod network setup to a CNI plugin

contrib/node/install-sdn.sh

@@ -1,36 +1,32 @@
 #!/bin/bash
 
 os::provision::install-sdn() {
-  local default_target="/usr"
-
   local deployed_root=$1
-  local target=${2:-${default_target}}
+  local target=${2:-}
+  local target_usrdir="${target}/usr"
+  local target_bindir="${target_usrdir}/bin"
+  local target_etcdir="${target}/etc"
+  local target_cnidir="${target}/opt/cni/bin"
 
-  if [ ! -d ${target} ]; then
-    mkdir -p ${target}
-  fi
+  mkdir -p ${target_usrdir}
+  mkdir -p ${target_bindir}
+  mkdir -p ${target_etcdir}
+  mkdir -p ${target_cnidir}
 
   local osdn_plugin_path="${deployed_root}/pkg/sdn/plugin"
-  mkdir -p "${target}/bin/"
   pushd "${osdn_plugin_path}" > /dev/null
-    install bin/openshift-sdn-ovs "${target}/bin/"
-    install bin/openshift-sdn-docker-setup.sh "${target}/bin/"
+    install sdn-cni-plugin/openshift-sdn-ovs "${target_bindir}"
   popd > /dev/null
 
-  # osdn plugin setup writes docker network options to
-  # /run/openshift-sdn/docker-network, make this file to be exported
-  # as part of docker service start.
-  local system_docker_path="${target}/lib/systemd/system/docker.service.d/"
-  mkdir -p "${system_docker_path}"
-  cat <<EOF > "${system_docker_path}/docker-sdn-ovs.conf"
-[Service]
-EnvironmentFile=-/run/openshift-sdn/docker-network
-EOF
+  # openshift-sdn places a CNI config file here
+  mkdir -p "${target_etcdir}/cni/net.d"
+
+  install "${OS_OUTPUT_BINPATH}/$(os::build::host_platform)/sdn-cni-plugin" "${target_cnidir}/openshift-sdn"
+  install "${OS_OUTPUT_BINPATH}/$(os::build::host_platform)/host-local" "${target_cnidir}/host-local"
 
-  # Assume a non-default target is an indication of deploying in an
-  # environment where openvswitch is managed in a separate container
-  # (e.g. atomic host).
-  if [[ "${target}" = "${default_target}" ]]; then
+  # Assume an empty/default target is an indication of deploying in an
+  # environment where openvswitch should be started by us
+  if [[ -z "${target}" ]]; then
     systemctl enable openvswitch
     systemctl start openvswitch
   fi

hack/common.sh

@@ -17,10 +17,15 @@ readonly OS_GO_PACKAGE=github.com/openshift/origin
 readonly OS_IMAGE_COMPILE_PLATFORMS=(
   linux/amd64
 )
+readonly OS_SDN_COMPILE_TARGETS_LINUX_AMD64=(
+  pkg/sdn/plugin/sdn-cni-plugin
+  vendor/github.com/containernetworking/cni/plugins/ipam/host-local
+)
 readonly OS_IMAGE_COMPILE_TARGETS=(
   images/pod
   cmd/dockerregistry
   cmd/gitserver
+  "${OS_SDN_COMPILE_TARGETS_LINUX_AMD64[@]}"
 )
 readonly OS_IMAGE_COMPILE_GOFLAGS="-tags include_gcs"
 readonly OS_SCRATCH_IMAGE_COMPILE_TARGETS=(
@@ -274,17 +279,7 @@ os::build::internal::build_binaries() {
       fi
     done
 
-    os::build::export_targets "$@"
-
-    local -a nonstatics=()
-    local -a tests=()
-    for binary in "${binaries[@]}"; do
-      if [[ "${binary}" =~ ".test"$ ]]; then
-        tests+=($binary)
-      else
-        nonstatics+=($binary)
-      fi
-    done
+    os::build::export_platforms
 
     # Temporarily enable swap for the duration of the build until we move
     # to Go 1.7
@@ -294,6 +289,18 @@ os::build::internal::build_binaries() {
     local host_platform=$(os::build::host_platform)
     local platform
     for platform in "${platforms[@]}"; do
+      os::build::export_targets_and_binaries "${platform}" "$@"
+
+      local -a nonstatics=()
+      local -a tests=()
+      for binary in "${binaries[@]}"; do
+        if [[ "${binary}" =~ ".test"$ ]]; then
+          tests+=($binary)
+        else
+          nonstatics+=($binary)
+        fi
+      done
+
       echo "++ Building go targets for ${platform}:" "${targets[@]}"
       mkdir -p "${OS_OUTPUT_BINPATH}/${platform}"
 
@@ -336,10 +343,13 @@ os::build::internal::build_binaries() {
 }
 readonly -f os::build::build_binaries
 
-# Generates the set of target packages, binaries, and platforms to build for.
-# Accepts binaries via $@, and platforms via OS_BUILD_PLATFORMS, or defaults to
-# the current platform.
-function os::build::export_targets() {
+# Generates the set of target packages and binaries (as full go package) to build
+# a for given platform.  First argument is platform, remaining arguments are
+# targets.  Targets can be given as full Go package path or as basenames.
+function os::build::export_targets_and_binaries() {
+  local platform=${1}
+  shift
+
   targets=()
   local arg
   for arg; do
@@ -349,17 +359,26 @@ function os::build::export_targets() {
   done
 
   if [[ ${#targets[@]} -eq 0 ]]; then
-    targets=("${OS_ALL_TARGETS[@]}")
+    if [[ "${platform}" == linux/amd64 ]]; then
+      targets=("${OS_ALL_TARGETS[@]}" "${OS_SDN_COMPILE_TARGETS_LINUX_AMD64[@]}")
+    else
+      targets=("${OS_ALL_TARGETS[@]}")
+    fi
   fi
 
   binaries=($(os::build::binaries_from_targets "${targets[@]}"))
+}
+readonly -f os::build::export_targets_and_binaries
 
+# Generates the set of target platforms to build for. Accepts platforms via
+# OS_BUILD_PLATFORMS, or defaults to the current platform.
+function os::build::export_platforms() {
   platforms=("${OS_BUILD_PLATFORMS[@]:+${OS_BUILD_PLATFORMS[@]}}")
   if [[ ${#platforms[@]} -eq 0 ]]; then
     platforms=("$(os::build::host_platform)")
   fi
 }
-readonly -f os::build::export_targets
+readonly -f os::build::export_platforms
 
 # This will take $@ from $GOPATH/bin and copy them to the appropriate
 # place in ${OS_OUTPUT_BINDIR}
@@ -384,8 +403,10 @@ function os::build::place_bins() {
       mkdir -p "${OS_LOCAL_RELEASEPATH}"
     fi
 
-    os::build::export_targets "$@"
+    os::build::export_platforms
     for platform in "${platforms[@]}"; do
+      os::build::export_targets_and_binaries "${platform}" "$@"
+
       # The substitution on platform_src below will replace all slashes with
       # underscores.  It'll transform darwin/amd64 -> darwin_amd64.
       local platform_src="/${platform//\//_}"
@@ -396,13 +417,13 @@ function os::build::place_bins() {
       fi
 
       # Create an array of binaries to release. Append .exe variants if the platform is windows.
-      local -a binaries=()
-      for binary in "${targets[@]}"; do
+      local -a binary_names=()
+      for binary in "${binaries[@]}"; do
         binary=$(basename $binary)
         if [[ $platform == "windows/amd64" ]]; then
-          binaries+=("${binary}.exe")
+          binary_names+=("${binary}.exe")
         else
-          binaries+=("${binary}")
+          binary_names+=("${binary}")
         fi
       done
 
@@ -413,7 +434,7 @@ function os::build::place_bins() {
 
       # Create a temporary bin directory containing only the binaries marked for release.
       local release_binpath=$(mktemp -d openshift.release.${OS_RELEASE_ARCHIVE}.XXX)
-      for binary in "${binaries[@]}"; do
+      for binary in "${binary_names[@]}"; do
         cp "${OS_OUTPUT_BINPATH}/${platform}/${binary}" "${release_binpath}/"
       done
 

images/node/Dockerfile

@@ -11,8 +11,8 @@
 FROM openshift/origin
 
 COPY bin/* /usr/bin/
+COPY opt/cni/bin/* /opt/cni/bin/
 COPY conf/openshift-sdn-ovs.conf /usr/lib/systemd/system/origin-node.service.d/
-COPY lib/systemd/system/docker.service.d/docker-sdn-ovs.conf /usr/lib/systemd/system/docker.service.d/docker-sdn-ovs.conf
 COPY scripts/* /usr/local/bin/
 
 RUN curl -L -o /etc/yum.repos.d/origin-next-epel-7.repo https://copr.fedoraproject.org/coprs/maxamillion/origin-next/repo/epel-7/maxamillion-origin-next-epel-7.repo && \

images/node/scripts/origin-node-run.sh

@@ -2,16 +2,10 @@
 
 set -eu
 
-hostetc=${HOST_ETC:-/rootfs/etc}
 conf=${CONFIG_FILE:-/etc/origin/node/node-config.yaml}
 opts=${OPTIONS:---loglevel=2}
 if [ "$#" -ne 0 ]; then
   opts=""
 fi
 
-if [ ! -f ${hostetc}/systemd/system/docker.service.d/docker-sdn-ovs.conf ]; then
-    mkdir -p ${hostetc}/systemd/system/docker.service.d
-    cp /usr/lib/systemd/system/docker.service.d/docker-sdn-ovs.conf ${hostetc}/systemd/system/docker.service.d
-fi
-
 exec /usr/bin/openshift start node "--config=${conf}" "${opts}" $@

origin.spec

@@ -267,12 +267,13 @@ mkdir -p %{buildroot}%{_sharedstatedir}/origin
 
 
 # Install sdn scripts
-install -d -m 0755 %{buildroot}%{_unitdir}/docker.service.d
-install -p -m 0644 contrib/systemd/docker-sdn-ovs.conf %{buildroot}%{_unitdir}/docker.service.d/
-pushd pkg/sdn/plugin/bin
+pushd pkg/sdn/plugin/sdn-cni-plugin
    install -p -m 755 openshift-sdn-ovs %{buildroot}%{_bindir}/openshift-sdn-ovs
-   install -p -m 755 openshift-sdn-docker-setup.sh %{buildroot}%{_bindir}/openshift-sdn-docker-setup.sh
 popd
+install -d -m 0755 %{buildroot}/opt/cni/bin
+install -p -m 755 _build/bin/sdn-cni-plugin %{buildroot}/opt/cni/bin/openshift-sdn
+install -p -m 755 _build/bin/host-local %{buildroot}/opt/cni/bin/host-local
+
 install -d -m 0755 %{buildroot}%{_unitdir}/%{name}-node.service.d
 install -p -m 0644 contrib/systemd/openshift-sdn-ovs.conf %{buildroot}%{_unitdir}/%{name}-node.service.d/openshift-sdn-ovs.conf
 
@@ -418,9 +419,7 @@ fi
 %dir %{_unitdir}/docker.service.d/
 %dir %{_unitdir}/%{name}-node.service.d/
 %{_bindir}/openshift-sdn-ovs
-%{_bindir}/openshift-sdn-docker-setup.sh
 %{_unitdir}/%{name}-node.service.d/openshift-sdn-ovs.conf
-%{_unitdir}/docker.service.d/docker-sdn-ovs.conf
 
 %posttrans sdn-ovs
 # This path was installed by older packages but the directory wasn't owned by

pkg/cmd/server/kubernetes/node_config.go

@@ -22,6 +22,7 @@ import (
 	clientadapter "k8s.io/kubernetes/pkg/client/unversioned/adapters/internalclientset"
 	"k8s.io/kubernetes/pkg/kubelet"
 	"k8s.io/kubernetes/pkg/kubelet/dockertools"
+	kubeletcni "k8s.io/kubernetes/pkg/kubelet/network/cni"
 	kubeletserver "k8s.io/kubernetes/pkg/kubelet/server"
 	kubelettypes "k8s.io/kubernetes/pkg/kubelet/types"
 	kcrypto "k8s.io/kubernetes/pkg/util/crypto"
@@ -77,7 +78,7 @@ type NodeConfig struct {
 	DNSServer *dns.Server
 
 	// SDNPlugin is an optional SDN plugin
-	SDNPlugin sdnpluginapi.OsdnNodePlugin
+	SDNPlugin *sdnplugin.OsdnNode
 	// EndpointsFilterer is an optional endpoints filterer
 	FilteringEndpointsHandler sdnpluginapi.FilteringEndpointsConfigHandler
 }
@@ -165,12 +166,6 @@ func BuildKubernetesNodeConfig(options configapi.NodeConfig, enableProxy, enable
 	}
 	server.DockerExecHandlerName = string(options.DockerConfig.ExecHandlerName)
 
-	if sdnplugin.IsOpenShiftNetworkPlugin(server.NetworkPluginName) {
-		// set defaults for openshift-sdn
-		server.HairpinMode = componentconfig.HairpinNone
-		server.ConfigureCBR0 = false
-	}
-
 	// prevents kube from generating certs
 	server.TLSCertFile = options.ServingInfo.ServerCert.CertFile
 	server.TLSPrivateKeyFile = options.ServingInfo.ServerCert.KeyFile
@@ -190,6 +185,23 @@ func BuildKubernetesNodeConfig(options configapi.NodeConfig, enableProxy, enable
 		return nil, err
 	}
 
+	// Initialize SDN before building kubelet config so it can modify options
+	iptablesSyncPeriod, err := time.ParseDuration(options.IPTablesSyncPeriod)
+	if err != nil {
+		return nil, fmt.Errorf("Cannot parse the provided ip-tables sync period (%s) : %v", options.IPTablesSyncPeriod, err)
+	}
+	sdnPlugin, err := sdnplugin.NewNodePlugin(options.NetworkConfig.NetworkPluginName, originClient, kubeClient, options.NodeName, options.NodeIP, iptablesSyncPeriod, options.NetworkConfig.MTU, options.MasterKubeConfig)
+	if err != nil {
+		return nil, fmt.Errorf("SDN initialization failed: %v", err)
+	}
+	if sdnPlugin != nil {
+		// SDN plugin pod setup/teardown is implemented as a CNI plugin
+		server.NetworkPluginName = kubeletcni.CNIPluginName
+		server.NetworkPluginDir = kubeletcni.DefaultNetDir
+		server.HairpinMode = componentconfig.HairpinNone
+		server.ConfigureCBR0 = false
+	}
+
 	deps, err := kubeletapp.UnsecuredKubeletDeps(server)
 	if err != nil {
 		return nil, err
@@ -250,18 +262,6 @@ func BuildKubernetesNodeConfig(options configapi.NodeConfig, enableProxy, enable
 		deps.TLSOptions = nil
 	}
 
-	iptablesSyncPeriod, err := time.ParseDuration(options.IPTablesSyncPeriod)
-	if err != nil {
-		return nil, fmt.Errorf("Cannot parse the provided ip-tables sync period (%s) : %v", options.IPTablesSyncPeriod, err)
-	}
-	sdnPlugin, err := sdnplugin.NewNodePlugin(options.NetworkConfig.NetworkPluginName, originClient, kubeClient, options.NodeName, options.NodeIP, iptablesSyncPeriod, options.NetworkConfig.MTU)
-	if err != nil {
-		return nil, fmt.Errorf("SDN initialization failed: %v", err)
-	}
-	if sdnPlugin != nil {
-		deps.NetworkPlugins = append(deps.NetworkPlugins, sdnPlugin)
-	}
-
 	endpointFilter, err := sdnplugin.NewProxyPlugin(options.NetworkConfig.NetworkPluginName, originClient, kubeClient)
 	if err != nil {
 		return nil, fmt.Errorf("SDN proxy initialization failed: %v", err)