initial windows port.

cmd/k3s/main.go

@@ -7,6 +7,7 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"runtime"
 	"strconv"
 	"strings"
 	"syscall"
@@ -46,8 +47,8 @@ func main() {
 	app := cmds.NewApp()
 	app.EnableBashCompletion = true
 	app.Commands = []cli.Command{
-		cmds.NewServerCommand(internalCLIAction(version.Program+"-server", dataDir, os.Args)),
-		cmds.NewAgentCommand(internalCLIAction(version.Program+"-agent", dataDir, os.Args)),
+		cmds.NewServerCommand(internalCLIAction(version.Program+"-server"+version.ProgramPostfix, dataDir, os.Args)),
+		cmds.NewAgentCommand(internalCLIAction(version.Program+"-agent"+version.ProgramPostfix, dataDir, os.Args)),
 		cmds.NewKubectlCommand(externalCLIAction("kubectl", dataDir)),
 		cmds.NewCRICTL(externalCLIAction("crictl", dataDir)),
 		cmds.NewCtrCommand(externalCLIAction("ctr", dataDir)),
@@ -158,7 +159,7 @@ func externalCLI(cli, dataDir string, args []string) error {
 			os.Setenv("CRI_CONFIG_FILE", findCriConfig(dataDir))
 		}
 	}
-	return stageAndRun(dataDir, cli, append([]string{cli}, args...))
+	return stageAndRun(dataDir, cli, append([]string{cli}, args...), false)
 }
 
 // internalCLIAction returns a function that will call a K3s internal command, be used as the Action of a cli.Command.
@@ -174,11 +175,11 @@ func internalCLIAction(cmd, dataDir string, args []string) func(ctx *cli.Context
 
 // stageAndRunCLI calls an external binary.
 func stageAndRunCLI(cli *cli.Context, cmd string, dataDir string, args []string) error {
-	return stageAndRun(dataDir, cmd, args)
+	return stageAndRun(dataDir, cmd, args, true)
 }
 
 // stageAndRun does the actual work of setting up and calling an external binary.
-func stageAndRun(dataDir, cmd string, args []string) error {
+func stageAndRun(dataDir, cmd string, args []string, calledAsInternal bool) error {
 	dir, err := extract(dataDir)
 	if err != nil {
 		return errors.Wrap(err, "extracting data")
@@ -187,9 +188,9 @@ func stageAndRun(dataDir, cmd string, args []string) error {
 
 	var pathEnv string
 	if findPreferBundledBin(args) {
-		pathEnv = filepath.Join(dir, "bin") + ":" + filepath.Join(dir, "bin/aux") + ":" + os.Getenv("PATH")
+		pathEnv = filepath.Join(dir, "bin") + string(os.PathListSeparator) + filepath.Join(dir, "bin/aux") + string(os.PathListSeparator) + os.Getenv("PATH")
 	} else {
-		pathEnv = filepath.Join(dir, "bin") + ":" + os.Getenv("PATH") + ":" + filepath.Join(dir, "bin/aux")
+		pathEnv = filepath.Join(dir, "bin") + string(os.PathListSeparator) + os.Getenv("PATH") + string(os.PathListSeparator) + filepath.Join(dir, "bin/aux")
 	}
 	if err := os.Setenv("PATH", pathEnv); err != nil {
 		return err
@@ -205,6 +206,20 @@ func stageAndRun(dataDir, cmd string, args []string) error {
 
 	logrus.Debugf("Running %s %v", cmd, args)
 
+	if runtime.GOOS == "windows" {
+		// syscall.Exec: not supported by windows
+		if calledAsInternal {
+			args = args[1:]
+		}
+		cmd := exec.Command(cmd, args...)
+		cmd.Stdout = os.Stdout
+		cmd.Stderr = os.Stderr
+		cmd.Stdin = os.Stdin
+		cmd.Env = os.Environ()
+		err := cmd.Run()
+		return err
+	}
+
 	return syscall.Exec(cmd, args, os.Environ())
 }
 
@@ -221,13 +236,13 @@ func getAssetAndDir(dataDir string) (string, string) {
 func extract(dataDir string) (string, error) {
 	// first look for global asset folder so we don't create a HOME version if not needed
 	_, dir := getAssetAndDir(datadir.DefaultDataDir)
-	if _, err := os.Stat(filepath.Join(dir, "bin", "k3s")); err == nil {
+	if _, err := os.Stat(filepath.Join(dir, "bin", "k3s"+version.ProgramPostfix)); err == nil {
 		return dir, nil
 	}
 
 	asset, dir := getAssetAndDir(dataDir)
 	// check if target content already exists
-	if _, err := os.Stat(filepath.Join(dir, "bin", "k3s")); err == nil {
+	if _, err := os.Stat(filepath.Join(dir, "bin", "k3s"+version.ProgramPostfix)); err == nil {
 		return dir, nil
 	}
 

go.mod

@@ -3,7 +3,7 @@ module github.com/k3s-io/k3s
 go 1.19
 
 replace (
-	github.com/Microsoft/hcsshim => github.com/Microsoft/hcsshim v0.8.22
+	github.com/Microsoft/hcsshim => github.com/Microsoft/hcsshim v0.9.5
 	github.com/Mirantis/cri-dockerd => github.com/k3s-io/cri-dockerd v0.3.2-0.20230123224936-bcd78c2d21d8 // k3s/release-1.26
 	github.com/cloudnativelabs/kube-router/v2 => github.com/k3s-io/kube-router/v2 v2.0.1-0.20230405162624-e18008d495ef
 	github.com/containerd/cgroups => github.com/containerd/cgroups v1.0.1

pkg/cli/crictl/crictl.go

@@ -1,11 +1,17 @@
 package crictl
 
 import (
+	"os"
+	"runtime"
+
 	"github.com/kubernetes-sigs/cri-tools/cmd/crictl"
 	"github.com/urfave/cli"
 )
 
 func Run(ctx *cli.Context) error {
+	if runtime.GOOS == "windows" {
+		os.Args = os.Args[1:]
+	}
 	crictl.Main()
 	return nil
 }

pkg/kubectl/main.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"math/rand"
 	"os"
+	"runtime"
 	"strings"
 	"time"
 
@@ -15,6 +16,9 @@ import (
 )
 
 func Main() {
+	if runtime.GOOS == "windows" {
+		os.Args = os.Args[1:]
+	}
 	kubenv := os.Getenv("KUBECONFIG")
 	for i, arg := range os.Args {
 		if strings.HasPrefix(arg, "--kubeconfig=") {

pkg/version/version_linux.go

@@ -0,0 +1,8 @@
+//go:build linux
+// +build linux
+
+package version
+
+var (
+	ProgramPostfix = ""
+)

pkg/version/version_windows.go

@@ -0,0 +1,8 @@
+//go:build windows
+// +build windows
+
+package version
+
+var (
+	ProgramPostfix = ".exe"
+)

scripts/build

@@ -68,6 +68,10 @@ TAGS="apparmor seccomp netcgo osusergo providerless urfave_cli_no_docs"
 RUNC_TAGS="apparmor seccomp"
 RUNC_STATIC="static"
 
+if [ ${OS} = windows ]; then
+    TAGS="netcgo osusergo providerless"
+fi
+
 if [ "$SELINUX" = "true" ]; then
     TAGS="$TAGS selinux"
     RUNC_TAGS="$RUNC_TAGS selinux"
@@ -92,21 +96,41 @@ if [ ${ARCH} = s390x ]; then
     export GOARCH="s390x"
 fi
 
-rm -f \
-    bin/k3s-agent \
-    bin/k3s-server \
-    bin/k3s-token \
-    bin/k3s-etcd-snapshot \
-    bin/k3s-secrets-encrypt \
-    bin/k3s-certificate \
-    bin/k3s-completion \
-    bin/kubectl \
-    bin/crictl \
-    bin/ctr \
-    bin/containerd \
-    bin/containerd-shim \
-    bin/containerd-shim-runc-v2 \
-    bin/runc
+k3s_binaries=(
+    "bin/k3s-agent"
+    "bin/k3s-server"
+    "bin/k3s-token"
+    "bin/k3s-etcd-snapshot"
+    "bin/k3s-secrets-encrypt"
+    "bin/k3s-certificate"
+    "bin/k3s-completion"
+    "bin/kubectl"
+    "bin/crictl"
+    "bin/ctr"
+)
+
+containerd_binaries=(
+    "bin/containerd"
+    "bin/containerd-shim"
+    "bin/containerd-shim-runc-v2"
+    "bin/runc"
+    "bin/containerd-shim-runhcs-v1"
+    "bin/runhcs"
+)
+
+for i in "${k3s_binaries[@]}"; do
+    if [ -f "$i${BINARY_POSTFIX}" ]; then
+        echo "Removing $i${BINARY_POSTFIX}"
+        rm -f "$i${BINARY_POSTFIX}"
+    fi
+done
+
+for i in "${containerd_binaries[@]}"; do
+    if [ -f "$i${BINARY_POSTFIX}" ]; then
+        echo "Removing $i${BINARY_POSTFIX}"
+        rm -f "$i${BINARY_POSTFIX}"
+    fi
+done
 
 cleanup() {
     exit_status=$?
@@ -115,44 +139,66 @@ cleanup() {
 }
 
 INSTALLBIN=$(pwd)/bin
-if [ ! -x ${INSTALLBIN}/cni ]; then
+PLUGINS_WINDOWS_PATCH=$(pwd)/scripts/plugins-windows.patch
+if [ ! -x ${INSTALLBIN}/cni${BINARY_POSTFIX} ]; then
 (
     echo Building cni
     TMPDIR=$(mktemp -d)
     trap cleanup EXIT
     WORKDIR=$TMPDIR/src/github.com/containernetworking/plugins
     git clone -b $VERSION_CNIPLUGINS https://github.com/rancher/plugins.git $WORKDIR
     cd $WORKDIR
-    GO111MODULE=off GOPATH=$TMPDIR CGO_ENABLED=0 "${GO}" build -tags "$TAGS" -gcflags="all=${GCFLAGS}" -ldflags "$VERSIONFLAGS $LDFLAGS $STATIC" -o $INSTALLBIN/cni
+    if [ ${OS} = windows ]; then
+        patch -p1 < $PLUGINS_WINDOWS_PATCH
+    fi
+    GO111MODULE=off GOPATH=$TMPDIR CGO_ENABLED=0 "${GO}" build -tags "$TAGS" -gcflags="all=${GCFLAGS}" -ldflags "$VERSIONFLAGS $LDFLAGS $STATIC" -o $INSTALLBIN/cni${BINARY_POSTFIX}
 )
 fi
 
 echo Building k3s
-CGO_ENABLED=1 "${GO}" build -tags "$TAGS" -gcflags="all=${GCFLAGS}" -ldflags "$VERSIONFLAGS $LDFLAGS $STATIC" -o bin/k3s ./cmd/server/main.go
-ln -s k3s ./bin/k3s-agent
-ln -s k3s ./bin/k3s-server
-ln -s k3s ./bin/k3s-token
-ln -s k3s ./bin/k3s-etcd-snapshot
-ln -s k3s ./bin/k3s-secrets-encrypt
-ln -s k3s ./bin/k3s-certificate
-ln -s k3s ./bin/k3s-completion
-ln -s k3s ./bin/kubectl
-ln -s k3s ./bin/crictl
-ln -s k3s ./bin/ctr
+CGO_ENABLED=1 "${GO}" build -tags "$TAGS" -gcflags="all=${GCFLAGS}" -ldflags "$VERSIONFLAGS $LDFLAGS $STATIC" -o bin/k3s${BINARY_POSTFIX} ./cmd/server/main.go
+
+for i in "${k3s_binaries[@]}"; do
+    ln -s "k3s${BINARY_POSTFIX}" "$i${BINARY_POSTFIX}"
+done
 
 export GOPATH=$(pwd)/build
 
-echo Building containerd
-pushd ./build/src/github.com/containerd/containerd
-TAGS="${TAGS/netcgo/netgo}"
-CGO_ENABLED=1 "${GO}" build -tags "$TAGS" -gcflags="all=${GCFLAGS}" -ldflags "$VERSIONFLAGS $LDFLAGS $STATIC" -o bin/containerd              ./cmd/containerd
-CGO_ENABLED=1 "${GO}" build -tags "$TAGS" -gcflags="all=${GCFLAGS}" -ldflags "$VERSIONFLAGS $LDFLAGS $STATIC" -o bin/containerd-shim-runc-v2 ./cmd/containerd-shim-runc-v2
-popd
-cp -vf ./build/src/github.com/containerd/containerd/bin/* ./bin/
-
-echo Building runc
-pushd ./build/src/github.com/opencontainers/runc
-rm -f runc
-make EXTRA_FLAGS="-gcflags=\"all=${GCFLAGS}\"" EXTRA_LDFLAGS="$LDFLAGS" BUILDTAGS="$RUNC_TAGS" $RUNC_STATIC
-popd
-cp -vf ./build/src/github.com/opencontainers/runc/runc ./bin/
+case ${OS} in
+  linux)
+    echo Building containerd
+    pushd ./build/src/github.com/containerd/containerd
+    TAGS="${TAGS/netcgo/netgo}"
+    CGO_ENABLED=1 "${GO}" build -tags "$TAGS" -gcflags="all=${GCFLAGS}" -ldflags "$VERSIONFLAGS $LDFLAGS $STATIC" -o bin/containerd ./cmd/containerd
+    CGO_ENABLED=1 "${GO}" build -tags "$TAGS" -gcflags="all=${GCFLAGS}" -ldflags "$VERSIONFLAGS $LDFLAGS $STATIC" -o bin/containerd-shim-runc-v2 ./cmd/containerd-shim-runc-v2
+    popd
+    cp -vf ./build/src/github.com/containerd/containerd/bin/* ./bin/
+
+    echo Building runc
+    pushd ./build/src/github.com/opencontainers/runc
+    rm -f runc
+    make EXTRA_FLAGS="-gcflags=\"all=${GCFLAGS}\"" EXTRA_LDFLAGS="$LDFLAGS" BUILDTAGS="$RUNC_TAGS" $RUNC_STATIC
+    popd
+    cp -vf ./build/src/github.com/opencontainers/runc/runc ./bin/
+    ;;
+  windows)
+    echo Building containerd
+    pushd ./build/src/github.com/containerd/containerd
+    TAGS="${TAGS/netcgo/netgo}"
+    CGO_ENABLED=1 "${GO}" build -tags "$TAGS" -gcflags="all=${GCFLAGS}" -ldflags "$VERSIONFLAGS $LDFLAGS $STATIC" -o bin/containerd${BINARY_POSTFIX} ./cmd/containerd
+    popd
+    cp -vf ./build/src/github.com/containerd/containerd/bin/*${BINARY_POSTFIX} ./bin/
+
+    echo Building containerd-shim-runhcs-v1
+    pushd ./build/src/github.com/microsoft/hcsshim
+    TAGS="${TAGS/netcgo/netgo}"
+    CGO_ENABLED=0 "${GO}" build -tags "$TAGS" -ldflags "$VERSIONFLAGS $LDFLAGS $STATIC" -o bin/containerd-shim-runhcs-v1${BINARY_POSTFIX} ./cmd/containerd-shim-runhcs-v1
+    CGO_ENABLED=0 "${GO}" build -tags "$TAGS" -ldflags "$VERSIONFLAGS $LDFLAGS $STATIC" -o bin/runhcs${BINARY_POSTFIX} ./cmd/runhcs
+    popd
+    cp -vf ./build/src/github.com/microsoft/hcsshim/bin/*${BINARY_POSTFIX} ./bin/
+    ;;
+  *)
+    echo "[ERROR] unrecognized opertaing system: ${OS}"
+    exit 1
+    ;;
+esac

scripts/download

@@ -10,25 +10,36 @@ CHARTS_URL=https://k3s.io/k3s-charts/assets
 CHARTS_DIR=build/static/charts
 RUNC_DIR=build/src/github.com/opencontainers/runc
 CONTAINERD_DIR=build/src/github.com/containerd/containerd
+HCSSHIM_DIR=build/src/github.com/microsoft/hcsshim
 DATA_DIR=build/data
 export TZ=UTC
 
 umask 022
 rm -rf ${CHARTS_DIR}
 rm -rf ${RUNC_DIR}
 rm -rf ${CONTAINERD_DIR}
+rm -rf ${HCSSHIM_DIR}
 mkdir -p ${CHARTS_DIR}
 mkdir -p ${DATA_DIR}
 
-curl --compressed -sfL https://github.com/k3s-io/k3s-root/releases/download/${VERSION_ROOT}/k3s-root-${ARCH}.tar | tar xf -
-
-git clone --single-branch --branch=${VERSION_RUNC} --depth=1 https://github.com/opencontainers/runc ${RUNC_DIR}
+case ${OS} in
+  linux)
+    git clone --single-branch --branch=${VERSION_RUNC} --depth=1 https://github.com/opencontainers/runc ${RUNC_DIR}
+    curl --compressed -sfL https://github.com/k3s-io/k3s-root/releases/download/${VERSION_ROOT}/k3s-root-${ARCH}.tar | tar xf -
+    cp scripts/wg-add.sh bin/aux
+    ;;
+  windows)
+    git clone --single-branch --branch=${VERSION_HCSSHIM} --depth=1 https://github.com/microsoft/hcsshim ${HCSSHIM_DIR}
+    ;;
+  *)
+    echo "[ERROR] unrecognized opertaing system: ${OS}"
+    exit 1
+    ;;
+esac
 
 git clone --single-branch --branch=${VERSION_CONTAINERD} --depth=1 https://github.com/k3s-io/containerd ${CONTAINERD_DIR}
 
 for CHART_FILE in $(grep -rlF HelmChart manifests/ | xargs yq eval --no-doc .spec.chart | xargs -n1 basename); do
   CHART_NAME=$(echo $CHART_FILE | grep -oE '^(-*[a-z])+')
   curl -sfL ${CHARTS_URL}/${CHART_NAME}/${CHART_FILE} -o ${CHARTS_DIR}/${CHART_FILE}
 done
-
-cp scripts/wg-add.sh bin/aux