From 9b131aa20a26c0e5bcfdcde1af6bb6ab01fd9b6e Mon Sep 17 00:00:00 2001 From: Sean Yen Date: Sun, 9 Apr 2023 10:57:45 -0700 Subject: [PATCH 01/11] initial windows port. Signed-off-by: Sean Yen --- cmd/k3s/main.go | 24 ++++---- cmd/k3s/main_linux.go | 20 +++++++ cmd/k3s/main_windows.go | 24 ++++++++ go.sum | 15 +++++ pkg/cli/crictl/crictl.go | 6 ++ pkg/kubectl/main.go | 4 ++ scripts/build | 118 +++++++++++++++++++++++++-------------- scripts/download | 21 +++++-- scripts/package-cli | 33 ++++++++--- scripts/validate | 2 +- scripts/version.sh | 11 ++++ 11 files changed, 209 insertions(+), 69 deletions(-) create mode 100644 cmd/k3s/main_linux.go create mode 100644 cmd/k3s/main_windows.go diff --git a/cmd/k3s/main.go b/cmd/k3s/main.go index 23eafa2185e3..7078999dac34 100644 --- a/cmd/k3s/main.go +++ b/cmd/k3s/main.go @@ -9,7 +9,6 @@ import ( "path/filepath" "strconv" "strings" - "syscall" "github.com/k3s-io/k3s/pkg/cli/cmds" "github.com/k3s-io/k3s/pkg/configfilearg" @@ -46,8 +45,8 @@ func main() { app := cmds.NewApp() app.EnableBashCompletion = true app.Commands = []cli.Command{ - cmds.NewServerCommand(internalCLIAction(version.Program+"-server", dataDir, os.Args)), - cmds.NewAgentCommand(internalCLIAction(version.Program+"-agent", dataDir, os.Args)), + cmds.NewServerCommand(internalCLIAction(version.Program+"-server"+programPostfix, dataDir, os.Args)), + cmds.NewAgentCommand(internalCLIAction(version.Program+"-agent"+programPostfix, dataDir, os.Args)), cmds.NewKubectlCommand(externalCLIAction("kubectl", dataDir)), cmds.NewCRICTL(externalCLIAction("crictl", dataDir)), cmds.NewCtrCommand(externalCLIAction("ctr", dataDir)), @@ -157,7 +156,7 @@ func externalCLI(cli, dataDir string, args []string) error { os.Setenv("CRI_CONFIG_FILE", findCriConfig(dataDir)) } } - return stageAndRun(dataDir, cli, append([]string{cli}, args...)) + return stageAndRun(dataDir, cli, append([]string{cli}, args...), false) } // internalCLIAction returns a function that will call a K3s internal command, be used as the Action of a cli.Command. @@ -173,11 +172,11 @@ func internalCLIAction(cmd, dataDir string, args []string) func(ctx *cli.Context // stageAndRunCLI calls an external binary. func stageAndRunCLI(cli *cli.Context, cmd string, dataDir string, args []string) error { - return stageAndRun(dataDir, cmd, args) + return stageAndRun(dataDir, cmd, args, true) } // stageAndRun does the actual work of setting up and calling an external binary. -func stageAndRun(dataDir, cmd string, args []string) error { +func stageAndRun(dataDir, cmd string, args []string, calledAsInternal bool) error { dir, err := extract(dataDir) if err != nil { return errors.Wrap(err, "extracting data") @@ -186,9 +185,9 @@ func stageAndRun(dataDir, cmd string, args []string) error { var pathEnv string if findPreferBundledBin(args) { - pathEnv = filepath.Join(dir, "bin") + ":" + filepath.Join(dir, "bin/aux") + ":" + os.Getenv("PATH") + pathEnv = filepath.Join(dir, "bin") + string(os.PathListSeparator) + filepath.Join(dir, "bin/aux") + string(os.PathListSeparator) + os.Getenv("PATH") } else { - pathEnv = filepath.Join(dir, "bin") + ":" + os.Getenv("PATH") + ":" + filepath.Join(dir, "bin/aux") + pathEnv = filepath.Join(dir, "bin") + string(os.PathListSeparator) + os.Getenv("PATH") + string(os.PathListSeparator) + filepath.Join(dir, "bin/aux") } if err := os.Setenv("PATH", pathEnv); err != nil { return err @@ -204,10 +203,7 @@ func stageAndRun(dataDir, cmd string, args []string) error { logrus.Debugf("Running %s %v", cmd, args) - if err := syscall.Exec(cmd, args, os.Environ()); err != nil { - return errors.Wrapf(err, "exec %s failed", cmd) - } - return nil + return runExec(cmd, args, calledAsInternal) } // getAssetAndDir returns the name of the bindata asset, along with a directory path @@ -223,7 +219,7 @@ func getAssetAndDir(dataDir string) (string, string) { func extract(dataDir string) (string, error) { // check if content already exists in requested data-dir asset, dir := getAssetAndDir(dataDir) - if _, err := os.Stat(filepath.Join(dir, "bin", "k3s")); err == nil { + if _, err := os.Stat(filepath.Join(dir, "bin", "k3s"+programPostfix)); err == nil { return dir, nil } @@ -232,7 +228,7 @@ func extract(dataDir string) (string, error) { // dir if the assets already exist in the default path. if dataDir != datadir.DefaultDataDir { _, defaultDir := getAssetAndDir(datadir.DefaultDataDir) - if _, err := os.Stat(filepath.Join(defaultDir, "bin", "k3s")); err == nil { + if _, err := os.Stat(filepath.Join(defaultDir, "bin", "k3s"+programPostfix)); err == nil { return defaultDir, nil } } diff --git a/cmd/k3s/main_linux.go b/cmd/k3s/main_linux.go new file mode 100644 index 000000000000..76a7b641795f --- /dev/null +++ b/cmd/k3s/main_linux.go @@ -0,0 +1,20 @@ +//go:build linux +// +build linux + +package main + +import ( + "os" + "syscall" + + "github.com/pkg/errors" +) + +const programPostfix = "" + +func runExec(cmd string, args []string, calledAsInternal bool) (err error) { + if err := syscall.Exec(cmd, args, os.Environ()); err != nil { + return errors.Wrapf(err, "exec %s failed", cmd) + } + return nil +} diff --git a/cmd/k3s/main_windows.go b/cmd/k3s/main_windows.go new file mode 100644 index 000000000000..cefca3c91e67 --- /dev/null +++ b/cmd/k3s/main_windows.go @@ -0,0 +1,24 @@ +//go:build windows +// +build windows + +package main + +import ( + "os" + "os/exec" +) + +const programPostfix = ".exe" + +func runExec(cmd string, args []string, calledAsInternal bool) (err error) { + // syscall.Exec: not supported by windows + if calledAsInternal { + args = args[1:] + } + cmdObj := exec.Command(cmd, args...) + cmdObj.Stdout = os.Stdout + cmdObj.Stderr = os.Stderr + cmdObj.Stdin = os.Stdin + cmdObj.Env = os.Environ() + return cmdObj.Run() +} diff --git a/go.sum b/go.sum index 2a11ccd29505..ee05d08217e3 100644 --- a/go.sum +++ b/go.sum @@ -263,6 +263,7 @@ github.com/JeffAshton/win_pdh v0.0.0-20161109143554-76bb4ee9f0ab/go.mod h1:3VYc5 github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c/go.mod h1:X0CRv0ky0k6m906ixxpzmDRLvX58TFUKS2eePweuyxk= github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ= github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE= +github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA= github.com/Microsoft/go-winio v0.4.15/go.mod h1:tTuCMEN+UleMWgg9dVx4Hu52b1bJo+59jBh3ajtinzw= github.com/Microsoft/go-winio v0.4.17/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= github.com/Microsoft/go-winio v0.5.1/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= @@ -490,6 +491,7 @@ github.com/distribution/distribution/v3 v3.0.0-20220526142353-ffbd94cbe269/go.mo github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E= github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= +github.com/docker/cli v0.0.0-20191017083524-a8ff7f821017/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/cli v20.10.10+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/cli v20.10.17+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/cli v23.0.1+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= @@ -838,6 +840,7 @@ github.com/jackc/pgx/v5 v5.4.2 h1:u1gmGDwbdRUZiwisBm/Ky2M14uQyUP65bG8+20nnyrg= github.com/jackc/pgx/v5 v5.4.2/go.mod h1:q6iHT8uDNXWiFNOlRqJzBTaSH3+2xCXkokxHZC5qWFY= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/joefitzgerald/rainbow-reporter v0.1.0/go.mod h1:481CNgqmVHQZzdIbN52CupLJyoVwB10FQ/IQlF1pdL8= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= github.com/jonboulle/clockwork v0.3.0 h1:9BSCMi8C+0qdApAp4auwX0RkLGUjs956h0EkuQymUhg= @@ -1134,6 +1137,8 @@ github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+ github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0HfGg= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= @@ -1154,6 +1159,7 @@ github.com/onsi/ginkgo/v2 v2.9.4/go.mod h1:gCQYp2Q+kSoIj7ykSVb9nskRSsR6PUj4AiLyw github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU= github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7kR0iZvM= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= @@ -1173,6 +1179,7 @@ github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3ev github.com/open-policy-agent/opa v0.42.2/go.mod h1:MrmoTi/BsKWT58kXlVayBb+rYVeaMwuBm3nYAN3923s= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= github.com/opencontainers/image-spec v1.0.2-0.20210730191737-8e42a01fb1b7/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= github.com/opencontainers/image-spec v1.0.2/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= @@ -1306,6 +1313,7 @@ github.com/ruudk/golang-pdf417 v0.0.0-20181029194003-1af4ab5afa58/go.mod h1:6lfF github.com/ruudk/golang-pdf417 v0.0.0-20201230142125-a7e3863a1245/go.mod h1:pQAZKsJ8yyVxGRWYNEm9oFB8ieLgKFnamEyDmSA0BRk= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/safchain/ethtool v0.2.0/go.mod h1:WkKB1DnNtvsMlDmQ50sgwowDJV/hGbJSOvJoEXs1AJQ= +github.com/sclevine/spec v1.2.0/go.mod h1:W4J29eT/Kzv7/b9IWLB055Z+qvVC9vt0Arko24q7p+U= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/seccomp/libseccomp-golang v0.9.2-0.20220502022130-f33da4d89646/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg= github.com/seccomp/libseccomp-golang v0.10.0 h1:aA4bp+/Zzi0BnWZ2F1wgNBs5gTpm+na2rWM6M9YjLpY= @@ -1407,6 +1415,7 @@ github.com/vbatts/tar-split v0.11.5/go.mod h1:yZbwRsSeGjusneWgA781EKej9HF8vme8ok github.com/vektah/gqlparser/v2 v2.4.5/go.mod h1:flJWIR04IMQPGz+BXLrORkrARBxv/rtyIAFvd/MceW0= github.com/veraison/go-cose v1.0.0-rc.1/go.mod h1:7ziE85vSq4ScFTg6wyoMXjucIGOf4JkFEZi/an96Ct4= github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= +github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= github.com/vishvananda/netlink v1.2.1-beta.2 h1:Llsql0lnQEbHj0I1OuKyp8otXp0r3q0mPkuhwHfStVs= github.com/vishvananda/netlink v1.2.1-beta.2/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= @@ -1666,6 +1675,7 @@ golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1688,6 +1698,7 @@ golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190706070813-72ffa07ba3db/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -1723,6 +1734,7 @@ golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= +golang.org/x/tools v0.0.0-20200916195026-c9a70fc28ce3/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201124115921-2c860bdd6e78/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= @@ -1908,6 +1920,8 @@ gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= +gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= gotest.tools/v3 v3.4.0 h1:ZazjZUfuVeZGLAmlKKuyv3IKP5orXcwtOwDQH6YVr6o= @@ -1922,6 +1936,7 @@ honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9 honnef.co/go/tools v0.1.3/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las= inet.af/tcpproxy v0.0.0-20200125044825-b6bb9b5b8252 h1:gmJCKidOfjKDUHF1jjke+I+2iQIyE3HNNxu2OKO/FUI= inet.af/tcpproxy v0.0.0-20200125044825-b6bb9b5b8252/go.mod h1:zq+R+tLcdHugi7Jt+FtIQY6m6wtX34lr2CdQVH2fhW0= +k8s.io/gengo v0.0.0-20201113003025-83324d819ded/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/gengo v0.0.0-20220902162205-c0856e24416d h1:U9tB195lKdzwqicbJvyJeOXV7Klv+wNAWENRnXEGi08= k8s.io/gengo v0.0.0-20220902162205-c0856e24416d/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= diff --git a/pkg/cli/crictl/crictl.go b/pkg/cli/crictl/crictl.go index 03b3b5b661c6..693032c3ce3c 100644 --- a/pkg/cli/crictl/crictl.go +++ b/pkg/cli/crictl/crictl.go @@ -1,11 +1,17 @@ package crictl import ( + "os" + "runtime" + "github.com/kubernetes-sigs/cri-tools/cmd/crictl" "github.com/urfave/cli" ) func Run(ctx *cli.Context) error { + if runtime.GOOS == "windows" { + os.Args = os.Args[1:] + } crictl.Main() return nil } diff --git a/pkg/kubectl/main.go b/pkg/kubectl/main.go index fd32c21cd34f..f3d77f24a11d 100644 --- a/pkg/kubectl/main.go +++ b/pkg/kubectl/main.go @@ -4,6 +4,7 @@ import ( "fmt" "math/rand" "os" + "runtime" "strings" "time" @@ -15,6 +16,9 @@ import ( ) func Main() { + if runtime.GOOS == "windows" { + os.Args = os.Args[1:] + } kubenv := os.Getenv("KUBECONFIG") for i, arg := range os.Args { if strings.HasPrefix(arg, "--kubeconfig=") { diff --git a/scripts/build b/scripts/build index 36bc9104dd14..9d237c52fb05 100755 --- a/scripts/build +++ b/scripts/build @@ -66,6 +66,10 @@ TAGS="ctrd apparmor seccomp netcgo osusergo providerless urfave_cli_no_docs" RUNC_TAGS="apparmor seccomp" RUNC_STATIC="static" +if [ ${OS} = windows ]; then + TAGS="netcgo osusergo providerless" +fi + if [ "$SELINUX" = "true" ]; then TAGS="$TAGS selinux" RUNC_TAGS="$RUNC_TAGS selinux" @@ -97,21 +101,41 @@ if [ ${ARCH} = s390x ]; then export GOARCH="s390x" fi -rm -f \ - bin/k3s-agent \ - bin/k3s-server \ - bin/k3s-token \ - bin/k3s-etcd-snapshot \ - bin/k3s-secrets-encrypt \ - bin/k3s-certificate \ - bin/k3s-completion \ - bin/kubectl \ - bin/crictl \ - bin/ctr \ - bin/containerd \ - bin/containerd-shim \ - bin/containerd-shim-runc-v2 \ - bin/runc +k3s_binaries=( + "bin/k3s-agent" + "bin/k3s-server" + "bin/k3s-token" + "bin/k3s-etcd-snapshot" + "bin/k3s-secrets-encrypt" + "bin/k3s-certificate" + "bin/k3s-completion" + "bin/kubectl" + "bin/crictl" + "bin/ctr" +) + +containerd_binaries=( + "bin/containerd" + "bin/containerd-shim" + "bin/containerd-shim-runc-v2" + "bin/runc" + "bin/containerd-shim-runhcs-v1" + "bin/runhcs" +) + +for i in "${k3s_binaries[@]}"; do + if [ -f "$i${BINARY_POSTFIX}" ]; then + echo "Removing $i${BINARY_POSTFIX}" + rm -f "$i${BINARY_POSTFIX}" + fi +done + +for i in "${containerd_binaries[@]}"; do + if [ -f "$i${BINARY_POSTFIX}" ]; then + echo "Removing $i${BINARY_POSTFIX}" + rm -f "$i${BINARY_POSTFIX}" + fi +done cleanup() { exit_status=$? @@ -120,7 +144,7 @@ cleanup() { } INSTALLBIN=$(pwd)/bin -if [ ! -x ${INSTALLBIN}/cni ]; then +if [ ! -x ${INSTALLBIN}/cni${BINARY_POSTFIX} ]; then ( echo Building cni TMPDIR=$(mktemp -d) @@ -128,36 +152,46 @@ if [ ! -x ${INSTALLBIN}/cni ]; then WORKDIR=$TMPDIR/src/github.com/containernetworking/plugins git clone -b $VERSION_CNIPLUGINS https://github.com/rancher/plugins.git $WORKDIR cd $WORKDIR - GO111MODULE=off GOPATH=$TMPDIR CGO_ENABLED=0 "${GO}" build -tags "$TAGS" -gcflags="all=${GCFLAGS}" -ldflags "$VERSIONFLAGS $LDFLAGS $STATIC" -o $INSTALLBIN/cni + GO111MODULE=off GOPATH=$TMPDIR CGO_ENABLED=0 "${GO}" build -tags "$TAGS" -gcflags="all=${GCFLAGS}" -ldflags "$VERSIONFLAGS $LDFLAGS $STATIC" -o $INSTALLBIN/cni${BINARY_POSTFIX} ) fi echo Building k3s -CGO_ENABLED=1 "${GO}" build $BLDFLAGS -tags "$TAGS" -buildvcs=false -gcflags="all=${GCFLAGS}" -ldflags "$VERSIONFLAGS $LDFLAGS $STATIC" -o bin/k3s ./cmd/server -ln -s k3s ./bin/containerd -ln -s k3s ./bin/crictl -ln -s k3s ./bin/ctr -ln -s k3s ./bin/k3s-agent -ln -s k3s ./bin/k3s-certificate -ln -s k3s ./bin/k3s-completion -ln -s k3s ./bin/k3s-etcd-snapshot -ln -s k3s ./bin/k3s-secrets-encrypt -ln -s k3s ./bin/k3s-server -ln -s k3s ./bin/k3s-token -ln -s k3s ./bin/kubectl +CGO_ENABLED=1 "${GO}" build $BLDFLAGS -tags "$TAGS" -buildvcs=false -gcflags="all=${GCFLAGS}" -ldflags "$VERSIONFLAGS $LDFLAGS $STATIC" -o bin/k3s${BINARY_POSTFIX} ./cmd/server + +for i in "${k3s_binaries[@]}"; do + ln -s "k3s${BINARY_POSTFIX}" "$i${BINARY_POSTFIX}" +done export GOPATH=$(pwd)/build -echo Building containerd -pushd ./build/src/github.com/containerd/containerd -TAGS="${TAGS/netcgo/netgo}" -CGO_ENABLED=1 "${GO}" build -tags "$TAGS" -gcflags="all=${GCFLAGS}" -ldflags "$VERSIONFLAGS $LDFLAGS $STATIC" -o bin/containerd-shim-runc-v2 ./cmd/containerd-shim-runc-v2 -popd -cp -vf ./build/src/github.com/containerd/containerd/bin/* ./bin/ - -echo Building runc -pushd ./build/src/github.com/opencontainers/runc -rm -f runc -make EXTRA_FLAGS="-gcflags=\"all=${GCFLAGS}\"" EXTRA_LDFLAGS="$LDFLAGS" BUILDTAGS="$RUNC_TAGS" $RUNC_STATIC -popd -cp -vf ./build/src/github.com/opencontainers/runc/runc ./bin/ +case ${OS} in + linux) + echo Building containerd-shim + pushd ./build/src/github.com/containerd/containerd + TAGS="${TAGS/netcgo/netgo}" + CGO_ENABLED=1 "${GO}" build -tags "$TAGS" -gcflags="all=${GCFLAGS}" -ldflags "$VERSIONFLAGS $LDFLAGS $STATIC" -o bin/containerd-shim-runc-v2 ./cmd/containerd-shim-runc-v2 + popd + cp -vf ./build/src/github.com/containerd/containerd/bin/* ./bin/ + + echo Building runc + pushd ./build/src/github.com/opencontainers/runc + rm -f runc + make EXTRA_FLAGS="-gcflags=\"all=${GCFLAGS}\"" EXTRA_LDFLAGS="$LDFLAGS" BUILDTAGS="$RUNC_TAGS" $RUNC_STATIC + popd + cp -vf ./build/src/github.com/opencontainers/runc/runc ./bin/ + ;; + windows) + echo Building containerd-shim-runhcs-v1 + pushd ./build/src/github.com/microsoft/hcsshim + TAGS="${TAGS/netcgo/netgo}" + CGO_ENABLED=0 "${GO}" build -tags "$TAGS" -ldflags "$VERSIONFLAGS $LDFLAGS $STATIC" -o bin/containerd-shim-runhcs-v1${BINARY_POSTFIX} ./cmd/containerd-shim-runhcs-v1 + CGO_ENABLED=0 "${GO}" build -tags "$TAGS" -ldflags "$VERSIONFLAGS $LDFLAGS $STATIC" -o bin/runhcs${BINARY_POSTFIX} ./cmd/runhcs + popd + cp -vf ./build/src/github.com/microsoft/hcsshim/bin/*${BINARY_POSTFIX} ./bin/ + ;; + *) + echo "[ERROR] unrecognized opertaing system: ${OS}" + exit 1 + ;; +esac diff --git a/scripts/download b/scripts/download index ac2c5d04be2a..0557eca715f7 100755 --- a/scripts/download +++ b/scripts/download @@ -10,6 +10,7 @@ CHARTS_URL=https://k3s.io/k3s-charts/assets CHARTS_DIR=build/static/charts RUNC_DIR=build/src/github.com/opencontainers/runc CONTAINERD_DIR=build/src/github.com/containerd/containerd +HCSSHIM_DIR=build/src/github.com/microsoft/hcsshim DATA_DIR=build/data export TZ=UTC @@ -17,12 +18,24 @@ umask 022 rm -rf ${CHARTS_DIR} rm -rf ${RUNC_DIR} rm -rf ${CONTAINERD_DIR} +rm -rf ${HCSSHIM_DIR} mkdir -p ${CHARTS_DIR} mkdir -p ${DATA_DIR} -curl --compressed -sfL https://github.com/k3s-io/k3s-root/releases/download/${VERSION_ROOT}/k3s-root-${ARCH}.tar | tar xf - - -git clone --single-branch --branch=${VERSION_RUNC} --depth=1 https://github.com/opencontainers/runc ${RUNC_DIR} +case ${OS} in + linux) + git clone --single-branch --branch=${VERSION_RUNC} --depth=1 https://github.com/opencontainers/runc ${RUNC_DIR} + curl --compressed -sfL https://github.com/k3s-io/k3s-root/releases/download/${VERSION_ROOT}/k3s-root-${ARCH}.tar | tar xf - + cp scripts/wg-add.sh bin/aux + ;; + windows) + git clone --single-branch --branch=${VERSION_HCSSHIM} --depth=1 https://github.com/microsoft/hcsshim ${HCSSHIM_DIR} + ;; + *) + echo "[ERROR] unrecognized opertaing system: ${OS}" + exit 1 + ;; +esac git clone --single-branch --branch=${VERSION_CONTAINERD} --depth=1 https://${PKG_CONTAINERD_K3S} ${CONTAINERD_DIR} @@ -30,5 +43,3 @@ for CHART_FILE in $(grep -rlF HelmChart manifests/ | xargs yq eval --no-doc .spe CHART_NAME=$(echo $CHART_FILE | grep -oE '^(-*[a-z])+') curl -sfL ${CHARTS_URL}/${CHART_NAME}/${CHART_FILE} -o ${CHARTS_DIR}/${CHART_FILE} done - -cp scripts/wg-add.sh bin/aux diff --git a/scripts/package-cli b/scripts/package-cli index 92fee50e1e7a..d65ad134680d 100755 --- a/scripts/package-cli +++ b/scripts/package-cli @@ -8,13 +8,32 @@ cd $(dirname $0)/.. GO=${GO-go} for i in containerd crictl kubectl k3s-agent k3s-server k3s-token k3s-etcd-snapshot k3s-secrets-encrypt k3s-certificate k3s-completion; do - rm -f bin/$i - ln -s k3s bin/$i + rm -f bin/$i${BINARY_POSTFIX} + ln -s k3s${BINARY_POSTFIX} bin/$i${BINARY_POSTFIX} done -for i in bandwidth bridge firewall flannel host-local loopback portmap; do - rm -f bin/$i - ln -s cni bin/$i +cni_binaries=( + "bandwidth" + "bridge" + "firewall" + "flannel" + "host-local" + "loopback" + "portmap" +) + +if [ ${OS} = windows ]; then + cni_binaries=( + "win-bridge" + "win-overlay" + "flannel" + "host-local" + ) +fi + +for i in "${cni_binaries[@]}"; do + rm -f bin/$i${BINARY_POSTFIX} + ln -s cni${BINARY_POSTFIX} bin/$i${BINARY_POSTFIX} done cp contrib/util/check-config.sh bin/check-config @@ -52,7 +71,7 @@ fi CMD_NAME=dist/artifacts/k3s${BIN_SUFFIX} -"${GO}" generate +GOOS=linux CC=gcc CXX=g++ "${GO}" generate LDFLAGS=" -X github.com/k3s-io/k3s/pkg/version.Version=$VERSION -X github.com/k3s-io/k3s/pkg/version.GitCommit=${COMMIT:0:8} @@ -60,7 +79,7 @@ LDFLAGS=" " TAGS="urfave_cli_no_docs" STATIC="-extldflags '-static'" -CGO_ENABLED=0 "${GO}" build -tags "$TAGS" -ldflags "$LDFLAGS $STATIC" -o ${CMD_NAME} ./cmd/k3s/main.go +CGO_ENABLED=0 "${GO}" build -tags "$TAGS" -ldflags "$LDFLAGS $STATIC" -o ${CMD_NAME} ./cmd/k3s stat ${CMD_NAME} diff --git a/scripts/validate b/scripts/validate index 74132598908f..9f69628000e2 100755 --- a/scripts/validate +++ b/scripts/validate @@ -15,7 +15,7 @@ echo Running: go mod tidy go mod tidy echo Running: go generate -go generate +GOOS=linux CC=gcc CXX=g++ go generate echo Running validation diff --git a/scripts/version.sh b/scripts/version.sh index 0520f055c4f0..9fbaf3cd93d8 100755 --- a/scripts/version.sh +++ b/scripts/version.sh @@ -2,6 +2,7 @@ GO=${GO-go} ARCH=${ARCH:-$("${GO}" env GOARCH)} +OS=${OS:-$("${GO}" env GOOS)} SUFFIX="-${ARCH}" GIT_TAG=$DRONE_TAG TREE_STATE=clean @@ -52,6 +53,11 @@ if [ -z "$VERSION_RUNC" ]; then VERSION_RUNC="v0.0.0" fi +VERSION_HCSSHIM=$(get-module-version github.com/Microsoft/hcsshim) +if [ -z "$VERSION_HCSSHIM" ]; then + VERSION_HCSSHIM="v0.0.0" +fi + VERSION_FLANNEL=$(get-module-version github.com/flannel-io/flannel) if [ -z "$VERSION_FLANNEL" ]; then VERSION_FLANNEL="v0.0.0" @@ -81,3 +87,8 @@ else VERSION="$VERSION_K8S+k3s-${COMMIT:0:8}$DIRTY" fi VERSION_TAG="$(sed -e 's/+/-/g' <<< "$VERSION")" + +BINARY_POSTFIX= +if [ ${OS} = windows ]; then + BINARY_POSTFIX=.exe +fi \ No newline at end of file From 27169848608b2d8df476401e6359fc58c7ab77c1 Mon Sep 17 00:00:00 2001 From: Derek Nola Date: Fri, 7 Jul 2023 11:44:42 -0700 Subject: [PATCH 02/11] Consolidate GH k3s build Signed-off-by: Derek Nola Signed-off-by: Sean Yen --- scripts/package-cli | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/package-cli b/scripts/package-cli index d65ad134680d..6e0f85058016 100755 --- a/scripts/package-cli +++ b/scripts/package-cli @@ -79,7 +79,7 @@ LDFLAGS=" " TAGS="urfave_cli_no_docs" STATIC="-extldflags '-static'" -CGO_ENABLED=0 "${GO}" build -tags "$TAGS" -ldflags "$LDFLAGS $STATIC" -o ${CMD_NAME} ./cmd/k3s +CGO_ENABLED=0 "${GO}" build -tags "$TAGS" -ldflags "$LDFLAGS $STATIC" -o ${CMD_NAME} ./cmd/k3s/main.go ./cmd/k3s/main_linux.go stat ${CMD_NAME} From 2091e7a53fdcdf143fed054f86e755d58ec8559b Mon Sep 17 00:00:00 2001 From: Wei Ran Date: Fri, 14 Jul 2023 16:58:44 +0800 Subject: [PATCH 03/11] Add Windows support in containerd in multi call binary Signed-off-by: Sean Yen --- ...{builtins_cri_linux.go => builtins_cri.go} | 0 pkg/containerd/builtins_windows.go | 27 +++++++++++++++++++ 2 files changed, 27 insertions(+) rename pkg/containerd/{builtins_cri_linux.go => builtins_cri.go} (100%) create mode 100644 pkg/containerd/builtins_windows.go diff --git a/pkg/containerd/builtins_cri_linux.go b/pkg/containerd/builtins_cri.go similarity index 100% rename from pkg/containerd/builtins_cri_linux.go rename to pkg/containerd/builtins_cri.go diff --git a/pkg/containerd/builtins_windows.go b/pkg/containerd/builtins_windows.go new file mode 100644 index 000000000000..aa8f01f1d7ab --- /dev/null +++ b/pkg/containerd/builtins_windows.go @@ -0,0 +1,27 @@ +//go:build ctrd +// +build ctrd + +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package containerd + +import ( + _ "github.com/containerd/containerd/diff/lcow" + _ "github.com/containerd/containerd/diff/windows" + _ "github.com/containerd/containerd/snapshots/lcow" + _ "github.com/containerd/containerd/snapshots/windows" +) From 63d50ef94c503271c79eecd4f09990447bcf9256 Mon Sep 17 00:00:00 2001 From: Sean Yen Date: Tue, 1 Aug 2023 23:25:05 -0700 Subject: [PATCH 04/11] integrated containerd changes. Signed-off-by: Sean Yen --- scripts/build | 4 ++-- scripts/package-cli | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/build b/scripts/build index 9d237c52fb05..12a9e0ecd0bd 100755 --- a/scripts/build +++ b/scripts/build @@ -67,7 +67,7 @@ RUNC_TAGS="apparmor seccomp" RUNC_STATIC="static" if [ ${OS} = windows ]; then - TAGS="netcgo osusergo providerless" + TAGS="ctrd netcgo osusergo providerless" fi if [ "$SELINUX" = "true" ]; then @@ -110,12 +110,12 @@ k3s_binaries=( "bin/k3s-certificate" "bin/k3s-completion" "bin/kubectl" + "bin/containerd" "bin/crictl" "bin/ctr" ) containerd_binaries=( - "bin/containerd" "bin/containerd-shim" "bin/containerd-shim-runc-v2" "bin/runc" diff --git a/scripts/package-cli b/scripts/package-cli index 6e0f85058016..7c419e73b5ea 100755 --- a/scripts/package-cli +++ b/scripts/package-cli @@ -69,7 +69,7 @@ elif [ ${ARCH} = s390x ]; then BIN_SUFFIX="-s390x" fi -CMD_NAME=dist/artifacts/k3s${BIN_SUFFIX} +CMD_NAME=dist/artifacts/k3s${BIN_SUFFIX}${BINARY_POSTFIX} GOOS=linux CC=gcc CXX=g++ "${GO}" generate LDFLAGS=" @@ -79,7 +79,7 @@ LDFLAGS=" " TAGS="urfave_cli_no_docs" STATIC="-extldflags '-static'" -CGO_ENABLED=0 "${GO}" build -tags "$TAGS" -ldflags "$LDFLAGS $STATIC" -o ${CMD_NAME} ./cmd/k3s/main.go ./cmd/k3s/main_linux.go +CGO_ENABLED=0 "${GO}" build -tags "$TAGS" -ldflags "$LDFLAGS $STATIC" -o ${CMD_NAME} ./cmd/k3s stat ${CMD_NAME} From 638b0c473964430e42a6af5ba7946115106f4444 Mon Sep 17 00:00:00 2001 From: Sean Yen Date: Wed, 2 Aug 2023 03:22:07 -0700 Subject: [PATCH 05/11] add flannel integration. Signed-off-by: Sean Yen --- go.mod | 1 + go.sum | 15 ----- pkg/agent/flannel/setup.go | 48 +++++---------- pkg/agent/flannel/setup_linux.go | 38 ++++++++++++ pkg/agent/flannel/setup_windows.go | 59 ++++++++++++++++++ pkg/daemons/agent/agent_windows.go | 1 + pkg/daemons/executor/embed.go | 4 +- pkg/daemons/executor/embed_linux.go | 16 +++++ pkg/daemons/executor/embed_windows.go | 89 +++++++++++++++++++++++++++ 9 files changed, 221 insertions(+), 50 deletions(-) create mode 100644 pkg/agent/flannel/setup_linux.go create mode 100644 pkg/agent/flannel/setup_windows.go create mode 100644 pkg/daemons/executor/embed_linux.go create mode 100644 pkg/daemons/executor/embed_windows.go diff --git a/go.mod b/go.mod index 829465266468..14c44a62c117 100644 --- a/go.mod +++ b/go.mod @@ -86,6 +86,7 @@ replace ( ) require ( + github.com/Microsoft/hcsshim v0.11.0 github.com/Mirantis/cri-dockerd v0.0.0-00010101000000-000000000000 github.com/blang/semver/v4 v4.0.0 github.com/cloudnativelabs/kube-router/v2 v2.0.0-00010101000000-000000000000 diff --git a/go.sum b/go.sum index ee05d08217e3..2a11ccd29505 100644 --- a/go.sum +++ b/go.sum @@ -263,7 +263,6 @@ github.com/JeffAshton/win_pdh v0.0.0-20161109143554-76bb4ee9f0ab/go.mod h1:3VYc5 github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c/go.mod h1:X0CRv0ky0k6m906ixxpzmDRLvX58TFUKS2eePweuyxk= github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ= github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE= -github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA= github.com/Microsoft/go-winio v0.4.15/go.mod h1:tTuCMEN+UleMWgg9dVx4Hu52b1bJo+59jBh3ajtinzw= github.com/Microsoft/go-winio v0.4.17/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= github.com/Microsoft/go-winio v0.5.1/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= @@ -491,7 +490,6 @@ github.com/distribution/distribution/v3 v3.0.0-20220526142353-ffbd94cbe269/go.mo github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E= github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= -github.com/docker/cli v0.0.0-20191017083524-a8ff7f821017/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/cli v20.10.10+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/cli v20.10.17+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/cli v23.0.1+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= @@ -840,7 +838,6 @@ github.com/jackc/pgx/v5 v5.4.2 h1:u1gmGDwbdRUZiwisBm/Ky2M14uQyUP65bG8+20nnyrg= github.com/jackc/pgx/v5 v5.4.2/go.mod h1:q6iHT8uDNXWiFNOlRqJzBTaSH3+2xCXkokxHZC5qWFY= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= -github.com/joefitzgerald/rainbow-reporter v0.1.0/go.mod h1:481CNgqmVHQZzdIbN52CupLJyoVwB10FQ/IQlF1pdL8= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= github.com/jonboulle/clockwork v0.3.0 h1:9BSCMi8C+0qdApAp4auwX0RkLGUjs956h0EkuQymUhg= @@ -1137,8 +1134,6 @@ github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+ github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0HfGg= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= @@ -1159,7 +1154,6 @@ github.com/onsi/ginkgo/v2 v2.9.4/go.mod h1:gCQYp2Q+kSoIj7ykSVb9nskRSsR6PUj4AiLyw github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU= github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7kR0iZvM= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= -github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= @@ -1179,7 +1173,6 @@ github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3ev github.com/open-policy-agent/opa v0.42.2/go.mod h1:MrmoTi/BsKWT58kXlVayBb+rYVeaMwuBm3nYAN3923s= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= -github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= github.com/opencontainers/image-spec v1.0.2-0.20210730191737-8e42a01fb1b7/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= github.com/opencontainers/image-spec v1.0.2/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= @@ -1313,7 +1306,6 @@ github.com/ruudk/golang-pdf417 v0.0.0-20181029194003-1af4ab5afa58/go.mod h1:6lfF github.com/ruudk/golang-pdf417 v0.0.0-20201230142125-a7e3863a1245/go.mod h1:pQAZKsJ8yyVxGRWYNEm9oFB8ieLgKFnamEyDmSA0BRk= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/safchain/ethtool v0.2.0/go.mod h1:WkKB1DnNtvsMlDmQ50sgwowDJV/hGbJSOvJoEXs1AJQ= -github.com/sclevine/spec v1.2.0/go.mod h1:W4J29eT/Kzv7/b9IWLB055Z+qvVC9vt0Arko24q7p+U= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/seccomp/libseccomp-golang v0.9.2-0.20220502022130-f33da4d89646/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg= github.com/seccomp/libseccomp-golang v0.10.0 h1:aA4bp+/Zzi0BnWZ2F1wgNBs5gTpm+na2rWM6M9YjLpY= @@ -1415,7 +1407,6 @@ github.com/vbatts/tar-split v0.11.5/go.mod h1:yZbwRsSeGjusneWgA781EKej9HF8vme8ok github.com/vektah/gqlparser/v2 v2.4.5/go.mod h1:flJWIR04IMQPGz+BXLrORkrARBxv/rtyIAFvd/MceW0= github.com/veraison/go-cose v1.0.0-rc.1/go.mod h1:7ziE85vSq4ScFTg6wyoMXjucIGOf4JkFEZi/an96Ct4= github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= -github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= github.com/vishvananda/netlink v1.2.1-beta.2 h1:Llsql0lnQEbHj0I1OuKyp8otXp0r3q0mPkuhwHfStVs= github.com/vishvananda/netlink v1.2.1-beta.2/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= @@ -1675,7 +1666,6 @@ golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1698,7 +1688,6 @@ golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190706070813-72ffa07ba3db/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -1734,7 +1723,6 @@ golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= -golang.org/x/tools v0.0.0-20200916195026-c9a70fc28ce3/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201124115921-2c860bdd6e78/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= @@ -1920,8 +1908,6 @@ gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= -gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= gotest.tools/v3 v3.4.0 h1:ZazjZUfuVeZGLAmlKKuyv3IKP5orXcwtOwDQH6YVr6o= @@ -1936,7 +1922,6 @@ honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9 honnef.co/go/tools v0.1.3/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las= inet.af/tcpproxy v0.0.0-20200125044825-b6bb9b5b8252 h1:gmJCKidOfjKDUHF1jjke+I+2iQIyE3HNNxu2OKO/FUI= inet.af/tcpproxy v0.0.0-20200125044825-b6bb9b5b8252/go.mod h1:zq+R+tLcdHugi7Jt+FtIQY6m6wtX34lr2CdQVH2fhW0= -k8s.io/gengo v0.0.0-20201113003025-83324d819ded/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/gengo v0.0.0-20220902162205-c0856e24416d h1:U9tB195lKdzwqicbJvyJeOXV7Klv+wNAWENRnXEGi08= k8s.io/gengo v0.0.0-20220902162205-c0856e24416d/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= diff --git a/pkg/agent/flannel/setup.go b/pkg/agent/flannel/setup.go index eb9228712e0d..28c2cd7c4d85 100644 --- a/pkg/agent/flannel/setup.go +++ b/pkg/agent/flannel/setup.go @@ -5,6 +5,7 @@ import ( "fmt" "net" "path/filepath" + goruntime "runtime" "strings" "github.com/k3s-io/k3s/pkg/agent/util" @@ -23,34 +24,6 @@ import ( ) const ( - cniConf = `{ - "name":"cbr0", - "cniVersion":"1.0.0", - "plugins":[ - { - "type":"flannel", - "delegate":{ - "hairpinMode":true, - "forceAddress":true, - "isDefaultGateway":true - } - }, - { - "type":"portmap", - "capabilities":{ - "portMappings":true - } - }, - { - "type":"bandwidth", - "capabilities":{ - "bandwidth":true - } - } - ] -} -` - flannelConf = `{ "Network": "%CIDR%", "EnableIPv6": %IPV6_ENABLED%, @@ -60,10 +33,6 @@ const ( } ` - vxlanBackend = `{ - "Type": "vxlan" -}` - hostGWBackend = `{ "Type": "host-gw" }` @@ -153,7 +122,20 @@ func createCNIConf(dir string, nodeConfig *config.Node) error { logrus.Debugf("Using %s as the flannel CNI conf", nodeConfig.AgentConfig.FlannelCniConfFile) return util.CopyFile(nodeConfig.AgentConfig.FlannelCniConfFile, p, false) } - return util.WriteFile(p, cniConf) + + cniConfJSON := cniConf + if goruntime.GOOS == "windows" { + extIface, err := LookupExtInterface(nodeConfig.FlannelIface, ipv4) + if err != nil { + return err + } + + cniConfJSON = strings.ReplaceAll(cniConfJSON, "%IPV4_ADDRESS%", extIface.IfaceAddr.String()) + cniConfJSON = strings.ReplaceAll(cniConfJSON, "%CLUSTER_CIDR%", nodeConfig.AgentConfig.ClusterCIDR.String()) + cniConfJSON = strings.ReplaceAll(cniConfJSON, "%SERVICE_CIDR%", nodeConfig.AgentConfig.ServiceCIDR.String()) + } + + return util.WriteFile(p, cniConfJSON) } func createFlannelConf(nodeConfig *config.Node) error { diff --git a/pkg/agent/flannel/setup_linux.go b/pkg/agent/flannel/setup_linux.go new file mode 100644 index 000000000000..36404a01c68f --- /dev/null +++ b/pkg/agent/flannel/setup_linux.go @@ -0,0 +1,38 @@ +//go:build linux +// +build linux + +package flannel + +const ( + cniConf = `{ + "name":"cbr0", + "cniVersion":"1.0.0", + "plugins":[ + { + "type":"flannel", + "delegate":{ + "hairpinMode":true, + "forceAddress":true, + "isDefaultGateway":true + } + }, + { + "type":"portmap", + "capabilities":{ + "portMappings":true + } + }, + { + "type":"bandwidth", + "capabilities":{ + "bandwidth":true + } + } + ] +} +` + + vxlanBackend = `{ + "Type": "vxlan" +}` +) diff --git a/pkg/agent/flannel/setup_windows.go b/pkg/agent/flannel/setup_windows.go new file mode 100644 index 000000000000..7b9c513cd364 --- /dev/null +++ b/pkg/agent/flannel/setup_windows.go @@ -0,0 +1,59 @@ +//go:build windows +// +build windows + +package flannel + +const ( + cniConf = `{ + "name":"flannel.4096", + "cniVersion":"1.0.0", + "plugins":[ + { + "type":"flannel", + "capabilities": { + "portMappings": true, + "dns": true + }, + "delegate": { + "type": "win-overlay", + "apiVersion": 2, + "Policies": [{ + "Name": "EndpointPolicy", + "Value": { + "Type": "OutBoundNAT", + "Settings": { + "Exceptions": [ + "%CLUSTER_CIDR%", "%SERVICE_CIDR%" + ] + } + } + }, { + "Name": "EndpointPolicy", + "Value": { + "Type": "SDNRoute", + "Settings": { + "DestinationPrefix": "%SERVICE_CIDR%", + "NeedEncap": true + } + } + }, { + "name": "EndpointPolicy", + "value": { + "Type": "ProviderAddress", + "Settings": { + "ProviderAddress": "%IPV4_ADDRESS%" + } + } + }] + } + } + ] +} +` + + vxlanBackend = `{ + "Type": "vxlan", + "VNI": 4096, + "Port": 4789 +}` +) diff --git a/pkg/daemons/agent/agent_windows.go b/pkg/daemons/agent/agent_windows.go index 36a5f505658a..3154d94e1bd3 100644 --- a/pkg/daemons/agent/agent_windows.go +++ b/pkg/daemons/agent/agent_windows.go @@ -122,5 +122,6 @@ func kubeletArgs(cfg *config.Agent) map[string]string { if cfg.ProtectKernelDefaults { argsMap["protect-kernel-defaults"] = "true" } + return argsMap } diff --git a/pkg/daemons/executor/embed.go b/pkg/daemons/executor/embed.go index 9f155bfde74b..0e02c3fd1929 100644 --- a/pkg/daemons/executor/embed.go +++ b/pkg/daemons/executor/embed.go @@ -96,9 +96,9 @@ func (e *Embedded) Kubelet(ctx context.Context, args []string) error { return nil } -func (*Embedded) KubeProxy(ctx context.Context, args []string) error { +func (e *Embedded) KubeProxy(ctx context.Context, args []string) error { command := proxy.NewProxyCommand() - command.SetArgs(args) + command.SetArgs(daemonconfig.GetArgs(platformKubeProxyArgs(e.nodeConfig), args)) go func() { defer func() { diff --git a/pkg/daemons/executor/embed_linux.go b/pkg/daemons/executor/embed_linux.go new file mode 100644 index 000000000000..0b9f4d5e78f9 --- /dev/null +++ b/pkg/daemons/executor/embed_linux.go @@ -0,0 +1,16 @@ +//go:build linux && !no_embedded_executor +// +build linux,!no_embedded_executor + +package executor + +import ( + daemonconfig "github.com/k3s-io/k3s/pkg/daemons/config" + + // registering k3s cloud provider + _ "github.com/k3s-io/k3s/pkg/cloudprovider" +) + +func platformKubeProxyArgs(nodeConfig *daemonconfig.Node) map[string]string { + argsMap := map[string]string{} + return argsMap +} diff --git a/pkg/daemons/executor/embed_windows.go b/pkg/daemons/executor/embed_windows.go new file mode 100644 index 000000000000..ac97273e689a --- /dev/null +++ b/pkg/daemons/executor/embed_windows.go @@ -0,0 +1,89 @@ +//go:build windows && !no_embedded_executor +// +build windows,!no_embedded_executor + +package executor + +import ( + "encoding/json" + "os" + "os/exec" + "strings" + "time" + + "github.com/Microsoft/hcsshim" + "github.com/sirupsen/logrus" + + // registering k3s cloud provider + _ "github.com/k3s-io/k3s/pkg/cloudprovider" + daemonconfig "github.com/k3s-io/k3s/pkg/daemons/config" +) + +const ( + networkName = "flannel.4096" +) + +type SourceVipResponse struct { + IP4 struct { + IP string `json:"ip"` + } `json:"ip4"` +} + +func platformKubeProxyArgs(nodeConfig *daemonconfig.Node) map[string]string { + argsMap := map[string]string{} + argsMap["network-name"] = networkName + if sourceVip := waitForSourceVip(networkName, nodeConfig); sourceVip != "" { + argsMap["source-vip"] = sourceVip + } + return argsMap +} + +func waitForSourceVip(networkName string, nodeConfig *daemonconfig.Node) string { + for range time.Tick(time.Second * 5) { + network, err := hcsshim.GetHNSNetworkByName(networkName) + if err != nil { + logrus.WithError(err).Warningf("can't find HNS network, retrying %s", networkName) + continue + } + if network.ManagementIP == "" { + continue + } + + subnet := network.Subnets[0].AddressPrefix + + configData := `{ + "cniVersion": "0.2.0", + "name": "vxlan0", + "ipam": { + "type": "host-local", + "ranges": [[{"subnet":"` + subnet + `"}]], + "dataDir": "/var/lib/cni/networks" + } + }` + + cmd := exec.Command("host-local.exe") + cmd.Env = append(os.Environ(), + "CNI_COMMAND=ADD", + "CNI_CONTAINERID=dummy", + "CNI_NETNS=dummy", + "CNI_IFNAME=dummy", + "CNI_PATH="+nodeConfig.AgentConfig.CNIBinDir, + ) + + cmd.Stdin = strings.NewReader(configData) + out, err := cmd.Output() + if err != nil { + logrus.WithError(err).Warning("Failed to execute host-local.exe") + continue + } + + var sourceVipResp SourceVipResponse + err = json.Unmarshal(out, &sourceVipResp) + if err != nil { + logrus.WithError(err).Warning("Failed to unmarshal sourceVip response") + continue + } + + return strings.TrimSpace(strings.Split(sourceVipResp.IP4.IP, "/")[0]) + } + return "" +} From 6c064b535ef4eeb2e10ad70085c907b03af50b28 Mon Sep 17 00:00:00 2001 From: Sean Yen Date: Mon, 25 Sep 2023 13:34:47 -0700 Subject: [PATCH 06/11] remove win-bridge. Signed-off-by: Sean Yen --- pkg/agent/flannel/setup.go | 10 ++++++++++ pkg/daemons/executor/embed_windows.go | 1 + scripts/package-cli | 1 - 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/pkg/agent/flannel/setup.go b/pkg/agent/flannel/setup.go index 28c2cd7c4d85..b8b7eb0b0b26 100644 --- a/pkg/agent/flannel/setup.go +++ b/pkg/agent/flannel/setup.go @@ -187,6 +187,16 @@ func createFlannelConf(nodeConfig *config.Node) error { var backendConf string backendOptions := make(map[string]string) + // precheck and error out unsupported flannel backends. + switch nodeConfig.FlannelBackend { + case config.FlannelBackendHostGW: + case config.FlannelBackendTailscale: + case config.FlannelBackendWireguardNative: + if goruntime.GOOS == "windows" { + return fmt.Errorf("unsupported flannel backend '%s' for Windows", nodeConfig.FlannelBackend) + } + } + switch nodeConfig.FlannelBackend { case config.FlannelBackendVXLAN: backendConf = vxlanBackend diff --git a/pkg/daemons/executor/embed_windows.go b/pkg/daemons/executor/embed_windows.go index ac97273e689a..e65913477c8f 100644 --- a/pkg/daemons/executor/embed_windows.go +++ b/pkg/daemons/executor/embed_windows.go @@ -45,6 +45,7 @@ func waitForSourceVip(networkName string, nodeConfig *daemonconfig.Node) string continue } if network.ManagementIP == "" { + logrus.WithError(err).Warningf("wait for management IP, retrying %s", networkName) continue } diff --git a/scripts/package-cli b/scripts/package-cli index 7c419e73b5ea..9266926c41a5 100755 --- a/scripts/package-cli +++ b/scripts/package-cli @@ -24,7 +24,6 @@ cni_binaries=( if [ ${OS} = windows ]; then cni_binaries=( - "win-bridge" "win-overlay" "flannel" "host-local" From 9691a4df3d3bf390c3b5b91f9757ab7063217f0c Mon Sep 17 00:00:00 2001 From: Sean Yen Date: Fri, 6 Oct 2023 13:36:33 -0700 Subject: [PATCH 07/11] Adding dual-stack and IPv6 precheck. Signed-off-by: Sean Yen --- pkg/agent/run.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pkg/agent/run.go b/pkg/agent/run.go index d8723b79f424..4d7cf4d856e6 100644 --- a/pkg/agent/run.go +++ b/pkg/agent/run.go @@ -6,6 +6,7 @@ import ( "net" "os" "path/filepath" + goruntime "runtime" "strconv" "strings" "time" @@ -83,6 +84,11 @@ func run(ctx context.Context, cfg cmds.Agent, proxy proxy.Proxy) error { enableIPv6 := dualCluster || clusterIPv6 enableIPv4 := dualCluster || clusterIPv4 + // dualStack or IPv6 are not supported on Windows node + if (goruntime.GOOS == "windows") && (enableIPv6 || dualNode) { + return fmt.Errorf("dual-stack or IPv6 are not supported on Windows node") + } + conntrackConfig, err := getConntrackConfig(nodeConfig) if err != nil { return errors.Wrap(err, "failed to validate kube-proxy conntrack configuration") From 72217fdafb4ada25e118eae63523b9857732194d Mon Sep 17 00:00:00 2001 From: Sean Yen Date: Mon, 9 Oct 2023 15:20:55 -0700 Subject: [PATCH 08/11] address the feedback Signed-off-by: Sean Yen --- pkg/agent/run.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/agent/run.go b/pkg/agent/run.go index 4d7cf4d856e6..735c41e70f98 100644 --- a/pkg/agent/run.go +++ b/pkg/agent/run.go @@ -85,7 +85,7 @@ func run(ctx context.Context, cfg cmds.Agent, proxy proxy.Proxy) error { enableIPv4 := dualCluster || clusterIPv4 // dualStack or IPv6 are not supported on Windows node - if (goruntime.GOOS == "windows") && (enableIPv6 || dualNode) { + if (goruntime.GOOS == "windows") && enableIPv6 { return fmt.Errorf("dual-stack or IPv6 are not supported on Windows node") } From 73636de21dfe5297cc645b2cc3c3e28ff1b3d9b6 Mon Sep 17 00:00:00 2001 From: Sean Yen Date: Mon, 9 Oct 2023 19:28:58 -0700 Subject: [PATCH 09/11] adding -buildvcs=false Signed-off-by: Sean Yen --- scripts/package-cli | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/package-cli b/scripts/package-cli index 9266926c41a5..8c6d6cb3cdc2 100755 --- a/scripts/package-cli +++ b/scripts/package-cli @@ -78,7 +78,7 @@ LDFLAGS=" " TAGS="urfave_cli_no_docs" STATIC="-extldflags '-static'" -CGO_ENABLED=0 "${GO}" build -tags "$TAGS" -ldflags "$LDFLAGS $STATIC" -o ${CMD_NAME} ./cmd/k3s +CGO_ENABLED=0 "${GO}" build -tags "$TAGS" -buildvcs=false -ldflags "$LDFLAGS $STATIC" -o ${CMD_NAME} ./cmd/k3s stat ${CMD_NAME} From 4ab76c0e9a7a5bde3b134476cd4affcb5b18f9b9 Mon Sep 17 00:00:00 2001 From: Sean Yen Date: Wed, 11 Oct 2023 22:23:14 -0700 Subject: [PATCH 10/11] ensure ./etc exists Signed-off-by: Sean Yen --- scripts/package-cli | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/package-cli b/scripts/package-cli index 8c6d6cb3cdc2..c790f04aab41 100755 --- a/scripts/package-cli +++ b/scripts/package-cli @@ -40,6 +40,7 @@ cp contrib/util/check-config.sh bin/check-config rm -rf build/data mkdir -p build/data build/out mkdir -p dist/artifacts +mkdir -p ./etc ( set +x From 95ad813921bd95ae9019ca5669b060e09100de7b Mon Sep 17 00:00:00 2001 From: Sean Yen Date: Mon, 16 Oct 2023 09:50:03 -0700 Subject: [PATCH 11/11] go mod tidy. Signed-off-by: Sean Yen --- go.mod | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 14c44a62c117..c661fca9a2ec 100644 --- a/go.mod +++ b/go.mod @@ -86,7 +86,7 @@ replace ( ) require ( - github.com/Microsoft/hcsshim v0.11.0 + github.com/Microsoft/hcsshim v0.11.1 github.com/Mirantis/cri-dockerd v0.0.0-00010101000000-000000000000 github.com/blang/semver/v4 v4.0.0 github.com/cloudnativelabs/kube-router/v2 v2.0.0-00010101000000-000000000000 @@ -185,7 +185,6 @@ require ( github.com/JeffAshton/win_pdh v0.0.0-20161109143554-76bb4ee9f0ab // indirect github.com/MakeNowJust/heredoc v1.0.0 // indirect github.com/Microsoft/go-winio v0.6.1 // indirect - github.com/Microsoft/hcsshim v0.11.1 // indirect github.com/NYTimes/gziphandler v1.1.1 // indirect github.com/Rican7/retry v0.1.0 // indirect github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df // indirect