diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index 67d36eaf5..268377521 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -77,8 +77,8 @@ jobs: token: ${{ secrets.HUB_JWT }} action: tag user: tf-autobuilder - name: ${{ steps.tag.outputs.reference }}/zos.flist - target: tf-autobuilder/zos:${{ steps.version.outputs.version }}.flist + name: ${{ steps.tag.outputs.reference }}/zos-light.flist + target: tf-autobuilder/zos-light:${{ steps.version.outputs.version }}.flist # only for main branch (devnet) # this basically releases this build to devnet @@ -89,5 +89,5 @@ jobs: token: ${{ secrets.HUB_JWT }} action: crosstag user: tf-zos - name: development + name: development-zos-light target: tf-autobuilder/${{ steps.tag.outputs.reference }} diff --git a/bootstrap/bootstrap/Cargo.lock b/bootstrap/bootstrap/Cargo.lock index 8f3f5b470..d8f21893d 100644 --- a/bootstrap/bootstrap/Cargo.lock +++ b/bootstrap/bootstrap/Cargo.lock @@ -155,6 +155,15 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" +[[package]] +name = "deranged" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" +dependencies = [ + "powerfmt", +] + [[package]] name = "encoding_rs" version = "0.8.32" @@ -528,6 +537,12 @@ dependencies = [ "void", ] +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + [[package]] name = "num_cpus" version = "1.15.0" @@ -576,7 +591,7 @@ checksum = "b501e44f11665960c7e7fcf062c7d96a14ade4aa98116c004b2e37b5be7d736c" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.107", ] [[package]] @@ -631,20 +646,26 @@ version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160" +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + [[package]] name = "proc-macro2" -version = "1.0.51" +version = "1.0.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d727cae5b39d21da60fa540906919ad737832fe0b1c165da3a34d6548c849d6" +checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" dependencies = [ "unicode-ident", ] [[package]] name = "quote" -version = "1.0.23" +version = "1.0.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8856d8364d252a14d474036ea1358d63c9e6965c8e5c1885c18f73d70bff9c7b" +checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" dependencies = [ "proc-macro2", ] @@ -877,22 +898,22 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.152" +version = "1.0.206" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb7d1f0d3021d347a83e556fc4683dea2ea09d87bccdf88ff5c12545d89d5efb" +checksum = "5b3e4cd94123dd520a128bcd11e34d9e9e423e7e3e50425cb1b4b1e3549d0284" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.152" +version = "1.0.206" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af487d118eecd09402d70a5d72551860e788df87b464af30e5ea6a38c75c541e" +checksum = "fabfb6138d2383ea8208cf98ccf69cdfb1aff4088460681d84189aa259762f97" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.74", ] [[package]] @@ -973,6 +994,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "2.0.74" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fceb41e3d546d0bd83421d3409b1460cc7444cd389341a4c880fe7a042cb3d7" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "tempfile" version = "3.3.0" @@ -998,13 +1030,16 @@ dependencies = [ [[package]] name = "time" -version = "0.3.19" +version = "0.3.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53250a3b3fed8ff8fd988587d8925d26a83ac3845d9e03b220b37f34c2b8d6c2" +checksum = "5dfd88e563464686c916c7e46e623e520ddc6d79fa6641390f2e3fa86e83e885" dependencies = [ + "deranged", "itoa", "libc", + "num-conv", "num_threads", + "powerfmt", "serde", "time-core", "time-macros", @@ -1012,16 +1047,17 @@ dependencies = [ [[package]] name = "time-core" -version = "0.1.0" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e153e1f1acaef8acc537e68b44906d2db6436e2b35ac2c6b42640fff91f00fd" +checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" [[package]] name = "time-macros" -version = "0.2.7" +version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a460aeb8de6dcb0f381e1ee05f1cd56fcf5a5f6eb8187ff3d8f0b11078d38b7c" +checksum = "3f252a68540fde3a3877aeea552b832b40ab9a69e318efd078774a01ddee1ccf" dependencies = [ + "num-conv", "time-core", ] @@ -1217,7 +1253,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn", + "syn 1.0.107", "wasm-bindgen-shared", ] @@ -1251,7 +1287,7 @@ checksum = "2aff81306fcac3c7515ad4e177f521b5c9a15f2b08f4e32d823066102f35a5f6" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.107", "wasm-bindgen-backend", "wasm-bindgen-shared", ] diff --git a/bootstrap/bootstrap/README.md b/bootstrap/bootstrap/README.md index 5374ba3a3..3ca3db384 100644 --- a/bootstrap/bootstrap/README.md +++ b/bootstrap/bootstrap/README.md @@ -17,7 +17,7 @@ will do a multiple stage bootstrap. Currently this is only two stages: ## How to works - Bootstrap is used by [0-initramfs](https://github.com/threefoldtech/0-initramfs/blob/development-zos-v3/packages/modules.sh) to basically add `internet` and `bootstrap` services to the base image -- After internet service is fully started, bootstrap will start to download flists needed to for zos node to work properly +- After internet service is fully started, bootstrap will start to download flists needed for zos node to work properly - As described above bootstrap run in two stages: - The first stage is used to update bootstrap itself, and it is done like that to avoid re-building the image if we only changed the bootstrap code. this update is basically done from `tf-autobuilder` repo in the [hub/tf-autobuilder](https://hub.grid.tf/tf-autobuilder) and download the latest bootstrap flist - For the second stage bootstrap will download the flists for that env. bootstrap cares about `runmode` argument that we pass during the start of the node. for example if we passed `runmode=dev` it will get the the tag `development` under [hub/tf-zos](https://hub.grid.tf/tf-zos) each tag is linked to a sub-directory where all flists for this env exists to be downloaded and installed on the node diff --git a/bootstrap/bootstrap/src/bootstrap.rs b/bootstrap/bootstrap/src/bootstrap.rs index 5e1c85ec8..a1dca58a0 100644 --- a/bootstrap/bootstrap/src/bootstrap.rs +++ b/bootstrap/bootstrap/src/bootstrap.rs @@ -26,15 +26,19 @@ fn bootstrap_zos(cfg: &config::Config) -> Result<()> { let flist = match &cfg.runmode { RunMode::Prod => match &cfg.version { Version::V3 => "zos:production-3:latest.flist", + _ => bail!("unsupported version in old style"), }, RunMode::Dev => match &cfg.version { Version::V3 => "zos:development-3:latest.flist", + _ => bail!("unsupported version in old style"), }, RunMode::Test => match &cfg.version { Version::V3 => "zos:testing-3:latest.flist", + _ => bail!("unsupported version in old style"), }, RunMode::QA => match &cfg.version { Version::V3 => "zos:qa-3:latest.flist", + _ => bail!("unsupported version in old style"), }, }; @@ -116,11 +120,16 @@ pub fn install(cfg: &config::Config) -> Result<()> { let repo = Repo::new(ZOS_REPO); let runmode = cfg.runmode.to_string(); - // we need to list all taglinks inside the repo + let mut listname = runmode.clone(); + match cfg.version { + Version::V3 => {} + Version::V4 => listname = format!("{}-v4", runmode), + } + // we need to list all taglinks let mut tag = None; for list in repo.list()? { - if list.kind == Kind::TagLink && list.name == runmode { + if list.kind == Kind::TagLink && list.name == listname { tag = Some(list); break; } @@ -163,6 +172,7 @@ pub fn install(cfg: &config::Config) -> Result<()> { fn install_packages_old(cfg: &config::Config) -> Result<()> { let name = match cfg.version { Version::V3 => BIN_REPO_V3, + _ => bail!("unsupported version for old style"), }; let repo = match cfg.runmode { diff --git a/bootstrap/bootstrap/src/config.rs b/bootstrap/bootstrap/src/config.rs index 1a918fc23..3e39ff23d 100644 --- a/bootstrap/bootstrap/src/config.rs +++ b/bootstrap/bootstrap/src/config.rs @@ -28,6 +28,7 @@ impl Display for RunMode { #[derive(Debug)] pub enum Version { V3, + V4, } fn runmode() -> Result { @@ -64,6 +65,7 @@ fn version() -> Result { Some(input) => match input { Some(input) => match input.as_ref() { "v3" => Version::V3, + "v4" => Version::V4, m => { bail!("unknown version: {}", m); } @@ -119,7 +121,7 @@ impl Config { } Ok(Config { - stage: stage, + stage, debug: matches.occurrences_of("debug") > 0, runmode: runmode()?, version: version()?, diff --git a/bootstrap/bootstrap/src/kparams.rs b/bootstrap/bootstrap/src/kparams.rs index d960adbd6..61e7f54be 100644 --- a/bootstrap/bootstrap/src/kparams.rs +++ b/bootstrap/bootstrap/src/kparams.rs @@ -40,10 +40,11 @@ mod tests { #[test] fn test_parse() -> Result<(), Error> { - let input: &str = "initrd=initramfs-linux.img root=UUID=10f9e7bb-ba63-4fbd-a95e-c78b5496cfbe rootflags=subvol=root rw b43.allhwsupport=1"; + let input: &str = "initrd=initramfs-linux.img version=v3 root=UUID=10f9e7bb-ba63-4fbd-a95e-c78b5496cfbe rootflags=subvol=root rw b43.allhwsupport=1"; let result = parse(input.as_bytes())?; - assert_eq!(result.len(), 5); + assert_eq!(result.len(), 6); assert_eq!(result["rw"], None); + assert_eq!(result["version"], Some(String::from("v3"))); assert_eq!(result["rootflags"], Some(String::from("subvol=root"))); Ok(()) } diff --git a/cmds/internet/main.go b/cmds/internet/main.go index d63e0b66c..300091ad6 100644 --- a/cmds/internet/main.go +++ b/cmds/internet/main.go @@ -13,12 +13,12 @@ import ( "github.com/threefoldtech/zos/pkg/app" "github.com/threefoldtech/zos/pkg/environment" - "github.com/threefoldtech/zos/pkg/network/bootstrap" - "github.com/threefoldtech/zos/pkg/network/bridge" - "github.com/threefoldtech/zos/pkg/network/dhcp" - "github.com/threefoldtech/zos/pkg/network/ifaceutil" - "github.com/threefoldtech/zos/pkg/network/options" - "github.com/threefoldtech/zos/pkg/network/types" + "github.com/threefoldtech/zos/pkg/netlight/bootstrap" + "github.com/threefoldtech/zos/pkg/netlight/bridge" + "github.com/threefoldtech/zos/pkg/netlight/dhcp" + "github.com/threefoldtech/zos/pkg/netlight/ifaceutil" + "github.com/threefoldtech/zos/pkg/netlight/options" + "github.com/threefoldtech/zos/pkg/netlight/types" "github.com/threefoldtech/zos/pkg/zinit" "github.com/threefoldtech/zos/pkg/version" diff --git a/cmds/modules/gateway/main.go b/cmds/modules/gateway/main.go deleted file mode 100644 index 15c91c55a..000000000 --- a/cmds/modules/gateway/main.go +++ /dev/null @@ -1,84 +0,0 @@ -package gateway - -import ( - "context" - - "github.com/pkg/errors" - "github.com/threefoldtech/zos/pkg/gateway" - "github.com/threefoldtech/zos/pkg/utils" - "github.com/urfave/cli/v2" - - "github.com/rs/zerolog/log" - - "github.com/threefoldtech/zbus" -) - -const ( - module = "gateway" -) - -// Module is entry point for module -var Module cli.Command = cli.Command{ - Name: "gateway", - Usage: "manage web gateway proxy", - Flags: []cli.Flag{ - &cli.StringFlag{ - Name: "root", - Usage: "`ROOT` working directory of the module", - Value: "/var/cache/modules/gateway", - }, - &cli.StringFlag{ - Name: "broker", - Usage: "connection string to the message `BROKER`", - Value: "unix:///var/run/redis.sock", - }, - &cli.UintFlag{ - Name: "workers", - Usage: "number of workers `N`", - Value: 1, - }, - }, - Action: action, -} - -func action(cli *cli.Context) error { - var ( - moduleRoot string = cli.String("root") - msgBrokerCon string = cli.String("broker") - workerNr uint = cli.Uint("workers") - ) - - server, err := zbus.NewRedisServer(module, msgBrokerCon, workerNr) - if err != nil { - return errors.Wrap(err, "fail to connect to message broker server") - } - - client, err := zbus.NewRedisClient(msgBrokerCon) - if err != nil { - return errors.Wrap(err, "failed to connect to zbus broker") - } - - mod, err := gateway.New(cli.Context, client, moduleRoot) - if err != nil { - return errors.Wrap(err, "failed to construct gateway object") - } - server.Register(zbus.ObjectID{Name: "manager", Version: "0.0.1"}, mod) - - ctx, cancel := utils.WithSignal(context.Background()) - defer cancel() - - log.Info(). - Str("broker", msgBrokerCon). - Uint("worker nr", workerNr). - Msg("starting gateway module") - - utils.OnDone(ctx, func(_ error) { - log.Info().Msg("shutting down") - }) - - if err := server.Run(ctx); err != nil && err != context.Canceled { - return errors.Wrap(err, "unexpected error") - } - - return nil -} diff --git a/cmds/modules/netlightd/main.go b/cmds/modules/netlightd/main.go new file mode 100644 index 000000000..1eb685c57 --- /dev/null +++ b/cmds/modules/netlightd/main.go @@ -0,0 +1,170 @@ +package netlightd + +import ( + "context" + "embed" + "fmt" + "net" + "os" + "os/exec" + "time" + + "github.com/oasisprotocol/curve25519-voi/primitives/x25519" + "github.com/pkg/errors" + "github.com/threefoldtech/zos/pkg/netlight" + "github.com/threefoldtech/zos/pkg/netlight/nft" + "github.com/threefoldtech/zos/pkg/netlight/resource" + "github.com/urfave/cli/v2" + + "github.com/cenkalti/backoff/v3" + "github.com/rs/zerolog/log" + "github.com/threefoldtech/zbus" + "github.com/threefoldtech/zos/pkg/netlight/bootstrap" + "github.com/threefoldtech/zos/pkg/stubs" + "github.com/threefoldtech/zos/pkg/utils" +) + +const ( + redisSocket = "unix:///var/run/redis.sock" + module = "netlight" +) + +//go:embed nft/rules.nft +var nftRules embed.FS + +// Module is entry point for module +var Module cli.Command = cli.Command{ + Name: "netlightd", + Usage: "handles network resources and user networks", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "root", + Usage: "`ROOT` working directory of the module", + Value: "/var/cache/modules/networkd", + }, + &cli.StringFlag{ + Name: "broker", + Usage: "connection string to the message `BROKER`", + Value: "unix:///var/run/redis.sock", + }, + &cli.UintFlag{ + Name: "workers", + Usage: "number of workers `N`", + Value: 1, + }, + }, + Action: action, +} + +func myceliumSeedFromIdentity(privKey []byte) []byte { + seed := x25519.PrivateKey(x25519.EdPrivateKeyToX25519([]byte(privKey))) + return seed[:] +} + +func action(cli *cli.Context) error { + var ( + root string = cli.String("root") + broker string = cli.String("broker") + workerNr uint = cli.Uint("workers") + ) + + if err := os.MkdirAll(root, 0755); err != nil { + return errors.Wrap(err, "fail to create module root") + } + + waitMyceliumBin() + + if err := bootstrap.DefaultBridgeValid(); err != nil { + return errors.Wrap(err, "invalid setup") + } + + client, err := zbus.NewRedisClient(broker) + if err != nil { + return errors.Wrap(err, "failed to connect to zbus broker") + } + + server, err := zbus.NewRedisServer(module, broker, workerNr) + if err != nil { + return errors.Wrap(err, "failed to connect to zbus broker") + } + + identity := stubs.NewIdentityManagerStub(client) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + ctx, _ = utils.WithSignal(ctx) + utils.OnDone(ctx, func(_ error) { + log.Info().Msg("shutting down") + }) + + rules, err := nftRules.Open("nft/rules.nft") + if err != nil { + return fmt.Errorf("failed to load rules.nft file") + } + + if err := nft.Apply(rules, ""); err != nil { + return fmt.Errorf("failed to apply host nft rules: %w", err) + } + + bridge, err := netlight.CreateNDMZBridge() + if err != nil { + return fmt.Errorf("failed to create ndmz bridge: %w", err) + } + + _, err = resource.Create("dmz", bridge, &net.IPNet{ + IP: net.ParseIP("100.127.0.2"), + Mask: net.CIDRMask(16, 32), + }, netlight.NDMZGwIP, nil, myceliumSeedFromIdentity(identity.PrivateKey(cli.Context))) + + if err != nil { + return fmt.Errorf("failed to create ndmz resource: %w", err) + } + + // create a test user network + // r, err := resource.Create("test", bridge, &net.IPNet{ + // IP: net.ParseIP("100.127.0.10"), + // Mask: net.CIDRMask(16, 32), + // }, netlight.NDMZGwIP, &net.IPNet{ + // IP: net.ParseIP("192.168.1.0"), + // Mask: net.CIDRMask(24, 32), + // }, zos.MustBytesFromHex("8ad7d29b81df3f3ef0a5ff95c25cc0824ef33137fbbcf22d2f23b0222ae3ac00")) + + // if err != nil { + // return fmt.Errorf("failed to create user resource: %w", err) + // } + // tap, err := r.AttachPrivate("123", &net.IPNet{ + // IP: net.ParseIP("192.168.1.15"), + // Mask: net.CIDRMask(24, 32), + // }) + + // if err != nil { + // return fmt.Errorf("failed to attach to private network: %w", err) + // } + // fmt.Println(tap) + + mod, err := netlight.NewNetworker() + if err != nil { + return fmt.Errorf("failed to create Networker: %w", err) + } + if err := server.Register(zbus.ObjectID{Name: module, Version: "0.0.1"}, mod); err != nil { + return fmt.Errorf("failed to register network light module: %w", err) + } + + if err := server.Run(ctx); err != nil && err != context.Canceled { + return errors.Wrap(err, "unexpected error") + } + return nil +} + +func waitMyceliumBin() { + log.Info().Msg("wait for mycelium binary to be available") + bo := backoff.NewExponentialBackOff() + bo.MaxElapsedTime = 0 // forever + _ = backoff.RetryNotify(func() error { + _, err := exec.LookPath("mycelium") + return err + }, bo, func(err error, d time.Duration) { + log.Warn().Err(err).Msgf("mycelium binary not found, retying in %s", d.String()) + }) +} diff --git a/cmds/modules/netlightd/nft/rules.nft b/cmds/modules/netlightd/nft/rules.nft new file mode 100644 index 000000000..68bdb5a47 --- /dev/null +++ b/cmds/modules/netlightd/nft/rules.nft @@ -0,0 +1,29 @@ +add table inet filter; +add table arp filter; +add table bridge filter; +add table nat; + + +add chain inet filter input { type filter hook input priority filter; policy accept; } +add chain inet filter forward { type filter hook forward priority filter; policy accept; } +add chain inet filter output { type filter hook output priority filter; policy accept; } +add chain inet filter prerouting { type filter hook prerouting priority filter; policy accept; } + +add chain arp filter input { type filter hook input priority filter; policy accept; } +add chain arp filter output { type filter hook output priority filter; policy accept; } + +add chain bridge filter input { type filter hook input priority filter; policy accept; } +add chain bridge filter forward { type filter hook forward priority filter; policy accept; } +add chain bridge filter prerouting { type filter hook prerouting priority filter; policy accept; } +add chain bridge filter postrouting { type filter hook postrouting priority filter; policy accept; } +add chain bridge filter output { type filter hook output priority filter; policy accept; } + +add chain nat postrouting { type nat hook postrouting priority 100 ; } + +flush chain bridge filter forward; +flush chain inet filter forward; +flush chain inet filter prerouting; +flush chain nat postrouting + +add rule inet filter prerouting iifname "b-*" tcp dport {25, 587, 465} reject with icmp type admin-prohibited; +add rule nat postrouting iifname gw masquerade fully-random; diff --git a/cmds/modules/networkd/main.go b/cmds/modules/networkd/main.go deleted file mode 100644 index 45f6c374d..000000000 --- a/cmds/modules/networkd/main.go +++ /dev/null @@ -1,271 +0,0 @@ -package networkd - -import ( - "context" - "fmt" - "os" - "os/exec" - "time" - - "github.com/pkg/errors" - "github.com/threefoldtech/zos/pkg/environment" - "github.com/threefoldtech/zos/pkg/network/dhcp" - "github.com/threefoldtech/zos/pkg/network/mycelium" - "github.com/threefoldtech/zos/pkg/network/public" - "github.com/threefoldtech/zos/pkg/network/types" - "github.com/threefoldtech/zos/pkg/zinit" - "github.com/urfave/cli/v2" - - "github.com/cenkalti/backoff/v3" - "github.com/rs/zerolog/log" - "github.com/threefoldtech/zbus" - "github.com/threefoldtech/zos/pkg" - "github.com/threefoldtech/zos/pkg/network" - "github.com/threefoldtech/zos/pkg/network/bootstrap" - "github.com/threefoldtech/zos/pkg/network/ndmz" - "github.com/threefoldtech/zos/pkg/network/yggdrasil" - "github.com/threefoldtech/zos/pkg/stubs" - "github.com/threefoldtech/zos/pkg/utils" -) - -const ( - redisSocket = "unix:///var/run/redis.sock" - module = "network" -) - -// Module is entry point for module -var Module cli.Command = cli.Command{ - Name: "networkd", - Usage: "handles network resources and user networks", - Flags: []cli.Flag{ - &cli.StringFlag{ - Name: "root", - Usage: "`ROOT` working directory of the module", - Value: "/var/cache/modules/networkd", - }, - &cli.StringFlag{ - Name: "broker", - Usage: "connection string to the message `BROKER`", - Value: "unix:///var/run/redis.sock", - }, - }, - Action: action, -} - -func action(cli *cli.Context) error { - var ( - root string = cli.String("root") - broker string = cli.String("broker") - ) - - if err := os.MkdirAll(root, 0755); err != nil { - return errors.Wrap(err, "fail to create module root") - } - - waitYggdrasilBin() - - if err := migrateOlderDHCPService(); err != nil { - return errors.Wrap(err, "failed to migrate older dhcp service") - } - - if err := bootstrap.DefaultBridgeValid(); err != nil { - return errors.Wrap(err, "invalid setup") - } - - client, err := zbus.NewRedisClient(broker) - if err != nil { - return errors.Wrap(err, "failed to connect to zbus broker") - } - - identity := stubs.NewIdentityManagerStub(client) - nodeID := identity.NodeID(cli.Context) - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - ctx, _ = utils.WithSignal(ctx) - utils.OnDone(ctx, func(_ error) { - log.Info().Msg("shutting down") - }) - - if err := ensureHostFw(ctx); err != nil { - return errors.Wrap(err, "failed to host firewall rules") - } - - public.SetPersistence(root) - - pub, err := public.LoadPublicConfig() - log.Debug().Err(err).Msgf("public interface configured: %+v", pub) - if err != nil && err != public.ErrNoPublicConfig { - return errors.Wrap(err, "failed to get node public_config") - } - - // EnsurePublicSetup knows how to handle a nil pub (in case of ErrNoPublicConfig) - master, err := public.EnsurePublicSetup(nodeID, environment.MustGet().PubVlan, pub) - if err != nil { - return errors.Wrap(err, "failed to setup public bridge") - } - - dmz := ndmz.New(nodeID.Identity(), master) - - if err := dmz.Create(ctx); err != nil { - return errors.Wrap(err, "failed to create ndmz") - } - - namespace := dmz.Namespace() - if public.HasPublicSetup() { - namespace = public.PublicNamespace - } - - log.Debug().Msg("starting yggdrasil") - ygg, err := setupYgg(ctx, namespace, dmz.Namespace(), identity.PrivateKey(cli.Context)) - if err != nil { - return err - } - - log.Debug().Msg("starting mycelium") - mycelium, err := setupMycelium(ctx, namespace, dmz.Namespace(), identity.PrivateKey(cli.Context)) - if err != nil { - return err - } - - networker, err := network.NewNetworker(identity, dmz, ygg, mycelium) - if err != nil { - return errors.Wrap(err, "error creating network manager") - } - - log.Info().Msg("start zbus server") - if err := startZBusServer(ctx, broker, networker); err != nil { - return errors.Wrap(err, "unexpected error") - } - - return nil -} - -func startZBusServer(ctx context.Context, broker string, networker pkg.Networker) error { - server, err := zbus.NewRedisServer(module, broker, 1) - if err != nil { - log.Error().Err(err).Msgf("fail to connect to message broker server") - } - - server.Register(zbus.ObjectID{Name: module, Version: "0.0.1"}, networker) - - log.Info(). - Str("broker", broker). - Uint("worker nr", 1). - Msg("starting networkd module") - - if err := server.Run(ctx); err != nil && err != context.Canceled { - return err - } - - return nil -} - -func waitYggdrasilBin() { - log.Info().Msg("wait for yggdrasil binary to be available") - bo := backoff.NewExponentialBackOff() - bo.MaxElapsedTime = 0 // forever - _ = backoff.RetryNotify(func() error { - _, err := exec.LookPath("yggdrasil") - return err - }, bo, func(err error, d time.Duration) { - log.Warn().Err(err).Msgf("yggdrasil binary not found, retying in %s", d.String()) - }) -} - -func migrateOlderDHCPService() error { - // only migrate dhcp service if it's using the older client - dhcpService := dhcp.NewService(types.DefaultBridge, "", zinit.Default()) - if dhcpService.IsUsingOlderClient() { - // only migrate if it's using older client - if err := dhcpService.Destroy(); err != nil { - return errors.Wrapf(err, "failed to destory older dhcp service") - } - if err := dhcpService.Create(); err != nil { - return errors.Wrapf(err, "failed to create newer dhcp service") - } - return dhcpService.Start() - } - - return nil -} - -func setupYgg(ctx context.Context, namespace, dmzNs string, privateKey []byte) (ygg *yggdrasil.YggServer, err error) { - yggNs, err := yggdrasil.NewYggdrasilNamespace(namespace) - if err != nil { - return ygg, errors.Wrap(err, "failed to create yggdrasil namespace") - } - - ygg, err = yggdrasil.EnsureYggdrasil(ctx, privateKey, yggNs) - if err != nil { - return ygg, errors.Wrap(err, "failed to start yggdrasil") - } - - if public.HasPublicSetup() { - // if yggdrasil is living inside public namespace - // we still need to setup ndmz to also have yggdrasil but we set the yggdrasil interface - // a different Ip that lives inside the yggdrasil range. - dmzYgg, err := yggdrasil.NewYggdrasilNamespace(dmzNs) - if err != nil { - return ygg, errors.Wrap(err, "failed to setup ygg for dmz namespace") - } - - ip, err := ygg.SubnetFor([]byte(fmt.Sprintf("ygg:%s", dmzNs))) - if err != nil { - return ygg, errors.Wrap(err, "failed to calculate ip for ygg inside dmz") - } - - gw, err := ygg.Gateway() - if err != nil { - return ygg, err - } - - if err := dmzYgg.SetYggIP(ip, gw.IP); err != nil { - return ygg, errors.Wrap(err, "failed to set yggdrasil ip for dmz") - } - } - return -} - -func setupMycelium(ctx context.Context, namespace, dmzNs string, privateKey []byte) (myc *mycelium.MyceliumServer, err error) { - myNs, err := mycelium.NewMyNamespace(namespace) - if err != nil { - return myc, errors.Wrap(err, "failed to create mycelium namespace") - } - - myc, err = mycelium.EnsureMycelium(ctx, privateKey, myNs) - if err != nil { - return myc, errors.Wrap(err, "failed to start mycelium") - } - - if public.HasPublicSetup() { - // if mycelium is living inside public namespace - // we still need to setup ndmz to also have mycelium but we set the mycelium interface - // a different Ip that lives inside the mycelium range. - dmzMy, err := mycelium.NewMyNamespace(dmzNs) - if err != nil { - return myc, errors.Wrap(err, "failed to setup mycelium for dmz namespace") - } - - inspcet, err := myc.InspectMycelium() - if err != nil { - return myc, err - } - - ip, err := inspcet.IPFor([]byte(fmt.Sprintf("my:%s", dmzNs))) - if err != nil { - return myc, errors.Wrap(err, "failed to calculate ip for mycelium inside dmz") - } - - gw, err := inspcet.Gateway() - if err != nil { - return myc, err - } - - if err := dmzMy.SetMyIP(ip, gw.IP); err != nil { - return myc, errors.Wrap(err, "failed to set mycelium ip for dmz") - } - } - return -} diff --git a/cmds/modules/networkd/nft.go b/cmds/modules/networkd/nft.go deleted file mode 100644 index db05f9b72..000000000 --- a/cmds/modules/networkd/nft.go +++ /dev/null @@ -1,61 +0,0 @@ -package networkd - -import ( - "context" - "os/exec" - - "github.com/pkg/errors" - "github.com/rs/zerolog/log" -) - -func ensureHostFw(ctx context.Context) error { - log.Info().Msg("ensuring existing host nft rules") - - cmd := exec.CommandContext(ctx, "/bin/sh", "-c", - ` -nft 'add table inet filter' -nft 'add table arp filter' -nft 'add table bridge filter' - -# duo to a bug we had we need to make sure those chains are -# deleted and then recreated later -nft 'delete chain inet filter input' -nft 'delete chain inet filter forward' -nft 'delete chain inet filter output' - -nft 'delete chain bridge filter input' -nft 'delete chain bridge filter forward' -nft 'delete chain bridge filter output' - -nft 'delete chain arp filter input' -nft 'delete chain arp filter output' - -# recreate chains correctly -nft 'add chain inet filter input { type filter hook input priority filter; policy accept; }' -nft 'add chain inet filter forward { type filter hook forward priority filter; policy accept; }' -nft 'add chain inet filter output { type filter hook output priority filter; policy accept; }' -nft 'add chain inet filter prerouting { type filter hook prerouting priority filter; policy accept; }' - -nft 'add chain arp filter input { type filter hook input priority filter; policy accept; }' -nft 'add chain arp filter output { type filter hook output priority filter; policy accept; }' - -nft 'add chain bridge filter input { type filter hook input priority filter; policy accept; }' -nft 'add chain bridge filter forward { type filter hook forward priority filter; policy accept; }' -nft 'add chain bridge filter prerouting { type filter hook prerouting priority filter; policy accept; }' -nft 'add chain bridge filter postrouting { type filter hook postrouting priority filter; policy accept; }' -nft 'add chain bridge filter output { type filter hook output priority filter; policy accept; }' - -nft 'flush chain bridge filter forward' -nft 'flush chain inet filter forward' -nft 'flush chain inet filter prerouting' - -# drop smtp traffic for hidden nodes -nft 'add rule inet filter prerouting iifname "b-*" tcp dport {25, 587, 465} reject with icmp type admin-prohibited' -`) - - if err := cmd.Run(); err != nil { - return errors.Wrap(err, "could not set up host nft rules") - } - - return nil -} diff --git a/cmds/modules/noded/main.go b/cmds/modules/noded/main.go index ccfd9a9cf..4289a7ae3 100644 --- a/cmds/modules/noded/main.go +++ b/cmds/modules/noded/main.go @@ -92,11 +92,6 @@ func action(cli *cli.Context) error { return errors.Wrap(err, "fail to connect to message broker server") } - consumer, err := events.NewConsumer(msgBrokerCon, module) - if err != nil { - return errors.Wrap(err, "failed to to create event consumer") - } - if printID { sysCl := stubs.NewSystemMonitorStub(redis) fmt.Println(sysCl.NodeID(cli.Context)) @@ -233,15 +228,6 @@ func action(cli *cli.Context) error { log.Info().Uint32("node", node).Uint32("twin", twin).Msg("node registered") - go func() { - for { - if err := public(ctx, node, redis, consumer); err != nil { - log.Error().Err(err).Msg("setting public config failed") - <-time.After(10 * time.Second) - } - } - }() - log.Info().Uint32("twin", twin).Msg("node has been registered") idStub := stubs.NewIdentityManagerStub(redis) fetchCtx, cancel := context.WithTimeout(ctx, 30*time.Second) diff --git a/cmds/modules/noded/public.go b/cmds/modules/noded/public.go deleted file mode 100644 index db06eb70b..000000000 --- a/cmds/modules/noded/public.go +++ /dev/null @@ -1,78 +0,0 @@ -package noded - -import ( - "context" - "time" - - "github.com/pkg/errors" - "github.com/rs/zerolog/log" - substrate "github.com/threefoldtech/tfchain/clients/tfchain-client-go" - "github.com/threefoldtech/zbus" - "github.com/threefoldtech/zos/pkg" - "github.com/threefoldtech/zos/pkg/events" - "github.com/threefoldtech/zos/pkg/stubs" -) - -func setPublicConfig(ctx context.Context, cl zbus.Client, cfg *substrate.PublicConfig) error { - log.Info().Msg("setting node public config") - netMgr := stubs.NewNetworkerStub(cl) - - if cfg == nil { - return netMgr.UnsetPublicConfig(ctx) - } - - pub, err := pkg.PublicConfigFrom(*cfg) - if err != nil { - return errors.Wrap(err, "failed to create public config from setup") - } - - return netMgr.SetPublicConfig(ctx, pub) -} - -// public sets and watches changes to public config on chain and tries to apply the provided setup -func public(ctx context.Context, nodeID uint32, cl zbus.Client, events *events.RedisConsumer) error { - ch, err := events.PublicConfig(ctx) - if err != nil { - return errors.Wrap(err, "failed to subscribe to node events") - } - - substrateGateway := stubs.NewSubstrateGatewayStub(cl) - -reapply: - for { - node, err := substrateGateway.GetNode(ctx, nodeID) - if err != nil { - return errors.Wrap(err, "failed to get node public config") - } - - var cfg *substrate.PublicConfig - if node.PublicConfig.HasValue { - cfg = &node.PublicConfig.AsValue - } - - if err := setPublicConfig(ctx, cl, cfg); err != nil { - return errors.Wrap(err, "failed to set public config (reapply)") - } - - for { - select { - case <-ctx.Done(): - return nil - case event := <-ch: - log.Info().Msgf("got a public config update: %+v", event.PublicConfig) - var cfg *substrate.PublicConfig - if event.PublicConfig.HasValue { - cfg = &event.PublicConfig.AsValue - } - if err := setPublicConfig(ctx, cl, cfg); err != nil { - return errors.Wrap(err, "failed to set public config") - } - case <-time.After(2 * time.Hour): - // last resort, if none of the events - // was received, it will be a good idea to just - // check every 2 hours for changes. - continue reapply - } - } - } -} diff --git a/cmds/modules/provisiond/main.go b/cmds/modules/provisiond/main.go index ad288bb65..5f724639c 100644 --- a/cmds/modules/provisiond/main.go +++ b/cmds/modules/provisiond/main.go @@ -10,7 +10,6 @@ import ( "path/filepath" "time" - "github.com/cenkalti/backoff/v3" "github.com/pkg/errors" substrate "github.com/threefoldtech/tfchain/clients/tfchain-client-go" "github.com/threefoldtech/zos/pkg" @@ -173,22 +172,6 @@ func action(cli *cli.Context) error { identity := stubs.NewIdentityManagerStub(cl) sk := ed25519.PrivateKey(identity.PrivateKey(ctx)) - // block until networkd is ready to serve request from zbus - // this is used to prevent uptime and online status to the explorer if the node is not in a fully ready - // https://github.com/threefoldtech/zos/issues/632 - // NOTE - UPDATE: this block of code should be deprecated - // since we do the waiting in zinit now since provisiond waits for networkd - // which has a 'test' condition in the zinit yaml file for networkd to wait - // for zbus - network := stubs.NewNetworkerStub(cl) - bo := backoff.NewExponentialBackOff() - bo.MaxElapsedTime = 0 - backoff.RetryNotify(func() error { - return network.Ready(cli.Context) - }, bo, func(err error, d time.Duration) { - log.Error().Err(err).Msg("networkd is not ready yet") - }) - // the v1 endpoint will be used by all components to register endpoints // that are specific for that component //v1 := router.PathPrefix("/api/v1").Subrouter() @@ -319,10 +302,10 @@ func action(cli *cli.Context) error { zos.ZMountType, zos.VolumeType, zos.QuantumSafeFSType, - zos.NetworkType, + zos.NetworkLightType, zos.PublicIPv4Type, zos.PublicIPType, - zos.ZMachineType, + zos.ZMachineLightType, zos.ZLogsType, //make sure zlogs comes after zmachine ), // if this is a node reboot, the node needs to diff --git a/cmds/modules/provisiond/reporter.go b/cmds/modules/provisiond/reporter.go index d7ea1b88f..8d765cbea 100644 --- a/cmds/modules/provisiond/reporter.go +++ b/cmds/modules/provisiond/reporter.go @@ -166,87 +166,16 @@ func (r *Reporter) getVmMetrics(ctx context.Context, slot rrd.Slot) error { return nil } -// getNetworkMetrics will collect network consumption for network resource and store it in the given slot -func (r *Reporter) getNetworkMetrics(ctx context.Context, slot rrd.Slot) error { - log.Debug().Msg("collecting networking metrics") - stub := stubs.NewNetworkerStub(r.cl) - - ctx, cancel := context.WithTimeout(ctx, 1*time.Minute) - defer cancel() - metrics, err := stub.Metrics(ctx) - if err != nil { - return err - } - - for wl, consumption := range metrics { - nu := consumption.Nu() - log.Debug().Str("network", wl).Uint64("computed", uint64(nu)).Msgf("consumption: %+v", consumption) - if err := slot.Counter(wl, float64(nu)); err != nil { - return errors.Wrapf(err, "failed to store metrics for '%s'", wl) - } - } - - return nil -} - -// getVmMetrics will collect network consumption every 5 min and store -// it in the rrd database. -func (r *Reporter) getGwMetrics(ctx context.Context, slot rrd.Slot) error { - log.Debug().Msg("collecting networking metrics") - gw := stubs.NewGatewayStub(r.cl) - - ctx, cancel := context.WithTimeout(ctx, 1*time.Minute) - defer cancel() - metrics, err := gw.Metrics(ctx) - if err != nil { - return err - } - requests := metrics.Request - responses := metrics.Response - - sums := make(map[string]float64) - for wl, v := range requests { - sums[wl] = v + responses[wl] - delete(responses, wl) - } - - for wl, v := range responses { - sums[wl] = v - } - - for wl, nu := range sums { - log.Debug().Str("gw", wl).Uint64("computed", uint64(nu)).Msg("consumption") - if err := slot.Counter(wl, float64(nu)); err != nil { - return errors.Wrapf(err, "failed to store metrics for '%s'", wl) - } - } - - return nil -} - func (r *Reporter) getMetrics(ctx context.Context) error { slot, err := r.rrd.Slot() if err != nil { return err } - // NOTICE: disable collecting traffic consumption - // for network resources. So ygg and wg traffic - // will not counter. - // To enable, uncomment the following section - // - // if err := r.getNetworkMetrics(ctx, slot); err != nil { - // log.Error().Err(err).Msg("failed to get network resource consumption") - // } - if err := r.getVmMetrics(ctx, slot); err != nil { log.Error().Err(err).Msg("failed to get vm public ip consumption") } - if err := r.getGwMetrics(ctx, slot); err != nil { - log.Error().Err(err).Msg("failed to get gateway network consumption") - } - return nil } diff --git a/cmds/modules/qsfsd/main.go b/cmds/modules/qsfsd/main.go deleted file mode 100644 index 73598e6f1..000000000 --- a/cmds/modules/qsfsd/main.go +++ /dev/null @@ -1,84 +0,0 @@ -package qsfsd - -import ( - "context" - - "github.com/pkg/errors" - "github.com/threefoldtech/zos/pkg/qsfsd" - "github.com/threefoldtech/zos/pkg/utils" - "github.com/urfave/cli/v2" - - "github.com/rs/zerolog/log" - - "github.com/threefoldtech/zbus" -) - -const ( - module = "qsfsd" -) - -// Module is entry point for module -var Module cli.Command = cli.Command{ - Name: "qsfsd", - Usage: "manage qsfsd", - Flags: []cli.Flag{ - &cli.StringFlag{ - Name: "root", - Usage: "`ROOT` working directory of the module", - Value: "/var/cache/modules/qsfsd", - }, - &cli.StringFlag{ - Name: "broker", - Usage: "connection string to the message `BROKER`", - Value: "unix:///var/run/redis.sock", - }, - &cli.UintFlag{ - Name: "workers", - Usage: "number of workers `N`", - Value: 1, - }, - }, - Action: action, -} - -func action(cli *cli.Context) error { - var ( - moduleRoot string = cli.String("root") - msgBrokerCon string = cli.String("broker") - workerNr uint = cli.Uint("workers") - ) - - server, err := zbus.NewRedisServer(module, msgBrokerCon, workerNr) - if err != nil { - return errors.Wrap(err, "fail to connect to message broker server") - } - - client, err := zbus.NewRedisClient(msgBrokerCon) - if err != nil { - return errors.Wrap(err, "failed to connect to zbus broker") - } - - ctx, cancel := utils.WithSignal(cli.Context) - defer cancel() - - mod, err := qsfsd.New(ctx, client, moduleRoot) - if err != nil { - return errors.Wrap(err, "failed to construct qsfsd object") - } - - server.Register(zbus.ObjectID{Name: "manager", Version: "0.0.1"}, mod) - log.Info(). - Str("broker", msgBrokerCon). - Uint("worker nr", workerNr). - Msg("starting qsfsd module") - - utils.OnDone(ctx, func(_ error) { - log.Info().Msg("shutting down") - }) - - if err := server.Run(ctx); err != nil && err != context.Canceled { - return errors.Wrap(err, "unexpected error") - } - - return nil -} diff --git a/cmds/modules/zbusdebug/main.go b/cmds/modules/zbusdebug/main.go index 7a3f534de..8123e34bc 100644 --- a/cmds/modules/zbusdebug/main.go +++ b/cmds/modules/zbusdebug/main.go @@ -24,7 +24,7 @@ var ( "identityd": {}, "vmd": {}, "flist": {}, - "network": {}, + "netlight": {}, "container": {}, "provision": {}, "gateway": {}, diff --git a/cmds/modules/zui/net.go b/cmds/modules/zui/net.go index 3d0e0cc61..aad981b71 100644 --- a/cmds/modules/zui/net.go +++ b/cmds/modules/zui/net.go @@ -2,8 +2,6 @@ package zui import ( "context" - "fmt" - "net" "strings" ui "github.com/gizak/termui/v3" @@ -11,7 +9,7 @@ import ( _ "github.com/pkg/errors" "github.com/threefoldtech/zbus" "github.com/threefoldtech/zos/pkg" - "github.com/threefoldtech/zos/pkg/network/types" + "github.com/threefoldtech/zos/pkg/netlight/types" "github.com/threefoldtech/zos/pkg/stubs" ) @@ -22,33 +20,15 @@ func addressRender(ctx context.Context, table *widgets.Table, client zbus.Client table.Rows = [][]string{ {"ZOS", loading}, - {"DMZ", loading}, - {"YGG", loading}, - {"PUB", loading}, {"DUL", loading}, } - - stub := stubs.NewNetworkerStub(client) + table.Rows[1][1] = "single" // light has only single stack setup + stub := stubs.NewNetworkerLightStub(client) zos, err := stub.ZOSAddresses(ctx) if err != nil { return err } - dmz, err := stub.DMZAddresses(ctx) - if err != nil { - return err - } - - ygg, err := stub.YggAddresses(ctx) - if err != nil { - return err - } - - pub, err := stub.PublicAddresses(ctx) - if err != nil { - return err - } - toString := func(al pkg.NetlinkAddresses) string { var buf strings.Builder for _, a := range al { @@ -77,28 +57,8 @@ func addressRender(ctx context.Context, table *widgets.Table, client zbus.Client for { render.Signal() table.ColumnWidths = []int{6, table.Size().X - 9} - select { - case a := <-zos: - table.Rows[0][1] = toString(a) - case a := <-dmz: - table.Rows[1][1] = toString(a) - case a := <-ygg: - table.Rows[2][1] = toString(a) - case a := <-pub: - str := "no public config" - if a.HasPublicConfig { - str = toString([]net.IPNet{a.IPv4.IPNet, a.IPv6.IPNet}) - } - table.Rows[3][1] = str - } - - exit, err := stub.GetPublicExitDevice(ctx) - dual := exit.String() - if err != nil { - dual = fmt.Sprintf("error: %s", err) - } + table.Rows[0][1] = toString(<-zos) - table.Rows[4][1] = dual } }() diff --git a/cmds/modules/zui/service.go b/cmds/modules/zui/service.go index b4e356a22..0dc251a8b 100644 --- a/cmds/modules/zui/service.go +++ b/cmds/modules/zui/service.go @@ -132,7 +132,7 @@ func getRegistrarStatus(ctx context.Context, client zbus.Client) string { } func getNetworkStatus(ctx context.Context, client zbus.Client) bool { - network := stubs.NewNetworkerStub(client) + network := stubs.NewNetworkerLightStub(client) if err := network.Ready(ctx); err != nil { return false diff --git a/cmds/zos/main.go b/cmds/zos/main.go index 5cf77aee1..4c9be0f2f 100644 --- a/cmds/zos/main.go +++ b/cmds/zos/main.go @@ -10,12 +10,10 @@ import ( apigateway "github.com/threefoldtech/zos/cmds/modules/api_gateway" "github.com/threefoldtech/zos/cmds/modules/contd" "github.com/threefoldtech/zos/cmds/modules/flistd" - "github.com/threefoldtech/zos/cmds/modules/gateway" - "github.com/threefoldtech/zos/cmds/modules/networkd" + "github.com/threefoldtech/zos/cmds/modules/netlightd" "github.com/threefoldtech/zos/cmds/modules/noded" "github.com/threefoldtech/zos/cmds/modules/powerd" "github.com/threefoldtech/zos/cmds/modules/provisiond" - "github.com/threefoldtech/zos/cmds/modules/qsfsd" "github.com/threefoldtech/zos/cmds/modules/storaged" "github.com/threefoldtech/zos/cmds/modules/vmd" "github.com/threefoldtech/zos/cmds/modules/zbusdebug" @@ -51,11 +49,9 @@ func main() { &contd.Module, &vmd.Module, &noded.Module, - &networkd.Module, + &netlightd.Module, &provisiond.Module, &zbusdebug.Module, - &gateway.Module, - &qsfsd.Module, &powerd.Module, &apigateway.Module, }, diff --git a/docs/internals/network-light/png/full.png b/docs/internals/network-light/png/full.png new file mode 100644 index 000000000..f7f1a0590 Binary files /dev/null and b/docs/internals/network-light/png/full.png differ diff --git a/docs/internals/network-light/png/host.png b/docs/internals/network-light/png/host.png new file mode 100644 index 000000000..75a78f9bf Binary files /dev/null and b/docs/internals/network-light/png/host.png differ diff --git a/docs/internals/network-light/png/ndmz.png b/docs/internals/network-light/png/ndmz.png new file mode 100644 index 000000000..2bb19059b Binary files /dev/null and b/docs/internals/network-light/png/ndmz.png differ diff --git a/docs/internals/network-light/png/nr.png b/docs/internals/network-light/png/nr.png new file mode 100644 index 000000000..a28277374 Binary files /dev/null and b/docs/internals/network-light/png/nr.png differ diff --git a/docs/internals/network-light/readme.md b/docs/internals/network-light/readme.md new file mode 100644 index 000000000..dbc15ef45 --- /dev/null +++ b/docs/internals/network-light/readme.md @@ -0,0 +1,85 @@ + +# Network Light + +> WIP: This is still a draft proposal + +Network light is a new version of network daemon that simplifies zos setup on most of cloud providers so ZOS can be deployed and hosted on the cloud. This version of networking has a small set of requirements and features as follows: + +- VMs have (Always) two IPs. + - A local private IP. This only allows the VM to have access to the public internet via NATting. So far it's not relevant if that IP subnet and IP is assigned by the user or ZOS itself + - A mycelium IP. The mycelium IP subnet is chosen by the user during deployment for his entire *network space* on that node (by providing the seed) and then he can chose individual IPs per VM that is deployed in that space. +- Different *network space* has isolated networks. VMs from different spaces can't communicate over their private Ips, but only over their mycelium Ips. +- If the user wants his VMs across multiple node to communicate it's up to him to setup mycelium ips white lists or other means of authentication. +- There is NO public IP support. +- ZDBs only have mycelium IPs. +- No yggdrasil +- No public config + +## Network layout + +This is to layout the simplified version of networkd. This version makes sure that ZOS gets only a single IP (via the cloud provider) and then all traffic from that node is basically NATted over this IP. + +### Host namespace + +![host](png/host.png) + +- The physical (only) nic is directly attached to the ZOS bridge. +- There is a dhcp client instance running on zos bridge that gets an IP from the physica network. +- There is created an **ndmz** bridge that is set to UP and statically assigned an IP address `100.127.0.1` + - This bridge will be wired later to other namespaces +- The `br-my` bridge is created. + - This bridge will be wired later to `zdb` namespaces so they can get mycelium IPs. +- nft rule to drop connection to `100.172.0.1` + +The usage of `br-ndmz` and `br-my` bridges will be clear later + +> **SAFETY:** All incoming connections to ip `100.127.0.1` is dropped via nft rule. This will prevent VMs from being able to connect to the node directly. + +## Ndmz namespace + +![ndmz](png/ndmz.png) +The ndmz namespace is a much simplified version of the previous layout. +The key difference here is that ndmz does **NOT** have an IP from the physical LAN instead it is also natted over the same node IP. + +The reason we need it is we can safely run services that need to be accessed remotely over the node mycelium IP. Like the NODE openRPC api. But also prevent attackers from reaching to other services directly running inside the host + +The node mycelium instance will also be running inside `ndmz` this process will then create the `my0` interface and give it the seed associated ip address. For example let's assume `my0` gets ip `58a:1059:b9a9:bc3:a1e8:5909:8957:d956/7` + +According to mycelium docs the entire `/64` prefix is assigned also to that node. We then can do the following: + +- Assign the IP `58a:1059:b9a9:bc3::1/64` to nmy +- make sure forwarding is enabled for ipv6 +- set routing of `58a:1059:b9a9:bc3::/64` over dev `nmy` +- make sure that `200::/7` is routed over `my0` +- make sure default gw is set to `100.127.0.1` (the br-ndmz on the host) +- enable ipv4 forwarding, and set any needed masquerading rules as needed. `oifname "public" masquerade fully-random` + +Later, containers wired to `br-my` can then get IPs inside the range `58a:1059:b9a9:bc3::/64` and they then can send traffic (and receive traffic) over mycelium network + +> Note: Again the reason we still have ndmz is to run mycelium isolated from the host, and run API services isolated from rest of the system. It's still possible to simply create the same setup on the host name space directly. Jan? + +## NR (Network Resource) + +![nr](png/nr.png) + +The network resource is yet a simplified version of NR namesapce on original setup but here it looks and behaves more like `ndmz` in the sense that it only to isolate VMs from rest of the system. All public traffic is natter over `public` interface, which is wired to br-ndmz + +The `my0`, `nmy`, and `m-` work in exactly similar way as the ones in `ndmz`. The only difference is that those are user network specific. The user provides the seed and hence control what IP range, and Ips assigned to the VMs. All routing rules has to be done here. + +The `b-` is where all VMs will be connected. Each VM must get an IP range from a private range as follows: + +- the private range doesn't really matter, it can be fixed by ZOS forever (say range `10.0.0.0/8`) or allowed to be chosen by the user. + - I personally prefer to make it fixed to `10.0.0.0/8` range and allow the user to choose individual VMs ips in the private range. +- `private` interface becomes the `gw` hence it can get the first ip in the range as per convention. So `10.0.0.1` +- make sure routing is set as follows: + - default gw is `100.127.0.1` (the br-ndmz on the host) +- enable ipv4 forwarding, and set any needed masquerading rules as needed. `oifname "public" masquerade fully-random` + +What happens now is: + +- VMs inside a single space can communicate directly over their bridge. +- Different networks resource can (and well) have conflicting IP and ranges but with no issue since each network is completely isolated from the other ones. + +### Full Picture + +![full](png/full.png) diff --git a/docs/internals/network-light/uml/full.wsd b/docs/internals/network-light/uml/full.wsd new file mode 100644 index 000000000..89b310fc7 --- /dev/null +++ b/docs/internals/network-light/uml/full.wsd @@ -0,0 +1,71 @@ +@startuml + + +component "Host" { + component NIC as nic + () ZOS as zos + note left of zos + ZOS bridge, get the Ip address + assigned by physical network (dhcp) + end note + + nic --> zos: attached + + () "br-ndmz" as brndmz + note right of brndmz + ndmz bridge will always have 100.127.0.1 + end note + zos .right. brndmz: Natted + + () "br-my" as brmy + () "b-" as bnr + () "m-" as bmy +} + +component "ndmz" { + component public as pub + pub -up->brndmz: macvlan + note bottom of pub + always assigned static IP 100.127.0.2 + end note + + component my0 + note bottom of my0 + created by mycelium process + end note + + + component nmy + + nmy -up-> brmy: macvlan +} + + +component "n-" { + component public as npub + npub -up->brndmz: macvlan + note bottom of npub + gets assigned static IP by ZOS in the range 100.127.0.0/16 + end note + + component my0 as nmy0 + note bottom of nmy0 + created by mycelium process + end note + + + component nmy as nnmy + nnmy -up-> bmy: macvlan + + component private + private -up->bnr: macvlan +} + +component "VM" + +VM --> bnr +VM --> bmy +@enduml + + +@enduml diff --git a/docs/internals/network-light/uml/host.wsd b/docs/internals/network-light/uml/host.wsd new file mode 100644 index 000000000..367df89d9 --- /dev/null +++ b/docs/internals/network-light/uml/host.wsd @@ -0,0 +1,23 @@ +@startuml + + +component "Host" { + component NIC as nic + () ZOS as zos + note left of zos + ZOS bridge, get the Ip address + assigned by physical network (dhcp) + end note + + nic --> zos: attached + + () "br-ndmz" as brndmz + note right of brndmz + ndmz bridge will always have 100.127.0.1 + end note + zos .right. brndmz: Natted + + () "br-my" as brmy +} + +@enduml diff --git a/docs/internals/network-light/uml/ndmz.wsd b/docs/internals/network-light/uml/ndmz.wsd new file mode 100644 index 000000000..56c20cbab --- /dev/null +++ b/docs/internals/network-light/uml/ndmz.wsd @@ -0,0 +1,24 @@ +@startuml + +() "br-ndmz" as brndmz +() "br-my" as brmy + +component "ndmz" { + component public as pub + pub -up->brndmz: macvlan + note bottom of pub + always assigned static IP 100.127.0.2 + end note + + component my0 + note bottom of my0 + created by mycelium process + end note + + + component nmy + + nmy -up-> brmy: macvlan +} + +@enduml diff --git a/docs/internals/network-light/uml/nr.wsd b/docs/internals/network-light/uml/nr.wsd new file mode 100644 index 000000000..ab566e491 --- /dev/null +++ b/docs/internals/network-light/uml/nr.wsd @@ -0,0 +1,30 @@ +@startuml + +() "br-ndmz" as brndmz +() "m-" as my +() "b-" as bnr + +component "n-" { + component public as pub + pub -up->brndmz: macvlan + note bottom of pub + gets assigned static IP by ZOS in the range 100.127.0.0/16 + end note + + component my0 + note bottom of my0 + created by mycelium process + end note + + + component nmy + nmy -up-> my: macvlan + + component private + private -up->bnr: macvlan +} + +component "VM" + +VM -right-> bnr +@enduml diff --git a/etc/zinit/gateway.yaml b/etc/zinit/gateway.yaml deleted file mode 100644 index c8f6be0e8..000000000 --- a/etc/zinit/gateway.yaml +++ /dev/null @@ -1,4 +0,0 @@ -exec: gateway --broker unix:///var/run/redis.sock --root /var/cache/modules/gateway -after: - - boot - - networkd diff --git a/etc/zinit/networkd.yaml b/etc/zinit/networkd.yaml index f28fc2b25..ba9abeaa1 100644 --- a/etc/zinit/networkd.yaml +++ b/etc/zinit/networkd.yaml @@ -1,4 +1,5 @@ -exec: networkd --broker unix:///var/run/redis.sock --root /var/cache/modules/networkd -test: zbusdebug --module network +exec: netlightd --broker unix:///var/run/redis.sock --root /var/cache/modules/networkd + +test: zbusdebug --module netlight after: - boot diff --git a/etc/zinit/qsfsd.yaml b/etc/zinit/qsfsd.yaml deleted file mode 100644 index ffda01320..000000000 --- a/etc/zinit/qsfsd.yaml +++ /dev/null @@ -1,4 +0,0 @@ -exec: qsfsd --broker unix:///var/run/redis.sock --root /var/cache/modules/qsfsd -after: - - boot - - contd \ No newline at end of file diff --git a/go.mod b/go.mod index b9a2910c7..8f35f5d66 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,6 @@ require ( github.com/BurntSushi/toml v1.1.0 github.com/ChainSafe/go-schnorrkel v1.1.0 github.com/agl/ed25519 v0.0.0-20170116200512-5312a6153412 - github.com/alexflint/go-filemutex v1.1.0 github.com/blang/semver v3.5.1+incompatible github.com/boltdb/bolt v1.3.1 github.com/cenkalti/backoff v2.2.1+incompatible @@ -20,7 +19,6 @@ require ( github.com/containernetworking/cni v0.8.1 github.com/containernetworking/plugins v0.9.1 github.com/dave/jennifer v1.3.0 - github.com/deckarep/golang-set v1.8.0 github.com/decred/base58 v1.0.5 github.com/diskfs/go-diskfs v1.2.0 github.com/g0rbe/go-chattr v0.0.0-20190906133247-aa435a6a0a37 @@ -39,6 +37,7 @@ require ( github.com/joncrlsn/dque v0.0.0-20200702023911-3e80e3146ce5 github.com/lestrrat-go/jwx v1.1.7 github.com/machinebox/graphql v0.2.2 + github.com/oasisprotocol/curve25519-voi v0.0.0-20230904125328-1f23a7beb09a github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417 github.com/patrickmn/go-cache v2.1.0+incompatible github.com/pkg/errors v0.9.1 @@ -54,17 +53,15 @@ require ( github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852 github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f github.com/whs/nacl-sealed-box v0.0.0-20180930164530-92b9ba845d8d - github.com/yggdrasil-network/yggdrasil-go v0.4.0 golang.org/x/crypto v0.23.0 golang.org/x/sys v0.20.0 - golang.zx2c4.com/wireguard/wgctrl v0.0.0-20200609130330-bd2cb7843e1b gopkg.in/yaml.v2 v2.4.0 - gotest.tools v2.2.0+incompatible ) require ( github.com/Microsoft/go-winio v0.5.2 // indirect github.com/Microsoft/hcsshim v0.8.25 // indirect + github.com/alexflint/go-filemutex v1.1.0 // indirect github.com/containerd/continuity v0.3.0 // indirect github.com/containerd/fifo v1.0.0 // indirect github.com/containerd/ttrpc v1.1.0 // indirect @@ -72,6 +69,7 @@ require ( github.com/cosmos/go-bip39 v1.0.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/deckarep/golang-set v1.8.0 // indirect github.com/decred/dcrd/crypto/blake256 v1.0.1 // indirect github.com/decred/dcrd/dcrec/secp256k1/v3 v3.0.0 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect @@ -88,7 +86,6 @@ require ( github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb // indirect - github.com/google/go-cmp v0.6.0 // indirect github.com/gorilla/websocket v1.5.1 // indirect github.com/gtank/ristretto255 v0.1.2 // indirect github.com/hanwen/go-fuse/v2 v2.3.0 // indirect @@ -96,7 +93,6 @@ require ( github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/holiman/uint256 v1.2.3 // indirect - github.com/josharian/native v0.0.0-20200817173448-b6b71def0850 // indirect github.com/klauspost/compress v1.16.7 // indirect github.com/lestrrat-go/backoff/v2 v2.0.7 // indirect github.com/lestrrat-go/blackmagic v1.0.0 // indirect @@ -107,15 +103,12 @@ require ( github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-runewidth v0.0.13 // indirect - github.com/mdlayher/genetlink v1.0.0 // indirect - github.com/mdlayher/netlink v1.4.0 // indirect github.com/mimoo/StrobeGo v0.0.0-20220103164710-9a04d6ca976b // indirect github.com/mitchellh/go-wordwrap v1.0.1 // indirect github.com/moby/locker v1.0.1 // indirect github.com/moby/sys/mountinfo v0.6.2 // indirect github.com/ncruces/go-strftime v0.1.9 // indirect github.com/nsf/termbox-go v1.1.1 // indirect - github.com/oasisprotocol/curve25519-voi v0.0.0-20230904125328-1f23a7beb09a // indirect github.com/onsi/ginkgo v1.16.4 // indirect github.com/onsi/gomega v1.16.0 // indirect github.com/op/go-logging v0.0.0-20160315200505-970db520ece7 // indirect @@ -149,7 +142,6 @@ require ( golang.org/x/net v0.22.0 // indirect golang.org/x/sync v0.3.0 // indirect golang.org/x/text v0.15.0 // indirect - golang.zx2c4.com/wireguard v0.0.20200320 // indirect gonum.org/v1/gonum v0.15.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20210917145530-b395a37504d4 // indirect diff --git a/go.sum b/go.sum index 2d5e7ab3c..49c2960f7 100644 --- a/go.sum +++ b/go.sum @@ -2,8 +2,6 @@ bazil.org/fuse v0.0.0-20160811212531-371fbbdaa898/go.mod h1:Xbm+BRKSBEpa4q4hTSxohYNQpsxXPbPry4JJWOB3LB8= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -github.com/Arceliar/ironwood v0.0.0-20210619124114-6ad55cae5031/go.mod h1:RP72rucOFm5udrnEzTmIWLRVGQiV/fSUAQXJ0RST/nk= -github.com/Arceliar/phony v0.0.0-20210209235338-dde1a8dca979/go.mod h1:6Lkn+/zJilRMsKmbmG1RPoamiArC6HS73xbwRyp3UyI= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v1.1.0 h1:ksErzDEI1khOiGPgpwuI7x2ebx/uXQNw7xJpn9Eq1+I= github.com/BurntSushi/toml v1.1.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= @@ -17,8 +15,6 @@ github.com/Microsoft/hcsshim v0.8.6/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXn github.com/Microsoft/hcsshim v0.8.25 h1:fRMwXiwk3qDwc0P05eHnh+y2v07JdtsfQ1fuAc69m9g= github.com/Microsoft/hcsshim v0.8.25/go.mod h1:4zegtUJth7lAvFyc6cH2gGQ5B3OFQim01nnU2M8jKDg= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA= -github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= github.com/agl/ed25519 v0.0.0-20170116200512-5312a6153412 h1:w1UutsfOrms1J05zt7ISrnJIXKzwaspym5BTKGx93EI= github.com/agl/ed25519 v0.0.0-20170116200512-5312a6153412/go.mod h1:WPjqKcmVOxf0XSf3YxCJs6N6AOSrOx3obionmG7T0y0= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= @@ -51,7 +47,6 @@ github.com/centrifuge/go-substrate-rpc-client/v4 v4.0.12/go.mod h1:5g1oM4Zu3BOaL github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/checkpoint-restore/go-criu/v5 v5.3.0/go.mod h1:E/eQpaFtUKGOOSEBZgmKAcn+zUUwWxqcaKZlF54wK8E= -github.com/cheggaaa/pb/v3 v3.0.8/go.mod h1:UICbiLec/XO6Hw6k+BHEtHeQFzzBH4i2/qk/ow1EJTA= github.com/cilium/ebpf v0.4.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= github.com/cilium/ebpf v0.7.0/go.mod h1:/oI2+1shJiTGAMgl6/RgJr36Eo1jzrRcAWbcXO2usCA= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= @@ -145,8 +140,6 @@ github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go. github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/ethereum/go-ethereum v1.11.6 h1:2VF8Mf7XiSUfmoNOy3D+ocfl9Qu8baQBrCNbo2CXQ8E= github.com/ethereum/go-ethereum v1.11.6/go.mod h1:+a8pUj1tOyJ2RinsNQD4326YS+leSoKGiG/uVVb0x6Y= -github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= -github.com/fatih/color v1.12.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= github.com/frankban/quicktest v1.11.3 h1:8sXhOn0uLys67V8EsXLc6eszDs8VXWxL3iRvebPhedY= github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= @@ -234,7 +227,6 @@ github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb h1:PBC98N2aIaM3XXiurYmW7fx4GZkL8feAMVq7nEjURHk= github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/gologme/log v1.2.0/go.mod h1:gq31gQ8wEHkR+WekdWsqDuf8pXTUZA9BnnzTuPz1Y9U= github.com/gomodule/redigo v1.8.9/go.mod h1:7ArFNvsTjH8GMMzB4uy1snslv2BwmginuMs06a1uzZE= github.com/gomodule/redigo v2.0.0+incompatible h1:K/R+8tc58AaqLkqG2Ol3Qk+DR/TlNuhuh457pBFPtt0= github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= @@ -244,7 +236,6 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= @@ -289,7 +280,6 @@ github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= -github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d h1:dg1dEPuWpEqDnvIw251EVy4zlP8gWbsGj4BsUKCRpYs= github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= @@ -297,7 +287,6 @@ github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyf github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hasura/go-graphql-client v0.10.0 h1:eQm/ap/rqxMG6yAGe6J+FkXu1VqJ9p21E63vz0A7zLQ= github.com/hasura/go-graphql-client v0.10.0/go.mod h1:z9UPkMmCBMuJjvBEtdE6F+oTR2r15AcjirVNq/8P+Ig= -github.com/hjson/hjson-go v3.1.0+incompatible/go.mod h1:qsetwF8NlsTsOTwZTApNlTCerV+b2GjYRRcIk4JMFio= github.com/holiman/uint256 v1.2.3 h1:K8UWO1HUJpRMXBxbmaY1Y8IAMZC/RsKB+ArEnnK4l5o= github.com/holiman/uint256 v1.2.3/go.mod h1:SC8Ryt4n+UBbPbIBKaG9zbbDlp4jOru9xFZmPzLUTxw= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= @@ -309,21 +298,10 @@ github.com/jgautheron/goconst v0.0.0-20170703170152-9740945f5dcb/go.mod h1:82Txj github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/joncrlsn/dque v0.0.0-20200702023911-3e80e3146ce5 h1:bo1aoO6l128nKJCBrFflOj9s+KPqMM7ErNyB5GGBNDs= github.com/joncrlsn/dque v0.0.0-20200702023911-3e80e3146ce5/go.mod h1:dNKs71rs2VJGBAmttu7fouEsRQlRjxy0p1Sx+T5wbpY= -github.com/josharian/native v0.0.0-20200817173448-b6b71def0850 h1:uhL5Gw7BINiiPAo24A2sxkcDI0Jt/sqp1v5xQCniEFA= -github.com/josharian/native v0.0.0-20200817173448-b6b71def0850/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w= -github.com/jsimonetti/rtnetlink v0.0.0-20190606172950-9527aa82566a/go.mod h1:Oz+70psSo5OFh8DBl0Zv2ACw7Esh6pPUphlvZG9x7uw= -github.com/jsimonetti/rtnetlink v0.0.0-20200117123717-f846d4f6c1f4/go.mod h1:WGuG/smIU4J/54PblvSbh+xvCZmpJnFgr3ds6Z55XMQ= -github.com/jsimonetti/rtnetlink v0.0.0-20201009170750-9c6f07d100c1/go.mod h1:hqoO/u39cqLeBLebZ8fWdE96O7FxrAsRYhnVOdgHxok= -github.com/jsimonetti/rtnetlink v0.0.0-20201216134343-bde56ed16391/go.mod h1:cR77jAZG3Y3bsb8hF6fHJbFoyFukLFOkQ98S0pQz3xw= -github.com/jsimonetti/rtnetlink v0.0.0-20201220180245-69540ac93943/go.mod h1:z4c53zj6Eex712ROyh8WI0ihysb5j2ROyV42iNogmAs= -github.com/jsimonetti/rtnetlink v0.0.0-20210122163228-8d122574c736/go.mod h1:ZXpIyOK59ZnN7J0BV99cZUPmsqDRZ3eq5X+st7u/oSA= -github.com/jsimonetti/rtnetlink v0.0.0-20210212075122-66c871082f2b h1:c3NTyLNozICy8B4mlMXemD3z/gXgQzVXZS/HqT+i3do= -github.com/jsimonetti/rtnetlink v0.0.0-20210212075122-66c871082f2b/go.mod h1:8w9Rh8m+aHZIG69YPGGem1i5VzoyRC8nw2kA8B+ik5U= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= -github.com/kardianos/minwinsvc v1.0.0/go.mod h1:Bgd0oc+D0Qo3bBytmNtyRKVlp85dAloLKhfxanPFFRc= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= @@ -361,48 +339,27 @@ github.com/lestrrat-go/option v1.0.0 h1:WqAWL8kh8VcSoD6xjSH34/1m8yxluXQbDeKNfvFe github.com/lestrrat-go/option v1.0.0/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I= github.com/lestrrat-go/pdebug/v3 v3.0.1 h1:3G5sX/aw/TbMTtVc9U7IHBWRZtMvwvBziF1e4HoQtv8= github.com/lestrrat-go/pdebug/v3 v3.0.1/go.mod h1:za+m+Ve24yCxTEhR59N7UlnJomWwCiIqbJRmKeiADU4= -github.com/lxn/walk v0.0.0-20210112085537-c389da54e794/go.mod h1:E23UucZGqpuUANJooIbHWCufXvOcT6E7Stq81gU+CSQ= -github.com/lxn/win v0.0.0-20210218163916-a377121e959e/go.mod h1:KxxjdtRkfNoYDCUP5ryK7XJJNTnpC8atvtmTheChOtk= github.com/machinebox/graphql v0.2.2 h1:dWKpJligYKhYKO5A2gvNhkJdQMNZeChZYyBbrZkBZfo= github.com/machinebox/graphql v0.2.2/go.mod h1:F+kbVMHuwrQ5tYgU9JXlnskM8nOaFxCAEolaQybkjWA= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/matryer/is v1.4.0 h1:sosSmIWwkYITGrxZ25ULNDeKiMNzFSr4V/eqBQP0PeE= github.com/matryer/is v1.4.0/go.mod h1:8I/i5uYgLzgsgEloJE1U6xx5HkBQpAZvepWuujKwMRU= -github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= -github.com/mattn/go-isatty v0.0.13/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= -github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU= github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-shellwords v1.0.3/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o= github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/mdlayher/ethtool v0.0.0-20210210192532-2b88debcdd43 h1:WgyLFv10Ov49JAQI/ZLUkCZ7VJS3r74hwFIGXJsgZlY= -github.com/mdlayher/ethtool v0.0.0-20210210192532-2b88debcdd43/go.mod h1:+t7E0lkKfbBsebllff1xdTmyJt8lH37niI6kwFk9OTo= -github.com/mdlayher/genetlink v1.0.0 h1:OoHN1OdyEIkScEmRgxLEe2M9U8ClMytqA5niynLtfj0= -github.com/mdlayher/genetlink v1.0.0/go.mod h1:0rJ0h4itni50A86M2kHcgS85ttZazNt7a8H2a2cw0Gc= -github.com/mdlayher/netlink v0.0.0-20190409211403-11939a169225/go.mod h1:eQB3mZE4aiYnlUsyGGCOpPETfdQq4Jhsgf1fk3cwQaA= -github.com/mdlayher/netlink v1.0.0/go.mod h1:KxeJAFOFLG6AjpyDkQ/iIhxygIUKD+vcwqcnu43w/+M= -github.com/mdlayher/netlink v1.1.0/go.mod h1:H4WCitaheIsdF9yOYu8CFmCgQthAPIWZmcKp9uZHgmY= -github.com/mdlayher/netlink v1.1.1/go.mod h1:WTYpFb/WTvlRJAyKhZL5/uy69TDDpHHu2VZmb2XgV7o= -github.com/mdlayher/netlink v1.2.0/go.mod h1:kwVW1io0AZy9A1E2YYgaD4Cj+C+GPkU6klXCMzIJ9p8= -github.com/mdlayher/netlink v1.2.1/go.mod h1:bacnNlfhqHqqLo4WsYeXSqfyXkInQ9JneWI68v1KwSU= -github.com/mdlayher/netlink v1.2.2-0.20210123213345-5cc92139ae3e/go.mod h1:bacnNlfhqHqqLo4WsYeXSqfyXkInQ9JneWI68v1KwSU= -github.com/mdlayher/netlink v1.3.0/go.mod h1:xK/BssKuwcRXHrtN04UBkwQ6dY9VviGGuriDdoPSWys= -github.com/mdlayher/netlink v1.4.0 h1:n3ARR+Fm0dDv37dj5wSWZXDKcy+U0zwcXS3zKMnSiT0= -github.com/mdlayher/netlink v1.4.0/go.mod h1:dRJi5IABcZpBD2A3D0Mv/AiX8I9uDEu5oGkAVrekmf8= github.com/mibk/dupl v1.0.0/go.mod h1:pCr4pNxxIbFGvtyCOi0c7LVjmV6duhKWV+ex5vh38ME= -github.com/mikioh/ipaddr v0.0.0-20190404000644-d465c8ab6721 h1:RlZweED6sbSArvlE924+mUcZuXKLBHA35U7LN621Bws= -github.com/mikioh/ipaddr v0.0.0-20190404000644-d465c8ab6721/go.mod h1:Ickgr2WtCLZ2MDGd4Gr0geeCH5HybhRJbonOgQpvSxc= github.com/mimoo/StrobeGo v0.0.0-20181016162300-f8f6d4d2b643/go.mod h1:43+3pMjjKimDBf5Kr4ZFNGbLql1zKkbImw+fZbw3geM= github.com/mimoo/StrobeGo v0.0.0-20220103164710-9a04d6ca976b h1:QrHweqAtyJ9EwCaGHBu1fghwxIPiopAHV06JlXrMHjk= github.com/mimoo/StrobeGo v0.0.0-20220103164710-9a04d6ca976b/go.mod h1:xxLb2ip6sSUts3g1irPVHyk/DGslwQsNOo9I7smJfNU= @@ -411,7 +368,6 @@ github.com/mitchellh/go-wordwrap v0.0.0-20150314170334-ad45545899c7/go.mod h1:ZX github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0= github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTSsCt+hzestvNj0= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/moby/locker v1.0.1 h1:fOXqR41zeveg4fFODix+1Ch4mj/gT0NE1XJbp/epuBg= github.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc= github.com/moby/sys/mountinfo v0.5.0/go.mod h1:3bMD3Rg+zkqx8MRYPi7Pyb0Ie97QEBmdxbhnCLlSvSU= @@ -493,7 +449,6 @@ github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= -github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= @@ -605,8 +560,6 @@ github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRT github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8= github.com/xxtea/xxtea-go v0.0.0-20170828040851-35c4b17eecf6 h1:S+0oS/OPAe0kdSpQ7GAnCmpcDL7Jh2iJMjZTV6mYbPo= github.com/xxtea/xxtea-go v0.0.0-20170828040851-35c4b17eecf6/go.mod h1:2uvuCBt0VXxijrX5ieiAeeNT2+2MIsrs1DI9iXz7OOQ= -github.com/yggdrasil-network/yggdrasil-go v0.4.0 h1:H2CS2pTjCTzNQMHFU7sEW3Ge59fQWQbUeh6fVUO1Gi0= -github.com/yggdrasil-network/yggdrasil-go v0.4.0/go.mod h1:/iMJjOrXRsjlFgqhWOPhecOKi7xHmHiY4/En3A42Fog= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= @@ -627,16 +580,10 @@ go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20191002192127-34f69633bfdc/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200204104054-c9f3fb736b72/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201217014255-9d1352758620/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= -golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= -golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= -golang.org/x/crypto v0.0.0-20210506145944-38f3c27a63bf/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= -golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -668,25 +615,15 @@ golang.org/x/net v0.0.0-20190514140710-3ec191127204/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191003171128-d98b1b443823/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191007182048-72f939374954/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201010224723-4f7140c49acb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201216054612-986b41b23924/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= -golang.org/x/net v0.0.0-20210510120150-4163338589ed/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210610132358-84b48f89b13b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc= golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -710,50 +647,31 @@ golang.org/x/sys v0.0.0-20181021155630-eda9bb28ed51/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190411185658-b44545bcd369/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190804053845-51ab0e2deafa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191003212358-c178f38b412c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191115151921-52ab43148777/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200728102440-3e129f6d46b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200916030750-2334cc1a136f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201009025420-dfb3f7c4e634/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201018230417-eeed37f84f13/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201117170446-d9b008d0a637/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201118182958-a01c418693c7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201218084310-7d0127a74742/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210110051926-789bb1bd4061/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210123111255-9b0068b26619/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210216163648-f7da38b97c65/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210309040221-94ec62e08169/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210611083646-a4fc73990273/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211116061358-0a5406a5449c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -771,7 +689,6 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7-0.20210503195748-5c7c50ebbd4f/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -801,14 +718,6 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.zx2c4.com/wireguard v0.0.0-20210510202332-9844c74f67ec/go.mod h1:a057zjmoc00UN7gVkaJt2sXVK523kMJcogDTEvPIasg= -golang.zx2c4.com/wireguard v0.0.0-20210604143328-f9b48a961cd2/go.mod h1:laHzsbfMhGSobUmruXWAyMKKHSqvIcrqZJMyHD+/3O8= -golang.zx2c4.com/wireguard v0.0.20200121/go.mod h1:P2HsVp8SKwZEufsnezXZA4GRX/T49/HlU7DGuelXsU4= -golang.zx2c4.com/wireguard v0.0.20200320 h1:1vE6zVeO7fix9cJX1Z9ZQ+ikPIIx7vIyU0o0tLDD88g= -golang.zx2c4.com/wireguard v0.0.20200320/go.mod h1:lDian4Sw4poJ04SgHh35nzMVwGSYlPumkdnHcucAQoY= -golang.zx2c4.com/wireguard/wgctrl v0.0.0-20200609130330-bd2cb7843e1b h1:l4mBVCYinjzZuR5DtxHuBD6wyd4348TGiavJ5vLrhEc= -golang.zx2c4.com/wireguard/wgctrl v0.0.0-20200609130330-bd2cb7843e1b/go.mod h1:UdS9frhv65KTfwxME1xE8+rHYoFpbm36gOud1GhBe9c= -golang.zx2c4.com/wireguard/windows v0.3.14/go.mod h1:3P4IEAsb+BjlKZmpUXgy74c0iX9AVwwr3WcVJ8nPgME= gonum.org/v1/gonum v0.15.0 h1:2lYxjRbTYyxkJxlhC+LvJIx3SsANPdRybu1tGj9/OrQ= gonum.org/v1/gonum v0.15.0/go.mod h1:xzZVBJBtS+Mz4q0Yl2LJTk+OxOg4jiXZ7qBoM0uISGo= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= @@ -880,8 +789,6 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= -gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0= gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/pkg/environment/environment.go b/pkg/environment/environment.go index 6888565aa..7dcad9488 100644 --- a/pkg/environment/environment.go +++ b/pkg/environment/environment.go @@ -39,9 +39,8 @@ type Environment struct { FlistURL string BinRepo string - FarmID pkg.FarmID - Orphan bool - + FarmID pkg.FarmID + Orphan bool FarmSecret string SubstrateURL []string // IMPORTANT NOTICE: @@ -321,6 +320,5 @@ func getEnvironmentFromParams(params kernel.Params) (Environment, error) { if e := os.Getenv("ZOS_BIN_REPO"); e != "" { env.BinRepo = e } - return env, nil } diff --git a/pkg/flist/flist.go b/pkg/flist/flist.go index f99b10ece..0fb5ef77d 100644 --- a/pkg/flist/flist.go +++ b/pkg/flist/flist.go @@ -23,7 +23,7 @@ import ( "github.com/threefoldtech/zos/pkg" "github.com/threefoldtech/zos/pkg/environment" "github.com/threefoldtech/zos/pkg/gridtypes" - "github.com/threefoldtech/zos/pkg/network/namespace" + "github.com/threefoldtech/zos/pkg/netlight/namespace" "github.com/threefoldtech/zos/pkg/stubs" ) diff --git a/pkg/gateway/flist.go b/pkg/gateway/flist.go deleted file mode 100644 index 88e04317c..000000000 --- a/pkg/gateway/flist.go +++ /dev/null @@ -1,34 +0,0 @@ -package gateway - -import ( - "context" - "fmt" - "path/filepath" - - "github.com/pkg/errors" - "github.com/threefoldtech/zbus" - "github.com/threefoldtech/zos/pkg" - "github.com/threefoldtech/zos/pkg/stubs" -) - -const ( - flist = "https://hub.grid.tf/tf-autobuilder/traefik-2.9.9.flist" -) - -// ensureTraefikBin makes sure traefik flist is mounted. -// TODO: we need to "update" traefik and restart the service -// if new version is available! -func ensureTraefikBin(ctx context.Context, cl zbus.Client) (string, error) { - const bin = "traefik" - flistd := stubs.NewFlisterStub(cl) - hash, err := flistd.FlistHash(ctx, flist) - if err != nil { - return "", errors.Wrap(err, "failed to get traefik flist hash") - } - mnt, err := flistd.Mount(ctx, fmt.Sprintf("%s:%s", bin, hash), flist, pkg.ReadOnlyMountOptions) - if err != nil { - return "", errors.Wrap(err, "failed to mount traefik flist") - } - - return filepath.Join(mnt, bin), nil -} diff --git a/pkg/gateway/gateway.go b/pkg/gateway/gateway.go deleted file mode 100644 index 33136241b..000000000 --- a/pkg/gateway/gateway.go +++ /dev/null @@ -1,752 +0,0 @@ -package gateway - -import ( - "context" - "fmt" - "io" - "net" - "os" - "path/filepath" - "regexp" - "strings" - "sync" - "time" - - "github.com/pkg/errors" - "github.com/rs/zerolog/log" - "github.com/threefoldtech/zbus" - "github.com/threefoldtech/zos/pkg" - "github.com/threefoldtech/zos/pkg/cache" - "github.com/threefoldtech/zos/pkg/gridtypes" - "github.com/threefoldtech/zos/pkg/gridtypes/zos" - "github.com/threefoldtech/zos/pkg/stubs" - "github.com/threefoldtech/zos/pkg/zinit" - "gopkg.in/yaml.v2" -) - -var ( - domainRe = regexp.MustCompile("^Host(?:SNI)?\\(`([^`]+)`\\)$") - traefikBinRegex = regexp.MustCompile("/var/cache/modules/flistd/mountpoint/([a-z0-9:]+)/traefik") -) - -const ( - traefikService = "traefik" - // letsEncryptEmail email need to customizable by he farmer. - letsEncryptEmail = "letsencrypt@threefold.tech" - // certResolver must match the one defined in static config - httpCertResolver = "resolver" - dnsCertResolver = "dnsresolver" - validationPeriod = 1 * time.Hour - - configDir = "proxy" - metaDir = "traefik" - zinitDir = "zinit" -) - -var ( - ErrTwinIDMismatch = fmt.Errorf("twin id mismatch") - ErrContractNotReserved = fmt.Errorf("a name contract with the given name must be reserved first") - ErrInvalidContractState = fmt.Errorf("the name contract must be in Created state") - - _ pkg.Gateway = (*gatewayModule)(nil) -) - -type gatewayModule struct { - volatile string - cl zbus.Client - resolver *net.Resolver - substrateGateway *stubs.SubstrateGatewayStub - // maps domain to workload id - reservedDomains map[string]string - domainLock sync.RWMutex - - staticConfigPath string - binPath string - certScriptPath string -} - -type ProxyConfig struct { - Http *HTTPConfig `yaml:"http,omitempty"` - TCP *HTTPConfig `yaml:"tcp,omitempty"` -} - -type HTTPConfig struct { - Routers map[string]Router - Services map[string]Service -} - -type Router struct { - Rule string - Service string - Tls *TlsConfig `yaml:"tls,omitempty"` -} -type TlsConfig struct { - CertResolver string `yaml:"certResolver,omitempty"` - Domains []Domain `yaml:"domains,omitempty"` - Passthrough string `yaml:"passthrough,omitempty"` -} - -type Domain struct { - Sans []string `yaml:"sans,omitempty"` - Main string `yaml:"main,omitempty"` -} - -type Service struct { - LoadBalancer LoadBalancer -} - -type LoadBalancer struct { - Servers []Server -} - -type Server struct { - Url string `yaml:"url,omitempty"` - Address string `yaml:"address,omitempty"` -} - -// domainFromRule gets domain from rules in the form Host(`domain`) or HostSNI(`domain`) -func domainFromRule(rule string) (string, error) { - m := domainRe.FindStringSubmatch(rule) - if len(m) == 2 { - return m[1], nil - } - // no match - return "", fmt.Errorf("failed to extract domain from routing rule '%s'", rule) -} - -// domainFromConfig returns workloadID, domain, error -func domainFromConfig(path string) (string, string, error) { - buf, err := os.ReadFile(path) - if err != nil { - return "", "", errors.Wrap(err, "failed to read file") - } - - var c ProxyConfig - err = yaml.Unmarshal(buf, &c) - if err != nil { - return "", "", errors.Wrap(err, "failed to unmarshal yaml file") - } - var routers map[string]Router - if c.TCP != nil { - routers = c.TCP.Routers - } else if c.Http != nil { - routers = c.Http.Routers - } else { - return "", "", fmt.Errorf("yaml file doesn't contain valid http or tcp config %s", path) - } - if len(routers) > 1 { - return "", "", fmt.Errorf("only one router expected, found more: %s", path) - } - - for _, router := range routers { - domain, err := domainFromRule(router.Rule) - return router.Service, domain, err - } - - return "", "", fmt.Errorf("no routes defined in: %s", path) -} - -func loadDomains(ctx context.Context, dir string) (map[string]string, error) { - domains := make(map[string]string) - entries, err := os.ReadDir(dir) - if err != nil { - return nil, errors.Wrap(err, "failed to read dir") - } - - for _, entry := range entries { - if !entry.IsDir() && strings.HasSuffix(entry.Name(), ".yaml") { - path := filepath.Join(dir, entry.Name()) - wlID, domain, err := domainFromConfig(path) - if err != nil { - log.Warn().Err(err).Str("path", path).Msg("failed to load domain from config file") - continue - } - domains[domain] = wlID - } - } - return domains, nil -} - -// migration moves config that was on persisted storage -// to volatile storage. -// gateway should NOT persist config because on node reboot -// the gw config files are re-created anyway. -func migrateFiles(root, volatile string) error { - // need to move any files from root/proxy to volatile/proxy - source := filepath.Join(root, configDir) - dest := filepath.Join(volatile, configDir) - entries, err := os.ReadDir(source) - if os.IsNotExist(err) { - return nil - } else if err != nil { - return err - } - - move := func(source, dest string) error { - src, err := os.Open(source) - if err != nil { - return err - } - defer src.Close() - dst, err := os.Create(dest) - if err != nil { - return err - } - defer dst.Close() - _, err = io.Copy(dst, src) - return err - } - - for _, entry := range entries { - if entry.IsDir() { - continue - } - - old := filepath.Join(source, entry.Name()) - new := filepath.Join(dest, entry.Name()) - if err := move(old, new); err != nil { - log.Error().Err(err).Str("name", entry.Name()).Msg("failed to migrate gw config") - continue - } - if err := os.Remove(old); err != nil { - log.Error().Err(err).Str("name", entry.Name()).Msg("failed to remove old gw config") - } - } - - return nil -} - -func New(ctx context.Context, cl zbus.Client, root string) (pkg.Gateway, error) { - // where should service-restart/node-reboot recovery be handled? - volatile, err := cache.VolatileDir("gateway", 10*cache.Megabyte) - if err != nil && !os.IsExist(err) { - return nil, fmt.Errorf("failed to create gateway cache directory: %w", err) - } - - // create persisted directories - for _, dir := range []string{metaDir} { - dir = filepath.Join(root, dir) - if err := os.MkdirAll(dir, 0644); err != nil { - return nil, errors.Wrapf(err, "failed to create directory '%s'", dir) - } - } - - // create volatile directories - for _, dir := range []string{configDir, zinitDir} { - dir = filepath.Join(volatile, dir) - if err := os.MkdirAll(dir, 0644); err != nil { - return nil, errors.Wrapf(err, "failed to create directory '%s'", dir) - } - } - - // (migration goes here) - if err := migrateFiles(root, volatile); err != nil { - log.Error().Err(err).Msg("failed doing gateway config migration") - } - - bin, err := ensureTraefikBin(ctx, cl) - if err != nil { - return nil, errors.Wrap(err, "failed to ensure traefik binary") - } - - dnsmasqCfgPath := filepath.Join(root, "dnsmasq.conf") - err = dnsmasqConfig(dnsmasqCfgPath) - if err != nil { - return nil, errors.Wrap(err, "failed to create dnsmasq config") - } - certScriptPath := filepath.Join(root, "cert.sh") - err = updateCertScript(certScriptPath, root) - if err != nil { - return nil, errors.Wrap(err, "failed to create cert script") - } - staticCfgPath := filepath.Join(root, "traefik.yaml") - updated, err := staticConfig(staticCfgPath, root, volatile, letsEncryptEmail) - if err != nil { - return nil, errors.Wrap(err, "failed to create static config") - } - - // we create the resolver to avoid the cache - resolver := &net.Resolver{ - PreferGo: true, - Dial: func(ctx context.Context, network, address string) (net.Conn, error) { - d := net.Dialer{ - Timeout: time.Millisecond * time.Duration(10000), - } - return d.DialContext(ctx, network, "8.8.8.8:53") - }, - } - - domains, err := loadDomains(ctx, filepath.Join(volatile, configDir)) - if err != nil { - return nil, errors.Wrap(err, "failed to load old domains") - } - substrateGateway := stubs.NewSubstrateGatewayStub(cl) - - gw := &gatewayModule{ - cl: cl, - resolver: resolver, - substrateGateway: substrateGateway, - volatile: volatile, - staticConfigPath: staticCfgPath, - certScriptPath: certScriptPath, - binPath: bin, - reservedDomains: domains, - domainLock: sync.RWMutex{}, - } - - // in case there are already active configurations we should always try to ensure running traefik - if _, err := gw.ensureGateway(ctx, updated); err != nil { - log.Error().Err(err).Msg("gateway is not supported") - // this is not a failure because supporting of the gateway can happen - // later if the farmer set the correct network configuration! - } - go gw.nameContractsValidator() - return gw, nil -} - -func (g *gatewayModule) getReservedDomain(domain string) (string, bool) { - v, ok := g.reservedDomains[domain] - return v, ok -} - -func (g *gatewayModule) setReservedDomain(domain string, wlID string) { - log.Debug(). - Str("domain", domain). - Str("wlID", wlID). - Msg("setting domain") - g.reservedDomains[domain] = wlID -} - -func (g *gatewayModule) deleteReservedDomain(domain string) { - log.Debug(). - Str("domain", domain). - Msg("deleting domain") - delete(g.reservedDomains, domain) -} - -func (g *gatewayModule) copyReservedDomain() map[string]string { - g.domainLock.Lock() - defer g.domainLock.Unlock() - - res := make(map[string]string, len(g.reservedDomains)) - for k, v := range g.reservedDomains { - res[k] = v - } - return res -} - -func (g *gatewayModule) validateNameContracts() error { - ctx, cancel := context.WithTimeout(context.Background(), validationPeriod/2) - defer cancel() - e := stubs.NewProvisionStub(g.cl) - networker := stubs.NewNetworkerStub(g.cl) - cfg, err := networker.GetPublicConfig(ctx) - if err != nil { - return nil - } - - baseDomain := cfg.Domain - if baseDomain == "" { - // domain doesn't exist so no name workloads exist - // or the domain was unset and name workloads will never be deleted - // should iterate over workloads instead? - return nil - } - reservedDomains := g.copyReservedDomain() - - for domain, id := range reservedDomains { - wlID := gridtypes.WorkloadID(id) - twinID, _, _, err := wlID.Parts() - if err != nil { - log.Error(). - Err(err). - Msgf("failed to parse wlID %s parts", id) - continue - } - if !strings.HasSuffix(domain, baseDomain) { - // a fqdn workload, skip validating it - continue - } - name := strings.TrimSuffix(domain, fmt.Sprintf(".%s", baseDomain)) - err = g.validateNameContract(name, twinID) - if errors.Is(err, ErrContractNotReserved) || errors.Is(err, ErrInvalidContractState) || errors.Is(err, ErrTwinIDMismatch) { - log.Debug(). - Str("reason", err.Error()). - Str("wlID", id). - Str("name", name). - Msg("removing domain in name contract validation") - if err := e.DecommissionCached(ctx, id, err.Error()); err != nil { - log.Error(). - Err(err). - Msgf("failed to decommission invalid gateway name workload %s", id) - } - } else if err != nil { - log.Error(). - Str("reason", err.Error()). - Str("wlID", id). - Str("name", name). - Msg("validating name contract failed because of a non-user error") - } - } - return nil -} - -func (g *gatewayModule) nameContractsValidator() { - // no context? - ticker := time.NewTicker(validationPeriod) - defer ticker.Stop() - for range ticker.C { - if err := g.validateNameContracts(); err != nil { - log.Error().Err(err).Msg("a round of failed name contract validation") - } - } -} - -func (g *gatewayModule) isTraefikStarted(z *zinit.Client) (bool, error) { - traefikStatus, err := z.Status(traefikService) - if errors.Is(err, zinit.ErrUnknownService) { - return false, nil - } else if err != nil { - return false, errors.Wrap(err, "failed to check traefik status") - } - - return traefikStatus.State.Is(zinit.ServiceStateRunning), nil -} - -func (g *gatewayModule) traefikBinary(ctx context.Context, z *zinit.Client) (string, string, error) { // path, name, err - info, err := z.Get(traefikService) - if err != nil { - return "", "", err - } - matches := traefikBinRegex.FindAllStringSubmatch(info.Exec, -1) - if len(matches) != 1 { - return "", "", errors.Wrapf(err, "find %d matches in %s", len(matches), info.Exec) - } - - return matches[0][0], matches[0][1], nil -} - -// ensureGateway makes sure that gateway infrastructure is in place and -// that it is supported. -func (g *gatewayModule) ensureGateway(ctx context.Context, forceResstart bool) (pkg.PublicConfig, error) { - var ( - networker = stubs.NewNetworkerStub(g.cl) - flistd = stubs.NewFlisterStub(g.cl) - ) - cfg, err := networker.GetPublicConfig(ctx) - if err != nil { - return pkg.PublicConfig{}, errors.Wrap(err, "gateway is not supported on this node") - } - - z := zinit.Default() - running, err := g.isTraefikStarted(z) - if err != nil { - return pkg.PublicConfig{}, errors.Wrap(err, "failed to check traefik status") - } - exists, err := z.Exists(traefikService) - if err != nil { - return pkg.PublicConfig{}, errors.Wrap(err, "couldn't get traefik service status") - } - if exists { - path, name, err := g.traefikBinary(ctx, z) - if err != nil { - return pkg.PublicConfig{}, errors.Wrap(err, "failed to get old traefik binary path") - } - if path != g.binPath { - if err := z.StopWait(10*time.Second, traefikService); err != nil { - return pkg.PublicConfig{}, errors.Wrap(err, "failed to stop old traefik") - } - running = false - if err := flistd.Unmount(ctx, name); err != nil { - log.Error().Err(err).Msg("failed to unmount old traefik") - } - } - - if !running { - if err := z.Forget(traefikService); err != nil { - return pkg.PublicConfig{}, errors.Wrap(err, "failed to forget old traefik") - } - } - } - - if running && forceResstart { - // note: a kill is basically a singal to traefik process to - // die. but zinit will restart it again anyway. so this is - // enough to force restart it. - if err := z.Kill(traefikService, zinit.SIGTERM); err != nil { - return pkg.PublicConfig{}, errors.Wrap(err, "failed to restart traefik") - } - } - - if running { - return cfg, nil - } - - //other wise we start traefik - return cfg, g.startTraefik(z) -} -func (g *gatewayModule) verifyDomainDestination(ctx context.Context, cfg pkg.PublicConfig, domain string) error { - ips, err := g.resolver.LookupHost(ctx, domain) - if err != nil { - return err - } - for _, ip := range ips { - if ip == cfg.IPv4.IP.String() || ip == cfg.IPv6.IP.String() { - return nil - } - } - return errors.New("host doesn't point to the gateway ip") -} - -func (g *gatewayModule) startTraefik(z *zinit.Client) error { - - cmd := fmt.Sprintf( - "ip netns exec public %s --configfile %s", - g.binPath, - g.staticConfigPath, - ) - - if err := zinit.AddService(traefikService, zinit.InitService{ - Exec: cmd, - Env: map[string]string{ - "EXEC_PATH": g.certScriptPath, - }, - }); err != nil { - return errors.Wrap(err, "failed to add traefik to zinit") - } - - if err := z.Monitor(traefikService); err != nil { - return errors.Wrap(err, "couldn't monitor traefik service") - } - - if err := z.StartWait(time.Second*20, traefikService); err != nil { - return errors.Wrap(err, "waiting for trafik start timed out") - } - - return nil -} - -func (g *gatewayModule) configPath(name string) string { - return filepath.Join(g.volatile, configDir, fmt.Sprintf("%s.yaml", name)) -} - -func (g *gatewayModule) validateNameContract(name string, twinID uint32) error { - - contractID, subErr := g.substrateGateway.GetContractIDByNameRegistration(context.Background(), string(name)) - if subErr.IsCode(pkg.CodeNotFound) { - return ErrContractNotReserved - } - if subErr.IsError() { - return subErr.Err - } - contract, subErr := g.substrateGateway.GetContract(context.Background(), contractID) - if subErr.IsCode(pkg.CodeNotFound) { - return fmt.Errorf("contract by name returned %d, but retrieving it results in 'not found' error", contractID) - } else if subErr.IsError() { - return subErr.Err - } - if !contract.State.IsCreated { - return ErrInvalidContractState - } - if uint32(contract.TwinID) != twinID { - return ErrTwinIDMismatch - } - return nil -} - -func (g *gatewayModule) SetNamedProxy(wlID string, config zos.GatewayNameProxy) (string, error) { - ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) - defer cancel() - - if len(config.Backends) != 1 { - return "", fmt.Errorf("only one backend is supported got '%d'", len(config.Backends)) - } - - twinID, _, _, err := gridtypes.WorkloadID(wlID).Parts() - if err != nil { - return "", errors.Wrap(err, "invalid workload id") - } - cfg, err := g.ensureGateway(ctx, false) - if err != nil { - return "", err - } - if cfg.Domain == "" { - return "", errors.New("node doesn't support name proxy (doesn't have a domain)") - } - - if err := g.validateNameContract(config.Name, twinID); err != nil { - return "", errors.Wrap(err, "failed to verify name contract") - } - - fqdn := fmt.Sprintf("%s.%s", config.Name, cfg.Domain) - - gatewayTLSConfig := TlsConfig{ - CertResolver: dnsCertResolver, - Domains: []Domain{ - { - Sans: []string{fmt.Sprintf("*.%s", cfg.Domain)}, - }, - }, - } - - if err := g.setupRouting(ctx, wlID, fqdn, gatewayTLSConfig, config.GatewayBase); err != nil { - return "", err - } - - return fqdn, nil -} - -func (g *gatewayModule) SetFQDNProxy(wlID string, config zos.GatewayFQDNProxy) error { - ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) - defer cancel() - - if len(config.Backends) != 1 { - return fmt.Errorf("only one backend is supported got '%d'", len(config.Backends)) - } - - cfg, err := g.ensureGateway(ctx, false) - if err != nil { - return err - } - - if cfg.Domain != "" && strings.HasSuffix(config.FQDN, cfg.Domain) { - return errors.New("can't create a fqdn workload with a subdomain of the gateway's managed domain") - } - if err := g.verifyDomainDestination(ctx, cfg, config.FQDN); err != nil { - return errors.Wrap(err, "failed to verify domain dns record") - } - gatewayTLSConfig := TlsConfig{ - CertResolver: httpCertResolver, - Domains: []Domain{ - { - Main: config.FQDN, - }, - }, - } - - return g.setupRouting(ctx, wlID, config.FQDN, gatewayTLSConfig, config.GatewayBase) -} - -func (g *gatewayModule) setupRouting(ctx context.Context, wlID string, fqdn string, tlsConfig TlsConfig, config zos.GatewayBase) error { - g.domainLock.Lock() - defer g.domainLock.Unlock() - - backend := config.Backends[0] - - if err := zos.Backend(backend).Valid(config.TLSPassthrough); err != nil { - return errors.Wrapf(err, "failed to validate backend '%s'", backend) - } - - if _, ok := g.getReservedDomain(fqdn); ok { - return errors.New("domain already registered") - } - - if config.Network == nil { - // not going over user private network - return g.setupRoutingGeneric(wlID, fqdn, tlsConfig, config) - } - - // otherwise we need to configure a nnc process - // to forward the user traffic. - - // first validate that network exist and get the network namespace - twinID, _, _, err := gridtypes.WorkloadID(wlID).Parts() - if err != nil { - return errors.Wrap(err, "invalid workload id") - } - // if network is set, means this ip need to be reached from inside the user NR - net := stubs.NewNetworkerStub(g.cl) - netID := zos.NetworkID(twinID, *config.Network) - if _, err := net.GetNet(ctx, netID); err != nil { - return errors.Wrap(err, "failed to get user network") - } - ns := net.Namespace(ctx, netID) - backend, err = g.nncEnsure(wlID, ns, config.Backends[0]) - if err != nil { - return errors.Wrap(err, "failed to ensure local gateway") - } - - if !config.TLSPassthrough { - // if tls passthrough is disabled traefik expecting backend - // to be in the format http://:port - backend = zos.Backend(fmt.Sprintf("http://%s", backend)) - } - - config.Backends = []zos.Backend{backend} - return g.setupRoutingGeneric(wlID, fqdn, tlsConfig, config) -} - -func (g *gatewayModule) setupRoutingGeneric(wlID string, fqdn string, tlsConfig TlsConfig, config zos.GatewayBase) error { - backend := config.Backends[0] - var rule string - if config.TLSPassthrough { - rule = fmt.Sprintf("HostSNI(`%s`)", fqdn) - tlsConfig = TlsConfig{ - Passthrough: "true", - } - } else { - rule = fmt.Sprintf("Host(`%s`)", fqdn) - } - - var server Server - if config.TLSPassthrough { - server = Server{Address: string(backend)} - } else { - server = Server{Url: string(backend)} - } - - route := fmt.Sprintf("%s-route", wlID) - proxyConfig := ProxyConfig{} - - routingconfig := &HTTPConfig{ - Routers: map[string]Router{ - route: { - Rule: rule, - Service: wlID, - Tls: &tlsConfig, - }, - }, - Services: map[string]Service{ - wlID: { - LoadBalancer: LoadBalancer{ - Servers: []Server{server}, - }, - }, - }, - } - if config.TLSPassthrough { - proxyConfig.TCP = routingconfig - } else { - proxyConfig.Http = routingconfig - } - yamlString, err := yaml.Marshal(&proxyConfig) - if err != nil { - return errors.Wrap(err, "failed to convert config to yaml") - } - log.Debug().Str("yaml-config", string(yamlString)).Msg("configuration file") - if err = os.WriteFile(g.configPath(wlID), yamlString, 0644); err != nil { - return errors.Wrap(err, "couldn't open config file for writing") - } - g.setReservedDomain(fqdn, wlID) - return nil -} - -func (g *gatewayModule) DeleteNamedProxy(wlID string) error { - g.destroyNNC(wlID) - - path := g.configPath(wlID) - _, domain, err := domainFromConfig(path) - if os.IsNotExist(err) { - return nil - } else if err != nil { - log.Error().Err(err).Str("path", path).Msg("failed to load domain from config file") - } - if err := os.Remove(path); err != nil && !os.IsNotExist(err) { - return errors.Wrap(err, "couldn't remove config file") - } - - if domain != "" { - g.deleteReservedDomain(domain) - } - return nil -} diff --git a/pkg/gateway/metrics.go b/pkg/gateway/metrics.go deleted file mode 100644 index 95ba251bb..000000000 --- a/pkg/gateway/metrics.go +++ /dev/null @@ -1,209 +0,0 @@ -package gateway - -import ( - "bufio" - "context" - "io" - "net" - "net/http" - "net/url" - "regexp" - "strconv" - "strings" - - "github.com/containernetworking/plugins/pkg/ns" - "github.com/pkg/errors" - "github.com/rs/zerolog/log" - "github.com/threefoldtech/zos/pkg" - "github.com/threefoldtech/zos/pkg/network/namespace" -) - -const ( - publicNS = "public" - metricsURL = "http://127.0.0.1:8082/metrics" - - metricRequest = "traefik_service_requests_bytes_total" - metricResponse = "traefik_service_responses_bytes_total" -) - -var ( - ErrMetricsNotAvailable = errors.New("metrics not available") - - metricM = regexp.MustCompile(`^(\w+)({[^}]+})? ([0-9\.e\+-]+)`) - tagsM = regexp.MustCompile(`([^,={]+)="([^"]+)"`) -) - -type value struct { - value float64 - labels map[string]string -} - -type metric struct { - key string - values []value -} - -// group group values by label, if multiple values in this -// metric has the same value for the label, values are added. -// applies the mapping function on the value of the label -func (m *metric) group(l string, mapping func(string) string) map[string]float64 { - result := make(map[string]float64) - if mapping == nil { - mapping = func(s string) string { return s } - } - for _, v := range m.values { - result[mapping(v.labels[l])] += v.value - } - return result -} - -func tags(s string) map[string]string { - matches := tagsM.FindAllStringSubmatch(s, -1) - if len(matches) == 0 { - return nil - } - result := make(map[string]string) - for _, m := range matches { - result[m[1]] = m[2] - } - return result -} - -func parseMetrics(in io.Reader) (map[string]*metric, error) { - scanner := bufio.NewScanner(in) - results := make(map[string]*metric) - - for scanner.Scan() { - line := strings.TrimSpace(scanner.Text()) - if strings.HasPrefix(line, "#") { - continue - } - - parts := metricM.FindStringSubmatch(line) - - if len(parts) == 0 { - // no match - continue - } - key := parts[1] - labels := tags(parts[2]) - valueStr := parts[3] - - valueF, err := strconv.ParseFloat(valueStr, 64) - if err != nil { - return nil, errors.Wrapf(err, "failed to parse value '%s' for metric '%s'", valueStr, key) - } - - v := value{ - value: valueF, - labels: labels, - } - - m, ok := results[key] - if !ok { - m = &metric{key: key} - results[key] = m - } - - m.values = append(m.values, v) - } - - return results, nil -} - -func metrics(rawUrl string) (map[string]*metric, error) { - // so. why this is done like this you may ask? - // and we don't have a straight answer to you. but here is the deal - // traefik is running (always) inside the public namespace - // this is why we call `metrics` method from inside that namespace - // we expected this to work since the go routine now is locked to - // the OS thread that is running inside this namespace. - // seems that is wrong, using the http client (default or custom) - // will always use the host namespace, so we assume there is a go routine - // spawned in the client somewhere which is not fixed to the same os thread. - // - // we tried to disable keep-alive so we create new connection always. - // and other tricks as well. but nothing worked. - // - // the only way was to create a tcp connection ourselves and then - // use this int he http client. - u, err := url.Parse(rawUrl) - if err != nil { - return nil, errors.Wrap(err, "failed to parse url") - } - - con, err := net.Dial("tcp", u.Host) - if err != nil { - return nil, errors.Wrap(ErrMetricsNotAvailable, err.Error()) - } - - defer con.Close() - - cl := http.Client{ - Transport: &http.Transport{ - DisableKeepAlives: true, - DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) { - return con, nil - }, - }, - } - - response, err := cl.Get(rawUrl) - - if err != nil { - return nil, errors.Wrap(err, "failed to get metrics") - } - - defer func() { - _, _ = io.ReadAll(response.Body) - _ = response.Body.Close() - }() - - if response.StatusCode != 200 { - return nil, errors.Wrapf(err, "got wrong status code for metrics: %s", response.Status) - } - - return parseMetrics(response.Body) -} - -func (g *gatewayModule) Metrics() (result pkg.GatewayMetrics, err error) { - // metric is only available if traefik is running. we can instead of doing - // all the checks, we can try to directly get the metrics and see if we - // can get it. we need to do these operations anyway. - pubNS, err := namespace.GetByName(publicNS) - if err != nil { - // gateway is not enabled - return result, nil - } - - defer pubNS.Close() - var values map[string]*metric - err = pubNS.Do(func(_ ns.NetNS) error { - log.Debug().Str("namespace", publicNS).Str("url", metricsURL).Msg("requesting metrics from traefik") - values, err = metrics(metricsURL) - - return err - }) - - if errors.Is(err, ErrMetricsNotAvailable) { - // traefik is not running because there - // are no gateway configured - return result, nil - } else if err != nil { - return result, err - } - - mapping := func(s string) string { - return strings.TrimSuffix(s, "@file") - } - if m, ok := values[metricRequest]; ok { - // sent metrics. - result.Request = m.group("service", mapping) - } - - if m, ok := values[metricResponse]; ok { - result.Response = m.group("service", mapping) - } - - return -} diff --git a/pkg/gateway/metrics_test.go b/pkg/gateway/metrics_test.go deleted file mode 100644 index afbe274a6..000000000 --- a/pkg/gateway/metrics_test.go +++ /dev/null @@ -1,149 +0,0 @@ -package gateway - -import ( - "bytes" - "fmt" - "strings" - "testing" - - "github.com/stretchr/testify/require" -) - -const ( - testBody = ` -# HELP process_virtual_memory_bytes Virtual memory size in bytes. -# TYPE process_virtual_memory_bytes gauge -process_virtual_memory_bytes 2.198962176e+09 -# HELP process_virtual_memory_max_bytes Maximum amount of virtual memory available in bytes. -# TYPE process_virtual_memory_max_bytes gauge -process_virtual_memory_max_bytes -1 -# HELP traefik_config_last_reload_failure Last config reload failure -# TYPE traefik_config_last_reload_failure gauge -traefik_config_last_reload_failure 0 -# HELP traefik_config_last_reload_success Last config reload success -# TYPE traefik_config_last_reload_success gauge -traefik_config_last_reload_success 1.631101061e+09 -# HELP traefik_config_reloads_failure_total Config failure reloads -# TYPE traefik_config_reloads_failure_total counter -traefik_config_reloads_failure_total 0 -# HELP traefik_config_reloads_total Config reloads -# TYPE traefik_config_reloads_total counter -traefik_config_reloads_total 1 -# HELP traefik_entrypoint_open_connections How many open connections exist on an entrypoint, partitioned by method and protocol. -# TYPE traefik_entrypoint_open_connections gauge -traefik_entrypoint_open_connections{entrypoint="metrics",method="GET",protocol="http"} 1 -traefik_entrypoint_open_connections{entrypoint="metrics",method="POST",protocol="http"} 0 -# HELP traefik_entrypoint_request_duration_seconds How long it took to process the request on an entrypoint, partitioned by status code, protocol, and method. -# TYPE traefik_entrypoint_request_duration_seconds histogram -traefik_entrypoint_request_duration_seconds_bucket{code="502",entrypoint="metrics",method="POST",protocol="http",le="0.1"} -1 -traefik_entrypoint_request_duration_seconds_bucket{code="502",entrypoint="metrics",method="POST",protocol="http",le="0.3"} 1 -traefik_entrypoint_request_duration_seconds_bucket{code="502",entrypoint="metrics",method="POST",protocol="http",le="1.2"} 1 -traefik_entrypoint_request_duration_seconds_bucket{code="502",entrypoint="metrics",method="POST",protocol="http",le="5"} 1 -traefik_entrypoint_request_duration_seconds_bucket{code="502",entrypoint="metrics",method="POST",protocol="http",le="+Inf"} 1 -traefik_entrypoint_request_duration_seconds_sum{code="502",entrypoint="metrics",method="POST",protocol="http"} 0.00067077 -traefik_entrypoint_request_duration_seconds_count{code="502",entrypoint="metrics",method="POST",protocol="http"} 1 -# HELP traefik_entrypoint_requests_total How many HTTP requests processed on an entrypoint, partitioned by status code, protocol, and method. -# TYPE traefik_entrypoint_requests_total counter -traefik_entrypoint_requests_total{code="502",entrypoint="metrics",method="POST",protocol="http"} 1 -# HELP traefik_service_open_connections How many open connections exist on a service, partitioned by method and protocol. -# TYPE traefik_service_open_connections gauge -traefik_service_open_connections{method="POST",protocol="http",service="foo@file"} 0 -# HELP traefik_service_request_duration_seconds How long it took to process the request on a service, partitioned by status code, protocol, and method. -# TYPE traefik_service_request_duration_seconds histogram -traefik_service_request_duration_seconds_bucket{code="502",method="POST",protocol="http",service="foo@file",le="0.1"} 1 -traefik_service_request_duration_seconds_bucket{code="502",method="POST",protocol="http",service="foo@file",le="0.3"} 1 -traefik_service_request_duration_seconds_bucket{code="502",method="POST",protocol="http",service="foo@file",le="1.2"} 1 -traefik_service_request_duration_seconds_bucket{code="502",method="POST",protocol="http",service="foo@file",le="5"} 1 -traefik_service_request_duration_seconds_bucket{code="502",method="POST",protocol="http",service="foo@file",le="+Inf"} 1 -traefik_service_request_duration_seconds_sum{code="502",method="POST",protocol="http",service="foo@file"} 0.000484654 -traefik_service_request_duration_seconds_count{code="502",method="POST",protocol="http",service="foo@file"} 1 -# HELP traefik_service_requests_total How many HTTP requests processed on a service, partitioned by status code, protocol, and method. -# TYPE traefik_service_requests_total counter -traefik_service_requests_total{code="502",method="POST",protocol="http",service="foo@file"} 1` - - bigNumber = ` -traefik_service_bytes_sent_total{service="50-715-gw@file"} 8.95470206e+08 -` - testValues = ` -# TYPE traefik_service_requests_bytes_total counter -traefik_service_requests_bytes_total{code="200",method="GET",protocol="http",service="10-123-gateway@file"} 100 -traefik_service_requests_bytes_total{code="404",method="GET",protocol="http",service="10-123-gateway@file"} 120 -traefik_service_requests_bytes_total{code="502",method="GET",protocol="http",service="10-123-gateway@file"} 10 -traefik_service_responses_bytes_total{code="200",method="GET",protocol="http",service="10-123-gateway@file"} 1307 -traefik_service_responses_bytes_total{code="404",method="GET",protocol="http",service="10-123-gateway@file"} 469 -traefik_service_responses_bytes_total{code="502",method="GET",protocol="http",service="10-123-gateway@file"} 22 - ` -) - -func TestParseMetrics(t *testing.T) { - - buf := bytes.NewBuffer([]byte(testBody)) - - results, err := parseMetrics(buf) - require.NoError(t, err) - - // single value, no tags - m, ok := results["traefik_config_reloads_total"] - require.True(t, ok) - - require.Equal(t, "traefik_config_reloads_total", m.key) - require.Len(t, m.values, 1) - require.EqualValues(t, m.values[0].value, 1) - - // multiple values - - m, ok = results["traefik_entrypoint_request_duration_seconds_bucket"] - require.True(t, ok) - - require.Equal(t, "traefik_entrypoint_request_duration_seconds_bucket", m.key) - require.Len(t, m.values, 5) - fst := m.values[0] - - require.EqualValues(t, fst.value, -1) - require.Len(t, fst.labels, 5) - - fmt.Println(fst.labels) - require.Equal(t, "502", fst.labels["code"]) - require.Equal(t, "0.1", fst.labels["le"]) -} - -func TestParseBigMetrics(t *testing.T) { - - buf := bytes.NewBuffer([]byte(bigNumber)) - - results, err := parseMetrics(buf) - require.NoError(t, err) - - // single value, no tags - m, ok := results["traefik_service_bytes_sent_total"] - require.True(t, ok) - - require.Equal(t, "traefik_service_bytes_sent_total", m.key) - require.Len(t, m.values, 1) - require.EqualValues(t, float64(8.95470206e+08), m.values[0].value) -} - -func TestMetrics(t *testing.T) { - - buf := bytes.NewBuffer([]byte(testValues)) - - results, err := parseMetrics(buf) - require.NoError(t, err) - - request := results[metricRequest] - responses := results[metricResponse] - - servicesRequests := request.group("service", func(s string) string { - return strings.TrimSuffix(s, "@file") - }) - - servicesResponses := responses.group("service", func(s string) string { - return strings.TrimSuffix(s, "@file") - }) - - require.Len(t, servicesRequests, 1) - require.Len(t, servicesResponses, 1) - - require.EqualValues(t, 230, servicesRequests["10-123-gateway"]) - require.EqualValues(t, 1798, servicesResponses["10-123-gateway"]) -} diff --git a/pkg/gateway/nnc.go b/pkg/gateway/nnc.go deleted file mode 100644 index 7cc8c2415..000000000 --- a/pkg/gateway/nnc.go +++ /dev/null @@ -1,239 +0,0 @@ -package gateway - -import ( - "fmt" - "math" - "math/rand" - "net" - "os" - "path/filepath" - "strconv" - "strings" - "time" - - "github.com/google/shlex" - "github.com/pkg/errors" - "github.com/threefoldtech/zos/pkg/gridtypes" - "github.com/threefoldtech/zos/pkg/gridtypes/zos" - "github.com/threefoldtech/zos/pkg/zinit" - "gopkg.in/yaml.v2" -) - -const ( - nncServicePrefix = "nnc-" - nncStartPort = 2000 -) - -// NNC holds (and extracts) information from nnc command -type NNC struct { - ID gridtypes.WorkloadID - Exec string -} - -func (n *NNC) arg(key string) (string, error) { - parts, err := shlex.Split(n.Exec) - if err != nil { - return "", err - } - - for i, part := range parts { - if part == key { - return parts[i+1], nil - } - } - - return "", fmt.Errorf("not found") -} - -// port return port number this nnc instance is listening on -func (n *NNC) port() (uint16, error) { - listen, err := n.arg("--listen") - if err != nil { - return 0, err - } - - _, port, err := net.SplitHostPort(listen) - if err != nil { - return 0, err - } - - value, err := strconv.Atoi(port) - if err != nil { - return 0, err - } - - return uint16(value), nil -} - -// nncZinitPath return path to the nnc zinit config file given the name -func (g *gatewayModule) nncZinitPath(name string) string { - return filepath.Join(g.volatile, zinitDir, fmt.Sprintf("%s.yaml", name)) -} - -// nncList lists all running instances of nnc. the map key is -// the instance configured listening port -func (g *gatewayModule) nncList() (map[uint16]NNC, error) { - cl := zinit.Default() - services, err := cl.List() - if err != nil { - return nil, err - } - nncs := make(map[uint16]NNC) - // note: should we just instead list the config - // from the known config directory? - for name := range services { - if !strings.HasPrefix(name, nncServicePrefix) { - continue - } - id := strings.TrimPrefix(name, nncServicePrefix) - - cfg, err := cl.Get(name) - if err != nil { - return nil, err - } - - nnc := NNC{ - ID: gridtypes.WorkloadID(id), - Exec: cfg.Exec, - } - - port, err := nnc.port() - if err != nil { - return nil, err - } - - nncs[port] = nnc - } - - return nncs, nil -} - -// nncGet return NNC instance by name -func (g *gatewayModule) nncGet(name string) (NNC, error) { - cl := zinit.Default() - cfg, err := cl.Get(name) - if err != nil { - return NNC{}, err - } - - return NNC{ - ID: gridtypes.WorkloadID(strings.TrimPrefix(name, nncServicePrefix)), - Exec: cfg.Exec, - }, nil - -} - -func (g *gatewayModule) nncName(id string) string { - return fmt.Sprintf("%s%s", nncServicePrefix, id) -} - -func (g *gatewayModule) nncFreePort() (uint16, error) { - // TODO: this need to call while holding some lock - // to avoid double allocation of the same port - current, err := g.nncList() - if err != nil { - return 0, err - } - - for { - port := uint16(rand.Intn(math.MaxUint16-nncStartPort) + nncStartPort) - if _, ok := current[port]; !ok { - return port, nil - } - } -} - -func (g *gatewayModule) nncCreateService(name string, service zinit.InitService) error { - path := g.nncZinitPath(name) - data, err := yaml.Marshal(service) - if err != nil { - return err - } - - if err := os.WriteFile(path, data, 0644); err != nil { - return errors.Wrap(err, "failed to create nnc service file") - } - - // link under zinit /etc/zinit - - if err := os.Symlink(path, filepath.Join("/etc/zinit", fmt.Sprintf("%s.yaml", name))); err != nil { - return errors.Wrap(err, "failed to create nnc service symlink") - } - - return nil -} - -// nncEnsure creates (or reuse) an nnc instance given the workload ID. the destination namespace and backend -// it return the backend that need to be configured in traefik. -func (g *gatewayModule) nncEnsure(wlID, namespace string, backend zos.Backend) (zos.Backend, error) { - name := g.nncName(wlID) - - // reuse or find a new free IP - var free uint16 - nnc, err := g.nncGet(name) - if err == nil { - // a service with the same name already exists - // verify the backend? - free, err = nnc.port() - // we always destroy the service if - // a one already exists with the same name - // to allow updating the gw code. - g.destroyNNC(wlID) - } else if errors.Is(err, zinit.ErrUnknownService) { - // if does not exist, just find a free port - free, err = g.nncFreePort() - } else if err != nil { - return "", err - } - - // this checks the error returned by the - // free port allocation - if err != nil { - return "", err - } - - target, err := backend.AsAddress() - if err != nil { - return "", err - } - - be := zos.Backend(fmt.Sprintf("127.0.0.1:%d", free)) - - cmd := []string{ - "ip", "netns", "exec", "public", - "nnc", - "--listen", string(be), - "--namespace", filepath.Join("/var/run/netns/", namespace), - "--target", target, - } - - service := zinit.InitService{ - Exec: strings.Join(cmd, " "), - } - - if err = g.nncCreateService(name, service); err != nil { - return "", err - } - - defer func() { - if err != nil { - g.destroyNNC(wlID) - } - }() - - if err = zinit.Default().Monitor(name); err != nil { - return "", errors.Wrap(err, "failed to start nnc service") - } - - return be, nil -} - -// destroyNNC stops and clean up nnc instances -func (g *gatewayModule) destroyNNC(wlID string) { - name := g.nncName(wlID) - path := g.nncZinitPath(name) - - cl := zinit.Default() - _ = cl.Destroy(10*time.Second, name) - _ = os.Remove(path) -} diff --git a/pkg/gateway/static.go b/pkg/gateway/static.go deleted file mode 100644 index c57e5ea5e..000000000 --- a/pkg/gateway/static.go +++ /dev/null @@ -1,45 +0,0 @@ -package gateway - -import ( - "bytes" - _ "embed" - "fmt" - "os" - - "github.com/pkg/errors" -) - -//go:embed static/config.yaml -var config string - -//go:embed static/dnsmasq.conf -var dConfig string - -//go:embed static/cert.sh -var certScript string - -// staticConfig write static config to file -func staticConfig(p, root, volatile, email string) (bool, error) { - config := fmt.Sprintf(config, root, email, volatile) - - var update bool - if oldConfig, err := os.ReadFile(p); os.IsNotExist(err) { - update = true - } else if err != nil { - return false, errors.Wrap(err, "failed to read traefik config") - } else { - // no errors - update = !bytes.Equal([]byte(config), oldConfig) - } - - return update, os.WriteFile(p, []byte(config), 0644) -} - -func updateCertScript(p, root string) error { - certScript := fmt.Sprintf(certScript, root) - return os.WriteFile(p, []byte(certScript), 0744) -} - -func dnsmasqConfig(p string) error { - return os.WriteFile(p, []byte(dConfig), 0644) -} diff --git a/pkg/gateway/static/cert.sh b/pkg/gateway/static/cert.sh deleted file mode 100644 index 678021a9b..000000000 --- a/pkg/gateway/static/cert.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/sh - -DOMAIN="$2" -DOMAIN=${DOMAIN#"*."} -DOMAIN=${DOMAIN%%"."} -TOKEN="$3" -IP=`ip -4 addr show public | grep inet | tr -s ' ' | cut -d' ' -f3 | cut -d'/' -f1 | tail -n 1` -PIDFILE="/tmp/certbot-dnsmasq.pid" -CONFFILE="/tmp/dnsmasq.conf" - -if [ -z "$IP" ]; then - echo no public ip found - exit 1 -fi - -if [ $1 == "present" ]; then - if [ -f $PIDFILE ]; then - PID=`cat $PIDFILE` - kill $PID - while kill -0 $PID; do - sleep 1 - done - fi - cp %[1]s/dnsmasq.conf $CONFFILE - sed -i "s/DOMAIN/$DOMAIN/g" $CONFFILE - sed -i "s/TOKEN/$TOKEN/g" $CONFFILE - sed -i "s/IP/$IP/g" $CONFFILE - dnsmasq -C $CONFFILE -elif [ $1 == "timeout" ]; then - echo '{"timeout": 30, "interval": 5}' -elif [ $1 == "cleanup" ]; then - # kill dnsmasq and remove related files - [ -f $PIDFILE ] && kill `cat $PIDFILE` - rm -rf $CONFFILE $PIDFILE -fi \ No newline at end of file diff --git a/pkg/gateway/static/config.yaml b/pkg/gateway/static/config.yaml deleted file mode 100644 index 373b76ff3..000000000 --- a/pkg/gateway/static/config.yaml +++ /dev/null @@ -1,34 +0,0 @@ -entryPoints: - web: - address: ":80" - http: - redirections: - entryPoint: - to: web-secure - scheme: https - web-secure: - address: ":443" - metrics: - # only listen on lo for metrics - address: "127.0.0.1:8082" -metrics: - prometheus: - entryPoint: metrics -providers: - file: - directory: "%[3]s/proxy" - watch: true -certificatesResolvers: - resolver: - acme: - email: "%[2]s" - storage: "%[1]s/traefik/acme.json" - httpChallenge: - # used during the challenge - entryPoint: web - dnsresolver: - acme: - email: "%[2]s" - storage: "%[1]s/traefik/acme2.json" - dnsChallenge: - provider: exec diff --git a/pkg/gateway/static/dnsmasq.conf b/pkg/gateway/static/dnsmasq.conf deleted file mode 100644 index b6b72b46b..000000000 --- a/pkg/gateway/static/dnsmasq.conf +++ /dev/null @@ -1,8 +0,0 @@ -no-hosts -pid-file=/tmp/certbot-dnsmasq.pid -auth-zone=DOMAIN,IP/24 -auth-server=DOMAIN,public -auth-soa=12345678,admin.DOMAIN. -host-record=DOMAIN,IP -caa-record=DOMAIN.,0,issue,letsencrypt.org -txt-record=DOMAIN.,"TOKEN" diff --git a/pkg/gridtypes/zos/network.go b/pkg/gridtypes/zos/network.go deleted file mode 100644 index 854cfbc41..000000000 --- a/pkg/gridtypes/zos/network.go +++ /dev/null @@ -1,245 +0,0 @@ -package zos - -import ( - "bytes" - "crypto/md5" - "fmt" - "io" - - "github.com/jbenet/go-base58" - "github.com/threefoldtech/zos/pkg/gridtypes" -) - -const ( - MyceliumKeyLen = 32 -) - -// NetID is a type defining the ID of a network -type NetID string - -func (i NetID) String() string { - return string(i) -} - -// NetworkID construct a network ID based on a userID and network name -func NetworkID(twin uint32, network gridtypes.Name) NetID { - buf := bytes.Buffer{} - buf.WriteString(fmt.Sprint(twin)) - buf.WriteString(":") - buf.WriteString(string(network)) - h := md5.Sum(buf.Bytes()) - b := base58.Encode(h[:]) - if len(b) > 13 { - b = b[:13] - } - return NetID(string(b)) -} - -func NetworkIDFromWorkloadID(wl gridtypes.WorkloadID) (NetID, error) { - twin, _, name, err := wl.Parts() - if err != nil { - return "", err - } - return NetworkID(twin, name), nil -} - -// Network is the description of a part of a network local to a specific node. -// A network workload defines a wireguard network that is usually spans multiple nodes. One of the nodes must work as an access node -// in other words, it must be reachable from other nodes, hence it needs to have a `PublicConfig`. -// Since the user library creates all deployments upfront then all wireguard keys, and ports must be pre-determinstic and must be -// also created upfront. -// A network structure basically must consist of -// - The network information (IP range) must be an ipv4 /16 range -// - The local (node) peer definition (subnet of the network ip range, wireguard secure key, wireguard port if any) -// - List of other peers that are part of the same network with their own config -// - For each PC or a laptop (for each wireguard peer) there must be a peer in the peer list (on all nodes) -// This is why this can get complicated. -type Network struct { - // IP range of the network, must be an IPv4 /16 - // for example a 10.1.0.0/16 - NetworkIPRange gridtypes.IPNet `json:"ip_range"` - - // IPV4 subnet for this network resource - // this must be a valid subnet of the entire network ip range. - // for example 10.1.1.0/24 - Subnet gridtypes.IPNet `json:"subnet"` - - // The private wg key of this node (this peer) which is installing this - // network workload right now. - // This has to be filled in by the user (and not generated for example) - // because other peers need to be installed as well (with this peer public key) - // hence it's easier to configure everything one time at the user side and then - // apply everything on all nodes at once - WGPrivateKey string `json:"wireguard_private_key"` - // WGListenPort is the wireguard listen port on this node. this has - // to be filled in by the user for same reason as private key (other nodes need to know about it) - // To find a free port you have to ask the node first by a call over RMB about which ports are possible - // to use. - WGListenPort uint16 `json:"wireguard_listen_port"` - - // Peers is a list of other peers in this network - Peers []Peer `json:"peers"` - - // Optional mycelium configuration. If provided - // VMs in this network can use the mycelium feature. - // if no mycelium configuration is provided, vms can't - // get mycelium IPs. - Mycelium *Mycelium `json:"mycelium,omitempty"` -} - -type MyceliumPeer string - -type Mycelium struct { - // Key is the key of the mycelium peer in the mycelium node - // associated with this network. - // It's provided by the user so it can be later moved to other nodes - // without losing the key. - Key Bytes `json:"hex_key"` - // An optional mycelium peer list to be used with this node, otherwise - // the default peer list is used. - Peers []MyceliumPeer `json:"peers"` -} - -func (c *Mycelium) Challenge(b io.Writer) error { - if _, err := fmt.Fprintf(b, "%x", c.Key); err != nil { - return err - } - - for _, peer := range c.Peers { - if _, err := fmt.Fprintf(b, "%s", peer); err != nil { - return err - } - } - - return nil -} - -func (c *Mycelium) Valid() error { - if len(c.Key) != MyceliumKeyLen { - return fmt.Errorf("invalid mycelium key length, expected %d", MyceliumKeyLen) - } - - // TODO: - // we are not supporting extra peers right now until - if len(c.Peers) != 0 { - return fmt.Errorf("user defined peers list is not supported right now") - } - return nil -} - -// Valid checks if the network resource is valid. -func (n Network) Valid(getter gridtypes.WorkloadGetter) error { - - if n.NetworkIPRange.Nil() { - return fmt.Errorf("network IP range cannot be empty") - } - - if len(n.Subnet.IP) == 0 { - return fmt.Errorf("network resource subnet cannot empty") - } - - if n.WGPrivateKey == "" { - return fmt.Errorf("network resource wireguard private key cannot empty") - } - - for _, peer := range n.Peers { - if err := peer.Valid(); err != nil { - return err - } - } - - if n.Mycelium != nil { - if err := n.Mycelium.Valid(); err != nil { - return err - } - } - - return nil -} - -// Challenge implements WorkloadData -func (n Network) Challenge(b io.Writer) error { - if _, err := fmt.Fprintf(b, "%s", n.NetworkIPRange.String()); err != nil { - return err - } - - if _, err := fmt.Fprintf(b, "%s", n.Subnet.String()); err != nil { - return err - } - - if _, err := fmt.Fprintf(b, "%s", n.WGPrivateKey); err != nil { - return err - } - - if _, err := fmt.Fprintf(b, "%d", n.WGListenPort); err != nil { - return err - } - - for _, p := range n.Peers { - if err := p.Challenge(b); err != nil { - return err - } - } - - if n.Mycelium != nil { - if err := n.Mycelium.Challenge(b); err != nil { - return err - } - } - - return nil -} - -// Capacity implementation -func (n Network) Capacity() (gridtypes.Capacity, error) { - return gridtypes.Capacity{}, nil -} - -// Peer is the description of a peer of a NetResource -type Peer struct { - // IPV4 subnet of the network resource of the peer - Subnet gridtypes.IPNet `json:"subnet"` - // WGPublicKey of the peer (driven from its private key) - WGPublicKey string `json:"wireguard_public_key"` - // Allowed Ips is related to his subnet. - // todo: remove and derive from subnet - AllowedIPs []gridtypes.IPNet `json:"allowed_ips"` - // Entrypoint of the peer - Endpoint string `json:"endpoint"` -} - -// Valid checks if peer is valid -func (p *Peer) Valid() error { - if p.Subnet.Nil() { - return fmt.Errorf("peer wireguard subnet cannot empty") - } - - if len(p.AllowedIPs) <= 0 { - return fmt.Errorf("peer wireguard allowedIPs cannot empty") - } - - if p.WGPublicKey == "" { - return fmt.Errorf("peer wireguard public key cannot empty") - } - - return nil -} - -// Challenge for peer -func (p Peer) Challenge(w io.Writer) error { - if _, err := fmt.Fprintf(w, "%s", p.WGPublicKey); err != nil { - return err - } - if _, err := fmt.Fprintf(w, "%s", p.Endpoint); err != nil { - return err - } - if _, err := fmt.Fprintf(w, "%s", p.Subnet.String()); err != nil { - return err - } - for _, ip := range p.AllowedIPs { - if _, err := fmt.Fprintf(w, "%s", ip.String()); err != nil { - return err - } - } - return nil -} diff --git a/pkg/gridtypes/zos/network_light.go b/pkg/gridtypes/zos/network_light.go new file mode 100644 index 000000000..6f443de56 --- /dev/null +++ b/pkg/gridtypes/zos/network_light.go @@ -0,0 +1,139 @@ +package zos + +import ( + "bytes" + "crypto/md5" + "fmt" + "io" + + "github.com/jbenet/go-base58" + "github.com/threefoldtech/zos/pkg/gridtypes" +) + +const ( + MyceliumKeyLen = 32 +) + +// NetID is a type defining the ID of a network +type NetID string + +func (i NetID) String() string { + return string(i) +} + +// NetworkID construct a network ID based on a userID and network name +func NetworkID(twin uint32, network gridtypes.Name) NetID { + buf := bytes.Buffer{} + buf.WriteString(fmt.Sprint(twin)) + buf.WriteString(":") + buf.WriteString(string(network)) + h := md5.Sum(buf.Bytes()) + b := base58.Encode(h[:]) + if len(b) > 13 { + b = b[:13] + } + return NetID(string(b)) +} + +func NetworkIDFromWorkloadID(wl gridtypes.WorkloadID) (NetID, error) { + twin, _, name, err := wl.Parts() + if err != nil { + return "", err + } + return NetworkID(twin, name), nil +} + +// NetworkLight is the description of a part of a network local to a specific node. +// A network workload defines a wireguard network that is usually spans multiple nodes. One of the nodes must work as an access node +// in other words, it must be reachable from other nodes, hence it needs to have a `PublicConfig`. +// Since the user library creates all deployments upfront then all wireguard keys, and ports must be pre-deterministic and must be +// also created upfront. +// A network structure basically must consist of +// - The network information (IP range) must be an ipv4 /16 range +// - The local (node) peer definition (subnet of the network ip range, wireguard secure key, wireguard port if any) +// - List of other peers that are part of the same network with their own config +// - For each PC or a laptop (for each wireguard peer) there must be a peer in the peer list (on all nodes) +// This is why this can get complicated. +type NetworkLight struct { + // IPV4 subnet for this network resource + // this must be a valid subnet of the entire network ip range. + // for example 10.1.1.0/24 + Subnet gridtypes.IPNet `json:"subnet"` + + // Optional mycelium configuration. If provided + // VMs in this network can use the mycelium feature. + // if no mycelium configuration is provided, vms can't + // get mycelium IPs. + Mycelium Mycelium `json:"mycelium,omitempty"` +} + +type MyceliumPeer string + +type Mycelium struct { + // Key is the key of the mycelium peer in the mycelium node + // associated with this network. + // It's provided by the user so it can be later moved to other nodes + // without losing the key. + Key Bytes `json:"hex_key"` + // An optional mycelium peer list to be used with this node, otherwise + // the default peer list is used. + Peers []MyceliumPeer `json:"peers"` +} + +func (c *Mycelium) Challenge(b io.Writer) error { + if _, err := fmt.Fprintf(b, "%x", c.Key); err != nil { + return err + } + + for _, peer := range c.Peers { + if _, err := fmt.Fprintf(b, "%s", peer); err != nil { + return err + } + } + + return nil +} + +func (c *Mycelium) Valid() error { + if len(c.Key) != MyceliumKeyLen { + return fmt.Errorf("invalid mycelium key length, expected %d", MyceliumKeyLen) + } + + // TODO: + // we are not supporting extra peers right now until + if len(c.Peers) != 0 { + return fmt.Errorf("user defined peers list is not supported right now") + } + return nil +} + +// Valid checks if the network resource is valid. +func (n NetworkLight) Valid(getter gridtypes.WorkloadGetter) error { + if len(n.Subnet.IP) == 0 { + return fmt.Errorf("network resource subnet cannot empty") + } + + if err := n.Mycelium.Valid(); err != nil { + return err + } + + return nil +} + +// Challenge implements WorkloadData +func (n NetworkLight) Challenge(b io.Writer) error { + if _, err := fmt.Fprintf(b, "%s", n.Subnet.String()); err != nil { + return err + } + + if err := n.Mycelium.Challenge(b); err != nil { + return err + } + + return nil +} + +// Capacity implementation +func (n NetworkLight) Capacity() (gridtypes.Capacity, error) { + return gridtypes.Capacity{}, nil +} diff --git a/pkg/gridtypes/zos/types.go b/pkg/gridtypes/zos/types.go index ce222a2c9..9032e4c9c 100644 --- a/pkg/gridtypes/zos/types.go +++ b/pkg/gridtypes/zos/types.go @@ -12,10 +12,14 @@ const ( ZMountType gridtypes.WorkloadType = "zmount" // NetworkType type NetworkType gridtypes.WorkloadType = "network" + // NetworkLightType type + NetworkLightType gridtypes.WorkloadType = "network-light" // ZDBType type ZDBType gridtypes.WorkloadType = "zdb" // ZMachineType type ZMachineType gridtypes.WorkloadType = "zmachine" + // ZMachineLightType type + ZMachineLightType gridtypes.WorkloadType = "zmachine-light" // VolumeType type VolumeType gridtypes.WorkloadType = "volume" //PublicIPv4Type type [deprecated] @@ -37,15 +41,13 @@ func init() { // network is a sharable type, which means for a single // twin, the network objects can be 'used' from different // deployments. - gridtypes.RegisterSharableType(NetworkType, Network{}) + gridtypes.RegisterType(ZMachineLightType, ZMachineLight{}) + gridtypes.RegisterSharableType(NetworkLightType, NetworkLight{}) + gridtypes.RegisterType(ZDBType, ZDB{}) gridtypes.RegisterType(ZMountType, ZMount{}) gridtypes.RegisterType(VolumeType, Volume{}) - gridtypes.RegisterType(ZDBType, ZDB{}) - gridtypes.RegisterType(ZMachineType, ZMachine{}) gridtypes.RegisterType(PublicIPv4Type, PublicIP4{}) gridtypes.RegisterType(PublicIPType, PublicIP{}) - gridtypes.RegisterType(GatewayNameProxyType, GatewayNameProxy{}) - gridtypes.RegisterType(GatewayFQDNProxyType, GatewayFQDNProxy{}) gridtypes.RegisterType(QuantumSafeFSType, QuantumSafeFS{}) gridtypes.RegisterType(ZLogsType, ZLogs{}) } diff --git a/pkg/gridtypes/zos/zdb.go b/pkg/gridtypes/zos/zdb.go index 3f2b1d5ed..f04ccda9d 100644 --- a/pkg/gridtypes/zos/zdb.go +++ b/pkg/gridtypes/zos/zdb.go @@ -7,6 +7,14 @@ import ( "github.com/threefoldtech/zos/pkg/gridtypes" ) +// ZDB namespace creation info +type ZDB struct { + Size gridtypes.Unit `json:"size"` + Mode ZDBMode `json:"mode"` + Password string `json:"password"` + Public bool `json:"public"` +} + // ZDBMode is the enumeration of the modes 0-db can operate in type ZDBMode string @@ -29,14 +37,6 @@ func (m ZDBMode) Valid() error { return nil } -// ZDB namespace creation info -type ZDB struct { - Size gridtypes.Unit `json:"size"` - Mode ZDBMode `json:"mode"` - Password string `json:"password"` - Public bool `json:"public"` -} - // Valid implementation func (z ZDB) Valid(getter gridtypes.WorkloadGetter) error { if z.Size == 0 { diff --git a/pkg/gridtypes/zos/zmachine.go b/pkg/gridtypes/zos/zmachine_light.go similarity index 70% rename from pkg/gridtypes/zos/zmachine.go rename to pkg/gridtypes/zos/zmachine_light.go index 3133ce531..5b8fbf3f3 100644 --- a/pkg/gridtypes/zos/zmachine.go +++ b/pkg/gridtypes/zos/zmachine_light.go @@ -1,14 +1,12 @@ package zos import ( - "encoding/json" "fmt" "io" "net" "sort" "strings" - "github.com/pkg/errors" "github.com/threefoldtech/zos/pkg/gridtypes" ) @@ -50,14 +48,8 @@ func (c *MyceliumIP) Challenge(w io.Writer) error { return nil } -// MachineNetwork structure -type MachineNetwork struct { - // PublicIP optional public IP attached to this machine. If set - // it must be a valid name of a PublicIP workload in the same deployment - PublicIP gridtypes.Name `json:"public_ip"` - // Planetary support planetary network - Planetary bool `json:"planetary"` - +// MachineNetworkLight structure +type MachineNetworkLight struct { // Mycelium IP config, if planetary is true, but Mycelium is not set we fall back // to yggdrasil support. Otherwise (if mycelium is set) a mycelium ip is used instead. Mycelium *MyceliumIP `json:"mycelium,omitempty"` @@ -67,15 +59,7 @@ type MachineNetwork struct { } // Challenge builder -func (n *MachineNetwork) Challenge(w io.Writer) error { - if _, err := fmt.Fprintf(w, "%s", n.PublicIP); err != nil { - return err - } - - if _, err := fmt.Fprintf(w, "%t", n.Planetary); err != nil { - return err - } - +func (n *MachineNetworkLight) Challenge(w io.Writer) error { for _, inf := range n.Interfaces { if _, err := fmt.Fprintf(w, "%s", inf.Network); err != nil { return err @@ -95,75 +79,12 @@ func (n *MachineNetwork) Challenge(w io.Writer) error { return nil } -// MachineCapacity structure -type MachineCapacity struct { - CPU uint8 `json:"cpu"` - Memory gridtypes.Unit `json:"memory"` -} - -func (c *MachineCapacity) String() string { - return fmt.Sprintf("cpu(%d)+mem(%d)", c.CPU, c.Memory) -} - -// Challenge builder -func (c *MachineCapacity) Challenge(w io.Writer) error { - if _, err := fmt.Fprintf(w, "%d", c.CPU); err != nil { - return err - } - - if _, err := fmt.Fprintf(w, "%d", c.Memory); err != nil { - return err - } - - return nil -} - -// MachineMount structure -type MachineMount struct { - // Name is name of a zmount. The name must be a valid zmount - // in the same deployment as the zmachine - Name gridtypes.Name `json:"name"` - // Mountpoint inside the container. Not used if the zmachine - // is running in a vm mode. - Mountpoint string `json:"mountpoint"` -} - -// Challenge builder -func (m *MachineMount) Challenge(w io.Writer) error { - if _, err := fmt.Fprintf(w, "%s", m.Name); err != nil { - return err - } - - if _, err := fmt.Fprintf(w, "%s", m.Mountpoint); err != nil { - return err - } - - return nil -} - -// GPU ID -// Used by a VM, a GPU id is in the format // -// This can be queried either from the node features on the chain -// or listed via the node rmb API. -// example of a valid gpu definition `0000:28:00.0/1002/731f“ -type GPU string - -func (g GPU) Parts() (slot, vendor, device string, err error) { - parts := strings.Split(string(g), "/") - if len(parts) != 3 { - err = fmt.Errorf("invalid GPU id format '%s'", g) - return - } - - return parts[0], parts[1], parts[2], nil -} - -// ZMachine reservation data -type ZMachine struct { +// ZMachineLight reservation data +type ZMachineLight struct { // Flist of the zmachine, must be a valid url to an flist. FList string `json:"flist"` // Network configuration for machine network - Network MachineNetwork `json:"network"` + Network MachineNetworkLight `json:"network"` // Size of zmachine disk Size gridtypes.Unit `json:"size"` // ComputeCapacity configuration for machine cpu+memory @@ -191,7 +112,7 @@ type ZMachine struct { GPU []GPU `json:"gpu,omitempty"` } -func (m *ZMachine) MinRootSize() gridtypes.Unit { +func (m *ZMachineLight) MinRootSize() gridtypes.Unit { // sru = (cpu * mem_in_gb) / 8 // each 1 SRU is 50GB of storage cu := gridtypes.Unit(m.ComputeCapacity.CPU) * m.ComputeCapacity.Memory / (8 * gridtypes.Gigabyte) @@ -203,7 +124,7 @@ func (m *ZMachine) MinRootSize() gridtypes.Unit { return 2 * gridtypes.Gigabyte } -func (m *ZMachine) RootSize() gridtypes.Unit { +func (m *ZMachineLight) RootSize() gridtypes.Unit { min := m.MinRootSize() if m.Size > min { return m.Size @@ -213,7 +134,7 @@ func (m *ZMachine) RootSize() gridtypes.Unit { } // Valid implementation -func (v ZMachine) Valid(getter gridtypes.WorkloadGetter) error { +func (v ZMachineLight) Valid(getter gridtypes.WorkloadGetter) error { if len(v.Network.Interfaces) != 1 { return fmt.Errorf("only one network private network is supported at the moment") } @@ -233,37 +154,6 @@ func (v ZMachine) Valid(getter gridtypes.WorkloadGetter) error { if v.Size != 0 && v.Size < minRoot { return fmt.Errorf("disk size can't be less that %d. Set to 0 for minimum", minRoot) } - if !v.Network.PublicIP.IsEmpty() { - wl, err := getter.Get(v.Network.PublicIP) - if err != nil { - return fmt.Errorf("public ip is not found") - } - - if wl.Type != PublicIPv4Type && wl.Type != PublicIPType { - return errors.Wrapf(err, "workload of name '%s' is not a public ip", v.Network.PublicIP) - } - - // also we need to make sure this public ip is not used by other vms in the same - // deployment. - allVMs := getter.ByType(ZMachineType) - count := 0 - for _, vm := range allVMs { - var data ZMachine - if err := json.Unmarshal(vm.Data, &data); err != nil { - return err - } - // we can only check the name because unfortunately we don't know - // `this` workload ID at this level. may be can b added later. - // for now, we can just count the number of this public ip workload - // name was referenced in all VMs and fail if it's more than one. - if data.Network.PublicIP == v.Network.PublicIP { - count += 1 - } - } - if count > 1 { - return fmt.Errorf("public ip is assigned to multiple vms") - } - } for _, ifc := range v.Network.Interfaces { if ifc.Network == "ygg" || ifc.Network == "pub" || ifc.Network == "mycelium" { //reserved temporary @@ -282,7 +172,7 @@ func (v ZMachine) Valid(getter gridtypes.WorkloadGetter) error { } // Capacity implementation -func (v ZMachine) Capacity() (gridtypes.Capacity, error) { +func (v ZMachineLight) Capacity() (gridtypes.Capacity, error) { return gridtypes.Capacity{ CRU: uint64(v.ComputeCapacity.CPU), MRU: v.ComputeCapacity.Memory, @@ -291,7 +181,7 @@ func (v ZMachine) Capacity() (gridtypes.Capacity, error) { } // Challenge creates signature challenge -func (v ZMachine) Challenge(b io.Writer) error { +func (v ZMachineLight) Challenge(b io.Writer) error { if _, err := fmt.Fprintf(b, "%s", v.FList); err != nil { return err } @@ -347,37 +237,72 @@ func (v ZMachine) Challenge(b io.Writer) error { return nil } -// ZMachineResult result returned by VM reservation -type ZMachineResult struct { - ID string `json:"id"` - IP string `json:"ip"` - PlanetaryIP string `json:"planetary_ip"` - MyceliumIP string `json:"mycelium_ip"` - ConsoleURL string `json:"console_url"` +// MachineCapacity structure +type MachineCapacity struct { + CPU uint8 `json:"cpu"` + Memory gridtypes.Unit `json:"memory"` } -func (r *ZMachineResult) UnmarshalJSON(data []byte) error { - var deprecated struct { - ID string `json:"id"` - IP string `json:"ip"` - YggIP string `json:"ygg_ip"` - PlanetaryIP string `json:"planetary_ip"` - MyceliumIP string `json:"mycelium_ip"` - ConsoleURL string `json:"console_url"` +func (c *MachineCapacity) String() string { + return fmt.Sprintf("cpu(%d)+mem(%d)", c.CPU, c.Memory) +} + +// Challenge builder +func (c *MachineCapacity) Challenge(w io.Writer) error { + if _, err := fmt.Fprintf(w, "%d", c.CPU); err != nil { + return err } - if err := json.Unmarshal(data, &deprecated); err != nil { + if _, err := fmt.Fprintf(w, "%d", c.Memory); err != nil { + return err + } + + return nil +} + +// MachineMount structure +type MachineMount struct { + // Name is name of a zmount. The name must be a valid zmount + // in the same deployment as the zmachine + Name gridtypes.Name `json:"name"` + // Mountpoint inside the container. Not used if the zmachine + // is running in a vm mode. + Mountpoint string `json:"mountpoint"` +} + +// Challenge builder +func (m *MachineMount) Challenge(w io.Writer) error { + if _, err := fmt.Fprintf(w, "%s", m.Name); err != nil { return err } - r.ID = deprecated.ID - r.IP = deprecated.IP - r.PlanetaryIP = deprecated.PlanetaryIP - if deprecated.YggIP != "" { - r.PlanetaryIP = deprecated.YggIP + if _, err := fmt.Fprintf(w, "%s", m.Mountpoint); err != nil { + return err } - r.MyceliumIP = deprecated.MyceliumIP - r.ConsoleURL = deprecated.ConsoleURL return nil } + +// GPU ID +// Used by a VM, a GPU id is in the format // +// This can be queried either from the node features on the chain +// or listed via the node rmb API. +// example of a valid gpu definition `0000:28:00.0/1002/731f“ +type GPU string + +func (g GPU) Parts() (slot, vendor, device string, err error) { + parts := strings.Split(string(g), "/") + if len(parts) != 3 { + err = fmt.Errorf("invalid GPU id format '%s'", g) + return + } + + return parts[0], parts[1], parts[2], nil +} + +// ZMachineLightResult result returned by VM reservation +type ZMachineLightResult struct { + ID string `json:"id"` + IP string `json:"ip"` + MyceliumIP string `json:"mycelium_ip"` +} diff --git a/pkg/gridtypes/zos/zmachine_test.go b/pkg/gridtypes/zos/zmachine_test.go index 9737bef1e..c85108766 100644 --- a/pkg/gridtypes/zos/zmachine_test.go +++ b/pkg/gridtypes/zos/zmachine_test.go @@ -1,7 +1,6 @@ package zos import ( - "encoding/json" "testing" "github.com/stretchr/testify/require" @@ -12,12 +11,12 @@ func TestZMachineSRU(t *testing.T) { type Case struct { Expected gridtypes.Unit - VM ZMachine + VM ZMachineLight } cases := []Case{ { Expected: 2 * gridtypes.Gigabyte, - VM: ZMachine{ + VM: ZMachineLight{ ComputeCapacity: MachineCapacity{ CPU: 1, @@ -27,7 +26,7 @@ func TestZMachineSRU(t *testing.T) { }, { Expected: 500 * gridtypes.Megabyte, - VM: ZMachine{ + VM: ZMachineLight{ ComputeCapacity: MachineCapacity{ CPU: 1, @@ -37,7 +36,7 @@ func TestZMachineSRU(t *testing.T) { }, { Expected: 2 * gridtypes.Gigabyte, - VM: ZMachine{ + VM: ZMachineLight{ ComputeCapacity: MachineCapacity{ CPU: 2, @@ -47,7 +46,7 @@ func TestZMachineSRU(t *testing.T) { }, { Expected: 2 * gridtypes.Gigabyte, - VM: ZMachine{ + VM: ZMachineLight{ ComputeCapacity: MachineCapacity{ CPU: 3, @@ -57,7 +56,7 @@ func TestZMachineSRU(t *testing.T) { }, { Expected: 2 * gridtypes.Gigabyte, - VM: ZMachine{ + VM: ZMachineLight{ ComputeCapacity: MachineCapacity{ CPU: 4, @@ -67,7 +66,7 @@ func TestZMachineSRU(t *testing.T) { }, { Expected: 500 * gridtypes.Megabyte, - VM: ZMachine{ + VM: ZMachineLight{ ComputeCapacity: MachineCapacity{ CPU: 1, @@ -77,7 +76,7 @@ func TestZMachineSRU(t *testing.T) { }, { Expected: 500 * gridtypes.Megabyte, - VM: ZMachine{ + VM: ZMachineLight{ ComputeCapacity: MachineCapacity{ CPU: 1, @@ -95,24 +94,3 @@ func TestZMachineSRU(t *testing.T) { }) } } - -func TestResultDeprecated(t *testing.T) { - raw := ` { - "id": "192-74881-testing2", - "ip": "10.20.2.2", - "ygg_ip": "32b:8310:9b03:5529:ff0f:37cd:de80:b322", - "console_url": "10.20.2.0:20002" - }` - - var result ZMachineResult - - err := json.Unmarshal([]byte(raw), &result) - require.NoError(t, err) - - require.EqualValues(t, ZMachineResult{ - ID: "192-74881-testing2", - IP: "10.20.2.2", - PlanetaryIP: "32b:8310:9b03:5529:ff0f:37cd:de80:b322", - ConsoleURL: "10.20.2.0:20002", - }, result) -} diff --git a/pkg/kernel/kernel.go b/pkg/kernel/kernel.go index 8ee9cd2f9..9dc8ba451 100644 --- a/pkg/kernel/kernel.go +++ b/pkg/kernel/kernel.go @@ -13,6 +13,8 @@ const ( // Debug means zos is running in debug mode // applications can handle this flag differently Debug = "zos-debug" + // Light means the node running in light mode + Light = "light" // VirtualMachine forces zos to think it's running // on a virtual machine. used mainly for development VirtualMachine = "zos-debug-vm" diff --git a/pkg/network/bootstrap/bootstrap.go b/pkg/netlight/bootstrap/bootstrap.go similarity index 97% rename from pkg/network/bootstrap/bootstrap.go rename to pkg/netlight/bootstrap/bootstrap.go index d822d9704..e6aa7d51f 100644 --- a/pkg/network/bootstrap/bootstrap.go +++ b/pkg/netlight/bootstrap/bootstrap.go @@ -12,14 +12,14 @@ import ( "sync" "time" - "github.com/threefoldtech/zos/pkg/network/dhcp" - "github.com/threefoldtech/zos/pkg/network/namespace" - "github.com/threefoldtech/zos/pkg/network/options" + "github.com/threefoldtech/zos/pkg/netlight/dhcp" + "github.com/threefoldtech/zos/pkg/netlight/namespace" + "github.com/threefoldtech/zos/pkg/netlight/options" "github.com/containernetworking/plugins/pkg/ns" "github.com/pkg/errors" "github.com/rs/zerolog/log" - "github.com/threefoldtech/zos/pkg/network/ifaceutil" + "github.com/threefoldtech/zos/pkg/netlight/ifaceutil" "github.com/vishvananda/netlink" ) diff --git a/pkg/network/bootstrap/bootstrap_test.go b/pkg/netlight/bootstrap/bootstrap_test.go similarity index 100% rename from pkg/network/bootstrap/bootstrap_test.go rename to pkg/netlight/bootstrap/bootstrap_test.go diff --git a/pkg/network/bootstrap/bridge.go b/pkg/netlight/bootstrap/bridge.go similarity index 91% rename from pkg/network/bootstrap/bridge.go rename to pkg/netlight/bootstrap/bridge.go index 249c7893a..50bec4fa1 100644 --- a/pkg/network/bootstrap/bridge.go +++ b/pkg/netlight/bootstrap/bridge.go @@ -3,13 +3,13 @@ package bootstrap import ( "fmt" - "github.com/threefoldtech/zos/pkg/network/ifaceutil" - "github.com/threefoldtech/zos/pkg/network/options" - "github.com/threefoldtech/zos/pkg/network/types" + "github.com/threefoldtech/zos/pkg/netlight/ifaceutil" + "github.com/threefoldtech/zos/pkg/netlight/options" + "github.com/threefoldtech/zos/pkg/netlight/types" "github.com/pkg/errors" "github.com/rs/zerolog/log" - "github.com/threefoldtech/zos/pkg/network/bridge" + "github.com/threefoldtech/zos/pkg/netlight/bridge" "github.com/vishvananda/netlink" ) diff --git a/pkg/network/bridge/bridge.go b/pkg/netlight/bridge/bridge.go similarity index 97% rename from pkg/network/bridge/bridge.go rename to pkg/netlight/bridge/bridge.go index e7a3230f8..e939292f2 100644 --- a/pkg/network/bridge/bridge.go +++ b/pkg/netlight/bridge/bridge.go @@ -6,8 +6,8 @@ import ( "github.com/pkg/errors" "github.com/rs/zerolog/log" - "github.com/threefoldtech/zos/pkg/network/ifaceutil" - "github.com/threefoldtech/zos/pkg/network/options" + "github.com/threefoldtech/zos/pkg/netlight/ifaceutil" + "github.com/threefoldtech/zos/pkg/netlight/options" "github.com/vishvananda/netlink" ) diff --git a/pkg/network/bridge/bridge_test.go b/pkg/netlight/bridge/bridge_test.go similarity index 100% rename from pkg/network/bridge/bridge_test.go rename to pkg/netlight/bridge/bridge_test.go diff --git a/pkg/network/dhcp/dhcp.go b/pkg/netlight/dhcp/dhcp.go similarity index 100% rename from pkg/network/dhcp/dhcp.go rename to pkg/netlight/dhcp/dhcp.go diff --git a/pkg/network/dhcp/service.go b/pkg/netlight/dhcp/service.go similarity index 100% rename from pkg/network/dhcp/service.go rename to pkg/netlight/dhcp/service.go diff --git a/pkg/network/ifaceutil/interface.go b/pkg/netlight/ifaceutil/interface.go similarity index 100% rename from pkg/network/ifaceutil/interface.go rename to pkg/netlight/ifaceutil/interface.go diff --git a/pkg/network/ifaceutil/ip.go b/pkg/netlight/ifaceutil/ip.go similarity index 100% rename from pkg/network/ifaceutil/ip.go rename to pkg/netlight/ifaceutil/ip.go diff --git a/pkg/network/ifaceutil/ip_test.go b/pkg/netlight/ifaceutil/ip_test.go similarity index 100% rename from pkg/network/ifaceutil/ip_test.go rename to pkg/netlight/ifaceutil/ip_test.go diff --git a/pkg/network/ifaceutil/mac.go b/pkg/netlight/ifaceutil/mac.go similarity index 83% rename from pkg/network/ifaceutil/mac.go rename to pkg/netlight/ifaceutil/mac.go index 19bc48624..a4f9ff7a5 100644 --- a/pkg/network/ifaceutil/mac.go +++ b/pkg/netlight/ifaceutil/mac.go @@ -4,6 +4,8 @@ import ( "bytes" "crypto/md5" "net" + + "github.com/decred/base58" ) // HardwareAddrFromInputBytes returns a deterministic hardware address @@ -37,6 +39,21 @@ outerLoop: } } +func DeviceNameFromInputBytes(input []byte) string { + // Unique generate a unique predetermined short name based + // on that ID and input value n. + // returns a max of 13 char str which is suitable + // to be used as devices and tap devices names. + + h := md5.Sum(input) + b := base58.Encode(h[:]) + if len(b) > 13 { + b = b[:13] + } + + return string(b) +} + func isHardwareAddrInValidRange(addr net.HardwareAddr) bool { // possible range 1 if bytes.Compare(addr[hwMacAddress-len(macPUR1):], macPUR1[:]) <= 0 { diff --git a/pkg/network/ifaceutil/mac_test.go b/pkg/netlight/ifaceutil/mac_test.go similarity index 100% rename from pkg/network/ifaceutil/mac_test.go rename to pkg/netlight/ifaceutil/mac_test.go diff --git a/pkg/network/ndmz/ipam.go b/pkg/netlight/ipam/ipam.go similarity index 84% rename from pkg/network/ndmz/ipam.go rename to pkg/netlight/ipam/ipam.go index 93a5a1471..38ed9e010 100644 --- a/pkg/network/ndmz/ipam.go +++ b/pkg/netlight/ipam/ipam.go @@ -1,4 +1,4 @@ -package ndmz +package ipam import ( "net" @@ -9,9 +9,9 @@ import ( "github.com/rs/zerolog/log" ) -// allocateIPv4 allocates a unique IPv4 for the entity defines by the given id (for example container id, or a vm). +// AllocateIPv4 allocates a unique IPv4 for the entity defines by the given id (for example container id, or a vm). // in the network with netID, and NetResource. -func allocateIPv4(networkID, leaseDir string) (*net.IPNet, error) { +func AllocateIPv4(networkID, leaseDir string) (*net.IPNet, error) { store, err := disk.New("ndmz", leaseDir) if err != nil { return nil, err @@ -20,7 +20,7 @@ func allocateIPv4(networkID, leaseDir string) (*net.IPNet, error) { defer store.Close() r := allocator.Range{ - RangeStart: net.ParseIP("100.127.0.2"), + RangeStart: net.ParseIP("100.127.0.3"), RangeEnd: net.ParseIP("100.127.255.254"), Subnet: types.IPNet(net.IPNet{ IP: net.ParseIP("100.127.0.0"), @@ -64,9 +64,7 @@ func allocateIPv4(networkID, leaseDir string) (*net.IPNet, error) { return &ipConfig.Address, nil } -// allocateIPv4 allocates a unique IPv4 for the entity defines by the given id (for example container id, or a vm). -// in the network with netID, and NetResource. -func deAllocateIPv4(networkID, leaseDir string) error { +func DeAllocateIPv4(networkID, leaseDir string) error { store, err := disk.New("ndmz", leaseDir) if err != nil { return err diff --git a/pkg/network/iperf/iperf.go b/pkg/netlight/iperf/iperf.go similarity index 100% rename from pkg/network/iperf/iperf.go rename to pkg/netlight/iperf/iperf.go diff --git a/pkg/network/macvlan/macvlan.go b/pkg/netlight/macvlan/macvlan.go similarity index 98% rename from pkg/network/macvlan/macvlan.go rename to pkg/netlight/macvlan/macvlan.go index 6529179de..598244bd2 100644 --- a/pkg/network/macvlan/macvlan.go +++ b/pkg/netlight/macvlan/macvlan.go @@ -7,7 +7,7 @@ import ( "os" "github.com/rs/zerolog/log" - "github.com/threefoldtech/zos/pkg/network/options" + "github.com/threefoldtech/zos/pkg/netlight/options" "github.com/containernetworking/plugins/pkg/ip" "github.com/containernetworking/plugins/pkg/ns" diff --git a/pkg/network/macvtap/tap.go b/pkg/netlight/macvtap/tap.go similarity index 97% rename from pkg/network/macvtap/tap.go rename to pkg/netlight/macvtap/tap.go index 730a13493..94cc6d7e1 100644 --- a/pkg/network/macvtap/tap.go +++ b/pkg/netlight/macvtap/tap.go @@ -5,7 +5,7 @@ import ( "net" "github.com/pkg/errors" - "github.com/threefoldtech/zos/pkg/network/options" + "github.com/threefoldtech/zos/pkg/netlight/options" "github.com/vishvananda/netlink" ) diff --git a/pkg/network/namespace/monitor.go b/pkg/netlight/namespace/monitor.go similarity index 100% rename from pkg/network/namespace/monitor.go rename to pkg/netlight/namespace/monitor.go diff --git a/pkg/network/namespace/namespace.go b/pkg/netlight/namespace/namespace.go similarity index 100% rename from pkg/network/namespace/namespace.go rename to pkg/netlight/namespace/namespace.go diff --git a/pkg/network/namespace/namespace_test.go b/pkg/netlight/namespace/namespace_test.go similarity index 100% rename from pkg/network/namespace/namespace_test.go rename to pkg/netlight/namespace/namespace_test.go diff --git a/pkg/netlight/network.go b/pkg/netlight/network.go new file mode 100644 index 000000000..9043788d7 --- /dev/null +++ b/pkg/netlight/network.go @@ -0,0 +1,378 @@ +package netlight + +import ( + "bytes" + "context" + "fmt" + "net" + "os" + "path/filepath" + "slices" + "time" + + "github.com/cenkalti/backoff" + "github.com/containernetworking/plugins/pkg/ns" + "github.com/pkg/errors" + "github.com/rs/zerolog/log" + "github.com/threefoldtech/zos/pkg" + "github.com/threefoldtech/zos/pkg/cache" + "github.com/threefoldtech/zos/pkg/netlight/bridge" + "github.com/threefoldtech/zos/pkg/netlight/ifaceutil" + "github.com/threefoldtech/zos/pkg/netlight/ipam" + "github.com/threefoldtech/zos/pkg/netlight/macvlan" + "github.com/threefoldtech/zos/pkg/netlight/namespace" + "github.com/threefoldtech/zos/pkg/netlight/options" + "github.com/threefoldtech/zos/pkg/netlight/resource" + "github.com/vishvananda/netlink" +) + +const ( + NDMZBridge = "br-ndmz" + NDMZGw = "gw" + mib = 1024 * 1024 + ipamLeaseDir = "ndmz-lease" + DefaultBridge = "zos" +) + +var ( + NDMZGwIP = &net.IPNet{ + IP: net.ParseIP("100.127.0.1"), + Mask: net.CIDRMask(16, 32), + } +) + +type networker struct { + ipamLease string +} + +var _ pkg.NetworkerLight = (*networker)(nil) + +func NewNetworker() (pkg.NetworkerLight, error) { + vd, err := cache.VolatileDir("networkd", 50*mib) + if err != nil && !os.IsExist(err) { + return nil, fmt.Errorf("failed to create networkd cache directory: %w", err) + } + + ipamLease := filepath.Join(vd, ipamLeaseDir) + return &networker{ipamLease: ipamLease}, nil +} + +func (n *networker) Create(name string, privateNet net.IPNet, seed []byte) error { + b, err := bridge.Get(NDMZBridge) + if err != nil { + return err + } + ip, err := ipam.AllocateIPv4(name, n.ipamLease) + if err != nil { + return err + } + + _, err = resource.Create(name, b, ip, NDMZGwIP, &privateNet, seed) + return err +} + +func (n *networker) Delete(name string) error { + if err := ipam.DeAllocateIPv4(name, n.ipamLease); err != nil { + return err + } + + return resource.Delete(name) + +} + +func (n *networker) AttachPrivate(name, id string, vmIp net.IP) (device pkg.TapDevice, err error) { + resource, err := resource.Get(name) + if err != nil { + return + } + return resource.AttachPrivate(id, vmIp) +} + +func (n *networker) AttachMycelium(name, id string, seed []byte) (device pkg.TapDevice, err error) { + resource, err := resource.Get(name) + if err != nil { + return + } + return resource.AttachMycelium(id, seed) +} + +// detach everything for this id +func (n *networker) Detach(id string) error { + // delete all tap devices for both mycelium and priv (if exists) + deviceName := ifaceutil.DeviceNameFromInputBytes([]byte(id)) + myName := fmt.Sprintf("m-%s", deviceName) + + if err := ifaceutil.Delete(myName, nil); err != nil { + return err + } + + tapName := fmt.Sprintf("b-%s", deviceName) + + return ifaceutil.Delete(tapName, nil) +} + +func (n *networker) AttachZDB(id string) (string, error) { + name := ifaceutil.DeviceNameFromInputBytes([]byte(id)) + nsName := fmt.Sprintf("n%s", name) + + ns, err := namespace.GetByName(nsName) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return "", err + } + if ns == nil { + ns, err = namespace.Create(nsName) + if err != nil { + return "", err + } + } + + r, err := resource.Get("dmz") + if err != nil { + return "", err + } + + return nsName, r.AttachMyceliumZDB(id, ns) +} + +func (n *networker) ZDBIPs(zdbNamespace string) ([]net.IP, error) { + ips := make([]net.IP, 0) + + netNs, err := namespace.GetByName(zdbNamespace) + if err != nil && errors.Is(err, os.ErrNotExist) { + return nil, err + } + err = netNs.Do(func(_ ns.NetNS) error { + links, err := netlink.LinkList() + if err != nil { + return err + } + for _, link := range links { + addrs, err := netlink.AddrList(link, netlink.FAMILY_ALL) + if err != nil { + return err + } + for _, addr := range addrs { + ips = append(ips, addr.IP) + } + } + return nil + }) + if err != nil { + return nil, err + } + + return ips, nil +} + +func (n *networker) Ready() error { + return nil +} + +func (n *networker) Namespace(id string) string { + return fmt.Sprintf("n-%s", id) +} + +func (n *networker) ZOSAddresses(ctx context.Context) <-chan pkg.NetlinkAddresses { + var index int + _ = backoff.Retry(func() error { + link, err := netlink.LinkByName(DefaultBridge) + if err != nil { + log.Error().Err(err).Msg("can't get defaut bridge") + return err + } + index = link.Attrs().Index + return nil + }, backoff.NewConstantBackOff(2*time.Second)) + + get := func() pkg.NetlinkAddresses { + var result pkg.NetlinkAddresses + link, err := netlink.LinkByName(DefaultBridge) + if err != nil { + log.Error().Err(err).Msgf("could not find the '%s' bridge", DefaultBridge) + return nil + } + values, err := netlink.AddrList(link, netlink.FAMILY_ALL) + if err != nil { + log.Error().Err(err).Msgf("could not list the '%s' bridge ips", DefaultBridge) + return nil + } + for _, value := range values { + result = append(result, *value.IPNet) + } + + slices.SortFunc(result, func(a, b net.IPNet) int { + return bytes.Compare(a.IP, b.IP) + }) + + return result + } + + updateChan := make(chan netlink.AddrUpdate) + if err := netlink.AddrSubscribe(updateChan, ctx.Done()); err != nil { + log.Error().Err(err).Msgf("could not subscribe to addresses updates") + return nil + } + + ch := make(chan pkg.NetlinkAddresses) + var current pkg.NetlinkAddresses + go func() { + defer close(ch) + + for { + select { + case <-ctx.Done(): + return + case update := <-updateChan: + if update.LinkIndex != index || !update.NewAddr { + continue + } + new := get() + if slices.CompareFunc(current, new, func(a, b net.IPNet) int { + return bytes.Compare(a.IP, b.IP) + }) == 0 { + // if the 2 sets of IPs are identitcal, we don't + // trigger the event + continue + } + current = new + ch <- new + } + } + }() + + return ch +} + +func (n *networker) Interfaces(iface string, netns string) (pkg.Interfaces, error) { + getter := func(iface string) ([]netlink.Link, error) { + if iface != "" { + l, err := netlink.LinkByName(iface) + if err != nil { + return nil, errors.Wrapf(err, "failed to get interface %s", iface) + } + return []netlink.Link{l}, nil + } + + all, err := netlink.LinkList() + if err != nil { + return nil, err + } + filtered := all[:0] + for _, l := range all { + name := l.Attrs().Name + + if name == "lo" || + (l.Type() != "device" && name != "zos") { + + continue + } + + filtered = append(filtered, l) + } + + return filtered, nil + } + + interfaces := make(map[string]pkg.Interface) + f := func(_ ns.NetNS) error { + links, err := getter(iface) + if err != nil { + return errors.Wrapf(err, "failed to get interfaces (query: '%s')", iface) + } + + for _, link := range links { + + addrs, err := netlink.AddrList(link, netlink.FAMILY_ALL) + if err != nil { + return errors.Wrapf(err, "failed to list addresses of interfaces %s", iface) + } + ips := make([]net.IPNet, 0, len(addrs)) + for _, addr := range addrs { + ip := addr.IP + if ip6 := ip.To16(); ip6 != nil { + // ipv6 + if !ip6.IsGlobalUnicast() || ifaceutil.IsULA(ip6) { + // skip if not global or is ula address + continue + } + } + + ips = append(ips, *addr.IPNet) + } + + interfaces[link.Attrs().Name] = pkg.Interface{ + Name: link.Attrs().Name, + Mac: link.Attrs().HardwareAddr.String(), + IPs: ips, + } + } + + return nil + } + + if netns != "" { + netNS, err := namespace.GetByName(netns) + if err != nil { + return pkg.Interfaces{}, errors.Wrapf(err, "failed to get network namespace %s", netns) + } + defer netNS.Close() + + if err := netNS.Do(f); err != nil { + return pkg.Interfaces{}, err + } + } else { + if err := f(nil); err != nil { + return pkg.Interfaces{}, err + } + } + + return pkg.Interfaces{Interfaces: interfaces}, nil +} + +func CreateNDMZBridge() (*netlink.Bridge, error) { + return createNDMZBridge(NDMZBridge, NDMZGw) +} + +func createNDMZBridge(name string, gw string) (*netlink.Bridge, error) { + if !bridge.Exists(name) { + if _, err := bridge.New(name); err != nil { + return nil, errors.Wrapf(err, "couldn't create bridge %s", name) + } + } + + if err := options.Set(name, options.IPv6Disable(true)); err != nil { + return nil, errors.Wrapf(err, "failed to disable ip6 on bridge %s", name) + } + + link, err := netlink.LinkByName(name) + if err != nil { + return nil, fmt.Errorf("failed to get ndmz bridge: %w", err) + } + + if link.Type() != "bridge" { + return nil, fmt.Errorf("ndmz is not a bridge") + } + + if !ifaceutil.Exists(gw, nil) { + gwLink, err := macvlan.Create(gw, name, nil) + if err != nil { + return nil, err + } + + err = netlink.AddrAdd(gwLink, &netlink.Addr{IPNet: NDMZGwIP}) + if err != nil && !os.IsExist(err) { + return nil, err + } + + if err := netlink.LinkSetUp(gwLink); err != nil { + return nil, err + } + + } + + if err := netlink.LinkSetUp(link); err != nil { + return nil, err + } + + return link.(*netlink.Bridge), nil +} diff --git a/pkg/network/nft/nft.go b/pkg/netlight/nft/nft.go similarity index 100% rename from pkg/network/nft/nft.go rename to pkg/netlight/nft/nft.go diff --git a/pkg/network/options/global.go b/pkg/netlight/options/global.go similarity index 100% rename from pkg/network/options/global.go rename to pkg/netlight/options/global.go diff --git a/pkg/network/options/interface.go b/pkg/netlight/options/interface.go similarity index 100% rename from pkg/network/options/interface.go rename to pkg/netlight/options/interface.go diff --git a/pkg/network/options/options.go b/pkg/netlight/options/options.go similarity index 100% rename from pkg/network/options/options.go rename to pkg/netlight/options/options.go diff --git a/pkg/network/public/persist.go b/pkg/netlight/public/persist.go similarity index 100% rename from pkg/network/public/persist.go rename to pkg/netlight/public/persist.go diff --git a/pkg/network/public/public.go b/pkg/netlight/public/public.go similarity index 97% rename from pkg/network/public/public.go rename to pkg/netlight/public/public.go index cf78bd7c1..6233e6c70 100644 --- a/pkg/network/public/public.go +++ b/pkg/netlight/public/public.go @@ -13,14 +13,14 @@ import ( "github.com/threefoldtech/zos/pkg" "github.com/threefoldtech/zos/pkg/environment" "github.com/threefoldtech/zos/pkg/gridtypes" - "github.com/threefoldtech/zos/pkg/network/bootstrap" - "github.com/threefoldtech/zos/pkg/network/bridge" - "github.com/threefoldtech/zos/pkg/network/ifaceutil" - "github.com/threefoldtech/zos/pkg/network/iperf" - "github.com/threefoldtech/zos/pkg/network/macvlan" - "github.com/threefoldtech/zos/pkg/network/namespace" - "github.com/threefoldtech/zos/pkg/network/options" - "github.com/threefoldtech/zos/pkg/network/types" + "github.com/threefoldtech/zos/pkg/netlight/bootstrap" + "github.com/threefoldtech/zos/pkg/netlight/bridge" + "github.com/threefoldtech/zos/pkg/netlight/ifaceutil" + "github.com/threefoldtech/zos/pkg/netlight/iperf" + "github.com/threefoldtech/zos/pkg/netlight/macvlan" + "github.com/threefoldtech/zos/pkg/netlight/namespace" + "github.com/threefoldtech/zos/pkg/netlight/options" + "github.com/threefoldtech/zos/pkg/netlight/types" "github.com/threefoldtech/zos/pkg/zinit" "github.com/vishvananda/netlink" ) diff --git a/pkg/network/public/public_test.go b/pkg/netlight/public/public_test.go similarity index 85% rename from pkg/network/public/public_test.go rename to pkg/netlight/public/public_test.go index a02ca570e..a2779e09e 100644 --- a/pkg/network/public/public_test.go +++ b/pkg/netlight/public/public_test.go @@ -7,8 +7,8 @@ import ( "github.com/stretchr/testify/require" "github.com/threefoldtech/zos/pkg" "github.com/threefoldtech/zos/pkg/gridtypes" - "github.com/threefoldtech/zos/pkg/network/namespace" - "github.com/threefoldtech/zos/pkg/network/types" + "github.com/threefoldtech/zos/pkg/netlight/namespace" + "github.com/threefoldtech/zos/pkg/netlight/types" ) func TestCreatePublicNS(t *testing.T) { diff --git a/pkg/netlight/resource/mycelium.go b/pkg/netlight/resource/mycelium.go new file mode 100644 index 000000000..10d8f8b11 --- /dev/null +++ b/pkg/netlight/resource/mycelium.go @@ -0,0 +1,252 @@ +package resource + +import ( + "encoding/json" + "fmt" + "net" + "os" + "os/exec" + "path/filepath" + "slices" + "strings" + "time" + + "github.com/containernetworking/plugins/pkg/ns" + "github.com/pkg/errors" + "github.com/threefoldtech/zos/pkg/netlight/resource/peers" + "github.com/threefoldtech/zos/pkg/zinit" +) + +const ( + myceliumBin = "mycelium" + myceliumSeedDir = "/tmp/network/mycelium" + + myceliumSeedLen = 6 +) + +var ( + myceliumIpBase = []byte{ + 0xff, 0x0f, + } + + invalidMyceliumSeeds = [][]byte{ + {0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + {0xff, 0xff, 0xff, 0xff, 0xff, 0xff}, + } +) + +type MyceliumInspection struct { + PublicKey string `json:"publicKey"` + Address net.IP `json:"address"` +} + +func inspectMycelium(seed []byte) (inspection MyceliumInspection, err error) { + // we check if the file exists before we do inspect because mycelium will create a random seed + // file if file does not exist + tmp, err := os.CreateTemp("", "my-inspect") + if err != nil { + return inspection, err + } + + defer os.RemoveAll(tmp.Name()) + + if _, err := tmp.Write(seed); err != nil { + return inspection, fmt.Errorf("failed to write seed: %w", err) + } + + tmp.Close() + + return inspectMyceliumFile(tmp.Name()) +} + +func inspectMyceliumFile(path string) (inspection MyceliumInspection, err error) { + output, err := exec.Command(myceliumBin, "inspect", "--json", "--key-file", path).Output() + if err != nil { + return inspection, fmt.Errorf("failed to inspect mycelium ip: %w", err) + } + + if err := json.Unmarshal(output, &inspection); err != nil { + return inspection, errors.Wrap(err, "failed to load mycelium information from key") + } + + return +} + +// IP return the address in the 400::/7 subnet allocated by mycelium +func (m *MyceliumInspection) IP() net.IP { + return net.IP(m.Address) +} + +// Subnet return the 400::/64 subnet allocated by mycelium +func (m *MyceliumInspection) Subnet() (subnet net.IPNet, err error) { + ipv6 := m.Address.To16() + if ipv6 == nil { + return subnet, errors.Errorf("invalid mycelium ip") + } + + ip := make(net.IP, net.IPv6len) + copy(ip[0:8], ipv6[0:8]) + + subnet = net.IPNet{ + IP: ip, + Mask: net.CIDRMask(64, 128), + } + + return +} + +// Gateway derive the gateway IP from the mycelium IP in the /64 range. +func (m *MyceliumInspection) Gateway() (gw net.IPNet, err error) { + subnet, err := m.Subnet() + if err != nil { + return gw, err + } + + ip := subnet.IP + ip[net.IPv6len-1] = 1 + + gw = net.IPNet{ + IP: ip, + Mask: net.CIDRMask(64, 128), + } + + return +} + +func (m *MyceliumInspection) IPFor(seed []byte) (ip net.IPNet, gw net.IPNet, err error) { + if len(seed) != myceliumSeedLen { + return ip, gw, fmt.Errorf("invalid seed length") + } + + if slices.ContainsFunc(invalidMyceliumSeeds, func(b []byte) bool { + return slices.Equal(seed, b) + }) { + return ip, gw, fmt.Errorf("invalid seed") + } + + // first find the base subnet. + gw, err = m.Gateway() + if err != nil { + return ip, gw, err + } + + ip = net.IPNet{ + IP: slices.Clone(gw.IP), + Mask: slices.Clone(gw.Mask), + } + + // the subnet already have the /64 part of the network (that's 8 bytes) + // we then add a fixed 2 bytes this will avoid reusing the same gw or + // the device ip + copy(ip.IP[8:10], myceliumIpBase) + // then finally we use the 6 bytes seed to build the rest of the IP + copy(ip.IP[10:16], seed) + + return +} + +func setupMycelium(netNS ns.NetNS, mycelium string, seed []byte) error { + if err := os.MkdirAll(myceliumSeedDir, 0744); err != nil { + return fmt.Errorf("failed to create seed temp location: %w", err) + } + + inspect, err := inspectMycelium(seed) + if err != nil { + return err + } + + gw, err := inspect.Gateway() + if err != nil { + return err + } + + err = netNS.Do(func(nn ns.NetNS) error { + return setLinkAddr(mycelium, &gw) + }) + + if err != nil { + return err + } + + // - fetch peers + // - write seed file + // - create zinit config + // - monitor + + list, err := peers.PeersList() + if err != nil { + return err + } + + name := filepath.Base(netNS.Path()) + if err := os.WriteFile(filepath.Join(myceliumSeedDir, name), seed, 0444); err != nil { + return fmt.Errorf("failed to create seed file '%s': %w", name, err) + } + + return ensureMyceliumService(zinit.Default(), name, list) +} + +// Start creates a mycelium zinit service and starts it +func ensureMyceliumService(z *zinit.Client, namespace string, peers []string) error { + zinitService := fmt.Sprintf("mycelium-%s", namespace) + // service found. + // better if we just stop, forget and start over to make + // sure we using the right exec params + if _, err := z.Status(zinitService); err == nil { + // not here we need to stop it + if err := z.StopWait(5*time.Second, zinitService); err != nil && !errors.Is(err, zinit.ErrUnknownService) { + return errors.Wrap(err, "failed to stop mycelium service") + } + if err := z.Forget(zinitService); err != nil && !errors.Is(err, zinit.ErrUnknownService) { + return errors.Wrap(err, "failed to forget mycelium service") + } + } + + bin, err := exec.LookPath(myceliumBin) + if err != nil { + return err + } + + args := []string{ + "ip", "netns", "exec", namespace, + bin, + "--silent", + "--key-file", filepath.Join(myceliumSeedDir, namespace), + "--tun-name", "my0", + "--peers", + } + + args = append(args, peers...) + + err = zinit.AddService(zinitService, zinit.InitService{ + Exec: strings.Join(args, " "), + After: []string{ + "node-ready", + }, + }) + + if err != nil { + return err + } + + if err := z.Monitor(zinitService); err != nil && !errors.Is(err, zinit.ErrAlreadyMonitored) { + return err + } + + return z.StartWait(time.Second*20, zinitService) +} + +func destroyMycelium(netNS ns.NetNS, z *zinit.Client) error { + name := filepath.Base(netNS.Path()) + + zinitService := fmt.Sprintf("mycelium-%s", name) + + if err := z.StopWait(5*time.Second, zinitService); err != nil && !errors.Is(err, zinit.ErrUnknownService) { + return fmt.Errorf("failed to stop service %q: %w", zinitService, err) + } + if err := z.Forget(zinitService); err != nil && !errors.Is(err, zinit.ErrUnknownService) { + return fmt.Errorf("failed to forget service %q: %w", zinitService, err) + } + + return os.Remove(filepath.Join(myceliumSeedDir, name)) +} diff --git a/pkg/netlight/resource/nft/rules.nft b/pkg/netlight/resource/nft/rules.nft new file mode 100644 index 000000000..6742cbdcc --- /dev/null +++ b/pkg/netlight/resource/nft/rules.nft @@ -0,0 +1,9 @@ +add table nat; +flush table nat; + +table ip nat { + chain postrouting { + type nat hook postrouting priority srcnat; policy accept; + iifname "private" masquerade; + } +} diff --git a/pkg/network/mycelium/public_peerlist.go b/pkg/netlight/resource/peers/peerlist.go similarity index 96% rename from pkg/network/mycelium/public_peerlist.go rename to pkg/netlight/resource/peers/peerlist.go index 30a6cc171..c2f56dd99 100644 --- a/pkg/network/mycelium/public_peerlist.go +++ b/pkg/netlight/resource/peers/peerlist.go @@ -1,4 +1,4 @@ -package mycelium +package peers import ( "net" @@ -45,7 +45,7 @@ func IPV4Only(ip net.IP) bool { return ip.To4() != nil } -func fetchZosMyList() (Peers, error) { +func PeersList() (Peers, error) { cfg, err := environment.GetConfig() if err != nil { return nil, err diff --git a/pkg/netlight/resource/resource.go b/pkg/netlight/resource/resource.go new file mode 100644 index 000000000..a43488178 --- /dev/null +++ b/pkg/netlight/resource/resource.go @@ -0,0 +1,399 @@ +package resource + +import ( + "crypto/rand" + "embed" + "fmt" + "net" + "os" + "path/filepath" + + "github.com/containernetworking/plugins/pkg/ns" + "github.com/pkg/errors" + "github.com/threefoldtech/zos/pkg" + "github.com/threefoldtech/zos/pkg/netlight/bridge" + "github.com/threefoldtech/zos/pkg/netlight/ifaceutil" + "github.com/threefoldtech/zos/pkg/netlight/macvlan" + "github.com/threefoldtech/zos/pkg/netlight/macvtap" + "github.com/threefoldtech/zos/pkg/netlight/namespace" + "github.com/threefoldtech/zos/pkg/netlight/nft" + "github.com/threefoldtech/zos/pkg/netlight/options" + "github.com/threefoldtech/zos/pkg/zinit" + "github.com/vishvananda/netlink" +) + +const ( + infPublic = "public" + infPrivate = "private" + infMycelium = "mycelium" +) + +//go:embed nft/rules.nft +var nftRules embed.FS + +type Resource struct { + name string +} + +// Create creates a network resource (please check docs) +// name: is the name of the network resource. The Create function is idempotent which means if the same name is used the function +// will not recreate the resource. +// master: Normally the br-ndmz bridge, this is the resource "way out" to the public internet. A `public` interface is created and wired +// to the master bridge +// ndmzIP: the IP assigned to the `public` interface. +// ndmzGwIP: the gw Ip for the resource. Normally this is the ip assigned to the master bridge. +// privateNet: optional private network range +// seed: mycelium seed +func Create(name string, master *netlink.Bridge, ndmzIP *net.IPNet, ndmzGwIP *net.IPNet, privateNet *net.IPNet, seed []byte) (*Resource, error) { + privateNetBr := fmt.Sprintf("r%s", name) + myBr := fmt.Sprintf("m%s", name) + nsName := fmt.Sprintf("n%s", name) + + bridges := []string{myBr} + if privateNet != nil { + bridges = append(bridges, privateNetBr) + } + + for _, name := range bridges { + if !bridge.Exists(name) { + if _, err := bridge.New(name); err != nil { + return nil, fmt.Errorf("couldn't create bridge %s: %w", name, err) + } + } + } + + netNS, err := namespace.GetByName(nsName) + if err != nil { + netNS, err = namespace.Create(nsName) + if err != nil { + return nil, fmt.Errorf("failed to create namespace %s", err) + } + } + + defer netNS.Close() + + if privateNet != nil { + if privateNet.IP.To4() == nil { + return nil, fmt.Errorf("private ip range must be IPv4 address") + } + + if !ifaceutil.Exists(infPrivate, netNS) { + if _, err = macvlan.Create(infPrivate, privateNetBr, netNS); err != nil { + return nil, fmt.Errorf("failed to create private link: %w", err) + } + } + } + + // create public interface and attach it to ndmz bridge + if !ifaceutil.Exists(infPublic, netNS) { + if _, err = macvlan.Create(infPublic, master.Name, netNS); err != nil { + return nil, fmt.Errorf("failed to create public link: %w", err) + } + } + + if !ifaceutil.Exists(infMycelium, netNS) { + if _, err = macvlan.Create(infMycelium, myBr, netNS); err != nil { + return nil, err + } + } + + err = netNS.Do(func(_ ns.NetNS) error { + if err := ifaceutil.SetLoUp(); err != nil { + return fmt.Errorf("failed to set lo up for namespace '%s': %w", nsName, err) + } + + if err := setLinkAddr(infPublic, ndmzIP); err != nil { + return fmt.Errorf("couldn't set link addr for public interface in namespace %s: %w", nsName, err) + } + if err := options.SetIPv6Forwarding(true); err != nil { + return fmt.Errorf("failed to enable ipv6 forwarding in namespace %q: %w", nsName, err) + } + + if privateNet != nil { + privGwIp := privateNet.IP.To4() + // this is to take the first IP of the private range + // and use it as the gw address for the entire private range + privGwIp[net.IPv4len-1] = 1 + // this IP is then set on the private interface + privateNet.IP = privGwIp + if err := setLinkAddr(infPrivate, privateNet); err != nil { + return fmt.Errorf("couldn't set link addr for private interface in namespace %s: %w", nsName, err) + } + } + + // if err := setLinkAddr(infPrivate, ) + if err := netlink.RouteAdd(&netlink.Route{ + Gw: ndmzGwIP.IP, + }); err != nil && !os.IsExist(err) { + return fmt.Errorf("failed to add ndmz routing") + } + + return nil + }) + + if err != nil { + return nil, err + } + + rules, err := nftRules.Open("nft/rules.nft") + if err != nil { + return nil, fmt.Errorf("failed to load nft.rules file") + } + + if err := nft.Apply(rules, nsName); err != nil { + return nil, fmt.Errorf("failed to apply nft rules for namespace '%s': %w", name, err) + } + + return &Resource{name}, setupMycelium(netNS, infMycelium, seed) +} + +func Delete(name string) error { + nsName := fmt.Sprintf("n%s", name) + netNS, err := namespace.GetByName(nsName) + if errors.Is(err, os.ErrNotExist) { + return nil + } + + if err != nil { + return err + } + + if err := destroyMycelium(netNS, zinit.Default()); err != nil { + return err + } + + if err := namespace.Delete(netNS); err != nil { + return err + } + + privateNetBr := fmt.Sprintf("r%s", name) + myBr := fmt.Sprintf("m%s", name) + + if err := bridge.Delete(privateNetBr); err != nil { + return err + } + + return bridge.Delete(myBr) + +} + +func setLinkAddr(name string, ip *net.IPNet) error { + link, err := netlink.LinkByName(name) + if err != nil { + return fmt.Errorf("failed to set link address: %w", err) + } + + // if err := options.Set(name, options.IPv6Disable(false)); err != nil { + // return fmt.Errorf("failed to enable ip6 on interface %s: %w", name, err) + // } + + addr := netlink.Addr{ + IPNet: ip, + } + err = netlink.AddrAdd(link, &addr) + if err != nil && !os.IsExist(err) { + return fmt.Errorf("failed to add ip address to link: %w", err) + } + + return netlink.LinkSetUp(link) +} + +// Get return resource handler +func Get(name string) (*Resource, error) { + nsName := fmt.Sprintf("n%s", name) + + if namespace.Exists(nsName) { + return &Resource{name}, nil + } + + return nil, fmt.Errorf("resource not found: %s", name) +} + +var networkResourceNet = net.IPNet{ + IP: net.ParseIP("100.64.0.0"), + Mask: net.IPv4Mask(0xff, 0xff, 0, 0), +} + +func (r *Resource) AttachPrivate(id string, vmIp net.IP) (device pkg.TapDevice, err error) { + nsName := fmt.Sprintf("n%s", r.name) + netNs, err := namespace.GetByName(nsName) + if err != nil { + return + } + if vmIp[3] == 1 { + return device, fmt.Errorf("ip %s is reserved", vmIp.String()) + } + + var addrs []netlink.Addr + + if err = netNs.Do(func(_ ns.NetNS) error { + link, err := netlink.LinkByName(infPrivate) + if err != nil { + return err + } + addrs, err = netlink.AddrList(link, netlink.FAMILY_V4) + if err != nil { + return err + } + return nil + }); err != nil { + return + } + if len(addrs) != 1 { + return device, fmt.Errorf("expect addresses on private interface to be 1 got %d", len(addrs)) + } + gw := addrs[0].IPNet + if !gw.Contains(vmIp) { + return device, fmt.Errorf("ip not in range") + } + + deviceName := ifaceutil.DeviceNameFromInputBytes([]byte(id)) + tapName := fmt.Sprintf("b-%s", deviceName) + + privateNetBr := fmt.Sprintf("r%s", r.name) + hw := ifaceutil.HardwareAddrFromInputBytes([]byte(tapName)) + + ip := &net.IPNet{ + IP: vmIp, + Mask: gw.Mask, + } + _, getLinkErr := netlink.LinkByName(tapName) + if getLinkErr != nil { + mtap, err := macvtap.CreateMACvTap(tapName, privateNetBr, hw) + if err != nil { + return pkg.TapDevice{}, err + } + + if err = netlink.AddrAdd(mtap, &netlink.Addr{ + IPNet: ip, + }); err != nil { + return pkg.TapDevice{}, err + } + } + + routes := []pkg.Route{ + { + Gateway: gw.IP, + }, + { + Net: networkResourceNet, + Gateway: gw.IP, + }, + } + return pkg.TapDevice{ + Name: tapName, + Mac: hw, + IP: ip, + Routes: routes, + }, nil +} + +func (r *Resource) AttachMycelium(id string, seed []byte) (device pkg.TapDevice, err error) { + nsName := fmt.Sprintf("n%s", r.name) + netNS, err := namespace.GetByName(nsName) + if err != nil { + return + } + name := filepath.Base(netNS.Path()) + netSeed, err := os.ReadFile(filepath.Join(myceliumSeedDir, name)) + if err != nil { + return + } + + inspect, err := inspectMycelium(netSeed) + if err != nil { + return + } + + ip, gw, err := inspect.IPFor(seed) + if err != nil { + return + } + + deviceName := ifaceutil.DeviceNameFromInputBytes([]byte(id)) + tapName := fmt.Sprintf("m-%s", deviceName) + + myBr := fmt.Sprintf("m%s", r.name) + hw := ifaceutil.HardwareAddrFromInputBytes([]byte(tapName)) + + _, err = macvtap.CreateMACvTap(tapName, myBr, hw) + if err != nil { + return + } + + route := pkg.Route{ + + Net: net.IPNet{ + IP: net.ParseIP("400::"), + Mask: net.CIDRMask(7, 128), + }, + Gateway: gw.IP, + } + return pkg.TapDevice{ + Name: tapName, + IP: &ip, + Routes: []pkg.Route{route}, + Mac: hw, + }, nil +} + +func (r *Resource) AttachMyceliumZDB(id string, zdbNS ns.NetNS) (err error) { + nsName := fmt.Sprintf("n%s", r.name) + netNS, err := namespace.GetByName(nsName) + if err != nil { + return + } + name := filepath.Base(netNS.Path()) + netSeed, err := os.ReadFile(filepath.Join(myceliumSeedDir, name)) + if err != nil { + return + } + + inspect, err := inspectMycelium(netSeed) + if err != nil { + return + } + seed := make([]byte, 6) + + _, err = rand.Read(seed) + if err != nil { + return + } + + ip, gw, err := inspect.IPFor(seed) + if err != nil { + return + } + routes := []*netlink.Route{ + { + Dst: &net.IPNet{ + IP: net.ParseIP("400::"), + Mask: net.CIDRMask(7, 128), + }, + Gw: gw.IP, + }, + } + + deviceName := ifaceutil.DeviceNameFromInputBytes([]byte(id)) + macvlanName := fmt.Sprintf("m-%s", deviceName) + + hw := ifaceutil.HardwareAddrFromInputBytes([]byte(macvlanName)) + + link, err := macvlan.Create(macvlanName, "mdmz", zdbNS) + if err != nil { + return + } + + return macvlan.Install(link, hw, []*net.IPNet{&ip}, routes, zdbNS) +} + +func (r *Resource) Seed() (seed []byte, err error) { + nsName := fmt.Sprintf("n%s", r.name) + netNS, err := namespace.GetByName(nsName) + if err != nil { + return + } + name := filepath.Base(netNS.Path()) + seed, err = os.ReadFile(filepath.Join(myceliumSeedDir, name)) + return +} diff --git a/pkg/network/types/constant.go b/pkg/netlight/types/constant.go similarity index 100% rename from pkg/network/types/constant.go rename to pkg/netlight/types/constant.go diff --git a/pkg/network/types/types.go b/pkg/netlight/types/types.go similarity index 100% rename from pkg/network/types/types.go rename to pkg/netlight/types/types.go diff --git a/pkg/network/types/types_test.go b/pkg/netlight/types/types_test.go similarity index 100% rename from pkg/network/types/types_test.go rename to pkg/netlight/types/types_test.go diff --git a/pkg/network.go b/pkg/network.go index f373cd215..f3a9aa80d 100644 --- a/pkg/network.go +++ b/pkg/network.go @@ -29,12 +29,6 @@ type PlanetaryTap struct { Gateway net.IPNet } -type Interface struct { - Name string - IPs []net.IPNet - Mac string -} - type ExitDevice struct { // IsSingle is set to true if br-pub // is connected to zos bridge @@ -220,7 +214,7 @@ type Networker interface { // Network type type Network struct { - zos.Network + zos.NetworkLight NetID NetID `json:"net_id"` } @@ -286,9 +280,3 @@ type OptionPublicConfig struct { PublicConfig HasPublicConfig bool } - -// Interfaces struct to bypass zbus generation error -// where it generate a stub with map as interface instead of map -type Interfaces struct { - Interfaces map[string]Interface -} diff --git a/pkg/network/mycelium/config.go b/pkg/network/mycelium/config.go deleted file mode 100644 index e80898539..000000000 --- a/pkg/network/mycelium/config.go +++ /dev/null @@ -1,47 +0,0 @@ -package mycelium - -import ( - "context" - "crypto/ed25519" - - "github.com/oasisprotocol/curve25519-voi/primitives/x25519" - "github.com/pkg/errors" -) - -type NodeConfig struct { - KeyFile string - TunName string - Peers []string - privateKey x25519.PrivateKey -} - -func (n *NodeConfig) FindPeers(ctx context.Context, filter ...Filter) error { - // fetching a peer list goes as this - // - Always include the list of peers from - peers, err := fetchZosMyList() - if err != nil { - return errors.Wrap(err, "failed to get zos public peer list") - } - - peers, err = peers.Ups(filter...) - if err != nil { - return errors.Wrap(err, "failed to filter out peer list") - } - - n.Peers = peers - return nil -} - -// GenerateConfig creates a new mycelium configuration -func GenerateConfig(privateKey ed25519.PrivateKey) (cfg NodeConfig) { - cfg = NodeConfig{ - KeyFile: confPath, - TunName: tunName, - } - - if privateKey != nil { - cfg.privateKey = x25519.PrivateKey(x25519.EdPrivateKeyToX25519([]byte(privateKey))) - } - - return -} diff --git a/pkg/network/mycelium/mycelium.go b/pkg/network/mycelium/mycelium.go deleted file mode 100644 index 482922029..000000000 --- a/pkg/network/mycelium/mycelium.go +++ /dev/null @@ -1,268 +0,0 @@ -package mycelium - -import ( - "context" - "crypto/ed25519" - "crypto/sha512" - "encoding/json" - "fmt" - "net" - "os" - "os/exec" - "path/filepath" - "strings" - "time" - - "github.com/oasisprotocol/curve25519-voi/primitives/x25519" - "github.com/pkg/errors" - "github.com/rs/zerolog/log" - "github.com/threefoldtech/zos/pkg/network/namespace" - "github.com/threefoldtech/zos/pkg/zinit" -) - -const ( - tunName = "my0" - myceliumBin = "mycelium" - zinitService = "mycelium" - confPath = "/tmp/mycelium_priv_key.bin" - MyListenTCP = 9651 -) - -// MyceliumServer represent a mycelium server -type MyceliumServer struct { - cfg *NodeConfig - ns string -} - -type MyceliumInspection struct { - PublicKey string `json:"publicKey"` - Address net.IP `json:"address"` -} - -// NewMyceliumServer create a new mycelium Server -func NewMyceliumServer(cfg *NodeConfig) *MyceliumServer { - return &MyceliumServer{ - cfg: cfg, - } -} - -func (s *MyceliumServer) Restart(z *zinit.Client) error { - return z.Kill(zinitService, zinit.SIGTERM) -} - -func (s *MyceliumServer) Reload(z *zinit.Client) error { - return z.Kill(zinitService, zinit.SIGHUP) -} - -// Start creates a mycelium zinit service and starts it -func (s *MyceliumServer) Ensure(z *zinit.Client, ns string) error { - if !namespace.Exists(ns) { - return fmt.Errorf("invalid namespace '%s'", ns) - } - s.ns = ns - - if err := writeKey(confPath, s.cfg.privateKey); err != nil { - return err - } - - // service found. - // better if we just stop, forget and start over to make - // sure we using the right exec params - if _, err := z.Status(zinitService); err == nil { - // not here we need to stop it - if err := z.StopWait(5*time.Second, zinitService); err != nil && !errors.Is(err, zinit.ErrUnknownService) { - return errors.Wrap(err, "failed to stop mycelium service") - } - if err := z.Forget(zinitService); err != nil && !errors.Is(err, zinit.ErrUnknownService) { - return errors.Wrap(err, "failed to forget mycelium service") - } - } - - bin, err := exec.LookPath(myceliumBin) - if err != nil { - return err - } - - args := []string{ - "ip", "netns", "exec", ns, - bin, - "--key-file", confPath, - "--tun-name", s.cfg.TunName, - "--peers", - } - - args = append(args, s.cfg.Peers...) - - err = zinit.AddService(zinitService, zinit.InitService{ - Exec: strings.Join(args, " "), - After: []string{ - "node-ready", - }, - }) - if err != nil { - return err - } - - if err := z.Monitor(zinitService); err != nil && !errors.Is(err, zinit.ErrAlreadyMonitored) { - return err - } - - return z.StartWait(time.Second*20, zinitService) -} - -func EnsureMycelium(ctx context.Context, privateKey ed25519.PrivateKey, ns MyceliumNamespace) (*MyceliumServer, error) { - // Filter out all the nodes from the same - // segment so we do not just connect locally - ips, err := ns.GetIPs() // returns ipv6 only - if err != nil { - return nil, errors.Wrap(err, "failed to get ndmz public ipv6") - } - - var ranges Ranges - for _, ip := range ips { - if ip.IP.IsGlobalUnicast() { - ranges = append(ranges, ip) - } - } - - log.Info().Msgf("filtering out peers from ranges: %s", ranges) - filter := Exclude(ranges) - z := zinit.Default() - - cfg := GenerateConfig(privateKey) - if err := cfg.FindPeers(ctx, filter); err != nil { - return nil, err - } - - server := NewMyceliumServer(&cfg) - if err := server.Ensure(z, ns.Name()); err != nil { - return nil, err - } - - myInspec, err := server.InspectMycelium() - if err != nil { - return nil, err - } - - gw, err := myInspec.Gateway() - if err != nil { - return nil, errors.Wrap(err, "fail read mycelium subnet") - } - - if err := ns.SetMyIP(gw, nil); err != nil { - return nil, errors.Wrap(err, "fail to configure mycelium subnet gateway IP") - } - - return server, nil -} - -func (s *MyceliumServer) InspectMycelium() (inspection MyceliumInspection, err error) { - // we check if the file exists before we do inspect because mycelium will create a random seed - // file if file does not exist - _, err = os.Stat(confPath) - if err != nil { - return inspection, err - } - - bin, err := exec.LookPath(myceliumBin) - if err != nil { - return inspection, err - } - - output, err := exec.Command("ip", "netns", "exec", s.ns, bin, "inspect", "--json", "--key-file", confPath).Output() - if err != nil { - return inspection, errors.Wrap(err, "failed to inspect mycelium ip") - } - - if err := json.Unmarshal(output, &inspection); err != nil { - return inspection, errors.Wrap(err, "failed to load mycelium information from key") - } - - return -} - -// IP return the address in the 400::/7 subnet allocated by mycelium -func (m *MyceliumInspection) IP() net.IP { - return net.IP(m.Address) -} - -// Subnet return the 400::/64 subnet allocated by mycelium -func (m *MyceliumInspection) Subnet() (subnet net.IPNet, err error) { - ipv6 := m.Address.To16() - if ipv6 == nil { - return subnet, errors.Errorf("invalid mycelium ip") - } - - ip := make(net.IP, net.IPv6len) - copy(ip[0:8], ipv6[0:8]) - - subnet = net.IPNet{ - IP: ip, - Mask: net.CIDRMask(64, 128), - } - - return -} - -// Gateway derive the gateway IP from the mycelium IP in the /64 range. -func (m *MyceliumInspection) Gateway() (gw net.IPNet, err error) { - subnet, err := m.Subnet() - if err != nil { - return gw, err - } - - ip := subnet.IP - ip[net.IPv6len-1] = 1 - - gw = net.IPNet{ - IP: ip, - Mask: net.CIDRMask(64, 128), - } - - return -} - -// Tun return the name of the TUN interface created by mycelium -func (s *MyceliumServer) Tun() string { - return s.cfg.TunName -} - -// IPFor return an IP address out of the node allocated subnet by hasing b and using it -// to generate the last 64 bits of the IPV6 address -func (m *MyceliumInspection) IPFor(b []byte) (net.IPNet, error) { - subnet, err := m.Subnet() - if err != nil { - return net.IPNet{}, err - } - - ip := make([]byte, net.IPv6len) - copy(ip, subnet.IP) - - subIP, err := subnetFor(ip, b) - if err != nil { - return net.IPNet{}, err - } - - return net.IPNet{ - IP: subIP, - Mask: net.CIDRMask(64, 128), - }, nil -} - -func subnetFor(prefix net.IP, b []byte) (net.IP, error) { - h := sha512.New() - if _, err := h.Write(b); err != nil { - return nil, err - } - digest := h.Sum(nil) - copy(prefix[8:], digest[:8]) - return prefix, nil -} - -func writeKey(path string, privateKey x25519.PrivateKey) error { - if err := os.MkdirAll(filepath.Dir(path), 0770); err != nil { - return err - } - - return os.WriteFile(path, privateKey[:], 0666) -} diff --git a/pkg/network/mycelium/namespace.go b/pkg/network/mycelium/namespace.go deleted file mode 100644 index a722c914d..000000000 --- a/pkg/network/mycelium/namespace.go +++ /dev/null @@ -1,269 +0,0 @@ -package mycelium - -import ( - "fmt" - "net" - "os" - - "github.com/containernetworking/plugins/pkg/ns" - "github.com/pkg/errors" - "github.com/rs/zerolog/log" - "github.com/threefoldtech/zos/pkg/network/bridge" - "github.com/threefoldtech/zos/pkg/network/ifaceutil" - "github.com/threefoldtech/zos/pkg/network/macvlan" - "github.com/threefoldtech/zos/pkg/network/namespace" - "github.com/threefoldtech/zos/pkg/network/types" - "github.com/vishvananda/netlink" -) - -const ( - // MyceliumNSInf inside the namespace - MyceliumNSInf = "nmy6" - myceliumBridge = types.MyceliumBridge -) - -var MyRange = net.IPNet{ - IP: net.ParseIP("400::"), - Mask: net.CIDRMask(7, 128), -} - -type MyceliumNamespace interface { - Name() string - // IsIPv4Only checks if namespace has NO public ipv6 on any of its interfaces - IsIPv4Only() (bool, error) - // GetIPs return a list of all IPv6 inside this namespace. - GetIPs() ([]net.IPNet, error) - // SetMyIP sets the mycelium ipv6 on the nmy6 iterface. - SetMyIP(ip net.IPNet, gw net.IP) error -} - -// ensureMy Plumbing this ensures that the mycelium plumbing is in place inside this namespace -func ensureMyPlumbing(netNS ns.NetNS) error { - if !bridge.Exists(myceliumBridge) { - if _, err := bridge.New(myceliumBridge); err != nil { - return errors.Wrapf(err, "couldn't create bridge %s", myceliumBridge) - } - } - - if err := dumdumHack(); err != nil { - log.Error().Err(err).Msg("failed to create the dummy hack for mycelium-bridge") - } - - if !ifaceutil.Exists(MyceliumNSInf, netNS) { - if _, err := macvlan.Create(MyceliumNSInf, myceliumBridge, netNS); err != nil { - return errors.Wrapf(err, "couldn't create %s inside", MyceliumNSInf) - } - } - - return netNS.Do(func(_ ns.NetNS) error { - link, err := netlink.LinkByName(MyceliumNSInf) - if err != nil { - return err - } - - return netlink.LinkSetUp(link) - }) -} - -func dumdumHack() error { - // dumdum hack. this hack to fix a weird issue with linux kernel - // 5.10.version 55 - // it seems that the macvlan on a bridge does not bring the bridge - // up. So we have to plug in a dummy device into myceliumBridge and set - // the device up to keep the bridge state UP. - br, err := bridge.Get(myceliumBridge) - if err != nil { - return errors.Wrap(err, "failed to get br-my") - } - - const name = "mydumdum" - link, err := netlink.LinkByName(name) - if _, ok := err.(netlink.LinkNotFoundError); ok { - if err := netlink.LinkAdd(&netlink.Dummy{ - LinkAttrs: netlink.LinkAttrs{ - NetNsID: -1, - TxQLen: -1, - Name: name, - }, - }); err != nil { - return err - } - - link, err = netlink.LinkByName(name) - if err != nil { - return errors.Wrap(err, "failed to get mydumdum device") - } - } else if err != nil { - return err - } - - if err := netlink.LinkSetMaster(link, br); err != nil { - return err - } - - return netlink.LinkSetUp(link) -} - -func NewMyNamespace(ns string) (MyceliumNamespace, error) { - myNs, err := namespace.GetByName(ns) - if err != nil { - return nil, errors.Wrapf(err, "namespace '%s' not found", ns) - } - if err := ensureMyPlumbing(myNs); err != nil { - return nil, errors.Wrapf(err, "failed to prepare namespace '%s' for mycelium", ns) - } - - return &myNS{ns}, nil -} - -type myNS struct { - ns string -} - -func (d *myNS) Name() string { - return d.ns -} - -func (d *myNS) setGw(gw net.IP) error { - ipv6routes, err := netlink.RouteList(nil, netlink.FAMILY_V6) - if err != nil { - return err - } - - for _, route := range ipv6routes { - if route.Dst == nil { - // default route! - continue - } - if route.Dst.String() == MyRange.String() { - // we found a match - if err := netlink.RouteDel(&route); err != nil { - return err - } - } - } - - // now add route - return netlink.RouteAdd(&netlink.Route{ - Dst: &MyRange, - Gw: gw, - }) -} - -func (d *myNS) SetMyIP(subnet net.IPNet, gw net.IP) error { - netns, err := namespace.GetByName(d.ns) - if err != nil { - return err - } - defer netns.Close() - - if ip6 := subnet.IP.To16(); ip6 == nil { - return fmt.Errorf("expecting ipv6 for mycelium interface") - } - - err = netns.Do(func(_ ns.NetNS) error { - link, err := netlink.LinkByName(MyceliumNSInf) - if err != nil { - return err - } - - ips, err := netlink.AddrList(link, netlink.FAMILY_V6) - if err != nil { - return err - } - - for _, ip := range ips { - if MyRange.Contains(ip.IP) { - _ = netlink.AddrDel(link, &ip) - } - } - - if err := netlink.AddrAdd(link, &netlink.Addr{ - IPNet: &subnet, - }); err != nil && !os.IsExist(err) { - return err - } - - if gw == nil { - return nil - } - // set gw for entire mycelium range - - return d.setGw(gw) - }) - return err -} - -func (n *myNS) GetIPs() ([]net.IPNet, error) { - netns, err := namespace.GetByName(n.ns) - if err != nil { - return nil, err - } - - defer netns.Close() - - var results []net.IPNet - err = netns.Do(func(_ ns.NetNS) error { - links, err := netlink.LinkList() - if err != nil { - return errors.Wrap(err, "failed to list interfaces") - } - - for _, link := range links { - ips, err := netlink.AddrList(link, netlink.FAMILY_V6) - if err != nil { - return err - } - - for _, ip := range ips { - results = append(results, *ip.IPNet) - } - } - - return nil - }) - - return results, err -} - -func (n *myNS) IsIPv4Only() (bool, error) { - // this is true if DMZPub6 only has local not routable ipv6 addresses - // DMZPub6 - netNS, err := namespace.GetByName(n.ns) - if err != nil { - return false, errors.Wrap(err, "failed to get ndmz namespace") - } - defer netNS.Close() - - var ipv4Only bool - err = netNS.Do(func(_ ns.NetNS) error { - links, err := netlink.LinkList() - if err != nil { - return errors.Wrap(err, "failed to list interfaces") - } - - for _, link := range links { - ips, err := netlink.AddrList(link, netlink.FAMILY_V6) - if err != nil { - return errors.Wrapf(err, "failed to list '%s' ips", link.Attrs().Name) - } - - for _, ip := range ips { - if MyRange.Contains(ip.IP) { - continue - } - - if ip.IP.IsGlobalUnicast() && !ip.IP.IsPrivate() { - // we found a public IPv6 so we are not ipv4 so mycelium can peer - // with other ipv6 peers - return nil - } - } - } - - ipv4Only = true - return nil - }) - - return ipv4Only, err -} diff --git a/pkg/network/ndmz/dhcp_monitor.go b/pkg/network/ndmz/dhcp_monitor.go deleted file mode 100644 index 5e5e97aa8..000000000 --- a/pkg/network/ndmz/dhcp_monitor.go +++ /dev/null @@ -1,136 +0,0 @@ -package ndmz - -import ( - "context" - "time" - - "github.com/containernetworking/plugins/pkg/ns" - "github.com/pkg/errors" - "github.com/rs/zerolog/log" - "github.com/threefoldtech/zos/pkg/network/dhcp" - "github.com/threefoldtech/zos/pkg/network/ifaceutil" - "github.com/threefoldtech/zos/pkg/network/namespace" - "github.com/threefoldtech/zos/pkg/zinit" - "github.com/vishvananda/netlink" -) - -// DHCPMon monitor a network interface status and force -// renew of DHCP lease if needed -type DHCPMon struct { - z *zinit.Client - service dhcp.ClientService -} - -// NewDHCPMon create a new DHCPMon object managing interface iface -// namespace is then network namespace name to use. it can be empty. -func NewDHCPMon(iface, namespace string, z *zinit.Client) *DHCPMon { - service := dhcp.NewService(iface, namespace, z) - return &DHCPMon{ - z: z, - service: service, - } -} - -// Start creates a zinit service for a DHCP client and start monitoring it -// this method is blocking, start is in a goroutine if needed. -// cancel the context to start it. -func (d *DHCPMon) Start(ctx context.Context) error { - - if err := d.startZinit(); err != nil { - return err - } - defer func() { - if err := d.stopZinit(); err != nil { - log.Error().Err(err).Msgf("error stopping %s zinit service", d.service.Name) - } - }() - - t := time.NewTicker(time.Minute) - defer t.Stop() - - for { - select { - case <-ctx.Done(): - return nil - - case <-t.C: - has, err := hasDefaultRoute(d.service.Iface, d.service.Namespace) - if err != nil { - log.Error().Str("iface", d.service.Iface).Err(err).Msg("error checking default gateway") - continue - } - - if !has { - log.Info().Msg("ndmz default route missing, waking up dhcpcd") - if err := d.wakeUp(); err != nil { - log.Error().Err(err).Msg("error while sending signal to service ") - } - } - } - } -} - -// wakeUp sends a signal to the dhcpcd daemon to force a release of the DHCP lease -func (d *DHCPMon) wakeUp() error { - err := d.z.Kill(d.service.Name, zinit.SIGUSR1) - if err != nil { - log.Error().Err(err).Msg("error while sending signal to service ") - } - return err -} - -// hasDefaultRoute checks if the network interface iface has a default route configured -// if netNS is not empty, switch to the network namespace named netNS before checking the routes -func hasDefaultRoute(iface, netNS string) (bool, error) { - var hasDefault bool - do := func(_ ns.NetNS) error { - link, err := netlink.LinkByName(iface) - if err != nil { - return err - } - hasDefault, _, err = ifaceutil.HasDefaultGW(link, netlink.FAMILY_V4) - return err - } - - var oerr error - if netNS != "" { - n, err := namespace.GetByName(netNS) - if err != nil { - return false, err - } - oerr = n.Do(do) - } else { - oerr = do(nil) - } - return hasDefault, oerr -} - -func (d *DHCPMon) startZinit() error { - if err := d.service.DestroyOlderService(); err != nil { - return err - } - - status, err := d.z.Status(d.service.Name) - if err != nil && err != zinit.ErrUnknownService { - log.Error().Err(err).Msgf("error checking zinit service %s status", d.service.Name) - return err - } - - if status.State.Exited() { - log.Info().Msgf("zinit service %s already exists but is stopped, starting it", d.service.Name) - return d.service.Start() - } else if status.State.Is(zinit.ServiceStateRunning) { - return nil - } - - return d.service.Create() -} - -// Stop stops a zinit background process -func (d *DHCPMon) stopZinit() error { - err := d.z.StopWait(time.Second*10, d.service.Name) - if err != nil { - return errors.Wrapf(err, "failed to stop zinit service %s", d.service.Name) - } - return d.z.Forget(d.service.Name) -} diff --git a/pkg/network/ndmz/dualstack.go b/pkg/network/ndmz/dualstack.go deleted file mode 100644 index 3e4058978..000000000 --- a/pkg/network/ndmz/dualstack.go +++ /dev/null @@ -1,573 +0,0 @@ -package ndmz - -import ( - "bytes" - "context" - "fmt" - "net" - "os" - "strconv" - "strings" - "time" - - "github.com/cenkalti/backoff/v3" - "github.com/threefoldtech/zos/pkg/gridtypes" - "github.com/threefoldtech/zos/pkg/kernel" - "github.com/threefoldtech/zos/pkg/network/bridge" - "github.com/threefoldtech/zos/pkg/network/dhcp" - "github.com/threefoldtech/zos/pkg/network/ifaceutil" - "github.com/threefoldtech/zos/pkg/network/nft" - "github.com/threefoldtech/zos/pkg/network/options" - "github.com/threefoldtech/zos/pkg/network/types" - "github.com/threefoldtech/zos/pkg/network/yggdrasil" - "github.com/threefoldtech/zos/pkg/zinit" - - "github.com/threefoldtech/zos/pkg/network/macvlan" - - "github.com/rs/zerolog/log" - "github.com/vishvananda/netlink" - - "github.com/containernetworking/plugins/pkg/ns" - "github.com/pkg/errors" - "github.com/threefoldtech/zos/pkg/network/namespace" -) - -const ( - - //NdmzBridge is the name of the ipv4 routing bridge in the host namespace - NdmzBridge = "br-ndmz" - - //dmzNamespace name of the dmz namespace - dmzNamespace = "ndmz" - - ndmzNsMACDerivationSuffix6 = "-ndmz6" - ndmzNsMACDerivationSuffix4 = "-ndmz4" - - // dmzPub4 ipv4 public interface - dmzPub4 = "npub4" - // dmzPub6 ipv6 public interface - dmzPub6 = "npub6" - - //nrPubIface is the name of the public interface in a network resource - nrPubIface = "public" - - toNrsIface = "tonrs" -) - -// dmzImpl implement DMZ interface using dual stack ipv4/ipv6 -type dmzImpl struct { - nodeID string - public *netlink.Bridge -} - -// New creates a new DMZ DualStack -func New(nodeID string, public *netlink.Bridge) DMZ { - return &dmzImpl{ - nodeID: nodeID, - public: public, - } -} - -func (d *dmzImpl) Namespace() string { - return dmzNamespace -} - -// Create create the NDMZ network namespace and configure its default routes and addresses -func (d *dmzImpl) Create(ctx context.Context) error { - // There are 2 options for the master: - // - use the interface directly - // - create a bridge and plug the interface into that one - // The second option is used by default, and the first one is now legacy. - // However to not break existing containers, we keep the old one if networkd - // is restarted but the node is not. In this case, ndmz will already be present. - // - // Now, it is possible that we are a 1 nic dualstack node, in which case - // master will actually be `zos`. In that case, we can't plug the physical - // iface, but need to create a veth pair between br-pub and zos. - - netNS, err := namespace.GetByName(dmzNamespace) - if err != nil { - netNS, err = namespace.Create(dmzNamespace) - if err != nil { - return errors.Wrap(err, "failed to create ndmz namespace") - } - } - - defer netNS.Close() - - if err := createRoutingBridge(NdmzBridge, netNS); err != nil { - return errors.Wrapf(err, "ndmz: createRoutingBridge error") - } - - if err := createPubIface6(dmzPub6, d.public, d.nodeID, netNS); err != nil { - return errors.Wrapf(err, "ndmz: could not node create pub iface 6") - } - - if err := createPubIface4(dmzPub4, d.nodeID, netNS); err != nil { - return errors.Wrapf(err, "ndmz: could not create pub iface 4") - } - - if err = applyFirewall(); err != nil { - return err - } - - err = netNS.Do(func(_ ns.NetNS) error { - if err := ifaceutil.SetLoUp(); err != nil { - return errors.Wrapf(err, "ndmz: couldn't bring lo up in ndmz namespace") - } - - if err := options.SetIPv6Forwarding(true); err != nil { - return errors.Wrapf(err, "failed to enable forwarding in ndmz") - } - - if err := waitIP4(); err != nil { - return err - } - - if err := waitIP6(); err != nil { - log.Error().Err(err).Msg("ndmz: no ipv6 found") - } - return nil - }) - if err != nil { - return err - } - - z := zinit.Default() - dhcpMon := NewDHCPMon(dmzPub4, dmzNamespace, z) - go func() { - _ = dhcpMon.Start(ctx) - }() - - return nil -} - -// Delete deletes the NDMZ network namespace -func (d *dmzImpl) Delete() error { - netNS, err := namespace.GetByName(dmzNamespace) - if err == nil { - if err := namespace.Delete(netNS); err != nil { - return errors.Wrap(err, "failed to delete ndmz network namespace") - } - } - - return nil -} - -func (d *dmzImpl) DetachNR(networkID, ipamLeaseDir string) error { - // so far this is only used to deallocate reserved IP - return deAllocateIPv4(networkID, ipamLeaseDir) -} - -// AttachNR links a network resource to the NDMZ -func (d *dmzImpl) AttachNR(networkID, nrNSName string, ipamLeaseDir string) error { - nrNS, err := namespace.GetByName(nrNSName) - if err != nil { - return err - } - - if !ifaceutil.Exists(nrPubIface, nrNS) { - if _, err = macvlan.Create(nrPubIface, NdmzBridge, nrNS); err != nil { - return err - } - } - - return nrNS.Do(func(_ ns.NetNS) error { - addr, err := allocateIPv4(networkID, ipamLeaseDir) - if err != nil { - return errors.Wrap(err, "ip allocation for network resource") - } - pubIface, err := netlink.LinkByName(nrPubIface) - if err != nil { - return err - } - - if err := netlink.AddrAdd(pubIface, &netlink.Addr{IPNet: addr}); err != nil && !os.IsExist(err) { - return err - } - - ipv6 := convertIpv4ToIpv6(addr.IP) - log.Debug().Msgf("ndmz: setting public NR ip to: %s from %s", ipv6.String(), addr.IP.String()) - - if err := netlink.AddrAdd(pubIface, &netlink.Addr{IPNet: &net.IPNet{ - IP: ipv6, - Mask: net.CIDRMask(64, 128), - }}); err != nil && !os.IsExist(err) { - return err - } - - if err = netlink.LinkSetUp(pubIface); err != nil { - return err - } - - err = netlink.RouteAdd(&netlink.Route{ - Dst: &net.IPNet{ - IP: net.ParseIP("0.0.0.0"), - Mask: net.CIDRMask(0, 32), - }, - Gw: net.ParseIP("100.127.0.1"), - LinkIndex: pubIface.Attrs().Index, - }) - if err != nil && !os.IsExist(err) { - return err - } - - err = netlink.RouteAdd(&netlink.Route{ - Dst: &net.IPNet{ - IP: net.ParseIP("::"), - Mask: net.CIDRMask(0, 128), - }, - Gw: net.ParseIP("fe80::1"), - LinkIndex: pubIface.Attrs().Index, - }) - if err != nil && !os.IsExist(err) { - return err - } - - return nil - }) -} - -func (d *dmzImpl) GetIPFor(inf string) ([]net.IPNet, error) { - - netns, err := namespace.GetByName(dmzNamespace) - if err != nil { - return nil, err - } - - defer netns.Close() - - var results []net.IPNet - err = netns.Do(func(_ ns.NetNS) error { - ln, err := netlink.LinkByName(inf) - if err != nil { - return err - } - - ips, err := netlink.AddrList(ln, netlink.FAMILY_ALL) - if err != nil { - return err - } - - for _, ip := range ips { - results = append(results, *ip.IPNet) - } - - return nil - }) - - return results, err -} - -func (d *dmzImpl) GetIP(family int) ([]net.IPNet, error) { - var links []string - if family == netlink.FAMILY_V4 || family == netlink.FAMILY_ALL { - links = append(links, dmzPub4) - } - if family == netlink.FAMILY_V6 || family == netlink.FAMILY_ALL { - links = append(links, dmzPub6) - } - - if len(links) == 0 { - return nil, nil - } - - netns, err := namespace.GetByName(dmzNamespace) - if err != nil { - return nil, err - } - - defer netns.Close() - - var results []net.IPNet - err = netns.Do(func(_ ns.NetNS) error { - for _, name := range links { - ln, err := netlink.LinkByName(name) - if err != nil { - return err - } - - ips, err := netlink.AddrList(ln, family) - if err != nil { - return err - } - - for _, ip := range ips { - results = append(results, *ip.IPNet) - } - } - - return nil - }) - - return results, err -} - -// Get gateway to given destination ip -func (d *dmzImpl) GetDefaultGateway(destination net.IP) (net.IP, error) { - netns, err := namespace.GetByName(dmzNamespace) - if err != nil { - return nil, err - } - - defer netns.Close() - - var gw net.IP - err = netns.Do(func(_ ns.NetNS) error { - routes, err := netlink.RouteGet(destination) - if err != nil { - return err - } - - for _, route := range routes { - if route.Gw != nil { - gw = route.Gw - return nil - } - } - - return nil - }) - - if gw == nil { - return nil, fmt.Errorf("default gateway not found") - } - - return gw, err -} - -// SupportsPubIPv4 implements DMZ interface -func (d *dmzImpl) SupportsPubIPv4() bool { - return true -} - -// Interfaces return information about dmz interfaces -func (d *dmzImpl) Interfaces() ([]types.IfaceInfo, error) { - var output []types.IfaceInfo - - f := func(_ ns.NetNS) error { - links, err := netlink.LinkList() - if err != nil { - log.Error().Err(err).Msgf("failed to list interfaces") - return err - } - for _, link := range links { - name := link.Attrs().Name - if name == toNrsIface { - continue - } - - addrs, err := netlink.AddrList(link, netlink.FAMILY_ALL) - if err != nil { - return err - } - - info := types.IfaceInfo{ - Name: name, - Addrs: make([]gridtypes.IPNet, len(addrs)), - MacAddress: types.MacAddress{HardwareAddr: link.Attrs().HardwareAddr}, - } - for i, addr := range addrs { - info.Addrs[i] = gridtypes.NewIPNet(*addr.IPNet) - } - - output = append(output, info) - - } - return nil - } - - // get the ndmz network namespace - ndmz, err := namespace.GetByName(dmzNamespace) - if err != nil { - return nil, err - } - defer ndmz.Close() - - err = ndmz.Do(f) - if err != nil { - return nil, err - } - - return output, nil -} - -// waitIP4 waits to receives some IPv4 from DHCP -// it returns the pid of the dhcp process or an error -func waitIP4() error { - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - - probe, err := dhcp.Probe(ctx, dmzPub4) - - if err != nil { - return errors.Wrapf(err, "error while proping interface '%s'", dmzPub4) - } - if len(probe.IP) != 0 && len(probe.Router) != 0 { - return nil - } - - return errors.Errorf("public interface in ndmz did not received an IP. make sure DHCP is working") -} - -func waitIP6() error { - // also, set kernel parameter that public always accepts an ra even when forwarding - if err := options.Set(dmzPub6, - options.AcceptRA(options.RAAcceptIfForwardingIsEnabled), - options.LearnDefaultRouteInRA(true), - options.ProxyArp(false)); err != nil { - return errors.Wrapf(err, "ndmz: failed to set ndmz pub6 flags") - } - - getRoutes := func() (err error) { - log.Info().Msg("wait for slaac to give ipv6") - // check if in the mean time SLAAC gave us an IPv6 deft gw, save it, and reapply after enabling forwarding - checkipv6 := net.ParseIP("2606:4700:4700::1111") - if _, err = netlink.RouteGet(checkipv6); err != nil { - return errors.Wrapf(err, "ndmz: failed to get the IPv6 routes in ndmz") - } - return nil - } - - bo := backoff.NewExponentialBackOff() - bo.MaxElapsedTime = 2 * time.Minute // default RA from router is every 60 secs - if kernel.GetParams().IsVirtualMachine() { - bo.MaxElapsedTime = 20 * time.Second - } - return backoff.Retry(getRoutes, bo) -} - -func createPubIface6(name string, master *netlink.Bridge, nodeID string, netNS ns.NetNS) error { - if !ifaceutil.Exists(name, netNS) { - if _, err := macvlan.Create(name, master.Name, netNS); err != nil { - return err - } - } - - return netNS.Do(func(_ ns.NetNS) error { - // set mac address to something static to make sure we receive the same IP from a DHCP server - mac := ifaceutil.HardwareAddrFromInputBytes([]byte(nodeID + ndmzNsMACDerivationSuffix6)) - log.Debug(). - Str("mac", mac.String()). - Str("interface", name). - Msg("set mac on ipv6 ndmz public iface") - - return ifaceutil.SetMAC(name, mac, nil) - }) -} - -func createPubIface4(name, nodeID string, netNS ns.NetNS) error { - if !ifaceutil.Exists(name, netNS) { - if _, err := macvlan.Create(name, types.DefaultBridge, netNS); err != nil { - return err - } - } - - return netNS.Do(func(_ ns.NetNS) error { - if err := options.Set(name, options.IPv6Disable(true)); err != nil { - return errors.Wrapf(err, "failed to disable ip6 on %s", name) - } - // set mac address to something static to make sure we receive the same IP from a DHCP server - mac := ifaceutil.HardwareAddrFromInputBytes([]byte(nodeID + ndmzNsMACDerivationSuffix4)) - log.Debug(). - Str("mac", mac.String()). - Str("interface", name). - Msg("set mac on ipv4 ndmz public iface") - - return ifaceutil.SetMAC(name, mac, nil) - }) -} - -func createRoutingBridge(name string, netNS ns.NetNS) error { - if !bridge.Exists(name) { - if _, err := bridge.New(name); err != nil { - return errors.Wrapf(err, "couldn't create bridge %s", name) - } - } - - if !ifaceutil.Exists(toNrsIface, netNS) { - if _, err := macvlan.Create(toNrsIface, name, netNS); err != nil { - return errors.Wrapf(err, "ndmz: couldn't create %s", toNrsIface) - } - } - - if err := options.Set(name, options.IPv6Disable(true)); err != nil { - return errors.Wrapf(err, "failed to disable ip6 on bridge %s", name) - } - - return netNS.Do(func(_ ns.NetNS) error { - - link, err := netlink.LinkByName(toNrsIface) - if err != nil { - return err - } - if err := options.Set(toNrsIface, options.IPv6Disable(false)); err != nil { - return errors.Wrapf(err, "failed to enable ip6 on interface %s", toNrsIface) - } - - addrs := []*netlink.Addr{ - { - IPNet: &net.IPNet{ - IP: net.ParseIP("100.127.0.1"), - Mask: net.CIDRMask(16, 32), - }, - }, - { - IPNet: &net.IPNet{ - IP: net.ParseIP("fe80::1"), - Mask: net.CIDRMask(64, 128), - }, - }, - { - IPNet: &net.IPNet{ - IP: net.ParseIP("fd00::1"), - Mask: net.CIDRMask(64, 128), - }, - }, - } - - for _, addr := range addrs { - err = netlink.AddrAdd(link, addr) - if err != nil && !os.IsExist(err) { - return err - } - } - - return netlink.LinkSetUp(link) - }) -} - -func applyFirewall() error { - buf := bytes.Buffer{} - - data := struct { - YggPorts string - }{ - YggPorts: strings.Join([]string{ - strconv.Itoa(yggdrasil.YggListenTCP), - strconv.Itoa(yggdrasil.YggListenTLS), - strconv.Itoa(yggdrasil.YggListenLinkLocal), - }, ","), - } - - if err := fwTmpl.Execute(&buf, data); err != nil { - return errors.Wrap(err, "failed to build nft rule set") - } - - if err := nft.Apply(&buf, dmzNamespace); err != nil { - return errors.Wrap(err, "failed to apply nft rule set") - } - - return nil -} - -func convertIpv4ToIpv6(ip net.IP) net.IP { - var ipv6 string - if len(ip) == net.IPv4len { - ipv6 = fmt.Sprintf("fd00::%02x%02x", ip[2], ip[3]) - } else { - ipv6 = fmt.Sprintf("fd00::%02x%02x", ip[14], ip[15]) - } - - return net.ParseIP(ipv6) -} diff --git a/pkg/network/ndmz/ndmz.go b/pkg/network/ndmz/ndmz.go deleted file mode 100644 index 68b8b96f6..000000000 --- a/pkg/network/ndmz/ndmz.go +++ /dev/null @@ -1,46 +0,0 @@ -package ndmz - -import ( - "context" - "net" - - "github.com/threefoldtech/zos/pkg/network/types" - "github.com/vishvananda/netlink" -) - -const ( - // FamilyAll get all IP address families - FamilyAll = netlink.FAMILY_ALL - //FamilyV4 gets all IPv4 addresses - FamilyV4 = netlink.FAMILY_V4 - //FamilyV6 gets all IPv6 addresses - FamilyV6 = netlink.FAMILY_V6 -) - -// DMZ is an interface used to create an DMZ network namespace -type DMZ interface { - Namespace() string - // create the ndmz network namespace and all requires network interfaces - Create(ctx context.Context) error - // delete the ndmz network namespace and clean up all network interfaces - Delete() error - // link a network resource from a user network to ndmz - AttachNR(networkID, nr, ipamLeaseDir string) error - - DetachNR(networkID, ipamLeaseDir string) error - // GetIP gets ndmz public ips from ndmz - GetIP(family int) ([]net.IPNet, error) - - // Get gateway to given destination ip - GetDefaultGateway(destination net.IP) (net.IP, error) - - // GetIPFor get the ip of an - GetIPFor(inf string) ([]net.IPNet, error) - //GetIP(family netlink.FAM) - // SupportsPubIPv4 indicates if the node supports public ipv4 addresses for - // workloads - SupportsPubIPv4() bool - - //Interfaces information about dmz interfaces - Interfaces() ([]types.IfaceInfo, error) -} diff --git a/pkg/network/ndmz/ndmz_test.go b/pkg/network/ndmz/ndmz_test.go deleted file mode 100644 index 52906f5b7..000000000 --- a/pkg/network/ndmz/ndmz_test.go +++ /dev/null @@ -1,86 +0,0 @@ -package ndmz - -import ( - "fmt" - "net" - "sync" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestIpv6(t *testing.T) { - - tt := []struct { - ipv4 net.IP - ipv6 net.IP - }{ - { - ipv4: net.ParseIP("100.127.0.3"), - ipv6: net.ParseIP("fd00::0000:0003"), - }, - { - ipv4: net.ParseIP("100.127.1.1"), - ipv6: net.ParseIP("fd00::101"), - }, - { - ipv4: net.ParseIP("100.127.255.254"), - ipv6: net.ParseIP("fd00::fffe"), - }, - } - for _, tc := range tt { - ipv6 := convertIpv4ToIpv6(tc.ipv4) - assert.Equal(t, tc.ipv6, ipv6) - } - -} - -func TestIPv4Allocate(t *testing.T) { - ipamPath := t.TempDir() - - addr, err := allocateIPv4("network1", ipamPath) - require.NoError(t, err) - - addr2, err := allocateIPv4("network1", ipamPath) - require.NoError(t, err) - - assert.Equal(t, addr.String(), addr2.String()) - - addr3, err := allocateIPv4("network2", ipamPath) - require.NoError(t, err) - assert.NotEqual(t, addr.String(), addr3.String()) -} - -func TestIPv4AllocateConcurent(t *testing.T) { - ipamPath := t.TempDir() - - wg := sync.WaitGroup{} - wg.Add(10) - - c := make(chan *net.IPNet) - - for i := 0; i < 10; i++ { - go func(c chan *net.IPNet, i int) { - defer wg.Done() - for y := 0; y < 10; y++ { - nw := fmt.Sprintf("network%d%d", i, y) - addr, err := allocateIPv4(nw, ipamPath) - require.NoError(t, err) - c <- addr - } - }(c, i) - } - - go func() { - addrs := map[*net.IPNet]struct{}{} - for addr := range c { - _, exists := addrs[addr] - require.False(t, exists) - addrs[addr] = struct{}{} - } - }() - - wg.Wait() - close(c) -} diff --git a/pkg/network/ndmz/nft.go b/pkg/network/ndmz/nft.go deleted file mode 100644 index 482b7a57d..000000000 --- a/pkg/network/ndmz/nft.go +++ /dev/null @@ -1,72 +0,0 @@ -package ndmz - -import ( - "text/template" -) - -var fwTmpl *template.Template - -func init() { - fwTmpl = template.Must(template.New("").Parse(_nft)) -} - -var _nft = ` -flush ruleset - -table inet nat { - chain prerouting { - type nat hook prerouting priority dstnat; policy accept; - } - - chain input { - type nat hook input priority 100; policy accept; - } - - chain output { - type nat hook output priority -100; policy accept; - } - - chain postrouting { - type nat hook postrouting priority srcnat; policy accept; - ip6 saddr 200::/7 accept - oifname "npub4" masquerade fully-random; - oifname "npub6" masquerade fully-random; - } -} - -table inet filter { - - chain base_checks { - # allow established/related connections - ct state {established, related} accept - # early drop of invalid connections - ct state invalid drop - } - - chain input { - type filter hook input priority 0; policy accept; - jump base_checks - ip6 daddr {ff02::/64,fe80::/64} accept - tcp dport { {{.YggPorts}} } accept - # port for provisiond - tcp dport 8051 accept - ip6 nexthdr icmpv6 accept - iifname "npub6" counter drop - iifname "npub4" counter drop - } - - chain forward { - type filter hook forward priority 0; policy accept; - # is there already an existing stream? (outgoing) - jump base_checks - # if not, verify if it's new and coming in from the br4-gw network - # if it is, drop it - iifname "npub6" counter drop - iifname "npub4" counter drop - } - - chain output { - type filter hook output priority 0; policy accept; - } -} -` diff --git a/pkg/network/network_test.go b/pkg/network/network_test.go deleted file mode 100644 index 1d6ac47fd..000000000 --- a/pkg/network/network_test.go +++ /dev/null @@ -1,44 +0,0 @@ -package network - -import ( - "crypto/rand" - "fmt" - "testing" - - "golang.org/x/crypto/ed25519" - - "golang.zx2c4.com/wireguard/wgctrl/wgtypes" - - "github.com/threefoldtech/zos/pkg/crypto" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestKeys(t *testing.T) { - wgKey, err := wgtypes.GenerateKey() - require.NoError(t, err) - - pk, sk, err := ed25519.GenerateKey(rand.Reader) - require.NoError(t, err) - - fmt.Println(wgKey.String()) - - encrypted, err := crypto.Encrypt([]byte(wgKey.String()), pk) - require.NoError(t, err) - - strEncrypted := fmt.Sprintf("%x", encrypted) - - strDecrypted := "" - fmt.Sscanf(strEncrypted, "%x", &strDecrypted) - - decrypted, err := crypto.Decrypt([]byte(strDecrypted), sk) - require.NoError(t, err) - - fmt.Println(string(decrypted)) - - wgKey2, err := wgtypes.ParseKey(string(decrypted)) - require.NoError(t, err) - - assert.Equal(t, wgKey, wgKey2) -} diff --git a/pkg/network/networker.go b/pkg/network/networker.go deleted file mode 100644 index ccef4a6d0..000000000 --- a/pkg/network/networker.go +++ /dev/null @@ -1,1510 +0,0 @@ -package network - -import ( - "bytes" - "context" - "crypto/ed25519" - "encoding/json" - "fmt" - "html/template" - "net" - "os" - "os/exec" - "path/filepath" - "slices" - "strings" - "time" - - "github.com/blang/semver" - "github.com/cenkalti/backoff/v3" - - "github.com/threefoldtech/zos/pkg/cache" - "github.com/threefoldtech/zos/pkg/environment" - "github.com/threefoldtech/zos/pkg/gridtypes" - "github.com/threefoldtech/zos/pkg/gridtypes/zos" - "github.com/threefoldtech/zos/pkg/network/bootstrap" - "github.com/threefoldtech/zos/pkg/network/iperf" - "github.com/threefoldtech/zos/pkg/network/mycelium" - "github.com/threefoldtech/zos/pkg/network/ndmz" - "github.com/threefoldtech/zos/pkg/network/public" - "github.com/threefoldtech/zos/pkg/network/tuntap" - "github.com/threefoldtech/zos/pkg/network/wireguard" - "github.com/threefoldtech/zos/pkg/network/yggdrasil" - "github.com/threefoldtech/zos/pkg/stubs" - "github.com/threefoldtech/zos/pkg/zinit" - - "github.com/vishvananda/netlink" - - "github.com/pkg/errors" - - "github.com/threefoldtech/zos/pkg/network/ifaceutil" - - "github.com/containernetworking/plugins/pkg/ns" - "github.com/threefoldtech/zos/pkg/network/macvlan" - "github.com/threefoldtech/zos/pkg/network/nr" - "github.com/threefoldtech/zos/pkg/network/types" - "github.com/threefoldtech/zos/pkg/set" - "github.com/threefoldtech/zos/pkg/versioned" - - "github.com/rs/zerolog/log" - - "github.com/threefoldtech/zos/pkg/network/namespace" - - "github.com/threefoldtech/zos/pkg" -) - -const ( - // PubIface is pub interface name of the interface used in the 0-db network namespace - PubIface = "eth0" - // ZDBYggIface is ygg interface name of the interface used in the 0-db network namespace - ZDBYggIface = "ygg0" - ZDBMyceliumIface = "my0" - - networkDir = "networks" - linkDir = "link" - ipamLeaseDir = "ndmz-lease" - myceliumKeyDir = "mycelium-key" - zdbNamespacePrefix = "zdb-ns-" - qsfsNamespacePrefix = "qfs-ns-" -) - -const ( - mib = 1024 * 1024 -) - -// NetworkSchemaLatestVersion last version -var NetworkSchemaLatestVersion = semver.MustParse("0.1.0") - -type networker struct { - identity *stubs.IdentityManagerStub - networkDir string - linkDir string - ipamLeaseDir string - myceliumKeyDir string - portSet *set.UIntSet - - ndmz ndmz.DMZ - ygg *yggdrasil.YggServer - mycelium *mycelium.MyceliumServer -} - -var _ pkg.Networker = (*networker)(nil) - -// NewNetworker create a new pkg.Networker that can be used over zbus -func NewNetworker(identity *stubs.IdentityManagerStub, ndmz ndmz.DMZ, ygg *yggdrasil.YggServer, myc *mycelium.MyceliumServer) (pkg.Networker, error) { - vd, err := cache.VolatileDir("networkd", 50*mib) - if err != nil && !os.IsExist(err) { - return nil, fmt.Errorf("failed to create networkd cache directory: %w", err) - } - - runtimeDir := filepath.Join(vd, networkDir) - linkDir := filepath.Join(runtimeDir, linkDir) - ipamLease := filepath.Join(vd, ipamLeaseDir) - myceliumKey := filepath.Join(vd, myceliumKeyDir) - - for _, dir := range []string{linkDir, ipamLease, myceliumKey} { - if err := os.MkdirAll(dir, 0755); err != nil { - return nil, errors.Wrapf(err, "failed to create directory: '%s'", dir) - } - } - - nw := &networker{ - identity: identity, - networkDir: runtimeDir, - linkDir: linkDir, - ipamLeaseDir: ipamLease, - myceliumKeyDir: myceliumKey, - portSet: set.NewInt(), - - ygg: ygg, - mycelium: myc, - ndmz: ndmz, - } - - // always add the reserved yggdrasil and mycelium ports to the port set so we make sure they are never - // picked for wireguard endpoints - // we also add http, https, and traefik metrics ports 8082 to the list. - for _, port := range []int{yggdrasil.YggListenTCP, yggdrasil.YggListenTLS, yggdrasil.YggListenLinkLocal, mycelium.MyListenTCP, iperf.IperfPort, 80, 443, 8082} { - if err := nw.portSet.Add(uint(port)); err != nil && errors.Is(err, set.ErrConflict{}) { - return nil, err - } - } - - if err := nw.syncWGPorts(); err != nil { - return nil, err - } - - return nw, nil -} - -var _ pkg.Networker = (*networker)(nil) - -func (n *networker) Ready() error { - return nil -} - -func (n *networker) WireguardPorts() ([]uint, error) { - return n.portSet.List() -} - -func (n *networker) attachYgg(id string, netNs ns.NetNS) (net.IPNet, error) { - // new hardware address for the ygg interface - hw := ifaceutil.HardwareAddrFromInputBytes([]byte("ygg:" + id)) - - ip, err := n.ygg.SubnetFor(hw) - if err != nil { - return net.IPNet{}, fmt.Errorf("failed to generate ygg subnet IP: %w", err) - } - - ips := []*net.IPNet{ - &ip, - } - - gw, err := n.ygg.Gateway() - if err != nil { - return net.IPNet{}, fmt.Errorf("failed to get ygg gateway IP: %w", err) - } - - routes := []*netlink.Route{ - { - Dst: &net.IPNet{ - IP: net.ParseIP("200::"), - Mask: net.CIDRMask(7, 128), - }, - Gw: gw.IP, - // LinkIndex:... this is set by macvlan.Install - }, - } - - if err := n.createMacVlan(ZDBYggIface, types.YggBridge, hw, ips, routes, netNs); err != nil { - return net.IPNet{}, errors.Wrap(err, "failed to setup zdb ygg interface") - } - - return ip, nil -} - -func (n *networker) detachYgg(id string, netNs ns.NetNS) error { - return netNs.Do(func(_ ns.NetNS) error { - link, err := netlink.LinkByName(ZDBYggIface) - if err != nil { - return err - } - if err := netlink.LinkDel(link); err != nil { - return errors.Wrap(err, "failed to delete zdb ygg interface") - } - return nil - }) -} - -func (n *networker) attachMycelium(id string, netNs ns.NetNS) (net.IPNet, error) { - // new hardware address for mycelium interface - hw := ifaceutil.HardwareAddrFromInputBytes([]byte("my:" + id)) - - myc, err := n.mycelium.InspectMycelium() - if err != nil { - return net.IPNet{}, err - } - - ip, err := myc.IPFor(hw) - log.Info().Msgf("mycelium IP is %s", ip.IP) - if err != nil { - return net.IPNet{}, fmt.Errorf("failed to generate mycelium IP: %w", err) - } - - ips := []*net.IPNet{ - &ip, - } - - gw, err := myc.Gateway() - if err != nil { - return net.IPNet{}, fmt.Errorf("failed to get mycelium gateway IP: %w", err) - } - - routes := []*netlink.Route{ - { - Dst: &net.IPNet{ - IP: net.ParseIP("400::"), - Mask: net.CIDRMask(7, 128), - }, - Gw: gw.IP, - // LinkIndex:... this is set by macvlan.Install - }, - } - - if err := n.createMacVlan(ZDBMyceliumIface, types.MyceliumBridge, hw, ips, routes, netNs); err != nil { - return net.IPNet{}, errors.Wrap(err, "failed to setup zdb mycelium interface") - } - - return ip, nil -} - -// ensurePrepare ensurets that a unique namespace is created (based on id) with "prefix" -// and make sure it's wired to the bridge on host namespace -func (n *networker) ensurePrepare(id, prefix, bridge string) (string, error) { - hw := ifaceutil.HardwareAddrFromInputBytes([]byte("pub:" + id)) - - netNSName := prefix + strings.Replace(hw.String(), ":", "", -1) - - netNs, err := createNetNS(netNSName) - if err != nil { - return "", err - } - defer netNs.Close() - - if err := n.createMacVlan(PubIface, bridge, hw, nil, nil, netNs); err != nil { - return "", errors.Wrap(err, "failed to setup zdb public interface") - } - - if n.ygg != nil { - _, err = n.attachYgg(id, netNs) - if err != nil { - return "", err - } - - } - - if n.mycelium != nil { - _, err = n.attachMycelium(id, netNs) - if err != nil { - return "", err - } - } - - return netNSName, nil -} - -func (n *networker) destroy(ns string) error { - if !namespace.Exists(ns) { - return nil - } - - nSpace, err := namespace.GetByName(ns) - if os.IsNotExist(err) { - return nil - } else if err != nil { - return errors.Wrapf(err, "failed to get namespace '%s'", ns) - } - - return namespace.Delete(nSpace) -} - -// func (n *networker) NSPrepare(id string, ) -// EnsureZDBPrepare sends a macvlan interface into the -// network namespace of a ZDB container -func (n *networker) EnsureZDBPrepare(id string) (string, error) { - return n.ensurePrepare(id, zdbNamespacePrefix, types.PublicBridge) -} - -// ZDBDestroy is the opposite of ZDPrepare, it makes sure network setup done -// for zdb is rewind. ns param is the namespace return by the ZDBPrepare -func (n *networker) ZDBDestroy(ns string) error { - panic("not implemented") - // if !strings.HasPrefix(ns, zdbNamespacePrefix) { - // return fmt.Errorf("invalid zdb namespace name '%s'", ns) - // } - - // return n.destroy(ns) -} - -func (n *networker) createMacVlan(iface string, master string, hw net.HardwareAddr, ips []*net.IPNet, routes []*netlink.Route, netNs ns.NetNS) error { - var macVlan *netlink.Macvlan - err := netNs.Do(func(_ ns.NetNS) error { - var err error - macVlan, err = macvlan.GetByName(iface) - return err - }) - - if _, ok := err.(netlink.LinkNotFoundError); ok { - macVlan, err = macvlan.Create(iface, master, netNs) - if err != nil { - return err - } - } else if err != nil { - return err - } - - log.Debug().Str("HW", hw.String()).Str("macvlan", macVlan.Name).Msg("setting hw address on link") - // we don't set any route or ip - if err := macvlan.Install(macVlan, hw, ips, routes, netNs); err != nil { - return err - } - - return nil -} - -// SetupTap interface in the network resource. We only allow 1 tap interface to be -// set up per NR currently -func (n *networker) SetupPrivTap(networkID pkg.NetID, name string) (ifc string, err error) { - log.Info().Str("network-id", string(networkID)).Msg("Setting up tap interface") - - localNR, err := n.networkOf(networkID) - if err != nil { - return "", errors.Wrapf(err, "couldn't load network with id (%s)", networkID) - } - - netRes := nr.New(localNR, n.myceliumKeyDir) - - bridgeName, err := netRes.BridgeName() - if err != nil { - return "", errors.Wrap(err, "could not get network namespace bridge") - } - - tapIface, err := tapName(name) - if err != nil { - return "", errors.Wrap(err, "could not get network namespace tap device name") - } - - _, err = tuntap.CreateTap(tapIface, bridgeName) - - return tapIface, err -} - -func (n *networker) TapExists(name string) (bool, error) { - log.Info().Str("tap-name", string(name)).Msg("Checking if tap interface exists") - - tapIface, err := tapName(name) - if err != nil { - return false, errors.Wrap(err, "could not get network namespace tap device name") - } - - return ifaceutil.Exists(tapIface, nil), nil -} - -// RemoveTap in the network resource. -func (n *networker) RemoveTap(name string) error { - log.Info().Str("tap-name", string(name)).Msg("Removing tap interface") - - tapIface, err := tapName(name) - if err != nil { - return errors.Wrap(err, "could not get network namespace tap device name") - } - - return ifaceutil.Delete(tapIface, nil) -} - -func (n *networker) PublicIPv4Support() bool { - return n.ndmz.SupportsPubIPv4() -} - -// SetupPubTap sets up a tap device in the host namespace for the public ip -// reservation id. It is hooked to the public bridge. The name of the tap -// interface is returned -func (n *networker) SetupPubTap(name string) (string, error) { - log.Info().Str("pubtap-name", string(name)).Msg("Setting up public tap interface") - - if !n.ndmz.SupportsPubIPv4() { - return "", errors.New("can't create public tap on this node") - } - - tapIface, err := pubTapName(name) - if err != nil { - return "", errors.Wrap(err, "could not get network namespace tap device name") - } - - _, err = tuntap.CreateTap(tapIface, public.PublicBridge) - - return tapIface, err -} - -// SetupMyceliumTap creates a new mycelium tap device attached to this network resource with deterministic IP address -func (n *networker) SetupMyceliumTap(name string, netID zos.NetID, config zos.MyceliumIP) (tap pkg.PlanetaryTap, err error) { - log.Info().Str("tap-name", string(name)).Msg("Setting up mycelium tap interface") - - network, err := n.networkOf(netID) - if err != nil { - return tap, errors.Wrapf(err, "failed to get network resource '%s'", netID) - } - - if network.Mycelium == nil { - return tap, fmt.Errorf("network resource does not support mycelium") - } - - tapIface, err := tapName(name) - if err != nil { - return tap, errors.Wrap(err, "could not get network namespace tap device name") - } - - tap.Name = tapIface - - // calculate the hw address that will be set INSIDE the vm (not the host) - hw := ifaceutil.HardwareAddrFromInputBytes([]byte("mycelium:" + name)) - tap.HW = hw - - netNR := nr.New(network, n.myceliumKeyDir) - - ip, gw, err := netNR.MyceliumIP(config.Seed) - if err != nil { - return tap, err - } - tap.IP = ip - tap.Gateway = gw - - if ifaceutil.Exists(tapIface, nil) { - return tap, nil - } - - if err := netNR.AttachMycelium(tapIface); err != nil { - return tap, err - } - - return tap, err -} - -// SetupYggTap sets up a tap device in the host namespace for the yggdrasil ip -func (n *networker) SetupYggTap(name string) (tap pkg.PlanetaryTap, err error) { - log.Info().Str("tap-name", string(name)).Msg("Setting up yggdrasil tap interface") - - tapIface, err := tapName(name) - if err != nil { - return tap, errors.Wrap(err, "could not get network namespace tap device name") - } - - tap.Name = tapIface - - hw := ifaceutil.HardwareAddrFromInputBytes([]byte("ygg:" + name)) - tap.HW = hw - ip, err := n.ygg.SubnetFor(hw) - if err != nil { - return tap, err - } - - tap.IP = ip - - gw, err := n.ygg.Gateway() - if err != nil { - return tap, err - } - - tap.Gateway = gw - if ifaceutil.Exists(tapIface, nil) { - return tap, nil - } - - _, err = tuntap.CreateTap(tapIface, types.YggBridge) - return tap, err -} - -// PubTapExists checks if the tap device for the public network exists already -func (n *networker) PubTapExists(name string) (bool, error) { - log.Info().Str("pubtap-name", string(name)).Msg("Checking if public tap interface exists") - - tapIface, err := pubTapName(name) - if err != nil { - return false, errors.Wrap(err, "could not get network namespace tap device name") - } - - return ifaceutil.Exists(tapIface, nil), nil -} - -// RemovePubTap removes the public tap device from the host namespace -// of the networkID -func (n *networker) RemovePubTap(name string) error { - log.Info().Str("pubtap-name", string(name)).Msg("Removing public tap interface") - - tapIface, err := pubTapName(name) - if err != nil { - return errors.Wrap(err, "could not get network namespace tap device name") - } - - return ifaceutil.Delete(tapIface, nil) -} - -var ( - pubIpTemplateSetup = template.Must(template.New("filter-setup").Parse( - `# add vm -# add a chain for the vm public interface in arp and bridge -nft 'add chain bridge filter {{.Name}}-pre' -nft 'add chain bridge filter {{.Name}}-post' - -# make nft jump to vm chain -nft 'add rule bridge filter prerouting iifname "{{.Iface}}" jump {{.Name}}-pre' -nft 'add rule bridge filter postrouting oifname "{{.Iface}}" jump {{.Name}}-post' - -nft 'add rule bridge filter {{.Name}}-pre ip saddr . ether saddr != { {{.IPv4}} . {{.Mac}} } counter drop' -# nft 'add rule bridge filter {{.Name}}-pre ip6 saddr . ether saddr != { {{.IPv6}} . {{.Mac}} } counter drop' - -nft 'add rule bridge filter {{.Name}}-pre arp operation reply arp saddr ip != {{.IPv4}} counter drop' -nft 'add rule bridge filter {{.Name}}-pre arp operation request arp saddr ip != {{.IPv4}} counter drop' - -nft 'add rule bridge filter {{.Name}}-post ip daddr . ether daddr != { {{.IPv4}} . {{.Mac}} } counter drop' -# nft 'add rule bridge filter {{.Name}}-post ip6 saddr . ether saddr != { {{.IPv6}} . {{.Mac}} } counter drop' -`)) - - pubIpTemplateDestroy = template.Must(template.New("filter-destroy").Parse( - `# in bridge table -nft 'flush chain bridge filter {{.Name}}-post' -nft 'flush chain bridge filter {{.Name}}-pre' - -# the .name rule is for backward compatibility -# to make sure older chains are deleted -nft 'flush chain bridge filter {{.Name}}' || true - -# we need to make sure this clean up can also work on older setup -# jump to chain rule -a=$( nft -a list table bridge filter | awk '/jump {{.Name}}-pre/{ print $NF}' ) -if [ -n "${a}" ]; then - nft delete rule bridge filter prerouting handle ${a} -fi -a=$( nft -a list table bridge filter | awk '/jump {{.Name}}-post/{ print $NF}' ) -if [ -n "${a}" ]; then - nft delete rule bridge filter postrouting handle ${a} -fi -a=$( nft -a list table bridge filter | awk '/jump {{.Name}}/{ print $NF}' ) -if [ -n "${a}" ]; then - nft delete rule bridge filter forward handle ${a} -fi - -# chain itself -for chain in $( nft -a list table bridge filter | awk '/chain {{.Name}}/{ print $NF}' ); do - nft delete chain bridge filter handle ${chain} -done - -# the next section is only for backward compatibility -# in arp table -nft 'flush chain arp filter {{.Name}}' -# jump to chain rule -a=$( nft -a list table arp filter | awk '/jump {{.Name}}/{ print $NF}' ) -if [ -n "${a}" ]; then - nft delete rule arp filter input handle ${a} -fi -# chain itself -a=$( nft -a list table arp filter | awk '/chain {{.Name}}/{ print $NF}' ) -if [ -n "${a}" ]; then - nft delete chain arp filter handle ${a} -fi -`)) -) - -// SetupPubIPFilter sets up filter for this public ip -func (n *networker) SetupPubIPFilter(filterName string, iface string, ipv4 net.IP, ipv6 net.IP, mac string) error { - if n.PubIPFilterExists(filterName) { - return nil - } - - ipv4 = ipv4.To4() - ipv6 = ipv6.To16() - // if no ipv4 or ipv6 provided, we make sure - // to use zero ip so the user can't just assign - // an ip to his vm to use. - if len(ipv4) == 0 { - ipv4 = net.IPv4zero - } - - if len(ipv6) == 0 { - ipv6 = net.IPv6zero - } - - data := struct { - Name string - Iface string - Mac string - IPv4 string - IPv6 string - }{ - Name: filterName, - Iface: iface, - Mac: mac, - IPv4: ipv4.String(), - IPv6: ipv6.String(), - } - - var buffer bytes.Buffer - if err := pubIpTemplateSetup.Execute(&buffer, data); err != nil { - return errors.Wrap(err, "failed to execute filter template") - } - - // TODO: use nft.Apply - cmd := exec.Command("/bin/sh", "-c", buffer.String()) - - output, err := cmd.CombinedOutput() - if err != nil { - return errors.Wrapf(err, "could not setup firewall rules for public ip\n%s", string(output)) - } - - return nil -} - -// PubIPFilterExists checks if pub ip filter -func (n *networker) PubIPFilterExists(filterName string) bool { - cmd := exec.Command( - "/bin/sh", - "-c", - fmt.Sprintf(`nft list table bridge filter | grep "chain %s"`, filterName), - ) - err := cmd.Run() - return err == nil -} - -// RemovePubIPFilter removes the filter setted up by SetupPubIPFilter -func (n *networker) RemovePubIPFilter(filterName string) error { - data := struct { - Name string - }{ - Name: filterName, - } - - var buffer bytes.Buffer - if err := pubIpTemplateDestroy.Execute(&buffer, data); err != nil { - return errors.Wrap(err, "failed to execute filter template") - } - - cmd := exec.Command("/bin/sh", "-c", buffer.String()) - - output, err := cmd.CombinedOutput() - if err != nil { - return errors.Wrapf(err, "could not tear down firewall rules for public ip\n%s", string(output)) - } - return nil -} - -// DisconnectPubTap disconnects the public tap from the network. The interface -// itself is not removed and will need to be cleaned up later -func (n *networker) DisconnectPubTap(name string) error { - log.Info().Str("pubtap-name", string(name)).Msg("Disconnecting public tap interface") - tapIfaceName, err := pubTapName(name) - if err != nil { - return errors.Wrap(err, "could not get network namespace tap device name") - } - tap, err := netlink.LinkByName(tapIfaceName) - if _, ok := err.(netlink.LinkNotFoundError); ok { - return nil - } else if err != nil { - if errors.Is(err, os.ErrNotExist) { - return nil - } - return errors.Wrap(err, "could not load tap device") - } - - return netlink.LinkSetNoMaster(tap) -} - -// GetPublicIPv6Subnet returns the IPv6 prefix op the public subnet of the host -func (n *networker) GetPublicIPv6Subnet() (net.IPNet, error) { - addrs, err := n.ndmz.GetIP(ndmz.FamilyV6) - if err != nil { - return net.IPNet{}, errors.Wrap(err, "could not get ips from ndmz") - } - - for _, addr := range addrs { - if addr.IP.IsGlobalUnicast() && !isULA(addr.IP) && !isYgg(addr.IP) { - return addr, nil - } - } - - return net.IPNet{}, fmt.Errorf("no public ipv6 found") -} - -func (n *networker) GetPublicIPV6Gateway() (net.IP, error) { - // simply find the default gw for a well known public ip. in this case - // we use the public google dns service - return n.ndmz.GetDefaultGateway(net.ParseIP("2001:4860:4860::8888")) -} - -// GetSubnet of a local network resource identified by the network ID, ipv4 and ipv6 -// subnet respectively -func (n *networker) GetSubnet(networkID pkg.NetID) (net.IPNet, error) { - localNR, err := n.networkOf(networkID) - if err != nil { - return net.IPNet{}, errors.Wrapf(err, "couldn't load network with id (%s)", networkID) - } - - return localNR.Subnet.IPNet, nil -} - -// GetNet of a network identified by the network ID -func (n *networker) GetNet(networkID pkg.NetID) (net.IPNet, error) { - localNR, err := n.networkOf(networkID) - if err != nil { - return net.IPNet{}, errors.Wrapf(err, "couldn't load network with id (%s)", networkID) - } - - return localNR.NetworkIPRange.IPNet, nil -} - -// GetDefaultGwIP returns the IPs of the default gateways inside the network -// resource identified by the network ID on the local node, for IPv4 and IPv6 -// respectively -func (n *networker) GetDefaultGwIP(networkID pkg.NetID) (net.IP, net.IP, error) { - localNR, err := n.networkOf(networkID) - if err != nil { - return nil, nil, errors.Wrapf(err, "couldn't load network with id (%s)", networkID) - } - - // only IP4 atm - ip := localNR.Subnet.IP.To4() - if ip == nil { - return nil, nil, errors.New("nr subnet is not valid IPv4") - } - - // defaut gw is currently implied to be at `x.x.x.1` - // also a subnet in a NR is assumed to be a /24 - ip[len(ip)-1] = 1 - - // ipv6 is derived from the ipv4 - return ip, nr.Convert4to6(string(networkID), ip), nil -} - -// GetIPv6From4 generates an IPv6 address from a given IPv4 address in a NR -func (n *networker) GetIPv6From4(networkID pkg.NetID, ip net.IP) (net.IPNet, error) { - if ip.To4() == nil { - return net.IPNet{}, errors.New("invalid IPv4 address") - } - return net.IPNet{IP: nr.Convert4to6(string(networkID), ip), Mask: net.CIDRMask(64, 128)}, nil -} - -func (n *networker) SetPublicExitDevice(iface string) error { - link, err := netlink.LinkByName(iface) - if err != nil { - return err - } - - return public.SetPublicExitLink(link) -} - -func (n *networker) Interfaces(iface string, netns string) (pkg.Interfaces, error) { - getter := func(iface string) ([]netlink.Link, error) { - if iface != "" { - l, err := netlink.LinkByName(iface) - if err != nil { - return nil, errors.Wrapf(err, "failed to get interface %s", iface) - } - return []netlink.Link{l}, nil - } - - all, err := netlink.LinkList() - if err != nil { - return nil, err - } - filtered := all[:0] - for _, l := range all { - name := l.Attrs().Name - - if name == "lo" || - (l.Type() != "device" && name != types.DefaultBridge) { - - continue - } - - filtered = append(filtered, l) - } - - return filtered, nil - } - - interfaces := make(map[string]pkg.Interface) - f := func(_ ns.NetNS) error { - links, err := getter(iface) - if err != nil { - return errors.Wrapf(err, "failed to get interfaces (query: '%s')", iface) - } - - for _, link := range links { - - addrs, err := netlink.AddrList(link, netlink.FAMILY_ALL) - if err != nil { - return errors.Wrapf(err, "failed to list addresses of interfaces %s", iface) - } - ips := make([]net.IPNet, 0, len(addrs)) - for _, addr := range addrs { - ip := addr.IP - if ip6 := ip.To16(); ip6 != nil { - // ipv6 - if !ip6.IsGlobalUnicast() || ifaceutil.IsULA(ip6) { - // skip if not global or is ula address - continue - } - } - - ips = append(ips, *addr.IPNet) - } - - interfaces[link.Attrs().Name] = pkg.Interface{ - Name: link.Attrs().Name, - Mac: link.Attrs().HardwareAddr.String(), - IPs: ips, - } - } - - return nil - } - - if netns != "" { - netNS, err := namespace.GetByName(netns) - if err != nil { - return pkg.Interfaces{}, errors.Wrapf(err, "failed to get network namespace %s", netns) - } - defer netNS.Close() - - if err := netNS.Do(f); err != nil { - return pkg.Interfaces{}, err - } - } else { - if err := f(nil); err != nil { - return pkg.Interfaces{}, err - } - } - - return pkg.Interfaces{Interfaces: interfaces}, nil -} - -// [obsolete] use Interfaces instead Addrs return the IP addresses of interface -func (n *networker) Addrs(iface string, netns string) (ips []net.IP, mac string, err error) { - if iface == "" { - return ips, mac, fmt.Errorf("iface cannot be empty") - } - interfaces, err := n.Interfaces(iface, netns) - if err != nil { - return nil, "", err - } - - inf := interfaces.Interfaces[iface] - mac = inf.Mac - for _, ip := range inf.IPs { - ips = append(ips, ip.IP) - } - return -} - -// CreateNR implements pkg.Networker interface -func (n *networker) CreateNR(wl gridtypes.WorkloadID, netNR pkg.Network) (string, error) { - log.Info().Str("network", string(netNR.NetID)).Msg("create network resource") - - if err := n.storeNetwork(wl, netNR); err != nil { - return "", errors.Wrap(err, "failed to store network object") - } - - // check if there is a reserved wireguard port for this NR already - // or if we need to update it - storedNR, err := n.networkOf(netNR.NetID) - if err != nil && !os.IsNotExist(err) { - return "", errors.Wrap(err, "failed to load previous network setup") - } - - if err == nil { - if err := n.releasePort(storedNR.WGListenPort); err != nil { - return "", err - } - } - - if err := n.reservePort(netNR.WGListenPort); err != nil { - return "", err - } - - netr := nr.New(netNR, n.myceliumKeyDir) - - cleanup := func() { - log.Error().Msg("clean up network resource") - if err := netr.Delete(); err != nil { - log.Error().Err(err).Msg("error during deletion of network resource after failed deployment") - } - if err := n.releasePort(netNR.WGListenPort); err != nil { - log.Error().Err(err).Msg("release wireguard port failed") - } - } - - defer func() { - if err != nil { - cleanup() - } - }() - - wgName, err := netr.WGName() - if err != nil { - return "", errors.Wrap(err, "failed to get wg interface name for network resource") - } - - log.Info().Msg("create network resource namespace") - if err = netr.Create(); err != nil { - return "", errors.Wrap(err, "failed to create network resource") - } - - // setup mycelium - if err := netr.SetMycelium(); err != nil { - return "", errors.Wrap(err, "failed to setup mycelium") - } - - exists, err := netr.HasWireguard() - if err != nil { - return "", errors.Wrap(err, "failed to check if network resource has wireguard setup") - } - - if !exists { - var wg *wireguard.Wireguard - wg, err = public.NewWireguard(wgName) - if err != nil { - return "", errors.Wrapf(err, "failed to create wg interface for network resource '%s'", netNR.NetID) - } - if err = netr.SetWireguard(wg); err != nil { - return "", errors.Wrap(err, "failed to setup wireguard interface for network resource") - } - } - - nsName, err := netr.Namespace() - if err != nil { - return "", errors.Wrap(err, "failed to get network resource namespace") - } - - if err = n.ndmz.AttachNR(string(netNR.NetID), nsName, n.ipamLeaseDir); err != nil { - return "", errors.Wrapf(err, "failed to attach network resource to DMZ bridge") - } - - if err = netr.ConfigureWG(netNR.WGPrivateKey); err != nil { - return "", errors.Wrap(err, "failed to configure network resource") - } - - return netr.Namespace() -} - -func (n *networker) rmNetwork(wl gridtypes.WorkloadID) error { - netID, err := zos.NetworkIDFromWorkloadID(wl) - if err != nil { - return err - } - - rm := []string{ - filepath.Join(n.networkDir, netID.String()), - filepath.Join(n.linkDir, wl.String()), - } - - for _, p := range rm { - if err := os.Remove(p); err != nil && !os.IsNotExist(err) { - log.Error().Err(err).Str("path", p).Msg("failed to delete file") - } - } - - return nil -} - -func (n *networker) storeNetwork(wl gridtypes.WorkloadID, network pkg.Network) error { - // map the network ID to the network namespace - path := filepath.Join(n.networkDir, string(network.NetID)) - file, err := os.Create(path) - if err != nil { - return err - } - defer file.Close() - - writer, err := versioned.NewWriter(file, NetworkSchemaLatestVersion) - if err != nil { - return err - } - - enc := json.NewEncoder(writer) - if err := enc.Encode(&network); err != nil { - return err - } - link := filepath.Join(n.linkDir, wl.String()) - if err := os.Symlink(filepath.Join("../", string(network.NetID)), link); err != nil && !os.IsExist(err) { - return errors.Wrap(err, "failed to create network symlink") - } - return nil -} - -// DeleteNR implements pkg.Networker interface -func (n *networker) DeleteNR(wl gridtypes.WorkloadID) error { - netID, err := zos.NetworkIDFromWorkloadID(wl) - if err != nil { - return err - } - netNR, err := n.networkOf(netID) - if err != nil { - return err - } - - nr := nr.New(netNR, n.myceliumKeyDir) - - if err := nr.Delete(); err != nil { - return errors.Wrap(err, "failed to delete network resource") - } - - if err := n.releasePort(netNR.WGListenPort); err != nil { - log.Error().Err(err).Msg("release wireguard port failed") - // TODO: should we return the error ? - } - - if err := n.ndmz.DetachNR(string(netNR.NetID), n.ipamLeaseDir); err != nil { - log.Error().Err(err).Msg("failed to detach network from ndmz") - } - - if err := n.rmNetwork(wl); err != nil { - log.Error().Err(err).Msg("failed to remove file mapping between network ID and namespace") - } - - return nil -} - -func (n *networker) Namespace(id zos.NetID) string { - return fmt.Sprintf("n-%s", id) -} - -func (n *networker) UnsetPublicConfig() error { - id := n.identity.NodeID(context.Background()) - _, err := public.EnsurePublicSetup(id, environment.MustGet().PubVlan, nil) - return err -} - -// Set node public namespace config -func (n *networker) SetPublicConfig(cfg pkg.PublicConfig) error { - if cfg.Equal(pkg.PublicConfig{}) { - return fmt.Errorf("public config cannot be unset, only modified") - } - - current, err := public.LoadPublicConfig() - if err != nil && err != public.ErrNoPublicConfig { - return errors.Wrapf(err, "failed to load current public configuration") - } - - if current != nil && current.Equal(cfg) { - // nothing to do - return nil - } - - id := n.identity.NodeID(context.Background()) - _, err = public.EnsurePublicSetup(id, environment.MustGet().PubVlan, &cfg) - if err != nil { - return errors.Wrap(err, "failed to apply public config") - } - - if err := public.SavePublicConfig(cfg); err != nil { - return errors.Wrap(err, "failed to store public config") - } - - // when public setup is updated. it can take a while but the capacityd - // will detect this change and take necessary actions to update the node - ctx := context.Background() - sk := ed25519.PrivateKey(n.identity.PrivateKey(ctx)) - ns, err := yggdrasil.NewYggdrasilNamespace(public.PublicNamespace) - if err != nil { - return errors.Wrap(err, "failed to setup public namespace for yggdrasil") - } - ygg, err := yggdrasil.EnsureYggdrasil(context.Background(), sk, ns) - if err != nil { - return err - } - - // since the public ip might have changed, it seems sometimes ygg needs to be - // restart to use new public ip - if err := ygg.Restart(zinit.Default()); err != nil { - log.Error().Err(err).Msg("failed to restart yggdrasil service") - } - - // if yggdrasil is living inside public namespace - // we still need to setup ndmz to also have yggdrasil but we set the yggdrasil interface - // a different Ip that lives inside the yggdrasil range. - dmzYgg, err := yggdrasil.NewYggdrasilNamespace(n.ndmz.Namespace()) - if err != nil { - return errors.Wrap(err, "failed to setup ygg for dmz namespace") - } - - ip, err := ygg.SubnetFor([]byte(fmt.Sprintf("ygg:%s", n.ndmz.Namespace()))) - if err != nil { - return errors.Wrap(err, "failed to calculate ip for ygg inside dmz") - } - - gw, err := ygg.Gateway() - if err != nil { - return err - } - - if err := dmzYgg.SetYggIP(ip, gw.IP); err != nil { - return errors.Wrap(err, "failed to set yggdrasil ip for dmz") - } - - return nil -} - -func (n *networker) GetPublicExitDevice() (pkg.ExitDevice, error) { - exit, err := public.GetCurrentPublicExitLink() - if err != nil { - return pkg.ExitDevice{}, err - } - - // if exit is over veth then we going over zos bridge - // hence it's a single nic setup - if ok, _ := bootstrap.VEthFilter(exit); ok { - return pkg.ExitDevice{IsSingle: true}, nil - } - - return pkg.ExitDevice{IsDual: true, AsDualInterface: exit.Attrs().Name}, nil -} - -// Get node public namespace config -func (n *networker) GetPublicConfig() (pkg.PublicConfig, error) { - // TODO: instea of loading, this actually must get - // from reality. - cfg, err := public.GetPublicSetup() - if err != nil { - return pkg.PublicConfig{}, err - } - return cfg, nil -} - -func (n *networker) networkOf(id zos.NetID) (nr pkg.Network, err error) { - path := filepath.Join(n.networkDir, string(id)) - file, err := os.OpenFile(path, os.O_RDWR, 0660) - if err != nil { - return nr, err - } - defer file.Close() - - reader, err := versioned.NewReader(file) - if versioned.IsNotVersioned(err) { - // old data that doesn't have any version information - if _, err := file.Seek(0, 0); err != nil { - return nr, err - } - - reader = versioned.NewVersionedReader(NetworkSchemaLatestVersion, file) - } else if err != nil { - return nr, err - } - - var net pkg.Network - dec := json.NewDecoder(reader) - - version := reader.Version() - // validV1 := versioned.MustParseRange(fmt.Sprintf("=%s", pkg.NetworkSchemaV1)) - validLatest := versioned.MustParseRange(fmt.Sprintf("<=%s", NetworkSchemaLatestVersion.String())) - - if validLatest(version) { - if err := dec.Decode(&net); err != nil { - return nr, err - } - } else { - return nr, fmt.Errorf("unknown network object version (%s)", version) - } - - return net, nil -} - -func (n *networker) reservePort(port uint16) error { - log.Debug().Uint16("port", port).Msg("reserve wireguard port") - err := n.portSet.Add(uint(port)) - if err != nil { - return errors.Wrap(err, "wireguard listen port already in use, pick another one") - } - - return nil -} - -func (n *networker) releasePort(port uint16) error { - log.Debug().Uint16("port", port).Msg("release wireguard port") - n.portSet.Remove(uint(port)) - return nil -} - -func (n *networker) DMZAddresses(ctx context.Context) <-chan pkg.NetlinkAddresses { - ch := make(chan pkg.NetlinkAddresses) - go func() { - for { - select { - case <-ctx.Done(): - return - case <-time.After(30 * time.Second): - ips, err := n.ndmz.GetIP(ndmz.FamilyAll) - if err != nil { - log.Error().Err(err).Msg("failed to get dmz IPs") - } - ch <- ips - } - } - }() - - return ch -} - -func (n *networker) Metrics() (pkg.NetResourceMetrics, error) { - links, err := os.ReadDir(n.linkDir) - if err != nil { - return nil, errors.Wrap(err, "failed to list networks") - } - - metrics := make(pkg.NetResourceMetrics) - for _, link := range links { - if link.IsDir() { - continue - } - - wl := link.Name() - logger := log.With().Str("workload", wl).Logger() - sym, err := os.Readlink(filepath.Join(n.linkDir, wl)) - if err != nil { - logger.Error().Err(err).Msg("failed to get network name from workload link") - continue - } - nsName := n.Namespace(zos.NetID(filepath.Base(sym))) - logger.Debug().Str("namespace", nsName).Msg("collecting namespace statistics") - nr, err := namespace.GetByName(nsName) - if err != nil { - // this happens on some node. it's weird the the namespace is suddenly gone - // while the workload is still active. - // TODO: investigate - // Note: I set it to debug because it shows error in logs of logs - logger.Debug(). - Str("namespace", nsName). - Err(err). - Msg("failed to get network namespace from workload") - continue - } - - defer nr.Close() - err = nr.Do(func(_ ns.NetNS) error { - // get stats of public interface. - m, err := metricsForNics("public") - if err != nil { - return err - } - - metrics[wl] = m - return nil - }) - if err != nil { - log.Error().Err(err).Msg("failed to collect metrics for network") - } - } - - return metrics, nil -} - -func (n *networker) YggAddresses(ctx context.Context) <-chan pkg.NetlinkAddresses { - ch := make(chan pkg.NetlinkAddresses) - go func() { - for { - select { - case <-ctx.Done(): - return - case <-time.After(30 * time.Second): - ips, err := n.ndmz.GetIPFor(yggdrasil.YggNSInf) - if err != nil { - log.Error().Err(err).Str("inf", yggdrasil.YggIface).Msg("failed to get public IPs") - } - filtered := ips[:0] - for _, ip := range ips { - if yggdrasil.YggRange.Contains(ip.IP) { - filtered = append(filtered, ip) - } - } - ch <- filtered - } - } - }() - - return ch -} - -func (n *networker) PublicAddresses(ctx context.Context) <-chan pkg.OptionPublicConfig { - ch := make(chan pkg.OptionPublicConfig) - go func() { - for { - select { - case <-ctx.Done(): - return - case <-time.After(30 * time.Second): - cfg, err := n.GetPublicConfig() - ch <- pkg.OptionPublicConfig{ - PublicConfig: cfg, - HasPublicConfig: err == nil, - } - } - } - }() - - return ch -} - -func (n *networker) ZOSAddresses(ctx context.Context) <-chan pkg.NetlinkAddresses { - var index int - _ = backoff.Retry(func() error { - link, err := netlink.LinkByName(types.DefaultBridge) - if err != nil { - log.Error().Err(err).Msg("can't get defaut bridge") - return err - } - index = link.Attrs().Index - return nil - }, backoff.NewConstantBackOff(2*time.Second)) - - get := func() pkg.NetlinkAddresses { - var result pkg.NetlinkAddresses - link, err := netlink.LinkByName(types.DefaultBridge) - if err != nil { - log.Error().Err(err).Msgf("could not find the '%s' bridge", types.DefaultBridge) - return nil - } - values, err := netlink.AddrList(link, netlink.FAMILY_ALL) - if err != nil { - log.Error().Err(err).Msgf("could not list the '%s' bridge ips", types.DefaultBridge) - return nil - } - for _, value := range values { - result = append(result, *value.IPNet) - } - - slices.SortFunc(result, func(a, b net.IPNet) int { - return bytes.Compare(a.IP, b.IP) - }) - - return result - } - - updateChan := make(chan netlink.AddrUpdate) - if err := netlink.AddrSubscribe(updateChan, ctx.Done()); err != nil { - log.Error().Err(err).Msgf("could not subscribe to addresses updates") - return nil - } - - ch := make(chan pkg.NetlinkAddresses) - var current pkg.NetlinkAddresses - go func() { - defer close(ch) - - for { - select { - case <-ctx.Done(): - return - case update := <-updateChan: - if update.LinkIndex != index || !update.NewAddr { - continue - } - new := get() - if slices.CompareFunc(current, new, func(a, b net.IPNet) int { - return bytes.Compare(a.IP, b.IP) - }) == 0 { - // if the 2 sets of IPs are identitcal, we don't - // trigger the event - continue - } - current = new - ch <- new - } - } - }() - - return ch -} - -func (n *networker) syncWGPorts() error { - names, err := namespace.List("n-") - if err != nil { - return err - } - - readPort := func(name string) (int, error) { - netNS, err := namespace.GetByName(name) - if err != nil { - return 0, err - } - defer netNS.Close() - - ifaceName := strings.Replace(name, "n-", "w-", 1) - - var port int - err = netNS.Do(func(_ ns.NetNS) error { - link, err := wireguard.GetByName(ifaceName) - if err != nil { - return err - } - d, err := link.Device() - if err != nil { - return err - } - - port = d.ListenPort - return nil - }) - if err != nil { - return 0, err - } - - return port, nil - } - - for _, name := range names { - port, err := readPort(name) - if err != nil { - log.Error().Err(err).Str("namespace", name).Msgf("failed to read port for network namespace") - continue - } - // skip error cause we don't care if there are some duplicate at this point - _ = n.portSet.Add(uint(port)) - } - - return nil -} - -// createNetNS create a network namespace and set lo interface up -func createNetNS(name string) (ns.NetNS, error) { - var netNs ns.NetNS - var err error - if namespace.Exists(name) { - netNs, err = namespace.GetByName(name) - } else { - netNs, err = namespace.Create(name) - } - - if err != nil { - return nil, fmt.Errorf("fail to create network namespace %s: %w", name, err) - } - - err = netNs.Do(func(_ ns.NetNS) error { - return ifaceutil.SetLoUp() - }) - if err != nil { - _ = namespace.Delete(netNs) - return nil, fmt.Errorf("failed to bring lo interface up in namespace %s: %w", name, err) - } - - return netNs, nil -} - -// tapName prefixes the tap name with a t- -func tapName(tname string) (string, error) { - name := fmt.Sprintf("t-%s", tname) - if len(name) > 15 { - return "", errors.Errorf("tap name too long %s", name) - } - return name, nil -} - -func pubTapName(resID string) (string, error) { - name := fmt.Sprintf("p-%s", resID) - if len(name) > 15 { - return "", errors.Errorf("tap name too long %s", name) - } - return name, nil -} - -var ulaPrefix = net.IPNet{ - IP: net.ParseIP("fc00::"), - Mask: net.CIDRMask(7, 128), -} - -func isULA(ip net.IP) bool { - return ulaPrefix.Contains(ip) -} - -var yggPrefix = net.IPNet{ - IP: net.ParseIP("200::"), - Mask: net.CIDRMask(7, 128), -} - -func isYgg(ip net.IP) bool { - return yggPrefix.Contains(ip) -} - -func metricsForNics(nics ...string) (m pkg.NetMetric, err error) { - for _, nic := range nics { - l, err := netlink.LinkByName(nic) - if err != nil { - return pkg.NetMetric{}, err - } - - stats := l.Attrs().Statistics - m.NetRxBytes += stats.RxBytes - m.NetTxBytes += stats.TxBytes - m.NetRxPackets += stats.RxPackets - m.NetTxPackets += stats.TxPackets - } - - return -} diff --git a/pkg/network/nr/net_resource.go b/pkg/network/nr/net_resource.go deleted file mode 100644 index ac5088c84..000000000 --- a/pkg/network/nr/net_resource.go +++ /dev/null @@ -1,882 +0,0 @@ -package nr - -import ( - "bytes" - "crypto/md5" - "encoding/json" - "fmt" - "net" - "os" - "os/exec" - "path/filepath" - "slices" - "strings" - "time" - - "github.com/threefoldtech/zos/pkg/environment" - "github.com/threefoldtech/zos/pkg/gridtypes/zos" - "github.com/threefoldtech/zos/pkg/network/ifaceutil" - "github.com/threefoldtech/zos/pkg/network/macvlan" - "github.com/threefoldtech/zos/pkg/network/options" - "github.com/threefoldtech/zos/pkg/network/tuntap" - "github.com/threefoldtech/zos/pkg/zinit" - - mapset "github.com/deckarep/golang-set" - - "github.com/pkg/errors" - - "github.com/containernetworking/plugins/pkg/ns" - "github.com/rs/zerolog/log" - "github.com/threefoldtech/zos/pkg" - "github.com/threefoldtech/zos/pkg/network/bridge" - "github.com/threefoldtech/zos/pkg/network/namespace" - "github.com/threefoldtech/zos/pkg/network/nft" - "github.com/threefoldtech/zos/pkg/network/wireguard" - "github.com/vishvananda/netlink" -) - -const ( - myceliumInterfaceName = "br-my" -) - -var ( - myceliumIpBase = []byte{ - 0xff, 0x0f, - } - - invalidMyceliumSeeds = [][]byte{ - {0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, - {0xff, 0xff, 0xff, 0xff, 0xff, 0xff}, - } -) - -type MyceliumInspection struct { - PublicHexKey string `json:"publicKey"` - Address net.IP `json:"address"` -} - -// Gateway derive the gateway IP from the mycelium IP in the /64 range. It also -// return the full /64 subnet. -func (m *MyceliumInspection) Gateway() (subnet net.IPNet, gw net.IPNet, err error) { - // here we need to return 2 things: - // - the IP range /64 for that IP - // - the gw address for that /64 range - - ipv6 := m.Address.To16() - if ipv6 == nil { - return gw, subnet, fmt.Errorf("invalid mycelium ip") - } - - ip := make(net.IP, net.IPv6len) - copy(ip[0:8], ipv6[0:8]) - - subnet = net.IPNet{ - IP: make(net.IP, net.IPv6len), - Mask: net.CIDRMask(64, 128), - } - copy(subnet.IP, ip) - - ip[net.IPv6len-1] = 1 - - gw = net.IPNet{ - IP: ip, - Mask: net.CIDRMask(64, 128), - } - - return -} - -func (m *MyceliumInspection) IP(seed zos.Bytes) (ip net.IPNet, gw net.IPNet, err error) { - - if slices.ContainsFunc(invalidMyceliumSeeds, func(b []byte) bool { - return slices.Equal(seed, b) - }) { - return ip, gw, fmt.Errorf("invalid seed") - } - - // first find the base subnet. - ip, gw, err = m.Gateway() - if err != nil { - return ip, gw, err - } - - // the subnet already have the /64 part of the network (that's 8 bytes) - // we then add a fixed 2 bytes this will avoid reusing the same gw or - // the device ip - copy(ip.IP[8:10], myceliumIpBase) - // then finally we use the 6 bytes seed to build the rest of the IP - copy(ip.IP[10:16], seed) - - return -} - -// NetResource holds the logic to configure an network resource -type NetResource struct { - id pkg.NetID - // local network resources - resource pkg.Network - // network IP range, usually a /16 - networkIPRange net.IPNet - - // keyDir location where keys can be stored - keyDir string -} - -// New creates a new NetResource object -// iprange is the full network subnet -// keyDir is the path where keys (mainly mycelium) -// is stored. -func New(nr pkg.Network, keyDir string) *NetResource { - return &NetResource{ - //fix here - id: nr.NetID, - resource: nr, - networkIPRange: nr.NetworkIPRange.IPNet, - keyDir: keyDir, - } -} - -func (nr *NetResource) String() string { - b, err := json.Marshal(nr.resource) - if err != nil { - panic(err) - } - return string(b) -} - -// ID returns the network ID in which the NetResource is defined -func (nr *NetResource) ID() string { - return string(nr.id) -} - -// BridgeName returns the name of the bridge to create for the network -// resource in the host network namespace -func (nr *NetResource) BridgeName() (string, error) { - name := fmt.Sprintf("b-%s", nr.id) - if len(name) > 15 { - return "", errors.Errorf("bridge namespace too long %s", name) - } - return name, nil -} - -func (nr *NetResource) myceliumBridgeName() (string, error) { - name := fmt.Sprintf("m-%s", nr.id) - if len(name) > 15 { - return "", errors.Errorf("bridge namespace too long %s", name) - } - return name, nil -} - -// Namespace returns the name of the network namespace to create for the network resource -func (nr *NetResource) Namespace() (string, error) { - name := fmt.Sprintf("n-%s", nr.id) - if len(name) > 15 { - return "", errors.Errorf("network namespace too long %s", name) - } - return name, nil -} - -// NRIface returns name of netresource local interface -func (nr *NetResource) NRIface() (string, error) { - name := fmt.Sprintf("n-%s", nr.id) - if len(name) > 15 { - return "", errors.Errorf("NR interface name too long %s", name) - } - return name, nil -} - -// WGName returns the name of the wireguard interface to create for the network resource -func (nr *NetResource) WGName() (string, error) { - wgName := fmt.Sprintf("w-%s", nr.id) - if len(wgName) > 15 { - return "", errors.Errorf("network namespace too long %s", wgName) - } - return wgName, nil -} - -// Create setup the basic components of the network resource -// network namespace, bridge, wireguard interface and veth pair -func (nr *NetResource) Create() error { - log.Debug().Str("nr", nr.String()).Msg("create network resource") - - if err := nr.ensureNRBridge(); err != nil { - return err - } - if err := nr.createNetNS(); err != nil { - return err - } - if err := nr.attachToNRBridge(); err != nil { - return err - } - - if err := nr.applyFirewall(); err != nil { - return err - } - - return nil -} - -func (nr *NetResource) myceliumServiceName() string { - return fmt.Sprintf("mycelium-%s", nr.ID()) -} - -func (nr *NetResource) MyceliumIP(seed zos.Bytes) (ip net.IPNet, gw net.IPNet, err error) { - if len(seed) != zos.MyceliumIPSeedLen { - return ip, gw, fmt.Errorf("invalid mycelium seed length") - } - - mycelium, err := nr.inspectMycelium(filepath.Join(nr.keyDir, nr.ID())) - if os.IsNotExist(err) { - return ip, gw, fmt.Errorf("mycelium is not configured for this network resource") - } else if err != nil { - return ip, gw, err - } - - return mycelium.IP(seed) -} - -// AttachMycelium attaches a tap device to mycelium, move it to the host namespace -// to it can be used by VMs later. -// It also return the IP that should be used with the interface -func (nr *NetResource) AttachMycelium(name string) (err error) { - brName, err := nr.myceliumBridgeName() - if err != nil { - return err - } - - _, err = tuntap.CreateTap(name, brName) - if err != nil { - return err - } - - return nil -} - -func (nr *NetResource) SetMycelium() (err error) { - if nr.resource.Mycelium == nil { - // no mycelium - return nil - } - - peers, err := environment.GetConfig() - if err != nil { - return errors.Wrap(err, "failed to get public mycelium peer list") - } - - config := nr.resource.Mycelium - // create the bridge. - if err := nr.ensureMyceliumBridge(); err != nil { - return err - } - - keyFile := filepath.Join(nr.keyDir, nr.ID()) - - defer func() { - if err != nil { - os.Remove(keyFile) - } - }() - - if err = os.WriteFile(keyFile, config.Key, 0444); err != nil { - return errors.Wrap(err, "failed to store mycelium key") - } - - if err := nr.ensureMyceliumNetwork(keyFile); err != nil { - return err - } - - name := nr.myceliumServiceName() - - init := zinit.Default() - exists, err := init.Exists(name) - if err != nil { - return errors.Wrap(err, "failed to check mycelium service") - } - - if exists { - return nil - } - - ns, err := nr.Namespace() - if err != nil { - return err - } - - args := []string{ - "ip", "netns", "exec", ns, - "mycelium", - "--silent", - "--key-file", keyFile, - "--tun-name", "my", - "--peers", - } - - // first append peers from user input. - // right now this is shadowed by Mycelium config validation - // which does not allow custom peer list. - args = AppendFunc(args, config.Peers, func(mp zos.MyceliumPeer) string { - return string(mp) - }) - - // global peers list - args = append(args, peers.Mycelium.Peers...) - - // todo: add custom peers requested by the user - - err = zinit.AddService(name, zinit.InitService{ - Exec: strings.Join(args, " "), - }) - - if err != nil { - return errors.Wrap(err, "failed to add mycelium service for nr") - } - - return init.Monitor(name) -} - -func (nr *NetResource) ensureMyceliumNetwork(keyFile string) error { - // applies network configuration for this mycelium instance inside the proper namespace - mycelium, err := nr.inspectMycelium(keyFile) - if err != nil { - return err - } - - subnet, gw, err := mycelium.Gateway() - if err != nil { - return err - } - - nsName, err := nr.Namespace() - if err != nil { - return err - } - - netNS, err := namespace.GetByName(nsName) - if err != nil { - return err - } - - defer netNS.Close() - - bridgeName, err := nr.myceliumBridgeName() - if err != nil { - return err - } - - if !ifaceutil.Exists(myceliumInterfaceName, netNS) { - log.Debug().Str("create macvlan", myceliumInterfaceName).Msg("attach mycelium to bridge") - if _, err := macvlan.Create(myceliumInterfaceName, bridgeName, netNS); err != nil { - return err - } - } - - return netNS.Do(func(_ ns.NetNS) error { - link, err := netlink.LinkByName(myceliumInterfaceName) - // this should not happen since it has been ensured before - if err != nil { - return err - } - // configure the bridge ip - addresses, err := netlink.AddrList(link, netlink.FAMILY_V6) - if err != nil { - return errors.Wrap(err, "failed to list my-br ip addresses") - } - - if !slices.ContainsFunc(addresses, func(a netlink.Addr) bool { - return slices.Equal(a.IP, gw.IP) - }) { - // If gw Ip is not configured, we set it up - if err := netlink.AddrAdd(link, &netlink.Addr{ - IPNet: &gw, - }); err != nil { - return errors.Wrap(err, "failed to setup mycelium bridge address") - } - } - - if err := netlink.LinkSetUp(link); err != nil { - return errors.Wrap(err, "failed to bring mycelium macvtap up") - } - - // also configure route to the subnet - routes, err := netlink.RouteList(link, netlink.FAMILY_V6) - if err != nil { - return errors.Wrap(err, "failed to list mycelium routes") - } - if !slices.ContainsFunc(routes, func(r netlink.Route) bool { - return r.Dst != nil && slices.Equal(r.Dst.IP, subnet.IP) - }) { - log.Debug().Str("gw", gw.IP.String()).Str("subnet", subnet.String()).Msg("adding mycelium route") - - if err := netlink.RouteAdd(&netlink.Route{ - Dst: &subnet, - LinkIndex: link.Attrs().Index, - }); err != nil { - return errors.Wrap(err, "failed to add mycelium route") - } - } - - return nil - }) - -} - -func (nr *NetResource) inspectMycelium(keyFile string) (inspection MyceliumInspection, err error) { - _, err = os.Stat(keyFile) - if err != nil { - return inspection, err - } - - // we check if the file exists before we do inspect because mycelium will create a random seed - // file if file does not exist - output, err := exec.Command("mycelium", "--key-file", keyFile, "inspect", "--json").Output() - if err != nil { - return inspection, errors.Wrap(err, "failed to inspect mycelium key") - } - - if err := json.Unmarshal(output, &inspection); err != nil { - return inspection, errors.Wrap(err, "failed to load mycelium information from key") - } - - return inspection, nil -} - -// only create the mycelium bridge inside the network resource. -// this is done anyway -func (nr *NetResource) ensureMyceliumBridge() error { - name, err := nr.myceliumBridgeName() - if err != nil { - return err - } - - if bridge.Exists(name) { - return nil - } - - log.Info().Str("bridge", name).Msg("create mycelium bridge") - - _, err = bridge.New(name) - if err != nil { - return err - } - - if err := options.Set(name, options.IPv6Disable(true)); err != nil { - return errors.Wrapf(err, "failed to disable ip6 on bridge %s", name) - } - return nil -} - -func wgIP(subnet *net.IPNet) *net.IPNet { - // example: 10.3.1.0 -> 100.64.3.1 - a := subnet.IP[len(subnet.IP)-3] - b := subnet.IP[len(subnet.IP)-2] - - return &net.IPNet{ - IP: net.IPv4(0x64, 0x40, a, b), - Mask: net.CIDRMask(16, 32), - } -} - -// ConfigureWG sets the routes and IP addresses on the -// wireguard interface of the network resources -func (nr *NetResource) ConfigureWG(privateKey string) error { - wgPeers, err := nr.wgPeers() - if err != nil { - return errors.Wrap(err, "failed to wireguard peer configuration") - } - - nsName, err := nr.Namespace() - if err != nil { - return err - } - netNS, err := namespace.GetByName(nsName) - if err != nil { - return fmt.Errorf("network namespace %s does not exits", nsName) - } - - handler := func(_ ns.NetNS) error { - - wgName, err := nr.WGName() - if err != nil { - return err - } - - wg, err := wireguard.GetByName(wgName) - if err != nil { - return errors.Wrapf(err, "failed to get wireguard interface %s", wgName) - } - - if err = wg.Configure(privateKey, int(nr.resource.WGListenPort), wgPeers); err != nil { - return errors.Wrap(err, "failed to configure wireguard interface") - } - - addrs, err := netlink.AddrList(wg, netlink.FAMILY_ALL) - if err != nil { - return err - } - curAddrs := mapset.NewSet() - for _, addr := range addrs { - curAddrs.Add(addr.IPNet.String()) - } - - newAddrs := mapset.NewSet() - newAddrs.Add(wgIP(&nr.resource.Subnet.IPNet).String()) - - toRemove := curAddrs.Difference(newAddrs) - toAdd := newAddrs.Difference(curAddrs) - - log.Info().Msgf("current %s", curAddrs.String()) - log.Info().Msgf("to add %s", toAdd.String()) - log.Info().Msgf("to remove %s", toRemove.String()) - - for addr := range toAdd.Iter() { - addr, _ := addr.(string) - log.Debug().Str("ip", addr).Msg("set ip on wireguard interface") - if err := wg.SetAddr(addr); err != nil && !os.IsExist(err) { - return errors.Wrapf(err, "failed to set address %s on wireguard interface %s", addr, wg.Attrs().Name) - } - } - - for addr := range toRemove.Iter() { - addr, _ := addr.(string) - log.Debug().Str("ip", addr).Msg("unset ip on wireguard interface") - if err := wg.UnsetAddr(addr); err != nil && !os.IsNotExist(err) { - return errors.Wrapf(err, "failed to unset address %s on wireguard interface %s", addr, wg.Attrs().Name) - } - } - - route := &netlink.Route{ - LinkIndex: wg.Attrs().Index, - Dst: &nr.networkIPRange, - } - if err := netlink.RouteAdd(route); err != nil && !os.IsExist(err) { - log.Error(). - Err(err). - Str("route", route.String()). - Msg("fail to set route") - return errors.Wrapf(err, "failed to add route %s", route.String()) - } - - return nil - } - - return netNS.Do(handler) -} - -// Delete removes all the interfaces and namespaces created by the Create method -func (nr *NetResource) Delete() error { - netnsName, err := nr.Namespace() - if err != nil { - return err - } - nrBrName, err := nr.BridgeName() - if err != nil { - return err - } - - myBrName, err := nr.myceliumBridgeName() - if err != nil { - return err - } - - myceliumName := nr.myceliumServiceName() - init := zinit.Default() - exists, err := init.Exists(myceliumName) - if err == nil && exists { - // we use StopMultiple instead of StopWait because multiple does an extra wait and - // verification after a service is sig-killed - if err := init.StopMultiple(10*time.Second, myceliumName); err != nil { - log.Error().Err(err).Msg("failed to stop mycelium for network resource") - } - - _ = init.Forget(myceliumName) - _ = zinit.RemoveService(myceliumName) - - keyFile := filepath.Join(nr.keyDir, nr.ID()) - _ = os.Remove(keyFile) - } - - if bridge.Exists(nrBrName) { - if err := bridge.Delete(nrBrName); err != nil { - log.Error(). - Err(err). - Str("bridge", nrBrName). - Msg("failed to delete network resource bridge") - return err - } - } - - if bridge.Exists(myBrName) { - if err := bridge.Delete(myBrName); err != nil { - log.Error(). - Err(err). - Str("bridge", myBrName). - Msg("failed to delete network mycelium bridge") - return err - } - } - - if namespace.Exists(netnsName) { - netResNS, err := namespace.GetByName(netnsName) - if err != nil { - return err - } - // don't explicitly netResNS.Close() the netResNS here, namespace.Delete will take care of it - if err := namespace.Delete(netResNS); err != nil { - log.Error(). - Err(err). - Str("namespace", netnsName). - Msg("failed to delete network resource namespace") - } - } - - return nil -} - -func (nr *NetResource) wgPeers() ([]*wireguard.Peer, error) { - - wgPeers := make([]*wireguard.Peer, 0, len(nr.resource.Peers)+1) - - for _, peer := range nr.resource.Peers { - - allowedIPs := make([]string, 0, len(peer.AllowedIPs)) - for _, ip := range peer.AllowedIPs { - allowedIPs = append(allowedIPs, ip.String()) - } - - wgPeer := &wireguard.Peer{ - PublicKey: string(peer.WGPublicKey), - AllowedIPs: allowedIPs, - Endpoint: peer.Endpoint, - } - - log.Info().Str("peer prefix", peer.Subnet.String()).Msg("generate wireguard configuration for peer") - wgPeers = append(wgPeers, wgPeer) - } - - return wgPeers, nil -} - -func (nr *NetResource) createNetNS() error { - name, err := nr.Namespace() - if err != nil { - return err - } - - if namespace.Exists(name) { - return nil - } - - log.Info().Str("namespace", name).Msg("Create namespace") - - netNS, err := namespace.Create(name) - if err != nil { - return err - } - defer netNS.Close() - err = netNS.Do(func(_ ns.NetNS) error { - if err := options.SetIPv6Forwarding(true); err != nil { - return err - } - if err := ifaceutil.SetLoUp(); err != nil { - return err - } - return nil - }) - return err -} - -// attachToNRBridge creates a macvlan interface in the NR namespace, and attaches -// it to the NR bridge -func (nr *NetResource) attachToNRBridge() error { - // are we really sure length is always <= 5 ? - nsName, err := nr.Namespace() - if err != nil { - return err - } - nrIfaceName, err := nr.NRIface() - if err != nil { - return err - } - - netNS, err := namespace.GetByName(nsName) - if err != nil { - return fmt.Errorf("network namespace %s does not exits", nsName) - } - defer netNS.Close() - - bridgeName, err := nr.BridgeName() - if err != nil { - return err - } - - if !ifaceutil.Exists(nrIfaceName, netNS) { - log.Debug().Str("create macvlan", nrIfaceName).Msg("attachNRToBridge") - if _, err := macvlan.Create(nrIfaceName, bridgeName, netNS); err != nil { - return err - } - } - - var handler = func(_ ns.NetNS) error { - link, err := netlink.LinkByName(nrIfaceName) - if err != nil { - return err - } - - ipnet := nr.resource.Subnet - ipnet.IP[len(ipnet.IP)-1] = 0x01 - log.Info().Str("addr", ipnet.String()).Msg("set address on macvlan interface") - - addr := &netlink.Addr{IPNet: &ipnet.IPNet, Label: ""} - if err = netlink.AddrAdd(link, addr); err != nil && !os.IsExist(err) { - return err - } - - ipv6 := Convert4to6(nr.ID(), ipnet.IP) - addr = &netlink.Addr{IPNet: &net.IPNet{ - IP: ipv6, - Mask: net.CIDRMask(64, 128), - }} - if err = netlink.AddrAdd(link, addr); err != nil && !os.IsExist(err) { - return err - } - - addr = &netlink.Addr{IPNet: &net.IPNet{ - IP: net.ParseIP("fe80::1"), - Mask: net.CIDRMask(64, 128), - }} - if err = netlink.AddrAdd(link, addr); err != nil && !os.IsExist(err) { - return err - } - - return netlink.LinkSetUp(link) - } - return netNS.Do(handler) -} - -// ensureNRBridge creates a bridge in the host namespace -// this bridge is used to connect containers from the name network -func (nr *NetResource) ensureNRBridge() error { - name, err := nr.BridgeName() - if err != nil { - return err - } - - if bridge.Exists(name) { - return nil - } - - log.Info().Str("bridge", name).Msg("Create bridge") - - _, err = bridge.New(name) - if err != nil { - return err - } - - if err := options.Set(name, options.IPv6Disable(true)); err != nil { - return errors.Wrapf(err, "failed to disable ip6 on bridge %s", name) - } - return nil -} - -// HasWireguard checks if network resource has wireguard setup up -func (nr *NetResource) HasWireguard() (bool, error) { - nsName, err := nr.Namespace() - if err != nil { - return false, err - } - - nrNetNS, err := namespace.GetByName(nsName) - if err != nil { - return false, err - } - - defer nrNetNS.Close() - - wgName, err := nr.WGName() - if err != nil { - return false, err - } - exist := false - err = nrNetNS.Do(func(_ ns.NetNS) error { - _, err = wireguard.GetByName(wgName) - - if errors.As(err, &netlink.LinkNotFoundError{}) { - return nil - } else if err != nil { - return err - } - - exist = true - return nil - }) - - return exist, err -} - -// SetWireguard sets wireguard of this network resource -func (nr *NetResource) SetWireguard(wg *wireguard.Wireguard) error { - nsName, err := nr.Namespace() - if err != nil { - return err - } - - nrNetNS, err := namespace.GetByName(nsName) - if err != nil { - return err - } - defer nrNetNS.Close() - - return netlink.LinkSetNsFd(wg, int(nrNetNS.Fd())) -} - -func (nr *NetResource) applyFirewall() error { - nsName, err := nr.Namespace() - if err != nil { - return err - } - - buf := bytes.Buffer{} - if err := fwTmpl.Execute(&buf, nil); err != nil { - return errors.Wrap(err, "failed to build nft rule set") - } - - if err := nft.Apply(&buf, nsName); err != nil { - return errors.Wrap(err, "failed to apply nft rule set") - } - - return nil -} - -// Convert4to6 converts a (private) ipv4 to the corresponding ipv6 -func Convert4to6(netID string, ip net.IP) net.IP { - h := md5.New() - md5NetID := h.Sum([]byte(netID)) - - // pick the last 2 bytes, handle ipv4 in both ipv6 form (leading 0 bytes) - // and ipv4 form - var lastbyte, secondtolastbyte byte - if len(ip) == net.IPv6len { - lastbyte = ip[15] - secondtolastbyte = ip[14] - } else if len(ip) == net.IPv4len { - lastbyte = ip[3] - secondtolastbyte = ip[2] - } - - ipv6 := fmt.Sprintf("fd%x:%x%x:%x%x", md5NetID[0], md5NetID[1], md5NetID[2], md5NetID[3], md5NetID[4]) - ipv6 = fmt.Sprintf("%s:%x::%x", ipv6, secondtolastbyte, lastbyte) - - return net.ParseIP(ipv6) -} - -// AppendFunc appends arrays with automatic map -func AppendFunc[A any, B any, S []A, D []B](d D, s S, f func(A) B) D { - d = slices.Grow(d, len(s)) - - for _, e := range s { - d = append(d, f(e)) - } - - return d - -} diff --git a/pkg/network/nr/net_resource_test.go b/pkg/network/nr/net_resource_test.go deleted file mode 100644 index 2679b73f3..000000000 --- a/pkg/network/nr/net_resource_test.go +++ /dev/null @@ -1,213 +0,0 @@ -package nr - -import ( - "fmt" - "net" - "reflect" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/threefoldtech/zos/pkg" - "github.com/vishvananda/netlink" -) - -type testIdentityManager struct { - id string - farm uint64 -} - -var _ pkg.IdentityManager = (*testIdentityManager)(nil) - -func (t *testIdentityManager) StoreKind() string { - return "test" -} - -// NodeID returns the node id (public key) -func (t *testIdentityManager) NodeID() pkg.StrIdentifier { - return pkg.StrIdentifier(t.id) -} - -func (t *testIdentityManager) Address() (pkg.Address, error) { - return pkg.Address(t.id), nil -} - -func (t *testIdentityManager) Farm() (string, error) { - return "test-farm", nil -} - -func (t *testIdentityManager) FarmSecret() (string, error) { - return "", nil -} - -// FarmID return the farm id this node is part of. this is usually a configuration -// that the node is booted with. An error is returned if the farmer id is not configured -func (t *testIdentityManager) FarmID() (pkg.FarmID, error) { - return pkg.FarmID(t.farm), nil -} - -// Sign signs the message with privateKey and returns a signature. -func (t *testIdentityManager) Sign(message []byte) ([]byte, error) { - return nil, fmt.Errorf("not implemented") -} - -// Verify reports whether sig is a valid signature of message by publicKey. -func (t *testIdentityManager) Verify(message, sig []byte) error { - return fmt.Errorf("not implemented") -} - -// Encrypt encrypts message with the public key of the node -func (t *testIdentityManager) Encrypt(message []byte) ([]byte, error) { - return nil, fmt.Errorf("not implemented") -} - -// Decrypt decrypts message with the private of the node -func (t *testIdentityManager) Decrypt(message []byte) ([]byte, error) { - return nil, fmt.Errorf("not implemented") -} - -// EncryptECDH aes encrypt msg using a shared key derived from private key of the node and public key of the other party using Elliptic curve Diffie Helman algorithm -// the nonce if prepended to the encrypted message -func (t *testIdentityManager) EncryptECDH(msg []byte, publicKey []byte) ([]byte, error) { - return nil, fmt.Errorf("not implemented") -} - -// DecryptECDH decrypt aes encrypted msg using a shared key derived from private key of the node and public key of the other party using Elliptic curve Diffie Helman algorithm -func (t *testIdentityManager) DecryptECDH(msg []byte, publicKey []byte) ([]byte, error) { - return nil, fmt.Errorf("not implemented") -} - -// PrivateKey sends the keypair -func (t *testIdentityManager) PrivateKey() []byte { - return nil -} - -func TestNamespace(t *testing.T) { - nr := New(pkg.Network{NetID: "networkd1"}, "") - - nsName, err := nr.Namespace() - require.NoError(t, err) - - brName, err := nr.BridgeName() - require.NoError(t, err) - - wgName, err := nr.WGName() - require.NoError(t, err) - - assert.Equal(t, "n-networkd1", nsName) - assert.Equal(t, "b-networkd1", brName) - assert.Equal(t, "w-networkd1", wgName) -} - -func TestCreateBridge(t *testing.T) { - nr := New(pkg.Network{}, "") - - brName, err := nr.BridgeName() - require.NoError(t, err) - - err = nr.ensureNRBridge() - require.NoError(t, err) - - l, err := netlink.LinkByName(brName) - assert.NoError(t, err) - _, ok := l.(*netlink.Bridge) - assert.True(t, ok) - - // cleanup - _ = netlink.LinkDel(l) -} - -func Test_wgIP(t *testing.T) { - type args struct { - subnet *net.IPNet - } - tests := []struct { - name string - args args - want *net.IPNet - }{ - { - name: "default", - args: args{ - subnet: &net.IPNet{ - IP: net.ParseIP("10.3.1.0"), - Mask: net.CIDRMask(16, 32), - }, - }, - want: &net.IPNet{ - IP: net.ParseIP("100.64.3.1"), - Mask: net.CIDRMask(16, 32), - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := wgIP(tt.args.subnet); !reflect.DeepEqual(got, tt.want) { - t.Errorf("wgIP() = %v, want %v", got, tt.want) - } - }) - } -} - -func Test_convert4to6(t *testing.T) { - type args struct { - netID string - ip net.IP - } - tests := []struct { - name string - args args - want net.IP - }{ - { - name: "valid", - args: args{ - netID: "networkdID", - ip: net.ParseIP("100.127.0.2"), - }, - want: net.ParseIP("fd6e:6574:776f:0000::2"), - }, - { - name: "valid", - args: args{ - netID: "networkdID", - ip: net.ParseIP("100.127.2.16"), - }, - want: net.ParseIP("fd6e:6574:776f:0002::0010"), - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - require.NotNil(t, tt.want) - got := Convert4to6(tt.args.netID, tt.args.ip) - require.NotNil(t, got) - assert.EqualValues(t, tt.want, got) - }) - } -} - -func TestMyceliumGw(t *testing.T) { - data := MyceliumInspection{ - Address: net.ParseIP("3b4:ca67:822d:b0c1:5d6f:c647:1ed8:6ced"), - } - - subnet, gw, err := data.Gateway() - require.NoError(t, err) - - require.Equal(t, net.ParseIP("3b4:ca67:822d:b0c1::1"), gw.IP) - require.Equal(t, "3b4:ca67:822d:b0c1::1/64", gw.String()) - require.Equal(t, "3b4:ca67:822d:b0c1::/64", subnet.String()) - -} -func TestMyceliumIP(t *testing.T) { - data := MyceliumInspection{ - Address: net.ParseIP("3b4:ca67:822d:b0c1:5d6f:c647:1ed8:6ced"), - } - - ip, gw, err := data.IP([]byte{0x00, 0x11, 0x22, 0x33, 0x44, 0x55}) - require.NoError(t, err) - - require.Equal(t, net.ParseIP("3b4:ca67:822d:b0c1:ff0f:11:2233:4455"), ip.IP) - require.Equal(t, "3b4:ca67:822d:b0c1::1/64", gw.String()) - -} diff --git a/pkg/network/nr/nft.go b/pkg/network/nr/nft.go deleted file mode 100644 index 0dce1a6e5..000000000 --- a/pkg/network/nr/nft.go +++ /dev/null @@ -1,62 +0,0 @@ -package nr - -import ( - "text/template" -) - -var fwTmpl *template.Template - -func init() { - fwTmpl = template.Must(template.New("nrfw").Parse(_nft)) -} - -var _nft = ` -flush ruleset - -table inet nat { - chain prerouting { - type nat hook prerouting priority dstnat; policy accept; - } - - chain input { - type nat hook input priority 100; policy accept; - } - - chain output { - type nat hook output priority -100; policy accept; - } - - chain postrouting { - type nat hook postrouting priority srcnat; policy accept; - oifname "public" masquerade fully-random; - } -} - -table inet filter { - chain base_checks { - # allow established/related connections - ct state {established, related} accept - # early drop of invalid connections - ct state invalid drop - } - chain input { - type filter hook input priority 0; policy accept; - jump base_checks - ip6 nexthdr icmpv6 accept - iifname "public" counter drop - } - - chain forward { - type filter hook forward priority 0; policy accept; - # is there already an existing stream? (outgoing) - jump base_checks - # if not, verify if it's new and coming in from the br4-gw network - # if it is, drop it - iifname "public" counter drop - } - - chain output { - type filter hook output priority 0; policy accept; - } -} -` diff --git a/pkg/network/portm/allocator.go b/pkg/network/portm/allocator.go deleted file mode 100644 index cff8c469f..000000000 --- a/pkg/network/portm/allocator.go +++ /dev/null @@ -1,98 +0,0 @@ -package portm - -import ( - "errors" - - "github.com/threefoldtech/zos/pkg/network/portm/backend" -) - -// ErrNoFreePort is returned when trying to reserve a port but all the -// the port of the range have been already reserved -var ErrNoFreePort = errors.New("no free port find") - -// PortRange hold the beginging and end of a range of port -// a PortAllocator can reserve -type PortRange struct { - Start int - End int -} - -// Allocator implements the PortAllocator interface -type Allocator struct { - pRange PortRange - store backend.Store -} - -var _ PortAllocator = (*Allocator)(nil) - -// NewAllocator return a PortAllocator -func NewAllocator(pRange PortRange, store backend.Store) *Allocator { - return &Allocator{ - pRange: pRange, - store: store, - } -} - -// Reserve implements PortAllocator interface -func (a *Allocator) Reserve(ns string) (int, error) { - if err := a.store.Lock(); err != nil { - return 0, err - } - defer func() { - _ = a.store.Unlock() - }() - - allocatedPorts, err := a.store.GetByNS(ns) - if err != nil { - return 0, err - } - - start, err := a.store.LastReserved(ns) - if err != nil { - return 0, err - } - - if start == -1 || start >= a.pRange.End { - // nothing reserve yet - // or we reach the end of the range - // then start from the start to re-use released port - start = a.pRange.Start - } - - for port := start; port <= a.pRange.End; port++ { - if contains(allocatedPorts, port) { - continue - } - - reserved, err := a.store.Reserve(ns, port) - if err != nil { - return 0, err - } - if reserved { - return port, nil - } - } - - return 0, ErrNoFreePort -} - -// Release implements PortAllocator interface -func (a *Allocator) Release(ns string, port int) error { - if err := a.store.Lock(); err != nil { - return err - } - defer func() { - _ = a.store.Unlock() - }() - - return a.store.Release(ns, port) -} - -func contains(s []int, port int) bool { - for i := range s { - if s[i] == port { - return true - } - } - return false -} diff --git a/pkg/network/portm/allocator_test.go b/pkg/network/portm/allocator_test.go deleted file mode 100644 index 838f658d8..000000000 --- a/pkg/network/portm/allocator_test.go +++ /dev/null @@ -1,203 +0,0 @@ -package portm - -import ( - "sync" - "testing" - - mapset "github.com/deckarep/golang-set" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -type testStore struct { - reserved map[string]mapset.Set - last map[string]int - sync.Mutex -} - -func newTestStore() *testStore { - return &testStore{ - reserved: make(map[string]mapset.Set), - last: make(map[string]int), - } -} - -func (s *testStore) Lock() error { - s.Mutex.Lock() - return nil -} -func (s *testStore) Unlock() error { - s.Mutex.Unlock() - return nil -} -func (s *testStore) Reserve(ns string, port int) (bool, error) { - set, ok := s.reserved[ns] - if !ok { - set = mapset.NewSet() - s.reserved[ns] = set - } - - if set.Contains(port) { - return false, nil - } - - s.last[ns] = port - return set.Add(port), nil -} -func (s *testStore) Release(ns string, port int) error { - set, ok := s.reserved[ns] - if !ok { - return nil - } - - set.Remove(port) - return nil -} - -func (s *testStore) GetByNS(ns string) ([]int, error) { - set, ok := s.reserved[ns] - if !ok { - return []int{}, nil - } - - ports := set.ToSlice() - output := make([]int, len(ports)) - for i, p := range ports { - output[i] = p.(int) - } - - return output, nil -} - -func (s *testStore) LastReserved(ns string) (int, error) { - port, ok := s.last[ns] - if !ok { - return -1, nil - } - return port, nil -} - -func (s *testStore) Close() error { - return nil -} - -func TestReserve(t *testing.T) { - store := newTestStore() - pRange := PortRange{ - Start: 1000, - End: 6000, - } - ns := "ns" - alloc := NewAllocator(pRange, store) - - p1, err := alloc.Reserve(ns) - require.NoError(t, err) - assert.True(t, store.reserved[ns].Contains(p1)) - assert.True(t, p1 >= pRange.Start) - assert.True(t, p1 <= pRange.End) - - p2, err := alloc.Reserve(ns) - require.NoError(t, err) - assert.True(t, store.reserved[ns].Contains(p2)) - assert.True(t, p1 != p2) - assert.True(t, p2 >= pRange.Start) - assert.True(t, p2 <= pRange.End) -} - -func TestReserveConcurent(t *testing.T) { - store := newTestStore() - pRange := PortRange{ - Start: 1000, - End: 6000, - } - ns := "ns" - N := 5 - wg := sync.WaitGroup{} - reserved := make([][]int, N) - - for i := 0; i < N; i++ { - wg.Add(1) - go func(reserved [][]int, i int) { - defer wg.Done() - alloc := NewAllocator(pRange, store) - reserved[i] = make([]int, 0, 20) - - for y := 0; y < 20; y++ { - p, err := alloc.Reserve(ns) - require.NoError(t, err) - reserved[i] = append(reserved[i], p) - } - }(reserved, i) - } - - wg.Wait() - // ensure all reserved ports are unique - allReserved := make(map[int]struct{}) - for i := 0; i < N; i++ { - for y := 0; y < 20; y++ { - _, exists := allReserved[reserved[i][y]] - assert.False(t, exists, "same port should not have been reserved twice") - allReserved[reserved[i][y]] = struct{}{} - } - } -} - -func TestReuseReleased(t *testing.T) { - store := newTestStore() - pRange := PortRange{ - Start: 1000, - End: 6000, - } - ns := "ns" - alloc := NewAllocator(pRange, store) - - for i := 0; i <= 5000; i++ { - _, err := alloc.Reserve(ns) - require.NoError(t, err) - } - - _, err := alloc.Reserve(ns) - assert.Equal(t, err, ErrNoFreePort) - - err = alloc.Release(ns, 1000) - require.NoError(t, err) - - port, err := alloc.Reserve(ns) - require.NoError(t, err) - assert.Equal(t, 1000, port) -} - -func TestRelease(t *testing.T) { - store := newTestStore() - _, _ = store.Reserve("ns", 1000) - _, _ = store.Reserve("ns", 1001) - - pRange := PortRange{ - Start: 1000, - End: 6000, - } - - alloc := NewAllocator(pRange, store) - - err := alloc.Release("ns", 1000) - require.NoError(t, err) - assert.False(t, store.reserved["ns"].Contains(1000)) -} - -func BenchmarkReserve(b *testing.B) { - store := newTestStore() - pRange := PortRange{ - Start: 1000, - End: 6000, - } - alloc := NewAllocator(pRange, store) - - for i := 0; i < b.N; i++ { - port, err := alloc.Reserve("ns") - if err == ErrNoFreePort { - break - } - require.NoError(b, err) - _ = port - } -} diff --git a/pkg/network/portm/backend/fs.go b/pkg/network/portm/backend/fs.go deleted file mode 100644 index d40af588a..000000000 --- a/pkg/network/portm/backend/fs.go +++ /dev/null @@ -1,107 +0,0 @@ -package backend - -import ( - "os" - "path/filepath" - "strconv" -) - -const lastPort = "last_reserved" - -type fsStore struct { - *FileLock - root string -} - -// NewFSStore creates a backend for port manager that stores -// the allocated port in a filesystem -func NewFSStore(root string) (Store, error) { - if err := os.MkdirAll(root, 0755); err != nil { - return nil, err - } - - lk, err := NewFileLock(root) - if err != nil { - return nil, err - } - - return &fsStore{ - FileLock: lk, - root: root, - }, nil -} - -func nsPath(root, ns string, port string) string { - return filepath.Join(root, ns, port) -} - -func (s *fsStore) Reserve(ns string, port int) (bool, error) { - fname := nsPath(s.root, ns, strconv.Itoa(port)) - - if err := os.MkdirAll(filepath.Dir(fname), 0755); err != nil { - return false, err - } - - f, err := os.OpenFile(fname, os.O_CREATE|os.O_EXCL|os.O_TRUNC, 0660) - if os.IsExist(err) { - return false, nil - } - if err != nil { - return false, err - } - - if err := f.Close(); err != nil { - os.Remove(f.Name()) - return false, err - } - - // store the reserved port in lastPort file - lastPortFile := nsPath(s.root, ns, lastPort) - err = os.WriteFile(lastPortFile, []byte(strconv.Itoa(port)), 0644) - if err != nil { - return false, err - } - return true, nil -} - -func (s *fsStore) Release(ns string, port int) error { - fname := nsPath(s.root, ns, strconv.Itoa(port)) - - return os.Remove(fname) -} - -func (s *fsStore) LastReserved(ns string) (int, error) { - lastPortFile := nsPath(s.root, ns, lastPort) - data, err := os.ReadFile(lastPortFile) - if os.IsNotExist(err) { - return -1, nil - } - if err != nil { - return 0, err - } - return strconv.Atoi(string(data)) -} - -func (s *fsStore) GetByNS(ns string) ([]int, error) { - var ports []int - dir := filepath.Join(s.root, ns) - - infos, err := os.ReadDir(dir) - if err != nil { - return nil, err - } - // walk through all ips in this network to get the ones which belong to a specific ID - for _, info := range infos { - if info.IsDir() { - continue - } - - p, err := strconv.Atoi(info.Name()) - if err != nil { - return nil, err - } - ports = append(ports, p) - } - - return ports, nil -} diff --git a/pkg/network/portm/backend/fs_test.go b/pkg/network/portm/backend/fs_test.go deleted file mode 100644 index 9d4a7bf6f..000000000 --- a/pkg/network/portm/backend/fs_test.go +++ /dev/null @@ -1,37 +0,0 @@ -package backend - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestReserve(t *testing.T) { - dir := t.TempDir() - - ns1 := "ns" - ns2 := "ns2" - - store, err := NewFSStore(dir) - require.NoError(t, err) - - reserved, err := store.Reserve(ns1, 1) - require.NoError(t, err) - assert.True(t, reserved) - - reserved, err = store.Reserve(ns1, 1) - require.NoError(t, err) - assert.False(t, reserved, "should not be able to reserve the same port twice") - - err = store.Release(ns1, 1) - require.NoError(t, err) - - reserved, err = store.Reserve(ns1, 1) - require.NoError(t, err) - assert.True(t, reserved, "should be able to reserve a released port") - - reserved, err = store.Reserve(ns2, 1) - require.NoError(t, err) - assert.True(t, reserved, "should be able to reserve same port in difference namespace") -} diff --git a/pkg/network/portm/backend/lock.go b/pkg/network/portm/backend/lock.go deleted file mode 100644 index b1410d3bc..000000000 --- a/pkg/network/portm/backend/lock.go +++ /dev/null @@ -1,32 +0,0 @@ -package backend - -import ( - "os" - "path" - - "github.com/alexflint/go-filemutex" -) - -// FileLock wraps os.File to be used as a lock using flock -type FileLock struct { - *filemutex.FileMutex -} - -// NewFileLock opens file/dir at path and returns unlocked FileLock object -func NewFileLock(lockPath string) (*FileLock, error) { - fi, err := os.Stat(lockPath) - if err != nil { - return nil, err - } - - if fi.IsDir() { - lockPath = path.Join(lockPath, "lock") - } - - f, err := filemutex.New(lockPath) - if err != nil { - return nil, err - } - - return &FileLock{f}, nil -} diff --git a/pkg/network/portm/backend/lock_test.go b/pkg/network/portm/backend/lock_test.go deleted file mode 100644 index 5d9726f02..000000000 --- a/pkg/network/portm/backend/lock_test.go +++ /dev/null @@ -1,42 +0,0 @@ -package backend - -import ( - "os" - "path/filepath" - "testing" - - "github.com/stretchr/testify/require" -) - -func TestLockOperations(t *testing.T) { - dir := t.TempDir() - - // create a dummy file to lock - path := filepath.Join(dir, "x") - f, err := os.OpenFile(path, os.O_RDONLY|os.O_CREATE, 0666) - require.NoError(t, err) - err = f.Close() - require.NoError(t, err) - - // now use it to lock - m, err := NewFileLock(path) - require.NoError(t, err) - - err = m.Lock() - require.NoError(t, err) - err = m.Unlock() - require.NoError(t, err) -} - -func TestLockFolderPath(t *testing.T) { - dir := t.TempDir() - - // use the folder to lock - m, err := NewFileLock(dir) - require.NoError(t, err) - - err = m.Lock() - require.NoError(t, err) - err = m.Unlock() - require.NoError(t, err) -} diff --git a/pkg/network/portm/backend/store.go b/pkg/network/portm/backend/store.go deleted file mode 100644 index 34f39458e..000000000 --- a/pkg/network/portm/backend/store.go +++ /dev/null @@ -1,13 +0,0 @@ -package backend - -// Store define the interface to implement to be used -// as a backend for a port Allocator -type Store interface { - Lock() error - Unlock() error - Reserve(ns string, port int) (bool, error) - Release(ns string, port int) error - LastReserved(ns string) (int, error) - GetByNS(ns string) ([]int, error) - Close() error -} diff --git a/pkg/network/portm/interface.go b/pkg/network/portm/interface.go deleted file mode 100644 index f1005ae17..000000000 --- a/pkg/network/portm/interface.go +++ /dev/null @@ -1,9 +0,0 @@ -package portm - -// PortAllocator is the interface that defines -// the behavior to reserve a port in a specific -// network namespace -type PortAllocator interface { - Reserve(ns string) (int, error) - Release(ns string, port int) error -} diff --git a/pkg/network/public/wireguard.go b/pkg/network/public/wireguard.go deleted file mode 100644 index 946817dde..000000000 --- a/pkg/network/public/wireguard.go +++ /dev/null @@ -1,37 +0,0 @@ -package public - -import ( - "github.com/containernetworking/plugins/pkg/ns" - "github.com/threefoldtech/zos/pkg/network/wireguard" - "github.com/vishvananda/netlink" -) - -// NewWireguard creates a new wireguard instance in public namespace -// if exits -func NewWireguard(name string) (wg *wireguard.Wireguard, err error) { - f := func(host ns.NetNS) error { - wg, err = wireguard.New(name) - if err != nil { - return err - } - - if host == nil { - return nil - } - - // we need to move it to the host - return netlink.LinkSetNsFd(wg, int(host.Fd())) - } - - ns := getPublicNamespace() - if ns != nil { - defer ns.Close() - if err := ns.Do(f); err != nil { - return nil, err - } - // load it from host ns - return wireguard.GetByName(wg.Attrs().Name) - } - - return wg, f(nil) -} diff --git a/pkg/network/qsfs.go b/pkg/network/qsfs.go deleted file mode 100644 index d88ffdb49..000000000 --- a/pkg/network/qsfs.go +++ /dev/null @@ -1,112 +0,0 @@ -package network - -import ( - "fmt" - "strings" - - "github.com/pkg/errors" - "github.com/rs/zerolog/log" - "github.com/threefoldtech/zos/pkg/network/ifaceutil" - "github.com/threefoldtech/zos/pkg/network/namespace" - "github.com/threefoldtech/zos/pkg/network/nft" -) - -var _nft = ` -flush ruleset - -table inet filter { - - chain base_checks { - # allow established/related connections - ct state {established, related} accept - # early drop of invalid connections - ct state invalid drop - } - - chain input { - type filter hook input priority 0; policy drop; - jump base_checks - # port for prometheus - tcp dport 9100 iifname ygg0 accept - # accept only locally generated packets - meta iif lo ct state new accept - ip6 nexthdr icmpv6 accept - } - - chain forward { - type filter hook forward priority 0; policy drop; - # is there already an existing stream? (outgoing) - jump base_checks - } -} - -` - -func applyQSFSFirewall(netns string) error { - if err := nft.Apply(strings.NewReader(_nft), netns); err != nil { - return errors.Wrap(err, "failed to apply nft rule set") - } - - return nil -} - -func (n networker) QSFSNamespace(id string) string { - netId := "qsfs:" + id - hw := ifaceutil.HardwareAddrFromInputBytes([]byte(netId)) - return qsfsNamespacePrefix + strings.Replace(hw.String(), ":", "", -1) -} -func (n networker) QSFSYggIP(id string) (string, error) { - hw := ifaceutil.HardwareAddrFromInputBytes([]byte("ygg:" + id)) - - ip, err := n.ygg.SubnetFor(hw) - if err != nil { - return "", fmt.Errorf("failed to get ygg subnet IP: %w", err) - } - return ip.IP.String(), nil -} -func (n networker) QSFSPrepare(id string) (string, string, error) { - netId := "qsfs:" + id - netNSName := n.QSFSNamespace(id) - netNs, err := createNetNS(netNSName) - if err != nil { - return "", "", err - } - defer netNs.Close() - if err := n.ndmz.AttachNR(netId, netNSName, n.ipamLeaseDir); err != nil { - return "", "", errors.Wrap(err, "failed to prepare qsfs namespace") - } - - if err := applyQSFSFirewall(netNSName); err != nil { - return "", "", err - } - - if n.ygg == nil { - return "", "", errors.New("no ygg server found") - } - ip, err := n.attachYgg(id, netNs) - if err != nil { - return "", "", err - } - - return netNSName, ip.IP.String(), err -} - -func (n networker) QSFSDestroy(id string) error { - netId := "qsfs:" + id - - netNSName := n.QSFSNamespace(id) - - if err := n.ndmz.DetachNR(netId, n.ipamLeaseDir); err != nil { - log.Err(err).Str("namespace", netNSName).Msg("failed to detach qsfs namespace from ndmz") - } - netNs, err := namespace.GetByName(netNSName) - if err != nil { - return errors.Wrap(err, "didn't find qsfs namespace") - } - defer netNs.Close() - if err := n.detachYgg(id, netNs); err != nil { - // log and continue cleaning up - log.Error().Err(err).Msg("couldn't detach ygg interface") - } - return n.destroy(netNSName) -} diff --git a/pkg/network/tuntap/tap.go b/pkg/network/tuntap/tap.go deleted file mode 100644 index e406cec0e..000000000 --- a/pkg/network/tuntap/tap.go +++ /dev/null @@ -1,79 +0,0 @@ -package tuntap - -import ( - "fmt" - - "github.com/pkg/errors" - "github.com/threefoldtech/zos/pkg/network/options" - "github.com/vishvananda/netlink" -) - -// CreateTap creates a new tap device with the given name, and sets the master interface -func CreateTap(name string, master string) (*netlink.Tuntap, error) { - masterIface, err := netlink.LinkByName(master) - if err != nil { - return nil, errors.Wrap(err, "failed to look up tap master") - } - - tap := &netlink.Tuntap{ - LinkAttrs: netlink.LinkAttrs{ - MTU: 1500, - Name: name, - ParentIndex: masterIface.Attrs().Index, - }, - Mode: netlink.TUNTAP_MODE_TAP, - } - - if err = netlink.LinkAdd(tap); err != nil { - return nil, errors.Wrap(err, "could not add tap device") - } - defer func() { - if err != nil { - _ = netlink.LinkDel(tap) - } - }() - - // Setting the master iface on the link attrs at creation time seems to not work - // (at least not always), so explicitly set the master again once the iface is added. - if err = netlink.LinkSetMaster(tap, masterIface); err != nil { - return nil, errors.Wrap(err, "could not set tap master") - } - - if err := options.Set(name, options.IPv6Disable(true)); err != nil { - return nil, errors.Wrap(err, "failed to disable ipv6 on interface host side") - } - - // Re-fetch tap to get all properties/attributes - var link netlink.Link - link, err = netlink.LinkByName(name) - if err != nil { - return nil, errors.Wrapf(err, "failed to refetch tap interface %q", name) - } - - var ok bool - tap, ok = link.(*netlink.Tuntap) - if !ok { - // right now, the netlink lib returns a `*GenericLink` for the tap interface, - // so assign properties to a blank tap - gl, ok := link.(*netlink.GenericLink) - if !ok { - return nil, fmt.Errorf("link %s should be of type tuntap", name) - } - tap = &netlink.Tuntap{LinkAttrs: gl.LinkAttrs, Mode: netlink.TUNTAP_MODE_TAP} - } else { - // make sure we have the right interface type - if tap.Mode != netlink.TUNTAP_MODE_TAP { - return nil, errors.New("tuntap iface does not have the expected 'tap' mode") - } - } - - if err = netlink.SetPromiscOn(tap); err != nil { - return nil, errors.Wrap(err, "could not bring set promsic on iface") - } - - if err = netlink.LinkSetUp(tap); err != nil { - return nil, errors.Wrap(err, "could not bring up tap iface") - } - - return tap, nil -} diff --git a/pkg/network/wireguard/wireguard.go b/pkg/network/wireguard/wireguard.go deleted file mode 100644 index 25d6d53ea..000000000 --- a/pkg/network/wireguard/wireguard.go +++ /dev/null @@ -1,233 +0,0 @@ -package wireguard - -import ( - "fmt" - "net" - "os" - "path/filepath" - "strconv" - "time" - - "github.com/pkg/errors" - "github.com/rs/zerolog/log" - - "github.com/vishvananda/netlink" - "golang.zx2c4.com/wireguard/wgctrl" - "golang.zx2c4.com/wireguard/wgctrl/wgtypes" -) - -// Wireguard is a netlink.Link of type wireguard -type Wireguard struct { - attrs *netlink.LinkAttrs -} - -// New create a new wireguard interface -func New(name string) (*Wireguard, error) { - attrs := netlink.NewLinkAttrs() - attrs.Name = name - attrs.MTU = 1420 - - wg := &Wireguard{attrs: &attrs} - if err := netlink.LinkAdd(wg); err != nil && !os.IsExist(err) { - return nil, err - } - - // always make sure we load the full attributes - return GetByName(name) -} - -// GetByName return a wireguard object by its name -func GetByName(name string) (*Wireguard, error) { - link, err := netlink.LinkByName(name) - if err != nil { - return nil, err - } - - if link.Type() != "wireguard" { - return nil, fmt.Errorf("link %s is not of type wireguard", name) - } - wg := &Wireguard{ - attrs: link.Attrs(), - } - return wg, nil -} - -// Type implements the netlink.Link interface -func (w *Wireguard) Type() string { - return "wireguard" -} - -// Attrs implements the netlink.Link interface -func (w *Wireguard) Attrs() *netlink.LinkAttrs { - return w.attrs -} - -// Device returns the detail of the configuration of the -// wireguard interface -func (w *Wireguard) Device() (*wgtypes.Device, error) { - wg, err := wgctrl.New() - if err != nil { - return nil, err - } - defer wg.Close() - - return wg.Device(w.attrs.Name) -} - -// SetAddr sets an IP address on the interface -func (w *Wireguard) SetAddr(cidr string) error { - addr, err := netlink.ParseAddr(cidr) - if err != nil { - return err - } - - if err := netlink.AddrAdd(w, addr); err != nil && !os.IsExist(err) { - return err - } - return nil -} - -// UnsetAddr removes an IP address from the interface -func (w *Wireguard) UnsetAddr(cidr string) error { - addr, err := netlink.ParseAddr(cidr) - if err != nil { - return err - } - - if err := netlink.AddrDel(w, addr); err != nil && !os.IsNotExist(err) { - return err - } - return nil -} - -// Peer represent a peer in a wireguard configuration -type Peer struct { - PublicKey string - Endpoint string - AllowedIPs []string -} - -// Configure configures the wiregard configuration -func (w *Wireguard) Configure(privateKey string, listentPort int, peers []*Peer) error { - - if err := netlink.LinkSetDown(w); err != nil { - return err - } - - wc, err := wgctrl.New() - if err != nil { - return err - } - defer wc.Close() - - peersConfig := make([]wgtypes.PeerConfig, len(peers)) - for i, peer := range peers { - p, err := newPeer(peer.PublicKey, peer.Endpoint, peer.AllowedIPs) - if err != nil { - return err - } - peersConfig[i] = p - } - - key, err := wgtypes.ParseKey(privateKey) - if err != nil { - return err - } - - config := wgtypes.Config{ - PrivateKey: &key, - Peers: peersConfig, - ListenPort: &listentPort, - ReplacePeers: true, - } - log.Info().Msg("configure wg device") - - if err := wc.ConfigureDevice(w.attrs.Name, config); err != nil { - return errors.Wrap(err, "failed to configure wireguard interface") - } - - if err := netlink.LinkSetUp(w); err != nil && !os.IsExist(err) { - return errors.Wrapf(err, "failed to bring wireguard interface %s up", w.Attrs().Name) - } - return nil -} - -func newPeer(pubkey, endpoint string, allowedIPs []string) (wgtypes.PeerConfig, error) { - peer := wgtypes.PeerConfig{ - ReplaceAllowedIPs: true, - } - var err error - - duration := time.Second * 20 - peer.PersistentKeepaliveInterval = &duration - - peer.PublicKey, err = wgtypes.ParseKey(pubkey) - if err != nil { - return peer, err - } - - if endpoint != "" { - host, p, err := net.SplitHostPort(endpoint) - if err != nil { - return peer, err - } - - port, err := strconv.Atoi(p) - if err != nil { - return peer, err - } - - peer.Endpoint = &net.UDPAddr{ - IP: net.ParseIP(host), - Port: port, - } - } - - for _, allowedIP := range allowedIPs { - ip, ipNet, err := net.ParseCIDR(allowedIP) - if err != nil { - return peer, err - } - ipNet.IP = ip - peer.AllowedIPs = append(peer.AllowedIPs, *ipNet) - } - - return peer, nil -} - -// GenerateKey generates a new private key. If key already exists -// in that location, that key is returned instead. -func GenerateKey(dir string) (wgtypes.Key, error) { - path := filepath.Join(dir, "key.priv") - data, err := os.ReadFile(path) - if err == nil { - //key already exists - return wgtypes.ParseKey(string(data)) - } else if !os.IsNotExist(err) { - //another error than not exist - return wgtypes.Key{}, err - } - - key, err := wgtypes.GeneratePrivateKey() - if err != nil { - return wgtypes.Key{}, err - } - if err := os.MkdirAll(dir, 0700); err != nil { - return wgtypes.Key{}, err - } - - if err := os.WriteFile(path, []byte(key.String()), 0400); err != nil { - return wgtypes.Key{}, err - } - return key, nil -} - -// LoadKey tries to read a private key from disk -func LoadKey(dir string) (wgtypes.Key, error) { - path := filepath.Join(dir, "key.priv") - b, err := os.ReadFile(path) - if err != nil { - return wgtypes.Key{}, err - } - return wgtypes.ParseKey(string(b)) -} diff --git a/pkg/network/wireguard/wireguard_test.go b/pkg/network/wireguard/wireguard_test.go deleted file mode 100644 index 6ccfa2783..000000000 --- a/pkg/network/wireguard/wireguard_test.go +++ /dev/null @@ -1,67 +0,0 @@ -package wireguard - -import ( - "testing" - - "github.com/vishvananda/netlink" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestNewPeer(t *testing.T) { - endpoint := "37.187.124.71:51820" - publicKey := "mR5fBXohKe2MZ6v+GLwlKwrvkFxo1VvV3bPNHDBhOAI=" - allowedIps := []string{"172.21.0.0/24", "fe80::f002/64"} - peer, err := newPeer(publicKey, endpoint, allowedIps) - require.NoError(t, err) - - require.Equal(t, endpoint, peer.Endpoint.String()) - require.Equal(t, publicKey, peer.PublicKey.String()) - tmp := make([]string, len(peer.AllowedIPs)) - for i, ip := range peer.AllowedIPs { - tmp[i] = ip.String() - } - require.Equal(t, allowedIps, tmp) -} - -func TestConfigure(t *testing.T) { - wg, err := New("test") - require.NoError(t, err) - - defer func() { - _ = netlink.LinkDel(wg) - }() - - privateKey := "4DwTbGRWECH8oqcTXdoWXGOaWWC952QKbFE1fMzBNmA=" - publicKey := "kDd5mB6L4gkd3U5W287JeQu7urFzBYH51JQZUrJd8Hg=" - endpoint := "37.187.124.71:51820" - peerPublicKey := "mR5fBXohKe2MZ6v+GLwlKwrvkFxo1VvV3bPNHDBhOAI=" - allowedIps := []string{"172.21.0.0/24", "192.168.1.10/32", "fe80::f002/128"} - - err = wg.Configure(privateKey, 1600, []*Peer{ - { - PublicKey: peerPublicKey, - AllowedIPs: allowedIps, - Endpoint: endpoint, - }, - }) - require.NoError(t, err) - - device, err := wg.Device() - require.NoError(t, err) - - assert.Equal(t, privateKey, device.PrivateKey.String()) - assert.Equal(t, publicKey, device.PrivateKey.PublicKey().String()) - assert.Equal(t, publicKey, device.PrivateKey.PublicKey().String()) - - for _, peer := range device.Peers { - assert.Equal(t, endpoint, peer.Endpoint.String()) - - actual := make([]string, len(peer.AllowedIPs)) - for y, ip := range peer.AllowedIPs { - actual[y] = ip.String() - } - assert.Equal(t, allowedIps, actual) - } -} diff --git a/pkg/network/yggdrasil/config.go b/pkg/network/yggdrasil/config.go deleted file mode 100644 index a9ffd7ad9..000000000 --- a/pkg/network/yggdrasil/config.go +++ /dev/null @@ -1,91 +0,0 @@ -package yggdrasil - -import ( - "context" - "crypto/ed25519" - "encoding/hex" - "fmt" - "net" - - "github.com/jbenet/go-base58" - "github.com/pkg/errors" - - "github.com/yggdrasil-network/yggdrasil-go/src/address" - "github.com/yggdrasil-network/yggdrasil-go/src/config" -) - -// List of port used by yggdrasil -const ( - YggListenTCP = 9943 - YggListenTLS = 9944 - YggListenLinkLocal = 9945 - - YggIface = "ygg0" -) - -// NodeConfig wrapper around yggdrasil node config -type NodeConfig config.NodeConfig - -// Address gets the address from the config -func (n *NodeConfig) Address() (net.IP, error) { - ip := make([]byte, net.IPv6len) - pk, err := hex.DecodeString(n.PublicKey) - if err != nil { - return nil, errors.Wrap(err, "failed to load public key") - } - copy(ip, address.AddrForKey(pk)[:]) - - return ip, nil -} - -func (n *NodeConfig) FindPeers(ctx context.Context, filter ...Filter) error { - // fetching a peer list goes as this - // - Always include the list of peers from - peers, err := fetchZosYggList() - if err != nil { - return errors.Wrap(err, "failed to get zos public peer list") - } - - peers, err = peers.Ups(filter...) - if err != nil { - return errors.Wrap(err, "failed to filter out peer list") - } - - n.Peers = peers - return nil -} - -// GenerateConfig creates a new yggdrasil configuration and generate the -// box and signing key from the ed25519 Private key of the node -// this creates a mapping between a yggdrasil identity and the TFGrid identity -func GenerateConfig(privateKey ed25519.PrivateKey) (cfg NodeConfig) { - cfg.IfMTU = 65535 - if privateKey != nil { - cfg.PrivateKey = hex.EncodeToString(privateKey) - - signingPublicKey := privateKey.Public().(ed25519.PublicKey) - cfg.PublicKey = hex.EncodeToString(signingPublicKey) - - cfg.NodeInfo = map[string]interface{}{ - "name": base58.Encode(signingPublicKey)[:6], - } - } - - cfg.MulticastInterfaces = []config.MulticastInterfaceConfig{ - { - Regex: ".*", - Listen: true, - Beacon: true, - Port: 0, - }, - } - - cfg.IfName = YggIface - - cfg.Listen = []string{ - fmt.Sprintf("tcp://[::]:%d", YggListenTCP), - fmt.Sprintf("tls://[::]:%d", YggListenTLS), - } - - return -} diff --git a/pkg/network/yggdrasil/namespace.go b/pkg/network/yggdrasil/namespace.go deleted file mode 100644 index 4f64f866a..000000000 --- a/pkg/network/yggdrasil/namespace.go +++ /dev/null @@ -1,272 +0,0 @@ -package yggdrasil - -import ( - "fmt" - "net" - "os" - - "github.com/containernetworking/plugins/pkg/ns" - "github.com/pkg/errors" - "github.com/rs/zerolog/log" - "github.com/threefoldtech/zos/pkg/network/bridge" - "github.com/threefoldtech/zos/pkg/network/ifaceutil" - "github.com/threefoldtech/zos/pkg/network/macvlan" - "github.com/threefoldtech/zos/pkg/network/namespace" - "github.com/threefoldtech/zos/pkg/network/types" - "github.com/vishvananda/netlink" -) - -const ( - // YggNSInf inside the namespace - YggNSInf = "nygg6" - yggBridge = types.YggBridge -) - -var ( - YggRange = net.IPNet{ - IP: net.ParseIP("200::"), - Mask: net.CIDRMask(7, 128), - } -) - -type YggdrasilNamespace interface { - Name() string - // IsIPv4Only checks if namespace has NO public ipv6 on any of its interfaces - IsIPv4Only() (bool, error) - // GetIPs return a list of all IPv6 inside this namespace. - GetIPs() ([]net.IPNet, error) - // SetYggIP sets the ygg ipv6 on the nygg6 iterface. - SetYggIP(ip net.IPNet, gw net.IP) error -} - -// ensureYggPlumbing this ensures that the yggdrasil plumbing is in place inside this namespace -func ensureYggPlumbing(netNS ns.NetNS) error { - if !bridge.Exists(yggBridge) { - if _, err := bridge.New(yggBridge); err != nil { - return errors.Wrapf(err, "couldn't create bridge %s", yggBridge) - } - } - - if err := dumdumHack(); err != nil { - log.Error().Err(err).Msg("failed to create the dummy hack for ygg-bridge") - } - - if !ifaceutil.Exists(YggNSInf, netNS) { - if _, err := macvlan.Create(YggNSInf, yggBridge, netNS); err != nil { - return errors.Wrapf(err, "couldn't create %s inside", YggNSInf) - } - } - - return netNS.Do(func(_ ns.NetNS) error { - link, err := netlink.LinkByName(YggNSInf) - if err != nil { - return err - } - - return netlink.LinkSetUp(link) - }) -} - -func dumdumHack() error { - // dumdum hack. this hack to fix a weird issue with linux kernel - // 5.10.version 55 - // it seems that the macvlan on a bridge does not bring the bridge - // up. So we have to plug in a dummy device into yggBridge and set - // the device up to keep the bridge state UP. - br, err := bridge.Get(yggBridge) - if err != nil { - return errors.Wrap(err, "failed to get br-ygg") - } - - const name = "dumdum" - link, err := netlink.LinkByName(name) - if _, ok := err.(netlink.LinkNotFoundError); ok { - if err := netlink.LinkAdd(&netlink.Dummy{ - LinkAttrs: netlink.LinkAttrs{ - NetNsID: -1, - TxQLen: -1, - Name: name, - }, - }); err != nil { - return err - } - - link, err = netlink.LinkByName(name) - if err != nil { - return errors.Wrap(err, "failed to get dumdum device") - } - } else if err != nil { - return err - } - - if err := netlink.LinkSetMaster(link, br); err != nil { - return err - } - - return netlink.LinkSetUp(link) - -} - -func NewYggdrasilNamespace(ns string) (YggdrasilNamespace, error) { - yggNs, err := namespace.GetByName(ns) - if err != nil { - return nil, errors.Wrapf(err, "namespace '%s' not found", ns) - } - if err := ensureYggPlumbing(yggNs); err != nil { - return nil, errors.Wrapf(err, "failed to prepare namespace '%s' for yggdrasil", ns) - } - - return &yggNS{ns}, nil -} - -type yggNS struct { - ns string -} - -func (d *yggNS) Name() string { - return d.ns -} - -func (d *yggNS) setGw(gw net.IP) error { - ipv6routes, err := netlink.RouteList(nil, netlink.FAMILY_V6) - if err != nil { - return err - } - - for _, route := range ipv6routes { - if route.Dst == nil { - //default route! - continue - } - if route.Dst.String() == YggRange.String() { - // we found a match - if err := netlink.RouteDel(&route); err != nil { - return err - } - } - } - - // now add route - return netlink.RouteAdd(&netlink.Route{ - Dst: &YggRange, - Gw: gw, - }) -} -func (d *yggNS) SetYggIP(subnet net.IPNet, gw net.IP) error { - netns, err := namespace.GetByName(d.ns) - if err != nil { - return err - } - defer netns.Close() - - if ip6 := subnet.IP.To16(); ip6 == nil { - return fmt.Errorf("expecting ipv6 for ygg interface") - } - - err = netns.Do(func(_ ns.NetNS) error { - link, err := netlink.LinkByName(YggNSInf) - if err != nil { - return err - } - - ips, err := netlink.AddrList(link, netlink.FAMILY_V6) - if err != nil { - return err - } - - for _, ip := range ips { - if YggRange.Contains(ip.IP) { - _ = netlink.AddrDel(link, &ip) - } - } - - if err := netlink.AddrAdd(link, &netlink.Addr{ - IPNet: &subnet, - }); err != nil && !os.IsExist(err) { - return err - } - - if gw == nil { - return nil - } - // set gw for entire ygg range - - return d.setGw(gw) - }) - return err -} - -func (n *yggNS) GetIPs() ([]net.IPNet, error) { - - netns, err := namespace.GetByName(n.ns) - if err != nil { - return nil, err - } - - defer netns.Close() - - var results []net.IPNet - err = netns.Do(func(_ ns.NetNS) error { - links, err := netlink.LinkList() - if err != nil { - return errors.Wrap(err, "failed to list interfaces") - } - - for _, link := range links { - ips, err := netlink.AddrList(link, netlink.FAMILY_V6) - if err != nil { - return err - } - - for _, ip := range ips { - results = append(results, *ip.IPNet) - } - } - - return nil - }) - - return results, err -} - -func (n *yggNS) IsIPv4Only() (bool, error) { - // this is true if DMZPub6 only has local not routable ipv6 addresses - //DMZPub6 - netNS, err := namespace.GetByName(n.ns) - if err != nil { - return false, errors.Wrap(err, "failed to get ndmz namespace") - } - defer netNS.Close() - - var ipv4Only bool - err = netNS.Do(func(_ ns.NetNS) error { - links, err := netlink.LinkList() - if err != nil { - return errors.Wrap(err, "failed to list interfaces") - } - - for _, link := range links { - ips, err := netlink.AddrList(link, netlink.FAMILY_V6) - if err != nil { - return errors.Wrapf(err, "failed to list '%s' ips", link.Attrs().Name) - } - - for _, ip := range ips { - if YggRange.Contains(ip.IP) { - continue - } - - if ip.IP.IsGlobalUnicast() && !ip.IP.IsPrivate() { - // we found a public IPv6 so we are not ipv4 so ygg can peer - // with other ipv6 peers - return nil - } - } - } - - ipv4Only = true - return nil - }) - - return ipv4Only, err -} diff --git a/pkg/network/yggdrasil/public_peerlist.go b/pkg/network/yggdrasil/public_peerlist.go deleted file mode 100644 index 1f10437bf..000000000 --- a/pkg/network/yggdrasil/public_peerlist.go +++ /dev/null @@ -1,91 +0,0 @@ -package yggdrasil - -import ( - "net" - "net/url" - - "github.com/rs/zerolog/log" - "github.com/threefoldtech/zos/pkg/environment" -) - -// Peers is a peers list -type Peers []string - -type Filter func(ip net.IP) bool - -// Ranges is a list of net.IPNet -type Ranges []net.IPNet - -// Exclude ranges, return IPs that are NOT in the given ranges -func Exclude(ranges Ranges) Filter { - return func(ip net.IP) bool { - for _, n := range ranges { - if n.Contains(ip) { - return false - } - } - return true - } -} - -// Include ranges, return IPs that are IN one of the given ranges -func Include(ranges Ranges) Filter { - return func(ip net.IP) bool { - for _, n := range ranges { - if n.Contains(ip) { - return true - } - } - return false - } -} - -// IPV4Only is an IPFilter function that filters out non IPv4 address -func IPV4Only(ip net.IP) bool { - return ip.To4() != nil -} - -func fetchZosYggList() (Peers, error) { - cfg, err := environment.GetConfig() - if err != nil { - return nil, err - } - - return cfg.Yggdrasil.Peers, nil - -} - -// Ups return all the peers that are marked up from the PeerList p -func (p Peers) Ups(filter ...Filter) (Peers, error) { - var peers Peers -next: - for _, n := range p { - if len(filter) == 0 { - peers = append(peers, n) - continue - } - // we have filters, we need to process the endpoint - u, err := url.Parse(n) - if err != nil { - log.Error().Err(err).Str("url", n).Msg("failed to parse url") - continue - } - ips, err := net.LookupIP(u.Hostname()) - if err != nil { - log.Error().Err(err).Str("url", n).Msg("failed to lookup ip") - continue - } - - for _, ip := range ips { - for _, f := range filter { - if !f(ip) { - continue next - } - } - } - - peers = append(peers, n) - } - - return peers, nil -} diff --git a/pkg/network/yggdrasil/utils.go b/pkg/network/yggdrasil/utils.go deleted file mode 100644 index 4765ffa32..000000000 --- a/pkg/network/yggdrasil/utils.go +++ /dev/null @@ -1,50 +0,0 @@ -package yggdrasil - -import ( - "context" - "crypto/ed25519" - - "github.com/pkg/errors" - "github.com/rs/zerolog/log" - "github.com/threefoldtech/zos/pkg/zinit" -) - -func EnsureYggdrasil(ctx context.Context, privateKey ed25519.PrivateKey, ns YggdrasilNamespace) (*YggServer, error) { - // Filter out all the nodes from the same - // segment so we do not just connect locally - ips, err := ns.GetIPs() // returns ipv6 only - if err != nil { - return nil, errors.Wrap(err, "failed to get ndmz public ipv6") - } - - var ranges Ranges - for _, ip := range ips { - if ip.IP.IsGlobalUnicast() { - ranges = append(ranges, ip) - } - } - log.Info().Msgf("filtering out peers from ranges: %s", ranges) - filter := Exclude(ranges) - z := zinit.Default() - - cfg := GenerateConfig(privateKey) - if err := cfg.FindPeers(ctx, filter); err != nil { - return nil, err - } - - server := NewYggServer(&cfg) - if err := server.Ensure(z, ns.Name()); err != nil { - return nil, err - } - - gw, err := server.Gateway() - if err != nil { - return nil, errors.Wrap(err, "fail read yggdrasil subnet") - } - - if err := ns.SetYggIP(gw, nil); err != nil { - return nil, errors.Wrap(err, "fail to configure yggdrasil subnet gateway IP") - } - - return server, nil -} diff --git a/pkg/network/yggdrasil/yygdrasil.go b/pkg/network/yggdrasil/yygdrasil.go deleted file mode 100644 index 64a4a7c1f..000000000 --- a/pkg/network/yggdrasil/yygdrasil.go +++ /dev/null @@ -1,198 +0,0 @@ -package yggdrasil - -import ( - "crypto/ed25519" - "crypto/sha512" - "encoding/hex" - "encoding/json" - "fmt" - "net" - "os" - "os/exec" - "path/filepath" - "time" - - "github.com/pkg/errors" - "github.com/threefoldtech/zos/pkg/network/namespace" - "github.com/threefoldtech/zos/pkg/zinit" - "github.com/yggdrasil-network/yggdrasil-go/src/address" -) - -const ( - zinitService = "yggdrasil" - confPath = "/var/cache/modules/networkd/yggdrasil.conf" -) - -// YggServer represent a yggdrasil server -type YggServer struct { - cfg *NodeConfig -} - -// NewYggServer create a new yggdrasil Server -func NewYggServer(cfg *NodeConfig) *YggServer { - return &YggServer{ - cfg: cfg, - } -} - -func (s *YggServer) Restart(z *zinit.Client) error { - return z.Kill(zinitService, zinit.SIGTERM) -} - -func (s *YggServer) Reload(z *zinit.Client) error { - return z.Kill(zinitService, zinit.SIGHUP) -} - -// Start creates an yggdrasil zinit service and starts it -func (s *YggServer) Ensure(z *zinit.Client, ns string) error { - if !namespace.Exists(ns) { - return fmt.Errorf("invalid namespace '%s'", ns) - } - - if err := writeConfig(confPath, s.cfg); err != nil { - return err - } - - // service found. - // better if we just stop, forget and start over to make - // sure we using the right exec params - if _, err := z.Status(zinitService); err == nil { - // not here we need to stop it - if err := z.StopWait(5*time.Second, zinitService); err != nil && !errors.Is(err, zinit.ErrUnknownService) { - return errors.Wrap(err, "failed to stop yggdrasil service") - } - if err := z.Forget(zinitService); err != nil && !errors.Is(err, zinit.ErrUnknownService) { - return errors.Wrap(err, "failed to forget yggdrasil service") - } - } - - bin, err := exec.LookPath("yggdrasil") - if err != nil { - return err - } - - cmd := `sh -c ' - ulimit -n 16384 - - exec ip netns exec %s %s -loglevel error -useconffile %s - '` - - err = zinit.AddService(zinitService, zinit.InitService{ - Exec: fmt.Sprintf(cmd, ns, bin, confPath), - After: []string{ - "node-ready", - }, - Test: "yggdrasilctl getself | grep -i coords", - }) - - if err != nil { - return err - } - - if err := z.Monitor(zinitService); err != nil && !errors.Is(err, zinit.ErrAlreadyMonitored) { - return err - } - - return z.StartWait(time.Second*20, zinitService) -} - -// NodeID returns the yggdrasil node ID of s -func (s *YggServer) NodeID() (ed25519.PublicKey, error) { - if s.cfg.PublicKey == "" { - panic("EncryptionPublicKey empty") - } - - return hex.DecodeString(s.cfg.PublicKey) -} - -// Address return the address in the 200::/7 subnet allocated by yggdrasil -func (s *YggServer) Address() (net.IP, error) { - nodeID, err := s.NodeID() - if err != nil { - return nil, err - } - - ip := make([]byte, net.IPv6len) - copy(ip, address.AddrForKey(nodeID)[:]) - - return ip, nil -} - -// Subnet return the 300::/64 subnet allocated by yggdrasil -func (s *YggServer) Subnet() (net.IPNet, error) { - nodeID, err := s.NodeID() - if err != nil { - return net.IPNet{}, err - } - - snet := *address.SubnetForKey(nodeID) - ipnet := net.IPNet{ - IP: append(snet[:], 0, 0, 0, 0, 0, 0, 0, 0), - Mask: net.CIDRMask(len(snet)*8, 128), - } - - return ipnet, nil -} - -// Gateway return the first IP of the 300::/64 subnet allocated by yggdrasil -func (s *YggServer) Gateway() (net.IPNet, error) { - - subnet, err := s.Subnet() - if err != nil { - return net.IPNet{}, err - } - subnet.IP[len(subnet.IP)-1] = 0x1 - - return subnet, nil -} - -// Tun return the name of the TUN interface created by yggdrasil -func (s *YggServer) Tun() string { - return s.cfg.IfName -} - -// SubnetFor return an IP address out of the node allocated subnet by hasing b and using it -// to generate the last 64 bits of the IPV6 address -func (s *YggServer) SubnetFor(b []byte) (net.IPNet, error) { - subnet, err := s.Subnet() - if err != nil { - return net.IPNet{}, err - } - - ip := make([]byte, net.IPv6len) - copy(ip, subnet.IP) - - subIP, err := subnetFor(ip, b) - if err != nil { - return net.IPNet{}, err - } - - return net.IPNet{ - IP: subIP, - Mask: net.CIDRMask(64, 128), - }, nil -} - -func subnetFor(prefix net.IP, b []byte) (net.IP, error) { - h := sha512.New() - if _, err := h.Write(b); err != nil { - return nil, err - } - digest := h.Sum(nil) - copy(prefix[8:], digest[:8]) - return prefix, nil -} - -func writeConfig(path string, cfg *NodeConfig) error { - if err := os.MkdirAll(filepath.Dir(path), 0770); err != nil { - return err - } - - f, err := os.Create(path) - if err != nil { - return err - } - defer f.Close() - - return json.NewEncoder(f).Encode(cfg) -} diff --git a/pkg/network/yggdrasil/yygdrasil_test.go b/pkg/network/yggdrasil/yygdrasil_test.go deleted file mode 100644 index 117767e73..000000000 --- a/pkg/network/yggdrasil/yygdrasil_test.go +++ /dev/null @@ -1,59 +0,0 @@ -package yggdrasil - -import ( - "crypto/ed25519" - "net" - "testing" - - "github.com/stretchr/testify/require" - "gotest.tools/assert" -) - -func TestAddresses(t *testing.T) { - sk := ed25519.NewKeyFromSeed([]byte("00000000000000000000000000000000")) - - cfg := GenerateConfig(sk) - s := NewYggServer(&cfg) - - ip, err := s.Address() - require.NoError(t, err) - - subnet, err := s.Subnet() - require.NoError(t, err) - - gw, err := s.Gateway() - require.NoError(t, err) - - assert.Equal(t, "203:45bf:8a48:8361:c04c:1321:ea32:50ad", ip.String()) - assert.Equal(t, "303:45bf:8a48:8361::/64", subnet.String()) - assert.Equal(t, "303:45bf:8a48:8361::1/64", gw.String()) -} - -func TestSubnetFor(t *testing.T) { - tests := []struct { - name string - prefix net.IP - ip net.IP - b []byte - }{ - { - name: "0000", - prefix: net.ParseIP("301:8d90:378f:a93::"), - ip: net.ParseIP("301:8d90:378f:a93:c600:1d5b:2ac3:df31"), - b: []byte("0000"), - }, - { - name: "bbbbb", - prefix: net.ParseIP("301:8d90:378f:a93::"), - ip: net.ParseIP("301:8d90:378f:a93:9b73:d9aa:7cec:73be"), - b: []byte("bbbb"), - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - ip, err := subnetFor(tt.prefix, tt.b) - require.NoError(t, err) - assert.Equal(t, tt.ip.String(), ip.String()) - }) - } -} diff --git a/pkg/network_light.go b/pkg/network_light.go new file mode 100644 index 000000000..6f3d9e055 --- /dev/null +++ b/pkg/network_light.go @@ -0,0 +1,43 @@ +package pkg + +import ( + "context" + "net" +) + +//go:generate mkdir -p stubs +//go:generate zbusc -module netlight -version 0.0.1 -name netlight -package stubs github.com/threefoldtech/zos/pkg+NetworkerLight stubs/network_light_stub.go + +// NetworkerLight is the interface for the network light module +type NetworkerLight interface { + Create(name string, privateNet net.IPNet, seed []byte) error + Delete(name string) error + AttachPrivate(name, id string, vmIp net.IP) (device TapDevice, err error) + AttachMycelium(name, id string, seed []byte) (device TapDevice, err error) + Detach(id string) error + Interfaces(iface string, netns string) (Interfaces, error) + AttachZDB(id string) (string, error) + ZDBIPs(namespace string) ([]net.IP, error) + Namespace(id string) string + Ready() error + ZOSAddresses(ctx context.Context) <-chan NetlinkAddresses +} + +type TapDevice struct { + Name string + Mac net.HardwareAddr + IP *net.IPNet + Routes []Route +} + +// Interfaces struct to bypass zbus generation error +// where it generate a stub with map as interface instead of map +type Interfaces struct { + Interfaces map[string]Interface +} + +type Interface struct { + Name string + IPs []net.IPNet + Mac string +} diff --git a/pkg/perf/iperf/iperf_task.go b/pkg/perf/iperf/iperf_task.go index 002b525d6..5fc10e7bb 100644 --- a/pkg/perf/iperf/iperf_task.go +++ b/pkg/perf/iperf/iperf_task.go @@ -14,7 +14,7 @@ import ( "github.com/pkg/errors" "github.com/rs/zerolog/log" "github.com/threefoldtech/zos/pkg/environment" - "github.com/threefoldtech/zos/pkg/network/iperf" + "github.com/threefoldtech/zos/pkg/netlight/iperf" "github.com/threefoldtech/zos/pkg/perf" "github.com/threefoldtech/zos/pkg/perf/graphql" ) diff --git a/pkg/perf/publicip/publicip_task.go b/pkg/perf/publicip/publicip_task.go index 5b68c1740..0b9cb75bc 100644 --- a/pkg/perf/publicip/publicip_task.go +++ b/pkg/perf/publicip/publicip_task.go @@ -15,8 +15,8 @@ import ( "github.com/rs/zerolog/log" substrate "github.com/threefoldtech/tfchain/clients/tfchain-client-go" "github.com/threefoldtech/zos/pkg/environment" - "github.com/threefoldtech/zos/pkg/network/macvlan" - "github.com/threefoldtech/zos/pkg/network/namespace" + "github.com/threefoldtech/zos/pkg/netlight/macvlan" + "github.com/threefoldtech/zos/pkg/netlight/namespace" "github.com/threefoldtech/zos/pkg/perf" "github.com/threefoldtech/zos/pkg/perf/graphql" "github.com/threefoldtech/zos/pkg/stubs" diff --git a/pkg/power/power.go b/pkg/power/power.go index 51465042c..a2f99f092 100644 --- a/pkg/power/power.go +++ b/pkg/power/power.go @@ -12,7 +12,7 @@ import ( substrate "github.com/threefoldtech/tfchain/clients/tfchain-client-go" "github.com/threefoldtech/zos/pkg" "github.com/threefoldtech/zos/pkg/events" - "github.com/threefoldtech/zos/pkg/network/bridge" + "github.com/threefoldtech/zos/pkg/netlight/bridge" "github.com/threefoldtech/zos/pkg/stubs" "github.com/threefoldtech/zos/pkg/zinit" ) diff --git a/pkg/primitives/network/network.go b/pkg/primitives/network-light/network.go similarity index 80% rename from pkg/primitives/network/network.go rename to pkg/primitives/network-light/network.go index df178c0a1..5a4e19671 100644 --- a/pkg/primitives/network/network.go +++ b/pkg/primitives/network-light/network.go @@ -1,4 +1,4 @@ -package network +package netlight import ( "context" @@ -7,9 +7,7 @@ import ( "github.com/pkg/errors" "github.com/rs/zerolog/log" - "github.com/threefoldtech/zbus" - "github.com/threefoldtech/zos/pkg" "github.com/threefoldtech/zos/pkg/gridtypes" "github.com/threefoldtech/zos/pkg/gridtypes/zos" "github.com/threefoldtech/zos/pkg/provision" @@ -33,19 +31,15 @@ func NewManager(zbus zbus.Client) *Manager { func (p *Manager) networkProvisionImpl(ctx context.Context, wl *gridtypes.WorkloadWithID) error { twin, _ := provision.GetDeploymentID(ctx) - var network zos.Network + var network zos.NetworkLight if err := json.Unmarshal(wl.Data, &network); err != nil { return fmt.Errorf("failed to unmarshal network from reservation: %w", err) } - mgr := stubs.NewNetworkerStub(p.zbus) + mgr := stubs.NewNetworkerLightStub(p.zbus) log.Debug().Str("network", fmt.Sprintf("%+v", network)).Msg("provision network") - _, err := mgr.CreateNR(ctx, wl.ID, pkg.Network{ - Network: network, - NetID: zos.NetworkID(twin, wl.Name), - }) - + err := mgr.Create(ctx, string(zos.NetworkID(twin, wl.Name)), network.Subnet.IPNet, network.Mycelium.Key) if err != nil { return errors.Wrapf(err, "failed to create network resource for network %s", wl.ID) } @@ -62,9 +56,10 @@ func (p *Manager) Update(ctx context.Context, wl *gridtypes.WorkloadWithID) (int } func (p *Manager) Deprovision(ctx context.Context, wl *gridtypes.WorkloadWithID) error { - mgr := stubs.NewNetworkerStub(p.zbus) + twin, _ := provision.GetDeploymentID(ctx) + mgr := stubs.NewNetworkerLightStub(p.zbus) - if err := mgr.DeleteNR(ctx, wl.ID); err != nil { + if err := mgr.Delete(ctx, string(zos.NetworkID(twin, wl.Name))); err != nil { return fmt.Errorf("failed to delete network resource: %w", err) } diff --git a/pkg/primitives/provisioner.go b/pkg/primitives/provisioner.go index 6062d99de..d2def0c26 100644 --- a/pkg/primitives/provisioner.go +++ b/pkg/primitives/provisioner.go @@ -4,11 +4,9 @@ import ( "github.com/threefoldtech/zbus" "github.com/threefoldtech/zos/pkg/gridtypes" "github.com/threefoldtech/zos/pkg/gridtypes/zos" - "github.com/threefoldtech/zos/pkg/primitives/gateway" - "github.com/threefoldtech/zos/pkg/primitives/network" - "github.com/threefoldtech/zos/pkg/primitives/pubip" + netlight "github.com/threefoldtech/zos/pkg/primitives/network-light" "github.com/threefoldtech/zos/pkg/primitives/qsfs" - "github.com/threefoldtech/zos/pkg/primitives/vm" + vmlight "github.com/threefoldtech/zos/pkg/primitives/vm-light" "github.com/threefoldtech/zos/pkg/primitives/volume" "github.com/threefoldtech/zos/pkg/primitives/zdb" "github.com/threefoldtech/zos/pkg/primitives/zlogs" @@ -19,17 +17,13 @@ import ( // NewPrimitivesProvisioner creates a new 0-OS provisioner func NewPrimitivesProvisioner(zbus zbus.Client) provision.Provisioner { managers := map[gridtypes.WorkloadType]provision.Manager{ - zos.ZMountType: zmount.NewManager(zbus), - zos.ZLogsType: zlogs.NewManager(zbus), - zos.QuantumSafeFSType: qsfs.NewManager(zbus), - zos.ZDBType: zdb.NewManager(zbus), - zos.NetworkType: network.NewManager(zbus), - zos.PublicIPType: pubip.NewManager(zbus), - zos.PublicIPv4Type: pubip.NewManager(zbus), // backward compatibility - zos.ZMachineType: vm.NewManager(zbus), - zos.VolumeType: volume.NewManager(zbus), - zos.GatewayNameProxyType: gateway.NewNameManager(zbus), - zos.GatewayFQDNProxyType: gateway.NewFQDNManager(zbus), + zos.ZMountType: zmount.NewManager(zbus), + zos.ZLogsType: zlogs.NewManager(zbus), + zos.QuantumSafeFSType: qsfs.NewManager(zbus), + zos.ZDBType: zdb.NewManager(zbus), + zos.NetworkLightType: netlight.NewManager(zbus), + zos.ZMachineLightType: vmlight.NewManager(zbus), + zos.VolumeType: volume.NewManager(zbus), } return provision.NewMapProvisioner(managers) diff --git a/pkg/primitives/pubip/public_ip.go b/pkg/primitives/pubip/public_ip.go deleted file mode 100644 index e9bb12a42..000000000 --- a/pkg/primitives/pubip/public_ip.go +++ /dev/null @@ -1,239 +0,0 @@ -package pubip - -import ( - "context" - "encoding/hex" - "encoding/json" - "fmt" - "net" - "strings" - - "github.com/pkg/errors" - "github.com/rs/zerolog/log" - "github.com/threefoldtech/zbus" - "github.com/threefoldtech/zos/pkg/gridtypes" - "github.com/threefoldtech/zos/pkg/gridtypes/zos" - "github.com/threefoldtech/zos/pkg/network/ifaceutil" - "github.com/threefoldtech/zos/pkg/provision" - "github.com/threefoldtech/zos/pkg/stubs" -) - -var ( - _ provision.Manager = (*Manager)(nil) -) - -type Manager struct { - zbus zbus.Client -} - -func NewManager(zbus zbus.Client) *Manager { - return &Manager{zbus} -} - -func (p *Manager) Provision(ctx context.Context, wl *gridtypes.WorkloadWithID) (interface{}, error) { - return p.publicIPProvisionImpl(ctx, wl) -} - -func (p *Manager) getAssignedPublicIP(ctx context.Context, wl *gridtypes.WorkloadWithID) (ip gridtypes.IPNet, gw net.IP, err error) { - //Okay, this implementation is tricky but will be simplified once we store the reserved IP - //on the contract. - - // Okay, so first (and easiest path) is that the Ip was already - // assigned, hence we can simply use it again. this is usually - // the case if the node is rerunning the same workload deployment for - // some reason. - if !wl.Result.IsNil() && wl.Result.State == gridtypes.StateOk { - var result zos.PublicIPResult - if err := wl.Result.Unmarshal(&result); err != nil { - return ip, gw, errors.Wrap(err, "failed to load public ip result") - } - - return result.IP, result.Gateway, nil - } - // otherwise we do the following: - - // - We need to get the contract and the farm object this node belongs to - deployment, err := provision.GetDeployment(ctx) - if err != nil { - return ip, gw, errors.Wrap(err, "failed to get deployment") - } - contract := provision.GetContract(ctx) - - // - now we find out ALL ips belonging to this contract - reserved := contract.PublicIPs - - // make sure we have enough reserved IPs - // we process both ipv4 type and ip type to be backward compatible - ipWorkloads := deployment.ByType(zos.PublicIPv4Type, zos.PublicIPType) - reservedCount := 0 - for _, wl := range ipWorkloads { - config, err := p.getPublicIPData(ctx, wl) - if err != nil { - return gridtypes.IPNet{}, nil, err - } - if config.V4 { - reservedCount += 1 - } - } - - if reservedCount > len(reserved) { - return ip, gw, fmt.Errorf("required %d ips while contract has %d ip reserved", len(ipWorkloads), len(reserved)) - } - - usedIPs := make(map[string]struct{}) - - for _, ipWl := range ipWorkloads { - if wl.Name == ipWl.Name { - // we don't need this. - continue - } - - if ipWl.Result.IsNil() || ipWl.Result.State != gridtypes.StateOk { - continue - } - - used, err := GetPubIPConfig(ipWl) - if err != nil { - return ip, gw, err - } - - usedIPs[used.IP.String()] = struct{}{} - } - - // otherwise we go over the list of IPs and take the first free one - for _, reservedIP := range reserved { - if _, ok := usedIPs[reservedIP.IP]; !ok { - // free ip. we can just take it - ip, err = gridtypes.ParseIPNet(reservedIP.IP) - if err != nil { - return ip, gw, fmt.Errorf("found a malformed ip address in contract object '%s'", ip.IP) - } - gw = net.ParseIP(reservedIP.Gateway) - if gw == nil { - return ip, gw, fmt.Errorf("found a malformed gateway address in farm object '%s'", reservedIP.Gateway) - } - - return ip, gw, nil - } - } - - return ip, gw, fmt.Errorf("could not allocate public IP address to workload") -} - -func (p *Manager) getPublicIPData(ctx context.Context, wl *gridtypes.WorkloadWithID) (result zos.PublicIP, err error) { - switch wl.Type { - case zos.PublicIPv4Type: - // backword compatibility with older ipv4 type - result.V4 = true - case zos.PublicIPType: - err = json.Unmarshal(wl.Data, &result) - default: - return result, fmt.Errorf("invalid workload type expecting (%s or %s) got '%s'", zos.PublicIPv4Type, zos.PublicIPType, wl.Type) - } - - return -} - -func (p *Manager) publicIPProvisionImpl(ctx context.Context, wl *gridtypes.WorkloadWithID) (result zos.PublicIPResult, err error) { - config, err := p.getPublicIPData(ctx, wl) - if err != nil { - return zos.PublicIPResult{}, err - } - - tapName := wl.ID.Unique("pub") - network := stubs.NewNetworkerStub(p.zbus) - fName := filterName(tapName) - - if network.PubIPFilterExists(ctx, fName) { - return result, provision.ErrNoActionNeeded - } - - var ipv6 gridtypes.IPNet - var ipv4 gridtypes.IPNet - var gw4 net.IP - - mac := ifaceutil.HardwareAddrFromInputBytes([]byte(tapName)) - if config.V6 { - pubIP6Base, err := network.GetPublicIPv6Subnet(ctx) - if err != nil { - return result, errors.Wrap(err, "could not look up ipv6 prefix") - } - - ipv6, err = predictedSlaac(pubIP6Base, mac.String()) - if err != nil { - return zos.PublicIPResult{}, errors.Wrap(err, "could not compute ipv6 valu") - } - } - - if config.V4 { - ipv4, gw4, err = p.getAssignedPublicIP(ctx, wl) - if err != nil { - return zos.PublicIPResult{}, err - } - } - - result.IP = ipv4 - result.IPv6 = ipv6 - result.Gateway = gw4 - - ifName := fmt.Sprintf("p-%s", tapName) // TODO: clean this up, needs to come form networkd - err = network.SetupPubIPFilter(ctx, fName, ifName, ipv4.IP, ipv6.IP, mac.String()) - - return -} - -func (p *Manager) Deprovision(ctx context.Context, wl *gridtypes.WorkloadWithID) error { - // Disconnect the public interface from the network if one exists - network := stubs.NewNetworkerStub(p.zbus) - tapName := wl.ID.Unique("pub") - fName := filterName(tapName) - if err := network.RemovePubIPFilter(ctx, fName); err != nil { - log.Error().Err(err).Msg("could not remove filter rules") - } - return network.DisconnectPubTap(ctx, tapName) -} - -func filterName(reservationID string) string { - return fmt.Sprintf("r-%s", reservationID) -} - -// modified version of: https://github.com/MalteJ/docker/blob/f09b7897d2a54f35a0b26f7cbe750b3c9383a553/daemon/networkdriver/bridge/driver.go#L585 -func predictedSlaac(base net.IPNet, mac string) (gridtypes.IPNet, error) { - // TODO: get pub ipv6 prefix - hx := strings.Replace(mac, ":", "", -1) - hw, err := hex.DecodeString(hx) - if err != nil { - return gridtypes.IPNet{}, errors.New("Could not parse MAC address " + mac) - } - - hw[0] ^= 0x2 - - base.IP[8] = hw[0] - base.IP[9] = hw[1] - base.IP[10] = hw[2] - base.IP[11] = 0xFF - base.IP[12] = 0xFE - base.IP[13] = hw[3] - base.IP[14] = hw[4] - base.IP[15] = hw[5] - - return gridtypes.IPNet{IPNet: base}, nil - -} - -// GetPubIPConfig get the public ip, and the gateway from the workload -func GetPubIPConfig(wl *gridtypes.WorkloadWithID) (result zos.PublicIPResult, err error) { - if wl.Type != zos.PublicIPv4Type && wl.Type != zos.PublicIPType { - return result, fmt.Errorf("workload for public IP is of wrong type") - } - - if wl.Result.State != gridtypes.StateOk { - return result, fmt.Errorf("public ip workload is not okay") - } - - if err := wl.Result.Unmarshal(&result); err != nil { - return result, errors.Wrap(err, "failed to load ip result") - } - - return result, nil -} diff --git a/pkg/primitives/statistics.go b/pkg/primitives/statistics.go index 41aa534bd..d4c4f0534 100644 --- a/pkg/primitives/statistics.go +++ b/pkg/primitives/statistics.go @@ -277,7 +277,7 @@ func (s *statsStream) ListGPUs() ([]pkg.GPUInfo, error) { if wl.Type != zos.ZMachineType { continue } - var vm zos.ZMachine + var vm zos.ZMachineLight if err := json.Unmarshal(wl.Data, &vm); err != nil { return nil, errors.Wrapf(err, "invalid workload data (%d.%s)", dl.ContractID, wl.Name) } diff --git a/pkg/primitives/vm/gpu.go b/pkg/primitives/vm-light/gpu.go similarity index 99% rename from pkg/primitives/vm/gpu.go rename to pkg/primitives/vm-light/gpu.go index 15e4cab0e..0a71ecc62 100644 --- a/pkg/primitives/vm/gpu.go +++ b/pkg/primitives/vm-light/gpu.go @@ -1,4 +1,4 @@ -package vm +package vmlight import ( "fmt" diff --git a/pkg/primitives/vm/pause.go b/pkg/primitives/vm-light/pause.go similarity index 97% rename from pkg/primitives/vm/pause.go rename to pkg/primitives/vm-light/pause.go index ec0e445ee..8c15a490a 100644 --- a/pkg/primitives/vm/pause.go +++ b/pkg/primitives/vm-light/pause.go @@ -1,4 +1,4 @@ -package vm +package vmlight import ( "context" diff --git a/pkg/primitives/vm/utils.go b/pkg/primitives/vm-light/utils.go similarity index 65% rename from pkg/primitives/vm/utils.go rename to pkg/primitives/vm-light/utils.go index 424fcdf36..a6c150882 100644 --- a/pkg/primitives/vm/utils.go +++ b/pkg/primitives/vm-light/utils.go @@ -1,4 +1,4 @@ -package vm +package vmlight import ( "context" @@ -14,23 +14,16 @@ import ( "github.com/threefoldtech/zos/pkg" "github.com/threefoldtech/zos/pkg/gridtypes" "github.com/threefoldtech/zos/pkg/gridtypes/zos" - "github.com/threefoldtech/zos/pkg/network/ifaceutil" - "github.com/threefoldtech/zos/pkg/primitives/pubip" "github.com/threefoldtech/zos/pkg/stubs" ) -var networkResourceNet = net.IPNet{ - IP: net.ParseIP("100.64.0.0"), - Mask: net.IPv4Mask(0xff, 0xff, 0, 0), -} - // fill up the VM (machine) object with write boot config for a full virtual machine (with a disk image) func (p *Manager) prepVirtualMachine( ctx context.Context, cloudImage string, imageInfo FListInfo, machine *pkg.VM, - config *zos.ZMachine, + config *zos.ZMachineLight, deployment *gridtypes.Deployment, wl *gridtypes.WorkloadWithID, ) error { @@ -81,7 +74,7 @@ func (p *Manager) prepContainer( cloudImage string, imageInfo FListInfo, machine *pkg.VM, - config *zos.ZMachine, + config *zos.ZMachineLight, deployment *gridtypes.Deployment, wl *gridtypes.WorkloadWithID, ) error { @@ -180,64 +173,23 @@ func (p *Manager) prepContainer( } func (p *Manager) newMyceliumNetworkInterface(ctx context.Context, dl gridtypes.Deployment, wl *gridtypes.WorkloadWithID, config *zos.MyceliumIP) (pkg.VMIface, error) { - network := stubs.NewNetworkerStub(p.zbus) + network := stubs.NewNetworkerLightStub(p.zbus) netID := zos.NetworkID(dl.TwinID, config.Network) - tapName := wl.ID.Unique("mycelium") - iface, err := network.SetupMyceliumTap(ctx, tapName, netID, *config) - - if err != nil { - return pkg.VMIface{}, errors.Wrap(err, "could not set up tap device") - } - - out := pkg.VMIface{ - Tap: iface.Name, - MAC: iface.HW.String(), - IPs: []net.IPNet{ - iface.IP, - }, - Routes: []pkg.Route{ - { - Net: net.IPNet{ - IP: net.ParseIP("400::"), - Mask: net.CIDRMask(7, 128), - }, - Gateway: iface.Gateway.IP, - }, - }, - PublicIPv4: false, - PublicIPv6: false, - } - - return out, nil -} + tapName := wl.ID.Unique(string(config.Network)) + iface, err := network.AttachMycelium(ctx, string(netID), tapName, config.Seed) -func (p *Manager) newYggNetworkInterface(ctx context.Context, wl *gridtypes.WorkloadWithID) (pkg.VMIface, error) { - network := stubs.NewNetworkerStub(p.zbus) - - // TODO: if we use `ygg` as a network name. this will conflict - // if the user has a network that is called `ygg`. - tapName := wl.ID.Unique("ygg") - iface, err := network.SetupYggTap(ctx, tapName) if err != nil { return pkg.VMIface{}, errors.Wrap(err, "could not set up tap device") } out := pkg.VMIface{ Tap: iface.Name, - MAC: iface.HW.String(), + MAC: iface.Mac.String(), IPs: []net.IPNet{ - iface.IP, - }, - Routes: []pkg.Route{ - { - Net: net.IPNet{ - IP: net.ParseIP("200::"), - Mask: net.CIDRMask(7, 128), - }, - Gateway: iface.Gateway.IP, - }, + *iface.IP, }, + Routes: iface.Routes, PublicIPv4: false, PublicIPv6: false, } @@ -246,137 +198,33 @@ func (p *Manager) newYggNetworkInterface(ctx context.Context, wl *gridtypes.Work } func (p *Manager) newPrivNetworkInterface(ctx context.Context, dl gridtypes.Deployment, wl *gridtypes.WorkloadWithID, inf zos.MachineInterface) (pkg.VMIface, error) { - network := stubs.NewNetworkerStub(p.zbus) + network := stubs.NewNetworkerLightStub(p.zbus) netID := zos.NetworkID(dl.TwinID, inf.Network) - subnet, err := network.GetSubnet(ctx, netID) - if err != nil { - return pkg.VMIface{}, errors.Wrapf(err, "could not get network resource subnet") - } - - inf.IP = inf.IP.To4() - if inf.IP == nil { - return pkg.VMIface{}, fmt.Errorf("invalid IPv4 supplied to wg interface") - } - - if !subnet.Contains(inf.IP) { - return pkg.VMIface{}, fmt.Errorf("IP %s is not part of local nr subnet %s", inf.IP.String(), subnet.String()) - } - - // always the .1/24 ip is reserved - if inf.IP[3] == 1 { - return pkg.VMIface{}, fmt.Errorf("ip %s is reserved", inf.IP.String()) - } - - privNet, err := network.GetNet(ctx, netID) - if err != nil { - return pkg.VMIface{}, errors.Wrapf(err, "could not get network range") - } - - addrCIDR := net.IPNet{ - IP: inf.IP, - Mask: subnet.Mask, - } - - gw4, gw6, err := network.GetDefaultGwIP(ctx, netID) - if err != nil { - return pkg.VMIface{}, errors.Wrap(err, "could not get network resource default gateway") - } - - privIP6, err := network.GetIPv6From4(ctx, netID, inf.IP) - if err != nil { - return pkg.VMIface{}, errors.Wrap(err, "could not convert private ipv4 to ipv6") - } - tapName := wl.ID.Unique(string(inf.Network)) - iface, err := network.SetupPrivTap(ctx, netID, tapName) + iface, err := network.AttachPrivate(ctx, string(netID), tapName, inf.IP) if err != nil { return pkg.VMIface{}, errors.Wrap(err, "could not set up tap device") } - // the mac address uses the global workload id - // this needs to be the same as how we get it in the actual IP reservation - mac := ifaceutil.HardwareAddrFromInputBytes([]byte(tapName)) - out := pkg.VMIface{ - Tap: iface, - MAC: mac.String(), + Tap: iface.Name, + MAC: iface.Mac.String(), IPs: []net.IPNet{ - addrCIDR, privIP6, + *iface.IP, + // privIP6, }, - Routes: []pkg.Route{ - {Net: privNet, Gateway: gw4}, - {Net: networkResourceNet, Gateway: gw4}, - }, - IP4DefaultGateway: net.IP(gw4), - IP6DefaultGateway: gw6, - PublicIPv4: false, - PublicIPv6: false, - NetID: netID, + Routes: iface.Routes, + IP4DefaultGateway: net.IP(iface.Routes[0].Gateway), + // IP6DefaultGateway: gw6, + PublicIPv4: false, + PublicIPv6: false, + NetID: netID, } return out, nil } -func (p *Manager) newPubNetworkInterface(ctx context.Context, deployment gridtypes.Deployment, cfg ZMachine) (pkg.VMIface, error) { - network := stubs.NewNetworkerStub(p.zbus) - ipWl, err := deployment.Get(cfg.Network.PublicIP) - if err != nil { - return pkg.VMIface{}, err - } - - tapName := ipWl.ID.Unique("pub") - - config, err := pubip.GetPubIPConfig(ipWl) - if err != nil { - return pkg.VMIface{}, errors.Wrap(err, "could not get public ip config") - } - - pubIface, err := network.SetupPubTap(ctx, tapName) - if err != nil { - return pkg.VMIface{}, errors.Wrap(err, "could not set up tap device for public network") - } - - // the mac address uses the global workload id - // this needs to be the same as how we get it in the actual IP reservation - mac := ifaceutil.HardwareAddrFromInputBytes([]byte(tapName)) - - // pubic ip config can has - // - reserved public ipv4 - // - public ipv6 - // - both - // in all cases we have ipv6 it's handed out via slaac, so we don't need - // to set the IP on the interface. We need to configure it ONLY for ipv4 - // hence: - var ips []net.IPNet - var gw4 net.IP - var gw6 net.IP - - if !config.IP.Nil() { - ips = append(ips, config.IP.IPNet) - gw4 = config.Gateway - } - - if !config.IPv6.Nil() { - ips = append(ips, config.IPv6.IPNet) - gw6, err = network.GetPublicIPV6Gateway(ctx) - log.Debug().IPAddr("gw", gw6).Msg("found gateway for ipv6") - if err != nil { - return pkg.VMIface{}, errors.Wrap(err, "failed to get the default gateway for ipv6") - } - } - - return pkg.VMIface{ - Tap: pubIface, - MAC: mac.String(), // mac so we always get the same IPv6 from slaac - IPs: ips, - IP4DefaultGateway: gw4, - IP6DefaultGateway: gw6, - PublicIPv4: config.HasIPv4(), - PublicIPv6: config.HasIPv6(), - }, nil -} - // FListInfo virtual machine details type FListInfo struct { ImagePath string @@ -490,7 +338,7 @@ func (e entry) Envs() map[string]string { // variables. this is used *with* the container configuration like this // - if no zmachine entry point is defined, use the one from .startup.toml // - if envs are defined in flist, merge with the env variables from the -func fListStartup(data *zos.ZMachine, path string) error { +func fListStartup(data *zos.ZMachineLight, path string) error { f, err := os.Open(path) if os.IsNotExist(err) { return nil diff --git a/pkg/primitives/vm/vm.go b/pkg/primitives/vm-light/vm.go similarity index 76% rename from pkg/primitives/vm/vm.go rename to pkg/primitives/vm-light/vm.go index f2a314032..88a6be4c7 100644 --- a/pkg/primitives/vm/vm.go +++ b/pkg/primitives/vm-light/vm.go @@ -1,4 +1,4 @@ -package vm +package vmlight import ( "context" @@ -24,7 +24,7 @@ const ( ) // ZMachine type -type ZMachine = zos.ZMachine +type ZMachine = zos.ZMachineLight var ( _ provision.Manager = (*Manager)(nil) @@ -118,9 +118,9 @@ func (p *Manager) mountQsfs(wl *gridtypes.WorkloadWithID, mount zos.MachineMount return nil } -func (p *Manager) virtualMachineProvisionImpl(ctx context.Context, wl *gridtypes.WorkloadWithID) (result zos.ZMachineResult, err error) { +func (p *Manager) virtualMachineProvisionImpl(ctx context.Context, wl *gridtypes.WorkloadWithID) (result zos.ZMachineLightResult, err error) { var ( - network = stubs.NewNetworkerStub(p.zbus) + network = stubs.NewNetworkerLightStub(p.zbus) flist = stubs.NewFlisterStub(p.zbus) vm = stubs.NewVMModuleStub(p.zbus) @@ -164,11 +164,6 @@ func (p *Manager) virtualMachineProvisionImpl(ctx context.Context, wl *gridtypes } netConfig := config.Network.Interfaces[0] - // check if public ipv4 is supported, should this be requested - if !config.Network.PublicIP.IsEmpty() && !network.PublicIPv4Support(ctx) { - return result, errors.New("public ipv4 is requested, but not supported on this node") - } - result.ID = wl.ID.String() result.IP = netConfig.IP.String() @@ -180,17 +175,10 @@ func (p *Manager) virtualMachineProvisionImpl(ctx context.Context, wl *gridtypes Nameservers: []net.IP{net.ParseIP("8.8.8.8"), net.ParseIP("1.1.1.1"), net.ParseIP("2001:4860:4860::8888")}, } - var ifs []string - var pubIf string - defer func() { + tapName := wl.ID.Unique(string(config.Network.Mycelium.Network)) if err != nil { - for _, nic := range ifs { - _ = network.RemoveTap(ctx, nic) - } - if pubIf != "" { - _ = network.DisconnectPubTap(ctx, pubIf) - } + _ = network.Detach(ctx, tapName) } }() @@ -199,34 +187,7 @@ func (p *Manager) virtualMachineProvisionImpl(ctx context.Context, wl *gridtypes if err != nil { return result, err } - ifs = append(ifs, wl.ID.Unique(string(nic.Network))) - networkInfo.Ifaces = append(networkInfo.Ifaces, inf) - } - - if !config.Network.PublicIP.IsEmpty() { - // some public access is required, we need to add a public - // interface to the machine with the right config. - inf, err := p.newPubNetworkInterface(ctx, deployment, config) - if err != nil { - return result, err - } - - ipWl, _ := deployment.Get(config.Network.PublicIP) - pubIf = ipWl.ID.Unique("pub") - ifs = append(ifs, pubIf) - networkInfo.Ifaces = append(networkInfo.Ifaces, inf) - } - - if config.Network.Planetary { - inf, err := p.newYggNetworkInterface(ctx, wl) - if err != nil { - return result, err - } - ifs = append(ifs, wl.ID.Unique("ygg")) - - log.Debug().Msgf("Planetary: %+v", inf) networkInfo.Ifaces = append(networkInfo.Ifaces, inf) - result.PlanetaryIP = inf.IPs[0].IP.String() } if config.Network.Mycelium != nil { @@ -234,7 +195,6 @@ func (p *Manager) virtualMachineProvisionImpl(ctx context.Context, wl *gridtypes if err != nil { return result, err } - ifs = append(ifs, wl.ID.Unique("mycelium")) networkInfo.Ifaces = append(networkInfo.Ifaces, inf) result.MyceliumIP = inf.IPs[0].IP.String() } @@ -256,7 +216,7 @@ func (p *Manager) virtualMachineProvisionImpl(ctx context.Context, wl *gridtypes // mount cloud-container flist (or reuse) which has kernel, initrd and also firmware hash, err := flist.FlistHash(ctx, cloudContainerFlist) if err != nil { - return zos.ZMachineResult{}, errors.Wrap(err, "failed to get cloud-container flist hash") + return zos.ZMachineLightResult{}, errors.Wrap(err, "failed to get cloud-container flist hash") } // if the name changes (because flist changed, a new mount will be created) @@ -284,13 +244,12 @@ func (p *Manager) virtualMachineProvisionImpl(ctx context.Context, wl *gridtypes machine.Environment = config.Env machine.Hostname = wl.Name.String() - machineInfo, err := vm.Run(ctx, machine) + _, err = vm.Run(ctx, machine) if err != nil { // attempt to delete the vm, should the process still be lingering log.Error().Err(err).Msg("cleaning up vm deployment duo to an error") _ = vm.Delete(ctx, wl.ID.String()) } - result.ConsoleURL = machineInfo.ConsoleURL return result, err } @@ -315,7 +274,7 @@ func (p *Manager) copyFile(srcPath string, destPath string, permissions os.FileM func (p *Manager) Deprovision(ctx context.Context, wl *gridtypes.WorkloadWithID) error { var ( flist = stubs.NewFlisterStub(p.zbus) - network = stubs.NewNetworkerStub(p.zbus) + network = stubs.NewNetworkerLightStub(p.zbus) vm = stubs.NewVMModuleStub(p.zbus) storage = stubs.NewStorageModuleStub(p.zbus) @@ -341,39 +300,10 @@ func (p *Manager) Deprovision(ctx context.Context, wl *gridtypes.WorkloadWithID) log.Error().Err(err).Str("name", volName).Msg("failed to delete rootfs volume") } - for _, inf := range cfg.Network.Interfaces { - tapName := wl.ID.Unique(string(inf.Network)) + tapName := wl.ID.Unique(string(cfg.Network.Mycelium.Network)) - if err := network.RemoveTap(ctx, tapName); err != nil { - return errors.Wrap(err, "could not clean up tap device") - } - } - - if cfg.Network.Planetary { - var tapName string - if cfg.Network.Mycelium == nil { - // yggdrasil network - tapName = wl.ID.Unique("ygg") - } else { - tapName = wl.ID.Unique("mycelium") - } - - if err := network.RemoveTap(ctx, tapName); err != nil { - return errors.Wrap(err, "could not clean up tap device") - } - } - - if len(cfg.Network.PublicIP) > 0 { - // TODO: we need to make sure workload status reflects the actual status by the engine - // this is not the case anymore. - ipWl, err := provision.GetWorkload(ctx, cfg.Network.PublicIP) - if err != nil { - return err - } - ifName := ipWl.ID.Unique("pub") - if err := network.RemovePubTap(ctx, ifName); err != nil { - return errors.Wrap(err, "could not clean up public tap device") - } + if err := network.Detach(ctx, tapName); err != nil { + return errors.Wrap(err, "could not clean up tap device") } return nil diff --git a/pkg/primitives/zdb/zdb.go b/pkg/primitives/zdb/zdb.go index d7bdff41a..4ab7084ce 100644 --- a/pkg/primitives/zdb/zdb.go +++ b/pkg/primitives/zdb/zdb.go @@ -21,7 +21,6 @@ import ( "github.com/pkg/errors" "github.com/rs/zerolog/log" "github.com/threefoldtech/zos/pkg" - nwmod "github.com/threefoldtech/zos/pkg/network" "github.com/threefoldtech/zos/pkg/stubs" ) @@ -261,7 +260,7 @@ func (p *Manager) createZdbContainer(ctx context.Context, device pkg.Device) err cont = stubs.NewContainerModuleStub(p.zbus) flist = stubs.NewFlisterStub(p.zbus) volumePath = device.Path - network = stubs.NewNetworkerStub(p.zbus) + network = stubs.NewNetworkerLightStub(p.zbus) slog = log.With().Str("containerID", string(name)).Logger() ) @@ -284,7 +283,7 @@ func (p *Manager) createZdbContainer(ctx context.Context, device pkg.Device) err } // create the network namespace and macvlan for the 0-db container - netNsName, err := network.EnsureZDBPrepare(ctx, device.ID) + netNsName, err := network.AttachZDB(ctx, device.ID) if err != nil { if err := flist.Unmount(ctx, string(name)); err != nil { slog.Error().Err(err).Str("path", rootFS).Msgf("failed to unmount") @@ -335,32 +334,9 @@ func (p *Manager) createZdbContainer(ctx context.Context, device pkg.Device) err return nil } -// dataMigration will make sure that we delete any data files from v1. This is -// hardly a data migration but at this early stage it's fine since there is still -// no real data loads live on the grid. All v2 zdbs, will be safe. -func (p *Manager) dataMigration(ctx context.Context, volume string) { - v1, _ := zdb.IsZDBVersion1(ctx, volume) - // TODO: what if there is an error? - if !v1 { - // it's eather a new volume, or already on version 2 - // so nothing to do. - return - } - - for _, sub := range []string{"data", "index"} { - if err := os.RemoveAll(filepath.Join(volume, sub)); err != nil { - log.Error().Err(err).Msg("failed to delete obsolete data directories") - } - } -} - func (p *Manager) zdbRun(ctx context.Context, name string, rootfs string, cmd string, netns string, volumepath string, socketdir string) error { cont := stubs.NewContainerModuleStub(p.zbus) - // we do data migration here because this is called - // on new zdb starts, or updating the runtime. - p.dataMigration(ctx, volumepath) - conf := pkg.Container{ Name: name, RootFS: rootfs, @@ -389,73 +365,22 @@ func (p *Manager) zdbRun(ctx context.Context, name string, rootfs string, cmd st } func (p *Manager) waitZDBIPs(ctx context.Context, namespace string, created time.Time) ([]net.IP, error) { - // TODO: this method need to be abstracted, since it's now depends on the knewledge - // of the networking daemon internal (interfaces names) - // may be at least just get all ips from all interfaces inside the namespace - // will be a slightly better solution + // TODO: is there a need for retrying anymore?? var ( - network = stubs.NewNetworkerStub(p.zbus) + network = stubs.NewNetworkerLightStub(p.zbus) containerIPs []net.IP ) log.Debug().Time("created-at", created).Str("namespace", namespace).Msg("checking zdb container ips") getIP := func() error { - // some older setups that might still be running has PubIface set to zdb0 not eth0 - // so we need to make sure that this we also try this older name - ips, _, err := network.Addrs(ctx, nwmod.PubIface, namespace) - if err != nil { - var err2 error - ips, _, err2 = network.Addrs(ctx, "zdb0", namespace) - if err2 != nil { - log.Debug().Err(err).Msg("no public ip found, waiting") - return err - } - } - - yggIps, _, err := network.Addrs(ctx, nwmod.ZDBYggIface, namespace) + ips, err := network.ZDBIPs(ctx, namespace) if err != nil { return err } - ips = append(ips, yggIps...) - - MyceliumIps, _, err := network.Addrs(ctx, nwmod.ZDBMyceliumIface, namespace) - if err != nil { - return err - } - ips = append(ips, MyceliumIps...) - - var ( - public = false - ygg = false - mycelium = false - ) - containerIPs = containerIPs[:0] - for _, ip := range ips { - if isPublic(ip) && !isYgg(ip) && !isMycelium(ip) { - log.Warn().IPAddr("ip", ip).Msg("0-db container public ip found") - public = true - containerIPs = append(containerIPs, ip) - } - if isYgg(ip) { - log.Warn().IPAddr("ip", ip).Msg("0-db container ygg ip found") - ygg = true - containerIPs = append(containerIPs, ip) - } - if isMycelium(ip) { - log.Warn().IPAddr("ip", ip).Msg("0-db container mycelium ip found") - mycelium = true - containerIPs = append(containerIPs, ip) - } + containerIPs = append(containerIPs, ip) } - - log.Warn().Msgf("public %v ygg: %v mycelium: %v", public, ygg, mycelium) - if public && ygg && mycelium || time.Since(created) > 2*time.Minute { - // if we have all ips detected or if the container is older than 2 minutes - // so it's safe we assume ips are final - return nil - } - return fmt.Errorf("waiting for more addresses") + return nil } bo := backoff.NewExponentialBackOff() @@ -754,43 +679,12 @@ func ipsToString(ips []net.IP) []string { return result } -// isPublic check if ip is a IPv6 public address -func isPublic(ip net.IP) bool { - if ip.To4() != nil { - return false - } - - return !(ip.IsLoopback() || - ip.IsLinkLocalUnicast() || - ip.IsLinkLocalMulticast() || - ip.IsInterfaceLocalMulticast()) -} - -// isPublic check if ip is a part of the yggdrasil 200::/7 range -var yggNet = net.IPNet{ - IP: net.ParseIP("200::"), - Mask: net.CIDRMask(7, 128), -} - -var myceliumNet = net.IPNet{ - IP: net.ParseIP("400::"), - Mask: net.CIDRMask(7, 128), -} - -func isYgg(ip net.IP) bool { - return yggNet.Contains(ip) -} - -func isMycelium(ip net.IP) bool { - return myceliumNet.Contains(ip) -} - // InitializeZDB makes sure all required zdbs are running func (p *Manager) Initialize(ctx context.Context) error { var ( storage = stubs.NewStorageModuleStub(p.zbus) contmod = stubs.NewContainerModuleStub(p.zbus) - network = stubs.NewNetworkerStub(p.zbus) + network = stubs.NewNetworkerLightStub(p.zbus) flistmod = stubs.NewFlisterStub(p.zbus) ) // fetching extected hash @@ -823,9 +717,9 @@ func (p *Manager) Initialize(ctx context.Context) error { } log.Debug().Str("container", string(container)).Msg("enusreing zdb network setup") - _, err := network.EnsureZDBPrepare(ctx, poolNames[string(container)].ID) + _, err := network.AttachZDB(ctx, poolNames[string(container)].ID) if err != nil { - log.Error().Err(err).Msg("failed to prepare zdb network") + log.Error().Err(err).Msg("failed to initialize zdb network") } delete(poolNames, string(container)) diff --git a/pkg/primitives/zlogs/zlogs.go b/pkg/primitives/zlogs/zlogs.go index c067f731a..d2f629487 100644 --- a/pkg/primitives/zlogs/zlogs.go +++ b/pkg/primitives/zlogs/zlogs.go @@ -29,7 +29,7 @@ func NewManager(zbus zbus.Client) *Manager { func (p *Manager) Provision(ctx context.Context, wl *gridtypes.WorkloadWithID) (interface{}, error) { var ( vm = stubs.NewVMModuleStub(p.zbus) - network = stubs.NewNetworkerStub(p.zbus) + network = stubs.NewNetworkerLightStub(p.zbus) ) var cfg zos.ZLogs @@ -46,7 +46,7 @@ func (p *Manager) Provision(ctx context.Context, wl *gridtypes.WorkloadWithID) ( return nil, errors.Wrapf(err, "machine state is not ok") } - var machineCfg zos.ZMachine + var machineCfg zos.ZMachineLight if err := json.Unmarshal(machine.Data, &machineCfg); err != nil { return nil, errors.Wrap(err, "failed to decode zlogs config") } @@ -60,10 +60,11 @@ func (p *Manager) Provision(ctx context.Context, wl *gridtypes.WorkloadWithID) ( } twin, _ := provision.GetDeploymentID(ctx) + netID := zos.NetworkID(twin, net) return nil, vm.StreamCreate(ctx, machine.ID.String(), pkg.Stream{ ID: wl.ID.String(), - Namespace: network.Namespace(ctx, zos.NetworkID(twin, net)), + Namespace: network.Namespace(ctx, netID.String()), Output: cfg.Output, }) diff --git a/pkg/primitives/zmount/zmount.go b/pkg/primitives/zmount/zmount.go index 9cf526d72..ba842823c 100644 --- a/pkg/primitives/zmount/zmount.go +++ b/pkg/primitives/zmount/zmount.go @@ -103,7 +103,7 @@ func (p *Manager) zMountUpdateImpl(ctx context.Context, wl *gridtypes.WorkloadWi continue } - var data zos.ZMachine + var data zos.ZMachineLight if err := json.Unmarshal(vm.Data, &data); err != nil { return vol, provision.UnChanged(errors.Wrap(err, "failed to load vm information")) } diff --git a/pkg/provision/engine.go b/pkg/provision/engine.go index 5880fa27e..53cbc6e3a 100644 --- a/pkg/provision/engine.go +++ b/pkg/provision/engine.go @@ -1126,7 +1126,7 @@ func (n *NativeEngine) ListPrivateIPs(twin uint32, network gridtypes.Name) ([]st if err != nil { return nil, err } - zmachine := data.(*zos.ZMachine) + zmachine := data.(*zos.ZMachineLight) for _, inf := range zmachine.Network.Interfaces { if inf.Network == network { ips = append(ips, inf.IP.String()) diff --git a/pkg/qsfsd/cleanup.go b/pkg/qsfsd/cleanup.go deleted file mode 100644 index 2a8a7a140..000000000 --- a/pkg/qsfsd/cleanup.go +++ /dev/null @@ -1,169 +0,0 @@ -package qsfsd - -import ( - "context" - "os" - "path" - "path/filepath" - "syscall" - "time" - - "github.com/containerd/containerd/errdefs" - "github.com/containernetworking/plugins/pkg/ns" - "github.com/pkg/errors" - "github.com/rs/zerolog/log" - "github.com/threefoldtech/zos/pkg" - "github.com/threefoldtech/zos/pkg/stubs" -) - -const ( - UploadLimit = 100 * 1024 * 1024 // kill it for less 100 MB every 10 minutes - checkPeriod = 10 * time.Minute -) - -type failedQSFSState struct { - lastUploadMap map[string]uint64 - metricsFailureCount map[string]uint -} - -func newState() *failedQSFSState { - return &failedQSFSState{ - lastUploadMap: make(map[string]uint64), - metricsFailureCount: make(map[string]uint), - } -} -func (f *failedQSFSState) delete(wlID string) { - delete(f.lastUploadMap, wlID) - delete(f.metricsFailureCount, wlID) -} - -func (q *QSFS) periodicCleanup(ctx context.Context) { - state := newState() - t := time.NewTicker(checkPeriod) - for { - select { - case <-t.C: - if err := q.checkDeadQSFSs(ctx, state); err != nil { - log.Error().Err(err).Msg("a failed qsfs cleanup round") - } - case <-ctx.Done(): - return - } - } -} - -func (q *QSFS) checkDeadQSFSs(ctx context.Context, state *failedQSFSState) error { - paths, err := filepath.Glob(filepath.Join(q.tombstonesPath, "*")) - if err != nil { - return errors.Wrap(err, "couldn't list deleted containers") - } - for _, path := range paths { - wlID := filepath.Base(path) - metrics, err := q.qsfsMetrics(ctx, string(wlID)) - if err != nil { - log.Err(err).Str("id", string(wlID)).Msg("couldn't get qsfs metrics") - state.metricsFailureCount[string(wlID)] += 1 - if state.metricsFailureCount[string(wlID)] >= 10 { - q.Unmount(wlID) - state.delete(wlID) - } - continue - } - uploaded := metrics.NetTxBytes - if lastUploaded, ok := state.lastUploadMap[wlID]; ok && uploaded-lastUploaded < UploadLimit { - // didn't upload enough => dead - q.Unmount(wlID) - state.delete(wlID) - } else { - // first time or uploaded a lot in the last 10 minutes - state.lastUploadMap[wlID] = uploaded - } - } - return nil -} - -func (q *QSFS) isMarkedForDeletion(ctx context.Context, wlID string) (bool, error) { - tombstonePath := q.tombstone(wlID) - _, err := os.Stat(tombstonePath) - if errors.Is(err, os.ErrNotExist) { - // not dead - return false, nil - } else if err != nil { - return false, errors.Wrap(err, "failed to check the container death mark") - } - return true, nil -} - -func (q *QSFS) isOldMarkedForDeletion(ctx context.Context, wlID string) (bool, error) { - contd := stubs.NewContainerModuleStub(q.cl) - contID := pkg.ContainerID(wlID) - cont, err := contd.Inspect(ctx, qsfsContainerNS, contID) - - if errors.Is(err, errdefs.ErrNotFound) { - // not found - // - return false, nil - } - if err != nil { - return false, errors.Wrap(err, "failed to fetch qsfs container for a cleanup check") - } - tombstonePath := filepath.Join(cont.RootFS, ".death.scheduled") - _, err = os.Stat(tombstonePath) - if errors.Is(err, os.ErrNotExist) { - // not dead - return false, nil - } else if err != nil { - return false, errors.Wrap(err, "failed to check the container death mark") - } - return true, nil -} - -func (q *QSFS) markDelete(ctx context.Context, wlID string) error { - tombstonePath := q.tombstone(wlID) - file, err := os.Create(tombstonePath) - if err != nil { - return errors.Wrap(err, "couldn't mark qsfs container for deletion") - } - file.Close() - return nil -} - -func (q *QSFS) tombstone(wlID string) string { - return path.Join(q.tombstonesPath, wlID) -} - -func (q *QSFS) Unmount(wlID string) { - networkd := stubs.NewNetworkerStub(q.cl) - flistd := stubs.NewFlisterStub(q.cl) - contd := stubs.NewContainerModuleStub(q.cl) - - ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) - defer cancel() - // listing all containers and matching the name looks like a lot of work - if err := contd.Delete(ctx, qsfsContainerNS, pkg.ContainerID(wlID)); err != nil { - log.Error().Err(err).Msg("failed to delete qsfs container") - } - mountPath := q.mountPath(wlID) - // unmount twice, once for the zdbfs and the self-mount - for i := 0; i < 2; i++ { - if err := syscall.Unmount(mountPath, 0); err != nil && !errors.Is(err, syscall.EINVAL) { - log.Error().Err(err).Msg("failed to unmount mount path 1st time") - } - } - if err := os.RemoveAll(mountPath); err != nil && !errors.Is(err, os.ErrNotExist) { - log.Error().Err(err).Msg("failed to remove mountpath dir") - } - if err := os.RemoveAll(q.tombstone(wlID)); err != nil && !errors.Is(err, os.ErrNotExist) { - log.Error().Err(err).Msg("failed to remove tombstone path") - } - if err := flistd.Unmount(ctx, wlID); err != nil { - log.Error().Err(err).Msg("failed to unmount flist") - } - - if err := networkd.QSFSDestroy(ctx, wlID); err != nil { - if _, ok := err.(ns.NSPathNotExistErr); !ok { - // log any error other than that the namespace doesn't exist - log.Error().Err(err).Msg("failed to destroy qsfs network") - } - } -} diff --git a/pkg/qsfsd/metrics.go b/pkg/qsfsd/metrics.go deleted file mode 100644 index be23900ca..000000000 --- a/pkg/qsfsd/metrics.go +++ /dev/null @@ -1,71 +0,0 @@ -package qsfsd - -import ( - "context" - "path/filepath" - "time" - - "github.com/containernetworking/plugins/pkg/ns" - "github.com/pkg/errors" - "github.com/rs/zerolog/log" - "github.com/threefoldtech/zos/pkg" - "github.com/threefoldtech/zos/pkg/network/namespace" - "github.com/threefoldtech/zos/pkg/stubs" - "github.com/vishvananda/netlink" -) - -// Metrics gets running qsfs network metrics -func (q *QSFS) Metrics() (pkg.QSFSMetrics, error) { - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) - defer cancel() - result := make(map[string]pkg.NetMetric) - - items, err := filepath.Glob(filepath.Join(q.mountsPath, "*")) - if err != nil { - return pkg.QSFSMetrics{}, errors.Wrap(err, "failed to list mounts directory") - } - for _, item := range items { - name := filepath.Base(item) - metrics, err := q.qsfsMetrics(ctx, name) - if err != nil { - log.Error().Err(err).Str("id", name).Msg("failed to get qsfs metrics") - continue - } - result[name] = metrics - } - return pkg.QSFSMetrics{Consumption: result}, nil -} -func (q *QSFS) qsfsMetrics(ctx context.Context, wlID string) (pkg.NetMetric, error) { - var m pkg.NetMetric - networker := stubs.NewNetworkerStub(q.cl) - nsName := networker.QSFSNamespace(ctx, wlID) - netNs, err := namespace.GetByName(nsName) - if err != nil { - return m, errors.Wrap(err, "didn't find qsfs namespace") - } - defer netNs.Close() - err = netNs.Do(func(_ ns.NetNS) error { - m, err = metricsForNics([]string{"public", "ygg0"}) - return err - }) - if err != nil { - return m, errors.Wrap(err, "failed to read metrics") - } - return m, nil -} -func metricsForNics(nics []string) (pkg.NetMetric, error) { - var m pkg.NetMetric - for _, nic := range nics { - link, err := netlink.LinkByName(nic) - if err != nil { - return m, errors.Wrapf(err, "couldn't get nic %s info", nic) - } - stats := link.Attrs().Statistics - m.NetRxBytes += stats.RxBytes - m.NetRxPackets += stats.RxPackets - m.NetTxBytes += stats.TxBytes - m.NetTxPackets += stats.TxPackets - } - - return m, nil -} diff --git a/pkg/qsfsd/qsfs.go b/pkg/qsfsd/qsfs.go deleted file mode 100644 index a77db2bc6..000000000 --- a/pkg/qsfsd/qsfs.go +++ /dev/null @@ -1,317 +0,0 @@ -package qsfsd - -import ( - "context" - "encoding/json" - "fmt" - "os" - "os/exec" - "path/filepath" - "syscall" - "time" - - "github.com/BurntSushi/toml" - "github.com/pkg/errors" - "github.com/rs/zerolog/log" - "github.com/threefoldtech/zbus" - "github.com/threefoldtech/zos/pkg" - "github.com/threefoldtech/zos/pkg/gridtypes" - "github.com/threefoldtech/zos/pkg/gridtypes/zos" - "github.com/threefoldtech/zos/pkg/stubs" -) - -const ( - qsfsFlist = "https://hub.grid.tf/tf-autobuilder/qsfs-0.2.0-rc2.flist" - qsfsContainerNS = "qsfs" - qsfsRootFsPropagation = pkg.RootFSPropagationSlave - zstorSocket = "/var/run/zstor.sock" - zstorZDBFSMountPoint = "/mnt" // hardcoded in the container - zstorMetricsPort = 9100 - zstorZDBDataDirPath = "/data/data/zdbfs-data" - tombstonesDir = "tombstones" -) - -type QSFS struct { - cl zbus.Client - - mountsPath string - tombstonesPath string -} - -type zstorConfig struct { - zos.QuantumSafeFSConfig - ZDBDataDirPath string `toml:"zdb_data_dir_path"` - Socket string `toml:"socket"` - MetricsPort uint32 `toml:"prometheus_port"` - ZDBFSMountpoint string `toml:"zdbfs_mountpoint"` -} - -func New(ctx context.Context, cl zbus.Client, root string) (pkg.QSFSD, error) { - mountPath := filepath.Join(root, "mounts") - err := os.MkdirAll(mountPath, 0644) - if err != nil { - return nil, errors.Wrap(err, "couldn't make qsfs mounts dir") - } - tombstonesPath := filepath.Join(root, tombstonesDir) - err = os.MkdirAll(tombstonesPath, 0644) - if err != nil { - return nil, errors.Wrap(err, "couldn't make qsfs tombstones dir") - } - - qsfs := &QSFS{ - cl: cl, - mountsPath: mountPath, - tombstonesPath: tombstonesPath, - } - if err := qsfs.migrateTombstones(ctx, cl); err != nil { - return nil, err - } - go qsfs.periodicCleanup(ctx) - return qsfs, nil -} - -func (q *QSFS) migrateTombstones(ctx context.Context, cl zbus.Client) error { - contd := stubs.NewContainerModuleStub(q.cl) - containers, err := contd.List(ctx, qsfsContainerNS) - if err != nil { - return errors.Wrap(err, "couldn't list qsfs containers") - } - for _, contID := range containers { - marked, err := q.isOldMarkedForDeletion(ctx, string(contID)) - if err != nil { - log.Error().Err(err).Str("id", string(contID)).Msg("failed to check container old mark") - } - if marked { - if err := q.markDelete(ctx, string(contID)); err != nil { - log.Error().Err(err).Str("id", string(contID)).Msg("failed to mark container for deletion") - } - } - } - return nil -} - -func setQSFSDefaults(cfg *zos.QuantumSafeFS) zstorConfig { - return zstorConfig{ - QuantumSafeFSConfig: cfg.Config, - Socket: zstorSocket, - MetricsPort: zstorMetricsPort, - ZDBFSMountpoint: zstorZDBFSMountPoint, - ZDBDataDirPath: zstorZDBDataDirPath, - } -} - -func (q *QSFS) Mount(wlID string, cfg zos.QuantumSafeFS) (info pkg.QSFSInfo, err error) { - defer func() { - if err != nil { - q.Unmount(wlID) - } - }() - zstorConfig := setQSFSDefaults(&cfg) - networkd := stubs.NewNetworkerStub(q.cl) - flistd := stubs.NewFlisterStub(q.cl) - contd := stubs.NewContainerModuleStub(q.cl) - - ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) - t := time.Now() - defer cancel() - marked, _ := q.isMarkedForDeletion(ctx, wlID) - if marked { - return info, errors.New("qsfs marked for deletion, try again later") - } - netns, yggIP, err := networkd.QSFSPrepare(ctx, wlID) - if err != nil { - return info, errors.Wrap(err, "failed to prepare qsfs") - } - flistPath, err := flistd.Mount(ctx, wlID, qsfsFlist, pkg.MountOptions{ - ReadOnly: false, - Limit: cfg.Cache, - }) - if err != nil { - err = errors.Wrap(err, "failed to mount qsfs flist") - return - } - if lerr := q.writeQSFSConfig(flistPath, zstorConfig); lerr != nil { - err = errors.Wrap(lerr, "couldn't write qsfs config") - return - } - mountPath := q.mountPath(wlID) - err = q.prepareMountPath(mountPath) - if err != nil { - err = errors.Wrap(err, "failed to prepare mount path") - return - } - cont := pkg.Container{ - Name: wlID, - RootFS: flistPath, - Entrypoint: "/sbin/zinit init --container", - Interactive: false, - Network: pkg.NetworkInfo{Namespace: netns}, - Memory: gridtypes.Gigabyte, - Mounts: []pkg.MountInfo{ - { - Source: mountPath, - Target: "/mnt", - }, - }, - Elevated: true, - // the default is rslave which recursively sets all mountpoints to slave - // we don't care about the rootfs propagation, it just has to be non-recursive - RootFsPropagation: qsfsRootFsPropagation, - } - _, err = contd.Run( - ctx, - qsfsContainerNS, - cont, - ) - log.Debug().Str("duration", time.Since(t).String()).Msg("time before waiting for qsfs mountpoint") - if lerr := q.waitUntilMounted(ctx, mountPath); lerr != nil { - logs, containerErr := contd.Logs(ctx, qsfsContainerNS, wlID) - if containerErr != nil { - log.Error().Err(containerErr).Msg("Failed to read container logs") - } - err = errors.Wrapf(lerr, fmt.Sprintf("Container Logs:\n%s", logs)) - return - } - log.Debug().Str("duration", time.Since(t).String()).Msg("waiting for qsfs deployment took") - info.Path = mountPath - info.MetricsEndpoint = fmt.Sprintf("http://[%s]:%d/metrics", yggIP, zstorMetricsPort) - - return -} - -func (f *QSFS) waitUntilMounted(ctx context.Context, path string) error { - ticker := time.NewTicker(1 * time.Second) - for { - select { - case <-ticker.C: - mounted, err := f.isMounted(path) - if err != nil { - return errors.Wrap(err, "failed to check if zdbfs is mounted") - } - if mounted { - return nil - } - case <-ctx.Done(): - return fmt.Errorf("waiting for zdbfs mount %s timedout: context cancelled", path) - } - } - -} - -func (f *QSFS) isMounted(path string) (bool, error) { - output, err := exec.Command("findmnt", "-J", path).Output() - if err, ok := err.(*exec.ExitError); ok && err != nil { - if err.ExitCode() == 1 { - return false, nil - } - } - var result struct { - Filesystems []struct { - Fstype string `json:"fstype"` - } `json:"filesystems"` - } - - if err := json.Unmarshal(output, &result); err != nil { - return false, errors.Wrap(err, "failed to parse findmnt output") - } - for _, fs := range result.Filesystems { - if fs.Fstype == "fuse.zdbfs" { - return true, nil - } - } - return false, nil -} - -func (q *QSFS) UpdateMount(wlID string, cfg zos.QuantumSafeFS) (pkg.QSFSInfo, error) { - var info pkg.QSFSInfo - zstorConfig := setQSFSDefaults(&cfg) - networkd := stubs.NewNetworkerStub(q.cl) - flistd := stubs.NewFlisterStub(q.cl) - contd := stubs.NewContainerModuleStub(q.cl) - - ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) - defer cancel() - - marked, err := q.isMarkedForDeletion(ctx, wlID) - if marked { - return info, errors.New("already marked for deletion") - } - if err != nil { - return info, errors.Wrap(err, "failed to check deletion mark") - } - yggIP, err := networkd.QSFSYggIP(ctx, wlID) - if err != nil { - return info, errors.Wrap(err, "failed to get ygg ip") - } - flistPath, err := flistd.UpdateMountSize(ctx, wlID, cfg.Cache) - if err != nil { - return info, errors.Wrap(err, "failed to get qsfs flist mountpoint") - } - if err := q.writeQSFSConfig(flistPath, zstorConfig); err != nil { - return info, errors.Wrap(err, "couldn't write qsfs config") - } - mountPath := q.mountPath(wlID) - - if err := contd.Exec(ctx, qsfsContainerNS, wlID, 10*time.Second, "/sbin/zinit", "kill", "zstor", "SIGINT"); err != nil { - return info, errors.Wrap(err, "failed to restart zstor process") - } - info.Path = mountPath - info.MetricsEndpoint = fmt.Sprintf("http://[%s]:%d/metrics", yggIP, zstorMetricsPort) - return info, nil -} - -func (q *QSFS) SignalDelete(wlID string) error { - contd := stubs.NewContainerModuleStub(q.cl) - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - marked, err := q.isMarkedForDeletion(ctx, wlID) - - if marked { - return nil - } - if err != nil { - return errors.Wrap(err, "failed to check deletion mark") - } - if err := q.markDelete(ctx, wlID); err != nil { - // container dead, no need to continue - return err - } - if err := contd.SignalDelete(ctx, qsfsContainerNS, pkg.ContainerID(wlID)); err != nil { - return errors.Wrap(err, "couldn't stop qsfs container") - } - return nil -} - -func (q *QSFS) mountPath(wlID string) string { - return filepath.Join(q.mountsPath, wlID) -} - -func (q *QSFS) prepareMountPath(path string) error { - if err := os.MkdirAll(path, 0644); err != nil { - return err - } - - // container mounts doesn't appear on the host if this is not mounted - if err := syscall.Mount(path, path, "bind", syscall.MS_BIND, ""); err != nil { - return err - } - if err := syscall.Mount("", path, "", syscall.MS_SHARED, ""); err != nil { - return err - } - return nil -} - -func (q *QSFS) writeQSFSConfig(root string, cfg zstorConfig) error { - cfgPath := filepath.Join(root, "data/zstor.toml") - f, err := os.OpenFile(cfgPath, os.O_WRONLY|os.O_CREATE, 0644) - if err != nil { - return errors.Wrap(err, "couldn't open zstor config file") - } - defer f.Close() - - if err := toml.NewEncoder(f).Encode(cfg); err != nil { - return errors.Wrap(err, "failed to convert config to yaml") - } - - return nil -} diff --git a/pkg/registrar/register.go b/pkg/registrar/register.go index 891d2ee37..eb9811258 100644 --- a/pkg/registrar/register.go +++ b/pkg/registrar/register.go @@ -4,7 +4,6 @@ import ( "context" "crypto/ed25519" "fmt" - "net" "time" "github.com/centrifuge/go-substrate-rpc-client/v4/types" @@ -106,24 +105,24 @@ func registerNode( ) (nodeID, twinID uint32, err error) { var ( mgr = stubs.NewIdentityManagerStub(cl) - netMgr = stubs.NewNetworkerStub(cl) + netMgr = stubs.NewNetworkerLightStub(cl) substrateGateway = stubs.NewSubstrateGatewayStub(cl) ) - zosIps, zosMac, err := netMgr.Addrs(ctx, "zos", "") + infs, err := netMgr.Interfaces(ctx, "zos", "") if err != nil { return 0, 0, errors.Wrap(err, "failed to get zos bridge information") } interfaces := []substrate.Interface{ { - Name: "zos", - Mac: zosMac, + Name: infs.Interfaces["zos"].Name, + Mac: infs.Interfaces["zos"].Mac, IPs: func() []string { - var ips []string - for _, ip := range zosIps { - ipV := net.IP(ip) - ips = append(ips, ipV.String()) + ips := make([]string, 0) + for _, ip := range infs.Interfaces["zos"].IPs { + + ips = append(ips, ip.IP.String()) } return ips }(), diff --git a/pkg/registrar/registrar.go b/pkg/registrar/registrar.go index 25e6756db..021e77eb6 100644 --- a/pkg/registrar/registrar.go +++ b/pkg/registrar/registrar.go @@ -132,7 +132,7 @@ func (r *Registrar) register(ctx context.Context, cl zbus.Client, env environmen register() - stub := stubs.NewNetworkerStub(cl) + stub := stubs.NewNetworkerLightStub(cl) addressesUpdate, err := stub.ZOSAddresses(ctx) if err != nil { log.Error().Err(err).Msg("failed to monitor ip changes") diff --git a/pkg/stubs/network_light_stub.go b/pkg/stubs/network_light_stub.go new file mode 100644 index 000000000..cad6ad091 --- /dev/null +++ b/pkg/stubs/network_light_stub.go @@ -0,0 +1,214 @@ +// GENERATED CODE +// -------------- +// please do not edit manually instead use the "zbusc" to regenerate + +package stubs + +import ( + "context" + zbus "github.com/threefoldtech/zbus" + pkg "github.com/threefoldtech/zos/pkg" + "net" +) + +type NetworkerLightStub struct { + client zbus.Client + module string + object zbus.ObjectID +} + +func NewNetworkerLightStub(client zbus.Client) *NetworkerLightStub { + return &NetworkerLightStub{ + client: client, + module: "netlight", + object: zbus.ObjectID{ + Name: "netlight", + Version: "0.0.1", + }, + } +} + +func (s *NetworkerLightStub) AttachMycelium(ctx context.Context, arg0 string, arg1 string, arg2 []uint8) (ret0 pkg.TapDevice, ret1 error) { + args := []interface{}{arg0, arg1, arg2} + result, err := s.client.RequestContext(ctx, s.module, s.object, "AttachMycelium", args...) + if err != nil { + panic(err) + } + result.PanicOnError() + ret1 = result.CallError() + loader := zbus.Loader{ + &ret0, + } + if err := result.Unmarshal(&loader); err != nil { + panic(err) + } + return +} + +func (s *NetworkerLightStub) AttachPrivate(ctx context.Context, arg0 string, arg1 string, arg2 []uint8) (ret0 pkg.TapDevice, ret1 error) { + args := []interface{}{arg0, arg1, arg2} + result, err := s.client.RequestContext(ctx, s.module, s.object, "AttachPrivate", args...) + if err != nil { + panic(err) + } + result.PanicOnError() + ret1 = result.CallError() + loader := zbus.Loader{ + &ret0, + } + if err := result.Unmarshal(&loader); err != nil { + panic(err) + } + return +} + +func (s *NetworkerLightStub) AttachZDB(ctx context.Context, arg0 string) (ret0 string, ret1 error) { + args := []interface{}{arg0} + result, err := s.client.RequestContext(ctx, s.module, s.object, "AttachZDB", args...) + if err != nil { + panic(err) + } + result.PanicOnError() + ret1 = result.CallError() + loader := zbus.Loader{ + &ret0, + } + if err := result.Unmarshal(&loader); err != nil { + panic(err) + } + return +} + +func (s *NetworkerLightStub) Create(ctx context.Context, arg0 string, arg1 net.IPNet, arg2 []uint8) (ret0 error) { + args := []interface{}{arg0, arg1, arg2} + result, err := s.client.RequestContext(ctx, s.module, s.object, "Create", args...) + if err != nil { + panic(err) + } + result.PanicOnError() + ret0 = result.CallError() + loader := zbus.Loader{} + if err := result.Unmarshal(&loader); err != nil { + panic(err) + } + return +} + +func (s *NetworkerLightStub) Delete(ctx context.Context, arg0 string) (ret0 error) { + args := []interface{}{arg0} + result, err := s.client.RequestContext(ctx, s.module, s.object, "Delete", args...) + if err != nil { + panic(err) + } + result.PanicOnError() + ret0 = result.CallError() + loader := zbus.Loader{} + if err := result.Unmarshal(&loader); err != nil { + panic(err) + } + return +} + +func (s *NetworkerLightStub) Detach(ctx context.Context, arg0 string) (ret0 error) { + args := []interface{}{arg0} + result, err := s.client.RequestContext(ctx, s.module, s.object, "Detach", args...) + if err != nil { + panic(err) + } + result.PanicOnError() + ret0 = result.CallError() + loader := zbus.Loader{} + if err := result.Unmarshal(&loader); err != nil { + panic(err) + } + return +} + +func (s *NetworkerLightStub) Interfaces(ctx context.Context, arg0 string, arg1 string) (ret0 pkg.Interfaces, ret1 error) { + args := []interface{}{arg0, arg1} + result, err := s.client.RequestContext(ctx, s.module, s.object, "Interfaces", args...) + if err != nil { + panic(err) + } + result.PanicOnError() + ret1 = result.CallError() + loader := zbus.Loader{ + &ret0, + } + if err := result.Unmarshal(&loader); err != nil { + panic(err) + } + return +} + +func (s *NetworkerLightStub) Namespace(ctx context.Context, arg0 string) (ret0 string) { + args := []interface{}{arg0} + result, err := s.client.RequestContext(ctx, s.module, s.object, "Namespace", args...) + if err != nil { + panic(err) + } + result.PanicOnError() + loader := zbus.Loader{ + &ret0, + } + if err := result.Unmarshal(&loader); err != nil { + panic(err) + } + return +} + +func (s *NetworkerLightStub) Ready(ctx context.Context) (ret0 error) { + args := []interface{}{} + result, err := s.client.RequestContext(ctx, s.module, s.object, "Ready", args...) + if err != nil { + panic(err) + } + result.PanicOnError() + ret0 = result.CallError() + loader := zbus.Loader{} + if err := result.Unmarshal(&loader); err != nil { + panic(err) + } + return +} + +func (s *NetworkerLightStub) ZDBIPs(ctx context.Context, arg0 string) (ret0 [][]uint8, ret1 error) { + args := []interface{}{arg0} + result, err := s.client.RequestContext(ctx, s.module, s.object, "ZDBIPs", args...) + if err != nil { + panic(err) + } + result.PanicOnError() + ret1 = result.CallError() + loader := zbus.Loader{ + &ret0, + } + if err := result.Unmarshal(&loader); err != nil { + panic(err) + } + return +} + +func (s *NetworkerLightStub) ZOSAddresses(ctx context.Context) (<-chan pkg.NetlinkAddresses, error) { + ch := make(chan pkg.NetlinkAddresses, 1) + recv, err := s.client.Stream(ctx, s.module, s.object, "ZOSAddresses") + if err != nil { + return nil, err + } + go func() { + defer close(ch) + for event := range recv { + var obj pkg.NetlinkAddresses + if err := event.Unmarshal(&obj); err != nil { + panic(err) + } + select { + case <-ctx.Done(): + return + case ch <- obj: + default: + } + } + }() + return ch, nil +} diff --git a/pkg/upgrade/upgrade.go b/pkg/upgrade/upgrade.go index 7f2f75398..fdabe6118 100644 --- a/pkg/upgrade/upgrade.go +++ b/pkg/upgrade/upgrade.go @@ -200,9 +200,13 @@ func (u *Upgrader) nextUpdate() time.Duration { func (u *Upgrader) remote() (remote hub.TagLink, err error) { mode := u.boot.RunMode() // find all taglinks that matches the same run mode (ex: development) + envName := mode.String() + if u.boot.Version().Major == 4 { + envName = fmt.Sprintf("%s-light", envName) + } matches, err := u.hub.Find( ZosRepo, - hub.MatchName(mode.String()), + hub.MatchName(envName), hub.MatchType(hub.TypeTagLink), ) if err != nil { @@ -210,7 +214,7 @@ func (u *Upgrader) remote() (remote hub.TagLink, err error) { } if len(matches) != 1 { - return remote, fmt.Errorf("can't find taglink that matches '%s'", mode.String()) + return remote, fmt.Errorf("can't find taglink that matches '%s'", envName) } return hub.NewTagLink(matches[0]), nil diff --git a/pkg/vm/ch.go b/pkg/vm/ch.go index 30ed5ac8b..913aceb71 100644 --- a/pkg/vm/ch.go +++ b/pkg/vm/ch.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "io" - "math" "net" "os" "os/exec" @@ -24,43 +23,6 @@ const ( cloudConsoleBin = "cloud-console" ) -// startCloudConsole Starts the cloud console for the vm on it's private network ip -func (m *Machine) startCloudConsole(ctx context.Context, namespace string, networkAddr net.IPNet, machineIP net.IPNet, ptyPath string, logs string) (string, error) { - ipv4 := machineIP.IP.To4() - if ipv4 == nil { - return "", fmt.Errorf("invalid vm ip address (%s) not ipv4", machineIP.IP.String()) - } - port := 20000 + uint16(ipv4[3]) - if port == math.MaxUint16 { - // this should be impossible since a byte max value is 512 hence 20_000 + 512 can never be over - // max of uint16 - return "", fmt.Errorf("couldn't start cloud console port number exceeds %d", port) - } - args := []string{ - "setsid", - "ip", - "netns", - "exec", namespace, - cloudConsoleBin, - ptyPath, - networkAddr.IP.String(), - fmt.Sprint(port), - logs, - } - - log.Debug().Msgf("running cloud-console : %+v", args) - - cmd := exec.CommandContext(ctx, "busybox", args...) - if err := cmd.Start(); err != nil { - return "", errors.Wrap(err, "failed to start cloud-hypervisor") - } - if err := m.release(cmd.Process); err != nil { - return "", err - } - consoleURL := fmt.Sprintf("%s:%d", networkAddr.IP.String(), port) - return consoleURL, nil -} - // Run run the machine with cloud-hypervisor func (m *Machine) Run(ctx context.Context, socket, logs string) (pkg.MachineInfo, error) { _ = os.Remove(socket) @@ -131,12 +93,16 @@ func (m *Machine) Run(ctx context.Context, socket, logs string) (pkg.MachineInfo for _, nic := range m.Interfaces { var typ InterfaceType - typ, _, err = nic.getType() + typ, idx, err := nic.getType() if err != nil { return pkg.MachineInfo{}, errors.Wrapf(err, "failed to detect interface type '%s'", nic.Tap) } if typ == InterfaceTAP { interfaces = append(interfaces, nic.asTap()) + } else if typ == InterfaceMacvtap { + fd := len(fds) + 3 + fds = append(fds, idx) + interfaces = append(interfaces, nic.asMACvTap(fd)) } else { err = fmt.Errorf("unsupported tap device type '%s'", nic.Tap) return pkg.MachineInfo{}, err @@ -222,22 +188,13 @@ func (m *Machine) Run(ctx context.Context, socket, logs string) (pkg.MachineInfo return pkg.MachineInfo{}, err } client := NewClient(socket) - vmData, err := client.Inspect(ctx) + _, err = client.Inspect(ctx) if err != nil { return pkg.MachineInfo{}, errors.Wrapf(err, "failed to Inspect vm with id: '%s'", m.ID) } - consoleURL := "" - for _, ifc := range m.Interfaces { - if ifc.Console != nil { - consoleURL, err = m.startCloudConsole(ctx, ifc.Console.Namespace, ifc.Console.ListenAddress, ifc.Console.VmAddress, vmData.PTYPath, logs) - if err != nil { - log.Error().Err(err).Str("vm", m.ID).Msg("failed to start cloud-console for vm") - } - } - } - return pkg.MachineInfo{ConsoleURL: consoleURL}, nil + return pkg.MachineInfo{}, nil } func (m *Machine) waitAndAdjOom(ctx context.Context, name string, socket string) error { diff --git a/pkg/vm/machine.go b/pkg/vm/machine.go index fa7c755b3..23193ea0e 100644 --- a/pkg/vm/machine.go +++ b/pkg/vm/machine.go @@ -48,7 +48,8 @@ type InterfaceType string const ( // InterfaceTAP tuntap type - InterfaceTAP InterfaceType = "tuntap" + InterfaceTAP InterfaceType = "tuntap" + InterfaceMacvtap InterfaceType = "macvtap" ) type Console struct { @@ -76,6 +77,17 @@ func (i Interface) asTap() string { return buf.String() } +// asMACvTap returns the command line argument for this interface as a macvtap +func (i Interface) asMACvTap(fd int) string { + var buf bytes.Buffer + buf.WriteString(fmt.Sprintf("fd=%d", fd)) + if len(i.Mac) > 0 { + buf.WriteString(fmt.Sprintf(",mac=%s", i.Mac)) + } + + return buf.String() +} + // getType detects the interface type func (i *Interface) getType() (InterfaceType, int, error) { link, err := netlink.LinkByName(i.Tap) @@ -87,8 +99,10 @@ func (i *Interface) getType() (InterfaceType, int, error) { switch InterfaceType(link.Type()) { case InterfaceTAP: return InterfaceTAP, link.Attrs().Index, nil + case InterfaceMacvtap: + return InterfaceMacvtap, link.Attrs().Index, nil default: - return "", 0, fmt.Errorf("unknown tap type") + return "", 0, fmt.Errorf("unknown tap type %s", link.Type()) } } diff --git a/pkg/vm/manager.go b/pkg/vm/manager.go index 42524dac2..5d341f121 100644 --- a/pkg/vm/manager.go +++ b/pkg/vm/manager.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "io" - "net" "os" "path/filepath" "strings" @@ -18,7 +17,6 @@ import ( "github.com/threefoldtech/zbus" "github.com/threefoldtech/zos/pkg" "github.com/threefoldtech/zos/pkg/gridtypes" - "github.com/threefoldtech/zos/pkg/stubs" "github.com/threefoldtech/zos/pkg/vm/cloudinit" ) @@ -148,31 +146,6 @@ func (m *Module) Exists(id string) bool { return err == nil } -func (m *Module) getConsoleConfig(ctx context.Context, ifc pkg.VMIface) (*Console, error) { - stub := stubs.NewNetworkerStub(m.client) - namespace := stub.Namespace(ctx, ifc.NetID) - - networkAddr, err := stub.GetSubnet(ctx, ifc.NetID) - if err != nil { - return nil, errors.Wrapf(err, "failed to get network '%s'", ifc.NetID) - } - - networkAddr.IP = networkAddr.IP.To4() - - if len(networkAddr.IP) != net.IPv4len { - return nil, fmt.Errorf("invalid network address: %s", networkAddr.IP.String()) - } - - // always listen on ip .1 - networkAddr.IP[3] = 1 - - return &Console{ - Namespace: namespace, - ListenAddress: networkAddr, - VmAddress: ifc.IPs[0], - }, nil -} - func (m *Module) makeNetwork(ctx context.Context, vm *pkg.VM, cfg *cloudinit.Configuration) ([]Interface, error) { // assume there is always at least 1 iface present @@ -202,14 +175,6 @@ func (m *Module) makeNetwork(ctx context.Context, vm *pkg.VM, cfg *cloudinit.Con Tap: ifcfg.Tap, Mac: ifcfg.MAC, } - if ifcfg.NetID != "" && len(ifcfg.IPs) > 0 { - // if NetID is set on this interface means it is a private network so we add console config to it. - console, err := m.getConsoleConfig(ctx, ifcfg) - if err != nil { - return nil, errors.Wrapf(err, "could not get console config for vm %s", vm.Name) - } - nic.Console = console - } nics = append(nics, nic) cinet := cloudinit.Ethernet{ diff --git a/pkg/zos_api/admin.go b/pkg/zos_api/admin.go index 4b53237e6..81d1292bf 100644 --- a/pkg/zos_api/admin.go +++ b/pkg/zos_api/admin.go @@ -13,7 +13,7 @@ func (g *ZosAPI) adminInterfacesHandler(ctx context.Context, payload []byte) (in Mac string `json:"mac"` } - interfaces, err := g.networkerStub.Interfaces(ctx, "", "") + interfaces, err := g.networkerLightStub.Interfaces(ctx, "", "") if err != nil { return nil, err } @@ -35,7 +35,7 @@ func (g *ZosAPI) adminInterfacesHandler(ctx context.Context, payload []byte) (in } func (g *ZosAPI) adminGetPublicNICHandler(ctx context.Context, payload []byte) (interface{}, error) { - return g.networkerStub.GetPublicExitDevice(ctx) + return nil, fmt.Errorf("not supported") } func (g *ZosAPI) adminSetPublicNICHandler(ctx context.Context, payload []byte) (interface{}, error) { @@ -43,5 +43,6 @@ func (g *ZosAPI) adminSetPublicNICHandler(ctx context.Context, payload []byte) ( if err := json.Unmarshal(payload, &iface); err != nil { return nil, fmt.Errorf("failed to decode input, expecting string: %w", err) } - return nil, g.networkerStub.SetPublicExitDevice(ctx, iface) + return nil, fmt.Errorf("not supported") + } diff --git a/pkg/zos_api/network.go b/pkg/zos_api/network.go index ecc524aa8..f763f2a5a 100644 --- a/pkg/zos_api/network.go +++ b/pkg/zos_api/network.go @@ -10,45 +10,21 @@ import ( "github.com/threefoldtech/zos/pkg/gridtypes" ) -func (g *ZosAPI) networkListWGPortsHandler(ctx context.Context, payload []byte) (interface{}, error) { - return g.networkerStub.WireguardPorts(ctx) -} -func (g *ZosAPI) networkPublicConfigGetHandler(ctx context.Context, payload []byte) (interface{}, error) { - return g.networkerStub.GetPublicConfig(ctx) -} func (g *ZosAPI) networkInterfacesHandler(ctx context.Context, payload []byte) (interface{}, error) { - results := make(map[string][]net.IP) - type q struct { - inf string - ns string - rename string - } - for _, i := range []q{{"zos", "", "zos"}, {"nygg6", "ndmz", "ygg"}} { - ips, _, err := g.networkerStub.Addrs(ctx, i.inf, i.ns) - if err != nil { - return nil, fmt.Errorf("failed to get ips for '%s' interface: %w", i, err) - } - - results[i.rename] = func() []net.IP { - list := make([]net.IP, 0, len(ips)) - for _, item := range ips { - ip := net.IP(item) - list = append(list, ip) - } - - return list - }() + results := make(map[string][]net.IPNet) + interfaces, err := g.networkerLightStub.Interfaces(ctx, "zos", "") + if err != nil { + return nil, fmt.Errorf("failed to get ips for 'zos' interface: %w", err) } + zosIfc := interfaces.Interfaces["zos"] + results["zos"] = zosIfc.IPs return results, nil } + func (g *ZosAPI) networkHasIPv6Handler(ctx context.Context, payload []byte) (interface{}, error) { - ipData, err := g.networkerStub.GetPublicIPv6Subnet(ctx) - hasIP := ipData.IP != nil && err == nil - return hasIP, err -} -func (g *ZosAPI) networkListPublicIPsHandler(ctx context.Context, payload []byte) (interface{}, error) { - return g.provisionStub.ListPublicIPs(ctx) + // networkd light + return false, nil } func (g *ZosAPI) networkListPrivateIPsHandler(ctx context.Context, payload []byte) (interface{}, error) { diff --git a/pkg/zos_api/routes.go b/pkg/zos_api/routes.go index 7fa70493a..ca2b13d43 100644 --- a/pkg/zos_api/routes.go +++ b/pkg/zos_api/routes.go @@ -25,11 +25,11 @@ func (g *ZosAPI) SetupRoutes(router *peer.Router) { storage.WithHandler("pools", g.storagePoolsHandler) network := root.SubRoute("network") - network.WithHandler("list_wg_ports", g.networkListWGPortsHandler) - network.WithHandler("public_config_get", g.networkPublicConfigGetHandler) + // network.WithHandler("list_wg_ports", g.networkListWGPortsHandler) + // network.WithHandler("public_config_get", g.networkPublicConfigGetHandler) network.WithHandler("interfaces", g.networkInterfacesHandler) network.WithHandler("has_ipv6", g.networkHasIPv6Handler) - network.WithHandler("list_public_ips", g.networkListPublicIPsHandler) + // network.WithHandler("list_public_ips", g.networkListPublicIPsHandler) network.WithHandler("list_private_ips", g.networkListPrivateIPsHandler) statistics := root.SubRoute("statistics") diff --git a/pkg/zos_api/zos_api.go b/pkg/zos_api/zos_api.go index 3e21ef61b..bd1fc876b 100644 --- a/pkg/zos_api/zos_api.go +++ b/pkg/zos_api/zos_api.go @@ -15,7 +15,7 @@ type ZosAPI struct { oracle *capacity.ResourceOracle versionMonitorStub *stubs.VersionMonitorStub provisionStub *stubs.ProvisionStub - networkerStub *stubs.NetworkerStub + networkerLightStub *stubs.NetworkerLightStub statisticsStub *stubs.StatisticsStub storageStub *stubs.StorageModuleStub performanceMonitorStub *stubs.PerformanceMonitorStub @@ -38,7 +38,7 @@ func NewZosAPI(manager substrate.Manager, client zbus.Client, msgBrokerCon strin oracle: capacity.NewResourceOracle(storageModuleStub), versionMonitorStub: stubs.NewVersionMonitorStub(client), provisionStub: stubs.NewProvisionStub(client), - networkerStub: stubs.NewNetworkerStub(client), + networkerLightStub: stubs.NewNetworkerLightStub(client), statisticsStub: stubs.NewStatisticsStub(client), storageStub: storageModuleStub, performanceMonitorStub: stubs.NewPerformanceMonitorStub(client), diff --git a/qemu/overlay.normal/bin/netlightd b/qemu/overlay.normal/bin/netlightd new file mode 120000 index 000000000..25eae5c70 --- /dev/null +++ b/qemu/overlay.normal/bin/netlightd @@ -0,0 +1 @@ +../../../bin/zos \ No newline at end of file diff --git a/tools/zos-update-worker/internal/update_worker.go b/tools/zos-update-worker/internal/update_worker.go index e540c786d..05d3c4d7a 100644 --- a/tools/zos-update-worker/internal/update_worker.go +++ b/tools/zos-update-worker/internal/update_worker.go @@ -2,6 +2,7 @@ package internal import ( "context" + "encoding/json" "fmt" "os" "path/filepath" @@ -103,12 +104,22 @@ func (w *Worker) updateZosVersion(network Network, manager client.Manager) error } defer con.Close() - currentZosVersion, err := con.GetZosVersion() + currentZosVersions, err := con.GetZosVersion() if err != nil { return err } - log.Debug().Msgf("getting substrate version %v for network %v", currentZosVersion, network) + type ZosVersions struct { + Zos string + ZosLight string + } + versions := ZosVersions{} + + err = json.Unmarshal([]byte(currentZosVersions), &versions) + if err != nil { + return err + } + log.Debug().Msgf("getting substrate version %v for network %v", versions, network) // now we need to find how dst is relative to src path, err := filepath.Rel(w.dst, w.src) @@ -116,42 +127,50 @@ func (w *Worker) updateZosVersion(network Network, manager client.Manager) error return fmt.Errorf("failed to get dst relative path to src: %w", err) } - zosCurrent := fmt.Sprintf("%v/.tag-%v", w.src, currentZosVersion) + //zos + zosCurrent := fmt.Sprintf("%v/.tag-%v", w.src, versions.Zos) zosLatest := fmt.Sprintf("%v/%v", w.dst, network) + // zos light + zosLightCurrent := fmt.Sprintf("%v/.tag-%v", w.src, versions.ZosLight) + zosLightLatest := fmt.Sprintf("%v/%v-light", w.dst, network) // the link is like zosCurrent but it has the path relative from the symlink // point of view (so relative to the symlink, how to reach zosCurrent) // hence the link is instead used in all calls to symlink - link := fmt.Sprintf("%v/.tag-%v", path, currentZosVersion) + zosLink := fmt.Sprintf("%v/.tag-%v", path, versions.Zos) + zosLightLink := fmt.Sprintf("%v/.tag-%v", path, versions.ZosLight) - // check if current exists - if _, err := os.Lstat(zosCurrent); err != nil { + // update links for both zos and zoslight + if err = w.updateLink(zosCurrent, zosLatest, zosLink); err != nil { return err } + return w.updateLink(zosLightCurrent, zosLightLatest, zosLightLink) +} +func (w *Worker) updateLink(current string, latest string, link string) error { // check if symlink exists - dst, err := os.Readlink(zosLatest) + dst, err := os.Readlink(latest) // if no symlink, then create it if os.IsNotExist(err) { - log.Info().Str("from", zosLatest).Str("to", zosCurrent).Msg("linking") - return os.Symlink(link, zosLatest) + log.Info().Str("from", latest).Str("to", current).Msg("linking") + return os.Symlink(link, latest) } else if err != nil { return err } // check if symlink is valid and exists - if filepath.Base(dst) == filepath.Base(zosCurrent) { - log.Debug().Msgf("symlink %v to %v already exists", zosCurrent, zosLatest) + if filepath.Base(dst) == filepath.Base(current) { + log.Debug().Msgf("symlink %v to %v already exists", current, latest) return nil } // remove symlink if it is not valid and exists - if err := os.Remove(zosLatest); err != nil { + if err := os.Remove(latest); err != nil { return err } - log.Info().Str("from", zosLatest).Str("to", zosCurrent).Msg("linking") - return os.Symlink(link, zosLatest) + log.Info().Str("from", latest).Str("to", current).Msg("linking") + return os.Symlink(link, latest) } // UpdateWithInterval updates the latest zos flist for a specific network with the updated zos version