From e73f2f7bb240eca4273c10a66093fee0d78453f5 Mon Sep 17 00:00:00 2001 From: Kristoffer Dalby Date: Thu, 2 Feb 2023 16:05:52 +0100 Subject: [PATCH] Add node expiry test This commits adds a test to verify that nodes get updated if a node in their network expires. Signed-off-by: Kristoffer Dalby --- .../test-integration-v2-TestExpireNode.yaml | 57 +++++++++++ integration/general_test.go | 99 ++++++++++++++++--- integration/scenario.go | 25 ++++- integration/tailscale.go | 3 +- integration/tsic/tsic.go | 68 +++++++++++-- integration/utils.go | 51 ++++++++++ 6 files changed, 281 insertions(+), 22 deletions(-) create mode 100644 .github/workflows/test-integration-v2-TestExpireNode.yaml create mode 100644 integration/utils.go diff --git a/.github/workflows/test-integration-v2-TestExpireNode.yaml b/.github/workflows/test-integration-v2-TestExpireNode.yaml new file mode 100644 index 0000000000..bfa4faa730 --- /dev/null +++ b/.github/workflows/test-integration-v2-TestExpireNode.yaml @@ -0,0 +1,57 @@ +# DO NOT EDIT, generated with cmd/gh-action-integration-generator/main.go +# To regenerate, run "go generate" in cmd/gh-action-integration-generator/ + +name: Integration Test v2 - TestExpireNode + +on: [pull_request] + +concurrency: + group: ${{ github.workflow }}-$${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + test: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 2 + + - name: Get changed files + id: changed-files + uses: tj-actions/changed-files@v34 + with: + files: | + *.nix + go.* + **/*.go + integration_test/ + config-example.yaml + + - uses: cachix/install-nix-action@v18 + if: ${{ env.ACT }} || steps.changed-files.outputs.any_changed == 'true' + + - name: Run general integration tests + if: steps.changed-files.outputs.any_changed == 'true' + run: | + nix develop --command -- docker run \ + --tty --rm \ + --volume ~/.cache/hs-integration-go:/go \ + --name headscale-test-suite \ + --volume $PWD:$PWD -w $PWD/integration \ + --volume /var/run/docker.sock:/var/run/docker.sock \ + --volume $PWD/control_logs:/tmp/control \ + golang:1 \ + go test ./... \ + -tags ts2019 \ + -failfast \ + -timeout 120m \ + -parallel 1 \ + -run "^TestExpireNode$" + + - uses: actions/upload-artifact@v3 + if: always() && steps.changed-files.outputs.any_changed == 'true' + with: + name: logs + path: "control_logs/*.log" diff --git a/integration/general_test.go b/integration/general_test.go index 10a8d03354..f3187e3a93 100644 --- a/integration/general_test.go +++ b/integration/general_test.go @@ -1,16 +1,19 @@ package integration import ( + "encoding/json" "fmt" "net/netip" "strings" "testing" "time" + v1 "github.com/juanfont/headscale/gen/go/headscale/v1" "github.com/juanfont/headscale/integration/hsic" "github.com/juanfont/headscale/integration/tsic" "github.com/rs/zerolog/log" "github.com/samber/lo" + "github.com/stretchr/testify/assert" ) func TestPingAllByIP(t *testing.T) { @@ -556,20 +559,92 @@ func TestResolveMagicDNS(t *testing.T) { } } -func pingAllHelper(t *testing.T, clients []TailscaleClient, addrs []string) int { - t.Helper() - success := 0 +func TestExpireNode(t *testing.T) { + IntegrationSkip(t) + t.Parallel() - for _, client := range clients { - for _, addr := range addrs { - err := client.Ping(addr) - if err != nil { - t.Errorf("failed to ping %s from %s: %s", addr, client.Hostname(), err) - } else { - success++ - } + scenario, err := NewScenario() + if err != nil { + t.Errorf("failed to create scenario: %s", err) + } + + spec := map[string]int{ + "user1": len(TailscaleVersions), + } + + err = scenario.CreateHeadscaleEnv(spec, []tsic.Option{}, hsic.WithTestName("expirenode")) + if err != nil { + t.Errorf("failed to create headscale environment: %s", err) + } + + allClients, err := scenario.ListTailscaleClients() + if err != nil { + t.Errorf("failed to get clients: %s", err) + } + + allIps, err := scenario.ListTailscaleClientsIPs() + if err != nil { + t.Errorf("failed to get clients: %s", err) + } + + err = scenario.WaitForTailscaleSync() + if err != nil { + t.Errorf("failed wait for tailscale clients to be in sync: %s", err) + } + + allAddrs := lo.Map(allIps, func(x netip.Addr, index int) string { + return x.String() + }) + + success := pingAllHelper(t, allClients, allAddrs) + t.Logf("before expire: %d successful pings out of %d", success, len(allClients)*len(allIps)) + + for _, client := range allClients { + status, err := client.Status() + assert.NoError(t, err) + + // Assert that we have the original count - self + assert.Len(t, status.Peers(), len(TailscaleVersions)-1) + } + + headscale, err := scenario.Headscale() + assert.NoError(t, err) + + // TODO(kradalby): This is Headscale specific and would not play nicely + // with other implementations of the ControlServer interface + result, err := headscale.Execute([]string{ + "headscale", "nodes", "expire", "--identifier", "0", "--output", "json", + }) + assert.NoError(t, err) + + var machine v1.Machine + err = json.Unmarshal([]byte(result), &machine) + assert.NoError(t, err) + + time.Sleep(30 * time.Second) + + // Verify that the expired not is no longer present in the Peer list + // of connected nodes. + for _, client := range allClients { + status, err := client.Status() + assert.NoError(t, err) + + for _, peerKey := range status.Peers() { + peerStatus := status.Peer[peerKey] + + peerPublicKey := strings.TrimPrefix(peerStatus.PublicKey.String(), "nodekey:") + + assert.NotEqual(t, machine.NodeKey, peerPublicKey) + } + + if client.Hostname() != machine.Name { + // Assert that we have the original count - self - expired node + assert.Len(t, status.Peers(), len(TailscaleVersions)-2) } } - return success + err = scenario.Shutdown() + if err != nil { + t.Errorf("failed to tear down scenario: %s", err) + } } diff --git a/integration/scenario.go b/integration/scenario.go index f44d4e61c8..cfdaa97ef3 100644 --- a/integration/scenario.go +++ b/integration/scenario.go @@ -26,6 +26,7 @@ const ( var ( errNoHeadscaleAvailable = errors.New("no headscale available") errNoUserAvailable = errors.New("no user available") + errNoClientFound = errors.New("client not found") // Tailscale started adding TS2021 support in CapabilityVersion>=28 (v1.24.0), but // proper support in Headscale was only added for CapabilityVersion>=39 clients (v1.30.0). @@ -203,7 +204,11 @@ func (s *Scenario) Headscale(opts ...hsic.Option) (ControlServer, error) { return headscale, nil } -func (s *Scenario) CreatePreAuthKey(user string, reusable bool, ephemeral bool) (*v1.PreAuthKey, error) { +func (s *Scenario) CreatePreAuthKey( + user string, + reusable bool, + ephemeral bool, +) (*v1.PreAuthKey, error) { if headscale, err := s.Headscale(); err == nil { key, err := headscale.CreateAuthKey(user, reusable, ephemeral) if err != nil { @@ -440,6 +445,24 @@ func (s *Scenario) ListTailscaleClients(users ...string) ([]TailscaleClient, err return allClients, nil } +func (s *Scenario) FindTailscaleClientByIP(ip netip.Addr) (TailscaleClient, error) { + clients, err := s.ListTailscaleClients() + if err != nil { + return nil, err + } + + for _, client := range clients { + ips, _ := client.IPs() + for _, ip2 := range ips { + if ip == ip2 { + return client, nil + } + } + } + + return nil, errNoClientFound +} + func (s *Scenario) ListTailscaleClientsIPs(users ...string) ([]netip.Addr, error) { var allIps []netip.Addr diff --git a/integration/tailscale.go b/integration/tailscale.go index 77a6b31a5f..5a7697a745 100644 --- a/integration/tailscale.go +++ b/integration/tailscale.go @@ -4,6 +4,7 @@ import ( "net/netip" "net/url" + "github.com/juanfont/headscale/integration/tsic" "tailscale.com/ipn/ipnstate" ) @@ -22,6 +23,6 @@ type TailscaleClient interface { WaitForReady() error WaitForLogout() error WaitForPeers(expected int) error - Ping(hostnameOrIP string) error + Ping(hostnameOrIP string, opts ...tsic.PingOption) error ID() string } diff --git a/integration/tsic/tsic.go b/integration/tsic/tsic.go index 768a0a9e2a..6c0c88ede4 100644 --- a/integration/tsic/tsic.go +++ b/integration/tsic/tsic.go @@ -7,7 +7,9 @@ import ( "log" "net/netip" "net/url" + "strconv" "strings" + "time" "github.com/cenkalti/backoff/v4" "github.com/juanfont/headscale" @@ -49,6 +51,7 @@ type TailscaleInContainer struct { headscaleCert []byte headscaleHostname string withSSH bool + withTags []string } type Option = func(c *TailscaleInContainer) @@ -85,6 +88,12 @@ func WithHeadscaleName(hsName string) Option { } } +func WithTags(tags []string) Option { + return func(tsic *TailscaleInContainer) { + tsic.withTags = tags + } +} + func WithSSH() Option { return func(tsic *TailscaleInContainer) { tsic.withSSH = true @@ -231,6 +240,12 @@ func (t *TailscaleInContainer) Up( command = append(command, "--ssh") } + if len(t.withTags) > 0 { + command = append(command, + fmt.Sprintf(`--advertise-tags=%s`, strings.Join(t.withTags, ",")), + ) + } + if _, _, err := t.Execute(command); err != nil { return fmt.Errorf("failed to join tailscale client: %w", err) } @@ -390,16 +405,53 @@ func (t *TailscaleInContainer) WaitForPeers(expected int) error { }) } +type PingOption = func(args *pingArgs) +type pingArgs struct { + timeout time.Duration + count int + direct bool +} + +func WithPingTimeout(timeout time.Duration) PingOption { + return func(args *pingArgs) { + args.timeout = timeout + } +} + +func WithPingCount(count int) PingOption { + return func(args *pingArgs) { + args.count = count + } +} + +func WithPingUntilDirect(direct bool) PingOption { + return func(args *pingArgs) { + args.direct = direct + } +} + // TODO(kradalby): Make multiping, go routine magic. -func (t *TailscaleInContainer) Ping(hostnameOrIP string) error { +func (t *TailscaleInContainer) Ping(hostnameOrIP string, opts ...PingOption) error { + args := pingArgs{ + timeout: time.Second, + count: 10, + direct: true, + } + + for _, opt := range opts { + opt(&args) + } + + command := []string{ + "tailscale", "ping", + fmt.Sprintf("--timeout=%s", args.timeout), + fmt.Sprintf("--c=%d", args.count), + fmt.Sprintf("--until-direct=%s", strconv.FormatBool(args.direct)), + } + + command = append(command, hostnameOrIP) + return t.pool.Retry(func() error { - command := []string{ - "tailscale", "ping", - "--timeout=1s", - "--c=10", - "--until-direct=true", - hostnameOrIP, - } result, _, err := t.Execute(command) if err != nil { diff --git a/integration/utils.go b/integration/utils.go new file mode 100644 index 0000000000..31635609b4 --- /dev/null +++ b/integration/utils.go @@ -0,0 +1,51 @@ +package integration + +import ( + "testing" + "time" + + "github.com/juanfont/headscale/integration/tsic" +) + +func pingAllHelper(t *testing.T, clients []TailscaleClient, addrs []string) int { + t.Helper() + success := 0 + + for _, client := range clients { + for _, addr := range addrs { + err := client.Ping(addr) + if err != nil { + t.Errorf("failed to ping %s from %s: %s", addr, client.Hostname(), err) + } else { + success++ + } + } + } + + return success +} + +// pingAllNegativeHelper is intended to have 1 or more nodes timeing out from the ping, +// it counts failures instead of successes. +func pingAllNegativeHelper(t *testing.T, clients []TailscaleClient, addrs []string) int { + t.Helper() + failures := 0 + + timeout := 100 + count := 3 + + for _, client := range clients { + for _, addr := range addrs { + err := client.Ping( + addr, + tsic.WithPingTimeout(time.Duration(timeout)*time.Millisecond), + tsic.WithPingCount(count), + ) + if err != nil { + failures++ + } + } + } + + return failures +}