From 6745390ee1ad30aa16dc13a91655d31267df354e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ivan=20Miri=C4=87?= Date: Thu, 27 Aug 2020 12:01:33 +0200 Subject: [PATCH 1/8] Update outdated comment --- js/bundle.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/js/bundle.go b/js/bundle.go index e0fcb5ce33c..a179dd107e0 100644 --- a/js/bundle.go +++ b/js/bundle.go @@ -249,7 +249,7 @@ func (b *Bundle) Instantiate(logger logrus.FieldLogger, vuID int64) (bi *BundleI } // Grab any exported functions that could be executed. These were - // already pre-validated in NewBundle(), just get them here. + // already pre-validated in cmd.validateScenarioConfig(), just get them here. exports := rt.Get("exports").ToObject(rt) for k := range b.exports { fn, _ := goja.AssertFunction(exports.Get(k)) From 5c97ddb7cebc4477018deec909b846a0f8e81a81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ivan=20Miri=C4=87?= Date: Thu, 27 Aug 2020 13:14:09 +0200 Subject: [PATCH 2/8] Make DNS resolver configurable, drop dnscache dependency --- cmd/config.go | 6 + cmd/config_consolidation_test.go | 91 ++++++++++- cmd/options.go | 12 ++ core/local/local_test.go | 104 ++++++++++++- go.mod | 1 - go.sum | 2 - js/initcontext_test.go | 13 +- js/runner.go | 68 +++++++- lib/dns_select_gen.go | 52 +++++++ lib/netext/dialer.go | 27 +--- lib/netext/dialer_test.go | 30 ++-- lib/netext/httpext/tracer_test.go | 6 +- lib/netext/resolver.go | 140 +++++++++++++++++ lib/netext/resolver_test.go | 96 ++++++++++++ lib/options.go | 147 ++++++++++++++++++ lib/testutils/httpmultibin/httpmultibin.go | 2 +- lib/testutils/mockresolver/resolver.go | 81 ++++++++++ .../github.com/viki-org/dnscache/dnscache.go | 77 --------- .../github.com/viki-org/dnscache/license.txt | 19 --- vendor/github.com/viki-org/dnscache/readme.md | 38 ----- vendor/modules.txt | 3 - 21 files changed, 815 insertions(+), 200 deletions(-) create mode 100644 lib/dns_select_gen.go create mode 100644 lib/netext/resolver.go create mode 100644 lib/netext/resolver_test.go create mode 100644 lib/testutils/mockresolver/resolver.go delete mode 100644 vendor/github.com/viki-org/dnscache/dnscache.go delete mode 100644 vendor/github.com/viki-org/dnscache/license.txt delete mode 100644 vendor/github.com/viki-org/dnscache/readme.md diff --git a/cmd/config.go b/cmd/config.go index 54c7cddbd22..46993266442 100644 --- a/cmd/config.go +++ b/cmd/config.go @@ -259,6 +259,12 @@ func applyDefault(conf Config) Config { if conf.Options.SummaryTrendStats == nil { conf.Options.SummaryTrendStats = lib.DefaultSummaryTrendStats } + if !conf.DNS.TTL.Valid { + conf.DNS.TTL = lib.DefaultDNSConfig().TTL + } + if !conf.DNS.Select.Valid { + conf.DNS.Select = lib.DefaultDNSConfig().Select + } return conf } diff --git a/cmd/config_consolidation_test.go b/cmd/config_consolidation_test.go index b11763c7197..2628af5e785 100644 --- a/cmd/config_consolidation_test.go +++ b/cmd/config_consolidation_test.go @@ -139,7 +139,7 @@ type file struct { func getFS(files []file) afero.Fs { fs := afero.NewMemMapFs() for _, f := range files { - must(afero.WriteFile(fs, f.filepath, []byte(f.contents), 0644)) // modes don't matter in the afero.MemMapFs + must(afero.WriteFile(fs, f.filepath, []byte(f.contents), 0o644)) // modes don't matter in the afero.MemMapFs } return fs } @@ -214,11 +214,13 @@ func getConfigConsolidationTestCases() []configConsolidationTestCase { {opts{cli: []string{"-u", "3", "-d", "30s"}}, exp{}, verifyConstLoopingVUs(I(3), 30*time.Second)}, {opts{cli: []string{"-u", "4", "--duration", "60s"}}, exp{}, verifyConstLoopingVUs(I(4), 1*time.Minute)}, { - opts{cli: []string{"--stage", "20s:10", "-s", "3m:5"}}, exp{}, + opts{cli: []string{"--stage", "20s:10", "-s", "3m:5"}}, + exp{}, verifyRampingVUs(null.NewInt(1, false), buildStages(20, 10, 180, 5)), }, { - opts{cli: []string{"-s", "1m6s:5", "--vus", "10"}}, exp{}, + opts{cli: []string{"-s", "1m6s:5", "--vus", "10"}}, + exp{}, verifyRampingVUs(null.NewInt(10, true), buildStages(66, 5)), }, {opts{cli: []string{"-u", "1", "-i", "6", "-d", "10s"}}, exp{}, func(t *testing.T, c Config) { @@ -248,11 +250,13 @@ func getConfigConsolidationTestCases() []configConsolidationTestCase { {opts{env: []string{"K6_VUS=5", "K6_ITERATIONS=15"}}, exp{}, verifySharedIters(I(5), I(15))}, {opts{env: []string{"K6_VUS=10", "K6_DURATION=20s"}}, exp{}, verifyConstLoopingVUs(I(10), 20*time.Second)}, { - opts{env: []string{"K6_STAGES=2m30s:11,1h1m:100"}}, exp{}, + opts{env: []string{"K6_STAGES=2m30s:11,1h1m:100"}}, + exp{}, verifyRampingVUs(null.NewInt(1, false), buildStages(150, 11, 3660, 100)), }, { - opts{env: []string{"K6_STAGES=100s:100,0m30s:0", "K6_VUS=0"}}, exp{}, + opts{env: []string{"K6_STAGES=100s:100,0m30s:0", "K6_VUS=0"}}, + exp{}, verifyRampingVUs(null.NewInt(0, true), buildStages(100, 100, 30, 0)), }, // Test if JSON configs work as expected @@ -275,14 +279,16 @@ func getConfigConsolidationTestCases() []configConsolidationTestCase { env: []string{"K6_DURATION=15s"}, cli: []string{"--stage", ""}, }, - exp{logWarning: true}, verifyOneIterPerOneVU, + exp{logWarning: true}, + verifyOneIterPerOneVU, }, { opts{ runner: &lib.Options{VUs: null.IntFrom(5), Duration: types.NullDurationFrom(50 * time.Second)}, cli: []string{"--stage", "5s:5"}, }, - exp{}, verifyRampingVUs(I(5), buildStages(5, 5)), + exp{}, + verifyRampingVUs(I(5), buildStages(5, 5)), }, { opts{ @@ -323,7 +329,8 @@ func getConfigConsolidationTestCases() []configConsolidationTestCase { env: []string{"K6_ITERATIONS=25"}, cli: []string{"--vus", "12"}, }, - exp{}, verifySharedIters(I(12), I(25)), + exp{}, + verifySharedIters(I(12), I(25)), }, // TODO: test the externally controlled executor @@ -375,6 +382,74 @@ func getConfigConsolidationTestCases() []configConsolidationTestCase { assert.Equal(t, []string{"avg", "p(90)", "count"}, c.Options.SummaryTrendStats) }, }, + {opts{cli: []string{}}, exp{}, func(t *testing.T, c Config) { + assert.Equal(t, lib.DNSConfig{ + TTL: null.NewString("5m", false), + Select: lib.NullDNSSelect{DNSSelect: lib.DNSRandom, Valid: false}, + }, c.Options.DNS) + }}, + {opts{env: []string{"K6_DNS=ttl=5,select=round-robin"}}, exp{}, func(t *testing.T, c Config) { + assert.Equal(t, lib.DNSConfig{ + TTL: null.StringFrom("5"), + Select: lib.NullDNSSelect{DNSSelect: lib.DNSRoundRobin, Valid: true}, + }, c.Options.DNS) + }}, + {opts{env: []string{"K6_DNS=ttl=inf,select=random,policy=preferIPv6"}}, exp{}, func(t *testing.T, c Config) { + assert.Equal(t, lib.DNSConfig{ + TTL: null.StringFrom("inf"), + Select: lib.NullDNSSelect{DNSSelect: lib.DNSRandom, Valid: true}, + }, c.Options.DNS) + }}, + // This is functionally invalid, but will error out in validation done in js.parseTTL(). + {opts{cli: []string{"--dns", "ttl=-1"}}, exp{}, func(t *testing.T, c Config) { + assert.Equal(t, lib.DNSConfig{ + TTL: null.StringFrom("-1"), + Select: lib.NullDNSSelect{DNSSelect: lib.DNSRandom, Valid: false}, + }, c.Options.DNS) + }}, + {opts{cli: []string{"--dns", "ttl=0,blah=nope"}}, exp{cliReadError: true}, nil}, + {opts{cli: []string{"--dns", "ttl=0"}}, exp{}, func(t *testing.T, c Config) { + assert.Equal(t, lib.DNSConfig{ + TTL: null.StringFrom("0"), + Select: lib.NullDNSSelect{DNSSelect: lib.DNSRandom, Valid: false}, + }, c.Options.DNS) + }}, + {opts{cli: []string{"--dns", "ttl=5s,select="}}, exp{cliReadError: true}, nil}, + {opts{fs: defaultConfig(`{"dns": {"ttl": "0", "select": "round-robin"}}`)}, exp{}, func(t *testing.T, c Config) { + assert.Equal(t, lib.DNSConfig{ + TTL: null.StringFrom("0"), + Select: lib.NullDNSSelect{DNSSelect: lib.DNSRoundRobin, Valid: true}, + }, c.Options.DNS) + }}, + { + opts{ + fs: defaultConfig(`{"dns": {"ttl": "0"}}`), + env: []string{"K6_DNS=ttl=30"}, + }, + exp{}, + func(t *testing.T, c Config) { + assert.Equal(t, lib.DNSConfig{ + TTL: null.StringFrom("30"), + Select: lib.NullDNSSelect{DNSSelect: lib.DNSRandom, Valid: false}, + }, c.Options.DNS) + }, + }, + { + // CLI overrides all, falling back to env + opts{ + fs: defaultConfig(`{"dns": {"ttl": "60", "select": "first"}}`), + env: []string{"K6_DNS=ttl=30,select=random"}, + cli: []string{"--dns", "ttl=5"}, + }, + exp{}, + func(t *testing.T, c Config) { + assert.Equal(t, lib.DNSConfig{ + TTL: null.StringFrom("5"), + Select: lib.NullDNSSelect{DNSSelect: lib.DNSRandom, Valid: true}, + }, c.Options.DNS) + }, + }, + // TODO: test for differences between flagsets // TODO: more tests in general, especially ones not related to execution parameters... } diff --git a/cmd/options.go b/cmd/options.go index 17bf1b477ca..6bca0fa6fca 100644 --- a/cmd/options.go +++ b/cmd/options.go @@ -93,6 +93,10 @@ func optionFlagSet() *pflag.FlagSet { flags.StringSlice("tag", nil, "add a `tag` to be applied to all samples, as `[name]=[value]`") flags.String("console-output", "", "redirects the console logging to the provided output file") flags.Bool("discard-response-bodies", false, "Read but don't process or save HTTP response bodies") + flags.String("dns", lib.DefaultDNSConfig().String(), "DNS configuration. Possible ttl values are: 'inf' "+ + "for a persistent cache, '0' to disable the cache,\nor a positive duration, e.g. '1s', '1m', etc. "+ + "Milliseconds are assumed if no unit is provided.\n"+ + "Possible select values to return a single IP are: 'first', 'random' or 'round-robin'.\n") return flags } @@ -248,6 +252,14 @@ func getOptions(flags *pflag.FlagSet) (lib.Options, error) { opts.ConsoleOutput = null.StringFrom(redirectConFile) } + if dns, err := flags.GetString("dns"); err != nil { + return opts, err + } else if dns != "" { + if err := opts.DNS.UnmarshalText([]byte(dns)); err != nil { + return opts, err + } + } + return opts, nil } diff --git a/core/local/local_test.go b/core/local/local_test.go index c7ace16d503..cc2e1f77c32 100644 --- a/core/local/local_test.go +++ b/core/local/local_test.go @@ -24,6 +24,7 @@ import ( "context" "errors" "fmt" + "io/ioutil" "net" "net/url" "reflect" @@ -47,6 +48,7 @@ import ( "github.com/loadimpact/k6/lib/testutils" "github.com/loadimpact/k6/lib/testutils/httpmultibin" "github.com/loadimpact/k6/lib/testutils/minirunner" + "github.com/loadimpact/k6/lib/testutils/mockresolver" "github.com/loadimpact/k6/lib/types" "github.com/loadimpact/k6/loader" "github.com/loadimpact/k6/stats" @@ -974,6 +976,103 @@ func TestExecutionSchedulerIsRunning(t *testing.T) { assert.NoError(t, <-err) } +// TestDNSResolver checks the DNS resolution behavior at the ExecutionScheduler level. +func TestDNSResolver(t *testing.T) { + tb := httpmultibin.NewHTTPMultiBin(t) + defer tb.Cleanup() + sr := tb.Replacer.Replace + script := sr(` + import http from "k6/http"; + import { sleep } from "k6"; + + export let options = { + vus: 1, + iterations: 8, + noConnectionReuse: true, + } + + export default function () { + const res = http.get("http://myhost:HTTPBIN_PORT/", { timeout: 50 }); + sleep(0.7); // somewhat uneven multiple of 0.5 to minimize races with asserts + }`) + + t.Run("cache", func(t *testing.T) { + testCases := map[string]struct { + opts lib.Options + expLogEntries int + }{ + "default": { // IPs are cached for 5m + lib.Options{DNS: lib.DefaultDNSConfig()}, 0, + }, + "0": { // cache is disabled, every request does a DNS lookup + lib.Options{DNS: lib.DNSConfig{ + TTL: null.StringFrom("0"), + Strategy: lib.NullDNSStrategy{DNSStrategy: lib.DNSFirst, Valid: true}, + }}, 5, + }, + "1000": { // cache IPs for 1s, check that unitless values are interpreted as ms + lib.Options{DNS: lib.DNSConfig{ + TTL: null.StringFrom("1000"), + Strategy: lib.NullDNSStrategy{DNSStrategy: lib.DNSFirst, Valid: true}, + }}, 4, + }, + "3s": { + lib.Options{DNS: lib.DNSConfig{ + TTL: null.StringFrom("3s"), + Strategy: lib.NullDNSStrategy{DNSStrategy: lib.DNSFirst, Valid: true}, + }}, 3, + }, + } + + expErr := sr(`dial tcp 127.0.0.254:HTTPBIN_PORT: connect: connection refused`) + if runtime.GOOS == "windows" { + expErr = "context deadline exceeded" + } + for name, tc := range testCases { + tc := tc + t.Run(name, func(t *testing.T) { + logger := logrus.New() + logger.SetOutput(ioutil.Discard) + logHook := testutils.SimpleLogrusHook{HookedLevels: []logrus.Level{logrus.WarnLevel}} + logger.AddHook(&logHook) + + runner, err := js.New(logger, &loader.SourceData{ + URL: &url.URL{Path: "/script.js"}, Data: []byte(script), + }, nil, lib.RuntimeOptions{}) + require.NoError(t, err) + + mr := mockresolver.New(nil, net.LookupIP) + runner.ActualResolver = mr.LookupIPAll + + ctx, cancel, execScheduler, samples := newTestExecutionScheduler(t, runner, logger, tc.opts) + defer cancel() + + mr.Set("myhost", sr("HTTPBIN_IP")) + time.AfterFunc(1700*time.Millisecond, func() { + mr.Set("myhost", "127.0.0.254") + }) + defer mr.Unset("myhost") + + errCh := make(chan error, 1) + go func() { errCh <- execScheduler.Run(ctx, ctx, samples) }() + + select { + case err := <-errCh: + require.NoError(t, err) + entries := logHook.Drain() + require.Len(t, entries, tc.expLogEntries) + for _, entry := range entries { + require.IsType(t, &url.Error{}, entry.Data["error"]) + assert.EqualError(t, entry.Data["error"].(*url.Error).Err, expErr) + } + case <-time.After(10 * time.Second): + t.Fatal("timed out") + } + }) + } + }) +} + func TestRealTimeAndSetupTeardownMetrics(t *testing.T) { if runtime.GOOS == "windows" { t.Skip() @@ -1100,7 +1199,10 @@ func TestRealTimeAndSetupTeardownMetrics(t *testing.T) { getDummyTrail := func(group string, emitIterations bool, addExpTags ...string) stats.SampleContainer { expTags := []string{"group", group} expTags = append(expTags, addExpTags...) - return netext.NewDialer(net.Dialer{}).GetTrail(time.Now(), time.Now(), + return netext.NewDialer( + net.Dialer{}, + netext.NewResolver(net.LookupIP, 0, lib.DNSFirst), + ).GetTrail(time.Now(), time.Now(), true, emitIterations, getTags(expTags...)) } diff --git a/go.mod b/go.mod index 407fbd792a7..b35b88bb09a 100644 --- a/go.mod +++ b/go.mod @@ -66,7 +66,6 @@ require ( github.com/urfave/negroni v0.3.1-0.20180130044549-22c5532ea862 github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/valyala/fasttemplate v0.0.0-20170224212429-dcecefd839c4 // indirect - github.com/viki-org/dnscache v0.0.0-20130720023526-c70c1f23c5d8 github.com/zyedidia/highlight v0.0.0-20170330143449-201131ce5cf5 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7 diff --git a/go.sum b/go.sum index 6dbf50f6750..b1301d267f7 100644 --- a/go.sum +++ b/go.sum @@ -153,8 +153,6 @@ github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6Kllzaw github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasttemplate v0.0.0-20170224212429-dcecefd839c4 h1:gKMu1Bf6QINDnvyZuTaACm9ofY+PRh+5vFz4oxBZeF8= github.com/valyala/fasttemplate v0.0.0-20170224212429-dcecefd839c4/go.mod h1:50wTf68f99/Zt14pr046Tgt3Lp2vLyFZKzbFXTOabXw= -github.com/viki-org/dnscache v0.0.0-20130720023526-c70c1f23c5d8 h1:EVObHAr8DqpoJCVv6KYTle8FEImKhtkfcZetNqxDoJQ= -github.com/viki-org/dnscache v0.0.0-20130720023526-c70c1f23c5d8/go.mod h1:dniwbG03GafCjFohMDmz6Zc6oCuiqgH6tGNyXTkHzXE= github.com/zyedidia/highlight v0.0.0-20170330143449-201131ce5cf5 h1:Zs6mpwXvlqpF9zHl5XaN0p5V4J9XvP+WBuiuXyIgqvc= github.com/zyedidia/highlight v0.0.0-20170330143449-201131ce5cf5/go.mod h1:c1r+Ob9tUTPB0FKWO1+x+Hsc/zNa45WdGq7Y38Ybip0= golang.org/x/crypto v0.0.0-20180308185624-c7dcf104e3a7 h1:c9Tyi4qyEZwEJ1+Zm6Fcqf+68wmUdMzfXYTp3s8Nzg8= diff --git a/js/initcontext_test.go b/js/initcontext_test.go index e6d73c041eb..d2bb5d0f104 100644 --- a/js/initcontext_test.go +++ b/js/initcontext_test.go @@ -388,11 +388,14 @@ func TestRequestWithBinaryFile(t *testing.T) { Logger: logger, Group: root, Transport: &http.Transport{ - DialContext: (netext.NewDialer(net.Dialer{ - Timeout: 10 * time.Second, - KeepAlive: 60 * time.Second, - DualStack: true, - })).DialContext, + DialContext: (netext.NewDialer( + net.Dialer{ + Timeout: 10 * time.Second, + KeepAlive: 60 * time.Second, + DualStack: true, + }, + netext.NewResolver(net.LookupIP, 0, lib.DNSFirst), + )).DialContext, }, BPool: bpool.NewBufferPool(1), Samples: make(chan stats.SampleContainer, 500), diff --git a/js/runner.go b/js/runner.go index eb584e64bed..1d4b73e9f61 100644 --- a/js/runner.go +++ b/js/runner.go @@ -36,7 +36,6 @@ import ( "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/spf13/afero" - "github.com/viki-org/dnscache" "golang.org/x/net/http2" "golang.org/x/time/rate" @@ -44,6 +43,7 @@ import ( "github.com/loadimpact/k6/lib" "github.com/loadimpact/k6/lib/consts" "github.com/loadimpact/k6/lib/netext" + "github.com/loadimpact/k6/lib/types" "github.com/loadimpact/k6/loader" "github.com/loadimpact/k6/stats" ) @@ -60,8 +60,10 @@ type Runner struct { defaultGroup *lib.Group BaseDialer net.Dialer - Resolver *dnscache.Resolver - RPSLimit *rate.Limiter + Resolver netext.Resolver + // TODO: Remove ActualResolver, it's a hack to simplify mocking in tests. + ActualResolver netext.MultiResolver + RPSLimit *rate.Limiter console *console setupData []byte @@ -104,8 +106,9 @@ func newFromBundle(logger *logrus.Logger, b *Bundle) (*Runner, error) { KeepAlive: 30 * time.Second, DualStack: true, }, - console: newConsole(logger), - Resolver: dnscache.New(0), + console: newConsole(logger), + Resolver: netext.NewResolver(net.LookupIP, 0, lib.DefaultDNSConfig().Select.DNSSelect), + ActualResolver: net.LookupIP, } err = r.SetOptions(r.Bundle.Options) @@ -319,9 +322,64 @@ func (r *Runner) SetOptions(opts lib.Options) error { r.console = c } + // FIXME: Resolver probably shouldn't be reset here... + // It's done because the js.Runner is created before the full + // configuration has been processed, at which point we don't have + // access to the DNSConfig, and need to wait for this SetOptions + // call that happens after all config has been assembled. + // We could make DNSConfig part of RuntimeOptions, but that seems + // conceptually wrong since the JS runtime doesn't care about it + // (it needs the actual resolver, not the config), and it would + // require an additional field on Bundle to pass the config through, + // which is arguably worse than this. + if err := r.setResolver(opts.DNS); err != nil { + return err + } + return nil } +func (r *Runner) setResolver(dns lib.DNSConfig) error { + ttl, err := parseTTL(dns.TTL.String) + if err != nil { + return err + } + + dnsSel := opts.DNS.Select.DNSSelect + if !dnsSel.IsADNSSelect() { + dnsSel = lib.DefaultDNSConfig().Select.DNSSelect + } + r.Resolver = netext.NewResolver(r.ActualResolver, ttl, dnsSel) + + return nil +} + +func parseTTL(ttlS string) (time.Duration, error) { + ttl := time.Duration(0) + switch ttlS { + case "inf": + // cache "infinitely" + ttl = time.Hour * 24 * 365 + case "0": + // disable cache + case "": + ttlS = lib.DefaultDNSConfig().TTL.String + fallthrough + default: + origTTLs := ttlS + // Treat unitless values as milliseconds + if t, err := strconv.ParseFloat(ttlS, 32); err == nil { + ttlS = fmt.Sprintf("%.2fms", t) + } + var err error + ttl, err = types.ParseExtendedDuration(ttlS) + if ttl < 0 || err != nil { + return ttl, fmt.Errorf("invalid DNS TTL: %s", origTTLs) + } + } + return ttl, nil +} + // Runs an exported function in its own temporary VU, optionally with an argument. Execution is // interrupted if the context expires. No error is returned if the part does not exist. func (r *Runner) runPart(ctx context.Context, out chan<- stats.SampleContainer, name string, arg interface{}) (goja.Value, error) { diff --git a/lib/dns_select_gen.go b/lib/dns_select_gen.go new file mode 100644 index 00000000000..b41f352ee61 --- /dev/null +++ b/lib/dns_select_gen.go @@ -0,0 +1,52 @@ +// Code generated by "enumer -type=DNSSelect -transform=kebab -trimprefix DNS -output dns_select_gen.go"; DO NOT EDIT. + +// +package lib + +import ( + "fmt" +) + +const _DNSSelectName = "firstround-robinrandom" + +var _DNSSelectIndex = [...]uint8{0, 5, 16, 22} + +func (i DNSSelect) String() string { + i -= 1 + if i >= DNSSelect(len(_DNSSelectIndex)-1) { + return fmt.Sprintf("DNSSelect(%d)", i+1) + } + return _DNSSelectName[_DNSSelectIndex[i]:_DNSSelectIndex[i+1]] +} + +var _DNSSelectValues = []DNSSelect{1, 2, 3} + +var _DNSSelectNameToValueMap = map[string]DNSSelect{ + _DNSSelectName[0:5]: 1, + _DNSSelectName[5:16]: 2, + _DNSSelectName[16:22]: 3, +} + +// DNSSelectString retrieves an enum value from the enum constants string name. +// Throws an error if the param is not part of the enum. +func DNSSelectString(s string) (DNSSelect, error) { + if val, ok := _DNSSelectNameToValueMap[s]; ok { + return val, nil + } + return 0, fmt.Errorf("%s does not belong to DNSSelect values", s) +} + +// DNSSelectValues returns all values of the enum +func DNSSelectValues() []DNSSelect { + return _DNSSelectValues +} + +// IsADNSSelect returns "true" if the value is listed in the enum definition. "false" otherwise +func (i DNSSelect) IsADNSSelect() bool { + for _, v := range _DNSSelectValues { + if i == v { + return true + } + } + return false +} diff --git a/lib/netext/dialer.go b/lib/netext/dialer.go index 050c4660067..9e940e008fb 100644 --- a/lib/netext/dialer.go +++ b/lib/netext/dialer.go @@ -28,27 +28,18 @@ import ( "sync/atomic" "time" - "github.com/pkg/errors" - "github.com/viki-org/dnscache" - "github.com/loadimpact/k6/lib" "github.com/loadimpact/k6/lib/metrics" "github.com/loadimpact/k6/lib/types" "github.com/loadimpact/k6/stats" ) -// dnsResolver is an interface that fetches dns information -// about a given address. -type dnsResolver interface { - FetchOne(address string) (net.IP, error) -} - // Dialer wraps net.Dialer and provides k6 specific functionality - // tracing, blacklists and DNS cache and aliases. type Dialer struct { net.Dialer - Resolver dnsResolver + Resolver Resolver Blacklist []*lib.IPNet BlockedHostnames *types.HostnameTrie Hosts map[string]*lib.HostAddress @@ -57,12 +48,8 @@ type Dialer struct { BytesWritten int64 } -// NewDialer constructs a new Dialer and initializes its cache. -func NewDialer(dialer net.Dialer) *Dialer { - return newDialerWithResolver(dialer, dnscache.New(0)) -} - -func newDialerWithResolver(dialer net.Dialer, resolver dnsResolver) *Dialer { +// NewDialer constructs a new Dialer with the given DNS resolver. +func NewDialer(dialer net.Dialer, resolver Resolver) *Dialer { return &Dialer{ Dialer: dialer, Resolver: resolver, @@ -191,17 +178,13 @@ func (d *Dialer) findRemote(addr string) (*lib.HostAddress, error) { return lib.NewHostAddress(ip, port) } - return d.fetchRemoteFromResolver(host, port) -} - -func (d *Dialer) fetchRemoteFromResolver(host, port string) (*lib.HostAddress, error) { - ip, err := d.Resolver.FetchOne(host) + ip, err = d.Resolver.LookupIP(host) if err != nil { return nil, err } if ip == nil { - return nil, errors.Errorf("lookup %s: no such host", host) + return nil, fmt.Errorf("lookup %s: no such host", host) } return lib.NewHostAddress(ip, port) diff --git a/lib/netext/dialer_test.go b/lib/netext/dialer_test.go index 71db79cc07e..46125005f3c 100644 --- a/lib/netext/dialer_test.go +++ b/lib/netext/dialer_test.go @@ -24,19 +24,15 @@ import ( "net" "testing" + "github.com/stretchr/testify/require" + "github.com/loadimpact/k6/lib" + "github.com/loadimpact/k6/lib/testutils/mockresolver" "github.com/loadimpact/k6/lib/types" - "github.com/stretchr/testify/require" ) -type testResolver struct { - hosts map[string]net.IP -} - -func (r testResolver) FetchOne(host string) (net.IP, error) { return r.hosts[host], nil } - func TestDialerAddr(t *testing.T) { - dialer := newDialerWithResolver(net.Dialer{}, newResolver()) + dialer := NewDialer(net.Dialer{}, newResolver()) dialer.Hosts = map[string]*lib.HostAddress{ "example.com": {IP: net.ParseIP("3.4.5.6")}, "example.com:443": {IP: net.ParseIP("3.4.5.6"), Port: 8443}, @@ -95,7 +91,7 @@ func TestDialerAddr(t *testing.T) { } func TestDialerAddrBlockHostnamesStar(t *testing.T) { - dialer := newDialerWithResolver(net.Dialer{}, newResolver()) + dialer := NewDialer(net.Dialer{}, newResolver()) dialer.Hosts = map[string]*lib.HostAddress{ "example.com": {IP: net.ParseIP("3.4.5.6")}, } @@ -129,12 +125,12 @@ func TestDialerAddrBlockHostnamesStar(t *testing.T) { } } -func newResolver() testResolver { - return testResolver{ - hosts: map[string]net.IP{ - "example-resolver.com": net.ParseIP("1.2.3.4"), - "example-deny-resolver.com": net.ParseIP("8.9.10.11"), - "example-ipv6-deny-resolver.com": net.ParseIP("::1"), - }, - } +func newResolver() *mockresolver.MockResolver { + return mockresolver.New( + map[string][]net.IP{ + "example-resolver.com": {net.ParseIP("1.2.3.4")}, + "example-deny-resolver.com": {net.ParseIP("8.9.10.11")}, + "example-ipv6-deny-resolver.com": {net.ParseIP("::1")}, + }, nil, + ) } diff --git a/lib/netext/httpext/tracer_test.go b/lib/netext/httpext/tracer_test.go index c2b56974180..812b261de81 100644 --- a/lib/netext/httpext/tracer_test.go +++ b/lib/netext/httpext/tracer_test.go @@ -40,6 +40,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/loadimpact/k6/lib" "github.com/loadimpact/k6/lib/metrics" "github.com/loadimpact/k6/lib/netext" "github.com/loadimpact/k6/stats" @@ -55,7 +56,10 @@ func TestTracer(t *testing.T) { transport, ok := srv.Client().Transport.(*http.Transport) assert.True(t, ok) - transport.DialContext = netext.NewDialer(net.Dialer{}).DialContext + transport.DialContext = netext.NewDialer( + net.Dialer{}, + netext.NewResolver(net.LookupIP, 0, lib.DNSFirst), + ).DialContext var prev int64 assertLaterOrZero := func(t *testing.T, val int64, canBeZero bool) { diff --git a/lib/netext/resolver.go b/lib/netext/resolver.go new file mode 100644 index 00000000000..858b0909c56 --- /dev/null +++ b/lib/netext/resolver.go @@ -0,0 +1,140 @@ +/* + * + * k6 - a next-generation load testing tool + * Copyright (C) 2020 Load Impact + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +package netext + +import ( + "math/rand" + "net" + "sync" + "time" + + "github.com/loadimpact/k6/lib" +) + +// MultiResolver returns all IP addresses for the given host. +type MultiResolver func(host string) ([]net.IP, error) + +// Resolver is an interface that returns DNS information about a given host. +type Resolver interface { + LookupIP(host string) (net.IP, error) +} + +type resolver struct { + resolve MultiResolver + selectIndex lib.DNSSelect + rrm *sync.Mutex + rand *rand.Rand + roundRobin map[string]uint8 +} + +type cacheRecord struct { + ips []net.IP + lastLookup time.Time +} + +type cacheResolver struct { + resolver + ttl time.Duration + cm *sync.Mutex + cache map[string]cacheRecord +} + +// NewResolver returns a new DNS resolver. If ttl is not 0, responses +// will be cached per host for the specified period. The IP returned from +// LookupIP() will be selected based on the given sel value. +func NewResolver(actRes MultiResolver, ttl time.Duration, sel lib.DNSSelect) Resolver { + r := rand.New(rand.NewSource(time.Now().UnixNano())) // nolint: gosec + res := resolver{ + resolve: actRes, + selectIndex: sel, + rrm: &sync.Mutex{}, + rand: r, + roundRobin: make(map[string]uint8), + } + if ttl == 0 { + return &res + } + return &cacheResolver{ + resolver: res, + ttl: ttl, + cm: &sync.Mutex{}, + cache: make(map[string]cacheRecord), + } +} + +// LookupIP returns a single IP resolved for host, selected by the +// configured select strategy. +func (r *resolver) LookupIP(host string) (net.IP, error) { + ips, err := r.resolve(host) + if err != nil { + return nil, err + } + return r.selectOne(host, ips), nil +} + +// LookupIP returns a single IP resolved for host, selected by the configured +// select strategy. Results are cached per host and will be refreshed if the +// last lookup time exceeds the configured TTL (not the TTL returned in the DNS +// record). +func (r *cacheResolver) LookupIP(host string) (net.IP, error) { + r.cm.Lock() + + var ips []net.IP + // TODO: Invalidate? When? + if d, ok := r.cache[host]; ok && time.Now().Before(d.lastLookup.Add(r.ttl)) { + ips = r.cache[host].ips + } else { + r.cm.Unlock() // The lookup could take some time, so unlock momentarily. + var err error + ips, err = r.resolve(host) + if err != nil { + return nil, err + } + r.cm.Lock() + r.cache[host] = cacheRecord{ips: ips, lastLookup: time.Now()} + } + + r.cm.Unlock() + return r.selectOne(host, ips), nil +} + +func (r *resolver) selectOne(host string, ips []net.IP) net.IP { + if len(ips) == 0 { + return nil + } + var ip net.IP + switch r.selectIndex { + case lib.DNSFirst: + return ips[0] + case lib.DNSRoundRobin: + r.rrm.Lock() + // NOTE: This index approach is not stable and might result in returning + // repeated or skipped IPs if the records change during a test run. + ip = ips[int(r.roundRobin[host])%len(ips)] + r.roundRobin[host]++ + r.rrm.Unlock() + case lib.DNSRandom: + r.rrm.Lock() + ip = ips[r.rand.Intn(len(ips))] + r.rrm.Unlock() + } + return ip +} diff --git a/lib/netext/resolver_test.go b/lib/netext/resolver_test.go new file mode 100644 index 00000000000..c6f3f8e17d5 --- /dev/null +++ b/lib/netext/resolver_test.go @@ -0,0 +1,96 @@ +/* + * + * k6 - a next-generation load testing tool + * Copyright (C) 2020 Load Impact + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +package netext + +import ( + "fmt" + "net" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/loadimpact/k6/lib" + "github.com/loadimpact/k6/lib/testutils/mockresolver" +) + +func TestResolver(t *testing.T) { + t.Parallel() + + host := "myhost" + mr := mockresolver.New(map[string][]net.IP{ + host: { + net.ParseIP("127.0.0.10"), + net.ParseIP("127.0.0.11"), + net.ParseIP("127.0.0.12"), + }, + }, nil) + + t.Run("LookupIP", func(t *testing.T) { + testCases := []struct { + ttl time.Duration + sel lib.DNSSelect + expIP []net.IP + }{ + {0, lib.DNSFirst, []net.IP{net.ParseIP("127.0.0.10")}}, + {time.Second, lib.DNSFirst, []net.IP{net.ParseIP("127.0.0.10")}}, + {0, lib.DNSRoundRobin, []net.IP{ + net.ParseIP("127.0.0.10"), + net.ParseIP("127.0.0.11"), + net.ParseIP("127.0.0.12"), + net.ParseIP("127.0.0.10"), + }}, + } + + for _, tc := range testCases { + tc := tc + t.Run(fmt.Sprintf("%s_%s", tc.ttl, tc.sel), func(t *testing.T) { + r := NewResolver(mr.LookupIPAll, tc.ttl, tc.sel) + ip, err := r.LookupIP(host) + require.NoError(t, err) + assert.Equal(t, tc.expIP[0], ip) + + if tc.ttl > 0 { + require.IsType(t, &cacheResolver{}, r) + cr := r.(*cacheResolver) + assert.Len(t, cr.cache, 1) + assert.Equal(t, tc.ttl, cr.ttl) + firstLookup := cr.cache[host].lastLookup + time.Sleep(cr.ttl + 100*time.Millisecond) + _, err = r.LookupIP(host) + require.NoError(t, err) + assert.True(t, cr.cache[host].lastLookup.After(firstLookup)) + } + + if tc.sel == lib.DNSRoundRobin { + ips := []net.IP{ip} + for i := 0; i < 3; i++ { + ip, err = r.LookupIP(host) + require.NoError(t, err) + ips = append(ips, ip) + } + assert.Equal(t, tc.expIP, ips) + } + }) + } + }) +} diff --git a/lib/options.go b/lib/options.go index 20555d4fd41..f89e06feb7e 100644 --- a/lib/options.go +++ b/lib/options.go @@ -21,6 +21,7 @@ package lib import ( + "bytes" "crypto/tls" "encoding/json" "fmt" @@ -28,6 +29,7 @@ import ( "reflect" "strconv" + "github.com/kubernetes/helm/pkg/strvals" "github.com/pkg/errors" "gopkg.in/guregu/null.v3" @@ -44,6 +46,142 @@ const DefaultScenarioName = "default" // nolint: gochecknoglobals var DefaultSummaryTrendStats = []string{"avg", "min", "med", "max", "p(90)", "p(95)"} +// DNSConfig is the DNS resolver configuration. +type DNSConfig struct { + // If positive, defines how long DNS lookups should be returned from the cache. + TTL null.String `json:"ttl"` + // Select specifies the strategy to use when picking a single IP if more than one is returned for a host name. + Select NullDNSSelect `json:"select"` + // FIXME: Valid is unused and is only added to satisfy some logic in + // lib.Options.ForEachSpecified(), otherwise it would panic with + // `reflect: call of reflect.Value.Bool on zero Value`. + Valid bool `json:"-"` +} + +// DNSSelect is the strategy to use when picking a single IP if more than one +// is returned for a host name. +//go:generate enumer -type=DNSSelect -transform=kebab -trimprefix DNS -output dns_select_gen.go +type DNSSelect uint8 + +const ( + // DNSFirst returns the first IP from the response. + DNSFirst DNSSelect = iota + 1 + // DNSRoundRobin rotates the IP returned on each lookup. + DNSRoundRobin + // DNSRandom returns a random IP from the response. + DNSRandom +) + +// UnmarshalJSON converts JSON data to a valid DNSSelect +func (d *DNSSelect) UnmarshalJSON(data []byte) error { + if bytes.Equal(data, []byte(`null`)) { + return nil + } + var s string + if err := json.Unmarshal(data, &s); err != nil { + return err + } + v, err := DNSSelectString(s) + if err != nil { + return err + } + *d = v + return nil +} + +// MarshalJSON returns the JSON representation of d +func (d DNSSelect) MarshalJSON() ([]byte, error) { + return json.Marshal(d.String()) +} + +// NullDNSSelect is a nullable wrapper around DNSSelect, required for the +// current configuration system. +type NullDNSSelect struct { + DNSSelect + Valid bool +} + +// UnmarshalJSON converts JSON data to a valid NullDNSStratey +func (d *NullDNSSelect) UnmarshalJSON(data []byte) error { + if bytes.Equal(data, []byte(`null`)) { + return nil + } + if err := json.Unmarshal(data, &d.DNSSelect); err != nil { + return err + } + d.Valid = true + return nil +} + +// MarshalJSON returns the JSON representation of d +func (d NullDNSSelect) MarshalJSON() ([]byte, error) { + if !d.Valid { + return []byte(`null`), nil + } + return json.Marshal(d.DNSSelect) +} + +// DefaultDNSConfig returns the default DNS configuration. +func DefaultDNSConfig() DNSConfig { + return DNSConfig{ + TTL: null.NewString("5m", false), + Select: NullDNSSelect{DNSRandom, false}, + } +} + +// String implements fmt.Stringer. +func (c DNSConfig) String() string { + return fmt.Sprintf("ttl=%s,select=%s", + c.TTL.String, c.Select.String()) +} + +// UnmarshalJSON implements json.Unmarshaler. +func (c *DNSConfig) UnmarshalJSON(data []byte) error { + var s struct { + TTL null.String `json:"ttl"` + Select NullDNSSelect `json:"select"` + } + if err := json.Unmarshal(data, &s); err != nil { + return err + } + c.TTL = s.TTL + c.Select = s.Select + return nil +} + +// UnmarshalText implements encoding.TextUnmarshaler. +func (c *DNSConfig) UnmarshalText(text []byte) error { + if string(text) == DefaultDNSConfig().String() { + *c = DefaultDNSConfig() + return nil + } + params, err := strvals.Parse(string(text)) + if err != nil { + return err + } + return c.unmarshal(params) +} + +func (c *DNSConfig) unmarshal(params map[string]interface{}) error { + for k, v := range params { + switch k { + case "select": + s, err := DNSSelectString(v.(string)) + if err != nil { + return err + } + c.Select.DNSSelect = s + c.Select.Valid = true + case "ttl": + ttlv := fmt.Sprintf("%v", v) + c.TTL = null.StringFrom(ttlv) + default: + return fmt.Errorf("unknown DNS configuration field: %s", k) + } + } + return nil +} + // Describes a TLS version. Serialised to/from JSON as a string, eg. "tls1.2". type TLSVersion int @@ -307,6 +445,9 @@ type Options struct { // Limit HTTP requests per second. RPS null.Int `json:"rps" envconfig:"K6_RPS"` + // DNS handling configuration. + DNS DNSConfig `json:"dns" envconfig:"K6_DNS"` + // How many HTTP redirects do we follow? MaxRedirects null.Int `json:"maxRedirects" envconfig:"K6_MAX_REDIRECTS"` @@ -539,6 +680,12 @@ func (o Options) Apply(opts Options) Options { if opts.ConsoleOutput.Valid { o.ConsoleOutput = opts.ConsoleOutput } + if opts.DNS.TTL.Valid { + o.DNS.TTL = opts.DNS.TTL + } + if opts.DNS.Select.Valid { + o.DNS.Select = opts.DNS.Select + } return o } diff --git a/lib/testutils/httpmultibin/httpmultibin.go b/lib/testutils/httpmultibin/httpmultibin.go index dfb3a741975..cbbfb0628fd 100644 --- a/lib/testutils/httpmultibin/httpmultibin.go +++ b/lib/testutils/httpmultibin/httpmultibin.go @@ -253,7 +253,7 @@ func NewHTTPMultiBin(t testing.TB) *HTTPMultiBin { Timeout: 2 * time.Second, KeepAlive: 10 * time.Second, DualStack: true, - }) + }, netext.NewResolver(net.LookupIP, 0, lib.DNSFirst)) dialer.Hosts = map[string]*lib.HostAddress{ httpDomain: httpDomainValue, httpsDomain: httpsDomainValue, diff --git a/lib/testutils/mockresolver/resolver.go b/lib/testutils/mockresolver/resolver.go new file mode 100644 index 00000000000..5e68d260893 --- /dev/null +++ b/lib/testutils/mockresolver/resolver.go @@ -0,0 +1,81 @@ +/* + * + * k6 - a next-generation load testing tool + * Copyright (C) 2020 Load Impact + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +package mockresolver + +import ( + "fmt" + "net" + "sync" +) + +// MockResolver implements netext.Resolver, and allows changing the host +// mapping at runtime. +type MockResolver struct { + m sync.RWMutex + hosts map[string][]net.IP + fallback func(host string) ([]net.IP, error) +} + +// New returns a new MockResolver. +func New(hosts map[string][]net.IP, fallback func(host string) ([]net.IP, error)) *MockResolver { + if hosts == nil { + hosts = make(map[string][]net.IP) + } + return &MockResolver{hosts: hosts, fallback: fallback} +} + +// LookupIP returns the first IP mapped for host. +func (r *MockResolver) LookupIP(host string) (net.IP, error) { + if ips, err := r.LookupIPAll(host); err != nil { + return nil, err + } else if len(ips) > 0 { + return ips[0], nil + } + return nil, nil +} + +// LookupIPAll returns all IPs mapped for host. It mimics the net.LookupIP +// signature so that it can be used to mock netext.LookupIP in tests. +func (r *MockResolver) LookupIPAll(host string) ([]net.IP, error) { + r.m.RLock() + defer r.m.RUnlock() + if ips, ok := r.hosts[host]; ok { + return ips, nil + } + if r.fallback != nil { + return r.fallback(host) + } + return nil, fmt.Errorf("lookup %s: no such host", host) +} + +// Set the host to resolve to ip. +func (r *MockResolver) Set(host, ip string) { + r.m.Lock() + defer r.m.Unlock() + r.hosts[host] = []net.IP{net.ParseIP(ip)} +} + +// Unset removes the host. +func (r *MockResolver) Unset(host string) { + r.m.Lock() + defer r.m.Unlock() + delete(r.hosts, host) +} diff --git a/vendor/github.com/viki-org/dnscache/dnscache.go b/vendor/github.com/viki-org/dnscache/dnscache.go deleted file mode 100644 index 74d6bd61fcb..00000000000 --- a/vendor/github.com/viki-org/dnscache/dnscache.go +++ /dev/null @@ -1,77 +0,0 @@ -package dnscache -// Package dnscache caches DNS lookups - -import ( - "net" - "sync" - "time" -) - -type Resolver struct { - lock sync.RWMutex - cache map[string][]net.IP -} - -func New(refreshRate time.Duration) *Resolver { - resolver := &Resolver { - cache: make(map[string][]net.IP, 64), - } - if refreshRate > 0 { - go resolver.autoRefresh(refreshRate) - } - return resolver -} - -func (r *Resolver) Fetch(address string) ([]net.IP, error) { - r.lock.RLock() - ips, exists := r.cache[address] - r.lock.RUnlock() - if exists { return ips, nil } - - return r.Lookup(address) -} - -func (r *Resolver) FetchOne(address string) (net.IP, error) { - ips, err := r.Fetch(address) - if err != nil || len(ips) == 0 { return nil, err} - return ips[0], nil -} - -func (r *Resolver) FetchOneString(address string) (string, error) { - ip, err := r.FetchOne(address) - if err != nil || ip == nil { return "", err } - return ip.String(), nil -} - -func (r *Resolver) Refresh() { - i := 0 - r.lock.RLock() - addresses := make([]string, len(r.cache)) - for key, _ := range r.cache { - addresses[i] = key - i++ - } - r.lock.RUnlock() - - for _, address := range addresses { - r.Lookup(address) - time.Sleep(time.Second * 2) - } -} - -func (r *Resolver) Lookup(address string) ([]net.IP, error) { - ips, err := net.LookupIP(address) - if err != nil { return nil, err } - - r.lock.Lock() - r.cache[address] = ips - r.lock.Unlock() - return ips, nil -} - -func (r *Resolver) autoRefresh(rate time.Duration) { - for { - time.Sleep(rate) - r.Refresh() - } -} diff --git a/vendor/github.com/viki-org/dnscache/license.txt b/vendor/github.com/viki-org/dnscache/license.txt deleted file mode 100644 index 8a7d969ed49..00000000000 --- a/vendor/github.com/viki-org/dnscache/license.txt +++ /dev/null @@ -1,19 +0,0 @@ -Copyright (c) 2013 Viki Inc. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. diff --git a/vendor/github.com/viki-org/dnscache/readme.md b/vendor/github.com/viki-org/dnscache/readme.md deleted file mode 100644 index 8c737aac35d..00000000000 --- a/vendor/github.com/viki-org/dnscache/readme.md +++ /dev/null @@ -1,38 +0,0 @@ -### A DNS cache for Go -CGO is used to lookup domain names. Given enough concurrent requests and the slightest hiccup in name resolution, it's quite easy to end up with blocked/leaking goroutines. - -The issue is documented at - -The Go team's singleflight solution (which isn't in stable yet) is rather elegant. However, it only eliminates concurrent lookups (thundering herd problems). Many systems can live with slightly stale resolve names, which means we can cacne DNS lookups and refresh them in the background. - -### Installation -Install using the "go get" command: - - go get github.com/viki-org/dnscache - -### Usage -The cache is thread safe. Create a new instance by specifying how long each entry should be cached (in seconds). Items will be refreshed in the background. - - //refresh items every 5 minutes - resolver := dnscache.New(time.Minute * 5) - - //get an array of net.IP - ips, _ := resolver.Fetch("api.viki.io") - - //get the first net.IP - ip, _ := resolver.FetchOne("api.viki.io") - - //get the first net.IP as string - ip, _ := resolver.FetchOneString("api.viki.io") - -If you are using an `http.Transport`, you can use this cache by speficifying a -`Dial` function: - - transport := &http.Transport { - MaxIdleConnsPerHost: 64, - Dial: func(network string, address string) (net.Conn, error) { - separator := strings.LastIndex(address, ":") - ip, _ := dnscache.FetchString(address[:separator]) - return net.Dial("tcp", ip + address[separator:]) - }, - } diff --git a/vendor/modules.txt b/vendor/modules.txt index 8c60f54c7e4..b1088edaf6b 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -225,9 +225,6 @@ github.com/valyala/bytebufferpool # github.com/valyala/fasttemplate v0.0.0-20170224212429-dcecefd839c4 ## explicit github.com/valyala/fasttemplate -# github.com/viki-org/dnscache v0.0.0-20130720023526-c70c1f23c5d8 -## explicit -github.com/viki-org/dnscache # github.com/zyedidia/highlight v0.0.0-20170330143449-201131ce5cf5 ## explicit github.com/zyedidia/highlight From b6f595cfe059447b7ce819ca95d268fae5cc2f6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ivan=20Miri=C4=87?= Date: Mon, 12 Oct 2020 17:26:05 +0200 Subject: [PATCH 3/8] Add DNS policy option --- cmd/config.go | 9 ++- cmd/config_consolidation_test.go | 28 +++++-- cmd/options.go | 3 +- core/local/local_test.go | 17 ++-- js/initcontext_test.go | 2 +- js/runner.go | 14 +++- lib/dns_policy_gen.go | 54 +++++++++++++ lib/netext/httpext/tracer_test.go | 2 +- lib/netext/resolver.go | 70 +++++++++++++--- lib/netext/resolver_test.go | 30 +++++-- lib/options.go | 93 ++++++++++++++++++++-- lib/testutils/httpmultibin/httpmultibin.go | 2 +- 12 files changed, 279 insertions(+), 45 deletions(-) create mode 100644 lib/dns_policy_gen.go diff --git a/cmd/config.go b/cmd/config.go index 46993266442..b75bb81bd00 100644 --- a/cmd/config.go +++ b/cmd/config.go @@ -259,12 +259,17 @@ func applyDefault(conf Config) Config { if conf.Options.SummaryTrendStats == nil { conf.Options.SummaryTrendStats = lib.DefaultSummaryTrendStats } + defDNS := lib.DefaultDNSConfig() if !conf.DNS.TTL.Valid { - conf.DNS.TTL = lib.DefaultDNSConfig().TTL + conf.DNS.TTL = defDNS.TTL } if !conf.DNS.Select.Valid { - conf.DNS.Select = lib.DefaultDNSConfig().Select + conf.DNS.Select = defDNS.Select } + if !conf.DNS.Policy.Valid { + conf.DNS.Policy = defDNS.Policy + } + return conf } diff --git a/cmd/config_consolidation_test.go b/cmd/config_consolidation_test.go index 2628af5e785..b73a5958f3f 100644 --- a/cmd/config_consolidation_test.go +++ b/cmd/config_consolidation_test.go @@ -386,18 +386,21 @@ func getConfigConsolidationTestCases() []configConsolidationTestCase { assert.Equal(t, lib.DNSConfig{ TTL: null.NewString("5m", false), Select: lib.NullDNSSelect{DNSSelect: lib.DNSRandom, Valid: false}, + Policy: lib.NullDNSPolicy{DNSPolicy: lib.DNSpreferIPv4, Valid: false}, }, c.Options.DNS) }}, {opts{env: []string{"K6_DNS=ttl=5,select=round-robin"}}, exp{}, func(t *testing.T, c Config) { assert.Equal(t, lib.DNSConfig{ TTL: null.StringFrom("5"), Select: lib.NullDNSSelect{DNSSelect: lib.DNSRoundRobin, Valid: true}, + Policy: lib.NullDNSPolicy{DNSPolicy: lib.DNSpreferIPv4, Valid: false}, }, c.Options.DNS) }}, {opts{env: []string{"K6_DNS=ttl=inf,select=random,policy=preferIPv6"}}, exp{}, func(t *testing.T, c Config) { assert.Equal(t, lib.DNSConfig{ TTL: null.StringFrom("inf"), Select: lib.NullDNSSelect{DNSSelect: lib.DNSRandom, Valid: true}, + Policy: lib.NullDNSPolicy{DNSPolicy: lib.DNSpreferIPv6, Valid: true}, }, c.Options.DNS) }}, // This is functionally invalid, but will error out in validation done in js.parseTTL(). @@ -405,6 +408,7 @@ func getConfigConsolidationTestCases() []configConsolidationTestCase { assert.Equal(t, lib.DNSConfig{ TTL: null.StringFrom("-1"), Select: lib.NullDNSSelect{DNSSelect: lib.DNSRandom, Valid: false}, + Policy: lib.NullDNSPolicy{DNSPolicy: lib.DNSpreferIPv4, Valid: false}, }, c.Options.DNS) }}, {opts{cli: []string{"--dns", "ttl=0,blah=nope"}}, exp{cliReadError: true}, nil}, @@ -412,25 +416,32 @@ func getConfigConsolidationTestCases() []configConsolidationTestCase { assert.Equal(t, lib.DNSConfig{ TTL: null.StringFrom("0"), Select: lib.NullDNSSelect{DNSSelect: lib.DNSRandom, Valid: false}, + Policy: lib.NullDNSPolicy{DNSPolicy: lib.DNSpreferIPv4, Valid: false}, }, c.Options.DNS) }}, {opts{cli: []string{"--dns", "ttl=5s,select="}}, exp{cliReadError: true}, nil}, - {opts{fs: defaultConfig(`{"dns": {"ttl": "0", "select": "round-robin"}}`)}, exp{}, func(t *testing.T, c Config) { - assert.Equal(t, lib.DNSConfig{ - TTL: null.StringFrom("0"), - Select: lib.NullDNSSelect{DNSSelect: lib.DNSRoundRobin, Valid: true}, - }, c.Options.DNS) - }}, + { + opts{fs: defaultConfig(`{"dns": {"ttl": "0", "select": "round-robin", "policy": "onlyIPv4"}}`)}, + exp{}, + func(t *testing.T, c Config) { + assert.Equal(t, lib.DNSConfig{ + TTL: null.StringFrom("0"), + Select: lib.NullDNSSelect{DNSSelect: lib.DNSRoundRobin, Valid: true}, + Policy: lib.NullDNSPolicy{DNSPolicy: lib.DNSonlyIPv4, Valid: true}, + }, c.Options.DNS) + }, + }, { opts{ fs: defaultConfig(`{"dns": {"ttl": "0"}}`), - env: []string{"K6_DNS=ttl=30"}, + env: []string{"K6_DNS=ttl=30,policy=any"}, }, exp{}, func(t *testing.T, c Config) { assert.Equal(t, lib.DNSConfig{ TTL: null.StringFrom("30"), Select: lib.NullDNSSelect{DNSSelect: lib.DNSRandom, Valid: false}, + Policy: lib.NullDNSPolicy{DNSPolicy: lib.DNSany, Valid: true}, }, c.Options.DNS) }, }, @@ -438,7 +449,7 @@ func getConfigConsolidationTestCases() []configConsolidationTestCase { // CLI overrides all, falling back to env opts{ fs: defaultConfig(`{"dns": {"ttl": "60", "select": "first"}}`), - env: []string{"K6_DNS=ttl=30,select=random"}, + env: []string{"K6_DNS=ttl=30,select=random,policy=any"}, cli: []string{"--dns", "ttl=5"}, }, exp{}, @@ -446,6 +457,7 @@ func getConfigConsolidationTestCases() []configConsolidationTestCase { assert.Equal(t, lib.DNSConfig{ TTL: null.StringFrom("5"), Select: lib.NullDNSSelect{DNSSelect: lib.DNSRandom, Valid: true}, + Policy: lib.NullDNSPolicy{DNSPolicy: lib.DNSany, Valid: true}, }, c.Options.DNS) }, }, diff --git a/cmd/options.go b/cmd/options.go index 6bca0fa6fca..6df83a4ac3a 100644 --- a/cmd/options.go +++ b/cmd/options.go @@ -96,7 +96,8 @@ func optionFlagSet() *pflag.FlagSet { flags.String("dns", lib.DefaultDNSConfig().String(), "DNS configuration. Possible ttl values are: 'inf' "+ "for a persistent cache, '0' to disable the cache,\nor a positive duration, e.g. '1s', '1m', etc. "+ "Milliseconds are assumed if no unit is provided.\n"+ - "Possible select values to return a single IP are: 'first', 'random' or 'round-robin'.\n") + "Possible select values to return a single IP are: 'first', 'random' or 'round-robin'.\n"+ + "Possible policy values are: 'preferIPv4', 'preferIPv6', 'onlyIPv4', 'onlyIPv6' or 'any'.\n") return flags } diff --git a/core/local/local_test.go b/core/local/local_test.go index cc2e1f77c32..f243704b0c6 100644 --- a/core/local/local_test.go +++ b/core/local/local_test.go @@ -1006,20 +1006,23 @@ func TestDNSResolver(t *testing.T) { }, "0": { // cache is disabled, every request does a DNS lookup lib.Options{DNS: lib.DNSConfig{ - TTL: null.StringFrom("0"), - Strategy: lib.NullDNSStrategy{DNSStrategy: lib.DNSFirst, Valid: true}, + TTL: null.StringFrom("0"), + Select: lib.NullDNSSelect{DNSSelect: lib.DNSFirst, Valid: true}, + Policy: lib.NullDNSPolicy{DNSPolicy: lib.DNSpreferIPv4, Valid: false}, }}, 5, }, "1000": { // cache IPs for 1s, check that unitless values are interpreted as ms lib.Options{DNS: lib.DNSConfig{ - TTL: null.StringFrom("1000"), - Strategy: lib.NullDNSStrategy{DNSStrategy: lib.DNSFirst, Valid: true}, + TTL: null.StringFrom("1000"), + Select: lib.NullDNSSelect{DNSSelect: lib.DNSFirst, Valid: true}, + Policy: lib.NullDNSPolicy{DNSPolicy: lib.DNSpreferIPv4, Valid: false}, }}, 4, }, "3s": { lib.Options{DNS: lib.DNSConfig{ - TTL: null.StringFrom("3s"), - Strategy: lib.NullDNSStrategy{DNSStrategy: lib.DNSFirst, Valid: true}, + TTL: null.StringFrom("3s"), + Select: lib.NullDNSSelect{DNSSelect: lib.DNSFirst, Valid: true}, + Policy: lib.NullDNSPolicy{DNSPolicy: lib.DNSpreferIPv4, Valid: false}, }}, 3, }, } @@ -1201,7 +1204,7 @@ func TestRealTimeAndSetupTeardownMetrics(t *testing.T) { expTags = append(expTags, addExpTags...) return netext.NewDialer( net.Dialer{}, - netext.NewResolver(net.LookupIP, 0, lib.DNSFirst), + netext.NewResolver(net.LookupIP, 0, lib.DNSFirst, lib.DNSpreferIPv4), ).GetTrail(time.Now(), time.Now(), true, emitIterations, getTags(expTags...)) } diff --git a/js/initcontext_test.go b/js/initcontext_test.go index d2bb5d0f104..eaeb3a1734b 100644 --- a/js/initcontext_test.go +++ b/js/initcontext_test.go @@ -394,7 +394,7 @@ func TestRequestWithBinaryFile(t *testing.T) { KeepAlive: 60 * time.Second, DualStack: true, }, - netext.NewResolver(net.LookupIP, 0, lib.DNSFirst), + netext.NewResolver(net.LookupIP, 0, lib.DNSFirst, lib.DNSpreferIPv4), )).DialContext, }, BPool: bpool.NewBufferPool(1), diff --git a/js/runner.go b/js/runner.go index 1d4b73e9f61..f6a3d5fd0c8 100644 --- a/js/runner.go +++ b/js/runner.go @@ -97,6 +97,7 @@ func newFromBundle(logger *logrus.Logger, b *Bundle) (*Runner, error) { return nil, err } + defDNS := lib.DefaultDNSConfig() r := &Runner{ Bundle: b, Logger: logger, @@ -106,8 +107,9 @@ func newFromBundle(logger *logrus.Logger, b *Bundle) (*Runner, error) { KeepAlive: 30 * time.Second, DualStack: true, }, - console: newConsole(logger), - Resolver: netext.NewResolver(net.LookupIP, 0, lib.DefaultDNSConfig().Select.DNSSelect), + console: newConsole(logger), + Resolver: netext.NewResolver( + net.LookupIP, 0, defDNS.Select.DNSSelect, defDNS.Policy.DNSPolicy), ActualResolver: net.LookupIP, } @@ -345,11 +347,15 @@ func (r *Runner) setResolver(dns lib.DNSConfig) error { return err } - dnsSel := opts.DNS.Select.DNSSelect + dnsSel := dns.Select.DNSSelect if !dnsSel.IsADNSSelect() { dnsSel = lib.DefaultDNSConfig().Select.DNSSelect } - r.Resolver = netext.NewResolver(r.ActualResolver, ttl, dnsSel) + dnsPol := dns.Policy.DNSPolicy + if !dnsPol.IsADNSPolicy() { + dnsPol = lib.DefaultDNSConfig().Policy.DNSPolicy + } + r.Resolver = netext.NewResolver(r.ActualResolver, ttl, dnsSel, dnsPol) return nil } diff --git a/lib/dns_policy_gen.go b/lib/dns_policy_gen.go new file mode 100644 index 00000000000..184e500c0ea --- /dev/null +++ b/lib/dns_policy_gen.go @@ -0,0 +1,54 @@ +// Code generated by "enumer -type=DNSPolicy -trimprefix DNS -output dns_policy_gen.go"; DO NOT EDIT. + +// +package lib + +import ( + "fmt" +) + +const _DNSPolicyName = "preferIPv4preferIPv6onlyIPv4onlyIPv6any" + +var _DNSPolicyIndex = [...]uint8{0, 10, 20, 28, 36, 39} + +func (i DNSPolicy) String() string { + i -= 1 + if i >= DNSPolicy(len(_DNSPolicyIndex)-1) { + return fmt.Sprintf("DNSPolicy(%d)", i+1) + } + return _DNSPolicyName[_DNSPolicyIndex[i]:_DNSPolicyIndex[i+1]] +} + +var _DNSPolicyValues = []DNSPolicy{1, 2, 3, 4, 5} + +var _DNSPolicyNameToValueMap = map[string]DNSPolicy{ + _DNSPolicyName[0:10]: 1, + _DNSPolicyName[10:20]: 2, + _DNSPolicyName[20:28]: 3, + _DNSPolicyName[28:36]: 4, + _DNSPolicyName[36:39]: 5, +} + +// DNSPolicyString retrieves an enum value from the enum constants string name. +// Throws an error if the param is not part of the enum. +func DNSPolicyString(s string) (DNSPolicy, error) { + if val, ok := _DNSPolicyNameToValueMap[s]; ok { + return val, nil + } + return 0, fmt.Errorf("%s does not belong to DNSPolicy values", s) +} + +// DNSPolicyValues returns all values of the enum +func DNSPolicyValues() []DNSPolicy { + return _DNSPolicyValues +} + +// IsADNSPolicy returns "true" if the value is listed in the enum definition. "false" otherwise +func (i DNSPolicy) IsADNSPolicy() bool { + for _, v := range _DNSPolicyValues { + if i == v { + return true + } + } + return false +} diff --git a/lib/netext/httpext/tracer_test.go b/lib/netext/httpext/tracer_test.go index 812b261de81..6ebe64ec2f1 100644 --- a/lib/netext/httpext/tracer_test.go +++ b/lib/netext/httpext/tracer_test.go @@ -58,7 +58,7 @@ func TestTracer(t *testing.T) { assert.True(t, ok) transport.DialContext = netext.NewDialer( net.Dialer{}, - netext.NewResolver(net.LookupIP, 0, lib.DNSFirst), + netext.NewResolver(net.LookupIP, 0, lib.DNSFirst, lib.DNSpreferIPv4), ).DialContext var prev int64 diff --git a/lib/netext/resolver.go b/lib/netext/resolver.go index 858b0909c56..614f6290a00 100644 --- a/lib/netext/resolver.go +++ b/lib/netext/resolver.go @@ -40,6 +40,7 @@ type Resolver interface { type resolver struct { resolve MultiResolver selectIndex lib.DNSSelect + policy lib.DNSPolicy rrm *sync.Mutex rand *rand.Rand roundRobin map[string]uint8 @@ -59,12 +60,15 @@ type cacheResolver struct { // NewResolver returns a new DNS resolver. If ttl is not 0, responses // will be cached per host for the specified period. The IP returned from -// LookupIP() will be selected based on the given sel value. -func NewResolver(actRes MultiResolver, ttl time.Duration, sel lib.DNSSelect) Resolver { +// LookupIP() will be selected based on the given sel and pol values. +func NewResolver( + actRes MultiResolver, ttl time.Duration, sel lib.DNSSelect, pol lib.DNSPolicy, +) Resolver { r := rand.New(rand.NewSource(time.Now().UnixNano())) // nolint: gosec res := resolver{ resolve: actRes, selectIndex: sel, + policy: pol, rrm: &sync.Mutex{}, rand: r, roundRobin: make(map[string]uint8), @@ -80,27 +84,29 @@ func NewResolver(actRes MultiResolver, ttl time.Duration, sel lib.DNSSelect) Res } } -// LookupIP returns a single IP resolved for host, selected by the -// configured select strategy. +// LookupIP returns a single IP resolved for host, selected according to the +// configured select and policy options. func (r *resolver) LookupIP(host string) (net.IP, error) { ips, err := r.resolve(host) if err != nil { return nil, err } + + ips = r.applyPolicy(ips) return r.selectOne(host, ips), nil } -// LookupIP returns a single IP resolved for host, selected by the configured -// select strategy. Results are cached per host and will be refreshed if the -// last lookup time exceeds the configured TTL (not the TTL returned in the DNS -// record). +// LookupIP returns a single IP resolved for host, selected according to the +// configured select and policy options. Results are cached per host and will be +// refreshed if the last lookup time exceeds the configured TTL (not the TTL +// returned in the DNS record). func (r *cacheResolver) LookupIP(host string) (net.IP, error) { r.cm.Lock() var ips []net.IP // TODO: Invalidate? When? - if d, ok := r.cache[host]; ok && time.Now().Before(d.lastLookup.Add(r.ttl)) { - ips = r.cache[host].ips + if cr, ok := r.cache[host]; ok && time.Now().Before(cr.lastLookup.Add(r.ttl)) { + ips = cr.ips } else { r.cm.Unlock() // The lookup could take some time, so unlock momentarily. var err error @@ -108,11 +114,13 @@ func (r *cacheResolver) LookupIP(host string) (net.IP, error) { if err != nil { return nil, err } + ips = r.applyPolicy(ips) r.cm.Lock() r.cache[host] = cacheRecord{ips: ips, lastLookup: time.Now()} } r.cm.Unlock() + return r.selectOne(host, ips), nil } @@ -120,6 +128,7 @@ func (r *resolver) selectOne(host string, ips []net.IP) net.IP { if len(ips) == 0 { return nil } + var ip net.IP switch r.selectIndex { case lib.DNSFirst: @@ -136,5 +145,46 @@ func (r *resolver) selectOne(host string, ips []net.IP) net.IP { ip = ips[r.rand.Intn(len(ips))] r.rrm.Unlock() } + return ip } + +func (r *resolver) applyPolicy(ips []net.IP) (retIPs []net.IP) { + if r.policy == lib.DNSany { + return ips + } + ip4, ip6 := groupByVersion(ips) + switch r.policy { + case lib.DNSpreferIPv4: + retIPs = ip4 + if len(retIPs) == 0 { + retIPs = ip6 + } + case lib.DNSpreferIPv6: + retIPs = ip6 + if len(retIPs) == 0 { + retIPs = ip4 + } + case lib.DNSonlyIPv4: + retIPs = ip4 + case lib.DNSonlyIPv6: + retIPs = ip6 + // Already checked above, but added to satisfy 'exhaustive' linter. + case lib.DNSany: + retIPs = ips + } + + return +} + +func groupByVersion(ips []net.IP) (ip4 []net.IP, ip6 []net.IP) { + for _, ip := range ips { + if ip.To4() != nil { + ip4 = append(ip4, ip) + } else { + ip6 = append(ip6, ip) + } + } + + return +} diff --git a/lib/netext/resolver_test.go b/lib/netext/resolver_test.go index c6f3f8e17d5..9416fd49d9e 100644 --- a/lib/netext/resolver_test.go +++ b/lib/netext/resolver_test.go @@ -42,6 +42,9 @@ func TestResolver(t *testing.T) { net.ParseIP("127.0.0.10"), net.ParseIP("127.0.0.11"), net.ParseIP("127.0.0.12"), + net.ParseIP("2001:db8::10"), + net.ParseIP("2001:db8::11"), + net.ParseIP("2001:db8::12"), }, }, nil) @@ -49,11 +52,28 @@ func TestResolver(t *testing.T) { testCases := []struct { ttl time.Duration sel lib.DNSSelect + pol lib.DNSPolicy expIP []net.IP }{ - {0, lib.DNSFirst, []net.IP{net.ParseIP("127.0.0.10")}}, - {time.Second, lib.DNSFirst, []net.IP{net.ParseIP("127.0.0.10")}}, - {0, lib.DNSRoundRobin, []net.IP{ + { + 0, lib.DNSFirst, lib.DNSpreferIPv4, + []net.IP{net.ParseIP("127.0.0.10")}, + }, + { + time.Second, lib.DNSFirst, lib.DNSpreferIPv4, + []net.IP{net.ParseIP("127.0.0.10")}, + }, + {0, lib.DNSRoundRobin, lib.DNSonlyIPv6, []net.IP{ + net.ParseIP("2001:db8::10"), + net.ParseIP("2001:db8::11"), + net.ParseIP("2001:db8::12"), + net.ParseIP("2001:db8::10"), + }}, + { + 0, lib.DNSFirst, lib.DNSpreferIPv6, + []net.IP{net.ParseIP("2001:db8::10")}, + }, + {0, lib.DNSRoundRobin, lib.DNSpreferIPv4, []net.IP{ net.ParseIP("127.0.0.10"), net.ParseIP("127.0.0.11"), net.ParseIP("127.0.0.12"), @@ -63,8 +83,8 @@ func TestResolver(t *testing.T) { for _, tc := range testCases { tc := tc - t.Run(fmt.Sprintf("%s_%s", tc.ttl, tc.sel), func(t *testing.T) { - r := NewResolver(mr.LookupIPAll, tc.ttl, tc.sel) + t.Run(fmt.Sprintf("%s_%s_%s", tc.ttl, tc.sel, tc.pol), func(t *testing.T) { + r := NewResolver(mr.LookupIPAll, tc.ttl, tc.sel, tc.pol) ip, err := r.LookupIP(host) require.NoError(t, err) assert.Equal(t, tc.expIP[0], ip) diff --git a/lib/options.go b/lib/options.go index f89e06feb7e..73410d0fca6 100644 --- a/lib/options.go +++ b/lib/options.go @@ -52,12 +52,82 @@ type DNSConfig struct { TTL null.String `json:"ttl"` // Select specifies the strategy to use when picking a single IP if more than one is returned for a host name. Select NullDNSSelect `json:"select"` + // Policy specifies how to handle returning of IPv4 or IPv6 addresses. + Policy NullDNSPolicy `json:"policy"` // FIXME: Valid is unused and is only added to satisfy some logic in // lib.Options.ForEachSpecified(), otherwise it would panic with // `reflect: call of reflect.Value.Bool on zero Value`. Valid bool `json:"-"` } +// DNSPolicy specifies the preference for handling IP versions in DNS resolutions. +//go:generate enumer -type=DNSPolicy -trimprefix DNS -output dns_policy_gen.go +type DNSPolicy uint8 + +// These are lower camel cased since enumer doesn't support it as a transform option. +// See https://github.com/alvaroloes/enumer/pull/60 . +const ( + // DNSpreferIPv4 returns an IPv4 address if available, falling back to IPv6 otherwise. + DNSpreferIPv4 DNSPolicy = iota + 1 + // DNSpreferIPv6 returns an IPv6 address if available, falling back to IPv4 otherwise. + DNSpreferIPv6 + // DNSonlyIPv4 only returns an IPv4 address and the resolution will fail if no IPv4 address is found. + DNSonlyIPv4 + // DNSonlyIPv6 only returns an IPv6 address and the resolution will fail if no IPv6 address is found. + DNSonlyIPv6 + // DNSany returns any resolved address regardless of version. + DNSany +) + +// UnmarshalJSON converts JSON data to a valid DNSPolicy +func (d *DNSPolicy) UnmarshalJSON(data []byte) error { + if bytes.Equal(data, []byte(`null`)) { + return nil + } + var s string + if err := json.Unmarshal(data, &s); err != nil { + return err + } + v, err := DNSPolicyString(s) + if err != nil { + return err + } + *d = v + return nil +} + +// MarshalJSON returns the JSON representation of d. +func (d DNSPolicy) MarshalJSON() ([]byte, error) { + return json.Marshal(d.String()) +} + +// NullDNSPolicy is a nullable wrapper around DNSPolicy, required for the +// current configuration system. +type NullDNSPolicy struct { + DNSPolicy + Valid bool +} + +// UnmarshalJSON converts JSON data to a valid NullDNSPolicy. +func (d *NullDNSPolicy) UnmarshalJSON(data []byte) error { + if bytes.Equal(data, []byte(`null`)) { + return nil + } + if err := json.Unmarshal(data, &d.DNSPolicy); err != nil { + return err + } + d.Valid = true + return nil +} + +// MarshalJSON returns the JSON representation of d. +func (d NullDNSPolicy) MarshalJSON() ([]byte, error) { + if !d.Valid { + return []byte(`null`), nil + } + return json.Marshal(d.DNSPolicy) +} + // DNSSelect is the strategy to use when picking a single IP if more than one // is returned for a host name. //go:generate enumer -type=DNSSelect -transform=kebab -trimprefix DNS -output dns_select_gen.go @@ -89,7 +159,7 @@ func (d *DNSSelect) UnmarshalJSON(data []byte) error { return nil } -// MarshalJSON returns the JSON representation of d +// MarshalJSON returns the JSON representation of d. func (d DNSSelect) MarshalJSON() ([]byte, error) { return json.Marshal(d.String()) } @@ -101,7 +171,7 @@ type NullDNSSelect struct { Valid bool } -// UnmarshalJSON converts JSON data to a valid NullDNSStratey +// UnmarshalJSON converts JSON data to a valid NullDNSSelect. func (d *NullDNSSelect) UnmarshalJSON(data []byte) error { if bytes.Equal(data, []byte(`null`)) { return nil @@ -113,7 +183,7 @@ func (d *NullDNSSelect) UnmarshalJSON(data []byte) error { return nil } -// MarshalJSON returns the JSON representation of d +// MarshalJSON returns the JSON representation of d. func (d NullDNSSelect) MarshalJSON() ([]byte, error) { if !d.Valid { return []byte(`null`), nil @@ -126,13 +196,14 @@ func DefaultDNSConfig() DNSConfig { return DNSConfig{ TTL: null.NewString("5m", false), Select: NullDNSSelect{DNSRandom, false}, + Policy: NullDNSPolicy{DNSpreferIPv4, false}, } } // String implements fmt.Stringer. func (c DNSConfig) String() string { - return fmt.Sprintf("ttl=%s,select=%s", - c.TTL.String, c.Select.String()) + return fmt.Sprintf("ttl=%s,select=%s,policy=%s", + c.TTL.String, c.Select.String(), c.Policy.String()) } // UnmarshalJSON implements json.Unmarshaler. @@ -140,12 +211,14 @@ func (c *DNSConfig) UnmarshalJSON(data []byte) error { var s struct { TTL null.String `json:"ttl"` Select NullDNSSelect `json:"select"` + Policy NullDNSPolicy `json:"policy"` } if err := json.Unmarshal(data, &s); err != nil { return err } c.TTL = s.TTL c.Select = s.Select + c.Policy = s.Policy return nil } @@ -165,6 +238,13 @@ func (c *DNSConfig) UnmarshalText(text []byte) error { func (c *DNSConfig) unmarshal(params map[string]interface{}) error { for k, v := range params { switch k { + case "policy": + p, err := DNSPolicyString(v.(string)) + if err != nil { + return err + } + c.Policy.DNSPolicy = p + c.Policy.Valid = true case "select": s, err := DNSSelectString(v.(string)) if err != nil { @@ -686,6 +766,9 @@ func (o Options) Apply(opts Options) Options { if opts.DNS.Select.Valid { o.DNS.Select = opts.DNS.Select } + if opts.DNS.Policy.Valid { + o.DNS.Policy = opts.DNS.Policy + } return o } diff --git a/lib/testutils/httpmultibin/httpmultibin.go b/lib/testutils/httpmultibin/httpmultibin.go index cbbfb0628fd..634a4445591 100644 --- a/lib/testutils/httpmultibin/httpmultibin.go +++ b/lib/testutils/httpmultibin/httpmultibin.go @@ -253,7 +253,7 @@ func NewHTTPMultiBin(t testing.TB) *HTTPMultiBin { Timeout: 2 * time.Second, KeepAlive: 10 * time.Second, DualStack: true, - }, netext.NewResolver(net.LookupIP, 0, lib.DNSFirst)) + }, netext.NewResolver(net.LookupIP, 0, lib.DNSFirst, lib.DNSpreferIPv4)) dialer.Hosts = map[string]*lib.HostAddress{ httpDomain: httpDomainValue, httpsDomain: httpsDomainValue, From a0bc1390c0f614632a9c3faee5cc58c376acb3b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ivan=20Miri=C4=87?= Date: Thu, 15 Oct 2020 16:24:04 +0200 Subject: [PATCH 4/8] Move DNS config types to lib/types Resolves https://github.com/loadimpact/k6/pull/1612#discussion_r504607892 --- cmd/config.go | 3 +- cmd/config_consolidation_test.go | 48 ++-- cmd/options.go | 2 +- core/local/local_test.go | 22 +- js/initcontext_test.go | 3 +- js/runner.go | 21 +- lib/netext/httpext/tracer_test.go | 4 +- lib/netext/resolver.go | 26 +-- lib/netext/resolver_test.go | 18 +- lib/options.go | 220 +----------------- lib/testutils/httpmultibin/httpmultibin.go | 3 +- lib/types/dns.go | 246 +++++++++++++++++++++ lib/{ => types}/dns_policy_gen.go | 2 +- lib/{ => types}/dns_select_gen.go | 2 +- 14 files changed, 326 insertions(+), 294 deletions(-) create mode 100644 lib/types/dns.go rename lib/{ => types}/dns_policy_gen.go (98%) rename lib/{ => types}/dns_select_gen.go (98%) diff --git a/cmd/config.go b/cmd/config.go index b75bb81bd00..de37f6a44bf 100644 --- a/cmd/config.go +++ b/cmd/config.go @@ -35,6 +35,7 @@ import ( "github.com/loadimpact/k6/lib" "github.com/loadimpact/k6/lib/executor" + "github.com/loadimpact/k6/lib/types" "github.com/loadimpact/k6/stats" "github.com/loadimpact/k6/stats/cloud" "github.com/loadimpact/k6/stats/csv" @@ -259,7 +260,7 @@ func applyDefault(conf Config) Config { if conf.Options.SummaryTrendStats == nil { conf.Options.SummaryTrendStats = lib.DefaultSummaryTrendStats } - defDNS := lib.DefaultDNSConfig() + defDNS := types.DefaultDNSConfig() if !conf.DNS.TTL.Valid { conf.DNS.TTL = defDNS.TTL } diff --git a/cmd/config_consolidation_test.go b/cmd/config_consolidation_test.go index b73a5958f3f..9998779d0d3 100644 --- a/cmd/config_consolidation_test.go +++ b/cmd/config_consolidation_test.go @@ -383,40 +383,40 @@ func getConfigConsolidationTestCases() []configConsolidationTestCase { }, }, {opts{cli: []string{}}, exp{}, func(t *testing.T, c Config) { - assert.Equal(t, lib.DNSConfig{ + assert.Equal(t, types.DNSConfig{ TTL: null.NewString("5m", false), - Select: lib.NullDNSSelect{DNSSelect: lib.DNSRandom, Valid: false}, - Policy: lib.NullDNSPolicy{DNSPolicy: lib.DNSpreferIPv4, Valid: false}, + Select: types.NullDNSSelect{DNSSelect: types.DNSRandom, Valid: false}, + Policy: types.NullDNSPolicy{DNSPolicy: types.DNSpreferIPv4, Valid: false}, }, c.Options.DNS) }}, {opts{env: []string{"K6_DNS=ttl=5,select=round-robin"}}, exp{}, func(t *testing.T, c Config) { - assert.Equal(t, lib.DNSConfig{ + assert.Equal(t, types.DNSConfig{ TTL: null.StringFrom("5"), - Select: lib.NullDNSSelect{DNSSelect: lib.DNSRoundRobin, Valid: true}, - Policy: lib.NullDNSPolicy{DNSPolicy: lib.DNSpreferIPv4, Valid: false}, + Select: types.NullDNSSelect{DNSSelect: types.DNSRoundRobin, Valid: true}, + Policy: types.NullDNSPolicy{DNSPolicy: types.DNSpreferIPv4, Valid: false}, }, c.Options.DNS) }}, {opts{env: []string{"K6_DNS=ttl=inf,select=random,policy=preferIPv6"}}, exp{}, func(t *testing.T, c Config) { - assert.Equal(t, lib.DNSConfig{ + assert.Equal(t, types.DNSConfig{ TTL: null.StringFrom("inf"), - Select: lib.NullDNSSelect{DNSSelect: lib.DNSRandom, Valid: true}, - Policy: lib.NullDNSPolicy{DNSPolicy: lib.DNSpreferIPv6, Valid: true}, + Select: types.NullDNSSelect{DNSSelect: types.DNSRandom, Valid: true}, + Policy: types.NullDNSPolicy{DNSPolicy: types.DNSpreferIPv6, Valid: true}, }, c.Options.DNS) }}, // This is functionally invalid, but will error out in validation done in js.parseTTL(). {opts{cli: []string{"--dns", "ttl=-1"}}, exp{}, func(t *testing.T, c Config) { - assert.Equal(t, lib.DNSConfig{ + assert.Equal(t, types.DNSConfig{ TTL: null.StringFrom("-1"), - Select: lib.NullDNSSelect{DNSSelect: lib.DNSRandom, Valid: false}, - Policy: lib.NullDNSPolicy{DNSPolicy: lib.DNSpreferIPv4, Valid: false}, + Select: types.NullDNSSelect{DNSSelect: types.DNSRandom, Valid: false}, + Policy: types.NullDNSPolicy{DNSPolicy: types.DNSpreferIPv4, Valid: false}, }, c.Options.DNS) }}, {opts{cli: []string{"--dns", "ttl=0,blah=nope"}}, exp{cliReadError: true}, nil}, {opts{cli: []string{"--dns", "ttl=0"}}, exp{}, func(t *testing.T, c Config) { - assert.Equal(t, lib.DNSConfig{ + assert.Equal(t, types.DNSConfig{ TTL: null.StringFrom("0"), - Select: lib.NullDNSSelect{DNSSelect: lib.DNSRandom, Valid: false}, - Policy: lib.NullDNSPolicy{DNSPolicy: lib.DNSpreferIPv4, Valid: false}, + Select: types.NullDNSSelect{DNSSelect: types.DNSRandom, Valid: false}, + Policy: types.NullDNSPolicy{DNSPolicy: types.DNSpreferIPv4, Valid: false}, }, c.Options.DNS) }}, {opts{cli: []string{"--dns", "ttl=5s,select="}}, exp{cliReadError: true}, nil}, @@ -424,10 +424,10 @@ func getConfigConsolidationTestCases() []configConsolidationTestCase { opts{fs: defaultConfig(`{"dns": {"ttl": "0", "select": "round-robin", "policy": "onlyIPv4"}}`)}, exp{}, func(t *testing.T, c Config) { - assert.Equal(t, lib.DNSConfig{ + assert.Equal(t, types.DNSConfig{ TTL: null.StringFrom("0"), - Select: lib.NullDNSSelect{DNSSelect: lib.DNSRoundRobin, Valid: true}, - Policy: lib.NullDNSPolicy{DNSPolicy: lib.DNSonlyIPv4, Valid: true}, + Select: types.NullDNSSelect{DNSSelect: types.DNSRoundRobin, Valid: true}, + Policy: types.NullDNSPolicy{DNSPolicy: types.DNSonlyIPv4, Valid: true}, }, c.Options.DNS) }, }, @@ -438,10 +438,10 @@ func getConfigConsolidationTestCases() []configConsolidationTestCase { }, exp{}, func(t *testing.T, c Config) { - assert.Equal(t, lib.DNSConfig{ + assert.Equal(t, types.DNSConfig{ TTL: null.StringFrom("30"), - Select: lib.NullDNSSelect{DNSSelect: lib.DNSRandom, Valid: false}, - Policy: lib.NullDNSPolicy{DNSPolicy: lib.DNSany, Valid: true}, + Select: types.NullDNSSelect{DNSSelect: types.DNSRandom, Valid: false}, + Policy: types.NullDNSPolicy{DNSPolicy: types.DNSany, Valid: true}, }, c.Options.DNS) }, }, @@ -454,10 +454,10 @@ func getConfigConsolidationTestCases() []configConsolidationTestCase { }, exp{}, func(t *testing.T, c Config) { - assert.Equal(t, lib.DNSConfig{ + assert.Equal(t, types.DNSConfig{ TTL: null.StringFrom("5"), - Select: lib.NullDNSSelect{DNSSelect: lib.DNSRandom, Valid: true}, - Policy: lib.NullDNSPolicy{DNSPolicy: lib.DNSany, Valid: true}, + Select: types.NullDNSSelect{DNSSelect: types.DNSRandom, Valid: true}, + Policy: types.NullDNSPolicy{DNSPolicy: types.DNSany, Valid: true}, }, c.Options.DNS) }, }, diff --git a/cmd/options.go b/cmd/options.go index 6df83a4ac3a..462404c49d5 100644 --- a/cmd/options.go +++ b/cmd/options.go @@ -93,7 +93,7 @@ func optionFlagSet() *pflag.FlagSet { flags.StringSlice("tag", nil, "add a `tag` to be applied to all samples, as `[name]=[value]`") flags.String("console-output", "", "redirects the console logging to the provided output file") flags.Bool("discard-response-bodies", false, "Read but don't process or save HTTP response bodies") - flags.String("dns", lib.DefaultDNSConfig().String(), "DNS configuration. Possible ttl values are: 'inf' "+ + flags.String("dns", types.DefaultDNSConfig().String(), "DNS resolver configuration. Possible ttl values are: 'inf' "+ "for a persistent cache, '0' to disable the cache,\nor a positive duration, e.g. '1s', '1m', etc. "+ "Milliseconds are assumed if no unit is provided.\n"+ "Possible select values to return a single IP are: 'first', 'random' or 'round-robin'.\n"+ diff --git a/core/local/local_test.go b/core/local/local_test.go index f243704b0c6..cb5c3c951e4 100644 --- a/core/local/local_test.go +++ b/core/local/local_test.go @@ -1002,27 +1002,27 @@ func TestDNSResolver(t *testing.T) { expLogEntries int }{ "default": { // IPs are cached for 5m - lib.Options{DNS: lib.DefaultDNSConfig()}, 0, + lib.Options{DNS: types.DefaultDNSConfig()}, 0, }, "0": { // cache is disabled, every request does a DNS lookup - lib.Options{DNS: lib.DNSConfig{ + lib.Options{DNS: types.DNSConfig{ TTL: null.StringFrom("0"), - Select: lib.NullDNSSelect{DNSSelect: lib.DNSFirst, Valid: true}, - Policy: lib.NullDNSPolicy{DNSPolicy: lib.DNSpreferIPv4, Valid: false}, + Select: types.NullDNSSelect{DNSSelect: types.DNSFirst, Valid: true}, + Policy: types.NullDNSPolicy{DNSPolicy: types.DNSpreferIPv4, Valid: false}, }}, 5, }, "1000": { // cache IPs for 1s, check that unitless values are interpreted as ms - lib.Options{DNS: lib.DNSConfig{ + lib.Options{DNS: types.DNSConfig{ TTL: null.StringFrom("1000"), - Select: lib.NullDNSSelect{DNSSelect: lib.DNSFirst, Valid: true}, - Policy: lib.NullDNSPolicy{DNSPolicy: lib.DNSpreferIPv4, Valid: false}, + Select: types.NullDNSSelect{DNSSelect: types.DNSFirst, Valid: true}, + Policy: types.NullDNSPolicy{DNSPolicy: types.DNSpreferIPv4, Valid: false}, }}, 4, }, "3s": { - lib.Options{DNS: lib.DNSConfig{ + lib.Options{DNS: types.DNSConfig{ TTL: null.StringFrom("3s"), - Select: lib.NullDNSSelect{DNSSelect: lib.DNSFirst, Valid: true}, - Policy: lib.NullDNSPolicy{DNSPolicy: lib.DNSpreferIPv4, Valid: false}, + Select: types.NullDNSSelect{DNSSelect: types.DNSFirst, Valid: true}, + Policy: types.NullDNSPolicy{DNSPolicy: types.DNSpreferIPv4, Valid: false}, }}, 3, }, } @@ -1204,7 +1204,7 @@ func TestRealTimeAndSetupTeardownMetrics(t *testing.T) { expTags = append(expTags, addExpTags...) return netext.NewDialer( net.Dialer{}, - netext.NewResolver(net.LookupIP, 0, lib.DNSFirst, lib.DNSpreferIPv4), + netext.NewResolver(net.LookupIP, 0, types.DNSFirst, types.DNSpreferIPv4), ).GetTrail(time.Now(), time.Now(), true, emitIterations, getTags(expTags...)) } diff --git a/js/initcontext_test.go b/js/initcontext_test.go index eaeb3a1734b..c9010d63691 100644 --- a/js/initcontext_test.go +++ b/js/initcontext_test.go @@ -43,6 +43,7 @@ import ( "github.com/loadimpact/k6/lib/consts" "github.com/loadimpact/k6/lib/netext" "github.com/loadimpact/k6/lib/testutils" + "github.com/loadimpact/k6/lib/types" "github.com/loadimpact/k6/stats" ) @@ -394,7 +395,7 @@ func TestRequestWithBinaryFile(t *testing.T) { KeepAlive: 60 * time.Second, DualStack: true, }, - netext.NewResolver(net.LookupIP, 0, lib.DNSFirst, lib.DNSpreferIPv4), + netext.NewResolver(net.LookupIP, 0, types.DNSFirst, types.DNSpreferIPv4), )).DialContext, }, BPool: bpool.NewBufferPool(1), diff --git a/js/runner.go b/js/runner.go index f6a3d5fd0c8..03859ef7822 100644 --- a/js/runner.go +++ b/js/runner.go @@ -97,7 +97,7 @@ func newFromBundle(logger *logrus.Logger, b *Bundle) (*Runner, error) { return nil, err } - defDNS := lib.DefaultDNSConfig() + defDNS := types.DefaultDNSConfig() r := &Runner{ Bundle: b, Logger: logger, @@ -341,21 +341,22 @@ func (r *Runner) SetOptions(opts lib.Options) error { return nil } -func (r *Runner) setResolver(dns lib.DNSConfig) error { +func (r *Runner) setResolver(dns types.DNSConfig) error { ttl, err := parseTTL(dns.TTL.String) if err != nil { return err } - dnsSel := dns.Select.DNSSelect - if !dnsSel.IsADNSSelect() { - dnsSel = lib.DefaultDNSConfig().Select.DNSSelect + dnsSel := dns.Select + if !dnsSel.Valid { + dnsSel = types.DefaultDNSConfig().Select } - dnsPol := dns.Policy.DNSPolicy - if !dnsPol.IsADNSPolicy() { - dnsPol = lib.DefaultDNSConfig().Policy.DNSPolicy + dnsPol := dns.Policy + if !dnsPol.Valid { + dnsPol = types.DefaultDNSConfig().Policy } - r.Resolver = netext.NewResolver(r.ActualResolver, ttl, dnsSel, dnsPol) + r.Resolver = netext.NewResolver( + r.ActualResolver, ttl, dnsSel.DNSSelect, dnsPol.DNSPolicy) return nil } @@ -369,7 +370,7 @@ func parseTTL(ttlS string) (time.Duration, error) { case "0": // disable cache case "": - ttlS = lib.DefaultDNSConfig().TTL.String + ttlS = types.DefaultDNSConfig().TTL.String fallthrough default: origTTLs := ttlS diff --git a/lib/netext/httpext/tracer_test.go b/lib/netext/httpext/tracer_test.go index 6ebe64ec2f1..a5231983909 100644 --- a/lib/netext/httpext/tracer_test.go +++ b/lib/netext/httpext/tracer_test.go @@ -40,9 +40,9 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/loadimpact/k6/lib" "github.com/loadimpact/k6/lib/metrics" "github.com/loadimpact/k6/lib/netext" + "github.com/loadimpact/k6/lib/types" "github.com/loadimpact/k6/stats" ) @@ -58,7 +58,7 @@ func TestTracer(t *testing.T) { assert.True(t, ok) transport.DialContext = netext.NewDialer( net.Dialer{}, - netext.NewResolver(net.LookupIP, 0, lib.DNSFirst, lib.DNSpreferIPv4), + netext.NewResolver(net.LookupIP, 0, types.DNSFirst, types.DNSpreferIPv4), ).DialContext var prev int64 diff --git a/lib/netext/resolver.go b/lib/netext/resolver.go index 614f6290a00..0c8ca6bebdf 100644 --- a/lib/netext/resolver.go +++ b/lib/netext/resolver.go @@ -26,7 +26,7 @@ import ( "sync" "time" - "github.com/loadimpact/k6/lib" + "github.com/loadimpact/k6/lib/types" ) // MultiResolver returns all IP addresses for the given host. @@ -39,8 +39,8 @@ type Resolver interface { type resolver struct { resolve MultiResolver - selectIndex lib.DNSSelect - policy lib.DNSPolicy + selectIndex types.DNSSelect + policy types.DNSPolicy rrm *sync.Mutex rand *rand.Rand roundRobin map[string]uint8 @@ -62,7 +62,7 @@ type cacheResolver struct { // will be cached per host for the specified period. The IP returned from // LookupIP() will be selected based on the given sel and pol values. func NewResolver( - actRes MultiResolver, ttl time.Duration, sel lib.DNSSelect, pol lib.DNSPolicy, + actRes MultiResolver, ttl time.Duration, sel types.DNSSelect, pol types.DNSPolicy, ) Resolver { r := rand.New(rand.NewSource(time.Now().UnixNano())) // nolint: gosec res := resolver{ @@ -131,16 +131,16 @@ func (r *resolver) selectOne(host string, ips []net.IP) net.IP { var ip net.IP switch r.selectIndex { - case lib.DNSFirst: + case types.DNSFirst: return ips[0] - case lib.DNSRoundRobin: + case types.DNSRoundRobin: r.rrm.Lock() // NOTE: This index approach is not stable and might result in returning // repeated or skipped IPs if the records change during a test run. ip = ips[int(r.roundRobin[host])%len(ips)] r.roundRobin[host]++ r.rrm.Unlock() - case lib.DNSRandom: + case types.DNSRandom: r.rrm.Lock() ip = ips[r.rand.Intn(len(ips))] r.rrm.Unlock() @@ -150,27 +150,27 @@ func (r *resolver) selectOne(host string, ips []net.IP) net.IP { } func (r *resolver) applyPolicy(ips []net.IP) (retIPs []net.IP) { - if r.policy == lib.DNSany { + if r.policy == types.DNSany { return ips } ip4, ip6 := groupByVersion(ips) switch r.policy { - case lib.DNSpreferIPv4: + case types.DNSpreferIPv4: retIPs = ip4 if len(retIPs) == 0 { retIPs = ip6 } - case lib.DNSpreferIPv6: + case types.DNSpreferIPv6: retIPs = ip6 if len(retIPs) == 0 { retIPs = ip4 } - case lib.DNSonlyIPv4: + case types.DNSonlyIPv4: retIPs = ip4 - case lib.DNSonlyIPv6: + case types.DNSonlyIPv6: retIPs = ip6 // Already checked above, but added to satisfy 'exhaustive' linter. - case lib.DNSany: + case types.DNSany: retIPs = ips } diff --git a/lib/netext/resolver_test.go b/lib/netext/resolver_test.go index 9416fd49d9e..2fa5d1b8d29 100644 --- a/lib/netext/resolver_test.go +++ b/lib/netext/resolver_test.go @@ -29,8 +29,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/loadimpact/k6/lib" "github.com/loadimpact/k6/lib/testutils/mockresolver" + "github.com/loadimpact/k6/lib/types" ) func TestResolver(t *testing.T) { @@ -51,29 +51,29 @@ func TestResolver(t *testing.T) { t.Run("LookupIP", func(t *testing.T) { testCases := []struct { ttl time.Duration - sel lib.DNSSelect - pol lib.DNSPolicy + sel types.DNSSelect + pol types.DNSPolicy expIP []net.IP }{ { - 0, lib.DNSFirst, lib.DNSpreferIPv4, + 0, types.DNSFirst, types.DNSpreferIPv4, []net.IP{net.ParseIP("127.0.0.10")}, }, { - time.Second, lib.DNSFirst, lib.DNSpreferIPv4, + time.Second, types.DNSFirst, types.DNSpreferIPv4, []net.IP{net.ParseIP("127.0.0.10")}, }, - {0, lib.DNSRoundRobin, lib.DNSonlyIPv6, []net.IP{ + {0, types.DNSRoundRobin, types.DNSonlyIPv6, []net.IP{ net.ParseIP("2001:db8::10"), net.ParseIP("2001:db8::11"), net.ParseIP("2001:db8::12"), net.ParseIP("2001:db8::10"), }}, { - 0, lib.DNSFirst, lib.DNSpreferIPv6, + 0, types.DNSFirst, types.DNSpreferIPv6, []net.IP{net.ParseIP("2001:db8::10")}, }, - {0, lib.DNSRoundRobin, lib.DNSpreferIPv4, []net.IP{ + {0, types.DNSRoundRobin, types.DNSpreferIPv4, []net.IP{ net.ParseIP("127.0.0.10"), net.ParseIP("127.0.0.11"), net.ParseIP("127.0.0.12"), @@ -101,7 +101,7 @@ func TestResolver(t *testing.T) { assert.True(t, cr.cache[host].lastLookup.After(firstLookup)) } - if tc.sel == lib.DNSRoundRobin { + if tc.sel == types.DNSRoundRobin { ips := []net.IP{ip} for i := 0; i < 3; i++ { ip, err = r.LookupIP(host) diff --git a/lib/options.go b/lib/options.go index 73410d0fca6..6dfec6ed4c7 100644 --- a/lib/options.go +++ b/lib/options.go @@ -21,7 +21,6 @@ package lib import ( - "bytes" "crypto/tls" "encoding/json" "fmt" @@ -29,7 +28,6 @@ import ( "reflect" "strconv" - "github.com/kubernetes/helm/pkg/strvals" "github.com/pkg/errors" "gopkg.in/guregu/null.v3" @@ -46,222 +44,6 @@ const DefaultScenarioName = "default" // nolint: gochecknoglobals var DefaultSummaryTrendStats = []string{"avg", "min", "med", "max", "p(90)", "p(95)"} -// DNSConfig is the DNS resolver configuration. -type DNSConfig struct { - // If positive, defines how long DNS lookups should be returned from the cache. - TTL null.String `json:"ttl"` - // Select specifies the strategy to use when picking a single IP if more than one is returned for a host name. - Select NullDNSSelect `json:"select"` - // Policy specifies how to handle returning of IPv4 or IPv6 addresses. - Policy NullDNSPolicy `json:"policy"` - // FIXME: Valid is unused and is only added to satisfy some logic in - // lib.Options.ForEachSpecified(), otherwise it would panic with - // `reflect: call of reflect.Value.Bool on zero Value`. - Valid bool `json:"-"` -} - -// DNSPolicy specifies the preference for handling IP versions in DNS resolutions. -//go:generate enumer -type=DNSPolicy -trimprefix DNS -output dns_policy_gen.go -type DNSPolicy uint8 - -// These are lower camel cased since enumer doesn't support it as a transform option. -// See https://github.com/alvaroloes/enumer/pull/60 . -const ( - // DNSpreferIPv4 returns an IPv4 address if available, falling back to IPv6 otherwise. - DNSpreferIPv4 DNSPolicy = iota + 1 - // DNSpreferIPv6 returns an IPv6 address if available, falling back to IPv4 otherwise. - DNSpreferIPv6 - // DNSonlyIPv4 only returns an IPv4 address and the resolution will fail if no IPv4 address is found. - DNSonlyIPv4 - // DNSonlyIPv6 only returns an IPv6 address and the resolution will fail if no IPv6 address is found. - DNSonlyIPv6 - // DNSany returns any resolved address regardless of version. - DNSany -) - -// UnmarshalJSON converts JSON data to a valid DNSPolicy -func (d *DNSPolicy) UnmarshalJSON(data []byte) error { - if bytes.Equal(data, []byte(`null`)) { - return nil - } - var s string - if err := json.Unmarshal(data, &s); err != nil { - return err - } - v, err := DNSPolicyString(s) - if err != nil { - return err - } - *d = v - return nil -} - -// MarshalJSON returns the JSON representation of d. -func (d DNSPolicy) MarshalJSON() ([]byte, error) { - return json.Marshal(d.String()) -} - -// NullDNSPolicy is a nullable wrapper around DNSPolicy, required for the -// current configuration system. -type NullDNSPolicy struct { - DNSPolicy - Valid bool -} - -// UnmarshalJSON converts JSON data to a valid NullDNSPolicy. -func (d *NullDNSPolicy) UnmarshalJSON(data []byte) error { - if bytes.Equal(data, []byte(`null`)) { - return nil - } - if err := json.Unmarshal(data, &d.DNSPolicy); err != nil { - return err - } - d.Valid = true - return nil -} - -// MarshalJSON returns the JSON representation of d. -func (d NullDNSPolicy) MarshalJSON() ([]byte, error) { - if !d.Valid { - return []byte(`null`), nil - } - return json.Marshal(d.DNSPolicy) -} - -// DNSSelect is the strategy to use when picking a single IP if more than one -// is returned for a host name. -//go:generate enumer -type=DNSSelect -transform=kebab -trimprefix DNS -output dns_select_gen.go -type DNSSelect uint8 - -const ( - // DNSFirst returns the first IP from the response. - DNSFirst DNSSelect = iota + 1 - // DNSRoundRobin rotates the IP returned on each lookup. - DNSRoundRobin - // DNSRandom returns a random IP from the response. - DNSRandom -) - -// UnmarshalJSON converts JSON data to a valid DNSSelect -func (d *DNSSelect) UnmarshalJSON(data []byte) error { - if bytes.Equal(data, []byte(`null`)) { - return nil - } - var s string - if err := json.Unmarshal(data, &s); err != nil { - return err - } - v, err := DNSSelectString(s) - if err != nil { - return err - } - *d = v - return nil -} - -// MarshalJSON returns the JSON representation of d. -func (d DNSSelect) MarshalJSON() ([]byte, error) { - return json.Marshal(d.String()) -} - -// NullDNSSelect is a nullable wrapper around DNSSelect, required for the -// current configuration system. -type NullDNSSelect struct { - DNSSelect - Valid bool -} - -// UnmarshalJSON converts JSON data to a valid NullDNSSelect. -func (d *NullDNSSelect) UnmarshalJSON(data []byte) error { - if bytes.Equal(data, []byte(`null`)) { - return nil - } - if err := json.Unmarshal(data, &d.DNSSelect); err != nil { - return err - } - d.Valid = true - return nil -} - -// MarshalJSON returns the JSON representation of d. -func (d NullDNSSelect) MarshalJSON() ([]byte, error) { - if !d.Valid { - return []byte(`null`), nil - } - return json.Marshal(d.DNSSelect) -} - -// DefaultDNSConfig returns the default DNS configuration. -func DefaultDNSConfig() DNSConfig { - return DNSConfig{ - TTL: null.NewString("5m", false), - Select: NullDNSSelect{DNSRandom, false}, - Policy: NullDNSPolicy{DNSpreferIPv4, false}, - } -} - -// String implements fmt.Stringer. -func (c DNSConfig) String() string { - return fmt.Sprintf("ttl=%s,select=%s,policy=%s", - c.TTL.String, c.Select.String(), c.Policy.String()) -} - -// UnmarshalJSON implements json.Unmarshaler. -func (c *DNSConfig) UnmarshalJSON(data []byte) error { - var s struct { - TTL null.String `json:"ttl"` - Select NullDNSSelect `json:"select"` - Policy NullDNSPolicy `json:"policy"` - } - if err := json.Unmarshal(data, &s); err != nil { - return err - } - c.TTL = s.TTL - c.Select = s.Select - c.Policy = s.Policy - return nil -} - -// UnmarshalText implements encoding.TextUnmarshaler. -func (c *DNSConfig) UnmarshalText(text []byte) error { - if string(text) == DefaultDNSConfig().String() { - *c = DefaultDNSConfig() - return nil - } - params, err := strvals.Parse(string(text)) - if err != nil { - return err - } - return c.unmarshal(params) -} - -func (c *DNSConfig) unmarshal(params map[string]interface{}) error { - for k, v := range params { - switch k { - case "policy": - p, err := DNSPolicyString(v.(string)) - if err != nil { - return err - } - c.Policy.DNSPolicy = p - c.Policy.Valid = true - case "select": - s, err := DNSSelectString(v.(string)) - if err != nil { - return err - } - c.Select.DNSSelect = s - c.Select.Valid = true - case "ttl": - ttlv := fmt.Sprintf("%v", v) - c.TTL = null.StringFrom(ttlv) - default: - return fmt.Errorf("unknown DNS configuration field: %s", k) - } - } - return nil -} - // Describes a TLS version. Serialised to/from JSON as a string, eg. "tls1.2". type TLSVersion int @@ -526,7 +308,7 @@ type Options struct { RPS null.Int `json:"rps" envconfig:"K6_RPS"` // DNS handling configuration. - DNS DNSConfig `json:"dns" envconfig:"K6_DNS"` + DNS types.DNSConfig `json:"dns" envconfig:"K6_DNS"` // How many HTTP redirects do we follow? MaxRedirects null.Int `json:"maxRedirects" envconfig:"K6_MAX_REDIRECTS"` diff --git a/lib/testutils/httpmultibin/httpmultibin.go b/lib/testutils/httpmultibin/httpmultibin.go index 634a4445591..a2c165c8b30 100644 --- a/lib/testutils/httpmultibin/httpmultibin.go +++ b/lib/testutils/httpmultibin/httpmultibin.go @@ -49,6 +49,7 @@ import ( "github.com/loadimpact/k6/lib" "github.com/loadimpact/k6/lib/netext" "github.com/loadimpact/k6/lib/netext/httpext" + "github.com/loadimpact/k6/lib/types" ) // GetTLSClientConfig returns a TLS config that trusts the supplied @@ -253,7 +254,7 @@ func NewHTTPMultiBin(t testing.TB) *HTTPMultiBin { Timeout: 2 * time.Second, KeepAlive: 10 * time.Second, DualStack: true, - }, netext.NewResolver(net.LookupIP, 0, lib.DNSFirst, lib.DNSpreferIPv4)) + }, netext.NewResolver(net.LookupIP, 0, types.DNSFirst, types.DNSpreferIPv4)) dialer.Hosts = map[string]*lib.HostAddress{ httpDomain: httpDomainValue, httpsDomain: httpsDomainValue, diff --git a/lib/types/dns.go b/lib/types/dns.go new file mode 100644 index 00000000000..244931c0149 --- /dev/null +++ b/lib/types/dns.go @@ -0,0 +1,246 @@ +/* + * + * k6 - a next-generation load testing tool + * Copyright (C) 2020 Load Impact + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +package types + +import ( + "bytes" + "encoding/json" + "fmt" + + "github.com/kubernetes/helm/pkg/strvals" + "gopkg.in/guregu/null.v3" +) + +// DNSConfig is the DNS resolver configuration. +type DNSConfig struct { + // If positive, defines how long DNS lookups should be returned from the cache. + TTL null.String `json:"ttl"` + // Select specifies the strategy to use when picking a single IP if more than one is returned for a host name. + Select NullDNSSelect `json:"select"` + // Policy specifies how to handle returning of IPv4 or IPv6 addresses. + Policy NullDNSPolicy `json:"policy"` + // FIXME: Valid is unused and is only added to satisfy some logic in + // lib.Options.ForEachSpecified(), otherwise it would panic with + // `reflect: call of reflect.Value.Bool on zero Value`. + Valid bool `json:"-"` +} + +// DefaultDNSConfig returns the default DNS configuration. +func DefaultDNSConfig() DNSConfig { + return DNSConfig{ + TTL: null.NewString("5m", false), + Select: NullDNSSelect{DNSRandom, false}, + Policy: NullDNSPolicy{DNSpreferIPv4, false}, + } +} + +// DNSPolicy specifies the preference for handling IP versions in DNS resolutions. +//go:generate enumer -type=DNSPolicy -trimprefix DNS -output dns_policy_gen.go +type DNSPolicy uint8 + +// These are lower camel cased since enumer doesn't support it as a transform option. +// See https://github.com/alvaroloes/enumer/pull/60 . +const ( + // DNSpreferIPv4 returns an IPv4 address if available, falling back to IPv6 otherwise. + DNSpreferIPv4 DNSPolicy = iota + 1 + // DNSpreferIPv6 returns an IPv6 address if available, falling back to IPv4 otherwise. + DNSpreferIPv6 + // DNSonlyIPv4 only returns an IPv4 address and the resolution will fail if no IPv4 address is found. + DNSonlyIPv4 + // DNSonlyIPv6 only returns an IPv6 address and the resolution will fail if no IPv6 address is found. + DNSonlyIPv6 + // DNSany returns any resolved address regardless of version. + DNSany +) + +// UnmarshalJSON converts JSON data to a valid DNSPolicy +func (d *DNSPolicy) UnmarshalJSON(data []byte) error { + if bytes.Equal(data, []byte(`null`)) { + return nil + } + var s string + if err := json.Unmarshal(data, &s); err != nil { + return err + } + v, err := DNSPolicyString(s) + if err != nil { + return err + } + *d = v + return nil +} + +// MarshalJSON returns the JSON representation of d. +func (d DNSPolicy) MarshalJSON() ([]byte, error) { + return json.Marshal(d.String()) +} + +// NullDNSPolicy is a nullable wrapper around DNSPolicy, required for the +// current configuration system. +type NullDNSPolicy struct { + DNSPolicy + Valid bool +} + +// UnmarshalJSON converts JSON data to a valid NullDNSPolicy. +func (d *NullDNSPolicy) UnmarshalJSON(data []byte) error { + if bytes.Equal(data, []byte(`null`)) { + return nil + } + if err := json.Unmarshal(data, &d.DNSPolicy); err != nil { + return err + } + d.Valid = true + return nil +} + +// MarshalJSON returns the JSON representation of d. +func (d NullDNSPolicy) MarshalJSON() ([]byte, error) { + if !d.Valid { + return []byte(`null`), nil + } + return json.Marshal(d.DNSPolicy) +} + +// DNSSelect is the strategy to use when picking a single IP if more than one +// is returned for a host name. +//go:generate enumer -type=DNSSelect -transform=kebab -trimprefix DNS -output dns_select_gen.go +type DNSSelect uint8 + +const ( + // DNSFirst returns the first IP from the response. + DNSFirst DNSSelect = iota + 1 + // DNSRoundRobin rotates the IP returned on each lookup. + DNSRoundRobin + // DNSRandom returns a random IP from the response. + DNSRandom +) + +// UnmarshalJSON converts JSON data to a valid DNSSelect +func (d *DNSSelect) UnmarshalJSON(data []byte) error { + if bytes.Equal(data, []byte(`null`)) { + return nil + } + var s string + if err := json.Unmarshal(data, &s); err != nil { + return err + } + v, err := DNSSelectString(s) + if err != nil { + return err + } + *d = v + return nil +} + +// MarshalJSON returns the JSON representation of d. +func (d DNSSelect) MarshalJSON() ([]byte, error) { + return json.Marshal(d.String()) +} + +// NullDNSSelect is a nullable wrapper around DNSSelect, required for the +// current configuration system. +type NullDNSSelect struct { + DNSSelect + Valid bool +} + +// UnmarshalJSON converts JSON data to a valid NullDNSSelect. +func (d *NullDNSSelect) UnmarshalJSON(data []byte) error { + if bytes.Equal(data, []byte(`null`)) { + return nil + } + if err := json.Unmarshal(data, &d.DNSSelect); err != nil { + return err + } + d.Valid = true + return nil +} + +// MarshalJSON returns the JSON representation of d. +func (d NullDNSSelect) MarshalJSON() ([]byte, error) { + if !d.Valid { + return []byte(`null`), nil + } + return json.Marshal(d.DNSSelect) +} + +// String implements fmt.Stringer. +func (c DNSConfig) String() string { + return fmt.Sprintf("ttl=%s,select=%s,policy=%s", + c.TTL.String, c.Select.String(), c.Policy.String()) +} + +// UnmarshalJSON implements json.Unmarshaler. +func (c *DNSConfig) UnmarshalJSON(data []byte) error { + var s struct { + TTL null.String `json:"ttl"` + Select NullDNSSelect `json:"select"` + Policy NullDNSPolicy `json:"policy"` + } + if err := json.Unmarshal(data, &s); err != nil { + return err + } + c.TTL = s.TTL + c.Select = s.Select + c.Policy = s.Policy + return nil +} + +// UnmarshalText implements encoding.TextUnmarshaler. +func (c *DNSConfig) UnmarshalText(text []byte) error { + if string(text) == DefaultDNSConfig().String() { + *c = DefaultDNSConfig() + return nil + } + params, err := strvals.Parse(string(text)) + if err != nil { + return err + } + return c.unmarshal(params) +} + +func (c *DNSConfig) unmarshal(params map[string]interface{}) error { + for k, v := range params { + switch k { + case "policy": + p, err := DNSPolicyString(v.(string)) + if err != nil { + return err + } + c.Policy.DNSPolicy = p + c.Policy.Valid = true + case "select": + s, err := DNSSelectString(v.(string)) + if err != nil { + return err + } + c.Select.DNSSelect = s + c.Select.Valid = true + case "ttl": + ttlv := fmt.Sprintf("%v", v) + c.TTL = null.StringFrom(ttlv) + default: + return fmt.Errorf("unknown DNS configuration field: %s", k) + } + } + return nil +} diff --git a/lib/dns_policy_gen.go b/lib/types/dns_policy_gen.go similarity index 98% rename from lib/dns_policy_gen.go rename to lib/types/dns_policy_gen.go index 184e500c0ea..85feb6d24a7 100644 --- a/lib/dns_policy_gen.go +++ b/lib/types/dns_policy_gen.go @@ -1,7 +1,7 @@ // Code generated by "enumer -type=DNSPolicy -trimprefix DNS -output dns_policy_gen.go"; DO NOT EDIT. // -package lib +package types import ( "fmt" diff --git a/lib/dns_select_gen.go b/lib/types/dns_select_gen.go similarity index 98% rename from lib/dns_select_gen.go rename to lib/types/dns_select_gen.go index b41f352ee61..88d893355e9 100644 --- a/lib/dns_select_gen.go +++ b/lib/types/dns_select_gen.go @@ -1,7 +1,7 @@ // Code generated by "enumer -type=DNSSelect -transform=kebab -trimprefix DNS -output dns_select_gen.go"; DO NOT EDIT. // -package lib +package types import ( "fmt" From 5dbb0cbe21fd6cb548b7d30ca3f54274e6d7e69e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ivan=20Miri=C4=87?= Date: Mon, 19 Oct 2020 15:39:49 +0200 Subject: [PATCH 5/8] Change DNSSelect options to camel case to align with DNSPolicy Resolves https://github.com/loadimpact/k6/pull/1612#discussion_r506133898 --- cmd/config_consolidation_test.go | 20 ++++++++++---------- cmd/options.go | 2 +- core/local/local_test.go | 8 ++++---- js/initcontext_test.go | 2 +- lib/netext/httpext/tracer_test.go | 2 +- lib/netext/resolver.go | 6 +++--- lib/netext/resolver_test.go | 12 ++++++------ lib/testutils/httpmultibin/httpmultibin.go | 2 +- lib/types/dns.go | 18 ++++++++++-------- lib/types/dns_select_gen.go | 10 +++++----- 10 files changed, 42 insertions(+), 40 deletions(-) diff --git a/cmd/config_consolidation_test.go b/cmd/config_consolidation_test.go index 9998779d0d3..40e01e88ff0 100644 --- a/cmd/config_consolidation_test.go +++ b/cmd/config_consolidation_test.go @@ -385,21 +385,21 @@ func getConfigConsolidationTestCases() []configConsolidationTestCase { {opts{cli: []string{}}, exp{}, func(t *testing.T, c Config) { assert.Equal(t, types.DNSConfig{ TTL: null.NewString("5m", false), - Select: types.NullDNSSelect{DNSSelect: types.DNSRandom, Valid: false}, + Select: types.NullDNSSelect{DNSSelect: types.DNSrandom, Valid: false}, Policy: types.NullDNSPolicy{DNSPolicy: types.DNSpreferIPv4, Valid: false}, }, c.Options.DNS) }}, - {opts{env: []string{"K6_DNS=ttl=5,select=round-robin"}}, exp{}, func(t *testing.T, c Config) { + {opts{env: []string{"K6_DNS=ttl=5,select=roundRobin"}}, exp{}, func(t *testing.T, c Config) { assert.Equal(t, types.DNSConfig{ TTL: null.StringFrom("5"), - Select: types.NullDNSSelect{DNSSelect: types.DNSRoundRobin, Valid: true}, + Select: types.NullDNSSelect{DNSSelect: types.DNSroundRobin, Valid: true}, Policy: types.NullDNSPolicy{DNSPolicy: types.DNSpreferIPv4, Valid: false}, }, c.Options.DNS) }}, {opts{env: []string{"K6_DNS=ttl=inf,select=random,policy=preferIPv6"}}, exp{}, func(t *testing.T, c Config) { assert.Equal(t, types.DNSConfig{ TTL: null.StringFrom("inf"), - Select: types.NullDNSSelect{DNSSelect: types.DNSRandom, Valid: true}, + Select: types.NullDNSSelect{DNSSelect: types.DNSrandom, Valid: true}, Policy: types.NullDNSPolicy{DNSPolicy: types.DNSpreferIPv6, Valid: true}, }, c.Options.DNS) }}, @@ -407,7 +407,7 @@ func getConfigConsolidationTestCases() []configConsolidationTestCase { {opts{cli: []string{"--dns", "ttl=-1"}}, exp{}, func(t *testing.T, c Config) { assert.Equal(t, types.DNSConfig{ TTL: null.StringFrom("-1"), - Select: types.NullDNSSelect{DNSSelect: types.DNSRandom, Valid: false}, + Select: types.NullDNSSelect{DNSSelect: types.DNSrandom, Valid: false}, Policy: types.NullDNSPolicy{DNSPolicy: types.DNSpreferIPv4, Valid: false}, }, c.Options.DNS) }}, @@ -415,18 +415,18 @@ func getConfigConsolidationTestCases() []configConsolidationTestCase { {opts{cli: []string{"--dns", "ttl=0"}}, exp{}, func(t *testing.T, c Config) { assert.Equal(t, types.DNSConfig{ TTL: null.StringFrom("0"), - Select: types.NullDNSSelect{DNSSelect: types.DNSRandom, Valid: false}, + Select: types.NullDNSSelect{DNSSelect: types.DNSrandom, Valid: false}, Policy: types.NullDNSPolicy{DNSPolicy: types.DNSpreferIPv4, Valid: false}, }, c.Options.DNS) }}, {opts{cli: []string{"--dns", "ttl=5s,select="}}, exp{cliReadError: true}, nil}, { - opts{fs: defaultConfig(`{"dns": {"ttl": "0", "select": "round-robin", "policy": "onlyIPv4"}}`)}, + opts{fs: defaultConfig(`{"dns": {"ttl": "0", "select": "roundRobin", "policy": "onlyIPv4"}}`)}, exp{}, func(t *testing.T, c Config) { assert.Equal(t, types.DNSConfig{ TTL: null.StringFrom("0"), - Select: types.NullDNSSelect{DNSSelect: types.DNSRoundRobin, Valid: true}, + Select: types.NullDNSSelect{DNSSelect: types.DNSroundRobin, Valid: true}, Policy: types.NullDNSPolicy{DNSPolicy: types.DNSonlyIPv4, Valid: true}, }, c.Options.DNS) }, @@ -440,7 +440,7 @@ func getConfigConsolidationTestCases() []configConsolidationTestCase { func(t *testing.T, c Config) { assert.Equal(t, types.DNSConfig{ TTL: null.StringFrom("30"), - Select: types.NullDNSSelect{DNSSelect: types.DNSRandom, Valid: false}, + Select: types.NullDNSSelect{DNSSelect: types.DNSrandom, Valid: false}, Policy: types.NullDNSPolicy{DNSPolicy: types.DNSany, Valid: true}, }, c.Options.DNS) }, @@ -456,7 +456,7 @@ func getConfigConsolidationTestCases() []configConsolidationTestCase { func(t *testing.T, c Config) { assert.Equal(t, types.DNSConfig{ TTL: null.StringFrom("5"), - Select: types.NullDNSSelect{DNSSelect: types.DNSRandom, Valid: true}, + Select: types.NullDNSSelect{DNSSelect: types.DNSrandom, Valid: true}, Policy: types.NullDNSPolicy{DNSPolicy: types.DNSany, Valid: true}, }, c.Options.DNS) }, diff --git a/cmd/options.go b/cmd/options.go index 462404c49d5..0276b4daaab 100644 --- a/cmd/options.go +++ b/cmd/options.go @@ -96,7 +96,7 @@ func optionFlagSet() *pflag.FlagSet { flags.String("dns", types.DefaultDNSConfig().String(), "DNS resolver configuration. Possible ttl values are: 'inf' "+ "for a persistent cache, '0' to disable the cache,\nor a positive duration, e.g. '1s', '1m', etc. "+ "Milliseconds are assumed if no unit is provided.\n"+ - "Possible select values to return a single IP are: 'first', 'random' or 'round-robin'.\n"+ + "Possible select values to return a single IP are: 'first', 'random' or 'roundRobin'.\n"+ "Possible policy values are: 'preferIPv4', 'preferIPv6', 'onlyIPv4', 'onlyIPv6' or 'any'.\n") return flags } diff --git a/core/local/local_test.go b/core/local/local_test.go index cb5c3c951e4..7b958df9028 100644 --- a/core/local/local_test.go +++ b/core/local/local_test.go @@ -1007,21 +1007,21 @@ func TestDNSResolver(t *testing.T) { "0": { // cache is disabled, every request does a DNS lookup lib.Options{DNS: types.DNSConfig{ TTL: null.StringFrom("0"), - Select: types.NullDNSSelect{DNSSelect: types.DNSFirst, Valid: true}, + Select: types.NullDNSSelect{DNSSelect: types.DNSfirst, Valid: true}, Policy: types.NullDNSPolicy{DNSPolicy: types.DNSpreferIPv4, Valid: false}, }}, 5, }, "1000": { // cache IPs for 1s, check that unitless values are interpreted as ms lib.Options{DNS: types.DNSConfig{ TTL: null.StringFrom("1000"), - Select: types.NullDNSSelect{DNSSelect: types.DNSFirst, Valid: true}, + Select: types.NullDNSSelect{DNSSelect: types.DNSfirst, Valid: true}, Policy: types.NullDNSPolicy{DNSPolicy: types.DNSpreferIPv4, Valid: false}, }}, 4, }, "3s": { lib.Options{DNS: types.DNSConfig{ TTL: null.StringFrom("3s"), - Select: types.NullDNSSelect{DNSSelect: types.DNSFirst, Valid: true}, + Select: types.NullDNSSelect{DNSSelect: types.DNSfirst, Valid: true}, Policy: types.NullDNSPolicy{DNSPolicy: types.DNSpreferIPv4, Valid: false}, }}, 3, }, @@ -1204,7 +1204,7 @@ func TestRealTimeAndSetupTeardownMetrics(t *testing.T) { expTags = append(expTags, addExpTags...) return netext.NewDialer( net.Dialer{}, - netext.NewResolver(net.LookupIP, 0, types.DNSFirst, types.DNSpreferIPv4), + netext.NewResolver(net.LookupIP, 0, types.DNSfirst, types.DNSpreferIPv4), ).GetTrail(time.Now(), time.Now(), true, emitIterations, getTags(expTags...)) } diff --git a/js/initcontext_test.go b/js/initcontext_test.go index c9010d63691..768aee5e960 100644 --- a/js/initcontext_test.go +++ b/js/initcontext_test.go @@ -395,7 +395,7 @@ func TestRequestWithBinaryFile(t *testing.T) { KeepAlive: 60 * time.Second, DualStack: true, }, - netext.NewResolver(net.LookupIP, 0, types.DNSFirst, types.DNSpreferIPv4), + netext.NewResolver(net.LookupIP, 0, types.DNSfirst, types.DNSpreferIPv4), )).DialContext, }, BPool: bpool.NewBufferPool(1), diff --git a/lib/netext/httpext/tracer_test.go b/lib/netext/httpext/tracer_test.go index a5231983909..fb97b8ea3c0 100644 --- a/lib/netext/httpext/tracer_test.go +++ b/lib/netext/httpext/tracer_test.go @@ -58,7 +58,7 @@ func TestTracer(t *testing.T) { assert.True(t, ok) transport.DialContext = netext.NewDialer( net.Dialer{}, - netext.NewResolver(net.LookupIP, 0, types.DNSFirst, types.DNSpreferIPv4), + netext.NewResolver(net.LookupIP, 0, types.DNSfirst, types.DNSpreferIPv4), ).DialContext var prev int64 diff --git a/lib/netext/resolver.go b/lib/netext/resolver.go index 0c8ca6bebdf..25795a2ef40 100644 --- a/lib/netext/resolver.go +++ b/lib/netext/resolver.go @@ -131,16 +131,16 @@ func (r *resolver) selectOne(host string, ips []net.IP) net.IP { var ip net.IP switch r.selectIndex { - case types.DNSFirst: + case types.DNSfirst: return ips[0] - case types.DNSRoundRobin: + case types.DNSroundRobin: r.rrm.Lock() // NOTE: This index approach is not stable and might result in returning // repeated or skipped IPs if the records change during a test run. ip = ips[int(r.roundRobin[host])%len(ips)] r.roundRobin[host]++ r.rrm.Unlock() - case types.DNSRandom: + case types.DNSrandom: r.rrm.Lock() ip = ips[r.rand.Intn(len(ips))] r.rrm.Unlock() diff --git a/lib/netext/resolver_test.go b/lib/netext/resolver_test.go index 2fa5d1b8d29..4ec75c2f87c 100644 --- a/lib/netext/resolver_test.go +++ b/lib/netext/resolver_test.go @@ -56,24 +56,24 @@ func TestResolver(t *testing.T) { expIP []net.IP }{ { - 0, types.DNSFirst, types.DNSpreferIPv4, + 0, types.DNSfirst, types.DNSpreferIPv4, []net.IP{net.ParseIP("127.0.0.10")}, }, { - time.Second, types.DNSFirst, types.DNSpreferIPv4, + time.Second, types.DNSfirst, types.DNSpreferIPv4, []net.IP{net.ParseIP("127.0.0.10")}, }, - {0, types.DNSRoundRobin, types.DNSonlyIPv6, []net.IP{ + {0, types.DNSroundRobin, types.DNSonlyIPv6, []net.IP{ net.ParseIP("2001:db8::10"), net.ParseIP("2001:db8::11"), net.ParseIP("2001:db8::12"), net.ParseIP("2001:db8::10"), }}, { - 0, types.DNSFirst, types.DNSpreferIPv6, + 0, types.DNSfirst, types.DNSpreferIPv6, []net.IP{net.ParseIP("2001:db8::10")}, }, - {0, types.DNSRoundRobin, types.DNSpreferIPv4, []net.IP{ + {0, types.DNSroundRobin, types.DNSpreferIPv4, []net.IP{ net.ParseIP("127.0.0.10"), net.ParseIP("127.0.0.11"), net.ParseIP("127.0.0.12"), @@ -101,7 +101,7 @@ func TestResolver(t *testing.T) { assert.True(t, cr.cache[host].lastLookup.After(firstLookup)) } - if tc.sel == types.DNSRoundRobin { + if tc.sel == types.DNSroundRobin { ips := []net.IP{ip} for i := 0; i < 3; i++ { ip, err = r.LookupIP(host) diff --git a/lib/testutils/httpmultibin/httpmultibin.go b/lib/testutils/httpmultibin/httpmultibin.go index a2c165c8b30..71c2cb93768 100644 --- a/lib/testutils/httpmultibin/httpmultibin.go +++ b/lib/testutils/httpmultibin/httpmultibin.go @@ -254,7 +254,7 @@ func NewHTTPMultiBin(t testing.TB) *HTTPMultiBin { Timeout: 2 * time.Second, KeepAlive: 10 * time.Second, DualStack: true, - }, netext.NewResolver(net.LookupIP, 0, types.DNSFirst, types.DNSpreferIPv4)) + }, netext.NewResolver(net.LookupIP, 0, types.DNSfirst, types.DNSpreferIPv4)) dialer.Hosts = map[string]*lib.HostAddress{ httpDomain: httpDomainValue, httpsDomain: httpsDomainValue, diff --git a/lib/types/dns.go b/lib/types/dns.go index 244931c0149..adcca440c5f 100644 --- a/lib/types/dns.go +++ b/lib/types/dns.go @@ -47,7 +47,7 @@ type DNSConfig struct { func DefaultDNSConfig() DNSConfig { return DNSConfig{ TTL: null.NewString("5m", false), - Select: NullDNSSelect{DNSRandom, false}, + Select: NullDNSSelect{DNSrandom, false}, Policy: NullDNSPolicy{DNSpreferIPv4, false}, } } @@ -122,16 +122,18 @@ func (d NullDNSPolicy) MarshalJSON() ([]byte, error) { // DNSSelect is the strategy to use when picking a single IP if more than one // is returned for a host name. -//go:generate enumer -type=DNSSelect -transform=kebab -trimprefix DNS -output dns_select_gen.go +//go:generate enumer -type=DNSSelect -trimprefix DNS -output dns_select_gen.go type DNSSelect uint8 +// These are lower camel cased since enumer doesn't support it as a transform option. +// See https://github.com/alvaroloes/enumer/pull/60 . const ( - // DNSFirst returns the first IP from the response. - DNSFirst DNSSelect = iota + 1 - // DNSRoundRobin rotates the IP returned on each lookup. - DNSRoundRobin - // DNSRandom returns a random IP from the response. - DNSRandom + // DNSfirst returns the first IP from the response. + DNSfirst DNSSelect = iota + 1 + // DNSroundRobin rotates the IP returned on each lookup. + DNSroundRobin + // DNSrandom returns a random IP from the response. + DNSrandom ) // UnmarshalJSON converts JSON data to a valid DNSSelect diff --git a/lib/types/dns_select_gen.go b/lib/types/dns_select_gen.go index 88d893355e9..347aaec7a61 100644 --- a/lib/types/dns_select_gen.go +++ b/lib/types/dns_select_gen.go @@ -1,4 +1,4 @@ -// Code generated by "enumer -type=DNSSelect -transform=kebab -trimprefix DNS -output dns_select_gen.go"; DO NOT EDIT. +// Code generated by "enumer -type=DNSSelect -trimprefix DNS -output dns_select_gen.go"; DO NOT EDIT. // package types @@ -7,9 +7,9 @@ import ( "fmt" ) -const _DNSSelectName = "firstround-robinrandom" +const _DNSSelectName = "firstroundRobinrandom" -var _DNSSelectIndex = [...]uint8{0, 5, 16, 22} +var _DNSSelectIndex = [...]uint8{0, 5, 15, 21} func (i DNSSelect) String() string { i -= 1 @@ -23,8 +23,8 @@ var _DNSSelectValues = []DNSSelect{1, 2, 3} var _DNSSelectNameToValueMap = map[string]DNSSelect{ _DNSSelectName[0:5]: 1, - _DNSSelectName[5:16]: 2, - _DNSSelectName[16:22]: 3, + _DNSSelectName[5:15]: 2, + _DNSSelectName[15:21]: 3, } // DNSSelectString retrieves an enum value from the enum constants string name. From 6cb1a2309a79b7fdbb1a42d9eb390e22c092a868 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ivan=20Miri=C4=87?= Date: Thu, 15 Oct 2020 12:35:29 +0200 Subject: [PATCH 6/8] Treat unitless duration values as milliseconds Resolves https://github.com/loadimpact/k6/pull/1612#discussion_r504758218 --- js/runner.go | 9 ++------- lib/types/types.go | 10 ++++++++++ 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/js/runner.go b/js/runner.go index 03859ef7822..370ebf17790 100644 --- a/js/runner.go +++ b/js/runner.go @@ -373,15 +373,10 @@ func parseTTL(ttlS string) (time.Duration, error) { ttlS = types.DefaultDNSConfig().TTL.String fallthrough default: - origTTLs := ttlS - // Treat unitless values as milliseconds - if t, err := strconv.ParseFloat(ttlS, 32); err == nil { - ttlS = fmt.Sprintf("%.2fms", t) - } var err error - ttl, err = types.ParseExtendedDuration(ttlS) + ttl, err = types.ParseExtendedDurationMs(ttlS) if ttl < 0 || err != nil { - return ttl, fmt.Errorf("invalid DNS TTL: %s", origTTLs) + return ttl, fmt.Errorf("invalid DNS TTL: %s", ttlS) } } return ttl, nil diff --git a/lib/types/types.go b/lib/types/types.go index e9ffeab037f..03609850e59 100644 --- a/lib/types/types.go +++ b/lib/types/types.go @@ -123,6 +123,16 @@ func ParseExtendedDuration(data string) (result time.Duration, err error) { return time.Duration(days)*24*time.Hour + hours, nil } +// ParseExtendedDurationMs wraps ParseExtendedDuration while assuming +// millisecond values if data is provided with no units. +// TODO: Merge this into ParseExtendedDuration once it's safe to do so globally. +func ParseExtendedDurationMs(data string) (result time.Duration, err error) { + if t, errp := strconv.ParseFloat(data, 32); errp == nil { + data = fmt.Sprintf("%.2fms", t) + } + return ParseExtendedDuration(data) +} + // UnmarshalText converts text data to Duration func (d *Duration) UnmarshalText(data []byte) error { v, err := ParseExtendedDuration(string(data)) From 85e475419c1ef4fe15f5da936c927d7a050dd003 Mon Sep 17 00:00:00 2001 From: Mihail Stoykov Date: Tue, 20 Oct 2020 12:33:16 +0300 Subject: [PATCH 7/8] Update regexp2 to fix a rare panic --- go.mod | 2 +- go.sum | 10 ++-------- vendor/github.com/dlclark/regexp2/README.md | 18 ++++++++++++++++++ .../dlclark/regexp2/syntax/prefix.go | 2 +- vendor/modules.txt | 2 +- 5 files changed, 23 insertions(+), 11 deletions(-) diff --git a/go.mod b/go.mod index b35b88bb09a..e73cb8c6b08 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/andybalholm/brotli v0.0.0-20190704151324-71eb68cc467c github.com/andybalholm/cascadia v1.0.0 // indirect github.com/daaku/go.zipexe v0.0.0-20150329023125-a5fe2436ffcb // indirect - github.com/dlclark/regexp2 v1.4.0 // indirect + github.com/dlclark/regexp2 v1.4.1-0.20201013204808-346446b5b182 // indirect github.com/dop251/goja v0.0.0-20201007100345-a8e472c705eb github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4 github.com/eapache/go-resiliency v1.1.0 // indirect diff --git a/go.sum b/go.sum index b1301d267f7..b6c198e9005 100644 --- a/go.sum +++ b/go.sum @@ -20,8 +20,8 @@ github.com/daaku/go.zipexe v0.0.0-20150329023125-a5fe2436ffcb h1:tUf55Po0vzOendQ github.com/daaku/go.zipexe v0.0.0-20150329023125-a5fe2436ffcb/go.mod h1:U0vRfAucUOohvdCxt5MWLF+TePIL0xbCkbKIiV8TQCE= github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dlclark/regexp2 v1.4.0 h1:F1rxgk7p4uKjwIQxBs9oAXe5CqrXlCduYEJvrF4u93E= -github.com/dlclark/regexp2 v1.4.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= +github.com/dlclark/regexp2 v1.4.1-0.20201013204808-346446b5b182 h1:P/IEt6bRdU/7F+i9VvzOgtLVg9MWMe4UODEllD/nH3o= +github.com/dlclark/regexp2 v1.4.1-0.20201013204808-346446b5b182/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= github.com/dop251/goja v0.0.0-20201007100345-a8e472c705eb h1:358PKppXCE79d1HvEQWYtc5Moihb1pURjNhPZkouSPM= github.com/dop251/goja v0.0.0-20201007100345-a8e472c705eb/go.mod h1:Mw6PkjjMXWbTj+nnj4s3QPXq1jaT0s5pC0iFD4+BOAA= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4 h1:qk/FSDDxo05wdJH28W+p5yivv7LuLYLRXPPD8KQCtZs= @@ -133,16 +133,10 @@ github.com/spf13/pflag v1.0.1 h1:aCvUg6QPl3ibpQUxyLkrEkCHtPqYJL4x9AuhqVqFis4= github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/stretchr/testify v1.2.1 h1:52QO5WkIUcHGIR7EnGagH88x1bUzqGXTC5/1bDTUQ7U= github.com/stretchr/testify v1.2.1/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/tidwall/gjson v1.1.3 h1:u4mspaByxY+Qk4U1QYYVzGFI8qxN/3jtEV0ZDb2vRic= -github.com/tidwall/gjson v1.1.3/go.mod h1:c/nTNbUr0E0OrXEhq1pwa8iEgc2DOt4ZZqAt1HtCkPA= github.com/tidwall/gjson v1.6.1 h1:LRbvNuNuvAiISWg6gxLEFuCe72UKy5hDqhxW/8183ws= github.com/tidwall/gjson v1.6.1/go.mod h1:BaHyNc5bjzYkPqgLq7mdVzeiRtULKULXLgZFKsxEHI0= -github.com/tidwall/match v1.0.0 h1:Ym1EcFkp+UQ4ptxfWlW+iMdq5cPH5nEuGzdf/Pb7VmI= -github.com/tidwall/match v1.0.0/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E= github.com/tidwall/match v1.0.1 h1:PnKP62LPNxHKTwvHHZZzdOAOCtsJTjo6dZLCwpKm5xc= github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E= -github.com/tidwall/pretty v0.0.0-20180105212114-65a9db5fad51 h1:BP2bjP495BBPaBcS5rmqviTfrOkN5rO5ceKAMRZCRFc= -github.com/tidwall/pretty v0.0.0-20180105212114-65a9db5fad51/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tidwall/pretty v1.0.2 h1:Z7S3cePv9Jwm1KwS0513MRaoUe3S01WPbLNV40pwWZU= github.com/tidwall/pretty v1.0.2/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/ugorji/go v0.0.0-20180112141927-9831f2c3ac10 h1:4zp+5ElNBLy5qmaDFrbVDolQSOtPmquw+W6EMNEpi+k= diff --git a/vendor/github.com/dlclark/regexp2/README.md b/vendor/github.com/dlclark/regexp2/README.md index 4e4abb4c6f5..f92f8b10bee 100644 --- a/vendor/github.com/dlclark/regexp2/README.md +++ b/vendor/github.com/dlclark/regexp2/README.md @@ -39,6 +39,24 @@ Group 0 is embedded in the Match. Group 0 is an automatically-assigned group th The __last__ capture is embedded in each group, so `g.String()` will return the same thing as `g.Capture.String()` and `g.Captures[len(g.Captures)-1].String()`. +If you want to find multiple matches from a single input string you should use the `FindNextMatch` method. For example, to implement a function similar to `regexp.FindAllString`: + +```go +func regexp2FindAllString(re *regexp2.Regexp, s string) []string { + var matches []string + m, _ := re.FindStringMatch(s) + for m != nil { + matches = append(matches, m.String()) + m, _ = re.FindNextMatch(m) + } + return matches +} +``` + +`FindNextMatch` is optmized so that it re-uses the underlying string/rune slice. + +The internals of `regexp2` always operate on `[]rune` so `Index` and `Length` data in a `Match` always reference a position in `rune`s rather than `byte`s (even if the input was given as a string). This is a dramatic difference between `regexp` and `regexp2`. It's advisable to use the provided `String()` methods to avoid having to work with indices. + ## Compare `regexp` and `regexp2` | Category | regexp | regexp2 | | --- | --- | --- | diff --git a/vendor/github.com/dlclark/regexp2/syntax/prefix.go b/vendor/github.com/dlclark/regexp2/syntax/prefix.go index 011ef0b4168..f6716886293 100644 --- a/vendor/github.com/dlclark/regexp2/syntax/prefix.go +++ b/vendor/github.com/dlclark/regexp2/syntax/prefix.go @@ -712,7 +712,7 @@ func (b *BmPrefix) Scan(text []rune, index, beglimit, endlimit int) int { if chTest != b.pattern[match] { advance = b.positive[match] - if (chTest & 0xFF80) == 0 { + if chTest < 128 { test2 = (match - startmatch) + b.negativeASCII[chTest] } else if chTest < 0xffff && len(b.negativeUnicode) > 0 { unicodeLookup = b.negativeUnicode[chTest>>8] diff --git a/vendor/modules.txt b/vendor/modules.txt index b1088edaf6b..b14028b943f 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -30,7 +30,7 @@ github.com/andybalholm/cascadia github.com/daaku/go.zipexe # github.com/davecgh/go-spew v1.1.1 => github.com/davecgh/go-spew v1.1.0 github.com/davecgh/go-spew/spew -# github.com/dlclark/regexp2 v1.4.0 +# github.com/dlclark/regexp2 v1.4.1-0.20201013204808-346446b5b182 ## explicit github.com/dlclark/regexp2 github.com/dlclark/regexp2/syntax From 300f6c85d79809f3e8e80332cb73adc4e90a1b7c Mon Sep 17 00:00:00 2001 From: Mihail Stoykov Date: Tue, 20 Oct 2020 12:35:09 +0300 Subject: [PATCH 8/8] Update goja to fix rare regexp panic --- go.mod | 2 +- go.sum | 4 ++-- vendor/github.com/dop251/goja/regexp.go | 4 +++- vendor/modules.txt | 2 +- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index e73cb8c6b08..ea75b32bda1 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/andybalholm/cascadia v1.0.0 // indirect github.com/daaku/go.zipexe v0.0.0-20150329023125-a5fe2436ffcb // indirect github.com/dlclark/regexp2 v1.4.1-0.20201013204808-346446b5b182 // indirect - github.com/dop251/goja v0.0.0-20201007100345-a8e472c705eb + github.com/dop251/goja v0.0.0-20201008094107-f97e50db25ec github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4 github.com/eapache/go-resiliency v1.1.0 // indirect github.com/eapache/go-xerial-snappy v0.0.0-20160609142408-bb955e01b934 // indirect diff --git a/go.sum b/go.sum index b6c198e9005..891dbf73e69 100644 --- a/go.sum +++ b/go.sum @@ -22,8 +22,8 @@ github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dlclark/regexp2 v1.4.1-0.20201013204808-346446b5b182 h1:P/IEt6bRdU/7F+i9VvzOgtLVg9MWMe4UODEllD/nH3o= github.com/dlclark/regexp2 v1.4.1-0.20201013204808-346446b5b182/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= -github.com/dop251/goja v0.0.0-20201007100345-a8e472c705eb h1:358PKppXCE79d1HvEQWYtc5Moihb1pURjNhPZkouSPM= -github.com/dop251/goja v0.0.0-20201007100345-a8e472c705eb/go.mod h1:Mw6PkjjMXWbTj+nnj4s3QPXq1jaT0s5pC0iFD4+BOAA= +github.com/dop251/goja v0.0.0-20201008094107-f97e50db25ec h1:I7aPaxeou6Da490jBeG2DuOa6mgj6N+uXi0AvidUMJc= +github.com/dop251/goja v0.0.0-20201008094107-f97e50db25ec/go.mod h1:Mw6PkjjMXWbTj+nnj4s3QPXq1jaT0s5pC0iFD4+BOAA= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4 h1:qk/FSDDxo05wdJH28W+p5yivv7LuLYLRXPPD8KQCtZs= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/eapache/go-resiliency v1.1.0 h1:1NtRmCAqadE2FN4ZcN6g90TP3uk8cg9rn9eNK2197aU= diff --git a/vendor/github.com/dop251/goja/regexp.go b/vendor/github.com/dop251/goja/regexp.go index e8fd7d17a19..2d020aa6821 100644 --- a/vendor/github.com/dop251/goja/regexp.go +++ b/vendor/github.com/dop251/goja/regexp.go @@ -467,7 +467,9 @@ func (r *regexpWrapper) findSubmatchIndexUnicode(s unicodeString, fullUnicode bo posMap, runes, _, _ := buildPosMap(&lenientUtf16Decoder{utf16Reader: s.utf16Reader(0)}, s.length(), 0) res := wrapped.FindReaderSubmatchIndex(&arrayRuneReader{runes: runes}) for i, item := range res { - res[i] = posMap[item] + if item >= 0 { + res[i] = posMap[item] + } } return res } diff --git a/vendor/modules.txt b/vendor/modules.txt index b14028b943f..77d290f921e 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -34,7 +34,7 @@ github.com/davecgh/go-spew/spew ## explicit github.com/dlclark/regexp2 github.com/dlclark/regexp2/syntax -# github.com/dop251/goja v0.0.0-20201007100345-a8e472c705eb +# github.com/dop251/goja v0.0.0-20201008094107-f97e50db25ec ## explicit github.com/dop251/goja github.com/dop251/goja/ast