slack-15.0: support v17+ --queryserver-config-* duration values (#…

…492)

* WIP

Signed-off-by: Tim Vaillancourt <tim@timvaillancourt.com>

* update all flags

Signed-off-by: Tim Vaillancourt <tim@timvaillancourt.com>

* update e2e flag test

Signed-off-by: Tim Vaillancourt <tim@timvaillancourt.com>

* 2 missing flags

Signed-off-by: Tim Vaillancourt <tim@timvaillancourt.com>

* rename Type()

Signed-off-by: Tim Vaillancourt <tim@timvaillancourt.com>

* empty commit to test ci

Signed-off-by: Tim Vaillancourt <tim@timvaillancourt.com>

* fakesqldb: Guard query log usage with lock (vitessio#12813)

* fakesqldb: Guard query log usage with lock

This lock is used around adding to the query log, but it means we also
need to use the lock when reading from it or when resetting it.

Signed-off-by: Dirkjan Bussink <d.bussink@gmail.com>

* grpcvtctldserver: Fix alias for loop reuse

Signed-off-by: Dirkjan Bussink <d.bussink@gmail.com>

* unit_race: Increase runtime

We've been adding a lot of tests for the evalengine and we're bumping up
to the timeout.

Signed-off-by: Dirkjan Bussink <d.bussink@gmail.com>

---------

Signed-off-by: Dirkjan Bussink <d.bussink@gmail.com>

* remove t.Parallel() like upstream did

Signed-off-by: Tim Vaillancourt <tim@timvaillancourt.com>

---------

Signed-off-by: Tim Vaillancourt <tim@timvaillancourt.com>
Signed-off-by: Dirkjan Bussink <d.bussink@gmail.com>
Co-authored-by: Dirkjan Bussink <d.bussink@gmail.com>

.github/workflows/unit_race.yml

@@ -90,7 +90,7 @@ jobs:
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.unit_tests == 'true'
       uses: nick-fields/retry@v2
       with:
-        timeout_minutes: 30
+        timeout_minutes: 45
         max_attempts: 3
         retry_on: error
         command: |

go/flagutil/flagutil.go

@@ -268,3 +268,24 @@ func (v *DurationOrIntVar) Type() string { return "duration" }
 
 // Value returns the underlying Duration value passed to the flag.
 func (v *DurationOrIntVar) Value() time.Duration { return v.val }
+
+type DurationOrSecondsFloatFlag float64
+
+func (set *DurationOrSecondsFloatFlag) Set(s string) error {
+	if dur, err := time.ParseDuration(s); err == nil {
+		*set = DurationOrSecondsFloatFlag(dur.Seconds())
+	} else {
+		f, err := strconv.ParseFloat(s, 64)
+		if err != nil {
+			return err
+		}
+		*set = DurationOrSecondsFloatFlag(f)
+	}
+	return nil
+}
+
+func (set *DurationOrSecondsFloatFlag) String() string {
+	return strconv.FormatFloat(float64(*set), 'f', -1, 64)
+}
+
+func (set *DurationOrSecondsFloatFlag) Type() string { return "DurationOrSecondsFloat" }

go/flagutil/flagutil_test.go

@@ -159,3 +159,49 @@ func TestDurationOrIntVar(t *testing.T) {
 		assert.Equal(t, tt.want, flag.Value())
 	}
 }
+
+func TestDurationOrSecondsFloatFlag(t *testing.T) {
+	testCases := []struct {
+		Set         string
+		Expected    float64
+		ExpectedErr string
+	}{
+		{
+			Set:      "1",
+			Expected: 1,
+		},
+		{
+			Set:      "0.5",
+			Expected: 0.5,
+		},
+		{
+			Set:      "1800",
+			Expected: 1800,
+		},
+		{
+			Set:      "50ms",
+			Expected: 0.05,
+		},
+		{
+			Set:      "42m",
+			Expected: 2520,
+		},
+		{
+			Set:         "wont-parse",
+			ExpectedErr: `strconv.ParseFloat: parsing "wont-parse": invalid syntax`,
+		},
+	}
+
+	for _, testCase := range testCases {
+		testCase := testCase
+		t.Run(testCase.Set, func(t *testing.T) {
+			t.Parallel()
+			var f DurationOrSecondsFloatFlag
+			err := f.Set(testCase.Set)
+			if testCase.ExpectedErr != "" {
+				assert.ErrorContains(t, err, testCase.ExpectedErr)
+			}
+			assert.Equal(t, testCase.Expected, float64(f))
+		})
+	}
+}

go/mysql/fakesqldb/server.go

@@ -613,11 +613,15 @@ func (db *DB) GetQueryCalledNum(query string) int {
 
 // QueryLog returns the query log in a semicomma separated string
 func (db *DB) QueryLog() string {
+	db.mu.Lock()
+	defer db.mu.Unlock()
 	return strings.Join(db.querylog, ";")
 }
 
 // ResetQueryLog resets the query log
 func (db *DB) ResetQueryLog() {
+	db.mu.Lock()
+	defer db.mu.Unlock()
 	db.querylog = nil
 }
 

go/vt/vtctl/grpcvtctldserver/server.go

@@ -4168,7 +4168,9 @@ func (s *VtctldServer) ValidateVersionShard(ctx context.Context, req *vtctldatap
 		}
 
 		wg.Add(1)
-		go s.diffVersion(ctx, primaryVersion.Version, shard.PrimaryAlias, alias, &wg, &er)
+		go func(alias *topodatapb.TabletAlias) {
+			s.diffVersion(ctx, primaryVersion.Version, shard.PrimaryAlias, alias, &wg, &er)
+		}(alias)
 	}
 
 	wg.Wait()

go/vt/vtctl/grpcvtctldserver/server_test.go

@@ -2705,7 +2705,7 @@ func TestDeleteShards(t *testing.T) {
 				defer func() {
 					topofactory.SetError(nil)
 
-					actualShards := []*vtctldatapb.Shard{}
+					var actualShards []*vtctldatapb.Shard
 
 					keyspaces, err := ts.GetKeyspaces(ctx)
 					require.NoError(t, err, "cannot get keyspace names to check remaining shards")
@@ -11372,8 +11372,6 @@ func TestValidateVersionShard(t *testing.T) {
 	for _, tt := range tests {
 		curT := tt
 		t.Run(tt.name, func(t *testing.T) {
-			t.Parallel()
-
 			curT.setup(&testSetupMu)
 			resp, err := vtctld.ValidateVersionShard(ctx, curT.req)
 			if curT.shouldErr {

go/vt/vttablet/tabletserver/tabletenv/config.go

@@ -128,7 +128,7 @@ func registerTabletEnvFlags(fs *pflag.FlagSet) {
 	fs.IntVar(&currentConfig.TxPool.PrefillParallelism, "queryserver-config-transaction-prefill-parallelism", defaultConfig.TxPool.PrefillParallelism, "Query server transaction prefill parallelism, a non-zero value will prefill the pool using the specified parallism.")
 	_ = fs.MarkDeprecated("queryserver-config-transaction-prefill-parallelism", "it will be removed in a future release.")
 	fs.IntVar(&currentConfig.MessagePostponeParallelism, "queryserver-config-message-postpone-cap", defaultConfig.MessagePostponeParallelism, "query server message postpone cap is the maximum number of messages that can be postponed at any given time. Set this number to substantially lower than transaction cap, so that the transaction pool isn't exhausted by the message subsystem.")
-	SecondsVar(fs, &currentConfig.Oltp.TxTimeoutSeconds, "queryserver-config-transaction-timeout", defaultConfig.Oltp.TxTimeoutSeconds, "query server transaction timeout (in seconds), a transaction will be killed if it takes longer than this value")
+	fs.Var((*flagutil.DurationOrSecondsFloatFlag)(&currentConfig.Oltp.TxTimeoutSeconds), "queryserver-config-transaction-timeout", "query server transaction timeout (in seconds), a transaction will be killed if it takes longer than this value")
 	SecondsVar(fs, &currentConfig.GracePeriods.ShutdownSeconds, "shutdown_grace_period", defaultConfig.GracePeriods.ShutdownSeconds, "how long to wait (in seconds) for queries and transactions to complete during graceful shutdown.")
 	fs.IntVar(&currentConfig.Oltp.MaxRows, "queryserver-config-max-result-size", defaultConfig.Oltp.MaxRows, "query server max result size, maximum number of rows allowed to return from vttablet for non-streaming queries.")
 	fs.IntVar(&currentConfig.Oltp.WarnRows, "queryserver-config-warn-result-size", defaultConfig.Oltp.WarnRows, "query server result size warning threshold, warn if number of rows returned from vttablet for non-streaming queries exceeds this")
@@ -138,15 +138,15 @@ func registerTabletEnvFlags(fs *pflag.FlagSet) {
 	fs.IntVar(&currentConfig.QueryCacheSize, "queryserver-config-query-cache-size", defaultConfig.QueryCacheSize, "query server query cache size, maximum number of queries to be cached. vttablet analyzes every incoming query and generate a query plan, these plans are being cached in a lru cache. This config controls the capacity of the lru cache.")
 	fs.Int64Var(&currentConfig.QueryCacheMemory, "queryserver-config-query-cache-memory", defaultConfig.QueryCacheMemory, "query server query cache size in bytes, maximum amount of memory to be used for caching. vttablet analyzes every incoming query and generate a query plan, these plans are being cached in a lru cache. This config controls the capacity of the lru cache.")
 	fs.BoolVar(&currentConfig.QueryCacheLFU, "queryserver-config-query-cache-lfu", defaultConfig.QueryCacheLFU, "query server cache algorithm. when set to true, a new cache algorithm based on a TinyLFU admission policy will be used to improve cache behavior and prevent pollution from sparse queries")
-	SecondsVar(fs, &currentConfig.SchemaReloadIntervalSeconds, "queryserver-config-schema-reload-time", defaultConfig.SchemaReloadIntervalSeconds, "query server schema reload time, how often vttablet reloads schemas from underlying MySQL instance in seconds. vttablet keeps table schemas in its own memory and periodically refreshes it from MySQL. This config controls the reload time.")
-	SecondsVar(fs, &currentConfig.SignalSchemaChangeReloadIntervalSeconds, "queryserver-config-schema-change-signal-interval", defaultConfig.SignalSchemaChangeReloadIntervalSeconds, "query server schema change signal interval defines at which interval the query server shall send schema updates to vtgate.")
+	fs.Var((*flagutil.DurationOrSecondsFloatFlag)(&currentConfig.SchemaReloadIntervalSeconds), "queryserver-config-schema-reload-time", "query server schema reload time, how often vttablet reloads schemas from underlying MySQL instance in seconds. vttablet keeps table schemas in its own memory and periodically refreshes it from MySQL. This config controls the reload time.")
+	fs.Var((*flagutil.DurationOrSecondsFloatFlag)(&currentConfig.SignalSchemaChangeReloadIntervalSeconds), "queryserver-config-schema-change-signal-interval", "query server schema change signal interval defines at which interval the query server shall send schema updates to vtgate.")
 	fs.BoolVar(&currentConfig.SignalWhenSchemaChange, "queryserver-config-schema-change-signal", defaultConfig.SignalWhenSchemaChange, "query server schema signal, will signal connected vtgates that schema has changed whenever this is detected. VTGates will need to have -schema_change_signal enabled for this to work")
-	SecondsVar(fs, &currentConfig.Olap.TxTimeoutSeconds, "queryserver-config-olap-transaction-timeout", defaultConfig.Olap.TxTimeoutSeconds, "query server transaction timeout (in seconds), after which a transaction in an OLAP session will be killed")
-	SecondsVar(fs, &currentConfig.Oltp.QueryTimeoutSeconds, "queryserver-config-query-timeout", defaultConfig.Oltp.QueryTimeoutSeconds, "query server query timeout (in seconds), this is the query timeout in vttablet side. If a query takes more than this timeout, it will be killed.")
-	SecondsVar(fs, &currentConfig.OltpReadPool.TimeoutSeconds, "queryserver-config-query-pool-timeout", defaultConfig.OltpReadPool.TimeoutSeconds, "query server query pool timeout (in seconds), it is how long vttablet waits for a connection from the query pool. If set to 0 (default) then the overall query timeout is used instead.")
-	SecondsVar(fs, &currentConfig.OlapReadPool.TimeoutSeconds, "queryserver-config-stream-pool-timeout", defaultConfig.OlapReadPool.TimeoutSeconds, "query server stream pool timeout (in seconds), it is how long vttablet waits for a connection from the stream pool. If set to 0 (default) then there is no timeout.")
-	SecondsVar(fs, &currentConfig.TxPool.TimeoutSeconds, "queryserver-config-txpool-timeout", defaultConfig.TxPool.TimeoutSeconds, "query server transaction pool timeout, it is how long vttablet waits if tx pool is full")
-	SecondsVar(fs, &currentConfig.OltpReadPool.IdleTimeoutSeconds, "queryserver-config-idle-timeout", defaultConfig.OltpReadPool.IdleTimeoutSeconds, "query server idle timeout (in seconds), vttablet manages various mysql connection pools. This config means if a connection has not been used in given idle timeout, this connection will be removed from pool. This effectively manages number of connection objects and optimize the pool performance.")
+	fs.Var((*flagutil.DurationOrSecondsFloatFlag)(&currentConfig.Olap.TxTimeoutSeconds), "queryserver-config-olap-transaction-timeout", "query server transaction timeout (in seconds), after which a transaction in an OLAP session will be killed")
+	fs.Var((*flagutil.DurationOrSecondsFloatFlag)(&currentConfig.Oltp.QueryTimeoutSeconds), "queryserver-config-query-timeout", "query server query timeout (in seconds), this is the query timeout in vttablet side. If a query takes more than this timeout, it will be killed.")
+	fs.Var((*flagutil.DurationOrSecondsFloatFlag)(&currentConfig.OltpReadPool.TimeoutSeconds), "queryserver-config-query-pool-timeout", "query server query pool timeout (in seconds), it is how long vttablet waits for a connection from the query pool. If set to 0 (default) then the overall query timeout is used instead.")
+	fs.Var((*flagutil.DurationOrSecondsFloatFlag)(&currentConfig.OlapReadPool.TimeoutSeconds), "queryserver-config-stream-pool-timeout", "query server stream pool timeout (in seconds), it is how long vttablet waits for a connection from the stream pool. If set to 0 (default) then there is no timeout.")
+	fs.Var((*flagutil.DurationOrSecondsFloatFlag)(&currentConfig.TxPool.TimeoutSeconds), "queryserver-config-txpool-timeout", "query server transaction pool timeout, it is how long vttablet waits if tx pool is full")
+	fs.Var((*flagutil.DurationOrSecondsFloatFlag)(&currentConfig.OltpReadPool.IdleTimeoutSeconds), "queryserver-config-idle-timeout", "query server idle timeout (in seconds), vttablet manages various mysql connection pools. This config means if a connection has not been used in given idle timeout, this connection will be removed from pool. This effectively manages number of connection objects and optimize the pool performance.")
 	fs.IntVar(&currentConfig.OltpReadPool.MaxWaiters, "queryserver-config-query-pool-waiter-cap", defaultConfig.OltpReadPool.MaxWaiters, "query server query pool waiter limit, this is the maximum number of queries that can be queued waiting to get a connection")
 	fs.IntVar(&currentConfig.OlapReadPool.MaxWaiters, "queryserver-config-stream-pool-waiter-cap", defaultConfig.OlapReadPool.MaxWaiters, "query server stream pool waiter limit, this is the maximum number of streaming queries that can be queued waiting to get a connection")
 	fs.IntVar(&currentConfig.TxPool.MaxWaiters, "queryserver-config-txpool-waiter-cap", defaultConfig.TxPool.MaxWaiters, "query server transaction pool waiter limit, this is the maximum number of transactions that can be queued waiting to get a connection")