diff --git a/.github/workflows/ci-sysbench-runner-tests.yaml b/.github/workflows/ci-sysbench-runner-tests.yaml index ea1ec9e5a1c..0b2535f8641 100644 --- a/.github/workflows/ci-sysbench-runner-tests.yaml +++ b/.github/workflows/ci-sysbench-runner-tests.yaml @@ -1,24 +1,24 @@ -name: Test Sysbench Runner Utility Works - -on: - pull_request: - branches: [ main ] - paths: - - 'go/**' - - 'integration-tests/**' - -concurrency: - group: ci-sysbench-runner-tests-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -jobs: - mysql_client_integrations_job: - runs-on: ubuntu-22.04 - name: Test Sysbench Runner - steps: - - name: Checkout - uses: actions/checkout@v3 - - name: Copy Dockerfile - run: cp -r ./go/performance/continuous_integration/. . - - name: Test sysbench runner - uses: ./.github/actions/sysbench-runner-tests +#name: Test Sysbench Runner Utility Works +# +#on: +# pull_request: +# branches: [ main ] +# paths: +# - 'go/**' +# - 'integration-tests/**' +# +#concurrency: +# group: ci-sysbench-runner-tests-${{ github.event.pull_request.number || github.ref }} +# cancel-in-progress: true +# +#jobs: +# mysql_client_integrations_job: +# runs-on: ubuntu-22.04 +# name: Test Sysbench Runner +# steps: +# - name: Checkout +# uses: actions/checkout@v3 +# - name: Copy Dockerfile +# run: cp -r ./go/performance/continuous_integration/. . +# - name: Test sysbench runner +# uses: ./.github/actions/sysbench-runner-tests diff --git a/go/performance/utils/sysbench_runner/README.md b/go/performance/utils/benchmark_runner/README.md similarity index 70% rename from go/performance/utils/sysbench_runner/README.md rename to go/performance/utils/benchmark_runner/README.md index 73aa8192cee..5d47db7395c 100644 --- a/go/performance/utils/sysbench_runner/README.md +++ b/go/performance/utils/benchmark_runner/README.md @@ -1,11 +1,6 @@ Sysbench runner is a tool for running sysbench tests against sql servers. Custom sysbench lua scripts used for benchmarking Dolt are [here](https://github.com/dolthub/sysbench-lua-scripts). -The tool requires a json config file to run: -```bash -$ sysbench_runner --config=config.json -``` - Configuration: ```json @@ -62,8 +57,6 @@ oltp_update_non_index `Port` is the server port. Defaults to **3306** for `dolt` and `mysql` Servers. (**Optional**) -`Server` is the server. Only `dolt` and `mysql` are supported. (**Required**) - `Version` is the server version. (**Required**) `ResultsFormat` is the format the results should be written in. Only `json` and `csv` are supported. (**Required**) @@ -99,3 +92,33 @@ oltp_update_non_index `sysbench [options]... [testname] [command]` Note: Be sure that all mysql processes are off when running this locally. + +# TPCC + +TPCC runner is a tool for running TPCC tests against sql servers. These tests run against the +Percona Labs repo [here](https://github.com/Percona-Lab/sysbench-tpcc). + +Note to this run this locally you need to have the TPCC repo cloned. The `ScriptDir` variable should then be linked +to the path of the cloned repo. + +Configuration: + +```json +{ + "Servers": "[...]", + "ScriptDir":"/Users/vinairachakonda/go/src/dolthub/sysbench-tpcc", + "ScaleFactors": [1] +} +``` + +`Servers`: The server defintions to run the benchmark against. Accepts Dolt and MySQL configuratiosn. + +`ScriptDir`: The directory of the TPCC testing scripts + +`ScaleFactors`: The number of warehouse to be generated in the test case. + +`NomsBinFormat`: The NomsBinFormat to use for this benchmark. + +Note that this configuration is still incomplete for the amount of the variable TPCC varies. This intentional as we +want expose small amounts of independent variables until Dolt gets more robust. See `config.go` to get a breakdown of all the +variables TPCC varies. diff --git a/go/performance/utils/benchmark_runner/benchmark.go b/go/performance/utils/benchmark_runner/benchmark.go new file mode 100644 index 00000000000..a2556180742 --- /dev/null +++ b/go/performance/utils/benchmark_runner/benchmark.go @@ -0,0 +1,21 @@ +// Copyright 2019-2022 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package benchmark_runner + +import "context" + +type Benchmarker interface { + Benchmark(ctx context.Context) (Results, error) +} diff --git a/go/performance/utils/benchmark_runner/config.go b/go/performance/utils/benchmark_runner/config.go new file mode 100644 index 00000000000..477a5613c08 --- /dev/null +++ b/go/performance/utils/benchmark_runner/config.go @@ -0,0 +1,39 @@ +// Copyright 2019-2022 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package benchmark_runner + +import "context" + +type Config interface { + GetRuns() int + GetScriptDir() string + GetNomsBinFormat() string + GetRuntimeOs() string + GetRuntimeGoArch() string + GetServerConfigs() []ServerConfig + Validate(ctx context.Context) error + ContainsServerOfType(server ServerType) bool +} + +type SysbenchConfig interface { + Config + GetTestOptions() []string + GetTestConfigs() []TestConfig +} + +type TpccConfig interface { + Config + GetScaleFactors() []int +} diff --git a/go/performance/utils/benchmark_runner/constants.go b/go/performance/utils/benchmark_runner/constants.go new file mode 100644 index 00000000000..389882e4be3 --- /dev/null +++ b/go/performance/utils/benchmark_runner/constants.go @@ -0,0 +1,209 @@ +// Copyright 2019-2022 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package benchmark_runner + +import "time" + +const ( + Dolt ServerType = "dolt" + Doltgres ServerType = "doltgres" + Postgres ServerType = "postgres" + MySql ServerType = "mysql" + + CsvFormat = "csv" + JsonFormat = "json" + + CsvExt = ".csv" + JsonExt = ".json" + + CpuServerProfile ServerProfile = "cpu" + + defaultHost = "127.0.0.1" + defaultDoltPort = 3306 + defaultMysqlPort = defaultDoltPort + defaultDoltgresPort = 5432 + defaultPostgresPort = defaultDoltgresPort + + defaultMysqlSocket = "/var/run/mysqld/mysqld.sock" + + tcpProtocol = "tcp" + unixProtocol = "unix" + + sysbenchUsername = "sysbench" + sysbenchUserLocal = "'sysbench'@'localhost'" + sysbenchPassLocal = "sysbenchpass" + sysbenchDbPsModeFlag = "--db-ps-mode" + sysbenchDbPsModeDisable = "disable" + sysbenchRandTypeFlag = "--rand-type" + sysbenchRandTypeUniform = "uniform" + sysbenchMysqlDbFlag = "--mysql-db" + sysbenchDbDriverFlag = "--db-driver" + sysbenchMysqlHostFlag = "--mysql-host" + sysbenchMysqlPortFlag = "--mysql-port" + sysbenchMysqlUserFlag = "--mysql-user" + sysbenchMysqlPasswordFlag = "--mysql-password" + sysbenchPostgresDbDriver = "pgsql" + sysbenchPostgresDbFlag = "--pgsql-db" + sysbenchPostgresHostFlag = "--pgsql-host" + sysbenchPostgresPortFlag = "--pgsql-port" + sysbenchPostgresUserFlag = "--pgsql-user" + + doltSqlServerCommand = "sql-server" + + userFlag = "--user" + hostFlag = "--host" + portFlag = "--port" + skipBinLogFlag = "--skip-log-bin" + profileFlag = "--prof" + profilePathFlag = "--prof-path" + cpuProfile = "cpu" + doltgresDataDirFlag = "--data-dir" + MysqlDataDirFlag = "--datadir" + MysqlInitializeInsecureFlag = "--initialize-insecure" + cpuProfileFilename = "cpu.pprof" + + sysbenchOltpReadOnlyTestName = "oltp_read_only" + sysbenchOltpInsertTestName = "oltp_insert" + sysbenchBulkInsertTestName = "bulk_insert" + sysbenchOltpPointSelectTestName = "oltp_point_select" + sysbenchSelectRandomPointsTestName = "select_random_points" + sysbenchSelectRandomRangesTestName = "select_random_ranges" + sysbenchOltpWriteOnlyTestName = "oltp_write_only" + sysbenchOltpReadWriteTestName = "oltp_read_write" + sysbenchOltpUpdateIndexTestName = "oltp_update_index" + sysbenchOltpUpdateNonIndexTestName = "oltp_update_non_index" + + sysbenchCoveringIndexScanLuaTestName = "covering_index_scan.lua" + sysbenchGroupByScanLuaTestName = "groupby_scan.lua" + sysbenchIndexJoinLuaTestName = "index_join.lua" + sysbenchIndexJoinScanLuaTestName = "index_join_scan.lua" + sysbenchIndexScanLuaTestName = "index_scan.lua" + sysbenchOltpDeleteInsertLuaTestName = "oltp_delete_insert.lua" + sysbenchTableScanLuaTestName = "table_scan.lua" + sysbenchTypesDeleteInsertLuaTestName = "types_delete_insert.lua" + sysbenchTypesTableScanLuaTestName = "types_table_scan.lua" + + sysbenchCoveringIndexScanPostgresLuaTestName = "covering_index_scan_postgres.lua" + sysbenchGroupByScanPostgresLuaTestName = "groupby_scan_postgres.lua" + sysbenchIndexJoinPostgresLuaTestName = "index_join_postgres.lua" + sysbenchIndexJoinScanPostgresLuaTestName = "index_join_scan_postgres.lua" + sysbenchIndexScanPostgresLuaTestName = "index_scan_postgres.lua" + sysbenchOltpDeleteInsertPostgresLuaTestName = "oltp_delete_insert_postgres.lua" + sysbenchTableScanPostgresLuaTestName = "table_scan_postgres.lua" + sysbenchTypesDeleteInsertPostgresLuaTestName = "types_delete_insert_postgres.lua" + sysbenchTypesTableScanPostgresLuaTestName = "types_table_scan_postgres.lua" + + doltConfigUsernameKey = "user.name" + doltConfigEmailKey = "user.email" + doltBenchmarkUser = "benchmark" + doltBenchmarkEmail = "benchmark@dolthub.com" + doltConfigCommand = "config" + doltConfigGlobalFlag = "--global" + doltConfigGetFlag = "--get" + doltConfigAddFlag = "--add" + doltCloneCommand = "clone" + doltVersionCommand = "version" + doltInitCommand = "init" + dbName = "test" + bigEmptyRepo = "max-hoffman/big-empty" + nbfEnvVar = "DOLT_DEFAULT_BIN_FORMAT" + + postgresDriver = "postgres" + doltgresUser = "doltgres" + doltDataDir = ".dolt" + createDatabaseTemplate = "create database %s;" + psqlDsnTemplate = "host=%s port=%d user=%s password=%s dbname=%s sslmode=disable" + + expectedServerKilledErrorMessage = "signal: killed" + expectedServerTerminatedErrorMessage = "signal: terminated" + + sysbenchCommand = "sysbench" + sysbenchVersionFlag = "--version" + sysbenchPrepareCommand = "prepare" + sysbenchRunCommand = "run" + sysbenchCleanupCommand = "cleanup" + luaPathEnvVarTemplate = "LUA_PATH=%s" + luaPath = "?.lua" + + defaultMysqlUser = "root" + + // Note this is built for the SysbenchDocker file. If you want to run locally you'll need to override these variables + // for your local MySQL setup. + tpccUserLocal = "'sysbench'@'localhost'" + tpccPassLocal = "sysbenchpass" + + tpccDbName = "sbt" + tpccScaleFactorTemplate = "tpcc-scale-factor-%d" + + tpccDbDriverFlag = "--db-driver" + tpccMysqlUsername = "sysbench" + tpccMysqlDbFlag = "--mysql-db" + tpccMysqlHostFlag = "--mysql-host" + tpccMysqlUserFlag = "--mysql-user" + tpccMysqlPasswordFlag = "--mysql-password" + tpccMysqlPortFlag = "--mysql-port" + tpccTimeFlag = "--time" + tpccThreadsFlag = "--threads" + tpccReportIntervalFlag = "--report_interval" + tpccTablesFlag = "--tables" + tpccScaleFlag = "--scale" + tpccTransactionLevelFlag = "--trx_level" + tpccReportCsv = "reportCsv" + tpccTransactionLevelRr = "RR" + tpccLuaFilename = "tpcc.lua" + + mysqlDriverName = "mysql" + mysqlRootTCPDsnTemplate = "root@tcp(%s:%d)/" + mysqlRootUnixDsnTemplate = "root@unix(%s)/" + mysqlDropDatabaseSqlTemplate = "DROP DATABASE IF EXISTS %s;" + mysqlCreateDatabaseSqlTemplate = "CREATE DATABASE %s;" + mysqlDropUserSqlTemplate = "DROP USER IF EXISTS %s;" + mysqlCreateUserSqlTemplate = "CREATE USER %s IDENTIFIED WITH mysql_native_password BY '%s';" + mysqlGrantPermissionsSqlTemplate = "GRANT ALL ON %s.* to %s;" + mysqlSetGlobalLocalInfileSql = "SET GLOBAL local_infile = 'ON';" + mysqlSetGlobalSqlModeSql = "SET GLOBAL sql_mode=(SELECT REPLACE(@@sql_mode,'ONLY_FULL_GROUP_BY',''));" + + postgresInitDbDataDirFlag = "--pgdata" + postgresUsernameFlag = "--username" + postgresUsername = "postgres" + postgresDataDirFlag = "-D" + postgresDropDatabaseSqlTemplate = "DROP DATABASE IF EXISTS %s;" + postgresDropUserSqlTemplate = "DROP USER IF EXISTS %s;" + postgresCreateUserSqlTemplate = "CREATE USER %s WITH PASSWORD '%s';" + postgresCreateDatabaseSqlTemplate = "CREATE DATABASE %s WITH OWNER %s;" + postgresLcAllEnvVarKey = "LC_ALL" + postgresLcAllEnvVarValue = "C" + + resultsDirname = "results" + stampFormat = time.RFC3339 + SqlStatsPrefix = "SQL statistics:" + read = "read" + write = "write" + other = "other" + totalQueries = "total" + totalEvents = "total number of events" + min = "min" + avg = "avg" + max = "max" + percentile = "percentile" + sum = "sum" + transactions = "transactions" + queriesPerSec = "queries" + ignoredErrors = "ignored errors" + reconnects = "reconnects" + totalTimeSecs = "total time" + + ResultFileTemplate = "%s_%s_%s_sysbench_performance%s" +) diff --git a/go/performance/utils/sysbench_runner/csv.go b/go/performance/utils/benchmark_runner/csv.go similarity index 99% rename from go/performance/utils/sysbench_runner/csv.go rename to go/performance/utils/benchmark_runner/csv.go index 83e9eefa26c..ba40ad66d0c 100644 --- a/go/performance/utils/sysbench_runner/csv.go +++ b/go/performance/utils/benchmark_runner/csv.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package sysbench_runner +package benchmark_runner import ( "encoding/csv" @@ -26,7 +26,7 @@ import ( // FromResultCsvHeaders returns supported csv headers for a Result func FromResultCsvHeaders() []string { return []string{ - "id", + "id", // todo: replace with constants "suite_id", "test_id", "runtime_os", diff --git a/go/performance/utils/sysbench_runner/csv_test.go b/go/performance/utils/benchmark_runner/csv_test.go similarity index 98% rename from go/performance/utils/sysbench_runner/csv_test.go rename to go/performance/utils/benchmark_runner/csv_test.go index 718c6a1b743..d530028828f 100644 --- a/go/performance/utils/sysbench_runner/csv_test.go +++ b/go/performance/utils/benchmark_runner/csv_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package sysbench_runner +package benchmark_runner import ( "fmt" diff --git a/go/performance/utils/sysbench_runner/debug.go b/go/performance/utils/benchmark_runner/debug.go similarity index 97% rename from go/performance/utils/sysbench_runner/debug.go rename to go/performance/utils/benchmark_runner/debug.go index d51f2a0ad01..2671d42347f 100644 --- a/go/performance/utils/sysbench_runner/debug.go +++ b/go/performance/utils/benchmark_runner/debug.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package sysbench_runner +package benchmark_runner import ( "context" diff --git a/go/performance/utils/benchmark_runner/dolt.go b/go/performance/utils/benchmark_runner/dolt.go new file mode 100644 index 00000000000..ec333a4471a --- /dev/null +++ b/go/performance/utils/benchmark_runner/dolt.go @@ -0,0 +1,172 @@ +// Copyright 2019-2022 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package benchmark_runner + +import ( + "context" + "errors" + "os" + "path/filepath" + "syscall" + "time" + + "github.com/dolthub/dolt/go/store/types" +) + +var ErrNotSysbenchTest = errors.New("sysbench test is required") + +var stampFunc = func() string { return time.Now().UTC().Format(stampFormat) } + +type doltBenchmarkerImpl struct { + dir string // cwd + config SysbenchConfig + serverConfig ServerConfig +} + +var _ Benchmarker = &doltBenchmarkerImpl{} + +func NewDoltBenchmarker(dir string, config SysbenchConfig, serverConfig ServerConfig) *doltBenchmarkerImpl { + return &doltBenchmarkerImpl{ + dir: dir, + config: config, + serverConfig: serverConfig, + } +} + +func (b *doltBenchmarkerImpl) updateGlobalConfig(ctx context.Context) error { + err := CheckSetDoltConfig(ctx, b.serverConfig.GetServerExec(), doltConfigUsernameKey, doltBenchmarkUser) + if err != nil { + return err + } + return CheckSetDoltConfig(ctx, b.serverConfig.GetServerExec(), doltConfigEmailKey, doltBenchmarkEmail) +} + +func (b *doltBenchmarkerImpl) checkInstallation(ctx context.Context) error { + version := ExecCommand(ctx, b.serverConfig.GetServerExec(), doltVersionCommand) + return version.Run() +} + +func (b *doltBenchmarkerImpl) initDoltRepo(ctx context.Context) (string, error) { + return InitDoltRepo(ctx, b.dir, b.serverConfig.GetServerExec(), b.config.GetNomsBinFormat(), dbName) +} + +func (b *doltBenchmarkerImpl) Benchmark(ctx context.Context) (Results, error) { + err := b.checkInstallation(ctx) + if err != nil { + return nil, err + } + + err = b.updateGlobalConfig(ctx) + if err != nil { + return nil, err + } + + testRepo, err := b.initDoltRepo(ctx) + if err != nil { + return nil, err + } + defer os.RemoveAll(testRepo) + + serverParams, err := b.serverConfig.GetServerArgs() + if err != nil { + return nil, err + } + + server := NewServer(ctx, testRepo, b.serverConfig, syscall.SIGTERM, serverParams) + err = server.Start() + if err != nil { + return nil, err + } + + tests, err := GetTests(b.config, b.serverConfig) + if err != nil { + return nil, err + } + + results := make(Results, 0) + runs := b.config.GetRuns() + for i := 0; i < runs; i++ { + for _, test := range tests { + t, ok := test.(SysbenchTest) + if !ok { + return nil, ErrNotSysbenchTest + } + tester := NewSysbenchTester(b.config, b.serverConfig, t, serverParams, stampFunc) + r, err := tester.Test(ctx) + if err != nil { + server.Stop() + return nil, err + } + results = append(results, r) + } + } + + err = server.Stop() + if err != nil { + return nil, err + } + + return results, nil +} + +// InitDoltRepo initializes a dolt database and returns its path +func InitDoltRepo(ctx context.Context, dir, serverExec, nomsBinFormat, dbName string) (string, error) { + testRepo := filepath.Join(dir, dbName) + if nomsBinFormat == types.Format_LD_1.VersionString() { + err := ExecCommand(ctx, serverExec, doltCloneCommand, bigEmptyRepo, dbName).Run() + if err != nil { + return "", err + } + return testRepo, nil + } + + err := os.MkdirAll(testRepo, os.ModePerm) + if err != nil { + return "", err + } + + if nomsBinFormat != "" { + if err = os.Setenv(nbfEnvVar, nomsBinFormat); err != nil { + return "", err + } + } + + doltInit := ExecCommand(ctx, serverExec, doltInitCommand) + doltInit.Dir = testRepo + err = doltInit.Run() + if err != nil { + return "", err + } + + return testRepo, nil +} + +// CheckSetDoltConfig checks the output of `dolt config --global --get` and sets the key, val if necessary +func CheckSetDoltConfig(ctx context.Context, serverExec, key, val string) error { + check := ExecCommand(ctx, serverExec, doltConfigCommand, doltConfigGlobalFlag, doltConfigGetFlag, key) + err := check.Run() + if err != nil { + // config get calls exit with 1 if not set + if err.Error() != "exit status 1" { + return err + } + set := ExecCommand(ctx, serverExec, doltConfigCommand, doltConfigGlobalFlag, doltConfigAddFlag, key, val) + err := set.Run() + if err != nil { + return err + } + } + return nil +} diff --git a/go/performance/utils/benchmark_runner/dolt_server_config.go b/go/performance/utils/benchmark_runner/dolt_server_config.go new file mode 100644 index 00000000000..6417fe3368a --- /dev/null +++ b/go/performance/utils/benchmark_runner/dolt_server_config.go @@ -0,0 +1,175 @@ +// Copyright 2019-2022 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package benchmark_runner + +import ( + "fmt" + "os" + + "github.com/google/uuid" +) + +type ServerProfile string + +type doltServerConfigImpl struct { + // Id is a unique id for this servers benchmarking + Id string + + // Host is the server host + Host string + + // Port is the server port + Port int + + // Version is the server version + Version string + + // ResultsFormat is the format the results should be written in + ResultsFormat string + + // ServerExec is the path to a server executable + ServerExec string + + // ServerUser is the user account that should start the server + ServerUser string + + // ServerArgs are the args used to start a server + ServerArgs []string + + // ServerProfile specifies the golang profile to take of a Dolt server + ServerProfile ServerProfile + + // ProfilePath path to directory where server profile will be written + ProfilePath string +} + +var _ ProfilingServerConfig = &doltServerConfigImpl{} + +func NewDoltServerConfig(version, serverExec, serverUser, host, resultsFormat, profilePath string, serverProfile ServerProfile, port int, serverArgs []string) *doltServerConfigImpl { + return &doltServerConfigImpl{ + Id: uuid.New().String(), + Host: host, + Port: port, + Version: version, + ResultsFormat: resultsFormat, + ServerExec: serverExec, + ServerUser: serverUser, + ServerArgs: serverArgs, + ServerProfile: serverProfile, + ProfilePath: profilePath, + } +} + +func (sc *doltServerConfigImpl) GetId() string { + return sc.Id +} + +func (sc *doltServerConfigImpl) GetHost() string { + return sc.Host +} + +func (sc *doltServerConfigImpl) GetPort() int { + return sc.Port +} + +func (sc *doltServerConfigImpl) GetVersion() string { + return sc.Version +} + +func (sc *doltServerConfigImpl) GetProfilePath() string { + return sc.ProfilePath +} + +func (sc *doltServerConfigImpl) GetServerProfile() ServerProfile { + return sc.ServerProfile +} + +func (sc *doltServerConfigImpl) GetServerType() ServerType { + return Dolt +} + +func (sc *doltServerConfigImpl) GetResultsFormat() string { + return sc.ResultsFormat +} + +func (sc *doltServerConfigImpl) GetServerExec() string { + return sc.ServerExec +} + +// GetServerArgs returns the args used to start a server +func (sc *doltServerConfigImpl) GetServerArgs() ([]string, error) { + params := make([]string, 0) + params = append(params, defaultDoltServerParams...) + if sc.Host != "" { + params = append(params, fmt.Sprintf("%s=%s", hostFlag, sc.Host)) + } + if sc.Port != 0 { + params = append(params, fmt.Sprintf("%s=%d", portFlag, sc.Port)) + } + params = append(params, sc.ServerArgs...) + return params, nil +} + +func (sc *doltServerConfigImpl) GetTestingParams(testConfig TestConfig) TestParams { + params := NewSysbenchTestParams() + params.Append(defaultSysbenchParams...) + params.Append(fmt.Sprintf("%s=%s", sysbenchMysqlDbFlag, dbName)) + params.Append(fmt.Sprintf("%s=%s", sysbenchDbDriverFlag, mysqlDriverName)) + params.Append(fmt.Sprintf("%s=%s", sysbenchMysqlHostFlag, sc.Host)) + params.Append(fmt.Sprintf("%s=%s", sysbenchMysqlUserFlag, defaultMysqlUser)) + if sc.Port != 0 { + params.Append(fmt.Sprintf("%s=%d", sysbenchMysqlPortFlag, sc.Port)) + } + params.Append(testConfig.GetOptions()...) + params.Append(testConfig.GetName()) + return params +} + +func (sc *doltServerConfigImpl) Validate() error { + if sc.Version == "" { + return getMustSupplyError("version") + } + if sc.ResultsFormat == "" { + return getMustSupplyError("results format") + } + if sc.ServerExec == "" { + return getMustSupplyError("server exec") + } + if sc.ServerProfile != "" { + if sc.ServerProfile != CpuServerProfile { + return fmt.Errorf("unsupported server profile: %s", sc.ServerProfile) + } + } + return CheckExec(sc.ServerExec, "server exec") +} + +func (sc *doltServerConfigImpl) SetDefaults() error { + if sc.Host == "" { + sc.Host = defaultHost + } + if sc.Port < 1 { + sc.Port = defaultDoltPort + } + if sc.ServerProfile != "" { + if sc.ProfilePath == "" { + cwd, err := os.Getwd() + if err != nil { + return err + } + sc.ProfilePath = cwd + } + } + return nil +} diff --git a/go/performance/utils/benchmark_runner/dolt_tpcc.go b/go/performance/utils/benchmark_runner/dolt_tpcc.go new file mode 100644 index 00000000000..e8b57c9ed95 --- /dev/null +++ b/go/performance/utils/benchmark_runner/dolt_tpcc.go @@ -0,0 +1,116 @@ +// Copyright 2019-2022 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package benchmark_runner + +import ( + "context" + "fmt" + "os" + "syscall" +) + +type doltTpccBenchmarkerImpl struct { + dir string // cwd + config TpccConfig + serverConfig ServerConfig +} + +var _ Benchmarker = &doltTpccBenchmarkerImpl{} + +func NewDoltTpccBenchmarker(dir string, config TpccConfig, serverConfig ServerConfig) *doltTpccBenchmarkerImpl { + return &doltTpccBenchmarkerImpl{ + dir: dir, + config: config, + serverConfig: serverConfig, + } +} + +func (b *doltTpccBenchmarkerImpl) updateGlobalConfig(ctx context.Context) error { + err := CheckSetDoltConfig(ctx, b.serverConfig.GetServerExec(), doltConfigUsernameKey, doltBenchmarkUser) + if err != nil { + return err + } + return CheckSetDoltConfig(ctx, b.serverConfig.GetServerExec(), doltConfigEmailKey, doltBenchmarkEmail) +} + +func (b *doltTpccBenchmarkerImpl) checkInstallation(ctx context.Context) error { + version := ExecCommand(ctx, b.serverConfig.GetServerExec(), doltVersionCommand) + return version.Run() +} + +func (b *doltTpccBenchmarkerImpl) initDoltRepo(ctx context.Context) (string, error) { + return InitDoltRepo(ctx, b.dir, b.serverConfig.GetServerExec(), b.config.GetNomsBinFormat(), tpccDbName) +} + +func (b *doltTpccBenchmarkerImpl) Benchmark(ctx context.Context) (Results, error) { + err := b.checkInstallation(ctx) + if err != nil { + return nil, err + } + + err = b.updateGlobalConfig(ctx) + if err != nil { + return nil, err + } + + testRepo, err := b.initDoltRepo(ctx) + if err != nil { + return nil, err + } + defer os.RemoveAll(testRepo) + + serverParams, err := b.serverConfig.GetServerArgs() + if err != nil { + return nil, err + } + + server := NewServer(ctx, testRepo, b.serverConfig, syscall.SIGTERM, serverParams) + err = server.Start() + if err != nil { + return nil, err + } + + tests := GetTpccTests(b.config) + + results := make(Results, 0) + for _, test := range tests { + tester := NewTpccTester(b.config, b.serverConfig, test, serverParams, stampFunc) + r, err := tester.Test(ctx) + if err != nil { + server.Stop() + return nil, err + } + results = append(results, r) + } + + err = server.Stop() + if err != nil { + return nil, err + } + + return results, nil +} + +// GetTpccTests creates a set of tests that the server needs to be executed on. +func GetTpccTests(config TpccConfig) []Test { + tests := make([]Test, 0) + for _, sf := range config.GetScaleFactors() { + params := NewDefaultTpccParams() + params.ScaleFactor = sf + test := NewTpccTest(fmt.Sprintf(tpccScaleFactorTemplate, sf), params) + tests = append(tests, test) + } + return tests +} diff --git a/go/performance/utils/benchmark_runner/doltgres.go b/go/performance/utils/benchmark_runner/doltgres.go new file mode 100644 index 00000000000..be77c38cece --- /dev/null +++ b/go/performance/utils/benchmark_runner/doltgres.go @@ -0,0 +1,172 @@ +// Copyright 2019-2022 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package benchmark_runner + +import ( + "context" + "database/sql" + "fmt" + "os" + "path/filepath" + "syscall" +) + +type doltgresBenchmarkerImpl struct { + dir string // cwd + config SysbenchConfig + serverConfig ServerConfig +} + +var _ Benchmarker = &doltgresBenchmarkerImpl{} + +func NewDoltgresBenchmarker(dir string, config SysbenchConfig, serverConfig ServerConfig) *doltgresBenchmarkerImpl { + return &doltgresBenchmarkerImpl{ + dir: dir, + config: config, + serverConfig: serverConfig, + } +} + +func (b *doltgresBenchmarkerImpl) checkInstallation(ctx context.Context) error { + version := ExecCommand(ctx, b.serverConfig.GetServerExec(), doltVersionCommand) + return version.Run() +} + +func (b *doltgresBenchmarkerImpl) createServerDir() (string, error) { + return CreateServerDir(dbName) +} + +func (b *doltgresBenchmarkerImpl) cleanupServerDir(dir string) error { + dataDir := filepath.Join(dir, doltDataDir) + defaultDir := filepath.Join(dir, doltgresUser) + testDir := filepath.Join(dir, dbName) + for _, d := range []string{dataDir, defaultDir, testDir} { + if _, err := os.Stat(d); !os.IsNotExist(err) { + err = os.RemoveAll(d) + if err != nil { + return err + } + } + } + return nil +} + +func (b *doltgresBenchmarkerImpl) createTestingDb(ctx context.Context) error { + psqlconn := fmt.Sprintf(psqlDsnTemplate, b.serverConfig.GetHost(), b.serverConfig.GetPort(), doltgresUser, "", dbName) + + // open database + db, err := sql.Open(postgresDriver, psqlconn) + if err != nil { + return err + } + + // close database + defer db.Close() + + // check db + err = db.PingContext(ctx) + if err != nil { + return err + } + + _, err = db.ExecContext(ctx, fmt.Sprintf(createDatabaseTemplate, dbName)) + return err +} + +func (b *doltgresBenchmarkerImpl) Benchmark(ctx context.Context) (results Results, err error) { + err = b.checkInstallation(ctx) + if err != nil { + return + } + + var serverDir string + serverDir, err = CreateServerDir(dbName) + if err != nil { + return + } + defer func() { + rerr := b.cleanupServerDir(serverDir) + if err == nil { + err = rerr + } + }() + + var serverParams []string + serverParams, err = b.serverConfig.GetServerArgs() + if err != nil { + return + } + + serverParams = append(serverParams, fmt.Sprintf("%s=%s", doltgresDataDirFlag, serverDir)) + + server := NewServer(ctx, serverDir, b.serverConfig, syscall.SIGTERM, serverParams) + err = server.Start() + if err != nil { + return + } + + err = b.createTestingDb(ctx) + if err != nil { + return + } + + var tests []Test + tests, err = GetTests(b.config, b.serverConfig) + if err != nil { + return + } + + results = make(Results, 0) + runs := b.config.GetRuns() + for i := 0; i < runs; i++ { + for _, test := range tests { + t, ok := test.(SysbenchTest) + if !ok { + return nil, ErrNotSysbenchTest + } + tester := NewSysbenchTester(b.config, b.serverConfig, t, serverParams, stampFunc) + var r *Result + r, err = tester.Test(ctx) + if err != nil { + server.Stop() + return + } + results = append(results, r) + } + } + + err = server.Stop() + if err != nil { + return + } + + return +} + +// CreateServerDir creates a server directory +func CreateServerDir(dbName string) (string, error) { + cwd, err := os.Getwd() + if err != nil { + return "", err + } + + serverDir := filepath.Join(cwd, dbName) + err = os.MkdirAll(serverDir, os.ModePerm) + if err != nil { + return "", err + } + + return serverDir, nil +} diff --git a/go/performance/utils/benchmark_runner/doltgres_server_config.go b/go/performance/utils/benchmark_runner/doltgres_server_config.go new file mode 100644 index 00000000000..caa0af6833e --- /dev/null +++ b/go/performance/utils/benchmark_runner/doltgres_server_config.go @@ -0,0 +1,140 @@ +// Copyright 2019-2022 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package benchmark_runner + +import ( + "fmt" + + "github.com/google/uuid" +) + +type doltgresServerConfigImpl struct { + // Id is a unique id for this servers benchmarking + Id string + + // Host is the server host + Host string + + // Port is the server port + Port int + + // Version is the server version + Version string + + // ResultsFormat is the format the results should be written in + ResultsFormat string + + // ServerExec is the path to a server executable + ServerExec string + + // ServerUser is the user account that should start the server + ServerUser string + + // ServerArgs are the args used to start a server + ServerArgs []string +} + +var _ ServerConfig = &doltgresServerConfigImpl{} + +func NewDoltgresServerConfig(version, serverExec, serverUser, host, resultsFormat string, port int, serverArgs []string) *doltgresServerConfigImpl { + return &doltgresServerConfigImpl{ + Id: uuid.New().String(), + Host: host, + Port: port, + Version: version, + ResultsFormat: resultsFormat, + ServerExec: serverExec, + ServerUser: serverUser, + ServerArgs: serverArgs, + } +} + +func (sc *doltgresServerConfigImpl) GetServerType() ServerType { + return Doltgres +} + +func (sc *doltgresServerConfigImpl) GetServerExec() string { + return sc.ServerExec +} + +func (sc *doltgresServerConfigImpl) GetResultsFormat() string { + return sc.ResultsFormat +} + +func (sc *doltgresServerConfigImpl) GetServerArgs() ([]string, error) { + params := make([]string, 0) + if sc.Host != "" { + params = append(params, fmt.Sprintf("%s=%s", hostFlag, sc.Host)) + } + if sc.Port != 0 { + params = append(params, fmt.Sprintf("%s=%d", portFlag, sc.Port)) + } + params = append(params, sc.ServerArgs...) + return params, nil +} + +func (sc *doltgresServerConfigImpl) GetTestingParams(testConfig TestConfig) TestParams { + params := NewSysbenchTestParams() + params.Append(defaultSysbenchParams...) + params.Append(fmt.Sprintf("%s=%s", sysbenchDbDriverFlag, sysbenchPostgresDbDriver)) + params.Append(fmt.Sprintf("%s=%s", sysbenchPostgresDbFlag, dbName)) + params.Append(fmt.Sprintf("%s=%s", sysbenchPostgresHostFlag, sc.Host)) + params.Append(fmt.Sprintf("%s=%s", sysbenchPostgresUserFlag, doltgresUser)) + if sc.Port != 0 { + params.Append(fmt.Sprintf("%s=%d", sysbenchPostgresPortFlag, sc.Port)) + } + params.Append(testConfig.GetOptions()...) + params.Append(testConfig.GetName()) + return params +} + +func (sc *doltgresServerConfigImpl) Validate() error { + if sc.Version == "" { + return getMustSupplyError("version") + } + if sc.ResultsFormat == "" { + return getMustSupplyError("results format") + } + if sc.ServerExec == "" { + return getMustSupplyError("server exec") + } + return CheckExec(sc.ServerExec, "server exec") +} + +func (sc *doltgresServerConfigImpl) SetDefaults() error { + if sc.Host == "" { + sc.Host = defaultHost + } + if sc.Port < 1 { + sc.Port = defaultDoltgresPort + } + return nil +} + +func (sc *doltgresServerConfigImpl) GetId() string { + return sc.Id +} + +func (sc *doltgresServerConfigImpl) GetHost() string { + return sc.Host +} + +func (sc *doltgresServerConfigImpl) GetPort() int { + return sc.Port +} + +func (sc *doltgresServerConfigImpl) GetVersion() string { + return sc.Version +} diff --git a/go/performance/utils/sysbench_runner/json.go b/go/performance/utils/benchmark_runner/json.go similarity index 98% rename from go/performance/utils/sysbench_runner/json.go rename to go/performance/utils/benchmark_runner/json.go index 389acd8c63f..48270d1372f 100644 --- a/go/performance/utils/sysbench_runner/json.go +++ b/go/performance/utils/benchmark_runner/json.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package sysbench_runner +package benchmark_runner import ( "encoding/json" diff --git a/go/performance/utils/sysbench_runner/json_test.go b/go/performance/utils/benchmark_runner/json_test.go similarity index 98% rename from go/performance/utils/sysbench_runner/json_test.go rename to go/performance/utils/benchmark_runner/json_test.go index c49b1b8d988..b3aaa1dabac 100644 --- a/go/performance/utils/sysbench_runner/json_test.go +++ b/go/performance/utils/benchmark_runner/json_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package sysbench_runner +package benchmark_runner import ( "fmt" diff --git a/go/performance/utils/benchmark_runner/mysql.go b/go/performance/utils/benchmark_runner/mysql.go new file mode 100644 index 00000000000..2f096838da1 --- /dev/null +++ b/go/performance/utils/benchmark_runner/mysql.go @@ -0,0 +1,177 @@ +// Copyright 2019-2022 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package benchmark_runner + +import ( + "context" + "database/sql" + "fmt" + "os" + "syscall" + + _ "github.com/go-sql-driver/mysql" +) + +type mysqlBenchmarkerImpl struct { + dir string // cwd + config SysbenchConfig + serverConfig ProtocolServerConfig +} + +var _ Benchmarker = &mysqlBenchmarkerImpl{} + +func NewMysqlBenchmarker(dir string, config SysbenchConfig, serverConfig ProtocolServerConfig) *mysqlBenchmarkerImpl { + return &mysqlBenchmarkerImpl{ + dir: dir, + config: config, + serverConfig: serverConfig, + } +} + +func (b *mysqlBenchmarkerImpl) getDsn() (string, error) { + return GetMysqlDsn(b.serverConfig.GetHost(), b.serverConfig.GetSocket(), b.serverConfig.GetConnectionProtocol(), b.serverConfig.GetPort()) +} + +func (b *mysqlBenchmarkerImpl) createTestingDb(ctx context.Context) error { + dsn, err := b.getDsn() + if err != nil { + return err + } + return CreateMysqlTestingDb(ctx, dsn, dbName) +} + +func (b *mysqlBenchmarkerImpl) Benchmark(ctx context.Context) (Results, error) { + serverDir, err := InitMysqlDataDir(ctx, b.serverConfig.GetServerExec(), dbName) + if err != nil { + return nil, err + } + + serverParams, err := b.serverConfig.GetServerArgs() + if err != nil { + return nil, err + } + serverParams = append(serverParams, fmt.Sprintf("%s=%s", MysqlDataDirFlag, serverDir)) + + server := NewServer(ctx, serverDir, b.serverConfig, syscall.SIGTERM, serverParams) + err = server.Start() + if err != nil { + return nil, err + } + + err = b.createTestingDb(ctx) + if err != nil { + return nil, err + } + + tests, err := GetTests(b.config, b.serverConfig) + if err != nil { + return nil, err + } + + results := make(Results, 0) + runs := b.config.GetRuns() + for i := 0; i < runs; i++ { + for _, test := range tests { + t, ok := test.(SysbenchTest) + if !ok { + return nil, ErrNotSysbenchTest + } + tester := NewSysbenchTester(b.config, b.serverConfig, t, serverParams, stampFunc) + r, err := tester.Test(ctx) + if err != nil { + server.Stop() + return nil, err + } + results = append(results, r) + } + } + + err = server.Stop() + if err != nil { + return nil, err + } + + return results, os.RemoveAll(serverDir) +} + +func InitMysqlDataDir(ctx context.Context, serverExec, dbName string) (string, error) { + serverDir, err := CreateServerDir(dbName) + if err != nil { + return "", err + } + + msInit := ExecCommand(ctx, serverExec, MysqlInitializeInsecureFlag, fmt.Sprintf("%s=%s", MysqlDataDirFlag, serverDir)) + err = msInit.Run() + if err != nil { + return "", err + } + + return serverDir, nil +} + +func CreateMysqlTestingDb(ctx context.Context, dsn, dbName string) (err error) { + var db *sql.DB + db, err = sql.Open(mysqlDriverName, dsn) + if err != nil { + return + } + defer func() { + rerr := db.Close() + if err == nil { + err = rerr + } + }() + + err = db.Ping() + if err != nil { + return + } + + stmts := []string{ + fmt.Sprintf(mysqlDropDatabaseSqlTemplate, dbName), + fmt.Sprintf(mysqlCreateDatabaseSqlTemplate, dbName), + fmt.Sprintf(mysqlDropUserSqlTemplate, sysbenchUserLocal), + fmt.Sprintf(mysqlCreateUserSqlTemplate, sysbenchUserLocal, sysbenchPassLocal), + fmt.Sprintf(mysqlGrantPermissionsSqlTemplate, dbName, sysbenchUserLocal), + mysqlSetGlobalLocalInfileSql, + mysqlSetGlobalSqlModeSql, // Required for running groupby_scan.lua without error + } + + for _, s := range stmts { + _, err = db.ExecContext(ctx, s) + if err != nil { + return + } + } + + return +} + +func GetMysqlDsn(host, socket, protocol string, port int) (string, error) { + var socketPath string + if socket != "" { + socketPath = socket + } else { + socketPath = defaultMysqlSocket + } + + if protocol == tcpProtocol { + return fmt.Sprintf(mysqlRootTCPDsnTemplate, host, port), nil + } else if protocol == unixProtocol { + return fmt.Sprintf(mysqlRootUnixDsnTemplate, socketPath), nil + } else { + return "", ErrUnsupportedConnectionProtocol + } +} diff --git a/go/performance/utils/benchmark_runner/mysql_server_config.go b/go/performance/utils/benchmark_runner/mysql_server_config.go new file mode 100644 index 00000000000..b2cd4f5fa43 --- /dev/null +++ b/go/performance/utils/benchmark_runner/mysql_server_config.go @@ -0,0 +1,168 @@ +// Copyright 2019-2022 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package benchmark_runner + +import ( + "fmt" + + "github.com/google/uuid" +) + +type mysqlServerConfigImpl struct { + // Id is a unique id for this servers benchmarking + Id string + + // Host is the server host + Host string + + // Port is the server port + Port int + + // Version is the server version + Version string + + // ResultsFormat is the format the results should be written in + ResultsFormat string + + // ServerExec is the path to a server executable + ServerExec string + + // ServerUser is the user account that should start the server + ServerUser string + + // SkipLogBin will skip bin logging + SkipLogBin bool + + // ServerArgs are the args used to start a server + ServerArgs []string + + // ConnectionProtocol defines the protocol for connecting to the server + ConnectionProtocol string + + // Socket is the path to the server socket + Socket string +} + +var _ ProtocolServerConfig = &mysqlServerConfigImpl{} + +func NewMysqlServerConfig(version, serverExec, serverUser, host, resultsFormat, protocol, socket string, port int, serverArgs []string, skipBinLog bool) *mysqlServerConfigImpl { + return &mysqlServerConfigImpl{ + Id: uuid.New().String(), + Host: host, + Port: port, + Version: version, + ResultsFormat: resultsFormat, + ServerExec: serverExec, + ServerUser: serverUser, + SkipLogBin: skipBinLog, + ServerArgs: serverArgs, + ConnectionProtocol: protocol, + Socket: socket, + } +} + +func (sc *mysqlServerConfigImpl) GetServerExec() string { + return sc.ServerExec +} + +func (sc *mysqlServerConfigImpl) GetId() string { + return sc.Id +} + +func (sc *mysqlServerConfigImpl) GetHost() string { + return sc.Host +} + +func (sc *mysqlServerConfigImpl) GetPort() int { + return sc.Port +} + +func (sc *mysqlServerConfigImpl) GetVersion() string { + return sc.Version +} + +func (sc *mysqlServerConfigImpl) GetServerType() ServerType { + return MySql +} + +func (sc *mysqlServerConfigImpl) GetResultsFormat() string { + return sc.ResultsFormat +} + +func (sc *mysqlServerConfigImpl) GetConnectionProtocol() string { + return sc.ConnectionProtocol +} + +func (sc *mysqlServerConfigImpl) GetSocket() string { + return sc.Socket +} + +func (sc *mysqlServerConfigImpl) GetServerArgs() ([]string, error) { + params := make([]string, 0) + if sc.ServerUser != "" { + params = append(params, fmt.Sprintf("%s=%s", userFlag, sc.ServerUser)) + } + if sc.SkipLogBin { + params = append(params, skipBinLogFlag) + } + if sc.Port != 0 { + params = append(params, fmt.Sprintf("%s=%d", portFlag, sc.Port)) + } + params = append(params, sc.ServerArgs...) + return params, nil +} + +func (sc *mysqlServerConfigImpl) GetTestingParams(testConfig TestConfig) TestParams { + params := NewSysbenchTestParams() + params.Append(defaultSysbenchParams...) + params.Append(fmt.Sprintf("%s=%s", sysbenchMysqlDbFlag, dbName)) + params.Append(fmt.Sprintf("%s=%s", sysbenchDbDriverFlag, mysqlDriverName)) + params.Append(fmt.Sprintf("%s=%s", sysbenchMysqlHostFlag, sc.Host)) + if sc.Port != 0 { + params.Append(fmt.Sprintf("%s=%d", sysbenchMysqlPortFlag, sc.Port)) + } + params.Append(fmt.Sprintf("%s=%s", sysbenchMysqlUserFlag, sysbenchCommand)) + params.Append(fmt.Sprintf("%s=%s", sysbenchMysqlPasswordFlag, sysbenchPassLocal)) + params.Append(testConfig.GetOptions()...) + params.Append(testConfig.GetName()) + return params +} + +func (sc *mysqlServerConfigImpl) Validate() error { + if sc.Version == "" { + return getMustSupplyError("version") + } + if sc.ResultsFormat == "" { + return getMustSupplyError("results format") + } + if sc.ServerExec == "" { + return getMustSupplyError("server exec") + } + err := CheckProtocol(sc.ConnectionProtocol) + if err != nil { + return err + } + return CheckExec(sc.ServerExec, "server exec") +} + +func (sc *mysqlServerConfigImpl) SetDefaults() error { + if sc.Host == "" { + sc.Host = defaultHost + } + if sc.Port < 1 { + sc.Port = defaultMysqlPort + } + return nil +} diff --git a/go/performance/utils/benchmark_runner/mysql_tpcc.go b/go/performance/utils/benchmark_runner/mysql_tpcc.go new file mode 100644 index 00000000000..93d577e5dcc --- /dev/null +++ b/go/performance/utils/benchmark_runner/mysql_tpcc.go @@ -0,0 +1,94 @@ +// Copyright 2019-2022 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package benchmark_runner + +import ( + "context" + "fmt" + "os" + "syscall" +) + +type mysqlTpccBenchmarkerImpl struct { + dir string // cwd + config TpccConfig + serverConfig ProtocolServerConfig +} + +var _ Benchmarker = &mysqlTpccBenchmarkerImpl{} + +func NewMysqlTpccBenchmarker(dir string, config TpccConfig, serverConfig ProtocolServerConfig) *mysqlTpccBenchmarkerImpl { + return &mysqlTpccBenchmarkerImpl{ + dir: dir, + config: config, + serverConfig: serverConfig, + } +} + +func (b *mysqlTpccBenchmarkerImpl) getDsn() (string, error) { + return GetMysqlDsn(b.serverConfig.GetHost(), b.serverConfig.GetSocket(), b.serverConfig.GetConnectionProtocol(), b.serverConfig.GetPort()) +} + +func (b *mysqlTpccBenchmarkerImpl) createTestingDb(ctx context.Context) error { + dsn, err := b.getDsn() + if err != nil { + return err + } + return CreateMysqlTestingDb(ctx, dsn, tpccDbName) +} + +func (b *mysqlTpccBenchmarkerImpl) Benchmark(ctx context.Context) (Results, error) { + serverDir, err := InitMysqlDataDir(ctx, b.serverConfig.GetServerExec(), tpccDbName) + if err != nil { + return nil, err + } + + serverParams, err := b.serverConfig.GetServerArgs() + if err != nil { + return nil, err + } + serverParams = append(serverParams, fmt.Sprintf("%s=%s", MysqlDataDirFlag, serverDir)) + + server := NewServer(ctx, serverDir, b.serverConfig, syscall.SIGTERM, serverParams) + err = server.Start() + if err != nil { + return nil, err + } + + err = b.createTestingDb(ctx) + if err != nil { + return nil, err + } + + tests := GetTpccTests(b.config) + + results := make(Results, 0) + for _, test := range tests { + tester := NewTpccTester(b.config, b.serverConfig, test, serverParams, stampFunc) + r, err := tester.Test(ctx) + if err != nil { + server.Stop() + return nil, err + } + results = append(results, r) + } + + err = server.Stop() + if err != nil { + return nil, err + } + + return results, os.RemoveAll(serverDir) +} diff --git a/go/performance/utils/benchmark_runner/postgres.go b/go/performance/utils/benchmark_runner/postgres.go new file mode 100644 index 00000000000..dcbe4af522f --- /dev/null +++ b/go/performance/utils/benchmark_runner/postgres.go @@ -0,0 +1,159 @@ +// Copyright 2019-2022 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package benchmark_runner + +import ( + "context" + "database/sql" + "fmt" + "os" + "syscall" + + _ "github.com/lib/pq" +) + +type postgresBenchmarkerImpl struct { + dir string // cwd + config SysbenchConfig + serverConfig InitServerConfig +} + +var _ Benchmarker = &postgresBenchmarkerImpl{} + +func NewPostgresBenchmarker(dir string, config SysbenchConfig, serverConfig InitServerConfig) *postgresBenchmarkerImpl { + return &postgresBenchmarkerImpl{ + dir: dir, + config: config, + serverConfig: serverConfig, + } +} + +func (b *postgresBenchmarkerImpl) initDataDir(ctx context.Context) (string, error) { + serverDir, err := CreateServerDir(dbName) + if err != nil { + return "", err + } + + pgInit := ExecCommand(ctx, b.serverConfig.GetInitDbExec(), fmt.Sprintf("%s=%s", postgresInitDbDataDirFlag, serverDir), fmt.Sprintf("%s=%s", postgresUsernameFlag, postgresUsername)) + err = pgInit.Run() + if err != nil { + return "", err + } + + return serverDir, nil +} + +func (b *postgresBenchmarkerImpl) createTestingDb(ctx context.Context) (err error) { + psqlconn := fmt.Sprintf(psqlDsnTemplate, b.serverConfig.GetHost(), b.serverConfig.GetPort(), postgresUsername, "", dbName) + + var db *sql.DB + db, err = sql.Open(postgresDriver, psqlconn) + if err != nil { + return + } + defer func() { + rerr := db.Close() + if err == nil { + err = rerr + } + }() + err = db.PingContext(ctx) + if err != nil { + return + } + + stmts := []string{ + fmt.Sprintf(postgresDropDatabaseSqlTemplate, dbName), + fmt.Sprintf(postgresDropUserSqlTemplate, sysbenchUsername), + fmt.Sprintf(postgresCreateUserSqlTemplate, sysbenchUsername, sysbenchPassLocal), + fmt.Sprintf(postgresCreateDatabaseSqlTemplate, dbName, sysbenchUsername), + } + + for _, s := range stmts { + _, err = db.ExecContext(ctx, s) + if err != nil { + return + } + } + + return +} + +func (b *postgresBenchmarkerImpl) Benchmark(ctx context.Context) (results Results, err error) { + var serverDir string + serverDir, err = b.initDataDir(ctx) + if err != nil { + return + } + defer func() { + rerr := os.RemoveAll(serverDir) + if err == nil { + err = rerr + } + }() + + var serverParams []string + serverParams, err = b.serverConfig.GetServerArgs() + if err != nil { + return + } + + serverParams = append(serverParams, postgresDataDirFlag, serverDir) + + server := NewServer(ctx, serverDir, b.serverConfig, syscall.SIGTERM, serverParams) + server.WithEnv(postgresLcAllEnvVarKey, postgresLcAllEnvVarValue) + + err = server.Start() + if err != nil { + return + } + + err = b.createTestingDb(ctx) + if err != nil { + return + } + + var tests []Test + tests, err = GetTests(b.config, b.serverConfig) + if err != nil { + return + } + + results = make(Results, 0) + runs := b.config.GetRuns() + for i := 0; i < runs; i++ { + for _, test := range tests { + t, ok := test.(SysbenchTest) + if !ok { + return nil, ErrNotSysbenchTest + } + tester := NewSysbenchTester(b.config, b.serverConfig, t, serverParams, stampFunc) + var r *Result + r, err = tester.Test(ctx) + if err != nil { + server.Stop() + return + } + results = append(results, r) + } + } + + err = server.Stop() + if err != nil { + return + } + + return +} diff --git a/go/performance/utils/benchmark_runner/postgres_server_config.go b/go/performance/utils/benchmark_runner/postgres_server_config.go new file mode 100644 index 00000000000..85d79d89e1f --- /dev/null +++ b/go/performance/utils/benchmark_runner/postgres_server_config.go @@ -0,0 +1,149 @@ +// Copyright 2019-2022 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package benchmark_runner + +import ( + "fmt" + + "github.com/google/uuid" +) + +type postgresServerConfigImpl struct { + // Id is a unique id for this servers benchmarking + Id string + + // Host is the server host + Host string + + // Port is the server port + Port int + + // Version is the server version + Version string + + // ResultsFormat is the format the results should be written in + ResultsFormat string + + // ServerExec is the path to a server executable + ServerExec string + + // InitExec is the path to the server init db executable + InitExec string + + // ServerUser is the user account that should start the server + ServerUser string + + // ServerArgs are the args used to start a server + ServerArgs []string +} + +var _ InitServerConfig = &postgresServerConfigImpl{} + +func NewPostgresServerConfig(version, serverExec, initDbExec, serverUser, host, resultsFormat string, port int, serverArgs []string) *postgresServerConfigImpl { + return &postgresServerConfigImpl{ + Id: uuid.New().String(), + Host: host, + Port: port, + Version: version, + ResultsFormat: resultsFormat, + ServerExec: serverExec, + InitExec: initDbExec, + ServerUser: serverUser, + ServerArgs: serverArgs, + } +} + +func (sc *postgresServerConfigImpl) GetServerExec() string { + return sc.ServerExec +} + +func (sc *postgresServerConfigImpl) GetInitDbExec() string { + return sc.InitExec +} + +func (sc *postgresServerConfigImpl) GetId() string { + return sc.Id +} + +func (sc *postgresServerConfigImpl) GetHost() string { + return sc.Host +} + +func (sc *postgresServerConfigImpl) GetPort() int { + return sc.Port +} + +func (sc *postgresServerConfigImpl) GetVersion() string { + return sc.Version +} + +func (sc *postgresServerConfigImpl) GetResultsFormat() string { + return sc.ResultsFormat +} + +func (sc *postgresServerConfigImpl) GetServerType() ServerType { + return Postgres +} + +func (sc *postgresServerConfigImpl) GetServerArgs() ([]string, error) { + params := make([]string, 0) + if sc.Port != 0 { + params = append(params, fmt.Sprintf("%s=%d", portFlag, sc.Port)) + } + params = append(params, sc.ServerArgs...) + return params, nil +} + +func (sc *postgresServerConfigImpl) GetTestingParams(testConfig TestConfig) TestParams { + params := NewSysbenchTestParams() + params.Append(defaultSysbenchParams...) + params.Append(fmt.Sprintf("%s=%s", sysbenchDbDriverFlag, sysbenchPostgresDbDriver)) + params.Append(fmt.Sprintf("%s=%s", sysbenchPostgresDbFlag, dbName)) + params.Append(fmt.Sprintf("%s=%s", sysbenchPostgresHostFlag, sc.Host)) + params.Append(fmt.Sprintf("%s=%s", sysbenchPostgresUserFlag, postgresUsername)) + if sc.Port != 0 { + params.Append(fmt.Sprintf("%s=%d", sysbenchPostgresPortFlag, sc.Port)) + } + params.Append(testConfig.GetOptions()...) + params.Append(testConfig.GetName()) + return params +} + +func (sc *postgresServerConfigImpl) Validate() error { + if sc.Version == "" { + return getMustSupplyError("version") + } + if sc.ResultsFormat == "" { + return getMustSupplyError("results format") + } + if sc.ServerExec == "" { + return getMustSupplyError("server exec") + } + err := CheckExec(sc.ServerExec, "server exec") + if err != nil { + return err + } + return CheckExec(sc.InitExec, "initdb exec") +} + +func (sc *postgresServerConfigImpl) SetDefaults() error { + if sc.Host == "" { + sc.Host = defaultHost + } + if sc.Port < 1 { + sc.Port = defaultPostgresPort + } + return nil +} diff --git a/go/performance/utils/benchmark_runner/profile.go b/go/performance/utils/benchmark_runner/profile.go new file mode 100644 index 00000000000..53da1c06212 --- /dev/null +++ b/go/performance/utils/benchmark_runner/profile.go @@ -0,0 +1,140 @@ +// Copyright 2019-2022 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package benchmark_runner + +import ( + "context" + "fmt" + "os" + "path/filepath" + "syscall" +) + +type Profiler interface { + Profile(ctx context.Context) error +} + +type doltProfilerImpl struct { + dir string // cwd + config SysbenchConfig + serverConfig ProfilingServerConfig +} + +var _ Profiler = &doltProfilerImpl{} + +func NewDoltProfiler(dir string, config SysbenchConfig, serverConfig ProfilingServerConfig) *doltProfilerImpl { + return &doltProfilerImpl{ + dir: dir, + config: config, + serverConfig: serverConfig, + } +} + +func (p *doltProfilerImpl) updateGlobalConfig(ctx context.Context) error { + err := CheckSetDoltConfig(ctx, p.serverConfig.GetServerExec(), doltConfigUsernameKey, doltBenchmarkUser) + if err != nil { + return err + } + return CheckSetDoltConfig(ctx, p.serverConfig.GetServerExec(), doltConfigEmailKey, doltBenchmarkEmail) +} + +func (p *doltProfilerImpl) checkInstallation(ctx context.Context) error { + version := ExecCommand(ctx, p.serverConfig.GetServerExec(), doltVersionCommand) + return version.Run() +} + +func (p *doltProfilerImpl) initDoltRepo(ctx context.Context) (string, error) { + return InitDoltRepo(ctx, p.dir, p.serverConfig.GetServerExec(), p.config.GetNomsBinFormat(), dbName) +} + +func (p *doltProfilerImpl) Profile(ctx context.Context) error { + err := p.checkInstallation(ctx) + if err != nil { + return err + } + + err = p.updateGlobalConfig(ctx) + if err != nil { + return err + } + + testRepo, err := p.initDoltRepo(ctx) + if err != nil { + return err + } + defer os.RemoveAll(testRepo) + + serverParams, err := p.serverConfig.GetServerArgs() + if err != nil { + return err + } + + profilePath, err := os.MkdirTemp("", "dolt_profile_path_*") + if err != nil { + return err + } + defer os.RemoveAll(profilePath) + + tempProfile := filepath.Join(profilePath, cpuProfileFilename) + profileParams := make([]string, 0) + profileParams = append(profileParams, profileFlag, cpuProfile, profilePathFlag, profilePath) + profileParams = append(profileParams, serverParams...) + + server := NewServer(ctx, testRepo, p.serverConfig, syscall.SIGTERM, profileParams) + err = server.Start() + if err != nil { + return err + } + + tests, err := GetTests(p.config, p.serverConfig) + if err != nil { + return err + } + + results := make(Results, 0) + runs := p.config.GetRuns() + for i := 0; i < runs; i++ { + for _, test := range tests { + t, ok := test.(SysbenchTest) + if !ok { + return ErrNotSysbenchTest + } + tester := NewSysbenchTester(p.config, p.serverConfig, t, profileParams, stampFunc) + r, err := tester.Test(ctx) + if err != nil { + server.Stop() + return err + } + results = append(results, r) + } + } + + err = server.Stop() + if err != nil { + return err + } + + info, err := os.Stat(tempProfile) + if err != nil { + return err + } + + if info.Size() < 1 { + return fmt.Errorf("failed to create profile: file was empty") + } + + finalProfile := filepath.Join(p.serverConfig.GetProfilePath(), fmt.Sprintf("%s_%s", p.serverConfig.GetId(), cpuProfileFilename)) + return os.Rename(tempProfile, finalProfile) +} diff --git a/go/performance/utils/sysbench_runner/results.go b/go/performance/utils/benchmark_runner/results.go similarity index 85% rename from go/performance/utils/sysbench_runner/results.go rename to go/performance/utils/benchmark_runner/results.go index 5ad7256196b..9d7fed6be05 100644 --- a/go/performance/utils/sysbench_runner/results.go +++ b/go/performance/utils/benchmark_runner/results.go @@ -12,41 +12,18 @@ // See the License for the specific language governing permissions and // limitations under the License. -package sysbench_runner +package benchmark_runner import ( "errors" "path/filepath" "strconv" "strings" - "time" "github.com/google/uuid" ) -const ( - stampFormat = time.RFC3339 - SqlStatsPrefix = "SQL statistics:" - read = "read" - write = "write" - other = "other" - totalQueries = "total" - totalEvents = "total number of events" - min = "min" - avg = "avg" - max = "max" - percentile = "percentile" - sum = "sum" - transactions = "transactions" - queriesPerSec = "queries" - ignoredErrors = "ignored errors" - reconnects = "reconnects" - totalTimeSecs = "total time" -) - var ( - ResultFileTemplate = "%s_%s_%s_sysbench_performance%s" - ErrUnableToParseOutput = errors.New("unable to parse output") ErrUnsupportedHeaderField = errors.New("unsupported header field") ) @@ -152,13 +129,7 @@ func (r *Result) Stamp(stampFunc func() string) { r.CreatedAt = stampFunc() } -// FromConfigsNewResult returns a new result with some fields set based on the provided configs -func FromConfigsNewResult(config *Config, serverConfig *ServerConfig, t *Test, suiteId string, idFunc func() string) (*Result, error) { - serverParams, err := serverConfig.GetServerArgs() - if err != nil { - return nil, err - } - +func NewResult(server ServerType, version, testName, testId, suiteId, runtimeOs, runtimeGoArch string, serverParams, testParams []string, idFunc func() string, fromScript bool) *Result { var getId func() string if idFunc == nil { getId = func() string { @@ -169,53 +140,26 @@ func FromConfigsNewResult(config *Config, serverConfig *ServerConfig, t *Test, s } var name string - if t.FromScript { - base := filepath.Base(t.Name) + if fromScript { + base := filepath.Base(testName) ext := filepath.Ext(base) name = strings.TrimSuffix(base, ext) } else { - name = t.Name + name = testName } return &Result{ Id: getId(), SuiteId: suiteId, - TestId: t.id, - RuntimeOS: config.RuntimeOS, - RuntimeGoArch: config.RuntimeGoArch, - ServerName: string(serverConfig.Server), - ServerVersion: serverConfig.Version, + TestId: testId, + RuntimeOS: runtimeOs, + RuntimeGoArch: runtimeGoArch, + ServerName: string(server), + ServerVersion: version, ServerParams: strings.Join(serverParams, " "), TestName: name, - TestParams: strings.Join(t.Params, " "), - }, nil -} - -// FromOutputResult accepts raw sysbench run output and returns the Result -func FromOutputResult(output []byte, config *Config, serverConfig *ServerConfig, test *Test, suiteId string, idFunc func() string) (*Result, error) { - result, err := FromConfigsNewResult(config, serverConfig, test, suiteId, idFunc) - if err != nil { - return nil, err - } - lines := strings.Split(string(output), "\n") - var process bool - for _, l := range lines { - trimmed := strings.TrimSpace(l) - if trimmed == "" { - continue - } - if strings.HasPrefix(trimmed, SqlStatsPrefix) { - process = true - continue - } - if process { - err := UpdateResult(result, trimmed) - if err != nil { - return result, err - } - } + TestParams: strings.Join(testParams, " "), } - return result, nil } // UpdateResult extracts the key and value from the given line and updates the given Result @@ -368,6 +312,31 @@ func updateResult(result *Result, key, val string) error { return nil } +func OutputToResult(output []byte, server ServerType, version, testName, testId, suiteId, runtimeOs, runtimeGoArch string, serverParams, testParams []string, idFunc func() string, fromScript bool) (*Result, error) { + result := NewResult(server, version, testName, testId, suiteId, runtimeOs, runtimeGoArch, serverParams, testParams, idFunc, fromScript) + + lines := strings.Split(string(output), "\n") + var process bool + for _, l := range lines { + trimmed := strings.TrimSpace(l) + if trimmed == "" { + continue + } + if strings.HasPrefix(trimmed, SqlStatsPrefix) { + process = true + continue + } + if process { + err := UpdateResult(result, trimmed) + if err != nil { + return result, err + } + } + } + + return result, nil +} + // FromValWithParens takes a string containing parens and // returns the value outside the parens first, and the value // inside the parens second diff --git a/go/performance/utils/sysbench_runner/results_test.go b/go/performance/utils/benchmark_runner/results_test.go similarity index 90% rename from go/performance/utils/sysbench_runner/results_test.go rename to go/performance/utils/benchmark_runner/results_test.go index b1e3cb5c345..4ed9a64844c 100644 --- a/go/performance/utils/sysbench_runner/results_test.go +++ b/go/performance/utils/benchmark_runner/results_test.go @@ -12,11 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. -package sysbench_runner +package benchmark_runner import ( "fmt" "math/rand" + "strings" "testing" "time" @@ -263,38 +264,43 @@ func fromResultSysbenchOutput(r *Result) string { } func TestFromOutputResults(t *testing.T) { + params := NewSysbenchTestParams() + params.Append(testTestParams) + + serverParams := defaultDoltServerParams + serverParams = append(serverParams, testServerParams) + tests := []struct { description string output []byte - config *Config - serverConfig *ServerConfig - test *Test + config Config + serverConfig ServerConfig + test SysbenchTest expectedResult *Result expectedError error }{ { description: "should parse output into result", output: []byte(sampleOutput1), - config: &Config{ + config: &sysbenchRunnerConfigImpl{ RuntimeOS: testOS, RuntimeGoArch: testGoArch, }, - serverConfig: &ServerConfig{ - Host: "localhost", - Server: ServerType(testServer), + serverConfig: &doltServerConfigImpl{ Version: testServerVersion, ServerExec: "test-exec", }, - test: &Test{ + test: &sysbenchTestImpl{ Name: testTestName, - Params: []string{testTestParams}, + Params: params, }, expectedResult: &Result{ Id: testId, SuiteId: testSuiteId, RuntimeOS: testOS, RuntimeGoArch: testGoArch, - ServerName: testServer, + ServerName: string(Dolt), + ServerParams: strings.Join(defaultDoltServerParams, " "), ServerVersion: testServerVersion, TestName: testTestName, TestParams: testTestParams, @@ -315,11 +321,12 @@ func TestFromOutputResults(t *testing.T) { }, }, } + for _, test := range tests { t.Run(test.description, func(t *testing.T) { - actual, err := FromOutputResult(test.output, test.config, test.serverConfig, test.test, testSuiteId, func() string { - return testId - }) + serverParams, err := test.serverConfig.GetServerArgs() + assert.NoError(t, err) + actual, err := OutputToResult(test.output, test.serverConfig.GetServerType(), test.serverConfig.GetVersion(), test.test.GetName(), test.test.GetId(), testSuiteId, test.config.GetRuntimeOs(), test.config.GetRuntimeGoArch(), serverParams, test.test.GetParamsToSlice(), func() string { return testId }, test.test.GetFromScript()) assert.Equal(t, test.expectedError, err) assert.Equal(t, test.expectedResult, actual) }) diff --git a/go/performance/utils/benchmark_runner/run.go b/go/performance/utils/benchmark_runner/run.go new file mode 100644 index 00000000000..a22964f41d6 --- /dev/null +++ b/go/performance/utils/benchmark_runner/run.go @@ -0,0 +1,141 @@ +// Copyright 2019-2022 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package benchmark_runner + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" +) + +var ErrNotProtocolServerConfig = errors.New("protocol server config required") +var ErrNotInitDbServerConfig = errors.New("init db server config required") + +// Run runs sysbench runner +func Run(ctx context.Context, config SysbenchConfig) error { + err := config.Validate(ctx) + if err != nil { + return err + } + + err = sysbenchVersion(ctx) + if err != nil { + return err + } + + cwd, err := os.Getwd() + if err != nil { + return err + } + + svs := config.GetServerConfigs() + for _, serverConfig := range svs { + var results Results + var b Benchmarker + st := serverConfig.GetServerType() + switch st { + case Dolt: + // handle a profiling run + sc, ok := serverConfig.(ProfilingServerConfig) + if ok { + if string(sc.GetServerProfile()) != "" { + fmt.Println("Profiling dolt while running sysbench tests") + p := NewDoltProfiler(cwd, config, sc) + return p.Profile(ctx) + } + } + + fmt.Println("Running dolt sysbench tests") + b = NewDoltBenchmarker(cwd, config, serverConfig) + case Doltgres: + fmt.Println("Running doltgres sysbench tests") + b = NewDoltgresBenchmarker(cwd, config, serverConfig) + case MySql: + sc, ok := serverConfig.(ProtocolServerConfig) + if !ok { + return ErrNotProtocolServerConfig + } + fmt.Println("Running mysql sysbench tests") + b = NewMysqlBenchmarker(cwd, config, sc) + case Postgres: + sc, ok := serverConfig.(InitServerConfig) + if !ok { + return ErrNotInitDbServerConfig + } + fmt.Println("Running postgres sysbench tests") + b = NewPostgresBenchmarker(cwd, config, sc) + default: + panic(fmt.Sprintf("unexpected server type: %s", st)) + } + + results, err = b.Benchmark(ctx) + if err != nil { + return err + } + + fmt.Printf("Successfuly finished %s\n", st) + + err = WriteResults(serverConfig, results) + if err != nil { + return err + } + + fmt.Printf("Successfuly wrote results for %s\n", st) + } + return nil +} + +func sysbenchVersion(ctx context.Context) error { + version := ExecCommand(ctx, sysbenchCommand, sysbenchVersionFlag) + return version.Run() +} + +func WriteResults(serverConfig ServerConfig, results Results) error { + st := serverConfig.GetServerType() + version := serverConfig.GetVersion() + id := serverConfig.GetId() + format := serverConfig.GetResultsFormat() + + cwd, err := os.Getwd() + if err != nil { + return err + } + + //var writePath string + switch format { + case CsvFormat, CsvExt: + writePath := filepath.Join( + cwd, + resultsDirname, + string(st), + version, + id, + fmt.Sprintf(ResultFileTemplate, id, st, version, CsvExt)) + return WriteResultsCsv(writePath, results) + case JsonFormat, JsonExt: + writePath := filepath.Join( + cwd, + resultsDirname, + string(st), + version, + id, + fmt.Sprintf(ResultFileTemplate, id, st, version, JsonExt)) + return WriteResultsJson(writePath, results) + default: + return fmt.Errorf("unsupported results format: %s", format) + } +} diff --git a/go/performance/utils/benchmark_runner/run_test.go b/go/performance/utils/benchmark_runner/run_test.go new file mode 100644 index 00000000000..f97d261e673 --- /dev/null +++ b/go/performance/utils/benchmark_runner/run_test.go @@ -0,0 +1,265 @@ +// Copyright 2022 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package benchmark_runner + +import ( + "context" + "errors" + "fmt" + "log" + "os" + "path/filepath" + "testing" +) + +var runTests = os.Getenv("RUN_BENCHMARK_RUNNER_TESTS") + +func TestRunner(t *testing.T) { + if runTests == "" { + t.Skip() + } + dir := t.TempDir() + log.Println(dir) + err := os.Chdir(dir) + if err != nil { + log.Fatal(err) + } + + conf := &sysbenchRunnerConfigImpl{ + Tests: selectTests("oltp_read_write", "oltp_update_index", "oltp_delete_insert"), + //Tests: selectTests("oltp_read_write", "oltp_update_index", "oltp_update_non_index", "oltp_insert", "bulk_insert", "oltp_write_only", "oltp_delete"), + Servers: []ServerConfig{ + &doltServerConfigImpl{ + Id: "test", + Version: "0.39.2", + ResultsFormat: CsvFormat, + ServerExec: "/Users/max-hoffman/go/bin/dolt", + }, + }, + ScriptDir: "/Users/max-hoffman/Documents/dolthub/sysbench-lua-scripts", + TestOptions: []string{ + "--rand-seed=1", + "--table-size=10000", + "--rand-type=uniform", + "--time=120", + "--percentile=50", + }, + InitBigRepo: true, + } + + err = Run(context.Background(), conf) + if err != nil { + log.Fatal(err) + } +} + +func selectTests(names ...string) []TestConfig { + tests := make([]TestConfig, len(names)) + for i := range names { + tests[i] = &testConfigImpl{Name: names[i], FromScript: false} + } + return tests +} + +func TestDoltMysqlSysbenchRunner(t *testing.T) { + if runTests == "" { + t.Skip() + } + dir := t.TempDir() + log.Println(dir) + err := os.Chdir(dir) + if err != nil { + log.Fatal(err) + } + + conf := &sysbenchRunnerConfigImpl{ + Tests: []TestConfig{ + NewTestConfig("oltp_read_write", nil, false), + NewTestConfig("oltp_update_index", nil, false), + NewTestConfig("oltp_delete_insert", nil, true), + }, + Servers: []ServerConfig{ + &doltServerConfigImpl{ + Id: "test-dolt", + Version: "1.33.0", + ResultsFormat: CsvFormat, + ServerExec: "/Users/dustin/go/bin/dolt", + }, + &mysqlServerConfigImpl{ + Id: "test-mysql", + Host: "127.0.0.1", + Port: 3606, + Version: "8.0.35", + ResultsFormat: CsvFormat, + ServerExec: "/opt/homebrew/bin/mysqld", + ServerUser: "root", + SkipLogBin: true, + ConnectionProtocol: "tcp", + }, + }, + ScriptDir: "/Users/dustin/src/sysbench-lua-scripts", + TestOptions: []string{ + "--rand-seed=1", + "--table-size=10000", + "--rand-type=uniform", + "--time=120", + "--percentile=50", + }, + InitBigRepo: true, + } + + err = Run(context.Background(), conf) + if err != nil { + log.Fatal(err) + } +} + +func TestDoltgresPostgresSysbenchRunner(t *testing.T) { + if runTests == "" { + t.Skip() + } + dir := t.TempDir() + log.Println(dir) + err := os.Chdir(dir) + if err != nil { + log.Fatal(err) + } + + conf := &sysbenchRunnerConfigImpl{ + Tests: []TestConfig{ + NewTestConfig("oltp_read_write", nil, false), + NewTestConfig("oltp_update_index", nil, false), + }, + Servers: []ServerConfig{ + &postgresServerConfigImpl{ + Id: "test-postgres", + Host: "127.0.0.1", + Version: "15.5", + ResultsFormat: CsvFormat, + ServerExec: "/opt/homebrew/opt/postgresql@15/bin/postgres", + InitExec: "/opt/homebrew/opt/postgresql@15/bin/initdb", + ServerUser: "root", + }, + &doltgresServerConfigImpl{ + Id: "test-doltgres", + Port: 4433, + Host: "127.0.0.1", + Version: "b139dfb", + ResultsFormat: CsvFormat, + ServerExec: "/Users/dustin/go/bin/doltgres", + }, + }, + ScriptDir: "/Users/dustin/src/sysbench-lua-scripts", + TestOptions: []string{ + "--rand-seed=1", + "--table-size=10000", + "--rand-type=uniform", + "--time=120", + "--percentile=50", + }, + InitBigRepo: true, + } + + err = Run(context.Background(), conf) + if err != nil { + log.Fatal(err) + } +} + +func TestDoltProfiler(t *testing.T) { + if runTests == "" { + t.Skip() + } + dir := t.TempDir() + log.Println(dir) + err := os.Chdir(dir) + if err != nil { + log.Fatal(err) + } + + id := "test-dolt-profile" + conf := &sysbenchRunnerConfigImpl{ + Tests: []TestConfig{ + NewTestConfig("oltp_read_write", nil, false), + }, + Servers: []ServerConfig{ + &doltServerConfigImpl{ + Id: id, + Version: "1.33.0", + ResultsFormat: CsvFormat, + ServerExec: "/Users/dustin/go/bin/dolt", + ServerProfile: CpuServerProfile, + ProfilePath: dir, + }, + }, + TestOptions: []string{ + "--rand-seed=1", + "--table-size=10000", + "--rand-type=uniform", + "--time=30", + "--percentile=50", + }, + } + + err = Run(context.Background(), conf) + if err != nil { + log.Fatal(err) + } + + expectedProfile := filepath.Join(dir, fmt.Sprintf("%s_%s", id, cpuProfileFilename)) + if _, err := os.Stat(expectedProfile); errors.Is(err, os.ErrNotExist) { + log.Fatal("failed to create dolt cpu profile") + } +} + +func TestDoltMysqlTpccRunner(t *testing.T) { + if runTests == "" { + t.Skip() + } + dir := t.TempDir() + log.Println(dir) + err := os.Chdir(dir) + if err != nil { + log.Fatal(err) + } + + conf := &tpccConfigImpl{ + Servers: []ServerConfig{ + &doltServerConfigImpl{ + Id: "test-dolt-tpcc", + Version: "1.33.0", + ResultsFormat: CsvFormat, + ServerExec: "/Users/dustin/go/bin/dolt", + }, + &mysqlServerConfigImpl{ + Id: "test-mysql-tpcc", + Host: "127.0.0.1", + Port: 3606, + Version: "8.0.35", + ResultsFormat: CsvFormat, + ServerExec: "/opt/homebrew/bin/mysqld", + ServerUser: "root", + SkipLogBin: true, + ConnectionProtocol: "tcp", + }, + }, + ScriptDir: "/Users/dustin/src/sysbench-tpcc", + } + + err = RunTpcc(context.Background(), conf) + if err != nil { + log.Fatal(err) + } +} diff --git a/go/performance/utils/benchmark_runner/run_tpcc.go b/go/performance/utils/benchmark_runner/run_tpcc.go new file mode 100644 index 00000000000..10a3710ca8e --- /dev/null +++ b/go/performance/utils/benchmark_runner/run_tpcc.go @@ -0,0 +1,69 @@ +// Copyright 2022 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package benchmark_runner + +import ( + "context" + "fmt" + "os" +) + +func RunTpcc(ctx context.Context, config TpccConfig) error { + err := config.Validate(ctx) + if err != nil { + return err + } + + cwd, err := os.Getwd() + if err != nil { + return err + } + + svs := config.GetServerConfigs() + for _, serverConfig := range svs { + var b Benchmarker + var results Results + st := serverConfig.GetServerType() + switch st { + case Dolt: + fmt.Println("Running dolt tpcc benchmarks") + b = NewDoltTpccBenchmarker(cwd, config, serverConfig) + case MySql: + sc, ok := serverConfig.(ProtocolServerConfig) + if !ok { + return ErrNotProtocolServerConfig + } + + fmt.Println("Running mysql tpcc benchmarks") + b = NewMysqlTpccBenchmarker(cwd, config, sc) + default: + panic(fmt.Sprintf("unexpected server type: %s", st)) + } + + results, err = b.Benchmark(ctx) + if err != nil { + return err + } + + err = WriteResults(serverConfig, results) + if err != nil { + return err + } + + fmt.Printf("Successfuly wrote results for %s\n", st) + } + + return nil +} diff --git a/go/performance/utils/benchmark_runner/server.go b/go/performance/utils/benchmark_runner/server.go new file mode 100644 index 00000000000..fbbca1f5c63 --- /dev/null +++ b/go/performance/utils/benchmark_runner/server.go @@ -0,0 +1,126 @@ +// Copyright 2019-2022 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package benchmark_runner + +import ( + "context" + "errors" + "fmt" + "os" + "os/exec" + "time" + + "golang.org/x/sync/errgroup" +) + +var ErrServerClosed = errors.New("server was previously closed") + +type Server interface { + Start() error + Stop() error + WithEnv(key, value string) +} + +type doltServerImpl struct { + dir string + serverConfig ServerConfig + serverCtx context.Context + serverCtxCancelFunc context.CancelFunc + server *exec.Cmd + serverEg *errgroup.Group + quit chan os.Signal + killSignal os.Signal +} + +var _ Server = &doltServerImpl{} + +func NewServer(ctx context.Context, dir string, serverConfig ServerConfig, killSignal os.Signal, serverParams []string) *doltServerImpl { + withKeyCtx, cancel := context.WithCancel(ctx) + gServer, serverCtx := errgroup.WithContext(withKeyCtx) + + server := ExecCommand(serverCtx, serverConfig.GetServerExec(), serverParams...) + server.Dir = dir + + quit := make(chan os.Signal, 1) + return &doltServerImpl{ + dir: dir, + serverConfig: serverConfig, + serverCtx: serverCtx, + server: server, + serverCtxCancelFunc: cancel, + serverEg: gServer, + quit: quit, + killSignal: killSignal, + } +} + +func (s *doltServerImpl) WithEnv(key, val string) { + if s.server != nil { + s.server.Env = append(s.server.Env, fmt.Sprintf("%s=%s", key, val)) + } +} + +func (s *doltServerImpl) Start() error { + if s.serverEg == nil || s.serverCtx == nil || s.quit == nil { + return ErrServerClosed + } + + s.serverEg.Go(func() error { + <-s.quit + return s.server.Process.Signal(s.killSignal) + }) + + s.serverEg.Go(func() error { + if Debug { + s.server.Stdout = os.Stdout + s.server.Stderr = os.Stderr + } + return s.server.Run() + }) + + // sleep to allow the server to start + time.Sleep(10 * time.Second) + fmt.Println("Successfully started database server") + return nil +} + +func (s *doltServerImpl) Stop() error { + defer s.serverCtxCancelFunc() + if s.serverEg != nil && s.serverCtx != nil && s.quit != nil { + // send signal to dolt server + s.quit <- s.killSignal + err := s.serverEg.Wait() + if err != nil { + // we expect a kill error + // we only exit in error if this is not the + // error + if err.Error() != expectedServerKilledErrorMessage && err.Error() != expectedServerTerminatedErrorMessage { + fmt.Println(err) + close(s.quit) + return err + } + } + + fmt.Println("Successfully killed database server") + close(s.quit) + + s.quit = nil + s.serverCtx = nil + s.serverEg = nil + s.serverCtxCancelFunc = func() {} + } + + return nil +} diff --git a/go/performance/utils/benchmark_runner/server_config.go b/go/performance/utils/benchmark_runner/server_config.go new file mode 100644 index 00000000000..37c02c41a77 --- /dev/null +++ b/go/performance/utils/benchmark_runner/server_config.go @@ -0,0 +1,48 @@ +// Copyright 2019-2022 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package benchmark_runner + +type ServerType string + +type ServerConfig interface { + GetId() string + GetHost() string + GetPort() int + GetVersion() string + GetServerExec() string + GetResultsFormat() string + GetServerType() ServerType + GetServerArgs() ([]string, error) + GetTestingParams(testConfig TestConfig) TestParams + Validate() error + SetDefaults() error +} + +type InitServerConfig interface { + ServerConfig + GetInitDbExec() string +} + +type ProtocolServerConfig interface { + ServerConfig + GetConnectionProtocol() string + GetSocket() string +} + +type ProfilingServerConfig interface { + ServerConfig + GetServerProfile() ServerProfile + GetProfilePath() string +} diff --git a/go/performance/utils/benchmark_runner/sysbench.go b/go/performance/utils/benchmark_runner/sysbench.go new file mode 100644 index 00000000000..71ff2230531 --- /dev/null +++ b/go/performance/utils/benchmark_runner/sysbench.go @@ -0,0 +1,160 @@ +// Copyright 2019-2022 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package benchmark_runner + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + + "github.com/google/uuid" +) + +type sysbenchTesterImpl struct { + test SysbenchTest + config Config + serverConfig ServerConfig + serverParams []string + stampFunc func() string + idFunc func() string + suiteId string +} + +var _ Tester = &sysbenchTesterImpl{} + +func NewSysbenchTester(config Config, serverConfig ServerConfig, test SysbenchTest, serverParams []string, stampFunc func() string) *sysbenchTesterImpl { + return &sysbenchTesterImpl{ + config: config, + serverParams: serverParams, + serverConfig: serverConfig, + test: test, + suiteId: serverConfig.GetId(), + stampFunc: stampFunc, + } +} + +func (t *sysbenchTesterImpl) newResult() (*Result, error) { + serverParams, err := t.serverConfig.GetServerArgs() + if err != nil { + return nil, err + } + + var getId func() string + if t.idFunc == nil { + getId = func() string { + return uuid.New().String() + } + } else { + getId = t.idFunc + } + + var name string + if t.test.GetFromScript() { + base := filepath.Base(t.test.GetName()) + ext := filepath.Ext(base) + name = strings.TrimSuffix(base, ext) + } else { + name = t.test.GetName() + } + + return &Result{ + Id: getId(), + SuiteId: t.suiteId, + TestId: t.test.GetId(), + RuntimeOS: t.config.GetRuntimeOs(), + RuntimeGoArch: t.config.GetRuntimeGoArch(), + ServerName: string(t.serverConfig.GetServerType()), + ServerVersion: t.serverConfig.GetVersion(), + ServerParams: strings.Join(serverParams, " "), + TestName: name, + TestParams: strings.Join(t.test.GetParamsToSlice(), " "), + }, nil +} + +func (t *sysbenchTesterImpl) outputToResult(output []byte) (*Result, error) { + return OutputToResult(output, t.serverConfig.GetServerType(), t.serverConfig.GetVersion(), t.test.GetName(), t.test.GetId(), t.suiteId, t.config.GetRuntimeOs(), t.config.GetRuntimeGoArch(), t.serverParams, t.test.GetParamsToSlice(), nil, t.test.GetFromScript()) +} + +func (t *sysbenchTesterImpl) prepare(ctx context.Context) error { + cmd := exec.CommandContext(ctx, sysbenchCommand, t.test.GetPrepareArgs(t.serverConfig)...) + if t.test.GetFromScript() { + lp := filepath.Join(t.config.GetScriptDir(), luaPath) + cmd.Env = os.Environ() + cmd.Env = append(cmd.Env, fmt.Sprintf(luaPathEnvVarTemplate, lp)) + } + out, err := cmd.Output() + if err != nil { + fmt.Println(string(out)) + return err + } + return nil +} + +func (t *sysbenchTesterImpl) run(ctx context.Context) (*Result, error) { + cmd := exec.CommandContext(ctx, sysbenchCommand, t.test.GetRunArgs(t.serverConfig)...) + if t.test.GetFromScript() { + lp := filepath.Join(t.config.GetScriptDir(), luaPath) + cmd.Env = os.Environ() + cmd.Env = append(cmd.Env, fmt.Sprintf(luaPathEnvVarTemplate, lp)) + } + + out, err := cmd.Output() + if err != nil { + fmt.Print(string(out)) + return nil, err + } + + if Debug == true { + fmt.Print(string(out)) + } + + rs, err := t.outputToResult(out) + if err != nil { + return nil, err + } + + rs.Stamp(t.stampFunc) + + return rs, nil +} + +func (t *sysbenchTesterImpl) cleanup(ctx context.Context) error { + cmd := ExecCommand(ctx, sysbenchCommand, t.test.GetCleanupArgs(t.serverConfig)...) + if t.test.GetFromScript() { + lp := filepath.Join(t.config.GetScriptDir(), luaPath) + cmd.Env = os.Environ() + cmd.Env = append(cmd.Env, fmt.Sprintf(luaPathEnvVarTemplate, lp)) + } + return cmd.Run() +} + +func (t *sysbenchTesterImpl) Test(ctx context.Context) (*Result, error) { + err := t.prepare(ctx) + if err != nil { + return nil, err + } + + fmt.Println("Running test", t.test.GetName()) + + rs, err := t.run(ctx) + if err != nil { + return nil, err + } + + return rs, t.cleanup(ctx) +} diff --git a/go/performance/utils/benchmark_runner/sysbench_config.go b/go/performance/utils/benchmark_runner/sysbench_config.go new file mode 100644 index 00000000000..dee682c8a33 --- /dev/null +++ b/go/performance/utils/benchmark_runner/sysbench_config.go @@ -0,0 +1,330 @@ +// Copyright 2019-2022 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package benchmark_runner + +import ( + "context" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "runtime" +) + +var ( + ErrTestNameNotDefined = errors.New("test name not defined") + ErrNoServersDefined = errors.New("servers not defined") + ErrTooManyServersDefined = errors.New("too many servers defined, two max") + ErrUnsupportedConnectionProtocol = errors.New("unsupported connection protocol") +) + +var defaultSysbenchParams = []string{ + fmt.Sprintf("%s=%s", sysbenchDbPsModeFlag, sysbenchDbPsModeDisable), + fmt.Sprintf("%s=%s", sysbenchRandTypeFlag, sysbenchRandTypeUniform), +} + +var defaultDoltServerParams = []string{doltSqlServerCommand} + +var defaultSysbenchTests = []TestConfig{ + NewTestConfig(sysbenchOltpReadOnlyTestName, []string{}, false), + NewTestConfig(sysbenchOltpInsertTestName, []string{}, false), + NewTestConfig(sysbenchBulkInsertTestName, []string{}, false), + NewTestConfig(sysbenchOltpPointSelectTestName, []string{}, false), + NewTestConfig(sysbenchSelectRandomPointsTestName, []string{}, false), + NewTestConfig(sysbenchSelectRandomRangesTestName, []string{}, false), + NewTestConfig(sysbenchOltpWriteOnlyTestName, []string{}, false), + NewTestConfig(sysbenchOltpReadWriteTestName, []string{}, false), + NewTestConfig(sysbenchOltpUpdateIndexTestName, []string{}, false), + NewTestConfig(sysbenchOltpUpdateNonIndexTestName, []string{}, false), +} + +var defaultDoltLuaScripts = map[string]string{ + sysbenchCoveringIndexScanLuaTestName: sysbenchCoveringIndexScanLuaTestName, + sysbenchGroupByScanLuaTestName: sysbenchGroupByScanLuaTestName, + sysbenchIndexJoinLuaTestName: sysbenchIndexJoinLuaTestName, + sysbenchIndexJoinScanLuaTestName: sysbenchIndexJoinScanLuaTestName, + sysbenchIndexScanLuaTestName: sysbenchIndexScanLuaTestName, + sysbenchOltpDeleteInsertLuaTestName: sysbenchOltpDeleteInsertLuaTestName, + sysbenchTableScanLuaTestName: sysbenchTableScanLuaTestName, + sysbenchTypesDeleteInsertLuaTestName: sysbenchTypesDeleteInsertLuaTestName, + sysbenchTypesTableScanLuaTestName: sysbenchTypesTableScanLuaTestName, +} + +// todo: check expressions need to be supported in doltgres for these +// todo: postgres does not have geometry types also +var defaultDoltgresLuaScripts = map[string]string{ + //sysbenchCoveringIndexScanPostgresLuaTestName: sysbenchCoveringIndexScanPostgresLuaTestName, + //sysbenchGroupByScanPostgresLuaTestName: sysbenchGroupByScanPostgresLuaTestName, + //sysbenchIndexJoinPostgresLuaTestName: sysbenchIndexJoinPostgresLuaTestName, + //sysbenchIndexJoinScanPostgresLuaTestName: sysbenchIndexJoinScanPostgresLuaTestName, + //sysbenchIndexScanPostgresLuaTestName: sysbenchIndexScanPostgresLuaTestName, + //sysbenchOltpDeleteInsertPostgresLuaTestName: sysbenchOltpDeleteInsertPostgresLuaTestName, + //sysbenchTableScanPostgresLuaTestName: sysbenchTableScanPostgresLuaTestName, + //sysbenchTypesDeleteInsertPostgresLuaTestName: sysbenchTypesDeleteInsertPostgresLuaTestName, + //sysbenchTypesTableScanPostgresLuaTestName: sysbenchTypesTableScanPostgresLuaTestName, +} + +// sysbenchRunnerConfigImpl is the configuration for a benchmarking run +type sysbenchRunnerConfigImpl struct { + // Runs is the number of times to run all tests + Runs int + // RuntimeOS is the platform the benchmarks ran on + RuntimeOS string + // RuntimeGoArch is the runtime architecture + RuntimeGoArch string + // Servers are the servers to benchmark + Servers []ServerConfig + // Tests are the tests to run. If no tests are provided, + // the default tests will be used + Tests []TestConfig + // TestOptions a list of sysbench test options to apply to all tests + TestOptions []string + // ScriptDir is a path to a directory of lua scripts + ScriptDir string + // InitBigRepo downloads a database with existing chunks and commits + InitBigRepo bool + // NomsBinFormat specifies the NomsBinFormat + NomsBinFormat string +} + +var _ SysbenchConfig = &sysbenchRunnerConfigImpl{} + +// NewRunnerConfig returns a new sysbenchRunnerConfigImpl +func NewRunnerConfig() *sysbenchRunnerConfigImpl { + return &sysbenchRunnerConfigImpl{ + Servers: make([]ServerConfig, 0), + } +} + +func (c *sysbenchRunnerConfigImpl) GetRuns() int { + return c.Runs +} + +func (c *sysbenchRunnerConfigImpl) GetScriptDir() string { + return c.ScriptDir +} + +func (c *sysbenchRunnerConfigImpl) GetNomsBinFormat() string { + return c.NomsBinFormat +} + +func (c *sysbenchRunnerConfigImpl) GetRuntimeOs() string { + return c.RuntimeOS +} + +func (c *sysbenchRunnerConfigImpl) GetRuntimeGoArch() string { + return c.RuntimeGoArch +} + +func (c *sysbenchRunnerConfigImpl) GetTestConfigs() []TestConfig { + return c.Tests +} + +func (c *sysbenchRunnerConfigImpl) GetTestOptions() []string { + return c.TestOptions +} + +func (c *sysbenchRunnerConfigImpl) GetServerConfigs() []ServerConfig { + return c.Servers +} + +// Validate checks the config for the required fields and sets defaults +// where necessary +func (c *sysbenchRunnerConfigImpl) Validate(ctx context.Context) error { + if len(c.Servers) < 1 { + return ErrNoServersDefined + } + if len(c.Servers) > 2 { + return ErrTooManyServersDefined + } + err := c.setDefaults() + if err != nil { + return err + } + return c.validateServerConfigs() +} + +// validateServerConfigs ensures the ServerConfigs are valid +func (c *sysbenchRunnerConfigImpl) validateServerConfigs() error { + portMap := make(map[int]ServerType) + for _, s := range c.Servers { + st := s.GetServerType() + if st != Dolt && st != MySql && st != Doltgres && st != Postgres { + return fmt.Errorf("unsupported server type: %s", st) + } + + err := s.Validate() + if err != nil { + return err + } + + err = s.SetDefaults() + if err != nil { + return err + } + + portMap, err = CheckUpdatePortMap(s, portMap) + if err != nil { + return err + } + } + + return nil +} + +func (c *sysbenchRunnerConfigImpl) ContainsServerOfType(st ServerType) bool { + for _, s := range c.Servers { + if s.GetServerType() == st { + return true + } + } + return false +} + +// setDefaults sets defaults on the sysbenchRunnerConfigImpl +func (c *sysbenchRunnerConfigImpl) setDefaults() error { + if c.RuntimeOS == "" { + c.RuntimeOS = runtime.GOOS + } + if c.RuntimeGoArch == "" { + c.RuntimeGoArch = runtime.GOARCH + } + if len(c.Tests) < 1 { + fmt.Printf("Preparing to benchmark against default tests\n") + if c.ScriptDir != "" { + abs, err := filepath.Abs(c.ScriptDir) + if err != nil { + return err + } + if _, err := os.Stat(abs); os.IsNotExist(err) { + return fmt.Errorf("script dir not found: %s", abs) + } + c.ScriptDir = abs + } + tests, err := c.getDefaultTests() + if err != nil { + return err + } + c.Tests = tests + } + if c.Runs < 1 { + c.Runs = 1 + } + return nil +} + +func (c *sysbenchRunnerConfigImpl) getLuaScriptTestsFromDir(toInclude map[string]string) ([]TestConfig, error) { + luaScripts := make([]TestConfig, 0) + abs, err := filepath.Abs(c.ScriptDir) + if err != nil { + return nil, err + } + err = filepath.Walk(abs, func(path string, info fs.FileInfo, err error) error { + if err != nil { + return err + } + + file := filepath.Base(path) + if _, ok := toInclude[file]; ok { + luaScripts = append(luaScripts, NewTestConfig(path, []string{}, true)) + } + return nil + }) + if err != nil { + return nil, err + } + return luaScripts, nil +} + +func (c *sysbenchRunnerConfigImpl) getDefaultTests() ([]TestConfig, error) { + defaultTests := make([]TestConfig, 0) + defaultTests = append(defaultTests, defaultSysbenchTests...) + if c.ScriptDir != "" { + var luaScriptTests []TestConfig + var err error + if !c.ContainsServerOfType(Doltgres) && !c.ContainsServerOfType(Postgres) { + luaScriptTests, err = c.getLuaScriptTestsFromDir(defaultDoltLuaScripts) + } else { + luaScriptTests, err = c.getLuaScriptTestsFromDir(defaultDoltgresLuaScripts) + } + if err != nil { + return nil, err + } + defaultTests = append(defaultTests, luaScriptTests...) + } + return defaultTests, nil +} + +// CheckUpdatePortMap returns an error if multiple servers have specified the same port +func CheckUpdatePortMap(serverConfig ServerConfig, portMap map[int]ServerType) (map[int]ServerType, error) { + port := serverConfig.GetPort() + st := serverConfig.GetServerType() + srv, ok := portMap[port] + if ok && srv != st { + return nil, fmt.Errorf("servers have port conflict on port: %d\n", port) + } + if !ok { + portMap[port] = st + } + return portMap, nil +} + +// CheckExec verifies the binary exists +func CheckExec(path, messageIfMissing string) error { + if path == "" { + return getMustSupplyError(messageIfMissing) + } + abs, err := filepath.Abs(path) + if err != nil { + return err + } + if _, err := os.Stat(abs); os.IsNotExist(err) { + return fmt.Errorf("exec not found: %s", abs) + } + return nil +} + +// CheckProtocol ensures the given protocol is supported +func CheckProtocol(protocol string) error { + if protocol == "" { + return getMustSupplyError("connection protocol") + } + if protocol == tcpProtocol || protocol == unixProtocol { + return nil + } + return ErrUnsupportedConnectionProtocol +} + +// GetTests returns a slice of Tests +func GetTests(config SysbenchConfig, serverConfig ServerConfig) ([]Test, error) { + flattened := make([]Test, 0) + for _, t := range config.GetTestConfigs() { + opts := config.GetTestOptions() + for _, o := range opts { + t.AppendOption(o) + } + tests, err := t.GetTests(serverConfig) + if err != nil { + return nil, err + } + flattened = append(flattened, tests...) + } + return flattened, nil +} + +func getMustSupplyError(name string) error { + return fmt.Errorf("Must supply %s", name) +} diff --git a/go/performance/utils/benchmark_runner/sysbench_config_test.go b/go/performance/utils/benchmark_runner/sysbench_config_test.go new file mode 100644 index 00000000000..a792a5f41c1 --- /dev/null +++ b/go/performance/utils/benchmark_runner/sysbench_config_test.go @@ -0,0 +1,133 @@ +// Copyright 2019-2022 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package benchmark_runner + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestConfigTestGetTests(t *testing.T) { + empty := &testConfigImpl{Name: "test_name"} + + one := &testConfigImpl{Name: "test_one", N: 3} + two := &testConfigImpl{Name: "test_two", N: 2} + three := &testConfigImpl{Name: "test_three", N: 1} + + opts := &testConfigImpl{ + Name: "test_options", + N: 1, + Options: []string{"--create_secondary=on", "--auto_inc=off"}, + } + + serverConfig := &doltServerConfigImpl{Version: "test-version", Host: "localhost", ResultsFormat: CsvFormat} + + tests := []struct { + description string + config SysbenchConfig + expectedTests []testTest + expectedError error + }{ + { + description: "should error if no test name is defined", + config: &sysbenchRunnerConfigImpl{ + Servers: []ServerConfig{serverConfig}, + Tests: []TestConfig{ + &testConfigImpl{Name: ""}, + }, + }, + expectedTests: nil, + expectedError: ErrTestNameNotDefined, + }, + { + description: "should create single test if N is < 1", + config: &sysbenchRunnerConfigImpl{ + Servers: []ServerConfig{serverConfig}, + Tests: []TestConfig{empty}, + }, + expectedTests: []testTest{ + &testTestImpl{ + &sysbenchTestImpl{ + Name: "test_name", + Params: serverConfig.GetTestingParams(empty), + }, + }, + }, + }, + { + description: "should return a test for each N defined on the TestConfigImpl", + config: &sysbenchRunnerConfigImpl{ + Servers: []ServerConfig{serverConfig}, + Tests: []TestConfig{one, two, three}, + }, + expectedTests: []testTest{ + &testTestImpl{&sysbenchTestImpl{Name: "test_one", Params: serverConfig.GetTestingParams(one)}}, + &testTestImpl{&sysbenchTestImpl{Name: "test_one", Params: serverConfig.GetTestingParams(one)}}, + &testTestImpl{&sysbenchTestImpl{Name: "test_one", Params: serverConfig.GetTestingParams(one)}}, + &testTestImpl{&sysbenchTestImpl{Name: "test_two", Params: serverConfig.GetTestingParams(two)}}, + &testTestImpl{&sysbenchTestImpl{Name: "test_two", Params: serverConfig.GetTestingParams(two)}}, + &testTestImpl{&sysbenchTestImpl{Name: "test_three", Params: serverConfig.GetTestingParams(three)}}, + }, + }, + { + description: "should apply user options to test params", + config: &sysbenchRunnerConfigImpl{ + Servers: []ServerConfig{serverConfig}, + Tests: []TestConfig{opts}, + }, + expectedTests: []testTest{ + &testTestImpl{&sysbenchTestImpl{Name: "test_options", Params: serverConfig.GetTestingParams(opts)}}, + }, + }, + } + for _, test := range tests { + t.Run(test.description, func(t *testing.T) { + svs := test.config.GetServerConfigs() + for _, s := range svs { + actual, err := GetTests(test.config, s) + assert.Equal(t, test.expectedError, err) + assert.Equal(t, len(test.expectedTests), len(actual)) + updatedExpected := make([]SysbenchTest, len(actual)) + for idx, a := range actual { + e := test.expectedTests[idx] + e.SetId(a.GetId()) + updatedExpected[idx] = e.GetSysbenchTest() + } + assert.ElementsMatch(t, updatedExpected, actual) + } + }) + } +} + +type testTest interface { + SetId(id string) + GetSysbenchTest() SysbenchTest + SysbenchTest +} + +type testTestImpl struct { + *sysbenchTestImpl +} + +func (t *testTestImpl) GetSysbenchTest() SysbenchTest { + return t.sysbenchTestImpl +} + +func (t *testTestImpl) SetId(id string) { + t.id = id +} + +var _ testTest = &testTestImpl{} diff --git a/go/performance/utils/benchmark_runner/sysbench_tests.go b/go/performance/utils/benchmark_runner/sysbench_tests.go new file mode 100644 index 00000000000..496bfea4caa --- /dev/null +++ b/go/performance/utils/benchmark_runner/sysbench_tests.go @@ -0,0 +1,95 @@ +// Copyright 2019-2022 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package benchmark_runner + +type sysbenchTestParamsImpl struct { + params []string +} + +var _ SysbenchTestParams = &sysbenchTestParamsImpl{} + +func (s *sysbenchTestParamsImpl) ToSlice() []string { + return s.params +} + +func (s *sysbenchTestParamsImpl) Append(params ...string) { + s.params = append(s.params, params...) +} + +func NewSysbenchTestParams() *sysbenchTestParamsImpl { + return &sysbenchTestParamsImpl{params: make([]string, 0)} +} + +// sysbenchTestImpl is a single sysbench test +type sysbenchTestImpl struct { + id string + + // Name is the test name + Name string + + // Params are the parameters passed to sysbench + Params TestParams + + // FromScript indicates if this test is from a lua script + FromScript bool +} + +var _ SysbenchTest = &sysbenchTestImpl{} + +func NewSysbenchTest(id, name string, params TestParams, fromScript bool) *sysbenchTestImpl { + return &sysbenchTestImpl{ + id: id, + Name: name, + Params: params, + FromScript: fromScript, + } +} + +func (t *sysbenchTestImpl) GetId() string { + return t.id +} + +func (t *sysbenchTestImpl) GetName() string { + return t.Name +} + +func (t *sysbenchTestImpl) GetParamsToSlice() []string { + return t.Params.ToSlice() +} + +func (t *sysbenchTestImpl) GetFromScript() bool { + return t.FromScript +} + +// PrepareArgs returns a test's args for sysbench's prepare step +func (t *sysbenchTestImpl) GetPrepareArgs(serverConfig ServerConfig) []string { + return withCommand(t.Params, sysbenchPrepareCommand) +} + +// Run returns a test's args for sysbench's run step +func (t *sysbenchTestImpl) GetRunArgs(serverConfig ServerConfig) []string { + return withCommand(t.Params, sysbenchRunCommand) +} + +// Cleanup returns a test's args for sysbench's cleanup step +func (t *sysbenchTestImpl) GetCleanupArgs(serverConfig ServerConfig) []string { + return withCommand(t.Params, sysbenchCleanupCommand) +} + +func withCommand(params TestParams, command string) []string { + c := make([]string, 0) + c = append(c, params.ToSlice()...) + return append(c, command) +} diff --git a/go/performance/utils/benchmark_runner/test_config.go b/go/performance/utils/benchmark_runner/test_config.go new file mode 100644 index 00000000000..95f3d4aef83 --- /dev/null +++ b/go/performance/utils/benchmark_runner/test_config.go @@ -0,0 +1,88 @@ +// Copyright 2019-2022 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package benchmark_runner + +import "github.com/google/uuid" + +type TestConfig interface { + GetName() string + GetOptions() []string + AppendOption(opt string) + GetTests(serverConfig ServerConfig) ([]Test, error) + NewId() string +} + +type testConfigImpl struct { + // Name is the test name + Name string + + // N is the number of times a test should run + N int + + // Options are additional sysbench test options a user can supply to run with this test + Options []string + + // FromScript is a boolean indicating that this test is from a lua script + FromScript bool +} + +var _ TestConfig = &testConfigImpl{} + +func NewTestConfig(name string, opts []string, fromScript bool) *testConfigImpl { + options := make([]string, 0) + options = append(options, opts...) + return &testConfigImpl{ + Name: name, + N: 1, + Options: options, + FromScript: fromScript, + } +} + +func (ct *testConfigImpl) NewId() string { + return uuid.New().String() +} + +func (ct *testConfigImpl) GetName() string { + return ct.Name +} + +func (ct *testConfigImpl) GetOptions() []string { + return ct.Options +} + +func (ct *testConfigImpl) AppendOption(opt string) { + ct.Options = append(ct.Options, opt) +} + +func (ct *testConfigImpl) GetTests(serverConfig ServerConfig) ([]Test, error) { + if ct.Name == "" { + return nil, ErrTestNameNotDefined + } + if ct.N < 1 { + ct.N = 1 + } + + params := serverConfig.GetTestingParams(ct) + tests := make([]Test, 0) + + for i := 0; i < ct.N; i++ { + //p := make([]string, params.Len()) + //copy(p, params) + tests = append(tests, NewSysbenchTest(ct.NewId(), ct.Name, params, ct.FromScript)) + } + + return tests, nil +} diff --git a/go/performance/utils/benchmark_runner/tester.go b/go/performance/utils/benchmark_runner/tester.go new file mode 100644 index 00000000000..7a2bed306cc --- /dev/null +++ b/go/performance/utils/benchmark_runner/tester.go @@ -0,0 +1,55 @@ +// Copyright 2019-2022 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package benchmark_runner + +import "context" + +type Tester interface { + Test(ctx context.Context) (*Result, error) +} + +type Test interface { + GetId() string + GetName() string + GetParamsToSlice() []string + GetPrepareArgs(serverConfig ServerConfig) []string + GetRunArgs(serverConfig ServerConfig) []string + GetCleanupArgs(serverConfig ServerConfig) []string +} + +type SysbenchTest interface { + Test + GetFromScript() bool +} + +type TestParams interface { + ToSlice() []string +} + +type SysbenchTestParams interface { + TestParams + Append(params ...string) +} + +type TpccTestParams interface { + TestParams + GetNumThreads() int + GetScaleFactor() int + GetTables() int + GetTrxLevel() string + GetReportCSV() bool + GetReportInterval() int + GetTime() int +} diff --git a/go/performance/utils/benchmark_runner/tpcc.go b/go/performance/utils/benchmark_runner/tpcc.go new file mode 100644 index 00000000000..e8dc59c33f1 --- /dev/null +++ b/go/performance/utils/benchmark_runner/tpcc.go @@ -0,0 +1,123 @@ +// Copyright 2019-2022 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package benchmark_runner + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" +) + +type tpccTesterImpl struct { + test Test + config Config + serverConfig ServerConfig + tpccCommand string + serverParams []string + stampFunc func() string + idFunc func() string + suiteId string +} + +var _ Tester = &tpccTesterImpl{} + +func NewTpccTester(config TpccConfig, serverConfig ServerConfig, test Test, serverParams []string, stampFunc func() string) *tpccTesterImpl { + return &tpccTesterImpl{ + tpccCommand: filepath.Join(config.GetScriptDir(), tpccLuaFilename), + config: config, + serverParams: serverParams, + serverConfig: serverConfig, + test: test, + suiteId: serverConfig.GetId(), + stampFunc: stampFunc, + } +} + +func (t *tpccTesterImpl) outputToResult(output []byte) (*Result, error) { + return OutputToResult(output, t.serverConfig.GetServerType(), t.serverConfig.GetVersion(), t.test.GetName(), t.test.GetId(), t.suiteId, t.config.GetRuntimeOs(), t.config.GetRuntimeGoArch(), t.serverParams, t.test.GetParamsToSlice(), nil, false) +} + +func (t *tpccTesterImpl) prepare(ctx context.Context) error { + args := t.test.GetPrepareArgs(t.serverConfig) + cmd := ExecCommand(ctx, t.tpccCommand, args...) + cmd = t.updateCmdEnv(cmd) + out, err := cmd.Output() + if err != nil { + fmt.Println(string(out)) + return err + } + return nil +} + +func (t *tpccTesterImpl) run(ctx context.Context) (*Result, error) { + args := t.test.GetRunArgs(t.serverConfig) + cmd := ExecCommand(ctx, t.tpccCommand, args...) + cmd = t.updateCmdEnv(cmd) + + out, err := cmd.Output() + if err != nil { + fmt.Print(string(out)) + return nil, err + } + + if Debug == true { + fmt.Print(string(out)) + } + + rs, err := t.outputToResult(out) + if err != nil { + return nil, err + } + + rs.Stamp(t.stampFunc) + + return rs, nil +} + +func (t *tpccTesterImpl) cleanup(ctx context.Context) error { + args := t.test.GetCleanupArgs(t.serverConfig) + cmd := ExecCommand(ctx, t.tpccCommand, args...) + cmd = t.updateCmdEnv(cmd) + err := cmd.Run() + if err != nil { + return err + } + return nil +} + +func (t *tpccTesterImpl) Test(ctx context.Context) (*Result, error) { + err := t.prepare(ctx) + if err != nil { + return nil, err + } + + fmt.Println("Running test", t.test.GetName()) + + rs, err := t.run(ctx) + if err != nil { + return nil, err + } + + return rs, t.cleanup(ctx) +} + +func (t *tpccTesterImpl) updateCmdEnv(cmd *exec.Cmd) *exec.Cmd { + lp := filepath.Join(t.config.GetScriptDir(), luaPath) + cmd.Env = os.Environ() + cmd.Env = append(cmd.Env, fmt.Sprintf(luaPathEnvVarTemplate, lp)) + return cmd +} diff --git a/go/performance/utils/benchmark_runner/tpcc_config.go b/go/performance/utils/benchmark_runner/tpcc_config.go new file mode 100644 index 00000000000..ab568b0177c --- /dev/null +++ b/go/performance/utils/benchmark_runner/tpcc_config.go @@ -0,0 +1,145 @@ +// Copyright 2022 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package benchmark_runner + +import ( + "context" + "fmt" + "runtime" +) + +var defaultTpccParams = []string{ + fmt.Sprintf("%s=%s", tpccMysqlDbFlag, tpccDbName), + fmt.Sprintf("%s=%s", tpccDbDriverFlag, mysqlDriverName), +} + +// tpccConfigImpl represents a configuration for an execution of the TPCC Benchmark. It executes a series of tests +// against different ServerConfigurations. +type tpccConfigImpl struct { + // RuntimeOS is the platform the benchmarks ran on + RuntimeOS string + + // RuntimeGoArch is the runtime architecture + RuntimeGoArch string + + // ScriptDir represents the location of the TPCC tests + ScriptDir string + + // Servers are the servers to benchmark. + Servers []ServerConfig + + // ScaleFactors represent the scale at which to run each TpccBenchmark at. + ScaleFactors []int + + // NomsBinFormat specifies the NomsBinFormat + NomsBinFormat string +} + +var _ TpccConfig = &tpccConfigImpl{} + +func NewTpccRunnerConfig() *tpccConfigImpl { + return &tpccConfigImpl{ + Servers: make([]ServerConfig, 0), + ScaleFactors: make([]int, 0), + } +} + +func (c *tpccConfigImpl) GetRuns() int { + return 1 +} + +func (c *tpccConfigImpl) GetScriptDir() string { + return c.ScriptDir +} + +func (c *tpccConfigImpl) GetNomsBinFormat() string { + return c.NomsBinFormat +} + +func (c *tpccConfigImpl) GetRuntimeOs() string { + return c.RuntimeOS +} + +func (c *tpccConfigImpl) GetRuntimeGoArch() string { + return c.RuntimeGoArch +} + +func (c *tpccConfigImpl) ContainsServerOfType(st ServerType) bool { + for _, s := range c.Servers { + if s.GetServerType() == st { + return true + } + } + return false +} + +func (c *tpccConfigImpl) GetScaleFactors() []int { + return c.ScaleFactors +} + +func (c *tpccConfigImpl) GetServerConfigs() []ServerConfig { + return c.Servers +} + +func (c *tpccConfigImpl) setDefaults() { + // TODO: Eventually we need to support scale factors all the way to 10 + if len(c.ScaleFactors) == 0 { + c.ScaleFactors = append(c.ScaleFactors, 1) + } + if c.RuntimeOS == "" { + c.RuntimeOS = runtime.GOOS + } + if c.RuntimeGoArch == "" { + c.RuntimeGoArch = runtime.GOARCH + } +} + +// validateServerConfigs ensures the ServerConfigs are valid +func (c *tpccConfigImpl) validateServerConfigs() error { + portMap := make(map[int]ServerType) + for _, s := range c.Servers { + st := s.GetServerType() + if st != Dolt && st != MySql && st != Doltgres && st != Postgres { + return fmt.Errorf("unsupported server type: %s", st) + } + + err := s.Validate() + if err != nil { + return err + } + + err = s.SetDefaults() + if err != nil { + return err + } + + portMap, err = CheckUpdatePortMap(s, portMap) + if err != nil { + return err + } + } + return nil +} + +func (c *tpccConfigImpl) Validate(ctx context.Context) error { + if len(c.Servers) < 1 { + return ErrNoServersDefined + } + if len(c.Servers) > 2 { + return ErrTooManyServersDefined + } + c.setDefaults() + return c.validateServerConfigs() +} diff --git a/go/performance/utils/benchmark_runner/tpcc_tests.go b/go/performance/utils/benchmark_runner/tpcc_tests.go new file mode 100644 index 00000000000..a7df1f10eb9 --- /dev/null +++ b/go/performance/utils/benchmark_runner/tpcc_tests.go @@ -0,0 +1,242 @@ +// Copyright 2019-2022 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package benchmark_runner + +import ( + "fmt" + + "github.com/google/uuid" +) + +type tpccTestParamsImpl struct { + // NumThreads represents the number of threads running queries concurrently. + NumThreads int + + // ScaleFactor represents the number of warehouse to test this at scale. + ScaleFactor int + + // Tables represents the number of tables created per warehouse. + Tables int + + // TrxLevel represents what transaction level to use + TrxLevel string + + // ReportCSV determines whether to report output as a csv. + ReportCSV bool + + // ReportInterval defines how often the tpcc benchmark outputs performance stats. + ReportInterval int + + // Time represents how long + Time int +} + +var _ TestParams = &tpccTestParamsImpl{} + +// NewDefaultTpccParams returns default TpccTestParams. +func NewDefaultTpccParams() *tpccTestParamsImpl { + return &tpccTestParamsImpl{ + NumThreads: 2, // TODO: When ready, expose as command line argument. + ScaleFactor: 1, + Tables: 1, + TrxLevel: tpccTransactionLevelRr, + ReportCSV: true, + ReportInterval: 1, + Time: 30, + } +} + +func (t *tpccTestParamsImpl) GetNumThreads() int { + return t.NumThreads +} + +func (t *tpccTestParamsImpl) GetScaleFactor() int { + return t.ScaleFactor +} + +func (t *tpccTestParamsImpl) GetTables() int { + return t.Tables +} + +func (t *tpccTestParamsImpl) GetTrxLevel() string { + return t.TrxLevel +} + +func (t *tpccTestParamsImpl) GetReportCSV() bool { + return t.ReportCSV +} + +func (t *tpccTestParamsImpl) GetReportInterval() int { + return t.ReportInterval +} + +func (t *tpccTestParamsImpl) GetTime() int { + return t.Time +} + +func (t *tpccTestParamsImpl) ToSlice() []string { + params := make([]string, 0) + params = append(params, fmt.Sprintf("%s=%d", tpccThreadsFlag, t.NumThreads)) + params = append(params, fmt.Sprintf("%s=%d", tpccScaleFlag, t.ScaleFactor)) + params = append(params, fmt.Sprintf("%s=%d", tpccTablesFlag, t.Tables)) + params = append(params, fmt.Sprintf("%s=%s", tpccTransactionLevelFlag, t.TrxLevel)) + params = append(params, fmt.Sprintf("%s=%t", tpccReportCsv, t.ReportCSV)) + params = append(params, fmt.Sprintf("%s=%d", tpccReportIntervalFlag, t.ReportInterval)) + params = append(params, fmt.Sprintf("%s=%d", tpccTimeFlag, t.Time)) + return params +} + +// tpccTestImpl encapsulates an End to End prepare, run, cleanup test case. +type tpccTestImpl struct { + // Id represents a unique test id + Id string + + // Name represents the name of the test case + Name string + + // Params are associated parameters this test runs with + Params TpccTestParams +} + +var _ Test = &tpccTestImpl{} + +// NewTpccTest instantiates and returns a TPCC test. +func NewTpccTest(name string, params TpccTestParams) *tpccTestImpl { + return &tpccTestImpl{ + Id: uuid.New().String(), + Name: name, + Params: params, + } +} + +func (t *tpccTestImpl) doltArgs(serverConfig ServerConfig) []string { + args := make([]string, 0) + args = append(args, defaultTpccParams...) + args = append(args, fmt.Sprintf("%s=%s", tpccMysqlHostFlag, serverConfig.GetHost())) + port := serverConfig.GetPort() + if port > 0 { + args = append(args, fmt.Sprintf("%s=%d", tpccMysqlPortFlag, serverConfig.GetPort())) + } + args = append(args, fmt.Sprintf("%s=%d", tpccMysqlPortFlag, serverConfig.GetPort())) + args = append(args, fmt.Sprintf("%s=%s", tpccMysqlUserFlag, defaultMysqlUser)) + args = append(args, fmt.Sprintf("%s=%d", tpccTimeFlag, t.Params.GetTime())) + args = append(args, fmt.Sprintf("%s=%d", tpccThreadsFlag, t.Params.GetNumThreads())) + args = append(args, fmt.Sprintf("%s=%d", tpccReportIntervalFlag, t.Params.GetReportInterval())) + args = append(args, fmt.Sprintf("%s=%d", tpccTablesFlag, t.Params.GetTables())) + args = append(args, fmt.Sprintf("%s=%d", tpccScaleFlag, t.Params.GetScaleFactor())) + args = append(args, fmt.Sprintf("%s=%s", tpccTransactionLevelFlag, t.Params.GetTrxLevel())) + return args +} + +func (t *tpccTestImpl) mysqlArgs(serverConfig ServerConfig) []string { + args := make([]string, 0) + args = append(args, defaultTpccParams...) + host := serverConfig.GetHost() + port := serverConfig.GetPort() + args = append(args, fmt.Sprintf("%s=%s", tpccMysqlHostFlag, host)) + if host == defaultHost { + args = append(args, fmt.Sprintf("%s=%s", tpccMysqlUserFlag, tpccMysqlUsername)) + args = append(args, fmt.Sprintf("%s=%s", tpccMysqlPasswordFlag, tpccPassLocal)) + } else { + args = append(args, fmt.Sprintf("%s=%s", tpccMysqlUserFlag, defaultMysqlUser)) + } + if port > 0 { + args = append(args, fmt.Sprintf("%s=%d", tpccMysqlPortFlag, serverConfig.GetPort())) + } + args = append(args, fmt.Sprintf("%s=%d", tpccTimeFlag, t.Params.GetTime())) + args = append(args, fmt.Sprintf("%s=%d", tpccThreadsFlag, t.Params.GetNumThreads())) + args = append(args, fmt.Sprintf("%s=%d", tpccReportIntervalFlag, t.Params.GetReportInterval())) + args = append(args, fmt.Sprintf("%s=%d", tpccTablesFlag, t.Params.GetTables())) + args = append(args, fmt.Sprintf("%s=%d", tpccScaleFlag, t.Params.GetScaleFactor())) + args = append(args, fmt.Sprintf("%s=%s", tpccTransactionLevelFlag, t.Params.GetTrxLevel())) + return args +} + +func (t *tpccTestImpl) doltgresArgs(serverConfig ServerConfig) []string { + args := make([]string, 0) + args = append(args, defaultTpccParams...) + args = append(args, fmt.Sprintf("%s=%d", tpccTimeFlag, t.Params.GetTime())) + args = append(args, fmt.Sprintf("%s=%d", tpccThreadsFlag, t.Params.GetNumThreads())) + args = append(args, fmt.Sprintf("%s=%d", tpccReportIntervalFlag, t.Params.GetReportInterval())) + args = append(args, fmt.Sprintf("%s=%d", tpccTablesFlag, t.Params.GetTables())) + args = append(args, fmt.Sprintf("%s=%d", tpccScaleFlag, t.Params.GetScaleFactor())) + args = append(args, fmt.Sprintf("%s=%s", tpccTransactionLevelFlag, t.Params.GetTrxLevel())) + return args +} + +func (t *tpccTestImpl) postgresArgs(serverConfig ServerConfig) []string { + args := make([]string, 0) + args = append(args, defaultTpccParams...) + args = append(args, fmt.Sprintf("%s=%d", tpccTimeFlag, t.Params.GetTime())) + args = append(args, fmt.Sprintf("%s=%d", tpccThreadsFlag, t.Params.GetNumThreads())) + args = append(args, fmt.Sprintf("%s=%d", tpccReportIntervalFlag, t.Params.GetReportInterval())) + args = append(args, fmt.Sprintf("%s=%d", tpccTablesFlag, t.Params.GetTables())) + args = append(args, fmt.Sprintf("%s=%d", tpccScaleFlag, t.Params.GetScaleFactor())) + args = append(args, fmt.Sprintf("%s=%s", tpccTransactionLevelFlag, t.Params.GetTrxLevel())) + return args +} + +// getArgs returns a test's args for all TPCC steps +func (t *tpccTestImpl) getArgs(serverConfig ServerConfig) []string { + st := serverConfig.GetServerType() + switch st { + case Dolt: + return t.doltArgs(serverConfig) + case Doltgres: + return t.doltgresArgs(serverConfig) + case Postgres: + return t.postgresArgs(serverConfig) + case MySql: + return t.mysqlArgs(serverConfig) + default: + panic(fmt.Sprintf("unexpected server type: %s", st)) + } +} + +func (t *tpccTestImpl) GetId() string { + return t.Id +} + +func (t *tpccTestImpl) GetName() string { + return t.Name +} + +func (t *tpccTestImpl) GetParamsToSlice() []string { + return t.Params.ToSlice() +} + +func (t *tpccTestImpl) GetPrepareArgs(serverConfg ServerConfig) []string { + args := make([]string, 0) + serverArgs := t.getArgs(serverConfg) + args = append(args, serverArgs...) + args = append(args, sysbenchPrepareCommand) + return args +} + +func (t *tpccTestImpl) GetRunArgs(serverConfg ServerConfig) []string { + args := make([]string, 0) + serverArgs := t.getArgs(serverConfg) + args = append(args, serverArgs...) + args = append(args, sysbenchRunCommand) + return args +} + +func (t *tpccTestImpl) GetCleanupArgs(serverConfg ServerConfig) []string { + args := make([]string, 0) + serverArgs := t.getArgs(serverConfg) + args = append(args, serverArgs...) + args = append(args, sysbenchCleanupCommand) + return args +} diff --git a/go/performance/utils/sysbench_runner/cmd/main.go b/go/performance/utils/sysbench_runner/cmd/main.go deleted file mode 100644 index 19bdb28d3fe..00000000000 --- a/go/performance/utils/sysbench_runner/cmd/main.go +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright 2019-2022 Dolthub, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "flag" - "log" - "os" - "path/filepath" - - runner "github.com/dolthub/dolt/go/performance/utils/sysbench_runner" -) - -var configFile = flag.String("config", "", "path to config file") - -func main() { - flag.Parse() - - if *configFile == "" { - log.Fatal("Must supply config") - } - - configPath, err := filepath.Abs(*configFile) - if err != nil { - log.Fatal(err) - } - if _, err = os.Stat(configPath); os.IsNotExist(err) { - log.Fatal(err) - } - - config, err := runner.FromFileConfig(configPath) - if err != nil { - log.Fatal(err) - } - - err = runner.Run(config) - if err != nil { - log.Fatal(err) - } - - os.Exit(0) -} diff --git a/go/performance/utils/sysbench_runner/config.go b/go/performance/utils/sysbench_runner/config.go deleted file mode 100644 index 39a69aceac4..00000000000 --- a/go/performance/utils/sysbench_runner/config.go +++ /dev/null @@ -1,616 +0,0 @@ -// Copyright 2019-2022 Dolthub, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package sysbench_runner - -import ( - "encoding/json" - "errors" - "fmt" - "io/fs" - "os" - "path/filepath" - "runtime" - - "github.com/google/uuid" -) - -const ( - Dolt ServerType = "dolt" - Doltgres ServerType = "doltgres" - Postgres ServerType = "postgres" - MySql ServerType = "mysql" - - CsvFormat = "csv" - JsonFormat = "json" - - CsvExt = ".csv" - JsonExt = ".json" - - defaultHost = "127.0.0.1" - defaultPort = 3306 - - defaultMysqlSocket = "/var/run/mysqld/mysqld.sock" - - tcpProtocol = "tcp" - unixProtocol = "unix" - - sysbenchUsername = "sysbench" - sysbenchUserLocal = "'sysbench'@'localhost'" - sysbenchPassLocal = "sysbenchpass" - - userFlag = "--user" - hostFlag = "--host" - portFlag = "--port" - skipBinLogFlag = "--skip-log-bin" - profileFlag = "--prof" - profilePathFlag = "--prof-path" - cpuProfile = "cpu" - doltgresDataDirFlag = "--data-dir" - MysqlDataDirFlag = "--datadir" - MysqlInitializeInsecureFlag = "--initialize-insecure" - cpuProfileFilename = "cpu.pprof" -) - -var ( - ErrTestNameNotDefined = errors.New("test name not defined") - ErrNoServersDefined = errors.New("servers not defined") - ErrTooManyServersDefined = errors.New("too many servers defined, two max") - ErrUnsupportedConnectionProtocol = errors.New("unsupported connection protocol") -) - -var defaultSysbenchParams = []string{ - "--db-ps-mode=disable", - "--rand-type=uniform", -} - -var defaultDoltServerParams = []string{"sql-server"} - -var defaultSysbenchTests = []*ConfigTest{ - NewConfigTest("oltp_read_only", []string{}, false), - NewConfigTest("oltp_insert", []string{}, false), - NewConfigTest("bulk_insert", []string{}, false), - NewConfigTest("oltp_point_select", []string{}, false), - NewConfigTest("select_random_points", []string{}, false), - NewConfigTest("select_random_ranges", []string{}, false), - NewConfigTest("oltp_write_only", []string{}, false), - NewConfigTest("oltp_read_write", []string{}, false), - NewConfigTest("oltp_update_index", []string{}, false), - NewConfigTest("oltp_update_non_index", []string{}, false), -} - -var defaultDoltLuaScripts = map[string]string{ - "covering_index_scan.lua": "covering_index_scan.lua", - "groupby_scan.lua": "groupby_scan.lua", - "index_join.lua": "index_join.lua", - "index_join_scan.lua": "index_join_scan.lua", - "index_scan.lua": "index_scan.lua", - "oltp_delete_insert.lua": "oltp_delete_insert.lua", - "table_scan.lua": "table_scan.lua", - "types_delete_insert.lua": "types_delete_insert.lua", - "types_table_scan.lua": "types_table_scan.lua", -} - -// todo: check expressions need to be supported in doltgres for these -// todo: postgres does not have geometry types also -var defaultDoltgresLuaScripts = map[string]string{ - //"covering_index_scan_postgres.lua": "covering_index_scan_postgres.lua", - //"groupby_scan_postgres.lua": "groupby_scan_postgres.lua", - //"index_join_postgres.lua": "index_join_postgres.lua", - //"index_join_scan_postgres.lua": "index_join_scan_postgres.lua", - //"index_scan_postgres.lua": "index_scan_postgres.lua", - //"oltp_delete_insert_postgres.lua": "oltp_delete_insert_postgres.lua", - //"table_scan_postgres.lua": "table_scan_postgres.lua", - //"types_delete_insert_postgres.lua": "types_delete_insert_postgres.lua", - //"types_table_scan_postgres.lua": "types_table_scan_postgres.lua", -} - -type ServerType string - -// Test is a single sysbench test -type Test struct { - id string - - // Name is the test name - Name string - - // Params are the parameters passed to sysbench - Params []string - - // FromScript indicates if this test is from a lua script - FromScript bool -} - -// Prepare returns a test's args for sysbench's prepare step -func (t *Test) Prepare() []string { - return withCommand(t.Params, "prepare") -} - -// Run returns a test's args for sysbench's run step -func (t *Test) Run() []string { - return withCommand(t.Params, "run") -} - -// Cleanup returns a test's args for sysbench's cleanup step -func (t *Test) Cleanup() []string { - return withCommand(t.Params, "cleanup") -} - -func withCommand(params []string, command string) []string { - c := make([]string, 0) - c = append(c, params...) - return append(c, command) -} - -// ConfigTest provides users a way to define a test for multiple tablesizes -type ConfigTest struct { - // Name is the test name - Name string - - // N is the number of times a test should run - N int - - // Options are additional sysbench test options a user can supply to run with this test - Options []string - - // FromScript is a boolean indicating that this test is from a lua script - FromScript bool -} - -// NewConfigTest returns a ConfigTest containing the supplied args -func NewConfigTest(name string, opts []string, fromScript bool) *ConfigTest { - options := make([]string, 0) - options = append(options, opts...) - return &ConfigTest{ - Name: name, - N: 1, - Options: options, - FromScript: fromScript, - } -} - -// GetTests returns a slice of Tests -func (ct *ConfigTest) GetTests(serverConfig *ServerConfig, testIdFunc func() string) ([]*Test, error) { - if ct.Name == "" { - return nil, ErrTestNameNotDefined - } - if ct.N < 1 { - ct.N = 1 - } - - params := fromConfigTestParams(ct, serverConfig) - tests := make([]*Test, 0) - - var idFunc func() string - if testIdFunc == nil { - idFunc = func() string { - return uuid.New().String() - } - } else { - idFunc = testIdFunc - } - - for i := 0; i < ct.N; i++ { - p := make([]string, len(params)) - copy(p, params) - tests = append(tests, &Test{ - id: idFunc(), - Name: ct.Name, - Params: p, - FromScript: ct.FromScript, - }) - } - return tests, nil -} - -// fromConfigTestParams returns params formatted for sysbench: -// `sysbench [options]... [testname] [command]` -func fromConfigTestParams(ct *ConfigTest, serverConfig *ServerConfig) []string { - params := make([]string, 0) - params = append(params, defaultSysbenchParams...) - if serverConfig.Server == MySql || serverConfig.Server == Dolt { - params = append(params, fmt.Sprintf("--mysql-db=%s", dbName)) - params = append(params, "--db-driver=mysql") - params = append(params, fmt.Sprintf("--mysql-host=%s", serverConfig.Host)) - if serverConfig.Port != 0 { - params = append(params, fmt.Sprintf("--mysql-port=%d", serverConfig.Port)) - } - } else if serverConfig.Server == Doltgres || serverConfig.Server == Postgres { - params = append(params, "--db-driver=pgsql") - params = append(params, fmt.Sprintf("--pgsql-db=%s", dbName)) - params = append(params, fmt.Sprintf("--pgsql-host=%s", serverConfig.Host)) - if serverConfig.Port != 0 { - params = append(params, fmt.Sprintf("--pgsql-port=%d", serverConfig.Port)) - } - } - - // handle sysbench user for local mysql server - if serverConfig.Server == MySql && serverConfig.Host == defaultHost { - params = append(params, "--mysql-user=sysbench") - params = append(params, fmt.Sprintf("--mysql-password=%s", sysbenchPassLocal)) - } else if serverConfig.Server == Dolt { - params = append(params, "--mysql-user=root") - } else if serverConfig.Server == Doltgres { - params = append(params, "--pgsql-user=doltgres") - } else if serverConfig.Server == Postgres { - params = append(params, "--pgsql-user=postgres") - } - - params = append(params, ct.Options...) - params = append(params, ct.Name) - return params -} - -// ServerConfig is the configuration for a server to test against -type ServerConfig struct { - // Id is a unique id for this servers benchmarking - Id string - - // Host is the server host - Host string - - // Port is the server port - Port int - - // Server is the type of server - Server ServerType - - // Version is the server version - Version string - - // ResultsFormat is the format the results should be written in - ResultsFormat string - - // ServerExec is the path to a server executable - ServerExec string - - // InitExec is the path to the server init db executable - InitExec string - - // ServerUser is the user account that should start the server - ServerUser string - - // SkipLogBin will skip bin logging - SkipLogBin bool - - // ServerArgs are the args used to start a server - ServerArgs []string - - // ConnectionProtocol defines the protocol for connecting to the server - ConnectionProtocol string - - // Socket is the path to the server socket - Socket string - - // ServerProfile specifies the golang profile to take of a Dolt server - ServerProfile string - - // ProfilePath path to directory where server profile will be written - ProfilePath string -} - -func (sc *ServerConfig) GetId() string { - if sc.Id == "" { - sc.Id = uuid.New().String() - } - return sc.Id -} - -// GetServerArgs returns the args used to start a server -func (sc *ServerConfig) GetServerArgs() ([]string, error) { - params := make([]string, 0) - - if sc.Server == Dolt { - params = append(params, defaultDoltServerParams...) - } else if sc.Server == MySql { - if sc.ServerUser != "" { - params = append(params, fmt.Sprintf("%s=%s", userFlag, sc.ServerUser)) - } - if sc.SkipLogBin { - params = append(params, skipBinLogFlag) - } - } - - if sc.Server == Dolt || sc.Server == Doltgres { - params = append(params, fmt.Sprintf("%s=%s", hostFlag, sc.Host)) - } - if sc.Port != 0 { - params = append(params, fmt.Sprintf("%s=%d", portFlag, sc.Port)) - } - - params = append(params, sc.ServerArgs...) - return params, nil -} - -// Config is the configuration for a benchmarking run -type Config struct { - // Runs is the number of times to run all tests - Runs int - // RuntimeOS is the platform the benchmarks ran on - RuntimeOS string - // RuntimeGoArch is the runtime architecture - RuntimeGoArch string - // Servers are the servers to benchmark - Servers []*ServerConfig - // Tests are the tests to run. If no tests are provided, - // the default tests will be used - Tests []*ConfigTest - // TestOptions a list of sysbench test options to apply to all tests - TestOptions []string - // ScriptDir is a path to a directory of lua scripts - ScriptDir string - // InitBigRepo downloads a database with existing chunks and commits - InitBigRepo bool - // NomsBinFormat specifies the NomsBinFormat - NomsBinFormat string -} - -// NewConfig returns a new Config -func NewConfig() *Config { - return &Config{ - Servers: make([]*ServerConfig, 0), - } -} - -// Validate checks the config for the required fields and sets defaults -// where necessary -func (c *Config) Validate() error { - if len(c.Servers) < 1 { - return ErrNoServersDefined - } - if len(c.Servers) > 2 { - return ErrTooManyServersDefined - } - err := c.setDefaults() - if err != nil { - return err - } - return c.validateServerConfigs() -} - -// validateServerConfigs ensures the ServerConfigs are valid -func (c *Config) validateServerConfigs() error { - portMap := make(map[int]ServerType) - for _, s := range c.Servers { - if s.Server != Dolt && s.Server != MySql && s.Server != Doltgres && s.Server != Postgres { - return fmt.Errorf("unsupported server type: %s", s.Server) - } - - err := ValidateRequiredFields(string(s.Server), s.Version, s.ResultsFormat) - if err != nil { - return err - } - - if s.Server == MySql { - err = CheckProtocol(s.ConnectionProtocol) - if err != nil { - return err - } - } - - if s.Host == "" { - s.Host = defaultHost - } - - portMap, err = CheckUpdatePortMap(s, portMap) - if err != nil { - return err - } - - err = CheckExec(s.ServerExec, "server exec") - if err != nil { - return err - } - - if s.Server == Postgres { - err = CheckExec(s.InitExec, "initdb exec") - if err != nil { - return err - } - } - - if s.Server != Dolt && s.ServerProfile != "" { - return fmt.Errorf("profiling can only be done against a dolt server") - } - - if s.Server == Dolt && s.ServerProfile != "" { - if s.ServerProfile != cpuProfile { - return fmt.Errorf("unsupported server profile: %s", s.ServerProfile) - } - if s.ProfilePath == "" { - cwd, err := os.Getwd() - if err != nil { - return err - } - s.ProfilePath = cwd - } - } - } - - return nil -} - -func (c *Config) Contains(st ServerType) bool { - for _, s := range c.Servers { - if s.Server == st { - return true - } - } - return false -} - -func ValidateRequiredFields(server, version, format string) error { - if server == "" { - return getMustSupplyError("server") - } - if version == "" { - return getMustSupplyError("version") - } - if format == "" { - return getMustSupplyError("results format") - } - return nil -} - -// setDefaults sets defaults on the Config -func (c *Config) setDefaults() error { - if c.RuntimeOS == "" { - c.RuntimeOS = runtime.GOOS - } - if c.RuntimeGoArch == "" { - c.RuntimeGoArch = runtime.GOARCH - } - if len(c.Tests) < 1 { - fmt.Printf("Preparing to benchmark against default tests\n") - if c.ScriptDir != "" { - abs, err := filepath.Abs(c.ScriptDir) - if err != nil { - return err - } - if _, err := os.Stat(abs); os.IsNotExist(err) { - return fmt.Errorf("script dir not found: %s", abs) - } - c.ScriptDir = abs - } - tests, err := getDefaultTests(c) - if err != nil { - return err - } - c.Tests = tests - } - if c.Runs < 1 { - c.Runs = 1 - } - return nil -} - -// CheckUpdatePortMap returns an error if multiple servers have specified the same port -func CheckUpdatePortMap(serverConfig *ServerConfig, portMap map[int]ServerType) (map[int]ServerType, error) { - if serverConfig.Port == 0 { - serverConfig.Port = defaultPort - } - srv, ok := portMap[serverConfig.Port] - if ok && srv != serverConfig.Server { - return nil, fmt.Errorf("servers have port conflict on port: %d\n", serverConfig.Port) - } - if !ok { - portMap[serverConfig.Port] = serverConfig.Server - } - return portMap, nil -} - -// CheckExec verifies the binary exists -func CheckExec(path, messageIfMissing string) error { - if path == "" { - return getMustSupplyError(messageIfMissing) - } - abs, err := filepath.Abs(path) - if err != nil { - return err - } - if _, err := os.Stat(abs); os.IsNotExist(err) { - return fmt.Errorf("exec not found: %s", abs) - } - return nil -} - -// CheckProtocol ensures the given protocol is supported -func CheckProtocol(protocol string) error { - if protocol == "" { - return getMustSupplyError("connection protocol") - } - if protocol == tcpProtocol || protocol == unixProtocol { - return nil - } - return ErrUnsupportedConnectionProtocol -} - -// GetTests returns a slice of Tests created from the -// defined ServerConfig.Tests -func GetTests(config *Config, serverConfig *ServerConfig, testIdFunc func() string) ([]*Test, error) { - flattened := make([]*Test, 0) - for _, t := range config.Tests { - if len(config.TestOptions) > 0 { - t.Options = append(t.Options, config.TestOptions...) - } - tests, err := t.GetTests(serverConfig, testIdFunc) - if err != nil { - return nil, err - } - flattened = append(flattened, tests...) - } - return flattened, nil -} - -// FromFileConfig returns a validated Config based on the config file at the configPath -func FromFileConfig(configPath string) (*Config, error) { - data, err := os.ReadFile(configPath) - if err != nil { - return nil, err - } - - config := NewConfig() - err = json.Unmarshal(data, config) - if err != nil { - return nil, err - } - - return config, nil -} - -func getMustSupplyError(name string) error { - return fmt.Errorf("Must supply %s", name) -} - -func getDefaultTests(config *Config) ([]*ConfigTest, error) { - defaultTests := make([]*ConfigTest, 0) - defaultTests = append(defaultTests, defaultSysbenchTests...) - if config.ScriptDir != "" { - var luaScriptTests []*ConfigTest - var err error - if !config.Contains(Doltgres) && !config.Contains(Postgres) { - luaScriptTests, err = getLuaScriptTestsFromDir(config.ScriptDir, defaultDoltLuaScripts) - } else { - luaScriptTests, err = getLuaScriptTestsFromDir(config.ScriptDir, defaultDoltgresLuaScripts) - } - if err != nil { - return nil, err - } - defaultTests = append(defaultTests, luaScriptTests...) - } - return defaultTests, nil -} - -func getLuaScriptTestsFromDir(dir string, toInclude map[string]string) ([]*ConfigTest, error) { - luaScripts := make([]*ConfigTest, 0) - abs, err := filepath.Abs(dir) - if err != nil { - return nil, err - } - err = filepath.Walk(abs, func(path string, info fs.FileInfo, err error) error { - if err != nil { - return err - } - - file := filepath.Base(path) - if _, ok := toInclude[file]; ok { - luaScripts = append(luaScripts, NewConfigTest(path, []string{}, true)) - } - return nil - }) - if err != nil { - return nil, err - } - return luaScripts, nil -} diff --git a/go/performance/utils/sysbench_runner/config_test.go b/go/performance/utils/sysbench_runner/config_test.go deleted file mode 100644 index 22a0a95037f..00000000000 --- a/go/performance/utils/sysbench_runner/config_test.go +++ /dev/null @@ -1,107 +0,0 @@ -// Copyright 2019-2022 Dolthub, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package sysbench_runner - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -var testIdFunc = func() string { return "id" } - -func TestConfigTestGetTests(t *testing.T) { - empty := &ConfigTest{Name: "test_name"} - - one := &ConfigTest{Name: "test_one", N: 3} - two := &ConfigTest{Name: "test_two", N: 2} - three := &ConfigTest{Name: "test_three", N: 1} - - opts := &ConfigTest{ - Name: "test_options", - N: 1, - Options: []string{"--create_secondary=on", "--auto_inc=off"}, - } - - serverConfig := &ServerConfig{Server: MySql, Version: "test-version", Host: "localhost", ResultsFormat: CsvFormat} - - tests := []struct { - description string - config *Config - expectedTests []*Test - expectedError error - }{ - { - description: "should error if no test name is defined", - config: &Config{ - Servers: []*ServerConfig{serverConfig}, - Tests: []*ConfigTest{ - {Name: ""}, - }, - }, - expectedTests: nil, - expectedError: ErrTestNameNotDefined, - }, - { - description: "should create single test if N is < 1", - config: &Config{ - Servers: []*ServerConfig{serverConfig}, - Tests: []*ConfigTest{empty}, - }, - expectedTests: []*Test{ - { - id: testIdFunc(), - Name: "test_name", - Params: fromConfigTestParams(empty, serverConfig), - }, - }, - }, - { - description: "should return a test for each N defined on the ConfigTest", - config: &Config{ - Servers: []*ServerConfig{serverConfig}, - Tests: []*ConfigTest{one, two, three}, - }, - expectedTests: []*Test{ - {id: testIdFunc(), Name: "test_one", Params: fromConfigTestParams(one, serverConfig)}, - {id: testIdFunc(), Name: "test_one", Params: fromConfigTestParams(one, serverConfig)}, - {id: testIdFunc(), Name: "test_one", Params: fromConfigTestParams(one, serverConfig)}, - {id: testIdFunc(), Name: "test_two", Params: fromConfigTestParams(two, serverConfig)}, - {id: testIdFunc(), Name: "test_two", Params: fromConfigTestParams(two, serverConfig)}, - {id: testIdFunc(), Name: "test_three", Params: fromConfigTestParams(three, serverConfig)}, - }, - }, - { - description: "should apply user options to test params", - config: &Config{ - Servers: []*ServerConfig{serverConfig}, - Tests: []*ConfigTest{opts}, - }, - expectedTests: []*Test{ - {id: testIdFunc(), Name: "test_options", Params: fromConfigTestParams(opts, serverConfig)}, - }, - }, - } - for _, test := range tests { - t.Run(test.description, func(t *testing.T) { - for _, s := range test.config.Servers { - actual, err := GetTests(test.config, s, testIdFunc) - assert.Equal(t, test.expectedError, err) - assert.Equal(t, len(test.expectedTests), len(actual)) - assert.ElementsMatch(t, test.expectedTests, actual) - } - }) - } -} diff --git a/go/performance/utils/sysbench_runner/dolt.go b/go/performance/utils/sysbench_runner/dolt.go deleted file mode 100644 index 3bae67be6b8..00000000000 --- a/go/performance/utils/sysbench_runner/dolt.go +++ /dev/null @@ -1,296 +0,0 @@ -// Copyright 2019-2022 Dolthub, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package sysbench_runner - -import ( - "context" - "fmt" - "os" - "os/exec" - "os/signal" - "path/filepath" - "sync" - "syscall" - "time" - - "github.com/dolthub/dolt/go/store/types" - - "golang.org/x/sync/errgroup" -) - -const ( - dbName = "test" - luaPath = "?.lua" - bigEmptyRepo = "max-hoffman/big-empty" - nbfEnvVar = "DOLT_DEFAULT_BIN_FORMAT" -) - -var stampFunc = func() string { return time.Now().UTC().Format(stampFormat) } - -// BenchmarkDolt benchmarks dolt based on the provided configurations -func BenchmarkDolt(ctx context.Context, config *Config, serverConfig *ServerConfig) (Results, error) { - serverParams, err := serverConfig.GetServerArgs() - if err != nil { - return nil, err - } - - err = DoltVersion(ctx, serverConfig.ServerExec) - if err != nil { - return nil, err - } - - err = UpdateDoltConfig(ctx, serverConfig.ServerExec) - if err != nil { - return nil, err - } - - testRepo, err := initDoltRepo(ctx, serverConfig, config.NomsBinFormat) - if err != nil { - return nil, err - } - - withKeyCtx, cancel := context.WithCancel(ctx) - gServer, serverCtx := errgroup.WithContext(withKeyCtx) - - server := getServer(serverCtx, serverConfig, testRepo, serverParams) - - // handle user interrupt - quit := make(chan os.Signal, 1) - signal.Notify(quit, os.Interrupt, syscall.SIGTERM) - var wg sync.WaitGroup - wg.Add(1) - go func() { - <-quit - defer wg.Done() - signal.Stop(quit) - cancel() - }() - - // launch the dolt server - gServer.Go(func() error { - return server.Run() - }) - - // sleep to allow the server to start - time.Sleep(5 * time.Second) - - tests, err := GetTests(config, serverConfig, nil) - if err != nil { - return nil, err - } - - results := make(Results, 0) - for i := 0; i < config.Runs; i++ { - for _, test := range tests { - r, err := benchmark(withKeyCtx, test, config, serverConfig, stampFunc, serverConfig.GetId()) - if err != nil { - close(quit) - wg.Wait() - return nil, err - } - results = append(results, r) - } - } - - // send signal to dolt server - quit <- syscall.SIGTERM - - err = gServer.Wait() - if err != nil { - // we expect a kill error - // we only exit in error if this is not the - // error - if err.Error() != "signal: killed" { - fmt.Println(err) - close(quit) - wg.Wait() - return nil, err - } - } - - fmt.Println("Successfully killed server") - close(quit) - wg.Wait() - - err = os.RemoveAll(testRepo) - if err != nil { - return nil, err - } - - return results, nil -} - -// DoltVersion ensures the dolt binary can run -func DoltVersion(ctx context.Context, serverExec string) error { - doltVersion := ExecCommand(ctx, serverExec, "version") - return doltVersion.Run() -} - -// initDoltRepo initializes a dolt repo and returns the repo path -func initDoltRepo(ctx context.Context, config *ServerConfig, nbf string) (string, error) { - cwd, err := os.Getwd() - if err != nil { - return "", err - } - - testRepo := filepath.Join(cwd, dbName) - if nbf == types.Format_LD_1.VersionString() { - err := ExecCommand(ctx, config.ServerExec, "clone", bigEmptyRepo, dbName).Run() - if err != nil { - return "", err - } - return testRepo, nil - } - err = os.MkdirAll(testRepo, os.ModePerm) - if err != nil { - return "", err - } - - if nbf != "" { - if err = os.Setenv(nbfEnvVar, nbf); err != nil { - return "", err - } - } - - doltInit := ExecCommand(ctx, config.ServerExec, "init") - doltInit.Dir = testRepo - err = doltInit.Run() - if err != nil { - return "", err - } - - return testRepo, nil -} - -// UpdateDoltConfig updates the dolt config if necessary -func UpdateDoltConfig(ctx context.Context, serverExec string) error { - err := checkSetDoltConfig(ctx, serverExec, "user.name", "benchmark") - if err != nil { - return err - } - return checkSetDoltConfig(ctx, serverExec, "user.email", "benchmark@dolthub.com") -} - -// checkSetDoltConfig checks the output of `dolt config --global --get` and sets the key, val if necessary -func checkSetDoltConfig(ctx context.Context, serverExec, key, val string) error { - check := ExecCommand(ctx, serverExec, "config", "--global", "--get", key) - err := check.Run() - if err != nil { - // config get calls exit with 1 if not set - if err.Error() != "exit status 1" { - return err - } - - set := ExecCommand(ctx, serverExec, "config", "--global", "--add", key, val) - err := set.Run() - if err != nil { - return err - } - } - - return nil -} - -// getServer returns a exec.Cmd for a dolt server -func getServer(ctx context.Context, config *ServerConfig, testRepo string, params []string) *exec.Cmd { - server := ExecCommand(ctx, config.ServerExec, params...) - server.Dir = testRepo - return server -} - -// sysbenchPrepare returns a exec.Cmd for running the sysbench prepare step -func sysbenchPrepare(ctx context.Context, test *Test, scriptDir string) *exec.Cmd { - cmd := exec.CommandContext(ctx, "sysbench", test.Prepare()...) - if test.FromScript { - lp := filepath.Join(scriptDir, luaPath) - cmd.Env = os.Environ() - cmd.Env = append(cmd.Env, fmt.Sprintf("LUA_PATH=%s", lp)) - } - return cmd -} - -// sysbenchRun returns a exec.Cmd for running the sysbench run step -func sysbenchRun(ctx context.Context, test *Test, scriptDir string) *exec.Cmd { - cmd := exec.CommandContext(ctx, "sysbench", test.Run()...) - if test.FromScript { - lp := filepath.Join(scriptDir, luaPath) - cmd.Env = os.Environ() - cmd.Env = append(cmd.Env, fmt.Sprintf("LUA_PATH=%s", lp)) - } - return cmd -} - -// sysbenchPrepare returns a exec.Cmd for running the sysbench cleanup step -func sysbenchCleanup(ctx context.Context, test *Test, scriptDir string) *exec.Cmd { - cmd := ExecCommand(ctx, "sysbench", test.Cleanup()...) - if test.FromScript { - lp := filepath.Join(scriptDir, luaPath) - cmd.Env = os.Environ() - cmd.Env = append(cmd.Env, fmt.Sprintf("LUA_PATH=%s", lp)) - } - return cmd -} - -// benchmark runs a sysbench benchmark against a server calling prepare, run, cleanup -func benchmark( - ctx context.Context, - test *Test, - config *Config, - serverConfig *ServerConfig, - stampFunc func() string, - suiteId string, -) (*Result, error) { - prepare := sysbenchPrepare(ctx, test, config.ScriptDir) - run := sysbenchRun(ctx, test, config.ScriptDir) - cleanup := sysbenchCleanup(ctx, test, config.ScriptDir) - - fmt.Println("Running test ", test.Name) - - out, err := prepare.Output() - if err != nil { - fmt.Println(string(out)) - return nil, err - } - - out, err = run.Output() - if err != nil { - fmt.Print(string(out)) - return nil, err - } - - if Debug == true { - fmt.Print(string(out)) - } - - r, err := FromOutputResult(out, config, serverConfig, test, suiteId, nil) - if err != nil { - return nil, err - } - - r.Stamp(stampFunc) - - return r, cleanup.Run() -} - -// fromChannelResults collects all Results from the given channel and returns them -func fromChannelResults(rc chan *Result) Results { - results := make(Results, 0) - for r := range rc { - if r != nil { - results = append(results, r) - } - } - return results -} diff --git a/go/performance/utils/sysbench_runner/doltgres.go b/go/performance/utils/sysbench_runner/doltgres.go deleted file mode 100644 index 7364fa4efef..00000000000 --- a/go/performance/utils/sysbench_runner/doltgres.go +++ /dev/null @@ -1,183 +0,0 @@ -// Copyright 2019-2022 Dolthub, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package sysbench_runner - -import ( - "context" - "fmt" - "os" - "os/signal" - "path/filepath" - "sync" - "syscall" - "time" - - "database/sql" - - _ "github.com/lib/pq" - "golang.org/x/sync/errgroup" -) - -// BenchmarkDoltgres benchmarks doltgres based on the provided configurations -func BenchmarkDoltgres(ctx context.Context, config *Config, serverConfig *ServerConfig) (Results, error) { - serverParams, err := serverConfig.GetServerArgs() - if err != nil { - return nil, err - } - - err = DoltVersion(ctx, serverConfig.ServerExec) - if err != nil { - return nil, err - } - - serverDir, err := createServerDir(dbName) - if err != nil { - return nil, err - } - defer func() { - cleanupDoltgresServerDir(serverDir) - }() - - serverParams = append(serverParams, fmt.Sprintf("%s=%s", doltgresDataDirFlag, serverDir)) - - withKeyCtx, cancel := context.WithCancel(ctx) - gServer, serverCtx := errgroup.WithContext(withKeyCtx) - - server := getServer(serverCtx, serverConfig, serverDir, serverParams) - - // handle user interrupt - quit := make(chan os.Signal, 1) - signal.Notify(quit, os.Interrupt, syscall.SIGTERM) - var wg sync.WaitGroup - wg.Add(1) - go func() { - <-quit - defer wg.Done() - signal.Stop(quit) - cancel() - }() - - // launch the dolt server - gServer.Go(func() error { - return server.Run() - }) - - // sleep to allow the server to start - time.Sleep(5 * time.Second) - - // create the db against the running server - err = createDb(ctx, serverConfig.Host, fmt.Sprintf("%d", serverConfig.Port), "doltgres", dbName) - if err != nil { - close(quit) - wg.Wait() - return nil, err - } - - tests, err := GetTests(config, serverConfig, nil) - if err != nil { - close(quit) - wg.Wait() - return nil, err - } - - results := make(Results, 0) - for i := 0; i < config.Runs; i++ { - for _, test := range tests { - r, err := benchmark(withKeyCtx, test, config, serverConfig, stampFunc, serverConfig.GetId()) - if err != nil { - close(quit) - wg.Wait() - return nil, err - } - results = append(results, r) - } - } - - // send signal to dolt server - quit <- syscall.SIGTERM - - err = gServer.Wait() - if err != nil { - // we expect a kill error - // we only exit in error if this is not the - // error - if err.Error() != "signal: killed" { - fmt.Println(err) - close(quit) - wg.Wait() - return nil, err - } - } - - fmt.Println("Successfully killed server") - close(quit) - wg.Wait() - - return results, nil -} - -func createDb(ctx context.Context, host, port, user, dbname string) error { - psqlconn := fmt.Sprintf("host=%s port=%s user=%s password=%s dbname=%s sslmode=disable", host, port, user, "", dbname) - - // open database - db, err := sql.Open("postgres", psqlconn) - if err != nil { - return err - } - - // close database - defer db.Close() - - // check db - err = db.PingContext(ctx) - if err != nil { - return err - } - - _, err = db.ExecContext(ctx, fmt.Sprintf("create database %s;", dbname)) - return err -} - -// createServerDir creates a server directory -func createServerDir(dbName string) (string, error) { - cwd, err := os.Getwd() - if err != nil { - return "", err - } - - serverDir := filepath.Join(cwd, dbName) - err = os.MkdirAll(serverDir, os.ModePerm) - if err != nil { - return "", err - } - - return serverDir, nil -} - -// cleanupDoltgresServerDir cleans up the doltgres assets in the provided dir -func cleanupDoltgresServerDir(dir string) error { - dataDir := filepath.Join(dir, ".dolt") - defaultDir := filepath.Join(dir, "doltgres") - testDir := filepath.Join(dir, dbName) - for _, d := range []string{dataDir, defaultDir, testDir} { - if _, err := os.Stat(d); !os.IsNotExist(err) { - err = os.RemoveAll(d) - if err != nil { - return err - } - } - } - return nil -} diff --git a/go/performance/utils/sysbench_runner/mysql.go b/go/performance/utils/sysbench_runner/mysql.go deleted file mode 100644 index 7a802a92099..00000000000 --- a/go/performance/utils/sysbench_runner/mysql.go +++ /dev/null @@ -1,248 +0,0 @@ -// Copyright 2019-2022 Dolthub, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package sysbench_runner - -import ( - "context" - "database/sql" - "fmt" - "log" - "os" - "os/exec" - "os/signal" - "sync" - "syscall" - "time" - - _ "github.com/go-sql-driver/mysql" - "golang.org/x/sync/errgroup" -) - -type MysqlConfig struct { - Socket string - ConnectionProtocol string - Port int - Host string -} - -// BenchmarkMysql benchmarks mysql based on the provided configurations -func BenchmarkMysql(ctx context.Context, config *Config, serverConfig *ServerConfig) (Results, error) { - withKeyCtx, cancel := context.WithCancel(ctx) - - var serverDir string - defer func() { - if serverDir != "" { - os.RemoveAll(serverDir) - } - }() - - var localServer bool - var gServer *errgroup.Group - var serverCtx context.Context - var server *exec.Cmd - var err error - if serverConfig.Host == defaultHost { - log.Println("Launching the default server") - localServer = true - - serverDir, err = InitMysqlDataDir(ctx, serverConfig) - if err != nil { - cancel() - return nil, err - } - - gServer, serverCtx = errgroup.WithContext(withKeyCtx) - var serverParams []string - serverParams, err = serverConfig.GetServerArgs() - if err != nil { - cancel() - return nil, err - } - serverParams = append(serverParams, fmt.Sprintf("%s=%s", MysqlDataDirFlag, serverDir)) - - server = getMysqlServer(serverCtx, serverConfig, serverParams) - - // launch the mysql server - gServer.Go(func() error { - return server.Run() - }) - - // sleep to allow the server to start - time.Sleep(10 * time.Second) - - // setup mysqldb - err := SetupDB(ctx, GetMysqlConnectionConfigFromServerConfig(serverConfig), dbName) - if err != nil { - cancel() - return nil, err - } - log.Println("Successfully set up the MySQL database") - } - - // handle user interrupt - quit := make(chan os.Signal, 1) - signal.Notify(quit, os.Interrupt, syscall.SIGTERM) - var wg sync.WaitGroup - wg.Add(1) - go func() { - <-quit - defer wg.Done() - signal.Stop(quit) - cancel() - }() - - tests, err := GetTests(config, serverConfig, nil) - if err != nil { - return nil, err - } - - results := make(Results, 0) - for i := 0; i < config.Runs; i++ { - for _, test := range tests { - r, err := benchmark(withKeyCtx, test, config, serverConfig, stampFunc, serverConfig.GetId()) - if err != nil { - close(quit) - wg.Wait() - return nil, err - } - results = append(results, r) - } - } - - // stop local mysql server - if localServer { - // send signal to server - quit <- syscall.SIGTERM - - err = gServer.Wait() - if err != nil { - // we expect a kill error - // we only exit in error if this is not the - // error - if err.Error() != "signal: killed" { - close(quit) - wg.Wait() - return nil, err - } - } - } - - fmt.Println("Successfully killed server") - close(quit) - wg.Wait() - - return results, nil -} - -// getMysqlServer returns a exec.Cmd for a dolt server -func getMysqlServer(ctx context.Context, config *ServerConfig, params []string) *exec.Cmd { - return ExecCommand(ctx, config.ServerExec, params...) -} - -// InitMysqlDataDir initializes a mysql data dir and returns the path -func InitMysqlDataDir(ctx context.Context, config *ServerConfig) (string, error) { - serverDir, err := createServerDir(dbName) - if err != nil { - return "", err - } - - msInit := ExecCommand(ctx, config.ServerExec, MysqlInitializeInsecureFlag, fmt.Sprintf("%s=%s", MysqlDataDirFlag, serverDir)) - err = msInit.Run() - if err != nil { - return "", err - } - - return serverDir, nil -} - -func SetupDB(ctx context.Context, mConfig MysqlConfig, databaseName string) (err error) { - dsn, err := FormatDsn(mConfig) - if err != nil { - return err - } - - // TODO make sure this can work on windows - db, err := sql.Open("mysql", dsn) - if err != nil { - return err - } - defer func() { - rerr := db.Close() - if err == nil { - err = rerr - } - }() - err = db.Ping() - if err != nil { - return err - } - _, err = db.ExecContext(ctx, fmt.Sprintf("DROP DATABASE IF EXISTS %s", databaseName)) - if err != nil { - return err - } - _, err = db.ExecContext(ctx, fmt.Sprintf("CREATE DATABASE %s", databaseName)) - if err != nil { - return err - } - _, err = db.ExecContext(ctx, fmt.Sprintf("DROP USER IF EXISTS %s", sysbenchUserLocal)) - if err != nil { - return err - } - _, err = db.ExecContext(ctx, fmt.Sprintf("CREATE USER %s IDENTIFIED WITH mysql_native_password BY '%s'", sysbenchUserLocal, sysbenchPassLocal)) - if err != nil { - return err - } - _, err = db.ExecContext(ctx, fmt.Sprintf("GRANT ALL ON %s.* to %s", databaseName, sysbenchUserLocal)) - if err != nil { - return err - } - _, err = db.ExecContext(ctx, "SET GLOBAL local_infile = 'ON'") - if err != nil { - return err - } - // Required for running groupby_scan.lua without error - _, err = db.ExecContext(ctx, "SET GLOBAL sql_mode=(SELECT REPLACE(@@sql_mode,'ONLY_FULL_GROUP_BY',''));") - if err != nil { - return err - } - - return -} - -func FormatDsn(mConfig MysqlConfig) (string, error) { - var socketPath string - if mConfig.Socket != "" { - socketPath = mConfig.Socket - } else { - socketPath = defaultMysqlSocket - } - - if mConfig.ConnectionProtocol == tcpProtocol { - return fmt.Sprintf("root@tcp(%s:%d)/", mConfig.Host, mConfig.Port), nil - } else if mConfig.ConnectionProtocol == unixProtocol { - return fmt.Sprintf("root@unix(%s)/", socketPath), nil - } else { - return "", ErrUnsupportedConnectionProtocol - } -} - -func GetMysqlConnectionConfigFromServerConfig(config *ServerConfig) MysqlConfig { - return MysqlConfig{ - Socket: config.Socket, - ConnectionProtocol: config.ConnectionProtocol, - Port: config.Port, - Host: defaultHost, - } -} diff --git a/go/performance/utils/sysbench_runner/postgres.go b/go/performance/utils/sysbench_runner/postgres.go deleted file mode 100644 index ce2e26be82e..00000000000 --- a/go/performance/utils/sysbench_runner/postgres.go +++ /dev/null @@ -1,200 +0,0 @@ -// Copyright 2019-2022 Dolthub, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package sysbench_runner - -import ( - "context" - "fmt" - "log" - "os" - "os/exec" - "os/signal" - "sync" - "syscall" - "time" - - "database/sql" - - _ "github.com/lib/pq" - "golang.org/x/sync/errgroup" -) - -type PostgresConfig struct { - Socket string - ConnectionProtocol string - Port int - Host string -} - -// BenchmarkPostgres benchmarks postgres based on the provided configurations -func BenchmarkPostgres(ctx context.Context, config *Config, serverConfig *ServerConfig) (Results, error) { - withKeyCtx, cancel := context.WithCancel(ctx) - - var serverDir string - defer func() { - if serverDir != "" { - os.RemoveAll(serverDir) - } - }() - - var localServer bool - var gServer *errgroup.Group - var serverCtx context.Context - var server *exec.Cmd - var err error - if serverConfig.Host == defaultHost { - log.Println("Launching the default server") - localServer = true - - serverDir, err = initPostgresDataDir(ctx, serverConfig) - if err != nil { - cancel() - return nil, err - } - gServer, serverCtx = errgroup.WithContext(withKeyCtx) - var serverParams []string - serverParams, err = serverConfig.GetServerArgs() - if err != nil { - cancel() - return nil, err - } - serverParams = append(serverParams, "-D", serverDir) - server = getMysqlServer(serverCtx, serverConfig, serverParams) - server.Env = append(server.Env, "LC_ALL=C") - - // launch the postgres server - gServer.Go(func() error { - return server.Run() - }) - - // sleep to allow the server to start - time.Sleep(10 * time.Second) - - // setup postgres db - err := setupPostgresDB(ctx, serverConfig.Host, fmt.Sprintf("%d", serverConfig.Port), "postgres", dbName) - if err != nil { - cancel() - return nil, err - } - - log.Println("Successfully set up the Postgres database") - } - - // handle user interrupt - quit := make(chan os.Signal, 1) - signal.Notify(quit, os.Interrupt, syscall.SIGTERM) - var wg sync.WaitGroup - wg.Add(1) - go func() { - <-quit - defer wg.Done() - signal.Stop(quit) - cancel() - }() - - tests, err := GetTests(config, serverConfig, nil) - if err != nil { - return nil, err - } - - results := make(Results, 0) - for i := 0; i < config.Runs; i++ { - for _, test := range tests { - r, err := benchmark(withKeyCtx, test, config, serverConfig, stampFunc, serverConfig.GetId()) - if err != nil { - close(quit) - wg.Wait() - return nil, err - } - results = append(results, r) - } - } - - // stop local mysql server - if localServer { - // send signal to server - quit <- syscall.SIGTERM - - err = gServer.Wait() - if err != nil { - // we expect a kill error - // we only exit in error if this is not the - // error - if err.Error() != "signal: killed" { - close(quit) - wg.Wait() - return nil, err - } - } - } - - fmt.Println("Successfully killed server") - close(quit) - wg.Wait() - - return results, nil -} - -// initPostgresDataDir initializes a postgres data dir and returns the path -func initPostgresDataDir(ctx context.Context, config *ServerConfig) (string, error) { - serverDir, err := createServerDir(dbName) - if err != nil { - return "", err - } - - pgInit := ExecCommand(ctx, config.InitExec, fmt.Sprintf("--pgdata=%s", serverDir), "--username=postgres") - err = pgInit.Run() - if err != nil { - return "", err - } - - return serverDir, nil -} - -func setupPostgresDB(ctx context.Context, host, port, user, dbname string) (err error) { - psqlconn := fmt.Sprintf("host=%s port=%s user=%s password=%s dbname=%s sslmode=disable", host, port, user, "", dbname) - - db, err := sql.Open("postgres", psqlconn) - if err != nil { - return err - } - defer func() { - rerr := db.Close() - if err == nil { - err = rerr - } - }() - err = db.PingContext(ctx) - if err != nil { - return err - } - _, err = db.ExecContext(ctx, fmt.Sprintf("DROP DATABASE IF EXISTS %s", dbname)) - if err != nil { - return err - } - _, err = db.ExecContext(ctx, fmt.Sprintf("DROP USER IF EXISTS %s", sysbenchUsername)) - if err != nil { - return err - } - _, err = db.ExecContext(ctx, fmt.Sprintf("CREATE USER %s WITH PASSWORD '%s'", sysbenchUsername, sysbenchPassLocal)) - if err != nil { - return err - } - _, err = db.ExecContext(ctx, fmt.Sprintf("CREATE DATABASE %s WITH OWNER %s", dbname, sysbenchUsername)) - if err != nil { - return err - } - return -} diff --git a/go/performance/utils/sysbench_runner/profile.go b/go/performance/utils/sysbench_runner/profile.go deleted file mode 100644 index feb27873e9b..00000000000 --- a/go/performance/utils/sysbench_runner/profile.go +++ /dev/null @@ -1,206 +0,0 @@ -// Copyright 2019-2022 Dolthub, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package sysbench_runner - -import ( - "context" - "fmt" - "io" - "os" - "os/signal" - "path/filepath" - "sync" - "syscall" - "time" - - "golang.org/x/sync/errgroup" -) - -// ProfileDolt profiles dolt while running the provided tests -func ProfileDolt(ctx context.Context, config *Config, serverConfig *ServerConfig) error { - serverParams, err := serverConfig.GetServerArgs() - if err != nil { - return err - } - - err = DoltVersion(ctx, serverConfig.ServerExec) - if err != nil { - return err - } - - err = UpdateDoltConfig(ctx, serverConfig.ServerExec) - if err != nil { - return err - } - - testRepo, err := initDoltRepo(ctx, serverConfig, config.NomsBinFormat) - if err != nil { - return err - } - - tests, err := GetTests(config, serverConfig, nil) - if err != nil { - return err - } - - tempProfilesDir, err := os.MkdirTemp("", "") - if err != nil { - return err - } - defer os.RemoveAll(tempProfilesDir) - - for i := 0; i < config.Runs; i++ { - for _, test := range tests { - _, err = profileTest(ctx, test, config, serverConfig, serverParams, testRepo, tempProfilesDir) - if err != nil { - return err - } - } - } - - profile, err := mergeProfiles(ctx, tempProfilesDir, serverConfig.ProfilePath) - if err != nil { - return err - } - - fmt.Println("Profile created at:", profile) - - err = os.RemoveAll(testRepo) - if err != nil { - return err - } - - return nil -} - -func profileTest(ctx context.Context, test *Test, config *Config, serverConfig *ServerConfig, serverParams []string, testRepo, profileDir string) (string, error) { - profilePath, err := os.MkdirTemp("", filepath.Base(test.Name)) - if err != nil { - return "", err - } - defer os.RemoveAll(profilePath) - - tempProfile := filepath.Join(profilePath, cpuProfileFilename) - profileParams := make([]string, 0) - profileParams = append(profileParams, profileFlag, cpuProfile, profilePathFlag, profilePath) - profileParams = append(profileParams, serverParams...) - - withKeyCtx, cancel := context.WithCancel(ctx) - defer cancel() - gServer, serverCtx := errgroup.WithContext(withKeyCtx) - server := getServer(serverCtx, serverConfig, testRepo, profileParams) - - // handle user interrupt - quit := make(chan os.Signal, 1) - signal.Notify(quit, os.Interrupt, syscall.SIGINT) - var wg sync.WaitGroup - wg.Add(1) - go func() { - s := <-quit - defer wg.Done() - server.Process.Signal(s) - signal.Stop(quit) - }() - - // launch the dolt server - gServer.Go(func() error { - return server.Run() - }) - - // sleep to allow the server to start - time.Sleep(5 * time.Second) - - _, err = benchmark(withKeyCtx, test, config, serverConfig, stampFunc, serverConfig.GetId()) - if err != nil { - close(quit) - wg.Wait() - return "", err - } - - // send signal to dolt server - quit <- syscall.SIGINT - - err = gServer.Wait() - if err != nil { - // we expect a kill error - // we only exit in error if this is not the - // error - if err.Error() != "signal: killed" { - fmt.Println(err) - close(quit) - wg.Wait() - return "", err - } - } - - fmt.Println("Successfully killed server") - close(quit) - wg.Wait() - - info, err := os.Stat(tempProfile) - if err != nil { - return "", err - } - if info.Size() < 1 { - return "", fmt.Errorf("failed to create profile: file was empty") - } - - finalProfile := filepath.Join(profileDir, fmt.Sprintf("%s_%s_%s", serverConfig.Id, test.Name, cpuProfileFilename)) - err = moveFile(tempProfile, finalProfile) - return finalProfile, err -} - -func mergeProfiles(ctx context.Context, sourceProfilesDir, destProfileDir string) (string, error) { - tmp, err := os.MkdirTemp("", "final_cpu_pprof") - if err != nil { - return "", err - } - defer os.RemoveAll(tmp) - outfile := filepath.Join(tmp, "cpu.pprof") - - merge := ExecCommand(ctx, "/bin/sh", "-c", fmt.Sprintf("go tool pprof -proto *.pprof > %s", outfile)) - merge.Dir = sourceProfilesDir - err = merge.Run() - if err != nil { - return "", err - } - - final := filepath.Join(destProfileDir, "cpu.pprof") - err = moveFile(outfile, final) - return final, err -} - -func moveFile(sourcePath, destPath string) error { - err := copyFile(sourcePath, destPath) - if err != nil { - return err - } - return os.Remove(sourcePath) -} - -func copyFile(sourcePath, destPath string) error { - inputFile, err := os.Open(sourcePath) - if err != nil { - return err - } - defer inputFile.Close() - outputFile, err := os.Create(destPath) - if err != nil { - return err - } - defer outputFile.Close() - _, err = io.Copy(outputFile, inputFile) - return err -} diff --git a/go/performance/utils/sysbench_runner/run.go b/go/performance/utils/sysbench_runner/run.go deleted file mode 100644 index 5e1e764ccad..00000000000 --- a/go/performance/utils/sysbench_runner/run.go +++ /dev/null @@ -1,110 +0,0 @@ -// Copyright 2019-2022 Dolthub, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package sysbench_runner - -import ( - "context" - "fmt" - "os" - "path/filepath" -) - -// Run runs sysbench runner -func Run(config *Config) error { - err := config.Validate() - if err != nil { - return err - } - - ctx := context.Background() - - err = sysbenchVersion(ctx) - if err != nil { - return err - } - - for _, serverConfig := range config.Servers { - var results Results - switch serverConfig.Server { - case Dolt: - // handle a profiling run - if serverConfig.ServerProfile != "" { - fmt.Println("Profiling dolt while running sysbench tests") - return ProfileDolt(ctx, config, serverConfig) - } - fmt.Println("Running dolt sysbench test") - results, err = BenchmarkDolt(ctx, config, serverConfig) - case Doltgres: - fmt.Println("Running doltgres sysbench test") - results, err = BenchmarkDoltgres(ctx, config, serverConfig) - case MySql: - fmt.Println("Running mysql sysbench test") - results, err = BenchmarkMysql(ctx, config, serverConfig) - case Postgres: - fmt.Println("Running postgres sysbench test") - results, err = BenchmarkPostgres(ctx, config, serverConfig) - default: - panic(fmt.Sprintf("unexpected server type: %s", serverConfig.Server)) - } - if err != nil { - return err - } - - fmt.Printf("Successfuly finished %s\n", serverConfig.Server) - - err = WriteResults(serverConfig, results) - if err != nil { - return err - } - - fmt.Printf("Successfuly wrote results for %s\n", serverConfig.Server) - } - return nil -} - -func sysbenchVersion(ctx context.Context) error { - sysbenchVersion := ExecCommand(ctx, "sysbench", "--version") - return sysbenchVersion.Run() -} - -func WriteResults(serverConfig *ServerConfig, results Results) error { - cwd, err := os.Getwd() - if err != nil { - return err - } - var writePath string - switch serverConfig.ResultsFormat { - case CsvFormat, CsvExt: - writePath = filepath.Join( - cwd, - "results", - string(serverConfig.Server), - serverConfig.Version, - serverConfig.GetId(), - fmt.Sprintf(ResultFileTemplate, serverConfig.GetId(), serverConfig.Server, serverConfig.Version, CsvExt)) - return WriteResultsCsv(writePath, results) - case JsonFormat, JsonExt: - writePath = filepath.Join( - cwd, - "results", - string(serverConfig.Server), - serverConfig.Version, - serverConfig.GetId(), - fmt.Sprintf(ResultFileTemplate, serverConfig.GetId(), serverConfig.Server, serverConfig.Version, JsonExt)) - return WriteResultsJson(writePath, results) - default: - } - return fmt.Errorf("unsupported results format: %s", serverConfig.ResultsFormat) -} diff --git a/go/performance/utils/sysbench_runner/run_test.go b/go/performance/utils/sysbench_runner/run_test.go deleted file mode 100644 index dd85db622c4..00000000000 --- a/go/performance/utils/sysbench_runner/run_test.go +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright 2022 Dolthub, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package sysbench_runner - -import ( - "log" - "os" - "testing" -) - -func TestRunner(t *testing.T) { - t.Skip() - dir := t.TempDir() - log.Println(dir) - err := os.Chdir(dir) - if err != nil { - log.Fatal(err) - } - - conf := &Config{ - Tests: selectTests("oltp_read_write", "oltp_update_index", "oltp_delete_insert"), - //Tests: selectTests("oltp_read_write", "oltp_update_index", "oltp_update_non_index", "oltp_insert", "bulk_insert", "oltp_write_only", "oltp_delete"), - Servers: []*ServerConfig{ - { - Id: "test", - Server: Dolt, - Version: "0.39.2", - ResultsFormat: CsvFormat, - ServerExec: "/Users/max-hoffman/go/bin/dolt", - }, - }, - ScriptDir: "/Users/max-hoffman/Documents/dolthub/sysbench-lua-scripts", - TestOptions: []string{ - "--rand-seed=1", - "--table-size=10000", - "--rand-type=uniform", - "--time=120", - "--percentile=50", - }, - InitBigRepo: true, - } - - err = Run(conf) - if err != nil { - log.Fatal(err) - } -} - -func selectTests(names ...string) []*ConfigTest { - tests := make([]*ConfigTest, len(names)) - for i := range names { - tests[i] = &ConfigTest{Name: names[i], FromScript: false} - } - return tests -} diff --git a/go/performance/utils/tpcc_runner/README.md b/go/performance/utils/tpcc_runner/README.md deleted file mode 100644 index cbbbf426395..00000000000 --- a/go/performance/utils/tpcc_runner/README.md +++ /dev/null @@ -1,33 +0,0 @@ -TPCC runner is a tool for running TPCC tests against sql servers. These tests run against the -Percona Labs repo [here](https://github.com/Percona-Lab/sysbench-tpcc). - -The tool requires a json config file to run. - -```bash -$ go run cmd/main.go --config=sample-tpcc-config.json -``` - -Note to this run this locally you need to have the TPCC repo cloned. The `ScriptDir` variable should then be linked -to the path of the cloned repo. - -Configuration: - -```json -{ - "Servers": "[...]", - "ScriptDir":"/Users/vinairachakonda/go/src/dolthub/sysbench-tpcc", - "ScaleFactors": [1] -} -``` - -`Servers`: The server defintions to run the benchmark against. Accepts Dolt and MySQL configuratiosn. - -`ScriptDir`: The directory of the TPCC testing scripts - -`ScaleFactors`: The number of warehouse to be generated in the test case. - -`NomsBinFormat`: The NomsBinFormat to use for this benchmark. - -Note that this configuration is still incomplete for the amount of the variable TPCC varies. This intentional as we -want expose small amounts of independent variables until Dolt gets more robust. See `config.go` to get a breakdown of all the -variables TPCC varies. diff --git a/go/performance/utils/tpcc_runner/cmd/main.go b/go/performance/utils/tpcc_runner/cmd/main.go deleted file mode 100644 index a81be869fdf..00000000000 --- a/go/performance/utils/tpcc_runner/cmd/main.go +++ /dev/null @@ -1,59 +0,0 @@ -// Copyright 2022 Dolthub, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "flag" - "fmt" - "log" - "os" - "path/filepath" - - tpcc_runner "github.com/dolthub/dolt/go/performance/utils/tpcc_runner" -) - -var configFile = flag.String("config", "", "path to config file q") - -func main() { - fmt.Println("Running the TPCC benchmark.") - - flag.Parse() - if *configFile == "" { - log.Fatal("Must supply config") - } - - configPath, err := filepath.Abs(*configFile) - if err != nil { - log.Fatal(err) - } - - if _, err = os.Stat(configPath); os.IsNotExist(err) { - log.Fatal(err) - } - - tpccBenchmarkConfig, err := tpcc_runner.FromFileConfig(configPath) - if err != nil { - log.Fatal(err) - } - - // Run the TPCC test - err = tpcc_runner.Run(tpccBenchmarkConfig) - if err != nil { - fmt.Println(err) - os.Exit(1) - } - - os.Exit(0) -} diff --git a/go/performance/utils/tpcc_runner/config.go b/go/performance/utils/tpcc_runner/config.go deleted file mode 100644 index 7ffeeeba874..00000000000 --- a/go/performance/utils/tpcc_runner/config.go +++ /dev/null @@ -1,255 +0,0 @@ -// Copyright 2022 Dolthub, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package tpcc_runner - -import ( - "context" - "encoding/json" - "fmt" - "os" - "os/exec" - "path/filepath" - "runtime" - - "github.com/google/uuid" - - "github.com/dolthub/dolt/go/performance/utils/sysbench_runner" -) - -const ( - defaultHost = "127.0.0.1" - defaultUser = "root" - - // Note this is built for the SysbenchDocker file. If you want to run locally you'll need to override these variables - // for your local MySQL setup. - tpccUserLocal = "'sysbench'@'localhost'" - tpccPassLocal = "sysbenchpass" -) - -var defaultTpccParams = []string{ - fmt.Sprintf("--mysql-db=%s", dbName), - "--db-driver=mysql", -} - -// TpccBenchmarkConfig represents a configuration for an execution of the TPCC Benchmark. It executes a series of tests -// against different ServerConfigurations. -type TpccBenchmarkConfig struct { - // RuntimeOS is the platform the benchmarks ran on - RuntimeOS string - - // RuntimeGoArch is the runtime architecture - RuntimeGoArch string - - // ScriptDir represents the location of the TPCC tests - ScriptDir string - - // Servers are the servers to benchmark. - Servers []*sysbench_runner.ServerConfig - - // ScaleFactors represent the scale at which to run each TpccBenchmark at. - ScaleFactors []int - - // NomsBinFormat specifies the NomsBinFormat - NomsBinFormat string -} - -func NewTpccConfig() *TpccBenchmarkConfig { - return &TpccBenchmarkConfig{ - Servers: make([]*sysbench_runner.ServerConfig, 0), - ScaleFactors: make([]int, 0), - } -} - -func (c *TpccBenchmarkConfig) updateDefaults() error { - if len(c.Servers) < 1 { - return sysbench_runner.ErrNoServersDefined - } - - // TODO: Eventually we need to support scale factors all the way to 10 - if len(c.ScaleFactors) == 0 { - c.ScaleFactors = append(c.ScaleFactors, 1) - } - - if c.RuntimeOS == "" { - c.RuntimeOS = runtime.GOOS - } - if c.RuntimeGoArch == "" { - c.RuntimeGoArch = runtime.GOARCH - } - - return c.validateServerConfigs() -} - -// validateServerConfigs ensures the ServerConfigs are valid -func (c *TpccBenchmarkConfig) validateServerConfigs() error { - portMap := make(map[int]sysbench_runner.ServerType) - for _, s := range c.Servers { - if s.Server != sysbench_runner.Dolt && s.Server != sysbench_runner.MySql { - return fmt.Errorf("unsupported server type: %s", s.Server) - } - - err := sysbench_runner.ValidateRequiredFields(string(s.Server), s.Version, s.ResultsFormat) - if err != nil { - return err - } - - if s.Server == sysbench_runner.MySql { - err = sysbench_runner.CheckProtocol(s.ConnectionProtocol) - if err != nil { - return err - } - } - - if s.Host == "" { - s.Host = defaultHost - } - - portMap, err = sysbench_runner.CheckUpdatePortMap(s, portMap) - if err != nil { - return err - } - - err = sysbench_runner.CheckExec(s.ServerExec, "server exec") - if err != nil { - return err - } - } - return nil -} - -// FromFileConfig returns a validated Config based on the config file at the configPath -func FromFileConfig(configPath string) (*TpccBenchmarkConfig, error) { - data, err := os.ReadFile(configPath) - if err != nil { - return nil, err - } - - config := NewTpccConfig() - err = json.Unmarshal(data, config) - if err != nil { - return nil, err - } - - return config, nil -} - -// TpccTest encapsulates an End to End prepare, run, cleanup test case. -type TpccTest struct { - // Id represents a unique test id - Id string - - // Name represents the name of the test case - Name string - - // Params are associated parameters this test runs with - Params *TpccTestParams -} - -type TpccTestParams struct { - // NumThreads represents the number of threads running queries concurrently. - NumThreads int - - // ScaleFactor represents the number of warehouse to test this at scale. - ScaleFactor int - - // Tables represents the number of tables created per warehouse. - Tables int - - // TrxLevel represents what transaction level to use - TrxLevel string - - // ReportCSV determines whether to report output as a csv. - ReportCSV bool - - // ReportInterval defines how often the tpcc benchmark outputs performance stats. - ReportInterval int - - // Time represents how long - Time int -} - -// NewDefaultTpccParams returns default TpccTestParams. -func NewDefaultTpccParams() *TpccTestParams { - return &TpccTestParams{ - NumThreads: 2, // TODO: When ready, expose as command line argument. - ScaleFactor: 1, - Tables: 1, - TrxLevel: "RR", - ReportCSV: true, - ReportInterval: 1, - Time: 30, - } -} - -// NewTpccTest instantiates and returns a TPCC test. -func NewTpccTest(name string, params *TpccTestParams) *TpccTest { - return &TpccTest{ - Id: uuid.New().String(), - Name: name, - Params: params, - } -} - -// getArgs returns a test's args for all TPCC steps -func (t *TpccTest) getArgs(serverConfig *sysbench_runner.ServerConfig) []string { - params := make([]string, 0) - params = append(params, defaultTpccParams...) - - params = append(params, fmt.Sprintf("--mysql-host=%s", serverConfig.Host)) - - // handle sysbench user for local mysql server - if serverConfig.Server == sysbench_runner.MySql && serverConfig.Host == defaultHost { - params = append(params, fmt.Sprintf("--mysql-user=%s", "sysbench")) - params = append(params, fmt.Sprintf("--mysql-password=%s", tpccPassLocal)) - } else { - params = append(params, fmt.Sprintf("--mysql-port=%d", serverConfig.Port)) - params = append(params, fmt.Sprintf("--mysql-user=%s", defaultUser)) - } - - params = append(params, fmt.Sprintf("--time=%d", t.Params.Time)) - params = append(params, fmt.Sprintf("--threads=%d", t.Params.NumThreads)) - params = append(params, fmt.Sprintf("--report_interval=%d", t.Params.ReportInterval)) - params = append(params, fmt.Sprintf("--tables=%d", t.Params.Tables)) - params = append(params, fmt.Sprintf("--scale=%d", t.Params.ScaleFactor)) - params = append(params, fmt.Sprintf("--trx_level=%s", t.Params.TrxLevel)) - - return params -} - -// TpccPrepare prepares the command executable for the Prepare step. -func (t *TpccTest) TpccPrepare(ctx context.Context, serverConfig *sysbench_runner.ServerConfig, scriptDir string) *exec.Cmd { - cmd := sysbench_runner.ExecCommand(ctx, scriptDir+"/tpcc.lua", append(t.getArgs(serverConfig), "prepare")...) - return addParamsToCmd(cmd, scriptDir) -} - -// TpccRun prepares the command executable for the Run step. -func (t *TpccTest) TpccRun(ctx context.Context, serverConfig *sysbench_runner.ServerConfig, scriptDir string) *exec.Cmd { - cmd := exec.CommandContext(ctx, scriptDir+"/tpcc.lua", append(t.getArgs(serverConfig), "run")...) - return addParamsToCmd(cmd, scriptDir) -} - -// TpccCleanup prepares the cleanup executable for the Cleanup step. -func (t *TpccTest) TpccCleanup(ctx context.Context, serverConfig *sysbench_runner.ServerConfig, scriptDir string) *exec.Cmd { - cmd := sysbench_runner.ExecCommand(ctx, scriptDir+"/tpcc.lua", append(t.getArgs(serverConfig), "cleanup")...) - return addParamsToCmd(cmd, scriptDir) -} - -func addParamsToCmd(cmd *exec.Cmd, scriptDir string) *exec.Cmd { - lp := filepath.Join(scriptDir, "?.lua") - cmd.Env = os.Environ() - cmd.Env = append(cmd.Env, fmt.Sprintf("LUA_PATH=%s", lp)) - - return cmd -} diff --git a/go/performance/utils/tpcc_runner/dolt.go b/go/performance/utils/tpcc_runner/dolt.go deleted file mode 100644 index e9ab273cf70..00000000000 --- a/go/performance/utils/tpcc_runner/dolt.go +++ /dev/null @@ -1,204 +0,0 @@ -// Copyright 2022 Dolthub, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package tpcc_runner - -import ( - "context" - "fmt" - "os" - "os/exec" - "os/signal" - "path/filepath" - "sync" - "syscall" - "time" - - "golang.org/x/sync/errgroup" - - "github.com/dolthub/dolt/go/performance/utils/sysbench_runner" -) - -const ( - dbName = "sbt" - nbfEnvVar = "DOLT_DEFAULT_BIN_FORMAT" -) - -// BenchmarkDolt executes a set of tpcc tests against a dolt server. -func BenchmarkDolt(ctx context.Context, tppcConfig *TpccBenchmarkConfig, serverConfig *sysbench_runner.ServerConfig) (sysbench_runner.Results, error) { - serverParams, err := serverConfig.GetServerArgs() - if err != nil { - return nil, err - } - - err = sysbench_runner.UpdateDoltConfig(ctx, serverConfig.ServerExec) - if err != nil { - return nil, err - } - - testRepo, err := initDoltRepo(ctx, serverConfig, tppcConfig.NomsBinFormat) - if err != nil { - return nil, err - } - - withKeyCtx, cancel := context.WithCancel(ctx) - gServer, serverCtx := errgroup.WithContext(withKeyCtx) - - server := getDoltServer(serverCtx, serverConfig, testRepo, serverParams) - - // handle user interrupt - quit := make(chan os.Signal, 1) - signal.Notify(quit, os.Interrupt, syscall.SIGTERM) - var wg sync.WaitGroup - wg.Add(1) - go func() { - <-quit - defer wg.Done() - signal.Stop(quit) - cancel() - }() - - // launch the dolt server - gServer.Go(func() error { - return server.Run() - }) - - // sleep to allow the server to start - time.Sleep(5 * time.Second) - - // GetTests and Benchmarks - tests := getTests(tppcConfig) - results := make(sysbench_runner.Results, 0) - - for _, test := range tests { - result, err := benchmark(ctx, test, serverConfig, tppcConfig) - if err != nil { - close(quit) - wg.Wait() - return nil, err - } - - results = append(results, result) - } - - // send signal to dolt server - quit <- syscall.SIGTERM - - err = gServer.Wait() - if err != nil { - // we expect a kill error - // we only exit in error if this is not the - // error - if err.Error() != "signal: killed" { - close(quit) - wg.Wait() - return nil, err - } - } - - close(quit) - wg.Wait() - - return results, os.RemoveAll(testRepo) -} - -// initDoltRepo initializes a dolt repo and returns the repo path -func initDoltRepo(ctx context.Context, config *sysbench_runner.ServerConfig, nbf string) (string, error) { - if nbf != "" { - if err := os.Setenv(nbfEnvVar, nbf); err != nil { - return "", err - } - } - - cwd, err := os.Getwd() - if err != nil { - return "", err - } - - testRepo := filepath.Join(cwd, dbName) - err = os.MkdirAll(testRepo, os.ModePerm) - if err != nil { - return "", err - } - - err = os.RemoveAll(filepath.Join(testRepo, ".dolt")) - if err != nil { - return "", err - } - - doltInit := sysbench_runner.ExecCommand(ctx, config.ServerExec, "init") - doltInit.Dir = testRepo - err = doltInit.Run() - if err != nil { - return "", err - } - - return testRepo, nil -} - -// getDoltServer returns a exec.Cmd for a dolt server -func getDoltServer(ctx context.Context, config *sysbench_runner.ServerConfig, testRepo string, params []string) *exec.Cmd { - server := sysbench_runner.ExecCommand(ctx, config.ServerExec, params...) - server.Dir = testRepo - return server -} - -// getTests creates a set of tests that the server needs to be executed on. -func getTests(config *TpccBenchmarkConfig) []*TpccTest { - tests := make([]*TpccTest, 0) - for _, sf := range config.ScaleFactors { - params := NewDefaultTpccParams() - params.ScaleFactor = sf - test := NewTpccTest(fmt.Sprintf("tpcc-scale-factor-%d", sf), params) - tests = append(tests, test) - } - - return tests -} - -// benchmark runs the relevant tpcc test against a server with a config. -func benchmark(ctx context.Context, test *TpccTest, serverConfig *sysbench_runner.ServerConfig, config *TpccBenchmarkConfig) (*sysbench_runner.Result, error) { - prepare := test.TpccPrepare(ctx, serverConfig, config.ScriptDir) - run := test.TpccRun(ctx, serverConfig, config.ScriptDir) - cleanup := test.TpccCleanup(ctx, serverConfig, config.ScriptDir) - - err := prepare.Run() - if err != nil { - return nil, err - } - - if run.Stdout != nil { - run.Stdout = nil - } - - out, err := run.Output() - if err != nil { - fmt.Print(string(out)) - return nil, err - } - - fmt.Print(string(out)) - - result, err := FromOutputResult(out, config, serverConfig, test, "tpcc", nil) - if err != nil { - return nil, err - } - - err = cleanup.Run() - if err != nil { - return nil, err - } - - return result, nil -} diff --git a/go/performance/utils/tpcc_runner/mysql.go b/go/performance/utils/tpcc_runner/mysql.go deleted file mode 100644 index 19a4e593a40..00000000000 --- a/go/performance/utils/tpcc_runner/mysql.go +++ /dev/null @@ -1,135 +0,0 @@ -// Copyright 2022 Dolthub, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package tpcc_runner - -import ( - "context" - "fmt" - "os" - "os/exec" - "os/signal" - "sync" - "syscall" - "time" - - "golang.org/x/sync/errgroup" - - "github.com/dolthub/dolt/go/performance/utils/sysbench_runner" -) - -// BenchmarkMysql benchmarks a mysql server based on the provided configurations. -func BenchmarkMysql(ctx context.Context, config *TpccBenchmarkConfig, serverConfig *sysbench_runner.ServerConfig) (sysbench_runner.Results, error) { - withKeyCtx, cancel := context.WithCancel(ctx) - - var serverDir string - defer func() { - if serverDir != "" { - os.RemoveAll(serverDir) - } - }() - - var localServer bool - var gServer *errgroup.Group - var serverCtx context.Context - var server *exec.Cmd - var err error - if serverConfig.Host == defaultHost { - localServer = true - serverDir, err = sysbench_runner.InitMysqlDataDir(ctx, serverConfig) - if err != nil { - cancel() - return nil, err - } - - gServer, serverCtx = errgroup.WithContext(withKeyCtx) - var serverParams []string - serverParams, err = serverConfig.GetServerArgs() - if err != nil { - cancel() - return nil, err - } - serverParams = append(serverParams, fmt.Sprintf("%s=%s", sysbench_runner.MysqlDataDirFlag, serverDir)) - server = getMysqlServer(serverCtx, serverConfig, serverParams) - - // launch the mysql server - gServer.Go(func() error { - return server.Run() - }) - - // sleep to allow the server to start - time.Sleep(10 * time.Second) - - // setup mysqldb - err := sysbench_runner.SetupDB(ctx, sysbench_runner.GetMysqlConnectionConfigFromServerConfig(serverConfig), dbName) - if err != nil { - cancel() - return nil, err - } - } - - // handle user interrupt - quit := make(chan os.Signal, 1) - signal.Notify(quit, os.Interrupt, syscall.SIGTERM) - var wg sync.WaitGroup - wg.Add(1) - go func() { - <-quit - defer wg.Done() - signal.Stop(quit) - cancel() - }() - - tests := getTests(config) - - results := make(sysbench_runner.Results, 0) - - for _, test := range tests { - r, err := benchmark(withKeyCtx, test, serverConfig, config) - if err != nil { - close(quit) - wg.Wait() - return nil, err - } - results = append(results, r) - } - - // stop local mysql server - if localServer { - // send signal to server - quit <- syscall.SIGTERM - - err := gServer.Wait() - if err != nil { - // we expect a kill error - // we only exit in error if this is not the - // error - if err.Error() != "signal: killed" { - close(quit) - wg.Wait() - return nil, err - } - } - } - - close(quit) - wg.Wait() - - return results, nil -} - -// getMysqlServer returns a exec.Cmd for a dolt server -func getMysqlServer(ctx context.Context, config *sysbench_runner.ServerConfig, params []string) *exec.Cmd { - return sysbench_runner.ExecCommand(ctx, config.ServerExec, params...) -} diff --git a/go/performance/utils/tpcc_runner/results.go b/go/performance/utils/tpcc_runner/results.go deleted file mode 100644 index 33e0a11e262..00000000000 --- a/go/performance/utils/tpcc_runner/results.go +++ /dev/null @@ -1,86 +0,0 @@ -// Copyright 2022 Dolthub, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package tpcc_runner - -import ( - "path/filepath" - "strings" - - "github.com/google/uuid" - - "github.com/dolthub/dolt/go/performance/utils/sysbench_runner" -) - -// FromConfigsNewResult returns a new result with some fields set based on the provided configs -func FromConfigsNewResult(config *TpccBenchmarkConfig, serverConfig *sysbench_runner.ServerConfig, test *TpccTest, suiteId string, idFunc func() string) (*sysbench_runner.Result, error) { - serverParams, err := serverConfig.GetServerArgs() - if err != nil { - return nil, err - } - - var getId func() string - if idFunc == nil { - getId = func() string { - return uuid.New().String() - } - } else { - getId = idFunc - } - - var name string - base := filepath.Base(test.Name) - ext := filepath.Ext(base) - name = strings.TrimSuffix(base, ext) - - return &sysbench_runner.Result{ - Id: getId(), - SuiteId: suiteId, - TestId: test.Id, - RuntimeOS: config.RuntimeOS, - RuntimeGoArch: config.RuntimeGoArch, - ServerName: string(serverConfig.Server), - ServerVersion: serverConfig.Version, - ServerParams: strings.Join(serverParams, " "), - TestName: name, - TestParams: strings.Join(test.getArgs(serverConfig), " "), - }, nil -} - -// FromOutputResult accepts raw sysbench run output and returns the Result -func FromOutputResult(output []byte, config *TpccBenchmarkConfig, serverConfig *sysbench_runner.ServerConfig, test *TpccTest, suiteId string, idFunc func() string) (*sysbench_runner.Result, error) { - result, err := FromConfigsNewResult(config, serverConfig, test, suiteId, idFunc) - if err != nil { - return nil, err - } - lines := strings.Split(string(output), "\n") - var process bool - for _, l := range lines { - trimmed := strings.TrimSpace(l) - if trimmed == "" { - continue - } - if strings.HasPrefix(trimmed, sysbench_runner.SqlStatsPrefix) { - process = true - continue - } - if process { - err := sysbench_runner.UpdateResult(result, trimmed) - if err != nil { - return result, err - } - } - } - return result, nil -} diff --git a/go/performance/utils/tpcc_runner/run.go b/go/performance/utils/tpcc_runner/run.go deleted file mode 100644 index 59f78db3e89..00000000000 --- a/go/performance/utils/tpcc_runner/run.go +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright 2022 Dolthub, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package tpcc_runner - -import ( - "context" - "fmt" - - "github.com/dolthub/dolt/go/performance/utils/sysbench_runner" -) - -func Run(config *TpccBenchmarkConfig) error { - err := config.updateDefaults() - if err != nil { - return err - } - - ctx := context.Background() - - for _, serverConfig := range config.Servers { - var results sysbench_runner.Results - var err error - switch serverConfig.Server { - case sysbench_runner.Dolt: - fmt.Println("Running Dolt Benchmark") - results, err = BenchmarkDolt(ctx, config, serverConfig) - if err != nil { - return err - } - case sysbench_runner.MySql: - fmt.Println("Running MySQL benchmark") - results, err = BenchmarkMysql(ctx, config, serverConfig) - if err != nil { - return err - } - default: - panic(fmt.Sprintf("unexpected server type: %s", serverConfig.Server)) - } - if err != nil { - return err - } - - err = sysbench_runner.WriteResults(serverConfig, results) - if err != nil { - return err - } - - fmt.Printf("Successfuly wrote results for %s\n", serverConfig.Server) - } - - return nil -} diff --git a/go/performance/utils/tpcc_runner/sample-tpcc-config.json b/go/performance/utils/tpcc_runner/sample-tpcc-config.json deleted file mode 100644 index 6acbfa64a23..00000000000 --- a/go/performance/utils/tpcc_runner/sample-tpcc-config.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "Servers": [ - { - "Host": "127.0.0.1", - "Port": 3307, - "Server": "dolt", - "Version": "HEAD", - "ResultsFormat": "csv", - "ServerExec": "/Users/vinairachakonda/go/bin/dolt" - }, - { - "Server": "mysql", - "Version": "8.0.22", - "ResultsFormat": "csv", - "ServerExec": "/usr/local/bin/mysqld", - "ConnectionProtocol": "tcp" - } - ], - "ScriptDir":"/Users/vinairachakonda/go/src/dolthub/sysbench-tpcc", - "ScaleFactors": [1], - "NomsBinFormat": "__DOLT__" -}