From 675abfbd6e69b98a24628f698e738e2a894a125a Mon Sep 17 00:00:00 2001 From: Allen Zhong Date: Wed, 22 Jul 2020 14:42:26 +0800 Subject: [PATCH 1/3] cluster/spec: check for multiple tispark workers on the same host --- pkg/cluster/spec/spec.go | 12 ++++++++++++ pkg/cluster/spec/spec_test.go | 19 +++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/pkg/cluster/spec/spec.go b/pkg/cluster/spec/spec.go index 8aba666439..9dd084afd6 100644 --- a/pkg/cluster/spec/spec.go +++ b/pkg/cluster/spec/spec.go @@ -41,6 +41,7 @@ var ( RoleTiSparkWorker = "tispark-worker" ErrNoTiSparkMaster = errors.New("there must be a Spark master node if you want to use the TiSpark component") ErrMultipleTiSparkMaster = errors.New("a TiSpark enabled cluster with more than 1 Spark master node is not supported") + ErrMultipleTisparkWorker = errors.New("multiple TiSpark workers on the same host is not supported by Spark") ) type ( @@ -515,6 +516,17 @@ func (s *Specification) validateTiSparkSpec() error { return ErrMultipleTiSparkMaster } + // Multiple workers on the same host is not supported by Spark + if len(s.TiSparkWorkers) > 1 { + cnt := make(map[string]int) + for _, w := range s.TiSparkWorkers { + if cnt[w.Host] > 0 { + return ErrMultipleTisparkWorker + } + cnt[w.Host]++ + } + } + return nil } diff --git a/pkg/cluster/spec/spec_test.go b/pkg/cluster/spec/spec_test.go index 013cdd29d4..d8390b2315 100644 --- a/pkg/cluster/spec/spec_test.go +++ b/pkg/cluster/spec/spec_test.go @@ -770,4 +770,23 @@ tispark_workers: `), &topo) c.Assert(err, NotNil) c.Assert(err.Error(), Equals, "there must be a Spark master node if you want to use the TiSpark component") + + err = yaml.Unmarshal([]byte(` +pd_servers: + - host: 172.16.5.138 + port: 1234 +tispark_masters: + - host: 172.16.5.138 + port: 1236 +tispark_workers: + - host: 172.16.5.138 + port: 1235 + - host: 172.16.5.139 + port: 1235 + - host: 172.16.5.139 + port: 1236 + web_port: 8089 +`), &topo) + c.Assert(err, NotNil) + c.Assert(err.Error(), Equals, "multiple TiSpark workers on the same host is not supported by Spark") } From bcd41a1029e6b54befb90d5249da74bf4b98d3b6 Mon Sep 17 00:00:00 2001 From: Allen Zhong Date: Wed, 22 Jul 2020 15:16:01 +0800 Subject: [PATCH 2/3] cluster: add more envs for tispark in templates --- pkg/cluster/embed/autogen_pkger.go | 4 +-- pkg/cluster/spec/tispark.go | 36 ++++++++++++++++++------- pkg/cluster/template/scripts/tispark.go | 19 ++++++++----- pkg/cluster/template/systemd/tispark.go | 4 ++- templates/scripts/spark-env.sh.tpl | 3 +++ templates/systemd/tispark.service.tpl | 5 +++- 6 files changed, 51 insertions(+), 20 deletions(-) diff --git a/pkg/cluster/embed/autogen_pkger.go b/pkg/cluster/embed/autogen_pkger.go index 0f1596841f..126f7240ff 100644 --- a/pkg/cluster/embed/autogen_pkger.go +++ b/pkg/cluster/embed/autogen_pkger.go @@ -41,8 +41,8 @@ func init() { autogenFiles["/templates/scripts/run_tidb.sh.tpl"] = "IyEvYmluL2Jhc2gKc2V0IC1lCgojIFdBUk5JTkc6IFRoaXMgZmlsZSB3YXMgYXV0by1nZW5lcmF0ZWQuIERvIG5vdCBlZGl0IQojICAgICAgICAgIEFsbCB5b3VyIGVkaXQgbWlnaHQgYmUgb3ZlcndyaXR0ZW4hCkRFUExPWV9ESVI9e3suRGVwbG95RGlyfX0KCmNkICIke0RFUExPWV9ESVJ9IiB8fCBleGl0IDEKCnt7LSBkZWZpbmUgIlBETGlzdCJ9fQogIHt7LSByYW5nZSAkaWR4LCAkcGQgOj0gLn19CiAgICB7ey0gaWYgZXEgJGlkeCAwfX0KICAgICAge3stICRwZC5JUH19Ont7JHBkLkNsaWVudFBvcnR9fQogICAge3stIGVsc2UgLX19CiAgICAgICx7eyRwZC5JUH19Ont7JHBkLkNsaWVudFBvcnR9fQogICAge3stIGVuZH19CiAge3stIGVuZH19Cnt7LSBlbmR9fQoKe3stIGlmIC5OdW1hTm9kZX19CmV4ZWMgbnVtYWN0bCAtLWNwdW5vZGViaW5kPXt7Lk51bWFOb2RlfX0gLS1tZW1iaW5kPXt7Lk51bWFOb2RlfX0gZW52IEdPREVCVUc9bWFkdmRvbnRuZWVkPTEgYmluL3RpZGItc2VydmVyIFwKe3stIGVsc2V9fQpleGVjIGVudiBHT0RFQlVHPW1hZHZkb250bmVlZD0xIGJpbi90aWRiLXNlcnZlciBcCnt7LSBlbmR9fQogICAgLVAge3suUG9ydH19IFwKICAgIC0tc3RhdHVzPSJ7ey5TdGF0dXNQb3J0fX0iIFwKICAgIC0taG9zdD0ie3suTGlzdGVuSG9zdH19IiBcCiAgICAtLWFkdmVydGlzZS1hZGRyZXNzPSJ7ey5JUH19IiBcCiAgICAtLXN0b3JlPSJ0aWt2IiBcCiAgICAtLWNvbmZpZz0iY29uZi90aWRiLnRvbWwiIFwKICAgIC0tcGF0aD0ie3t0ZW1wbGF0ZSAiUERMaXN0IiAuRW5kcG9pbnRzfX0iIFwKICAgIC0tbG9nLXNsb3ctcXVlcnk9ImxvZy90aWRiX3Nsb3dfcXVlcnkubG9nIiBcCiAgICAtLWNvbmZpZz1jb25mL3RpZGIudG9tbCBcCiAgICAtLWxvZy1maWxlPSJ7ey5Mb2dEaXJ9fS90aWRiLmxvZyIgMj4+ICJ7ey5Mb2dEaXJ9fS90aWRiX3N0ZGVyci5sb2ciCg==" autogenFiles["/templates/scripts/run_tiflash.sh.tpl"] = "IyEvYmluL2Jhc2gKc2V0IC1lCgojIFdBUk5JTkc6IFRoaXMgZmlsZSB3YXMgYXV0by1nZW5lcmF0ZWQuIERvIG5vdCBlZGl0IQojICAgICAgICAgIEFsbCB5b3VyIGVkaXQgbWlnaHQgYmUgb3ZlcndyaXR0ZW4hCmNkICJ7ey5EZXBsb3lEaXJ9fSIgfHwgZXhpdCAxCgpleHBvcnQgUlVTVF9CQUNLVFJBQ0U9MQoKZXhwb3J0IFRaPSR7VFo6LS9ldGMvbG9jYWx0aW1lfQpleHBvcnQgTERfTElCUkFSWV9QQVRIPXt7LkRlcGxveURpcn19L2Jpbi90aWZsYXNoOiRMRF9MSUJSQVJZX1BBVEgKCmVjaG8gLW4gJ3N5bmMgLi4uICcKc3RhdD0kKHRpbWUgc3luYykKZWNobyBvawplY2hvICRzdGF0Cgp7ey0gaWYgLk51bWFOb2RlfX0KZXhlYyBudW1hY3RsIC0tY3B1bm9kZWJpbmQ9e3suTnVtYU5vZGV9fSAtLW1lbWJpbmQ9e3suTnVtYU5vZGV9fSAgXAp7ey0gZWxzZX19CmV4ZWMgXAp7ey0gZW5kfX0KICAgIGJpbi90aWZsYXNoL3RpZmxhc2ggc2VydmVyIC0tY29uZmlnLWZpbGUgY29uZi90aWZsYXNoLnRvbWw=" autogenFiles["/templates/scripts/run_tikv.sh.tpl"] = "IyEvYmluL2Jhc2gKc2V0IC1lCgojIFdBUk5JTkc6IFRoaXMgZmlsZSB3YXMgYXV0by1nZW5lcmF0ZWQuIERvIG5vdCBlZGl0IQojICAgICAgICAgIEFsbCB5b3VyIGVkaXQgbWlnaHQgYmUgb3ZlcndyaXR0ZW4hCmNkICJ7ey5EZXBsb3lEaXJ9fSIgfHwgZXhpdCAxCgplY2hvIC1uICdzeW5jIC4uLiAnCnN0YXQ9JCh0aW1lIHN5bmMgfHwgc3luYykKZWNobyBvawplY2hvICRzdGF0Cgp7ey0gZGVmaW5lICJQRExpc3QifX0KICB7ey0gcmFuZ2UgJGlkeCwgJHBkIDo9IC59fQogICAge3stIGlmIGVxICRpZHggMH19CiAgICAgIHt7LSAkcGQuSVB9fTp7eyRwZC5DbGllbnRQb3J0fX0KICAgIHt7LSBlbHNlIC19fQogICAgICAse3skcGQuSVB9fTp7eyRwZC5DbGllbnRQb3J0fX0KICAgIHt7LSBlbmR9fQogIHt7LSBlbmR9fQp7ey0gZW5kfX0KCnt7LSBpZiAuTnVtYU5vZGV9fQpleGVjIG51bWFjdGwgLS1jcHVub2RlYmluZD17ey5OdW1hTm9kZX19IC0tbWVtYmluZD17ey5OdW1hTm9kZX19IGJpbi90aWt2LXNlcnZlciBcCnt7LSBlbHNlfX0KZXhlYyBiaW4vdGlrdi1zZXJ2ZXIgXAp7ey0gZW5kfX0KICAgIC0tYWRkciAie3suTGlzdGVuSG9zdH19Ont7LlBvcnR9fSIgXAogICAgLS1hZHZlcnRpc2UtYWRkciAie3suSVB9fTp7ey5Qb3J0fX0iIFwKICAgIC0tc3RhdHVzLWFkZHIgInt7LklQfX06e3suU3RhdHVzUG9ydH19IiBcCiAgICAtLXBkICJ7e3RlbXBsYXRlICJQRExpc3QiIC5FbmRwb2ludHN9fSIgXAogICAgLS1kYXRhLWRpciAie3suRGF0YURpcn19IiBcCiAgICAtLWNvbmZpZyBjb25mL3Rpa3YudG9tbCBcCiAgICAtLWxvZy1maWxlICJ7ey5Mb2dEaXJ9fS90aWt2LmxvZyIgMj4+ICJ7ey5Mb2dEaXJ9fS90aWt2X3N0ZGVyci5sb2ciCg==" - autogenFiles["/templates/scripts/spark-env.sh.tpl"] = "IyEvdXNyL2Jpbi9lbnYgYmFzaAoKIwojIExpY2Vuc2VkIHRvIHRoZSBBcGFjaGUgU29mdHdhcmUgRm91bmRhdGlvbiAoQVNGKSB1bmRlciBvbmUgb3IgbW9yZQojIGNvbnRyaWJ1dG9yIGxpY2Vuc2UgYWdyZWVtZW50cy4gIFNlZSB0aGUgTk9USUNFIGZpbGUgZGlzdHJpYnV0ZWQgd2l0aAojIHRoaXMgd29yayBmb3IgYWRkaXRpb25hbCBpbmZvcm1hdGlvbiByZWdhcmRpbmcgY29weXJpZ2h0IG93bmVyc2hpcC4KIyBUaGUgQVNGIGxpY2Vuc2VzIHRoaXMgZmlsZSB0byBZb3UgdW5kZXIgdGhlIEFwYWNoZSBMaWNlbnNlLCBWZXJzaW9uIDIuMAojICh0aGUgIkxpY2Vuc2UiKTsgeW91IG1heSBub3QgdXNlIHRoaXMgZmlsZSBleGNlcHQgaW4gY29tcGxpYW5jZSB3aXRoCiMgdGhlIExpY2Vuc2UuICBZb3UgbWF5IG9idGFpbiBhIGNvcHkgb2YgdGhlIExpY2Vuc2UgYXQKIwojICAgIGh0dHA6Ly93d3cuYXBhY2hlLm9yZy9saWNlbnNlcy9MSUNFTlNFLTIuMAojCiMgVW5sZXNzIHJlcXVpcmVkIGJ5IGFwcGxpY2FibGUgbGF3IG9yIGFncmVlZCB0byBpbiB3cml0aW5nLCBzb2Z0d2FyZQojIGRpc3RyaWJ1dGVkIHVuZGVyIHRoZSBMaWNlbnNlIGlzIGRpc3RyaWJ1dGVkIG9uIGFuICJBUyBJUyIgQkFTSVMsCiMgV0lUSE9VVCBXQVJSQU5USUVTIE9SIENPTkRJVElPTlMgT0YgQU5ZIEtJTkQsIGVpdGhlciBleHByZXNzIG9yIGltcGxpZWQuCiMgU2VlIHRoZSBMaWNlbnNlIGZvciB0aGUgc3BlY2lmaWMgbGFuZ3VhZ2UgZ292ZXJuaW5nIHBlcm1pc3Npb25zIGFuZAojIGxpbWl0YXRpb25zIHVuZGVyIHRoZSBMaWNlbnNlLgojCgojIFRoaXMgZmlsZSBpcyBzb3VyY2VkIHdoZW4gcnVubmluZyB2YXJpb3VzIFNwYXJrIHByb2dyYW1zLgojIENvcHkgaXQgYXMgc3BhcmstZW52LnNoIGFuZCBlZGl0IHRoYXQgdG8gY29uZmlndXJlIFNwYXJrIGZvciB5b3VyIHNpdGUuCgojIE9wdGlvbnMgcmVhZCB3aGVuIGxhdW5jaGluZyBwcm9ncmFtcyBsb2NhbGx5IHdpdGgKIyAuL2Jpbi9ydW4tZXhhbXBsZSBvciAuL2Jpbi9zcGFyay1zdWJtaXQKIyAtIEhBRE9PUF9DT05GX0RJUiwgdG8gcG9pbnQgU3BhcmsgdG93YXJkcyBIYWRvb3AgY29uZmlndXJhdGlvbiBmaWxlcwojIC0gU1BBUktfTE9DQUxfSVAsIHRvIHNldCB0aGUgSVAgYWRkcmVzcyBTcGFyayBiaW5kcyB0byBvbiB0aGlzIG5vZGUKIyAtIFNQQVJLX1BVQkxJQ19ETlMsIHRvIHNldCB0aGUgcHVibGljIGRucyBuYW1lIG9mIHRoZSBkcml2ZXIgcHJvZ3JhbQojIC0gU1BBUktfQ0xBU1NQQVRILCBkZWZhdWx0IGNsYXNzcGF0aCBlbnRyaWVzIHRvIGFwcGVuZAoKIyBPcHRpb25zIHJlYWQgYnkgZXhlY3V0b3JzIGFuZCBkcml2ZXJzIHJ1bm5pbmcgaW5zaWRlIHRoZSBjbHVzdGVyCiMgLSBTUEFSS19MT0NBTF9JUCwgdG8gc2V0IHRoZSBJUCBhZGRyZXNzIFNwYXJrIGJpbmRzIHRvIG9uIHRoaXMgbm9kZQojIC0gU1BBUktfUFVCTElDX0ROUywgdG8gc2V0IHRoZSBwdWJsaWMgRE5TIG5hbWUgb2YgdGhlIGRyaXZlciBwcm9ncmFtCiMgLSBTUEFSS19DTEFTU1BBVEgsIGRlZmF1bHQgY2xhc3NwYXRoIGVudHJpZXMgdG8gYXBwZW5kCiMgLSBTUEFSS19MT0NBTF9ESVJTLCBzdG9yYWdlIGRpcmVjdG9yaWVzIHRvIHVzZSBvbiB0aGlzIG5vZGUgZm9yIHNodWZmbGUgYW5kIFJERCBkYXRhCiMgLSBNRVNPU19OQVRJVkVfSkFWQV9MSUJSQVJZLCB0byBwb2ludCB0byB5b3VyIGxpYm1lc29zLnNvIGlmIHlvdSB1c2UgTWVzb3MKCiMgT3B0aW9ucyByZWFkIGluIFlBUk4gY2xpZW50IG1vZGUKIyAtIEhBRE9PUF9DT05GX0RJUiwgdG8gcG9pbnQgU3BhcmsgdG93YXJkcyBIYWRvb3AgY29uZmlndXJhdGlvbiBmaWxlcwojIC0gU1BBUktfRVhFQ1VUT1JfSU5TVEFOQ0VTLCBOdW1iZXIgb2YgZXhlY3V0b3JzIHRvIHN0YXJ0IChEZWZhdWx0OiAyKQojIC0gU1BBUktfRVhFQ1VUT1JfQ09SRVMsIE51bWJlciBvZiBjb3JlcyBmb3IgdGhlIGV4ZWN1dG9ycyAoRGVmYXVsdDogMSkuCiMgLSBTUEFSS19FWEVDVVRPUl9NRU1PUlksIE1lbW9yeSBwZXIgRXhlY3V0b3IgKGUuZy4gMTAwME0sIDJHKSAoRGVmYXVsdDogMUcpCiMgLSBTUEFSS19EUklWRVJfTUVNT1JZLCBNZW1vcnkgZm9yIERyaXZlciAoZS5nLiAxMDAwTSwgMkcpIChEZWZhdWx0OiAxRykKCiMgT3B0aW9ucyBmb3IgdGhlIGRhZW1vbnMgdXNlZCBpbiB0aGUgc3RhbmRhbG9uZSBkZXBsb3kgbW9kZQojIC0gU1BBUktfTUFTVEVSX0hPU1QsIHRvIGJpbmQgdGhlIG1hc3RlciB0byBhIGRpZmZlcmVudCBJUCBhZGRyZXNzIG9yIGhvc3RuYW1lCiMgLSBTUEFSS19NQVNURVJfUE9SVCAvIFNQQVJLX01BU1RFUl9XRUJVSV9QT1JULCB0byB1c2Ugbm9uLWRlZmF1bHQgcG9ydHMgZm9yIHRoZSBtYXN0ZXIKIyAtIFNQQVJLX01BU1RFUl9PUFRTLCB0byBzZXQgY29uZmlnIHByb3BlcnRpZXMgb25seSBmb3IgdGhlIG1hc3RlciAoZS5nLiAiLUR4PXkiKQojIC0gU1BBUktfV09SS0VSX0NPUkVTLCB0byBzZXQgdGhlIG51bWJlciBvZiBjb3JlcyB0byB1c2Ugb24gdGhpcyBtYWNoaW5lCiMgLSBTUEFSS19XT1JLRVJfTUVNT1JZLCB0byBzZXQgaG93IG11Y2ggdG90YWwgbWVtb3J5IHdvcmtlcnMgaGF2ZSB0byBnaXZlIGV4ZWN1dG9ycyAoZS5nLiAxMDAwbSwgMmcpCiMgLSBTUEFSS19XT1JLRVJfUE9SVCAvIFNQQVJLX1dPUktFUl9XRUJVSV9QT1JULCB0byB1c2Ugbm9uLWRlZmF1bHQgcG9ydHMgZm9yIHRoZSB3b3JrZXIKIyAtIFNQQVJLX1dPUktFUl9JTlNUQU5DRVMsIHRvIHNldCB0aGUgbnVtYmVyIG9mIHdvcmtlciBwcm9jZXNzZXMgcGVyIG5vZGUKIyAtIFNQQVJLX1dPUktFUl9ESVIsIHRvIHNldCB0aGUgd29ya2luZyBkaXJlY3Rvcnkgb2Ygd29ya2VyIHByb2Nlc3NlcwojIC0gU1BBUktfV09SS0VSX09QVFMsIHRvIHNldCBjb25maWcgcHJvcGVydGllcyBvbmx5IGZvciB0aGUgd29ya2VyIChlLmcuICItRHg9eSIpCiMgLSBTUEFSS19EQUVNT05fTUVNT1JZLCB0byBhbGxvY2F0ZSB0byB0aGUgbWFzdGVyLCB3b3JrZXIgYW5kIGhpc3Rvcnkgc2VydmVyIHRoZW1zZWx2ZXMgKGRlZmF1bHQ6IDFnKS4KIyAtIFNQQVJLX0hJU1RPUllfT1BUUywgdG8gc2V0IGNvbmZpZyBwcm9wZXJ0aWVzIG9ubHkgZm9yIHRoZSBoaXN0b3J5IHNlcnZlciAoZS5nLiAiLUR4PXkiKQojIC0gU1BBUktfU0hVRkZMRV9PUFRTLCB0byBzZXQgY29uZmlnIHByb3BlcnRpZXMgb25seSBmb3IgdGhlIGV4dGVybmFsIHNodWZmbGUgc2VydmljZSAoZS5nLiAiLUR4PXkiKQojIC0gU1BBUktfREFFTU9OX0pBVkFfT1BUUywgdG8gc2V0IGNvbmZpZyBwcm9wZXJ0aWVzIGZvciBhbGwgZGFlbW9ucyAoZS5nLiAiLUR4PXkiKQojIC0gU1BBUktfUFVCTElDX0ROUywgdG8gc2V0IHRoZSBwdWJsaWMgZG5zIG5hbWUgb2YgdGhlIG1hc3RlciBvciB3b3JrZXJzCgojIEdlbmVyaWMgb3B0aW9ucyBmb3IgdGhlIGRhZW1vbnMgdXNlZCBpbiB0aGUgc3RhbmRhbG9uZSBkZXBsb3kgbW9kZQojIC0gU1BBUktfQ09ORl9ESVIgICAgICBBbHRlcm5hdGUgY29uZiBkaXIuIChEZWZhdWx0OiAke1NQQVJLX0hPTUV9L2NvbmYpCiMgLSBTUEFSS19MT0dfRElSICAgICAgIFdoZXJlIGxvZyBmaWxlcyBhcmUgc3RvcmVkLiAgKERlZmF1bHQ6ICR7U1BBUktfSE9NRX0vbG9ncykKIyAtIFNQQVJLX1BJRF9ESVIgICAgICAgV2hlcmUgdGhlIHBpZCBmaWxlIGlzIHN0b3JlZC4gKERlZmF1bHQ6IC90bXApCiMgLSBTUEFSS19JREVOVF9TVFJJTkcgIEEgc3RyaW5nIHJlcHJlc2VudGluZyB0aGlzIGluc3RhbmNlIG9mIHNwYXJrLiAoRGVmYXVsdDogJFVTRVIpCiMgLSBTUEFSS19OSUNFTkVTUyAgICAgIFRoZSBzY2hlZHVsaW5nIHByaW9yaXR5IGZvciBkYWVtb25zLiAoRGVmYXVsdDogMCkKIyAtIFNQQVJLX05PX0RBRU1PTklaRSAgUnVuIHRoZSBwcm9wb3NlZCBjb21tYW5kIGluIHRoZSBmb3JlZ3JvdW5kLiBJdCB3aWxsIG5vdCBvdXRwdXQgYSBQSUQgZmlsZS4KCiNleHBvcnQgSkFWQV9IT01FLCB0byBzZXQgamRrIGhvbWUKCnt7IHJhbmdlICRrLCAkdiA6PSAuQ3VzdG9tRW52c319Cnt7ICRrIH19PXt7ICR2IH19Cnt7LSBlbmQgfX0KCnt7LSBpZiAuVGlTcGFya01hc3Rlcn19ClNQQVJLX01BU1RFUl9IT1NUPXt7LlRpU3BhcmtNYXN0ZXJ9fQp7ey0gZW5kfX0Ke3stIGlmIG5lIC5NYXN0ZXJQb3J0IDB9fQpTUEFSS19NQVNURVJfUE9SVD17ey5NYXN0ZXJQb3J0fX0Ke3stIGVuZH19Cnt7LSBpZiBuZSAuTWFzdGVyVUlQb3J0IDB9fQpTUEFSS19NQVNURVJfV0VCVUlfUE9SVD17ey5NYXN0ZXJVSVBvcnR9fQp7ey0gZW5kfX0Ke3stIGlmIG5lIC5Xb3JrZXJQb3J0IDB9fQpTUEFSS19XT1JLRVJfUE9SVD17ey5Xb3JrZXJQb3J0fX0Ke3stIGVuZH19Cnt7LSBpZiBuZSAuV29ya2VyVUlQb3J0IDB9fQpTUEFSS19XT1JLRVJfV0VCVUlfUE9SVD17ey5Xb3JrZXJVSVBvcnR9fQp7ey0gZW5kfX0K" + autogenFiles["/templates/scripts/spark-env.sh.tpl"] = "IyEvdXNyL2Jpbi9lbnYgYmFzaAoKIwojIExpY2Vuc2VkIHRvIHRoZSBBcGFjaGUgU29mdHdhcmUgRm91bmRhdGlvbiAoQVNGKSB1bmRlciBvbmUgb3IgbW9yZQojIGNvbnRyaWJ1dG9yIGxpY2Vuc2UgYWdyZWVtZW50cy4gIFNlZSB0aGUgTk9USUNFIGZpbGUgZGlzdHJpYnV0ZWQgd2l0aAojIHRoaXMgd29yayBmb3IgYWRkaXRpb25hbCBpbmZvcm1hdGlvbiByZWdhcmRpbmcgY29weXJpZ2h0IG93bmVyc2hpcC4KIyBUaGUgQVNGIGxpY2Vuc2VzIHRoaXMgZmlsZSB0byBZb3UgdW5kZXIgdGhlIEFwYWNoZSBMaWNlbnNlLCBWZXJzaW9uIDIuMAojICh0aGUgIkxpY2Vuc2UiKTsgeW91IG1heSBub3QgdXNlIHRoaXMgZmlsZSBleGNlcHQgaW4gY29tcGxpYW5jZSB3aXRoCiMgdGhlIExpY2Vuc2UuICBZb3UgbWF5IG9idGFpbiBhIGNvcHkgb2YgdGhlIExpY2Vuc2UgYXQKIwojICAgIGh0dHA6Ly93d3cuYXBhY2hlLm9yZy9saWNlbnNlcy9MSUNFTlNFLTIuMAojCiMgVW5sZXNzIHJlcXVpcmVkIGJ5IGFwcGxpY2FibGUgbGF3IG9yIGFncmVlZCB0byBpbiB3cml0aW5nLCBzb2Z0d2FyZQojIGRpc3RyaWJ1dGVkIHVuZGVyIHRoZSBMaWNlbnNlIGlzIGRpc3RyaWJ1dGVkIG9uIGFuICJBUyBJUyIgQkFTSVMsCiMgV0lUSE9VVCBXQVJSQU5USUVTIE9SIENPTkRJVElPTlMgT0YgQU5ZIEtJTkQsIGVpdGhlciBleHByZXNzIG9yIGltcGxpZWQuCiMgU2VlIHRoZSBMaWNlbnNlIGZvciB0aGUgc3BlY2lmaWMgbGFuZ3VhZ2UgZ292ZXJuaW5nIHBlcm1pc3Npb25zIGFuZAojIGxpbWl0YXRpb25zIHVuZGVyIHRoZSBMaWNlbnNlLgojCgojIFRoaXMgZmlsZSBpcyBzb3VyY2VkIHdoZW4gcnVubmluZyB2YXJpb3VzIFNwYXJrIHByb2dyYW1zLgojIENvcHkgaXQgYXMgc3BhcmstZW52LnNoIGFuZCBlZGl0IHRoYXQgdG8gY29uZmlndXJlIFNwYXJrIGZvciB5b3VyIHNpdGUuCgojIE9wdGlvbnMgcmVhZCB3aGVuIGxhdW5jaGluZyBwcm9ncmFtcyBsb2NhbGx5IHdpdGgKIyAuL2Jpbi9ydW4tZXhhbXBsZSBvciAuL2Jpbi9zcGFyay1zdWJtaXQKIyAtIEhBRE9PUF9DT05GX0RJUiwgdG8gcG9pbnQgU3BhcmsgdG93YXJkcyBIYWRvb3AgY29uZmlndXJhdGlvbiBmaWxlcwojIC0gU1BBUktfTE9DQUxfSVAsIHRvIHNldCB0aGUgSVAgYWRkcmVzcyBTcGFyayBiaW5kcyB0byBvbiB0aGlzIG5vZGUKIyAtIFNQQVJLX1BVQkxJQ19ETlMsIHRvIHNldCB0aGUgcHVibGljIGRucyBuYW1lIG9mIHRoZSBkcml2ZXIgcHJvZ3JhbQojIC0gU1BBUktfQ0xBU1NQQVRILCBkZWZhdWx0IGNsYXNzcGF0aCBlbnRyaWVzIHRvIGFwcGVuZAoKIyBPcHRpb25zIHJlYWQgYnkgZXhlY3V0b3JzIGFuZCBkcml2ZXJzIHJ1bm5pbmcgaW5zaWRlIHRoZSBjbHVzdGVyCiMgLSBTUEFSS19MT0NBTF9JUCwgdG8gc2V0IHRoZSBJUCBhZGRyZXNzIFNwYXJrIGJpbmRzIHRvIG9uIHRoaXMgbm9kZQojIC0gU1BBUktfUFVCTElDX0ROUywgdG8gc2V0IHRoZSBwdWJsaWMgRE5TIG5hbWUgb2YgdGhlIGRyaXZlciBwcm9ncmFtCiMgLSBTUEFSS19DTEFTU1BBVEgsIGRlZmF1bHQgY2xhc3NwYXRoIGVudHJpZXMgdG8gYXBwZW5kCiMgLSBTUEFSS19MT0NBTF9ESVJTLCBzdG9yYWdlIGRpcmVjdG9yaWVzIHRvIHVzZSBvbiB0aGlzIG5vZGUgZm9yIHNodWZmbGUgYW5kIFJERCBkYXRhCiMgLSBNRVNPU19OQVRJVkVfSkFWQV9MSUJSQVJZLCB0byBwb2ludCB0byB5b3VyIGxpYm1lc29zLnNvIGlmIHlvdSB1c2UgTWVzb3MKCiMgT3B0aW9ucyByZWFkIGluIFlBUk4gY2xpZW50IG1vZGUKIyAtIEhBRE9PUF9DT05GX0RJUiwgdG8gcG9pbnQgU3BhcmsgdG93YXJkcyBIYWRvb3AgY29uZmlndXJhdGlvbiBmaWxlcwojIC0gU1BBUktfRVhFQ1VUT1JfSU5TVEFOQ0VTLCBOdW1iZXIgb2YgZXhlY3V0b3JzIHRvIHN0YXJ0IChEZWZhdWx0OiAyKQojIC0gU1BBUktfRVhFQ1VUT1JfQ09SRVMsIE51bWJlciBvZiBjb3JlcyBmb3IgdGhlIGV4ZWN1dG9ycyAoRGVmYXVsdDogMSkuCiMgLSBTUEFSS19FWEVDVVRPUl9NRU1PUlksIE1lbW9yeSBwZXIgRXhlY3V0b3IgKGUuZy4gMTAwME0sIDJHKSAoRGVmYXVsdDogMUcpCiMgLSBTUEFSS19EUklWRVJfTUVNT1JZLCBNZW1vcnkgZm9yIERyaXZlciAoZS5nLiAxMDAwTSwgMkcpIChEZWZhdWx0OiAxRykKCiMgT3B0aW9ucyBmb3IgdGhlIGRhZW1vbnMgdXNlZCBpbiB0aGUgc3RhbmRhbG9uZSBkZXBsb3kgbW9kZQojIC0gU1BBUktfTUFTVEVSX0hPU1QsIHRvIGJpbmQgdGhlIG1hc3RlciB0byBhIGRpZmZlcmVudCBJUCBhZGRyZXNzIG9yIGhvc3RuYW1lCiMgLSBTUEFSS19NQVNURVJfUE9SVCAvIFNQQVJLX01BU1RFUl9XRUJVSV9QT1JULCB0byB1c2Ugbm9uLWRlZmF1bHQgcG9ydHMgZm9yIHRoZSBtYXN0ZXIKIyAtIFNQQVJLX01BU1RFUl9PUFRTLCB0byBzZXQgY29uZmlnIHByb3BlcnRpZXMgb25seSBmb3IgdGhlIG1hc3RlciAoZS5nLiAiLUR4PXkiKQojIC0gU1BBUktfV09SS0VSX0NPUkVTLCB0byBzZXQgdGhlIG51bWJlciBvZiBjb3JlcyB0byB1c2Ugb24gdGhpcyBtYWNoaW5lCiMgLSBTUEFSS19XT1JLRVJfTUVNT1JZLCB0byBzZXQgaG93IG11Y2ggdG90YWwgbWVtb3J5IHdvcmtlcnMgaGF2ZSB0byBnaXZlIGV4ZWN1dG9ycyAoZS5nLiAxMDAwbSwgMmcpCiMgLSBTUEFSS19XT1JLRVJfUE9SVCAvIFNQQVJLX1dPUktFUl9XRUJVSV9QT1JULCB0byB1c2Ugbm9uLWRlZmF1bHQgcG9ydHMgZm9yIHRoZSB3b3JrZXIKIyAtIFNQQVJLX1dPUktFUl9JTlNUQU5DRVMsIHRvIHNldCB0aGUgbnVtYmVyIG9mIHdvcmtlciBwcm9jZXNzZXMgcGVyIG5vZGUKIyAtIFNQQVJLX1dPUktFUl9ESVIsIHRvIHNldCB0aGUgd29ya2luZyBkaXJlY3Rvcnkgb2Ygd29ya2VyIHByb2Nlc3NlcwojIC0gU1BBUktfV09SS0VSX09QVFMsIHRvIHNldCBjb25maWcgcHJvcGVydGllcyBvbmx5IGZvciB0aGUgd29ya2VyIChlLmcuICItRHg9eSIpCiMgLSBTUEFSS19EQUVNT05fTUVNT1JZLCB0byBhbGxvY2F0ZSB0byB0aGUgbWFzdGVyLCB3b3JrZXIgYW5kIGhpc3Rvcnkgc2VydmVyIHRoZW1zZWx2ZXMgKGRlZmF1bHQ6IDFnKS4KIyAtIFNQQVJLX0hJU1RPUllfT1BUUywgdG8gc2V0IGNvbmZpZyBwcm9wZXJ0aWVzIG9ubHkgZm9yIHRoZSBoaXN0b3J5IHNlcnZlciAoZS5nLiAiLUR4PXkiKQojIC0gU1BBUktfU0hVRkZMRV9PUFRTLCB0byBzZXQgY29uZmlnIHByb3BlcnRpZXMgb25seSBmb3IgdGhlIGV4dGVybmFsIHNodWZmbGUgc2VydmljZSAoZS5nLiAiLUR4PXkiKQojIC0gU1BBUktfREFFTU9OX0pBVkFfT1BUUywgdG8gc2V0IGNvbmZpZyBwcm9wZXJ0aWVzIGZvciBhbGwgZGFlbW9ucyAoZS5nLiAiLUR4PXkiKQojIC0gU1BBUktfUFVCTElDX0ROUywgdG8gc2V0IHRoZSBwdWJsaWMgZG5zIG5hbWUgb2YgdGhlIG1hc3RlciBvciB3b3JrZXJzCgojIEdlbmVyaWMgb3B0aW9ucyBmb3IgdGhlIGRhZW1vbnMgdXNlZCBpbiB0aGUgc3RhbmRhbG9uZSBkZXBsb3kgbW9kZQojIC0gU1BBUktfQ09ORl9ESVIgICAgICBBbHRlcm5hdGUgY29uZiBkaXIuIChEZWZhdWx0OiAke1NQQVJLX0hPTUV9L2NvbmYpCiMgLSBTUEFSS19MT0dfRElSICAgICAgIFdoZXJlIGxvZyBmaWxlcyBhcmUgc3RvcmVkLiAgKERlZmF1bHQ6ICR7U1BBUktfSE9NRX0vbG9ncykKIyAtIFNQQVJLX1BJRF9ESVIgICAgICAgV2hlcmUgdGhlIHBpZCBmaWxlIGlzIHN0b3JlZC4gKERlZmF1bHQ6IC90bXApCiMgLSBTUEFSS19JREVOVF9TVFJJTkcgIEEgc3RyaW5nIHJlcHJlc2VudGluZyB0aGlzIGluc3RhbmNlIG9mIHNwYXJrLiAoRGVmYXVsdDogJFVTRVIpCiMgLSBTUEFSS19OSUNFTkVTUyAgICAgIFRoZSBzY2hlZHVsaW5nIHByaW9yaXR5IGZvciBkYWVtb25zLiAoRGVmYXVsdDogMCkKIyAtIFNQQVJLX05PX0RBRU1PTklaRSAgUnVuIHRoZSBwcm9wb3NlZCBjb21tYW5kIGluIHRoZSBmb3JlZ3JvdW5kLiBJdCB3aWxsIG5vdCBvdXRwdXQgYSBQSUQgZmlsZS4KCiNleHBvcnQgSkFWQV9IT01FLCB0byBzZXQgamRrIGhvbWUKCnt7IHJhbmdlICRrLCAkdiA6PSAuQ3VzdG9tRW52c319Cnt7ICRrIH19PXt7ICR2IH19Cnt7LSBlbmQgfX0KCnt7LSBpZiAuVGlTcGFya01hc3Rlcn19ClNQQVJLX01BU1RFUl9IT1NUPXt7LlRpU3BhcmtNYXN0ZXJ9fQp7ey0gZW5kfX0Ke3stIGlmIG5lIC5NYXN0ZXJQb3J0IDB9fQpTUEFSS19NQVNURVJfUE9SVD17ey5NYXN0ZXJQb3J0fX0Ke3stIGVuZH19Cnt7LSBpZiBuZSAuTWFzdGVyVUlQb3J0IDB9fQpTUEFSS19NQVNURVJfV0VCVUlfUE9SVD17ey5NYXN0ZXJVSVBvcnR9fQp7ey0gZW5kfX0Ke3stIGlmIG5lIC5Xb3JrZXJQb3J0IDB9fQpTUEFSS19XT1JLRVJfUE9SVD17ey5Xb3JrZXJQb3J0fX0Ke3stIGVuZH19Cnt7LSBpZiBuZSAuV29ya2VyVUlQb3J0IDB9fQpTUEFSS19XT1JLRVJfV0VCVUlfUE9SVD17ey5Xb3JrZXJVSVBvcnR9fQp7ey0gZW5kfX0Ke3stIGlmIG5lIC5UaVNwYXJrTG9jYWxJUCAiIn19ClNQQVJLX0xPQ0FMX0lQPXt7LlRpU3BhcmtMb2NhbElQfX0Ke3stIGVuZH19Cg==" autogenFiles["/templates/scripts/start_tispark_slave.sh.tpl"] = "IyEvdXNyL2Jpbi9lbnYgYmFzaAoKIwojIExpY2Vuc2VkIHRvIHRoZSBBcGFjaGUgU29mdHdhcmUgRm91bmRhdGlvbiAoQVNGKSB1bmRlciBvbmUgb3IgbW9yZQojIGNvbnRyaWJ1dG9yIGxpY2Vuc2UgYWdyZWVtZW50cy4gIFNlZSB0aGUgTk9USUNFIGZpbGUgZGlzdHJpYnV0ZWQgd2l0aAojIHRoaXMgd29yayBmb3IgYWRkaXRpb25hbCBpbmZvcm1hdGlvbiByZWdhcmRpbmcgY29weXJpZ2h0IG93bmVyc2hpcC4KIyBUaGUgQVNGIGxpY2Vuc2VzIHRoaXMgZmlsZSB0byBZb3UgdW5kZXIgdGhlIEFwYWNoZSBMaWNlbnNlLCBWZXJzaW9uIDIuMAojICh0aGUgIkxpY2Vuc2UiKTsgeW91IG1heSBub3QgdXNlIHRoaXMgZmlsZSBleGNlcHQgaW4gY29tcGxpYW5jZSB3aXRoCiMgdGhlIExpY2Vuc2UuICBZb3UgbWF5IG9idGFpbiBhIGNvcHkgb2YgdGhlIExpY2Vuc2UgYXQKIwojICAgIGh0dHA6Ly93d3cuYXBhY2hlLm9yZy9saWNlbnNlcy9MSUNFTlNFLTIuMAojCiMgVW5sZXNzIHJlcXVpcmVkIGJ5IGFwcGxpY2FibGUgbGF3IG9yIGFncmVlZCB0byBpbiB3cml0aW5nLCBzb2Z0d2FyZQojIGRpc3RyaWJ1dGVkIHVuZGVyIHRoZSBMaWNlbnNlIGlzIGRpc3RyaWJ1dGVkIG9uIGFuICJBUyBJUyIgQkFTSVMsCiMgV0lUSE9VVCBXQVJSQU5USUVTIE9SIENPTkRJVElPTlMgT0YgQU5ZIEtJTkQsIGVpdGhlciBleHByZXNzIG9yIGltcGxpZWQuCiMgU2VlIHRoZSBMaWNlbnNlIGZvciB0aGUgc3BlY2lmaWMgbGFuZ3VhZ2UgZ292ZXJuaW5nIHBlcm1pc3Npb25zIGFuZAojIGxpbWl0YXRpb25zIHVuZGVyIHRoZSBMaWNlbnNlLgojCgojIFN0YXJ0cyBhIHNsYXZlIG9uIHRoZSBtYWNoaW5lIHRoaXMgc2NyaXB0IGlzIGV4ZWN1dGVkIG9uLgojCiMgRW52aXJvbm1lbnQgVmFyaWFibGVzCiMKIyAgIFNQQVJLX1dPUktFUl9JTlNUQU5DRVMgIFRoZSBudW1iZXIgb2Ygd29ya2VyIGluc3RhbmNlcyB0byBydW4gb24gdGhpcwojICAgICAgICAgICAgICAgICAgICAgICAgICAgc2xhdmUuICBEZWZhdWx0IGlzIDEuCiMgICBTUEFSS19XT1JLRVJfUE9SVCAgICAgICBUaGUgYmFzZSBwb3J0IG51bWJlciBmb3IgdGhlIGZpcnN0IHdvcmtlci4gSWYgc2V0LAojICAgICAgICAgICAgICAgICAgICAgICAgICAgc3Vic2VxdWVudCB3b3JrZXJzIHdpbGwgaW5jcmVtZW50IHRoaXMgbnVtYmVyLiAgSWYKIyAgICAgICAgICAgICAgICAgICAgICAgICAgIHVuc2V0LCBTcGFyayB3aWxsIGZpbmQgYSB2YWxpZCBwb3J0IG51bWJlciwgYnV0CiMgICAgICAgICAgICAgICAgICAgICAgICAgICB3aXRoIG5vIGd1YXJhbnRlZSBvZiBhIHByZWRpY3RhYmxlIHBhdHRlcm4uCiMgICBTUEFSS19XT1JLRVJfV0VCVUlfUE9SVCBUaGUgYmFzZSBwb3J0IGZvciB0aGUgd2ViIGludGVyZmFjZSBvZiB0aGUgZmlyc3QKIyAgICAgICAgICAgICAgICAgICAgICAgICAgIHdvcmtlci4gIFN1YnNlcXVlbnQgd29ya2VycyB3aWxsIGluY3JlbWVudCB0aGlzCiMgICAgICAgICAgICAgICAgICAgICAgICAgICBudW1iZXIuICBEZWZhdWx0IGlzIDgwODEuCgppZiBbIC16ICIke1NQQVJLX0hPTUV9IiBdOyB0aGVuCiAgZXhwb3J0IFNQQVJLX0hPTUU9IiQoY2QgImBkaXJuYW1lICIkMCJgIi8uLjsgcHdkKSIKZmkKCiMgTk9URTogVGhpcyBleGFjdCBjbGFzcyBuYW1lIGlzIG1hdGNoZWQgZG93bnN0cmVhbSBieSBTcGFya1N1Ym1pdC4KIyBBbnkgY2hhbmdlcyBuZWVkIHRvIGJlIHJlZmxlY3RlZCB0aGVyZS4KQ0xBU1M9Im9yZy5hcGFjaGUuc3BhcmsuZGVwbG95Lndvcmtlci5Xb3JrZXIiCgppZiBbWyAiJEAiID0gKi0taGVscCBdXSB8fCBbWyAiJEAiID0gKi1oIF1dOyB0aGVuCiAgZWNobyAiVXNhZ2U6IC4vc2Jpbi9zdGFydC1zbGF2ZS5zaCBbb3B0aW9uc10gPG1hc3Rlcj4iCiAgcGF0dGVybj0iVXNhZ2U6IgogIHBhdHRlcm4rPSJcfFVzaW5nIFNwYXJrJ3MgZGVmYXVsdCBsb2c0aiBwcm9maWxlOiIKICBwYXR0ZXJuKz0iXHxSZWdpc3RlcmVkIHNpZ25hbCBoYW5kbGVycyBmb3IiCgogICIke1NQQVJLX0hPTUV9Ii9iaW4vc3BhcmstY2xhc3MgJENMQVNTIC0taGVscCAyPiYxIHwgZ3JlcCAtdiAiJHBhdHRlcm4iIDE+JjIKICBleGl0IDEKZmkKCi4gIiR7U1BBUktfSE9NRX0vc2Jpbi9zcGFyay1jb25maWcuc2giCgouICIke1NQQVJLX0hPTUV9L2Jpbi9sb2FkLXNwYXJrLWVudi5zaCIKCiMgRmlyc3QgYXJndW1lbnQgc2hvdWxkIGJlIHRoZSBtYXN0ZXI7IHdlIG5lZWQgdG8gc3RvcmUgaXQgYXNpZGUgYmVjYXVzZSB3ZSBtYXkKIyBuZWVkIHRvIGluc2VydCBhcmd1bWVudHMgYmV0d2VlbiBpdCBhbmQgdGhlIG90aGVyIGFyZ3VtZW50cwoKe3stIGlmIC5UaVNwYXJrTWFzdGVyfX0KTUFTVEVSPXNwYXJrOi8ve3suVGlTcGFya01hc3Rlcn19Ont7Lk1hc3RlclBvcnR9fQpzaGlmdAp7ey0gZW5kfX0KCiMgRGV0ZXJtaW5lIGRlc2lyZWQgd29ya2VyIHBvcnQKaWYgWyAiJFNQQVJLX1dPUktFUl9XRUJVSV9QT1JUIiA9ICIiIF07IHRoZW4KICBTUEFSS19XT1JLRVJfV0VCVUlfUE9SVD04MDgxCmZpCgojIFN0YXJ0IHVwIHRoZSBhcHByb3ByaWF0ZSBudW1iZXIgb2Ygd29ya2VycyBvbiB0aGlzIG1hY2hpbmUuCiMgcXVpY2sgbG9jYWwgZnVuY3Rpb24gdG8gc3RhcnQgYSB3b3JrZXIKZnVuY3Rpb24gc3RhcnRfaW5zdGFuY2UgewogIFdPUktFUl9OVU09JDEKICBzaGlmdAoKICBpZiBbICIkU1BBUktfV09SS0VSX1BPUlQiID0gIiIgXTsgdGhlbgogICAgUE9SVF9GTEFHPQogICAgUE9SVF9OVU09CiAgZWxzZQogICAgUE9SVF9GTEFHPSItLXBvcnQiCiAgICBQT1JUX05VTT0kKCggJFNQQVJLX1dPUktFUl9QT1JUICsgJFdPUktFUl9OVU0gLSAxICkpCiAgZmkKICBXRUJVSV9QT1JUPSQoKCAkU1BBUktfV09SS0VSX1dFQlVJX1BPUlQgKyAkV09SS0VSX05VTSAtIDEgKSkKCiAgIiR7U1BBUktfSE9NRX0vc2JpbiIvc3BhcmstZGFlbW9uLnNoIHN0YXJ0ICRDTEFTUyAkV09SS0VSX05VTSBcCiAgICAgLS13ZWJ1aS1wb3J0ICIkV0VCVUlfUE9SVCIgJFBPUlRfRkxBRyAkUE9SVF9OVU0gJE1BU1RFUiAiJEAiCn0KCmlmIFsgIiRTUEFSS19XT1JLRVJfSU5TVEFOQ0VTIiA9ICIiIF07IHRoZW4KICBzdGFydF9pbnN0YW5jZSAxICIkQCIKZWxzZQogIGZvciAoKGk9MDsgaTwkU1BBUktfV09SS0VSX0lOU1RBTkNFUzsgaSsrKSk7IGRvCiAgICBzdGFydF9pbnN0YW5jZSAkKCggMSArICRpICkpICIkQCIKICBkb25lCmZpCg==" autogenFiles["/templates/systemd/system.service.tpl"] = "W1VuaXRdCkRlc2NyaXB0aW9uPXt7LlNlcnZpY2VOYW1lfX0gc2VydmljZQpBZnRlcj1zeXNsb2cudGFyZ2V0IG5ldHdvcmsudGFyZ2V0IHJlbW90ZS1mcy50YXJnZXQgbnNzLWxvb2t1cC50YXJnZXQKCltTZXJ2aWNlXQp7ey0gaWYgLk1lbW9yeUxpbWl0fX0KTWVtb3J5TGltaXQ9e3suTWVtb3J5TGltaXR9fQp7ey0gZW5kfX0Ke3stIGlmIC5DUFVRdW90YX19CkNQVVF1b3RhPXt7LkNQVVF1b3RhfX0Ke3stIGVuZH19Cnt7LSBpZiAuSU9SZWFkQmFuZHdpZHRoTWF4fX0KSU9SZWFkQmFuZHdpZHRoTWF4PXt7LklPUmVhZEJhbmR3aWR0aE1heH19Cnt7LSBlbmR9fQp7ey0gaWYgLklPV3JpdGVCYW5kd2lkdGhNYXh9fQpJT1dyaXRlQmFuZHdpZHRoTWF4PXt7LklPV3JpdGVCYW5kd2lkdGhNYXh9fQp7ey0gZW5kfX0KTGltaXROT0ZJTEU9MTAwMDAwMAojTGltaXRDT1JFPWluZmluaXR5CkxpbWl0U1RBQ0s9MTA0ODU3NjAKClVzZXI9e3suVXNlcn19CkV4ZWNTdGFydD17ey5EZXBsb3lEaXJ9fS9zY3JpcHRzL3J1bl97ey5TZXJ2aWNlTmFtZX19LnNoCgp7ey0gaWYgLlJlc3RhcnR9fQpSZXN0YXJ0PXt7LlJlc3RhcnR9fQp7e2Vsc2V9fQpSZXN0YXJ0PWFsd2F5cwp7e2VuZH19ClJlc3RhcnRTZWM9MTVzCnt7LSBpZiAuRGlzYWJsZVNlbmRTaWdraWxsfX0KU2VuZFNJR0tJTEw9bm8Ke3stIGVuZH19CgpbSW5zdGFsbF0KV2FudGVkQnk9bXVsdGktdXNlci50YXJnZXQK" - autogenFiles["/templates/systemd/tispark.service.tpl"] = "W1VuaXRdCkRlc2NyaXB0aW9uPXt7LlNlcnZpY2VOYW1lfX0gc2VydmljZQpBZnRlcj1zeXNsb2cudGFyZ2V0IG5ldHdvcmsudGFyZ2V0IHJlbW90ZS1mcy50YXJnZXQgbnNzLWxvb2t1cC50YXJnZXQKCltTZXJ2aWNlXQpVc2VyPXt7LlVzZXJ9fQpFeGVjU3RhcnQ9e3suRGVwbG95RGlyfX0vc2Jpbi9zdGFydC17ey5TZXJ2aWNlTmFtZX19LnNoCkV4ZWNTdG9wPXt7LkRlcGxveURpcn19L3NiaW4vc3RvcC17ey5TZXJ2aWNlTmFtZX19LnNoClR5cGU9Zm9ya2luZwp7ey0gaWYgLlJlc3RhcnR9fQpSZXN0YXJ0PXt7LlJlc3RhcnR9fQp7e2Vsc2V9fQpSZXN0YXJ0PWFsd2F5cwp7e2VuZH19ClJlc3RhcnRTZWM9MTVzClNlbmRTSUdLSUxMPW5vCgpbSW5zdGFsbF0KV2FudGVkQnk9bXVsdGktdXNlci50YXJnZXQK" + autogenFiles["/templates/systemd/tispark.service.tpl"] = "W1VuaXRdCkRlc2NyaXB0aW9uPXt7LlNlcnZpY2VOYW1lfX0gc2VydmljZQpBZnRlcj1zeXNsb2cudGFyZ2V0IG5ldHdvcmsudGFyZ2V0IHJlbW90ZS1mcy50YXJnZXQgbnNzLWxvb2t1cC50YXJnZXQKCltTZXJ2aWNlXQpVc2VyPXt7LlVzZXJ9fQp7ey0gaWYgbmUgLkphdmFIb21lICIifX0KRW52aXJvbm1lbnQ9IkpBVkFfSE9NRT17ey5KYXZhSG9tZX19Igp7ey0gZW5kfX0KRXhlY1N0YXJ0PXt7LkRlcGxveURpcn19L3NiaW4vc3RhcnQte3suU2VydmljZU5hbWV9fS5zaApFeGVjU3RvcD17ey5EZXBsb3lEaXJ9fS9zYmluL3N0b3Ate3suU2VydmljZU5hbWV9fS5zaApUeXBlPWZvcmtpbmcKe3stIGlmIC5SZXN0YXJ0fX0KUmVzdGFydD17ey5SZXN0YXJ0fX0Ke3tlbHNlfX0KUmVzdGFydD1hbHdheXMKe3stIGVuZH19ClJlc3RhcnRTZWM9MTVzClNlbmRTSUdLSUxMPW5vCgpbSW5zdGFsbF0KV2FudGVkQnk9bXVsdGktdXNlci50YXJnZXQK" } diff --git a/pkg/cluster/spec/tispark.go b/pkg/cluster/spec/tispark.go index 0f8274c496..21eda6ff13 100644 --- a/pkg/cluster/spec/tispark.go +++ b/pkg/cluster/spec/tispark.go @@ -32,11 +32,13 @@ import ( // TiSparkMasterSpec is the topology specification for TiSpark master node type TiSparkMasterSpec struct { Host string `yaml:"host"` + ListenHost string `yaml:"listen_host,omitempty"` SSHPort int `yaml:"ssh_port,omitempty" validate:"ssh_port:editable"` Imported bool `yaml:"imported,omitempty"` Port int `yaml:"port" default:"7077"` WebPort int `yaml:"web_port" default:"8080"` DeployDir string `yaml:"deploy_dir,omitempty"` + JavaHome string `yaml:"java_home,omitempty" validate:"java_home:editable"` SparkConfigs map[string]interface{} `yaml:"spark_config,omitempty" validate:"spark_config:editable"` SparkEnvs map[string]string `yaml:"spark_env,omitempty" validate:"spark_env:editable"` Arch string `yaml:"arch,omitempty"` @@ -71,14 +73,16 @@ func (s TiSparkMasterSpec) Status(pdList ...string) string { // TiSparkWorkerSpec is the topology specification for TiSpark slave nodes type TiSparkWorkerSpec struct { - Host string `yaml:"host"` - SSHPort int `yaml:"ssh_port,omitempty" validate:"ssh_port:editable"` - Imported bool `yaml:"imported,omitempty"` - Port int `yaml:"port" default:"7078"` - WebPort int `yaml:"web_port" default:"8081"` - DeployDir string `yaml:"deploy_dir,omitempty"` - Arch string `yaml:"arch,omitempty"` - OS string `yaml:"os,omitempty"` + Host string `yaml:"host"` + ListenHost string `yaml:"listen_host,omitempty"` + SSHPort int `yaml:"ssh_port,omitempty" validate:"ssh_port:editable"` + Imported bool `yaml:"imported,omitempty"` + Port int `yaml:"port" default:"7078"` + WebPort int `yaml:"web_port" default:"8081"` + DeployDir string `yaml:"deploy_dir,omitempty"` + JavaHome string `yaml:"java_home,omitempty" validate:"java_home:editable"` + Arch string `yaml:"arch,omitempty"` + OS string `yaml:"os,omitempty"` } // Role returns the component role of the instance @@ -165,6 +169,11 @@ func (i *TiSparkMasterInstance) GetCustomEnvs() map[string]string { return v.Interface().(map[string]string) } +// GetJavaHome returns the java_home value in spec +func (i *TiSparkMasterInstance) GetJavaHome() string { + return reflect.ValueOf(i.InstanceSpec).FieldByName("JavaHome").String() +} + // InitConfig implement Instance interface func (i *TiSparkMasterInstance) InitConfig(e executor.Executor, clusterName, clusterVersion, deployUser string, paths meta.DirPaths) error { // generate systemd service to invoke spark's start/stop scripts @@ -173,7 +182,7 @@ func (i *TiSparkMasterInstance) InitConfig(e executor.Executor, clusterName, clu port := i.GetPort() sysCfg := filepath.Join(paths.Cache, fmt.Sprintf("%s-%s-%d.service", comp, host, port)) - systemCfg := system.NewTiSparkConfig(comp, deployUser, paths.Deploy) + systemCfg := system.NewTiSparkConfig(comp, deployUser, paths.Deploy, i.GetJavaHome()) if err := systemCfg.ConfigToFile(sysCfg); err != nil { return errors.Trace(err) @@ -210,6 +219,7 @@ func (i *TiSparkMasterInstance) InitConfig(e executor.Executor, clusterName, clu } env := scripts.NewTiSparkEnv(host). + WithLocalIP(i.GetListenHost()). WithMasterPorts(i.usedPorts[0], i.usedPorts[1]). WithCustomEnv(i.GetCustomEnvs()) // transfer spark-env.sh file @@ -285,6 +295,11 @@ type TiSparkWorkerInstance struct { instance } +// GetJavaHome returns the java_home value in spec +func (i *TiSparkWorkerInstance) GetJavaHome() string { + return reflect.ValueOf(i.InstanceSpec).FieldByName("JavaHome").String() +} + // InitConfig implement Instance interface func (i *TiSparkWorkerInstance) InitConfig(e executor.Executor, clusterName, clusterVersion, deployUser string, paths meta.DirPaths) error { // generate systemd service to invoke spark's start/stop scripts @@ -293,7 +308,7 @@ func (i *TiSparkWorkerInstance) InitConfig(e executor.Executor, clusterName, clu port := i.GetPort() sysCfg := filepath.Join(paths.Cache, fmt.Sprintf("%s-%s-%d.service", comp, host, port)) - systemCfg := system.NewTiSparkConfig(comp, deployUser, paths.Deploy) + systemCfg := system.NewTiSparkConfig(comp, deployUser, paths.Deploy, i.GetJavaHome()) if err := systemCfg.ConfigToFile(sysCfg); err != nil { return errors.Trace(err) @@ -330,6 +345,7 @@ func (i *TiSparkWorkerInstance) InitConfig(e executor.Executor, clusterName, clu } env := scripts.NewTiSparkEnv(i.topo.TiSparkMasters[0].Host). + WithLocalIP(i.GetListenHost()). WithMasterPorts(i.topo.TiSparkMasters[0].Port, i.topo.TiSparkMasters[0].WebPort). WithWorkerPorts(i.usedPorts[0], i.usedPorts[1]). WithCustomEnv(i.instance.topo.TiSparkMasters[0].SparkEnvs) diff --git a/pkg/cluster/template/scripts/tispark.go b/pkg/cluster/template/scripts/tispark.go index 61ca1ca97f..eb2cd23100 100644 --- a/pkg/cluster/template/scripts/tispark.go +++ b/pkg/cluster/template/scripts/tispark.go @@ -21,12 +21,13 @@ import ( // TiSparkEnv represent the data to generate TiSpark environment config type TiSparkEnv struct { - TiSparkMaster string - MasterPort int - WorkerPort int - MasterUIPort int - WorkerUIPort int - CustomEnvs map[string]string + TiSparkMaster string + TiSparkLocalIP string + MasterPort int + WorkerPort int + MasterUIPort int + WorkerUIPort int + CustomEnvs map[string]string } // NewTiSparkEnv returns a TiSparkConfig @@ -40,6 +41,12 @@ func (c *TiSparkEnv) WithCustomEnv(m map[string]string) *TiSparkEnv { return c } +// WithLocalIP sets custom setting fields +func (c *TiSparkEnv) WithLocalIP(ip string) *TiSparkEnv { + c.TiSparkLocalIP = ip + return c +} + // WithMasterPorts sets port for masters func (c *TiSparkEnv) WithMasterPorts(port, ui int) *TiSparkEnv { c.MasterPort = port diff --git a/pkg/cluster/template/systemd/tispark.go b/pkg/cluster/template/systemd/tispark.go index 38632b2572..8620505a7e 100644 --- a/pkg/cluster/template/systemd/tispark.go +++ b/pkg/cluster/template/systemd/tispark.go @@ -28,13 +28,14 @@ type TiSparkConfig struct { ServiceName string User string DeployDir string + JavaHome string // Takes one of no, on-success, on-failure, on-abnormal, on-watchdog, on-abort, or always. // The Template set as always if this is not setted. Restart string } // NewTiSparkConfig returns a Config with given arguments -func NewTiSparkConfig(service, user, deployDir string) *TiSparkConfig { +func NewTiSparkConfig(service, user, deployDir, javaHome string) *TiSparkConfig { if strings.Contains(service, "master") { service = "master" } else if strings.Contains(service, "worker") { @@ -44,6 +45,7 @@ func NewTiSparkConfig(service, user, deployDir string) *TiSparkConfig { ServiceName: service, User: user, DeployDir: deployDir, + JavaHome: javaHome, } } diff --git a/templates/scripts/spark-env.sh.tpl b/templates/scripts/spark-env.sh.tpl index 330f46f934..aa841da3a6 100755 --- a/templates/scripts/spark-env.sh.tpl +++ b/templates/scripts/spark-env.sh.tpl @@ -86,3 +86,6 @@ SPARK_WORKER_PORT={{.WorkerPort}} {{- if ne .WorkerUIPort 0}} SPARK_WORKER_WEBUI_PORT={{.WorkerUIPort}} {{- end}} +{{- if ne .TiSparkLocalIP ""}} +SPARK_LOCAL_IP={{.TiSparkLocalIP}} +{{- end}} diff --git a/templates/systemd/tispark.service.tpl b/templates/systemd/tispark.service.tpl index 80e0624699..c3545c97bc 100644 --- a/templates/systemd/tispark.service.tpl +++ b/templates/systemd/tispark.service.tpl @@ -4,6 +4,9 @@ After=syslog.target network.target remote-fs.target nss-lookup.target [Service] User={{.User}} +{{- if ne .JavaHome ""}} +Environment="JAVA_HOME={{.JavaHome}}" +{{- end}} ExecStart={{.DeployDir}}/sbin/start-{{.ServiceName}}.sh ExecStop={{.DeployDir}}/sbin/stop-{{.ServiceName}}.sh Type=forking @@ -11,7 +14,7 @@ Type=forking Restart={{.Restart}} {{else}} Restart=always -{{end}} +{{- end}} RestartSec=15s SendSIGKILL=no From f885bc81d1f5d2294c20fab2ef2559777975a094 Mon Sep 17 00:00:00 2001 From: Allen Zhong Date: Wed, 22 Jul 2020 16:19:20 +0800 Subject: [PATCH 3/3] cluster/spec: adjust tispark validation error message --- pkg/cluster/spec/spec.go | 2 +- pkg/cluster/spec/spec_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/cluster/spec/spec.go b/pkg/cluster/spec/spec.go index 9dd084afd6..cebbd61e27 100644 --- a/pkg/cluster/spec/spec.go +++ b/pkg/cluster/spec/spec.go @@ -521,7 +521,7 @@ func (s *Specification) validateTiSparkSpec() error { cnt := make(map[string]int) for _, w := range s.TiSparkWorkers { if cnt[w.Host] > 0 { - return ErrMultipleTisparkWorker + return errors.Annotatef(ErrMultipleTisparkWorker, "the host %s is duplicated", w.Host) } cnt[w.Host]++ } diff --git a/pkg/cluster/spec/spec_test.go b/pkg/cluster/spec/spec_test.go index d8390b2315..cfb21c7fb1 100644 --- a/pkg/cluster/spec/spec_test.go +++ b/pkg/cluster/spec/spec_test.go @@ -788,5 +788,5 @@ tispark_workers: web_port: 8089 `), &topo) c.Assert(err, NotNil) - c.Assert(err.Error(), Equals, "multiple TiSpark workers on the same host is not supported by Spark") + c.Assert(err.Error(), Equals, "the host 172.16.5.139 is duplicated: multiple TiSpark workers on the same host is not supported by Spark") }