From 1180a922a7ae29958073b229e1010705bd74e8c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20P=C3=A9rez-Aradros=20Herce?= Date: Wed, 14 Jun 2017 15:17:01 +0200 Subject: [PATCH 01/10] Add deprecation messages for deprecated flags --- libbeat/beat/beat.go | 5 +++++ libbeat/cmd/root.go | 2 -- libbeat/cmd/run.go | 4 ++++ 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/libbeat/beat/beat.go b/libbeat/beat/beat.go index 71aff07f725..d231fd14bf4 100644 --- a/libbeat/beat/beat.go +++ b/libbeat/beat/beat.go @@ -289,6 +289,7 @@ func (b *Beat) launch(bt Creator) error { // If -configtest was specified, exit now prior to run. if cfgfile.IsTestConfig() { + logp.Deprecate("6.0", "-configtest flag has been deprecated, use configtest subcommand") fmt.Println("Config OK") return GracefulExit } @@ -296,6 +297,9 @@ func (b *Beat) launch(bt Creator) error { svc.HandleSignals(beater.Stop) // TODO Deprecate this in favor of setup subcommand (7.0) + if *setup { + logp.Deprecate("6.0", "-setup flag has been deprectad, use setup subcommand") + } err = b.loadDashboards(false) if err != nil { return err @@ -414,6 +418,7 @@ func (b *Beat) handleFlags() error { flag.Parse() if *printVersion { + logp.Deprecate("6.0", "-version flag has been deprectad, use version subcommand") fmt.Printf("%s version %s (%s), libbeat %s\n", b.Info.Beat, b.Info.Version, runtime.GOARCH, version.GetDefaultVersion()) return GracefulExit diff --git a/libbeat/cmd/root.go b/libbeat/cmd/root.go index 9d7392dd5ad..fec60efbbab 100644 --- a/libbeat/cmd/root.go +++ b/libbeat/cmd/root.go @@ -9,14 +9,12 @@ import ( "github.com/spf13/pflag" "github.com/elastic/beats/libbeat/beat" - "github.com/elastic/beats/libbeat/logp" ) func init() { // backwards compatibility workaround, convert -flags to --flags: for i, arg := range os.Args[1:] { if strings.HasPrefix(arg, "-") && !strings.HasPrefix(arg, "--") && len(arg) > 2 { - logp.Deprecate("6.0", "Argument %s should be -%s", arg, arg) os.Args[1+i] = "-" + arg } } diff --git a/libbeat/cmd/run.go b/libbeat/cmd/run.go index 6c636896718..9b0c2a318fc 100644 --- a/libbeat/cmd/run.go +++ b/libbeat/cmd/run.go @@ -33,6 +33,10 @@ func genRunCmd(name, version string, beatCreator beat.Creator, runFlags *pflag.F runCmd.Flags().AddGoFlag(flag.CommandLine.Lookup("setup")) runCmd.Flags().AddGoFlag(flag.CommandLine.Lookup("version")) + runCmd.Flags().MarkDeprecated("version", "version flag has been deprectad, use version subcommand") + runCmd.Flags().MarkDeprecated("setup", "setup flag has been deprectad, use setup subcommand") + runCmd.Flags().MarkDeprecated("configtest", "setup flag has been deprectad, use configtest subcommand") + if runFlags != nil { runCmd.Flags().AddFlagSet(runFlags) } From af6cbdeddab03c18eb6757a860c88742371741fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20P=C3=A9rez-Aradros=20Herce?= Date: Wed, 14 Jun 2017 17:24:50 +0200 Subject: [PATCH 02/10] Ensure RootCmd works when imported somewhere outside main.go --- filebeat/cmd/root.go | 2 +- heartbeat/cmd/root.go | 9 +++++++-- heartbeat/main.go | 3 --- metricbeat/cmd/root.go | 4 +++- metricbeat/main.go | 2 -- packetbeat/cmd/root.go | 4 +++- packetbeat/main.go | 3 --- 7 files changed, 14 insertions(+), 13 deletions(-) diff --git a/filebeat/cmd/root.go b/filebeat/cmd/root.go index 4598e064758..4b685b657ef 100644 --- a/filebeat/cmd/root.go +++ b/filebeat/cmd/root.go @@ -23,6 +23,6 @@ func init() { RootCmd = cmd.GenRootCmdWithRunFlags(Name, "", beater.New, runFlags) RootCmd.PersistentFlags().AddGoFlag(flag.CommandLine.Lookup("M")) - RootCmd.ConfigTest.Flags().AddGoFlag(flag.CommandLine.Lookup("modules")) + RootCmd.ConfigTestCmd.Flags().AddGoFlag(flag.CommandLine.Lookup("modules")) RootCmd.SetupCmd.Flags().AddGoFlag(flag.CommandLine.Lookup("modules")) } diff --git a/heartbeat/cmd/root.go b/heartbeat/cmd/root.go index 3d80a5bd17f..651cfa28683 100644 --- a/heartbeat/cmd/root.go +++ b/heartbeat/cmd/root.go @@ -1,7 +1,12 @@ package cmd -import cmd "github.com/elastic/beats/libbeat/cmd" -import "github.com/elastic/beats/heartbeat/beater" +import ( + // register default heartbeat monitors + _ "github.com/elastic/beats/heartbeat/monitors/defaults" + + "github.com/elastic/beats/heartbeat/beater" + cmd "github.com/elastic/beats/libbeat/cmd" +) // Name of this beat var Name = "heartbeat" diff --git a/heartbeat/main.go b/heartbeat/main.go index a4d8416d56b..6458a0e6bfb 100644 --- a/heartbeat/main.go +++ b/heartbeat/main.go @@ -4,9 +4,6 @@ import ( "os" "github.com/elastic/beats/heartbeat/cmd" - - // register default heartbeat monitors - _ "github.com/elastic/beats/heartbeat/monitors/defaults" ) func main() { diff --git a/metricbeat/cmd/root.go b/metricbeat/cmd/root.go index d987a9d748b..bb01846fb1e 100644 --- a/metricbeat/cmd/root.go +++ b/metricbeat/cmd/root.go @@ -5,9 +5,11 @@ import ( "github.com/spf13/pflag" - "github.com/elastic/beats/metricbeat/beater" + // import modules + _ "github.com/elastic/beats/metricbeat/include" cmd "github.com/elastic/beats/libbeat/cmd" + "github.com/elastic/beats/metricbeat/beater" ) // Name of this beat diff --git a/metricbeat/main.go b/metricbeat/main.go index e3f95ecdc15..78a9589dd3f 100644 --- a/metricbeat/main.go +++ b/metricbeat/main.go @@ -10,8 +10,6 @@ package main import ( "os" - _ "github.com/elastic/beats/metricbeat/include" - "github.com/elastic/beats/metricbeat/cmd" ) diff --git a/packetbeat/cmd/root.go b/packetbeat/cmd/root.go index 22af21583f0..42403d10356 100644 --- a/packetbeat/cmd/root.go +++ b/packetbeat/cmd/root.go @@ -5,9 +5,11 @@ import ( "github.com/spf13/pflag" - "github.com/elastic/beats/packetbeat/beater" + // import protocol modules + _ "github.com/elastic/beats/packetbeat/include" cmd "github.com/elastic/beats/libbeat/cmd" + "github.com/elastic/beats/packetbeat/beater" ) // Name of this beat diff --git a/packetbeat/main.go b/packetbeat/main.go index 73943a0a25c..dd32f2c9ad6 100644 --- a/packetbeat/main.go +++ b/packetbeat/main.go @@ -3,9 +3,6 @@ package main import ( "os" - // import protocol modules - _ "github.com/elastic/beats/packetbeat/include" - "github.com/elastic/beats/packetbeat/cmd" ) From f4923aec31b092e62aa4f3f3c52c86b6b9e47cc9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20P=C3=A9rez-Aradros=20Herce?= Date: Wed, 14 Jun 2017 17:35:07 +0200 Subject: [PATCH 03/10] Add `completion` command --- libbeat/cmd/completion.go | 32 ++++++++++++++++++++++++++++++++ libbeat/cmd/root.go | 15 +++++++++------ 2 files changed, 41 insertions(+), 6 deletions(-) create mode 100644 libbeat/cmd/completion.go diff --git a/libbeat/cmd/completion.go b/libbeat/cmd/completion.go new file mode 100644 index 00000000000..fe8d3c35c0a --- /dev/null +++ b/libbeat/cmd/completion.go @@ -0,0 +1,32 @@ +package cmd + +import ( + "fmt" + "os" + + "github.com/spf13/cobra" +) + +func genCompletionCmd(name, version string, rootCmd *BeatsRootCmd) *cobra.Command { + completionCmd := cobra.Command{ + Use: "completion SHELL", + Short: "Output shell completion code for the specified shell (bash only by the moment)", + // We don't want to expose this one in help: + Hidden: true, + Run: func(cmd *cobra.Command, args []string) { + if len(args) != 1 { + fmt.Println("Expected one argument with the desired shell") + os.Exit(1) + } + + switch args[0] { + case "bash": + rootCmd.GenBashCompletion(os.Stdout) + default: + fmt.Printf("Unknown shell %s, only bash is available\n", args[0]) + } + }, + } + + return &completionCmd +} diff --git a/libbeat/cmd/root.go b/libbeat/cmd/root.go index fec60efbbab..e296990ed6c 100644 --- a/libbeat/cmd/root.go +++ b/libbeat/cmd/root.go @@ -24,10 +24,11 @@ func init() { // flags and runs subcommands type BeatsRootCmd struct { cobra.Command - RunCmd *cobra.Command - SetupCmd *cobra.Command - VersionCmd *cobra.Command - ConfigTest *cobra.Command + RunCmd *cobra.Command + SetupCmd *cobra.Command + VersionCmd *cobra.Command + ConfigTestCmd *cobra.Command + CompletionCmd *cobra.Command } // GenRootCmd returns the root command to use for your beat. It takes @@ -48,7 +49,8 @@ func GenRootCmdWithRunFlags(name, version string, beatCreator beat.Creator, runF rootCmd.RunCmd = genRunCmd(name, version, beatCreator, runFlags) rootCmd.SetupCmd = genSetupCmd(name, version, beatCreator) rootCmd.VersionCmd = genVersionCmd(name, version) - rootCmd.ConfigTest = genConfigTestCmd(name, version, beatCreator) + rootCmd.ConfigTestCmd = genConfigTestCmd(name, version, beatCreator) + rootCmd.CompletionCmd = genCompletionCmd(name, version, rootCmd) // Root command is an alias for run rootCmd.Run = rootCmd.RunCmd.Run @@ -73,7 +75,8 @@ func GenRootCmdWithRunFlags(name, version string, beatCreator beat.Creator, runF rootCmd.AddCommand(rootCmd.RunCmd) rootCmd.AddCommand(rootCmd.SetupCmd) rootCmd.AddCommand(rootCmd.VersionCmd) - rootCmd.AddCommand(rootCmd.ConfigTest) + rootCmd.AddCommand(rootCmd.ConfigTestCmd) + rootCmd.AddCommand(rootCmd.CompletionCmd) return rootCmd } From b57ac771eaa8ac8fd7b8289f2086694b853fe0fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20P=C3=A9rez-Aradros=20Herce?= Date: Thu, 15 Jun 2017 17:39:45 +0200 Subject: [PATCH 04/10] Add `export config` subcommand --- libbeat/beat/beat.go | 8 +++--- libbeat/cmd/export.go | 19 ++++++++++++++ libbeat/cmd/export/config.go | 45 ++++++++++++++++++++++++++++++++ libbeat/cmd/root.go | 3 +++ libbeat/template/config.go | 3 ++- libbeat/template/load.go | 2 +- libbeat/tests/system/test_cmd.py | 19 +++++++++++++- 7 files changed, 92 insertions(+), 7 deletions(-) create mode 100644 libbeat/cmd/export.go create mode 100644 libbeat/cmd/export/config.go diff --git a/libbeat/beat/beat.go b/libbeat/beat/beat.go index d231fd14bf4..fba1b94426a 100644 --- a/libbeat/beat/beat.go +++ b/libbeat/beat/beat.go @@ -195,7 +195,7 @@ func New(name, v string) (*Beat, error) { } // init does initialization of things common to all actions (read confs, flags) -func (b *Beat) init() error { +func (b *Beat) Init() error { err := b.handleFlags() if err != nil { return err @@ -266,7 +266,7 @@ func (b *Beat) createBeater(bt Creator) (Beater, error) { } func (b *Beat) launch(bt Creator) error { - err := b.init() + err := b.Init() if err != nil { return err } @@ -325,7 +325,7 @@ func (b *Beat) launch(bt Creator) error { // TestConfig check all settings are ok and the beat can be run func (b *Beat) TestConfig(bt Creator) error { return handleError(func() error { - err := b.init() + err := b.Init() if err != nil { return err } @@ -344,7 +344,7 @@ func (b *Beat) TestConfig(bt Creator) error { // Setup registers ES index template and kibana dashboards func (b *Beat) Setup(bt Creator, template, dashboards, machineLearning bool) error { return handleError(func() error { - err := b.init() + err := b.Init() if err != nil { return err } diff --git a/libbeat/cmd/export.go b/libbeat/cmd/export.go new file mode 100644 index 00000000000..5d87915c2de --- /dev/null +++ b/libbeat/cmd/export.go @@ -0,0 +1,19 @@ +package cmd + +import ( + "github.com/spf13/cobra" + + "github.com/elastic/beats/libbeat/beat" + "github.com/elastic/beats/libbeat/cmd/export" +) + +func genExportCmd(name, beatVersion string, beatCreator beat.Creator) *cobra.Command { + exportCmd := &cobra.Command{ + Use: "export", + Short: "Export current config or index template", + } + + exportCmd.AddCommand(export.GenExportConfigCmd(name, beatVersion, beatCreator)) + + return exportCmd +} diff --git a/libbeat/cmd/export/config.go b/libbeat/cmd/export/config.go new file mode 100644 index 00000000000..062657b475f --- /dev/null +++ b/libbeat/cmd/export/config.go @@ -0,0 +1,45 @@ +package export + +import ( + "fmt" + "os" + + "github.com/spf13/cobra" + "gopkg.in/yaml.v2" + + "github.com/elastic/beats/libbeat/beat" +) + +func GenExportConfigCmd(name, beatVersion string, beatCreator beat.Creator) *cobra.Command { + return &cobra.Command{ + Use: "config", + Short: "Export current config to stdout", + Run: func(cmd *cobra.Command, args []string) { + b, err := beat.New(name, beatVersion) + if err != nil { + fmt.Fprintf(os.Stderr, "Error initializing beat") + os.Exit(1) + } + + err = b.Init() + if err != nil { + fmt.Fprintf(os.Stderr, "Error initializing beat") + os.Exit(1) + } + + var config map[string]interface{} + err = b.RawConfig.Unpack(&config) + if err != nil { + fmt.Fprintf(os.Stderr, "Error unpacking config") + os.Exit(1) + } + res, err := yaml.Marshal(config) + if err != nil { + fmt.Fprintf(os.Stderr, "Error converting config to YAML format") + os.Exit(1) + } + + os.Stdout.Write(res) + }, + } +} diff --git a/libbeat/cmd/root.go b/libbeat/cmd/root.go index e296990ed6c..b2505f4dd5b 100644 --- a/libbeat/cmd/root.go +++ b/libbeat/cmd/root.go @@ -29,6 +29,7 @@ type BeatsRootCmd struct { VersionCmd *cobra.Command ConfigTestCmd *cobra.Command CompletionCmd *cobra.Command + ExportCmd *cobra.Command } // GenRootCmd returns the root command to use for your beat. It takes @@ -51,6 +52,7 @@ func GenRootCmdWithRunFlags(name, version string, beatCreator beat.Creator, runF rootCmd.VersionCmd = genVersionCmd(name, version) rootCmd.ConfigTestCmd = genConfigTestCmd(name, version, beatCreator) rootCmd.CompletionCmd = genCompletionCmd(name, version, rootCmd) + rootCmd.ExportCmd = genExportCmd(name, version, beatCreator) // Root command is an alias for run rootCmd.Run = rootCmd.RunCmd.Run @@ -77,6 +79,7 @@ func GenRootCmdWithRunFlags(name, version string, beatCreator beat.Creator, runF rootCmd.AddCommand(rootCmd.VersionCmd) rootCmd.AddCommand(rootCmd.ConfigTestCmd) rootCmd.AddCommand(rootCmd.CompletionCmd) + rootCmd.AddCommand(rootCmd.ExportCmd) return rootCmd } diff --git a/libbeat/template/config.go b/libbeat/template/config.go index 60707e882bf..bb0f401ffd4 100644 --- a/libbeat/template/config.go +++ b/libbeat/template/config.go @@ -22,7 +22,8 @@ type TemplateSettings struct { } var ( - defaultConfig = TemplateConfig{ + // DefaultConfig for index template + DefaultConfig = TemplateConfig{ Enabled: true, Fields: "fields.yml", } diff --git a/libbeat/template/load.go b/libbeat/template/load.go index fe67d08f035..e1bf6f86e49 100644 --- a/libbeat/template/load.go +++ b/libbeat/template/load.go @@ -26,7 +26,7 @@ type Loader struct { func NewLoader(cfg *common.Config, client ESClient, beatInfo common.BeatInfo) (*Loader, error) { - config := defaultConfig + config := DefaultConfig err := cfg.Unpack(&config) if err != nil { diff --git a/libbeat/tests/system/test_cmd.py b/libbeat/tests/system/test_cmd.py index e51cba40271..008d29b6d30 100644 --- a/libbeat/tests/system/test_cmd.py +++ b/libbeat/tests/system/test_cmd.py @@ -81,7 +81,6 @@ def test_configtest(self): extra_args=["configtest"], config="libbeat.yml") - print(self.get_log()) assert exit_code == 0 assert self.log_contains("Config OK") @@ -97,5 +96,23 @@ def test_configtest_bad_config(self): assert exit_code == 1 assert self.log_contains("Config OK") is False + def test_export_config(self): + """ + Test export config works + """ + self.render_config_template("mockbeat", + os.path.join(self.working_dir, + "libbeat.yml"), + metrics_period=1234) + + exit_code = self.run_beat( + logging_args=[], + extra_args=["export", "config"], + config="libbeat.yml") + + assert exit_code == 0 + assert self.log_contains("filename: mockbeat") + assert self.log_contains("period: 1234") + def get_host(self): return os.getenv('ES_HOST', 'localhost') + ':' + os.getenv('ES_PORT', '9200') From c2a9e136dc8c8983cf506f8a993f8bf025352ea3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20P=C3=A9rez-Aradros=20Herce?= Date: Thu, 15 Jun 2017 20:30:23 +0200 Subject: [PATCH 05/10] Add `export template` subcommand --- libbeat/cmd/export.go | 1 + libbeat/cmd/export/template.go | 68 +++++++++++++++++++++ libbeat/tests/system/config/mockbeat.yml.j2 | 7 +++ libbeat/tests/system/test_cmd.py | 18 ++++++ 4 files changed, 94 insertions(+) create mode 100644 libbeat/cmd/export/template.go diff --git a/libbeat/cmd/export.go b/libbeat/cmd/export.go index 5d87915c2de..20ee115868a 100644 --- a/libbeat/cmd/export.go +++ b/libbeat/cmd/export.go @@ -14,6 +14,7 @@ func genExportCmd(name, beatVersion string, beatCreator beat.Creator) *cobra.Com } exportCmd.AddCommand(export.GenExportConfigCmd(name, beatVersion, beatCreator)) + exportCmd.AddCommand(export.GenTemplateConfigCmd(name, beatVersion, beatCreator)) return exportCmd } diff --git a/libbeat/cmd/export/template.go b/libbeat/cmd/export/template.go new file mode 100644 index 00000000000..1f8ed7e3722 --- /dev/null +++ b/libbeat/cmd/export/template.go @@ -0,0 +1,68 @@ +package export + +import ( + "fmt" + "os" + + "github.com/spf13/cobra" + + "github.com/elastic/beats/libbeat/beat" + "github.com/elastic/beats/libbeat/template" +) + +func GenTemplateConfigCmd(name, beatVersion string, beatCreator beat.Creator) *cobra.Command { + genTemplateConfigCmd := &cobra.Command{ + Use: "template", + Short: "Export index template to stdout", + Run: func(cmd *cobra.Command, args []string) { + version, _ := cmd.Flags().GetString("es.version") + index, _ := cmd.Flags().GetString("index") + + // Make it compatible with the sem versioning + if version == "2x" { + version = "2.0.0" + } + + b, err := beat.New(name, beatVersion) + if err != nil { + fmt.Fprintf(os.Stderr, "Error initializing beat") + os.Exit(1) + } + err = b.Init() + if err != nil { + fmt.Fprintf(os.Stderr, "Error initializing beat") + os.Exit(1) + } + + cfg := template.DefaultConfig + err = b.Config.Template.Unpack(&cfg) + if err != nil { + fmt.Fprintf(os.Stderr, "Error getting template settings: %+v", err) + os.Exit(1) + } + + tmpl, err := template.New(b.Info.Version, version, index, cfg.Settings) + if err != nil { + fmt.Fprintf(os.Stderr, "Error generating template: %+v", err) + os.Exit(1) + } + + templateString, err := tmpl.Load(cfg.Fields) + if err != nil { + fmt.Fprintf(os.Stderr, "Error generating template: %+v", err) + os.Exit(1) + } + + _, err = os.Stdout.WriteString(templateString.StringToPrint()) + if err != nil { + fmt.Fprintf(os.Stderr, "Error writing template: %+v", err) + os.Exit(1) + } + }, + } + + genTemplateConfigCmd.Flags().String("es.version", beatVersion, "Elasticsearch version") + genTemplateConfigCmd.Flags().String("index", name, "Base index name") + + return genTemplateConfigCmd +} diff --git a/libbeat/tests/system/config/mockbeat.yml.j2 b/libbeat/tests/system/config/mockbeat.yml.j2 index 11355a34a98..289edb80581 100644 --- a/libbeat/tests/system/config/mockbeat.yml.j2 +++ b/libbeat/tests/system/config/mockbeat.yml.j2 @@ -64,6 +64,13 @@ output: #number_of_files: 7 {%- endif %} +#==================== Elasticsearch template setting ========================== +setup.template: + fields: {{ fields|default("fields.yml") }} + settings: + index.number_of_shards: 1 + index.codec: best_compression + #================================ Logging ===================================== {% if metrics_period -%} diff --git a/libbeat/tests/system/test_cmd.py b/libbeat/tests/system/test_cmd.py index 008d29b6d30..33c37191779 100644 --- a/libbeat/tests/system/test_cmd.py +++ b/libbeat/tests/system/test_cmd.py @@ -114,5 +114,23 @@ def test_export_config(self): assert self.log_contains("filename: mockbeat") assert self.log_contains("period: 1234") + def test_export_template(self): + """ + Test export template works + """ + self.render_config_template("mockbeat", + os.path.join(self.working_dir, "mockbeat.yml"), + fields=os.path.join(self.working_dir, "fields.yml")) + shutil.copy(self.beat_path + "/fields.yml", + os.path.join(self.working_dir, "fields.yml")) + exit_code = self.run_beat( + logging_args=[], + extra_args=["export", "template"], + config="mockbeat.yml") + + assert exit_code == 0 + assert self.log_contains('"mockbeat-9.9.9-*"') + assert self.log_contains('"codec": "best_compression"') + def get_host(self): return os.getenv('ES_HOST', 'localhost') + ':' + os.getenv('ES_PORT', '9200') From 8e0f2429175fb6f5fe1917795883e6069094b6d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20P=C3=A9rez-Aradros=20Herce?= Date: Wed, 21 Jun 2017 11:48:27 +0200 Subject: [PATCH 06/10] Be more verbose on cmd errors --- libbeat/cmd/configtest.go | 2 ++ libbeat/cmd/export/config.go | 4 ++-- libbeat/cmd/export/template.go | 4 ++-- libbeat/cmd/setup.go | 2 ++ libbeat/cmd/version.go | 1 + 5 files changed, 9 insertions(+), 4 deletions(-) diff --git a/libbeat/cmd/configtest.go b/libbeat/cmd/configtest.go index 6bd637a0cf3..734b1973414 100644 --- a/libbeat/cmd/configtest.go +++ b/libbeat/cmd/configtest.go @@ -1,6 +1,7 @@ package cmd import ( + "fmt" "os" "github.com/spf13/cobra" @@ -15,6 +16,7 @@ func genConfigTestCmd(name, version string, beatCreator beat.Creator) *cobra.Com Run: func(cmd *cobra.Command, args []string) { b, err := beat.New(name, version) if err != nil { + fmt.Fprintf(os.Stderr, "Error initializing beat: %s\n", err) os.Exit(1) } diff --git a/libbeat/cmd/export/config.go b/libbeat/cmd/export/config.go index 062657b475f..64dd47b9c42 100644 --- a/libbeat/cmd/export/config.go +++ b/libbeat/cmd/export/config.go @@ -17,13 +17,13 @@ func GenExportConfigCmd(name, beatVersion string, beatCreator beat.Creator) *cob Run: func(cmd *cobra.Command, args []string) { b, err := beat.New(name, beatVersion) if err != nil { - fmt.Fprintf(os.Stderr, "Error initializing beat") + fmt.Fprintf(os.Stderr, "Error initializing beat: %s\n", err) os.Exit(1) } err = b.Init() if err != nil { - fmt.Fprintf(os.Stderr, "Error initializing beat") + fmt.Fprintf(os.Stderr, "Error initializing beat: %s\n", err) os.Exit(1) } diff --git a/libbeat/cmd/export/template.go b/libbeat/cmd/export/template.go index 1f8ed7e3722..a3bfd0e1600 100644 --- a/libbeat/cmd/export/template.go +++ b/libbeat/cmd/export/template.go @@ -25,12 +25,12 @@ func GenTemplateConfigCmd(name, beatVersion string, beatCreator beat.Creator) *c b, err := beat.New(name, beatVersion) if err != nil { - fmt.Fprintf(os.Stderr, "Error initializing beat") + fmt.Fprintf(os.Stderr, "Error initializing beat: %s\n", err) os.Exit(1) } err = b.Init() if err != nil { - fmt.Fprintf(os.Stderr, "Error initializing beat") + fmt.Fprintf(os.Stderr, "Error initializing beat: %s\n", err) os.Exit(1) } diff --git a/libbeat/cmd/setup.go b/libbeat/cmd/setup.go index c55cbac5e3a..7d1ddb1ed26 100644 --- a/libbeat/cmd/setup.go +++ b/libbeat/cmd/setup.go @@ -1,6 +1,7 @@ package cmd import ( + "fmt" "os" "github.com/spf13/cobra" @@ -20,6 +21,7 @@ func genSetupCmd(name, version string, beatCreator beat.Creator) *cobra.Command Run: func(cmd *cobra.Command, args []string) { beat, err := beat.New(name, version) if err != nil { + fmt.Fprintf(os.Stderr, "Error initializing beat: %s\n", err) os.Exit(1) } diff --git a/libbeat/cmd/version.go b/libbeat/cmd/version.go index 8f10494a779..dcb7b0c2278 100644 --- a/libbeat/cmd/version.go +++ b/libbeat/cmd/version.go @@ -18,6 +18,7 @@ func genVersionCmd(name, beatVersion string) *cobra.Command { Run: func(cmd *cobra.Command, args []string) { beat, err := beat.New(name, beatVersion) if err != nil { + fmt.Fprintf(os.Stderr, "Error initializing beat: %s\n", err) os.Exit(1) } From 11df39188ab19e5098b88eea1cd207e26381c150 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20P=C3=A9rez-Aradros=20Herce?= Date: Thu, 22 Jun 2017 12:40:08 +0200 Subject: [PATCH 07/10] Do not fail when no template settings are given --- libbeat/cmd/export/template.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/libbeat/cmd/export/template.go b/libbeat/cmd/export/template.go index a3bfd0e1600..f716f3dcac3 100644 --- a/libbeat/cmd/export/template.go +++ b/libbeat/cmd/export/template.go @@ -35,10 +35,12 @@ func GenTemplateConfigCmd(name, beatVersion string, beatCreator beat.Creator) *c } cfg := template.DefaultConfig - err = b.Config.Template.Unpack(&cfg) - if err != nil { - fmt.Fprintf(os.Stderr, "Error getting template settings: %+v", err) - os.Exit(1) + if b.Config.Template.Enabled() { + err = b.Config.Template.Unpack(&cfg) + if err != nil { + fmt.Fprintf(os.Stderr, "Error getting template settings: %+v", err) + os.Exit(1) + } } tmpl, err := template.New(b.Info.Version, version, index, cfg.Settings) @@ -53,7 +55,7 @@ func GenTemplateConfigCmd(name, beatVersion string, beatCreator beat.Creator) *c os.Exit(1) } - _, err = os.Stdout.WriteString(templateString.StringToPrint()) + _, err = os.Stdout.WriteString(templateString.StringToPrint() + "\n") if err != nil { fmt.Fprintf(os.Stderr, "Error writing template: %+v", err) os.Exit(1) From 155f07a349bc0838d0f3db761a91762e245ae9f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20P=C3=A9rez-Aradros=20Herce?= Date: Thu, 22 Jun 2017 21:40:38 +0200 Subject: [PATCH 08/10] Do not deprecate `-setup` flag --- libbeat/cmd/run.go | 1 - 1 file changed, 1 deletion(-) diff --git a/libbeat/cmd/run.go b/libbeat/cmd/run.go index 9b0c2a318fc..d68cd19fa3e 100644 --- a/libbeat/cmd/run.go +++ b/libbeat/cmd/run.go @@ -34,7 +34,6 @@ func genRunCmd(name, version string, beatCreator beat.Creator, runFlags *pflag.F runCmd.Flags().AddGoFlag(flag.CommandLine.Lookup("version")) runCmd.Flags().MarkDeprecated("version", "version flag has been deprectad, use version subcommand") - runCmd.Flags().MarkDeprecated("setup", "setup flag has been deprectad, use setup subcommand") runCmd.Flags().MarkDeprecated("configtest", "setup flag has been deprectad, use configtest subcommand") if runFlags != nil { From 310bb5248575a2f2c0eebfd4688d7234e0a584c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20P=C3=A9rez-Aradros=20Herce?= Date: Fri, 23 Jun 2017 10:59:28 +0200 Subject: [PATCH 09/10] Remove version mangling in `export template` --- libbeat/cmd/export/template.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/libbeat/cmd/export/template.go b/libbeat/cmd/export/template.go index f716f3dcac3..7c5ad8ec751 100644 --- a/libbeat/cmd/export/template.go +++ b/libbeat/cmd/export/template.go @@ -18,11 +18,6 @@ func GenTemplateConfigCmd(name, beatVersion string, beatCreator beat.Creator) *c version, _ := cmd.Flags().GetString("es.version") index, _ := cmd.Flags().GetString("index") - // Make it compatible with the sem versioning - if version == "2x" { - version = "2.0.0" - } - b, err := beat.New(name, beatVersion) if err != nil { fmt.Fprintf(os.Stderr, "Error initializing beat: %s\n", err) From d7d5e0af9d559790068ec5cdef1e01abd574501d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20P=C3=A9rez-Aradros=20Herce?= Date: Mon, 26 Jun 2017 12:21:28 +0200 Subject: [PATCH 10/10] Remove `output_to_file` template settings Deprecated in favor of `beat export template` --- libbeat/beat/beat.go | 14 --- libbeat/beat/setup.go | 11 +-- libbeat/docs/template-config.asciidoc | 22 ----- libbeat/template/config.go | 18 +--- libbeat/template/load.go | 39 -------- libbeat/template/load_test.go | 135 -------------------------- libbeat/tests/system/test_template.py | 47 --------- 7 files changed, 10 insertions(+), 276 deletions(-) delete mode 100644 libbeat/template/load_test.go delete mode 100644 libbeat/tests/system/test_template.py diff --git a/libbeat/beat/beat.go b/libbeat/beat/beat.go index fba1b94426a..56802e8a2b5 100644 --- a/libbeat/beat/beat.go +++ b/libbeat/beat/beat.go @@ -600,20 +600,6 @@ func (b *Beat) registerTemplateLoading() error { if err != nil { return fmt.Errorf("unpacking template config fails: %v", err) } - if len(cfg.OutputToFile.Path) > 0 { - // output to file is enabled - loader, err := template.NewLoader(b.Config.Template, nil, b.Info) - if err != nil { - return fmt.Errorf("Error creating Elasticsearch template loader: %v", err) - } - err = loader.Generate() - if err != nil { - return fmt.Errorf("Error generating template: %v", err) - } - - // XXX: Should we kill the Beat here or just continue? - return fmt.Errorf("Stopping after successfully writing the template to the file.") - } } // Loads template by default if esOutput is enabled diff --git a/libbeat/beat/setup.go b/libbeat/beat/setup.go index 061e9091270..1cb5c2d4a41 100644 --- a/libbeat/beat/setup.go +++ b/libbeat/beat/setup.go @@ -1,10 +1,9 @@ package beat type TemplateConfig struct { - Enabled bool `config:"enabled"` - Name string `config:"name"` - Fields string `config:"fields"` - Overwrite bool `config:"overwrite"` - OutputToFile string `config:"output_to_file"` - Settings map[string]string `config:"settings"` + Enabled bool `config:"enabled"` + Name string `config:"name"` + Fields string `config:"fields"` + Overwrite bool `config:"overwrite"` + Settings map[string]string `config:"settings"` } diff --git a/libbeat/docs/template-config.asciidoc b/libbeat/docs/template-config.asciidoc index 3a554590dd1..a27b2fd9ebb 100644 --- a/libbeat/docs/template-config.asciidoc +++ b/libbeat/docs/template-config.asciidoc @@ -49,25 +49,3 @@ setup.template.overwrite: false setup.template.settings: _source.enabled: false ---------------------------------------------------------------------- - -*`output_to_file.path`*:: If this option is set, {beatname_uc} generates the Elasticsearch template - JSON object and writes into a file at the specified path. Immediately after writing the file, - {beatname_uc} exists with the exit code 1. - -For example, you can generate a template file ready to be uploaded to Elasticsearch like this: - -["source","yaml",subs="attributes,callouts"] ----------------------------------------------------------------------- -./{beatname_lc} -e -E "setup.template.output_to_file.path={beatname_lc}.template.json" ----------------------------------------------------------------------- - -*`output_to_file.version`*:: The Elasticsearch version for which to generate the template file. By -default, the {beatname_uc} version is used. This setting is only used if `output_to_file.path` is -also set. - -For example, the following generates a template file for Elasticsearch 5.4: - -["source","yaml",subs="attributes,callouts"] ----------------------------------------------------------------------- -./{beatname_lc} -e -E "setup.template.output_to_file.path={beatname_lc}.template.json" -E "setup.template.output_to_file.version=5.4.0" ----------------------------------------------------------------------- diff --git a/libbeat/template/config.go b/libbeat/template/config.go index bb0f401ffd4..9aacc7e997a 100644 --- a/libbeat/template/config.go +++ b/libbeat/template/config.go @@ -1,19 +1,11 @@ package template type TemplateConfig struct { - Enabled bool `config:"enabled"` - Name string `config:"name"` - Fields string `config:"fields"` - Overwrite bool `config:"overwrite"` - Settings TemplateSettings `config:"settings"` - OutputToFile OutputToFile `config:"output_to_file"` -} - -// OutputToFile contains the configuration options for generating -// and writing the template into a file. -type OutputToFile struct { - Path string `config:"path"` - Version string `config:"version"` + Enabled bool `config:"enabled"` + Name string `config:"name"` + Fields string `config:"fields"` + Overwrite bool `config:"overwrite"` + Settings TemplateSettings `config:"settings"` } type TemplateSettings struct { diff --git a/libbeat/template/load.go b/libbeat/template/load.go index e1bf6f86e49..d362c7dfd84 100644 --- a/libbeat/template/load.go +++ b/libbeat/template/load.go @@ -1,9 +1,7 @@ package template import ( - "encoding/json" "fmt" - "io/ioutil" "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/logp" @@ -82,43 +80,6 @@ func (l *Loader) Load() error { return nil } -// Generate generates the template and writes it to a file based on the configuration -// from `output_to_file`. -func (l *Loader) Generate() error { - if l.config.OutputToFile.Version == "" { - l.config.OutputToFile.Version = l.beatInfo.Version - } - - if l.config.Name == "" { - l.config.Name = l.beatInfo.Beat - } - - tmpl, err := New(l.beatInfo.Version, l.config.OutputToFile.Version, l.config.Name, l.config.Settings) - if err != nil { - return fmt.Errorf("error creating template instance: %v", err) - } - - fieldsPath := paths.Resolve(paths.Config, l.config.Fields) - - output, err := tmpl.Load(fieldsPath) - if err != nil { - return fmt.Errorf("error creating template from file %s: %v", fieldsPath, err) - } - - jsonBytes, err := json.MarshalIndent(output, "", " ") - if err != nil { - return fmt.Errorf("error marshaling template: %v", err) - } - - err = ioutil.WriteFile(l.config.OutputToFile.Path, jsonBytes, 0644) - if err != nil { - return fmt.Errorf("error writing to file %s: %v", l.config.OutputToFile.Path, err) - } - - logp.Info("Template for Elasticsearch %s written to: %s", l.config.OutputToFile.Version, l.config.OutputToFile.Path) - return nil -} - // LoadTemplate loads a template into Elasticsearch overwriting the existing // template if it exists. If you wish to not overwrite an existing template // then use CheckTemplate prior to calling this method. diff --git a/libbeat/template/load_test.go b/libbeat/template/load_test.go deleted file mode 100644 index d6347d66387..00000000000 --- a/libbeat/template/load_test.go +++ /dev/null @@ -1,135 +0,0 @@ -// +build !integration - -package template - -import ( - "encoding/json" - "io/ioutil" - "os" - "path/filepath" - "testing" - - "github.com/elastic/beats/libbeat/common" - "github.com/elastic/beats/libbeat/version" - "github.com/stretchr/testify/assert" -) - -func TestGenerateTemplate(t *testing.T) { - // Load template - absPath, err := filepath.Abs("../") - if err != nil { - t.Fatal(err) - } - - beatInfo := common.BeatInfo{ - Beat: "testbeat", - Version: version.GetDefaultVersion(), - } - - dir, err := ioutil.TempDir("", "test-template") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(dir) - - outputFile := filepath.Join(dir, "template.json") - - config := newConfigFrom(t, TemplateConfig{ - Enabled: true, - Fields: filepath.Join(absPath, "fields.yml"), - OutputToFile: OutputToFile{ - Path: outputFile, - }, - }) - - loader, err := NewLoader(config, nil, beatInfo) - if err != nil { - t.Fatal(err) - } - - if err = loader.Generate(); err != nil { - t.Fatal("generate failed", err) - } - - // Read it back to check it - fp, err := os.Open(outputFile) - if err != nil { - t.Fatal(err) - } - - jsonParser := json.NewDecoder(fp) - var parsed common.MapStr - if err = jsonParser.Decode(&parsed); err != nil { - t.Fatal("decoding failed", err) - } - - val, err := parsed.GetValue("mappings._default_._meta.version") - if err != nil { - t.Fatal(err) - } - assert.Equal(t, val.(string), version.GetDefaultVersion()) -} - -func TestGenerateTemplateWithVersion(t *testing.T) { - // Load template - absPath, err := filepath.Abs("../") - if err != nil { - t.Fatal(err) - } - - beatInfo := common.BeatInfo{ - Beat: "testbeat", - Version: version.GetDefaultVersion(), - } - - dir, err := ioutil.TempDir("", "test-template") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(dir) - - outputFile := filepath.Join(dir, "template.json") - - config := newConfigFrom(t, TemplateConfig{ - Enabled: true, - Fields: filepath.Join(absPath, "fields.yml"), - OutputToFile: OutputToFile{ - Path: outputFile, - Version: "2.4.0", - }, - }) - - loader, err := NewLoader(config, nil, beatInfo) - if err != nil { - t.Fatal(err) - } - - if err = loader.Generate(); err != nil { - t.Fatal("generate failed", err) - } - - // Read it back to check it - fp, err := os.Open(outputFile) - if err != nil { - t.Fatal(err) - } - - jsonParser := json.NewDecoder(fp) - var parsed common.MapStr - if err = jsonParser.Decode(&parsed); err != nil { - t.Fatal("decoding failed", err) - } - - // Check a setting specific to that version. - val, err := parsed.GetValue("mappings._default_._all.norms.enabled") - if err != nil { - t.Fatal(err) - } - assert.Equal(t, val.(bool), false) -} - -func newConfigFrom(t *testing.T, from interface{}) *common.Config { - cfg, err := common.NewConfigFrom(from) - assert.NoError(t, err) - return cfg -} diff --git a/libbeat/tests/system/test_template.py b/libbeat/tests/system/test_template.py deleted file mode 100644 index 41ea5359124..00000000000 --- a/libbeat/tests/system/test_template.py +++ /dev/null @@ -1,47 +0,0 @@ -from base import BaseTest - -import os -import json - - -class Test(BaseTest): - - def test_generate_templates(self): - """ - Generates templates from other Beats. - """ - self.render_config_template() - - output_json = os.path.join(self.working_dir, "template.json") - fields_yml = "../../../../fields.yml" - - exit_code = self.run_beat(extra_args=[ - "-E", "setup.template.output_to_file.path={}".format(output_json), - "-E", "setup.template.fields={}".format(fields_yml)]) - assert exit_code == 1 - - # check json file - with open(output_json) as f: - tmpl = json.load(f) - assert "mappings" in tmpl - - def test_generate_templates_v5(self): - """ - Generates templates from other Beats. - """ - self.render_config_template() - - output_json = os.path.join(self.working_dir, "template-5x.json") - fields_yml = "../../../../fields.yml" - - exit_code = self.run_beat(extra_args=[ - "-E", "setup.template.output_to_file.path={}".format(output_json), - "-E", "setup.template.output_to_file.version=5.0.0".format(output_json), - "-E", "setup.template.fields={}".format(fields_yml)]) - assert exit_code == 1 - - # check json file - with open(output_json) as f: - tmpl = json.load(f) - assert "mappings" in tmpl - assert tmpl["mappings"]["_default_"]["_all"]["norms"]["enabled"] is False