Add functionbeat to the build process (#8639)

* This add beatless/functionbeat to the build processes and do the
following changes:

    Use a custom packages.yml to only build some of the artifact.
    Each distribution also include a linux binary to be run on AWS lambda
    Some refactoring is done in the mage.go file to allow to specify a
    specs file more easily.
    Make sure that beatless is added to build/distributions/beatless

* fix the issue with license on the fields.go

* Adding minimal docs to satisfy DTD requirements and not fail the build

Makefile

@@ -1,6 +1,6 @@
 BUILD_DIR=$(CURDIR)/build
 COVERAGE_DIR=$(BUILD_DIR)/coverage
-BEATS?=auditbeat filebeat heartbeat metricbeat packetbeat winlogbeat
+BEATS?=auditbeat filebeat heartbeat metricbeat packetbeat winlogbeat x-pack/beatless
 PROJECTS=libbeat $(BEATS)
 PROJECTS_ENV=libbeat filebeat metricbeat
 PYTHON_ENV?=$(BUILD_DIR)/python-env
@@ -11,6 +11,7 @@ GOLINT_REPO=github.com/golang/lint/golint
 REVIEWDOG=reviewdog
 REVIEWDOG_OPTIONS?=-diff "git diff master"
 REVIEWDOG_REPO=github.com/haya14busa/reviewdog/cmd/reviewdog
+XPACK_SUFFIX=x-pack/
 
 # Runs complete testsuites (unit, system, integration) for all beats with coverage and race detection.
 # Also it builds the docs and the generators
@@ -152,7 +153,7 @@ release: beats-dashboards
 	@$(foreach var,$(BEATS),$(MAKE) -C $(var) release || exit 1;)
 	@$(foreach var,$(BEATS), \
       test -d $(var)/build/distributions && test -n "$$(ls $(var)/build/distributions)" || exit 0; \
-      mkdir -p build/distributions/$(var) && mv -f $(var)/build/distributions/* build/distributions/$(var)/ || exit 1;)
+      mkdir -p build/distributions/$(subst $(XPACK_SUFFIX),'',$(var)) && mv -f $(var)/build/distributions/* build/distributions/$(subst $(XPACK_SUFFIX),'',$(var))/ || exit 1;)
 
 # Builds a snapshot release. The Go version defined in .go-version will be
 # installed and used for the build.

dev-tools/mage/pkgspecs.go

@@ -35,31 +35,15 @@ var Packages []OSPackageArgs
 // UseCommunityBeatPackaging configures the package target to build packages for
 // a community Beat.
 func UseCommunityBeatPackaging() {
-	beatsDir, err := ElasticBeatsDir()
-	if err != nil {
-		panic(err)
-	}
-
-	err = LoadNamedSpec("community_beat", filepath.Join(beatsDir, packageSpecFile))
-	if err != nil {
-		panic(err)
-	}
+	MustUsePackaging("community_beat", packageSpecFile)
 }
 
 // UseElasticBeatPackaging configures the package target to build packages for
 // an Elastic Beat. This means it will generate two sets of packages -- one
 // that is purely OSS under Apache 2.0 and one that is licensed under the
 // Elastic License and may contain additional X-Pack features.
 func UseElasticBeatPackaging() {
-	beatsDir, err := ElasticBeatsDir()
-	if err != nil {
-		panic(err)
-	}
-
-	err = LoadNamedSpec("elastic_beat", filepath.Join(beatsDir, packageSpecFile))
-	if err != nil {
-		panic(err)
-	}
+	MustUsePackaging("elastic_beat", packageSpecFile)
 }
 
 // UseElasticBeatWithoutXPackPackaging configures the package target to build packages for
@@ -71,12 +55,20 @@ func UseElasticBeatPackaging() {
 // a temporary packaging target for projects that depends on beat but do have concrete x-pack
 // binaries.
 func UseElasticBeatWithoutXPackPackaging() {
+	MustUsePackaging("elastic_beat_without_xpack", packageSpecFile)
+}
+
+// MustUsePackaging will load a named spec from a named file, if any errors occurs when loading
+// the specs it will panic.
+//
+// NOTE: we assume that specFile is relative to the beatsDir.
+func MustUsePackaging(specName, specFile string) {
 	beatsDir, err := ElasticBeatsDir()
 	if err != nil {
 		panic(err)
 	}
 
-	err = LoadNamedSpec("elastic_beat_without_xpack", filepath.Join(beatsDir, packageSpecFile))
+	err = LoadNamedSpec(specName, filepath.Join(beatsDir, specFile))
 	if err != nil {
 		panic(err)
 	}

dev-tools/packaging/package_test.go

@@ -48,7 +48,7 @@ const (
 )
 
 var (
-	configFilePattern      = regexp.MustCompile(`.*beat\.yml|apm-server\.yml`)
+	configFilePattern      = regexp.MustCompile(`.*beat\.yml|beatless\.yml|apm-server\.yml`)
 	manifestFilePattern    = regexp.MustCompile(`manifest.yml`)
 	modulesDirPattern      = regexp.MustCompile(`module/.+`)
 	modulesDDirPattern     = regexp.MustCompile(`modules.d/$`)

libbeat/scripts/Makefile

@@ -337,7 +337,7 @@ endif
 
 ifneq ($(shell [[ $(BEAT_NAME) == libbeat || $(BEAT_NAME) == metricbeat ]] && echo true ),true)
 	mkdir -p include
-	go run  ${ES_BEATS}/dev-tools/cmd/asset/asset.go -pkg include -in fields.yml -out include/fields.go $(BEAT_NAME)
+	go run  ${ES_BEATS}/dev-tools/cmd/asset/asset.go -license $(LICENSE) -pkg include -in fields.yml -out include/fields.go $(BEAT_NAME)
 endif
 
 ifneq ($(shell [[ $(BEAT_NAME) == libbeat ]] && echo true ),true)

magefile.go

@@ -34,6 +34,7 @@ var (
 		"metricbeat",
 		"packetbeat",
 		"winlogbeat",
+		"x-pack/beatless",
 	}
 )
 
@@ -68,5 +69,5 @@ func PackageBeatDashboards() error {
 
 // DumpVariables writes the template variables and values to stdout.
 func DumpVariables() error {
-    return mage.DumpVariables()
+	return mage.DumpVariables()
 }

x-pack/beatless/dev-tools/packaging/packages.yml

@@ -0,0 +1,101 @@
+---
+
+# This file contains the package specifications for Beatless.
+
+shared:
+  - &common
+    name: '{{.BeatName}}'
+    service_name: '{{.BeatServiceName}}'
+    os: '{{.GOOS}}'
+    arch: '{{.PackageArch}}'
+    vendor: '{{.BeatVendor}}'
+    version: '{{ beat_version }}'
+    license: '{{.BeatLicense}}'
+    url: '{{.BeatURL}}'
+    description: '{{.BeatDescription}}'
+
+  - &binary_files
+    '{{.BeatName}}{{.BinaryExt}}':
+      source: build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}}
+      mode: 0755
+    fields.yml:
+      source: fields.yml
+      mode: 0644
+    LICENSE.txt:
+      source: '{{ repo.RootDir }}/LICENSE.txt'
+      mode: 0644
+    NOTICE.txt:
+      source: '{{ repo.RootDir }}/NOTICE.txt'
+      mode: 0644
+    README.md:
+      template: '{{ elastic_beats_dir }}/dev-tools/packaging/templates/common/README.md.tmpl'
+      mode: 0644
+    .build_hash.txt:
+      content: >
+        {{ commit }}
+      mode: 0644
+    '{{.BeatName}}.reference.yml':
+      source: '{{.BeatName}}.reference.yml'
+      mode: 0644
+    '{{.BeatName}}.yml':
+      source: '{{.BeatName}}.yml'
+      mode: 0600
+      config: true
+    kibana:
+      source: _meta/kibana.generated
+      mode: 0644
+
+  # Binary package spec (tar.gz for linux/darwin)
+  - &binary_spec
+    <<: *common
+    files:
+      <<: *binary_files
+
+  #
+  # License modifiers for the Elastic License
+  #
+  - &elastic_license_for_binaries
+    license: "Elastic License"
+    files:
+      LICENSE.txt:
+        source: '{{ repo.RootDir }}/licenses/ELASTIC-LICENSE.txt'
+        mode: 0644
+  #
+  # Binaries used to run the function.
+  #
+  - &beatless_binaries
+    files:
+      pkg/beatless:
+        source: 'build/golang-crossbuild/{{.BeatName}}-linux-amd64'
+        mode: 0755
+# specs is a list of named packaging "flavors".
+specs:
+  beatless:
+    ###
+    # Elastic Licensed Packages
+    ###
+    - os: windows
+      types: [zip]
+      spec:
+        <<: *binary_spec
+        <<: *beatless_binaries
+        <<: *elastic_license_for_binaries
+        '{{.BeatName}}{{.BinaryExt}}':
+          source: build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}}
+
+    - os: darwin
+      types: [tgz]
+      spec:
+        <<: *binary_spec
+        <<: *beatless_binaries
+        <<: *elastic_license_for_binaries
+
+    - os: linux
+      types: [tgz]
+      spec:
+        <<: *binary_spec
+        <<: *beatless_binaries
+        <<: *elastic_license_for_binaries
+        files:
+          '{{.BeatName}}{{.BinaryExt}}':
+            source: build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}}

x-pack/beatless/docs/index.asciidoc

@@ -1,5 +1,20 @@
-= Beatless Docs
+= Beatless Reference
 
-Welcome to the Beatless documentation.
+include::../../../libbeat/docs/version.asciidoc[]
 
+include::{asciidoc-dir}/../../shared/attributes.asciidoc[]
+
+:version: {stack-version}
+:beatname_lc: beatless
+:beatname_uc: Beatless
+:beatname_pkg: {beatname_lc}
+:github_repo_name: beats
+:discuss_forum: beats/{beatname_lc}
+:beat_default_index_prefix: {beatname_lc}
+:has_ml_jobs: no
+
+
+include::../../../libbeat/docs/shared-beats-attributes.asciidoc[]
+
+include::./overview.asciidoc[]
 

x-pack/beatless/docs/overview.asciidoc

@@ -0,0 +1,10 @@
+[id="{beatname_lc}-overview"]
+== {beatname_uc} overview
+
+++++
+<titleabbrev>Overview</titleabbrev>
+++++
+
+{beatname_uc} is a lightweight shipper that you can install as a function on
+a serverless provider.For example, you can use {beatname_uc} to collect and
+centralize log events from the Cloudwatchlogs to your Elasticsearch cluster.

x-pack/beatless/magefile.go

@@ -58,6 +58,8 @@ func Package() {
 	start := time.Now()
 	defer func() { fmt.Println("package ran for", time.Since(start)) }()
 
+	mage.MustUsePackaging("beatless", "x-pack/beatless/dev-tools/packaging/packages.yml")
+
 	mg.Deps(Update)
 	mg.Deps(CrossBuild, CrossBuildGoDaemon)
 	mg.SerialDeps(mage.Package, TestPackages)