Skip to content

Commit

Permalink
Cherry-pick elastic#16927 to 7.x: Fix issue when building Agent snaps…
Browse files Browse the repository at this point in the history
…hot and enable building the agent. (elastic#17574)

* Fix issue when building Agent snapshot and enable building the agent. (elastic#16927)

An extension in the generated files ("SNAPSHOT") was preventing us to
correctly generate the agent artifact with the packaged
metricbeat/filebeat.

Fixes: elastic#16908
(cherry picked from commit 62d3683)

* [Agent] Fix installers (elastic#17077)

[Agent] Fix installers for snapshots (elastic#17077)

* [Agent] fixed mage package for agent (elastic#17368)

[Agent] fixed mage package for agent (elastic#17368)

* [Agent] Make default config OS agnostic (elastic#17016)

[Agent] Make default config OS agnostic (elastic#17016)

* [Agent] Docker image for integration tests (elastic#16898)

[Agent] Docker image for integration tests (elastic#16898)

Co-authored-by: Michal Pristas <michal.pristas@gmail.com>
  • Loading branch information
ph and michalpristas authored Apr 7, 2020
1 parent 0921532 commit 284b202
Show file tree
Hide file tree
Showing 27 changed files with 337 additions and 84 deletions.
2 changes: 1 addition & 1 deletion Makefile
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
BUILD_DIR=$(CURDIR)/build
COVERAGE_DIR=$(BUILD_DIR)/coverage
BEATS?=auditbeat filebeat heartbeat journalbeat metricbeat packetbeat winlogbeat x-pack/functionbeat
BEATS?=auditbeat filebeat heartbeat journalbeat metricbeat packetbeat winlogbeat x-pack/functionbeat x-pack/agent
PROJECTS=libbeat $(BEATS)
PROJECTS_ENV=libbeat filebeat metricbeat
PYTHON_ENV?=$(BUILD_DIR)/python-env
Expand Down
1 change: 1 addition & 0 deletions dev-tools/mage/crossbuild.go
Original file line number Diff line number Diff line change
Expand Up @@ -268,6 +268,7 @@ func (b GolangCrossBuilder) Build() error {
"--rm",
"--env", "MAGEFILE_VERBOSE="+verbose,
"--env", "MAGEFILE_TIMEOUT="+EnvOr("MAGEFILE_TIMEOUT", ""),
"--env", fmt.Sprintf("SNAPSHOT=%v", Snapshot),
"-v", repoInfo.RootDir+":"+mountPoint,
"-w", workDir,
image,
Expand Down
1 change: 1 addition & 0 deletions dev-tools/mage/dmgbuilder.go
Original file line number Diff line number Diff line change
Expand Up @@ -244,6 +244,7 @@ func (b *dmgBuilder) buildDMG() error {
"create",
"-volname", b.MustExpand("{{.BeatName | title}} {{.Version}}{{if .Snapshot}}-SNAPSHOT{{end}}"),
"-srcfolder", b.dmgDir,
"-size", "500m", // allowing extra space
"-ov",
createDir(dmgFile),
}
Expand Down
50 changes: 48 additions & 2 deletions dev-tools/mage/dockerbuilder.go
Original file line number Diff line number Diff line change
Expand Up @@ -120,8 +120,8 @@ func (b *dockerBuilder) prepareBuild() error {
"ModulesDirs": b.modulesDirs(),
}

return filepath.Walk(templatesDir, func(path string, info os.FileInfo, _ error) error {
if !info.IsDir() {
err = filepath.Walk(templatesDir, func(path string, info os.FileInfo, _ error) error {
if !info.IsDir() && !isDockerFile(path) {
target := strings.TrimSuffix(
filepath.Join(b.buildDir, filepath.Base(path)),
".tmpl",
Expand All @@ -134,6 +134,52 @@ func (b *dockerBuilder) prepareBuild() error {
}
return nil
})

if err != nil {
return err
}

return b.expandDockerfile(templatesDir, data)
}

func isDockerFile(path string) bool {
path = filepath.Base(path)
return strings.HasPrefix(path, "Dockerfile") || strings.HasPrefix(path, "docker-entrypoint")
}

func (b *dockerBuilder) expandDockerfile(templatesDir string, data map[string]interface{}) error {
// has specific dockerfile
dockerfile := fmt.Sprintf("Dockerfile.%s.tmpl", b.imageName)
_, err := os.Stat(filepath.Join(templatesDir, dockerfile))
if err != nil {
// specific missing fallback to generic
dockerfile = "Dockerfile.tmpl"
}

entrypoint := fmt.Sprintf("docker-entrypoint.%s.tmpl", b.imageName)
_, err = os.Stat(filepath.Join(templatesDir, entrypoint))
if err != nil {
// specific missing fallback to generic
entrypoint = "docker-entrypoint.tmpl"
}

type fileExpansion struct {
source string
target string
}
for _, file := range []fileExpansion{{dockerfile, "Dockerfile.tmpl"}, {entrypoint, "docker-entrypoint.tmpl"}} {
target := strings.TrimSuffix(
filepath.Join(b.buildDir, file.target),
".tmpl",
)
path := filepath.Join(templatesDir, file.source)
err = b.ExpandFile(path, target, data)
if err != nil {
return errors.Wrapf(err, "expanding template '%s' to '%s'", path, target)
}
}

return nil
}

func (b *dockerBuilder) dockerBuild() (string, error) {
Expand Down
34 changes: 17 additions & 17 deletions dev-tools/packaging/packages.yml
Original file line number Diff line number Diff line change
Expand Up @@ -54,11 +54,11 @@ shared:
/etc/init.d/{{.BeatServiceName}}:
template: '{{ elastic_beats_dir }}/dev-tools/packaging/templates/{{.PackageType}}/init.sh.tmpl'
mode: 0755
/etc/{{.BeatName}}/beats/filebeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.tar.gz:
source: '{{ elastic_beats_dir }}/x-pack/filebeat/build/distributions/filebeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.tar.gz'
/etc/{{.BeatName}}/data/downloads/filebeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.tar.gz:
source: '{{ elastic_beats_dir }}/x-pack/filebeat/build/distributions/filebeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz'
mode: 0644
/etc/{{.BeatName}}/beats/metricbeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.tar.gz:
source: '{{ elastic_beats_dir }}/x-pack/metricbeat/build/distributions/metricbeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.tar.gz'
/etc/{{.BeatName}}/data/downloads/metricbeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.tar.gz:
source: '{{ elastic_beats_dir }}/x-pack/metricbeat/build/distributions/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz'
mode: 0644


Expand Down Expand Up @@ -97,11 +97,11 @@ shared:
source: 'agent.yml'
mode: 0600
config: true
/etc/{{.BeatName}}/beats/filebeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.tar.gz:
source: '{{ elastic_beats_dir }}/x-pack/filebeat/build/distributions/filebeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.tar.gz'
/etc/{{.BeatName}}/data/downloads/filebeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.tar.gz:
source: '{{ elastic_beats_dir }}/x-pack/filebeat/build/distributions/filebeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz'
mode: 0644
/etc/{{.BeatName}}/beats/metricbeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.tar.gz:
source: '{{ elastic_beats_dir }}/x-pack/metricbeat/build/distributions/metricbeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.tar.gz'
/etc/{{.BeatName}}/data/downloads/metricbeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.tar.gz:
source: '{{ elastic_beats_dir }}/x-pack/metricbeat/build/distributions/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz'
mode: 0644

- &agent_binary_files
Expand Down Expand Up @@ -131,11 +131,11 @@ shared:
<<: *common
files:
<<: *agent_binary_files
'beats/filebeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.tar.gz':
source: '{{ elastic_beats_dir }}/x-pack/filebeat/build/distributions/filebeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.tar.gz'
'data/downloads/filebeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.tar.gz':
source: '{{ elastic_beats_dir }}/x-pack/filebeat/build/distributions/filebeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz'
mode: 0644
'beats/metricbeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.tar.gz':
source: '{{ elastic_beats_dir }}/x-pack/metricbeat/build/distributions/metricbeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.tar.gz'
'data/downloads/metricbeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.tar.gz':
source: '{{ elastic_beats_dir }}/x-pack/metricbeat/build/distributions/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz'
mode: 0644

# Binary package spec (zip for windows) for community beats.
Expand All @@ -149,18 +149,18 @@ shared:
uninstall-service-{{.BeatName}}.ps1:
template: '{{ elastic_beats_dir }}/dev-tools/packaging/templates/windows/uninstall-service.ps1.tmpl'
mode: 0755
'beats/filebeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.zip':
source: '{{ elastic_beats_dir }}/x-pack/filebeat/build/distributions/filebeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.zip'
'data/downloads/filebeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.zip':
source: '{{ elastic_beats_dir }}/x-pack/filebeat/build/distributions/filebeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip'
mode: 0644
'beats/metricbeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.zip':
source: '{{ elastic_beats_dir }}/x-pack/metricbeat/build/distributions/metricbeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.zip'
'data/downloads/metricbeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.zip':
source: '{{ elastic_beats_dir }}/x-pack/metricbeat/build/distributions/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip'
mode: 0644

- &agent_docker_spec
<<: *agent_binary_spec
extra_vars:
from: 'centos:7'
user: '{{ .BeatName }}'
user: 'root'
linux_capabilities: ''
files:
'agent.yml':
Expand Down
52 changes: 52 additions & 0 deletions dev-tools/packaging/templates/docker/Dockerfile.agent.tmpl
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
{{- $beatHome := printf "%s/%s" "/usr/share" .BeatName }}
{{- $beatBinary := printf "%s/%s" $beatHome .BeatName }}
{{- $repoInfo := repo }}

FROM {{ .from }}

LABEL \
org.label-schema.build-date="{{ date }}" \
org.label-schema.schema-version="1.0" \
org.label-schema.vendor="{{ .BeatVendor }}" \
org.label-schema.license="{{ .License }}" \
org.label-schema.name="{{ .BeatName }}" \
org.label-schema.version="{{ beat_version }}" \
org.label-schema.url="{{ .BeatURL }}" \
org.label-schema.vcs-url="{{ $repoInfo.RootImportPath }}" \
org.label-schema.vcs-ref="{{ commit }}" \
license="{{ .License }}" \
description="{{ .BeatDescription }}"

ENV ELASTIC_CONTAINER "true"
ENV PATH={{ $beatHome }}:$PATH

COPY beat {{ $beatHome }}
COPY docker-entrypoint /usr/local/bin/docker-entrypoint
RUN chmod 755 /usr/local/bin/docker-entrypoint

RUN groupadd --gid 1000 {{ .BeatName }}

RUN mkdir -p {{ $beatHome }}/data {{ $beatHome }}/logs && \
chown -R root:{{ .BeatName }} {{ $beatHome }} && \
find {{ $beatHome }} -type d -exec chmod 0750 {} \; && \
find {{ $beatHome }} -type f -exec chmod 0640 {} \; && \
chmod 0750 {{ $beatBinary }} && \
{{- if .linux_capabilities }}
setcap {{ .linux_capabilities }} {{ $beatBinary }} && \
{{- end }}
{{- range $i, $modulesd := .ModulesDirs }}
chmod 0770 {{ $beatHome}}/{{ $modulesd }} && \
{{- end }}
chmod 0770 {{ $beatHome }}/data {{ $beatHome }}/logs

{{- if ne .user "root" }}
RUN useradd -M --uid 1000 --gid 1000 --home {{ $beatHome }} {{ .user }}
{{- end }}
USER {{ .user }}

{{- range $i, $port := .ExposePorts }}
EXPOSE {{ $port }}
{{- end }}

WORKDIR {{ $beatHome }}
ENTRYPOINT ["/usr/local/bin/docker-entrypoint"]
55 changes: 55 additions & 0 deletions dev-tools/packaging/templates/docker/docker-entrypoint.agent.tmpl
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
#!/bin/bash

set -eo pipefail

# Environment variables used
# FLEET_ADMIN_PASSWORD - used for new fleet user [elastic]
# FLEET_ADMIN_USERNAME - used for new fleet user [changeme]
# FLEET_CONFIG_ID - config related to new token [defaul]
# FLEET_ENROLLMENT_TOKEN - existing enrollment token to be used for enroll
# FLEET_ENROLL - if set to 1 enroll will be performed
# FLEET_SETUP - if set to 1 fleet setup will be performed
# FLEET_TOKEN_NAME - token name for a token to be created
# KIBANA_HOST - actual kibana host [http://localhost:5601]
# KIBANA_PASSWORD - password for accessing kibana API [changeme]
# KIBANA_USERNAME - username for accessing kibana API [elastic]

function setup(){
curl -X POST ${KIBANA_HOST:-http://localhost:5601}/api/ingest_manager/setup -H 'kbn-xsrf: true' -u ${KIBANA_USERNAME:-elastic}:${KIBANA_PASSWORD:-changeme}
curl -X POST ${KIBANA_HOST:-http://localhost:5601}/api/ingest_manager/fleet/setup \
-H 'Content-Type: application/json' \
-H 'kbn-xsrf: true' \
-d '{"admin_username":"'"${FLEET_ADMIN_USERNAME:-elastic}"'","admin_password":"'"${FLEET_ADMIN_PASSWORD:-changeme}"'"}' \
-u ${KIBANA_USERNAME:-elastic}:${KIBANA_PASSWORD:-changeme}
}

function enroll(){
local enrollResp
local apiKey

if [[ -n "${FLEET_ENROLLMENT_TOKEN}" ]] && [[ ${FLEET_ENROLLMENT_TOKEN} == 1 ]]; then
apikey = "${FLEET_ENROLLMENT_TOKEN}"
else
enrollResp=$(curl -X POST ${KIBANA_HOST:-http://localhost:5601}/api/ingest_manager/fleet/enrollment-api-keys \
-H 'Content-Type: application/json' \
-H 'kbn-xsrf: true' \
-u ${KIBANA_USERNAME:-elastic}:${KIBANA_PASSWORD:-changeme} \
-d '{"name":"'"${FLEET_TOKEN_NAME:-demotoken}"'","config_id":"'"${FLEET_CONFIG_ID:-default}"'"}')

local exitCode=$?
if [ $exitCode -ne 0 ]; then
exit $exitCode
fi

apikey=$(echo $enrollResp | jq -r '.item.api_key')
fi

./{{ .BeatName }} enroll ${KIBANA_HOST:-http://localhost:5601} $apikey -f
}
yum install -y epel-release
yum install -y jq

if [[ -n "${FLEET_SETUP}" ]] && [[ ${FLEET_SETUP} == 1 ]]; then setup; fi
if [[ -n "${FLEET_ENROLL}" ]] && [[ ${FLEET_ENROLL} == 1 ]]; then enroll; fi

exec {{ .BeatName }} run "$@"
21 changes: 21 additions & 0 deletions x-pack/agent/CHANGELOG.asciidoc
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
// Use these for links to issue and pulls. Note issues and pulls redirect one to
// each other on Github, so don't worry too much on using the right prefix.
:issue: https://github.com/elastic/beats/issues/
:pull: https://github.com/elastic/beats/pull/


[[release-notes-8.0.0]]
=== Agent version 8.0.0


==== Breaking changes

==== Bugfixes

- Fixed tests on windows {pull}16922[16922]
- Fixed installers for SNAPSHOTs and windows {pull}17077[17077]

==== New features

- Generate index name in a format type-dataset-namespace {pull}16903[16903]
- OS agnostic default configuration {pull}17016[17016]
12 changes: 6 additions & 6 deletions x-pack/agent/_meta/agent.docker.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,9 @@
outputs:
default:
type: elasticsearch
hosts: [127.0.0.1:9200, 127.0.0.1:9300]
username: elastic
password: changeme
hosts: '${ELASTICSEARCH_HOSTS:http://elasticsearch:9200}'
username: '${ELASTICSEARCH_USERNAME:elastic}'
password: '${ELASTICSEARCH_PASSWORD:changeme}'

streams:
- type: event/file
Expand Down Expand Up @@ -44,15 +44,15 @@ download:
# e.g /windows-x86.zip
sourceURI: "https://artifacts.elastic.co/downloads/beats/"
# path to the directory containing downloaded packages
target_directory: "/home/elastic/downloads"
target_directory: "${path.data}/downloads"
# timeout for downloading package
timeout: 30s
# file path to a public key used for verifying downloaded artifacts
# if not file is present agent will try to load public key from elastic.co website.
pgpfile: "/home/elastic/elastic.pgp"
pgpfile: "${path.data}/elastic.pgp"
# install_path describes the location of installed packages/programs. It is also used
# for reading program specifications.
install_path: "/home/elastic/install"
install_path: "${path.data}/install"

process:
# minimal port number for spawned processes
Expand Down
6 changes: 3 additions & 3 deletions x-pack/agent/_meta/agent.fleet.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,15 +10,15 @@ download:
# e.g /windows-x86.zip
sourceURI: "https://artifacts.elastic.co/downloads/beats/"
# path to the directory containing downloaded packages
target_directory: "/home/elastic/downloads"
target_directory: "${path.data}/downloads"
# timeout for downloading package
timeout: 30s
# file path to a public key used for verifying downloaded artifacts
# if not file is present agent will try to load public key from elastic.co website.
pgpfile: "/home/elastic/elastic.pgp"
pgpfile: "${path.data}/elastic.pgp"
# install_path describes the location of installed packages/programs. It is also used
# for reading program specifications.
install_path: "/home/elastic/install"
install_path: "${path.data}/install"

process:
# minimal port number for spawned processes
Expand Down
8 changes: 4 additions & 4 deletions x-pack/agent/_meta/agent.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
outputs:
default:
type: elasticsearch
hosts: [127.0.0.1:9200, 127.0.0.1:9300]
hosts: [127.0.0.1:9200]
username: elastic
password: changeme

Expand Down Expand Up @@ -45,15 +45,15 @@ download:
# e.g /windows-x86.zip
sourceURI: "https://artifacts.elastic.co/downloads/beats/"
# path to the directory containing downloaded packages
target_directory: "/home/elastic/downloads"
target_directory: "${path.data}/downloads"
# timeout for downloading package
timeout: 30s
# file path to a public key used for verifying downloaded artifacts
# if not file is present agent will try to load public key from elastic.co website.
pgpfile: "/home/elastic/elastic.pgp"
pgpfile: "${path.data}/elastic.pgp"
# install_path describes the location of installed packages/programs. It is also used
# for reading program specifications.
install_path: "/home/elastic/install"
install_path: "${path.data}/install"

process:
# minimal port number for spawned processes
Expand Down
8 changes: 4 additions & 4 deletions x-pack/agent/_meta/common.p2.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
outputs:
default:
type: elasticsearch
hosts: [127.0.0.1:9200, 127.0.0.1:9300]
hosts: [127.0.0.1:9200]
username: elastic
password: changeme

Expand Down Expand Up @@ -70,15 +70,15 @@ download:
# e.g /windows-x86.zip
sourceURI: "https://artifacts.elastic.co/downloads/beats/"
# path to the directory containing downloaded packages
target_directory: "/home/elastic/downloads"
target_directory: "${path.data}/downloads"
# timeout for downloading package
timeout: 30s
# file path to a public key used for verifying downloaded artifacts
# if not file is present agent will try to load public key from elastic.co website.
pgpfile: "/home/elastic/elastic.pgp"
pgpfile: "${path.data}/elastic.pgp"
# install_path describes the location of installed packages/programs. It is also used
# for reading program specifications.
install_path: "/home/elastic/install"
install_path: "${path.data}/install"

process:
# minimal port number for spawned processes
Expand Down
Loading

0 comments on commit 284b202

Please sign in to comment.