diff --git a/dev-tools/packaging/packages.yml b/dev-tools/packaging/packages.yml index 283273178bc..d6d9026f055 100644 --- a/dev-tools/packaging/packages.yml +++ b/dev-tools/packaging/packages.yml @@ -60,6 +60,12 @@ shared: /etc/{{.BeatName}}/data/downloads/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz: source: '{{ elastic_beats_dir }}/x-pack/metricbeat/build/distributions/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz' mode: 0644 + /etc/{{.BeatName}}/data/downloads/filebeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512: + source: '{{ elastic_beats_dir }}/x-pack/filebeat/build/distributions/filebeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512' + mode: 0644 + /etc/{{.BeatName}}/data/downloads/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512: + source: '{{ elastic_beats_dir }}/x-pack/metricbeat/build/distributions/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512' + mode: 0644 # MacOS pkg spec for community beats. @@ -103,6 +109,12 @@ shared: /etc/{{.BeatName}}/data/downloads/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz: source: '{{ elastic_beats_dir }}/x-pack/metricbeat/build/distributions/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz' mode: 0644 + /etc/{{.BeatName}}/data/downloads/filebeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512: + source: '{{ elastic_beats_dir }}/x-pack/filebeat/build/distributions/filebeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512' + mode: 0644 + /etc/{{.BeatName}}/data/downloads/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512: + source: '{{ elastic_beats_dir }}/x-pack/metricbeat/build/distributions/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512' + mode: 0644 - &agent_binary_files '{{.BeatName}}{{.BinaryExt}}': @@ -137,6 +149,13 @@ shared: 'data/downloads/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz': source: '{{ elastic_beats_dir }}/x-pack/metricbeat/build/distributions/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz' mode: 0644 + <<: *agent_binary_files + 'data/downloads/filebeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512': + source: '{{ elastic_beats_dir }}/x-pack/filebeat/build/distributions/filebeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512' + mode: 0644 + 'data/downloads/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512': + source: '{{ elastic_beats_dir }}/x-pack/metricbeat/build/distributions/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512' + mode: 0644 # Binary package spec (zip for windows) for community beats. - &agent_windows_binary_spec @@ -155,6 +174,12 @@ shared: 'data/downloads/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip': source: '{{ elastic_beats_dir }}/x-pack/metricbeat/build/distributions/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip' mode: 0644 + 'data/downloads/filebeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip.sha512': + source: '{{ elastic_beats_dir }}/x-pack/filebeat/build/distributions/filebeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip.sha512' + mode: 0644 + 'data/downloads/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip.sha512': + source: '{{ elastic_beats_dir }}/x-pack/metricbeat/build/distributions/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip.sha512' + mode: 0644 - &agent_docker_spec <<: *agent_binary_spec diff --git a/x-pack/elastic-agent/CHANGELOG.asciidoc b/x-pack/elastic-agent/CHANGELOG.asciidoc index b4842c79788..c6d6c7af732 100644 --- a/x-pack/elastic-agent/CHANGELOG.asciidoc +++ b/x-pack/elastic-agent/CHANGELOG.asciidoc @@ -67,3 +67,4 @@ - Pick up version from libbeat {pull}18350[18350] - Use shorter hash for application differentiator {pull}18770[18770] - When not port are specified and the https is used fallback to 443 {pull}18844[18844] +- Agent verifies packages before using them {pull}18876[18876] diff --git a/x-pack/elastic-agent/pkg/agent/application/stream.go b/x-pack/elastic-agent/pkg/agent/application/stream.go index 5d8880a9e07..e53529c910b 100644 --- a/x-pack/elastic-agent/pkg/agent/application/stream.go +++ b/x-pack/elastic-agent/pkg/agent/application/stream.go @@ -78,6 +78,11 @@ func newOperator(ctx context.Context, log *logger.Logger, id routingKey, config } fetcher := downloader.NewDownloader(operatorConfig.DownloadConfig) + verifier, err := downloader.NewVerifier(operatorConfig.DownloadConfig) + if err != nil { + return nil, errors.New(err, "initiating verifier") + } + installer, err := install.NewInstaller(operatorConfig.DownloadConfig) if err != nil { return nil, errors.New(err, "initiating installer") @@ -94,6 +99,7 @@ func newOperator(ctx context.Context, log *logger.Logger, id routingKey, config id, config, fetcher, + verifier, installer, stateResolver, r, diff --git a/x-pack/elastic-agent/pkg/agent/operation/common_test.go b/x-pack/elastic-agent/pkg/agent/operation/common_test.go index 10a3aab90d1..4576674f735 100644 --- a/x-pack/elastic-agent/pkg/agent/operation/common_test.go +++ b/x-pack/elastic-agent/pkg/agent/operation/common_test.go @@ -46,6 +46,7 @@ func getTestOperator(t *testing.T, installPath string) (*Operator, *operatorCfg. l := getLogger() fetcher := &DummyDownloader{} + verifier := &DummyVerifier{} installer := &DummyInstaller{} stateResolver, err := stateresolver.NewStateResolver(l) @@ -53,7 +54,7 @@ func getTestOperator(t *testing.T, installPath string) (*Operator, *operatorCfg. t.Fatal(err) } - operator, err := NewOperator(context.Background(), l, "p1", cfg, fetcher, installer, stateResolver, nil, noop.NewMonitor()) + operator, err := NewOperator(context.Background(), l, "p1", cfg, fetcher, verifier, installer, stateResolver, nil, noop.NewMonitor()) if err != nil { t.Fatal(err) } @@ -81,8 +82,7 @@ type TestConfig struct { TestFile string } -type DummyDownloader struct { -} +type DummyDownloader struct{} func (*DummyDownloader) Download(_ context.Context, p, v string) (string, error) { return "", nil @@ -90,9 +90,16 @@ func (*DummyDownloader) Download(_ context.Context, p, v string) (string, error) var _ download.Downloader = &DummyDownloader{} -type DummyInstaller struct { +type DummyVerifier struct{} + +func (*DummyVerifier) Verify(p, v string) (bool, error) { + return true, nil } +var _ download.Verifier = &DummyVerifier{} + +type DummyInstaller struct{} + func (*DummyInstaller) Install(p, v, _ string) error { return nil } diff --git a/x-pack/elastic-agent/pkg/agent/operation/monitoring_test.go b/x-pack/elastic-agent/pkg/agent/operation/monitoring_test.go index 2db5aeb0bad..371ddbda308 100644 --- a/x-pack/elastic-agent/pkg/agent/operation/monitoring_test.go +++ b/x-pack/elastic-agent/pkg/agent/operation/monitoring_test.go @@ -116,6 +116,7 @@ func getMonitorableTestOperator(t *testing.T, installPath string, m monitoring.M l := getLogger() fetcher := &DummyDownloader{} + verifier := &DummyVerifier{} installer := &DummyInstaller{} stateResolver, err := stateresolver.NewStateResolver(l) @@ -124,7 +125,7 @@ func getMonitorableTestOperator(t *testing.T, installPath string, m monitoring.M } ctx := context.Background() - operator, err := NewOperator(ctx, l, "p1", cfg, fetcher, installer, stateResolver, nil, m) + operator, err := NewOperator(ctx, l, "p1", cfg, fetcher, verifier, installer, stateResolver, nil, m) if err != nil { t.Fatal(err) } diff --git a/x-pack/elastic-agent/pkg/agent/operation/operation_verify.go b/x-pack/elastic-agent/pkg/agent/operation/operation_verify.go index b0957ef2060..b3ffbee3dcd 100644 --- a/x-pack/elastic-agent/pkg/agent/operation/operation_verify.go +++ b/x-pack/elastic-agent/pkg/agent/operation/operation_verify.go @@ -6,18 +6,35 @@ package operation import ( "context" + "fmt" + "os" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/errors" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/operation/config" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/artifact" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/artifact/download" ) // operationVerify verifies downloaded artifact for correct signature // skips if artifact is already installed type operationVerify struct { eventProcessor callbackHooks + program Descriptor + operatorConfig *config.Config + verifier download.Verifier } -func newOperationVerify(eventProcessor callbackHooks) *operationVerify { - return &operationVerify{eventProcessor: eventProcessor} +func newOperationVerify( + program Descriptor, + operatorConfig *config.Config, + verifier download.Verifier, + eventProcessor callbackHooks) *operationVerify { + return &operationVerify{ + program: program, + operatorConfig: operatorConfig, + eventProcessor: eventProcessor, + verifier: verifier, + } } // Name is human readable name identifying an operation @@ -30,7 +47,18 @@ func (o *operationVerify) Name() string { // - Start does not need to run if process is running // - Fetch does not need to run if package is already present func (o *operationVerify) Check() (bool, error) { - return false, nil + downloadConfig := o.operatorConfig.DownloadConfig + fullPath, err := artifact.GetArtifactPath(o.program.BinaryName(), o.program.Version(), downloadConfig.OS(), downloadConfig.Arch(), downloadConfig.TargetDirectory) + if err != nil { + return false, err + } + + if _, err := os.Stat(fullPath); os.IsNotExist(err) { + return false, errors.New(errors.TypeApplication, + fmt.Sprintf("%s.%s package does not exist in %s. Skipping operation %s", o.program.BinaryName(), o.program.Version(), fullPath, o.Name())) + } + + return true, err } // Run runs the operation @@ -45,5 +73,18 @@ func (o *operationVerify) Run(ctx context.Context, application Application) (err } }() + isVerified, err := o.verifier.Verify(o.program.BinaryName(), o.program.Version()) + if err != nil { + return errors.New(err, + fmt.Sprintf("operation '%s' failed to verify %s.%s", o.Name(), o.program.BinaryName(), o.program.Version()), + errors.TypeSecurity) + } + + if !isVerified { + return errors.New(err, + fmt.Sprintf("operation '%s' marked '%s.%s' corrupted", o.Name(), o.program.BinaryName(), o.program.Version()), + errors.TypeSecurity) + } + return nil } diff --git a/x-pack/elastic-agent/pkg/agent/operation/operator.go b/x-pack/elastic-agent/pkg/agent/operation/operator.go index b0c7d42c886..221750bf156 100644 --- a/x-pack/elastic-agent/pkg/agent/operation/operator.go +++ b/x-pack/elastic-agent/pkg/agent/operation/operator.go @@ -51,6 +51,7 @@ type Operator struct { appsLock sync.Mutex downloader download.Downloader + verifier download.Verifier installer install.Installer } @@ -63,6 +64,7 @@ func NewOperator( pipelineID string, config *config.Config, fetcher download.Downloader, + verifier download.Verifier, installer install.Installer, stateResolver *stateresolver.StateResolver, eventProcessor callbackHooks, @@ -87,6 +89,7 @@ func NewOperator( pipelineID: pipelineID, logger: logger, downloader: fetcher, + verifier: verifier, installer: installer, stateResolver: stateResolver, apps: make(map[string]Application), @@ -155,7 +158,7 @@ func (o *Operator) HandleConfig(cfg configrequest.Request) error { func (o *Operator) start(p Descriptor, cfg map[string]interface{}) (err error) { flow := []operation{ newOperationFetch(o.logger, p, o.config, o.downloader, o.eventProcessor), - newOperationVerify(o.eventProcessor), + newOperationVerify(p, o.config, o.verifier, o.eventProcessor), newOperationInstall(o.logger, p, o.config, o.installer, o.eventProcessor), newOperationStart(o.logger, p, o.config, cfg, o.eventProcessor), newOperationConfig(o.logger, o.config, cfg, o.eventProcessor), diff --git a/x-pack/elastic-agent/pkg/artifact/download/fs/downloader.go b/x-pack/elastic-agent/pkg/artifact/download/fs/downloader.go index cbbb0c2319a..1f4c2183883 100644 --- a/x-pack/elastic-agent/pkg/artifact/download/fs/downloader.go +++ b/x-pack/elastic-agent/pkg/artifact/download/fs/downloader.go @@ -50,8 +50,10 @@ func (e *Downloader) Download(_ context.Context, programName, version string) (s path, err := e.download(e.config.OS(), programName, version) if err != nil { os.Remove(path) + return "", err } + _, err = e.downloadHash(e.config.OS(), programName, version) return path, err } @@ -66,6 +68,27 @@ func (e *Downloader) download(operatingSystem, programName, version string) (str return "", errors.New(err, "generating package path failed") } + return e.downloadFile(filename, fullPath) +} + +func (e *Downloader) downloadHash(operatingSystem, programName, version string) (string, error) { + filename, err := artifact.GetArtifactName(programName, version, operatingSystem, e.config.Arch()) + if err != nil { + return "", errors.New(err, "generating package name failed") + } + + fullPath, err := artifact.GetArtifactPath(programName, version, operatingSystem, e.config.Arch(), e.config.TargetDirectory) + if err != nil { + return "", errors.New(err, "generating package path failed") + } + + filename = filename + ".sha512" + fullPath = fullPath + ".sha512" + + return e.downloadFile(filename, fullPath) +} + +func (e *Downloader) downloadFile(filename, fullPath string) (string, error) { sourcePath := filepath.Join(e.dropPath, filename) sourceFile, err := os.Open(sourcePath) if err != nil { diff --git a/x-pack/elastic-agent/pkg/artifact/download/fs/verifier.go b/x-pack/elastic-agent/pkg/artifact/download/fs/verifier.go index 1c06dc4b033..85f1813c992 100644 --- a/x-pack/elastic-agent/pkg/artifact/download/fs/verifier.go +++ b/x-pack/elastic-agent/pkg/artifact/download/fs/verifier.go @@ -5,11 +5,16 @@ package fs import ( + "bufio" "bytes" + "crypto/sha512" "fmt" + "io" "io/ioutil" "os" "path/filepath" + "strings" + "sync" "golang.org/x/crypto/openpgp" @@ -35,10 +40,6 @@ func NewVerifier(config *artifact.Config) (*Verifier, error) { config: config, } - if err := v.loadPGP(config.PgpFile); err != nil { - return nil, errors.New(err, "loading PGP") - } - return v, nil } @@ -52,6 +53,63 @@ func (v *Verifier) Verify(programName, version string) (bool, error) { fullPath := filepath.Join(v.config.TargetDirectory, filename) + return v.verifyHash(filename, fullPath) +} + +func (v *Verifier) verifyHash(filename, fullPath string) (bool, error) { + hashFilePath := fullPath + ".sha512" + hashFileHandler, err := os.Open(hashFilePath) + if err != nil { + return false, err + } + defer hashFileHandler.Close() + + // get hash + // content of a file is in following format + // hash filename + var expectedHash string + scanner := bufio.NewScanner(hashFileHandler) + for scanner.Scan() { + line := scanner.Text() + if !strings.HasSuffix(line, filename) { + continue + } + + expectedHash = strings.TrimSpace(strings.TrimSuffix(line, filename)) + } + + if expectedHash == "" { + return false, fmt.Errorf("hash for '%s' not found", filename) + } + + // compute file hash + fileReader, err := os.OpenFile(fullPath, os.O_RDONLY, 0666) + if err != nil { + return false, errors.New(err, errors.TypeFilesystem, errors.M(errors.MetaKeyPath, fullPath)) + } + defer fileReader.Close() + + hash := sha512.New() + if _, err := io.Copy(hash, fileReader); err != nil { + return false, err + } + computedHash := fmt.Sprintf("%x", hash.Sum(nil)) + + return expectedHash == computedHash, nil +} + +func (v *Verifier) verifyAsc(filename, fullPath string) (bool, error) { + var err error + var pgpBytesLoader sync.Once + + pgpBytesLoader.Do(func() { + err = v.loadPGP(v.config.PgpFile) + }) + + if err != nil { + return false, errors.New(err, "loading PGP") + } + ascBytes, err := v.getPublicAsc(filename) if err != nil { return false, err diff --git a/x-pack/elastic-agent/pkg/artifact/download/fs/verifier_test.go b/x-pack/elastic-agent/pkg/artifact/download/fs/verifier_test.go new file mode 100644 index 00000000000..5cf98fcc244 --- /dev/null +++ b/x-pack/elastic-agent/pkg/artifact/download/fs/verifier_test.go @@ -0,0 +1,100 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package fs + +import ( + "context" + "crypto/sha512" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "testing" + "time" + + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/artifact" +) + +const ( + beatName = "filebeat" + version = "7.5.1" + sourcePattern = "/downloads/beats/filebeat/" +) + +type testCase struct { + system string + arch string +} + +func TestVerify(t *testing.T) { + targetDir, err := ioutil.TempDir(os.TempDir(), "") + if err != nil { + t.Fatal(err) + } + + timeout := 30 * time.Second + + config := &artifact.Config{ + TargetDirectory: targetDir, + DropPath: filepath.Join(targetDir, "drop"), + Timeout: timeout, + OperatingSystem: "linux", + Architecture: "32", + } + + if err := prepareTestCase(beatName, version, config); err != nil { + t.Fatal(err) + } + + testClient := NewDownloader(config) + artifact, err := testClient.Download(context.Background(), beatName, version) + if err != nil { + t.Fatal(err) + } + + _, err = os.Stat(artifact) + if err != nil { + t.Fatal(err) + } + + testVerifier, err := NewVerifier(config) + if err != nil { + t.Fatal(err) + } + + isOk, err := testVerifier.Verify(beatName, version) + if err != nil { + t.Fatal(err) + } + + if !isOk { + t.Fatal("verify failed") + } + + os.Remove(artifact) + os.Remove(artifact + ".sha512") + os.RemoveAll(config.DropPath) +} + +func prepareTestCase(beatName, version string, cfg *artifact.Config) error { + filename, err := artifact.GetArtifactName(beatName, version, cfg.OperatingSystem, cfg.Architecture) + if err != nil { + return err + } + + if err := os.MkdirAll(cfg.DropPath, 0777); err != nil { + return err + } + + content := []byte("sample content") + hash := sha512.Sum512(content) + hashContent := fmt.Sprintf("%x %s", hash, filename) + + if err := ioutil.WriteFile(filepath.Join(cfg.DropPath, filename), []byte(content), 0644); err != nil { + return err + } + + return ioutil.WriteFile(filepath.Join(cfg.DropPath, filename+".sha512"), []byte(hashContent), 0644) +} diff --git a/x-pack/elastic-agent/pkg/artifact/download/http/downloader.go b/x-pack/elastic-agent/pkg/artifact/download/http/downloader.go index 7429f3fc137..e882a2194de 100644 --- a/x-pack/elastic-agent/pkg/artifact/download/http/downloader.go +++ b/x-pack/elastic-agent/pkg/artifact/download/http/downloader.go @@ -56,8 +56,10 @@ func (e *Downloader) Download(ctx context.Context, programName, version string) path, err := e.download(ctx, e.config.OS(), programName, version) if err != nil { os.Remove(path) + return "", err } + _, err = e.downloadHash(ctx, e.config.OS(), programName, version) return path, err } @@ -89,6 +91,27 @@ func (e *Downloader) download(ctx context.Context, operatingSystem, programName, return "", errors.New(err, "generating package path failed") } + return e.downloadFile(ctx, programName, filename, fullPath) +} + +func (e *Downloader) downloadHash(ctx context.Context, operatingSystem, programName, version string) (string, error) { + filename, err := artifact.GetArtifactName(programName, version, operatingSystem, e.config.Arch()) + if err != nil { + return "", errors.New(err, "generating package name failed") + } + + fullPath, err := artifact.GetArtifactPath(programName, version, operatingSystem, e.config.Arch(), e.config.TargetDirectory) + if err != nil { + return "", errors.New(err, "generating package path failed") + } + + filename = filename + ".sha512" + fullPath = fullPath + ".sha512" + + return e.downloadFile(ctx, programName, filename, fullPath) +} + +func (e *Downloader) downloadFile(ctx context.Context, programName, filename, fullPath string) (string, error) { sourceURI, err := e.composeURI(programName, filename) if err != nil { return "", err diff --git a/x-pack/elastic-agent/pkg/artifact/download/http/elastic_test.go b/x-pack/elastic-agent/pkg/artifact/download/http/elastic_test.go index ba0f1cffbda..8087fc28823 100644 --- a/x-pack/elastic-agent/pkg/artifact/download/http/elastic_test.go +++ b/x-pack/elastic-agent/pkg/artifact/download/http/elastic_test.go @@ -6,6 +6,7 @@ package http import ( "context" + "crypto/sha512" "fmt" "io/ioutil" "math/rand" @@ -13,6 +14,7 @@ import ( "net/http" "net/http/httptest" "os" + "strings" "testing" "time" @@ -70,9 +72,6 @@ func TestDownload(t *testing.T) { } func TestVerify(t *testing.T) { - // skip so beats are not fetched from upstream, test only locally when change is made - t.Skip() - targetDir, err := ioutil.TempDir(os.TempDir(), "") if err != nil { t.Fatal(err) @@ -80,6 +79,7 @@ func TestVerify(t *testing.T) { timeout := 30 * time.Second testCases := getRandomTestCases() + elasticClient := getElasticCoClient() config := &artifact.Config{ BeatsSourceURI: source, @@ -93,8 +93,7 @@ func TestVerify(t *testing.T) { config.OperatingSystem = testCase.system config.Architecture = testCase.arch - testClient := NewDownloader(config) - + testClient := NewDownloaderWithClient(config, elasticClient) artifact, err := testClient.Download(context.Background(), beatName, version) if err != nil { t.Fatal(err) @@ -120,6 +119,7 @@ func TestVerify(t *testing.T) { } os.Remove(artifact) + os.Remove(artifact + ".sha512") }) } } @@ -164,11 +164,20 @@ func getElasticCoClient() http.Client { handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { packageName := r.URL.Path[len(sourcePattern):] + isShaReq := strings.HasSuffix(packageName, ".sha512") + packageName = strings.TrimSuffix(packageName, ".sha512") + if _, ok := correctValues[packageName]; !ok { w.WriteHeader(http.StatusInternalServerError) } - w.Write([]byte(packageName)) + content := []byte(packageName) + if isShaReq { + hash := sha512.Sum512(content) + w.Write([]byte(fmt.Sprintf("%x %s", hash, packageName))) + } else { + w.Write(content) + } }) server := httptest.NewServer(handler) diff --git a/x-pack/elastic-agent/pkg/artifact/download/http/verifier.go b/x-pack/elastic-agent/pkg/artifact/download/http/verifier.go index f1f6475c1b6..4c08b5049e8 100644 --- a/x-pack/elastic-agent/pkg/artifact/download/http/verifier.go +++ b/x-pack/elastic-agent/pkg/artifact/download/http/verifier.go @@ -5,14 +5,18 @@ package http import ( + "bufio" "bytes" + "crypto/sha512" "fmt" + "io" "io/ioutil" "net/http" "net/url" "os" "path" "strings" + "sync" "golang.org/x/crypto/openpgp" @@ -44,16 +48,82 @@ func NewVerifier(config *artifact.Config) (*Verifier, error) { client: client, } - if err := v.loadPGP(config.PgpFile); err != nil { - return nil, errors.New(err, "loading PGP") - } - return v, nil } // Verify checks downloaded package on preconfigured // location agains a key stored on elastic.co website. func (v *Verifier) Verify(programName, version string) (bool, error) { + // TODO: think about verifying asc for prepacked beats + + filename, err := artifact.GetArtifactName(programName, version, v.config.OS(), v.config.Arch()) + if err != nil { + return false, errors.New(err, "retrieving package name") + } + + fullPath, err := artifact.GetArtifactPath(programName, version, v.config.OS(), v.config.Arch(), v.config.TargetDirectory) + if err != nil { + return false, errors.New(err, "retrieving package path") + } + + return v.verifyHash(filename, fullPath) +} + +func (v *Verifier) verifyHash(filename, fullPath string) (bool, error) { + hashFilePath := fullPath + ".sha512" + hashFileHandler, err := os.Open(hashFilePath) + if err != nil { + return false, err + } + defer hashFileHandler.Close() + + // get hash + // content of a file is in following format + // hash filename + var expectedHash string + scanner := bufio.NewScanner(hashFileHandler) + for scanner.Scan() { + line := scanner.Text() + if !strings.HasSuffix(line, filename) { + continue + } + + expectedHash = strings.TrimSpace(strings.TrimSuffix(line, filename)) + } + + if expectedHash == "" { + return false, fmt.Errorf("hash for '%s' not found", filename) + } + + // compute file hash + fileReader, err := os.OpenFile(fullPath, os.O_RDONLY, 0666) + if err != nil { + return false, errors.New(err, errors.TypeFilesystem, errors.M(errors.MetaKeyPath, fullPath)) + } + defer fileReader.Close() + + // compute file hash + hash := sha512.New() + if _, err := io.Copy(hash, fileReader); err != nil { + return false, err + } + computedHash := fmt.Sprintf("%x", hash.Sum(nil)) + + return expectedHash == computedHash, nil +} + +func (v *Verifier) verifyAsc(programName, version string) (bool, error) { + var err error + var pgpBytesLoader sync.Once + + pgpBytesLoader.Do(func() { + err = v.loadPGP(v.config.PgpFile) + }) + + if err != nil { + return false, errors.New(err, "loading PGP") + } + filename, err := artifact.GetArtifactName(programName, version, v.config.OS(), v.config.Arch()) if err != nil { return false, errors.New(err, "retrieving package name") @@ -92,6 +162,7 @@ func (v *Verifier) Verify(programName, version string) (bool, error) { } return true, nil + } func (v *Verifier) composeURI(programName, filename string) (string, error) {