Test modules and modules.d folder contents in resulting packages (ela…

…stic#8107)

This change ensures that `modules` (Filebeat) and `modules.d` (Filebeat & Metricbeat) folders contain files in the right place.

Permissions and ownership were already tested

Part of elastic#7488

dev-tools/mage/pkg.go

@@ -96,16 +96,52 @@ func (b packageBuilder) Build() error {
 		b.Spec.Name, b.Type, b.Platform.Name)
 }
 
+type testPackagesParams struct {
+	HasModules  bool
+	HasModulesD bool
+}
+
+// TestPackagesOption defines a option to the TestPackages target.
+type TestPackagesOption func(params *testPackagesParams)
+
+// WithModules enables modules folder contents testing
+func WithModules() func(params *testPackagesParams) {
+	return func(params *testPackagesParams) {
+		params.HasModules = true
+	}
+}
+
+// WithModulesD enables modules.d folder contents testing
+func WithModulesD() func(params *testPackagesParams) {
+	return func(params *testPackagesParams) {
+		params.HasModulesD = true
+	}
+}
+
 // TestPackages executes the package tests on the produced binaries. These tests
 // inspect things like file ownership and mode.
-func TestPackages() error {
+func TestPackages(options ...TestPackagesOption) error {
+	params := testPackagesParams{}
+	for _, opt := range options {
+		opt(&params)
+	}
+
 	fmt.Println(">> Testing package contents")
 	goTest := sh.OutCmd("go", "test")
 
 	var args []string
 	if mg.Verbose() {
 		args = append(args, "-v")
 	}
+
+	if params.HasModules {
+		args = append(args, "--modules")
+	}
+
+	if params.HasModulesD {
+		args = append(args, "--modules.d")
+	}
+
 	args = append(args,
 		MustExpand("{{ elastic_beats_dir }}/dev-tools/packaging/package_test.go"),
 		"-files",

dev-tools/packaging/package_test.go

@@ -26,6 +26,7 @@ import (
 	"bytes"
 	"compress/gzip"
 	"flag"
+	"fmt"
 	"io"
 	"os"
 	"path/filepath"
@@ -55,7 +56,9 @@ var (
 )
 
 var (
-	files = flag.String("files", "../build/distributions/*/*", "filepath glob containing package files")
+	files    = flag.String("files", "../build/distributions/*/*", "filepath glob containing package files")
+	modules  = flag.Bool("modules", false, "check modules folder contents")
+	modulesd = flag.Bool("modules.d", false, "check modules.d folder contents")
 )
 
 func TestRPM(t *testing.T) {
@@ -101,6 +104,7 @@ func checkRPM(t *testing.T, file string) {
 	checkManifestPermissions(t, p)
 	checkManifestOwner(t, p)
 	checkModulesPermissions(t, p)
+	checkModulesPresent(t, "/etc/", p)
 	checkModulesOwner(t, p)
 	checkSystemdUnitPermissions(t, p)
 }
@@ -116,6 +120,7 @@ func checkDeb(t *testing.T, file string, buf *bytes.Buffer) {
 	checkConfigOwner(t, p)
 	checkManifestPermissions(t, p)
 	checkManifestOwner(t, p)
+	checkModulesPresent(t, "/etc/", p)
 	checkModulesPermissions(t, p)
 	checkModulesOwner(t, p)
 	checkSystemdUnitPermissions(t, p)
@@ -131,6 +136,7 @@ func checkTar(t *testing.T, file string) {
 	checkConfigPermissions(t, p)
 	checkConfigOwner(t, p)
 	checkManifestPermissions(t, p)
+	checkModulesPresent(t, "", p)
 	checkModulesPermissions(t, p)
 	checkModulesOwner(t, p)
 }
@@ -144,6 +150,7 @@ func checkZip(t *testing.T, file string) {
 
 	checkConfigPermissions(t, p)
 	checkManifestPermissions(t, p)
+	checkModulesPresent(t, "", p)
 	checkModulesPermissions(t, p)
 }
 
@@ -268,6 +275,35 @@ func checkSystemdUnitPermissions(t *testing.T, p *packageFile) {
 	})
 }
 
+// Verify that modules.d folder is present and has module files in
+func checkModulesPresent(t *testing.T, prefix string, p *packageFile) {
+	minExpectedModules := 4
+
+	test := func(name string, r *regexp.Regexp) {
+		t.Run(fmt.Sprintf("%s %s contents", p.Name, name), func(t *testing.T) {
+			total := 0
+			for _, entry := range p.Contents {
+				if strings.HasPrefix(entry.File, prefix) && r.MatchString(entry.File) {
+					total++
+				}
+			}
+
+			if total < minExpectedModules {
+				t.Errorf("not enough modules found under %s: actual=%d, expected>=%d",
+					name, total, minExpectedModules)
+			}
+		})
+	}
+
+	if *modules {
+		test("modules", modulesFilePattern)
+	}
+
+	if *modulesd {
+		test("modules.d", modulesDirPattern)
+	}
+}
+
 // Helpers
 
 type packageFile struct {

filebeat/magefile.go

@@ -89,7 +89,7 @@ func Package() {
 
 // TestPackages tests the generated packages (i.e. file modes, owners, groups).
 func TestPackages() error {
-	return mage.TestPackages()
+	return mage.TestPackages(mage.WithModules(), mage.WithModulesD())
 }
 
 // Update updates the generated files (aka make update).

metricbeat/magefile.go

@@ -89,7 +89,7 @@ func Package() {
 
 // TestPackages tests the generated packages (i.e. file modes, owners, groups).
 func TestPackages() error {
-	return mage.TestPackages()
+	return mage.TestPackages(mage.WithModulesD())
 }
 
 // Update updates the generated files (aka make update).