Cherry-pick #15957 to 7.x: Fix packaging of OSS light modules

metricbeat/magefile.go

@@ -79,7 +79,13 @@ func Package() {
 
 // TestPackages tests the generated packages (i.e. file modes, owners, groups).
 func TestPackages() error {
-	return devtools.TestPackages(devtools.WithModulesD())
+	return devtools.TestPackages(
+		devtools.WithModulesD(),
+		devtools.WithModules(),
+
+		// To be increased or removed when more light modules are added
+		devtools.MinModules(1),
+	)
 }
 
 // Dashboards collects all the dashboards and generates index patterns.

metricbeat/scripts/mage/package.go

@@ -18,6 +18,7 @@
 package mage
 
 import (
+	"fmt"
 	"io"
 	"os"
 	"path/filepath"
@@ -31,6 +32,7 @@ import (
 )
 
 const (
+	dirModulesGenerated  = "build/package/module"
 	dirModulesDGenerated = "build/package/modules.d"
 )
 
@@ -39,6 +41,8 @@ const (
 // not supported. You must declare a dependency on either
 // PrepareModulePackagingOSS or PrepareModulePackagingXPack.
 func CustomizePackaging() {
+	mg.Deps(customizeLightModulesPackaging)
+
 	var (
 		modulesDTarget = "modules.d"
 		modulesD       = devtools.PackageFile{
@@ -101,6 +105,10 @@ func CustomizePackaging() {
 // PrepareModulePackagingOSS generates build/package/modules and
 // build/package/modules.d directories for use in packaging.
 func PrepareModulePackagingOSS() error {
+	err := prepareLightModulesPackaging("module")
+	if err != nil {
+		return err
+	}
 	return prepareModulePackaging([]struct{ Src, Dst string }{
 		{devtools.OSSBeatDir("modules.d"), dirModulesDGenerated},
 	}...)
@@ -109,6 +117,10 @@ func PrepareModulePackagingOSS() error {
 // PrepareModulePackagingXPack generates build/package/modules and
 // build/package/modules.d directories for use in packaging.
 func PrepareModulePackagingXPack() error {
+	err := prepareLightModulesPackaging("module", devtools.OSSBeatDir("module"))
+	if err != nil {
+		return err
+	}
 	return prepareModulePackaging([]struct{ Src, Dst string }{
 		{devtools.OSSBeatDir("modules.d"), dirModulesDGenerated},
 		{"modules.d", dirModulesDGenerated},
@@ -190,6 +202,80 @@ func GenerateDirModulesD() error {
 	return nil
 }
 
+// customizeLightModulesPackaging customizes packaging to add light modules
+func customizeLightModulesPackaging() error {
+	var (
+		moduleTarget = "module"
+		module       = devtools.PackageFile{
+			Mode:   0644,
+			Source: dirModulesGenerated,
+		}
+	)
+
+	for _, args := range devtools.Packages {
+		pkgType := args.Types[0]
+		switch pkgType {
+		case devtools.TarGz, devtools.Zip, devtools.Docker:
+			args.Spec.Files[moduleTarget] = module
+		case devtools.Deb, devtools.RPM:
+			args.Spec.Files["/usr/share/{{.BeatName}}/"+moduleTarget] = module
+		case devtools.DMG:
+			args.Spec.Files["/Library/Application Support/{{.BeatVendor}}/{{.BeatName}}/"+moduleTarget] = module
+		default:
+			return fmt.Errorf("unhandled package type: %v", pkgType)
+		}
+	}
+	return nil
+}
+
+// prepareLightModulesPackaging generates light modules
+func prepareLightModulesPackaging(paths ...string) error {
+	err := devtools.Clean([]string{dirModulesGenerated})
+	if err != nil {
+		return err
+	}
+	if err := os.MkdirAll(dirModulesGenerated, 0755); err != nil {
+		return err
+	}
+
+	filePatterns := []string{
+		"*/module.yml",
+		"*/*/manifest.yml",
+	}
+
+	var tasks []devtools.CopyTask
+	for _, path := range paths {
+		for _, pattern := range filePatterns {
+			matches, err := filepath.Glob(filepath.Join(path, pattern))
+			if err != nil {
+				return err
+			}
+
+			for _, file := range matches {
+				rel, _ := filepath.Rel(path, file)
+				dest := filepath.Join(dirModulesGenerated, rel)
+				tasks = append(tasks, devtools.CopyTask{
+					Source:  file,
+					Dest:    dest,
+					Mode:    0644,
+					DirMode: 0755,
+				})
+			}
+		}
+	}
+
+	if len(tasks) == 0 {
+		return fmt.Errorf("no light modules found")
+	}
+
+	for _, task := range tasks {
+		if err := task.Execute(); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 // moduleConfigParts obtain the moduleName and the configName from a config path.
 // The configName includes the flavor
 func moduleConfigParts(f string) (moduleName string, configName string, ok bool) {

x-pack/metricbeat/magefile.go

@@ -9,8 +9,6 @@ package main
 import (
 	"context"
 	"fmt"
-	"os"
-	"path/filepath"
 	"time"
 
 	"github.com/magefile/mage/mg"
@@ -24,10 +22,6 @@ import (
 	_ "github.com/elastic/beats/dev-tools/mage/target/compose"
 )
 
-const (
-	dirModulesGenerated = "build/package/module"
-)
-
 func init() {
 	common.RegisterCheckDeps(Update)
 
@@ -78,7 +72,6 @@ func Package() {
 	devtools.UseElasticBeatXPackPackaging()
 	metricbeat.CustomizePackaging()
 	devtools.PackageKibanaDashboardsFromBuildDir()
-	packageLightModules()
 
 	mg.Deps(Update, metricbeat.PrepareModulePackagingXPack)
 	mg.Deps(CrossBuild, CrossBuildGoDaemon)
@@ -92,7 +85,8 @@ func TestPackages() error {
 		devtools.WithModules(),
 
 		// To be increased or removed when more light modules are added
-		devtools.MinModules(3))
+		devtools.MinModules(5),
+	)
 }
 
 // Fields generates a fields.yml and fields.go for each module.
@@ -148,78 +142,6 @@ func PythonUnitTest() error {
 	return devtools.PythonNoseTest(devtools.DefaultPythonTestUnitArgs())
 }
 
-// prepareLightModules generates light modules
-func prepareLightModules(path string) error {
-	err := devtools.Clean([]string{dirModulesGenerated})
-	if err != nil {
-		return err
-	}
-	if err := os.MkdirAll(dirModulesGenerated, 0755); err != nil {
-		return err
-	}
-
-	filePatterns := []string{
-		"*/module.yml",
-		"*/*/manifest.yml",
-	}
-
-	var files []string
-	for _, pattern := range filePatterns {
-		matches, err := filepath.Glob(filepath.Join(path, pattern))
-		if err != nil {
-			return err
-		}
-		files = append(files, matches...)
-	}
-
-	if len(files) == 0 {
-		return fmt.Errorf("no light modules found")
-	}
-
-	for _, file := range files {
-		rel, _ := filepath.Rel(path, file)
-		dest := filepath.Join(dirModulesGenerated, rel)
-		err := (&devtools.CopyTask{
-			Source:  file,
-			Dest:    dest,
-			Mode:    0644,
-			DirMode: 0755,
-		}).Execute()
-		if err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-// packageLightModules customizes packaging to add light modules
-func packageLightModules() error {
-	prepareLightModules("module")
-
-	var (
-		moduleTarget = "module"
-		module       = devtools.PackageFile{
-			Mode:   0644,
-			Source: dirModulesGenerated,
-		}
-	)
-
-	for _, args := range devtools.Packages {
-		pkgType := args.Types[0]
-		switch pkgType {
-		case devtools.TarGz, devtools.Zip, devtools.Docker:
-			args.Spec.Files[moduleTarget] = module
-		case devtools.Deb, devtools.RPM:
-			args.Spec.Files["/usr/share/{{.BeatName}}/"+moduleTarget] = module
-		case devtools.DMG:
-			args.Spec.Files["/Library/Application Support/{{.BeatVendor}}/{{.BeatName}}/"+moduleTarget] = module
-		default:
-			return fmt.Errorf("unhandled package type: %v", pkgType)
-		}
-	}
-	return nil
-}
-
 // IntegTest executes integration tests (it uses Docker to run the tests).
 func IntegTest() {
 	devtools.AddIntegTestUsage()