ci: fix warnings with wildcards and archive system-tests (elastic#18695)

.ci/scripts/pre_archive_test.py

@@ -0,0 +1,20 @@
+#!/usr/bin/env python3
+
+import os
+import distutils
+from distutils import dir_util
+
+
+if __name__ == "__main__":
+
+    if not os.path.exists('build'):
+        os.makedirs('build')
+
+    # Top level folders to be excluded
+    EXCLUDE = set(['.ci', '.git', '.github', 'vendor', 'dev-tools'])
+    for root, dirs, files in os.walk('.'):
+        dirs[:] = [d for d in dirs if d not in EXCLUDE]
+        if root.endswith(('build')) and not root.startswith((".{}build".format(os.sep))):
+            dest = os.path.join('build', root.replace(".{}".format(os.sep), ''))
+            print("Copy {} into {}".format(root, dest))
+            distutils.dir_util.copy_tree(root, dest, preserve_symlinks=1)

.ci/scripts/search_system_tests.py

@@ -0,0 +1,10 @@
+#!/usr/bin/env python3
+
+import os
+
+
+if __name__ == "__main__":
+
+    for root, dirs, files in os.walk('build'):
+        if root.endswith(('system-tests')):
+            print(root.replace(".{}".format(os.sep), ''))

Jenkinsfile

@@ -901,17 +901,39 @@ def withBeatsEnv(Map args = [:], Closure body) {
         }
       } finally {
         if (archive) {
-          catchError(buildResult: 'SUCCESS', stageResult: 'UNSTABLE') {
-            junitAndStore(allowEmptyResults: true, keepLongStdio: true, testResults: "**/build/TEST*.xml")
-            archiveArtifacts(allowEmptyArchive: true, artifacts: '**/build/TEST*.out')
-          }
+          archiveTestOutput(testResults: '**/build/TEST*.xml', artifacts: '**/build/TEST*.out')
         }
         reportCoverage()
       }
     }
   }
 }
 
+/**
+  This method archives and report the tests output, for such, it searches in certain folders
+  to bypass some issues when working with big repositories.
+*/
+def archiveTestOutput(Map args = [:]) {
+  catchError(buildResult: 'SUCCESS', stageResult: 'UNSTABLE') {
+    if (isUnix()) {
+      fixPermissions("${WORKSPACE}")
+    }
+    cmd(label: 'Prepare test output', script: 'python .ci/scripts/pre_archive_test.py')
+    dir('build') {
+      junitAndStore(allowEmptyResults: true, keepLongStdio: true, testResults: args.testResults)
+      archiveArtifacts(allowEmptyArchive: true, artifacts: args.artifacts)
+    }
+    catchError(buildResult: 'SUCCESS', message: 'Failed to archive the build test results', stageResult: 'SUCCESS') {
+      def folder = cmd(label: 'Find system-tests', returnStdout: true, script: 'python .ci/scripts/search_system_tests.py').trim()
+      log(level: 'INFO', text: "system-tests='${folder}'. If no empty then let's create a tarball")
+      if (folder.trim()) {
+        def name = folder.replaceAll('/', '-').replaceAll('\\\\', '-').replaceAll('build', '').replaceAll('^-', '') + '-' + goos()
+        tar(file: "${name}.tgz", archive: true, dir: folder)
+      }
+    }
+  }
+}
+
 def withBeatsEnvWin(Map args = [:], Closure body) {
   def withModule = args.get('withModule', false)
   def directory = args.get('directory', '')
@@ -946,10 +968,7 @@ def withBeatsEnvWin(Map args = [:], Closure body) {
           body()
         }
       } finally {
-        catchError(buildResult: 'SUCCESS', stageResult: 'UNSTABLE') {
-          junitAndStore(allowEmptyResults: true, keepLongStdio: true, testResults: "**\\build\\TEST*.xml")
-          archiveArtifacts(allowEmptyArchive: true, artifacts: '**\\build\\TEST*.out')
-        }
+        archiveTestOutput(testResults: "**\\build\\TEST*.xml", artifacts: "**\\build\\TEST*.out")
       }
     }
   }