From f89a58f6f3df02ce6bb7cf424d33a0ae8a0f5402 Mon Sep 17 00:00:00 2001
From: Victor Martinez <victormartinezrubio@gmail.com>
Date: Mon, 20 Jul 2020 14:09:51 +0100
Subject: [PATCH] ci: fix warnings with wildcards and archive system-tests
 (#18695)

---
 .ci/scripts/pre_archive_test.py    | 20 +++++++++++++++++
 .ci/scripts/search_system_tests.py | 10 +++++++++
 Jenkinsfile                        | 35 +++++++++++++++++++++++-------
 3 files changed, 57 insertions(+), 8 deletions(-)
 create mode 100755 .ci/scripts/pre_archive_test.py
 create mode 100755 .ci/scripts/search_system_tests.py

diff --git a/.ci/scripts/pre_archive_test.py b/.ci/scripts/pre_archive_test.py
new file mode 100755
index 000000000000..8fd8cb75ea1e
--- /dev/null
+++ b/.ci/scripts/pre_archive_test.py
@@ -0,0 +1,20 @@
+#!/usr/bin/env python3
+
+import os
+import distutils
+from distutils import dir_util
+
+
+if __name__ == "__main__":
+
+    if not os.path.exists('build'):
+        os.makedirs('build')
+
+    # Top level folders to be excluded
+    EXCLUDE = set(['.ci', '.git', '.github', 'vendor', 'dev-tools'])
+    for root, dirs, files in os.walk('.'):
+        dirs[:] = [d for d in dirs if d not in EXCLUDE]
+        if root.endswith(('build')) and not root.startswith((".{}build".format(os.sep))):
+            dest = os.path.join('build', root.replace(".{}".format(os.sep), ''))
+            print("Copy {} into {}".format(root, dest))
+            distutils.dir_util.copy_tree(root, dest, preserve_symlinks=1)
diff --git a/.ci/scripts/search_system_tests.py b/.ci/scripts/search_system_tests.py
new file mode 100755
index 000000000000..0e3896d9ff4c
--- /dev/null
+++ b/.ci/scripts/search_system_tests.py
@@ -0,0 +1,10 @@
+#!/usr/bin/env python3
+
+import os
+
+
+if __name__ == "__main__":
+
+    for root, dirs, files in os.walk('build'):
+        if root.endswith(('system-tests')):
+            print(root.replace(".{}".format(os.sep), ''))
diff --git a/Jenkinsfile b/Jenkinsfile
index 0569da706ed3..c5dcc1f6e646 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -901,10 +901,7 @@ def withBeatsEnv(Map args = [:], Closure body) {
         }
       } finally {
         if (archive) {
-          catchError(buildResult: 'SUCCESS', stageResult: 'UNSTABLE') {
-            junitAndStore(allowEmptyResults: true, keepLongStdio: true, testResults: "**/build/TEST*.xml")
-            archiveArtifacts(allowEmptyArchive: true, artifacts: '**/build/TEST*.out')
-          }
+          archiveTestOutput(testResults: '**/build/TEST*.xml', artifacts: '**/build/TEST*.out')
         }
         reportCoverage()
       }
@@ -912,6 +909,31 @@ def withBeatsEnv(Map args = [:], Closure body) {
   }
 }
 
+/**
+  This method archives and report the tests output, for such, it searches in certain folders
+  to bypass some issues when working with big repositories.
+*/
+def archiveTestOutput(Map args = [:]) {
+  catchError(buildResult: 'SUCCESS', stageResult: 'UNSTABLE') {
+    if (isUnix()) {
+      fixPermissions("${WORKSPACE}")
+    }
+    cmd(label: 'Prepare test output', script: 'python .ci/scripts/pre_archive_test.py')
+    dir('build') {
+      junitAndStore(allowEmptyResults: true, keepLongStdio: true, testResults: args.testResults)
+      archiveArtifacts(allowEmptyArchive: true, artifacts: args.artifacts)
+    }
+    catchError(buildResult: 'SUCCESS', message: 'Failed to archive the build test results', stageResult: 'SUCCESS') {
+      def folder = cmd(label: 'Find system-tests', returnStdout: true, script: 'python .ci/scripts/search_system_tests.py').trim()
+      log(level: 'INFO', text: "system-tests='${folder}'. If no empty then let's create a tarball")
+      if (folder.trim()) {
+        def name = folder.replaceAll('/', '-').replaceAll('\\\\', '-').replaceAll('build', '').replaceAll('^-', '') + '-' + goos()
+        tar(file: "${name}.tgz", archive: true, dir: folder)
+      }
+    }
+  }
+}
+
 def withBeatsEnvWin(Map args = [:], Closure body) {
   def withModule = args.get('withModule', false)
   def directory = args.get('directory', '')
@@ -946,10 +968,7 @@ def withBeatsEnvWin(Map args = [:], Closure body) {
           body()
         }
       } finally {
-        catchError(buildResult: 'SUCCESS', stageResult: 'UNSTABLE') {
-          junitAndStore(allowEmptyResults: true, keepLongStdio: true, testResults: "**\\build\\TEST*.xml")
-          archiveArtifacts(allowEmptyArchive: true, artifacts: '**\\build\\TEST*.out')
-        }
+        archiveTestOutput(testResults: "**\\build\\TEST*.xml", artifacts: "**\\build\\TEST*.out")
       }
     }
   }