Merge pull request #1360 from msimberg/batch-logstash

Batch logstash uploads to speed up CI jobs

.gitlab/scripts/collect_ctest_metrics.sh

@@ -17,6 +17,7 @@ metadata_file=$(mktemp --tmpdir metadata.XXXXXXXXXX.json)
 create_metadata_file "${metadata_file}"
 
 # Submit individual test data
+result_file_all=$(mktemp --tmpdir "ctest.XXXXXXXXXX.json")
 num_tests=$(xq . "${ctest_xml}" | jq '.testsuite.testcase | length')
 for i in $(seq 1 ${num_tests}); do
     result_file=$(mktemp --tmpdir "ctest_${i}.XXXXXXXXXX.json")
@@ -27,8 +28,9 @@ for i in $(seq 1 ${num_tests}); do
     json_add_value_json "${result_file}" "ctest.testcase" "${ctest_object}"
 
     json_merge "${metadata_file}" "${result_file}" "${result_file}"
-    submit_logstash "${result_file}"
+    cat "${result_file}" >>"${result_file_all}"
 done
+submit_logstash "${result_file_all}"
 
 result_file=$(mktemp --tmpdir "ctest.XXXXXXXXXX.json")
 echo '{}' >"${result_file}"

.gitlab/scripts/collect_file_sizes.sh

@@ -20,6 +20,7 @@ function submit_filesizes {
     directory="${1}"
     IFS='
 '
+    result_file_all=$(mktemp --tmpdir "filesizes.XXXXXXXXXX.json")
     for line in $(ls --time-style long-iso -l $(find "${directory}" -maxdepth 1 -type f)); do
         filename=$(echo "${line}" | awk '{ print $8 }')
         filesize=$(echo "${line}" | awk '{ print $5 }')
@@ -31,8 +32,9 @@ function submit_filesizes {
         json_add_value_number "${result_file}" "files.size" "${filesize}"
 
         json_merge "${metadata_file}" "${result_file}" "${result_file}"
-        submit_logstash "${result_file}"
+        cat "${result_file}" >>"${result_file_all}"
     done
+    submit_logstash "${result_file_all}"
 }
 
 # Submit file name and size for files under lib and bin in the build directory

.gitlab/scripts/run_performance_benchmarks.sh

@@ -111,6 +111,7 @@ pika_test_options=(
 
 index=0
 failures=0
+result_file_all=$(mktemp --tmpdir "benchmarks.XXXXXXXXXX.json")
 for executable in "${pika_targets[@]}"; do
     test_opts=${pika_test_options[$index]}
     raw_result_file=$(mktemp --tmpdir "${executable}_raw.XXXXXXXXXX.json")
@@ -140,11 +141,12 @@ for executable in "${pika_targets[@]}"; do
         json_add_value_json "${result_file}" "metric.benchmark.series" "${benchmark_series}"
 
         json_merge "${metadata_file}" "${result_file}" "${result_file}"
-        submit_logstash "${result_file}"
+        cat "${result_file}" >>"${result_file_all}"
     fi
 
     index=$((index + 1))
 done
+submit_logstash "${result_file_all}"
 
 if ((failures > 0)); then
     exit 1

.gitlab/scripts/utilities.sh

@@ -10,12 +10,14 @@ function submit_logstash {
         jq . "${1}"
     fi
 
+    array_of_objects_file=$(mktemp --tmpdir metadata.XXXXXXXXXX.json)
+    jq --null-input '[inputs]' <"${1}" >"${array_of_objects_file}"
     curl \
         --silent \
         --show-error \
         --request POST \
         --header "Content-Type: application/json" \
-        --data "@${1}" \
+        --data "@${array_of_objects_file}" \
         "${CSCS_LOGSTASH_URL}" ||
         true
 }