Improve test reliability, output test results consumable by Jenkins

Signed-off-by: Addisu Z. Taddese <addisu@openrobotics.org>
gazebosim · Oct 22, 2021 · 311d588 · 311d588
1 parent cb9c102
commit 311d588
Show file tree

Hide file tree

Showing 7 changed files with 71 additions and 32 deletions.
diff --git a/cmake/run_test.cmake b/cmake/run_test.cmake
@@ -0,0 +1,27 @@
+# This script is part of the unit test mechanism for ign-tools.
+execute_process(COMMAND ruby ${IGN_SCRIPT} 
+  ${TEST_NAME}
+  OUTPUT_VARIABLE TEST_OUTPUT
+  ERROR_VARIABLE TEST_ERROR)
+
+# Copy failure result everytime we run test so we don't accidentally use
+# previous results.
+execute_process(COMMAND
+  ${CMAKE_COMMAND} -E copy
+  "${TEST_NAME}_fail.xml"
+  "../test_results/${TEST_NAME}.xml"
+)
+
+if (${TEST_OUTPUT} MATCHES "Test executed" AND NOT 
+    ((${TEST_OUTPUT} MATCHES "Library error") OR 
+    (${TEST_ERROR} MATCHES "Library error")))
+  execute_process(COMMAND
+    ${CMAKE_COMMAND} -E copy
+    "${TEST_NAME}_pass.xml"
+    "../test_results/${TEST_NAME}.xml"
+  )
+else()
+  message(FATAL_ERROR "Failed test.
+  Output: ${TEST_OUTPUT}
+  Error: ${TEST_ERROR}")
+endif()
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
@@ -59,46 +59,44 @@ if(BUILD_TESTING)
   configure_file(ign.in ${ign_script_configured_test})
   file(GENERATE OUTPUT ${ign_script_generated_test} INPUT ${ign_script_configured_test})
 
-  set(IGN_DESIGNIATION tools)
-  set(cmd_script_configured_test "${CMAKE_CURRENT_BINARY_DIR}/cmd${IGN_DESIGNIATION}_TEST.rb.configured")
-  set(cmd_script_generated_test "${PROJECT_BINARY_DIR}/test/lib/$<CONFIG>/ruby/ignition/cmd${IGN_DESIGNIATION}_TEST.rb")
+  set(TEST_NAME UNIT_ign_TEST)
 
-  configure_file(
-    "cmd${IGN_DESIGNIATION}_TEST.rb.in"
-    "${cmd_script_configured_test}"
-    @ONLY)
-
-  file(GENERATE
-    OUTPUT "${cmd_script_generated_test}"
-    INPUT  "${cmd_script_configured_test}")
+  set(ign_library_path  "${PROJECT_SOURCE_DIR}/src/cmd${TEST_NAME}.rb")
 
-  set(ign_library_path ${cmd_script_generated_test})
-
-  set(yaml_output_dir "${PROJECT_BINARY_DIR}/test/conf/" )
-  set(yaml_output_configured_test "${yaml_output_dir}/${IGN_DESIGNIATION}_TEST.yaml.configured" )
-  set(yaml_output_generated_test "${yaml_output_dir}/$<CONFIG>/${IGN_DESIGNIATION}_TEST.yaml" )
+  set(yaml_output_dir "${PROJECT_BINARY_DIR}/test/conf" )
+  set(yaml_output_configured_test "${yaml_output_dir}/${TEST_NAME}.yaml.configured")
+  set(yaml_output_generated_test "${yaml_output_dir}/$<CONFIG>/${TEST_NAME}.yaml")
   configure_file(
-    "${IGN_DESIGNIATION}_TEST.yaml.in"
-    ${yaml_output_configured_test} @ONLY)
+    "${TEST_NAME}.yaml.in"
+    "${yaml_output_configured_test}" @ONLY)
 
   file(GENERATE
     OUTPUT "${yaml_output_generated_test}"
     INPUT "${yaml_output_configured_test}")
 
-  add_test(NAME UNIT_ign_TEST COMMAND ruby ${ign_script_generated_test} tools_TEST)
+  configure_file(
+    "${PROJECT_SOURCE_DIR}/tools/junit_pass.xml.in"
+    "${CMAKE_CURRENT_BINARY_DIR}/${TEST_NAME}_pass.xml"
+    @ONLY)
 
-  set_tests_properties(
-    UNIT_ign_TEST
-    PROPERTIES
-    ENVIRONMENT
-    "IGN_CONFIG_PATH=${yaml_output_dir}/$<CONFIG>"
-  )
-  set_tests_properties(UNIT_ign_TEST PROPERTIES
-    PASS_REGULAR_EXPRESSION "Test passed"
+  configure_file(
+    "${PROJECT_SOURCE_DIR}/tools/junit_fail.xml.in"
+    "${CMAKE_CURRENT_BINARY_DIR}/${TEST_NAME}_fail.xml"
+    @ONLY)
+
+  # This simply executes ign. The output will be tested in the check_* test
+  add_test(
+    NAME ${TEST_NAME}
+    COMMAND ${CMAKE_COMMAND}
+    -DTEST_NAME=${TEST_NAME}
+    -DIGN_SCRIPT=${ign_script_generated_test} 
+    -P ${PROJECT_SOURCE_DIR}/cmake/run_test.cmake
   )
 
-  set_tests_properties(UNIT_ign_TEST PROPERTIES
-    FAIL_REGULAR_EXPRESSION "Library error"
+  set_tests_properties(
+    ${TEST_NAME}
+    PROPERTIES
+    ENVIRONMENT "IGN_CONFIG_PATH=${yaml_output_dir}/$<CONFIG>"
   )
 endif()
 # END TEST

diff --git a/src/tools_TEST.yaml.in → src/UNIT_ign_TEST.yaml.in b/src/tools_TEST.yaml.in → src/UNIT_ign_TEST.yaml.in
@@ -4,6 +4,6 @@ library_name: @PROJECT_NAME@
 library_version: ''
 library_path: @ign_library_path@
 commands:
-    - tools_TEST   : Test command
+    - UNIT_ign_TEST   : Test command
 ---
 
diff --git a/src/cmdtools_TEST.rb.in → src/cmdUNIT_ign_TEST.rb b/src/cmdtools_TEST.rb.in → src/cmdUNIT_ign_TEST.rb
@@ -24,7 +24,7 @@ class Cmd
   # Execute the command
   #
   def execute(args)
-    puts "Test passed"
+    puts "Test executed"
   end
 end
 
diff --git a/src/ign.in b/src/ign.in
@@ -285,7 +285,7 @@ end
 begin
   SharedLibInterface::Importer.dlload '@backward_library_name@'
 rescue SharedLibInterface::DLError
-  puts 'Library error: @backward_library_name@ not found. Improved backtrace'\
+  warn 'Library error: @backward_library_name@ not found. Improved backtrace'\
        ' generation will be disabled'
 end
 

diff --git a/tools/junit_fail.xml.in b/tools/junit_fail.xml.in
@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuites tests="1" failures="1" disabled="0" errors="0" timestamp="@TEST_TIME@" time="0" name="AllTests">
+  <testsuite name="@TEST_NAME@" tests="1" failures="1" disabled="0" errors="0" time="0">
+    <testcase name="ign_script" status="run" time="0" classname="@TEST_NAME@">
+      <failure type="Standard" message="@TEST_NAME@ failed." />
+    </testcase>
+  </testsuite>
+</testsuites>
diff --git a/tools/junit_pass.xml.in b/tools/junit_pass.xml.in
@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuites tests="1" failures="0" disabled="0" errors="0" timestamp="@TEST_TIME@" time="0" name="AllTests">
+  <testsuite name="@TEST_NAME@" tests="1" failures="0" disabled="0" errors="0" time="0">
+    <testcase name="ign_script" status="run" time="0" classname="@TEST_NAME@" />
+  </testsuite>
+</testsuites>