Skip to content

Commit

Permalink
Merge pull request PixarAnimationStudios#787 from autodesk-forks/ehle…
Browse files Browse the repository at this point in the history
…nl/OGSMOD-4406

* OGSMOD-4406 run all the unit test, and the filter by TestSkipList
* OGSMOD-4406 adjust Ctest timeout and number of jobs. Archive Testing folder on failures and junit now works well on reports generated from xcode
* OGSMOD-4406 skip test testSdfListOp on Windows, it takes too long to run on Debug.
* OGSMOD-4406 check xcresultparser checksum

---------

Co-authored-by: Luis Ehlen <lehlen@evernote.com>
  • Loading branch information
2 people authored and GitHub Enterprise committed Jan 18, 2024
2 parents 73ab6f6 + 0e4ce2a commit a137597
Show file tree
Hide file tree
Showing 4 changed files with 137 additions and 47 deletions.
109 changes: 62 additions & 47 deletions Jenkinsfile
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ DEPLOY_CREDENTIALS = [$class: 'StringBinding',
// -R - a regular expression used to list desired tests rather than all tests (TBD)
// -E - a regular expression used to remove the list of tests which're known to fail
// common flags are listed here and skipped cases are listed on adsk-build-scripts/TestSkipList_{os}_{platform}.txt
USDTESTINGFLAGS = "--timeout 400 --repeat until-pass:3 --output-on-failure"
USDTESTINGFLAGS = "--timeout 120 --repeat until-pass:3 -j4"

// Necessary flags to pass to build_usd.py
USDBUILDINGFLAGS = "--materialx --ptex --openvdb --tests --textsystem --python --openimageio --build-args USD,\"-DPXR_ENABLE_PRECOMPILED_HEADERS=OFF -DPXR_USE_PYTHON_3=ON\""
Expand Down Expand Up @@ -666,32 +666,13 @@ def getConnectedDeviceName()
return realDeviceName
}

def readSkippedTests(buildInfo){
// Returns the location of the skipped tests file for your specific platform
def getSkippedTestsFile(buildInfo){
def suffix = "${buildInfo.Platform}".replaceAll("_${buildInfo.CMakeConfig}", "")
def skipfile = "${env.WORKSPACE}/adsk-build-scripts/TestSkipList_${suffix}.txt"
def skipCaseList = []
def skipfileContent = readFile(skipfile)
for (item in "${skipfileContent}".split('\n'))
{
item = item.replaceAll("\\s+", "")
if (item.startsWith('#')) {
continue
}
if (item != "") {
skipCaseList.add(item)
}
}

def skipFlag = ""
if (skipCaseList) {
def skips = skipCaseList.join('|')
skipFlag = " -E \"${skips}\""
}

return skipFlag
return skipfile
}


///////////////////////////////////////////////////
// Checkout functions

Expand Down Expand Up @@ -878,6 +859,23 @@ def linuxBuild(buildInfo) {
///////////////////////////////////////////////////
// Test functions

// Will consume the junit report and use the skip tests list to determinate the result
def jUnitProcessing(buildInfo, usdinst, junit_file){
def skipFile = getSkippedTestsFile(buildInfo)

COMMONSHELL.shell """${buildInfo.Python} adsk-build-scripts/junit_utils.py --input_file ${usdinst}/${junit_file} --skip_file ${skipFile} --output_file output_${junit_file} --skipped_output_file skipped_${junit_file}"""

junit "output_${junit_file}*"
archiveArtifacts artifacts: "skipped_${junit_file}"

// Fail the build if the name of the output file ends with '.has_failures'
if (fileExists("output_${junit_file}.has_failures")) {
currentBuild.result = "FAILURE"
postStatus('FAILURE', "${buildInfo.Platform}_unittest")
throw new Exception("failed tests found")
}
}

// Generic test stage. Takes the platform-specific test steps as a parameter.
def testStage(buildInfo, runTest) {
try {
Expand All @@ -886,6 +884,7 @@ def testStage(buildInfo, runTest) {
}
catch (err) {
echo "*** TEST FAILURE on ${buildInfo.Platform}.\nInternal error message: ${err.message} ***"
archiveArtifacts artifacts: "**/Testing/**"
isLastTestsFailed(buildInfo)
postStatus('FAILURE', "${buildInfo.Platform}_unittest")
throw err
Expand Down Expand Up @@ -920,12 +919,10 @@ def runGenericTest(buildInfo, label, runTestSteps) {
// Step 2 to download test package
usdinst = downloadTestPackage(buildInfo)

// Step 3 to read tests skip list
def skipFlag = readSkippedTests(buildInfo)

// Step 4 to run platform specific test steps
def normLabel = "${label}".replaceAll(" && ", "_")
stage("Test ${buildInfo.Platform} on ${normLabel}") {

try{
runTestSteps(buildInfo, usdinst, skipFlag)
} finally {
Expand All @@ -947,33 +944,33 @@ def windowsTest(buildInfo) {
parallel tests
}

def runWinTestSteps(buildInfo, usdinst, skipFlag) {
// Execlude testUsdImagingGL for their known problem to run with mutltiple thread,
// and have a separate step to run them with single thread
// TODO: Fix and run testUsdImagingGL with multiple thread
def skipFlag_mt = "${skipFlag}".replaceAll("testUsdviewNavigationKeys", "testUsdviewNavigationKeys|testUsdImagingGL")

def runWinTestSteps(buildInfo, usdinst) {
// Clean up %temp% to avoid potential impact from previous runs
bat """
rd %temp% /s /q
"""

// Always set EnvVar("PXR_MTLX_STDLIB_SEARCH_PATHS") to use MaterialX libraies from USD installed path.
// Exclude testUsdImagingGL for their known problem to run with mutltiple thread,
// and have a separate step to run them with single thread
// TODO: Fix and run testUsdImagingGL with multiple thread
bat """
call set PXR_MTLX_STDLIB_SEARCH_PATHS=${usdinst}\\libraries;
call set PATH=%PATH%;${usdinst}\\lib;${usdinst}\\bin;${usdinst}\\share\\usd\\examples\\plugin;
call set PYTHONPATH=${usdinst}\\lib\\python
call cd /d ${usdinst}
ctest -C ${buildInfo.CMakeConfig} ${USDTESTINGFLAGS} ${skipFlag_mt}
set PXR_WORK_THREAD_LIMIT=1 && ctest -C ${buildInfo.CMakeConfig} ${USDTESTINGFLAGS} ${skipFlag} -R \"testUsdImagingGL\"
ctest -C ${buildInfo.CMakeConfig} ${USDTESTINGFLAGS} -E testUsdImagingGL --output-junit mt_junit_results.xml
set PXR_WORK_THREAD_LIMIT=1 && ctest -C ${buildInfo.CMakeConfig} ${USDTESTINGFLAGS} -R \"testUsdImagingGL\" --output-junit st_junit_results.xml
ver > nul
"""

jUnitProcessing(buildInfo, usdinst, 'mt_junit_results.xml')
jUnitProcessing(buildInfo, usdinst, 'st_junit_results.xml')
}

def macTest(buildInfo) {
if ("${buildInfo.Platform}".contains("ios")) {
stage("Test ${buildInfo.Platform}") {
def skipFlag = readSkippedTests(buildInfo)

// Run test cases
def testpath = "${buildInfo.USDgen_build}/iOSTests"
dir("${testpath}") {
Expand All @@ -982,14 +979,22 @@ def macTest(buildInfo) {
def matcher = testCmd =~ /xcodebuild test -scheme (\w+) -destination .*$/
def casename = matcher[0][1]
matcher = null
if(!"${skipFlag}".contains(casename)){
sh "${testCmd}"
}
else{
echo "Test case ${casename} is skipped."
}
// We don't want the macTest() function to exit if a test fail, we just want to keep testing
// todo that we always reutorn 0 by adding '||true' at the end
sh "${testCmd} -resultBundlePath ${testpath}/resultbundle/${casename} || true"
}
sh """xcrun xcresulttool merge \$(find . -name '*.xcresult') --output-path=merged.xcresult"""
sh """
pushd ${env.WORKSPACE}/adsk-build-scripts/
curl https://github.com/a7ex/xcresultparser/releases/download/1.5.2/xcresultparser.zip -O -J -L -s
unzip xcresultparser.zip && chmod +x product/xcresultparser && mv product/xcresultparser .
shasum -c xcresultparser.checksum
popd
${env.WORKSPACE}/adsk-build-scripts/xcresultparser merged.xcresult --output-format junit > ios_junit_results.xml
"""
}
jUnitProcessing(buildInfo, testpath, 'ios_junit_results.xml')

}
}
else {
Expand All @@ -1000,11 +1005,10 @@ def macTest(buildInfo) {
}
}

def runMacTestSteps(buildInfo, usdinst, skipFlag) {
def runMacTestSteps(buildInfo, usdinst) {
sh """
install_name_tool -add_rpath ${usdinst}/lib ${usdinst}/bin/idiff
"""

// Always set EnvVar("PXR_MTLX_STDLIB_SEARCH_PATHS") to use MaterialX libraies from USD installed path.
sh """
# PySide6 is required for Apple Silicon support
Expand All @@ -1014,8 +1018,13 @@ def runMacTestSteps(buildInfo, usdinst, skipFlag) {
export PXR_MTLX_STDLIB_SEARCH_PATHS=${usdinst}/libraries
export PATH=$PATH:${usdinst}/lib:
export PYTHONPATH=${usdinst}/lib/python:
cd ${usdinst} && ctest -C ${buildInfo.CMakeConfig} ${USDTESTINGFLAGS} ${skipFlag}
# We don't want the shell to exit if a test fail
# todo that we always reutorn 0 by adding '||true' at the end
set +e
cd ${usdinst} && ctest -C ${buildInfo.CMakeConfig} ${USDTESTINGFLAGS} --output-junit mac_junit_results.xml || true
"""
jUnitProcessing(buildInfo, usdinst, 'mac_junit_results.xml')
}

def linuxTest(buildInfo) {
Expand All @@ -1024,19 +1033,25 @@ def linuxTest(buildInfo) {
parallel tests
}

def runLnxTestSteps(buildInfo, usdinst, skipFlag) {
def runLnxTestSteps(buildInfo, usdinst) {
// Always set EnvVar("PXR_MTLX_STDLIB_SEARCH_PATHS") to use MaterialX libraies from USD installed path.
withEnv(["PATH=${usdinst}/lib:/usr/lib:/usr/bin/cmake/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
"LD_LIBRARY_PATH=${usdinst}/lib",
"PXR_MTLX_STDLIB_SEARCH_PATHS=${usdinst}/libraries",
"PYTHONPATH=${usdinst}/lib/python"]) {
def buildImage = "autodesk-docker.art-toucan.autodesk.com/gfx/gfx-vulkan-builder-ubuntu-20.04"
docker.image("${buildImage}").pull()
def ctest_cmd = "ctest -C ${buildInfo.CMakeConfig} ${USDTESTINGFLAGS} --output-junit Testing/lnx_junit_results.xml"
sh """
set -x
mkdir ${usdinst}/Testing
chmod -R a+w ${usdinst}/Testing
docker run --rm -t --network=host --gpus all -e PATH -e LD_LIBRARY_PATH -e PYTHONPATH -e PXR_MTLX_STDLIB_SEARCH_PATHS -e NVIDIA_DRIVER_CAPABILITIES=all -e DISPLAY=:0 -v /tmp/.X11-unix:/tmp/.X11-unix:ro -u agpdevops -v ${env.WORKSPACE}:${env.WORKSPACE}:rw,z -v ${usdinst}:${usdinst}:rw,z -w ${usdinst} ${buildImage} ctest -C ${buildInfo.CMakeConfig} ${USDTESTINGFLAGS} ${skipFlag}
# We don't want the shell to exit if a test fail
# todo that we always return 0 by adding '||true' at the end
docker run --rm -t --network=host --gpus all -e PATH -e LD_LIBRARY_PATH -e PYTHONPATH -e PXR_MTLX_STDLIB_SEARCH_PATHS -e NVIDIA_DRIVER_CAPABILITIES=all -e DISPLAY=:0 -v /tmp/.X11-unix:/tmp/.X11-unix:ro -u agpdevops -v ${env.WORKSPACE}:${env.WORKSPACE}:rw,z -v ${usdinst}:${usdinst}:rw,z -w ${usdinst} ${buildImage} ${ctest_cmd} || true
ls -l ${usdinst}/Testing
"""
jUnitProcessing(buildInfo, "${usdinst}/Testing", 'lnx_junit_results.xml')
}
}

Expand Down
3 changes: 3 additions & 0 deletions adsk-build-scripts/TestSkipList_windows_x64.txt
Original file line number Diff line number Diff line change
Expand Up @@ -21,3 +21,6 @@ testUsdImagingGLPointInstancer_SceneIndex_pi_pi_varying
testUsdImagingGLMaterialXadsklibNodes_shaderball
testUsdImagingGLGeomSubsets_High
testUsdImagingGLGeomSubsets_VeryHigh

# Skip for OGSMOD-4406
testSdfListOp
71 changes: 71 additions & 0 deletions adsk-build-scripts/junit_utils.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
import xml.etree.ElementTree as ET
import argparse
import copy

def remove_elements_by_name(junit_file, tests_to_skip, output_file, skipped_tests_output_file):
# Parse the JUnit XML file
tree = ET.parse(junit_file)
root = tree.getroot()

# Create a new tree for skipped tests
removed_tree = ET.ElementTree(ET.Element('testsuite'))
removed_root = removed_tree.getroot()

# go over all the tests and put the skipped ones in on file and the rest in another
has_failures = False
for testcase in root.findall('.//testcase'):
test_name = testcase.get('name')
if test_name in tests_to_skip:
print(f"skiping: {test_name}")
removed_root.append(copy.deepcopy(testcase))
root.remove(testcase)
elif not has_failures :
test_status = testcase.get('status')
# we have only 2 status: run or fail
if test_status == 'fail':
has_failures = True

# for ios, we need to check on failures in testsuite as it has no status
for testsuite in root.findall('.//testsuite'):
testsuite_name = testsuite.get('name')
if testsuite_name in tests_to_skip:
print(f"skiping: {testsuite_name}")
removed_root.append(copy.deepcopy(testsuite))
root.remove(testsuite)
elif not has_failures :
failures = testsuite.get('failures')
if failures != '0':
has_failures = True


# Change output file name if fail were found in the junit file
if has_failures:
print(f"the junit file contains failures")
output_file=output_file + '.has_failures'

# Save both new files
print(f"Writing non skipped test to {output_file}")
print(f"Writing skipped test to {skipped_tests_output_file}")
tree.write(output_file)
removed_tree.write(skipped_tests_output_file)

if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Takes a junit file and split it based on a list of test to skip.")
parser.add_argument("--input_file", help="Path to the JUnit XML file")
parser.add_argument("--skip_file", help="Path to the TestSkipList file")
parser.add_argument("--output_file", help="Output jUnit file with the non skipped tests")
parser.add_argument("--skipped_output_file", help="Output jUnit file with the skipped tests")

args = parser.parse_args()

junit_file_path = args.input_file
skip_file_path = args.skip_file
output_file_path = args.output_file
skipped_output_file_path = args.skipped_output_file

# Step 1: Read the list of test to skip, ignoring comments
with open(skip_file_path, 'r') as names_file:
tests_to_skip = [line.strip().rstrip('$') for line in names_file.readlines() if not line.startswith("#")]

# Step 2: Remove specified 'testcase' elements, save the modified XML, and create a new file with removed elements
remove_elements_by_name(junit_file_path, tests_to_skip, output_file_path, skipped_output_file_path)
1 change: 1 addition & 0 deletions adsk-build-scripts/xcresultparser.checksum
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
80fb0b5bff1e4d6b8a021e56d3ab3097b2daa914 xcresultparser

0 comments on commit a137597

Please sign in to comment.