Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Check JUnit and XUnit against the XSD schemas #3154

Closed
wants to merge 2 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 18 additions & 0 deletions tests/report/junit/data/main.fmf
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,15 @@
/timeout:
test: ./runtest.sh timeout
duration: 2s
/skip:
test: ./runtest.sh pass
result: skip
/info:
test: ./runtest.sh pass
result: info
/warn:
test: ./runtest.sh pass
result: warn

/shell:
framework: shell
Expand All @@ -27,3 +36,12 @@
/timeout:
test: sleep 10
duration: 2s
/skip:
test: "true"
result: skip
/info:
test: "true"
result: info
/warn:
test: "false"
result: warn
79 changes: 79 additions & 0 deletions tests/report/junit/data/tmt-report-junit.xsd
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
<?xml version="1.0" encoding="UTF-8" ?>

<!--
This schema supports only a subset of the features provided by the
`xml-junit` library. Additionally, many attributes are explicitly set as
required. This is intentional to limit the currently supported features of
the TMT JUnit report plugin.

For example, TMT always creates only one `testsuite` within a `testsuites`,
and this schema enforces that constraint.
-->

<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema">

<xs:element name="failure">
<xs:complexType mixed="true">
<xs:attribute name="type" type="xs:string" use="required"/>
<xs:attribute name="message" type="xs:string" use="required"/>
</xs:complexType>
</xs:element>

<xs:element name="error">
<xs:complexType mixed="true">
<xs:attribute name="type" type="xs:string" use="required"/>
<xs:attribute name="message" type="xs:string" use="required"/>
</xs:complexType>
</xs:element>

<xs:element name="skipped">
<xs:complexType mixed="true">
<xs:attribute name="type" type="xs:string" use="required"/>
<xs:attribute name="message" type="xs:string" use="required"/>
</xs:complexType>
</xs:element>
<xs:element name="system-err" type="xs:string"/>
<xs:element name="system-out" type="xs:string"/>

<xs:element name="testcase">
<xs:complexType>
<xs:sequence>
<xs:element ref="skipped" minOccurs="0" maxOccurs="1"/>
<xs:element ref="error" minOccurs="0" maxOccurs="1"/>
<xs:element ref="failure" minOccurs="0" maxOccurs="1"/>
<xs:element ref="system-out" minOccurs="0" maxOccurs="1"/>
<xs:element ref="system-err" minOccurs="0" maxOccurs="1"/>
</xs:sequence>
<xs:attribute name="name" type="xs:string" use="required"/>
<xs:attribute name="time" type="xs:string" use="optional"/>
</xs:complexType>
</xs:element>

<xs:element name="testsuite">
<xs:complexType>
<xs:sequence>
<xs:element ref="testcase" minOccurs="0" maxOccurs="unbounded"/>
</xs:sequence>
<xs:attribute name="name" type="xs:string" use="required"/>
<xs:attribute name="tests" type="xs:string" use="required"/>
<xs:attribute name="failures" type="xs:string" use="required"/>
<xs:attribute name="errors" type="xs:string" use="required"/>
<xs:attribute name="disabled" type="xs:string" use="required"/>
<xs:attribute name="skipped" type="xs:string" use="required"/>
<xs:attribute name="time" type="xs:string" use="required"/>
</xs:complexType>
</xs:element>

<xs:element name="testsuites">
<xs:complexType>
<xs:sequence>
<xs:element ref="testsuite" minOccurs="1" maxOccurs="1"/>
</xs:sequence>
<xs:attribute name="time" type="xs:string" use="required"/>
<xs:attribute name="tests" type="xs:string" use="optional"/>
<xs:attribute name="failures" type="xs:string" use="optional"/>
<xs:attribute name="disabled" type="xs:string" use="optional"/>
<xs:attribute name="errors" type="xs:string" use="optional"/>
</xs:complexType>
</xs:element>
</xs:schema>
3 changes: 3 additions & 0 deletions tests/report/junit/main.fmf
Original file line number Diff line number Diff line change
@@ -1 +1,4 @@
summary: Check for JUnit output
require+:
# To support xmllint tool
- libxml2
6 changes: 4 additions & 2 deletions tests/report/junit/test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,11 @@ rlJournalStart
for method in tmt; do
rlPhaseStartTest "$method"
rlRun "tmt run -avr execute -h $method report -h junit --file junit.xml 2>&1 >/dev/null | tee output" 2
rlAssertGrep "2 tests passed, 2 tests failed and 2 errors" "output"
rlAssertGrep '<testsuite disabled="0" errors="2" failures="2" name="/plan" skipped="0" tests="6"' "junit.xml"
rlAssertGrep "2 tests passed, 2 tests failed, 2 tests skipped, 2 infos, 2 warns and 2 errors" "output"
rlAssertGrep '<testsuite disabled="0" errors="4" failures="2" name="/plan" skipped="2" tests="12"' "junit.xml"
rlAssertGrep 'fail</failure>' "junit.xml"

rlRun "xmllint --noout --schema 'tmt-report-junit.xsd' 'junit.xml'" 0 "Checking generated JUnit against the XSD schema"
rlPhaseEnd
done

Expand Down
97 changes: 97 additions & 0 deletions tests/report/polarion/data/tmt-report-polarion.xsd
Original file line number Diff line number Diff line change
@@ -0,0 +1,97 @@
<?xml version="1.0" encoding="UTF-8" ?>

<!--
This schema supports only a subset of the features provided by the
`xml-junit` library. Additionally, many attributes are explicitly set as
required. This is intentional to limit the currently supported features of
the TMT Polarion report plugin .

The Polarion `xunit.xml` is almost the same as default output of junit
report plugin but it must allow definition of `properties` inside of
`testsuites` (NOT `testsuite`) and `testcase`.
-->

<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema">

<xs:element name="failure">
<xs:complexType mixed="true">
<xs:attribute name="type" type="xs:string" use="required"/>
<xs:attribute name="message" type="xs:string" use="required"/>
</xs:complexType>
</xs:element>

<xs:element name="error">
<xs:complexType mixed="true">
<xs:attribute name="type" type="xs:string" use="required"/>
<xs:attribute name="message" type="xs:string" use="required"/>
</xs:complexType>
</xs:element>

<xs:element name="skipped">
<xs:complexType mixed="true">
<xs:attribute name="type" type="xs:string" use="required"/>
<xs:attribute name="message" type="xs:string" use="required"/>
</xs:complexType>
</xs:element>
<xs:element name="system-err" type="xs:string"/>
<xs:element name="system-out" type="xs:string"/>

<xs:element name="properties">
<xs:complexType>
<xs:sequence>
<xs:element ref="property" maxOccurs="unbounded"/>
</xs:sequence>
</xs:complexType>
</xs:element>

<xs:element name="property">
<xs:complexType>
<xs:attribute name="name" type="xs:string" use="required"/>
<xs:attribute name="value" type="xs:string" use="required"/>
</xs:complexType>
</xs:element>

<xs:element name="testcase">
<xs:complexType>
<xs:sequence>
<xs:element ref="skipped" minOccurs="0" maxOccurs="1"/>
<xs:element ref="error" minOccurs="0" maxOccurs="1"/>
<xs:element ref="failure" minOccurs="0" maxOccurs="1"/>
<xs:element ref="system-out" minOccurs="0" maxOccurs="1"/>
<xs:element ref="system-err" minOccurs="0" maxOccurs="1"/>
<xs:element ref="properties" minOccurs="0" maxOccurs="1"/>
</xs:sequence>
<xs:attribute name="name" type="xs:string" use="required"/>
<xs:attribute name="time" type="xs:string" use="optional"/>
</xs:complexType>
</xs:element>

<xs:element name="testsuite">
<xs:complexType>
<xs:sequence>
<xs:element ref="testcase" minOccurs="0" maxOccurs="unbounded"/>
</xs:sequence>
<xs:attribute name="name" type="xs:string" use="required"/>
<xs:attribute name="tests" type="xs:string" use="required"/>
<xs:attribute name="failures" type="xs:string" use="required"/>
<xs:attribute name="errors" type="xs:string" use="required"/>
<xs:attribute name="disabled" type="xs:string" use="required"/>
<xs:attribute name="skipped" type="xs:string" use="required"/>
<xs:attribute name="time" type="xs:string" use="required"/>
</xs:complexType>
</xs:element>

<xs:element name="testsuites">
<xs:complexType>
<xs:sequence>
<xs:element ref="testsuite" minOccurs="1" maxOccurs="1"/>
<xs:element ref="properties" minOccurs="0" maxOccurs="1"/>
</xs:sequence>
<xs:attribute name="time" type="xs:string" use="required"/>
<xs:attribute name="tests" type="xs:string" use="optional"/>
<xs:attribute name="failures" type="xs:string" use="optional"/>
<xs:attribute name="disabled" type="xs:string" use="optional"/>
<xs:attribute name="errors" type="xs:string" use="optional"/>
</xs:complexType>
</xs:element>
</xs:schema>
3 changes: 3 additions & 0 deletions tests/report/polarion/main.fmf
Original file line number Diff line number Diff line change
@@ -1,2 +1,5 @@
summary: Check for xUnit output
enabled: false # needs pylero, so just for local testing
require+:
# To support xmllint tool
- libxml2
2 changes: 2 additions & 0 deletions tests/report/polarion/test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@ rlJournalStart
rlAssertGrep '<property name="polarion-project-id" value="RHELBASEOS" />' "xunit.xml"
rlAssertGrep '<property name="polarion-testcase-id" value="BASEOS-10914" />' "xunit.xml"
rlAssertGrep '<property name="polarion-custom-plannedin" value="RHEL-9.1.0" />' "xunit.xml"

rlRun "xmllint --noout --schema 'tmt-report-polarion.xsd' 'xunit.xml'" 0 "Checking generated XUnit against the XSD schema"
rlPhaseEnd

rlPhaseStartCleanup
Expand Down
Loading