esp-idf/tools/ci/python_packages/ttfw_idf/unity_test_parser.py

378 lines
13 KiB
Python
Raw Normal View History

2022-06-15 10:46:55 -04:00
# SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD
# SPDX-License-Identifier: Apache-2.0
"""
Modification version of https://github.com/ETCLabs/unity-test-parser/blob/develop/unity_test_parser.py
since only python 3.6 or higher version have ``enum.auto()``
unity_test_parser.py
Parse the output of the Unity Test Framework for C. Parsed results are held in the TestResults
object format, which can then be converted to various XML formats.
"""
import enum
import re
import junit_xml
_NORMAL_TEST_REGEX = re.compile(r'(?P<file>.+):(?P<line>\d+):(?P<test_name>[^\s:]+):(?P<result>PASS|FAIL|IGNORE)(?:: (?P<message>.+))?')
_UNITY_FIXTURE_VERBOSE_PREFIX_REGEX = re.compile(r'(?P<prefix>TEST\((?P<test_group>[^\s,]+), (?P<test_name>[^\s\)]+)\))(?P<remainder>.+)?$')
_UNITY_FIXTURE_REMAINDER_REGEX = re.compile(r'^(?P<file>.+):(?P<line>\d+)::(?P<result>PASS|FAIL|IGNORE)(?:: (?P<message>.+))?')
_TEST_SUMMARY_BLOCK_REGEX = re.compile(
r'^(?P<num_tests>\d+) Tests (?P<num_failures>\d+) Failures (?P<num_ignored>\d+) Ignored\s*\r?\n(?P<overall_result>OK|FAIL)(?:ED)?', re.MULTILINE
)
_TEST_RESULT_ENUM = ['PASS', 'FAIL', 'IGNORE']
class TestFormat(enum.Enum):
"""Represents the flavor of Unity used to produce a given output."""
UNITY_BASIC = 0
# UNITY_FIXTURE = enum.auto()
UNITY_FIXTURE_VERBOSE = 1
globals().update(TestFormat.__members__)
class TestStats:
"""Statistics about a test collection"""
def __init__(self):
self.total = 0
self.passed = 0
self.failed = 0
self.ignored = 0
def __eq__(self, other):
if isinstance(other, self.__class__):
return (self.total == other.total
and self.passed == other.passed
and self.failed == other.failed
and self.ignored == other.ignored)
return False
class TestResult:
"""
Class representing the result of a single test.
Contains the test name, its result (either PASS, FAIL or IGNORE), the file and line number if
the test result was not PASS, and an optional message.
"""
def __init__(
self,
test_name,
result,
group='default',
file='',
line=0,
message='',
full_line='',
):
if result not in _TEST_RESULT_ENUM:
raise ValueError('result must be one of {}.'.format(_TEST_RESULT_ENUM))
self._test_name = test_name
self._result = result
self._group = group
self._message = message
self._full_line = full_line
if result != 'PASS':
self._file = file
self._line = line
else:
self._file = ''
self._line = 0
def file(self):
"""The file name - returns empty string if the result is PASS."""
return self._file
def line(self):
"""The line number - returns 0 if the result is PASS."""
return self._line
def name(self):
"""The test name."""
return self._test_name
def result(self):
"""The test result, one of PASS, FAIL or IGNORED."""
return self._result
def group(self):
"""
The test group, if applicable.
For basic Unity output, this will always be "default".
"""
return self._group
def message(self):
"""The accompanying message - returns empty string if the result is PASS."""
return self._message
def full_line(self):
"""The original, full line of unit test output that this object was created from."""
return self._full_line
class TestResults:
"""
Class representing Unity test results.
After being initialized with raw test output, it parses the output and represents it as a list
of TestResult objects which can be inspected or converted to other types of output, e.g. JUnit
XML.
"""
def __init__(self, test_output, test_format=TestFormat.UNITY_BASIC):
"""
Create a new TestResults object from Unity test output.
Keyword arguments:
test_output -- The full test console output, must contain the overall result and summary
block at the bottom.
Optional arguments:
test_format -- TestFormat enum representing the flavor of Unity used to create the output.
Exceptions:
ValueError, if the test output is not formatted properly.
"""
self._tests = []
self._test_stats = self._find_summary_block(test_output)
if test_format is TestFormat.UNITY_BASIC:
self._parse_unity_basic(test_output)
elif test_format is TestFormat.UNITY_FIXTURE_VERBOSE:
self._parse_unity_fixture_verbose(test_output)
else:
raise ValueError(
'test_format must be one of UNITY_BASIC or UNITY_FIXTURE_VERBOSE.'
)
def num_tests(self):
"""The total number of tests parsed."""
return self._test_stats.total
def num_passed(self):
"""The number of tests with result PASS."""
return self._test_stats.passed
def num_failed(self):
"""The number of tests with result FAIL."""
return self._test_stats.failed
def num_ignored(self):
"""The number of tests with result IGNORE."""
return self._test_stats.ignored
def test_iter(self):
"""Get an iterator for iterating over individual tests.
Returns an iterator over TestResult objects.
Example:
for test in unity_results.test_iter():
print(test.name())
"""
return iter(self._tests)
def tests(self):
"""Get a list of all the tests (TestResult objects)."""
return self._tests
def to_junit(
self, suite_name='all_tests',
):
"""
Convert the tests to JUnit XML.
Returns a junit_xml.TestSuite containing all of the test cases. One test suite will be
generated with the name given in suite_name. Unity Fixture test groups are mapped to the
classname attribute of test cases; for basic Unity output there will be one class named
"default".
Optional arguments:
suite_name -- The name to use for the "name" and "package" attributes of the testsuite element.
Sample output:
<testsuite disabled="0" errors="0" failures="1" name="[suite_name]" package="[suite_name]" skipped="0" tests="8" time="0">
<testcase classname="test_group_1" name="group_1_test" />
<testcase classname="test_group_2" name="group_2_test" />
</testsuite>
"""
test_case_list = []
for test in self._tests:
if test.result() == 'PASS':
test_case_list.append(
junit_xml.TestCase(name=test.name(), classname=test.group())
)
else:
junit_tc = junit_xml.TestCase(
name=test.name(),
classname=test.group(),
file=test.file(),
line=test.line(),
)
if test.result() == 'FAIL':
junit_tc.add_failure_info(
message=test.message(), output=test.full_line()
)
elif test.result() == 'IGNORE':
junit_tc.add_skipped_info(
message=test.message(), output=test.full_line()
)
test_case_list.append(junit_tc)
return junit_xml.TestSuite(
name=suite_name, package=suite_name, test_cases=test_case_list
)
def _find_summary_block(self, unity_output):
"""
Find and parse the test summary block.
Unity prints a test summary block at the end of a test run of the form:
-----------------------
X Tests Y Failures Z Ignored
[PASS|FAIL]
Returns the contents of the test summary block as a TestStats object.
"""
match = _TEST_SUMMARY_BLOCK_REGEX.search(unity_output)
if not match:
raise ValueError('A Unity test summary block was not found.')
try:
stats = TestStats()
stats.total = int(match.group('num_tests'))
stats.failed = int(match.group('num_failures'))
stats.ignored = int(match.group('num_ignored'))
stats.passed = stats.total - stats.failed - stats.ignored
return stats
except ValueError:
raise ValueError('The Unity test summary block was not valid.')
def _parse_unity_basic(self, unity_output):
"""
Parse basic unity output.
This is of the form file:line:test_name:result[:optional_message]
"""
found_test_stats = TestStats()
for test in _NORMAL_TEST_REGEX.finditer(unity_output):
try:
new_test = TestResult(
test.group('test_name'),
test.group('result'),
file=test.group('file'),
line=int(test.group('line')),
message=test.group('message')
if test.group('message') is not None
else '',
full_line=test.group(0),
)
except ValueError:
continue
self._add_new_test(new_test, found_test_stats)
if len(self._tests) == 0:
raise ValueError('No tests were found.')
if found_test_stats != self._test_stats:
raise ValueError('Test output does not match summary block.')
def _parse_unity_fixture_verbose(self, unity_output):
"""
Parse the output of the unity_fixture add-in invoked with the -v flag.
This is a more complex operation than basic unity output, because the output for a single
test can span multiple lines. There is a prefix of the form "TEST(test_group, test_name)"
that always exists on the first line for a given test. Immediately following that can be a
pass or fail message, or some number of diagnostic messages followed by a pass or fail
message.
"""
found_test_stats = TestStats()
line_iter = iter(unity_output.splitlines())
try:
line = next(line_iter)
while True:
prefix_match = _UNITY_FIXTURE_VERBOSE_PREFIX_REGEX.search(line)
line = next(line_iter)
if prefix_match:
# Handle the remaining portion of a test case line after the unity_fixture
# prefix.
remainder = prefix_match.group('remainder')
if remainder:
self._parse_unity_fixture_remainder(
prefix_match, remainder, found_test_stats
)
# Handle any subsequent lines with more information on the same test case.
while not _UNITY_FIXTURE_VERBOSE_PREFIX_REGEX.search(line):
self._parse_unity_fixture_remainder(
prefix_match, line, found_test_stats
)
line = next(line_iter)
except StopIteration:
pass
if len(self._tests) == 0:
raise ValueError('No tests were found.')
if found_test_stats != self._test_stats:
raise ValueError('Test output does not match summary block.')
def _parse_unity_fixture_remainder(self, prefix_match, remainder, test_stats):
"""
Parse the remainder of a Unity Fixture test case.
Can be on the same line as the prefix or on subsequent lines.
"""
new_test = None
if remainder == ' PASS':
new_test = TestResult(
prefix_match.group('test_name'),
'PASS',
group=prefix_match.group('test_group'),
full_line=prefix_match.group(0),
)
else:
remainder_match = _UNITY_FIXTURE_REMAINDER_REGEX.match(remainder)
if remainder_match:
new_test = TestResult(
prefix_match.group('test_name'),
remainder_match.group('result'),
group=prefix_match.group('test_group'),
file=remainder_match.group('file'),
line=int(remainder_match.group('line')),
message=remainder_match.group('message')
if remainder_match.group('message') is not None
else '',
full_line=prefix_match.group('prefix') + remainder_match.group(0),
)
if new_test is not None:
self._add_new_test(new_test, test_stats)
def _add_new_test(self, new_test, test_stats):
"""Add a new test and increment the proper members of test_stats."""
test_stats.total += 1
if new_test.result() == 'PASS':
test_stats.passed += 1
elif new_test.result() == 'FAIL':
test_stats.failed += 1
else:
test_stats.ignored += 1
self._tests.append(new_test)