From 6d3e06a9a3a3e1ef17b6fb6b6c15083f1b3f543d Mon Sep 17 00:00:00 2001 From: Angus Gratton Date: Tue, 5 Jan 2021 10:38:39 +1100 Subject: [PATCH] ttfw: Move TestCaseFailed exception and handle differently to other exceptions But also ensure the string form of this exception is never empty, if it ends up somewhere else. --- tools/ci/python_packages/tiny_test_fw/TinyFW.py | 16 ++++++++++++++++ tools/unit-test-app/unit_test.py | 17 ++++++++--------- 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/tools/ci/python_packages/tiny_test_fw/TinyFW.py b/tools/ci/python_packages/tiny_test_fw/TinyFW.py index c45d678205..d9e6abb10b 100644 --- a/tools/ci/python_packages/tiny_test_fw/TinyFW.py +++ b/tools/ci/python_packages/tiny_test_fw/TinyFW.py @@ -27,6 +27,20 @@ from . import App from . import Utility +class TestCaseFailed(AssertionError): + def __init__(self, *cases): + """ + Raise this exception if one or more test cases fail in a 'normal' way (ie the test runs but fails, no unexpected exceptions) + + This will avoid dumping the Python stack trace, because the assumption is the junit error info and full job log already has + enough information for a developer to debug. + + 'cases' argument is the names of one or more test cases + """ + message = "Test case{} failed: {}".format("s" if len(cases) > 1 else "", ", ".join(str(c) for c in cases)) + super(TestCaseFailed, self).__init__(self, message) + + class DefaultEnvConfig(object): """ default test configs. There're 3 places to set configs, priority is (high -> low): @@ -193,6 +207,8 @@ def test_method(**kwargs): test_func(env_inst, extra_data) # if finish without exception, test result is True result = True + except TestCaseFailed as e: + junit_test_case.add_failure_info(str(e)) except Exception as e: Utility.handle_unexpected_exception(junit_test_case, e) finally: diff --git a/tools/unit-test-app/unit_test.py b/tools/unit-test-app/unit_test.py index 565ff46de0..8f9c827596 100755 --- a/tools/unit-test-app/unit_test.py +++ b/tools/unit-test-app/unit_test.py @@ -24,6 +24,7 @@ import argparse import threading from tiny_test_fw import TinyFW, Utility, Env, DUT +from tiny_test_fw.TinyFW import TestCaseFailed from tiny_test_fw.Utility import handle_unexpected_exception import ttfw_idf @@ -72,10 +73,6 @@ def reset_reason_matches(reported_str, expected_str): return False -class TestCaseFailed(AssertionError): - pass - - def format_test_case_config(test_case_data): """ convert the test case data to unified format. @@ -222,7 +219,7 @@ def run_one_normal_case(dut, one_case, junit_test_case): else: Utility.console_log("Failed: " + format_case_name(one_case), color="red") junit_test_case.add_failure_info(output) - raise TestCaseFailed() + raise TestCaseFailed(format_case_name(one_case)) def handle_exception_reset(data): """ @@ -331,7 +328,7 @@ def run_unit_test_cases(env, extra_data): Utility.console_log("Failed Cases:", color="red") for _case_name in failed_cases: Utility.console_log("\t" + _case_name, color="red") - raise AssertionError("Unit Test Failed") + raise TestCaseFailed(*failed_cases) class Handler(threading.Thread): @@ -517,6 +514,8 @@ def run_multiple_devices_cases(env, extra_data): try: result = run_one_multiple_devices_case(duts, ut_config, env, one_case, one_case.get('app_bin'), junit_test_case) + except TestCaseFailed: + pass # result is False, this is handled by the finally block except Exception as e: handle_unexpected_exception(junit_test_case, e) finally: @@ -535,7 +534,7 @@ def run_multiple_devices_cases(env, extra_data): Utility.console_log("Failed Cases:", color="red") for _case_name in failed_cases: Utility.console_log("\t" + _case_name, color="red") - raise AssertionError("Unit Test Failed") + raise TestCaseFailed(*failed_cases) def run_one_multiple_stage_case(dut, one_case, junit_test_case): @@ -590,7 +589,7 @@ def run_one_multiple_stage_case(dut, one_case, junit_test_case): else: Utility.console_log("Failed: " + format_case_name(one_case), color="red") junit_test_case.add_failure_info(output) - raise TestCaseFailed() + raise TestCaseFailed(format_case_name(one_case)) stage_finish.append("break") def handle_exception_reset(data): @@ -691,7 +690,7 @@ def run_multiple_stage_cases(env, extra_data): Utility.console_log("Failed Cases:", color="red") for _case_name in failed_cases: Utility.console_log("\t" + _case_name, color="red") - raise AssertionError("Unit Test Failed") + raise TestCaseFailed(*failed_cases) def detect_update_unit_test_info(env, extra_data, app_bin):