mirror of
https://github.com/espressif/esp-idf.git
synced 2024-10-05 20:47:46 -04:00
unit-test: fix 2 issues in unit test script:
1. Gitlab will only regard failure (not error) as failed case. Use log failure instead of log error, to display all error msg in test result 2. fix failed case not recognized by test script when failed during reset DUT
This commit is contained in:
He Yin Ling
bot
parent
7c59207351
commit
431653bba8
@ -64,7 +64,11 @@ MULTI_DEVICE_ID = 2
|
||||
DEFAULT_TIMEOUT = 20
|
||||
|
||||
DUT_STARTUP_CHECK_RETRY_COUNT = 5
|
||||
TEST_HISTROY_CHECK_TIMEOUT = 1
|
||||
TEST_HISTORY_CHECK_TIMEOUT = 1
|
||||
|
||||
|
||||
class TestCaseFailed(AssertionError):
|
||||
pass
|
||||
|
||||
|
||||
def format_test_case_config(test_case_data):
|
||||
@ -163,7 +167,7 @@ def reset_dut(dut):
|
||||
for _ in range(DUT_STARTUP_CHECK_RETRY_COUNT):
|
||||
dut.write("-")
|
||||
try:
|
||||
dut.expect("0 Tests 0 Failures 0 Ignored", timeout=TEST_HISTROY_CHECK_TIMEOUT)
|
||||
dut.expect("0 Tests 0 Failures 0 Ignored", timeout=TEST_HISTORY_CHECK_TIMEOUT)
|
||||
break
|
||||
except ExpectTimeout:
|
||||
pass
|
||||
@ -171,7 +175,7 @@ def reset_dut(dut):
|
||||
raise AssertionError("Reset {} ({}) failed!".format(dut.name, dut.port))
|
||||
|
||||
|
||||
def run_one_normal_case(dut, one_case, junit_test_case, failed_cases):
|
||||
def run_one_normal_case(dut, one_case, junit_test_case):
|
||||
|
||||
reset_dut(dut)
|
||||
|
||||
@ -194,9 +198,9 @@ def run_one_normal_case(dut, one_case, junit_test_case, failed_cases):
|
||||
if result:
|
||||
Utility.console_log("Success: " + one_case["name"], color="green")
|
||||
else:
|
||||
failed_cases.append(one_case["name"])
|
||||
Utility.console_log("Failed: " + one_case["name"], color="red")
|
||||
junit_test_case.add_failure_info(output)
|
||||
raise TestCaseFailed()
|
||||
|
||||
def handle_exception_reset(data):
|
||||
"""
|
||||
@ -242,7 +246,7 @@ def run_one_normal_case(dut, one_case, junit_test_case, failed_cases):
|
||||
timeout=one_case["timeout"])
|
||||
except ExpectTimeout:
|
||||
Utility.console_log("Timeout in expect", color="orange")
|
||||
junit_test_case.add_error_info("timeout")
|
||||
junit_test_case.add_failure_info("timeout")
|
||||
one_case_finish(False)
|
||||
break
|
||||
|
||||
@ -284,10 +288,12 @@ def run_unit_test_cases(env, extra_data):
|
||||
# create junit report test case
|
||||
junit_test_case = TinyFW.JunitReport.create_test_case("[{}] {}".format(ut_config, one_case["name"]))
|
||||
try:
|
||||
run_one_normal_case(dut, one_case, junit_test_case, failed_cases)
|
||||
TinyFW.JunitReport.test_case_finish(junit_test_case)
|
||||
run_one_normal_case(dut, one_case, junit_test_case)
|
||||
except TestCaseFailed:
|
||||
failed_cases.append(one_case["name"])
|
||||
except Exception as e:
|
||||
junit_test_case.add_error_info("Unexpected exception: " + str(e))
|
||||
junit_test_case.add_failure_info("Unexpected exception: " + str(e))
|
||||
finally:
|
||||
TinyFW.JunitReport.test_case_finish(junit_test_case)
|
||||
|
||||
# raise exception if any case fails
|
||||
@ -476,9 +482,9 @@ def run_multiple_devices_cases(env, extra_data):
|
||||
try:
|
||||
run_one_multiple_devices_case(duts, ut_config, env, one_case, failed_cases,
|
||||
one_case.get('app_bin'), junit_test_case)
|
||||
TinyFW.JunitReport.test_case_finish(junit_test_case)
|
||||
except Exception as e:
|
||||
junit_test_case.add_error_info("Unexpected exception: " + str(e))
|
||||
junit_test_case.add_failure_info("Unexpected exception: " + str(e))
|
||||
finally:
|
||||
TinyFW.JunitReport.test_case_finish(junit_test_case)
|
||||
|
||||
if failed_cases:
|
||||
@ -488,7 +494,7 @@ def run_multiple_devices_cases(env, extra_data):
|
||||
raise AssertionError("Unit Test Failed")
|
||||
|
||||
|
||||
def run_one_multiple_stage_case(dut, one_case, failed_cases, junit_test_case):
|
||||
def run_one_multiple_stage_case(dut, one_case, junit_test_case):
|
||||
reset_dut(dut)
|
||||
|
||||
dut.start_capture_raw_data()
|
||||
@ -523,7 +529,7 @@ def run_one_multiple_stage_case(dut, one_case, failed_cases, junit_test_case):
|
||||
err_msg = "Reset Check Failed: \r\n\tExpected: {}\r\n\tGet: {}".format(one_case["reset"],
|
||||
exception_reset_list)
|
||||
Utility.console_log(err_msg, color="orange")
|
||||
junit_test_case.add_error_info(err_msg)
|
||||
junit_test_case.add_failure_info(err_msg)
|
||||
else:
|
||||
# we allow omit reset in multi stage cases
|
||||
result = True
|
||||
@ -538,9 +544,9 @@ def run_one_multiple_stage_case(dut, one_case, failed_cases, junit_test_case):
|
||||
if result:
|
||||
Utility.console_log("Success: " + one_case["name"], color="green")
|
||||
else:
|
||||
failed_cases.append(one_case["name"])
|
||||
Utility.console_log("Failed: " + one_case["name"], color="red")
|
||||
junit_test_case.add_failure_info(output)
|
||||
raise TestCaseFailed()
|
||||
stage_finish.append("break")
|
||||
|
||||
def handle_exception_reset(data):
|
||||
@ -620,10 +626,12 @@ def run_multiple_stage_cases(env, extra_data):
|
||||
for one_case in case_config[ut_config]:
|
||||
junit_test_case = TinyFW.JunitReport.create_test_case("[{}] {}".format(ut_config, one_case["name"]))
|
||||
try:
|
||||
run_one_multiple_stage_case(dut, one_case, failed_cases, junit_test_case)
|
||||
TinyFW.JunitReport.test_case_finish(junit_test_case)
|
||||
run_one_multiple_stage_case(dut, one_case, junit_test_case)
|
||||
except TestCaseFailed:
|
||||
failed_cases.append(one_case["name"])
|
||||
except Exception as e:
|
||||
junit_test_case.add_error_info("Unexpected exception: " + str(e))
|
||||
junit_test_case.add_failure_info("Unexpected exception: " + str(e))
|
||||
finally:
|
||||
TinyFW.JunitReport.test_case_finish(junit_test_case)
|
||||
|
||||
# raise exception if any case fails
|
||||
|
||||
Loading…
Reference in New Issue
Block a user