diff --git a/tools/ci/python_packages/tiny_test_fw/TinyFW.py b/tools/ci/python_packages/tiny_test_fw/TinyFW.py index 45dbefe23c..37b8b8ed5b 100644 --- a/tools/ci/python_packages/tiny_test_fw/TinyFW.py +++ b/tools/ci/python_packages/tiny_test_fw/TinyFW.py @@ -166,21 +166,6 @@ def test_method(**kwargs): case_info["name"] = case_info["ID"] = test_func.__name__ case_info["junit_report_by_case"] = False - def _filter_ci_target(target, ci_target): - if not ci_target: - return target - if isinstance(target, str): - if isinstance(ci_target, str) and target == ci_target: - return ci_target - else: - if isinstance(ci_target, str) and ci_target in target: - return ci_target - elif isinstance(ci_target, list) and set(ci_target).issubset(set(target)): - return ci_target - raise ValueError('ci_target must be a subset of target') - - if os.getenv('CI_JOB_NAME') and 'ci_target' in kwargs: - kwargs['target'] = _filter_ci_target(kwargs['target'], kwargs['ci_target']) case_info.update(kwargs) @functools.wraps(test_func) @@ -230,7 +215,7 @@ def test_method(**kwargs): dut_dict = kwargs['dut_dict'] if target not in dut_dict: - raise Exception('target can only be {%s}' % ', '.join(dut_dict.keys())) + raise Exception('target can only be {%s} (case insensitive)' % ', '.join(dut_dict.keys())) dut = dut_dict[target] try: diff --git a/tools/ci/python_packages/tiny_test_fw/Utility/CIAssignTest.py b/tools/ci/python_packages/tiny_test_fw/Utility/CIAssignTest.py index 6b584c31c0..b89379992f 100644 --- a/tools/ci/python_packages/tiny_test_fw/Utility/CIAssignTest.py +++ b/tools/ci/python_packages/tiny_test_fw/Utility/CIAssignTest.py @@ -143,6 +143,7 @@ class AssignTest(object): DEFAULT_FILTER = { "category": "function", "ignore": False, + "supported_in_ci": True, } def __init__(self, test_case_path, ci_config_file, case_group=Group): diff --git a/tools/ci/python_packages/tiny_test_fw/Utility/CaseConfig.py b/tools/ci/python_packages/tiny_test_fw/Utility/CaseConfig.py index 6205a8e851..d016043aba 100644 --- a/tools/ci/python_packages/tiny_test_fw/Utility/CaseConfig.py +++ b/tools/ci/python_packages/tiny_test_fw/Utility/CaseConfig.py @@ -127,7 +127,6 @@ def filter_test_cases(test_methods, case_filter): * user case filter is ``chip: ["esp32", "esp32c"]``, case attribute is ``chip: "esp32"`` * user case filter is ``chip: "esp32"``, case attribute is ``chip: "esp32"`` - :param test_methods: a list of test methods functions :param case_filter: case filter :return: filtered test methods diff --git a/tools/ci/python_packages/tiny_test_fw/Utility/SearchCases.py b/tools/ci/python_packages/tiny_test_fw/Utility/SearchCases.py index 075f7a1cca..e9a8e29b49 100644 --- a/tools/ci/python_packages/tiny_test_fw/Utility/SearchCases.py +++ b/tools/ci/python_packages/tiny_test_fw/Utility/SearchCases.py @@ -106,6 +106,16 @@ class Search(object): key = replicate_config.pop() replicated_cases = _replicate_for_key(replicated_cases, key, case.case_info[key]) + # mark the cases with targets not in ci_target + for case in replicated_cases: + ci_target = case.case_info['ci_target'] + if isinstance(ci_target, str): + ci_target = [ci_target] + if not ci_target or case.case_info['target'] in ci_target: + case.case_info['supported_in_ci'] = True + else: + case.case_info['supported_in_ci'] = False + return replicated_cases @classmethod diff --git a/tools/ci/python_packages/ttfw_idf/__init__.py b/tools/ci/python_packages/ttfw_idf/__init__.py index 5aa4f3426e..d5d94956da 100644 --- a/tools/ci/python_packages/ttfw_idf/__init__.py +++ b/tools/ci/python_packages/ttfw_idf/__init__.py @@ -11,9 +11,12 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import functools import os import re +from typing import Union + from tiny_test_fw import TinyFW, Utility from .IDFApp import IDFApp, Example, LoadableElfTestApp, UT, TestApp # noqa: export all Apps for users from .IDFDUT import IDFDUT, ESP32DUT, ESP32S2DUT, ESP8266DUT, ESP32QEMUDUT # noqa: export DUTs for users @@ -30,6 +33,28 @@ def format_case_id(chip, case_name): return "{}.{}".format(chip, case_name) +def upper_list(text): # type: (Union[str, unicode, list]) -> list + if isinstance(text, basestring): # It's not working in python3 + res = [text.upper()] + else: + res = [item.upper() for item in text] + return res + + +def ci_target_check(func): + @functools.wraps(func) + def wrapper(**kwargs): + target = upper_list(kwargs.get('target', [])) + ci_target = upper_list(kwargs.get('ci_target', [])) + if not set(ci_target).issubset(set(target)): + raise ValueError('ci_target must be a subset of target') + + return func(**kwargs) + + return wrapper + + +@ci_target_check def idf_example_test(app=Example, target="ESP32", ci_target=None, module="examples", execution_time=1, level="example", erase_nvs=True, config_name=None, **kwargs): """ @@ -48,7 +73,7 @@ def idf_example_test(app=Example, target="ESP32", ci_target=None, module="exampl """ def test(func): - original_method = TinyFW.test_method(app=app, target=target, ci_target=ci_target, module=module, + original_method = TinyFW.test_method(app=app, target=upper_list(target), ci_target=upper_list(ci_target), module=module, execution_time=execution_time, level=level, dut_dict=TARGET_DUT_CLS_DICT, erase_nvs=erase_nvs, **kwargs) test_func = original_method(func) @@ -58,6 +83,7 @@ def idf_example_test(app=Example, target="ESP32", ci_target=None, module="exampl return test +@ci_target_check def idf_unit_test(app=UT, target="ESP32", ci_target=None, module="unit-test", execution_time=1, level="unit", erase_nvs=True, **kwargs): """ @@ -75,7 +101,7 @@ def idf_unit_test(app=UT, target="ESP32", ci_target=None, module="unit-test", ex """ def test(func): - original_method = TinyFW.test_method(app=app, target=target, ci_target=ci_target, module=module, + original_method = TinyFW.test_method(app=app, target=upper_list(target), ci_target=upper_list(ci_target), module=module, execution_time=execution_time, level=level, dut_dict=TARGET_DUT_CLS_DICT, erase_nvs=erase_nvs, **kwargs) test_func = original_method(func) @@ -85,6 +111,7 @@ def idf_unit_test(app=UT, target="ESP32", ci_target=None, module="unit-test", ex return test +@ci_target_check def idf_custom_test(app=TestApp, target="ESP32", ci_target=None, module="misc", execution_time=1, level="integration", erase_nvs=True, config_name=None, group="test-apps", **kwargs): """ @@ -104,7 +131,7 @@ def idf_custom_test(app=TestApp, target="ESP32", ci_target=None, module="misc", """ def test(func): - original_method = TinyFW.test_method(app=app, target=target, ci_target=ci_target, module=module, + original_method = TinyFW.test_method(app=app, target=upper_list(target), ci_target=upper_list(ci_target), module=module, execution_time=execution_time, level=level, dut_dict=TARGET_DUT_CLS_DICT, erase_nvs=erase_nvs, **kwargs) test_func = original_method(func)