From 7e0bb1dabd9da07ea337db84ab9124c8a3bc701a Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Sun, 22 May 2022 00:38:17 +0800 Subject: [PATCH] ci: use tags as markers --- .gitlab/ci/target-test.yml | 145 +++++++------------------ conftest.py | 73 ++++++++++--- tools/ci/python_packages/gitlab_api.py | 13 +++ tools/ci/utils.sh | 15 ++- 4 files changed, 118 insertions(+), 128 deletions(-) diff --git a/.gitlab/ci/target-test.yml b/.gitlab/ci/target-test.yml index a8bd3cdde8..0c4e93b299 100644 --- a/.gitlab/ci/target-test.yml +++ b/.gitlab/ci/target-test.yml @@ -3,7 +3,6 @@ stage: target_test timeout: 1 hour extends: .before_script_pytest - tags: [$TARGET, $ENV_MARKER] artifacts: when: always paths: @@ -14,7 +13,11 @@ expire_in: 1 week script: - retry_failed git clone $KNOWN_FAILURE_CASES_REPO known_failure_cases - - pytest $TEST_DIR --target $TARGET -m $ENV_MARKER --junitxml=XUNIT_RESULT.xml --known-failure-cases-file known_failure_cases/known_failure_cases.txt + # using runner tags as markers to filter the test cases + # Runner tags are comma separated, replace the comma with " and " for markers + - job_tags=$(python tools/ci/python_packages/gitlab_api.py get_job_tags $CI_PROJECT_ID --job_id $CI_JOB_ID) + - markers=$(echo $job_tags | sed -e "s/,/ and /g") + - run_cmd pytest $TEST_DIR -m \"${markers}\" --junitxml=XUNIT_RESULT.xml --known-failure-cases-file known_failure_cases/known_failure_cases.txt .pytest_examples_dir_template: extends: .pytest_template @@ -27,9 +30,7 @@ example_test_pytest_esp32_generic: - .rules:test:example_test-esp32 needs: - build_pytest_examples_esp32 - variables: - TARGET: ESP32 - ENV_MARKER: generic + tags: [ esp32, generic ] example_test_pytest_esp32_ir_transceiver: extends: @@ -37,9 +38,7 @@ example_test_pytest_esp32_ir_transceiver: - .rules:test:example_test-esp32 needs: - build_pytest_examples_esp32 - variables: - TARGET: ESP32 - ENV_MARKER: ir_transceiver + tags: [ esp32, ir_transceiver ] example_test_pytest_esp32s2_generic: extends: @@ -47,9 +46,7 @@ example_test_pytest_esp32s2_generic: - .rules:test:example_test-esp32s2 needs: - build_pytest_examples_esp32s2 - variables: - TARGET: ESP32S2 - ENV_MARKER: generic + tags: [ esp32s2, generic ] example_test_pytest_esp32s3_generic: extends: @@ -57,9 +54,7 @@ example_test_pytest_esp32s3_generic: - .rules:test:example_test-esp32s3 needs: - build_pytest_examples_esp32s3 - variables: - TARGET: ESP32S3 - ENV_MARKER: generic + tags: [ esp32s3, generic ] example_test_pytest_esp32c2_generic: extends: @@ -67,9 +62,7 @@ example_test_pytest_esp32c2_generic: - .rules:test:example_test-esp32c2 needs: - build_pytest_examples_esp32c2 - variables: - TARGET: ESP32C2 - ENV_MARKER: generic + tags: [ esp32c2, generic ] example_test_pytest_esp32c3_generic: extends: @@ -77,9 +70,7 @@ example_test_pytest_esp32c3_generic: - .rules:test:example_test-esp32c3 needs: - build_pytest_examples_esp32c3 - variables: - TARGET: ESP32C3 - ENV_MARKER: generic + tags: [ esp32c3, generic ] example_test_pytest_esp32c3_flash_suspend: extends: @@ -87,9 +78,7 @@ example_test_pytest_esp32c3_flash_suspend: - .rules:test:example_test-esp32c3 needs: - build_pytest_examples_esp32c3 - variables: - TARGET: ESP32C3 - ENV_MARKER: flash_suspend + tags: [ esp32c3, flash_suspend ] example_test_pytest_esp32_ethernet_ota: extends: @@ -97,9 +86,7 @@ example_test_pytest_esp32_ethernet_ota: - .rules:test:example_test-esp32 needs: - build_pytest_examples_esp32 - variables: - TARGET: ESP32 - ENV_MARKER: ethernet_ota + tags: [ esp32, ethernet_ota ] example_test_pytest_esp32_wifi_ota: extends: @@ -107,9 +94,7 @@ example_test_pytest_esp32_wifi_ota: - .rules:test:example_test-esp32 needs: - build_pytest_examples_esp32 - variables: - TARGET: ESP32 - ENV_MARKER: wifi_ota + tags: [ esp32, wifi_ota ] example_test_pytest_esp32_flash_encryption_ota: extends: @@ -117,9 +102,7 @@ example_test_pytest_esp32_flash_encryption_ota: - .rules:test:example_test-esp32 needs: - build_pytest_examples_esp32 - variables: - TARGET: ESP32 - ENV_MARKER: flash_encryption_ota + tags: [ esp32, flash_encryption_ota ] example_test_pytest_esp32c3_flash_encryption_wifi_ota: extends: @@ -127,9 +110,7 @@ example_test_pytest_esp32c3_flash_encryption_wifi_ota: - .rules:test:example_test-esp32c3 needs: - build_pytest_examples_esp32c3 - variables: - TARGET: ESP32C3 - ENV_MARKER: flash_encryption_wifi_ota + tags: [ esp32c3, flash_encryption_wifi_ota ] example_test_pytest_esp32_ethernet: extends: @@ -137,9 +118,7 @@ example_test_pytest_esp32_ethernet: - .rules:test:example_test-esp32 needs: - build_pytest_examples_esp32 - variables: - TARGET: ESP32 - ENV_MARKER: ethernet + tags: [ esp32, ethernet] example_test_pytest_esp32_8mb_flash: extends: @@ -147,9 +126,7 @@ example_test_pytest_esp32_8mb_flash: - .rules:test:example_test-esp32 needs: - build_pytest_examples_esp32 - variables: - TARGET: ESP32 - ENV_MARKER: ethernet_flash_8m + tags: [ esp32, ethernet_flash_8m ] example_test_pytest_esp32_wifi: extends: @@ -157,9 +134,7 @@ example_test_pytest_esp32_wifi: - .rules:test:example_test-esp32 needs: - build_pytest_examples_esp32 - variables: - TARGET: ESP32 - ENV_MARKER: wifi + tags: [ esp32, wifi ] example_test_pytest_esp32_wifi_bt: extends: @@ -167,9 +142,7 @@ example_test_pytest_esp32_wifi_bt: - .rules:test:example_test-esp32 needs: - build_pytest_examples_esp32 - variables: - TARGET: ESP32 - ENV_MARKER: wifi_bt + tags: [ esp32, wifi_bt ] example_test_pytest_esp32_ethernet_ip101: extends: @@ -177,9 +150,7 @@ example_test_pytest_esp32_ethernet_ip101: - .rules:test:example_test-esp32 needs: - build_pytest_examples_esp32 - variables: - TARGET: ESP32 - ENV_MARKER: ip101 + tags: [ esp32, ip101 ] example_test_pytest_esp32_flash_encryption: extends: @@ -187,9 +158,7 @@ example_test_pytest_esp32_flash_encryption: - .rules:test:example_test-esp32 needs: - build_pytest_examples_esp32 - variables: - TARGET: ESP32 - ENV_MARKER: flash_encryption + tags: [ esp32, flash_encryption ] example_test_pytest_esp32_multi_dut_generic: extends: @@ -197,9 +166,7 @@ example_test_pytest_esp32_multi_dut_generic: - .rules:test:example_test-esp32 needs: - build_pytest_examples_esp32 - variables: - TARGET: ESP32 - ENV_MARKER: multi_dut_generic + tags: [ esp32, multi_dut_generic ] example_test_pytest_esp32c3_flash_encryption: extends: @@ -207,9 +174,7 @@ example_test_pytest_esp32c3_flash_encryption: - .rules:test:example_test-esp32c3 needs: - build_pytest_examples_esp32c3 - variables: - TARGET: ESP32C3 - ENV_MARKER: flash_encryption + tags: [ esp32c3, flash_encryption ] example_test_pytest_esp32s2_deepsleep: # Temp tag, will be removed IDF-5213 extends: @@ -217,9 +182,7 @@ example_test_pytest_esp32s2_deepsleep: # Temp tag, will be removed IDF-5213 - .rules:test:example_test-esp32s2 needs: - build_pytest_examples_esp32s2 - variables: - TARGET: ESP32S2 - ENV_MARKER: deepsleep_temp_tag + tags: [ esp32s2, deepsleep_temp_tag ] example_test_pytest_esp32s3_deepsleep: # Temp tag, will be removed IDF-5213 extends: @@ -227,9 +190,7 @@ example_test_pytest_esp32s3_deepsleep: # Temp tag, will be removed IDF-5213 - .rules:test:example_test-esp32s3 needs: - build_pytest_examples_esp32s3 - variables: - TARGET: ESP32S3 - ENV_MARKER: deepsleep_temp_tag + tags: [esp32s3, deepsleep_temp_tag ] .pytest_components_dir_template: extends: .pytest_template @@ -242,9 +203,7 @@ component_ut_pytest_esp32_generic: - .rules:test:component_ut-esp32 needs: - build_pytest_components_esp32 - variables: - TARGET: ESP32 - ENV_MARKER: generic + tags: [ esp32, generic ] component_ut_pytest_esp32_ip101: extends: @@ -252,9 +211,7 @@ component_ut_pytest_esp32_ip101: - .rules:test:component_ut-esp32 needs: - build_pytest_components_esp32 - variables: - TARGET: ESP32 - ENV_MARKER: ip101 + tags: [ esp32, ip101 ] component_ut_pytest_esp32_lan8720: extends: @@ -262,9 +219,7 @@ component_ut_pytest_esp32_lan8720: - .rules:labels-protected:lan8720 # FIXME: IDFCI-1176 needs: - build_pytest_components_esp32 - variables: - TARGET: ESP32 - ENV_MARKER: lan8720 + tags: [ esp32, lan8720 ] component_ut_pytest_esp32s2_generic: extends: @@ -272,9 +227,7 @@ component_ut_pytest_esp32s2_generic: - .rules:test:component_ut-esp32s2 needs: - build_pytest_components_esp32s2 - variables: - TARGET: ESP32S2 - ENV_MARKER: generic + tags: [ esp32s2, generic ] component_ut_pytest_esp32s3_generic: extends: @@ -282,9 +235,7 @@ component_ut_pytest_esp32s3_generic: - .rules:test:component_ut-esp32s3 needs: - build_pytest_components_esp32s3 - variables: - TARGET: ESP32S3 - ENV_MARKER: generic + tags: [ esp32s3, generic ] component_ut_pytest_esp32s3_octal_psram: extends: @@ -292,9 +243,7 @@ component_ut_pytest_esp32s3_octal_psram: - .rules:test:component_ut-esp32s3 needs: - build_pytest_components_esp32s3 - variables: - TARGET: ESP32S3 - ENV_MARKER: octal_psram + tags: [ esp32s3, octal_psram ] component_ut_pytest_esp32c2_generic: extends: @@ -302,9 +251,7 @@ component_ut_pytest_esp32c2_generic: - .rules:test:component_ut-esp32c2 needs: - build_pytest_components_esp32c2 - variables: - TARGET: ESP32C2 - ENV_MARKER: generic + tags: [ esp32c2, generic ] component_ut_pytest_esp32c3_generic: extends: @@ -312,9 +259,7 @@ component_ut_pytest_esp32c3_generic: - .rules:test:component_ut-esp32c3 needs: - build_pytest_components_esp32c3 - variables: - TARGET: ESP32C3 - ENV_MARKER: generic + tags: [ esp32c3, generic ] .pytest_test_apps_dir_template: extends: .pytest_template @@ -327,9 +272,8 @@ test_app_test_pytest_esp32_generic: - .rules:test:custom_test-esp32 needs: - build_pytest_test_apps_esp32 + tags: [ esp32, generic ] variables: - TARGET: ESP32 - ENV_MARKER: generic SETUP_TOOLS: "1" # need gdb test_app_test_pytest_esp32s2_generic: @@ -338,9 +282,8 @@ test_app_test_pytest_esp32s2_generic: - .rules:test:custom_test-esp32s2 needs: - build_pytest_test_apps_esp32s2 + tags: [ esp32s2, generic ] variables: - TARGET: ESP32S2 - ENV_MARKER: generic SETUP_TOOLS: "1" # need gdb test_app_test_pytest_esp32s3_generic: @@ -349,9 +292,7 @@ test_app_test_pytest_esp32s3_generic: - .rules:test:custom_test-esp32s3 needs: - build_pytest_test_apps_esp32s3 - variables: - TARGET: ESP32S3 - ENV_MARKER: generic + tags: [ esp32s3, generic ] test_app_test_pytest_esp32c2_generic: extends: @@ -359,9 +300,7 @@ test_app_test_pytest_esp32c2_generic: - .rules:test:custom_test-esp32c2 needs: - build_pytest_test_apps_esp32c2 - variables: - TARGET: ESP32C2 - ENV_MARKER: generic + tags: [ esp32c2, generic ] test_app_test_pytest_esp32c3_generic: extends: @@ -369,9 +308,7 @@ test_app_test_pytest_esp32c3_generic: - .rules:test:custom_test-esp32c3 needs: - build_pytest_test_apps_esp32c3 - variables: - TARGET: ESP32C3 - ENV_MARKER: generic + tags: [ esp32c3, generic ] test_app_test_pytest_esp32s2_usb_host: extends: @@ -379,9 +316,7 @@ test_app_test_pytest_esp32s2_usb_host: - .rules:test:custom_test-esp32s2 needs: - build_pytest_test_apps_esp32s2 - variables: - TARGET: ESP32S2 - ENV_MARKER: usb_host + tags: [ esp32s2, usb_host ] # for parallel jobs, CI_JOB_NAME will be "job_name index/total" (for example, "IT_001 1/2") # we need to convert to pattern "job_name_index.yml" diff --git a/conftest.py b/conftest.py index 29c08a87f1..ec1534d4a8 100644 --- a/conftest.py +++ b/conftest.py @@ -42,10 +42,7 @@ DEFAULT_SDKCONFIG = 'default' # Help Functions # ################## def is_target_marker(marker: str) -> bool: - if marker.startswith('esp32'): - return True - - if marker.startswith('esp8'): + if marker.startswith('esp32') or marker.startswith('esp8') or marker == 'linux': return True return False @@ -59,6 +56,26 @@ def item_marker_names(item: Item) -> List[str]: return [marker.name for marker in item.iter_markers()] +def get_target_marker(markexpr: str) -> str: + candidates = set() + # we use `-m "esp32 and generic"` in our CI to filter the test cases + for marker in markexpr.split('and'): + marker = marker.strip() + if is_target_marker(marker): + candidates.add(marker) + + if len(candidates) > 1: + raise ValueError( + f'Specified more than one target markers: {candidates}. Please specify no more than one.' + ) + elif len(candidates) == 1: + return candidates.pop() + else: + raise ValueError( + 'Please specify one target marker via "--target [TARGET]" or via "-m [TARGET]"' + ) + + ############ # Fixtures # ############ @@ -79,15 +96,19 @@ def session_tempdir() -> str: def log_minimum_free_heap_size(dut: IdfDut, config: str) -> Callable[..., None]: def real_func() -> None: res = dut.expect(r'Minimum free heap size: (\d+) bytes') - logging.info('\n------ heap size info ------\n' - '[app_name] {}\n' - '[config_name] {}\n' - '[target] {}\n' - '[minimum_free_heap_size] {} Bytes\n' - '------ heap size end ------'.format(os.path.basename(dut.app.app_path), - config, - dut.target, - res.group(1).decode('utf8'))) + logging.info( + '\n------ heap size info ------\n' + '[app_name] {}\n' + '[config_name] {}\n' + '[target] {}\n' + '[minimum_free_heap_size] {} Bytes\n' + '------ heap size end ------'.format( + os.path.basename(dut.app.app_path), + config, + dut.target, + res.group(1).decode('utf8'), + ) + ) return real_func @@ -183,8 +204,20 @@ _idf_pytest_embedded_key = pytest.StashKey['IdfPytestEmbedded'] def pytest_configure(config: Config) -> None: + # cli option "--target" + target = config.getoption('target') or '' + + help_commands = ['--help', '--fixtures', '--markers', '--version'] + for cmd in help_commands: + if cmd in config.invocation_params.args: + target = 'unneeded' + break + + if not target: # also could specify through markexpr via "-m" + target = get_target_marker(config.getoption('markexpr') or '') + config.stash[_idf_pytest_embedded_key] = IdfPytestEmbedded( - target=config.getoption('target'), + target=target, sdkconfig=config.getoption('sdkconfig'), known_failure_cases_file=config.getoption('known_failure_cases_file'), ) @@ -218,7 +251,11 @@ class IdfPytestEmbedded: @property def failed_cases(self) -> List[str]: - return [case for case, is_known, is_xfail in self._failed_cases if not is_known and not is_xfail] + return [ + case + for case, is_known, is_xfail in self._failed_cases + if not is_known and not is_xfail + ] @property def known_failure_cases(self) -> List[str]: @@ -268,13 +305,13 @@ class IdfPytestEmbedded: # add markers for special markers for item in items: - if 'supported_targets' in item_marker_names(item): + if 'supported_targets' in item.keywords: for _target in SUPPORTED_TARGETS: item.add_marker(_target) - if 'preview_targets' in item_marker_names(item): + if 'preview_targets' in item.keywords: for _target in PREVIEW_TARGETS: item.add_marker(_target) - if 'all_targets' in item_marker_names(item): + if 'all_targets' in item.keywords: for _target in [*SUPPORTED_TARGETS, *PREVIEW_TARGETS]: item.add_marker(_target) diff --git a/tools/ci/python_packages/gitlab_api.py b/tools/ci/python_packages/gitlab_api.py index ba2b5e4ef3..62f20185e4 100644 --- a/tools/ci/python_packages/gitlab_api.py +++ b/tools/ci/python_packages/gitlab_api.py @@ -204,6 +204,16 @@ class Gitlab(object): return os.path.join(os.path.realpath(destination), root_name) + def get_job_tags(self, job_id: int) -> str: + """ + Get tags of a job + + :param job_id: job id + :return: comma-separated tags of the job + """ + job = self.project.jobs.get(job_id) + return ','.join(job.tag_list) + def main() -> None: parser = argparse.ArgumentParser() @@ -231,6 +241,9 @@ def main() -> None: elif args.action == 'get_project_id': ret = gitlab_inst.get_project_id(args.project_name) print('project id: {}'.format(ret)) + elif args.action == 'get_job_tags': + ret = gitlab_inst.get_job_tags(args.job_id) + print(ret) if __name__ == '__main__': diff --git a/tools/ci/utils.sh b/tools/ci/utils.sh index 00a0768da1..f7cb597781 100644 --- a/tools/ci/utils.sh +++ b/tools/ci/utils.sh @@ -62,17 +62,22 @@ function warning() { } function run_cmd() { + local args=( "$@" ) + local cmd="${args[@]}" local start=$(date +%s) - eval "$@" + + info "\$ ${cmd}" + eval "${cmd}" + local ret=$? local end=$(date +%s) - local duration=$((end - start)) + local runtime=$((end-start)) if [[ $ret -eq 0 ]]; then - info "(\$ $*) succeeded in ${duration} seconds." + info "==> '\$ ${cmd}' succeeded in ${runtime} seconds." return 0 - else - error "(\$ $*) failed in ${duration} seconds." + echo + error "==> '\$ ${cmd}' failed (${ret}) in ${runtime} seconds." return $ret fi }