ci: use tags as markers

.gitlab/ci/target-test.yml

@@ -3,7 +3,6 @@
   stage: target_test
   timeout: 1 hour
   extends: .before_script_pytest
-  tags: [$TARGET, $ENV_MARKER]
   artifacts:
     when: always
     paths:
@@ -14,7 +13,11 @@
     expire_in: 1 week
   script:
     - retry_failed git clone $KNOWN_FAILURE_CASES_REPO known_failure_cases
-    - pytest $TEST_DIR --target $TARGET -m $ENV_MARKER --junitxml=XUNIT_RESULT.xml --known-failure-cases-file known_failure_cases/known_failure_cases.txt
+    # using runner tags as markers to filter the test cases
+    # Runner tags are comma separated, replace the comma with " and " for markers
+    - job_tags=$(python tools/ci/python_packages/gitlab_api.py get_job_tags $CI_PROJECT_ID --job_id $CI_JOB_ID)
+    - markers=$(echo $job_tags | sed -e "s/,/ and /g")
+    - run_cmd pytest $TEST_DIR -m \"${markers}\" --junitxml=XUNIT_RESULT.xml --known-failure-cases-file known_failure_cases/known_failure_cases.txt
 
 .pytest_examples_dir_template:
   extends: .pytest_template
@@ -27,9 +30,7 @@ example_test_pytest_esp32_generic:
     - .rules:test:example_test-esp32
   needs:
     - build_pytest_examples_esp32
-  variables:
+  tags: [ esp32, generic ]
-    TARGET: ESP32
-    ENV_MARKER: generic
 
 example_test_pytest_esp32_ir_transceiver:
   extends:
@@ -37,9 +38,7 @@ example_test_pytest_esp32_ir_transceiver:
     - .rules:test:example_test-esp32
   needs:
     - build_pytest_examples_esp32
-  variables:
+  tags: [ esp32, ir_transceiver ]
-    TARGET: ESP32
-    ENV_MARKER: ir_transceiver
 
 example_test_pytest_esp32s2_generic:
   extends:
@@ -47,9 +46,7 @@ example_test_pytest_esp32s2_generic:
     - .rules:test:example_test-esp32s2
   needs:
     - build_pytest_examples_esp32s2
-  variables:
+  tags: [ esp32s2, generic ]
-    TARGET: ESP32S2
-    ENV_MARKER: generic
 
 example_test_pytest_esp32s3_generic:
   extends:
@@ -57,9 +54,7 @@ example_test_pytest_esp32s3_generic:
     - .rules:test:example_test-esp32s3
   needs:
     - build_pytest_examples_esp32s3
-  variables:
+  tags: [ esp32s3, generic ]
-    TARGET: ESP32S3
-    ENV_MARKER: generic
 
 example_test_pytest_esp32c2_generic:
   extends:
@@ -67,9 +62,7 @@ example_test_pytest_esp32c2_generic:
     - .rules:test:example_test-esp32c2
   needs:
     - build_pytest_examples_esp32c2
-  variables:
+  tags: [ esp32c2, generic ]
-    TARGET: ESP32C2
-    ENV_MARKER: generic
 
 example_test_pytest_esp32c3_generic:
   extends:
@@ -77,9 +70,7 @@ example_test_pytest_esp32c3_generic:
     - .rules:test:example_test-esp32c3
   needs:
     - build_pytest_examples_esp32c3
-  variables:
+  tags: [ esp32c3, generic ]
-    TARGET: ESP32C3
-    ENV_MARKER: generic
 
 example_test_pytest_esp32c3_flash_suspend:
   extends:
@@ -87,9 +78,7 @@ example_test_pytest_esp32c3_flash_suspend:
     - .rules:test:example_test-esp32c3
   needs:
     - build_pytest_examples_esp32c3
-  variables:
+  tags: [ esp32c3, flash_suspend ]
-    TARGET: ESP32C3
-    ENV_MARKER: flash_suspend
 
 example_test_pytest_esp32_ethernet_ota:
   extends:
@@ -97,9 +86,7 @@ example_test_pytest_esp32_ethernet_ota:
     - .rules:test:example_test-esp32
   needs:
     - build_pytest_examples_esp32
-  variables:
+  tags: [ esp32, ethernet_ota ]
-    TARGET: ESP32
-    ENV_MARKER: ethernet_ota
 
 example_test_pytest_esp32_wifi_ota:
   extends:
@@ -107,9 +94,7 @@ example_test_pytest_esp32_wifi_ota:
     - .rules:test:example_test-esp32
   needs:
     - build_pytest_examples_esp32
-  variables:
+  tags: [ esp32, wifi_ota ]
-    TARGET: ESP32
-    ENV_MARKER: wifi_ota
 
 example_test_pytest_esp32_flash_encryption_ota:
   extends:
@@ -117,9 +102,7 @@ example_test_pytest_esp32_flash_encryption_ota:
     - .rules:test:example_test-esp32
   needs:
     - build_pytest_examples_esp32
-  variables:
+  tags: [ esp32, flash_encryption_ota ]
-    TARGET: ESP32
-    ENV_MARKER: flash_encryption_ota
 
 example_test_pytest_esp32c3_flash_encryption_wifi_ota:
   extends:
@@ -127,9 +110,7 @@ example_test_pytest_esp32c3_flash_encryption_wifi_ota:
     - .rules:test:example_test-esp32c3
   needs:
     - build_pytest_examples_esp32c3
-  variables:
+  tags: [ esp32c3, flash_encryption_wifi_ota ]
-    TARGET: ESP32C3
-    ENV_MARKER: flash_encryption_wifi_ota
 
 example_test_pytest_esp32_ethernet:
   extends:
@@ -137,9 +118,7 @@ example_test_pytest_esp32_ethernet:
     - .rules:test:example_test-esp32
   needs:
     - build_pytest_examples_esp32
-  variables:
+  tags: [ esp32, ethernet]
-    TARGET: ESP32
-    ENV_MARKER: ethernet
 
 example_test_pytest_esp32_8mb_flash:
   extends:
@@ -147,9 +126,7 @@ example_test_pytest_esp32_8mb_flash:
     - .rules:test:example_test-esp32
   needs:
     - build_pytest_examples_esp32
-  variables:
+  tags: [ esp32, ethernet_flash_8m ]
-    TARGET: ESP32
-    ENV_MARKER: ethernet_flash_8m
 
 example_test_pytest_esp32_wifi:
   extends:
@@ -157,9 +134,7 @@ example_test_pytest_esp32_wifi:
     - .rules:test:example_test-esp32
   needs:
     - build_pytest_examples_esp32
-  variables:
+  tags: [ esp32, wifi ]
-    TARGET: ESP32
-    ENV_MARKER: wifi
 
 example_test_pytest_esp32_wifi_bt:
   extends:
@@ -167,9 +142,7 @@ example_test_pytest_esp32_wifi_bt:
     - .rules:test:example_test-esp32
   needs:
     - build_pytest_examples_esp32
-  variables:
+  tags: [ esp32, wifi_bt ]
-    TARGET: ESP32
-    ENV_MARKER: wifi_bt
 
 example_test_pytest_esp32_ethernet_ip101:
   extends:
@@ -177,9 +150,7 @@ example_test_pytest_esp32_ethernet_ip101:
     - .rules:test:example_test-esp32
   needs:
     - build_pytest_examples_esp32
-  variables:
+  tags: [ esp32, ip101 ]
-    TARGET: ESP32
-    ENV_MARKER: ip101
 
 example_test_pytest_esp32_flash_encryption:
   extends:
@@ -187,9 +158,7 @@ example_test_pytest_esp32_flash_encryption:
     - .rules:test:example_test-esp32
   needs:
     - build_pytest_examples_esp32
-  variables:
+  tags: [ esp32, flash_encryption ]
-    TARGET: ESP32
-    ENV_MARKER: flash_encryption
 
 example_test_pytest_esp32_multi_dut_generic:
   extends:
@@ -197,9 +166,7 @@ example_test_pytest_esp32_multi_dut_generic:
     - .rules:test:example_test-esp32
   needs:
     - build_pytest_examples_esp32
-  variables:
+  tags: [ esp32, multi_dut_generic ]
-    TARGET: ESP32
-    ENV_MARKER: multi_dut_generic
 
 example_test_pytest_esp32c3_flash_encryption:
   extends:
@@ -207,9 +174,7 @@ example_test_pytest_esp32c3_flash_encryption:
     - .rules:test:example_test-esp32c3
   needs:
     - build_pytest_examples_esp32c3
-  variables:
+  tags: [ esp32c3, flash_encryption ]
-    TARGET: ESP32C3
-    ENV_MARKER: flash_encryption
 
 example_test_pytest_esp32s2_deepsleep: # Temp tag, will be removed IDF-5213
   extends:
@@ -217,9 +182,7 @@ example_test_pytest_esp32s2_deepsleep: # Temp tag, will be removed IDF-5213
     - .rules:test:example_test-esp32s2
   needs:
     - build_pytest_examples_esp32s2
-  variables:
+  tags: [ esp32s2, deepsleep_temp_tag ]
-    TARGET: ESP32S2
-    ENV_MARKER: deepsleep_temp_tag
 
 example_test_pytest_esp32s3_deepsleep: # Temp tag, will be removed IDF-5213
   extends:
@@ -227,9 +190,7 @@ example_test_pytest_esp32s3_deepsleep: # Temp tag, will be removed IDF-5213
     - .rules:test:example_test-esp32s3
   needs:
     - build_pytest_examples_esp32s3
-  variables:
+  tags: [esp32s3, deepsleep_temp_tag ]
-    TARGET: ESP32S3
-    ENV_MARKER: deepsleep_temp_tag
 
 .pytest_components_dir_template:
   extends: .pytest_template
@@ -242,9 +203,7 @@ component_ut_pytest_esp32_generic:
     - .rules:test:component_ut-esp32
   needs:
     - build_pytest_components_esp32
-  variables:
+  tags: [ esp32, generic ]
-    TARGET: ESP32
-    ENV_MARKER: generic
 
 component_ut_pytest_esp32_ip101:
   extends:
@@ -252,9 +211,7 @@ component_ut_pytest_esp32_ip101:
     - .rules:test:component_ut-esp32
   needs:
     - build_pytest_components_esp32
-  variables:
+  tags: [ esp32, ip101 ]
-    TARGET: ESP32
-    ENV_MARKER: ip101
 
 component_ut_pytest_esp32_lan8720:
   extends:
@@ -262,9 +219,7 @@ component_ut_pytest_esp32_lan8720:
     - .rules:labels-protected:lan8720  # FIXME: IDFCI-1176
   needs:
     - build_pytest_components_esp32
-  variables:
+  tags: [ esp32, lan8720 ]
-    TARGET: ESP32
-    ENV_MARKER: lan8720
 
 component_ut_pytest_esp32s2_generic:
   extends:
@@ -272,9 +227,7 @@ component_ut_pytest_esp32s2_generic:
     - .rules:test:component_ut-esp32s2
   needs:
     - build_pytest_components_esp32s2
-  variables:
+  tags: [ esp32s2, generic ]
-    TARGET: ESP32S2
-    ENV_MARKER: generic
 
 component_ut_pytest_esp32s3_generic:
   extends:
@@ -282,9 +235,7 @@ component_ut_pytest_esp32s3_generic:
     - .rules:test:component_ut-esp32s3
   needs:
     - build_pytest_components_esp32s3
-  variables:
+  tags: [ esp32s3, generic ]
-    TARGET: ESP32S3
-    ENV_MARKER: generic
 
 component_ut_pytest_esp32s3_octal_psram:
   extends:
@@ -292,9 +243,7 @@ component_ut_pytest_esp32s3_octal_psram:
     - .rules:test:component_ut-esp32s3
   needs:
     - build_pytest_components_esp32s3
-  variables:
+  tags: [ esp32s3, octal_psram ]
-    TARGET: ESP32S3
-    ENV_MARKER: octal_psram
 
 component_ut_pytest_esp32c2_generic:
   extends:
@@ -302,9 +251,7 @@ component_ut_pytest_esp32c2_generic:
     - .rules:test:component_ut-esp32c2
   needs:
     - build_pytest_components_esp32c2
-  variables:
+  tags: [ esp32c2, generic ]
-    TARGET: ESP32C2
-    ENV_MARKER: generic
 
 component_ut_pytest_esp32c3_generic:
   extends:
@@ -312,9 +259,7 @@ component_ut_pytest_esp32c3_generic:
     - .rules:test:component_ut-esp32c3
   needs:
     - build_pytest_components_esp32c3
-  variables:
+  tags: [ esp32c3, generic ]
-    TARGET: ESP32C3
-    ENV_MARKER: generic
 
 .pytest_test_apps_dir_template:
   extends: .pytest_template
@@ -327,9 +272,8 @@ test_app_test_pytest_esp32_generic:
     - .rules:test:custom_test-esp32
   needs:
     - build_pytest_test_apps_esp32
+  tags: [ esp32, generic ]
   variables:
-    TARGET: ESP32
-    ENV_MARKER: generic
     SETUP_TOOLS: "1"  # need gdb
 
 test_app_test_pytest_esp32s2_generic:
@@ -338,9 +282,8 @@ test_app_test_pytest_esp32s2_generic:
     - .rules:test:custom_test-esp32s2
   needs:
     - build_pytest_test_apps_esp32s2
+  tags: [ esp32s2, generic ]
   variables:
-    TARGET: ESP32S2
-    ENV_MARKER: generic
     SETUP_TOOLS: "1"  # need gdb
 
 test_app_test_pytest_esp32s3_generic:
@@ -349,9 +292,7 @@ test_app_test_pytest_esp32s3_generic:
     - .rules:test:custom_test-esp32s3
   needs:
     - build_pytest_test_apps_esp32s3
-  variables:
+  tags: [ esp32s3, generic ]
-    TARGET: ESP32S3
-    ENV_MARKER: generic
 
 test_app_test_pytest_esp32c2_generic:
   extends:
@@ -359,9 +300,7 @@ test_app_test_pytest_esp32c2_generic:
     - .rules:test:custom_test-esp32c2
   needs:
     - build_pytest_test_apps_esp32c2
-  variables:
+  tags: [ esp32c2, generic ]
-    TARGET: ESP32C2
-    ENV_MARKER: generic
 
 test_app_test_pytest_esp32c3_generic:
   extends:
@@ -369,9 +308,7 @@ test_app_test_pytest_esp32c3_generic:
     - .rules:test:custom_test-esp32c3
   needs:
     - build_pytest_test_apps_esp32c3
-  variables:
+  tags: [ esp32c3, generic ]
-    TARGET: ESP32C3
-    ENV_MARKER: generic
 
 test_app_test_pytest_esp32s2_usb_host:
   extends:
@@ -379,9 +316,7 @@ test_app_test_pytest_esp32s2_usb_host:
     - .rules:test:custom_test-esp32s2
   needs:
     - build_pytest_test_apps_esp32s2
-  variables:
+  tags: [ esp32s2, usb_host ]
-    TARGET: ESP32S2
-    ENV_MARKER: usb_host
 
 # for parallel jobs, CI_JOB_NAME will be "job_name index/total" (for example, "IT_001 1/2")
 # we need to convert to pattern "job_name_index.yml"

conftest.py

@@ -42,10 +42,7 @@ DEFAULT_SDKCONFIG = 'default'
 # Help Functions #
 ##################
 def is_target_marker(marker: str) -> bool:
-    if marker.startswith('esp32'):
+    if marker.startswith('esp32') or marker.startswith('esp8') or marker == 'linux':
-        return True
-
-    if marker.startswith('esp8'):
         return True
 
     return False
@@ -59,6 +56,26 @@ def item_marker_names(item: Item) -> List[str]:
     return [marker.name for marker in item.iter_markers()]
 
 
+def get_target_marker(markexpr: str) -> str:
+    candidates = set()
+    # we use `-m "esp32 and generic"` in our CI to filter the test cases
+    for marker in markexpr.split('and'):
+        marker = marker.strip()
+        if is_target_marker(marker):
+            candidates.add(marker)
+
+    if len(candidates) > 1:
+        raise ValueError(
+            f'Specified more than one target markers: {candidates}. Please specify no more than one.'
+        )
+    elif len(candidates) == 1:
+        return candidates.pop()
+    else:
+        raise ValueError(
+            'Please specify one target marker via "--target [TARGET]" or via "-m [TARGET]"'
+        )
+
+
 ############
 # Fixtures #
 ############
@@ -79,15 +96,19 @@ def session_tempdir() -> str:
 def log_minimum_free_heap_size(dut: IdfDut, config: str) -> Callable[..., None]:
     def real_func() -> None:
         res = dut.expect(r'Minimum free heap size: (\d+) bytes')
-        logging.info('\n------ heap size info ------\n'
+        logging.info(
-                     '[app_name] {}\n'
+            '\n------ heap size info ------\n'
-                     '[config_name] {}\n'
+            '[app_name] {}\n'
-                     '[target] {}\n'
+            '[config_name] {}\n'
-                     '[minimum_free_heap_size] {} Bytes\n'
+            '[target] {}\n'
-                     '------ heap size end ------'.format(os.path.basename(dut.app.app_path),
+            '[minimum_free_heap_size] {} Bytes\n'
-                                                          config,
+            '------ heap size end ------'.format(
-                                                          dut.target,
+                os.path.basename(dut.app.app_path),
-                                                          res.group(1).decode('utf8')))
+                config,
+                dut.target,
+                res.group(1).decode('utf8'),
+            )
+        )
 
     return real_func
 
@@ -183,8 +204,20 @@ _idf_pytest_embedded_key = pytest.StashKey['IdfPytestEmbedded']
 
 
 def pytest_configure(config: Config) -> None:
+    # cli option "--target"
+    target = config.getoption('target') or ''
+
+    help_commands = ['--help', '--fixtures', '--markers', '--version']
+    for cmd in help_commands:
+        if cmd in config.invocation_params.args:
+            target = 'unneeded'
+            break
+
+    if not target:  # also could specify through markexpr via "-m"
+        target = get_target_marker(config.getoption('markexpr') or '')
+
     config.stash[_idf_pytest_embedded_key] = IdfPytestEmbedded(
-        target=config.getoption('target'),
+        target=target,
         sdkconfig=config.getoption('sdkconfig'),
         known_failure_cases_file=config.getoption('known_failure_cases_file'),
     )
@@ -218,7 +251,11 @@ class IdfPytestEmbedded:
 
     @property
     def failed_cases(self) -> List[str]:
-        return [case for case, is_known, is_xfail in self._failed_cases if not is_known and not is_xfail]
+        return [
+            case
+            for case, is_known, is_xfail in self._failed_cases
+            if not is_known and not is_xfail
+        ]
 
     @property
     def known_failure_cases(self) -> List[str]:
@@ -268,13 +305,13 @@ class IdfPytestEmbedded:
 
         # add markers for special markers
         for item in items:
-            if 'supported_targets' in item_marker_names(item):
+            if 'supported_targets' in item.keywords:
                 for _target in SUPPORTED_TARGETS:
                     item.add_marker(_target)
-            if 'preview_targets' in item_marker_names(item):
+            if 'preview_targets' in item.keywords:
                 for _target in PREVIEW_TARGETS:
                     item.add_marker(_target)
-            if 'all_targets' in item_marker_names(item):
+            if 'all_targets' in item.keywords:
                 for _target in [*SUPPORTED_TARGETS, *PREVIEW_TARGETS]:
                     item.add_marker(_target)
 

tools/ci/python_packages/gitlab_api.py

@@ -204,6 +204,16 @@ class Gitlab(object):
 
         return os.path.join(os.path.realpath(destination), root_name)
 
+    def get_job_tags(self, job_id: int) -> str:
+        """
+        Get tags of a job
+
+        :param job_id: job id
+        :return: comma-separated tags of the job
+        """
+        job = self.project.jobs.get(job_id)
+        return ','.join(job.tag_list)
+
 
 def main() -> None:
     parser = argparse.ArgumentParser()
@@ -231,6 +241,9 @@ def main() -> None:
     elif args.action == 'get_project_id':
         ret = gitlab_inst.get_project_id(args.project_name)
         print('project id: {}'.format(ret))
+    elif args.action == 'get_job_tags':
+        ret = gitlab_inst.get_job_tags(args.job_id)
+        print(ret)
 
 
 if __name__ == '__main__':

tools/ci/utils.sh

@@ -62,17 +62,22 @@ function warning() {
 }
 
 function run_cmd() {
+  local args=( "$@" )
+  local cmd="${args[@]}"
   local start=$(date +%s)
-  eval "$@"
+
+  info "\$ ${cmd}"
+  eval "${cmd}"
+
   local ret=$?
   local end=$(date +%s)
-  local duration=$((end - start))
+  local runtime=$((end-start))
 
   if [[ $ret -eq 0 ]]; then
-    info "(\$ $*) succeeded in ${duration} seconds."
+    info "==> '\$ ${cmd}' succeeded in ${runtime} seconds."
     return 0
-  else
+  echo
-    error "(\$ $*) failed in ${duration} seconds."
+    error "==> '\$ ${cmd}' failed (${ret}) in ${runtime} seconds."
     return $ret
   fi
 }