Merge branch 'ci/renable_s3_tests' into 'master'

ci: renable s3 tests

See merge request espressif/esp-idf!17574

.gitlab/ci/target-test.yml

@@ -41,16 +41,15 @@ example_test_pytest_esp32s2_generic:
     TARGET: ESP32S2
     ENV_MARKER: generic
 
-# Temporary disabled until S3 runners are added, IDFCI-1171
-# example_test_pytest_esp32s3_generic:
-#   extends:
-#     - .pytest_examples_dir_template
-#     - .rules:test:example_test-esp32s3
-#   needs:
-#     - build_pytest_examples_esp32s3
-#   variables:
-#     TARGET: ESP32S3
-#     ENV_MARKER: generic
+example_test_pytest_esp32s3_generic:
+  extends:
+    - .pytest_examples_dir_template
+    - .rules:test:example_test-esp32s3
+  needs:
+    - build_pytest_examples_esp32s3
+  variables:
+    TARGET: ESP32S3
+    ENV_MARKER: generic_s3_fixme
 
 example_test_pytest_esp32c3_generic:
   extends:
@@ -127,16 +126,15 @@ component_ut_pytest_esp32s2_generic:
     TARGET: ESP32S2
     ENV_MARKER: generic
 
-# Temporary disabled until S3 runners are added, IDFCI-1171
-# component_ut_pytest_esp32s3_generic:
-#   extends:
-#     - .pytest_components_dir_template
-#     - .rules:test:component_ut-esp32s3
-#   needs:
-#     - build_pytest_components_esp32s3
-#   variables:
-#     TARGET: ESP32S3
-#     ENV_MARKER: generic
+component_ut_pytest_esp32s3_generic:
+  extends:
+    - .pytest_components_dir_template
+    - .rules:test:component_ut-esp32s3
+  needs:
+    - build_pytest_components_esp32s3
+  variables:
+    TARGET: ESP32S3
+    ENV_MARKER: generic_s3_fixme
 
 component_ut_pytest_esp32s3_octal_psram:
   extends:
@@ -185,16 +183,15 @@ test_app_test_pytest_esp32s2_generic:
     ENV_MARKER: generic
     SETUP_TOOLS: "1"  # need gdb
 
-# Temporary disabled until S3 runners are added, IDFCI-1171
-# test_app_test_pytest_esp32s3_generic:
-#   extends:
-#     - .pytest_test_apps_dir_template
-#     - .rules:test:custom_test-esp32s3
-#   needs:
-#     - build_pytest_test_apps_esp32s3
-#   variables:
-#     TARGET: ESP32S3
-#     ENV_MARKER: generic
+test_app_test_pytest_esp32s3_generic:
+  extends:
+    - .pytest_test_apps_dir_template
+    - .rules:test:custom_test-esp32s3
+  needs:
+    - build_pytest_test_apps_esp32s3
+  variables:
+    TARGET: ESP32S3
+    ENV_MARKER: generic_s3_fixme
 
 test_app_test_pytest_esp32c3_generic:
   extends:
@@ -325,7 +322,7 @@ example_test_001B_V3:
 
 example_test_001C:
   extends: .example_test_esp32_template
-  parallel: 4
+  parallel: 2
   tags:
     - ESP32
     - Example_GENERIC
@@ -479,7 +476,7 @@ example_test_017:
 
 example_test_C3_GENERIC:
   extends: .example_test_esp32c3_template
-  parallel: 3
+  parallel: 2
   tags:
     - ESP32C3
     - Example_GENERIC
@@ -502,13 +499,6 @@ example_test_ESP32_SDSPI:
     - ESP32
     - UT_T1_SPIMODE
 
-# Temporary disabled until S3 runners are added, IDFCI-1171
-# example_test_S3_GENERIC:
-#   extends: .example_test_esp32s3_template
-#   tags:
-#     - ESP32S3
-#     - Example_GENERIC
-
 example_test_ESP32S2_SDSPI:
   extends: .example_test_esp32s2_template
   tags:
@@ -883,13 +873,12 @@ UT_C3_SDSPI:
     - ESP32C3_IDF
     - UT_T1_SPIMODE
 
-# Temporary disabled until S3 runners are added, IDFCI-1171
-# UT_S3:
-#   extends: .unit_test_esp32s3_template
-#   parallel: 44
-#   tags:
-#     - ESP32S3_IDF
-#     - UT_T1_1
+UT_S3:
+  extends: .unit_test_esp32s3_template
+  parallel: 44
+  tags:
+    - ESP32S3_IDF
+    - UT_T1_1
 
 UT_S3_SPI_DUAL:
   extends: .unit_test_esp32s3_template

conftest.py

@@ -244,6 +244,9 @@ class IdfPytestEmbedded:
             if 'all_targets' in item_marker_names(item):
                 for _target in [*SUPPORTED_TARGETS, *PREVIEW_TARGETS]:
                     item.add_marker(_target)
+            # FIXME: temporarily modify the s3 runner tag "generic" to "s3_generic" due to the deep sleep bug
+            if 'generic' in item_marker_names(item) and 'esp32s3' in item_marker_names(item):
+                item.add_marker('generic_s3_fixme')
 
         # filter all the test cases with "--target"
         if self.target:

pytest.ini

@@ -25,6 +25,7 @@ markers =
 
   # env markers
   generic: tests should be run on generic runners
+  generic_s3_fixme: temporary workaround s3 runner
   flash_suspend: support flash suspend feature
   ip101: connected via wired 10/100M ethernet
   lan8720: connected via LAN8720 ethernet transceiver

tools/ci/python_packages/tiny_test_fw/Utility/CIAssignTest.py

@@ -312,21 +312,11 @@ class AssignTest(object):
             console_log('\t{}'.format(job['name']), 'O')
 
         # failures
-        real_failure_group = []
         if failed_to_assign:
-            for group in failed_to_assign:
-                if ('ESP32S3_IDF' in group.filters.values()  # for example_test and custom_test
-                        or ['ESP32S3_IDF'] in group.filters.values()):  # for unit_test
-                    console_log('Bypassing Tag "ESP32S3_IDF" missing jobs for now!!!', 'O')  # FIXME
-                    continue
-
-                real_failure_group.append(group)
-
-        if real_failure_group:
             console_log('Too many test cases vs jobs to run. '
                         'Please increase parallel count in .gitlab/ci/target-test.yml '
                         'for jobs with specific tags:', 'R')
-            failed_group_count = self._count_groups_by_keys(real_failure_group)
+            failed_group_count = self._count_groups_by_keys(failed_to_assign)
             for tags in failed_group_count:
                 console_log('\t{}: {}'.format(tags, failed_group_count[tags]), 'R')
             raise RuntimeError('Failed to assign test case to CI jobs')