Merge branch 'ci/sort_pytest_cases_by_app_path' into 'master'

ci: sort pytest test cases by app path

See merge request espressif/esp-idf!26611

.gitlab/ci/rules.yml

@@ -24,6 +24,7 @@
   - "tools/ci/sonar_exclude_list.txt"
 
 .patterns-example_test: &patterns-example_test
+  - "tools/ci/idf_pytest/**/*"
   - "tools/ci/python_packages/gitlab_api.py"
   - "tools/ci/python_packages/idf_http_server_test/**/*"
   - "tools/ci/python_packages/idf_iperf_test_util/**/*"
@@ -70,6 +71,7 @@
   - "tools/test_sbom/*"
 
 .patterns-custom_test: &patterns-custom_test
+  - "tools/ci/idf_pytest/**/*"
   - "tools/ci/python_packages/gitlab_api.py"
   - "tools/ci/python_packages/common_test_methods.py"
 
@@ -77,6 +79,7 @@
   - "tools/ldgen/**/*"
 
 .patterns-component_ut: &patterns-component_ut
+  - "tools/ci/idf_pytest/**/*"
   - "tools/ci/python_packages/gitlab_api.py"
   - "tools/ci/python_packages/common_test_methods.py"
   - "tools/test_apps/configs/sdkconfig.debug_helpers"

tools/ci/exclude_check_tools_files.txt

@@ -39,6 +39,5 @@ tools/templates/sample_component/CMakeLists.txt
 tools/templates/sample_component/include/main.h
 tools/templates/sample_component/main.c
 tools/ci/cleanup_ignore_lists.py
-tools/ci/idf_pytest/**/*
 tools/ci/artifacts_handler.py
 tools/unit-test-app/**/*

tools/ci/idf_pytest/plugin.py

@@ -49,7 +49,7 @@ class IdfPytestEmbedded:
         # implement like this since this is a limitation of pytest, couldn't get fixture values while collecting
         # https://github.com/pytest-dev/pytest/discussions/9689
         if not hasattr(item, 'callspec'):
-            raise ValueError(f'Function {item} does not have params')
+            return default
 
         return item.callspec.params.get(key, default) or default
 
@@ -177,7 +177,11 @@ class IdfPytestEmbedded:
             # finally!
             filtered_items.append(item)
 
-        items[:] = filtered_items[:]
+        # sort the test cases with (app folder, config)
+        items[:] = sorted(
+            filtered_items,
+            key=lambda x: (os.path.dirname(x.path), self.get_param(x, 'config', DEFAULT_SDKCONFIG))
+        )
 
     def pytest_report_collectionfinish(self, items: t.List[Function]) -> None:
         for item in items: