ci(pytest): fix multi dut wrongly be picked issue

This issue will happen to multi-dut test cases
- without `target` defined in param
- and with `app_path` defined in param
- and with `pytest.mark.target` markers

tools/ci/idf_pytest/constants.py

@@ -5,6 +5,7 @@ Pytest Related Constants. Don't import third-party packages here.
 """
 import os
 import typing as t
+import warnings
 from dataclasses import dataclass
 from enum import Enum
 from functools import cached_property
@@ -174,6 +175,7 @@ class PytestCase:
     apps: t.List[PytestApp]
 
     item: Function
+    multi_dut_without_param: bool
 
     def __hash__(self) -> int:
         return hash((self.path, self.name, self.apps, self.all_markers))
@@ -188,8 +190,23 @@ class PytestCase:
 
     @cached_property
     def targets(self) -> t.List[str]:
+        if not self.multi_dut_without_param:
             return [app.target for app in self.apps]
 
+        # multi-dut test cases without parametrize
+        skip = True
+        for _t in [app.target for app in self.apps]:
+            if _t in self.target_markers:
+                skip = False
+                warnings.warn(f'`pytest.mark.[TARGET]` defined in parametrize for multi-dut test cases is deprecated. '
+                              f'Please use parametrize instead for test case {self.item.nodeid}')
+                break
+
+        if not skip:
+            return [app.target for app in self.apps]
+
+        return [''] * len(self.apps)  # this will help to filter these cases out later
+
     @cached_property
     def is_single_dut_test_case(self) -> bool:
         return True if len(self.apps) == 1 else False

tools/ci/idf_pytest/plugin.py

@@ -104,7 +104,7 @@ class IdfPytestEmbedded:
 
         return item.callspec.params.get(key, default) or default
 
-    def item_to_pytest_case(self, item: Function) -> PytestCase:
+    def item_to_pytest_case(self, item: Function) -> t.Optional[PytestCase]:
         """
         Turn pytest item to PytestCase
         """
@@ -113,10 +113,23 @@ class IdfPytestEmbedded:
         # default app_path is where the test script locates
         app_paths = to_list(parse_multi_dut_args(count, self.get_param(item, 'app_path', os.path.dirname(item.path))))
         configs = to_list(parse_multi_dut_args(count, self.get_param(item, 'config', DEFAULT_SDKCONFIG)))
-        targets = to_list(parse_multi_dut_args(count, self.get_param(item, 'target', self.target[0])))
+        targets = to_list(parse_multi_dut_args(count, self.get_param(item, 'target')))
+
+        multi_dut_without_param = False
+        if count > 1 and targets == [None] * count:
+            multi_dut_without_param = True
+            try:
+                targets = to_list(parse_multi_dut_args(count, '|'.join(self.target)))  # check later while collecting
+            except ValueError:  # count doesn't match
+                return None
+
+        elif targets is None:
+            targets = self.target
 
         return PytestCase(
-            [PytestApp(app_paths[i], targets[i], configs[i]) for i in range(count)], item
+            apps=[PytestApp(app_paths[i], targets[i], configs[i]) for i in range(count)],
+            item=item,
+            multi_dut_without_param=multi_dut_without_param
         )
 
     @pytest.hookimpl(tryfirst=True)
@@ -167,7 +180,11 @@ class IdfPytestEmbedded:
         # 2. Add markers according to special markers
         item_to_case_dict: t.Dict[Function, PytestCase] = {}
         for item in items:
-            item.stash[ITEM_PYTEST_CASE_KEY] = item_to_case_dict[item] = self.item_to_pytest_case(item)
+            case = self.item_to_pytest_case(item)
+            if case is None:
+                continue
+
+            item.stash[ITEM_PYTEST_CASE_KEY] = item_to_case_dict[item] = case
             if 'supported_targets' in item.keywords:
                 for _target in SUPPORTED_TARGETS:
                     item.add_marker(_target)
@@ -177,6 +194,7 @@ class IdfPytestEmbedded:
             if 'all_targets' in item.keywords:
                 for _target in [*SUPPORTED_TARGETS, *PREVIEW_TARGETS]:
                     item.add_marker(_target)
+        items[:] = [_item for _item in items if _item in item_to_case_dict]
 
         # 3.1. CollectMode.SINGLE_SPECIFIC, like `pytest --target esp32`
         if self.collect_mode == CollectMode.SINGLE_SPECIFIC:

tools/ci/idf_pytest/tests/test_get_pytest_cases.py

@@ -1,5 +1,6 @@
 # SPDX-FileCopyrightText: 2023-2024 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
+import textwrap
 from pathlib import Path
 
 from idf_pytest.constants import CollectMode
@@ -86,3 +87,34 @@ def test_get_pytest_cases_all(work_dirpath: Path) -> None:
 
     assert cases[5].targets == ['esp32s2']
     assert cases[5].name == 'test_foo_single'
+
+
+def test_multi_with_marker_and_app_path(work_dirpath: Path) -> None:
+    script = work_dirpath / 'pytest_multi_with_marker_and_app_path.py'
+    script.write_text(
+        textwrap.dedent(
+            '''
+        import pytest
+
+        @pytest.mark.esp32c2
+        @pytest.mark.parametrize(
+            'count,app_path', [
+                (2, 'foo|bar'),
+                (3, 'foo|bar|baz'),
+            ], indirect=True
+        )
+        def test_foo_multi_with_marker_and_app_path(dut):
+            pass
+    '''
+        )
+    )
+    cases = get_pytest_cases([str(work_dirpath)], 'esp32c3,esp32c3')
+    assert len(cases) == 0
+
+    cases = get_pytest_cases([str(work_dirpath)], 'esp32c2,esp32c2')
+    assert len(cases) == 1
+    assert cases[0].targets == ['esp32c2', 'esp32c2']
+
+    cases = get_pytest_cases([str(work_dirpath)], 'esp32c2,esp32c2,esp32c2')
+    assert len(cases) == 1
+    assert cases[0].targets == ['esp32c2', 'esp32c2', 'esp32c2']