Merge branch 'ci/improve_build_scripts_for_multi_targets' into 'master'

CI: improve build scripts for multi targets

See merge request espressif/esp-idf!17895

.gitlab-ci.yml

@@ -73,7 +73,7 @@ variables:
   TEST_ENV_CONFIG_REPO: "https://gitlab-ci-token:${BOT_TOKEN}@${CI_SERVER_HOST}:${CI_SERVER_PORT}/qa/ci-test-runner-configs.git"
   CI_AUTO_TEST_SCRIPT_REPO_URL: "https://gitlab-ci-token:${BOT_TOKEN}@${CI_SERVER_HOST}:${CI_SERVER_PORT}/qa/auto_test_script.git"
   CI_AUTO_TEST_SCRIPT_REPO_BRANCH: "ci/v4.1"
-  PYTEST_EMBEDDED_VERSION: "0.6.0"
+  PYTEST_EMBEDDED_VERSION: "0.7.0"
 
   # cache python dependencies
   PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"

.gitlab/ci/target-test.yml

@@ -111,6 +111,16 @@ example_test_pytest_esp32_flash_encryption:
     TARGET: ESP32
     ENV_MARKER: flash_encryption
 
+example_test_pytest_esp32_multi_dut_generic:
+  extends:
+    - .pytest_examples_dir_template
+    - .rules:test:example_test-esp32
+  needs:
+    - build_pytest_examples_esp32
+  variables:
+    TARGET: ESP32
+    ENV_MARKER: multi_dut_generic
+
 example_test_pytest_esp32c3_flash_encryption:
   extends:
     - .pytest_examples_dir_template

conftest.py

@@ -29,7 +29,7 @@ from _pytest.python import Function
 from _pytest.reports import TestReport
 from _pytest.runner import CallInfo
 from _pytest.terminal import TerminalReporter
-from pytest_embedded.plugin import apply_count, parse_configuration
+from pytest_embedded.plugin import multi_dut_argument, multi_dut_fixture
 from pytest_embedded.utils import find_by_suffix
 
 SUPPORTED_TARGETS = ['esp32', 'esp32s2', 'esp32c3', 'esp32s3', 'esp32c2']
@@ -75,7 +75,7 @@ def session_tempdir() -> str:
 
 
 @pytest.fixture
-@parse_configuration
+@multi_dut_argument
 def config(request: FixtureRequest) -> str:
     return getattr(request, 'param', None) or DEFAULT_SDKCONFIG
 
@@ -91,7 +91,7 @@ def test_case_name(request: FixtureRequest, target: str, config: str) -> str:
 
 
 @pytest.fixture
-@apply_count
+@multi_dut_fixture
 def build_dir(app_path: str, target: Optional[str], config: Optional[str]) -> str:
     """
     Check local build dir with the following priority:
@@ -137,7 +137,7 @@ def build_dir(app_path: str, target: Optional[str], config: Optional[str]) -> st
 
 
 @pytest.fixture(autouse=True)
-@apply_count
+@multi_dut_fixture
 def junit_properties(
     test_case_name: str, record_xml_attribute: Callable[[str, object], None]
 ) -> None:

docs/en/contribute/esp-idf-tests-with-pytest.rst

@@ -19,6 +19,11 @@ In ESP-IDF, we use the following plugins by default:
 
 All the introduced concepts and usages are based on the default behavior in ESP-IDF. Not all of them are available in vanilla pytest.
 
+Installation
+------------
+
+``$ pip install -U pytest-embedded-serial-esp~=0.7.0 pytest-embedded-idf~=0.7.0``
+
 Basic Concepts
 --------------
 
@@ -95,12 +100,12 @@ Pytest Execution Process
 
    1. Get all the python files with the prefix ``pytest_``
    2. Get all the test functions with the prefix ``test_``
-   3. Apply the `params <https://docs.pytest.org/en/latest/how-to/parametrize.html>`__, duplicate the test functions.
-   4. Filter the test cases with CLI options. Introduced detail usages `here <#filter-the-test-cases>`__
+   3. Apply the `params <https://docs.pytest.org/en/latest/how-to/parametrize.html>`__, and duplicate the test functions.
+   4. Filter the test cases with CLI options. Introduced detailed usages `here <#filter-the-test-cases>`__
 
 3. Test Running Phase
 
-   1. Construct the `fixtures <https://docs.pytest.org/en/latest/how-to/fixtures.html>`__. In ESP-IDF, the common fixtures are initialized with this order:
+   1. Construct the `fixtures <https://docs.pytest.org/en/latest/how-to/fixtures.html>`__. In ESP-IDF, the common fixtures are initialized in this order:
 
       1. ``pexpect_proc``: `pexpect <https://github.com/pexpect/pexpect>`__ instance
 
@@ -116,7 +121,7 @@ Pytest Execution Process
 
    2. Run the real test function
 
-   3. Deconstruct the fixtures with this order:
+   3. Deconstruct the fixtures in this order:
 
       1. ``dut``
 
@@ -198,7 +203,7 @@ You can use ``pytest.mark.parametrize`` with “config” to apply the same test
        'nohistory',   # <-- run with app built by sdkconfig.ci.nohistory
    ], indirect=True)  # <-- `indirect=True` is required
 
-Overall, this test case would be duplicated to 4 test functions:
+Overall, this test function would be replicated to 4 test cases:
 
 -  esp32.history.test_console_advanced
 -  esp32.nohistory.test_console_advanced
@@ -208,6 +213,80 @@ Overall, this test case would be duplicated to 4 test functions:
 Advanced Examples
 ~~~~~~~~~~~~~~~~~
 
+Multi Dut Tests with the Same App
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+This code example is taken from :idf_file:`pytest_usb_host.py <tools/test_apps/peripherals/usb/pytest_usb_host.py>`.
+
+.. code:: python
+
+    @pytest.mark.esp32s2
+    @pytest.mark.esp32s3
+    @pytest.mark.usb_host
+    @pytest.mark.parametrize('count', [
+        2,
+    ], indirect=True)
+    def test_usb_host(dut: Tuple[IdfDut, IdfDut]) -> None:
+        device = dut[0]  # <-- assume the first dut is the device
+        host = dut[1]    # <-- and the second dut is the host
+        ...
+
+After setting the param ``count`` to 2, all these fixtures are changed into tuples.
+
+Multi Dut Tests with Different Apps
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+This code example is taken from :idf_file:`pytest_wifi_getting_started.py <examples/wifi/getting_started/pytest_wifi_getting_started.py>`.
+
+.. code:: python
+
+    @pytest.mark.esp32
+    @pytest.mark.multi_dut_generic
+    @pytest.mark.parametrize(
+        'count, app_path', [
+            (2,
+             f'{os.path.join(os.path.dirname(__file__), "softAP")}|{os.path.join(os.path.dirname(__file__), "station")}'),
+        ], indirect=True
+    )
+    def test_wifi_getting_started(dut: Tuple[IdfDut, IdfDut]) -> None:
+        softap = dut[0]
+        station = dut[1]
+        ...
+
+Here the first dut was flashed with the app :idf_file:`softap <examples/wifi/getting_started/softAP/main/softap_example_main.c>`, and the second dut was flashed with the app :idf_file:`station <examples/wifi/getting_started/station/main/station_example_main.c>`.
+
+.. note::
+
+   Here the ``app_path`` should be set with absolute path. the ``__file__`` macro in python would return the absolute path of the test script itself.
+
+Multi Dut Tests with Different Apps, and Targets
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+This code example is taken from :idf_file:`pytest_wifi_getting_started.py <examples/wifi/getting_started/pytest_wifi_getting_started.py>`. As the comment says, for now it's not running in the ESP-IDF CI.
+
+.. code:: python
+
+    @pytest.mark.parametrize(
+        'count, app_path, target', [
+            (2,
+             f'{os.path.join(os.path.dirname(__file__), "softAP")}|{os.path.join(os.path.dirname(__file__), "station")}',
+             'esp32|esp32s2'),
+            (2,
+             f'{os.path.join(os.path.dirname(__file__), "softAP")}|{os.path.join(os.path.dirname(__file__), "station")}',
+             'esp32s2|esp32'),
+        ],
+        indirect=True,
+    )
+    def test_wifi_getting_started(dut: Tuple[IdfDut, IdfDut]) -> None:
+        softap = dut[0]
+        station = dut[1]
+        ...
+
+Overall, this test function would be replicated to 2 test cases:
+
+- softap with esp32 target, and station with esp32s2 target
+- softap with esp32s2 target, and station with esp32 target
+
 Support different targets with different sdkconfig files
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
@@ -278,7 +357,7 @@ Mark Known Failure Cases
 Sometimes a test couldn't pass for the following reasons:
 
 - Has a bug
-- Success ratio too low because of environment issue, such as network issue. Retry couldn't help
+- The success ratio is too low because of environment issue, such as network issue. Retry couldn't help
 
 Now you may mark this test case with marker `xfail <https://docs.pytest.org/en/latest/how-to/skipping.html#xfail-mark-test-functions-as-expected-to-fail>`__ with a user-friendly readable reason.
 
@@ -397,7 +476,7 @@ You can call pytest with ``--junitxml <filepath>`` to generate the JUnit report.
 Skip Auto Flash Binary
 ~~~~~~~~~~~~~~~~~~~~~~
 
-Skipping auto-flash binary everytime would be useful when you're debugging your test script.
+Skipping auto-flash binary every time would be useful when you're debugging your test script.
 
 You can call pytest with ``--skip-autoflash y`` to achieve it.
 

examples/provisioning/wifi_prov_mgr/pytest_wifi_prov_mgr.py

@@ -6,8 +6,15 @@
 from __future__ import print_function
 
 import logging
+import os
+import sys
+
+try:
+    import esp_prov
+except ImportError:
+    sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', '..', 'tools'))
+    import esp_prov
 
-import esp_prov
 import pytest
 from pytest_embedded import Dut
 

examples/wifi/getting_started/pytest_wifi_getting_started.py

@@ -0,0 +1,42 @@
+# SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: CC0-1.0
+
+import os.path
+from typing import Tuple
+
+import pytest
+from pytest_embedded_idf.dut import IdfDut
+
+# @pytest.mark.supported_targets
+# This test should support all targets, even between different target types
+# For now our CI only support multi dut with esp32
+# If you want to enable different target type, please use the following param
+# @pytest.mark.parametrize(
+#     'count, app_path, target', [
+#         (2,
+#          f'{os.path.join(os.path.dirname(__file__), "softAP")}|{os.path.join(os.path.dirname(__file__), "station")}',
+#          'esp32|esp32s2'),
+#     ],
+#     indirect=True,
+# )
+
+
+@pytest.mark.esp32
+@pytest.mark.multi_dut_generic
+@pytest.mark.parametrize(
+    'count, app_path', [
+        (2,
+         f'{os.path.join(os.path.dirname(__file__), "softAP")}|{os.path.join(os.path.dirname(__file__), "station")}'),
+    ], indirect=True
+)
+def test_wifi_getting_started(dut: Tuple[IdfDut, IdfDut]) -> None:
+    softap = dut[0]
+    station = dut[1]
+
+    ssid = 'myssid'
+    password = 'mypassword'
+    tag = 'wifi station'
+
+    station.expect(f'{tag}: got ip:', timeout=60)
+    station.expect(f'{tag}: connected to ap SSID:{ssid} password:{password}', timeout=60)
+    softap.expect('station .+ join, AID=', timeout=60)

pytest.ini

@@ -35,6 +35,9 @@ markers =
   flash_encryption: Flash Encryption runners
   ir_transceiver: runners with a pair of IR transmitter and receiver
 
+  ## multi-dut markers
+  multi_dut_generic: tests should be run on generic runners, at least have two duts connected.
+
 # log related
 log_cli = True
 log_cli_level = INFO

tools/ci/build_pytest_apps.py

@@ -13,7 +13,7 @@ import sys
 from collections import defaultdict
 from typing import List
 
-from idf_ci_utils import IDF_PATH, get_pytest_cases
+from idf_ci_utils import IDF_PATH, PytestCase, get_pytest_cases
 
 try:
     from build_apps import build_apps
@@ -28,15 +28,16 @@ except ImportError:
 
 
 def main(args: argparse.Namespace) -> None:
-    pytest_cases = []
+    pytest_cases: List[PytestCase] = []
     for path in args.paths:
         pytest_cases += get_pytest_cases(path, args.target)
 
     paths = set()
     app_configs = defaultdict(set)
     for case in pytest_cases:
-        paths.add(case.app_path)
-        app_configs[case.app_path].add(case.config)
+        for app in case.apps:
+            paths.add(app.path)
+            app_configs[app.path].add(app.config)
 
     app_dirs = list(paths)
     if not app_dirs:

tools/ci/idf_ci_utils.py

@@ -1,19 +1,22 @@
 # internal use only for CI
 # some CI related util functions
 #
-# SPDX-FileCopyrightText: 2020-2021 Espressif Systems (Shanghai) CO LTD
+# SPDX-FileCopyrightText: 2020-2022 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
 #
+
 import io
 import logging
 import os
 import subprocess
 import sys
 from contextlib import redirect_stdout
-from typing import TYPE_CHECKING, List
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Any, List, Set
 
 if TYPE_CHECKING:
-    from _pytest.nodes import Function
+    from _pytest.python import Function
+
 
 IDF_PATH = os.path.abspath(
     os.getenv('IDF_PATH', os.path.join(os.path.dirname(__file__), '..', '..'))
@@ -113,39 +116,83 @@ def is_in_directory(file_path: str, folder: str) -> bool:
     return os.path.realpath(file_path).startswith(os.path.realpath(folder) + os.sep)
 
 
-class PytestCase:
-    def __init__(self, test_path: str, target: str, config: str, case: str):
-        self.app_path = os.path.dirname(test_path)
-        self.test_path = test_path
-        self.target = target
-        self.config = config
-        self.case = case
+def to_list(s: Any) -> List[Any]:
+    if isinstance(s, (set, tuple)):
+        return list(s)
 
-    def __repr__(self) -> str:
-        return f'{self.test_path}: {self.target}.{self.config}.{self.case}'
+    if isinstance(s, list):
+        return s
+
+    return [s]
+
+
+@dataclass
+class PytestApp:
+    path: str
+    target: str
+    config: str
+
+    def __hash__(self) -> int:
+        return hash((self.path, self.target, self.config))
+
+
+@dataclass
+class PytestCase:
+    path: str
+    name: str
+    apps: Set[PytestApp]
+
+    def __hash__(self) -> int:
+        return hash((self.path, self.name, self.apps))
 
 
 class PytestCollectPlugin:
     def __init__(self, target: str) -> None:
         self.target = target
-        self.nodes: List[PytestCase] = []
+        self.cases: List[PytestCase] = []
+
+    @staticmethod
+    def get_param(item: 'Function', key: str, default: Any = None) -> Any:
+        if not hasattr(item, 'callspec'):
+            raise ValueError(f'Function {item} does not have params')
+
+        return item.callspec.params.get(key, default) or default
 
     def pytest_collection_modifyitems(self, items: List['Function']) -> None:
+        from pytest_embedded.plugin import parse_multi_dut_args
+
         for item in items:
-            try:
-                file_path = str(item.path)
-            except AttributeError:
-                # pytest 6.x
-                file_path = item.fspath
-
-            target = self.target
-            if hasattr(item, 'callspec'):
-                config = item.callspec.params.get('config', 'default')
-            else:
-                config = 'default'
+            count = 1
+            case_path = str(item.path)
             case_name = item.originalname
+            target = self.target
+            # funcargs is not calculated while collection
+            if hasattr(item, 'callspec'):
+                count = item.callspec.params.get('count', 1)
+                app_paths = to_list(
+                    parse_multi_dut_args(
+                        count,
+                        self.get_param(item, 'app_path', os.path.dirname(case_path)),
+                    )
+                )
+                configs = to_list(
+                    parse_multi_dut_args(
+                        count, self.get_param(item, 'config', 'default')
+                    )
+                )
+                targets = to_list(
+                    parse_multi_dut_args(count, self.get_param(item, 'target', target))
+                )
+            else:
+                app_paths = [os.path.dirname(case_path)]
+                configs = ['default']
+                targets = [target]
 
-            self.nodes.append(PytestCase(file_path, target, config, case_name))
+            case_apps = set()
+            for i in range(count):
+                case_apps.add(PytestApp(app_paths[i], targets[i], configs[i]))
+
+            self.cases.append(PytestCase(case_path, case_name, case_apps))
 
 
 def get_pytest_cases(folder: str, target: str) -> List[PytestCase]:
@@ -156,18 +203,23 @@ def get_pytest_cases(folder: str, target: str) -> List[PytestCase]:
 
     with io.StringIO() as buf:
         with redirect_stdout(buf):
-            res = pytest.main(['--collect-only', folder, '-q', '--target', target], plugins=[collector])
+            res = pytest.main(
+                ['--collect-only', folder, '-q', '--target', target],
+                plugins=[collector],
+            )
         if res.value != ExitCode.OK:
             if res.value == ExitCode.NO_TESTS_COLLECTED:
-                print(f'WARNING: no pytest app found for target {target} under folder {folder}')
+                print(
+                    f'WARNING: no pytest app found for target {target} under folder {folder}'
+                )
             else:
                 print(buf.getvalue())
                 raise RuntimeError('pytest collection failed')
 
-    return collector.nodes
+    return collector.cases
 
 
-def get_pytest_app_paths(folder: str, target: str) -> List[str]:
-    nodes = get_pytest_cases(folder, target)
+def get_pytest_app_paths(folder: str, target: str) -> Set[str]:
+    cases = get_pytest_cases(folder, target)
 
-    return list({node.app_path for node in nodes})
+    return set({app.path for case in cases for app in case.apps})

tools/test_apps/system/panic/conftest.py

@@ -138,7 +138,7 @@ class PanicTestDut(IdfDut):
         """Extract the core dump from flash, run espcoredump on it"""
         coredump_file_name = os.path.join(self.logdir, 'coredump_data.bin')
         logging.info('Writing flash binary core dump to %s', coredump_file_name)
-        self.serial.dump_flash(coredump_file_name, partition='coredump')
+        self.serial.dump_flash(partition='coredump', output=coredump_file_name)
 
         output_file_name = os.path.join(self.logdir, 'coredump_flash_result.txt')
         self._call_espcoredump(