Merge branch 'ci/qemu_respect_all_test_triggerer' into 'master'

ci: fix idf_relpath issue while searching the apps

See merge request espressif/esp-idf!28645

conftest.py

@@ -38,7 +38,7 @@ from artifacts_handler import ArtifactType
 from dynamic_pipelines.constants import TEST_RELATED_APPS_DOWNLOAD_URLS_FILENAME
 from idf_ci.app import import_apps_from_txt
 from idf_ci.uploader import AppDownloader, AppUploader
-from idf_ci_utils import IDF_PATH
+from idf_ci_utils import IDF_PATH, idf_relpath
 from idf_pytest.constants import DEFAULT_SDKCONFIG, ENV_MARKERS, SPECIAL_MARKERS, TARGET_MARKERS, PytestCase
 from idf_pytest.plugin import IDF_PYTEST_EMBEDDED_KEY, ITEM_PYTEST_CASE_KEY, IdfPytestEmbedded
 from idf_pytest.utils import format_case_id
@@ -203,7 +203,7 @@ def build_dir(
     case: PytestCase = request._pyfuncitem.stash[ITEM_PYTEST_CASE_KEY]
     if app_downloader:
         # somehow hardcoded...
-        app_build_path = os.path.join(os.path.relpath(app_path, IDF_PATH), f'build_{target}_{config}')
+        app_build_path = os.path.join(idf_relpath(app_path), f'build_{target}_{config}')
         if case.requires_elf_or_map:
             app_downloader.download_app(app_build_path)
         else:

tools/ci/idf_ci_utils.py

@@ -248,3 +248,16 @@ def sanitize_job_name(name: str) -> str:
     :return: sanitized job name
     """
     return re.sub(r' \d+/\d+', '', name)
+
+
+def idf_relpath(p: str) -> str:
+    """
+    Turn all paths under IDF_PATH to relative paths
+    :param p: path
+    :return: relpath to IDF_PATH, or absolute path if not under IDF_PATH
+    """
+    abs_path = os.path.abspath(p)
+    if abs_path.startswith(IDF_PATH):
+        return os.path.relpath(abs_path, IDF_PATH)
+    else:
+        return abs_path

tools/ci/idf_pytest/constants.py

@@ -12,6 +12,7 @@ from pathlib import Path
 
 from _pytest.python import Function
 from idf_ci_utils import IDF_PATH
+from idf_ci_utils import idf_relpath
 from pytest_embedded.utils import to_list
 
 SUPPORTED_TARGETS = ['esp32', 'esp32s2', 'esp32c3', 'esp32s3', 'esp32c2', 'esp32c6', 'esp32h2', 'esp32p4']
@@ -151,11 +152,14 @@ class CollectMode(str, Enum):
     ALL = 'all'
 
 
-@dataclass
 class PytestApp:
-    path: str
+    """
-    target: str
+    Pytest App with relative path to IDF_PATH
-    config: str
+    """
+    def __init__(self, path: str, target: str, config: str) -> None:
+        self.path = idf_relpath(path)
+        self.target = target
+        self.config = config
 
     def __hash__(self) -> int:
         return hash((self.path, self.target, self.config))
@@ -247,7 +251,7 @@ class PytestCase:
         if 'jtag' in self.env_markers or 'usb_serial_jtag' in self.env_markers:
             return True
 
-        if any('panic' in Path(app.path).resolve().parts for app in self.apps):
+        if any('panic' in Path(app.path).parts for app in self.apps):
             return True
 
         return False

tools/ci/idf_pytest/plugin.py

@@ -13,6 +13,7 @@ from _pytest.python import Function
 from _pytest.runner import CallInfo
 from idf_build_apps import App
 from idf_build_apps.constants import BuildStatus
+from idf_ci_utils import idf_relpath
 from pytest_embedded import Dut
 from pytest_embedded.plugin import parse_multi_dut_args
 from pytest_embedded.utils import find_by_suffix
@@ -73,7 +74,7 @@ class IdfPytestEmbedded:
         self._single_target_duplicate_mode = single_target_duplicate_mode
 
         self.apps_list = (
-            [os.path.join(app.app_dir, app.build_dir) for app in apps if app.build_status == BuildStatus.SUCCESS]
+            [os.path.join(idf_relpath(app.app_dir), app.build_dir) for app in apps if app.build_status == BuildStatus.SUCCESS]
             if apps
             else None
         )
@@ -114,14 +115,8 @@ class IdfPytestEmbedded:
         configs = to_list(parse_multi_dut_args(count, self.get_param(item, 'config', DEFAULT_SDKCONFIG)))
         targets = to_list(parse_multi_dut_args(count, self.get_param(item, 'target', self.target[0])))
 
-        def abspath_or_relpath(s: str) -> str:
-            if os.path.abspath(s) and s.startswith(os.getcwd()):
-                return os.path.relpath(s)
-
-            return s
-
         return PytestCase(
-            [PytestApp(abspath_or_relpath(app_paths[i]), targets[i], configs[i]) for i in range(count)], item
+            [PytestApp(app_paths[i], targets[i], configs[i]) for i in range(count)], item
         )
 
     @pytest.hookimpl(tryfirst=True)

tools/ci/idf_pytest/script.py

@@ -1,6 +1,5 @@
-# SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD
+# SPDX-FileCopyrightText: 2023-2024 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
-
 import fnmatch
 import io
 import logging
@@ -11,14 +10,22 @@ from pathlib import Path
 
 import pytest
 from _pytest.config import ExitCode
-from idf_build_apps import App, find_apps
+from idf_build_apps import App
-from idf_build_apps.constants import SUPPORTED_TARGETS, BuildStatus
+from idf_build_apps import find_apps
+from idf_build_apps.constants import BuildStatus
+from idf_build_apps.constants import SUPPORTED_TARGETS
 from idf_ci.app import IdfCMakeApp
-from idf_ci_utils import IDF_PATH, get_all_manifest_files, to_list
+from idf_ci_utils import get_all_manifest_files
+from idf_ci_utils import IDF_PATH
+from idf_ci_utils import idf_relpath
+from idf_ci_utils import to_list
 from idf_py_actions.constants import PREVIEW_TARGETS as TOOLS_PREVIEW_TARGETS
 from idf_py_actions.constants import SUPPORTED_TARGETS as TOOLS_SUPPORTED_TARGETS
 
-from .constants import DEFAULT_BUILD_LOG_FILENAME, DEFAULT_CONFIG_RULES_STR, CollectMode, PytestCase
+from .constants import CollectMode
+from .constants import DEFAULT_BUILD_LOG_FILENAME
+from .constants import DEFAULT_CONFIG_RULES_STR
+from .constants import PytestCase
 from .plugin import IdfPytestEmbedded
 
 
@@ -48,6 +55,8 @@ def get_pytest_cases(
     apps: t.Optional[t.List[App]] = None,
 ) -> t.List[PytestCase]:
     """
+    Return the list of test cases
+
     For single-dut test cases, `target` could be
     - [TARGET], e.g. `esp32`, to get the test cases for the given target
     - or `single_all`, to get all single-dut test cases
@@ -136,24 +145,27 @@ def get_all_apps(
     :param ignore_app_dependencies_filepatterns: ignore app dependencies filepatterns
     :return: tuple of test-required apps and non-test-related apps
     """
-    all_apps = find_apps(
+    # target could be comma separated list
-        paths,
+    all_apps: t.List[App] = []
-        target,
+    for _t in set(target.split(',')):
-        build_system=IdfCMakeApp,
+        all_apps.extend(find_apps(
-        recursive=True,
+            paths,
-        build_dir='build_@t_@w',
+            _t,
-        config_rules_str=config_rules_str or DEFAULT_CONFIG_RULES_STR,
+            build_system=IdfCMakeApp,
-        build_log_filename=DEFAULT_BUILD_LOG_FILENAME,
+            recursive=True,
-        size_json_filename='size.json',
+            build_dir='build_@t_@w',
-        check_warnings=True,
+            config_rules_str=config_rules_str or DEFAULT_CONFIG_RULES_STR,
-        manifest_rootpath=IDF_PATH,
+            build_log_filename=DEFAULT_BUILD_LOG_FILENAME,
-        manifest_files=get_all_manifest_files(),
+            size_json_filename='size.json',
-        default_build_targets=SUPPORTED_TARGETS + (extra_default_build_targets or []),
+            check_warnings=True,
-        modified_components=modified_components,
+            manifest_rootpath=IDF_PATH,
-        modified_files=modified_files,
+            manifest_files=get_all_manifest_files(),
-        ignore_app_dependencies_filepatterns=ignore_app_dependencies_filepatterns,
+            default_build_targets=SUPPORTED_TARGETS + (extra_default_build_targets or []),
-        include_skipped_apps=True,
+            modified_components=modified_components,
-    )
+            modified_files=modified_files,
+            ignore_app_dependencies_filepatterns=ignore_app_dependencies_filepatterns,
+            include_skipped_apps=True,
+        ))
 
     pytest_cases = get_pytest_cases(
         paths,
@@ -176,27 +188,30 @@ def get_all_apps(
             )
 
     # app_path, target, config
-    pytest_app_path_tuple_dict: t.Dict[t.Tuple[Path, str, str], PytestCase] = {}
+    pytest_app_path_tuple_dict: t.Dict[t.Tuple[str, str, str], PytestCase] = {}
     for case in pytest_cases:
         for app in case.apps:
-            pytest_app_path_tuple_dict[(Path(app.path), app.target, app.config)] = case
+            pytest_app_path_tuple_dict[(app.path, app.target, app.config)] = case
 
-    modified_pytest_app_path_tuple_dict: t.Dict[t.Tuple[Path, str, str], PytestCase] = {}
+    modified_pytest_app_path_tuple_dict: t.Dict[t.Tuple[str, str, str], PytestCase] = {}
     for case in modified_pytest_cases:
         for app in case.apps:
-            modified_pytest_app_path_tuple_dict[(Path(app.path), app.target, app.config)] = case
+            modified_pytest_app_path_tuple_dict[(app.path, app.target, app.config)] = case
 
     test_related_apps: t.Set[App] = set()
     non_test_related_apps: t.Set[App] = set()
     for app in all_apps:
+        # PytestCase.app.path is idf_relpath
+        app_path = idf_relpath(app.app_dir)
+
         # override build_status if test script got modified
-        if case := modified_pytest_app_path_tuple_dict.get((Path(app.app_dir), app.target, app.config_name)):
+        if case := modified_pytest_app_path_tuple_dict.get((app_path, app.target, app.config_name)):
             test_related_apps.add(app)
             app.build_status = BuildStatus.SHOULD_BE_BUILT
             app.preserve = True
             logging.debug('Found app: %s - required by modified test case %s', app, case.path)
         elif app.build_status != BuildStatus.SKIPPED:
-            if case := pytest_app_path_tuple_dict.get((Path(app.app_dir), app.target, app.config_name)):
+            if case := pytest_app_path_tuple_dict.get((app_path, app.target, app.config_name)):
                 test_related_apps.add(app)
                 # should be built if
                 app.build_status = BuildStatus.SHOULD_BE_BUILT

tools/ci/idf_pytest/tests/conftest.py

@@ -1,8 +1,11 @@
 # SPDX-FileCopyrightText: 2023-2024 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
+import logging
 import os
+import shutil
 import sys
 import tempfile
+import typing as t
 from pathlib import Path
 
 import pytest
@@ -53,7 +56,15 @@ void app_main(void) {}
 
 
 @pytest.fixture
-def tmp_path() -> Path:
+def work_dirpath() -> t.Generator[Path, None, None]:
     os.makedirs(os.path.join(IDF_PATH, 'pytest_embedded_log'), exist_ok=True)
 
-    return Path(tempfile.mkdtemp(prefix=os.path.join(IDF_PATH, 'pytest_embedded_log') + os.sep))
+    p = Path(tempfile.mkdtemp(prefix=os.path.join(IDF_PATH, 'pytest_embedded_log') + os.sep))
+
+    try:
+        yield p
+    except Exception:
+        logging.critical('Test is failing, Please check the log in %s', p)
+        raise
+    else:
+        shutil.rmtree(p)

tools/ci/idf_pytest/tests/test_get_all_apps.py

@@ -8,19 +8,19 @@ from idf_pytest.script import SUPPORTED_TARGETS
 from conftest import create_project
 
 
-def test_get_all_apps_non(tmp_path: Path) -> None:
+def test_get_all_apps_non(work_dirpath: Path) -> None:
-    create_project('foo', tmp_path)
+    create_project('foo', work_dirpath)
-    create_project('bar', tmp_path)
+    create_project('bar', work_dirpath)
 
-    test_related_apps, non_test_related_apps = get_all_apps([str(tmp_path)])
+    test_related_apps, non_test_related_apps = get_all_apps([str(work_dirpath)])
 
     assert test_related_apps == set()
     assert len(non_test_related_apps) == 2 * len(SUPPORTED_TARGETS)
 
 
-def test_get_all_apps_single_dut_test_script(tmp_path: Path) -> None:
+def test_get_all_apps_single_dut_test_script(work_dirpath: Path) -> None:
-    create_project('foo', tmp_path)
+    create_project('foo', work_dirpath)
-    with open(tmp_path / 'foo' / 'pytest_get_all_apps_single_dut_test_script.py', 'w') as fw:
+    with open(work_dirpath / 'foo' / 'pytest_get_all_apps_single_dut_test_script.py', 'w') as fw:
         fw.write(
             """import pytest
 
@@ -30,18 +30,18 @@ def test_foo(dut):
     pass
 """
         )
-    create_project('bar', tmp_path)
+    create_project('bar', work_dirpath)
 
-    test_related_apps, non_test_related_apps = get_all_apps([str(tmp_path)], target='all')
+    test_related_apps, non_test_related_apps = get_all_apps([str(work_dirpath)], target='all')
 
     assert len(test_related_apps) == 2
     assert len(non_test_related_apps) == 2 * len(SUPPORTED_TARGETS) - 2
 
 
-def test_get_all_apps_multi_dut_with_markers_test_script(tmp_path: Path) -> None:
+def test_get_all_apps_multi_dut_with_markers_test_script(work_dirpath: Path) -> None:
-    create_project('foo', tmp_path)
+    create_project('foo', work_dirpath)
 
-    (tmp_path / 'foo' / 'pytest_get_all_apps_multi_dut_with_markers_test_script.py').write_text(
+    (work_dirpath / 'foo' / 'pytest_get_all_apps_multi_dut_with_markers_test_script.py').write_text(
         """import pytest
 
 @pytest.mark.esp32
@@ -52,15 +52,15 @@ def test_foo(dut):
         encoding='utf-8',
     )
 
-    test_related_apps, non_test_related_apps = get_all_apps([str(tmp_path)], target='all')
+    test_related_apps, non_test_related_apps = get_all_apps([str(work_dirpath)], target='all')
 
     assert len(test_related_apps) == 1
     assert len(non_test_related_apps) == len(SUPPORTED_TARGETS) - 1
 
 
-def test_get_all_apps_multi_dut_test_script(tmp_path: Path) -> None:
+def test_get_all_apps_multi_dut_test_script(work_dirpath: Path) -> None:
-    create_project('foo', tmp_path)
+    create_project('foo', work_dirpath)
-    with open(tmp_path / 'foo' / 'pytest_get_all_apps_multi_dut_test_script.py', 'w') as fw:
+    with open(work_dirpath / 'foo' / 'pytest_get_all_apps_multi_dut_test_script.py', 'w') as fw:
         fw.write(
             """import pytest
 
@@ -75,17 +75,28 @@ def test_foo(dut):
 """
         )
 
-    test_related_apps, non_test_related_apps = get_all_apps([str(tmp_path)], target='all')
+    test_related_apps, non_test_related_apps = get_all_apps([str(work_dirpath)], target='esp32s2,esp32s3')
+    assert len(test_related_apps) == 2
+    assert len(non_test_related_apps) == 0
 
-    assert len(test_related_apps) == 3  # 32, s2, s3
+    test_related_apps, non_test_related_apps = get_all_apps([str(work_dirpath)], target='esp32,esp32s3,esp32')
+    assert len(test_related_apps) == 2
+    assert len(non_test_related_apps) == 0
+
+    test_related_apps, non_test_related_apps = get_all_apps([str(work_dirpath)], target='all')
+    assert len(test_related_apps) == 3
     assert len(non_test_related_apps) == len(SUPPORTED_TARGETS) - 3
 
+    test_related_apps, non_test_related_apps = get_all_apps([str(work_dirpath)], target='foo,bar')
+    assert len(test_related_apps) == 0
+    assert len(non_test_related_apps) == 0
 
-def test_get_all_apps_modified_pytest_script(tmp_path: Path) -> None:
-    create_project('foo', tmp_path)
-    create_project('bar', tmp_path)
 
-    (tmp_path / 'pytest_get_all_apps_modified_pytest_script.py').write_text(
+def test_get_all_apps_modified_pytest_script(work_dirpath: Path) -> None:
+    create_project('foo', work_dirpath)
+    create_project('bar', work_dirpath)
+
+    (work_dirpath / 'pytest_get_all_apps_modified_pytest_script.py').write_text(
         """import pytest
 import os
 
@@ -100,20 +111,20 @@ def test_multi_foo_bar(dut):
         encoding='utf-8',
     )
 
-    test_related_apps, non_test_related_apps = get_all_apps([str(tmp_path)], target='all')
+    test_related_apps, non_test_related_apps = get_all_apps([str(work_dirpath)], target='all')
     assert len(test_related_apps) == 2  # foo-esp32, bar-esp32
     assert len(non_test_related_apps) == 2 * len(SUPPORTED_TARGETS) - 2
 
     test_related_apps, non_test_related_apps = get_all_apps(
-        [str(tmp_path)], target='all', modified_files=[], modified_components=[]
+        [str(work_dirpath)], target='all', modified_files=[], modified_components=[]
     )
     assert len(test_related_apps) == 0
     assert len(non_test_related_apps) == 0
 
     test_related_apps, non_test_related_apps = get_all_apps(
-        [str(tmp_path)],
+        [str(work_dirpath)],
         target='all',
-        modified_files=[str(tmp_path / 'pytest_get_all_apps_modified_pytest_script.py')],
+        modified_files=[str(work_dirpath / 'pytest_get_all_apps_modified_pytest_script.py')],
         modified_components=[],
     )
     assert len(test_related_apps) == 2

tools/ci/idf_pytest/tests/test_get_pytest_cases.py

@@ -32,41 +32,41 @@ def test_foo_multi_with_marker(dut):
 '''
 
 
-def test_get_pytest_cases_single_specific(tmp_path: Path) -> None:
+def test_get_pytest_cases_single_specific(work_dirpath: Path) -> None:
-    script = tmp_path / 'pytest_get_pytest_cases_single_specific.py'
+    script = work_dirpath / 'pytest_get_pytest_cases_single_specific.py'
     script.write_text(TEMPLATE_SCRIPT)
-    cases = get_pytest_cases([str(tmp_path)], 'esp32')
+    cases = get_pytest_cases([str(work_dirpath)], 'esp32')
 
     assert len(cases) == 1
     assert cases[0].targets == ['esp32']
 
 
-def test_get_pytest_cases_multi_specific(tmp_path: Path) -> None:
+def test_get_pytest_cases_multi_specific(work_dirpath: Path) -> None:
-    script = tmp_path / 'pytest_get_pytest_cases_multi_specific.py'
+    script = work_dirpath / 'pytest_get_pytest_cases_multi_specific.py'
     script.write_text(TEMPLATE_SCRIPT)
-    cases = get_pytest_cases([str(tmp_path)], 'esp32s2,esp32s2, esp32s3')
+    cases = get_pytest_cases([str(work_dirpath)], 'esp32s2,esp32s2, esp32s3')
 
     assert len(cases) == 1
     assert cases[0].targets == ['esp32s2', 'esp32s2', 'esp32s3']
 
-    cases = get_pytest_cases([str(tmp_path)], 'esp32s3,esp32s2,esp32s2')  # order matters
+    cases = get_pytest_cases([str(work_dirpath)], 'esp32s3,esp32s2,esp32s2')  # order matters
     assert len(cases) == 0
 
 
-def test_get_pytest_cases_multi_all(tmp_path: Path) -> None:
+def test_get_pytest_cases_multi_all(work_dirpath: Path) -> None:
-    script = tmp_path / 'pytest_get_pytest_cases_multi_all.py'
+    script = work_dirpath / 'pytest_get_pytest_cases_multi_all.py'
     script.write_text(TEMPLATE_SCRIPT)
-    cases = get_pytest_cases([str(tmp_path)], CollectMode.MULTI_ALL_WITH_PARAM)
+    cases = get_pytest_cases([str(work_dirpath)], CollectMode.MULTI_ALL_WITH_PARAM)
 
     assert len(cases) == 2
     assert cases[0].targets == ['esp32', 'esp32s2']
     assert cases[1].targets == ['esp32s2', 'esp32s2', 'esp32s3']
 
 
-def test_get_pytest_cases_all(tmp_path: Path) -> None:
+def test_get_pytest_cases_all(work_dirpath: Path) -> None:
-    script = tmp_path / 'pytest_get_pytest_cases_all.py'
+    script = work_dirpath / 'pytest_get_pytest_cases_all.py'
     script.write_text(TEMPLATE_SCRIPT)
-    cases = get_pytest_cases([str(tmp_path)], CollectMode.ALL)
+    cases = get_pytest_cases([str(work_dirpath)], CollectMode.ALL)
 
     assert len(cases) == 6
     assert cases[0].targets == ['esp32', 'esp32s2']