Merge branch 'ci/remove_pytest_dir_non_test_apps' into 'master'

ci: build_pytest_app will now remove the non-test apps

Closes IDFCI-1075

See merge request espressif/esp-idf!16984

.gitlab/ci/build.yml

@@ -42,63 +42,65 @@ build_pytest_examples_esp32:
     - .build_pytest_template
     - .rules:build:example_test-esp32
   script:
-    - python tools/ci/build_pytest_apps.py --all-pytest-apps --under-dir examples --target esp32 --size-info $SIZE_INFO_LOCATION -vv
+    - run_cmd python tools/ci/build_pytest_apps.py examples --target esp32 --size-info $SIZE_INFO_LOCATION -vv
 
 build_pytest_examples_esp32s2:
   extends:
     - .build_pytest_template
     - .rules:build:example_test-esp32s2
   script:
-    - python tools/ci/build_pytest_apps.py --all-pytest-apps --under-dir examples --target esp32s2 --size-info $SIZE_INFO_LOCATION -vv
+    - run_cmd python tools/ci/build_pytest_apps.py examples --target esp32s2 --size-info $SIZE_INFO_LOCATION -vv
 
 build_pytest_examples_esp32s3:
   extends:
     - .build_pytest_template
     - .rules:build:example_test-esp32s3
   script:
-    - python tools/ci/build_pytest_apps.py --all-pytest-apps --under-dir examples --target esp32s3 --size-info $SIZE_INFO_LOCATION -vv
+    - run_cmd python tools/ci/build_pytest_apps.py examples --target esp32s3 --size-info $SIZE_INFO_LOCATION -vv
 
 build_pytest_examples_esp32c3:
   extends:
     - .build_pytest_template
     - .rules:build:example_test-esp32c3
   script:
-    - python tools/ci/build_pytest_apps.py --all-pytest-apps --under-dir examples --target esp32c3 --size-info $SIZE_INFO_LOCATION -vv
+    - run_cmd python tools/ci/build_pytest_apps.py examples --target esp32c3 --size-info $SIZE_INFO_LOCATION -vv
 
 build_pytest_components_esp32:
   extends:
     - .build_pytest_template
     - .rules:build:component_ut-esp32
   script:
-    - python tools/ci/build_pytest_apps.py --all-pytest-apps --under-dir components --target esp32 --size-info $SIZE_INFO_LOCATION -vv
+    - run_cmd python tools/ci/build_pytest_apps.py components --target esp32 --size-info $SIZE_INFO_LOCATION -vv
 
 build_pytest_components_esp32s2:
   extends:
     - .build_pytest_template
     - .rules:build:component_ut-esp32s2
   script:
-    - python tools/ci/build_pytest_apps.py --all-pytest-apps --under-dir components --target esp32s2 --size-info $SIZE_INFO_LOCATION -vv
+    - run_cmd python tools/ci/build_pytest_apps.py components --target esp32s2 --size-info $SIZE_INFO_LOCATION -vv
 
 build_pytest_components_esp32s3:
   extends:
     - .build_pytest_template
     - .rules:build:component_ut-esp32s3
   script:
-    - python tools/ci/build_pytest_apps.py --all-pytest-apps --under-dir components --target esp32s3 --size-info $SIZE_INFO_LOCATION -vv
+    - run_cmd python tools/ci/build_pytest_apps.py components --target esp32s3 --size-info $SIZE_INFO_LOCATION -vv
-
-build_pytest_components_esp32c2:
-  extends:
-    - .build_pytest_template
-    - .rules:build:component_ut-esp32c2
-  script:
-    - python tools/ci/build_pytest_apps.py --all-pytest-apps --under-dir components --target esp32c2 --size-info $SIZE_INFO_LOCATION -vv
 
 build_pytest_components_esp32c3:
   extends:
     - .build_pytest_template
     - .rules:build:component_ut-esp32c3
   script:
-    - python tools/ci/build_pytest_apps.py --all-pytest-apps --under-dir components --target esp32c3 --size-info $SIZE_INFO_LOCATION -vv
+    - run_cmd python tools/ci/build_pytest_apps.py components --target esp32c3 --size-info $SIZE_INFO_LOCATION -vv
+
+build_non_test_components_apps:
+  extends:
+    - .build_template
+    - .build_test_apps_template
+  variables:
+    IDF_TARGET: all
+    TEST_PREFIX: component_ut
+    TEST_RELATIVE_DIR: component_ut
 
 .build_template_app_template:
   extends: .build_template
@@ -289,7 +291,7 @@ build_examples_cmake_esp32:
   extends:
     - .build_examples_cmake_template
     - .rules:build:example_test-esp32
-  parallel: 10
+  parallel: 12
   variables:
     IDF_TARGET: esp32
 
@@ -337,7 +339,7 @@ build_test_apps_esp32:
   extends:
     - .build_test_apps_template
     - .rules:build:custom_test-esp32
-  parallel: 8
+  parallel: 2
   variables:
     IDF_TARGET: esp32
 
@@ -345,7 +347,7 @@ build_test_apps_esp32s2:
   extends:
     - .build_test_apps_template
     - .rules:build:custom_test-esp32s2
-  parallel: 8
+  parallel: 2
   variables:
     IDF_TARGET: esp32s2
 
@@ -353,7 +355,7 @@ build_test_apps_esp32s3:
   extends:
     - .build_test_apps_template
     - .rules:build:custom_test-esp32s3
-  parallel: 8
+  parallel: 2
   variables:
     IDF_TARGET: esp32s3
 
@@ -361,7 +363,7 @@ build_test_apps_esp32c3:
   extends:
     - .build_test_apps_template
     - .rules:build:custom_test-esp32c3
-  parallel: 8
+  parallel: 2
   variables:
     IDF_TARGET: esp32c3
 
@@ -369,7 +371,6 @@ build_test_apps_esp32c2:
   extends:
     - .build_test_apps_template
     - .rules:build:custom_test-esp32c2
-  parallel: 8
   variables:
     IDF_TARGET: esp32c2
 

.gitlab/ci/pre_check.yml

@@ -142,6 +142,7 @@ check_esp_err_to_name:
 scan_tests:
   extends:
     - .pre_check_base_template
+    - .before_script_pytest
     - .rules:build:target_test
   image: $CI_DOCKER_REGISTRY/ubuntu-test-env$BOT_DOCKER_IMAGE_TAG
   tags:
@@ -150,19 +151,24 @@ scan_tests:
     paths:
       - $EXAMPLE_TEST_OUTPUT_DIR
       - $TEST_APPS_OUTPUT_DIR
+      - $COMPONENT_UT_OUTPUT_DIR
   variables:
     EXAMPLE_TEST_DIR: ${CI_PROJECT_DIR}/examples
     EXAMPLE_TEST_OUTPUT_DIR: ${CI_PROJECT_DIR}/examples/test_configs
     TEST_APPS_TEST_DIR: ${CI_PROJECT_DIR}/tools/test_apps
     TEST_APPS_OUTPUT_DIR: ${CI_PROJECT_DIR}/tools/test_apps/test_configs
+    COMPONENT_UT_OUTPUT_DIR: ${CI_PROJECT_DIR}/component_ut/test_configs
     CI_SCAN_TESTS_PY: ${CI_PROJECT_DIR}/tools/ci/python_packages/ttfw_idf/CIScanTests.py
     EXTRA_TEST_DIRS: >-
       examples/bluetooth/esp_ble_mesh/ble_mesh_console
       examples/bluetooth/hci/controller_hci_uart_esp32
       examples/wifi/iperf
+    EXTRA_EVALUATE_ARGS: '--evaluate-parallel-count --config "sdkconfig.ci=default" --config "sdkconfig.ci.*=" --config "=default"'
   script:
-    - run_cmd python $CI_SCAN_TESTS_PY example_test $EXAMPLE_TEST_DIR -b cmake --exclude examples/build_system/idf_as_lib -c $CI_TARGET_TEST_CONFIG_FILE -o $EXAMPLE_TEST_OUTPUT_DIR --extra_test_dirs $EXTRA_TEST_DIRS
+    - run_cmd python $CI_SCAN_TESTS_PY example_test $EXAMPLE_TEST_DIR -b cmake --exclude examples/build_system/idf_as_lib -c $CI_TARGET_TEST_CONFIG_FILE -o $EXAMPLE_TEST_OUTPUT_DIR --extra_test_dirs $EXTRA_TEST_DIRS $EXTRA_EVALUATE_ARGS
-    - run_cmd python $CI_SCAN_TESTS_PY test_apps $TEST_APPS_TEST_DIR -c $CI_TARGET_TEST_CONFIG_FILE -o $TEST_APPS_OUTPUT_DIR
+    - run_cmd python $CI_SCAN_TESTS_PY test_apps $TEST_APPS_TEST_DIR -c $CI_TARGET_TEST_CONFIG_FILE -o $TEST_APPS_OUTPUT_DIR $EXTRA_EVALUATE_ARGS
+    - set_component_ut_vars
+    - run_cmd python $CI_SCAN_TESTS_PY component_ut $COMPONENT_UT_DIRS --exclude $COMPONENT_UT_EXCLUDES -c $CI_TARGET_TEST_CONFIG_FILE -o $COMPONENT_UT_OUTPUT_DIR --combine-all-targets --except-targets linux $EXTRA_EVALUATE_ARGS
 
 # For release tag pipelines only, make sure the tag was created with 'git tag -a' so it will update
 # the version returned by 'git describe'

tools/ci/build_pytest_apps.py

@@ -9,39 +9,39 @@ import argparse
 import logging
 import os
 import sys
+from collections import defaultdict
 from typing import List
 
-from idf_ci_utils import IDF_PATH, get_pytest_dirs
+from idf_ci_utils import IDF_PATH, get_pytest_cases
 
 try:
     from build_apps import build_apps
-    from find_apps import find_apps, find_builds_for_app
+    from find_apps import find_builds_for_app
-    from find_build_apps import BuildItem, CMakeBuildSystem, config_rules_from_str, setup_logging
+    from find_build_apps import BuildItem, config_rules_from_str, setup_logging
 except ImportError:
     sys.path.append(os.path.join(IDF_PATH, 'tools'))
 
     from build_apps import build_apps
-    from find_apps import find_apps, find_builds_for_app
+    from find_apps import find_builds_for_app
-    from find_build_apps import BuildItem, CMakeBuildSystem, config_rules_from_str, setup_logging
+    from find_build_apps import BuildItem, config_rules_from_str, setup_logging
 
 
 def main(args: argparse.Namespace) -> None:
-    if args.all_pytest_apps:
+    pytest_cases = []
-        paths = get_pytest_dirs(args.under_dir)
+    for path in args.paths:
-        args.recursive = True
+        pytest_cases += get_pytest_cases(path, args.target)
-    elif args.paths is None:
-        paths = [os.getcwd()]
-    else:
-        paths = args.paths
 
-    app_dirs = []
+    paths = set()
-    for path in paths:
+    app_configs = defaultdict(set)
-        app_dirs += find_apps(CMakeBuildSystem, path, args.recursive, [], args.target)
+    for case in pytest_cases:
+        paths.add(case.app_path)
+        app_configs[case.app_path].add(case.config)
+
+    app_dirs = list(paths)
     if not app_dirs:
-        logging.error('No apps found')
+        raise RuntimeError('No apps found')
-        sys.exit(1)
 
-    logging.info('Found {} apps'.format(len(app_dirs)))
+    logging.info(f'Found {len(app_dirs)} apps')
     app_dirs.sort()
 
     # Find compatible configurations of each app, collect them as BuildItems
@@ -50,61 +50,58 @@ def main(args: argparse.Namespace) -> None:
     for app_dir in app_dirs:
         app_dir = os.path.realpath(app_dir)
         build_items += find_builds_for_app(
-            app_dir,
+            app_path=app_dir,
-            app_dir,
+            work_dir=app_dir,
-            'build_@t_@w',
+            build_dir='build_@t_@w',
-            f'{app_dir}/build_@t_@w/build.log',
+            build_log=f'{app_dir}/build_@t_@w/build.log',
-            args.target,
+            target_arg=args.target,
-            'cmake',
+            build_system='cmake',
-            config_rules,
+            config_rules=config_rules,
-            True,
         )
-    logging.info('Found {} builds'.format(len(build_items)))
+    logging.info(f'Found {len(build_items)} builds')
     build_items.sort(key=lambda x: x.build_path)  # type: ignore
 
-    build_apps(build_items, args.parallel_count, args.parallel_index, False, args.build_verbose, True, None,
+    # auto clean up the binaries if no flag --preserve-all
-               args.size_info)
+    if args.preserve_all is False:
+        for item in build_items:
+            if item.config_name not in app_configs[item.app_dir]:
+                item.preserve = False
+
+    build_apps(
+        build_items=build_items,
+        parallel_count=args.parallel_count,
+        parallel_index=args.parallel_index,
+        dry_run=False,
+        build_verbose=args.build_verbose,
+        keep_going=True,
+        output_build_list=None,
+        size_info=args.size_info,
+    )
 
 
 if __name__ == '__main__':
-    parser = argparse.ArgumentParser(description='Tool to generate build steps for IDF apps')
+    parser = argparse.ArgumentParser(
-    parser.add_argument(
+        description='Build all the pytest apps under specified paths. Will auto remove those non-test apps binaries'
-        '--recursive',
-        action='store_true',
-        help='Look for apps in the specified directories recursively.',
     )
     parser.add_argument('--target', required=True, help='Build apps for given target.')
     parser.add_argument(
         '--config',
         default=['sdkconfig.ci=default', 'sdkconfig.ci.*=', '=default'],
         action='append',
-        help='Adds configurations (sdkconfig file names) to build. This can either be ' +
+        help='Adds configurations (sdkconfig file names) to build. This can either be '
-             'FILENAME[=NAME] or FILEPATTERN. FILENAME is the name of the sdkconfig file, ' +
+        + 'FILENAME[=NAME] or FILEPATTERN. FILENAME is the name of the sdkconfig file, '
-             'relative to the project directory, to be used. Optional NAME can be specified, ' +
+        + 'relative to the project directory, to be used. Optional NAME can be specified, '
-             'which can be used as a name of this configuration. FILEPATTERN is the name of ' +
+        + 'which can be used as a name of this configuration. FILEPATTERN is the name of '
-             'the sdkconfig file, relative to the project directory, with at most one wildcard. ' +
+        + 'the sdkconfig file, relative to the project directory, with at most one wildcard. '
-             'The part captured by the wildcard is used as the name of the configuration.',
+        + 'The part captured by the wildcard is used as the name of the configuration.',
     )
     parser.add_argument(
-        '-p', '--paths',
+        'paths',
-        nargs='*',
+        nargs='+',
-        help='One or more app paths. Will use the current path if not specified.'
+        help='One or more app paths. Will use the current path if not specified.',
     )
     parser.add_argument(
-        '--all-pytest-apps',
+        '--parallel-count', default=1, type=int, help='Number of parallel build jobs.'
-        action='store_true',
-        help='Look for all pytest apps. "--paths" would be ignored if specify this flag.'
-    )
-    parser.add_argument(
-        '--under-dir',
-        help='Build only the pytest apps under this directory if specified. '
-             'Would be ignored if "--all-pytest-apps" is unflagged.'
-    )
-    parser.add_argument(
-        '--parallel-count',
-        default=1,
-        type=int,
-        help='Number of parallel build jobs.'
     )
     parser.add_argument(
         '--parallel-index',
@@ -115,7 +112,7 @@ if __name__ == '__main__':
     parser.add_argument(
         '--size-info',
         type=argparse.FileType('a'),
-        help='If specified, the test case name and size info json will be written to this file'
+        help='If specified, the test case name and size info json will be written to this file',
     )
     parser.add_argument(
         '-v',
@@ -128,6 +125,11 @@ if __name__ == '__main__':
         action='store_true',
         help='Enable verbose output from build system.',
     )
+    parser.add_argument(
+        '--preserve-all',
+        action='store_true',
+        help='add this flag to preserve the binaries for all apps',
+    )
     arguments = parser.parse_args()
     setup_logging(arguments)
     main(arguments)

tools/ci/idf_ci_utils.py

@@ -4,13 +4,20 @@
 # SPDX-FileCopyrightText: 2020-2021 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
 #
+import io
 import logging
 import os
 import subprocess
 import sys
-from typing import List
+from contextlib import redirect_stdout
+from typing import TYPE_CHECKING, List
 
-IDF_PATH = os.path.abspath(os.getenv('IDF_PATH', os.path.join(os.path.dirname(__file__), '..', '..')))
+if TYPE_CHECKING:
+    from _pytest.nodes import Function
+
+IDF_PATH = os.path.abspath(
+    os.getenv('IDF_PATH', os.path.join(os.path.dirname(__file__), '..', '..'))
+)
 
 
 def get_submodule_dirs(full_path: bool = False) -> List:
@@ -21,9 +28,21 @@ def get_submodule_dirs(full_path: bool = False) -> List:
     """
     dirs = []
     try:
-        lines = subprocess.check_output(
+        lines = (
-            ['git', 'config', '--file', os.path.realpath(os.path.join(IDF_PATH, '.gitmodules')),
+            subprocess.check_output(
-             '--get-regexp', 'path']).decode('utf8').strip().split('\n')
+                [
+                    'git',
+                    'config',
+                    '--file',
+                    os.path.realpath(os.path.join(IDF_PATH, '.gitmodules')),
+                    '--get-regexp',
+                    'path',
+                ]
+            )
+            .decode('utf8')
+            .strip()
+            .split('\n')
+        )
         for line in lines:
             _, path = line.split(' ')
             if full_path:
@@ -38,7 +57,11 @@ def get_submodule_dirs(full_path: bool = False) -> List:
 
 def _check_git_filemode(full_path):  # type: (str) -> bool
     try:
-        stdout = subprocess.check_output(['git', 'ls-files', '--stage', full_path]).strip().decode('utf-8')
+        stdout = (
+            subprocess.check_output(['git', 'ls-files', '--stage', full_path])
+            .strip()
+            .decode('utf-8')
+        )
     except subprocess.CalledProcessError:
         return True
 
@@ -74,8 +97,12 @@ def get_git_files(path: str = IDF_PATH, full_path: bool = False) -> List[str]:
         # folder if no `.git` folder found in `cwd`.
         workaround_env = os.environ.copy()
         workaround_env.pop('GIT_DIR', None)
-        files = subprocess.check_output(['git', 'ls-files'], cwd=path, env=workaround_env) \
+        files = (
-            .decode('utf8').strip().split('\n')
+            subprocess.check_output(['git', 'ls-files'], cwd=path, env=workaround_env)
+            .decode('utf8')
+            .strip()
+            .split('\n')
+        )
     except Exception as e:  # pylint: disable=W0703
         logging.warning(str(e))
         files = []
@@ -86,30 +113,61 @@ def is_in_directory(file_path: str, folder: str) -> bool:
     return os.path.realpath(file_path).startswith(os.path.realpath(folder) + os.sep)
 
 
-def get_pytest_dirs(folder: str) -> List[str]:
+class PytestCase:
+    def __init__(self, test_path: str, target: str, config: str, case: str):
+        self.app_path = os.path.dirname(test_path)
+        self.test_path = test_path
+        self.target = target
+        self.config = config
+        self.case = case
+
+    def __repr__(self) -> str:
+        return f'{self.test_path}: {self.target}.{self.config}.{self.case}'
+
+
+class PytestCollectPlugin:
+    def __init__(self, target: str) -> None:
+        self.target = target
+        self.nodes: List[PytestCase] = []
+
+    def pytest_collection_modifyitems(self, items: List['Function']) -> None:
+        for item in items:
+            try:
+                file_path = str(item.path)
+            except AttributeError:
+                # pytest 6.x
+                file_path = item.fspath
+
+            target = self.target
+            if hasattr(item, 'callspec'):
+                config = item.callspec.params.get('config', 'default')
+            else:
+                config = 'default'
+            case_name = item.originalname
+
+            self.nodes.append(PytestCase(file_path, target, config, case_name))
+
+
+def get_pytest_cases(folder: str, target: str) -> List[PytestCase]:
     import pytest
-    from _pytest.nodes import Item
+    from _pytest.config import ExitCode
 
-    class CollectPlugin:
+    collector = PytestCollectPlugin(target)
-        def __init__(self) -> None:
-            self.nodes: List[Item] = []
 
-        def pytest_collection_modifyitems(self, items: List[Item]) -> None:
+    with io.StringIO() as buf:
-            for item in items:
+        with redirect_stdout(buf):
-                self.nodes.append(item)
+            res = pytest.main(['--collect-only', folder, '-q', '--target', target], plugins=[collector])
+        if res.value != ExitCode.OK:
+            if res.value == ExitCode.NO_TESTS_COLLECTED:
+                print(f'WARNING: no pytest app found for target {target} under folder {folder}')
+            else:
+                print(buf.getvalue())
+                raise RuntimeError('pytest collection failed')
 
-    collector = CollectPlugin()
+    return collector.nodes
 
-    res = pytest.main(['--collect-only', '-q', folder], plugins=[collector])
-    if res.value != 0:
-        raise RuntimeError('pytest collection failed')
 
-    sys.stdout.flush()  # print instantly
+def get_pytest_app_paths(folder: str, target: str) -> List[str]:
+    nodes = get_pytest_cases(folder, target)
 
-    try:
+    return list({node.app_path for node in nodes})
-        test_file_paths = set(node.path for node in collector.nodes)
-    except AttributeError:
-        # pytest 6.x
-        test_file_paths = set(node.fspath for node in collector.nodes)
-
-    return [os.path.dirname(file) for file in test_file_paths]

tools/ci/python_packages/ttfw_idf/CIScanTests.py

@@ -5,14 +5,11 @@ import logging
 import os
 from collections import defaultdict
 from copy import deepcopy
+from typing import Any
 
-try:
+from ci.idf_ci_utils import get_pytest_app_paths
-    from typing import Any
+from find_apps import find_apps, find_builds_for_app
-except ImportError:
+from find_build_apps import BUILD_SYSTEM_CMAKE, BUILD_SYSTEMS, config_rules_from_str
-    # Only used for type annotations
-    pass
-from find_apps import find_apps
-from find_build_apps import BUILD_SYSTEM_CMAKE, BUILD_SYSTEMS
 from idf_py_actions.constants import PREVIEW_TARGETS, SUPPORTED_TARGETS
 from ttfw_idf.IDFAssignTest import ExampleAssignTest, TestAppsAssignTest
 
@@ -32,6 +29,8 @@ BUILD_ALL_LABELS = [
     'BOT_LABEL_WEEKEND_TEST',
 ]
 
+BUILD_PER_JOB = 30  # each build takes 1 mins around
+
 
 def _has_build_all_label():  # type: () -> bool
     for label in BUILD_ALL_LABELS:
@@ -100,6 +99,23 @@ def main():  # type: () -> None
                         help='add this flag to preserve artifacts for all apps')
     parser.add_argument('--build-all', action='store_true',
                         help='add this flag to build all apps')
+    parser.add_argument('--combine-all-targets', action='store_true',
+                        help='add this flag to combine all target jsons into one')
+    parser.add_argument('--except-targets', nargs='+',
+                        help='only useful when "--combine-all-targets". Specified targets would be skipped.')
+    parser.add_argument(
+        '--config',
+        action='append',
+        help='Only useful when "--evaluate-parallel-count" is flagged.'
+             'Adds configurations (sdkconfig file names) to build. This can either be ' +
+             'FILENAME[=NAME] or FILEPATTERN. FILENAME is the name of the sdkconfig file, ' +
+             'relative to the project directory, to be used. Optional NAME can be specified, ' +
+             'which can be used as a name of this configuration. FILEPATTERN is the name of ' +
+             'the sdkconfig file, relative to the project directory, with at most one wildcard. ' +
+             'The part captured by the wildcard is used as the name of the configuration.',
+    )
+    parser.add_argument('--evaluate-parallel-count', action='store_true',
+                        help='suggest parallel count according to build items')
 
     args = parser.parse_args()
     build_test_case_apps, build_standalone_apps = _judge_build_or_not(args.test_type, args.build_all)
@@ -173,9 +189,19 @@ def main():  # type: () -> None
         else:
             scan_info_dict[target]['standalone_apps'] = set()
     test_case_apps_preserve_default = True if build_system == 'cmake' else False
+    output_files = []
+    build_items_total_count = 0
     for target in SUPPORTED_TARGETS:
+        # get pytest apps paths
+        pytest_app_paths = set()
+        for path in paths:
+            pytest_app_paths.update(get_pytest_app_paths(path, target))
+
         apps = []
         for app_dir in scan_info_dict[target]['test_case_apps']:
+            if app_dir in pytest_app_paths:
+                print(f'WARNING: has pytest script: {app_dir}')
+                continue
             apps.append({
                 'app_dir': app_dir,
                 'build_system': args.build_system,
@@ -183,6 +209,9 @@ def main():  # type: () -> None
                 'preserve': args.preserve_all or test_case_apps_preserve_default
             })
         for app_dir in scan_info_dict[target]['standalone_apps']:
+            if app_dir in pytest_app_paths:
+                print(f'Skipping pytest app: {app_dir}')
+                continue
             apps.append({
                 'app_dir': app_dir,
                 'build_system': args.build_system,
@@ -191,8 +220,49 @@ def main():  # type: () -> None
             })
         output_path = os.path.join(args.output_path, 'scan_{}_{}.json'.format(target.lower(), build_system))
         with open(output_path, 'w') as fw:
+            if args.evaluate_parallel_count:
+                build_items = []
+                config_rules = config_rules_from_str(args.config or [])
+                for app in apps:
+                    build_items += find_builds_for_app(
+                        app['app_dir'],
+                        app['app_dir'],
+                        'build',
+                        '',
+                        app['target'],
+                        app['build_system'],
+                        config_rules,
+                        app['preserve'],
+                    )
+                print('Found {} builds'.format(len(build_items)))
+                if args.combine_all_targets:
+                    if (args.except_targets and target not in [t.lower() for t in args.except_targets]) \
+                            or (not args.except_targets):
+                        build_items_total_count += len(build_items)
+                else:
+                    print(f'suggest set parallel count for target {target} to {len(build_items) // BUILD_PER_JOB + 1}')
             fw.writelines([json.dumps(app) + '\n' for app in apps])
 
+            if args.combine_all_targets:
+                if (args.except_targets and target not in [t.lower() for t in args.except_targets]) \
+                        or (not args.except_targets):
+                    output_files.append(output_path)
+                else:
+                    print(f'skipping combining target {target}')
+
+    if args.combine_all_targets:
+        scan_all_json = os.path.join(args.output_path, f'scan_all_{build_system}.json')
+        lines = []
+        for file in output_files:
+            with open(file) as fr:
+                lines.extend([line for line in fr.readlines() if line.strip()])
+        with open(scan_all_json, 'w') as fw:
+            fw.writelines(lines)
+        print(f'combined into file: {scan_all_json}')
+
+        if args.evaluate_parallel_count:
+            print(f'Total build: {build_items_total_count}. Suggest set parallel count for all target to {build_items_total_count // BUILD_PER_JOB + 1}')
+
 
 if __name__ == '__main__':
     main()

tools/ci/utils.sh

@@ -42,6 +42,13 @@ function get_all_submodules() {
   git config --file .gitmodules --get-regexp path | awk '{ print $2 }' | sed -e 's|$|/**|' | xargs | sed -e 's/ /,/g'
 }
 
+function set_component_ut_vars() {
+  local exclude_list_fp="${IDF_PATH}/tools/ci/component_ut_excludes.txt"
+  export COMPONENT_UT_DIRS=$(find components/ -name test_apps -type d | xargs)
+  export COMPONENT_UT_EXCLUDES=$([ -r $exclude_list_fp ] && cat $exclude_list_fp | xargs)
+  echo "exported variables COMPONENT_UT_DIRS, COMPONENT_UT_EXCLUDES"
+}
+
 function error() {
   printf "\033[0;31m%s\n\033[0m" "${1}" >&2
 }

tools/find_build_apps/cmake.py

@@ -63,6 +63,7 @@ class CMakeBuildSystem(BuildSystem):
             build_stderr = log_file
 
         try:
+            os.environ['IDF_TARGET'] = build_item.target
             subprocess.check_call(args, stdout=build_stdout, stderr=build_stderr)
         except subprocess.CalledProcessError as e:
             raise BuildError('Build failed with exit code {}'.format(e.returncode))