diff --git a/.gitlab/ci/build.yml b/.gitlab/ci/build.yml index 8e814b5755..af1d3f455e 100644 --- a/.gitlab/ci/build.yml +++ b/.gitlab/ci/build.yml @@ -42,63 +42,65 @@ build_pytest_examples_esp32: - .build_pytest_template - .rules:build:example_test-esp32 script: - - python tools/ci/build_pytest_apps.py --all-pytest-apps --under-dir examples --target esp32 --size-info $SIZE_INFO_LOCATION -vv + - run_cmd python tools/ci/build_pytest_apps.py examples --target esp32 --size-info $SIZE_INFO_LOCATION -vv build_pytest_examples_esp32s2: extends: - .build_pytest_template - .rules:build:example_test-esp32s2 script: - - python tools/ci/build_pytest_apps.py --all-pytest-apps --under-dir examples --target esp32s2 --size-info $SIZE_INFO_LOCATION -vv + - run_cmd python tools/ci/build_pytest_apps.py examples --target esp32s2 --size-info $SIZE_INFO_LOCATION -vv build_pytest_examples_esp32s3: extends: - .build_pytest_template - .rules:build:example_test-esp32s3 script: - - python tools/ci/build_pytest_apps.py --all-pytest-apps --under-dir examples --target esp32s3 --size-info $SIZE_INFO_LOCATION -vv + - run_cmd python tools/ci/build_pytest_apps.py examples --target esp32s3 --size-info $SIZE_INFO_LOCATION -vv build_pytest_examples_esp32c3: extends: - .build_pytest_template - .rules:build:example_test-esp32c3 script: - - python tools/ci/build_pytest_apps.py --all-pytest-apps --under-dir examples --target esp32c3 --size-info $SIZE_INFO_LOCATION -vv + - run_cmd python tools/ci/build_pytest_apps.py examples --target esp32c3 --size-info $SIZE_INFO_LOCATION -vv build_pytest_components_esp32: extends: - .build_pytest_template - .rules:build:component_ut-esp32 script: - - python tools/ci/build_pytest_apps.py --all-pytest-apps --under-dir components --target esp32 --size-info $SIZE_INFO_LOCATION -vv + - run_cmd python tools/ci/build_pytest_apps.py components --target esp32 --size-info $SIZE_INFO_LOCATION -vv build_pytest_components_esp32s2: extends: - .build_pytest_template - .rules:build:component_ut-esp32s2 script: - - python tools/ci/build_pytest_apps.py --all-pytest-apps --under-dir components --target esp32s2 --size-info $SIZE_INFO_LOCATION -vv + - run_cmd python tools/ci/build_pytest_apps.py components --target esp32s2 --size-info $SIZE_INFO_LOCATION -vv build_pytest_components_esp32s3: extends: - .build_pytest_template - .rules:build:component_ut-esp32s3 script: - - python tools/ci/build_pytest_apps.py --all-pytest-apps --under-dir components --target esp32s3 --size-info $SIZE_INFO_LOCATION -vv - -build_pytest_components_esp32c2: - extends: - - .build_pytest_template - - .rules:build:component_ut-esp32c2 - script: - - python tools/ci/build_pytest_apps.py --all-pytest-apps --under-dir components --target esp32c2 --size-info $SIZE_INFO_LOCATION -vv + - run_cmd python tools/ci/build_pytest_apps.py components --target esp32s3 --size-info $SIZE_INFO_LOCATION -vv build_pytest_components_esp32c3: extends: - .build_pytest_template - .rules:build:component_ut-esp32c3 script: - - python tools/ci/build_pytest_apps.py --all-pytest-apps --under-dir components --target esp32c3 --size-info $SIZE_INFO_LOCATION -vv + - run_cmd python tools/ci/build_pytest_apps.py components --target esp32c3 --size-info $SIZE_INFO_LOCATION -vv + +build_non_test_components_apps: + extends: + - .build_template + - .build_test_apps_template + variables: + IDF_TARGET: all + TEST_PREFIX: component_ut + TEST_RELATIVE_DIR: component_ut .build_template_app_template: extends: .build_template @@ -289,7 +291,7 @@ build_examples_cmake_esp32: extends: - .build_examples_cmake_template - .rules:build:example_test-esp32 - parallel: 10 + parallel: 12 variables: IDF_TARGET: esp32 @@ -337,7 +339,7 @@ build_test_apps_esp32: extends: - .build_test_apps_template - .rules:build:custom_test-esp32 - parallel: 8 + parallel: 2 variables: IDF_TARGET: esp32 @@ -345,7 +347,7 @@ build_test_apps_esp32s2: extends: - .build_test_apps_template - .rules:build:custom_test-esp32s2 - parallel: 8 + parallel: 2 variables: IDF_TARGET: esp32s2 @@ -353,7 +355,7 @@ build_test_apps_esp32s3: extends: - .build_test_apps_template - .rules:build:custom_test-esp32s3 - parallel: 8 + parallel: 2 variables: IDF_TARGET: esp32s3 @@ -361,7 +363,7 @@ build_test_apps_esp32c3: extends: - .build_test_apps_template - .rules:build:custom_test-esp32c3 - parallel: 8 + parallel: 2 variables: IDF_TARGET: esp32c3 @@ -369,7 +371,6 @@ build_test_apps_esp32c2: extends: - .build_test_apps_template - .rules:build:custom_test-esp32c2 - parallel: 8 variables: IDF_TARGET: esp32c2 diff --git a/.gitlab/ci/pre_check.yml b/.gitlab/ci/pre_check.yml index 14c9c6a4d0..aa69df9abe 100644 --- a/.gitlab/ci/pre_check.yml +++ b/.gitlab/ci/pre_check.yml @@ -142,6 +142,7 @@ check_esp_err_to_name: scan_tests: extends: - .pre_check_base_template + - .before_script_pytest - .rules:build:target_test image: $CI_DOCKER_REGISTRY/ubuntu-test-env$BOT_DOCKER_IMAGE_TAG tags: @@ -150,19 +151,24 @@ scan_tests: paths: - $EXAMPLE_TEST_OUTPUT_DIR - $TEST_APPS_OUTPUT_DIR + - $COMPONENT_UT_OUTPUT_DIR variables: EXAMPLE_TEST_DIR: ${CI_PROJECT_DIR}/examples EXAMPLE_TEST_OUTPUT_DIR: ${CI_PROJECT_DIR}/examples/test_configs TEST_APPS_TEST_DIR: ${CI_PROJECT_DIR}/tools/test_apps TEST_APPS_OUTPUT_DIR: ${CI_PROJECT_DIR}/tools/test_apps/test_configs + COMPONENT_UT_OUTPUT_DIR: ${CI_PROJECT_DIR}/component_ut/test_configs CI_SCAN_TESTS_PY: ${CI_PROJECT_DIR}/tools/ci/python_packages/ttfw_idf/CIScanTests.py EXTRA_TEST_DIRS: >- examples/bluetooth/esp_ble_mesh/ble_mesh_console examples/bluetooth/hci/controller_hci_uart_esp32 examples/wifi/iperf + EXTRA_EVALUATE_ARGS: '--evaluate-parallel-count --config "sdkconfig.ci=default" --config "sdkconfig.ci.*=" --config "=default"' script: - - run_cmd python $CI_SCAN_TESTS_PY example_test $EXAMPLE_TEST_DIR -b cmake --exclude examples/build_system/idf_as_lib -c $CI_TARGET_TEST_CONFIG_FILE -o $EXAMPLE_TEST_OUTPUT_DIR --extra_test_dirs $EXTRA_TEST_DIRS - - run_cmd python $CI_SCAN_TESTS_PY test_apps $TEST_APPS_TEST_DIR -c $CI_TARGET_TEST_CONFIG_FILE -o $TEST_APPS_OUTPUT_DIR + - run_cmd python $CI_SCAN_TESTS_PY example_test $EXAMPLE_TEST_DIR -b cmake --exclude examples/build_system/idf_as_lib -c $CI_TARGET_TEST_CONFIG_FILE -o $EXAMPLE_TEST_OUTPUT_DIR --extra_test_dirs $EXTRA_TEST_DIRS $EXTRA_EVALUATE_ARGS + - run_cmd python $CI_SCAN_TESTS_PY test_apps $TEST_APPS_TEST_DIR -c $CI_TARGET_TEST_CONFIG_FILE -o $TEST_APPS_OUTPUT_DIR $EXTRA_EVALUATE_ARGS + - set_component_ut_vars + - run_cmd python $CI_SCAN_TESTS_PY component_ut $COMPONENT_UT_DIRS --exclude $COMPONENT_UT_EXCLUDES -c $CI_TARGET_TEST_CONFIG_FILE -o $COMPONENT_UT_OUTPUT_DIR --combine-all-targets --except-targets linux $EXTRA_EVALUATE_ARGS # For release tag pipelines only, make sure the tag was created with 'git tag -a' so it will update # the version returned by 'git describe' diff --git a/tools/ci/build_pytest_apps.py b/tools/ci/build_pytest_apps.py index f26e92fdb6..e5683d17fa 100644 --- a/tools/ci/build_pytest_apps.py +++ b/tools/ci/build_pytest_apps.py @@ -9,39 +9,39 @@ import argparse import logging import os import sys +from collections import defaultdict from typing import List -from idf_ci_utils import IDF_PATH, get_pytest_dirs +from idf_ci_utils import IDF_PATH, get_pytest_cases try: from build_apps import build_apps - from find_apps import find_apps, find_builds_for_app - from find_build_apps import BuildItem, CMakeBuildSystem, config_rules_from_str, setup_logging + from find_apps import find_builds_for_app + from find_build_apps import BuildItem, config_rules_from_str, setup_logging except ImportError: sys.path.append(os.path.join(IDF_PATH, 'tools')) from build_apps import build_apps - from find_apps import find_apps, find_builds_for_app - from find_build_apps import BuildItem, CMakeBuildSystem, config_rules_from_str, setup_logging + from find_apps import find_builds_for_app + from find_build_apps import BuildItem, config_rules_from_str, setup_logging def main(args: argparse.Namespace) -> None: - if args.all_pytest_apps: - paths = get_pytest_dirs(args.under_dir) - args.recursive = True - elif args.paths is None: - paths = [os.getcwd()] - else: - paths = args.paths + pytest_cases = [] + for path in args.paths: + pytest_cases += get_pytest_cases(path, args.target) - app_dirs = [] - for path in paths: - app_dirs += find_apps(CMakeBuildSystem, path, args.recursive, [], args.target) + paths = set() + app_configs = defaultdict(set) + for case in pytest_cases: + paths.add(case.app_path) + app_configs[case.app_path].add(case.config) + + app_dirs = list(paths) if not app_dirs: - logging.error('No apps found') - sys.exit(1) + raise RuntimeError('No apps found') - logging.info('Found {} apps'.format(len(app_dirs))) + logging.info(f'Found {len(app_dirs)} apps') app_dirs.sort() # Find compatible configurations of each app, collect them as BuildItems @@ -50,61 +50,58 @@ def main(args: argparse.Namespace) -> None: for app_dir in app_dirs: app_dir = os.path.realpath(app_dir) build_items += find_builds_for_app( - app_dir, - app_dir, - 'build_@t_@w', - f'{app_dir}/build_@t_@w/build.log', - args.target, - 'cmake', - config_rules, - True, + app_path=app_dir, + work_dir=app_dir, + build_dir='build_@t_@w', + build_log=f'{app_dir}/build_@t_@w/build.log', + target_arg=args.target, + build_system='cmake', + config_rules=config_rules, ) - logging.info('Found {} builds'.format(len(build_items))) + logging.info(f'Found {len(build_items)} builds') build_items.sort(key=lambda x: x.build_path) # type: ignore - build_apps(build_items, args.parallel_count, args.parallel_index, False, args.build_verbose, True, None, - args.size_info) + # auto clean up the binaries if no flag --preserve-all + if args.preserve_all is False: + for item in build_items: + if item.config_name not in app_configs[item.app_dir]: + item.preserve = False + + build_apps( + build_items=build_items, + parallel_count=args.parallel_count, + parallel_index=args.parallel_index, + dry_run=False, + build_verbose=args.build_verbose, + keep_going=True, + output_build_list=None, + size_info=args.size_info, + ) if __name__ == '__main__': - parser = argparse.ArgumentParser(description='Tool to generate build steps for IDF apps') - parser.add_argument( - '--recursive', - action='store_true', - help='Look for apps in the specified directories recursively.', + parser = argparse.ArgumentParser( + description='Build all the pytest apps under specified paths. Will auto remove those non-test apps binaries' ) parser.add_argument('--target', required=True, help='Build apps for given target.') parser.add_argument( '--config', default=['sdkconfig.ci=default', 'sdkconfig.ci.*=', '=default'], action='append', - help='Adds configurations (sdkconfig file names) to build. This can either be ' + - 'FILENAME[=NAME] or FILEPATTERN. FILENAME is the name of the sdkconfig file, ' + - 'relative to the project directory, to be used. Optional NAME can be specified, ' + - 'which can be used as a name of this configuration. FILEPATTERN is the name of ' + - 'the sdkconfig file, relative to the project directory, with at most one wildcard. ' + - 'The part captured by the wildcard is used as the name of the configuration.', + help='Adds configurations (sdkconfig file names) to build. This can either be ' + + 'FILENAME[=NAME] or FILEPATTERN. FILENAME is the name of the sdkconfig file, ' + + 'relative to the project directory, to be used. Optional NAME can be specified, ' + + 'which can be used as a name of this configuration. FILEPATTERN is the name of ' + + 'the sdkconfig file, relative to the project directory, with at most one wildcard. ' + + 'The part captured by the wildcard is used as the name of the configuration.', ) parser.add_argument( - '-p', '--paths', - nargs='*', - help='One or more app paths. Will use the current path if not specified.' + 'paths', + nargs='+', + help='One or more app paths. Will use the current path if not specified.', ) parser.add_argument( - '--all-pytest-apps', - action='store_true', - help='Look for all pytest apps. "--paths" would be ignored if specify this flag.' - ) - parser.add_argument( - '--under-dir', - help='Build only the pytest apps under this directory if specified. ' - 'Would be ignored if "--all-pytest-apps" is unflagged.' - ) - parser.add_argument( - '--parallel-count', - default=1, - type=int, - help='Number of parallel build jobs.' + '--parallel-count', default=1, type=int, help='Number of parallel build jobs.' ) parser.add_argument( '--parallel-index', @@ -115,7 +112,7 @@ if __name__ == '__main__': parser.add_argument( '--size-info', type=argparse.FileType('a'), - help='If specified, the test case name and size info json will be written to this file' + help='If specified, the test case name and size info json will be written to this file', ) parser.add_argument( '-v', @@ -128,6 +125,11 @@ if __name__ == '__main__': action='store_true', help='Enable verbose output from build system.', ) + parser.add_argument( + '--preserve-all', + action='store_true', + help='add this flag to preserve the binaries for all apps', + ) arguments = parser.parse_args() setup_logging(arguments) main(arguments) diff --git a/tools/ci/idf_ci_utils.py b/tools/ci/idf_ci_utils.py index 6327e4fa44..9866a5c23d 100644 --- a/tools/ci/idf_ci_utils.py +++ b/tools/ci/idf_ci_utils.py @@ -4,13 +4,20 @@ # SPDX-FileCopyrightText: 2020-2021 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 # +import io import logging import os import subprocess import sys -from typing import List +from contextlib import redirect_stdout +from typing import TYPE_CHECKING, List -IDF_PATH = os.path.abspath(os.getenv('IDF_PATH', os.path.join(os.path.dirname(__file__), '..', '..'))) +if TYPE_CHECKING: + from _pytest.nodes import Function + +IDF_PATH = os.path.abspath( + os.getenv('IDF_PATH', os.path.join(os.path.dirname(__file__), '..', '..')) +) def get_submodule_dirs(full_path: bool = False) -> List: @@ -21,9 +28,21 @@ def get_submodule_dirs(full_path: bool = False) -> List: """ dirs = [] try: - lines = subprocess.check_output( - ['git', 'config', '--file', os.path.realpath(os.path.join(IDF_PATH, '.gitmodules')), - '--get-regexp', 'path']).decode('utf8').strip().split('\n') + lines = ( + subprocess.check_output( + [ + 'git', + 'config', + '--file', + os.path.realpath(os.path.join(IDF_PATH, '.gitmodules')), + '--get-regexp', + 'path', + ] + ) + .decode('utf8') + .strip() + .split('\n') + ) for line in lines: _, path = line.split(' ') if full_path: @@ -38,7 +57,11 @@ def get_submodule_dirs(full_path: bool = False) -> List: def _check_git_filemode(full_path): # type: (str) -> bool try: - stdout = subprocess.check_output(['git', 'ls-files', '--stage', full_path]).strip().decode('utf-8') + stdout = ( + subprocess.check_output(['git', 'ls-files', '--stage', full_path]) + .strip() + .decode('utf-8') + ) except subprocess.CalledProcessError: return True @@ -74,8 +97,12 @@ def get_git_files(path: str = IDF_PATH, full_path: bool = False) -> List[str]: # folder if no `.git` folder found in `cwd`. workaround_env = os.environ.copy() workaround_env.pop('GIT_DIR', None) - files = subprocess.check_output(['git', 'ls-files'], cwd=path, env=workaround_env) \ - .decode('utf8').strip().split('\n') + files = ( + subprocess.check_output(['git', 'ls-files'], cwd=path, env=workaround_env) + .decode('utf8') + .strip() + .split('\n') + ) except Exception as e: # pylint: disable=W0703 logging.warning(str(e)) files = [] @@ -86,30 +113,61 @@ def is_in_directory(file_path: str, folder: str) -> bool: return os.path.realpath(file_path).startswith(os.path.realpath(folder) + os.sep) -def get_pytest_dirs(folder: str) -> List[str]: +class PytestCase: + def __init__(self, test_path: str, target: str, config: str, case: str): + self.app_path = os.path.dirname(test_path) + self.test_path = test_path + self.target = target + self.config = config + self.case = case + + def __repr__(self) -> str: + return f'{self.test_path}: {self.target}.{self.config}.{self.case}' + + +class PytestCollectPlugin: + def __init__(self, target: str) -> None: + self.target = target + self.nodes: List[PytestCase] = [] + + def pytest_collection_modifyitems(self, items: List['Function']) -> None: + for item in items: + try: + file_path = str(item.path) + except AttributeError: + # pytest 6.x + file_path = item.fspath + + target = self.target + if hasattr(item, 'callspec'): + config = item.callspec.params.get('config', 'default') + else: + config = 'default' + case_name = item.originalname + + self.nodes.append(PytestCase(file_path, target, config, case_name)) + + +def get_pytest_cases(folder: str, target: str) -> List[PytestCase]: import pytest - from _pytest.nodes import Item + from _pytest.config import ExitCode - class CollectPlugin: - def __init__(self) -> None: - self.nodes: List[Item] = [] + collector = PytestCollectPlugin(target) - def pytest_collection_modifyitems(self, items: List[Item]) -> None: - for item in items: - self.nodes.append(item) + with io.StringIO() as buf: + with redirect_stdout(buf): + res = pytest.main(['--collect-only', folder, '-q', '--target', target], plugins=[collector]) + if res.value != ExitCode.OK: + if res.value == ExitCode.NO_TESTS_COLLECTED: + print(f'WARNING: no pytest app found for target {target} under folder {folder}') + else: + print(buf.getvalue()) + raise RuntimeError('pytest collection failed') - collector = CollectPlugin() + return collector.nodes - res = pytest.main(['--collect-only', '-q', folder], plugins=[collector]) - if res.value != 0: - raise RuntimeError('pytest collection failed') - sys.stdout.flush() # print instantly +def get_pytest_app_paths(folder: str, target: str) -> List[str]: + nodes = get_pytest_cases(folder, target) - try: - test_file_paths = set(node.path for node in collector.nodes) - except AttributeError: - # pytest 6.x - test_file_paths = set(node.fspath for node in collector.nodes) - - return [os.path.dirname(file) for file in test_file_paths] + return list({node.app_path for node in nodes}) diff --git a/tools/ci/python_packages/ttfw_idf/CIScanTests.py b/tools/ci/python_packages/ttfw_idf/CIScanTests.py index 08bb1f1d7b..729ac24919 100644 --- a/tools/ci/python_packages/ttfw_idf/CIScanTests.py +++ b/tools/ci/python_packages/ttfw_idf/CIScanTests.py @@ -5,14 +5,11 @@ import logging import os from collections import defaultdict from copy import deepcopy +from typing import Any -try: - from typing import Any -except ImportError: - # Only used for type annotations - pass -from find_apps import find_apps -from find_build_apps import BUILD_SYSTEM_CMAKE, BUILD_SYSTEMS +from ci.idf_ci_utils import get_pytest_app_paths +from find_apps import find_apps, find_builds_for_app +from find_build_apps import BUILD_SYSTEM_CMAKE, BUILD_SYSTEMS, config_rules_from_str from idf_py_actions.constants import PREVIEW_TARGETS, SUPPORTED_TARGETS from ttfw_idf.IDFAssignTest import ExampleAssignTest, TestAppsAssignTest @@ -32,6 +29,8 @@ BUILD_ALL_LABELS = [ 'BOT_LABEL_WEEKEND_TEST', ] +BUILD_PER_JOB = 30 # each build takes 1 mins around + def _has_build_all_label(): # type: () -> bool for label in BUILD_ALL_LABELS: @@ -100,6 +99,23 @@ def main(): # type: () -> None help='add this flag to preserve artifacts for all apps') parser.add_argument('--build-all', action='store_true', help='add this flag to build all apps') + parser.add_argument('--combine-all-targets', action='store_true', + help='add this flag to combine all target jsons into one') + parser.add_argument('--except-targets', nargs='+', + help='only useful when "--combine-all-targets". Specified targets would be skipped.') + parser.add_argument( + '--config', + action='append', + help='Only useful when "--evaluate-parallel-count" is flagged.' + 'Adds configurations (sdkconfig file names) to build. This can either be ' + + 'FILENAME[=NAME] or FILEPATTERN. FILENAME is the name of the sdkconfig file, ' + + 'relative to the project directory, to be used. Optional NAME can be specified, ' + + 'which can be used as a name of this configuration. FILEPATTERN is the name of ' + + 'the sdkconfig file, relative to the project directory, with at most one wildcard. ' + + 'The part captured by the wildcard is used as the name of the configuration.', + ) + parser.add_argument('--evaluate-parallel-count', action='store_true', + help='suggest parallel count according to build items') args = parser.parse_args() build_test_case_apps, build_standalone_apps = _judge_build_or_not(args.test_type, args.build_all) @@ -173,9 +189,19 @@ def main(): # type: () -> None else: scan_info_dict[target]['standalone_apps'] = set() test_case_apps_preserve_default = True if build_system == 'cmake' else False + output_files = [] + build_items_total_count = 0 for target in SUPPORTED_TARGETS: + # get pytest apps paths + pytest_app_paths = set() + for path in paths: + pytest_app_paths.update(get_pytest_app_paths(path, target)) + apps = [] for app_dir in scan_info_dict[target]['test_case_apps']: + if app_dir in pytest_app_paths: + print(f'WARNING: has pytest script: {app_dir}') + continue apps.append({ 'app_dir': app_dir, 'build_system': args.build_system, @@ -183,6 +209,9 @@ def main(): # type: () -> None 'preserve': args.preserve_all or test_case_apps_preserve_default }) for app_dir in scan_info_dict[target]['standalone_apps']: + if app_dir in pytest_app_paths: + print(f'Skipping pytest app: {app_dir}') + continue apps.append({ 'app_dir': app_dir, 'build_system': args.build_system, @@ -191,8 +220,49 @@ def main(): # type: () -> None }) output_path = os.path.join(args.output_path, 'scan_{}_{}.json'.format(target.lower(), build_system)) with open(output_path, 'w') as fw: + if args.evaluate_parallel_count: + build_items = [] + config_rules = config_rules_from_str(args.config or []) + for app in apps: + build_items += find_builds_for_app( + app['app_dir'], + app['app_dir'], + 'build', + '', + app['target'], + app['build_system'], + config_rules, + app['preserve'], + ) + print('Found {} builds'.format(len(build_items))) + if args.combine_all_targets: + if (args.except_targets and target not in [t.lower() for t in args.except_targets]) \ + or (not args.except_targets): + build_items_total_count += len(build_items) + else: + print(f'suggest set parallel count for target {target} to {len(build_items) // BUILD_PER_JOB + 1}') fw.writelines([json.dumps(app) + '\n' for app in apps]) + if args.combine_all_targets: + if (args.except_targets and target not in [t.lower() for t in args.except_targets]) \ + or (not args.except_targets): + output_files.append(output_path) + else: + print(f'skipping combining target {target}') + + if args.combine_all_targets: + scan_all_json = os.path.join(args.output_path, f'scan_all_{build_system}.json') + lines = [] + for file in output_files: + with open(file) as fr: + lines.extend([line for line in fr.readlines() if line.strip()]) + with open(scan_all_json, 'w') as fw: + fw.writelines(lines) + print(f'combined into file: {scan_all_json}') + + if args.evaluate_parallel_count: + print(f'Total build: {build_items_total_count}. Suggest set parallel count for all target to {build_items_total_count // BUILD_PER_JOB + 1}') + if __name__ == '__main__': main() diff --git a/tools/ci/utils.sh b/tools/ci/utils.sh index 42b2a83373..426758a957 100644 --- a/tools/ci/utils.sh +++ b/tools/ci/utils.sh @@ -42,6 +42,13 @@ function get_all_submodules() { git config --file .gitmodules --get-regexp path | awk '{ print $2 }' | sed -e 's|$|/**|' | xargs | sed -e 's/ /,/g' } +function set_component_ut_vars() { + local exclude_list_fp="${IDF_PATH}/tools/ci/component_ut_excludes.txt" + export COMPONENT_UT_DIRS=$(find components/ -name test_apps -type d | xargs) + export COMPONENT_UT_EXCLUDES=$([ -r $exclude_list_fp ] && cat $exclude_list_fp | xargs) + echo "exported variables COMPONENT_UT_DIRS, COMPONENT_UT_EXCLUDES" +} + function error() { printf "\033[0;31m%s\n\033[0m" "${1}" >&2 } diff --git a/tools/find_build_apps/cmake.py b/tools/find_build_apps/cmake.py index 610c1bc4c0..5527dc8648 100644 --- a/tools/find_build_apps/cmake.py +++ b/tools/find_build_apps/cmake.py @@ -63,6 +63,7 @@ class CMakeBuildSystem(BuildSystem): build_stderr = log_file try: + os.environ['IDF_TARGET'] = build_item.target subprocess.check_call(args, stdout=build_stdout, stderr=build_stderr) except subprocess.CalledProcessError as e: raise BuildError('Build failed with exit code {}'.format(e.returncode))