# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 """ This file is used in CI generate binary files for different kinds of apps """ import argparse import os import sys import typing as t import unittest from collections import defaultdict from pathlib import Path import yaml from idf_build_apps import LOGGER, App, build_apps, find_apps, setup_logging from idf_build_apps.constants import SUPPORTED_TARGETS from idf_ci_utils import IDF_PATH, PytestApp, get_pytest_cases, get_ttfw_app_paths CI_ENV_VARS = { 'EXTRA_CFLAGS': '-Werror -Werror=deprecated-declarations -Werror=unused-variable ' '-Werror=unused-but-set-variable -Werror=unused-function -Wstrict-prototypes', 'EXTRA_CXXFLAGS': '-Werror -Werror=deprecated-declarations -Werror=unused-variable ' '-Werror=unused-but-set-variable -Werror=unused-function', 'LDGEN_CHECK_MAPPING': '1', } def get_pytest_apps( paths: t.List[str], target: str, config_rules_str: t.List[str], marker_expr: str, filter_expr: str, preserve_all: bool = False, extra_default_build_targets: t.Optional[t.List[str]] = None, modified_components: t.Optional[t.List[str]] = None, modified_files: t.Optional[t.List[str]] = None, ignore_app_dependencies_filepatterns: t.Optional[t.List[str]] = None, ) -> t.List[App]: pytest_cases = get_pytest_cases(paths, target, marker_expr, filter_expr) _paths: t.Set[str] = set() test_related_app_configs = defaultdict(set) for case in pytest_cases: for app in case.apps: _paths.add(app.path) if os.getenv('INCLUDE_NIGHTLY_RUN') == '1': test_related_app_configs[app.path].add(app.config) elif os.getenv('NIGHTLY_RUN') == '1': if case.nightly_run: test_related_app_configs[app.path].add(app.config) else: if not case.nightly_run: test_related_app_configs[app.path].add(app.config) if not extra_default_build_targets: extra_default_build_targets = [] app_dirs = list(_paths) if not app_dirs: raise RuntimeError('No apps found') LOGGER.info(f'Found {len(app_dirs)} apps') app_dirs.sort() apps = find_apps( app_dirs, target=target, build_dir='build_@t_@w', config_rules_str=config_rules_str, build_log_path='build_log.txt', size_json_path='size.json', check_warnings=True, manifest_rootpath=IDF_PATH, manifest_files=[str(p) for p in Path(IDF_PATH).glob('**/.build-test-rules.yml')], default_build_targets=SUPPORTED_TARGETS + extra_default_build_targets, modified_components=modified_components, modified_files=modified_files, ignore_app_dependencies_filepatterns=ignore_app_dependencies_filepatterns, ) for app in apps: is_test_related = app.config_name in test_related_app_configs[app.app_dir] if not preserve_all and not is_test_related: app.preserve = False if app.target == 'linux': app._size_json_path = None # no esp_idf_size for linux target return apps # type: ignore def get_cmake_apps( paths: t.List[str], target: str, config_rules_str: t.List[str], preserve_all: bool = False, extra_default_build_targets: t.Optional[t.List[str]] = None, modified_components: t.Optional[t.List[str]] = None, modified_files: t.Optional[t.List[str]] = None, ignore_app_dependencies_filepatterns: t.Optional[t.List[str]] = None, ) -> t.List[App]: ttfw_app_dirs = get_ttfw_app_paths(paths, target) apps = find_apps( paths, recursive=True, target=target, build_dir='build_@t_@w', config_rules_str=config_rules_str, build_log_path='build_log.txt', size_json_path='size.json', check_warnings=True, preserve=False, manifest_rootpath=IDF_PATH, manifest_files=[str(p) for p in Path(IDF_PATH).glob('**/.build-test-rules.yml')], default_build_targets=SUPPORTED_TARGETS + extra_default_build_targets, modified_components=modified_components, modified_files=modified_files, ignore_app_dependencies_filepatterns=ignore_app_dependencies_filepatterns, ) apps_for_build = [] pytest_cases_apps = [app for case in get_pytest_cases(paths, target) for app in case.apps] for app in apps: if preserve_all or app.app_dir in ttfw_app_dirs: # relpath app.preserve = True if PytestApp(os.path.realpath(app.app_dir), app.target, app.config_name) in pytest_cases_apps: LOGGER.debug('Skipping build app with pytest scripts: %s', app) continue if app.target == 'linux': app._size_json_path = None # no esp_idf_size for linux target apps_for_build.append(app) return apps_for_build APPS_BUILD_PER_JOB = 30 def main(args: argparse.Namespace) -> None: extra_default_build_targets: t.List[str] = [] if args.default_build_test_rules: with open(args.default_build_test_rules) as fr: configs = yaml.safe_load(fr) if configs: extra_default_build_targets = configs.get('extra_default_build_targets') or [] if args.pytest_apps: LOGGER.info('Only build apps with pytest scripts') apps = get_pytest_apps( args.paths, args.target, args.config, args.marker_expr, args.filter_expr, args.preserve_all, extra_default_build_targets, args.modified_components, args.modified_files, args.ignore_app_dependencies_filepatterns, ) else: LOGGER.info('build apps. will skip pytest apps with pytest scripts') apps = get_cmake_apps( args.paths, args.target, args.config, args.preserve_all, extra_default_build_targets, args.modified_components, args.modified_files, args.ignore_app_dependencies_filepatterns, ) LOGGER.info('Found %d apps after filtering', len(apps)) LOGGER.info( 'Suggest setting the parallel count to %d for this build job', len(apps) // APPS_BUILD_PER_JOB + 1, ) if args.extra_preserve_dirs: for app in apps: if app.preserve: continue for extra_preserve_dir in args.extra_preserve_dirs: abs_extra_preserve_dir = Path(extra_preserve_dir).resolve() abs_app_dir = Path(app.app_dir).resolve() if abs_extra_preserve_dir == abs_app_dir or abs_extra_preserve_dir in abs_app_dir.parents: app.preserve = True res = build_apps( apps, parallel_count=args.parallel_count, parallel_index=args.parallel_index, dry_run=False, build_verbose=args.build_verbose, keep_going=True, collect_size_info='size_info.txt', collect_app_info=args.collect_app_info, ignore_warning_strs=args.ignore_warning_str, ignore_warning_file=args.ignore_warning_file, copy_sdkconfig=args.copy_sdkconfig, modified_components=args.modified_components, modified_files=args.modified_files, ignore_app_dependencies_filepatterns=args.ignore_app_dependencies_filepatterns, ) if isinstance(res, tuple): sys.exit(res[0]) else: sys.exit(res) if __name__ == '__main__': parser = argparse.ArgumentParser( description='Build all the apps for different test types. Will auto remove those non-test apps binaries', formatter_class=argparse.ArgumentDefaultsHelpFormatter, ) parser.add_argument('paths', nargs='+', help='Paths to the apps to build.') parser.add_argument( '-t', '--target', default='all', help='Build apps for given target', ) parser.add_argument( '--config', default=['sdkconfig.ci=default', 'sdkconfig.ci.*=', '=default'], nargs='+', help='Adds configurations (sdkconfig file names) to build. This can either be ' 'FILENAME[=NAME] or FILEPATTERN. FILENAME is the name of the sdkconfig file, ' 'relative to the project directory, to be used. Optional NAME can be specified, ' 'which can be used as a name of this configuration. FILEPATTERN is the name of ' 'the sdkconfig file, relative to the project directory, with at most one wildcard. ' 'The part captured by the wildcard is used as the name of the configuration.', ) parser.add_argument( '-v', '--verbose', action='count', help='Increase the LOGGER level of the script. Can be specified multiple times.', ) parser.add_argument( '--build-verbose', action='store_true', help='Enable verbose output from build system.', ) parser.add_argument( '--preserve-all', action='store_true', help='Preserve the binaries for all apps when specified.', ) parser.add_argument('--parallel-count', default=1, type=int, help='Number of parallel build jobs.') parser.add_argument( '--parallel-index', default=1, type=int, help='Index (1-based) of the job, out of the number specified by --parallel-count.', ) parser.add_argument( '--collect-app-info', default='list_job_@p.txt', help='If specified, the test case name and app info json will be written to this file', ) parser.add_argument( '--ignore-warning-str', nargs='+', help='Ignore the warning string that match the specified regex in the build output. space-separated list', ) parser.add_argument( '--ignore-warning-file', default=os.path.join(IDF_PATH, 'tools', 'ci', 'ignore_build_warnings.txt'), type=argparse.FileType('r'), help='Ignore the warning strings in the specified file. Each line should be a regex string.', ) parser.add_argument( '--copy-sdkconfig', action='store_true', help='Copy the sdkconfig file to the build directory.', ) parser.add_argument( '--extra-preserve-dirs', nargs='+', help='also preserve binaries of the apps under the specified dirs', ) parser.add_argument( '--pytest-apps', action='store_true', help='Only build apps with pytest scripts. Will build apps without pytest scripts if this flag is unspecified.', ) parser.add_argument( '-m', '--marker-expr', default='not host_test', # host_test apps would be built and tested under the same job help='only build tests matching given mark expression. For example: -m "host_test and generic". Works only' 'for pytest', ) parser.add_argument( '-k', '--filter-expr', help='only build tests matching given filter expression. For example: -k "test_hello_world". Works only' 'for pytest', ) parser.add_argument( '--default-build-test-rules', default=os.path.join(IDF_PATH, '.gitlab', 'ci', 'default-build-test-rules.yml'), help='default build test rules config file', ) parser.add_argument( '--skip-setting-flags', action='store_true', help='by default this script would set the build flags exactly the same as the CI ones. ' 'Set this flag to use your local build flags.', ) parser.add_argument( '--modified-components', nargs='*', default=None, help='space-separated list which specifies the modified components. app with `depends_components` set in the ' 'corresponding manifest files would only be built if depends on any of the specified components.', ) parser.add_argument( '--modified-files', nargs='*', default=None, help='space-separated list which specifies the modified files. app with `depends_filepatterns` set in the ' 'corresponding manifest files would only be built if any of the specified file pattern matches any of the ' 'specified modified files.', ) parser.add_argument( '-if', '--ignore-app-dependencies-filepatterns', nargs='*', default=None, help='space-separated list which specifies the file patterns used for ignoring checking the app dependencies. ' 'The `depends_components` and `depends_filepatterns` set in the manifest files will be ignored when any of the ' 'specified file patterns matches any of the modified files. Must be used together with --modified-files', ) arguments = parser.parse_args() setup_logging(arguments.verbose) # skip setting flags in CI if not arguments.skip_setting_flags and not os.getenv('CI_JOB_ID'): for _k, _v in CI_ENV_VARS.items(): os.environ[_k] = _v LOGGER.info(f'env var {_k} set to "{_v}"') if os.getenv('IS_MR_PIPELINE') == '0' or os.getenv('BUILD_AND_TEST_ALL_APPS') == '1': # if it's not MR pipeline or env var BUILD_AND_TEST_ALL_APPS=1, # remove component dependency related arguments if 'modified_components' in arguments: arguments.modified_components = None if 'modified_files' in arguments: arguments.modified_files = None # file patterns to tigger full build if 'modified_components' in arguments and not arguments.ignore_app_dependencies_filepatterns: arguments.ignore_app_dependencies_filepatterns = [ # tools 'tools/cmake/**/*', 'tools/tools.json', # components 'components/cxx/**/*', 'components/esp_common/**/*', 'components/esp_hw_support/**/*', 'components/esp_rom/**/*', 'components/esp_system/**/*', 'components/esp_timer/**/*', 'components/freertos/**/*', 'components/hal/**/*', 'components/heap/**/*', 'components/log/**/*', 'components/newlib/**/*', 'components/riscv/**/*', 'components/soc/**/*', 'components/xtensa/**/*', ] main(arguments) class TestParsingShellScript(unittest.TestCase): """ This test case is run in CI jobs to make sure the CI build flags is the same as the ones recorded in CI_ENV_VARS """ def test_parse_result(self) -> None: for k, v in CI_ENV_VARS.items(): self.assertEqual(os.getenv(k), v)