esp-idf/tools/ci/build_pytest_apps.py
2022-05-21 00:11:59 +08:00

160 lines
5.5 KiB
Python

# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD
# SPDX-License-Identifier: Apache-2.0
"""
This file is used to generate binary files for the given path.
"""
import argparse
import copy
import logging
import os
import sys
from collections import defaultdict
from typing import List
from idf_ci_utils import IDF_PATH, PytestCase, get_pytest_cases
try:
from build_apps import build_apps
from find_apps import find_builds_for_app
from find_build_apps import BuildItem, CMakeBuildSystem, config_rules_from_str, setup_logging
except ImportError:
sys.path.append(os.path.join(IDF_PATH, 'tools'))
from build_apps import build_apps
from find_apps import find_builds_for_app
from find_build_apps import BuildItem, CMakeBuildSystem, config_rules_from_str, setup_logging
def main(args: argparse.Namespace) -> None:
pytest_cases: List[PytestCase] = []
for path in args.paths:
pytest_cases += get_pytest_cases(path, args.target, args.marker_expr)
paths = set()
app_configs = defaultdict(set)
for case in pytest_cases:
for app in case.apps:
paths.add(app.path)
app_configs[app.path].add(app.config)
app_dirs = list(paths)
if not app_dirs:
raise RuntimeError('No apps found')
logging.info(f'Found {len(app_dirs)} apps')
app_dirs.sort()
# Find compatible configurations of each app, collect them as BuildItems
build_items: List[BuildItem] = []
config_rules = config_rules_from_str(args.config or [])
for app_dir in app_dirs:
app_dir = os.path.realpath(app_dir)
if args.target in CMakeBuildSystem.supported_targets(app_dir):
build_items += find_builds_for_app(
app_path=app_dir,
work_dir=app_dir,
build_dir='build_@t_@w',
build_log=f'{app_dir}/build_@t_@w/build.log',
target_arg=args.target,
build_system='cmake',
config_rules=config_rules,
)
modified_build_items = []
# auto clean up the binaries if no flag --preserve-all
for item in build_items:
is_test_related = item.config_name in app_configs[item.app_dir]
if args.test_only and not is_test_related:
logging.info(f'Skipping non-test app: {item}')
continue
copied_item = copy.deepcopy(item)
if not args.preserve_all and not is_test_related:
copied_item.preserve = False
modified_build_items.append(copied_item)
logging.info(f'Found {len(modified_build_items)} builds')
modified_build_items.sort(key=lambda x: x.build_path) # type: ignore
build_apps(
build_items=modified_build_items,
parallel_count=args.parallel_count,
parallel_index=args.parallel_index,
dry_run=False,
build_verbose=args.build_verbose,
keep_going=True,
output_build_list=None,
size_info=args.size_info,
)
if __name__ == '__main__':
parser = argparse.ArgumentParser(
description='Build all the pytest apps under specified paths. Will auto remove those non-test apps binaries'
)
parser.add_argument(
'-t', '--target', required=True, help='Build apps for given target.'
)
parser.add_argument(
'-m',
'--marker-expr',
default='not host_test', # host_test apps would be built and tested under the same job
help='only build tests matching given mark expression. For example: -m "host_test and generic".',
)
parser.add_argument(
'--config',
default=['sdkconfig.ci=default', 'sdkconfig.ci.*=', '=default'],
action='append',
help='Adds configurations (sdkconfig file names) to build. This can either be '
+ 'FILENAME[=NAME] or FILEPATTERN. FILENAME is the name of the sdkconfig file, '
+ 'relative to the project directory, to be used. Optional NAME can be specified, '
+ 'which can be used as a name of this configuration. FILEPATTERN is the name of '
+ 'the sdkconfig file, relative to the project directory, with at most one wildcard. '
+ 'The part captured by the wildcard is used as the name of the configuration.',
)
parser.add_argument(
'paths',
nargs='+',
help='One or more app paths. Will use the current path if not specified.',
)
parser.add_argument(
'--parallel-count', default=1, type=int, help='Number of parallel build jobs.'
)
parser.add_argument(
'--parallel-index',
default=1,
type=int,
help='Index (1-based) of the job, out of the number specified by --parallel-count.',
)
parser.add_argument(
'--size-info',
type=argparse.FileType('a'),
help='If specified, the test case name and size info json will be written to this file',
)
parser.add_argument(
'-v',
'--verbose',
action='count',
help='Increase the logging level of the script. Can be specified multiple times.',
)
parser.add_argument(
'--build-verbose',
action='store_true',
help='Enable verbose output from build system.',
)
parser.add_argument(
'--preserve-all',
action='store_true',
help='Preserve the binaries for all apps when specified.',
)
parser.add_argument(
'--test-only',
action='store_true',
help='Build only test related app when specified.',
)
arguments = parser.parse_args()
setup_logging(arguments)
main(arguments)