esp-idf/tools/build_apps.py
Ivan Grokhotkov 9d15c1fd40 tools: add {find,build}_apps.py, scripts to build multiple apps
This commit adds a pair of scripts, find_apps.py and build_apps.py.
These scripts are intended to be used in various CI jobs, building
multiple applications with different configurations and targets.

The first script, find_apps.py, is used to prepare the list of builds:
1. It finds apps for the given build system.
2. For each app, it finds configurations (sdkconfig files) which need
   to be built.
3. It filters out the apps and configurations which are not compatible
   with the given target.
4. It outputs the list of builds into stdout or a file. Currently the
   format is a list of lines, each line a JSON string. In the future,
   the tool can be updated to output YAML files.

The lists of builds can be concatenated and processed with standard
command line tools, like sed.

The second script, build_apps.py, executes the builds from the list.
It can execute a subset of builds based on --parallel-count and
--parallel-index arguments.

These two scripts are intended to replace build_examples_make,
build_examples_cmake, and the custom unit-test-app logic (in the
Makefile and idf_ext.py).

Closes IDF-641
2019-11-21 15:36:06 +01:00

130 lines
4.3 KiB
Python
Executable File

#!/usr/bin/env python
# coding=utf-8
#
# ESP-IDF helper script to build multiple applications. Consumes the input of find_apps.py.
#
import argparse
import sys
import logging
from find_build_apps import BuildItem, BuildError, setup_logging, BUILD_SYSTEMS
def main():
parser = argparse.ArgumentParser(description="ESP-IDF app builder")
parser.add_argument(
"-v",
"--verbose",
action="count",
help="Increase the logging level of the script. Can be specified multiple times.",
)
parser.add_argument(
"--build-verbose",
action="store_true",
help="Enable verbose output from build system.",
)
parser.add_argument(
"--log-file",
type=argparse.FileType("w"),
help="Write the script log to the specified file, instead of stderr",
)
parser.add_argument(
"--parallel-count",
default=1,
type=int,
help="Number of parallel build jobs. Note that this script doesn't start the jobs, " +
"it needs to be executed multiple times with same value of --parallel-count and " +
"different values of --parallel-index.",
)
parser.add_argument(
"--parallel-index",
default=1,
type=int,
help="Index (1-based) of the job, out of the number specified by --parallel-count.",
)
parser.add_argument(
"--format",
default="json",
choices=["json"],
help="Format to read the list of builds",
)
parser.add_argument(
"--dry-run",
action="store_true",
help="Don't actually build, only print the build commands",
)
parser.add_argument(
"--keep-going",
action="store_true",
help="Don't exit immediately when a build fails.",
)
parser.add_argument(
"--output-build-list",
type=argparse.FileType("w"),
help="If specified, the list of builds (with all the placeholders expanded) will be written to this file.",
)
parser.add_argument(
"build_list",
type=argparse.FileType("r"),
nargs="?",
default=sys.stdin,
help="Name of the file to read the list of builds from. If not specified, read from stdin.",
)
args = parser.parse_args()
setup_logging(args)
build_items = [BuildItem.from_json(line) for line in args.build_list]
if not build_items:
logging.error("Empty build list!")
raise SystemExit(1)
num_builds = len(build_items)
num_jobs = args.parallel_count
job_index = args.parallel_index - 1 # convert to 0-based index
num_builds_per_job = (num_builds + num_jobs - 1) // num_jobs
min_job_index = num_builds_per_job * job_index
if min_job_index >= num_builds:
logging.warn("Nothing to do for job {} (build total: {}, per job: {})".format(
job_index + 1, num_builds, num_builds_per_job))
raise SystemExit(0)
max_job_index = min(num_builds_per_job * (job_index + 1) - 1, num_builds - 1)
logging.info("Total {} builds, max. {} builds per job, running builds {}-{}".format(
num_builds, num_builds_per_job, min_job_index + 1, max_job_index + 1))
builds_for_current_job = build_items[min_job_index:max_job_index + 1]
for i, build_info in enumerate(builds_for_current_job):
index = i + min_job_index + 1
build_info.index = index
build_info.dry_run = args.dry_run
build_info.verbose = args.build_verbose
build_info.keep_going = args.keep_going
logging.debug(" Build {}: {}".format(index, repr(build_info)))
if args.output_build_list:
args.output_build_list.write(build_info.to_json_expanded() + "\n")
failed_builds = []
for build_info in builds_for_current_job:
logging.info("Running build {}: {}".format(build_info.index, repr(build_info)))
build_system_class = BUILD_SYSTEMS[build_info.build_system]
try:
build_system_class.build(build_info)
except BuildError as e:
logging.error(e.message)
if args.keep_going:
failed_builds.append(build_info)
else:
raise SystemExit(1)
if failed_builds:
logging.error("The following build have failed:")
for build in failed_builds:
logging.error(" {}".format(build))
raise SystemExit(1)
if __name__ == "__main__":
main()