alex/esp-idf
alex/esp-idf
1
0
Fork 0
mirror of https://github.com/espressif/esp-idf.git synced 2024-10-05 20:47:46 -04:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

ci: build and test only modified components related test cases

Browse Source
This commit is contained in:
Fu Hanxi 2023-05-24 10:53:57 +08:00
parent 2cbcafc4d3
commit 22e2738f78
12 changed files with 274 additions and 100 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@ -97,3 +97,5 @@ managed_components
# pytest log
pytest_embedded_log/
list_job_*.txt
size_info.txt

2
.gitlab-ci.yml
View File

@ -20,9 +20,11 @@ workflow:
- if: $CI_OPEN_MERGE_REQUESTS != null
variables:
PIPELINE_COMMIT_SHA: $CI_MERGE_REQUEST_SOURCE_BRANCH_SHA
IS_MR_PIPELINE: 1
- if: $CI_OPEN_MERGE_REQUESTS == null
variables:
PIPELINE_COMMIT_SHA: $CI_COMMIT_SHA
IS_MR_PIPELINE: 0
- when: always
variables:

13
.gitlab/ci/README.md
View File

@ -21,11 +21,7 @@
- [Shell Script Related](#shell-script-related)
- [Manifest File to Control the Build/Test apps](#manifest-file-to-control-the-buildtest-apps)
- [Grammar](#grammar)
- [Operands](#operands)
- [Operators](#operators)
- [Limitation:](#limitation)
- [How does it work?](#how-does-it-work)
- [Example](#example)
- [Special Rules](#special-rules)
## General Workflow
@ -242,3 +238,10 @@ We're using the latest version of [idf-build-apps][idf-build-apps]. Please refer
[idf-build-apps]: https://github.com/espressif/idf-build-apps
[manifest-doc]: https://docs.espressif.com/projects/idf-build-apps/en/latest/manifest.html
### Special Rules
In ESP-IDF CI, there's a few more special rules are additionally supported to disable the check app dependencies feature:
- Add MR labels `BUILD_AND_TEST_ALL_APPS`
- Run in protected branches

55
.gitlab/ci/build.yml
View File

@ -21,6 +21,8 @@
needs:
- job: fast_template_app
artifacts: false
- job: mr_variables
optional: true # only MR pipelines would have this
artifacts:
paths:
- "**/build*/size.json"
@ -37,7 +39,7 @@
- "**/build*/sdkconfig"
- "**/build*/bootloader/*.bin"
- "**/build*/partition_table/*.bin"
- list_job_*.json
- list_job_*.txt
- size_info.txt
# unit test specific
- components/idf_test/unit_test/*.yml
@ -66,18 +68,18 @@
# would be clean up after 4 days
- mc share download shiny-s3/idf-artifacts/${CI_PIPELINE_ID}/${CI_JOB_ID}.zip --expire=96h
script:
# CI specific options start from "--collect-size-info xxx". could ignore when running locally
# CI specific options start from "--parallel-count xxx". could ignore when running locally
- run_cmd python tools/ci/ci_build_apps.py $TEST_DIR -v
-t $IDF_TARGET
--copy-sdkconfig
--collect-size-info size_info.txt
--collect-app-info list_job_${CI_NODE_INDEX:-1}.json
--parallel-count ${CI_NODE_TOTAL:-1}
--parallel-index ${CI_NODE_INDEX:-1}
--extra-preserve-dirs
examples/bluetooth/esp_ble_mesh/ble_mesh_console
examples/bluetooth/hci/controller_hci_uart_esp32
examples/wifi/iperf
--modified-components ${MR_MODIFIED_COMPONENTS}
--modified-files ${MR_MODIFIED_FILES}
.build_cmake_clang_template:
extends:
@ -87,14 +89,14 @@
TEST_BUILD_OPTS_EXTRA: ""
TEST_DIR: tools/test_apps/system/cxx_pthread_bluetooth
script:
# CI specific options start from "--collect-size-info xxx". could ignore when running locally
# CI specific options start from "--parallel-count xxx". could ignore when running locally
- run_cmd python tools/ci/ci_build_apps.py $TEST_DIR -v
-t $IDF_TARGET
--copy-sdkconfig
--collect-size-info size_info.txt
--collect-app-info list_job_${CI_NODE_INDEX:-1}.json
--parallel-count ${CI_NODE_TOTAL:-1}
--parallel-index ${CI_NODE_INDEX:-1}
--modified-components ${MR_MODIFIED_COMPONENTS}
--modified-files ${MR_MODIFIED_FILES}
$TEST_BUILD_OPTS_EXTRA
.build_pytest_template:
@ -114,30 +116,32 @@
- "**/build*/config/sdkconfig.json"
- "**/build*/bootloader/*.bin"
- "**/build*/partition_table/*.bin"
- list_job_*.json
- list_job_*.txt
- size_info.txt
when: always
expire_in: 4 days
script:
# CI specific options start from "--collect-size-info xxx". could ignore when running locally
# CI specific options start from "--parallel-count xxx". could ignore when running locally
- run_cmd python tools/ci/ci_build_apps.py $TEST_DIR -v
-t $IDF_TARGET
--pytest-apps
--collect-size-info size_info.txt
--parallel-count ${CI_NODE_TOTAL:-1}
--parallel-index ${CI_NODE_INDEX:-1}
--modified-components ${MR_MODIFIED_COMPONENTS}
--modified-files ${MR_MODIFIED_FILES}
.build_pytest_no_jtag_template:
extends: .build_pytest_template
script:
# CI specific options start from "--collect-size-info xxx". could ignore when running locally
# CI specific options start from "--parallel-count xxx". could ignore when running locally
- run_cmd python tools/ci/ci_build_apps.py $TEST_DIR -v
-t $IDF_TARGET
-m \"not host_test and not jtag\"
--pytest-apps
--collect-size-info size_info.txt
--parallel-count ${CI_NODE_TOTAL:-1}
--parallel-index ${CI_NODE_INDEX:-1}
--modified-components ${MR_MODIFIED_COMPONENTS}
--modified-files ${MR_MODIFIED_FILES}
.build_pytest_jtag_template:
extends:
@ -156,19 +160,20 @@
- "**/build*/config/sdkconfig.json"
- "**/build*/bootloader/*.bin"
- "**/build*/partition_table/*.bin"
- list_job_*.json
- list_job_*.txt
- size_info.txt
when: always
expire_in: 4 days
script:
# CI specific options start from "--collect-size-info xxx". could ignore when running locally
# CI specific options start from "--parallel-count xxx". could ignore when running locally
- run_cmd python tools/ci/ci_build_apps.py $TEST_DIR -v
-t $IDF_TARGET
-m \"not host_test and jtag\"
--pytest-apps
--collect-size-info size_info.txt
--parallel-count ${CI_NODE_TOTAL:-1}
--parallel-index ${CI_NODE_INDEX:-1}
--modified-components ${MR_MODIFIED_COMPONENTS}
--modified-files ${MR_MODIFIED_FILES}
build_pytest_examples_esp32:
extends:
@ -310,13 +315,13 @@ build_only_components_apps:
parallel: 5
script:
- set_component_ut_vars
# CI specific options start from "--collect-size-info xxx". could ignore when running locally
# CI specific options start from "--parallel-count xxx". could ignore when running locally
- run_cmd python tools/ci/ci_build_apps.py $COMPONENT_UT_DIRS -v
-t all
--collect-size-info size_info.txt
--collect-app-info list_job_${CI_NODE_INDEX:-1}.json
--parallel-count ${CI_NODE_TOTAL:-1}
--parallel-index ${CI_NODE_INDEX:-1}
--modified-components ${MR_MODIFIED_COMPONENTS}
--modified-files ${MR_MODIFIED_FILES}
.build_pytest_test_apps_template:
extends: .build_pytest_template
@ -336,7 +341,7 @@ build_only_components_apps:
- "**/build*/bootloader/*.bin"
- "**/build*/partition_table/*.bin"
- "**/build*/project_description.json"
- list_job_*.json
- list_job_*.txt
- size_info.txt
when: always
expire_in: 4 days
@ -404,13 +409,13 @@ build_only_tools_test_apps:
- .rules:build:custom_test
parallel: 9
script:
# CI specific options start from "--collect-size-info xxx". could ignore when running locally
# CI specific options start from "--parallel-count xxx". could ignore when running locally
- run_cmd python tools/ci/ci_build_apps.py tools/test_apps -v
-t all
--collect-size-info size_info.txt
--collect-app-info list_job_${CI_NODE_INDEX:-1}.json
--parallel-count ${CI_NODE_TOTAL:-1}
--parallel-index ${CI_NODE_INDEX:-1}
--modified-components ${MR_MODIFIED_COMPONENTS}
--modified-files ${MR_MODIFIED_FILES}
.build_template_app_template:
extends:
@ -529,20 +534,18 @@ build_ssc_esp32h2:
- "**/build*/sdkconfig"
- "**/build*/bootloader/*.bin"
- "**/build*/partition_table/*.bin"
- list_job_*.json
- list_job_*.txt
- size_info.txt
- components/idf_test/unit_test/*.yml
when: always
expire_in: 4 days
script:
# CI specific options start from "--collect-size-info xxx". could ignore when running locally
# CI specific options start from "--parallel-count xxx". could ignore when running locally
- run_cmd python tools/ci/ci_build_apps.py tools/unit-test-app -v
-t $IDF_TARGET
--config "configs/*="
--copy-sdkconfig
--preserve-all
--collect-size-info size_info.txt
--collect-app-info list_job_${CI_NODE_INDEX:-1}.json
--parallel-count ${CI_NODE_TOTAL:-1}
--parallel-index ${CI_NODE_INDEX:-1}
- run_cmd python tools/unit-test-app/tools/UnitTestParser.py tools/unit-test-app ${CI_NODE_INDEX:-1}

19
.gitlab/ci/pre_check.yml
View File

@ -186,3 +186,22 @@ check_configure_ci_environment_parsing:
script:
- cd tools/ci
- python -m unittest ci_build_apps.py
mr_variables:
extends:
- .pre_check_template
- .rules:mr
- .before_script_minimal
tags:
- build
script:
- echo "MR_MODIFIED_FILES=$(python tools/ci/ci_get_mr_info.py files ${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME} | xargs)" >> mr.env
- echo "MR_MODIFIED_COMPONENTS=$(python tools/ci/ci_get_mr_info.py components ${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME} | xargs)" >> mr.env
- >
if echo "$CI_MERGE_REQUEST_LABELS" | egrep "^([^,\n\r]+,)*BUILD_AND_TEST_ALL_APPS(,[^,\n\r]+)*$"; then
echo "BUILD_AND_TEST_ALL_APPS=1" >> mr.env
fi
artifacts:
reports:
dotenv: mr.env
expire_in: 4 days

44
.gitlab/ci/rules.yml
View File

@ -374,6 +374,7 @@
#########
# Rules #
#########
### Branches ###
.rules:protected:
rules:
- <<: *if-protected
@ -382,6 +383,30 @@
rules:
- <<: *if-protected-no_label
.rules:dev:
rules:
- <<: *if-trigger
- <<: *if-dev-push
.rules:mr:
rules:
- <<: *if-dev-push
.rules:tag:release:
rules:
- <<: *if-tag-release
.rules:ref:master-schedule:
rules:
- <<: *if-ref-master
- <<: *if-schedule
.rules:ref:master-always:
rules:
- <<: *if-ref-master
when: always
### Patterns ###
.rules:patterns:python-cache:
rules:
- *if-schedule
@ -404,25 +429,6 @@
- <<: *if-dev-push
changes: *patterns-danger-npm
.rules:dev:
rules:
- <<: *if-trigger
- <<: *if-dev-push
.rules:tag:release:
rules:
- <<: *if-tag-release
.rules:ref:master-schedule:
rules:
- <<: *if-ref-master
- <<: *if-schedule
.rules:ref:master-always:
rules:
- <<: *if-ref-master
when: always
.rules:patterns:clang_tidy:
rules:
- <<: *if-protected

1
.gitlab/ci/target-test.yml
View File

@ -39,6 +39,7 @@
--parallel-count ${CI_NODE_TOTAL:-1}
--parallel-index ${CI_NODE_INDEX:-1}
${PYTEST_EXTRA_FLAGS}
--app-info-filepattern \"list_job_*.txt\"
.pytest_examples_dir_template:
extends: .pytest_template

2
.pre-commit-config.yaml
View File

@ -143,7 +143,7 @@ repos:
require_serial: true
additional_dependencies:
- PyYAML == 5.3.1
- idf_build_apps
- idf_build_apps~=1.0
- id: sort-build-test-rules-ymls
name: sort .build-test-rules.yml files
entry: tools/ci/check_build_test_rules.py sort-yaml

89
conftest.py
View File

@ -13,6 +13,8 @@
# This is an experimental feature, and if you found any bug or have any question, please report to
# https://github.com/espressif/pytest-embedded/issues
import glob
import json
import logging
import os
import re
@ -36,11 +38,11 @@ from pytest_embedded.utils import find_by_suffix
from pytest_embedded_idf.dut import IdfDut
try:
from idf_ci_utils import to_list
from idf_ci_utils import IDF_PATH, to_list
from idf_unity_tester import CaseTester
except ImportError:
sys.path.append(os.path.join(os.path.dirname(__file__), 'tools', 'ci'))
from idf_ci_utils import to_list
from idf_ci_utils import IDF_PATH, to_list
from idf_unity_tester import CaseTester
try:
@ -252,7 +254,7 @@ def test_case_name(request: FixtureRequest, target: str, config: str) -> str:
@pytest.fixture
@multi_dut_fixture
def build_dir(app_path: str, target: Optional[str], config: Optional[str]) -> str:
def build_dir(request: FixtureRequest, app_path: str, target: Optional[str], config: Optional[str]) -> str:
"""
Check local build dir with the following priority:
@ -261,11 +263,6 @@ def build_dir(app_path: str, target: Optional[str], config: Optional[str]) -> st
3. build_<config>
4. build
Args:
app_path: app path
target: target
config: config
Returns:
valid build directory
"""
@ -278,6 +275,25 @@ def build_dir(app_path: str, target: Optional[str], config: Optional[str]) -> st
check_dirs.append(f'build_{config}')
check_dirs.append('build')
idf_pytest_embedded = request.config.stash[_idf_pytest_embedded_key]
build_dir = None
if idf_pytest_embedded.apps_list is not None:
for check_dir in check_dirs:
binary_path = os.path.join(app_path, check_dir)
if binary_path in idf_pytest_embedded.apps_list:
build_dir = check_dir
break
if build_dir is None:
pytest.skip(
f'app path {app_path} with target {target} and config {config} is not listed in app info list files'
)
return '' # not reachable, to fool mypy
if build_dir:
check_dirs = [build_dir]
for check_dir in check_dirs:
binary_path = os.path.join(app_path, check_dir)
if os.path.isdir(binary_path):
@ -286,9 +302,8 @@ def build_dir(app_path: str, target: Optional[str], config: Optional[str]) -> st
logging.warning('checking binary path: %s... missing... try another place', binary_path)
recommend_place = check_dirs[0]
raise ValueError(
f'no build dir valid. Please build the binary via "idf.py -B {recommend_place} build" and run pytest again'
f'no build dir valid. Please build the binary via "idf.py -B {check_dirs[0]} build" and run pytest again'
)
@ -412,20 +427,32 @@ def dev_user(request: FixtureRequest) -> str:
# Hook functions #
##################
def pytest_addoption(parser: pytest.Parser) -> None:
base_group = parser.getgroup('idf')
base_group.addoption(
idf_group = parser.getgroup('idf')
idf_group.addoption(
'--sdkconfig',
help='sdkconfig postfix, like sdkconfig.ci.<config>. (Default: None, which would build all found apps)',
)
base_group.addoption('--known-failure-cases-file', help='known failure cases file path')
base_group.addoption(
idf_group.addoption('--known-failure-cases-file', help='known failure cases file path')
idf_group.addoption(
'--dev-user',
help='user name associated with some specific device/service used during the test execution',
)
base_group.addoption(
idf_group.addoption(
'--dev-passwd',
help='password associated with some specific device/service used during the test execution',
)
idf_group.addoption(
'--app-info-basedir',
default=IDF_PATH,
help='app info base directory. specify this value when you\'re building under a '
'different IDF_PATH. (Default: $IDF_PATH)',
)
idf_group.addoption(
'--app-info-filepattern',
help='glob pattern to specify the files that include built app info generated by '
'`idf-build-apps --collect-app-info ...`. will not raise ValueError when binary '
'paths not exist in local file system if not listed recorded in the app info.',
)
_idf_pytest_embedded_key = pytest.StashKey['IdfPytestEmbedded']()
@ -446,10 +473,34 @@ def pytest_configure(config: Config) -> None:
if not target: # also could specify through markexpr via "-m"
target = get_target_marker_from_expr(config.getoption('markexpr') or '')
apps_list = None
app_info_basedir = config.getoption('app_info_basedir')
app_info_filepattern = config.getoption('app_info_filepattern')
if app_info_filepattern:
apps_list = []
for file in glob.glob(os.path.join(IDF_PATH, app_info_filepattern)):
with open(file) as fr:
for line in fr.readlines():
if not line.strip():
continue
# each line is a valid json
app_info = json.loads(line.strip())
if app_info_basedir and app_info['app_dir'].startswith(app_info_basedir):
relative_app_dir = os.path.relpath(app_info['app_dir'], app_info_basedir)
apps_list.append(os.path.join(IDF_PATH, os.path.join(relative_app_dir, app_info['build_dir'])))
print('Detected app: ', apps_list[-1])
else:
print(
f'WARNING: app_info base dir {app_info_basedir} not recognizable in {app_info["app_dir"]}, skipping...'
)
continue
config.stash[_idf_pytest_embedded_key] = IdfPytestEmbedded(
target=target,
sdkconfig=config.getoption('sdkconfig'),
known_failure_cases_file=config.getoption('known_failure_cases_file'),
apps_list=apps_list,
)
config.pluginmanager.register(config.stash[_idf_pytest_embedded_key])
@ -470,11 +521,13 @@ class IdfPytestEmbedded:
target: str,
sdkconfig: Optional[str] = None,
known_failure_cases_file: Optional[str] = None,
apps_list: Optional[List[str]] = None,
):
# CLI options to filter the test cases
self.target = target.lower()
self.sdkconfig = sdkconfig
self.known_failure_patterns = self._parse_known_failure_cases_file(known_failure_cases_file)
self.apps_list = apps_list
self._failed_cases: List[Tuple[str, bool, bool]] = [] # (test_case_name, is_known_failure_cases, is_xfail)
@ -599,7 +652,11 @@ class IdfPytestEmbedded:
test_case_name = item.funcargs.get('test_case_name', '')
if test_case_name:
self._failed_cases.append(
(test_case_name, self._is_known_failure(test_case_name), report.keywords.get('xfail', False))
(
test_case_name,
self._is_known_failure(test_case_name),
report.keywords.get('xfail', False),
)
)
return report

2
pytest.ini
View File

@ -5,7 +5,7 @@ python_files = pytest_*.py
# ignore PytestExperimentalApiWarning for record_xml_attribute
# set traceback to "short" to prevent the overwhelming tracebacks
addopts =
-s
-s -vv
--embedded-services esp,idf
--tb short
--strict-markers

143
tools/ci/ci_build_apps.py
View File

@ -8,10 +8,10 @@ This file is used in CI generate binary files for different kinds of apps
import argparse
import os
import sys
import typing as t
import unittest
from collections import defaultdict
from pathlib import Path
from typing import List, Optional, Set
import yaml
from idf_build_apps import LOGGER, App, build_apps, find_apps, setup_logging
@ -20,25 +20,28 @@ from idf_ci_utils import IDF_PATH, PytestApp, get_pytest_cases, get_ttfw_app_pat
CI_ENV_VARS = {
'EXTRA_CFLAGS': '-Werror -Werror=deprecated-declarations -Werror=unused-variable '
'-Werror=unused-but-set-variable -Werror=unused-function -Wstrict-prototypes',
'-Werror=unused-but-set-variable -Werror=unused-function -Wstrict-prototypes',
'EXTRA_CXXFLAGS': '-Werror -Werror=deprecated-declarations -Werror=unused-variable '
'-Werror=unused-but-set-variable -Werror=unused-function',
'-Werror=unused-but-set-variable -Werror=unused-function',
'LDGEN_CHECK_MAPPING': '1',
}
def get_pytest_apps(
paths: List[str],
paths: t.List[str],
target: str,
config_rules_str: List[str],
config_rules_str: t.List[str],
marker_expr: str,
filter_expr: str,
preserve_all: bool = False,
extra_default_build_targets: Optional[List[str]] = None,
) -> List[App]:
extra_default_build_targets: t.Optional[t.List[str]] = None,
modified_components: t.Optional[t.List[str]] = None,
modified_files: t.Optional[t.List[str]] = None,
ignore_app_dependencies_filepatterns: t.Optional[t.List[str]] = None,
) -> t.List[App]:
pytest_cases = get_pytest_cases(paths, target, marker_expr, filter_expr)
_paths: Set[str] = set()
_paths: t.Set[str] = set()
test_related_app_configs = defaultdict(set)
for case in pytest_cases:
for app in case.apps:
@ -53,6 +56,9 @@ def get_pytest_apps(
if not case.nightly_run:
test_related_app_configs[app.path].add(app.config)
if not extra_default_build_targets:
extra_default_build_targets = []
app_dirs = list(_paths)
if not app_dirs:
raise RuntimeError('No apps found')
@ -68,9 +74,12 @@ def get_pytest_apps(
build_log_path='build_log.txt',
size_json_path='size.json',
check_warnings=True,
manifest_rootpath=IDF_PATH,
manifest_files=[str(p) for p in Path(IDF_PATH).glob('**/.build-test-rules.yml')],
default_build_targets=SUPPORTED_TARGETS + extra_default_build_targets,
manifest_rootpath=IDF_PATH,
modified_components=modified_components,
modified_files=modified_files,
ignore_app_dependencies_filepatterns=ignore_app_dependencies_filepatterns,
)
for app in apps:
@ -85,12 +94,15 @@ def get_pytest_apps(
def get_cmake_apps(
paths: List[str],
paths: t.List[str],
target: str,
config_rules_str: List[str],
config_rules_str: t.List[str],
preserve_all: bool = False,
extra_default_build_targets: Optional[List[str]] = None,
) -> List[App]:
extra_default_build_targets: t.Optional[t.List[str]] = None,
modified_components: t.Optional[t.List[str]] = None,
modified_files: t.Optional[t.List[str]] = None,
ignore_app_dependencies_filepatterns: t.Optional[t.List[str]] = None,
) -> t.List[App]:
ttfw_app_dirs = get_ttfw_app_paths(paths, target)
apps = find_apps(
@ -103,9 +115,12 @@ def get_cmake_apps(
size_json_path='size.json',
check_warnings=True,
preserve=False,
manifest_rootpath=IDF_PATH,
manifest_files=[str(p) for p in Path(IDF_PATH).glob('**/.build-test-rules.yml')],
default_build_targets=SUPPORTED_TARGETS + extra_default_build_targets,
manifest_rootpath=IDF_PATH,
modified_components=modified_components,
modified_files=modified_files,
ignore_app_dependencies_filepatterns=ignore_app_dependencies_filepatterns,
)
apps_for_build = []
@ -130,7 +145,7 @@ APPS_BUILD_PER_JOB = 30
def main(args: argparse.Namespace) -> None:
extra_default_build_targets: List[str] = []
extra_default_build_targets: t.List[str] = []
if args.default_build_test_rules:
with open(args.default_build_test_rules) as fr:
configs = yaml.safe_load(fr)
@ -148,6 +163,9 @@ def main(args: argparse.Namespace) -> None:
args.filter_expr,
args.preserve_all,
extra_default_build_targets,
args.modified_components,
args.modified_files,
args.ignore_app_dependencies_filepatterns,
)
else:
LOGGER.info('build apps. will skip pytest apps with pytest scripts')
@ -157,6 +175,9 @@ def main(args: argparse.Namespace) -> None:
args.config,
args.preserve_all,
extra_default_build_targets,
args.modified_components,
args.modified_files,
args.ignore_app_dependencies_filepatterns,
)
LOGGER.info('Found %d apps after filtering', len(apps))
@ -175,22 +196,28 @@ def main(args: argparse.Namespace) -> None:
if abs_extra_preserve_dir == abs_app_dir or abs_extra_preserve_dir in abs_app_dir.parents:
app.preserve = True
sys.exit(
build_apps(
apps,
parallel_count=args.parallel_count,
parallel_index=args.parallel_index,
dry_run=False,
build_verbose=args.build_verbose,
keep_going=True,
collect_size_info=args.collect_size_info,
collect_app_info=args.collect_app_info,
ignore_warning_strs=args.ignore_warning_str,
ignore_warning_file=args.ignore_warning_file,
copy_sdkconfig=args.copy_sdkconfig,
)
res = build_apps(
apps,
parallel_count=args.parallel_count,
parallel_index=args.parallel_index,
dry_run=False,
build_verbose=args.build_verbose,
keep_going=True,
collect_size_info='size_info.txt',
collect_app_info='list_job_@p.txt',
ignore_warning_strs=args.ignore_warning_str,
ignore_warning_file=args.ignore_warning_file,
copy_sdkconfig=args.copy_sdkconfig,
modified_components=args.modified_components,
modified_files=args.modified_files,
ignore_app_dependencies_filepatterns=args.ignore_app_dependencies_filepatterns,
)
if isinstance(res, tuple):
sys.exit(res[0])
else:
sys.exit(res)
if __name__ == '__main__':
parser = argparse.ArgumentParser(
@ -249,8 +276,7 @@ if __name__ == '__main__':
parser.add_argument(
'--ignore-warning-str',
nargs='+',
help='Ignore the warning string that match the specified regex in the build output. '
'Can be specified multiple times.',
help='Ignore the warning string that match the specified regex in the build output. space-separated list',
)
parser.add_argument(
'--ignore-warning-file',
@ -298,6 +324,30 @@ if __name__ == '__main__':
help='by default this script would set the build flags exactly the same as the CI ones. '
'Set this flag to use your local build flags.',
)
parser.add_argument(
'--modified-components',
nargs='*',
default=None,
help='space-separated list which specifies the modified components. app with `depends_components` set in the '
'corresponding manifest files would only be built if depends on any of the specified components.',
)
parser.add_argument(
'--modified-files',
nargs='*',
default=None,
help='space-separated list which specifies the modified files. app with `depends_filepatterns` set in the '
'corresponding manifest files would only be built if any of the specified file pattern matches any of the '
'specified modified files.',
)
parser.add_argument(
'-if',
'--ignore-app-dependencies-filepatterns',
nargs='*',
default=None,
help='space-separated list which specifies the file patterns used for ignoring checking the app dependencies. '
'The `depends_components` and `depends_filepatterns` set in the manifest files will be ignored when any of the '
'specified file patterns matches any of the modified files. Must be used together with --modified-files',
)
arguments = parser.parse_args()
@ -309,6 +359,37 @@ if __name__ == '__main__':
os.environ[_k] = _v
LOGGER.info(f'env var {_k} set to "{_v}"')
if os.getenv('IS_MR_PIPELINE') == '0' or os.getenv('BUILD_AND_TEST_ALL_APPS') == '1':
# if it's not MR pipeline or env var BUILD_AND_TEST_ALL_APPS=1,
# remove component dependency related arguments
if 'modified_components' in arguments:
arguments.modified_components = None
if 'modified_files' in arguments:
arguments.modified_files = None
# file patterns to tigger full build
if 'modified_components' in arguments and not arguments.ignore_app_dependencies_filepatterns:
arguments.ignore_app_dependencies_filepatterns = [
# tools
'tools/cmake/**/*',
'tools/tools.json',
# components
'components/cxx/**/*',
'components/esp_common/**/*',
'components/esp_hw_support/**/*',
'components/esp_rom/**/*',
'components/esp_system/**/*',
'components/esp_timer/**/*',
'components/freertos/**/*',
'components/hal/**/*',
'components/heap/**/*',
'components/log/**/*',
'components/newlib/**/*',
'components/riscv/**/*',
'components/soc/**/*',
'components/xtensa/**/*',
]
main(arguments)

2
tools/ci/python_packages/ttfw_idf/IDFAssignTest.py
View File

@ -54,7 +54,7 @@ class IDFAssignTest(CIAssignTest.AssignTest):
super(IDFAssignTest, self).__init__(test_case_path, ci_config_file, case_group)
def format_build_log_path(self, parallel_num):
return 'list_job_{}.json'.format(parallel_num)
return 'list_job_{}.txt'.format(parallel_num)
def create_artifact_index_file(self, project_id=None, pipeline_id=None):
if project_id is None: