ci: always set expire_in and when with artifacts

set it under `default` does not work due to gitlab bugs.
will use `default` back once the bugs are resolved.
This commit is contained in:
Fu Hanxi 2024-02-16 13:43:53 +01:00
parent dee34c2a48
commit b70d65444a
No known key found for this signature in database
GPG Key ID: 19399699CF3C4B16
10 changed files with 75 additions and 25 deletions

View File

@ -32,6 +32,8 @@
# keep the size info to help track the binary size
- size_info.txt
- "**/build*/size.json"
expire_in: 1 week
when: always
script:
# CI specific options start from "--parallel-count xxx". could ignore when running locally
- run_cmd python tools/ci/ci_build_apps.py $TEST_DIR -v
@ -85,6 +87,8 @@
- log_template_app/*
- size_info.txt
- build_template_app/**/size.json
expire_in: 1 week
when: always
script:
# Set the variable for 'esp-idf-template' testing
- ESP_IDF_TEMPLATE_GIT=${ESP_IDF_TEMPLATE_GIT:-"https://github.com/espressif/esp-idf-template.git"}
@ -178,6 +182,14 @@ build_clang_test_apps_esp32c6:
- job: fast_template_app
artifacts: false
optional: true
artifacts:
reports:
junit: XUNIT_RESULT.xml
paths:
- XUNIT_RESULT.xml
- test_build_system
expire_in: 1 week
when: always
script:
- ${IDF_PATH}/tools/ci/test_configure_ci_environment.sh
- cd ${IDF_PATH}/tools/test_build_system
@ -189,13 +201,6 @@ build_clang_test_apps_esp32c6:
pytest_build_system:
extends: .test_build_system_template
parallel: 3
artifacts:
paths:
- XUNIT_RESULT.xml
- test_build_system
expire_in: 2 days
reports:
junit: XUNIT_RESULT.xml
pytest_build_system_macos:
extends:
@ -205,13 +210,6 @@ pytest_build_system_macos:
tags:
- macos_shell
parallel: 3
artifacts:
paths:
- XUNIT_RESULT.xml
- test_build_system
expire_in: 2 days
reports:
junit: XUNIT_RESULT.xml
build_docker:
extends:
@ -264,6 +262,8 @@ generate_build_child_pipeline:
- build_child_pipeline.yml
- test_related_apps.txt
- non_test_related_apps.txt
expire_in: 1 week
when: always
script:
- run_cmd python tools/ci/dynamic_pipelines/scripts/generate_build_child_pipeline.py
--modified-components ${MR_MODIFIED_COMPONENTS}

View File

@ -353,9 +353,12 @@ default:
- *setup_tools_and_idf_python_venv
- add_gitlab_ssh_keys
- fetch_submodules
artifacts:
expire_in: 1 week
when: always
# gitlab bug, setting them here doesn't work
# - expire_in: https://gitlab.com/gitlab-org/gitlab/-/issues/404563
# - when: https://gitlab.com/gitlab-org/gitlab/-/issues/440672
# artifacts:
# expire_in: 1 week
# when: always
retry:
max: 2
when:

View File

@ -96,6 +96,9 @@ check_docs_lang_sync:
- if [ -n "${BREATHE_ALT_INSTALL_URL}" ]; then pip uninstall -y breathe && pip install -U ${BREATHE_ALT_INSTALL_URL}; fi
- cd docs
- build-docs -t $DOCTGT -bs $DOC_BUILDERS -l $DOCLANG build
artifacts:
expire_in: 4 days
when: always
parallel:
matrix:
- DOCLANG: ["en", "zh_CN"]
@ -124,7 +127,6 @@ build_docs_html_full:
paths:
- docs/_build/*/*/*.txt
- docs/_build/*/*/html/*
expire_in: 4 days
variables:
DOC_BUILDERS: "html"
@ -137,7 +139,6 @@ build_docs_html_full_prod:
paths:
- docs/_build/*/*/*.txt
- docs/_build/*/*/html/*
expire_in: 4 days
variables:
DOC_BUILDERS: "html"
@ -153,7 +154,6 @@ build_docs_html_partial:
paths:
- docs/_build/*/*/*.txt
- docs/_build/*/*/html/*
expire_in: 4 days
variables:
DOC_BUILDERS: "html"
parallel:
@ -174,7 +174,6 @@ build_docs_pdf:
artifacts:
paths:
- docs/_build/*/*/latex/*
expire_in: 4 days
variables:
DOC_BUILDERS: "latex"
@ -186,7 +185,6 @@ build_docs_pdf_prod:
artifacts:
paths:
- docs/_build/*/*/latex/*
expire_in: 4 days
variables:
DOC_BUILDERS: "latex"

View File

@ -14,6 +14,9 @@
optional: true
artifacts: false
- pipeline_variables
artifacts:
expire_in: 1 week
when: always
test_nvs_on_host:
extends: .host_test_template

View File

@ -46,6 +46,7 @@ gen_integration_pipeline:
paths:
- idf-integration-ci/child_pipeline/
expire_in: 2 weeks
when: always
script:
- add_gitlab_ssh_keys
- retry_failed git clone ${CI_GEN_INTEGRATION_PIPELINE_REPO} idf-integration-ci

View File

@ -72,6 +72,7 @@ check_chip_support_components:
paths:
- esp_hw_support_part.h
- bootloader_support_part.h
expire_in: 1 week
script:
- python tools/ci/check_soc_headers_leak.py
- find ${IDF_PATH}/components/soc/*/include/soc/ -name "*_struct.h" -print0 | xargs -0 -n1 ./tools/ci/check_soc_struct_headers.py
@ -85,6 +86,7 @@ check_esp_err_to_name:
when: on_failure
paths:
- components/esp_common/esp_err_to_name.c
expire_in: 1 week
script:
- cd ${IDF_PATH}/tools/
- ./gen_esp_err_to_name.py
@ -157,6 +159,10 @@ pipeline_variables:
artifacts:
reports:
dotenv: pipeline.env
paths:
- pipeline.env
expire_in: 1 week
when: always
check_test_cases_env_markers_and_required_runners:
extends:

View File

@ -6,6 +6,8 @@ clang_tidy_check:
artifacts:
paths:
- clang_tidy_reports/
expire_in: 1 week
when: always
variables:
IDF_TOOLCHAIN: clang
script:
@ -23,6 +25,10 @@ check_pylint:
artifacts:
reports:
codequality: pylint.json
paths:
- pylint.json
expire_in: 1 week
when: always
script:
- |
if [ -n "$CI_MERGE_REQUEST_IID" ]; then
@ -70,6 +76,8 @@ check_pylint:
artifacts:
paths:
- $REPORT_PATTERN
expire_in: 1 week
when: always
dependencies: # Here is not a hard dependency relationship, could be skipped when only python files changed. so we do not use "needs" here.
- clang_tidy_check

View File

@ -26,6 +26,7 @@ test_cli_installer_win:
paths:
- tools/tools.new.json
- tools/test_idf_tools/test_python_env_logs.txt
expire_in: 1 week
variables:
IDF_PATH: "$CI_PROJECT_DIR"
script:
@ -46,6 +47,8 @@ test_tools_win:
- ${IDF_PATH}/XUNIT_*.xml
reports:
junit: ${IDF_PATH}/XUNIT_*.xml
expire_in: 1 week
when: always
variables:
LC_ALL: C.UTF-8
PYTHONPATH: "$PYTHONPATH;$IDF_PATH\\tools;$IDF_PATH\\tools\\esp_app_trace;$IDF_PATH\\components\\partition_table;$IDF_PATH\\tools\\ci\\python_packages"
@ -90,3 +93,4 @@ pytest_build_system_win:
expire_in: 2 days
reports:
junit: XUNIT_RESULT.xml
when: always

View File

@ -104,6 +104,21 @@ class YmlLinter:
f'Please set `dependencies:` (to null) explicitly to avoid missing artifacts issue'
)
def _lint_artifacts_expire_in_and_when(self) -> None:
"""
Set `artifacts: expire_in` and `artifacts: when` together since gitlab has bugs:
- https://gitlab.com/gitlab-org/gitlab/-/issues/404563 (expire_in)
- https://gitlab.com/gitlab-org/gitlab/-/issues/440672 (when)
"""
for job_name, d in self.yml_config.jobs.items():
if 'artifacts' in d:
if 'expire_in' not in d['artifacts']:
self._errors.append(f'job {job_name} missing `artifacts: expire_in`. (suggest to set to `1 week`)')
if 'when' not in d['artifacts']:
self._errors.append(f'job {job_name} missing `artifacts: when`. (suggest to set to `always`)')
if __name__ == '__main__':
parser = argparse.ArgumentParser()

View File

@ -202,22 +202,34 @@ class GitlabYmlConfig:
for k in self._job_keys:
self._expand_extends(k)
def _merge_dict(self, d1: t.Dict[str, t.Any], d2: t.Dict[str, t.Any]) -> t.Any:
for k, v in d2.items():
if k in d1:
if isinstance(v, dict) and isinstance(d1[k], dict):
d1[k] = self._merge_dict(d1[k], v)
else:
d1[k] = v
else:
d1[k] = v
return d1
def _expand_extends(self, name: str) -> t.Dict[str, t.Any]:
extends = to_list(self.config[name].pop('extends', None))
original_d = self.config[name].copy()
if not extends:
return self.config[name] # type: ignore
original_d = self.config[name].copy()
d = {}
while extends:
self._used_template_keys.update(extends)
self._used_template_keys.update(extends) # for tracking
for i in extends:
d.update(self._expand_extends(i))
extends = to_list(self.config[name].pop('extends', None))
self.config[name] = {**d, **original_d}
self.config[name] = self._merge_dict(d, original_d)
return self.config[name] # type: ignore