Merge branch 'ci/fix_default_artifacts' into 'master'

ci: always set `expire_in` and `when` with `artifacts`

See merge request espressif/esp-idf!29083

.gitlab/ci/build.yml

@@ -32,6 +32,8 @@
       # keep the size info to help track the binary size
       - size_info.txt
       - "**/build*/size.json"
+    expire_in: 1 week
+    when: always
   script:
     # CI specific options start from "--parallel-count xxx". could ignore when running locally
     - run_cmd python tools/ci/ci_build_apps.py $TEST_DIR -v
@@ -85,6 +87,8 @@
       - log_template_app/*
       - size_info.txt
       - build_template_app/**/size.json
+    expire_in: 1 week
+    when: always
   script:
     # Set the variable for 'esp-idf-template' testing
     - ESP_IDF_TEMPLATE_GIT=${ESP_IDF_TEMPLATE_GIT:-"https://github.com/espressif/esp-idf-template.git"}
@@ -178,6 +182,14 @@ build_clang_test_apps_esp32c6:
     - job: fast_template_app
       artifacts: false
       optional: true
+  artifacts:
+    reports:
+      junit: XUNIT_RESULT.xml
+    paths:
+      - XUNIT_RESULT.xml
+      - test_build_system
+    expire_in: 1 week
+    when: always
   script:
     - ${IDF_PATH}/tools/ci/test_configure_ci_environment.sh
     - cd ${IDF_PATH}/tools/test_build_system
@@ -189,13 +201,6 @@ build_clang_test_apps_esp32c6:
 pytest_build_system:
   extends: .test_build_system_template
   parallel: 3
-  artifacts:
-    paths:
-      - XUNIT_RESULT.xml
-      - test_build_system
-    expire_in: 2 days
-    reports:
-      junit: XUNIT_RESULT.xml
 
 pytest_build_system_macos:
   extends:
@@ -205,13 +210,6 @@ pytest_build_system_macos:
   tags:
     - macos_shell
   parallel: 3
-  artifacts:
-    paths:
-      - XUNIT_RESULT.xml
-      - test_build_system
-    expire_in: 2 days
-    reports:
-      junit: XUNIT_RESULT.xml
 
 build_docker:
   extends:
@@ -264,6 +262,8 @@ generate_build_child_pipeline:
       - build_child_pipeline.yml
       - test_related_apps.txt
       - non_test_related_apps.txt
+    expire_in: 1 week
+    when: always
   script:
     - run_cmd python tools/ci/dynamic_pipelines/scripts/generate_build_child_pipeline.py
       --modified-components ${MR_MODIFIED_COMPONENTS}

.gitlab/ci/common.yml

@@ -353,9 +353,12 @@ default:
     - *setup_tools_and_idf_python_venv
     - add_gitlab_ssh_keys
     - fetch_submodules
-  artifacts:
-    expire_in: 1 week
-    when: always
+# gitlab bug, setting them here doesn't work
+# - expire_in: https://gitlab.com/gitlab-org/gitlab/-/issues/404563
+# - when: https://gitlab.com/gitlab-org/gitlab/-/issues/440672
+#  artifacts:
+#    expire_in: 1 week
+#    when: always
   retry:
     max: 2
     when:

.gitlab/ci/docs.yml

@@ -96,6 +96,9 @@ check_docs_lang_sync:
     - if [ -n "${BREATHE_ALT_INSTALL_URL}" ]; then pip uninstall -y breathe && pip install -U ${BREATHE_ALT_INSTALL_URL}; fi
     - cd docs
     - build-docs -t $DOCTGT -bs $DOC_BUILDERS -l $DOCLANG build
+  artifacts:
+    expire_in: 4 days
+    when: always
   parallel:
     matrix:
       - DOCLANG: ["en", "zh_CN"]
@@ -124,7 +127,6 @@ build_docs_html_full:
     paths:
       - docs/_build/*/*/*.txt
       - docs/_build/*/*/html/*
-    expire_in: 4 days
   variables:
     DOC_BUILDERS: "html"
 
@@ -137,7 +139,6 @@ build_docs_html_full_prod:
     paths:
       - docs/_build/*/*/*.txt
       - docs/_build/*/*/html/*
-    expire_in: 4 days
   variables:
     DOC_BUILDERS: "html"
 
@@ -153,7 +154,6 @@ build_docs_html_partial:
     paths:
       - docs/_build/*/*/*.txt
       - docs/_build/*/*/html/*
-    expire_in: 4 days
   variables:
     DOC_BUILDERS: "html"
   parallel:
@@ -174,7 +174,6 @@ build_docs_pdf:
   artifacts:
     paths:
       - docs/_build/*/*/latex/*
-    expire_in: 4 days
   variables:
     DOC_BUILDERS: "latex"
 
@@ -186,7 +185,6 @@ build_docs_pdf_prod:
   artifacts:
     paths:
       - docs/_build/*/*/latex/*
-    expire_in: 4 days
   variables:
     DOC_BUILDERS: "latex"
 

.gitlab/ci/host-test.yml

@@ -14,6 +14,9 @@
       optional: true
       artifacts: false
     - pipeline_variables
+  artifacts:
+    expire_in: 1 week
+    when: always
 
 test_nvs_on_host:
   extends: .host_test_template

.gitlab/ci/integration_test.yml

@@ -46,6 +46,7 @@ gen_integration_pipeline:
     paths:
       - idf-integration-ci/child_pipeline/
     expire_in: 2 weeks
+    when: always
   script:
     - add_gitlab_ssh_keys
     - retry_failed git clone ${CI_GEN_INTEGRATION_PIPELINE_REPO} idf-integration-ci

.gitlab/ci/pre_check.yml

@@ -72,6 +72,7 @@ check_chip_support_components:
     paths:
       - esp_hw_support_part.h
       - bootloader_support_part.h
+    expire_in: 1 week
   script:
     - python tools/ci/check_soc_headers_leak.py
     - find ${IDF_PATH}/components/soc/*/include/soc/ -name "*_struct.h" -print0 | xargs -0 -n1 ./tools/ci/check_soc_struct_headers.py
@@ -85,6 +86,7 @@ check_esp_err_to_name:
     when: on_failure
     paths:
       - components/esp_common/esp_err_to_name.c
+    expire_in: 1 week
   script:
     - cd ${IDF_PATH}/tools/
     - ./gen_esp_err_to_name.py
@@ -157,6 +159,10 @@ pipeline_variables:
   artifacts:
     reports:
       dotenv: pipeline.env
+    paths:
+      - pipeline.env
+    expire_in: 1 week
+    when: always
 
 check_test_cases_env_markers_and_required_runners:
   extends:

.gitlab/ci/static-code-analysis.yml

@@ -6,6 +6,8 @@ clang_tidy_check:
   artifacts:
     paths:
       - clang_tidy_reports/
+    expire_in: 1 week
+    when: always
   variables:
     IDF_TOOLCHAIN: clang
   script:
@@ -23,6 +25,10 @@ check_pylint:
   artifacts:
     reports:
       codequality: pylint.json
+    paths:
+      - pylint.json
+    expire_in: 1 week
+    when: always
   script:
     - |
       if [ -n "$CI_MERGE_REQUEST_IID" ]; then
@@ -70,6 +76,8 @@ check_pylint:
   artifacts:
     paths:
       - $REPORT_PATTERN
+    expire_in: 1 week
+    when: always
   dependencies:  # Here is not a hard dependency relationship, could be skipped when only python files changed. so we do not use "needs" here.
     - clang_tidy_check
 

.gitlab/ci/test-win.yml

@@ -26,6 +26,7 @@ test_cli_installer_win:
     paths:
       - tools/tools.new.json
       - tools/test_idf_tools/test_python_env_logs.txt
+    expire_in: 1 week
   variables:
     IDF_PATH: "$CI_PROJECT_DIR"
   script:
@@ -46,6 +47,8 @@ test_tools_win:
       - ${IDF_PATH}/XUNIT_*.xml
     reports:
       junit: ${IDF_PATH}/XUNIT_*.xml
+    expire_in: 1 week
+    when: always
   variables:
     LC_ALL: C.UTF-8
     PYTHONPATH: "$PYTHONPATH;$IDF_PATH\\tools;$IDF_PATH\\tools\\esp_app_trace;$IDF_PATH\\components\\partition_table;$IDF_PATH\\tools\\ci\\python_packages"
@@ -90,3 +93,4 @@ pytest_build_system_win:
     expire_in: 2 days
     reports:
       junit: XUNIT_RESULT.xml
+    when: always

tools/ci/gitlab_yaml_linter.py

@@ -104,6 +104,21 @@ class YmlLinter:
                         f'Please set `dependencies:` (to null) explicitly to avoid missing artifacts issue'
                     )
 
+    def _lint_artifacts_expire_in_and_when(self) -> None:
+        """
+        Set `artifacts: expire_in` and `artifacts: when` together since gitlab has bugs:
+
+        - https://gitlab.com/gitlab-org/gitlab/-/issues/404563 (expire_in)
+        - https://gitlab.com/gitlab-org/gitlab/-/issues/440672 (when)
+        """
+        for job_name, d in self.yml_config.jobs.items():
+            if 'artifacts' in d:
+                if 'expire_in' not in d['artifacts']:
+                    self._errors.append(f'job {job_name} missing `artifacts: expire_in`. (suggest to set to `1 week`)')
+
+                if 'when' not in d['artifacts']:
+                    self._errors.append(f'job {job_name} missing `artifacts: when`. (suggest to set to `always`)')
+
 
 if __name__ == '__main__':
     parser = argparse.ArgumentParser()

tools/ci/idf_ci_utils.py

@@ -202,22 +202,34 @@ class GitlabYmlConfig:
         for k in self._job_keys:
             self._expand_extends(k)
 
+    def _merge_dict(self, d1: t.Dict[str, t.Any], d2: t.Dict[str, t.Any]) -> t.Any:
+        for k, v in d2.items():
+            if k in d1:
+                if isinstance(v, dict) and isinstance(d1[k], dict):
+                    d1[k] = self._merge_dict(d1[k], v)
+                else:
+                    d1[k] = v
+            else:
+                d1[k] = v
+
+        return d1
+
     def _expand_extends(self, name: str) -> t.Dict[str, t.Any]:
         extends = to_list(self.config[name].pop('extends', None))
-        original_d = self.config[name].copy()
         if not extends:
             return self.config[name]  # type: ignore
 
+        original_d = self.config[name].copy()
         d = {}
         while extends:
-            self._used_template_keys.update(extends)
+            self._used_template_keys.update(extends)  # for tracking
 
             for i in extends:
                 d.update(self._expand_extends(i))
 
             extends = to_list(self.config[name].pop('extends', None))
 
-        self.config[name] = {**d, **original_d}
+        self.config[name] = self._merge_dict(d, original_d)
         return self.config[name]  # type: ignore