From b52b49c8b60af8c2b38df896d538b647525b3ed4 Mon Sep 17 00:00:00 2001
From: Roland Dobai <roland@espressif.com>
Date: Tue, 12 Nov 2019 08:59:11 +0100
Subject: [PATCH] CI: Do check first in order to save build time

---
 .gitlab-ci.yml                                |  10 +-
 tools/ci/config/build.yml                     |  77 ----------
 tools/ci/config/{check.yml => post_check.yml} | 116 +++-----------
 tools/ci/config/pre_check.yml                 | 142 ++++++++++++++++++
 4 files changed, 173 insertions(+), 172 deletions(-)
 rename tools/ci/config/{check.yml => post_check.yml} (56%)
 create mode 100644 tools/ci/config/pre_check.yml

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index f3e8f7c11b..3897135ce8 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,9 +1,10 @@
 stages:
+  - pre_check
   - build
   - assign_test
   - host_test
   - target_test
-  - check
+  - post_check
   - deploy
   - post_check
 
@@ -120,7 +121,7 @@ after_script:
   - *cleanup_custom_toolchain
 
 .check_job_template:
-  stage: check
+  stage: pre_check
   image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
   tags:
     - host_test
@@ -128,7 +129,7 @@ after_script:
   extends: .before_script_lesser_nofilter
 
 .check_job_template_with_filter:
-  stage: check
+  stage: pre_check
   image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
   tags:
     - host_test
@@ -147,9 +148,10 @@ after_script:
     - source tools/ci/configure_ci_environment.sh
 
 include:
+  - '/tools/ci/config/pre_check.yml'
   - '/tools/ci/config/build.yml'
   - '/tools/ci/config/assign-test.yml'
   - '/tools/ci/config/host-test.yml'
   - '/tools/ci/config/target-test.yml'
-  - '/tools/ci/config/check.yml'
+  - '/tools/ci/config/post_check.yml'
   - '/tools/ci/config/deploy.yml'
diff --git a/tools/ci/config/build.yml b/tools/ci/config/build.yml
index fd714a7c63..b758e5e422 100644
--- a/tools/ci/config/build.yml
+++ b/tools/ci/config/build.yml
@@ -22,72 +22,6 @@
       - $BOT_LABEL_UNIT_TEST
       - $BOT_LABEL_REGULAR_TEST
 
-
-.build_with_make_and_cmake: &build_with_make_and_cmake |
-  echo -e "section_end:"`date +%s`":build_script\r\e[0Ksection_start:"`date +%s`":build_make\r\e[0KBuild with Make"
-  make defconfig
-  make all
-  make clean
-  echo -e "section_end:"`date +%s`":build_make\r\e[0Ksection_start:"`date +%s`":build_cmake\r\e[0KBuild with CMake"
-  rm -rf build sdkconfig
-  idf.py build
-  echo -e "section_end:"`date +%s`":build_cmake\r\e[0Ksection_start:"`date +%s`":build_script\r\e[0K"
-
-build_template_app:
-  stage: build
-  image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
-  tags:
-    - build
-  variables:
-    BATCH_BUILD: "1"
-  only:
-    variables:
-      - $BOT_TRIGGER_WITH_LABEL == null
-      - $BOT_LABEL_BUILD
-      - $BOT_LABEL_REGULAR_TEST
-  script:
-    # Set the variable for 'esp-idf-template' testing
-    - ESP_IDF_TEMPLATE_GIT=${ESP_IDF_TEMPLATE_GIT:-"https://github.com/espressif/esp-idf-template.git"}
-    - git clone ${ESP_IDF_TEMPLATE_GIT}
-    # Try to use the same branch name for esp-idf-template that we're
-    # using on esp-idf. If it doesn't exist then just stick to the default branch
-    - python $CHECKOUT_REF_SCRIPT esp-idf-template esp-idf-template
-    - cd esp-idf-template
-    - export PATH="$IDF_PATH/tools:$PATH"
-    - export EXTRA_CFLAGS=${PEDANTIC_CFLAGS}
-    - export EXTRA_CXXFLAGS=${PEDANTIC_CXXFLAGS}
-
-    # CONFIG_COMPILER_OPTIMIZATION_DEFAULT with flag -Og
-    - echo "CONFIG_COMPILER_OPTIMIZATION_DEFAULT=y" > sdkconfig.defaults
-    - *build_with_make_and_cmake
-
-    # CONFIG_COMPILER_OPTIMIZATION_NONE with flag -O0
-    - echo "CONFIG_COMPILER_OPTIMIZATION_NONE=y" > sdkconfig.defaults
-    - *build_with_make_and_cmake
-
-    # CONFIG_COMPILER_OPTIMIZATION_SIZE with flag -Os
-    - echo "CONFIG_COMPILER_OPTIMIZATION_SIZE=y" > sdkconfig.defaults
-    - *build_with_make_and_cmake
-
-    # CONFIG_COMPILER_OPTIMIZATION_PERF with flag -O2
-    - echo "CONFIG_COMPILER_OPTIMIZATION_PERF=y" > sdkconfig.defaults
-    - *build_with_make_and_cmake
-
-    # Same as above, but also disable assertions.
-    - echo "CONFIG_COMPILER_OPTIMIZATION_ASSERTIONS_DISABLE=y" >> sdkconfig.defaults
-    # Don't error out on -Wunused, when assertions are disabled
-    - export EXTRA_CFLAGS=${PEDANTIC_CFLAGS/-Werror=unused-variable -Werror=unused-but-set-variable -Werror=unused-function/}
-    - export EXTRA_CXXFLAGS=${PEDANTIC_CXXFLAGS/-Werror=unused-variable -Werror=unused-but-set-variable -Werror=unused-function/}
-    - *build_with_make_and_cmake
-    - export EXTRA_CFLAGS=${PEDANTIC_CFLAGS}
-    - export EXTRA_CXXFLAGS=${PEDANTIC_CXXFLAGS}
-
-    # Check if there are any stray printf/ets_printf references in WiFi libs
-    - pushd ../components/esp_wifi/lib
-    - for dir in esp32 esp32s2beta; do test $(xtensa-esp32-elf-nm $dir/*.a | grep -w printf | wc -l) -eq 0; done;
-    - for dir in esp32 esp32s2beta; do test $(xtensa-esp32-elf-nm $dir/*.a | grep -w ets_printf | wc -l) -eq 0; done;
-    - popd
-
 build_ssc:
   extends: .build_template
   parallel: 3
@@ -279,17 +213,6 @@ build_docs:
     - make html
     - ../check_doc_warnings.sh
 
-verify_cmake_style:
-  extends: .check_job_template
-  stage: build
-  only:
-    variables:
-      - $BOT_TRIGGER_WITH_LABEL == null
-      - $BOT_LABEL_BUILD
-      - $BOT_LABEL_REGULAR_TEST
-  script:
-    tools/cmake/run_cmake_lint.sh
-
 test_build_system:
   extends: .build_template
   script:
diff --git a/tools/ci/config/check.yml b/tools/ci/config/post_check.yml
similarity index 56%
rename from tools/ci/config/check.yml
rename to tools/ci/config/post_check.yml
index 23746404ba..2f9dacc125 100644
--- a/tools/ci/config/check.yml
+++ b/tools/ci/config/post_check.yml
@@ -1,100 +1,10 @@
-
 # copy from .gitlab-ci.yml as anchor is not global
 .show_submodule_urls: &show_submodule_urls |
   git config --get-regexp '^submodule\..*\.url$' || true
 
-check_line_endings:
-  extends: .check_job_template
-  script:
-    - tools/ci/check-line-endings.sh ${IDF_PATH}
-
-check_commit_msg:
-  extends: .check_job_template
-  script:
-    - git status
-    - git log -n10 --oneline
-    # commit start with "WIP: " need to be squashed before merge
-    - 'git log --pretty=%s master.. -- | grep "^WIP: " && exit 1 || exit 0'
-
-check_permissions:
-  extends: .check_job_template
-  script:
-    - tools/ci/check-executable.sh
-
-check_version:
-  extends: .check_job_template
-  # Don't run this for feature/bugfix branches, so that it is possible to modify
-  # esp_idf_version.h in a branch before tagging the next version.
-  only:
-    - master
-    - /^release\/v/
-    - /^v\d+\.\d+(\.\d+)?($|-)/
-  script:
-    - export IDF_PATH=$PWD
-    - tools/ci/check_idf_version.sh
-
-check_examples_cmake_make:
-  extends: .check_job_template_with_filter
-  except:
-    - master
-    - /^release\/v/
-    - /^v\d+\.\d+(\.\d+)?($|-)/
-  script:
-    - tools/ci/check_examples_cmake_make.sh
-
-check_examples_rom_header:
-  extends: .check_job_template_with_filter
-  script:
-    - tools/ci/check_examples_rom_header.sh
-
-check_python_style:
-  extends: .check_job_template_with_filter
-  artifacts:
-    when: on_failure
-    paths:
-      - flake8_output.txt
-    expire_in: 1 week
-  script:
-    # run it only under Python 3 (it is very slow under Python 2)
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.4.8 python -m flake8 --config=$IDF_PATH/.flake8 --output-file=flake8_output.txt --tee --benchmark $IDF_PATH
-
-check_kconfigs:
-  extends: .check_job_template_with_filter
-  artifacts:
-    when: on_failure
-    paths:
-      - components/*/Kconfig*.new
-      - examples/*/*/*/Kconfig*.new
-      - examples/*/*/*/*/Kconfig*.new
-      - tools/*/Kconfig*.new
-      - tools/*/*/Kconfig*.new
-      - tools/*/*/*/Kconfig*.new
-    expire_in: 1 week
-  script:
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ${IDF_PATH}/tools/test_check_kconfigs.py
-    - ${IDF_PATH}/tools/check_kconfigs.py
-
-check_deprecated_kconfig_options:
-  extends: .check_job_template_with_filter
-  script:
-    - ${IDF_PATH}/tools/ci/check_deprecated_kconfigs.py
-
-check_ut_cmake_make:
-  extends: .check_job_template_with_filter
-  stage: check
-  image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
-  tags:
-    - build
-  except:
-    - master
-    - /^release\/v/
-    - /^v\d+\.\d+(\.\d+)?($|-)/
-  dependencies: []
-  script:
-    - tools/ci/check_ut_cmake_make.sh
-
 check_submodule_sync:
   extends: .check_job_template
+  stage: post_check
   tags:
     - github_sync
   retry: 2
@@ -125,8 +35,23 @@ check_wifi_lib_md5:
     - IDF_TARGET=esp32 $IDF_PATH/components/esp_wifi/test_md5/test_md5.sh
     - IDF_TARGET=esp32s2beta $IDF_PATH/components/esp_wifi/test_md5/test_md5.sh
 
+check_ut_cmake_make:
+  extends: .check_job_template_with_filter
+  stage: post_check
+  image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
+  tags:
+    - build
+  except:
+    - master
+    - /^release\/v/
+    - /^v\d+\.\d+(\.\d+)?($|-)/
+  dependencies: []
+  script:
+    - tools/ci/check_ut_cmake_make.sh
+
 check_artifacts_expire_time:
   extends: .check_job_template
+  stage: post_check
   script:
     # check if we have set expire time for all artifacts
     - python tools/ci/check_artifacts_expire_time.py
@@ -141,3 +66,12 @@ check_pipeline_triggered_by_label:
     # If the pipeline is triggered with label, the pipeline will only succeeded if "regular_test" label is added.
     # We want to make sure some jobs are always executed to detect regression.
     - test "$BOT_LABEL_REGULAR_TEST" = "true" || { echo "CI can only pass if 'regular_test' label is included"; exit -1; }
+
+check_commit_msg:
+  extends: .check_job_template
+  stage: post_check
+  script:
+    - git status
+    - git log -n10 --oneline
+    # commit start with "WIP: " need to be squashed before merge
+    - 'git log --pretty=%s master.. -- | grep "^WIP: " && exit 1 || exit 0'
diff --git a/tools/ci/config/pre_check.yml b/tools/ci/config/pre_check.yml
new file mode 100644
index 0000000000..7b67e917e1
--- /dev/null
+++ b/tools/ci/config/pre_check.yml
@@ -0,0 +1,142 @@
+check_line_endings:
+  extends: .check_job_template
+  script:
+    - tools/ci/check-line-endings.sh ${IDF_PATH}
+
+check_permissions:
+  extends: .check_job_template
+  script:
+    - tools/ci/check-executable.sh
+
+check_version:
+  extends: .check_job_template
+  # Don't run this for feature/bugfix branches, so that it is possible to modify
+  # esp_idf_version.h in a branch before tagging the next version.
+  only:
+    - master
+    - /^release\/v/
+    - /^v\d+\.\d+(\.\d+)?($|-)/
+  script:
+    - export IDF_PATH=$PWD
+    - tools/ci/check_idf_version.sh
+
+check_examples_cmake_make:
+  extends: .check_job_template_with_filter
+  except:
+    - master
+    - /^release\/v/
+    - /^v\d+\.\d+(\.\d+)?($|-)/
+  script:
+    - tools/ci/check_examples_cmake_make.sh
+
+check_examples_rom_header:
+  extends: .check_job_template_with_filter
+  script:
+    - tools/ci/check_examples_rom_header.sh
+
+check_python_style:
+  extends: .check_job_template_with_filter
+  artifacts:
+    when: on_failure
+    paths:
+      - flake8_output.txt
+    expire_in: 1 week
+  script:
+    # run it only under Python 3 (it is very slow under Python 2)
+    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.4.8 python -m flake8 --config=$IDF_PATH/.flake8 --output-file=flake8_output.txt --tee --benchmark $IDF_PATH
+
+check_kconfigs:
+  extends: .check_job_template_with_filter
+  artifacts:
+    when: on_failure
+    paths:
+      - components/*/Kconfig*.new
+      - examples/*/*/*/Kconfig*.new
+      - examples/*/*/*/*/Kconfig*.new
+      - tools/*/Kconfig*.new
+      - tools/*/*/Kconfig*.new
+      - tools/*/*/*/Kconfig*.new
+    expire_in: 1 week
+  script:
+    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ${IDF_PATH}/tools/test_check_kconfigs.py
+    - ${IDF_PATH}/tools/check_kconfigs.py
+
+check_deprecated_kconfig_options:
+  extends: .check_job_template_with_filter
+  script:
+    - ${IDF_PATH}/tools/ci/check_deprecated_kconfigs.py
+
+check_cmake_style:
+  extends: .check_job_template
+  only:
+    variables:
+      - $BOT_TRIGGER_WITH_LABEL == null
+      - $BOT_LABEL_BUILD
+      - $BOT_LABEL_REGULAR_TEST
+  script:
+    tools/cmake/run_cmake_lint.sh
+
+.build_with_make_and_cmake: &build_with_make_and_cmake |
+  echo -e "section_end:"`date +%s`":build_script\r\e[0Ksection_start:"`date +%s`":build_make\r\e[0KBuild with Make"
+  make defconfig
+  make all
+  make clean
+  echo -e "section_end:"`date +%s`":build_make\r\e[0Ksection_start:"`date +%s`":build_cmake\r\e[0KBuild with CMake"
+  rm -rf build sdkconfig
+  idf.py build
+  echo -e "section_end:"`date +%s`":build_cmake\r\e[0Ksection_start:"`date +%s`":build_script\r\e[0K"
+
+build_template_app:
+  stage: pre_check
+  image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
+  tags:
+    - build
+  variables:
+    BATCH_BUILD: "1"
+  only:
+    variables:
+      - $BOT_TRIGGER_WITH_LABEL == null
+      - $BOT_LABEL_BUILD
+      - $BOT_LABEL_REGULAR_TEST
+  script:
+    # Set the variable for 'esp-idf-template' testing
+    - ESP_IDF_TEMPLATE_GIT=${ESP_IDF_TEMPLATE_GIT:-"https://github.com/espressif/esp-idf-template.git"}
+    - git clone ${ESP_IDF_TEMPLATE_GIT}
+    # Try to use the same branch name for esp-idf-template that we're
+    # using on esp-idf. If it doesn't exist then just stick to the default branch
+    - python $CHECKOUT_REF_SCRIPT esp-idf-template esp-idf-template
+    - cd esp-idf-template
+    - export PATH="$IDF_PATH/tools:$PATH"
+    - export EXTRA_CFLAGS=${PEDANTIC_CFLAGS}
+    - export EXTRA_CXXFLAGS=${PEDANTIC_CXXFLAGS}
+
+    # CONFIG_COMPILER_OPTIMIZATION_DEFAULT with flag -Og
+    - echo "CONFIG_COMPILER_OPTIMIZATION_DEFAULT=y" > sdkconfig.defaults
+    - *build_with_make_and_cmake
+
+    # CONFIG_COMPILER_OPTIMIZATION_NONE with flag -O0
+    - echo "CONFIG_COMPILER_OPTIMIZATION_NONE=y" > sdkconfig.defaults
+    - *build_with_make_and_cmake
+
+    # CONFIG_COMPILER_OPTIMIZATION_SIZE with flag -Os
+    - echo "CONFIG_COMPILER_OPTIMIZATION_SIZE=y" > sdkconfig.defaults
+    - *build_with_make_and_cmake
+
+    # CONFIG_COMPILER_OPTIMIZATION_PERF with flag -O2
+    - echo "CONFIG_COMPILER_OPTIMIZATION_PERF=y" > sdkconfig.defaults
+    - *build_with_make_and_cmake
+
+    # Same as above, but also disable assertions.
+    - echo "CONFIG_COMPILER_OPTIMIZATION_ASSERTIONS_DISABLE=y" >> sdkconfig.defaults
+    # Don't error out on -Wunused, when assertions are disabled
+    - export EXTRA_CFLAGS=${PEDANTIC_CFLAGS/-Werror=unused-variable -Werror=unused-but-set-variable -Werror=unused-function/}
+    - export EXTRA_CXXFLAGS=${PEDANTIC_CXXFLAGS/-Werror=unused-variable -Werror=unused-but-set-variable -Werror=unused-function/}
+    - *build_with_make_and_cmake
+    - export EXTRA_CFLAGS=${PEDANTIC_CFLAGS}
+    - export EXTRA_CXXFLAGS=${PEDANTIC_CXXFLAGS}
+
+    # Check if there are any stray printf/ets_printf references in WiFi libs
+    - pushd ../components/esp_wifi/lib
+    - for dir in esp32 esp32s2beta; do test $(xtensa-esp32-elf-nm $dir/*.a | grep -w printf | wc -l) -eq 0; done;
+    - for dir in esp32 esp32s2beta; do test $(xtensa-esp32-elf-nm $dir/*.a | grep -w ets_printf | wc -l) -eq 0; done;
+    - popd