ci: enable clang-tidy check for more components

Use internal project clang-tidy-runner

Move sonarqube related configs to sonar-project.properties

.gitlab-ci.yml

@@ -51,7 +51,7 @@ variables:
   ESP_IDF_DOC_ENV_IMAGE: "$CI_DOCKER_REGISTRY/esp-idf-doc-env:v4.4-1-v2"
   ESP_ENV_IMAGE: "$CI_DOCKER_REGISTRY/esp-env:v4.4-1"
   AFL_FUZZER_TEST_IMAGE: "$CI_DOCKER_REGISTRY/afl-fuzzer-test:v4.4-1-1"
-  CLANG_STATIC_ANALYSIS_IMAGE: "${CI_DOCKER_REGISTRY}/clang-static-analysis:v4.4-1-1"
+  CLANG_STATIC_ANALYSIS_IMAGE: "${CI_DOCKER_REGISTRY}/clang-static-analysis:v4.4-1-2"
 
   # target test config file, used by assign test job
   CI_TARGET_TEST_CONFIG_FILE: "$CI_PROJECT_DIR/.gitlab/ci/target-test.yml"

.gitlab/CODEOWNERS

@@ -57,6 +57,7 @@
 /export.*                             @esp-idf-codeowners/tools
 /install.*                            @esp-idf-codeowners/tools
 /sdkconfig.rename                     @esp-idf-codeowners/build-config
+/sonar-project.properties             @esp-idf-codeowners/ci
 
 # sort-order-reset
 

.gitlab/ci/rules.yml

@@ -3,7 +3,10 @@
 ############
 .patterns-c-files: &patterns-c-files
   - ".gitlab/ci/static-code-analysis.yml"
+
   - "tools/ci/static-analysis-rules.yml"
+  - "tools/ci/clang_tidy_dirs.txt"
+
   - "**/*.{c,C}"
   - "**/*.{h,H}"
   - "components/**/Kconfig"

.gitlab/ci/static-code-analysis.yml

@@ -5,17 +5,46 @@ clang_tidy_check:
     - .rules:patterns:clang_tidy
   image: ${CLANG_STATIC_ANALYSIS_IMAGE}
   artifacts:
-    reports:
+    paths:
-      junit: $IDF_PATH/output.xml
+      - $OUTPUT_DIR
+    when: always
+    expire_in: 1 day
+  variables:
+    CLANG_TIDY_RUNNER_PROJ: 2107  # idf/clang-tidy-runner
+    CLANG_TIDY_DIRS_TXT: ${CI_PROJECT_DIR}/tools/ci/clang_tidy_dirs.txt
+    RULES_FILE: ${CI_PROJECT_DIR}/tools/ci/static-analysis-rules.yml
+    OUTPUT_DIR: ${CI_PROJECT_DIR}/clang_tidy_reports
+  script:
+    - python -m pip install -U pip
+    - internal_pip_install $CLANG_TIDY_RUNNER_PROJ pyclang
+    - export PATH=$PATH:$(python -c "import sys; print(sys.executable.rsplit('/', 1)[0])")
+    - dirs=$(cat ${CLANG_TIDY_DIRS_TXT} | while read line; do echo ${CI_PROJECT_DIR}/${line}; done | xargs)
+    - run_cmd idf_clang ${dirs}
+      --output-path ${OUTPUT_DIR}
+      --limit-file ${RULES_FILE}
+      --xtensa-include-dir
+      --run-clang-tidy-py ${RUN_CLANG_TIDY_PY}
+
+check_pylint:
+  extends:
+    - .pre_check_base_template
+    - .rules:patterns:python-files
+    - .before_script_minimal
+  image: $CI_DOCKER_REGISTRY/sonarqube-scanner:2
+  artifacts:
     when: always
     paths:
-      - $IDF_PATH/examples/get-started/hello_world/tidybuild/report/*
+      - pylint-report.txt
-    expire_in: 1 day
+    expire_in: 1 week
   script:
-    - retry_failed git clone $IDF_ANALYSIS_UTILS static_analysis_utils && cd static_analysis_utils
+    - export PYTHONPATH="$IDF_PATH/tools:$IDF_PATH/tools/ci/python_packages:$PYTHONPATH"
-    # Setup parameters of triggered/regular job
+    - |
-    - export TARGET_BRANCH=${BOT_CUSTOMIZED_REVISION-}
+      if [ -n "$CI_MERGE_REQUEST_IID" ]; then
-    - ./analyze.sh $IDF_PATH/examples/get-started/hello_world/ $IDF_PATH/tools/ci/static-analysis-rules.yml $IDF_PATH/output.xml
+        export files=$(python ${CI_PROJECT_DIR}/tools/ci/ci_get_mr_info.py files ${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME} | grep ".py");
+      else
+        export files=$(find . -iname "*.py" -print);
+      fi
+    - pylint --rcfile=.pylintrc $files -r n --output-format=parseable > pylint-report.txt || exit 0
 
 # build stage
 # Sonarqube related jobs put here for this reason:
@@ -38,21 +67,18 @@ clang_tidy_check:
     - source tools/ci/utils.sh
     - export PYTHONPATH="$CI_PROJECT_DIR/tools:$CI_PROJECT_DIR/tools/ci/python_packages:$PYTHONPATH"
     - fetch_submodules
-    # Exclude the submodules, all paths ends with /**
-    - export SUBMODULES=$(get_all_submodules)
-    # get all exclude paths specified in tools/ci/sonar_exclude_list.txt | ignore lines start with # | xargs | replace all <space> to <comma>
-    - export CUSTOM_EXCLUDES=$(cat $CI_PROJECT_DIR/tools/ci/sonar_exclude_list.txt | grep -v '^#' | xargs | sed -e 's/ /,/g')
-    # Exclude the report dir as well
-    - export EXCLUSIONS="$CUSTOM_EXCLUDES,$SUBMODULES,$REPORT_DIR/**,docs/_static/**,**/*.png,**/*.jpg"
-    - python $NORMALIZE_CLANGTIDY_PY $CI_PROJECT_DIR/$REPORT_DIR/warnings.txt $CI_PROJECT_DIR/$REPORT_DIR/clang_tidy_report.txt $CI_PROJECT_DIR
   variables:
     GIT_DEPTH: 0
-    NORMALIZE_CLANGTIDY_PY: $CI_PROJECT_DIR/tools/ci/normalize_clangtidy_path.py
+    REPORT_PATTERN: clang_tidy_reports/*.txt
-    REPORT_DIR: examples/get-started/hello_world/tidybuild/report
+  artifacts:
+    when: always
+    paths:
+      - $REPORT_PATTERN
   tags:
     - host_test
   dependencies:  # Here is not a hard dependency relationship, could be skipped when only python files changed. so we do not use "needs" here.
     - clang_tidy_check
+    - check_pylint
 
 code_quality_check:
   extends:
@@ -67,23 +93,14 @@ code_quality_check:
     - test -n "$CI_MERGE_REQUEST_COMMITS" || exit 0
     - sonar-scanner
       -Dsonar.analysis.mode=preview
-      -Dsonar.branch.name=$CI_COMMIT_REF_NAME
+      -Dsonar.branch.name=$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME
-      -Dsonar.cxx.clangtidy.reportPath=$REPORT_DIR/clang_tidy_report.txt
+      -Dsonar.cxx.clangtidy.reportPath=$REPORT_PATTERN
-      -Dsonar.cxx.includeDirectories=components,/usr/include
-      -Dsonar.exclusions=$EXCLUSIONS
       -Dsonar.gitlab.ci_merge_request_iid=$CI_MERGE_REQUEST_IID
       -Dsonar.gitlab.commit_sha=$CI_MERGE_REQUEST_COMMITS
-      -Dsonar.gitlab.failure_notification_mode=exit-code
+      -Dsonar.gitlab.ref_name=$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME
-      -Dsonar.gitlab.merge_request_discussion=true
-      -Dsonar.gitlab.project_id=$CI_PROJECT_ID
-      -Dsonar.gitlab.ref_name=$CI_COMMIT_REF_NAME
       -Dsonar.host.url=$SONAR_HOST_URL
       -Dsonar.login=$SONAR_LOGIN
-      -Dsonar.projectBaseDir=$CI_PROJECT_DIR
+      -Dsonar.python.pylint.reportPath=pylint-report.txt
-      -Dsonar.projectKey=esp-idf
-      -Dsonar.python.pylint_config=.pylintrc
-      -Dsonar.sourceEncoding=UTF-8
-      -Dsonar.sources=$CI_PROJECT_DIR
 
 code_quality_report:
   extends:
@@ -94,41 +111,9 @@ code_quality_report:
   script:
     - sonar-scanner
       -Dsonar.branch.name=$CI_COMMIT_REF_NAME
-      -Dsonar.cxx.clangtidy.reportPath=$REPORT_DIR/clang_tidy_report.txt
+      -Dsonar.cxx.clangtidy.reportPath=$REPORT_PATTERN
-      -Dsonar.cxx.includeDirectories=components,/usr/include
-      -Dsonar.exclusions=$EXCLUSIONS
       -Dsonar.gitlab.commit_sha=$CI_COMMIT_SHA
-      -Dsonar.gitlab.failure_notification_mode=exit-code
-      -Dsonar.gitlab.project_id=$CI_PROJECT_ID
       -Dsonar.gitlab.ref_name=$CI_COMMIT_REF_NAME
       -Dsonar.host.url=$SONAR_HOST_URL
       -Dsonar.login=$SONAR_LOGIN
-      -Dsonar.projectBaseDir=$CI_PROJECT_DIR
+      -Dsonar.python.pylint.reportPath=pylint-report.txt
-      -Dsonar.projectKey=esp-idf
-      -Dsonar.python.pylint_config=.pylintrc
-      -Dsonar.sourceEncoding=UTF-8
-      -Dsonar.sources=$CI_PROJECT_DIR
-
-# deploy stage
-clang_tidy_deploy:
-  extends:
-    - .deploy_job_template
-    - .rules:patterns:clang_tidy
-  needs:
-    - clang_tidy_check
-  tags:
-    - deploy
-    - shiny
-  script:
-    - add_doc_server_ssh_keys $DOCS_DEPLOY_KEY $DOCS_SERVER $DOCS_SERVER_USER
-    - export GIT_VER=$(git describe --always)
-    - cd $IDF_PATH/examples/get-started/hello_world/tidybuild
-    - mv report $GIT_VER
-    - tar czvf $GIT_VER.tar.gz $GIT_VER
-    - export STATIC_REPORT_PATH="web/static_analysis/esp-idf/"
-    - ssh $DOCS_SERVER -x "mkdir -p $STATIC_REPORT_PATH/clang-tidy"
-    - scp $GIT_VER.tar.gz $DOCS_SERVER:$STATIC_REPORT_PATH/clang-tidy
-    - ssh $DOCS_SERVER -x "cd $STATIC_REPORT_PATH/clang-tidy && tar xzvf $GIT_VER.tar.gz && rm -f latest && ln -s $GIT_VER latest"
-    # add link to view the report
-    - echo "[static analysis][clang tidy] $CI_DOCKER_REGISTRY/static_analysis/esp-idf/clang-tidy/${GIT_VER}/index.html"
-    - test ! -e ${GIT_VER}/FAILED_RULES || { echo 'Failed static analysis rules!'; cat ${GIT_VER}/FAILED_RULES; exit 1; }

sonar-project.properties

@@ -0,0 +1,37 @@
+sonar.projectKey=esp-idf
+sonar.projectName=Espressif IoT Development Framework
+sonar.projectVersion=4.4
+
+sonar.sources=.
+sonar.sourceEncoding=UTF-8
+
+# exclude list for sonarqube
+# Here're some examples: (relative with the idf_path)
+# test/**
+# test/*/*.c
+sonar.exclusion=\
+  components/freertos/*.c,\
+  components/freertos/include/freertos/*.h,\
+  components/wpa_supplicant/src/**,\
+  components/wpa_supplicant/include/**
+
+sonar.inclusions=\
+  **/*.c,**/*.cpp,**/*.h,**/*.hpp,\
+  **/*.py
+
+sonar.python.pylint_config=.pylintrc
+
+####################
+# sonar-cxx-plugin #
+####################
+
+sonar.cxx.includeDirectories=/usr/include
+sonar.cxx.file.suffixes=.cxx,.cpp,.cc,.c,.hxx,.hpp,.hh,.h
+
+#######################
+# sonar-gitlab-plugin #
+#######################
+
+sonar.gitlab.failure_notification_mode=exit-code
+sonar.gitlab.merge_request_discussion=true
+sonar.gitlab.project_id=103

tools/ci/clang_tidy_dirs.txt

@@ -0,0 +1,2 @@
+examples/get-started/hello_world
+examples/bluetooth/blufi

tools/ci/executable-list.txt

@@ -63,7 +63,6 @@ tools/ci/get-full-sources.sh
 tools/ci/get_supported_examples.sh
 tools/ci/mirror-submodule-update.sh
 tools/ci/multirun_with_pyenv.sh
-tools/ci/normalize_clangtidy_path.py
 tools/ci/push_to_github.sh
 tools/ci/test_autocomplete.py
 tools/ci/test_build_system.sh

tools/ci/normalize_clangtidy_path.py

@@ -1,33 +0,0 @@
-#!/usr/bin/env python
-import argparse
-import re
-from os.path import dirname, exists, join, normpath, relpath
-
-CLANG_TIDY_REGEX = re.compile(r'(.+|[a-zA-Z]:\\\\.+):([0-9]+):([0-9]+): ([^:]+): (.+)')
-
-
-def normalize_clang_tidy_path(file_path, output_path, base_dir):
-    if not exists(file_path):
-        print('Skipping normalizing. This could only happen when skipping clang-tidy check '
-              'because of no c file modified. Please double check')
-        return
-
-    with open(output_path, 'w') as fw:
-        for line in open(file_path):
-            result = CLANG_TIDY_REGEX.match(line)
-            if result:
-                path = result.group(1)
-                abs_path = normpath(join(dirname(file_path), path))
-                rel_path = relpath(abs_path, base_dir)
-                line = line.replace(path, rel_path)
-            fw.write(line)
-
-
-if __name__ == '__main__':
-    parser = argparse.ArgumentParser()
-    parser.add_argument('file', help='clang tidy report path')
-    parser.add_argument('output_file', help='normalized clang tidy report path')
-    parser.add_argument('base_dir', help='relative path base dir')
-    args = parser.parse_args()
-
-    normalize_clang_tidy_path(args.file, args.output_file, args.base_dir)

tools/ci/sonar_exclude_list.txt

@@ -1,15 +0,0 @@
-# exclude list for sonarqube
-# Here're some examples: (relative with the idf_path)
-# test/**
-# test/*/*.c
-# test.c
-#
-# lines start with # will be ignored
-
-# FreeRTOS upstream code (don't include our port files here)
-components/freertos/*.c
-components/freertos/include/freertos/*.h
-
-# wpa_supplicant upstream code
-components/wpa_supplicant/src/**
-components/wpa_supplicant/include/**

tools/ci/static-analysis-rules.yml

@@ -7,17 +7,43 @@ ignore:
   - "llvm-include-order"
 
 skip:
-  - "components/mbedtls/mbedtls"
+  # submodules and third-party code
-  - "components/lwip/lwip"
   - "components/asio/asio"
   - "components/bootloader/subproject/components/micro-ecc/micro-ecc"
+  - "components/bt/controller/lib_esp32"
+  - "components/bt/controller/lib_esp32c3_family"
+  - "components/bt/host/nimble/nimble"
   - "components/bt/lib"
+  - "components/cbor/tinycbor"
+  - "components/cmock/CMock"
   - "components/coap"
+  - "components/esp_phy/lib"
+  - "components/esp_wifi/lib"
   - "components/esp_wifi/lib_esp32"
+  - "components/esptool_py/esptool"
   - "components/expat/expat"
+  - "components/ieee802154/lib"
   - "components/json/cJSON"
   - "components/libsodium/libsodium"
+  - "components/lwip/lwip"
+  - "components/mbedtls/mbedtls"
+  - "components/mqtt/esp-mqtt"
   - "components/nghttp/nghttp2"
+  - "components/openthread/lib"
+  - "components/openthread/openthread"
   - "components/protobuf-c/protobuf-c"
   - "components/spiffs/spiffs"
+  - "components/tinyusb/tinyusb"
   - "components/unity/unity"
+  - "examples/build_system/cmake/import_lib/main/lib/tinyxml2"
+  - "examples/peripherals/secure_element/atecc608_ecdsa/components/esp-cryptoauthlib"
+
+  # disabled temporarily to pass the CI
+  - "components/bt/common/btc/core/btc_task.c"
+  - "components/bt/host/bluedroid/btc/profile/std/gap/btc_gap_ble.c"
+  - "components/bt/host/bluedroid/stack/btm/btm_acl.c"
+  - "components/bt/host/bluedroid/stack/btm/btm_ble_gap.c"
+  - "components/bt/host/bluedroid/stack/btm/btm_dev.c"
+  - "components/bt/host/bluedroid/stack/gatt/att_protocol.c"
+  - "components/bt/host/bluedroid/stack/gatt/gatt_db.c"
+  - "components/wifi_provisioning/src/scheme_ble.c"

tools/ci/utils.sh

@@ -112,3 +112,13 @@ function retry_failed() {
   fi
   return $exitCode
 }
+
+function internal_pip_install() {
+    project=$1
+    package=$2
+    token_name=${3:-${BOT_TOKEN_NAME}}
+    token=${4:-${BOT_TOKEN}}
+    python=${5:-python}
+
+    $python -m pip install --index-url https://${token_name}:${token}@${GITLAB_HTTPS_HOST}/api/v4/projects/${project}/packages/pypi/simple --force-reinstall --no-deps ${package}
+}