diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4a57cdb4c6..3037b3da1d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -51,7 +51,7 @@ variables: ESP_IDF_DOC_ENV_IMAGE: "$CI_DOCKER_REGISTRY/esp-idf-doc-env:v4.4-1-v2" ESP_ENV_IMAGE: "$CI_DOCKER_REGISTRY/esp-env:v4.4-1" AFL_FUZZER_TEST_IMAGE: "$CI_DOCKER_REGISTRY/afl-fuzzer-test:v4.4-1-1" - CLANG_STATIC_ANALYSIS_IMAGE: "${CI_DOCKER_REGISTRY}/clang-static-analysis:v4.4-1-1" + CLANG_STATIC_ANALYSIS_IMAGE: "${CI_DOCKER_REGISTRY}/clang-static-analysis:v4.4-1-2" # target test config file, used by assign test job CI_TARGET_TEST_CONFIG_FILE: "$CI_PROJECT_DIR/.gitlab/ci/target-test.yml" diff --git a/.gitlab/CODEOWNERS b/.gitlab/CODEOWNERS index 123538a09b..07f880ebd4 100644 --- a/.gitlab/CODEOWNERS +++ b/.gitlab/CODEOWNERS @@ -57,6 +57,7 @@ /export.* @esp-idf-codeowners/tools /install.* @esp-idf-codeowners/tools /sdkconfig.rename @esp-idf-codeowners/build-config +/sonar-project.properties @esp-idf-codeowners/ci # sort-order-reset diff --git a/.gitlab/ci/rules.yml b/.gitlab/ci/rules.yml index 172563df51..d9e8056d1c 100644 --- a/.gitlab/ci/rules.yml +++ b/.gitlab/ci/rules.yml @@ -3,7 +3,10 @@ ############ .patterns-c-files: &patterns-c-files - ".gitlab/ci/static-code-analysis.yml" + - "tools/ci/static-analysis-rules.yml" + - "tools/ci/clang_tidy_dirs.txt" + - "**/*.{c,C}" - "**/*.{h,H}" - "components/**/Kconfig" diff --git a/.gitlab/ci/static-code-analysis.yml b/.gitlab/ci/static-code-analysis.yml index ecf0c319ef..74a7f67bb8 100644 --- a/.gitlab/ci/static-code-analysis.yml +++ b/.gitlab/ci/static-code-analysis.yml @@ -5,17 +5,46 @@ clang_tidy_check: - .rules:patterns:clang_tidy image: ${CLANG_STATIC_ANALYSIS_IMAGE} artifacts: - reports: - junit: $IDF_PATH/output.xml + paths: + - $OUTPUT_DIR + when: always + expire_in: 1 day + variables: + CLANG_TIDY_RUNNER_PROJ: 2107 # idf/clang-tidy-runner + CLANG_TIDY_DIRS_TXT: ${CI_PROJECT_DIR}/tools/ci/clang_tidy_dirs.txt + RULES_FILE: ${CI_PROJECT_DIR}/tools/ci/static-analysis-rules.yml + OUTPUT_DIR: ${CI_PROJECT_DIR}/clang_tidy_reports + script: + - python -m pip install -U pip + - internal_pip_install $CLANG_TIDY_RUNNER_PROJ pyclang + - export PATH=$PATH:$(python -c "import sys; print(sys.executable.rsplit('/', 1)[0])") + - dirs=$(cat ${CLANG_TIDY_DIRS_TXT} | while read line; do echo ${CI_PROJECT_DIR}/${line}; done | xargs) + - run_cmd idf_clang ${dirs} + --output-path ${OUTPUT_DIR} + --limit-file ${RULES_FILE} + --xtensa-include-dir + --run-clang-tidy-py ${RUN_CLANG_TIDY_PY} + +check_pylint: + extends: + - .pre_check_base_template + - .rules:patterns:python-files + - .before_script_minimal + image: $CI_DOCKER_REGISTRY/sonarqube-scanner:2 + artifacts: when: always paths: - - $IDF_PATH/examples/get-started/hello_world/tidybuild/report/* - expire_in: 1 day + - pylint-report.txt + expire_in: 1 week script: - - retry_failed git clone $IDF_ANALYSIS_UTILS static_analysis_utils && cd static_analysis_utils - # Setup parameters of triggered/regular job - - export TARGET_BRANCH=${BOT_CUSTOMIZED_REVISION-} - - ./analyze.sh $IDF_PATH/examples/get-started/hello_world/ $IDF_PATH/tools/ci/static-analysis-rules.yml $IDF_PATH/output.xml + - export PYTHONPATH="$IDF_PATH/tools:$IDF_PATH/tools/ci/python_packages:$PYTHONPATH" + - | + if [ -n "$CI_MERGE_REQUEST_IID" ]; then + export files=$(python ${CI_PROJECT_DIR}/tools/ci/ci_get_mr_info.py files ${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME} | grep ".py"); + else + export files=$(find . -iname "*.py" -print); + fi + - pylint --rcfile=.pylintrc $files -r n --output-format=parseable > pylint-report.txt || exit 0 # build stage # Sonarqube related jobs put here for this reason: @@ -38,21 +67,18 @@ clang_tidy_check: - source tools/ci/utils.sh - export PYTHONPATH="$CI_PROJECT_DIR/tools:$CI_PROJECT_DIR/tools/ci/python_packages:$PYTHONPATH" - fetch_submodules - # Exclude the submodules, all paths ends with /** - - export SUBMODULES=$(get_all_submodules) - # get all exclude paths specified in tools/ci/sonar_exclude_list.txt | ignore lines start with # | xargs | replace all to - - export CUSTOM_EXCLUDES=$(cat $CI_PROJECT_DIR/tools/ci/sonar_exclude_list.txt | grep -v '^#' | xargs | sed -e 's/ /,/g') - # Exclude the report dir as well - - export EXCLUSIONS="$CUSTOM_EXCLUDES,$SUBMODULES,$REPORT_DIR/**,docs/_static/**,**/*.png,**/*.jpg" - - python $NORMALIZE_CLANGTIDY_PY $CI_PROJECT_DIR/$REPORT_DIR/warnings.txt $CI_PROJECT_DIR/$REPORT_DIR/clang_tidy_report.txt $CI_PROJECT_DIR variables: GIT_DEPTH: 0 - NORMALIZE_CLANGTIDY_PY: $CI_PROJECT_DIR/tools/ci/normalize_clangtidy_path.py - REPORT_DIR: examples/get-started/hello_world/tidybuild/report + REPORT_PATTERN: clang_tidy_reports/*.txt + artifacts: + when: always + paths: + - $REPORT_PATTERN tags: - host_test dependencies: # Here is not a hard dependency relationship, could be skipped when only python files changed. so we do not use "needs" here. - clang_tidy_check + - check_pylint code_quality_check: extends: @@ -67,23 +93,14 @@ code_quality_check: - test -n "$CI_MERGE_REQUEST_COMMITS" || exit 0 - sonar-scanner -Dsonar.analysis.mode=preview - -Dsonar.branch.name=$CI_COMMIT_REF_NAME - -Dsonar.cxx.clangtidy.reportPath=$REPORT_DIR/clang_tidy_report.txt - -Dsonar.cxx.includeDirectories=components,/usr/include - -Dsonar.exclusions=$EXCLUSIONS + -Dsonar.branch.name=$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME + -Dsonar.cxx.clangtidy.reportPath=$REPORT_PATTERN -Dsonar.gitlab.ci_merge_request_iid=$CI_MERGE_REQUEST_IID -Dsonar.gitlab.commit_sha=$CI_MERGE_REQUEST_COMMITS - -Dsonar.gitlab.failure_notification_mode=exit-code - -Dsonar.gitlab.merge_request_discussion=true - -Dsonar.gitlab.project_id=$CI_PROJECT_ID - -Dsonar.gitlab.ref_name=$CI_COMMIT_REF_NAME + -Dsonar.gitlab.ref_name=$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME -Dsonar.host.url=$SONAR_HOST_URL -Dsonar.login=$SONAR_LOGIN - -Dsonar.projectBaseDir=$CI_PROJECT_DIR - -Dsonar.projectKey=esp-idf - -Dsonar.python.pylint_config=.pylintrc - -Dsonar.sourceEncoding=UTF-8 - -Dsonar.sources=$CI_PROJECT_DIR + -Dsonar.python.pylint.reportPath=pylint-report.txt code_quality_report: extends: @@ -94,41 +111,9 @@ code_quality_report: script: - sonar-scanner -Dsonar.branch.name=$CI_COMMIT_REF_NAME - -Dsonar.cxx.clangtidy.reportPath=$REPORT_DIR/clang_tidy_report.txt - -Dsonar.cxx.includeDirectories=components,/usr/include - -Dsonar.exclusions=$EXCLUSIONS + -Dsonar.cxx.clangtidy.reportPath=$REPORT_PATTERN -Dsonar.gitlab.commit_sha=$CI_COMMIT_SHA - -Dsonar.gitlab.failure_notification_mode=exit-code - -Dsonar.gitlab.project_id=$CI_PROJECT_ID -Dsonar.gitlab.ref_name=$CI_COMMIT_REF_NAME -Dsonar.host.url=$SONAR_HOST_URL -Dsonar.login=$SONAR_LOGIN - -Dsonar.projectBaseDir=$CI_PROJECT_DIR - -Dsonar.projectKey=esp-idf - -Dsonar.python.pylint_config=.pylintrc - -Dsonar.sourceEncoding=UTF-8 - -Dsonar.sources=$CI_PROJECT_DIR - -# deploy stage -clang_tidy_deploy: - extends: - - .deploy_job_template - - .rules:patterns:clang_tidy - needs: - - clang_tidy_check - tags: - - deploy - - shiny - script: - - add_doc_server_ssh_keys $DOCS_DEPLOY_KEY $DOCS_SERVER $DOCS_SERVER_USER - - export GIT_VER=$(git describe --always) - - cd $IDF_PATH/examples/get-started/hello_world/tidybuild - - mv report $GIT_VER - - tar czvf $GIT_VER.tar.gz $GIT_VER - - export STATIC_REPORT_PATH="web/static_analysis/esp-idf/" - - ssh $DOCS_SERVER -x "mkdir -p $STATIC_REPORT_PATH/clang-tidy" - - scp $GIT_VER.tar.gz $DOCS_SERVER:$STATIC_REPORT_PATH/clang-tidy - - ssh $DOCS_SERVER -x "cd $STATIC_REPORT_PATH/clang-tidy && tar xzvf $GIT_VER.tar.gz && rm -f latest && ln -s $GIT_VER latest" - # add link to view the report - - echo "[static analysis][clang tidy] $CI_DOCKER_REGISTRY/static_analysis/esp-idf/clang-tidy/${GIT_VER}/index.html" - - test ! -e ${GIT_VER}/FAILED_RULES || { echo 'Failed static analysis rules!'; cat ${GIT_VER}/FAILED_RULES; exit 1; } + -Dsonar.python.pylint.reportPath=pylint-report.txt diff --git a/sonar-project.properties b/sonar-project.properties new file mode 100644 index 0000000000..164c54522c --- /dev/null +++ b/sonar-project.properties @@ -0,0 +1,37 @@ +sonar.projectKey=esp-idf +sonar.projectName=Espressif IoT Development Framework +sonar.projectVersion=4.4 + +sonar.sources=. +sonar.sourceEncoding=UTF-8 + +# exclude list for sonarqube +# Here're some examples: (relative with the idf_path) +# test/** +# test/*/*.c +sonar.exclusion=\ + components/freertos/*.c,\ + components/freertos/include/freertos/*.h,\ + components/wpa_supplicant/src/**,\ + components/wpa_supplicant/include/** + +sonar.inclusions=\ + **/*.c,**/*.cpp,**/*.h,**/*.hpp,\ + **/*.py + +sonar.python.pylint_config=.pylintrc + +#################### +# sonar-cxx-plugin # +#################### + +sonar.cxx.includeDirectories=/usr/include +sonar.cxx.file.suffixes=.cxx,.cpp,.cc,.c,.hxx,.hpp,.hh,.h + +####################### +# sonar-gitlab-plugin # +####################### + +sonar.gitlab.failure_notification_mode=exit-code +sonar.gitlab.merge_request_discussion=true +sonar.gitlab.project_id=103 diff --git a/tools/ci/clang_tidy_dirs.txt b/tools/ci/clang_tidy_dirs.txt new file mode 100644 index 0000000000..e89ef79590 --- /dev/null +++ b/tools/ci/clang_tidy_dirs.txt @@ -0,0 +1,2 @@ +examples/get-started/hello_world +examples/bluetooth/blufi diff --git a/tools/ci/executable-list.txt b/tools/ci/executable-list.txt index 00dff0ed42..2fe79b6f1b 100644 --- a/tools/ci/executable-list.txt +++ b/tools/ci/executable-list.txt @@ -63,7 +63,6 @@ tools/ci/get-full-sources.sh tools/ci/get_supported_examples.sh tools/ci/mirror-submodule-update.sh tools/ci/multirun_with_pyenv.sh -tools/ci/normalize_clangtidy_path.py tools/ci/push_to_github.sh tools/ci/test_autocomplete.py tools/ci/test_build_system.sh diff --git a/tools/ci/normalize_clangtidy_path.py b/tools/ci/normalize_clangtidy_path.py deleted file mode 100755 index f6998554fb..0000000000 --- a/tools/ci/normalize_clangtidy_path.py +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env python -import argparse -import re -from os.path import dirname, exists, join, normpath, relpath - -CLANG_TIDY_REGEX = re.compile(r'(.+|[a-zA-Z]:\\\\.+):([0-9]+):([0-9]+): ([^:]+): (.+)') - - -def normalize_clang_tidy_path(file_path, output_path, base_dir): - if not exists(file_path): - print('Skipping normalizing. This could only happen when skipping clang-tidy check ' - 'because of no c file modified. Please double check') - return - - with open(output_path, 'w') as fw: - for line in open(file_path): - result = CLANG_TIDY_REGEX.match(line) - if result: - path = result.group(1) - abs_path = normpath(join(dirname(file_path), path)) - rel_path = relpath(abs_path, base_dir) - line = line.replace(path, rel_path) - fw.write(line) - - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - parser.add_argument('file', help='clang tidy report path') - parser.add_argument('output_file', help='normalized clang tidy report path') - parser.add_argument('base_dir', help='relative path base dir') - args = parser.parse_args() - - normalize_clang_tidy_path(args.file, args.output_file, args.base_dir) diff --git a/tools/ci/sonar_exclude_list.txt b/tools/ci/sonar_exclude_list.txt index a91e445f52..e69de29bb2 100644 --- a/tools/ci/sonar_exclude_list.txt +++ b/tools/ci/sonar_exclude_list.txt @@ -1,15 +0,0 @@ -# exclude list for sonarqube -# Here're some examples: (relative with the idf_path) -# test/** -# test/*/*.c -# test.c -# -# lines start with # will be ignored - -# FreeRTOS upstream code (don't include our port files here) -components/freertos/*.c -components/freertos/include/freertos/*.h - -# wpa_supplicant upstream code -components/wpa_supplicant/src/** -components/wpa_supplicant/include/** diff --git a/tools/ci/static-analysis-rules.yml b/tools/ci/static-analysis-rules.yml index d96b0a0aa9..0b2f3603c2 100644 --- a/tools/ci/static-analysis-rules.yml +++ b/tools/ci/static-analysis-rules.yml @@ -7,17 +7,43 @@ ignore: - "llvm-include-order" skip: - - "components/mbedtls/mbedtls" - - "components/lwip/lwip" + # submodules and third-party code - "components/asio/asio" - "components/bootloader/subproject/components/micro-ecc/micro-ecc" + - "components/bt/controller/lib_esp32" + - "components/bt/controller/lib_esp32c3_family" + - "components/bt/host/nimble/nimble" - "components/bt/lib" + - "components/cbor/tinycbor" + - "components/cmock/CMock" - "components/coap" + - "components/esp_phy/lib" + - "components/esp_wifi/lib" - "components/esp_wifi/lib_esp32" + - "components/esptool_py/esptool" - "components/expat/expat" + - "components/ieee802154/lib" - "components/json/cJSON" - "components/libsodium/libsodium" + - "components/lwip/lwip" + - "components/mbedtls/mbedtls" + - "components/mqtt/esp-mqtt" - "components/nghttp/nghttp2" + - "components/openthread/lib" + - "components/openthread/openthread" - "components/protobuf-c/protobuf-c" - "components/spiffs/spiffs" + - "components/tinyusb/tinyusb" - "components/unity/unity" + - "examples/build_system/cmake/import_lib/main/lib/tinyxml2" + - "examples/peripherals/secure_element/atecc608_ecdsa/components/esp-cryptoauthlib" + + # disabled temporarily to pass the CI + - "components/bt/common/btc/core/btc_task.c" + - "components/bt/host/bluedroid/btc/profile/std/gap/btc_gap_ble.c" + - "components/bt/host/bluedroid/stack/btm/btm_acl.c" + - "components/bt/host/bluedroid/stack/btm/btm_ble_gap.c" + - "components/bt/host/bluedroid/stack/btm/btm_dev.c" + - "components/bt/host/bluedroid/stack/gatt/att_protocol.c" + - "components/bt/host/bluedroid/stack/gatt/gatt_db.c" + - "components/wifi_provisioning/src/scheme_ble.c" diff --git a/tools/ci/utils.sh b/tools/ci/utils.sh index 2d6b4a8ac4..3db3537224 100644 --- a/tools/ci/utils.sh +++ b/tools/ci/utils.sh @@ -112,3 +112,13 @@ function retry_failed() { fi return $exitCode } + +function internal_pip_install() { + project=$1 + package=$2 + token_name=${3:-${BOT_TOKEN_NAME}} + token=${4:-${BOT_TOKEN}} + python=${5:-python} + + $python -m pip install --index-url https://${token_name}:${token}@${GITLAB_HTTPS_HOST}/api/v4/projects/${project}/packages/pypi/simple --force-reinstall --no-deps ${package} +}