ci: add build_non_test_component_apps

.gitlab/ci/build.yml

@@ -93,6 +93,15 @@ build_pytest_components_esp32c3:
   script:
     - run_cmd python tools/ci/build_pytest_apps.py components --target esp32c3 --size-info $SIZE_INFO_LOCATION -vv
 
+build_non_test_components_apps:
+  extends:
+    - .build_template
+    - .build_test_apps_template
+  variables:
+    IDF_TARGET: all
+    TEST_PREFIX: component_ut
+    TEST_RELATIVE_DIR: component_ut
+
 .build_template_app_template:
   extends: .build_template
   variables:

.gitlab/ci/pre_check.yml

@@ -151,11 +151,13 @@ scan_tests:
     paths:
       - $EXAMPLE_TEST_OUTPUT_DIR
       - $TEST_APPS_OUTPUT_DIR
+      - $COMPONENT_UT_OUTPUT_DIR
   variables:
     EXAMPLE_TEST_DIR: ${CI_PROJECT_DIR}/examples
     EXAMPLE_TEST_OUTPUT_DIR: ${CI_PROJECT_DIR}/examples/test_configs
     TEST_APPS_TEST_DIR: ${CI_PROJECT_DIR}/tools/test_apps
     TEST_APPS_OUTPUT_DIR: ${CI_PROJECT_DIR}/tools/test_apps/test_configs
+    COMPONENT_UT_OUTPUT_DIR: ${CI_PROJECT_DIR}/component_ut/test_configs
     CI_SCAN_TESTS_PY: ${CI_PROJECT_DIR}/tools/ci/python_packages/ttfw_idf/CIScanTests.py
     EXTRA_TEST_DIRS: >-
       examples/bluetooth/esp_ble_mesh/ble_mesh_console
@@ -164,6 +166,8 @@ scan_tests:
   script:
     - run_cmd python $CI_SCAN_TESTS_PY example_test $EXAMPLE_TEST_DIR -b cmake --exclude examples/build_system/idf_as_lib -c $CI_TARGET_TEST_CONFIG_FILE -o $EXAMPLE_TEST_OUTPUT_DIR --extra_test_dirs $EXTRA_TEST_DIRS
     - run_cmd python $CI_SCAN_TESTS_PY test_apps $TEST_APPS_TEST_DIR -c $CI_TARGET_TEST_CONFIG_FILE -o $TEST_APPS_OUTPUT_DIR
+    - set_component_ut_vars
+    - run_cmd python $CI_SCAN_TESTS_PY component_ut $COMPONENT_UT_DIRS --exclude $COMPONENT_UT_EXCLUDES -c $CI_TARGET_TEST_CONFIG_FILE -o $COMPONENT_UT_OUTPUT_DIR --combine-all-targets --except-targets linux
 
 # For release tag pipelines only, make sure the tag was created with 'git tag -a' so it will update
 # the version returned by 'git describe'

tools/ci/python_packages/ttfw_idf/CIScanTests.py

@@ -97,6 +97,10 @@ def main():  # type: () -> None
                         help='add this flag to preserve artifacts for all apps')
     parser.add_argument('--build-all', action='store_true',
                         help='add this flag to build all apps')
+    parser.add_argument('--combine-all-targets', action='store_true',
+                        help='add this flag to combine all target jsons into one')
+    parser.add_argument('--except-targets', nargs='+',
+                        help='only useful when "--combine-all-targets". Specified targets would be skipped.')
 
     args = parser.parse_args()
     build_test_case_apps, build_standalone_apps = _judge_build_or_not(args.test_type, args.build_all)
@@ -170,6 +174,7 @@ def main():  # type: () -> None
         else:
             scan_info_dict[target]['standalone_apps'] = set()
     test_case_apps_preserve_default = True if build_system == 'cmake' else False
+    output_files = []
     for target in SUPPORTED_TARGETS:
         # get pytest apps paths
         pytest_app_paths = set()
@@ -201,6 +206,24 @@ def main():  # type: () -> None
         with open(output_path, 'w') as fw:
             fw.writelines([json.dumps(app) + '\n' for app in apps])
 
+        if args.combine_all_targets:
+            if (args.except_targets and target not in [t.lower() for t in args.except_targets]) \
+                    or (not args.except_targets):
+                output_files.append(output_path)
+                build_items_total_count += len(build_items)
+            else:
+                print(f'skipping combining target {target}')
+
+    if args.combine_all_targets:
+        scan_all_json = os.path.join(args.output_path, f'scan_all_{build_system}.json')
+        lines = []
+        for file in output_files:
+            with open(file) as fr:
+                lines.extend([line for line in fr.readlines() if line.strip()])
+        with open(scan_all_json, 'w') as fw:
+            fw.writelines(lines)
+        print(f'combined into file: {scan_all_json}')
+
 
 if __name__ == '__main__':
     main()

tools/ci/utils.sh

@@ -42,6 +42,13 @@ function get_all_submodules() {
   git config --file .gitmodules --get-regexp path | awk '{ print $2 }' | sed -e 's|$|/**|' | xargs | sed -e 's/ /,/g'
 }
 
+function set_component_ut_vars() {
+  local exclude_list_fp="${IDF_PATH}/tools/ci/component_ut_excludes.txt"
+  export COMPONENT_UT_DIRS=$(find components/ -name test_apps -type d | xargs)
+  export COMPONENT_UT_EXCLUDES=$([ -r $exclude_list_fp ] && cat $exclude_list_fp | xargs)
+  echo "exported variables COMPONENT_UT_DIRS, COMPONENT_UT_EXCLUDES"
+}
+
 function error() {
   printf "\033[0;31m%s\n\033[0m" "${1}" >&2
 }

tools/find_build_apps/cmake.py

@@ -63,6 +63,7 @@ class CMakeBuildSystem(BuildSystem):
             build_stderr = log_file
 
         try:
+            os.environ['IDF_TARGET'] = build_item.target
             subprocess.check_call(args, stdout=build_stdout, stderr=build_stderr)
         except subprocess.CalledProcessError as e:
             raise BuildError('Build failed with exit code {}'.format(e.returncode))