CI: assign and target-test stages updated to run test-apps in the ci

tools/ci/build_test_apps.sh

@@ -57,7 +57,7 @@ echo "build_examples running for target $IDF_TARGET"
 
 cd ${IDF_PATH}
 
-# This part of the script produces the same result for all the example build jobs. It may be moved to a separate stage
+# This part of the script produces the same result for all the test app build jobs. It may be moved to a separate stage
 # (pre-build) later, then the build jobs will receive ${BUILD_LIST_JSON} file as an artifact.
 
 # If changing the work-dir or build-dir, remember to update the "artifacts" in gitlab-ci configs, and IDFApp.py.
@@ -68,7 +68,8 @@ ${IDF_PATH}/tools/find_apps.py tools/test_apps \
     --build-system cmake \
     --target ${IDF_TARGET} \
     --recursive \
-    --build-dir "\${IDF_PATH}/${BUILD_PATH}/@f/@w/@t/build" \
+    --work-dir "${BUILD_PATH}/@f/@w/@t" \
+    --build-dir build \
     --build-log "${LOG_PATH}/@f.txt" \
     --output ${ALL_BUILD_LIST_JSON} \
     --config 'sdkconfig.ci=default' \
@@ -79,6 +80,11 @@ ${IDF_PATH}/tools/find_apps.py tools/test_apps \
 # 1. If sdkconfig.ci exists, use it build the example with configuration name "default"
 # 2. If sdkconfig.ci.* exists, use it to build the "*" configuration
 # 3. If none of the above exist, build the default configuration under the name "default"
+# --work-dir and --build-log above uses "placeholders" @x:
+# - @f: full path to the test with slashes replaced with underscores
+# - @w: wildcard used as config name
+# - @t: target name
+# so the workdir .../@f/@w/@t would expand to e.g. tools_test_apps_system_startup/default/esp32
 
 # The part below is where the actual builds happen
 

tools/ci/config/assign-test.yml

@@ -9,7 +9,6 @@ assign_test:
   dependencies:
     - build_ssc_esp32
     - build_esp_idf_tests_cmake
-    - build_test_apps_esp32
   variables:
     SUBMODULES_TO_FETCH: "components/esptool_py/esptool"
     EXAMPLE_CONFIG_OUTPUT_PATH: "$CI_PROJECT_DIR/examples/test_configs"
@@ -22,6 +21,7 @@ assign_test:
       - $EXAMPLE_CONFIG_OUTPUT_PATH
       - $TEST_APP_CONFIG_OUTPUT_PATH
       - build_examples/artifact_index.json
+      - build_test_apps/artifact_index.json
     expire_in: 1 week
   only:
     variables:
@@ -29,11 +29,12 @@ assign_test:
       - $BOT_LABEL_UNIT_TEST
       - $BOT_LABEL_INTEGRATION_TEST
       - $BOT_LABEL_EXAMPLE_TEST
+      - $BOT_LABEL_CUSTOM_TEST
   script:
     # assign example tests
     - python tools/ci/python_packages/ttfw_idf/CIAssignExampleTest.py $IDF_PATH/examples $CI_TARGET_TEST_CONFIG_FILE $EXAMPLE_CONFIG_OUTPUT_PATH
     # assign test apps
-    - python tools/ci/python_packages/ttfw_idf/CIAssignExampleTest.py --job-prefix test_app_test_ $IDF_PATH/tools/test_apps $CI_TARGET_TEST_CONFIG_FILE $TEST_APP_CONFIG_OUTPUT_PATH
+    - python tools/ci/python_packages/ttfw_idf/CIAssignExampleTest.py --custom-group test-apps --job-prefix test_app_test_ $IDF_PATH/tools/test_apps $CI_TARGET_TEST_CONFIG_FILE $TEST_APP_CONFIG_OUTPUT_PATH
     # assign unit test cases
     - python tools/ci/python_packages/ttfw_idf/CIAssignUnitTest.py $UNIT_TEST_CASE_FILE $CI_TARGET_TEST_CONFIG_FILE $IDF_PATH/components/idf_test/unit_test/CIConfigs
     # clone test script to assign tests

tools/ci/config/build.yml

@@ -186,8 +186,7 @@ build_examples_cmake_esp32s2:
 
 .build_test_apps: &build_test_apps
   extends: .build_template
-  parallel: 2
+  stage: build
-  stage: pre_build
   artifacts:
     when: always
     paths:
@@ -225,7 +224,7 @@ build_test_apps_esp32:
 build_test_apps_esp32s2:
   extends: .build_test_apps
   variables:
-    IDF_TARGET: esp32s2beta
+    IDF_TARGET: esp32s2
 
 
 # If you want to add new build example jobs, please add it into dependencies of `.example_test_template`

tools/ci/config/target-test.yml

@@ -85,27 +85,26 @@
 
 .test_app_template:
   extends: .example_test_template
-  stage: pre_target_test
+  stage: target_test
   dependencies:
     - assign_test
-    - build_test_apps_esp32
+  only:
+    refs:
+      - master
+      - /^release\/v/
+      - /^v\d+\.\d+(\.\d+)?($|-)/
+      - triggers
+      - schedules
+    variables:
+      - $BOT_TRIGGER_WITH_LABEL == null
+      - $BOT_LABEL_CUSTOM_TEST
+      - $BOT_LABEL_EXAMPLE_TEST
   variables:
     TEST_FW_PATH: "$CI_PROJECT_DIR/tools/tiny-test-fw"
     TEST_CASE_PATH: "$CI_PROJECT_DIR/tools/test_apps"
     CONFIG_FILE_PATH: "${CI_PROJECT_DIR}/tools/test_apps/test_configs"
     LOG_PATH: "$CI_PROJECT_DIR/TEST_LOGS"
     ENV_FILE: "$CI_PROJECT_DIR/ci-test-runner-configs/$CI_RUNNER_DESCRIPTION/EnvConfig.yml"
-  script:
-    - *define_config_file_name
-    # first test if config file exists, if not exist, exit 0
-    - test -e $CONFIG_FILE || exit 0
-    # clone test env configs
-    - git clone $TEST_ENV_CONFIG_REPOSITORY
-    - python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs
-    - cd $TEST_FW_PATH
-    # run test
-    - python Runner.py $TEST_CASE_PATH -c $CONFIG_FILE -e $ENV_FILE
-
 
 .unit_test_template:
   extends: .example_test_template

tools/ci/python_packages/ttfw_idf/CIAssignExampleTest.py

@@ -25,25 +25,34 @@ import json
 import gitlab_api
 from tiny_test_fw.Utility import CIAssignTest
 
-
-EXAMPLE_BUILD_JOB_NAMES = ["build_examples_cmake_esp32", "build_examples_cmake_esp32s2"]
 IDF_PATH_FROM_ENV = os.getenv("IDF_PATH")
-if IDF_PATH_FROM_ENV:
-    ARTIFACT_INDEX_FILE = os.path.join(IDF_PATH_FROM_ENV,
-                                       "build_examples", "artifact_index.json")
-else:
-    ARTIFACT_INDEX_FILE = "artifact_index.json"
 
 
 class ExampleGroup(CIAssignTest.Group):
     SORT_KEYS = CI_JOB_MATCH_KEYS = ["env_tag", "chip"]
+    BUILD_LOCAL_DIR = "build_examples"
+    BUILD_JOB_NAMES = ["build_examples_cmake_esp32", "build_examples_cmake_esp32s2"]
+
+
+class TestAppsGroup(ExampleGroup):
+    BUILD_LOCAL_DIR = "build_test_apps"
+    BUILD_JOB_NAMES = ["build_test_apps_esp32", "build_test_apps_esp32s2"]
 
 
 class CIExampleAssignTest(CIAssignTest.AssignTest):
     CI_TEST_JOB_PATTERN = re.compile(r"^example_test_.+")
 
 
-def create_artifact_index_file(project_id=None, pipeline_id=None):
+def get_artifact_index_file(case_group=ExampleGroup):
+    if IDF_PATH_FROM_ENV:
+        artifact_index_file = os.path.join(IDF_PATH_FROM_ENV,
+                                           case_group.BUILD_LOCAL_DIR, "artifact_index.json")
+    else:
+        artifact_index_file = "artifact_index.json"
+    return artifact_index_file
+
+
+def create_artifact_index_file(project_id=None, pipeline_id=None, case_group=ExampleGroup):
     if project_id is None:
         project_id = os.getenv("CI_PROJECT_ID")
     if pipeline_id is None:
@@ -52,9 +61,10 @@ def create_artifact_index_file(project_id=None, pipeline_id=None):
     artifact_index_list = []
 
     def format_build_log_path():
-        return "build_examples/list_job_{}.json".format(job_info["parallel_num"])
+        parallel = job_info["parallel_num"]    # Could be None if "parallel_num" not defined for the job
+        return "{}/list_job_{}.json".format(case_group.BUILD_LOCAL_DIR, parallel or 1)
 
-    for build_job_name in EXAMPLE_BUILD_JOB_NAMES:
+    for build_job_name in case_group.BUILD_JOB_NAMES:
         job_info_list = gitlab_inst.find_job_id(build_job_name, pipeline_id=pipeline_id)
         for job_info in job_info_list:
             raw_data = gitlab_inst.download_artifact(job_info["id"], [format_build_log_path()])[0]
@@ -62,13 +72,14 @@ def create_artifact_index_file(project_id=None, pipeline_id=None):
             for build_info in build_info_list:
                 build_info["ci_job_id"] = job_info["id"]
                 artifact_index_list.append(build_info)
+    artifact_index_file = get_artifact_index_file(case_group=case_group)
     try:
-        os.makedirs(os.path.dirname(ARTIFACT_INDEX_FILE))
+        os.makedirs(os.path.dirname(artifact_index_file))
     except OSError:
         # already created
         pass
 
-    with open(ARTIFACT_INDEX_FILE, "w") as f:
+    with open(artifact_index_file, "w") as f:
         json.dump(artifact_index_list, f)
 
 
@@ -86,16 +97,18 @@ if __name__ == '__main__':
                         help="prefix of the test job name in CI yml file")
     parser.add_argument("--test-case-file-pattern",
                         help="file name pattern used to find Python test case files")
+    parser.add_argument('--custom-group',
+                        help='select custom-group for the test cases, if other than ExampleTest',
+                        choices=['example','test-apps'], default='example')
+
     args = parser.parse_args()
 
     if args.job_prefix:
         CIExampleAssignTest.CI_TEST_JOB_PATTERN = re.compile(r"^{}.+".format(args.job_prefix))
 
-    assign_test = CIExampleAssignTest(args.test_case, args.ci_config_file, case_group=ExampleGroup)
+    case_group = ExampleGroup if args.custom_group == 'example' else TestAppsGroup
-    if args.test_case_file_pattern:
-        assign_test.test_case_file_pattern = args.test_case_file_pattern
 
-    assign_test = CIExampleAssignTest(args.test_case, args.ci_config_file, case_group=ExampleGroup)
+    assign_test = CIExampleAssignTest(args.test_case, args.ci_config_file, case_group=case_group)
     assign_test.assign_cases()
     assign_test.output_configs(args.output_path)
-    create_artifact_index_file()
+    create_artifact_index_file(case_group=case_group)

tools/ci/python_packages/ttfw_idf/CIAssignUnitTest.py

@@ -139,7 +139,7 @@ class UnitTestAssignTest(CIAssignTest.AssignTest):
     def __init__(self, test_case_path, ci_config_file):
         CIAssignTest.AssignTest.__init__(self, test_case_path, ci_config_file, case_group=Group)
 
-    def _search_cases(self, test_case_path, case_filter=None):
+    def _search_cases(self, test_case_path, case_filter=None, test_case_file_pattern=None):
         """
         For unit test case, we don't search for test functions.
         The unit test cases is stored in a yaml file which is created in job build-idf-test.

tools/ci/python_packages/ttfw_idf/IDFApp.py

@@ -310,7 +310,7 @@ class Example(IDFApp):
         """
         return [os.path.join(self.binary_path, "..", "sdkconfig")]
 
-    def _try_get_binary_from_local_fs(self, app_path, config_name=None, target=None):
+    def _try_get_binary_from_local_fs(self, app_path, config_name=None, target=None, local_build_dir="build_examples"):
         # build folder of example path
         path = os.path.join(self.idf_path, app_path, "build")
         if os.path.exists(path):
@@ -327,7 +327,7 @@ class Example(IDFApp):
         # (see tools/ci/build_examples_cmake.sh)
         # For example: $IDF_PATH/build_examples/examples_get-started_blink/default/esp32
         app_path_underscored = app_path.replace(os.path.sep, "_")
-        example_path = os.path.join(self.idf_path, "build_examples")
+        example_path = os.path.join(self.idf_path, local_build_dir)
         for dirpath in os.listdir(example_path):
             if os.path.basename(dirpath) == app_path_underscored:
                 path = os.path.join(example_path, dirpath, config_name, target, "build")
@@ -341,7 +341,8 @@ class Example(IDFApp):
         if path:
             return path
         else:
-            artifacts = Artifacts(self.idf_path, CIAssignExampleTest.ARTIFACT_INDEX_FILE,
+            artifacts = Artifacts(self.idf_path,
+                                  CIAssignExampleTest.get_artifact_index_file(case_group=CIAssignExampleTest.ExampleGroup),
                                   app_path, config_name, target)
             path = artifacts.download_artifacts()
             if path:
@@ -369,7 +370,8 @@ class LoadableElfExample(Example):
         if path:
             return path
         else:
-            artifacts = Artifacts(self.idf_path, CIAssignExampleTest.ARTIFACT_INDEX_FILE,
+            artifacts = Artifacts(self.idf_path,
+                                  CIAssignExampleTest.get_artifact_index_file(case_group=CIAssignExampleTest.ExampleGroup),
                                   app_path, config_name, target)
             path = artifacts.download_artifact_files(self.app_files)
             if path:
@@ -402,34 +404,20 @@ class UT(IDFApp):
         raise OSError("Failed to get unit-test-app binary path")
 
 
-class TestApp(IDFApp):
+class TestApp(Example):
-    def _get_sdkconfig_paths(self):
+    def get_binary_path(self, app_path, config_name=None, target=None):
-        """
+        path = self._try_get_binary_from_local_fs(app_path, config_name, target, local_build_dir="build_test_apps")
-        overrides the parent method to provide exact path of sdkconfig for example tests
+        if path:
-        """
-        return [os.path.join(self.binary_path, "..", "sdkconfig")]
-
-    def get_binary_path(self, app_path, config_name=None):
-        # local build folder
-        path = os.path.join(self.idf_path, app_path, "build")
-        if os.path.exists(path):
             return path
-
+        else:
-        if not config_name:
+            artifacts = Artifacts(self.idf_path,
-            config_name = "default"
+                                  CIAssignExampleTest.get_artifact_index_file(case_group=CIAssignExampleTest.TestAppsGroup),
-
+                                  app_path, config_name, target)
-        # Search for CI build folders.
+            path = artifacts.download_artifacts()
-        # Path format: $IDF_PATH/build_test_apps/app_path_with_underscores/config/target
+            if path:
-        # (see tools/ci/build_test_apps.sh)
+                return os.path.join(self.idf_path, path)
-        # For example: $IDF_PATH/build_test_apps/startup/default/esp32
+            else:
-        app_path_underscored = app_path.replace(os.path.sep, "_")
+                raise OSError("Failed to find example binary")
-        build_root = os.path.join(self.idf_path, "build_test_apps")
-        for dirpath in os.listdir(build_root):
-            if os.path.basename(dirpath) == app_path_underscored:
-                path = os.path.join(build_root, dirpath, config_name, self.target, "build")
-                return path
-
-        raise OSError("Failed to find test app binary")
 
 
 class SSC(IDFApp):

tools/ci/python_packages/ttfw_idf/__init__.py

@@ -88,10 +88,11 @@ def idf_unit_test(app=UT, dut=IDFDUT, chip="ESP32", module="unit-test", executio
     return test
 
 
-def idf_test_app_test(app=TestApp, dut=IDFDUT, chip="ESP32", module="misc", execution_time=1,
+def idf_custom_test(app=TestApp, dut=IDFDUT, chip="ESP32", module="misc", execution_time=1,
-                      level="integration", erase_nvs=True, **kwargs):
+                    level="integration", erase_nvs=True, config_name=None, group="test-apps", **kwargs):
+
     """
-    decorator for testing idf unit tests (with default values for some keyword args).
+    decorator for idf custom tests (with default values for some keyword args).
 
     :param app: test application class
     :param dut: dut class
@@ -100,6 +101,8 @@ def idf_test_app_test(app=TestApp, dut=IDFDUT, chip="ESP32", module="misc", exec
     :param execution_time: execution time in minutes, int
     :param level: test level, could be used to filter test cases, string
     :param erase_nvs: if need to erase_nvs in DUT.start_app()
+    :param config_name: if specified, name of the app configuration
+    :param group: identifier to group custom tests (unused for now, defaults to "test-apps")
     :param kwargs: other keyword args
     :return: test method
     """

tools/test_apps/startup/app_test.py

@@ -1,27 +1,15 @@
 #!/usr/bin/env python
 
-import re
 import os
-import sys
 import glob
-
+import ttfw_idf
-try:
+from tiny_test_fw import Utility
-    import IDF
-except ImportError:
-    # This environment variable is expected on the host machine
-    test_fw_path = os.getenv("TEST_FW_PATH")
-    if test_fw_path and test_fw_path not in sys.path:
-        sys.path.insert(0, test_fw_path)
-
-    import IDF
-
-import Utility
 
 
-@IDF.idf_test_app_test(env_tag="test_jtag_arm")
+@ttfw_idf.idf_test_app_test(env_tag="test_jtag_arm")
 def test_startup(env, extra_data):
     config_files = glob.glob(os.path.join(os.path.dirname(__file__), "sdkconfig.ci.*"))
-    config_names = [s.replace("sdkconfig.ci.", "") for s in config_files]
+    config_names = [os.path.basename(s).replace("sdkconfig.ci.", "") for s in config_files]
     for name in config_names:
         Utility.console_log("Checking config \"{}\"... ".format(name), end="")
         dut = env.get_dut("startup", "tools/test_apps/startup", app_config_name=name)