esp-idf/tools/ci/config/target-test.yml

# for parallel jobs, CI_JOB_NAME will be "job_name index/total" (for example, "IT_001 1/2")
# we need to convert to pattern "job_name_index.yml"
.define_config_file_name: &define_config_file_name |
    JOB_NAME_PREFIX=$(echo ${CI_JOB_NAME} | awk '{print $1}')
    JOB_FULL_NAME="${JOB_NAME_PREFIX}_${CI_NODE_INDEX}"
    CONFIG_FILE="${CONFIG_FILE_PATH}/${JOB_FULL_NAME}.yml"    

.example_test_template:
  stage: target_test
  when: on_success
  only:
    refs:
      - master
      - /^release\/v/
      - /^v\d+\.\d+(\.\d+)?($|-)/
      - triggers
      - schedules
    variables:
      - $BOT_TRIGGER_WITH_LABEL == null
      - $BOT_LABEL_EXAMPLE_TEST
  dependencies:
    - assign_test
  artifacts:
    when: always
    paths:
      - "**/*.log"
      - $LOG_PATH
    expire_in: 1 week
    reports:
        junit: $LOG_PATH/*/XUNIT_RESULT.xml
  variables:
    TEST_CASE_PATH: "$CI_PROJECT_DIR/examples"
    CONFIG_FILE_PATH: "${CI_PROJECT_DIR}/examples/test_configs"
    LOG_PATH: "$CI_PROJECT_DIR/TEST_LOGS"
    ENV_FILE: "$CI_PROJECT_DIR/ci-test-runner-configs/$CI_RUNNER_DESCRIPTION/EnvConfig.yml"
    SUBMODULES_TO_FETCH: "components/esptool_py/esptool"
    PYTHON_VER: 3
  script:
    - *define_config_file_name
    # first test if config file exists, if not exist, exit 0
    - test -e $CONFIG_FILE || exit 0
    # clone test env configs
    - ./tools/ci/retry_failed.sh git clone $TEST_ENV_CONFIG_REPO
    - python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs
    # git clone the known failure cases repo, run test
    - ./tools/ci/retry_failed.sh git clone $KNOWN_FAILURE_CASES_REPO known_failure_cases
    # run test
    - cd tools/ci/python_packages/tiny_test_fw/bin
    - python Runner.py $TEST_CASE_PATH -c $CONFIG_FILE -e $ENV_FILE --known_failure_cases_file $CI_PROJECT_DIR/known_failure_cases/known_failure_cases.txt

.example_debug_template:
  stage: target_test
  when: on_success
  only:
    refs:
      - master
      - /^release\/v/
      - /^v\d+\.\d+(\.\d+)?($|-)/
      - triggers
      - schedules
    variables:
      - $BOT_TRIGGER_WITH_LABEL == null
      - $BOT_LABEL_EXAMPLE_TEST
  dependencies:
    - assign_test
  artifacts:
    when: always
    paths:
      - "**/*.log"
      - $LOG_PATH
    expire_in: 1 week
    reports:
        junit: $LOG_PATH/*/XUNIT_RESULT.xml
  variables:
    TEST_CASE_PATH: "$CI_PROJECT_DIR/examples"
    CONFIG_FILE_PATH: "${CI_PROJECT_DIR}/examples/test_configs"
    LOG_PATH: "$CI_PROJECT_DIR/TEST_LOGS"
    ENV_FILE: "$CI_PROJECT_DIR/ci-test-runner-configs/$CI_RUNNER_DESCRIPTION/EnvConfig.yml"
  script:
    - *define_config_file_name
    # first test if config file exists, if not exist, exit 0
    - test -e $CONFIG_FILE || exit 0
    # clone test env configs
    - ./tools/ci/retry_failed.sh git clone $TEST_ENV_CONFIG_REPO
    - python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs
    # git clone the known failure cases repo, run test
    - ./tools/ci/retry_failed.sh git clone $KNOWN_FAILURE_CASES_REPO known_failure_cases
    # run test
    - cd tools/ci/python_packages/tiny_test_fw/bin
    - python Runner.py $TEST_CASE_PATH -c $CONFIG_FILE -e $ENV_FILE --known_failure_cases_file $CI_PROJECT_DIR/known_failure_cases/known_failure_cases.txt

.test_app_template:
  extends: .example_test_template
  stage: target_test
  dependencies:
    - assign_test
  only:
    refs:
      - master
      - /^release\/v/
      - /^v\d+\.\d+(\.\d+)?($|-)/
      - triggers
      - schedules
    variables:
      - $BOT_TRIGGER_WITH_LABEL == null
      - $BOT_LABEL_CUSTOM_TEST
  variables:
    TEST_FW_PATH: "$CI_PROJECT_DIR/tools/tiny-test-fw"
    TEST_CASE_PATH: "$CI_PROJECT_DIR/tools/test_apps"
    CONFIG_FILE_PATH: "${CI_PROJECT_DIR}/tools/test_apps/test_configs"
    LOG_PATH: "$CI_PROJECT_DIR/TEST_LOGS"
    ENV_FILE: "$CI_PROJECT_DIR/ci-test-runner-configs/$CI_RUNNER_DESCRIPTION/EnvConfig.yml"

.unit_test_template:
  extends: .example_test_template
  stage: target_test
  dependencies:
    - assign_test
    - build_esp_idf_tests_cmake_esp32
  only:
    refs:
      - master
      - /^release\/v/
      - /^v\d+\.\d+(\.\d+)?($|-)/
      - triggers
      - schedules
    variables:
      - $BOT_TRIGGER_WITH_LABEL == null
      - $BOT_LABEL_UNIT_TEST
  variables:
    TEST_FW_PATH: "$CI_PROJECT_DIR/tools/tiny-test-fw"
    TEST_CASE_PATH: "$CI_PROJECT_DIR/tools/unit-test-app"
    CONFIG_FILE_PATH: "${CI_PROJECT_DIR}/components/idf_test/unit_test/CIConfigs"
    LOG_PATH: "$CI_PROJECT_DIR/TEST_LOGS"
    ENV_FILE: "$CI_PROJECT_DIR/ci-test-runner-configs/$CI_RUNNER_DESCRIPTION/EnvConfig.yml"
    PYTHON_VER: 3

.test_template:
  stage: target_test
  when: on_success
  only:
    refs:
      - master
      - /^release\/v/
      - /^v\d+\.\d+(\.\d+)?($|-)/
      - triggers
      - schedules
    variables:
      - $BOT_TRIGGER_WITH_LABEL == null
      - $BOT_LABEL_INTEGRATION_TEST
  dependencies:
    - assign_test
    - build_ssc_esp32
  artifacts:
    when: always
    reports:
        junit: $LOG_PATH/*/XUNIT_RESULT.xml
    paths:
      - "**/*.log"
      - $LOG_PATH
    expire_in: 1 week
  variables:
    SUBMODULES_TO_FETCH: "components/esptool_py/esptool"
    LOCAL_ENV_CONFIG_PATH: "$CI_PROJECT_DIR/ci-test-runner-configs/$CI_RUNNER_DESCRIPTION/ESP32_IDF"
    LOG_PATH: "${CI_PROJECT_DIR}/TEST_LOGS"
    TEST_CASE_FILE_PATH: "$CI_PROJECT_DIR/auto_test_script/TestCaseFiles"
    MODULE_UPDATE_FILE: "$CI_PROJECT_DIR/components/idf_test/ModuleDefinition.yml"
    CONFIG_FILE_PATH: "${CI_PROJECT_DIR}/components/idf_test/integration_test/CIConfigs"
    KNOWN_ISSUE_FILE: "${CI_PROJECT_DIR}/components/idf_test/integration_test/KnownIssues"
    PYTHON_VER: 3.7.7
  script:
    - *define_config_file_name
    # first test if config file exists, if not exist, exit 0
    - test -e $CONFIG_FILE || exit 0
    # clone local test env configs
    - ./tools/ci/retry_failed.sh git clone $TEST_ENV_CONFIG_REPO
    - python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs
    # clone auto test repo
    - ./tools/ci/retry_failed.sh git clone ${CI_AUTO_TEST_SCRIPT_REPO_URL} -b empty_branch
    - ./tools/ci/retry_failed.sh git -C auto_test_script checkout -f ${CI_AUTO_TEST_SCRIPT_REPO_BRANCH}
    - python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script
    - export PYTHONPATH="${CI_PROJECT_DIR}/auto_test_script/packages:$PYTHONPATH"
    - cd auto_test_script
    # run test
    - cat ${KNOWN_ISSUE_FILE} >> ${TEST_CASE_FILE_PATH}/KnownIssues
    - python bin/CIRunner.py -l "$LOG_PATH/$JOB_FULL_NAME" -c $CONFIG_FILE -e $LOCAL_ENV_CONFIG_PATH -t $TEST_CASE_FILE_PATH

test_weekend_mqtt:
  extends: .example_test_template
  stage: target_test
  tags:
    - ESP32
    - Example_WIFI
  only:
    variables:
      - $BOT_LABEL_WEEKEND_TEST
  variables:
    TEST_CASE_PATH: "$CI_PROJECT_DIR/components/mqtt/weekend_test"
    LOG_PATH: "$CI_PROJECT_DIR/TEST_LOGS"
    ENV_FILE: "$CI_PROJECT_DIR/components/mqtt/weekend_test/env.yml"
    CONFIG_FILE_PATH: "$CI_PROJECT_DIR/components/mqtt/weekend_test"

test_weekend_network:
  extends: .example_test_template
  stage: target_test
  image: $CI_DOCKER_REGISTRY/rpi-net-suite$BOT_DOCKER_IMAGE_TAG
  tags:
    - ESP32
    - Example_WIFI
  only:
    variables:
      - $BOT_LABEL_WEEKEND_TEST
  variables:
    TEST_CASE_PATH: "$CI_PROJECT_DIR/components/lwip/weekend_test"
    LOG_PATH: "$CI_PROJECT_DIR/TEST_LOGS"
    ENV_FILE: "$CI_PROJECT_DIR/components/lwip/weekend_test/env.yml"
    CONFIG_FILE_PATH: "$CI_PROJECT_DIR/components/lwip/weekend_test"

example_test_001A:
  extends: .example_test_template
  parallel: 4
  tags:
    - ESP32
    - Example_WIFI

example_test_001B:
  extends: .example_test_template
  tags:
    - ESP32
    - Example_EthKitV1

example_test_001C:
  extends: .example_test_template
  parallel: 2
  tags:
    - ESP32
    - Example_GENERIC

example_test_002:
  extends: .example_test_template
  image: $CI_DOCKER_REGISTRY/ubuntu-test-env$BOT_DOCKER_IMAGE_TAG
  tags:
    - ESP32
    - Example_ShieldBox_Basic
  script:
    - *define_config_file_name
    # first test if config file exists, if not exist, exit 0
    - test -e $CONFIG_FILE || exit 0
    # clone test env configs
    - ./tools/ci/retry_failed.sh git clone $TEST_ENV_CONFIG_REPO
    - python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs
    - cd tools/ci/python_packages/tiny_test_fw/bin
    # run test
    - python Runner.py $TEST_CASE_PATH -c $CONFIG_FILE -e $ENV_FILE


.example_test_003:
  extends: .example_test_template
  tags:
    - ESP32
    - Example_SDIO

example_test_004A:
  extends: .example_test_template
  tags:
    - ESP32
    - Example_TWAI1

example_test_004B:
  extends: .example_test_template
  tags:
    - ESP32
    - Example_TWAI2

example_test_005:
  extends: .example_test_template
  tags:
    - ESP32
    - Example_WIFI_BT

example_test_006:
  extends: .example_test_template
  image: $CI_DOCKER_REGISTRY/ubuntu-test-env$BOT_DOCKER_IMAGE_TAG
  only:
    variables:
      - $BOT_LABEL_IPERF_STRESS_TEST
  tags:
    - ESP32
    - Example_ShieldBox

example_test_007:
  extends: .example_test_template
  tags:
    - ESP32
    - Example_I2C_CCS811_SENSOR

example_test_008:
  extends: .example_test_template
  tags:
    - ESP32
    - Example_Flash_Encryption

example_test_009:
  extends: .example_test_template
  tags:
    - ESP32
    - test_jtag_arm
  variables:
    SETUP_TOOLS: "1"

example_test_010:
  extends: .example_test_template
  tags:
    - ESP32
    - Example_ExtFlash

example_test_011:
  extends: .example_debug_template
  tags:
    - ESP32
    - UT_T2_RS485
  variables:
    SETUP_TOOLS: "1"

example_test_012:
  extends: .example_test_template
  tags:
    - ESP32
    - Example_RMT_IR_PROTOCOLS

example_test_013:
  extends: .example_test_template
  tags:
    - ESP32
    - UT_T1_SDMODE

example_test_014:
  extends: .example_test_template
  tags:
    - ESP32
    - 8Mpsram

example_test_015:
  extends: .example_test_template
  tags:
    - ESP32
    - Example_PPP

test_app_test_esp32_generic:
  extends: .test_app_template
  tags:
    - ESP32
    - Example_GENERIC
  variables:
    SETUP_TOOLS: "1"

test_app_test_001:
  extends: .test_app_template
  tags:
    - ESP32
    - test_jtag_arm
  variables:
    SETUP_TOOLS: "1"

test_app_test_002:
  extends: .test_app_template
  tags:
    - ESP32
    - Example_WIFI

test_app_test_003:
  extends: .test_app_template
  tags:
    - ESP32
    - Example_PPP

UT_001:
  extends: .unit_test_template
  parallel: 39
  tags:
    - ESP32_IDF
    - UT_T1_1

# Max. allowed value of 'parallel' is 50.

UT_002:
  extends: .unit_test_template
  parallel: 15
  tags:
    - ESP32_IDF
    - UT_T1_1
    - psram

UT_003:
  extends: .unit_test_template
  tags:
    - ESP32_IDF
    - UT_T1_SDMODE

UT_004:
  extends: .unit_test_template
  tags:
    - ESP32_IDF
    - UT_T1_SPIMODE

UT_005:
  extends: .unit_test_template
  tags:
    - ESP32_IDF
    - UT_T1_SDMODE
    - psram

UT_006:
  extends: .unit_test_template
  tags:
    - ESP32_IDF
    - UT_T1_SPIMODE
    - psram

UT_008:
  extends: .unit_test_template
  tags:
    - ESP32_IDF
    - UT_T1_GPIO
    - psram

UT_012:
  extends: .unit_test_template
  tags:
    - ESP32_IDF
    - UT_T1_LEDC
    - psram

UT_014:
  extends: .unit_test_template
  tags:
    - ESP32_IDF
    - UT_T2_RS485
    - psram

UT_015:
  extends: .unit_test_template
  tags:
    - ESP32_IDF
    - UT_T1_RMT

UT_016:
  extends: .unit_test_template
  tags:
    - ESP32_IDF
    - UT_T1_RMT
    - psram

UT_017:
  extends: .unit_test_template
  tags:
    - ESP32_IDF
    - EMMC

UT_018:
  extends: .unit_test_template
  parallel: 5
  tags:
    - ESP32_IDF
    - UT_T1_1
    - 8Mpsram

UT_020:
  extends: .unit_test_template
  tags:
    - ESP32_IDF
    - Example_SPI_Multi_device
    - psram

UT_021:
  extends: .unit_test_template
  parallel: 2
  tags:
    - ESP32_IDF
    - psram
    - UT_T1_FlashEncryption

UT_022:
  extends: .unit_test_template
  tags:
    - ESP32_IDF
    - UT_T2_I2C
    - psram

UT_023:
  extends: .unit_test_template
  parallel: 2
  tags:
    - ESP32_IDF
    - UT_T1_MCPWM

UT_024:
  extends: .unit_test_template
  tags:
    - ESP32_IDF
    - UT_T1_MCPWM
    - psram

UT_028:
  extends: .unit_test_template
  tags:
    - ESP32_IDF
    - UT_T2_1
    - psram

UT_031:
  extends: .unit_test_template
  tags:
    - ESP32_IDF
    - UT_T1_FlashEncryption

UT_033:
  extends: .unit_test_template
  tags:
    - ESP32_IDF
    - UT_T2_Ethernet
    - psram

UT_034:
  extends: .unit_test_template
  tags:
    - ESP32_IDF
    - UT_T1_ESP_FLASH

.unit_test_s2_template:
  extends: .unit_test_template
  dependencies:
    - assign_test
    - build_esp_idf_tests_cmake_esp32s2
  only:
    refs:
      # Due to lack of runners, the tests are only done by manual trigger
      # - master
      # - /^release\/v/
      # - /^v\d+\.\d+(\.\d+)?($|-)/
      - triggers
      - schedules
    variables:
      # Due to lack of runners, S2 tests will only be triggered with label (@bot test will not trigger)
      # - $BOT_TRIGGER_WITH_LABEL == null
      - $BOT_LABEL_UNIT_TEST_S2

UT_035:
  extends: .unit_test_s2_template
  parallel: 38
  tags:
    - ESP32S2_IDF
    - UT_T1_1

UT_036:
  extends: .unit_test_template
  tags:
    - ESP32_IDF
    - UT_T1_PSRAMV0
    - psram

# ToDo: re-enable this job when ESP32-S2 LEDC runner installed
# UT_037:
#   extends: .unit_test_template
#   tags:
#     - ESP32S2_IDF
#     - UT_T1_LEDC

UT_041:
  extends: .unit_test_template
  tags:
    - ESP32_IDF
    - UT_T1_no32kXTAL
    - psram

UT_043:
  extends: .unit_test_template
  tags:
    - ESP32_IDF
    - UT_T1_32kXTAL
    - psram

UT_044:
  extends: .unit_test_template
  tags:
    - ESP32_IDF
    - UT_SDIO

UT_045:
  extends: .unit_test_template
  tags:
    - ESP32_IDF
    - UT_SDIO
    - psram


nvs_compatible_test:
  extends: .test_template
  artifacts:
    when: always
    paths:
      - $LOG_PATH
      - nvs_wifi.bin
    expire_in: 1 mos
  tags:
    - ESP32_IDF
    - NVS_Compatible
  script:
    - *define_config_file_name
    # first test if config file exists, if not exist, exit 0
    - test -e $CONFIG_FILE || exit 0
    # clone local test env configs
    - ./tools/ci/retry_failed.sh git clone $TEST_ENV_CONFIG_REPO
    - python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs
    # clone test bench
    - ./tools/ci/retry_failed.sh git clone ${CI_AUTO_TEST_SCRIPT_REPO_URL} -b empty_branch
    - ./tools/ci/retry_failed.sh git -C auto_test_script checkout -f ${CI_AUTO_TEST_SCRIPT_REPO_BRANCH}
    - python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script
    - export PYTHONPATH="${CI_PROJECT_DIR}/auto_test_script/packages:$PYTHONPATH"
    - cd auto_test_script
    # prepare nvs bins
    - ./tools/prepare_nvs_bin.sh
    # run test
    - python bin/CIRunner.py -l "$LOG_PATH/$JOB_FULL_NAME" -c $CONFIG_FILE -e $LOCAL_ENV_CONFIG_PATH -t $TEST_CASE_FILE_PATH

IT_001:
  extends: .test_template
  parallel: 2
  tags:
    - ESP32_IDF
    - SSC_T1_4

IT_002:
  extends: .test_template
  tags:
    - ESP32_IDF
    - SSC_T1_2

IT_003:
  extends: .test_template
  parallel: 9
  tags:
    - ESP32_IDF
    - SSC_T2_5

IT_005:
  extends: .test_template
  tags:
    - ESP32_IDF
    - SSC_T1_5

IT_006:
  extends: .test_template
  parallel: 5
  tags:
    - ESP32_IDF
    - SSC_T1_6

IT_007:
  extends: .test_template
  tags:
    - ESP32_IDF
    - SSC_T1_7

IT_008:
  extends: .test_template
  tags:
    - ESP32_IDF
    - SSC_T1_8

IT_011:
  extends: .test_template
  tags:
    - ESP32_IDF
    - SSC_T1_MESH1

IT_012:
  extends: .test_template
  parallel: 2
  tags:
    - ESP32_IDF
    - SSC_T2_MESH1

IT_013:
  extends: .test_template
  tags:
    - ESP32_IDF
    - SSC_T3_MESH1

IT_014:
  extends: .test_template
  tags:
    - ESP32_IDF
    - SSC_T6_MESH1

IT_015:
  extends: .test_template
  tags:
    - ESP32_IDF
    - SSC_T12_MESH1

IT_016:
  extends: .test_template
  allow_failure: true
  tags:
    - ESP32_IDF
    - SSC_T50_MESH1

IT_017:
  extends: .test_template
  tags:
    - ESP32_IDF
    - SSC_T1_MESH2

IT_018:
  extends: .test_template
  tags:
    - ESP32_IDF
    - SSC_T1_9

IT_019:
  extends: .test_template
  tags:
    - ESP32_IDF
    - SSC_T2_2

IT_020:
  extends: .test_template
  tags:
    - ESP32_IDF
    - SSC_T2_3

IT_022:
  extends: .test_template
  tags:
    - ESP32_IDF
    - SSC_T3_2