esp-idf/.gitlab/ci/target-test.yml

.target_test_template:
  image: $TARGET_TEST_ENV_IMAGE
  stage: target_test
  timeout: 1 hour
  variables:
    GIT_DEPTH: 1
    SUBMODULES_TO_FETCH: "none"
  cache:
    # Usually do not need submodule-cache in target_test
    - key: pip-cache
      paths:
        - .cache/pip
      policy: pull

.pytest_template:
  extends:
    - .target_test_template
  artifacts:
    when: always
    paths:
      - XUNIT_RESULT.xml
      - pytest_embedded_log/
    reports:
      junit: XUNIT_RESULT.xml
    expire_in: 1 week
  script:
    - retry_failed git clone $KNOWN_FAILURE_CASES_REPO known_failure_cases
    # get runner env config file
    - retry_failed git clone $TEST_ENV_CONFIG_REPO
    - python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs
    # using runner tags as markers to filter the test cases
    # Runner tags are comma separated, replace the comma with " and " for markers
    - job_tags=$(python tools/ci/python_packages/gitlab_api.py get_job_tags $CI_PROJECT_ID --job_id $CI_JOB_ID)
    - markers=$(echo $job_tags | sed -e "s/,/ and /g")
    - run_cmd pytest $TEST_DIR
      -m \"${markers}\"
      --junitxml=XUNIT_RESULT.xml
      --known-failure-cases-file known_failure_cases/known_failure_cases.txt
      --parallel-count ${CI_NODE_TOTAL:-1}
      --parallel-index ${CI_NODE_INDEX:-1}
      ${PYTEST_EXTRA_FLAGS}

.pytest_examples_dir_template:
  extends: .pytest_template
  variables:
    TEST_DIR: examples

example_test_pytest_esp32_generic:
  extends:
    - .pytest_examples_dir_template
    - .rules:test:example_test-esp32
  needs:
    - build_pytest_examples_esp32
  tags: [ esp32, generic ]

example_test_pytest_esp32_ir_transceiver:
  extends:
    - .pytest_examples_dir_template
    - .rules:test:example_test-esp32
  needs:
    - build_pytest_examples_esp32
  tags: [ esp32, ir_transceiver ]

example_test_pytest_esp32_twai_transceiver:
  extends:
    - .pytest_examples_dir_template
    - .rules:test:example_test-esp32
  needs:
    - build_pytest_examples_esp32
  tags: [ esp32, twai_transceiver ]

example_test_pytest_esp32_twai_network:
  extends:
    - .pytest_examples_dir_template
    - .rules:test:example_test-esp32
  needs:
    - build_pytest_examples_esp32
  tags: [ esp32, twai_network ]

example_test_pytest_esp32_jtag:
  extends:
    - .pytest_examples_dir_template
    - .rules:test:example_test-esp32
  needs:
    - build_pytest_examples_jtag
  tags: [ esp32, jtag ]
  variables:
    SETUP_TOOLS: "1"  # need gdb openocd
    PYTEST_EXTRA_FLAGS: "--log-cli-level DEBUG"

example_test_pytest_esp32s2_generic:
  extends:
    - .pytest_examples_dir_template
    - .rules:test:example_test-esp32s2
  needs:
    - build_pytest_examples_esp32s2
  tags: [ esp32s2, generic ]

example_test_pytest_esp32s3_generic:
  extends:
    - .pytest_examples_dir_template
    - .rules:test:example_test-esp32s3
  needs:
    - build_pytest_examples_esp32s3
  tags: [ esp32s3, generic ]

example_test_pytest_esp32s3_f4r8:
  extends:
    - .pytest_examples_dir_template
    - .rules:test:example_test-esp32s3
  needs:
    - build_pytest_examples_esp32s3
  tags: [ esp32s3, MSPI_F4R8 ]

example_test_pytest_esp32c2_generic:
  extends:
    - .pytest_examples_dir_template
    - .rules:test:example_test-esp32c2
  needs:
    - build_pytest_examples_esp32c2
  tags: [ esp32c2, generic, xtal_40mhz ]

example_test_pytest_esp32c2_26mhz:
  extends:
    - .pytest_examples_dir_template
    - .rules:test:example_test-esp32c2
  needs:
    - build_pytest_examples_esp32c2
  tags: [ esp32c2, xtal_26mhz ]

example_test_pytest_esp32c3_generic:
  extends:
    - .pytest_examples_dir_template
    - .rules:test:example_test-esp32c3
  needs:
    - build_pytest_examples_esp32c3
  tags: [ esp32c3, generic ]

example_test_pytest_esp32c3_flash_suspend:
  extends:
    - .pytest_examples_dir_template
    - .rules:test:example_test-esp32c3
  needs:
    - build_pytest_examples_esp32c3
  tags: [ esp32c3, flash_suspend ]

example_test_pytest_esp32c6_generic:
  extends:
    - .pytest_examples_dir_template
    - .rules:test:example_test-esp32c6
  needs:
    - build_pytest_examples_esp32c6
  tags: [ esp32c6, generic ]

example_test_pytest_esp32_ethernet_ota:
  extends:
    - .pytest_examples_dir_template
    - .rules:test:example_test-esp32-ethernet
  needs:
    - build_pytest_examples_esp32
  tags: [ esp32, ethernet_ota ]

example_test_pytest_esp32_wifi_high_traffic:
  extends:
    - .pytest_examples_dir_template
    - .rules:test:example_test-esp32-wifi
  needs:
    - build_pytest_examples_esp32
  tags: [ esp32, wifi_high_traffic ]

example_test_pytest_esp32_flash_encryption_wifi_high_traffic:
  extends:
    - .pytest_examples_dir_template
    - .rules:test:example_test-esp32-include_nightly_run-rule
  needs:
    - build_pytest_examples_esp32
  tags: [ esp32, flash_encryption_wifi_high_traffic ]

example_test_pytest_esp32c3_flash_encryption_wifi_high_traffic:
  extends:
    - .pytest_examples_dir_template
    - .rules:test:example_test-esp32c3-include_nightly_run-rule
  needs:
    - build_pytest_examples_esp32c3
  tags: [ esp32c3, flash_encryption_wifi_high_traffic ]

example_test_pytest_esp32_ethernet:
  extends:
    - .pytest_examples_dir_template
    - .rules:test:example_test-esp32-ethernet
  needs:
    - build_pytest_examples_esp32
  tags: [ esp32, ethernet]

example_test_pytest_esp32_8mb_flash:
  extends:
    - .pytest_examples_dir_template
    - .rules:test:example_test-esp32
  needs:
    - build_pytest_examples_esp32
  tags: [ esp32, ethernet_flash_8m ]

example_test_pytest_esp32_wifi_ap:
  extends:
    - .pytest_examples_dir_template
    - .rules:test:example_test-esp32-wifi
  needs:
    - build_pytest_examples_esp32
  tags: [ esp32, wifi_ap ]

example_test_pytest_esp32_wifi_router:
  extends:
    - .pytest_examples_dir_template
    - .rules:test:example_test-esp32-wifi
  needs:
    - build_pytest_examples_esp32
  tags: [ esp32, wifi_router ]

example_test_pytest_esp32_wifi_wlan:
  extends:
    - .pytest_examples_dir_template
    - .rules:test:example_test-esp32-wifi
  needs:
    - build_pytest_examples_esp32
  tags: [ esp32, wifi_wlan ]

example_test_pytest_esp32_ethernet_ip101:
  extends:
    - .pytest_examples_dir_template
    - .rules:test:example_test-esp32
  needs:
    - build_pytest_examples_esp32
  tags: [ esp32, ip101 ]

example_test_pytest_esp32_flash_encryption:
  extends:
    - .pytest_examples_dir_template
    - .rules:test:example_test-esp32
  needs:
    - build_pytest_examples_esp32
  tags: [ esp32, flash_encryption ]

example_test_pytest_esp32_wifi_two_dut:
  extends:
    - .pytest_examples_dir_template
    - .rules:test:example_test-esp32-wifi
  needs:
    - build_pytest_examples_esp32
  tags: [ esp32, wifi_two_dut ]

example_test_pytest_esp32c3_flash_encryption:
  extends:
    - .pytest_examples_dir_template
    - .rules:test:example_test-esp32c3
  needs:
    - build_pytest_examples_esp32c3
  tags: [ esp32c3, flash_encryption ]

example_test_pytest_esp32s2_usb_device:
  extends:
    - .pytest_examples_dir_template
    - .rules:test:example_test-esp32s2-usb
  needs:
    - build_pytest_examples_esp32s2
  tags: [ esp32s2, usb_device ]

example_test_pytest_esp32_sdmmc:
  extends:
    - .pytest_examples_dir_template
    - .rules:test:example_test-esp32
  needs:
    - build_pytest_examples_esp32
  tags: [ esp32, sdcard_sdmode ]

example_test_pytest_esp32_sdspi:
  extends:
    - .pytest_examples_dir_template
    - .rules:test:example_test-esp32
  needs:
    - build_pytest_examples_esp32
  tags: [ esp32, sdcard_spimode ]

example_test_pytest_esp32s2_sdspi:
  extends:
    - .pytest_examples_dir_template
    - .rules:test:example_test-esp32s2
  needs:
    - build_pytest_examples_esp32s2
  tags: [ esp32s2, sdcard_spimode ]

example_test_pytest_esp32c3_sdspi:
  extends:
    - .pytest_examples_dir_template
    - .rules:test:example_test-esp32c3
  needs:
    - build_pytest_examples_esp32c3
  tags: [ esp32c3, sdcard_spimode ]

example_test_pytest_esp32_extflash:
  extends:
    - .pytest_examples_dir_template
    - .rules:test:example_test-esp32
  needs:
    - build_pytest_examples_esp32
  tags: [ esp32, external_flash ]

.pytest_components_dir_template:
  extends: .pytest_template
  variables:
    TEST_DIR: components

component_ut_pytest_esp32_generic:
  extends:
    - .pytest_components_dir_template
    - .rules:test:component_ut-esp32
  needs:
    - build_pytest_components_esp32
  tags: [ esp32, generic ]
  parallel: 2

component_ut_pytest_esp32_generic_multi_device:
  extends:
    - .pytest_components_dir_template
    - .rules:test:component_ut-esp32
  needs:
    - build_pytest_components_esp32
  tags: [ esp32, generic_multi_device ]

component_ut_pytest_esp32_wifi_two_dut:
  extends:
    - .pytest_components_dir_template
    - .rules:test:component_ut-esp32-wifi
  needs:
    - build_pytest_components_esp32
  tags: [ esp32, wifi_two_dut ]

component_ut_pytest_esp32_adc:
  extends:
    - .pytest_components_dir_template
    - .rules:test:component_ut-esp32-adc
  needs:
    - build_pytest_components_esp32
  tags: [ esp32, adc ]

component_ut_pytest_esp32_sdio:
  extends:
    - .pytest_components_dir_template
    - .rules:test:unit_test-esp32-sdio
  needs:
    - build_pytest_components_esp32
  tags: [ esp32, sdio_master_slave ]

component_ut_pytest_esp32_ip101:
  extends:
    - .pytest_components_dir_template
    - .rules:test:component_ut-esp32
  needs:
    - build_pytest_components_esp32
  tags: [ esp32, ip101 ]

component_ut_pytest_esp32_lan8720:
  extends:
    - .pytest_components_dir_template
    - .rules:labels-protected:lan8720  # FIXME: IDFCI-1176
  needs:
    - build_pytest_components_esp32
  tags: [ esp32, lan8720 ]

component_ut_pytest_esp32_ethernet:
  extends:
    - .pytest_components_dir_template
    - .rules:test:component_ut-esp32
  needs:
    - build_pytest_components_esp32
  tags: [ esp32, ethernet ]

component_ut_pytest_esp32_flash_encryption:
  extends:
    - .pytest_components_dir_template
    - .rules:test:component_ut-esp32
  needs:
    - build_pytest_components_esp32
  tags: [ esp32, flash_encryption ]

component_ut_pytest_esp32_flash_multi:
  extends:
    - .pytest_components_dir_template
    - .rules:test:component_ut-esp32
  needs:
    - build_pytest_components_esp32
  tags: [ esp32, flash_mutli ]

component_ut_pytest_esp32_xtal32k:
  extends:
    - .pytest_components_dir_template
    - .rules:test:component_ut-esp32
  needs:
    - build_pytest_components_esp32
  tags: [ esp32, xtal32k ]

component_ut_pytest_esp32_no32kXtal:
  extends:
    - .pytest_components_dir_template
    - .rules:test:component_ut-esp32
  needs:
    - build_pytest_components_esp32
  tags: [ esp32, no32kXtal ]

component_ut_pytest_esp32_rs485_multi:
  extends:
    - .pytest_components_dir_template
    - .rules:test:component_ut-esp32
  needs:
    - build_pytest_components_esp32
  tags: [ esp32, multi_dut_modbus_rs485 ]

component_ut_pytest_esp32_psramv0:
  extends:
    - .pytest_components_dir_template
    - .rules:test:component_ut-esp32
  needs:
    - build_pytest_components_esp32
  tags: [ esp32, psramv0 ]


component_ut_pytest_esp32s2_generic:
  extends:
    - .pytest_components_dir_template
    - .rules:test:component_ut-esp32s2
  needs:
    - build_pytest_components_esp32s2
  tags: [ esp32s2, generic ]
  parallel: 2

component_ut_pytest_esp32s2_generic_multi_device:
  extends:
    - .pytest_components_dir_template
    - .rules:test:component_ut-esp32s2
  needs:
    - build_pytest_components_esp32s2
  tags: [ esp32s2, generic_multi_device ]

component_ut_pytest_esp32s2_adc:
  extends:
    - .pytest_components_dir_template
    - .rules:test:component_ut-esp32s2-adc
  needs:
    - build_pytest_components_esp32s2
  tags: [ esp32s2, adc ]

component_ut_pytest_esp32s2_flash_multi:
  extends:
    - .pytest_components_dir_template
    - .rules:test:component_ut-esp32s2
  needs:
    - build_pytest_components_esp32s2
  tags: [ esp32s2, flash_mutli ]

component_ut_pytest_esp32s3_generic:
  extends:
    - .pytest_components_dir_template
    - .rules:test:component_ut-esp32s3
  needs:
    - build_pytest_components_esp32s3
  tags: [ esp32s3, generic ]
  parallel: 2

component_ut_pytest_esp32s3_generic_multi_device:
  extends:
    - .pytest_components_dir_template
    - .rules:test:component_ut-esp32s3
  needs:
    - build_pytest_components_esp32s3
  tags: [ esp32s3, generic_multi_device ]

component_ut_pytest_esp32s3_adc:
  extends:
    - .pytest_components_dir_template
    - .rules:test:component_ut-esp32s3-adc
  needs:
    - build_pytest_components_esp32s3
  tags: [ esp32s3, adc ]

component_ut_pytest_esp32s3_octal_psram:
  extends:
    - .pytest_components_dir_template
    - .rules:test:component_ut-esp32s3
  needs:
    - build_pytest_components_esp32s3
  tags: [ esp32s3, octal_psram ]

component_ut_pytest_esp32s3_flash_encryption_f4r8:
  extends:
    - .pytest_components_dir_template
    - .rules:test:component_ut-esp32s3
  needs:
    - build_pytest_components_esp32s3
  tags: [ esp32s3, flash_encryption_f4r8 ]

component_ut_pytest_esp32s3_flash_encryption_f8r8:
  extends:
    - .pytest_components_dir_template
    - .rules:test:component_ut-esp32s3
  needs:
    - build_pytest_components_esp32s3
  tags: [ esp32s3, flash_encryption_f8r8 ]

component_ut_pytest_esp32s3_flash_multi:
  extends:
    - .pytest_components_dir_template
    - .rules:test:component_ut-esp32s3
  needs:
    - build_pytest_components_esp32s3
  tags: [ esp32s3, flash_mutli ]

component_ut_pytest_esp32c2_generic:
  extends:
    - .pytest_components_dir_template
    - .rules:test:component_ut-esp32c2
  needs:
    - build_pytest_components_esp32c2
  tags: [ esp32c2, generic, xtal_40mhz ]

component_ut_pytest_esp32c2_adc:
  extends:
    - .pytest_components_dir_template
    - .rules:test:component_ut-esp32c2-adc
  needs:
    - build_pytest_components_esp32c2
  tags: [ esp32c2, adc ]

component_ut_pytest_esp32c2_generic_multi_device:
  extends:
    - .pytest_components_dir_template
    - .rules:test:component_ut-esp32c2
  needs:
    - build_pytest_components_esp32c2
  tags: [ esp32c2, generic_multi_device, xtal_40mhz ]

component_ut_pytest_esp32c2_xtal_26mhz:
  extends:
    - .pytest_components_dir_template
    - .rules:test:component_ut-esp32c2
  needs:
    - build_pytest_components_esp32c2
  tags: [ esp32c2, xtal_26mhz ]

component_ut_pytest_esp32c3_generic:
  extends:
    - .pytest_components_dir_template
    - .rules:test:component_ut-esp32c3
  needs:
    - build_pytest_components_esp32c3
  tags: [ esp32c3, generic ]

component_ut_pytest_esp32c3_generic_multi_device:
  extends:
    - .pytest_components_dir_template
    - .rules:test:component_ut-esp32c3
  needs:
    - build_pytest_components_esp32c3
  tags: [ esp32c3, generic_multi_device ]

component_ut_pytest_esp32c3_wifi_two_dut:
  extends:
    - .pytest_components_dir_template
    - .rules:test:component_ut-esp32c3-wifi
  needs:
    - build_pytest_components_esp32c3
  tags: [ esp32c3, wifi_two_dut ]

component_ut_pytest_esp32c3_adc:
  extends:
    - .pytest_components_dir_template
    - .rules:test:component_ut-esp32c3-adc
  needs:
    - build_pytest_components_esp32c3
  tags: [ esp32c3, adc ]

component_ut_pytest_esp32c3_flash_encryption:
  extends:
    - .pytest_components_dir_template
    - .rules:test:component_ut-esp32c3
  needs:
    - build_pytest_components_esp32c3
  tags: [ esp32c3, flash_encryption ]

component_ut_pytest_esp32c3_flash_multi:
  extends:
    - .pytest_components_dir_template
    - .rules:test:component_ut-esp32c3
  needs:
    - build_pytest_components_esp32c3
  tags: [ esp32c3, flash_mutli ]

component_ut_pytest_esp32_sdmmc:
  extends:
    - .pytest_components_dir_template
    - .rules:test:component_ut-esp32
  needs:
    - build_pytest_components_esp32
  tags: [ esp32, sdcard_sdmode ]

component_ut_pytest_esp32_sdspi:
  extends:
    - .pytest_components_dir_template
    - .rules:test:component_ut-esp32
  needs:
    - build_pytest_components_esp32
  tags: [ esp32, sdcard_spimode ]

component_ut_pytest_esp32s2_sdspi:
  extends:
    - .pytest_components_dir_template
    - .rules:test:component_ut-esp32s2
  needs:
    - build_pytest_components_esp32s2
  tags: [ esp32s2, sdcard_spimode ]

component_ut_pytest_esp32c3_sdspi:
  extends:
    - .pytest_components_dir_template
    - .rules:test:component_ut-esp32c3
  needs:
    - build_pytest_components_esp32c3
  tags: [ esp32c3, sdcard_spimode ]

component_ut_pytest_esp32c6_generic:
  extends:
    - .pytest_components_dir_template
    - .rules:test:component_ut-esp32c6
  needs:
    - build_pytest_components_esp32c6
  tags: [ esp32c6, generic ]

component_ut_pytest_esp32c6_generic_multi_device:
  extends:
    - .pytest_components_dir_template
    - .rules:test:component_ut-esp32c6
  needs:
    - build_pytest_components_esp32c6
  tags: [ esp32c6, generic_multi_device ]

component_ut_pytest_esp32h4_i154:
  extends:
    - .pytest_components_dir_template
    - .rules:test:component_ut-esp32h4-i154
  needs:
    - build_pytest_components_esp32h4
  tags: [ esp32h4, ieee802154 ]

example_test_pytest_openthread_br:
  extends:
    - .pytest_examples_dir_template
    - .rules:test:example_test-i154
  needs:
    - build_pytest_examples_esp32s3
    - build_pytest_examples_esp32h4
  tags:
    - esp32h4
    - i154_multi_dut

component_ut_pytest_esp32s3_usb_host:
  extends:
    - .pytest_components_dir_template
    - .rules:test:component_ut-esp32s3-usb
  needs:
    - build_pytest_components_esp32s3
  tags: [ esp32s3, usb_host_flash_disk ]

.pytest_test_apps_dir_template:
  extends: .pytest_template
  variables:
    TEST_DIR: tools/test_apps

test_app_test_pytest_esp32_generic:
  extends:
    - .pytest_test_apps_dir_template
    - .rules:test:custom_test-esp32
  needs:
    - build_pytest_test_apps_esp32
  tags: [ esp32, generic ]
  variables:
    SETUP_TOOLS: "1"  # need gdb

test_app_test_pytest_esp32_jtag:
  extends:
    - .pytest_test_apps_dir_template
    - .rules:test:custom_test-esp32
  needs:
    - build_pytest_test_apps_esp32
  tags: [ esp32, jtag]
  variables:
    SETUP_TOOLS: "1"  # need gdb
    PYTEST_EXTRA_FLAGS: "--log-cli-level DEBUG"

test_app_test_pytest_esp32s2_generic:
  extends:
    - .pytest_test_apps_dir_template
    - .rules:test:custom_test-esp32s2
  needs:
    - build_pytest_test_apps_esp32s2
  tags: [ esp32s2, generic ]
  variables:
    SETUP_TOOLS: "1"  # need gdb

test_app_test_pytest_esp32s3_generic:
  extends:
    - .pytest_test_apps_dir_template
    - .rules:test:custom_test-esp32s3
  needs:
    - build_pytest_test_apps_esp32s3
  tags: [ esp32s3, generic ]

test_app_test_pytest_esp32c2_generic:
  extends:
    - .pytest_test_apps_dir_template
    - .rules:test:custom_test-esp32c2
  needs:
    - build_pytest_test_apps_esp32c2
  tags: [ esp32c2, generic, xtal_40mhz ]

test_app_test_pytest_esp32c2_26mhz:
  extends:
    - .pytest_test_apps_dir_template
    - .rules:test:custom_test-esp32c2
  needs:
    - build_pytest_test_apps_esp32c2
  tags: [ esp32c2, xtal_26mhz ]

test_app_test_pytest_esp32c3_generic:
  extends:
    - .pytest_test_apps_dir_template
    - .rules:test:custom_test-esp32c3
  needs:
    - build_pytest_test_apps_esp32c3
  tags: [ esp32c3, generic ]

test_app_test_pytest_esp32c6_generic:
  extends:
    - .pytest_test_apps_dir_template
    - .rules:test:custom_test-esp32c6
  needs:
    - build_pytest_test_apps_esp32c6
  tags: [ esp32c6, generic ]

test_app_test_pytest_esp32s3_mspi_f8r8:
  extends:
    - .pytest_test_apps_dir_template
    - .rules:test:custom_test-esp32s3
  needs:
    - build_pytest_test_apps_esp32s3
  tags: [ esp32s3, MSPI_F8R8 ]

test_app_test_pytest_esp32s3_mspi_f4r8:
  extends:
    - .pytest_test_apps_dir_template
    - .rules:test:custom_test-esp32s3
  needs:
    - build_pytest_test_apps_esp32s3
  tags: [ esp32s3, MSPI_F4R8 ]

test_app_test_pytest_esp32s3_mspi_f4r4:
  extends:
    - .pytest_test_apps_dir_template
    - .rules:test:custom_test-esp32s3
  needs:
    - build_pytest_test_apps_esp32s3
  tags: [ esp32s3, MSPI_F4R4 ]

# for parallel jobs, CI_JOB_NAME will be "job_name index/total" (for example, "IT_001 1/2")
# we need to convert to pattern "job_name_index.yml"
.define_config_file_name: &define_config_file_name |
  JOB_NAME_PREFIX=$(echo ${CI_JOB_NAME} | awk '{print $1}')
  JOB_FULL_NAME="${JOB_NAME_PREFIX}_${CI_NODE_INDEX}"
  CONFIG_FILE="${CONFIG_FILE_PATH}/${JOB_FULL_NAME}.yml"

.target_test_job_template:
  extends:
    - .target_test_template
  artifacts:
    when: always
    paths:
      - "**/*.log"
      - $LOG_PATH
    exclude:
      - .git/**/*
    reports:
      junit: $LOG_PATH/*/XUNIT_RESULT.xml
    expire_in: 1 week
  variables:
    TEST_FW_PATH: "$CI_PROJECT_DIR/tools/tiny-test-fw"
    LOG_PATH: "$CI_PROJECT_DIR/TEST_LOGS"
    ENV_FILE: "$CI_PROJECT_DIR/ci-test-runner-configs/$CI_RUNNER_DESCRIPTION/EnvConfig.yml"
  script:
    - *define_config_file_name
    # first test if config file exists, if not exist, exit 0
    - |
      { [[ -e $CONFIG_FILE ]]; } || { echo 'No config file found. Consider decreasing the parallel count of this job in ".gitlab/ci/target-test.yml"'; exit 0; }
    # clone test env configs
    - retry_failed git clone $TEST_ENV_CONFIG_REPO
    - python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs
    # git clone the known failure cases repo, run test
    - retry_failed git clone $KNOWN_FAILURE_CASES_REPO known_failure_cases
    # run test
    - cd tools/ci/python_packages/tiny_test_fw/bin
    - run_cmd python Runner.py $TEST_CASE_PATH -c $CONFIG_FILE -e $ENV_FILE --known_failure_cases_file $CI_PROJECT_DIR/known_failure_cases/known_failure_cases.txt

.example_test_template:
  extends: .target_test_job_template
  needs:
    - assign_example_test
  variables:
    TEST_CASE_PATH: "$CI_PROJECT_DIR/examples"
    CONFIG_FILE_PATH: "${CI_PROJECT_DIR}/examples/test_configs"

test_weekend_mqtt:
  extends:
    - .test_app_esp32_template
    - .rules:labels:weekend_test
  tags:
    - ESP32
    - ethernet_router
  script:
    - export MQTT_PUBLISH_TEST=1
    - export TEST_PATH=$CI_PROJECT_DIR/tools/test_apps/protocols/mqtt/publish_connect_test
    - cd $IDF_PATH/tools/ci/python_packages/tiny_test_fw/bin
    - run_cmd python Runner.py $TEST_PATH -c $TEST_PATH/publish_connect_mqtt_.yml

.example_test_esp32_template:
  extends:
    - .example_test_template
    - .rules:test:example_test-esp32

.example_test_esp32s2_template:
  extends:
    - .example_test_template
    - .rules:test:example_test-esp32s2

.example_test_esp32c2_template:
  extends:
    - .example_test_template
    - .rules:test:example_test-esp32c2

.example_test_esp32c3_template:
  extends:
    - .example_test_template
    - .rules:test:example_test-esp32c3

.example_test_esp32c6_template:
  extends:
    - .example_test_template
    - .rules:test:example_test-esp32c6

.example_test_esp32s3_template:
  extends:
    - .example_test_template
    - .rules:test:example_test-esp32s3

example_test_001B_V3:
  extends: .example_test_esp32_template
  tags:
    - ESP32
    - Example_EthKitV12

example_test_001C:
  extends: .example_test_esp32_template
  tags:
    - ESP32
    - Example_GENERIC

example_test_002:
  extends:
    - .example_test_esp32_template
    - .rules:test:example_test-esp32-wifi
  tags:
    - ESP32
    - Example_ShieldBox_Basic

example_test_ethernet_router:
  extends:
    - .example_test_esp32_template
    - .rules:test:example_test-esp32-ethernet
  tags:
    - ESP32
    - ethernet_router

.example_test_003:
  extends: .example_test_esp32_template
  tags:
    - ESP32
    - Example_SDIO

example_test_005:
  extends:
    - .example_test_esp32_template
    - .rules:test:example_test-esp32-bt
  tags:
    - ESP32
    - Example_WIFI_BT

example_test_006:
  extends:
    - .example_test_esp32_template
    - .rules:labels:iperf_stress_test
  tags:
    - ESP32
    - Example_ShieldBox

example_test_007:
  extends: .example_test_esp32_template
  tags:
    - ESP32
    - Example_I2C_CCS811_SENSOR

example_test_011:
  extends: .example_test_esp32_template
  tags:
    - ESP32
    - Example_T2_RS485

example_test_016:
  extends: .example_test_esp32_template
  tags:
    - ESP32
    - Example_Modbus_TCP

example_test_017:
  extends: .example_test_esp32s2_template
  tags:
    - ESP32S2
    - Example_GENERIC

example_test_C3_GENERIC:
  extends: .example_test_esp32c3_template
  tags:
    - ESP32C3
    - Example_GENERIC

example_test_C2_GENERIC:
  extends: .example_test_esp32c2_template
  tags:
    - ESP32C2
    - Example_GENERIC

example_test_C6_GENERIC:
  extends: .example_test_esp32c6_template
  tags:
    - ESP32C6
    - Example_GENERIC

.test_app_template:
  extends: .target_test_job_template
  needs:
    - assign_custom_test
  variables:
    GIT_DEPTH: 1
    TEST_CASE_PATH: "$CI_PROJECT_DIR/tools/test_apps"
    CONFIG_FILE_PATH: "${CI_PROJECT_DIR}/tools/test_apps/test_configs"

.test_app_esp32_template:
  extends:
    - .test_app_template
    - .rules:test:custom_test-esp32

.test_app_esp32s2_template:
  extends:
    - .test_app_template
    - .rules:test:custom_test-esp32s2

.test_app_esp32c2_template:
  extends:
    - .test_app_template
    - .rules:test:custom_test-esp32c2

.test_app_esp32c3_template:
  extends:
    - .test_app_template
    - .rules:test:custom_test-esp32c3

.test_app_esp32c6_template:
  extends:
    - .test_app_template
    - .rules:test:custom_test-esp32c6

.test_app_esp32s3_template:
  extends:
    - .test_app_template
    - .rules:test:custom_test-esp32s3

test_app_test_eth:
  extends: .test_app_esp32_template
  tags:
    - ESP32
    - ethernet_router

.unit_test_template:
  extends: .target_test_job_template
  needs: # the assign already needs all the build jobs
    - assign_unit_test
  variables:
    GIT_DEPTH: 1
    TEST_CASE_PATH: "$CI_PROJECT_DIR/tools/unit-test-app"
    CONFIG_FILE_PATH: "${CI_PROJECT_DIR}/components/idf_test/unit_test/test_configs"

.unit_test_esp32_template:
  extends:
    - .unit_test_template
    - .rules:test:unit_test-esp32

.unit_test_esp32s2_template:
  extends:
    - .unit_test_template
    - .rules:test:unit_test-esp32s2

.unit_test_esp32s3_template:
  extends:
    - .unit_test_template
    - .rules:test:unit_test-esp32s3

.unit_test_esp32c2_template:
  extends:
    - .unit_test_template
    - .rules:test:unit_test-esp32c2

.unit_test_esp32c3_template:
  extends:
    - .unit_test_template
    - .rules:test:unit_test-esp32c3

.unit_test_esp32c6_template:
  extends:
    - .unit_test_template
    - .rules:test:unit_test-esp32c6

UT_001:
  extends: .unit_test_esp32_template
  parallel: 17
  tags:
    - ESP32_IDF
    - UT_T1_1

# Max. allowed value of 'parallel' is 50.

UT_002:
  extends: .unit_test_esp32_template
  parallel: 7
  tags:
    - ESP32_IDF
    - UT_T1_1
    - psram

UT_003:
  extends: .unit_test_esp32_template
  tags:
    - ESP32_IDF
    - UT_T1_SDMODE

UT_004:
  extends: .unit_test_esp32_template
  tags:
    - ESP32_IDF
    - UT_T1_SPIMODE

UT_005:
  extends: .unit_test_esp32_template
  tags:
    - ESP32_IDF
    - UT_T1_SDMODE
    - psram

UT_006:
  extends: .unit_test_esp32_template
  tags:
    - ESP32_IDF
    - UT_T1_SPIMODE
    - psram

UT_017:
  extends: .unit_test_esp32_template
  tags:
    - ESP32_IDF
    - EMMC

UT_022:
  extends: .unit_test_esp32_template
  tags:
    - ESP32_IDF
    - UT_T2_I2C
    - psram

UT_028:
  extends: .unit_test_esp32_template
  tags:
    - ESP32_IDF
    - UT_T2_1
    - psram

UT_033:
  extends: .unit_test_esp32_template
  tags:
    - ESP32_IDF
    - UT_T2_Ethernet
    - psram

UT_034:
  extends: .unit_test_esp32_template
  tags:
    - ESP32_IDF
    - UT_T1_ESP_FLASH

UT_035:
  extends: .unit_test_esp32s2_template
  parallel: 17
  tags:
    - ESP32S2_IDF
    - UT_T1_1

UT_038:
  extends: .unit_test_esp32s2_template
  tags:
    - ESP32S2_IDF
    - UT_T1_ESP_FLASH

UT_S2_SDSPI:
  extends: .unit_test_esp32s2_template
  tags:
    - ESP32S2_IDF
    - UT_T1_SPIMODE

UT_C2:
  extends: .unit_test_esp32c2_template
  parallel: 8
  tags:
    - ESP32C2_IDF
    - UT_T1_1
    - xtal_40mhz

UT_C2_FLASH:
  extends:
    - .unit_test_esp32c2_template
  tags:
    - ESP32C2_IDF
    - UT_T1_ESP_FLASH
    - xtal_40mhz

UT_C3:
  extends: .unit_test_esp32c3_template
  parallel: 11
  tags:
    - ESP32C3_IDF
    - UT_T1_1

UT_C3_FLASH:
  extends: .unit_test_esp32c3_template
  tags:
    - ESP32C3_IDF
    - UT_T1_ESP_FLASH

UT_C3_I2C:
  extends: .unit_test_esp32c3_template
  tags:
    - ESP32C3_IDF
    - UT_T2_I2C

UT_C3_SDSPI:
  extends: .unit_test_esp32c3_template
  tags:
    - ESP32C3_IDF
    - UT_T1_SPIMODE

UT_C6:
  extends: .unit_test_esp32c6_template
  parallel: 12
  tags:
    - ESP32C6_IDF
    - UT_T1_1

UT_S3:
  extends: .unit_test_esp32s3_template
  parallel: 10
  tags:
    - ESP32S3_IDF
    - UT_T1_1

UT_S3_FLASH:
  extends: .unit_test_esp32s3_template
  tags:
    - ESP32S3_IDF
    - UT_T1_ESP_FLASH


UT_S3_QUAD_PSRAM:
  extends: .unit_test_esp32s3_template
  parallel: 2
  tags:
    - ESP32S3_IDF
    - UT_T1_1
    - quad_psram

.integration_test_template:
  extends:
    - .target_test_job_template
    - .rules:test:integration_test
    - .before_script_minimal
  image: ${CI_INTEGRATION_TEST_ENV_IMAGE}
  cache: []
  needs: # the assign already needs all the build jobs
    - assign_integration_test
  variables:
    LOCAL_ENV_CONFIG_PATH: "$CI_PROJECT_DIR/ci-test-runner-configs/$CI_RUNNER_DESCRIPTION/ESP32_IDF"
    LOG_PATH: "${CI_PROJECT_DIR}/TEST_LOGS"
    TEST_CASE_FILE_PATH: "$CI_PROJECT_DIR/auto_test_script/TestCaseFiles"
    CONFIG_FILE_PATH: "${CI_PROJECT_DIR}/tools/ci/integration_test/test_configs"
    KNOWN_ISSUE_FILE: "${CI_PROJECT_DIR}/tools/ci/integration_test/KnownIssues"
    CI_RUNNER_SCRIPT: "${CI_PROJECT_DIR}/auto_test_script/bin/CIRunner.py"
    PREPARE_TEST_BIN_SCRIPT: "${CI_PROJECT_DIR}/auto_test_script/tools/ci/idf_prepare_test_bins.py"
    PYTHONPATH: "${CI_PROJECT_DIR}/auto_test_script/packages:${CI_PROJECT_DIR}/tools/ci/python_packages:${PYTHONPATH}"
    INITIAL_CONDITION_RETRY_COUNT: "1"
    GIT_LFS_SKIP_SMUDGE: 1
  script:
    - *define_config_file_name
    # first test if config file exists, if not exist, exit 0
    - test -e $CONFIG_FILE || exit 0
    # clone local test env configs
    - add_gitlab_ssh_keys
    - retry_failed git clone $TEST_ENV_CONFIG_REPO
    - python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs
    # clone test bench
    - retry_failed git clone ${CI_AUTO_TEST_SCRIPT_REPO_URL} auto_test_script
    - python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script
    - cd auto_test_script
    - ./tools/ci/setup_idfci.sh
    # Merge known issues
    - cat ${KNOWN_ISSUE_FILE} >> ${TEST_CASE_FILE_PATH}/KnownIssues
    # run test
    - python ${PREPARE_TEST_BIN_SCRIPT} $CONFIG_FILE
    - python ${CI_RUNNER_SCRIPT} -l "$LOG_PATH/$JOB_FULL_NAME" -c $CONFIG_FILE -e $LOCAL_ENV_CONFIG_PATH -t $TEST_CASE_FILE_PATH

.integration_test_esp32c3_template:
  extends:
    - .integration_test_template
  variables:
    LOCAL_ENV_CONFIG_PATH: "$CI_PROJECT_DIR/ci-test-runner-configs/$CI_RUNNER_DESCRIPTION/ESP32C3_IDF"

.integration_test_esp32c2_template:
  extends:
    - .integration_test_template
  variables:
    LOCAL_ENV_CONFIG_PATH: "$CI_PROJECT_DIR/ci-test-runner-configs/$CI_RUNNER_DESCRIPTION/ESP32C2_IDF"

nvs_compatible_test:
  extends:
    - .integration_test_template
    - .rules:test:integration_test
  artifacts:
    when: always
    paths:
      - $LOG_PATH
      - nvs_wifi.bin
    expire_in: 1 mos
  tags:
    - ESP32_IDF
    - NVS_Compatible
  script:
    - *define_config_file_name
    # first test if config file exists, if not exist, exit 0
    - test -e $CONFIG_FILE || exit 0
    # clone local test env configs
    - add_gitlab_ssh_keys
    - retry_failed git clone $TEST_ENV_CONFIG_REPO
    - python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs
    # clone test bench
    - retry_failed git clone ${CI_AUTO_TEST_SCRIPT_REPO_URL} auto_test_script
    - python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script
    - cd auto_test_script
    - ./tools/ci/setup_idfci.sh
    # prepare nvs bins
    - ./tools/prepare_nvs_bin.sh
    # run test
    - python ${PREPARE_TEST_BIN_SCRIPT} $CONFIG_FILE
    - python ${CI_RUNNER_SCRIPT} -l "$LOG_PATH/$JOB_FULL_NAME" -c $CONFIG_FILE -e $LOCAL_ENV_CONFIG_PATH -t $TEST_CASE_FILE_PATH

IT_T1_Simple:
  extends:
    - .integration_test_template
    - .rules:test:integration_test_wifi
  parallel: 2
  tags:
    - ESP32_IDF
    - SSC_T1_Simple

IT_T1_WAP:
  extends:
    - .integration_test_template
    - .rules:test:integration_test_wifi
  tags:
    - ESP32_IDF
    - SSC_T1_WAP

IT_T2_Simple:
  extends:
    - .integration_test_template
    - .rules:test:integration_test_wifi
  parallel: 9
  tags:
    - ESP32_IDF
    - SSC_T2_Simple

IT_T5_BT_Simple:
  extends:
    - .integration_test_template
    - .rules:test:integration_test_ble
  parallel: 3
  tags:
    - ESP32_IDF
    - SSC_T5_BT_Simple

IT_T2_BT_Simple:
  extends:
    - .integration_test_template
    - .rules:test:integration_test_ble
  parallel: 6
  tags:
    - ESP32_IDF
    - SSC_T2_BT_Simple

IT_T1_BT_Dongle:
  extends:
    - .integration_test_template
    - .rules:test:integration_test_ble
  parallel: 2
  tags:
    - ESP32_IDF
    - SSC_T1_BT_Dongle

IT_T1_AP:
  extends:
    - .integration_test_template
    - .rules:test:integration_test_wifi
  tags:
    - ESP32_IDF
    - SSC_T1_AP

IT_T1_AP_E:
  extends:
    - .integration_test_template
    - .rules:test:integration_test_wifi
  parallel: 5
  tags:
    - ESP32_IDF
    - SSC_T1_AP_E

IT_T1_WNIC_AP:
  extends:
    - .integration_test_template
    - .rules:test:integration_test_wifi
  tags:
    - ESP32_IDF
    - SSC_T1_WNIC_AP

IT_T1_WNIC_AP_E:
  extends:
    - .integration_test_template
    - .rules:test:integration_test_wifi
  tags:
    - ESP32_IDF
    - SSC_T1_WNIC_AP_E

IT_T1_MESH1:
  extends:
    - .integration_test_template
    - .rules:test:integration_test_wifi
  tags:
    - ESP32_IDF
    - SSC_T1_MESH1

IT_T2_MESH1:
  extends:
    - .integration_test_template
    - .rules:test:integration_test_wifi
  parallel: 2
  tags:
    - ESP32_IDF
    - SSC_T2_MESH1

IT_T3_MESH1:
  extends:
    - .integration_test_template
    - .rules:test:integration_test_wifi
  tags:
    - ESP32_IDF
    - SSC_T3_MESH1

IT_T6_MESH1:
  extends:
    - .integration_test_template
    - .rules:test:integration_test_wifi
  tags:
    - ESP32_IDF
    - SSC_T6_MESH1

IT_T12_MESH1:
  extends:
    - .integration_test_template
    - .rules:test:integration_test_wifi
  tags:
    - ESP32_IDF
    - SSC_T12_MESH1

IT_T50_MESH1:
  extends:
    - .integration_test_template
    - .rules:test:integration_test_wifi
  allow_failure: true
  tags:
    - ESP32_IDF
    - SSC_T50_MESH1

IT_T1_MESH2:
  extends:
    - .integration_test_template
    - .rules:test:integration_test_wifi
  tags:
    - ESP32_IDF
    - SSC_T1_MESH2

IT_T2_WNIC:
  extends:
   - .integration_test_template
   - .rules:test:integration_test_wifi
  tags:
    - ESP32_IDF
    - SSC_T2_WNIC

IT_T2_AP:
  extends:
    - .integration_test_template
    - .rules:test:integration_test_wifi
  tags:
    - ESP32_IDF
    - SSC_T2_AP

IT_T3_Simple:
  extends:
    - .integration_test_template
    - .rules:test:integration_test_wifi
  tags:
    - ESP32_IDF
    - SSC_T3_Simple

IT_C3_T2_BT_Simple:
  extends:
    - .integration_test_esp32c3_template
    - .rules:test:integration_test_ble
  parallel: 11
  tags:
    - ESP32C3_IDF
    - SSC_T2_BT_Simple

IT_C3_T5_BT_Simple:
  extends:
    - .integration_test_esp32c3_template
    - .rules:test:integration_test_ble
  parallel: 5
  tags:
    - ESP32C3_IDF
    - SSC_T5_BT_Simple

IT_C3_T1_BT_Dongle:
  extends:
    - .integration_test_esp32c3_template
    - .rules:test:integration_test_ble
  image: gitlab.espressif.cn:5050/qa/dockerfiles/integration-test-env-dbg:1
  parallel: 2
  tags:
    - ESP32C3_IDF
    - SSC_T1_BT_Dongle

IT_C2_T2_BT_Simple:
  extends:
    - .integration_test_esp32c2_template
    - .rules:test:integration_test_ble
  parallel: 9
  tags:
    - ESP32C2_IDF
    - SSC_T2_BT_Simple

IT_C2_T1_BT_Dongle:
  extends:
    - .integration_test_esp32c2_template
    - .rules:test:integration_test_ble
  image: gitlab.espressif.cn:5050/qa/dockerfiles/integration-test-env-dbg:1
  parallel: 2
  tags:
    - ESP32C2_IDF
    - SSC_T1_BT_Dongle