esp-idf/.gitlab/ci/host-test.yml

.host_test_template:
  extends: .rules:test:host_test
  stage: host_test
  image: $ESP_ENV_IMAGE
  tags:
    - host_test
  dependencies: # set dependencies to null to avoid missing artifacts issue
  # run host_test jobs immediately, only after upload cache
  needs:
    - job: upload-pip-cache
      optional: true
      artifacts: false
    - job: upload-submodules-cache
      optional: true
      artifacts: false
    - pipeline_variables

test_nvs_on_host:
  extends: .host_test_template
  script:
    - cd components/nvs_flash/test_nvs_host
    - make test

test_nvs_coverage:
  extends:
    - .host_test_template
    - .rules:labels:nvs_coverage
  artifacts:
    paths:
      - components/nvs_flash/test_nvs_host/coverage_report
    expire_in: 1 week
  script:
    - cd components/nvs_flash/test_nvs_host
    - make coverage_report
  # the 'long' host tests take approx 11 hours on our current runners. Adding some margin here for possible CPU contention
  timeout: 18 hours

test_partition_table_on_host:
  extends: .host_test_template
  script:
    - cd components/partition_table/test_gen_esp32part_host
    - ./gen_esp32part_tests.py

test_ldgen_on_host:
  extends: .host_test_template
  script:
    - cd tools/ldgen/test
    - export PYTHONPATH=$PYTHONPATH:..
    - python -m unittest
  variables:
    LC_ALL: C.UTF-8

test_reproducible_build:
  extends: .host_test_template
  script:
    - ./tools/ci/test_reproducible_build.sh
  artifacts:
    when: on_failure
    paths:
      - "**/sdkconfig"
      - "**/build*/*.bin"
      - "**/build*/*.elf"
      - "**/build*/*.map"
      - "**/build*/flasher_args.json"
      - "**/build*/*.bin"
      - "**/build*/bootloader/*.bin"
      - "**/build*/partition_table/*.bin"
    expire_in: 1 week

test_spiffs_on_host:
  extends: .host_test_template
  script:
    - cd components/spiffs/test_spiffsgen/
    - ./test_spiffsgen.py

test_fatfsgen_on_host:
  extends: .host_test_template
  script:
    - cd components/fatfs/test_fatfsgen/
    - ./test_fatfsgen.py
    - ./test_wl_fatfsgen.py
    - ./test_fatfsparse.py

test_multi_heap_on_host:
  extends: .host_test_template
  script:
    - cd components/heap/test_multi_heap_host
    - ./test_all_configs.sh

test_certificate_bundle_on_host:
  extends: .host_test_template
  script:
    - cd components/mbedtls/esp_crt_bundle/test_gen_crt_bundle/
    - ./test_gen_crt_bundle.py

test_gdbstub_on_host:
  extends: .host_test_template
  script:
    - cd components/esp_gdbstub/test_gdbstub_host
    - make test

# Test for create virtualenv. It must be invoked from Python, not from virtualenv.
# Use docker image system python without any extra dependencies
test_cli_installer:
  extends:
    - .host_test_template
    - .before_script:minimal
  artifacts:
    when: on_failure
    paths:
      - tools/tools.new.json
      - tools/test_idf_tools/test_python_env_logs.txt
    expire_in: 1 week
  image:
    name: $ESP_ENV_IMAGE
    entrypoint: [""]  # use system python3. no extra pip package installed
  script:
    # Tools must be downloaded for testing
    - python3 ${IDF_PATH}/tools/idf_tools.py download required qemu-riscv32 qemu-xtensa
    - cd ${IDF_PATH}/tools/test_idf_tools
    - python3 -m pip install jsonschema
    - python3 ./test_idf_tools.py -v
    - python3 ./test_idf_tools_python_env.py

.test_efuse_table_on_host_template:
  extends: .host_test_template
  variables:
    IDF_TARGET: "esp32"
  artifacts:
    when: on_failure
    paths:
      - components/efuse/${IDF_TARGET}/esp_efuse_table.c
    expire_in: 1 week
  script:
    - cd ${IDF_PATH}/components/efuse/
    - ./efuse_table_gen.py -t "${IDF_TARGET}" ${IDF_PATH}/components/efuse/${IDF_TARGET}/esp_efuse_table.csv
    - git diff --exit-code -- ${IDF_TARGET}/esp_efuse_table.c || { echo 'Differences found for ${IDF_TARGET} target. Please run idf.py efuse-common-table and commit the changes.'; exit 1; }
    - cd ${IDF_PATH}/components/efuse/test_efuse_host
    - ./efuse_tests.py

test_efuse_table_on_host_esp32:
  extends: .test_efuse_table_on_host_template

test_efuse_table_on_host_esp32s2:
  extends: .test_efuse_table_on_host_template
  variables:
    IDF_TARGET: esp32s2

test_efuse_table_on_host_esp32s3:
  extends: .test_efuse_table_on_host_template
  variables:
    IDF_TARGET: esp32s3

test_efuse_table_on_host_esp32c3:
  extends: .test_efuse_table_on_host_template
  variables:
    IDF_TARGET: esp32c3

test_efuse_table_on_host_esp32h2:
  extends: .test_efuse_table_on_host_template
  variables:
    IDF_TARGET: esp32h2

test_efuse_table_on_host_esp32c6:
  extends: .test_efuse_table_on_host_template
  variables:
    IDF_TARGET: esp32c6

test_logtrace_proc:
  extends: .host_test_template
  artifacts:
    when: on_failure
    paths:
      - tools/esp_app_trace/test/logtrace/output
      - tools/esp_app_trace/test/logtrace/.coverage
    expire_in: 1 week
  script:
    - cd ${IDF_PATH}/tools/esp_app_trace/test/logtrace
    - ./test.sh

test_sysviewtrace_proc:
  extends: .host_test_template
  artifacts:
    when: on_failure
    paths:
      - tools/esp_app_trace/test/sysview/output
      - tools/esp_app_trace/test/sysview/.coverage
    expire_in: 1 week
  script:
    - cd ${IDF_PATH}/tools/esp_app_trace/test/sysview
    - ./test.sh

test_tools:
  extends:
    - .host_test_template
  artifacts:
    when: always
    paths:
      - ${IDF_PATH}/*.out
      - ${IDF_PATH}/XUNIT_*.xml
    reports:
      junit: ${IDF_PATH}/XUNIT_*.xml
    expire_in: 1 week
  variables:
    LC_ALL: C.UTF-8
    INSTALL_QEMU: 1  # for test_idf_qemu.py
  script:
    - stat=0
    - cd ${IDF_PATH}/tools/ci/test_autocomplete
    - pytest --noconftest test_autocomplete.py --junitxml=${IDF_PATH}/XUNIT_AUTOCOMP.xml                  || stat=1
    - cd ${IDF_PATH}/tools/test_idf_py
    - pytest --noconftest test_idf_py.py       --junitxml=${IDF_PATH}/XUNIT_IDF_PY.xml                    || stat=1
    - pytest --noconftest test_hints.py        --junitxml=${IDF_PATH}/XUNIT_HINTS.xml                     || stat=1
    - pytest --noconftest test_idf_qemu.py     --junitxml=${IDF_PATH}/XUNIT_IDF_PY_QEMU.xml               || stat=1
    - cd ${IDF_PATH}/tools/test_mkdfu
    - pytest --noconftest test_mkdfu.py        --junitxml=${IDF_PATH}/XUNIT_MKDFU.xml                     || stat=1
    - cd ${IDF_PATH}
    - shellcheck -s sh tools/detect_python.sh                                                             || stat=1
    - shellcheck -s bash tools/detect_python.sh                                                           || stat=1
    - shellcheck -s dash tools/detect_python.sh                                                           || stat=1
    - "bash -c '. tools/detect_python.sh && echo Our Python: ${ESP_PYTHON?Python is not set}'"
    - "dash -c '. tools/detect_python.sh && echo Our Python: ${ESP_PYTHON?Python is not set}'"
    - "zsh -c '. tools/detect_python.sh && echo Our Python: ${ESP_PYTHON?Python is not set}'"
    - "fish -c 'source tools/detect_python.fish && echo Our Python: $ESP_PYTHON'"
    - exit "$stat"

test_split_path_by_spaces:
  extends: .host_test_template
  script:
    - cd ${IDF_PATH}/tools
    - python -m unittest split_paths_by_spaces.py

test_mqtt_on_host:
  extends: .host_test_template
  script:
    - cd ${IDF_PATH}/components/mqtt/esp-mqtt/host_test
    - idf.py build
    - LSAN_OPTIONS=verbosity=1:log_threads=1 build/host_mqtt_client_test.elf

test_transport_on_host:
  extends: .host_test_template
  script:
    - cd ${IDF_PATH}/components/tcp_transport/host_test
    - idf.py build
    - LSAN_OPTIONS=verbosity=1:log_threads=1 build/host_tcp_transport_test.elf

test_sockets_on_host:
  extends: .host_test_template
  script:
    # test the tcp-client example with system sockets
    - cd ${IDF_PATH}/examples/protocols/sockets/tcp_client
    - echo 'CONFIG_EXAMPLE_IPV4_ADDR="127.0.0.1"' >> sdkconfig.defaults
    - idf.py --preview set-target linux
    - idf.py build
    - timeout 5 ./build/tcp_client.elf >test.log || true
    - grep "Socket unable to connect" test.log
    # test the udp-client example with lwip sockets
    - cd ${IDF_PATH}/examples/protocols/sockets/udp_client
    - idf.py --preview set-target linux
    - cat sdkconfig.ci.linux > sdkconfig
    - idf.py build
    - timeout 5 ./build/udp_client.elf >test.log || true
    - grep "Message sent" test.log

test_eh_frame_parser:
  extends: .host_test_template
  script:
    - cd ${IDF_PATH}/components/esp_system/test_eh_frame_parser
    - make
    - ./eh_frame_test

test_gen_soc_caps_kconfig:
  extends: .host_test_template
  script:
    - cd ${IDF_PATH}/tools/gen_soc_caps_kconfig/
    - ./test/test_gen_soc_caps_kconfig.py

test_pytest_qemu:
  extends:
    - .host_test_template
    - .before_script:build
  artifacts:
    when: always
    paths:
      - XUNIT_RESULT.xml
      - pytest_embedded_log/
    reports:
      junit: XUNIT_RESULT.xml
    expire_in: 1 week
  allow_failure: true # IDFCI-1752
  parallel:
    matrix:
      - IDF_TARGET: [esp32, esp32c3]
  variables:
    INSTALL_QEMU: 1
  script:
    - run_cmd python tools/ci/ci_build_apps.py . -vv
      --target $IDF_TARGET
      --pytest-apps
      -m qemu
      --collect-app-info "list_job_${CI_JOB_NAME_SLUG}.txt"
      --modified-components ${MODIFIED_COMPONENTS}
      --modified-files ${MODIFIED_FILES}
    - retry_failed git clone $KNOWN_FAILURE_CASES_REPO known_failure_cases
    - run_cmd pytest
      --target $IDF_TARGET
      -m qemu
      --embedded-services idf,qemu
      --junitxml=XUNIT_RESULT.xml
      --ignore-result-files known_failure_cases/known_failure_cases.txt
      --app-info-filepattern \"list_job_*.txt\"

test_pytest_linux:
  extends:
    - .host_test_template
    - .before_script:build
  artifacts:
    when: always
    paths:
      - XUNIT_RESULT.xml
      - pytest_embedded_log/
      - "**/build*/build_log.txt"
    reports:
      junit: XUNIT_RESULT.xml
    expire_in: 1 week
  script:
    - run_cmd python tools/ci/ci_build_apps.py components examples tools/test_apps -vv
      --target linux
      --pytest-apps
      -m host_test
      --collect-app-info "list_job_${CI_JOB_NAME_SLUG}.txt"
      --modified-components ${MODIFIED_COMPONENTS}
      --modified-files ${MODIFIED_FILES}
    - retry_failed git clone $KNOWN_FAILURE_CASES_REPO known_failure_cases
    - run_cmd pytest
      --target linux
      -m host_test
      --junitxml=XUNIT_RESULT.xml
      --ignore-result-files known_failure_cases/known_failure_cases.txt
      --app-info-filepattern \"list_job_*.txt\"