Merge branch 'bugfix/improve_ota_test_success_rate_v4.4' into 'release/v4.4'

OTA examples: Reduce example test failure rate (v4.4) See merge request espressif/esp-idf!16613
2024-09-20 10:46:02 -04:00 · 2022-02-10 05:52:27 +00:00 · 2022-02-10 05:52:27 +00:00 · 14b52e9d83
commit 14b52e9d83
parent c7b9f94733 4ecd0ef9fc
7 changed files with 70 additions and 34 deletions
--- a/.gitlab/ci/target-test.yml
+++ b/.gitlab/ci/target-test.yml
@ -117,10 +117,9 @@ example_test_001D:

 example_test_OTA:
  extends: .example_test_esp32_template
-  parallel: 2
  tags:
    - ESP32
-    - Example_WIFI_OTA
+    - EXAMPLE_ETH_OTA

 example_test_protocols:
  extends: .example_test_esp32_template
@ -129,6 +128,13 @@ example_test_protocols:
    - ESP32
    - Example_WIFI_Protocols

+# This job is only triggered by env var `NIGHTLY_RUN`, please do NOT remove
+example_test_esp32_WIFI_OTA:
+  extends: .example_test_esp32_template
+  tags:
+    - ESP32
+    - Example_WIFI_OTA
+
 example_test_002:
  extends: .example_test_esp32_template
  image: $CI_DOCKER_REGISTRY/ubuntu-test-env$BOT_DOCKER_IMAGE_TAG
--- a/examples/system/ota/advanced_https_ota/example_test.py
+++ b/examples/system/ota/advanced_https_ota/example_test.py
@ -105,7 +105,7 @@ def start_redirect_server(ota_image_dir, server_ip, server_port, redirection_por
    httpd.serve_forever()


-@ttfw_idf.idf_example_test(env_tag='Example_WIFI_OTA')
+@ttfw_idf.idf_example_test(env_tag='EXAMPLE_ETH_OTA')
 def test_examples_protocol_advanced_https_ota_example(env, extra_data):
    """
    This is a positive test case, which downloads complete binary file multiple number of times.
@ -135,7 +135,7 @@ def test_examples_protocol_advanced_https_ota_example(env, extra_data):
    for i in range(iterations):
        dut1.expect('Loaded app from partition at offset', timeout=30)
        try:
-            ip_address = dut1.expect(re.compile(r' sta ip: ([^,]+),'), timeout=30)
+            ip_address = dut1.expect(re.compile(r' (sta|eth) ip: ([^,]+),'), timeout=30)
            print('Connected to AP with IP: {}'.format(ip_address))
        except DUT.ExpectTimeout:
            raise ValueError('ENV_TEST_FAILURE: Cannot connect to AP')
@ -150,7 +150,7 @@ def test_examples_protocol_advanced_https_ota_example(env, extra_data):
    thread1.terminate()


-@ttfw_idf.idf_example_test(env_tag='Example_WIFI_OTA')
+@ttfw_idf.idf_example_test(env_tag='EXAMPLE_ETH_OTA')
 def test_examples_protocol_advanced_https_ota_example_truncated_bin(env, extra_data):
    """
    Working of OTA if binary file is truncated is validated in this test case.
@ -189,7 +189,7 @@ def test_examples_protocol_advanced_https_ota_example_truncated_bin(env, extra_d
    dut1.start_app()
    dut1.expect('Loaded app from partition at offset', timeout=30)
    try:
-        ip_address = dut1.expect(re.compile(r' sta ip: ([^,]+),'), timeout=30)
+        ip_address = dut1.expect(re.compile(r' (sta|eth) ip: ([^,]+),'), timeout=30)
        print('Connected to AP with IP: {}'.format(ip_address))
    except DUT.ExpectTimeout:
        raise ValueError('ENV_TEST_FAILURE: Cannot connect to AP')
@ -203,7 +203,7 @@ def test_examples_protocol_advanced_https_ota_example_truncated_bin(env, extra_d
    thread1.terminate()


-@ttfw_idf.idf_example_test(env_tag='Example_WIFI_OTA')
+@ttfw_idf.idf_example_test(env_tag='EXAMPLE_ETH_OTA')
 def test_examples_protocol_advanced_https_ota_example_truncated_header(env, extra_data):
    """
    Working of OTA if headers of binary file are truncated is vaildated in this test case.
@ -241,7 +241,7 @@ def test_examples_protocol_advanced_https_ota_example_truncated_header(env, extr
    dut1.start_app()
    dut1.expect('Loaded app from partition at offset', timeout=30)
    try:
-        ip_address = dut1.expect(re.compile(r' sta ip: ([^,]+),'), timeout=30)
+        ip_address = dut1.expect(re.compile(r' (sta|eth) ip: ([^,]+),'), timeout=30)
        print('Connected to AP with IP: {}'.format(ip_address))
    except DUT.ExpectTimeout:
        raise ValueError('ENV_TEST_FAILURE: Cannot connect to AP')
@ -255,7 +255,7 @@ def test_examples_protocol_advanced_https_ota_example_truncated_header(env, extr
    thread1.terminate()


-@ttfw_idf.idf_example_test(env_tag='Example_WIFI_OTA')
+@ttfw_idf.idf_example_test(env_tag='EXAMPLE_ETH_OTA')
 def test_examples_protocol_advanced_https_ota_example_random(env, extra_data):
    """
    Working of OTA if random data is added in binary file are validated in this test case.
@ -292,7 +292,7 @@ def test_examples_protocol_advanced_https_ota_example_random(env, extra_data):
    dut1.start_app()
    dut1.expect('Loaded app from partition at offset', timeout=30)
    try:
-        ip_address = dut1.expect(re.compile(r' sta ip: ([^,]+),'), timeout=30)
+        ip_address = dut1.expect(re.compile(r' (sta|eth) ip: ([^,]+),'), timeout=30)
        print('Connected to AP with IP: {}'.format(ip_address))
    except DUT.ExpectTimeout:
        raise ValueError('ENV_TEST_FAILURE: Cannot connect to AP')
@ -306,7 +306,7 @@ def test_examples_protocol_advanced_https_ota_example_random(env, extra_data):
    thread1.terminate()


-@ttfw_idf.idf_example_test(env_tag='Example_WIFI_OTA')
+@ttfw_idf.idf_example_test(env_tag='EXAMPLE_ETH_OTA')
 def test_examples_protocol_advanced_https_ota_example_chunked(env, extra_data):
    """
    This is a positive test case, which downloads complete binary file multiple number of times.
@ -329,7 +329,7 @@ def test_examples_protocol_advanced_https_ota_example_chunked(env, extra_data):
    dut1.start_app()
    dut1.expect('Loaded app from partition at offset', timeout=30)
    try:
-        ip_address = dut1.expect(re.compile(r' sta ip: ([^,]+),'), timeout=30)
+        ip_address = dut1.expect(re.compile(r' (sta|eth) ip: ([^,]+),'), timeout=30)
        print('Connected to AP with IP: {}'.format(ip_address))
    except DUT.ExpectTimeout:
        raise ValueError('ENV_TEST_FAILURE: Cannot connect to AP')
@ -342,7 +342,7 @@ def test_examples_protocol_advanced_https_ota_example_chunked(env, extra_data):
    chunked_server.kill()


-@ttfw_idf.idf_example_test(env_tag='Example_WIFI_OTA')
+@ttfw_idf.idf_example_test(env_tag='EXAMPLE_ETH_OTA')
 def test_examples_protocol_advanced_https_ota_example_redirect_url(env, extra_data):
    """
    This is a positive test case, which starts a server and a redirection server.
@ -375,7 +375,7 @@ def test_examples_protocol_advanced_https_ota_example_redirect_url(env, extra_da
    dut1.start_app()
    dut1.expect('Loaded app from partition at offset', timeout=30)
    try:
-        ip_address = dut1.expect(re.compile(r' sta ip: ([^,]+),'), timeout=30)
+        ip_address = dut1.expect(re.compile(r' (sta|eth) ip: ([^,]+),'), timeout=30)
        print('Connected to AP with IP: {}'.format(ip_address))
    except DUT.ExpectTimeout:
        raise ValueError('ENV_TEST_FAILURE: Cannot connect to AP')
@ -436,7 +436,7 @@ def test_examples_protocol_advanced_https_ota_example_anti_rollback(env, extra_d
    # Positive Case
    dut1.expect('Loaded app from partition at offset', timeout=30)
    try:
-        ip_address = dut1.expect(re.compile(r' eth ip: ([^,]+),'), timeout=30)
+        ip_address = dut1.expect(re.compile(r' (sta|eth) ip: ([^,]+),'), timeout=30)
        print('Connected to AP with IP: {}'.format(ip_address))
    except DUT.ExpectTimeout:
        raise ValueError('ENV_TEST_FAILURE: Cannot connect to AP')
@ -447,7 +447,7 @@ def test_examples_protocol_advanced_https_ota_example_anti_rollback(env, extra_d
    print('writing to device: {}'.format('https://' + host_ip + ':' + str(server_port) + '/' + bin_name))
    dut1.write('https://' + host_ip + ':' + str(server_port) + '/' + bin_name)
    dut1.expect('Loaded app from partition at offset', timeout=60)
-    dut1.expect(re.compile(r' eth ip: ([^,]+),'), timeout=30)
+    dut1.expect(re.compile(r' (sta|eth) ip: ([^,]+),'), timeout=30)
    dut1.expect('App is valid, rollback cancelled successfully', 30)

    # Negative Case
@ -460,7 +460,7 @@ def test_examples_protocol_advanced_https_ota_example_anti_rollback(env, extra_d
    thread1.terminate()


-@ttfw_idf.idf_example_test(env_tag='Example_WIFI_OTA')
+@ttfw_idf.idf_example_test(env_tag='EXAMPLE_ETH_OTA')
 def test_examples_protocol_advanced_https_ota_example_partial_request(env, extra_data):
    """
    This is a positive test case, to test OTA workflow with Range HTTP header.
@ -489,7 +489,7 @@ def test_examples_protocol_advanced_https_ota_example_partial_request(env, extra
    dut1.start_app()
    dut1.expect('Loaded app from partition at offset', timeout=30)
    try:
-        ip_address = dut1.expect(re.compile(r' sta ip: ([^,]+),'), timeout=30)
+        ip_address = dut1.expect(re.compile(r' (sta|eth) ip: ([^,]+),'), timeout=30)
        print('Connected to AP with IP: {}'.format(ip_address))
    except DUT.ExpectTimeout:
        Utility.console_log('ENV_TEST_FAILURE: Cannot connect to AP')
@ -507,7 +507,7 @@ def test_examples_protocol_advanced_https_ota_example_partial_request(env, extra
    thread1.terminate()


-@ttfw_idf.idf_example_test(env_tag='Example_WIFI_OTA')
+@ttfw_idf.idf_example_test(env_tag='Example_WIFI_OTA', nightly_run=True)
 def test_examples_protocol_advanced_https_ota_example_nimble_gatts(env, extra_data):
    """
    Run an OTA image update while a BLE GATT Server is running in background. This GATT server will be using NimBLE Host stack.
@ -552,7 +552,7 @@ def test_examples_protocol_advanced_https_ota_example_nimble_gatts(env, extra_da
    thread1.terminate()


-@ttfw_idf.idf_example_test(env_tag='Example_WIFI_OTA')
+@ttfw_idf.idf_example_test(env_tag='Example_WIFI_OTA', nightly_run=True)
 def test_examples_protocol_advanced_https_ota_example_bluedroid_gatts(env, extra_data):
    """
    Run an OTA image update while a BLE GATT Server is running in background. This GATT server will be using Bluedroid Host stack.
@ -597,7 +597,7 @@ def test_examples_protocol_advanced_https_ota_example_bluedroid_gatts(env, extra
    thread1.terminate()


-@ttfw_idf.idf_example_test(env_tag='Example_WIFI_OTA')
+@ttfw_idf.idf_example_test(env_tag='EXAMPLE_ETH_OTA')
 def test_examples_protocol_advanced_https_ota_example_openssl_aligned_bin(env, extra_data):
    """
    This is a test case for esp_http_client_read with binary size multiple of 289 bytes
@ -630,7 +630,7 @@ def test_examples_protocol_advanced_https_ota_example_openssl_aligned_bin(env, e
    dut1.start_app()
    dut1.expect('Loaded app from partition at offset', timeout=30)
    try:
-        ip_address = dut1.expect(re.compile(r' sta ip: ([^,]+),'), timeout=30)
+        ip_address = dut1.expect(re.compile(r' (sta|eth) ip: ([^,]+),'), timeout=30)
        print('Connected to AP with IP: {}'.format(ip_address))
    except DUT.ExpectTimeout:
        raise ValueError('ENV_TEST_FAILURE: Cannot connect to AP')
--- a/examples/system/ota/advanced_https_ota/sdkconfig.ci
+++ b/examples/system/ota/advanced_https_ota/sdkconfig.ci
@ -2,3 +2,13 @@ CONFIG_EXAMPLE_FIRMWARE_UPGRADE_URL="FROM_STDIN"
 CONFIG_EXAMPLE_SKIP_COMMON_NAME_CHECK=y
 CONFIG_EXAMPLE_SKIP_VERSION_CHECK=y
 CONFIG_EXAMPLE_OTA_RECV_TIMEOUT=3000
+
+CONFIG_EXAMPLE_CONNECT_ETHERNET=y
+CONFIG_EXAMPLE_CONNECT_WIFI=n
+CONFIG_EXAMPLE_USE_INTERNAL_ETHERNET=y
+CONFIG_EXAMPLE_ETH_PHY_IP101=y
+CONFIG_EXAMPLE_ETH_MDC_GPIO=23
+CONFIG_EXAMPLE_ETH_MDIO_GPIO=18
+CONFIG_EXAMPLE_ETH_PHY_RST_GPIO=5
+CONFIG_EXAMPLE_ETH_PHY_ADDR=1
+CONFIG_EXAMPLE_CONNECT_IPV6=y
--- a/examples/system/ota/advanced_https_ota/sdkconfig.ci.partial_download
+++ b/examples/system/ota/advanced_https_ota/sdkconfig.ci.partial_download
@ -5,3 +5,13 @@ CONFIG_EXAMPLE_OTA_RECV_TIMEOUT=3000
 CONFIG_EXAMPLE_ENABLE_PARTIAL_HTTP_DOWNLOAD=y

 CONFIG_LOG_DEFAULT_LEVEL_DEBUG=y
+
+CONFIG_EXAMPLE_CONNECT_ETHERNET=y
+CONFIG_EXAMPLE_CONNECT_WIFI=n
+CONFIG_EXAMPLE_USE_INTERNAL_ETHERNET=y
+CONFIG_EXAMPLE_ETH_PHY_IP101=y
+CONFIG_EXAMPLE_ETH_MDC_GPIO=23
+CONFIG_EXAMPLE_ETH_MDIO_GPIO=18
+CONFIG_EXAMPLE_ETH_PHY_RST_GPIO=5
+CONFIG_EXAMPLE_ETH_PHY_ADDR=1
+CONFIG_EXAMPLE_CONNECT_IPV6=y
--- a/examples/system/ota/native_ota_example/example_test.py
+++ b/examples/system/ota/native_ota_example/example_test.py
@ -134,7 +134,7 @@ def start_chunked_server(ota_image_dir, server_port):
    return chunked_server


-@ttfw_idf.idf_example_test(env_tag='Example_WIFI_OTA')
+@ttfw_idf.idf_example_test(env_tag='EXAMPLE_ETH_OTA')
 def test_examples_protocol_native_ota_example(env, extra_data):
    """
    This is a positive test case, which downloads complete binary file multiple number of times.
@ -164,7 +164,7 @@ def test_examples_protocol_native_ota_example(env, extra_data):
    for i in range(iterations):
        dut1.expect('Loaded app from partition at offset', timeout=30)
        try:
-            ip_address = dut1.expect(re.compile(r' sta ip: ([^,]+),'), timeout=30)
+            ip_address = dut1.expect(re.compile(r' (sta|eth) ip: ([^,]+),'), timeout=30)
            print('Connected to AP with IP: {}'.format(ip_address))
        except DUT.ExpectTimeout:
            raise ValueError('ENV_TEST_FAILURE: Cannot connect to AP')
@ -179,7 +179,7 @@ def test_examples_protocol_native_ota_example(env, extra_data):
    thread1.terminate()


-@ttfw_idf.idf_example_test(env_tag='Example_WIFI_OTA')
+@ttfw_idf.idf_example_test(env_tag='EXAMPLE_ETH_OTA')
 def test_examples_protocol_native_ota_example_truncated_bin(env, extra_data):
    """
    Working of OTA if binary file is truncated is validated in this test case.
@ -218,7 +218,7 @@ def test_examples_protocol_native_ota_example_truncated_bin(env, extra_data):
    dut1.start_app()
    dut1.expect('Loaded app from partition at offset', timeout=30)
    try:
-        ip_address = dut1.expect(re.compile(r' sta ip: ([^,]+),'), timeout=60)
+        ip_address = dut1.expect(re.compile(r' (sta|eth) ip: ([^,]+),'), timeout=60)
        print('Connected to AP with IP: {}'.format(ip_address))
    except DUT.ExpectTimeout:
        raise ValueError('ENV_TEST_FAILURE: Cannot connect to AP')
@ -232,7 +232,7 @@ def test_examples_protocol_native_ota_example_truncated_bin(env, extra_data):
    thread1.terminate()


-@ttfw_idf.idf_example_test(env_tag='Example_WIFI_OTA')
+@ttfw_idf.idf_example_test(env_tag='EXAMPLE_ETH_OTA')
 def test_examples_protocol_native_ota_example_truncated_header(env, extra_data):
    """
    Working of OTA if headers of binary file are truncated is vaildated in this test case.
@ -270,7 +270,7 @@ def test_examples_protocol_native_ota_example_truncated_header(env, extra_data):
    dut1.start_app()
    dut1.expect('Loaded app from partition at offset', timeout=30)
    try:
-        ip_address = dut1.expect(re.compile(r' sta ip: ([^,]+),'), timeout=60)
+        ip_address = dut1.expect(re.compile(r' (sta|eth) ip: ([^,]+),'), timeout=60)
        print('Connected to AP with IP: {}'.format(ip_address))
    except DUT.ExpectTimeout:
        raise ValueError('ENV_TEST_FAILURE: Cannot connect to AP')
@ -284,7 +284,7 @@ def test_examples_protocol_native_ota_example_truncated_header(env, extra_data):
    thread1.terminate()


-@ttfw_idf.idf_example_test(env_tag='Example_WIFI_OTA')
+@ttfw_idf.idf_example_test(env_tag='EXAMPLE_ETH_OTA')
 def test_examples_protocol_native_ota_example_random(env, extra_data):
    """
    Working of OTA if random data is added in binary file are validated in this test case.
@ -321,7 +321,7 @@ def test_examples_protocol_native_ota_example_random(env, extra_data):
    dut1.start_app()
    dut1.expect('Loaded app from partition at offset', timeout=30)
    try:
-        ip_address = dut1.expect(re.compile(r' sta ip: ([^,]+),'), timeout=60)
+        ip_address = dut1.expect(re.compile(r' (sta|eth) ip: ([^,]+),'), timeout=60)
        print('Connected to AP with IP: {}'.format(ip_address))
    except DUT.ExpectTimeout:
        raise ValueError('ENV_TEST_FAILURE: Cannot connect to AP')
@ -335,7 +335,7 @@ def test_examples_protocol_native_ota_example_random(env, extra_data):
    thread1.terminate()


-@ttfw_idf.idf_example_test(env_tag='Example_WIFI_OTA')
+@ttfw_idf.idf_example_test(env_tag='EXAMPLE_ETH_OTA')
 def test_examples_protocol_native_ota_example_chunked(env, extra_data):
    """
    This is a positive test case, which downloads complete binary file multiple number of times.
@ -358,7 +358,7 @@ def test_examples_protocol_native_ota_example_chunked(env, extra_data):
    dut1.start_app()
    dut1.expect('Loaded app from partition at offset', timeout=30)
    try:
-        ip_address = dut1.expect(re.compile(r' sta ip: ([^,]+),'), timeout=30)
+        ip_address = dut1.expect(re.compile(r' (sta|eth) ip: ([^,]+),'), timeout=30)
        print('Connected to AP with IP: {}'.format(ip_address))
    except DUT.ExpectTimeout:
        raise ValueError('ENV_TEST_FAILURE: Cannot connect to AP')
--- a/examples/system/ota/native_ota_example/sdkconfig.ci
+++ b/examples/system/ota/native_ota_example/sdkconfig.ci
@ -2,3 +2,13 @@ CONFIG_EXAMPLE_FIRMWARE_UPG_URL="FROM_STDIN"
 CONFIG_EXAMPLE_SKIP_COMMON_NAME_CHECK=y
 CONFIG_EXAMPLE_SKIP_VERSION_CHECK=y
 CONFIG_EXAMPLE_OTA_RECV_TIMEOUT=3000
+
+CONFIG_EXAMPLE_CONNECT_ETHERNET=y
+CONFIG_EXAMPLE_CONNECT_WIFI=n
+CONFIG_EXAMPLE_USE_INTERNAL_ETHERNET=y
+CONFIG_EXAMPLE_ETH_PHY_IP101=y
+CONFIG_EXAMPLE_ETH_MDC_GPIO=23
+CONFIG_EXAMPLE_ETH_MDIO_GPIO=18
+CONFIG_EXAMPLE_ETH_PHY_RST_GPIO=5
+CONFIG_EXAMPLE_ETH_PHY_ADDR=1
+CONFIG_EXAMPLE_CONNECT_IPV6=y
--- a/examples/system/ota/simple_ota_example/example_test.py
+++ b/examples/system/ota/simple_ota_example/example_test.py
@ -123,7 +123,7 @@ def calc_all_sha256(dut):
    return sha256_bootloader, sha256_app


-@ttfw_idf.idf_example_test(env_tag='Example_WIFI_OTA')
+@ttfw_idf.idf_example_test(env_tag='Example_WIFI_OTA', nightly_run=True)
 def test_examples_protocol_simple_ota_example(env, extra_data):
    """
    steps: |
@ -239,7 +239,7 @@ def test_examples_protocol_simple_ota_example_with_flash_encryption(env, extra_d
    thread1.terminate()


-@ttfw_idf.idf_example_test(env_tag='Example_Flash_Encryption_OTA_WiFi', target=['esp32c3'])
+@ttfw_idf.idf_example_test(env_tag='Example_Flash_Encryption_OTA_WiFi', target=['esp32c3'], nightly_run=True)
 def test_examples_protocol_simple_ota_example_with_flash_encryption_wifi(env, extra_data):
    """
    steps: |