Merge branch 'feature/split_example_tests' into 'master'

Split example_tests with Example_WIFI tag group into Example_OTA and Example_Protocols

See merge request espressif/esp-idf!12973

.gitlab/ci/target-test.yml

@@ -89,7 +89,6 @@ test_weekend_network:
 
 example_test_001A:
   extends: .example_test_esp32_template
-  parallel: 4
   tags:
     - ESP32
     - Example_WIFI
@@ -119,6 +118,19 @@ example_test_001D:
     - ESP32
     - Example_8Mflash_Ethernet
 
+example_test_OTA:
+  extends: .example_test_esp32_template
+  tags:
+    - ESP32
+    - Example_WIFI_OTA
+
+example_test_protocols:
+  extends: .example_test_esp32_template
+  parallel: 2
+  tags:
+    - ESP32
+    - Example_WIFI_Protocols
+
 example_test_002:
   extends: .example_test_esp32_template
   image: $CI_DOCKER_REGISTRY/ubuntu-test-env$BOT_DOCKER_IMAGE_TAG

examples/protocols/asio/chat_client/asio_chat_client_test.py

@@ -44,7 +44,7 @@ def chat_server_sketch(my_ip):
     print('server closed')
 
 
-@ttfw_idf.idf_example_test(env_tag='Example_WIFI')
+@ttfw_idf.idf_example_test(env_tag='Example_WIFI_Protocols')
 def test_examples_protocol_asio_chat_client(env, extra_data):
     """
     steps: |

examples/protocols/asio/chat_server/asio_chat_server_test.py

@@ -5,7 +5,7 @@ import socket
 import ttfw_idf
 
 
-@ttfw_idf.idf_example_test(env_tag='Example_WIFI')
+@ttfw_idf.idf_example_test(env_tag='Example_WIFI_Protocols')
 def test_examples_protocol_asio_chat_server(env, extra_data):
     """
     steps: |

examples/protocols/asio/tcp_echo_server/asio_tcp_server_test.py

@@ -5,7 +5,7 @@ import socket
 import ttfw_idf
 
 
-@ttfw_idf.idf_example_test(env_tag='Example_WIFI')
+@ttfw_idf.idf_example_test(env_tag='Example_WIFI_Protocols')
 def test_examples_protocol_asio_tcp_server(env, extra_data):
     """
     steps: |

examples/protocols/asio/udp_echo_server/asio_udp_server_test.py

@@ -5,7 +5,7 @@ import socket
 import ttfw_idf
 
 
-@ttfw_idf.idf_example_test(env_tag='Example_WIFI')
+@ttfw_idf.idf_example_test(env_tag='Example_WIFI_Protocols')
 def test_examples_protocol_asio_udp_server(env, extra_data):
     """
     steps: |

examples/protocols/cbor/example_test.py

@@ -6,7 +6,7 @@ import textwrap
 import ttfw_idf
 
 
-@ttfw_idf.idf_example_test(env_tag='Example_WIFI')
+@ttfw_idf.idf_example_test(env_tag='Example_WIFI_Protocols')
 def test_examples_cbor(env, extra_data):
 
     dut = env.get_dut('cbor', 'examples/protocols/cbor')

examples/protocols/esp_local_ctrl/example_test.py

@@ -7,7 +7,7 @@ import sys
 import ttfw_idf
 
 
-@ttfw_idf.idf_example_test(env_tag='Example_WIFI')
+@ttfw_idf.idf_example_test(env_tag='Example_WIFI_Protocols')
 def test_examples_esp_local_ctrl(env, extra_data):
 
     rel_project_path = os.path.join('examples', 'protocols', 'esp_local_ctrl')

examples/protocols/http_server/advanced_tests/http_server_advanced_test.py

@@ -35,7 +35,7 @@ from tiny_test_fw import Utility
 # features to this component.
 
 
-@ttfw_idf.idf_example_test(env_tag='Example_WIFI')
+@ttfw_idf.idf_example_test(env_tag='Example_WIFI_Protocols')
 def test_examples_protocol_http_server_advanced(env, extra_data):
     # Acquire DUT
     dut1 = env.get_dut('http_server', 'examples/protocols/http_server/advanced_tests', dut_class=ttfw_idf.ESP32DUT)

examples/protocols/http_server/file_serving/http_server_file_serving_test.py

@@ -25,7 +25,7 @@ from idf_http_server_test import adder as client
 from tiny_test_fw import Utility
 
 
-@ttfw_idf.idf_example_test(env_tag='Example_WIFI')
+@ttfw_idf.idf_example_test(env_tag='Example_WIFI_Protocols')
 def test_examples_protocol_http_server_file_serving(env, extra_data):  # type: (tiny_test_fw.Env.Env, None) -> None # pylint: disable=unused-argument
     # Acquire DUT
     dut1 = env.get_dut('http file_serving', 'examples/protocols/http_server/file_serving', dut_class=ttfw_idf.ESP32DUT)

examples/protocols/http_server/persistent_sockets/http_server_persistence_test.py

@@ -30,7 +30,7 @@ from tiny_test_fw import Utility
 # > make print_flash_cmd | tail -n 1 > build/download.config
 
 
-@ttfw_idf.idf_example_test(env_tag='Example_WIFI')
+@ttfw_idf.idf_example_test(env_tag='Example_WIFI_Protocols')
 def test_examples_protocol_http_server_persistence(env, extra_data):
     # Acquire DUT
     dut1 = env.get_dut('http_server', 'examples/protocols/http_server/persistent_sockets',

examples/protocols/http_server/simple/http_server_simple_test.py

@@ -62,7 +62,7 @@ class http_client_thread(threading.Thread):
 # > make print_flash_cmd | tail -n 1 > build/download.config
 
 
-@ttfw_idf.idf_example_test(env_tag='Example_WIFI')
+@ttfw_idf.idf_example_test(env_tag='Example_WIFI_Protocols')
 def test_examples_protocol_http_server_simple(env, extra_data):
     # Acquire DUT
     dut1 = env.get_dut('http_server', 'examples/protocols/http_server/simple', dut_class=ttfw_idf.ESP32DUT)
@@ -129,7 +129,7 @@ def test_examples_protocol_http_server_simple(env, extra_data):
     dut1.expect('Found URL query => ' + query, timeout=30)
 
 
-@ttfw_idf.idf_example_test(env_tag='Example_WIFI')
+@ttfw_idf.idf_example_test(env_tag='Example_WIFI_Protocols')
 def test_examples_protocol_http_server_lru_purge_enable(env, extra_data):
     # Acquire DUT
     dut1 = env.get_dut('http_server', 'examples/protocols/http_server/simple', dut_class=ttfw_idf.ESP32DUT)

examples/protocols/http_server/ws_echo_server/ws_server_example_test.py

@@ -53,7 +53,7 @@ class WsClient:
         return self.ws.send(data)
 
 
-@ttfw_idf.idf_example_test(env_tag='Example_WIFI')
+@ttfw_idf.idf_example_test(env_tag='Example_WIFI_Protocols')
 def test_examples_protocol_http_ws_echo_server(env, extra_data):
     # Acquire DUT
     dut1 = env.get_dut('http_server', 'examples/protocols/http_server/ws_echo_server', dut_class=ttfw_idf.ESP32DUT)

examples/protocols/https_server/simple/example_test.py

@@ -46,7 +46,7 @@ server_cert_pem = '-----BEGIN CERTIFICATE-----\n'\
 success_response = '<h1>Hello Secure World!</h1>'
 
 
-@ttfw_idf.idf_example_test(env_tag='Example_WIFI')
+@ttfw_idf.idf_example_test(env_tag='Example_WIFI_Protocols')
 def test_examples_protocol_https_server_simple(env, extra_data):  # type: (tiny_test_fw.Env.Env, None) -> None # pylint: disable=unused-argument
     """
     steps: |

examples/protocols/https_server/wss_server/wss_server_example_test.py

@@ -117,7 +117,7 @@ def test_multiple_client_keep_alive_and_async_response(ip, port, ca_file):  # ty
         t.join()
 
 
-@ttfw_idf.idf_example_test(env_tag='Example_WIFI')
+@ttfw_idf.idf_example_test(env_tag='Example_WIFI_Protocols')
 def test_examples_protocol_https_wss_server(env, extra_data):  # type: (tiny_test_fw.Env.Env, None) -> None # pylint: disable=unused-argument
 
     # Acquire DUT

examples/protocols/https_x509_bundle/example_test.py

@@ -4,7 +4,7 @@ import re
 import ttfw_idf
 
 
-@ttfw_idf.idf_example_test(env_tag='Example_WIFI', ignore=True)
+@ttfw_idf.idf_example_test(env_tag='Example_WIFI_Protocols', ignore=True)
 def test_examples_protocol_https_x509_bundle(env, extra_data):
     """
     steps: |

examples/protocols/icmp_echo/example_test.py

@@ -6,7 +6,7 @@ import re
 import ttfw_idf
 
 
-@ttfw_idf.idf_example_test(env_tag='Example_WIFI')
+@ttfw_idf.idf_example_test(env_tag='Example_WIFI_Protocols')
 def test_examples_icmp_echo(env, extra_data):
 
     dut = env.get_dut('icmp_echo', 'examples/protocols/icmp_echo')

examples/protocols/mdns/mdns_example_test.py

@@ -87,7 +87,7 @@ def mdns_server(esp_host):
             continue
 
 
-@ttfw_idf.idf_example_test(env_tag='Example_WIFI')
+@ttfw_idf.idf_example_test(env_tag='Example_WIFI_Protocols')
 def test_examples_protocol_mdns(env, extra_data):
     global stop_mdns_server
     """

examples/protocols/mqtt/ssl/mqtt_ssl_example_test.py

@@ -58,7 +58,7 @@ def on_message(client, userdata, msg):
     message_log += 'Received data:' + msg.topic + ' ' + payload + '\n'
 
 
-@ttfw_idf.idf_example_test(env_tag='Example_WIFI')
+@ttfw_idf.idf_example_test(env_tag='Example_WIFI_Protocols')
 def test_examples_protocol_mqtt_ssl(env, extra_data):
     broker_url = ''
     broker_port = 0

examples/protocols/mqtt/tcp/mqtt_tcp_example_test.py

@@ -53,7 +53,7 @@ def mqqt_server_sketch(my_ip, port):
     print('server closed')
 
 
-@ttfw_idf.idf_example_test(env_tag='Example_WIFI')
+@ttfw_idf.idf_example_test(env_tag='Example_WIFI_Protocols')
 def test_examples_protocol_mqtt_qos1(env, extra_data):
     global msgid
     """

examples/protocols/mqtt/ws/mqtt_ws_example_test.py

@@ -39,7 +39,7 @@ def on_message(client, userdata, msg):
     message_log += 'Received data:' + msg.topic + ' ' + payload + '\n'
 
 
-@ttfw_idf.idf_example_test(env_tag='Example_WIFI')
+@ttfw_idf.idf_example_test(env_tag='Example_WIFI_Protocols')
 def test_examples_protocol_mqtt_ws(env, extra_data):
     broker_url = ''
     broker_port = 0

examples/protocols/mqtt/wss/mqtt_wss_example_test.py

@@ -40,7 +40,7 @@ def on_message(client, userdata, msg):
     message_log += 'Received data:' + msg.topic + ' ' + payload + '\n'
 
 
-@ttfw_idf.idf_example_test(env_tag='Example_WIFI')
+@ttfw_idf.idf_example_test(env_tag='Example_WIFI_Protocols')
 def test_examples_protocol_mqtt_wss(env, extra_data):
     broker_url = ''
     broker_port = 0

examples/protocols/sntp/example_test.py

@@ -7,7 +7,7 @@ import ttfw_idf
 from tiny_test_fw import Utility
 
 
-@ttfw_idf.idf_example_test(env_tag='Example_WIFI')
+@ttfw_idf.idf_example_test(env_tag='Example_WIFI_Protocols')
 def test_examples_sntp(env, extra_data):
 
     dut = env.get_dut('sntp', 'examples/protocols/sntp')

examples/protocols/sockets/tcp_client/example_test.py

@@ -85,7 +85,7 @@ class TcpServer:
                 break
 
 
-@ttfw_idf.idf_example_test(env_tag='Example_WIFI')
+@ttfw_idf.idf_example_test(env_tag='Example_WIFI_Protocols')
 def test_examples_protocol_socket_tcpclient(env, extra_data):
     """
     steps:

examples/protocols/sockets/tcp_server/example_test.py

@@ -46,7 +46,7 @@ def tcp_client(address, payload):
     return data.decode()
 
 
-@ttfw_idf.idf_example_test(env_tag='Example_WIFI')
+@ttfw_idf.idf_example_test(env_tag='Example_WIFI_Protocols')
 def test_examples_protocol_socket_tcpserver(env, extra_data):
     MESSAGE = 'Data to ESP'
     """

examples/protocols/sockets/udp_client/example_test.py

@@ -78,7 +78,7 @@ class UdpServer:
                 break
 
 
-@ttfw_idf.idf_example_test(env_tag='Example_WIFI')
+@ttfw_idf.idf_example_test(env_tag='Example_WIFI_Protocols')
 def test_examples_protocol_socket_udpclient(env, extra_data):
     """
     steps:

examples/protocols/sockets/udp_server/example_test.py

@@ -44,7 +44,7 @@ def udp_client(address, payload):
     return reply.decode()
 
 
-@ttfw_idf.idf_example_test(env_tag='Example_WIFI')
+@ttfw_idf.idf_example_test(env_tag='Example_WIFI_Protocols')
 def test_examples_protocol_socket_udpserver(env, extra_data):
     MESSAGE = 'Data to ESP'
     """

examples/protocols/websocket/example_test.py

@@ -97,7 +97,7 @@ def test_recv_long_msg(dut, websocket, msg_len, repeats):
                             \nreceived: {}\nwith length {}'.format(send_msg, len(send_msg), recv_msg, len(recv_msg)))
 
 
-@ttfw_idf.idf_example_test(env_tag='Example_WIFI')
+@ttfw_idf.idf_example_test(env_tag='Example_WIFI_Protocols')
 def test_examples_protocol_websocket(env, extra_data):
     """
     steps:

examples/system/ota/advanced_https_ota/example_test.py

@@ -169,7 +169,7 @@ def start_redirect_server(ota_image_dir, server_ip, server_port, redirection_por
     httpd.serve_forever()
 
 
-@ttfw_idf.idf_example_test(env_tag='Example_WIFI')
+@ttfw_idf.idf_example_test(env_tag='Example_WIFI_OTA')
 def test_examples_protocol_advanced_https_ota_example(env, extra_data):
     """
     This is a positive test case, which downloads complete binary file multiple number of times.
@@ -213,7 +213,7 @@ def test_examples_protocol_advanced_https_ota_example(env, extra_data):
         dut1.reset()
 
 
-@ttfw_idf.idf_example_test(env_tag='Example_WIFI')
+@ttfw_idf.idf_example_test(env_tag='Example_WIFI_OTA')
 def test_examples_protocol_advanced_https_ota_example_truncated_bin(env, extra_data):
     """
     Working of OTA if binary file is truncated is validated in this test case.
@@ -264,7 +264,7 @@ def test_examples_protocol_advanced_https_ota_example_truncated_bin(env, extra_d
     os.remove(binary_file)
 
 
-@ttfw_idf.idf_example_test(env_tag='Example_WIFI')
+@ttfw_idf.idf_example_test(env_tag='Example_WIFI_OTA')
 def test_examples_protocol_advanced_https_ota_example_truncated_header(env, extra_data):
     """
     Working of OTA if headers of binary file are truncated is vaildated in this test case.
@@ -314,7 +314,7 @@ def test_examples_protocol_advanced_https_ota_example_truncated_header(env, extr
     os.remove(binary_file)
 
 
-@ttfw_idf.idf_example_test(env_tag='Example_WIFI')
+@ttfw_idf.idf_example_test(env_tag='Example_WIFI_OTA')
 def test_examples_protocol_advanced_https_ota_example_random(env, extra_data):
     """
     Working of OTA if random data is added in binary file are validated in this test case.
@@ -363,7 +363,7 @@ def test_examples_protocol_advanced_https_ota_example_random(env, extra_data):
     os.remove(binary_file)
 
 
-@ttfw_idf.idf_example_test(env_tag='Example_WIFI')
+@ttfw_idf.idf_example_test(env_tag='Example_WIFI_OTA')
 def test_examples_protocol_advanced_https_ota_example_chunked(env, extra_data):
     """
     This is a positive test case, which downloads complete binary file multiple number of times.
@@ -401,7 +401,7 @@ def test_examples_protocol_advanced_https_ota_example_chunked(env, extra_data):
     os.remove(os.path.join(dut1.app.binary_path, 'server_key.pem'))
 
 
-@ttfw_idf.idf_example_test(env_tag='Example_WIFI')
+@ttfw_idf.idf_example_test(env_tag='Example_WIFI_OTA')
 def test_examples_protocol_advanced_https_ota_example_redirect_url(env, extra_data):
     """
     This is a positive test case, which starts a server and a redirection server.
@@ -512,7 +512,7 @@ def test_examples_protocol_advanced_https_ota_example_anti_rollback(env, extra_d
     os.remove(anti_rollback_bin_name)
 
 
-@ttfw_idf.idf_example_test(env_tag='Example_WIFI')
+@ttfw_idf.idf_example_test(env_tag='Example_WIFI_OTA')
 def test_examples_protocol_advanced_https_ota_example_partial_request(env, extra_data):
     """
     This is a positive test case, to test OTA workflow with Range HTTP header.

examples/system/ota/native_ota_example/example_test.py

@@ -135,7 +135,7 @@ def start_chunked_server(ota_image_dir, server_port):
     return chunked_server
 
 
-@ttfw_idf.idf_example_test(env_tag='Example_WIFI')
+@ttfw_idf.idf_example_test(env_tag='Example_WIFI_OTA')
 def test_examples_protocol_native_ota_example(env, extra_data):
     """
     This is a positive test case, which downloads complete binary file multiple number of times.
@@ -179,7 +179,7 @@ def test_examples_protocol_native_ota_example(env, extra_data):
         dut1.reset()
 
 
-@ttfw_idf.idf_example_test(env_tag='Example_WIFI')
+@ttfw_idf.idf_example_test(env_tag='Example_WIFI_OTA')
 def test_examples_protocol_native_ota_example_truncated_bin(env, extra_data):
     """
     Working of OTA if binary file is truncated is validated in this test case.
@@ -230,7 +230,7 @@ def test_examples_protocol_native_ota_example_truncated_bin(env, extra_data):
     os.remove(binary_file)
 
 
-@ttfw_idf.idf_example_test(env_tag='Example_WIFI')
+@ttfw_idf.idf_example_test(env_tag='Example_WIFI_OTA')
 def test_examples_protocol_native_ota_example_truncated_header(env, extra_data):
     """
     Working of OTA if headers of binary file are truncated is vaildated in this test case.
@@ -280,7 +280,7 @@ def test_examples_protocol_native_ota_example_truncated_header(env, extra_data):
     os.remove(binary_file)
 
 
-@ttfw_idf.idf_example_test(env_tag='Example_WIFI')
+@ttfw_idf.idf_example_test(env_tag='Example_WIFI_OTA')
 def test_examples_protocol_native_ota_example_random(env, extra_data):
     """
     Working of OTA if random data is added in binary file are validated in this test case.
@@ -329,7 +329,7 @@ def test_examples_protocol_native_ota_example_random(env, extra_data):
     os.remove(binary_file)
 
 
-@ttfw_idf.idf_example_test(env_tag='Example_WIFI')
+@ttfw_idf.idf_example_test(env_tag='Example_WIFI_OTA')
 def test_examples_protocol_native_ota_example_chunked(env, extra_data):
     """
     This is a positive test case, which downloads complete binary file multiple number of times.

examples/system/ota/simple_ota_example/example_test.py

@@ -92,7 +92,7 @@ def start_https_server(ota_image_dir, server_ip, server_port, server_file=None,
     httpd.serve_forever()
 
 
-@ttfw_idf.idf_example_test(env_tag='Example_WIFI')
+@ttfw_idf.idf_example_test(env_tag='Example_WIFI_OTA')
 def test_examples_protocol_simple_ota_example(env, extra_data):
     """
     steps: |