Merge branch 'feature/mqtt_weekend_tests' into 'master'

introducing weekend tests for mqtt and fuzzer test (mdns, LWIP) Closes IDF-540 See merge request idf/esp-idf!4230
2024-10-05 20:47:46 -04:00 · 2019-03-27 19:12:33 +08:00 · 2019-03-27 19:12:33 +08:00 · da90966828
commit da90966828
parent 18946a5421 bc60bbbeaf
19 changed files with 697 additions and 14 deletions
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@ -278,6 +278,7 @@ build_esp_idf_tests_cmake:
      - $BOT_LABEL_BUILD
      - $BOT_LABEL_EXAMPLE_TEST
      - $BOT_LABEL_REGULAR_TEST
+      - $BOT_LABEL_WEEKEND_TEST
  script:
    # it's not possible to build 100% out-of-tree and have the "artifacts"
    # mechanism work, but this is the next best thing
@ -310,6 +311,7 @@ build_esp_idf_tests_cmake:
      - $BOT_LABEL_BUILD
      - $BOT_LABEL_EXAMPLE_TEST
      - $BOT_LABEL_REGULAR_TEST
+      - $BOT_LABEL_WEEKEND_TEST
  script:
    # it's not possible to build 100% out-of-tree and have the "artifacts"
    # mechanism work, but this is the next best thing
@ -504,11 +506,9 @@ test_ldgen_on_host:
      - ${FUZZER_TEST_DIR}/fuzz_output.log
    expire_in: 1 week
  only:
-    refs:
-      # can only be triggered
-      - triggers
    variables:
      - $BOT_LABEL_FUZZER_TEST
+      - $BOT_LABEL_WEEKEND_TEST
  script:
    - export AFL_I_DONT_CARE_ABOUT_MISSING_CRASHES=1 && export AFL_SKIP_CPUFREQ=1
    - cd ${FUZZER_TEST_DIR}
@ -521,27 +521,23 @@ test_ldgen_on_host:
 test_mdns_fuzzer_on_host:
  <<: *host_fuzzer_test_template
  variables:
-    BOT_NEEDS_TRIGGER_BY_NAME: 1
    FUZZER_TEST_DIR: components/mdns/test_afl_fuzz_host

 test_lwip_dns_fuzzer_on_host:
  <<: *host_fuzzer_test_template
  variables:
-    BOT_NEEDS_TRIGGER_BY_NAME: 1
    FUZZER_TEST_DIR: components/lwip/test_afl_host
    FUZZER_PARAMS: MODE=dns

 test_lwip_dhcp_fuzzer_on_host:
  <<: *host_fuzzer_test_template
  variables:
-    BOT_NEEDS_TRIGGER_BY_NAME: 1
    FUZZER_TEST_DIR: components/lwip/test_afl_host
    FUZZER_PARAMS: MODE=dhcp_client

 test_lwip_dhcps_fuzzer_on_host:
  <<: *host_fuzzer_test_template
  variables:
-    BOT_NEEDS_TRIGGER_BY_NAME: 1
    FUZZER_TEST_DIR: components/lwip/test_afl_host
    FUZZER_PARAMS: MODE=dhcp_server

@ -951,6 +947,22 @@ assign_test:
    LOG_PATH: "$CI_PROJECT_DIR/TEST_LOGS"
    ENV_FILE: "$CI_PROJECT_DIR/ci-test-runner-configs/$CI_RUNNER_DESCRIPTION/EnvConfig.yml"

+test_weekend_mqtt:
+  <<: *example_test_template
+  stage: target_test
+  tags:
+    - ESP32
+    - Example_WIFI
+  only:
+    variables:
+      - $BOT_LABEL_WEEKEND_TEST
+  variables:
+    TEST_CASE_PATH: "$CI_PROJECT_DIR/components/mqtt/weekend_test"
+    TEST_FW_PATH: "$CI_PROJECT_DIR/tools/tiny-test-fw"
+    LOG_PATH: "$CI_PROJECT_DIR/TEST_LOGS"
+    ENV_FILE: "$CI_PROJECT_DIR/components/mqtt/weekend_test/env.yml"
+    CONFIG_FILE: "$CI_PROJECT_DIR/components/mqtt/weekend_test/config.yml"
+
 .test_template: &test_template
  stage: target_test
  when: on_success
--- a/components/lwip/test_afl_host/Makefile
+++ b/components/lwip/test_afl_host/Makefile
@ -1,7 +1,7 @@
 COMPONENTS_DIR=../..
 CFLAGS=-std=gnu99 -Og -ggdb -ffunction-sections -fdata-sections -nostdlib -Wall  -Werror=all -Wno-int-to-pointer-cast -Wno-error=unused-function -Wno-error=unused-variable -Wno-error=deprecated-declarations -Wextra \
 -Wno-unused-parameter -Wno-sign-compare -Wno-address   -Wno-unused-variable -DESP_PLATFORM -D IDF_VER=\"v3.1\" -MMD -MP -DWITH_POSIX
-INC_DIRS=-I . -I ./build/config -I $(COMPONENTS_DIR)/newlib/platform_include -I $(COMPONENTS_DIR)/newlib/include -I $(COMPONENTS_DIR)/driver/include -I $(COMPONENTS_DIR)/esp32/include -I $(COMPONENTS_DIR)/ethernet/include -I $(COMPONENTS_DIR)/freertos/include -I $(COMPONENTS_DIR)/heap/include -I $(COMPONENTS_DIR)/lwip/lwip/src/include  -I $(COMPONENTS_DIR)/lwip/include/apps -I $(COMPONENTS_DIR)/lwip/lwip/src/include/netif -I $(COMPONENTS_DIR)/lwip/lwip/src/include/posix -I $(COMPONENTS_DIR)/lwip/port/esp32/include -I $(COMPONENTS_DIR)/lwip/lwip/src/include/posix -I $(COMPONENTS_DIR)/lwip/include/apps/ping -I $(COMPONENTS_DIR)/soc/esp32/include -I $(COMPONENTS_DIR)/soc/include -I $(COMPONENTS_DIR)/tcpip_adapter/include  -I $(COMPONENTS_DIR)/xtensa-debug-module/include
+INC_DIRS=-I . -I ./build/config -I $(COMPONENTS_DIR)/newlib/platform_include -I $(COMPONENTS_DIR)/newlib/include -I $(COMPONENTS_DIR)/driver/include -I $(COMPONENTS_DIR)/esp32/include -I $(COMPONENTS_DIR)/ethernet/include -I $(COMPONENTS_DIR)/freertos/include -I $(COMPONENTS_DIR)/heap/include -I $(COMPONENTS_DIR)/lwip/lwip/src/include  -I $(COMPONENTS_DIR)/lwip/include/apps -I $(COMPONENTS_DIR)/lwip/lwip/src/include/netif -I $(COMPONENTS_DIR)/lwip/lwip/src/include/posix -I $(COMPONENTS_DIR)/lwip/port/esp32/include -I $(COMPONENTS_DIR)/lwip/lwip/src/include/posix -I $(COMPONENTS_DIR)/lwip/include/apps/ping -I $(COMPONENTS_DIR)/soc/esp32/include -I $(COMPONENTS_DIR)/soc/include -I $(COMPONENTS_DIR)/tcpip_adapter/include -I $(COMPONENTS_DIR)/esp_rom/include  -I $(COMPONENTS_DIR)/xtensa-debug-module/include
 TEST_NAME=test
 FUZZ=afl-fuzz
 GEN_CFG=generate_config
--- a/components/mdns/test_afl_fuzz_host/Makefile
+++ b/components/mdns/test_afl_fuzz_host/Makefile
@ -1,7 +1,9 @@
 TEST_NAME=test
 FUZZ=afl-fuzz
 COMPONENTS_DIR=../..
-CFLAGS=-g -DMDNS_TEST_MODE -I. -I.. -I../include -I../private_include -I$(COMPONENTS_DIR)/tcpip_adapter/include -I$(COMPONENTS_DIR)/esp32/include -include esp32_compat.h
+CFLAGS=-g -DHOOK_MALLOC_FAILED -DESP_EVENT_H_ -D__ESP_LOG_H__ -DMDNS_TEST_MODE \
+				-I. -I.. -I../include -I../private_include -include esp32_compat.h \
+				-I$(COMPONENTS_DIR)/tcpip_adapter/include -I$(COMPONENTS_DIR)/esp_event/include -I$(COMPONENTS_DIR)/log/include
 MDNS_C_DEPENDENCY_INJECTION=-include mdns_di.h
 ifeq ($(INSTR),off)
    CC=gcc
--- a/components/mdns/test_afl_fuzz_host/esp32_compat.h
+++ b/components/mdns/test_afl_fuzz_host/esp32_compat.h
@ -57,6 +57,7 @@

 #define portMAX_DELAY               0xFFFFFFFF
 #define portTICK_PERIOD_MS          1
+#define ESP_LOGD(a,b)

 #define xSemaphoreTake(s,d)
 #define xTaskDelete(a)
@ -65,6 +66,7 @@
 #define _mdns_pcb_init(a,b)         true
 #define _mdns_pcb_deinit(a,b)         true
 #define xSemaphoreCreateMutex()     malloc(1)
+#define xSemaphoreCreateBinary()    malloc(1)
 #define vSemaphoreDelete(s)         free(s)
 #define xTaskCreatePinnedToCore(a,b,c,d,e,f,g)     *(f) = malloc(1)
 #define vTaskDelay(m)               usleep((m)*0)
--- a/components/mqtt/Kconfig
+++ b/components/mqtt/Kconfig
@ -78,6 +78,14 @@ menu "ESP-MQTT Configurations"
        help
            MQTT task stack size

+    config MQTT_DISABLE_API_LOCKS
+        bool "Disable API locks"
+        default n
+        depends on MQTT_USE_CUSTOM_CONFIG
+        help
+            Default config employs API locks to protect internal structures. It is possible to disable
+            these locks if the user code doesn't access MQTT API from multiple concurrent tasks
+
    config MQTT_TASK_CORE_SELECTION_ENABLED
        bool "Enable MQTT task core selection"
        default false
--- a/components/mqtt/weekend_test/config.yml
+++ b/components/mqtt/weekend_test/config.yml
@ -0,0 +1,3 @@
+CaseConfig:
+- name: test_weekend_mqtt_publish
+
--- a/components/mqtt/weekend_test/env.yml
+++ b/components/mqtt/weekend_test/env.yml
--- a/components/mqtt/weekend_test/mqtt_publish_test.py
+++ b/components/mqtt/weekend_test/mqtt_publish_test.py
@ -0,0 +1,190 @@
+from __future__ import print_function
+from __future__ import unicode_literals
+from builtins import str
+import re
+import os
+import sys
+import ssl
+import paho.mqtt.client as mqtt
+from threading import Thread, Event
+import time
+import string
+import random
+
+try:
+    import IDF
+except ImportError:
+    # this is a test case write with tiny-test-fw.
+    # to run test cases outside tiny-test-fw,
+    # we need to set environment variable `TEST_FW_PATH`,
+    # then get and insert `TEST_FW_PATH` to sys path before import FW module
+    test_fw_path = os.getenv("TEST_FW_PATH")
+    if test_fw_path and test_fw_path not in sys.path:
+        sys.path.insert(0, test_fw_path)
+    import IDF
+
+import DUT
+
+
+event_client_connected = Event()
+event_stop_client = Event()
+event_client_received_correct = Event()
+message_log = ""
+broker_host = {}
+broker_port = {}
+expected_data = ""
+subscribe_topic = ""
+publish_topic = ""
+expected_count = 0
+
+
+# The callback for when the client receives a CONNACK response from the server.
+def on_connect(client, userdata, flags, rc):
+    print("Connected with result code " + str(rc))
+    event_client_connected.set()
+    client.subscribe("/topic/qos0")
+
+
+def mqtt_client_task(client):
+    while not event_stop_client.is_set():
+        client.loop()
+
+
+def get_host_port_from_dut(dut1, config_option):
+    value = re.search(r'\:\/\/([^:]+)\:([0-9]+)', dut1.app.get_sdkconfig()[config_option])
+    return value.group(1), int(value.group(2))
+
+
+# The callback for when a PUBLISH message is received from the server.
+def on_message(client, userdata, msg):
+    global message_log
+    global expected_count
+    payload = msg.payload.decode()
+    if payload == expected_data:
+        expected_count += 1
+        print("[{}] Received...".format(msg.mid))
+    message_log += "Received data:" + msg.topic + " " + payload + "\n"
+
+
+def test_single_config(dut, transport, qos, repeat, published):
+    global expected_count
+    global expected_data
+    global message_log
+    sample_string = ''.join(random.choice(string.ascii_uppercase + string.ascii_lowercase + string.digits) for _ in range(16))
+    print("PUBLISH TEST: transport:{}, qos:{}, sequence:{}, sample msg:{}".format(transport, qos, published, sample_string))
+    event_client_connected.clear()
+    expected_count = 0
+    message_log = ""
+    expected_data = sample_string * repeat
+    client = None
+    try:
+        if transport in ["ws", "wss"]:
+            client = mqtt.Client(transport="websockets")
+            client.ws_set_options(path="/ws", headers=None)
+        else:
+            client = mqtt.Client()
+        client.on_connect = on_connect
+        client.on_message = on_message
+        if transport in ["ssl", "wss"]:
+            client.tls_set(None, None, None, cert_reqs=ssl.CERT_NONE, tls_version=ssl.PROTOCOL_TLSv1, ciphers=None)
+            client.tls_insecure_set(True)
+        print("Connecting...")
+        client.connect(broker_host[transport], broker_port[transport], 60)
+    except Exception:
+        print("ENV_TEST_FAILURE: Unexpected error while connecting to broker {}: {}:".format(broker_host[transport], sys.exc_info()[0]))
+        raise
+    # Starting a py-client in a separate thread
+    thread1 = Thread(target=mqtt_client_task, args=(client,))
+    thread1.start()
+    print("Connecting py-client to broker {}:{}...".format(broker_host[transport], broker_port[transport]))
+    if not event_client_connected.wait(timeout=30):
+        raise ValueError("ENV_TEST_FAILURE: Test script cannot connect to broker: {}".format(broker_host[transport]))
+    client.subscribe(subscribe_topic, qos)
+    dut.write("{} {} {} {} {}".format(transport, sample_string, repeat, published, qos), eol="\n")
+    try:
+        # waiting till subscribed to defined topic
+        dut.expect(re.compile(r"MQTT_EVENT_SUBSCRIBED"), timeout=30)
+        for i in range(published):
+            client.publish(publish_topic, sample_string * repeat, qos)
+            print("Publishing...")
+        print("Checking esp-client received msg published from py-client...")
+        dut.expect(re.compile(r"Correct pattern received exactly x times"), timeout=60)
+        start = time.time()
+        while expected_count < published and time.time() - start <= 60:
+            time.sleep(1)
+        # Note: tolerate that messages qos=1 to be received more than once
+        if expected_count == published or (expected_count > published and qos == 1):
+            print("All data received from ESP32...")
+        else:
+            raise ValueError("Not all data received from ESP32: Expected:{}x{}, Received:{}x{}".format(expected_data, published, message_log, expected_count))
+    finally:
+        event_stop_client.set()
+        thread1.join()
+    client.disconnect()
+    event_stop_client.clear()
+
+
+@IDF.idf_example_test(env_tag="Example_WIFI")
+def test_weekend_mqtt_publish(env, extra_data):
+    # Using broker url dictionary for different transport
+    global broker_host
+    global broker_port
+    global publish_topic
+    global subscribe_topic
+    """
+    steps: |
+      1. join AP and connects to ssl broker
+      2. Test connects a client to the same broker
+      3. Test evaluates python client received correct qos0 message
+      4. Test ESP32 client received correct qos0 message
+    """
+    dut1 = env.get_dut("mqtt_publish", "examples/protocols/mqtt/publish_test")
+    # check and log bin size
+    binary_file = os.path.join(dut1.app.binary_path, "mqtt_publish.bin")
+    bin_size = os.path.getsize(binary_file)
+    IDF.log_performance("mqtt_publish_bin_size", "{}KB"
+                        .format(bin_size // 1024))
+    IDF.check_performance("mqtt_publish_size", bin_size // 1024)
+    # Look for host:port in sdkconfig
+    try:
+        # python client subscribes to the topic to which esp client publishes and vice versa
+        publish_topic = dut1.app.get_sdkconfig()["CONFIG_SUBSCIBE_TOPIC"].replace('"','')
+        subscribe_topic = dut1.app.get_sdkconfig()["CONFIG_PUBLISH_TOPIC"].replace('"','')
+        broker_host["ssl"], broker_port["ssl"] = get_host_port_from_dut(dut1, "CONFIG_BROKER_SSL_URI")
+        broker_host["tcp"], broker_port["tcp"] = get_host_port_from_dut(dut1, "CONFIG_BROKER_TCP_URI")
+        broker_host["ws"], broker_port["ws"] = get_host_port_from_dut(dut1, "CONFIG_BROKER_WS_URI")
+        broker_host["wss"], broker_port["wss"] = get_host_port_from_dut(dut1, "CONFIG_BROKER_WSS_URI")
+    except Exception:
+        print('ENV_TEST_FAILURE: Cannot find broker url in sdkconfig')
+        raise
+    dut1.start_app()
+    try:
+        ip_address = dut1.expect(re.compile(r" sta ip: ([^,]+),"), timeout=30)
+        print("Connected to AP with IP: {}".format(ip_address))
+    except DUT.ExpectTimeout:
+        print('ENV_TEST_FAILURE: Cannot connect to AP')
+        raise
+    for qos in [0, 1, 2]:
+        for transport in ["tcp", "ssl", "ws", "wss"]:
+            # decide on broker what level of test will pass (local broker works the best)
+            if broker_host[transport].startswith("192.168"):
+                # medium size, medium repeated
+                test_single_config(dut1, transport, qos, 5, 50)
+                # long data
+                test_single_config(dut1, transport, qos, 1000, 10)
+                # short data, many repeats
+                test_single_config(dut1, transport, qos, 2, 200)
+            elif transport in ["ws", "wss"]:
+                # more relaxed criteria for websockets!
+                test_single_config(dut1, transport, qos, 2, 5)
+                test_single_config(dut1, transport, qos, 50, 1)
+                test_single_config(dut1, transport, qos, 10, 20)
+            else:
+                # common configuration should be good for most public mosquittos
+                test_single_config(dut1, transport, qos, 5, 10)
+                test_single_config(dut1, transport, qos, 500, 3)
+                test_single_config(dut1, transport, qos, 1, 50)
+
+
+if __name__ == '__main__':
+    test_weekend_mqtt_publish()
--- a/components/tcp_transport/transport_ws.c
+++ b/components/tcp_transport/transport_ws.c
@ -230,15 +230,17 @@ static int ws_read(esp_transport_handle_t t, char *buffer, int len, int timeout_
        data_ptr += 8;
        payload_len_buff = len - 10;
    }
+
+    if (payload_len > payload_len_buff) {
+        ESP_LOGD(TAG, "Actual data to receive (%d) are longer than ws buffer (%d)", payload_len, payload_len_buff);
+        payload_len = payload_len_buff;
+    }
+
    // Then receive and process payload
    if ((rlen = esp_transport_read(ws->parent, data_ptr, payload_len, timeout_ms)) <= 0) {
        ESP_LOGE(TAG, "Error read data");
        return rlen;
    }
-    if (payload_len > payload_len_buff) {
-        ESP_LOGD(TAG, "Actual data received (%d) are longer than mqtt buffer (%d)", payload_len, payload_len_buff);
-        payload_len = payload_len_buff;
-    }

    if (mask) {
        mask_key = data_ptr;
--- a/examples/protocols/mqtt/publish_test/CMakeLists.txt
+++ b/examples/protocols/mqtt/publish_test/CMakeLists.txt
@ -0,0 +1,9 @@
+# The following four lines of boilerplate have to be in your project's CMakeLists
+# in this exact order for cmake to work correctly
+cmake_minimum_required(VERSION 3.5)
+
+include($ENV{IDF_PATH}/tools/cmake/project.cmake)
+
+project(mqtt_publish)
+
+target_add_binary_data(mqtt_publish.elf "main/iot_eclipse_org.pem" TEXT)
--- a/examples/protocols/mqtt/publish_test/Makefile
+++ b/examples/protocols/mqtt/publish_test/Makefile
@ -0,0 +1,7 @@
+#
+# This is a project Makefile. It is assumed the directory this Makefile resides in is a
+# project subdirectory.
+#
+PROJECT_NAME := mqtt_publish
+
+include $(IDF_PATH)/make/project.mk
--- a/examples/protocols/mqtt/publish_test/README.md
+++ b/examples/protocols/mqtt/publish_test/README.md
@ -0,0 +1,91 @@
+# ESP-MQTT advanced published test
+(See the README.md file in the upper level 'examples' directory for more information about examples.)
+
+Main purpose of this example is to test the MQTT library to correctly publish and receive messages (of different size and sequences) over different transports. 
+It is possible to run this example manually without any test to exercise how the MQTT library deals with
+
+- reception of fragmented messages
+- runtime updates of URI
+
+## How to use example
+
+This example waits for user input to provide these parameters:
+- transport: string parameter, one of: tcp, ssl, ws, wss
+- pattern: sample string to be transmitted as message
+- pattern repeats: number of repeats of pattern in one MQTT message
+- repeated: number of repeats ESP32 publishes the message, also ESP32 expects to receive the same message the same number of repeats
+- qos: number specifying qos, one of: 0, 1, 2
+
+### Hardware Required
+
+This example can be executed on any ESP32 board, the only required interface is WiFi and connection to internet.
+
+### Configure the project
+
+```
+make menuconfig
+```
+
+* Set serial port under Serial Flasher Options.
+
+* Set ssid and password for the board to connect to AP.
+
+* Set brokers for all 4 transports (TCP, SSL, WS, WSS), also set certificate if needed
+
+* Set topics for publishing from and to ESP32
+
+### Build and Flash
+
+Build the project and flash it to the board, then run monitor tool to view serial output:
+
+```
+make -j4 flash monitor
+```
+
+(To exit the serial monitor, type ``Ctrl-]``.)
+
+See the Getting Started Guide for full steps to configure and use ESP-IDF to build projects.
+
+## Example Output
+
+```
+I (4730) event: sta ip: 192.168.0.125, mask: 255.255.255.0, gw: 192.168.0.2
+I (4730) PUBLISH_TEST: [APP] Free memory: 236728 bytes
+I (4730) system_api: Base MAC address is not set, read default base MAC address from BLK0 of EFUSE
+D (4740) MQTT_CLIENT: MQTT client_id=ESP32_09885C
+I (31360) PUBLISH_TEST: PATTERN:1234 REPEATED:10 PUBLISHED:10
+```
+- User enters "tcp 1234 10 10 1"
+```
+EXPECTED STRING 1234123412341234123412341234123412341234, SIZE:40
+W (31360) MQTT_CLIENT: Client asked to stop, but was not started
+I (31360) PUBLISH_TEST: [TCP transport] Startup..
+D (31370) MQTT_CLIENT: Core selection disabled
+I (31370) PUBLISH_TEST: Note free memory: 224652 bytes
+I (31370) PUBLISH_TEST: Other event id:7
+D (31390) MQTT_CLIENT: Transport connected to mqtt://192.168.0.163:1883
+I (31400) MQTT_CLIENT: Sending MQTT CONNECT message, type: 1, id: 0000
+D (31410) MQTT_CLIENT: Connected
+I (31410) PUBLISH_TEST: MQTT_EVENT_CONNECTED
+D (31410) MQTT_CLIENT: mqtt_enqueue id: 31184, type=8 successful
+D (31410) OUTBOX: ENQUEUE msgid=31184, msg_type=8, len=20, size=20
+D (31420) MQTT_CLIENT: Sent subscribe topic=/xxx.topic123, id: 31184, type=8 successful
+I (31430) PUBLISH_TEST: sent subscribe successful, msg_id=31184
+D (31440) MQTT_CLIENT: mqtt_enqueue id: 16584, type=3 successful
+D (31440) OUTBOX: ENQUEUE msgid=16584, msg_type=3, len=59, size=79
+I (31450) PUBLISH_TEST: [16584] Publishing...
+D (31450) MQTT_CLIENT: msg_type=9, msg_id=31184
+D (31460) MQTT_CLIENT: pending_id=16584, pending_msg_count = 2
+D (31460) OUTBOX: DELETED msgid=31184, msg_type=8, remain size=59
+D (31470) MQTT_CLIENT: Subscribe successful
+I (31470) PUBLISH_TEST: MQTT_EVENT_SUBSCRIBED, msg_id=31184
+D (31480) MQTT_CLIENT: msg_type=4, msg_id=16584
+D (31480) MQTT_CLIENT: pending_id=16584, pending_msg_count = 1
+D (31490) OUTBOX: DELETED msgid=16584, msg_type=3, remain size=0
+D (31500) MQTT_CLIENT: received MQTT_MSG_TYPE_PUBACK, finish QoS1 publish
+I (31500) PUBLISH_TEST: MQTT_EVENT_PUBLISHED, msg_id=16584
+D (31510) MQTT_CLIENT: mqtt_enqueue id: 44615, type=3 successful
+D (31520) OUTBOX: ENQUEUE msgid=44615, msg_type=3, len=59, size=59
+I (31530) PUBLISH_TEST: [44615] Publishing...
+...
+```
--- a/examples/protocols/mqtt/publish_test/main/CMakeLists.txt
+++ b/examples/protocols/mqtt/publish_test/main/CMakeLists.txt
@ -0,0 +1,4 @@
+set(COMPONENT_SRCS "publish_test.c")
+set(COMPONENT_ADD_INCLUDEDIRS ".")
+
+register_component()
--- a/examples/protocols/mqtt/publish_test/main/Kconfig.projbuild
+++ b/examples/protocols/mqtt/publish_test/main/Kconfig.projbuild
@ -0,0 +1,62 @@
+menu "Example Configuration"
+
+    config WIFI_SSID
+        string "WiFi SSID"
+        default "myssid"
+        help
+            SSID (network name) for the example to connect to.
+
+    config WIFI_PASSWORD
+        string "WiFi Password"
+        default "mypassword"
+        help
+            WiFi password (WPA or WPA2) for the example to use.
+
+    config BROKER_SSL_URI
+        string "Broker SSL URL"
+        default "mqtts://iot.eclipse.org:8883"
+        help
+            URL of an mqtt broker for ssl transport
+
+    config BROKER_TCP_URI
+        string "Broker TCP URL"
+        default "mqtt://iot.eclipse.org:1883"
+        help
+            URL of an mqtt broker for tcp transport
+
+    config BROKER_WS_URI
+        string "Broker WS URL"
+        default "ws://iot.eclipse.org:80/ws"
+        help
+            URL of an mqtt broker for ws transport
+
+    config BROKER_WSS_URI
+        string "Broker WSS URL"
+        default "wss://iot.eclipse.org:443/ws"
+        help
+            URL of an mqtt broker for wss transport
+
+    config PUBLISH_TOPIC
+        string "publish topic"
+        default "/topic/publish/esp2py"
+        help
+            topic to which esp32 client publishes
+
+    config SUBSCIBE_TOPIC
+        string "subscribe topic"
+        default "/topic/subscribe/py2esp"
+        help
+            topic to which esp32 client subsribes (and expects data)
+
+    config BROKER_CERTIFICATE_OVERRIDE
+        string "Broker certificate override"
+        default ""
+        help
+            Please leave empty if broker certificate included from a textfile; otherwise fill in a base64 part of PEM
+            format certificate
+
+    config BROKER_CERTIFICATE_OVERRIDDEN
+        bool
+        default y if BROKER_CERTIFICATE_OVERRIDE != ""
+
+endmenu
--- a/examples/protocols/mqtt/publish_test/main/component.mk
+++ b/examples/protocols/mqtt/publish_test/main/component.mk
@ -0,0 +1 @@
+COMPONENT_EMBED_TXTFILES := iot_eclipse_org.pem
--- a/examples/protocols/mqtt/publish_test/main/iot_eclipse_org.pem
+++ b/examples/protocols/mqtt/publish_test/main/iot_eclipse_org.pem
@ -0,0 +1,27 @@
+-----BEGIN CERTIFICATE-----
+MIIEkjCCA3qgAwIBAgIQCgFBQgAAAVOFc2oLheynCDANBgkqhkiG9w0BAQsFADA/
+MSQwIgYDVQQKExtEaWdpdGFsIFNpZ25hdHVyZSBUcnVzdCBDby4xFzAVBgNVBAMT
+DkRTVCBSb290IENBIFgzMB4XDTE2MDMxNzE2NDA0NloXDTIxMDMxNzE2NDA0Nlow
+SjELMAkGA1UEBhMCVVMxFjAUBgNVBAoTDUxldCdzIEVuY3J5cHQxIzAhBgNVBAMT
+GkxldCdzIEVuY3J5cHQgQXV0aG9yaXR5IFgzMIIBIjANBgkqhkiG9w0BAQEFAAOC
+AQ8AMIIBCgKCAQEAnNMM8FrlLke3cl03g7NoYzDq1zUmGSXhvb418XCSL7e4S0EF
+q6meNQhY7LEqxGiHC6PjdeTm86dicbp5gWAf15Gan/PQeGdxyGkOlZHP/uaZ6WA8
+SMx+yk13EiSdRxta67nsHjcAHJyse6cF6s5K671B5TaYucv9bTyWaN8jKkKQDIZ0
+Z8h/pZq4UmEUEz9l6YKHy9v6Dlb2honzhT+Xhq+w3Brvaw2VFn3EK6BlspkENnWA
+a6xK8xuQSXgvopZPKiAlKQTGdMDQMc2PMTiVFrqoM7hD8bEfwzB/onkxEz0tNvjj
+/PIzark5McWvxI0NHWQWM6r6hCm21AvA2H3DkwIDAQABo4IBfTCCAXkwEgYDVR0T
+AQH/BAgwBgEB/wIBADAOBgNVHQ8BAf8EBAMCAYYwfwYIKwYBBQUHAQEEczBxMDIG
+CCsGAQUFBzABhiZodHRwOi8vaXNyZy50cnVzdGlkLm9jc3AuaWRlbnRydXN0LmNv
+bTA7BggrBgEFBQcwAoYvaHR0cDovL2FwcHMuaWRlbnRydXN0LmNvbS9yb290cy9k
+c3Ryb290Y2F4My5wN2MwHwYDVR0jBBgwFoAUxKexpHsscfrb4UuQdf/EFWCFiRAw
+VAYDVR0gBE0wSzAIBgZngQwBAgEwPwYLKwYBBAGC3xMBAQEwMDAuBggrBgEFBQcC
+ARYiaHR0cDovL2Nwcy5yb290LXgxLmxldHNlbmNyeXB0Lm9yZzA8BgNVHR8ENTAz
+MDGgL6AthitodHRwOi8vY3JsLmlkZW50cnVzdC5jb20vRFNUUk9PVENBWDNDUkwu
+Y3JsMB0GA1UdDgQWBBSoSmpjBH3duubRObemRWXv86jsoTANBgkqhkiG9w0BAQsF
+AAOCAQEA3TPXEfNjWDjdGBX7CVW+dla5cEilaUcne8IkCJLxWh9KEik3JHRRHGJo
+uM2VcGfl96S8TihRzZvoroed6ti6WqEBmtzw3Wodatg+VyOeph4EYpr/1wXKtx8/
+wApIvJSwtmVi4MFU5aMqrSDE6ea73Mj2tcMyo5jMd6jmeWUHK8so/joWUoHOUgwu
+X4Po1QYz+3dszkDqMp4fklxBwXRsW10KXzPMTZ+sOPAveyxindmjkW8lGy+QsRlG
+PfZ+G6Z6h7mjem0Y+iWlkYcV4PIWL1iwBi8saCbGS5jN2p8M+X+Q7UNKEkROb3N6
+KOqkqm57TH2H3eDJAkSnh6/DNFu0Qg==
+-----END CERTIFICATE-----
--- a/examples/protocols/mqtt/publish_test/main/publish_test.c
+++ b/examples/protocols/mqtt/publish_test/main/publish_test.c
@ -0,0 +1,258 @@
+/* MQTT publish test
+
+   This example code is in the Public Domain (or CC0 licensed, at your option.)
+
+   Unless required by applicable law or agreed to in writing, this
+   software is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+   CONDITIONS OF ANY KIND, either express or implied.
+*/
+#include <stdio.h>
+#include <stdint.h>
+#include <stddef.h>
+#include <string.h>
+#include "esp_wifi.h"
+#include "esp_system.h"
+#include "nvs_flash.h"
+#include "esp_event_loop.h"
+
+#include "freertos/FreeRTOS.h"
+#include "freertos/task.h"
+#include "freertos/semphr.h"
+#include "freertos/queue.h"
+#include "freertos/event_groups.h"
+
+#include "lwip/sockets.h"
+#include "lwip/dns.h"
+#include "lwip/netdb.h"
+
+#include "esp_log.h"
+#include "mqtt_client.h"
+
+static const char *TAG = "PUBLISH_TEST";
+
+static EventGroupHandle_t wifi_event_group;
+static EventGroupHandle_t mqtt_event_group;
+const static int CONNECTED_BIT = BIT0;
+
+static esp_mqtt_client_handle_t mqtt_client = NULL;
+
+static char *expected_data = NULL;
+static char *actual_data = NULL;
+static size_t expected_size = 0;
+static size_t expected_published = 0;
+static size_t actual_published = 0;
+static int qos_test = 0;
+
+static esp_err_t wifi_event_handler(void *ctx, system_event_t *event)
+{
+    switch (event->event_id) {
+    case SYSTEM_EVENT_STA_START:
+        esp_wifi_connect();
+        break;
+    case SYSTEM_EVENT_STA_GOT_IP:
+        xEventGroupSetBits(wifi_event_group, CONNECTED_BIT);
+
+        break;
+    case SYSTEM_EVENT_STA_DISCONNECTED:
+        esp_wifi_connect();
+        xEventGroupClearBits(wifi_event_group, CONNECTED_BIT);
+        break;
+    default:
+        break;
+    }
+    return ESP_OK;
+}
+
+static void wifi_init(void)
+{
+    tcpip_adapter_init();
+    wifi_event_group = xEventGroupCreate();
+    ESP_ERROR_CHECK(esp_event_loop_init(wifi_event_handler, NULL));
+    wifi_init_config_t cfg = WIFI_INIT_CONFIG_DEFAULT();
+    ESP_ERROR_CHECK(esp_wifi_init(&cfg));
+    ESP_ERROR_CHECK(esp_wifi_set_storage(WIFI_STORAGE_RAM));
+    wifi_config_t wifi_config = {
+        .sta = {
+            .ssid = CONFIG_WIFI_SSID,
+            .password = CONFIG_WIFI_PASSWORD,
+        },
+    };
+    ESP_ERROR_CHECK(esp_wifi_set_mode(WIFI_MODE_STA));
+    ESP_ERROR_CHECK(esp_wifi_set_config(ESP_IF_WIFI_STA, &wifi_config));
+    ESP_LOGI(TAG, "start the WIFI SSID:[%s]", CONFIG_WIFI_SSID);
+    ESP_ERROR_CHECK(esp_wifi_start());
+    ESP_LOGI(TAG, "Waiting for wifi");
+    xEventGroupWaitBits(wifi_event_group, CONNECTED_BIT, false, true, portMAX_DELAY);
+}
+
+#if CONFIG_BROKER_CERTIFICATE_OVERRIDDEN == 1
+static const uint8_t iot_eclipse_org_pem_start[]  = "-----BEGIN CERTIFICATE-----\n" CONFIG_BROKER_CERTIFICATE_OVERRIDE "\n-----END CERTIFICATE-----";
+#else
+extern const uint8_t iot_eclipse_org_pem_start[]   asm("_binary_iot_eclipse_org_pem_start");
+#endif
+extern const uint8_t iot_eclipse_org_pem_end[]   asm("_binary_iot_eclipse_org_pem_end");
+
+static esp_err_t mqtt_event_handler(esp_mqtt_event_handle_t event)
+{
+    esp_mqtt_client_handle_t client = event->client;
+    static int msg_id = 0;
+    static int actual_len = 0;
+    // your_context_t *context = event->context;
+    switch (event->event_id) {
+    case MQTT_EVENT_CONNECTED:
+        ESP_LOGI(TAG, "MQTT_EVENT_CONNECTED");
+        xEventGroupSetBits(mqtt_event_group, CONNECTED_BIT);
+        msg_id = esp_mqtt_client_subscribe(client, CONFIG_SUBSCIBE_TOPIC, qos_test);
+        ESP_LOGI(TAG, "sent subscribe successful, msg_id=%d", msg_id);
+
+        break;
+    case MQTT_EVENT_DISCONNECTED:
+        ESP_LOGI(TAG, "MQTT_EVENT_DISCONNECTED");
+        break;
+
+    case MQTT_EVENT_SUBSCRIBED:
+        ESP_LOGI(TAG, "MQTT_EVENT_SUBSCRIBED, msg_id=%d", event->msg_id);
+        break;
+    case MQTT_EVENT_UNSUBSCRIBED:
+        ESP_LOGI(TAG, "MQTT_EVENT_UNSUBSCRIBED, msg_id=%d", event->msg_id);
+        break;
+    case MQTT_EVENT_PUBLISHED:
+        ESP_LOGI(TAG, "MQTT_EVENT_PUBLISHED, msg_id=%d", event->msg_id);
+        break;
+    case MQTT_EVENT_DATA:
+        ESP_LOGI(TAG, "MQTT_EVENT_DATA");
+        printf("TOPIC=%.*s\r\n", event->topic_len, event->topic);
+        printf("DATA=%.*s\r\n", event->data_len, event->data);
+        printf("ID=%d, total_len=%d, data_len=%d, current_data_offset=%d\n", event->msg_id, event->total_data_len, event->data_len, event->current_data_offset);
+        if (event->topic) {
+            actual_len = event->data_len;
+            msg_id = event->msg_id;
+        } else {
+            actual_len += event->data_len;
+            // check consisency with msg_id across multiple data events for single msg
+            if (msg_id != event->msg_id) {
+                ESP_LOGI(TAG, "Wrong msg_id in chunked message %d != %d", msg_id, event->msg_id);
+                abort();
+            }
+        }
+        memcpy(actual_data + event->current_data_offset, event->data, event->data_len);
+        if (actual_len == event->total_data_len) {
+            if (0 == memcmp(actual_data, expected_data, expected_size)) {
+                printf("OK!");
+                memset(actual_data, 0, expected_size);
+                actual_published ++;
+                if (actual_published == expected_published) {
+                    printf("Correct pattern received exactly x times\n");
+                    ESP_LOGI(TAG, "Test finished correctly!");
+                }
+            } else {
+                printf("FAILED!");
+                abort();
+            }
+        }
+        break;
+    case MQTT_EVENT_ERROR:
+        ESP_LOGI(TAG, "MQTT_EVENT_ERROR");
+        break;
+    default:
+        ESP_LOGI(TAG, "Other event id:%d", event->event_id);
+        break;
+    }
+    return ESP_OK;
+}
+
+
+static void mqtt_app_start(void)
+{
+    mqtt_event_group = xEventGroupCreate();
+    const esp_mqtt_client_config_t mqtt_cfg = {
+        .event_handle = mqtt_event_handler,
+        .cert_pem = (const char *)iot_eclipse_org_pem_start,
+    };
+
+    ESP_LOGI(TAG, "[APP] Free memory: %d bytes", esp_get_free_heap_size());
+    mqtt_client = esp_mqtt_client_init(&mqtt_cfg);
+}
+
+static void get_string(char *line, size_t size)
+{
+
+    int count = 0;
+    while (count < size) {
+        int c = fgetc(stdin);
+        if (c == '\n') {
+            line[count] = '\0';
+            break;
+        } else if (c > 0 && c < 127) {
+            line[count] = c;
+            ++count;
+        }
+        vTaskDelay(10 / portTICK_PERIOD_MS);
+    }
+}
+
+void app_main()
+{
+    char line[256];
+    char pattern[32];
+    char transport[32];
+    int repeat = 0;
+
+    ESP_LOGI(TAG, "[APP] Free memory: %d bytes", esp_get_free_heap_size());
+    ESP_LOGI(TAG, "[APP] IDF version: %s", esp_get_idf_version());
+
+    esp_log_level_set("*", ESP_LOG_INFO);
+    esp_log_level_set("MQTT_CLIENT", ESP_LOG_VERBOSE);
+    esp_log_level_set("TRANSPORT_TCP", ESP_LOG_VERBOSE);
+    esp_log_level_set("TRANSPORT_SSL", ESP_LOG_VERBOSE);
+    esp_log_level_set("TRANSPORT", ESP_LOG_VERBOSE);
+    esp_log_level_set("OUTBOX", ESP_LOG_VERBOSE);
+
+    nvs_flash_init();
+    wifi_init();
+    mqtt_app_start();
+
+    while (1) {
+        get_string(line, sizeof(line));
+        sscanf(line, "%s %s %d %d %d", transport, pattern, &repeat, &expected_published, &qos_test);
+        ESP_LOGI(TAG, "PATTERN:%s REPEATED:%d PUBLISHED:%d\n", pattern, repeat, expected_published);
+        int pattern_size = strlen(pattern);
+        free(expected_data);
+        free(actual_data);
+        actual_published = 0;
+        expected_size = pattern_size * repeat;
+        expected_data = malloc(expected_size);
+        actual_data = malloc(expected_size);
+        for (int i = 0; i < repeat; i++) {
+            memcpy(expected_data + i * pattern_size, pattern, pattern_size);
+        }
+        printf("EXPECTED STRING %.*s, SIZE:%d\n", expected_size, expected_data, expected_size);
+        esp_mqtt_client_stop(mqtt_client);
+
+        if (0 == strcmp(transport, "tcp")) {
+            ESP_LOGI(TAG, "[TCP transport] Startup..");
+            esp_mqtt_client_set_uri(mqtt_client, CONFIG_BROKER_TCP_URI);
+        } else if (0 == strcmp(transport, "ssl")) {
+            ESP_LOGI(TAG, "[SSL transport] Startup..");
+            esp_mqtt_client_set_uri(mqtt_client, CONFIG_BROKER_SSL_URI);
+        } else if (0 == strcmp(transport, "ws")) {
+            ESP_LOGI(TAG, "[WS transport] Startup..");
+            esp_mqtt_client_set_uri(mqtt_client, CONFIG_BROKER_WS_URI);
+        } else if (0 == strcmp(transport, "wss")) {
+            ESP_LOGI(TAG, "[WSS transport] Startup..");
+            esp_mqtt_client_set_uri(mqtt_client, CONFIG_BROKER_WSS_URI);
+        } else {
+            ESP_LOGE(TAG, "Unexpected transport");
+            abort();
+        }
+        xEventGroupClearBits(mqtt_event_group, CONNECTED_BIT);
+        esp_mqtt_client_start(mqtt_client);
+        ESP_LOGI(TAG, "Note free memory: %d bytes", esp_get_free_heap_size());
+        xEventGroupWaitBits(mqtt_event_group, CONNECTED_BIT, false, true, portMAX_DELAY);
+
+        for (int i = 0; i < expected_published; i++) {
+            int msg_id = esp_mqtt_client_publish(mqtt_client, CONFIG_PUBLISH_TOPIC, expected_data, expected_size, qos_test, 0);
+            ESP_LOGI(TAG, "[%d] Publishing...", msg_id);
+        }
+    }
+}
--- a/examples/protocols/mqtt/publish_test/sdkconfig.ci
+++ b/examples/protocols/mqtt/publish_test/sdkconfig.ci
@ -0,0 +1,5 @@
+CONFIG_BROKER_SSL_URI="mqtts://${EXAMPLE_MQTT_BROKER_SSL}"
+CONFIG_BROKER_TCP_URI="mqtt://${EXAMPLE_MQTT_BROKER_TCP}"
+CONFIG_BROKER_WS_URI="ws://${EXAMPLE_MQTT_BROKER_WS}/ws"
+CONFIG_BROKER_WSS_URI="wss://${EXAMPLE_MQTT_BROKER_WSS}/ws"
+CONFIG_BROKER_CERTIFICATE_OVERRIDE="${EXAMPLE_MQTT_BROKER_CERTIFICATE}"
--- a/tools/tiny-test-fw/IDF/IDFApp.py
+++ b/tools/tiny-test-fw/IDF/IDFApp.py
@ -81,7 +81,7 @@ class IDFApp(App.BaseApp):
            for line in f:
                configs = line.split('=')
                if len(configs) == 2:
-                    d[configs[0]] = configs[1]
+                    d[configs[0]] = configs[1].rstrip()
        return d

    def get_binary_path(self, app_path):
				`@ -0,0 +1 @@`
				`COMPONENT_EMBED_TXTFILES := iot_eclipse_org.pem`