Merge branch 'example/non_blocking_sockets' into 'master'

examples: Added non blockinng sockets example tests

Closes IDFGH-4879

See merge request espressif/esp-idf!12928

examples/protocols/sockets/non_blocking/CMakeLists.txt

@@ -0,0 +1,10 @@
+# The following five lines of boilerplate have to be in your project's
+# CMakeLists in this exact order for cmake to work correctly
+cmake_minimum_required(VERSION 3.5)
+
+# (Not part of the boilerplate)
+# This example uses an extra component for common functions such as Wi-Fi and Ethernet connection.
+set(EXTRA_COMPONENT_DIRS $ENV{IDF_PATH}/examples/common_components/protocol_examples_common)
+
+include($ENV{IDF_PATH}/tools/cmake/project.cmake)
+project(non_blocking_socket)

examples/protocols/sockets/non_blocking/Makefile

@@ -0,0 +1,10 @@
+#
+# This is a project Makefile. It is assumed the directory this Makefile resides in is a
+# project subdirectory.
+#
+
+PROJECT_NAME := non_blocking_socket
+
+EXTRA_COMPONENT_DIRS = $(IDF_PATH)/examples/common_components/protocol_examples_common
+
+include $(IDF_PATH)/make/project.mk

examples/protocols/sockets/non_blocking/README.md

@@ -0,0 +1,70 @@
+
+# TCP non-blocking client and server examples
+
+(See the README.md file in the upper level 'examples' directory for more information about examples.)
+
+The application aims to demonstrate a simple use of TCP sockets in a nonblocking mode.
+It could be configured to run either a TCP server, or a TCP client, or both, in the project configuration settings.
+
+## How to use example
+
+The example is configured by default as the TCP client. 
+
+Note that the example uses string representation of IP addresses and ports and thus
+could be used on both IPv4 and IPv6 protocols.
+
+### TCP Client
+
+In the client mode, the example connects to a configured hostname or address, sends the specified payload data and waits for a response,
+then closes the connection. By default, it connects to a public http server and performs a simple http `GET` request.
+
+### TCP Server
+
+The server example creates a non-blocking TCP socket with the specified port number and polls till
+a connection request from the client arrives.
+After accepting a request from the client, a connection between server and client is
+established, and the application polls for some data to be received from the client.
+Received data are printed as ASCII text and retransmitted back to the client.
+
+The server could listen on the specified interface (by the configured bound address) and serves multiple clients.
+It resumes to listening for new connections when the client's socket gets closed.
+
+## Hardware Required
+
+This example can be run on any commonly available ESP32 development board.
+
+## Configure the project
+
+```
+idf.py menuconfig
+```
+
+Set following parameters under Example Configuration Options:
+
+* Set `EXAMPLE_TCP_SERVER` to use the example as a non-blocking TCP server
+    * Configure `EXAMPLE_TCP_SERVER_BIND_ADDRESS` to a string representation of the address to bind the server socket to.
+    * Configure `EXAMPLE_TCP_SERVER_BIND_PORT` to the port number.
+
+* Set `EXAMPLE_TCP_CLIENT` to use the example as a non-blocking TCP client
+    * Configure `EXAMPLE_TCP_CLIENT_CONNECT_ADDRESS` to a string representation of the address to connect the client to.
+    * Configure `EXAMPLE_TCP_CLIENT_CONNECT_PORT` to the port number.
+
+* Configure Wi-Fi or Ethernet under "Example Connection Configuration" menu. See "Establishing Wi-Fi or Ethernet Connection" section in [examples/protocols/README.md](../../README.md) for more details.
+
+## Build and Flash
+
+Build the project and flash it to the board, then run monitor tool to view serial output:
+
+```
+idf.py -p PORT flash monitor
+```
+
+(To exit the serial monitor, type ``Ctrl-]``.)
+
+See the Getting Started Guide for full steps to configure and use ESP-IDF to build projects.
+
+
+## Troubleshooting
+
+Follow the same troubleshooting instruction as for the standard [TCP sever](../tcp_server/README.md) and the [TCP client](../tcp_client/README.md),
+using the host tools and scripts as descibed in the upper level documentation on [BSD socket examples](../README.md).

examples/protocols/sockets/non_blocking/example_test.py

@@ -0,0 +1,26 @@
+# This example code is in the Public Domain (or CC0 licensed, at your option.)
+
+# Unless required by applicable law or agreed to in writing, this
+# software is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+# CONDITIONS OF ANY KIND, either express or implied.
+
+# -*- coding: utf-8 -*-
+
+from __future__ import print_function, unicode_literals
+
+import re
+
+import ttfw_idf
+
+
+@ttfw_idf.idf_example_test(env_tag='Example_GENERIC')
+def test_examples_protocol_socket_non_block(env, _):
+    dut = env.get_dut('non_blocking_socket', 'examples/protocols/sockets/non_blocking', dut_class=ttfw_idf.ESP32DUT)
+
+    # start the test and expect the client to receive back it's original data
+    dut.start_app()
+    dut.expect(re.compile(r'nonblocking-socket-client: Received: GET / HTTP/1.1'), timeout=30)
+
+
+if __name__ == '__main__':
+    test_examples_protocol_socket_non_block()

examples/protocols/sockets/non_blocking/main/CMakeLists.txt

@@ -0,0 +1,2 @@
+idf_component_register(SRCS "non_blocking_socket_example.c"
+                    INCLUDE_DIRS ".")

examples/protocols/sockets/non_blocking/main/Kconfig.projbuild

@@ -0,0 +1,48 @@
+menu "Example Configuration"
+
+    config EXAMPLE_TCP_SERVER
+        bool "TCP server"
+        default n
+        help
+            This example will setup a tcp server, binds it to the specified address
+            and starts listening
+
+    if EXAMPLE_TCP_SERVER
+        config EXAMPLE_TCP_SERVER_BIND_ADDRESS
+            string "Server bind address"
+            default "0.0.0.0"
+            help
+                Server listener's socket would be bound to this address. This address could be
+                either IPv4 or IPv6 address
+
+        config EXAMPLE_TCP_SERVER_BIND_PORT
+            string "Server bind port"
+            default "3344"
+            help
+                Server listener's socket would be bound to this port.
+
+    endif
+
+    config EXAMPLE_TCP_CLIENT
+        bool "TCP client"
+        default y
+        help
+            This example will setup a tcp client, connects to the specified address
+            and sends the data.
+
+    if EXAMPLE_TCP_CLIENT
+        config EXAMPLE_TCP_CLIENT_CONNECT_ADDRESS
+            string "Client connection address or hostname"
+            default "www.google.com"
+            help
+                Client's socket would connect to this address/host.
+
+        config EXAMPLE_TCP_CLIENT_CONNECT_PORT
+            string "Client connection port"
+            default "80"
+            help
+                Client connection port.
+
+    endif
+
+endmenu

examples/protocols/sockets/non_blocking/main/component.mk

@@ -0,0 +1,4 @@
+#
+# "main" pseudo-component makefile.
+#
+# (Uses default behaviour of compiling all source files in directory, adding 'include' to include path.)

examples/protocols/sockets/non_blocking/main/non_blocking_socket_example.c

@@ -0,0 +1,401 @@
+/* BSD non-blocking socket example
+
+   This example code is in the Public Domain (or CC0 licensed, at your option.)
+
+   Unless required by applicable law or agreed to in writing, this
+   software is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+   CONDITIONS OF ANY KIND, either express or implied.
+*/
+#include <string.h>
+#include "freertos/FreeRTOS.h"
+#include "freertos/task.h"
+#include "sys/socket.h"
+#include "netdb.h"
+#include "errno.h"
+#include "esp_system.h"
+#include "esp_event.h"
+#include "esp_log.h"
+#include "nvs_flash.h"
+#include "protocol_examples_common.h"
+
+/**
+ * @brief Indicates that the file descriptor represents an invalid (uninitialized or closed) socket
+ *
+ * Used in the TCP server structure `sock[]` which holds list of active clients we serve.
+ */
+#define INVALID_SOCK (-1)
+
+/**
+ * @brief Time in ms to yield to all tasks when a non-blocking socket would block
+ *
+ * Non-blocking socket operations are typically executed in a separate task validating
+ * the socket status. Whenever the socket returns `EAGAIN` (idle status, i.e. would block)
+ * we have to yield to all tasks to prevent lower priority tasks from starving.
+ */
+#define YIELD_TO_ALL_MS 50
+
+/**
+ * @brief Utility to log socket errors
+ *
+ * @param[in] tag Logging tag
+ * @param[in] sock Socket number
+ * @param[in] err Socket errno
+ * @param[in] message Message to print
+ */
+static void log_socket_error(const char *tag, const int sock, const int err, const char *message)
+{
+    ESP_LOGE(tag, "[sock=%d]: %s\n"
+                  "error=%d: %s", sock, message, err, strerror(err));
+}
+
+/**
+ * @brief Tries to receive data from specified sockets in a non-blocking way,
+ *        i.e. returns immediately if no data.
+ *
+ * @param[in] tag Logging tag
+ * @param[in] sock Socket for reception
+ * @param[out] data Data pointer to write the received data
+ * @param[in] max_len Maximum size of the allocated space for receiving data
+ * @return
+ *          >0 : Size of received data
+ *          =0 : No data available
+ *          -1 : Error occurred during socket read operation
+ *          -2 : Socket is not connected, to distinguish between an actual socket error and active disconnection
+ */
+static int try_receive(const char *tag, const int sock, char * data, size_t max_len)
+{
+    int len = recv(sock, data, max_len, 0);
+    if (len < 0) {
+        if (errno == EINPROGRESS || errno == EAGAIN || errno == EWOULDBLOCK) {
+            return 0;   // Not an error
+        }
+        if (errno == ENOTCONN) {
+            ESP_LOGW(tag, "[sock=%d]: Connection closed", sock);
+            return -2;  // Socket has been disconnected
+        }
+        log_socket_error(tag, sock, errno, "Error occurred during receiving");
+        return -1;
+    }
+
+    return len;
+}
+
+/**
+ * @brief Sends the specified data to the socket. This function blocks until all bytes got sent.
+ *
+ * @param[in] tag Logging tag
+ * @param[in] sock Socket to write data
+ * @param[in] data Data to be written
+ * @param[in] len Length of the data
+ * @return
+ *          >0 : Size the written data
+ *          -1 : Error occurred during socket write operation
+ */
+static int socket_send(const char *tag, const int sock, const char * data, const size_t len)
+{
+    int to_write = len;
+    while (to_write > 0) {
+        int written = send(sock, data + (len - to_write), to_write, 0);
+        if (written < 0 && errno != EINPROGRESS && errno != EAGAIN && errno != EWOULDBLOCK) {
+            log_socket_error(tag, sock, errno, "Error occurred during sending");
+            return -1;
+        }
+        to_write -= written;
+    }
+    return len;
+}
+
+
+#ifdef CONFIG_EXAMPLE_TCP_CLIENT
+
+static void tcp_client_task(void *pvParameters)
+{
+    static const char *TAG = "nonblocking-socket-client";
+    static const char *payload = "GET / HTTP/1.1\r\n\r\n";
+    static char rx_buffer[128];
+
+    struct addrinfo hints = { .ai_socktype = SOCK_STREAM };
+    struct addrinfo *address_info;
+    int sock = INVALID_SOCK;
+
+    int res = getaddrinfo(CONFIG_EXAMPLE_TCP_CLIENT_CONNECT_ADDRESS, CONFIG_EXAMPLE_TCP_CLIENT_CONNECT_PORT, &hints, &address_info);
+    if (res != 0 || address_info == NULL) {
+        ESP_LOGE(TAG, "couldn't get hostname for `%s` "
+                      "getaddrinfo() returns %d, addrinfo=%p", CONFIG_EXAMPLE_TCP_CLIENT_CONNECT_ADDRESS, res, address_info);
+        goto error;
+    }
+
+    // Creating client's socket
+    sock = socket(address_info->ai_family, address_info->ai_socktype, address_info->ai_protocol);
+    if (sock < 0) {
+        log_socket_error(TAG, sock, errno, "Unable to create socket");
+        goto error;
+    }
+    ESP_LOGI(TAG, "Socket created, connecting to %s:%s", CONFIG_EXAMPLE_TCP_CLIENT_CONNECT_ADDRESS, CONFIG_EXAMPLE_TCP_CLIENT_CONNECT_PORT);
+
+    // Marking the socket as non-blocking
+    int flags = fcntl(sock, F_GETFL);
+    if (fcntl(sock, F_SETFL, flags | O_NONBLOCK) == -1) {
+        log_socket_error(TAG, sock, errno, "Unable to set socket non blocking");
+    }
+
+    if (connect(sock, address_info->ai_addr, address_info->ai_addrlen) != 0) {
+        if (errno == EINPROGRESS) {
+            ESP_LOGD(TAG, "connection in progress");
+            fd_set fdset;
+            FD_ZERO(&fdset);
+            FD_SET(sock, &fdset);
+
+            // Connection in progress -> have to wait until the connecting socket is marked as writable, i.e. connection completes
+            res = select(sock+1, NULL, &fdset, NULL, NULL);
+            if (res < 0) {
+                log_socket_error(TAG, sock, errno, "Error during connection: select for socket to be writable");
+                goto error;
+            } else if (res == 0) {
+                log_socket_error(TAG, sock, errno, "Connection timeout: select for socket to be writable");
+                goto error;
+            } else {
+                int sockerr;
+                socklen_t len = (socklen_t)sizeof(int);
+
+                if (getsockopt(sock, SOL_SOCKET, SO_ERROR, (void*)(&sockerr), &len) < 0) {
+                    log_socket_error(TAG, sock, errno, "Error when getting socket error using getsockopt()");
+                    goto error;
+                }
+                if (sockerr) {
+                    log_socket_error(TAG, sock, sockerr, "Connection error");
+                    goto error;
+                }
+            }
+        } else {
+            log_socket_error(TAG, sock, errno, "Socket is unable to connect");
+            goto error;
+        }
+    }
+
+    ESP_LOGI(TAG, "Client sends data to the server...");
+    int len = socket_send(TAG, sock, payload, strlen(payload));
+    if (len < 0) {
+        ESP_LOGE(TAG, "Error occurred during socket_send");
+        goto error;
+    }
+    ESP_LOGI(TAG, "Written: %.*s", len, payload);
+
+    // Keep receiving until we have a reply
+    do {
+        len = try_receive(TAG, sock, rx_buffer, sizeof(rx_buffer));
+        if (len < 0) {
+            ESP_LOGE(TAG, "Error occurred during try_receive");
+            goto error;
+        }
+        vTaskDelay(pdMS_TO_TICKS(YIELD_TO_ALL_MS));
+    } while (len == 0);
+    ESP_LOGI(TAG, "Received: %.*s", len, rx_buffer);
+
+error:
+    if (sock != INVALID_SOCK) {
+        close(sock);
+    }
+    free(address_info);
+    vTaskDelete(NULL);
+
+}
+#endif // CONFIG_EXAMPLE_TCP_CLIENT
+
+
+#ifdef CONFIG_EXAMPLE_TCP_SERVER
+
+/**
+ * @brief Returns the string representation of client's address (accepted on this server)
+ */
+static inline char* get_clients_address(struct sockaddr_storage *source_addr)
+{
+    static char address_str[128];
+    char *res = NULL;
+    // Convert ip address to string
+    if (source_addr->ss_family == PF_INET) {
+        res = inet_ntoa_r(((struct sockaddr_in *)source_addr)->sin_addr, address_str, sizeof(address_str) - 1);
+    }
+#ifdef CONFIG_LWIP_IPV6
+    else if (source_addr->ss_family == PF_INET6) {
+        res = inet6_ntoa_r(((struct sockaddr_in6 *)source_addr)->sin6_addr, address_str, sizeof(address_str) - 1);
+    }
+#endif
+    if (!res) {
+        address_str[0] = '\0'; // Returns empty string if conversion didn't succeed
+    }
+    return address_str;
+}
+
+static void tcp_server_task(void *pvParameters)
+{
+    static char rx_buffer[128];
+    static const char *TAG = "nonblocking-socket-server";
+    xSemaphoreHandle *server_ready = pvParameters;
+    struct addrinfo hints = { .ai_socktype = SOCK_STREAM };
+    struct addrinfo *address_info;
+    int listen_sock = INVALID_SOCK;
+    const size_t max_socks = CONFIG_LWIP_MAX_SOCKETS - 1;
+    static int sock[CONFIG_LWIP_MAX_SOCKETS - 1];
+
+    // Prepare a list of file descriptors to hold client's sockets, mark all of them as invalid, i.e. available
+    for (int i=0; i<max_socks; ++i) {
+        sock[i] = INVALID_SOCK;
+    }
+
+    // Translating the hostname or a string representation of an IP to address_info
+    int res = getaddrinfo(CONFIG_EXAMPLE_TCP_SERVER_BIND_ADDRESS, CONFIG_EXAMPLE_TCP_SERVER_BIND_PORT, &hints, &address_info);
+    if (res != 0 || address_info == NULL) {
+        ESP_LOGE(TAG, "couldn't get hostname for `%s` "
+                      "getaddrinfo() returns %d, addrinfo=%p", CONFIG_EXAMPLE_TCP_SERVER_BIND_ADDRESS, res, address_info);
+        goto error;
+    }
+
+    // Creating a listener socket
+    listen_sock = socket(address_info->ai_family, address_info->ai_socktype, address_info->ai_protocol);
+
+    if (listen_sock < 0) {
+        log_socket_error(TAG, listen_sock, errno, "Unable to create socket");
+        goto error;
+    }
+    ESP_LOGI(TAG, "Listener socket created");
+
+    // Marking the socket as non-blocking
+    int flags = fcntl(listen_sock, F_GETFL);
+    if (fcntl(listen_sock, F_SETFL, flags | O_NONBLOCK) == -1) {
+        log_socket_error(TAG, listen_sock, errno, "Unable to set socket non blocking");
+        goto error;
+    }
+    ESP_LOGI(TAG, "Socket marked as non blocking");
+
+    // Binding socket to the given address
+    int err = bind(listen_sock, address_info->ai_addr, address_info->ai_addrlen);
+    if (err != 0) {
+        log_socket_error(TAG, listen_sock, errno, "Socket unable to bind");
+        goto error;
+    }
+    ESP_LOGI(TAG, "Socket bound on %s:%s", CONFIG_EXAMPLE_TCP_SERVER_BIND_ADDRESS, CONFIG_EXAMPLE_TCP_SERVER_BIND_PORT);
+
+    // Set queue (backlog) of pending connections to one (can be more)
+    err = listen(listen_sock, 1);
+    if (err != 0) {
+        log_socket_error(TAG, listen_sock, errno, "Error occurred during listen");
+        goto error;
+    }
+    ESP_LOGI(TAG, "Socket listening");
+    xSemaphoreGive(*server_ready);
+
+    // Main loop for accepting new connections and serving all connected clients
+    while (1) {
+        struct sockaddr_storage source_addr; // Large enough for both IPv4 or IPv6
+        socklen_t addr_len = sizeof(source_addr);
+
+        // Find a free socket
+        int new_sock_index = 0;
+        for (new_sock_index=0; new_sock_index<max_socks; ++new_sock_index) {
+            if (sock[new_sock_index] == INVALID_SOCK) {
+                break;
+            }
+        }
+
+        // We accept a new connection only if we have a free socket
+        if (new_sock_index < max_socks) {
+            // Try to accept a new connections
+            sock[new_sock_index] = accept(listen_sock, (struct sockaddr *)&source_addr, &addr_len);
+
+            if (sock[new_sock_index] < 0) {
+                if (errno == EWOULDBLOCK) { // The listener socket did not accepts any connection
+                                            // continue to serve open connections and try to accept again upon the next iteration
+                    ESP_LOGV(TAG, "No pending connections...");
+                } else {
+                    log_socket_error(TAG, listen_sock, errno, "Error when accepting connection");
+                    goto error;
+                }
+            } else {
+                // We have a new client connected -> print it's address
+                ESP_LOGI(TAG, "[sock=%d]: Connection accepted from IP:%s", sock[new_sock_index], get_clients_address(&source_addr));
+
+                // ...and set the client's socket non-blocking
+                flags = fcntl(sock[new_sock_index], F_GETFL);
+                if (fcntl(sock[new_sock_index], F_SETFL, flags | O_NONBLOCK) == -1) {
+                    log_socket_error(TAG, sock[new_sock_index], errno, "Unable to set socket non blocking");
+                    goto error;
+                }
+                ESP_LOGI(TAG, "[sock=%d]: Socket marked as non blocking", sock[new_sock_index]);
+            }
+        }
+
+        // We serve all the connected clients in this loop
+        for (int i=0; i<max_socks; ++i) {
+            if (sock[i] != INVALID_SOCK) {
+
+                // This is an open socket -> try to serve it
+                int len = try_receive(TAG, sock[i], rx_buffer, sizeof(rx_buffer));
+                if (len < 0) {
+                    // Error occurred within this client's socket -> close and mark invalid
+                    ESP_LOGI(TAG, "[sock=%d]: try_receive() returned %d -> closing the socket", sock[i], len);
+                    close(sock[i]);
+                    sock[i] = INVALID_SOCK;
+                } else if (len > 0) {
+                    // Received some data -> echo back
+                    ESP_LOGI(TAG, "[sock=%d]: Received %.*s", sock[i], len, rx_buffer);
+
+                    len = socket_send(TAG, sock[i], rx_buffer, len);
+                    if (len < 0) {
+                        // Error occurred on write to this socket -> close it and mark invalid
+                        ESP_LOGI(TAG, "[sock=%d]: socket_send() returned %d -> closing the socket", sock[i], len);
+                        close(sock[i]);
+                        sock[i] = INVALID_SOCK;
+                    } else {
+                        // Successfully echoed to this socket
+                        ESP_LOGI(TAG, "[sock=%d]: Written %.*s", sock[i], len, rx_buffer);
+                    }
+                }
+
+            } // one client's socket
+        } // for all sockets
+
+        // Yield to other tasks
+        vTaskDelay(pdMS_TO_TICKS(YIELD_TO_ALL_MS));
+    }
+
+error:
+    if (listen_sock != INVALID_SOCK) {
+        close(listen_sock);
+    }
+
+    for (int i=0; i<max_socks; ++i) {
+        if (sock[i] != INVALID_SOCK) {
+            close(sock[i]);
+        }
+    }
+
+    free(address_info);
+    vTaskDelete(NULL);
+}
+#endif  // CONFIG_EXAMPLE_TCP_SERVER
+
+void app_main(void)
+{
+    ESP_ERROR_CHECK(nvs_flash_init());
+    ESP_ERROR_CHECK(esp_netif_init());
+    ESP_ERROR_CHECK(esp_event_loop_create_default());
+
+    /* This helper function configures Wi-Fi or Ethernet, as selected in menuconfig.
+     * Read "Establishing Wi-Fi or Ethernet Connection" section in
+     * examples/protocols/README.md for more information about this function.
+     */
+    ESP_ERROR_CHECK(example_connect());
+
+#ifdef CONFIG_EXAMPLE_TCP_SERVER
+    xSemaphoreHandle server_ready = xSemaphoreCreateBinary();
+    assert(server_ready);
+    xTaskCreate(tcp_server_task, "tcp_server", 4096, &server_ready, 5, NULL);
+    xSemaphoreTake(server_ready, portMAX_DELAY);
+    vSemaphoreDelete(server_ready);
+#endif // CONFIG_EXAMPLE_TCP_SERVER
+
+#ifdef CONFIG_EXAMPLE_TCP_CLIENT
+    xTaskCreate(tcp_client_task, "tcp_client", 4096, NULL, 5, NULL);
+#endif // CONFIG_EXAMPLE_TCP_CLIENT
+}

examples/protocols/sockets/non_blocking/sdkconfig.ci

@@ -0,0 +1,5 @@
+CONFIG_EXAMPLE_CONNECT_WIFI=n
+CONFIG_EXAMPLE_CONNECT_ETHERNET=n
+CONFIG_EXAMPLE_TCP_SERVER=y
+CONFIG_EXAMPLE_TCP_CLIENT_CONNECT_ADDRESS="localhost"
+CONFIG_EXAMPLE_TCP_CLIENT_CONNECT_PORT="3344"

tools/ci/mypy_ignore_list.txt

@@ -97,6 +97,7 @@ examples/protocols/openssl_client/example_test.py
 examples/protocols/openssl_server/example_test.py
 examples/protocols/pppos_client/example_test.py
 examples/protocols/sntp/example_test.py
+examples/protocols/sockets/non_blocking/example_test.py
 examples/protocols/sockets/tcp_client/example_test.py
 examples/protocols/sockets/tcp_server/example_test.py
 examples/protocols/sockets/udp_client/example_test.py