examples/protocols: Added URI encoding/decoding feature

- http_server/simple: Decoding received query
  - esp_http_client: Sending encoded query

examples/common_components/protocol_examples_common/CMakeLists.txt

@@ -1,7 +1,8 @@
 set(srcs "stdin_out.c"
          "addr_from_stdin.c"
          "connect.c"
-         "wifi_connect.c")
+         "wifi_connect.c"
+         "protocol_examples_utils.c")
 
 if(CONFIG_EXAMPLE_PROVIDE_WIFI_CONSOLE_CMD)
     list(APPEND srcs "console_cmd.c")

examples/common_components/protocol_examples_common/include/protocol_examples_utils.h

@@ -0,0 +1,49 @@
+/*
+ * Utility functions for protocol examples
+ *
+ * SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Unlicense OR CC0-1.0
+ */
+
+#pragma once
+
+#include <stdint.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+
+/**
+ * @brief Encode an URI
+ *
+ * @param dest       a destination memory location
+ * @param src        the source string
+ * @param len        the length of the source string
+ * @return uint32_t  the count of escaped characters
+ *
+ * @note Please allocate the destination buffer keeping in mind that encoding a
+ *       special character will take up 3 bytes (for '%' and two hex digits).
+ *       In the worst-case scenario, the destination buffer will have to be 3 times
+ *       that of the source string.
+ */
+uint32_t example_uri_encode(char *dest, const char *src, size_t len);
+
+/**
+ * @brief Decode an URI
+ *
+ * @param dest  a destination memory location
+ * @param src   the source string
+ * @param len   the length of the source string
+ *
+ * @note Please allocate the destination buffer keeping in mind that a decoded
+ *       special character will take up 2 less bytes than its encoded form.
+ *       In the worst-case scenario, the destination buffer will have to be
+ *       the same size that of the source string.
+ */
+void example_uri_decode(char *dest, const char *src, size_t len);
+
+#ifdef __cplusplus
+}
+#endif

examples/common_components/protocol_examples_common/protocol_examples_utils.c

@@ -0,0 +1,388 @@
+/*
+ * Utility functions for protocol examples
+ *
+ * SPDX-FileCopyrightText: 2002-2021 Igor Sysoev
+ *                         2011-2022 Nginx, Inc.
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * SPDX-FileContributor: 2023 Espressif Systems (Shanghai) CO LTD
+ */
+/*
+ * Copyright (C) 2002-2021 Igor Sysoev
+ * Copyright (C) 2011-2022 Nginx, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/types.h>
+
+#include "protocol_examples_utils.h"
+
+/* Type of Escape algorithms to be used */
+#define NGX_ESCAPE_URI            (0)
+#define NGX_ESCAPE_ARGS           (1)
+#define NGX_ESCAPE_URI_COMPONENT  (2)
+#define NGX_ESCAPE_HTML           (3)
+#define NGX_ESCAPE_REFRESH        (4)
+#define NGX_ESCAPE_MEMCACHED      (5)
+#define NGX_ESCAPE_MAIL_AUTH      (6)
+
+/* Type of Unescape algorithms to be used */
+#define NGX_UNESCAPE_URI          (1)
+#define NGX_UNESCAPE_REDIRECT     (2)
+
+
+uintptr_t ngx_escape_uri(u_char *dst, u_char *src, size_t size, unsigned int type)
+{
+    unsigned int      n;
+    uint32_t       *escape;
+    static u_char   hex[] = "0123456789ABCDEF";
+
+    /*
+     * Per RFC 3986 only the following chars are allowed in URIs unescaped:
+     *
+     * unreserved    = ALPHA / DIGIT / "-" / "." / "_" / "~"
+     * gen-delims    = ":" / "/" / "?" / "#" / "[" / "]" / "@"
+     * sub-delims    = "!" / "$" / "&" / "'" / "(" / ")"
+     *               / "*" / "+" / "," / ";" / "="
+     *
+     * And "%" can appear as a part of escaping itself.  The following
+     * characters are not allowed and need to be escaped: %00-%1F, %7F-%FF,
+     * " ", """, "<", ">", "\", "^", "`", "{", "|", "}".
+     */
+
+    /* " ", "#", "%", "?", not allowed */
+
+    static uint32_t   uri[] = {
+        0xffffffff, /* 1111 1111 1111 1111  1111 1111 1111 1111 */
+
+        /* ?>=< ;:98 7654 3210  /.-, +*)( '&%$ #"!  */
+        0xd000002d, /* 1101 0000 0000 0000  0000 0000 0010 1101 */
+
+        /* _^]\ [ZYX WVUT SRQP  ONML KJIH GFED CBA@ */
+        0x50000000, /* 0101 0000 0000 0000  0000 0000 0000 0000 */
+
+        /*  ~}| {zyx wvut srqp  onml kjih gfed cba` */
+        0xb8000001, /* 1011 1000 0000 0000  0000 0000 0000 0001 */
+
+        0xffffffff, /* 1111 1111 1111 1111  1111 1111 1111 1111 */
+        0xffffffff, /* 1111 1111 1111 1111  1111 1111 1111 1111 */
+        0xffffffff, /* 1111 1111 1111 1111  1111 1111 1111 1111 */
+        0xffffffff  /* 1111 1111 1111 1111  1111 1111 1111 1111 */
+    };
+
+    /* " ", "#", "%", "&", "+", ";", "?", not allowed */
+
+    static uint32_t   args[] = {
+        0xffffffff, /* 1111 1111 1111 1111  1111 1111 1111 1111 */
+
+        /* ?>=< ;:98 7654 3210  /.-, +*)( '&%$ #"!  */
+        0xd800086d, /* 1101 1000 0000 0000  0000 1000 0110 1101 */
+
+        /* _^]\ [ZYX WVUT SRQP  ONML KJIH GFED CBA@ */
+        0x50000000, /* 0101 0000 0000 0000  0000 0000 0000 0000 */
+
+        /*  ~}| {zyx wvut srqp  onml kjih gfed cba` */
+        0xb8000001, /* 1011 1000 0000 0000  0000 0000 0000 0001 */
+
+        0xffffffff, /* 1111 1111 1111 1111  1111 1111 1111 1111 */
+        0xffffffff, /* 1111 1111 1111 1111  1111 1111 1111 1111 */
+        0xffffffff, /* 1111 1111 1111 1111  1111 1111 1111 1111 */
+        0xffffffff  /* 1111 1111 1111 1111  1111 1111 1111 1111 */
+    };
+
+    /* not ALPHA, DIGIT, "-", ".", "_", "~" */
+
+    static uint32_t   uri_component[] = {
+        0xffffffff, /* 1111 1111 1111 1111  1111 1111 1111 1111 */
+
+        /* ?>=< ;:98 7654 3210  /.-, +*)( '&%$ #"!  */
+        0xfc009fff, /* 1111 1100 0000 0000  1001 1111 1111 1111 */
+
+        /* _^]\ [ZYX WVUT SRQP  ONML KJIH GFED CBA@ */
+        0x78000001, /* 0111 1000 0000 0000  0000 0000 0000 0001 */
+
+        /*  ~}| {zyx wvut srqp  onml kjih gfed cba` */
+        0xb8000001, /* 1011 1000 0000 0000  0000 0000 0000 0001 */
+
+        0xffffffff, /* 1111 1111 1111 1111  1111 1111 1111 1111 */
+        0xffffffff, /* 1111 1111 1111 1111  1111 1111 1111 1111 */
+        0xffffffff, /* 1111 1111 1111 1111  1111 1111 1111 1111 */
+        0xffffffff  /* 1111 1111 1111 1111  1111 1111 1111 1111 */
+    };
+
+    /* " ", "#", """, "%", "'", not allowed */
+
+    static uint32_t   html[] = {
+        0xffffffff, /* 1111 1111 1111 1111  1111 1111 1111 1111 */
+
+        /* ?>=< ;:98 7654 3210  /.-, +*)( '&%$ #"!  */
+        0x500000ad, /* 0101 0000 0000 0000  0000 0000 1010 1101 */
+
+        /* _^]\ [ZYX WVUT SRQP  ONML KJIH GFED CBA@ */
+        0x50000000, /* 0101 0000 0000 0000  0000 0000 0000 0000 */
+
+        /*  ~}| {zyx wvut srqp  onml kjih gfed cba` */
+        0xb8000001, /* 1011 1000 0000 0000  0000 0000 0000 0001 */
+
+        0xffffffff, /* 1111 1111 1111 1111  1111 1111 1111 1111 */
+        0xffffffff, /* 1111 1111 1111 1111  1111 1111 1111 1111 */
+        0xffffffff, /* 1111 1111 1111 1111  1111 1111 1111 1111 */
+        0xffffffff  /* 1111 1111 1111 1111  1111 1111 1111 1111 */
+    };
+
+    /* " ", """, "'", not allowed */
+
+    static uint32_t   refresh[] = {
+        0xffffffff, /* 1111 1111 1111 1111  1111 1111 1111 1111 */
+
+        /* ?>=< ;:98 7654 3210  /.-, +*)( '&%$ #"!  */
+        0x50000085, /* 0101 0000 0000 0000  0000 0000 1000 0101 */
+
+        /* _^]\ [ZYX WVUT SRQP  ONML KJIH GFED CBA@ */
+        0x50000000, /* 0101 0000 0000 0000  0000 0000 0000 0000 */
+
+        /*  ~}| {zyx wvut srqp  onml kjih gfed cba` */
+        0xd8000001, /* 1011 1000 0000 0000  0000 0000 0000 0001 */
+
+        0xffffffff, /* 1111 1111 1111 1111  1111 1111 1111 1111 */
+        0xffffffff, /* 1111 1111 1111 1111  1111 1111 1111 1111 */
+        0xffffffff, /* 1111 1111 1111 1111  1111 1111 1111 1111 */
+        0xffffffff  /* 1111 1111 1111 1111  1111 1111 1111 1111 */
+    };
+
+    /* " ", "%", %00-%1F */
+
+    static uint32_t   memcached[] = {
+        0xffffffff, /* 1111 1111 1111 1111  1111 1111 1111 1111 */
+
+        /* ?>=< ;:98 7654 3210  /.-, +*)( '&%$ #"!  */
+        0x00000021, /* 0000 0000 0000 0000  0000 0000 0010 0001 */
+
+        /* _^]\ [ZYX WVUT SRQP  ONML KJIH GFED CBA@ */
+        0x00000000, /* 0000 0000 0000 0000  0000 0000 0000 0000 */
+
+        /*  ~}| {zyx wvut srqp  onml kjih gfed cba` */
+        0x00000000, /* 0000 0000 0000 0000  0000 0000 0000 0000 */
+
+        0x00000000, /* 0000 0000 0000 0000  0000 0000 0000 0000 */
+        0x00000000, /* 0000 0000 0000 0000  0000 0000 0000 0000 */
+        0x00000000, /* 0000 0000 0000 0000  0000 0000 0000 0000 */
+        0x00000000, /* 0000 0000 0000 0000  0000 0000 0000 0000 */
+    };
+
+    /* mail_auth is the same as memcached */
+
+    static uint32_t  *map[] =
+    { uri, args, uri_component, html, refresh, memcached, memcached };
+
+
+    escape = map[type];
+
+    if (dst == NULL) {
+
+        /* find the number of the characters to be escaped */
+
+        n = 0;
+
+        while (size) {
+            if (escape[*src >> 5] & (1U << (*src & 0x1f))) {
+                n++;
+            }
+            src++;
+            size--;
+        }
+
+        return (uintptr_t) n;
+    }
+
+    while (size) {
+        if (escape[*src >> 5] & (1U << (*src & 0x1f))) {
+            *dst++ = '%';
+            *dst++ = hex[*src >> 4];
+            *dst++ = hex[*src & 0xf];
+            src++;
+
+        } else {
+            *dst++ = *src++;
+        }
+        size--;
+    }
+
+    return (uintptr_t) dst;
+}
+
+
+void ngx_unescape_uri(u_char **dst, u_char **src, size_t size, unsigned int type)
+{
+    u_char  *d, *s, ch, c, decoded;
+    enum {
+        sw_usual = 0,
+        sw_quoted,
+        sw_quoted_second
+    } state;
+
+    d = *dst;
+    s = *src;
+
+    state = 0;
+    decoded = 0;
+
+    while (size--) {
+
+        ch = *s++;
+
+        switch (state) {
+        case sw_usual:
+            if (ch == '?'
+                    && (type & (NGX_UNESCAPE_URI | NGX_UNESCAPE_REDIRECT))) {
+                *d++ = ch;
+                goto done;
+            }
+
+            if (ch == '%') {
+                state = sw_quoted;
+                break;
+            }
+
+            *d++ = ch;
+            break;
+
+        case sw_quoted:
+
+            if (ch >= '0' && ch <= '9') {
+                decoded = (u_char) (ch - '0');
+                state = sw_quoted_second;
+                break;
+            }
+
+            c = (u_char) (ch | 0x20);
+            if (c >= 'a' && c <= 'f') {
+                decoded = (u_char) (c - 'a' + 10);
+                state = sw_quoted_second;
+                break;
+            }
+
+            /* the invalid quoted character */
+
+            state = sw_usual;
+
+            *d++ = ch;
+
+            break;
+
+        case sw_quoted_second:
+
+            state = sw_usual;
+
+            if (ch >= '0' && ch <= '9') {
+                ch = (u_char) ((decoded << 4) + (ch - '0'));
+
+                if (type & NGX_UNESCAPE_REDIRECT) {
+                    if (ch > '%' && ch < 0x7f) {
+                        *d++ = ch;
+                        break;
+                    }
+
+                    *d++ = '%'; *d++ = *(s - 2); *d++ = *(s - 1);
+
+                    break;
+                }
+
+                *d++ = ch;
+
+                break;
+            }
+
+            c = (u_char) (ch | 0x20);
+            if (c >= 'a' && c <= 'f') {
+                ch = (u_char) ((decoded << 4) + (c - 'a') + 10);
+
+                if (type & NGX_UNESCAPE_URI) {
+                    if (ch == '?') {
+                        *d++ = ch;
+                        goto done;
+                    }
+
+                    *d++ = ch;
+                    break;
+                }
+
+                if (type & NGX_UNESCAPE_REDIRECT) {
+                    if (ch == '?') {
+                        *d++ = ch;
+                        goto done;
+                    }
+
+                    if (ch > '%' && ch < 0x7f) {
+                        *d++ = ch;
+                        break;
+                    }
+
+                    *d++ = '%'; *d++ = *(s - 2); *d++ = *(s - 1);
+                    break;
+                }
+
+                *d++ = ch;
+
+                break;
+            }
+
+            /* the invalid quoted character */
+
+            break;
+        }
+    }
+
+done:
+
+    *dst = d;
+    *src = s;
+}
+
+
+uint32_t example_uri_encode(char *dest, const char *src, size_t len)
+{
+    if (!src || !dest) {
+        return 0;
+    }
+
+    uintptr_t ret = ngx_escape_uri((unsigned char *)dest, (unsigned char *)src, len, NGX_ESCAPE_URI_COMPONENT);
+    return (uint32_t)(ret - (uintptr_t)dest);
+}
+
+
+void example_uri_decode(char *dest, const char *src, size_t len)
+{
+    if (!src || !dest) {
+        return;
+    }
+
+    unsigned char *src_ptr = (unsigned char *)src;
+    unsigned char *dst_ptr = (unsigned char *)dest;
+    ngx_unescape_uri(&dst_ptr, &src_ptr, len, NGX_UNESCAPE_URI);
+}

examples/protocols/esp_http_client/main/esp_http_client_example.c

@@ -10,11 +10,13 @@
 #include <string.h>
 #include <sys/param.h>
 #include <stdlib.h>
+#include <ctype.h>
 #include "esp_log.h"
 #include "nvs_flash.h"
 #include "esp_event.h"
 #include "esp_netif.h"
 #include "protocol_examples_common.h"
+#include "protocol_examples_utils.h"
 #include "esp_tls.h"
 #if CONFIG_MBEDTLS_CERTIFICATE_BUNDLE
 #include "esp_crt_bundle.h"
@@ -431,6 +433,34 @@ static void https_with_hostname_path(void)
     esp_http_client_cleanup(client);
 }
 
+static void http_encoded_query(void)
+{
+    esp_http_client_config_t config = {
+        .host = "httpbin.org",
+        .path = "/get",
+        .event_handler = _http_event_handler,
+    };
+
+    static const char query_val[] = "ABC xyz!012@#%&";
+    char query_val_enc[64] = {0};
+
+    uint32_t enc_len = example_uri_encode(query_val_enc, query_val, strlen(query_val));
+    if (enc_len > 0) {
+        ESP_LOG_BUFFER_HEXDUMP(TAG, query_val_enc, enc_len, ESP_LOG_DEBUG);
+        config.query = query_val_enc;
+    }
+
+    esp_http_client_handle_t client = esp_http_client_init(&config);
+    esp_err_t err = esp_http_client_perform(client);
+    if (err == ESP_OK) {
+        ESP_LOGI(TAG, "HTTP GET Status = %d, content_length = %"PRIu64,
+                esp_http_client_get_status_code(client),
+                esp_http_client_get_content_length(client));
+    } else {
+        ESP_LOGE(TAG, "HTTP GET request failed: %s", esp_err_to_name(err));
+    }
+}
+
 static void http_relative_redirect(void)
 {
     esp_http_client_config_t config = {
@@ -743,6 +773,7 @@ static void http_test_task(void *pvParameters)
 #if CONFIG_ESP_HTTP_CLIENT_ENABLE_DIGEST_AUTH
     http_auth_digest();
 #endif
+    http_encoded_query();
     http_relative_redirect();
     http_absolute_redirect();
     http_absolute_redirect_manual();

examples/protocols/http_server/simple/main/main.c

@@ -17,9 +17,12 @@
 #include "esp_netif.h"
 #include "esp_eth.h"
 #include "protocol_examples_common.h"
+#include "protocol_examples_utils.h"
 #include "esp_tls_crypto.h"
 #include <esp_http_server.h>
 
+#define EXAMPLE_HTTP_QUERY_KEY_MAX_LEN  (64)
+
 /* A simple example that demonstrates how to create GET and POST
  * handlers for the web server.
  */
@@ -188,16 +191,22 @@ static esp_err_t hello_get_handler(httpd_req_t *req)
         buf = malloc(buf_len);
         if (httpd_req_get_url_query_str(req, buf, buf_len) == ESP_OK) {
             ESP_LOGI(TAG, "Found URL query => %s", buf);
-            char param[32];
+            char param[EXAMPLE_HTTP_QUERY_KEY_MAX_LEN], dec_param[EXAMPLE_HTTP_QUERY_KEY_MAX_LEN] = {0};
             /* Get value of expected key from query string */
             if (httpd_query_key_value(buf, "query1", param, sizeof(param)) == ESP_OK) {
                 ESP_LOGI(TAG, "Found URL query parameter => query1=%s", param);
+                example_uri_decode(dec_param, param, strnlen(param, EXAMPLE_HTTP_QUERY_KEY_MAX_LEN));
+                ESP_LOGI(TAG, "Decoded query parameter => %s", dec_param);
             }
             if (httpd_query_key_value(buf, "query3", param, sizeof(param)) == ESP_OK) {
                 ESP_LOGI(TAG, "Found URL query parameter => query3=%s", param);
+                example_uri_decode(dec_param, param, strnlen(param, EXAMPLE_HTTP_QUERY_KEY_MAX_LEN));
+                ESP_LOGI(TAG, "Decoded query parameter => %s", dec_param);
             }
             if (httpd_query_key_value(buf, "query2", param, sizeof(param)) == ESP_OK) {
                 ESP_LOGI(TAG, "Found URL query parameter => query2=%s", param);
+                example_uri_decode(dec_param, param, strnlen(param, EXAMPLE_HTTP_QUERY_KEY_MAX_LEN));
+                ESP_LOGI(TAG, "Decoded query parameter => %s", dec_param);
             }
         }
         free(buf);

examples/protocols/http_server/simple/pytest_http_server_simple.py

@@ -115,17 +115,18 @@ def test_examples_protocol_http_server_simple(dut: Dut) -> None:
     if not client.test_post_handler(got_ip, got_port, random_data):
         raise RuntimeError
 
-    query = 'http://foobar'
+    queries = 'query1=http%3A%2F%2Ffoobar&query3=abcd%2B1234%20xyz&query2=Esp%21%40%20%23%2471'
-    logging.info('Test /hello with custom query : {}'.format(query))
+    logging.info('Test /hello with custom query')
-    if not client.test_custom_uri_query(got_ip, got_port, query):
+    if not client.test_custom_uri_query(got_ip, got_port, queries):
         raise RuntimeError
-    dut.expect('Found URL query => ' + query, timeout=30)
 
-    query = 'abcd+1234%20xyz'
+    dut.expect_exact('Found URL query => query1=http%3A%2F%2Ffoobar&query3=abcd%2B1234%20xyz&query2=Esp%21%40%20%23%2471', timeout=30)
-    logging.info('Test /hello with custom query : {}'.format(query))
+    dut.expect_exact('Found URL query parameter => query1=http%3A%2F%2Ffoobar', timeout=30)
-    if not client.test_custom_uri_query(got_ip, got_port, query):
+    dut.expect_exact('Decoded query parameter => http://foobar', timeout=30)
-        raise RuntimeError
+    dut.expect_exact('Found URL query parameter => query3=abcd%2B1234%20xyz', timeout=30)
-    dut.expect_exact('Found URL query => ' + query, timeout=30)
+    dut.expect_exact('Decoded query parameter => abcd+1234 xyz', timeout=30)
+    dut.expect_exact('Found URL query parameter => query2=Esp%21%40%20%23%2471', timeout=30)
+    dut.expect_exact('Decoded query parameter => Esp!@ #$71', timeout=30)
 
 
 @pytest.mark.esp32

tools/ci/check_copyright_config.yaml

@@ -156,6 +156,16 @@ tinyusb:
     - Unlicense
     - CC0-1.0
 
+protocol_examples_common_component:
+  include:
+    - 'examples/common_components/protocol_examples_common/'
+  allowed_licenses:
+    - Apache-2.0
+    - BSD-2-Clause  # Contains URI encoding/decoding code from nginx
+    - Unlicense
+    - CC0-1.0
+  license_for_new_files: Unlicense OR CC0-1.0
+
 # files matching this section do not perform the check
 # file patterns starting with ! are negated, meaning files matching them won't match the section.
 ignore: