Merge branch 'feature/unit_test_tag_leaks' into 'master'

Add support a [leaks] tag for UT

Closes IDF-143

See merge request idf/esp-idf!4333

.gitlab-ci.yml

@@ -1528,6 +1528,20 @@ UT_004_17:
     - UT_T1_1
     - psram
 
+UT_004_18:
+  <<: *unit_test_template
+  tags:
+    - ESP32_IDF
+    - UT_T1_1
+    - psram
+
+UT_004_19:
+  <<: *unit_test_template
+  tags:
+    - ESP32_IDF
+    - UT_T1_1
+    - psram
+
 UT_005_01:
   <<: *unit_test_template
   tags:

components/driver/test/test_uart.c

@@ -136,7 +136,7 @@ TEST_CASE("test uart get baud-rate","[uart]")
     ESP_LOGI(UART_TAG, "get baud-rate test passed  ....\n");
 }
 
-TEST_CASE("test uart tx data with break","[uart]")
+TEST_CASE("test uart tx data with break","[uart][leaks=2192]")
 {
     const int buf_len = 200;
     const int send_len = 128;

components/esp_event/test/test_event.c

@@ -868,7 +868,7 @@ TEST_CASE("performance test - dedicated task", "[event]")
     performance_test(true);
 }
 
-TEST_CASE("performance test - no dedicated task", "[event]")
+TEST_CASE("performance test - no dedicated task", "[event][leaks=2736]")
 {
     performance_test(false);
 }

components/esp_wifi/test/test_phy_rtc.c

@@ -77,7 +77,7 @@ static IRAM_ATTR void test_phy_rtc_cache_task(void *arg)
     vTaskDelete(NULL);
 }
 
-TEST_CASE("Test PHY/RTC functions called when cache is disabled", "[phy_rtc][cache_disabled]")
+TEST_CASE("Test PHY/RTC functions called when cache is disabled", "[phy_rtc][cache_disabled][leaks=1216]")
 {
     semphr_done = xSemaphoreCreateCounting(1, 0);
 

components/fatfs/test/test_fatfs_rawflash.c

@@ -69,7 +69,7 @@ TEST_CASE("(raw) can read file", "[fatfs]")
     test_teardown();
 }
 
-TEST_CASE("(raw) can open maximum number of files", "[fatfs]")
+TEST_CASE("(raw) can open maximum number of files", "[fatfs][leaks=2028]")
 {
     size_t max_files = FOPEN_MAX - 3; /* account for stdin, stdout, stderr */
     test_setup(max_files);

components/fatfs/test/test_fatfs_sdmmc.c

@@ -158,7 +158,7 @@ TEST_CASE("(SD) opendir, readdir, rewinddir, seekdir work as expected", "[fatfs]
     test_teardown();
 }
 
-TEST_CASE("(SD) multiple tasks can use same volume", "[fatfs][test_env=UT_T1_SDMODE]")
+TEST_CASE("(SD) multiple tasks can use same volume", "[fatfs][test_env=UT_T1_SDMODE][leaks=1264]")
 {
     test_setup();
     test_fatfs_concurrent("/sdcard/f");
@@ -167,7 +167,7 @@ TEST_CASE("(SD) multiple tasks can use same volume", "[fatfs][test_env=UT_T1_SDM
 
 static void speed_test(void* buf, size_t buf_size, size_t file_size, bool write);
 
-TEST_CASE("(SD) write/read speed test", "[fatfs][sd][test_env=UT_T1_SDMODE][timeout=60]")
+TEST_CASE("(SD) write/read speed test", "[fatfs][sd][test_env=UT_T1_SDMODE][timeout=60][leaks=1080]")
 {
     size_t heap_size;
     HEAP_SIZE_CAPTURE(heap_size);

components/fatfs/test/test_fatfs_spiflash.c

@@ -70,7 +70,7 @@ TEST_CASE("(WL) can read file", "[fatfs][wear_levelling]")
     test_teardown();
 }
 
-TEST_CASE("(WL) can open maximum number of files", "[fatfs][wear_levelling]")
+TEST_CASE("(WL) can open maximum number of files", "[fatfs][wear_levelling][leaks=2460]")
 {
     size_t max_files = FOPEN_MAX - 3; /* account for stdin, stdout, stderr */
     esp_vfs_fat_sdmmc_mount_config_t mount_config = {
@@ -152,14 +152,14 @@ TEST_CASE("(WL) opendir, readdir, rewinddir, seekdir work as expected", "[fatfs]
     test_teardown();
 }
 
-TEST_CASE("(WL) multiple tasks can use same volume", "[fatfs][wear_levelling]")
+TEST_CASE("(WL) multiple tasks can use same volume", "[fatfs][wear_levelling][leaks=1340]")
 {
     test_setup();
     test_fatfs_concurrent("/spiflash/f");
     test_teardown();
 }
 
-TEST_CASE("(WL) write/read speed test", "[fatfs][wear_levelling][timeout=60]")
+TEST_CASE("(WL) write/read speed test", "[fatfs][wear_levelling][timeout=60][leaks=1156]")
 {
     /* Erase partition before running the test to get consistent results */
     const esp_partition_t* part = get_test_data_partition();

components/heap/test/test_leak.c

@@ -0,0 +1,60 @@
+/*
+ Tests for a leak tag
+*/
+
+#include <stdio.h>
+#include "unity.h"
+#include "esp_heap_caps_init.h"
+#include "esp_system.h"
+#include <stdlib.h>
+
+
+static char* check_calloc(int size)
+{
+    char *arr = calloc(size, sizeof(char));
+    TEST_ASSERT_NOT_NULL(arr);
+    return arr;
+}
+
+TEST_CASE("Check for leaks (no leak)", "[heap]")
+{
+    char *arr = check_calloc(7000);
+    free(arr);
+}
+
+TEST_CASE("Check for leaks (leak)", "[heap][ignore]")
+{
+    check_calloc(7000);
+}
+
+TEST_CASE("Not check for leaks", "[heap][leaks]")
+{
+    check_calloc(7000);
+}
+
+TEST_CASE("Set a leak level = 7016", "[heap][leaks=7016]")
+{
+    check_calloc(7000);
+}
+
+static void test_fn(void)
+{
+    check_calloc(7000);
+}
+
+TEST_CASE_MULTIPLE_STAGES("Not check for leaks in MULTIPLE_STAGES mode", "[heap][leaks]", test_fn, test_fn, test_fn);
+
+TEST_CASE_MULTIPLE_STAGES("Check for leaks in MULTIPLE_STAGES mode (leak)", "[heap][ignore]", test_fn, test_fn, test_fn);
+
+static void test_fn2(void)
+{
+    check_calloc(7000);
+    esp_restart();
+}
+
+static void test_fn3(void)
+{
+    check_calloc(7000);
+}
+
+TEST_CASE_MULTIPLE_STAGES("Check for leaks in MULTIPLE_STAGES mode (manual reset)", "[heap][leaks][reset=SW_CPU_RESET, SW_CPU_RESET]", test_fn2, test_fn2, test_fn3);

components/spiffs/test/test_spiffs.c

@@ -531,7 +531,7 @@ TEST_CASE("can read file", "[spiffs]")
     test_teardown();
 }
 
-TEST_CASE("can open maximum number of files", "[spiffs]")
+TEST_CASE("can open maximum number of files", "[spiffs][leaks=2244]")
 {
     size_t max_files = FOPEN_MAX - 3; /* account for stdin, stdout, stderr */
     esp_vfs_spiffs_conf_t conf = {
@@ -602,7 +602,7 @@ TEST_CASE("readdir with large number of files", "[spiffs][timeout=30]")
     test_teardown();
 }
 
-TEST_CASE("multiple tasks can use same volume", "[spiffs]")
+TEST_CASE("multiple tasks can use same volume", "[spiffs][leaks=1128]")
 {
     test_setup();
     test_spiffs_concurrent("/spiffs/f");

components/unity/unity_runner.c

@@ -18,6 +18,7 @@
 #include <ctype.h>
 #include <stdio.h>
 #include "unity.h"
+#include "esp_system.h"
 
 /* similar to UNITY_PRINT_EOL */
 #define UNITY_PRINT_TAB() UNITY_OUTPUT_CHAR('\t')
@@ -61,7 +62,29 @@ static void print_multiple_function_test_menu(const test_desc_t *test_ms)
     }
 }
 
-static void multiple_function_option(const test_desc_t *test_ms)
+/*
+ * This function looks like UnityDefaultTestRun function only without UNITY_CLR_DETAILS.
+ * void UnityDefaultTestRun(UnityTestFunction Func, const char* FuncName, const int FuncLineNum)
+ * was moved from `components/unity/unity/src/unity.c` to here.
+*/
+static void unity_default_test_run(UnityTestFunction Func, const char* FuncName, const int FuncLineNum)
+{
+    Unity.CurrentTestName = FuncName;
+    Unity.CurrentTestLineNumber = (UNITY_LINE_TYPE)FuncLineNum;
+    Unity.NumberOfTests++;
+    if (TEST_PROTECT())
+    {
+        setUp();
+        Func();
+    }
+    if (TEST_PROTECT())
+    {
+        tearDown();
+    }
+    UnityConcludeTest();
+}
+
+static int multiple_function_option(const test_desc_t *test_ms)
 {
     int selection;
     char cmdline[256] = {0};
@@ -76,12 +99,13 @@ static void multiple_function_option(const test_desc_t *test_ms)
     }
     selection = atoi((const char *) cmdline) - 1;
     if (selection >= 0 && selection < test_ms->test_fn_count) {
-        UnityDefaultTestRun(test_ms->fn[selection], test_ms->name, test_ms->line);
+        unity_default_test_run(test_ms->fn[selection], test_ms->name, test_ms->line);
     } else {
         UnityPrint("Invalid selection, your should input number 1-");
         UnityPrintNumber(test_ms->test_fn_count);
         UNITY_PRINT_EOL();
     }
+    return selection;
 }
 
 static void unity_run_single_test(const test_desc_t *test)
@@ -95,10 +119,27 @@ static void unity_run_single_test(const test_desc_t *test)
 
     Unity.TestFile = test->file;
     Unity.CurrentDetail1 = test->desc;
+    bool reset_after_test = strstr(Unity.CurrentDetail1, "[leaks") != NULL;
+    bool multi_device = strstr(Unity.CurrentDetail1, "[multi_device]") != NULL;
     if (test->test_fn_count == 1) {
-        UnityDefaultTestRun(test->fn[0], test->name, test->line);
+        unity_default_test_run(test->fn[0], test->name, test->line);
     } else {
-        multiple_function_option(test);
+        int selection = multiple_function_option(test);
+        if (reset_after_test && multi_device == false) {
+            if (selection != (test->test_fn_count - 1)) {
+                // to do a reset for all stages except the last stage.
+                esp_restart();
+            }
+        }
+    }
+
+    if (reset_after_test) {
+        // print a result of test before to do reset for the last stage.
+        UNITY_END();
+        UnityPrint("Enter next test, or 'enter' to see menu");
+        UNITY_PRINT_EOL();
+        UNITY_OUTPUT_FLUSH();
+        esp_restart();
     }
 }
 

tools/unit-test-app/README.md

@@ -56,6 +56,15 @@ When we add new test case, it will construct a structure to save case data durin
 2. the rest tags should be [type=value]. Tags could have default value and omitted value. For example, reset tag default value is "POWERON_RESET", omitted value is "" (do not reset) :
     * "[reset]" equal to [reset=POWERON_RESET]
     * if reset tag doesn't exist, then it equals to [reset=""]
+3. the `[leaks]` tag is used to disable the leak checking. A specific maximum memory leakage can be set as follows: `[leaks=500]`. This allows no more than 500 bytes of heap to be leaked. Also there is a special function to set the critical level of leakage not through a tag, just directly in the test code ``test_utils_set_critical_leak_level()``. 
+
+The priority of using leakage level is as follows:
+
+1. Setting by tag `[leaks=500]`.
+2. Setting by ``test_utils_set_critical_leak_level()`` function.
+3. Setting by default leakage in Kconfig ``CONFIG_UNITY_CRITICAL_LEAK_LEVEL_GENERAL``.
+
+Tests marked as `[leaks]` or `[leaks=xxx]` reset the device after completion (or after each stage in multistage tests).
 
 `TagDefinition.yml` defines how we should parse the description. In `TagDefinition.yml`, we declare the tags we are interested in, their default value and omitted value. Parser will parse the properities of test cases according to this file, and add them as test case attributes.
 

tools/unit-test-app/components/test_utils/Kconfig

@@ -12,4 +12,16 @@ menu "IDF unit test"
         int "Stack size of Unity test task, in bytes"
         default 8192
 
+    config UNITY_WARN_LEAK_LEVEL_GENERAL
+        int "Leak warning level"
+        default 255
+
+    config UNITY_CRITICAL_LEAK_LEVEL_GENERAL
+        int "Critical leak"
+        default 1024
+
+    config UNITY_CRITICAL_LEAK_LEVEL_LWIP
+        int "Critical leak for UT which use LWIP component"
+        default 4095
+
 endmenu

tools/unit-test-app/components/test_utils/include/test_utils.h

@@ -196,3 +196,43 @@ static inline void unity_send_signal(const char* signal_name)
  * @param[out] mac_addr store converted MAC address
  */
 bool unity_util_convert_mac_from_string(const char* mac_str, uint8_t *mac_addr);
+
+/**
+ * @brief Leak for components
+ */
+typedef enum {
+    COMP_LEAK_GENERAL = 0,  /**< Leak by default */
+    COMP_LEAK_LWIP,         /**< Leak for LWIP */
+    COMP_LEAK_NVS,          /**< Leak for NVS */
+    COMP_LEAK_ALL,          /**< Use for getting the summary leak level */
+} esp_comp_leak_t;
+
+/**
+ * @brief Type of leak
+ */
+typedef enum {
+    TYPE_LEAK_WARNING = 0,  /**< Warning level of leak */
+    TYPE_LEAK_CRITICAL,     /**< Critical level of leak */
+    TYPE_LEAK_MAX,          /**< Max number of leak levels */
+} esp_type_leak_t;
+
+/**
+ * @brief Set a leak level for the required type and component.
+ *
+ * @param[in] leak_level Level of leak
+ * @param[in] type       Type of leak
+ * @param[in] component  Name of component
+ *
+ * return ESP_OK: Successful.
+ *        ESP_ERR_INVALID_ARG: Invalid argument.
+ */
+esp_err_t test_utils_set_leak_level(size_t leak_level, esp_type_leak_t type, esp_comp_leak_t component);
+
+/**
+ * @brief Get a leak level for the required type and component.
+ *
+ * @param[in] type       Type of leak.
+ * @param[in] component  Name of component. If COMP_LEAK_ALL, then the level will be summarized for all components.
+ * return Leak level
+ */
+size_t test_utils_get_leak_level(esp_type_leak_t type, esp_comp_leak_t component);

tools/unit-test-app/components/test_utils/test_runner.c

@@ -13,6 +13,7 @@
 // limitations under the License.
 
 #include <stdio.h>
+#include "string.h"
 #include "esp_heap_caps.h"
 #include "freertos/FreeRTOS.h"
 #include "freertos/task.h"
@@ -26,14 +27,8 @@
 static size_t before_free_8bit;
 static size_t before_free_32bit;
 
-/* Each unit test is allowed to "leak" this many bytes.
+static size_t warn_leak_threshold;
-
+static size_t critical_leak_threshold;
-   TODO: Make this value editable by the test.
-
-   Will always need to be some value here, as fragmentation can reduce free space even when no leak is occurring.
-*/
-const size_t WARN_LEAK_THRESHOLD = 256;
-const size_t CRITICAL_LEAK_THRESHOLD = 4096;
 
 static void unity_task(void *pvParameters)
 {
@@ -77,6 +72,8 @@ void setUp(void)
     get_test_data_partition();  /* allocate persistent partition table structures */
 
     unity_reset_leak_checks();
+    test_utils_set_leak_level(CONFIG_UNITY_CRITICAL_LEAK_LEVEL_GENERAL, TYPE_LEAK_CRITICAL, COMP_LEAK_GENERAL);
+    test_utils_set_leak_level(CONFIG_UNITY_WARN_LEAK_LEVEL_GENERAL, TYPE_LEAK_WARNING, COMP_LEAK_GENERAL);
 }
 
 static void check_leak(size_t before_free, size_t after_free, const char *type)
@@ -85,16 +82,37 @@ static void check_leak(size_t before_free, size_t after_free, const char *type)
         return;
     }
     size_t leaked = before_free - after_free;
-    if (leaked < WARN_LEAK_THRESHOLD) {
+    if (leaked <= warn_leak_threshold) {
         return;
     }
 
     printf("MALLOC_CAP_%s %s leak: Before %u bytes free, After %u bytes free (delta %u)\n",
            type,
-           leaked < CRITICAL_LEAK_THRESHOLD ? "potential" : "critical",
+           leaked <= critical_leak_threshold ? "potential" : "critical",
            before_free, after_free, leaked);
     fflush(stdout);
-    TEST_ASSERT_MESSAGE(leaked < CRITICAL_LEAK_THRESHOLD, "The test leaked too much memory");
+    TEST_ASSERT_MESSAGE(leaked <= critical_leak_threshold, "The test leaked too much memory");
+}
+
+static bool leak_check_required()
+{
+    warn_leak_threshold = test_utils_get_leak_level(TYPE_LEAK_WARNING, COMP_LEAK_ALL);
+    critical_leak_threshold = test_utils_get_leak_level(TYPE_LEAK_CRITICAL, COMP_LEAK_ALL);
+    if (Unity.CurrentDetail1 != NULL) {
+        const char *leaks = "[leaks";
+        const int len_leaks = strlen(leaks);
+        const char *sub_leaks = strstr(Unity.CurrentDetail1, leaks);
+        if (sub_leaks != NULL) {
+            if (sub_leaks[len_leaks] == ']') {
+                return true;
+            } else if (sub_leaks[len_leaks] == '=') {
+                critical_leak_threshold = strtol(&sub_leaks[len_leaks + 1], NULL, 10);
+                warn_leak_threshold = critical_leak_threshold;
+                return false;
+            }
+        }
+    }
+    return false;
 }
 
 /* tearDown runs after every test */
@@ -103,6 +121,8 @@ void tearDown(void)
     /* some FreeRTOS stuff is cleaned up by idle task */
     vTaskDelay(5);
 
+    size_t after_free_8bit = heap_caps_get_free_size(MALLOC_CAP_8BIT);
+    size_t after_free_32bit = heap_caps_get_free_size(MALLOC_CAP_32BIT);
     /* We want the teardown to have this file in the printout if TEST_ASSERT fails */
     const char *real_testfile = Unity.TestFile;
     Unity.TestFile = __FILE__;
@@ -115,11 +135,11 @@ void tearDown(void)
     heap_trace_stop();
     heap_trace_dump();
 #endif
-    size_t after_free_8bit = heap_caps_get_free_size(MALLOC_CAP_8BIT);
-    size_t after_free_32bit = heap_caps_get_free_size(MALLOC_CAP_32BIT);
 
-    check_leak(before_free_8bit, after_free_8bit, "8BIT");
+    if (leak_check_required() == false) {
-    check_leak(before_free_32bit, after_free_32bit, "32BIT");
+        check_leak(before_free_8bit, after_free_8bit, "8BIT");
+        check_leak(before_free_32bit, after_free_32bit, "32BIT");
+    }
 
     Unity.TestFile = real_testfile; // go back to the real filename
 }

tools/unit-test-app/components/test_utils/test_utils.c

@@ -55,6 +55,8 @@ void test_case_uses_tcpip()
 
     // Reset the leak checker as LWIP allocates a lot of memory on first run
     unity_reset_leak_checks();
+    test_utils_set_leak_level(0, TYPE_LEAK_CRITICAL, COMP_LEAK_GENERAL);
+    test_utils_set_leak_level(CONFIG_UNITY_CRITICAL_LEAK_LEVEL_LWIP, TYPE_LEAK_CRITICAL, COMP_LEAK_LWIP);
 }
 
 // wait user to send "Enter" key or input parameter
@@ -114,3 +116,30 @@ bool unity_util_convert_mac_from_string(const char* mac_str, uint8_t *mac_addr)
     return true;
 }
 
+static size_t test_unity_leak_level[TYPE_LEAK_MAX][COMP_LEAK_ALL] = { 0 };
+
+esp_err_t test_utils_set_leak_level(size_t leak_level, esp_type_leak_t type_of_leak, esp_comp_leak_t component)
+{
+    if (type_of_leak >= TYPE_LEAK_MAX || component >= COMP_LEAK_ALL) {
+        return ESP_ERR_INVALID_ARG;
+    }
+    test_unity_leak_level[type_of_leak][component] = leak_level;
+    return ESP_OK;
+}
+
+size_t test_utils_get_leak_level(esp_type_leak_t type_of_leak, esp_comp_leak_t component)
+{
+    size_t leak_level = 0;
+    if (type_of_leak >= TYPE_LEAK_MAX || component > COMP_LEAK_ALL) {
+        leak_level = 0;
+    } else {
+        if (component == COMP_LEAK_ALL) {
+            for (int comp = 0; comp < COMP_LEAK_ALL; ++comp) {
+                leak_level += test_unity_leak_level[type_of_leak][comp];
+            }
+        } else {
+            leak_level = test_unity_leak_level[type_of_leak][component];
+        }
+    }
+    return leak_level;
+}