idf: Support a custom toolchain with time_t wide 64-bits

Allows resolving the Y2K38 problem.

Closes: IDF-350

Closes: https://github.com/espressif/esp-idf/issues/584

Kconfig

@@ -69,6 +69,22 @@ mainmenu "Espressif IoT Development Framework Configuration"
 
                 (Note: this option is used with the legacy GNU Make build system only.)
 
+        config SDK_TOOLCHAIN_SUPPORTS_TIME_WIDE_64_BITS
+            bool "Toolchain supports time_t wide 64-bits"
+            default n
+            help
+                Enable this option in case you have a custom toolchain which supports time_t wide 64-bits.
+                This option checks time_t is 64-bits and disables ROM time functions
+                to use the time functions from the toolchain instead.
+                This option allows resolving the Y2K38 problem.
+                See "Setup Linux Toolchain from Scratch" to build
+                a custom toolchain which supports 64-bits time_t.
+
+                Note: ESP-IDF does not currently come with any pre-compiled toolchain
+                that supports 64-bit wide time_t.
+                This will change in a future major release,
+                but currently 64-bit time_t requires a custom built toolchain.
+
     endmenu  # SDK tool configuration
 
     menu "Build type"

components/esp_rom/CMakeLists.txt

@@ -24,6 +24,12 @@ else() # Regular app build
 
         if(NOT CONFIG_SPIRAM_CACHE_WORKAROUND)
             list(APPEND scripts "esp32/ld/esp32.rom.newlib-funcs.ld")
+            if(NOT CONFIG_SDK_TOOLCHAIN_SUPPORTS_TIME_WIDE_64_BITS)
+                # If SDK_TOOLCHAIN_SUPPORTS_TIME_WIDE_64_BITS option is defined
+                # then all time functions from the ROM memory will not be linked.
+                # Instead, those functions can be used from the toolchain by ESP-IDF.
+                target_linker_script(${COMPONENT_LIB} INTERFACE "esp32/ld/esp32.rom.newlib-funcs-time.ld")
+            endif()
         endif()
 
         if(CONFIG_NEWLIB_NANO_FORMAT)

components/esp_rom/esp32/ld/esp32.rom.newlib-funcs-time.ld

@@ -0,0 +1,21 @@
+/* These are the newlib functions present in ESP32 ROM.
+   They should not be used when you need to solve the Y2K38 problem.
+   Because these functions were compiled with 32-bit width for the time_t structure.
+ */
+
+asctime = 0x40059588;
+asctime_r = 0x40000ec8;
+ctime = 0x400595b0;
+ctime_r = 0x400595c4;
+__gettzinfo = 0x40001fcc;
+__get_current_time_locale = 0x40001834;
+gmtime = 0x40059848;
+gmtime_r = 0x40059868;
+localtime = 0x400595dc;
+localtime_r = 0x400595fc;
+mktime = 0x4005a5e8;
+strftime = 0x40059ab4;
+time = 0x40001844;
+__time_load_locale = 0x4000183c;
+tzset = 0x40001a1c;
+_tzset_r = 0x40001a28;

components/esp_rom/esp32/ld/esp32.rom.newlib-funcs.ld

@@ -8,12 +8,12 @@
    weak symbols, newlib related functions are exported using assignment,
    which declares strong symbols. This is done so that ROM functions are always
    used instead of the ones provided by libc.a.
+
+   Time functions were moved to the esp32.rom.newlib-funcs-time.ld file.
  */
 
 abs = 0x40056340;
 __ascii_wctomb = 0x40058ef0;
-asctime = 0x40059588;
-asctime_r = 0x40000ec8;
 atoi = 0x400566c4;
 _atoi_r = 0x400566d4;
 atol = 0x400566ec;
@@ -22,8 +22,6 @@ bzero = 0x4000c1f4;
 _cleanup = 0x40001df8;
 _cleanup_r = 0x40001d48;
 creat = 0x40000e8c;
-ctime = 0x400595b0;
-ctime_r = 0x400595c4;
 div = 0x40056348;
 __dummy_lock = 0x4000c728;
 __dummy_lock_try = 0x4000c730;
@@ -41,11 +39,7 @@ fputwc = 0x40058ea8;
 _fputwc_r = 0x40058e4c;
 _fwalk = 0x4000c738;
 _fwalk_reent = 0x4000c770;
-__get_current_time_locale = 0x40001834;
 _getenv_r = 0x40001fbc;
-__gettzinfo = 0x40001fcc;
-gmtime = 0x40059848;
-gmtime_r = 0x40059868;
 isalnum = 0x40000f04;
 isalpha = 0x40000f18;
 isascii = 0x4000c20c;
@@ -63,8 +57,6 @@ __itoa = 0x40056678;
 itoa = 0x400566b4;
 labs = 0x40056370;
 ldiv = 0x40056378;
-localtime = 0x400595dc;
-localtime_r = 0x400595fc;
 longjmp = 0x400562cc;
 memccpy = 0x4000c220;
 memchr = 0x4000c244;
@@ -73,7 +65,6 @@ memcpy = 0x4000c2c8;
 memmove = 0x4000c3c0;
 memrchr = 0x4000c400;
 memset = 0x4000c44c;
-mktime = 0x4005a5e8;
 qsort = 0x40056424;
 rand = 0x40001058;
 rand_r = 0x400010d4;
@@ -105,7 +96,6 @@ strcpy = 0x400013ac;
 strcspn = 0x4000c558;
 strdup = 0x4000143c;
 _strdup_r = 0x40001450;
-strftime = 0x40059ab4;
 strlcat = 0x40001470;
 strlcpy = 0x4000c584;
 strlen = 0x400014c0;
@@ -133,15 +123,11 @@ __swbuf = 0x40058cb4;
 __swbuf_r = 0x40058bec;
 __swrite = 0x40001150;
 __swsetup_r = 0x40058cc8;
-time = 0x40001844;
-__time_load_locale = 0x4000183c;
 toascii = 0x4000c720;
 tolower = 0x40001868;
 toupper = 0x40001884;
 __tzcalc_limits = 0x400018a0;
 __tz_lock = 0x40001a04;
-tzset = 0x40001a1c;
-_tzset_r = 0x40001a28;
 __tz_unlock = 0x40001a10;
 ungetc = 0x400590f4;
 _ungetc_r = 0x40058fa0;

components/newlib/select.c

@@ -44,7 +44,7 @@ int select(int nfds, fd_set *readfds, fd_set *writefds, fd_set *errorfds, struct
 #ifdef CONFIG_LWIP_USE_ONLY_LWIP_SELECT
     ESP_LOGD(TAG, "lwip_select starts with nfds = %d", nfds);
     if (timeout) {
-        ESP_LOGD(TAG, "timeout is %lds + %ldus", timeout->tv_sec, timeout->tv_usec);
+        ESP_LOGD(TAG, "timeout is %lds + %ldus", (long)timeout->tv_sec, timeout->tv_usec);
     }
     log_fd_set("readfds", readfds);
     log_fd_set("writefds", writefds);

components/newlib/test/test_time.c

@@ -11,6 +11,7 @@
 #include "soc/rtc.h"
 #include "esp_system.h"
 #include "test_utils.h"
+#include "esp_log.h"
 
 #if portNUM_PROCESSORS == 2
 
@@ -420,3 +421,92 @@ TEST_CASE("test posix_timers clock_... functions", "[newlib]")
 {
     test_posix_timers_clock();
 }
+
+#ifdef CONFIG_SDK_TOOLCHAIN_SUPPORTS_TIME_WIDE_64_BITS
+#include <string.h>
+
+static struct timeval get_time(const char *desc, char *buffer)
+{
+    struct timeval timestamp;
+    gettimeofday(&timestamp, NULL);
+    struct tm* tm_info = localtime(&timestamp.tv_sec);
+    strftime(buffer, 32, "%c", tm_info);
+    ESP_LOGI("TAG", "%s: %016llX (%s)", desc, timestamp.tv_sec, buffer);
+    return timestamp;
+}
+
+TEST_CASE("test time_t wide 64 bits", "[newlib]")
+{
+    static char buffer[32];
+    ESP_LOGI("TAG", "sizeof(time_t): %d (%d-bit)", sizeof(time_t), sizeof(time_t)*8);
+    TEST_ASSERT_EQUAL(8, sizeof(time_t));
+
+    struct tm tm = {4, 14, 3, 19, 0, 138, 0, 0, 0};
+    struct timeval timestamp = { mktime(&tm), 0 };
+    ESP_LOGI("TAG", "timestamp: %016llX", timestamp.tv_sec);
+    settimeofday(&timestamp, NULL);
+    get_time("Set time", buffer);
+
+    while (timestamp.tv_sec < 0x80000003LL) {
+        vTaskDelay(1000 / portTICK_PERIOD_MS);
+        timestamp = get_time("Time now", buffer);
+    }
+    TEST_ASSERT_EQUAL_MEMORY("Tue Jan 19 03:14:11 2038", buffer, strlen(buffer));
+}
+
+TEST_CASE("test time functions wide 64 bits", "[newlib]")
+{
+    static char origin_buffer[32];
+    char strftime_buf[64];
+
+    int year = 2018;
+    struct tm tm = {0, 14, 3, 19, 0, year - 1900, 0, 0, 0};
+    time_t t = mktime(&tm);
+    while (year < 2119) {
+        struct timeval timestamp = { t, 0 };
+        ESP_LOGI("TAG", "year: %d", year);
+        settimeofday(&timestamp, NULL);
+        get_time("Time now", origin_buffer);
+        vTaskDelay(10 / portTICK_PERIOD_MS);
+        t += 86400 * 366;
+        struct tm timeinfo = { 0 };
+        time_t now;
+        time(&now);
+        localtime_r(&now, &timeinfo);
+
+        time_t t = mktime(&timeinfo);
+        ESP_LOGI("TAG", "Test mktime(). Time: %016llX", t);
+        TEST_ASSERT_EQUAL(timestamp.tv_sec, t);
+        // mktime() has error in newlib-3.0.0. It fixed in newlib-3.0.0.20180720
+        TEST_ASSERT_EQUAL((timestamp.tv_sec >> 32), (t >> 32));
+
+        strftime(strftime_buf, sizeof(strftime_buf), "%c", &timeinfo);
+        ESP_LOGI("TAG", "Test time() and localtime_r(). Time: %s", strftime_buf);
+        TEST_ASSERT_EQUAL(timeinfo.tm_year, year - 1900);
+        TEST_ASSERT_EQUAL_MEMORY(origin_buffer, strftime_buf, strlen(origin_buffer));
+
+        struct tm *tm2 = localtime(&now);
+        strftime(strftime_buf, sizeof(strftime_buf), "%c", tm2);
+        ESP_LOGI("TAG", "Test localtime(). Time: %s", strftime_buf);
+        TEST_ASSERT_EQUAL(tm2->tm_year, year - 1900);
+        TEST_ASSERT_EQUAL_MEMORY(origin_buffer, strftime_buf, strlen(origin_buffer));
+
+        struct tm *gm = gmtime(&now);
+        strftime(strftime_buf, sizeof(strftime_buf), "%c", gm);
+        ESP_LOGI("TAG", "Test gmtime(). Time: %s", strftime_buf);
+        TEST_ASSERT_EQUAL_MEMORY(origin_buffer, strftime_buf, strlen(origin_buffer));
+
+        const char* time_str1 = ctime(&now);
+        ESP_LOGI("TAG", "Test ctime(). Time: %s", time_str1);
+        TEST_ASSERT_EQUAL_MEMORY(origin_buffer, time_str1, strlen(origin_buffer));
+
+        const char* time_str2 = asctime(&timeinfo);
+        ESP_LOGI("TAG", "Test asctime(). Time: %s", time_str2);
+        TEST_ASSERT_EQUAL_MEMORY(origin_buffer, time_str2, strlen(origin_buffer));
+
+        printf("\n");
+        ++year;
+    }
+}
+
+#endif // CONFIG_SDK_TOOLCHAIN_SUPPORTS_TIME_WIDE_64_BITS

components/newlib/time.c

@@ -43,6 +43,12 @@
 #include "esp32s2beta/rom/ets_sys.h"
 #endif
 
+#ifdef CONFIG_SDK_TOOLCHAIN_SUPPORTS_TIME_WIDE_64_BITS
+_Static_assert(sizeof(time_t) == 8, "The toolchain does not support time_t wide 64-bits");
+#else
+_Static_assert(sizeof(time_t) == 4, "The toolchain supports time_t wide 64-bits. Please enable CONFIG_SDK_TOOLCHAIN_SUPPORTS_TIME_WIDE_64_BITS.");
+#endif
+
 #if defined( CONFIG_ESP32_TIME_SYSCALL_USE_RTC ) || defined( CONFIG_ESP32_TIME_SYSCALL_USE_RTC_FRC1 ) || defined( CONFIG_ESP32S2_TIME_SYSCALL_USE_RTC ) || defined( CONFIG_ESP32S2_TIME_SYSCALL_USE_RTC_FRC1 )
 #define WITH_RTC 1
 #endif

components/spiffs/Kconfig

@@ -123,6 +123,19 @@ menu "SPIFFS Configuration"
             stat/fstat functions.
             Modification time is updated when the file is opened.
 
+    config SPIFFS_MTIME_WIDE_64_BITS
+        bool "The time field occupies 64 bits in the image instead of 32 bits"
+        default n
+        depends on SPIFFS_META_LENGTH >= 8
+        help
+            If this option is not set, the time field is 32 bits (up to 2106 year),
+            otherwise it is 64 bits and make sure it matches SPIFFS_META_LENGTH.
+            If the chip already has the spiffs image with the time field = 32 bits
+            then this option cannot be applied in this case.
+            Erase it first before using this option.
+            To resolve the Y2K38 problem for the spiffs, use a toolchain with support
+            time_t 64 bits (see SDK_TOOLCHAIN_SUPPORTS_TIME_WIDE_64_BITS).
+
     menu "Debug Configuration"
 
         config SPIFFS_DBG

components/spiffs/esp_spiffs.c

@@ -35,8 +35,13 @@
 static const char* TAG = "SPIFFS";
 
 #ifdef CONFIG_SPIFFS_USE_MTIME
-_Static_assert(CONFIG_SPIFFS_META_LENGTH >= sizeof(time_t),
+#ifdef CONFIG_SPIFFS_MTIME_WIDE_64_BITS
-        "SPIFFS_META_LENGTH size should be >= sizeof(time_t)");
+typedef time_t spiffs_time_t;
+#else
+typedef unsigned long spiffs_time_t;
+#endif
+_Static_assert(CONFIG_SPIFFS_META_LENGTH >= sizeof(spiffs_time_t),
+        "SPIFFS_META_LENGTH size should be >= sizeof(spiffs_time_t)");
 #endif //CONFIG_SPIFFS_USE_MTIME
 
 /**
@@ -726,7 +731,7 @@ static int vfs_spiffs_link(void* ctx, const char* n1, const char* n2)
 static void vfs_spiffs_update_mtime(spiffs *fs, spiffs_file fd)
 {
 #ifdef CONFIG_SPIFFS_USE_MTIME
-    time_t t = time(NULL);
+    spiffs_time_t t = (spiffs_time_t)time(NULL);
     spiffs_stat s;
     int ret = SPIFFS_OK;
     if (CONFIG_SPIFFS_META_LENGTH > sizeof(t)) {
@@ -744,15 +749,15 @@ static void vfs_spiffs_update_mtime(spiffs *fs, spiffs_file fd)
 
 static time_t vfs_spiffs_get_mtime(const spiffs_stat* s)
 {
-    time_t t = 0;
+    spiffs_time_t t = 0;
 #ifdef CONFIG_SPIFFS_USE_MTIME
     memcpy(&t, s->meta, sizeof(t));
 #endif
-    return t;
+    return (time_t)t;
 }
 
 #ifdef CONFIG_SPIFFS_USE_MTIME
-static int vfs_spiffs_update_mtime_value(spiffs *fs, const char *path, time_t t)
+static int vfs_spiffs_update_mtime_value(spiffs *fs, const char *path, spiffs_time_t t)
 {
     int ret = SPIFFS_OK;
     spiffs_stat s;
@@ -776,13 +781,13 @@ static int vfs_spiffs_utime(void *ctx, const char *path, const struct utimbuf *t
     assert(path);
 
     esp_spiffs_t *efs = (esp_spiffs_t *) ctx;
-    time_t t;
+    spiffs_time_t t;
 
     if (times) {
-        t = times->modtime;
+        t = (spiffs_time_t)times->modtime;
     } else {
         // use current time
-        t = time(NULL);
+        t = (spiffs_time_t)time(NULL);
     }
 
     int ret = vfs_spiffs_update_mtime_value(efs->fs, path, t);

components/vfs/vfs.c

@@ -855,7 +855,7 @@ int esp_vfs_select(int nfds, fd_set *readfds, fd_set *writefds, fd_set *errorfds
 
     ESP_LOGD(TAG, "esp_vfs_select starts with nfds = %d", nfds);
     if (timeout) {
-        ESP_LOGD(TAG, "timeout is %lds + %ldus", timeout->tv_sec, timeout->tv_usec);
+        ESP_LOGD(TAG, "timeout is %lds + %ldus", (long)timeout->tv_sec, timeout->tv_usec);
     }
     esp_vfs_log_fd_set("readfds", readfds);
     esp_vfs_log_fd_set("writefds", writefds);

components/wpa_supplicant/port/include/os.h

@@ -21,7 +21,7 @@
 #include "esp_err.h"
 // #include "esp32/rom/ets_sys.h"
 
-typedef long os_time_t;
+typedef time_t os_time_t;
 
 /**
  * os_sleep - Sleep (sec, usec)
@@ -32,7 +32,7 @@ void os_sleep(os_time_t sec, os_time_t usec);
 
 struct os_time {
 	os_time_t sec;
-	os_time_t usec;
+	suseconds_t usec;
 };
 
 /**

components/wpa_supplicant/port/os_xtensa.c

@@ -31,7 +31,11 @@
 
 int os_get_time(struct os_time *t)
 {
-    return gettimeofday((struct timeval*) t, NULL);
+    struct timeval tv;
+    int ret = gettimeofday(&tv, NULL);
+    t->sec = (os_time_t) tv.tv_sec;
+    t->usec = tv.tv_usec;
+    return ret;
 }
 
 unsigned long os_random(void)

components/wpa_supplicant/src/tls/x509v3.c

@@ -1785,7 +1785,7 @@ int x509_certificate_chain_validate(struct x509_certificate *trusted,
 		     (unsigned long) cert->not_after)) {
 			wpa_printf(MSG_INFO, "X509: Certificate not valid "
 				   "(now=%lu not_before=%lu not_after=%lu)",
-				   now.sec, cert->not_before, cert->not_after);
+				   (unsigned long)now.sec, (unsigned long)cert->not_before, (unsigned long)cert->not_after);
 			*reason = X509_VALIDATE_CERTIFICATE_EXPIRED;
 			return -1;
 		}

docs/en/get-started/linux-setup-scratch.rst

@@ -6,6 +6,8 @@ Setup Linux Toolchain from Scratch
 
 The following instructions are alternative to downloading binary toolchain from Espressif website. To quickly setup the binary toolchain, instead of compiling it yourself, backup and proceed to section :doc:`linux-setup`.
 
+.. note:: The reason you might need to build your own toolchain is to solve the Y2K38 problem (time_t expand to 64 bits instead of 32 bits).
+
 Install Prerequisites
 =====================
 
@@ -60,6 +62,8 @@ Download ``crosstool-NG`` and build it:
 
 .. include:: /_build/inc/scratch-build-code.inc
 
+.. note:: To create a toolchain with support for 64-bit time_t, you need to remove the ``--enable-newlib-long-time_t`` option from the ``crosstool-NG/samples/xtensa-esp32-elf/crosstool.config`` file in 33 and 43 lines.
+
 Build the toolchain::
 
     ./ct-ng xtensa-esp32-elf

examples/protocols/sntp/main/sntp_example_main.c

@@ -104,7 +104,7 @@ void app_main(void)
         while (sntp_get_sync_status() == SNTP_SYNC_STATUS_IN_PROGRESS) {
             adjtime(NULL, &outdelta);
             ESP_LOGI(TAG, "Waiting for adjusting time ... outdelta = %li sec: %li ms: %li us",
-                        outdelta.tv_sec,
+                        (long)outdelta.tv_sec,
                         outdelta.tv_usec/1000,
                         outdelta.tv_usec%1000);
             vTaskDelay(2000 / portTICK_PERIOD_MS);