diff --git a/components/esp_hw_support/port/esp32c3/rtc_sleep.c b/components/esp_hw_support/port/esp32c3/rtc_sleep.c
index 328b8db76e..831c9bb379 100644
--- a/components/esp_hw_support/port/esp32c3/rtc_sleep.c
+++ b/components/esp_hw_support/port/esp32c3/rtc_sleep.c
@@ -222,6 +222,8 @@ void rtc_sleep_init(rtc_sleep_config_t cfg)
     }
     /* mem force pu */
     SET_PERI_REG_MASK(RTC_CNTL_DIG_PWC_REG, RTC_CNTL_LSLP_MEM_FORCE_PU);
+    SET_PERI_REG_MASK(RTC_CNTL_DIG_PWC_REG, RTC_CNTL_FASTMEM_FORCE_LPU);
+
     REG_SET_FIELD(RTC_CNTL_REG, RTC_CNTL_REGULATOR_FORCE_PU, cfg.rtc_regulator_fpu);
     if (!cfg.int_8m_pd_en) {
         SET_PERI_REG_MASK(RTC_CNTL_CLK_CONF_REG, RTC_CNTL_CK8M_FORCE_PU);
diff --git a/components/esp_hw_support/port/esp32s3/rtc_sleep.c b/components/esp_hw_support/port/esp32s3/rtc_sleep.c
index 4033f2f781..8f7d2e0fcb 100644
--- a/components/esp_hw_support/port/esp32s3/rtc_sleep.c
+++ b/components/esp_hw_support/port/esp32s3/rtc_sleep.c
@@ -233,6 +233,8 @@ void rtc_sleep_init(rtc_sleep_config_t cfg)
     }
     /* mem force pu */
     SET_PERI_REG_MASK(RTC_CNTL_DIG_PWC_REG, RTC_CNTL_LSLP_MEM_FORCE_PU);
+    SET_PERI_REG_MASK(RTC_CNTL_PWC_REG, RTC_CNTL_FASTMEM_FORCE_LPU);
+    SET_PERI_REG_MASK(RTC_CNTL_PWC_REG, RTC_CNTL_SLOWMEM_FORCE_LPU);
 
     REG_SET_FIELD(RTC_CNTL_REG, RTC_CNTL_REGULATOR_FORCE_PU, cfg.rtc_regulator_fpu);
     if (!cfg.int_8m_pd_en) {