diff --git a/components/esp_hw_support/sleep_modes.c b/components/esp_hw_support/sleep_modes.c
index dbec0b3ce8..dd410bfa4f 100644
--- a/components/esp_hw_support/sleep_modes.c
+++ b/components/esp_hw_support/sleep_modes.c
@@ -551,9 +551,6 @@ static uint32_t IRAM_ATTR esp_sleep_start(uint32_t pd_flags, esp_sleep_mode_t mo
         pd_flags &= ~RTC_SLEEP_PD_INT_8M;
     }
 
-    //turn down MSPI speed
-    mspi_timing_change_speed_mode_cache_safe(true);
-
     // Sleep UART prepare
     if (deep_sleep) {
         flush_uarts();
@@ -561,6 +558,9 @@ static uint32_t IRAM_ATTR esp_sleep_start(uint32_t pd_flags, esp_sleep_mode_t mo
         should_skip_sleep = light_sleep_uart_prepare(pd_flags, sleep_duration);
     }
 
+    // Will switch to XTAL turn down MSPI speed
+    mspi_timing_change_speed_mode_cache_safe(true);
+
     // Save current frequency and switch to XTAL
     rtc_cpu_freq_config_t cpu_freq_config;
     rtc_clk_cpu_freq_get_config(&cpu_freq_config);
@@ -743,8 +743,10 @@ static uint32_t IRAM_ATTR esp_sleep_start(uint32_t pd_flags, esp_sleep_mode_t mo
         rtc_clk_cpu_freq_set_config(&cpu_freq_config);
     }
 
-    //restore MSPI speed
-    mspi_timing_change_speed_mode_cache_safe(false);
+    if (cpu_freq_config.source == SOC_CPU_CLK_SRC_PLL) {
+        // Turn up MSPI speed if switch to PLL
+        mspi_timing_change_speed_mode_cache_safe(false);
+    }
 
     if (!deep_sleep) {
         s_config.ccount_ticks_record = esp_cpu_get_cycle_count();
diff --git a/components/esp_pm/pm_impl.c b/components/esp_pm/pm_impl.c
index 9b7f6e608a..c33912dbbb 100644
--- a/components/esp_pm/pm_impl.c
+++ b/components/esp_pm/pm_impl.c
@@ -31,6 +31,10 @@
 #include "xtensa/core-macros.h"
 #endif
 
+#if SOC_SPI_MEM_SUPPORT_TIMING_TUNING
+#include "esp_private/mspi_timing_tuning.h"
+#endif
+
 #include "esp_private/pm_impl.h"
 #include "esp_private/pm_trace.h"
 #include "esp_private/esp_timer_private.h"
@@ -475,7 +479,17 @@ static void IRAM_ATTR do_switch(pm_mode_t new_mode)
         if (switch_down) {
             on_freq_update(old_ticks_per_us, new_ticks_per_us);
         }
-        rtc_clk_cpu_freq_set_config_fast(&new_config);
+        if (new_config.source == SOC_CPU_CLK_SRC_PLL) {
+            rtc_clk_cpu_freq_set_config_fast(&new_config);
+#if SOC_SPI_MEM_SUPPORT_TIMING_TUNING
+            mspi_timing_change_speed_mode_cache_safe(false);
+#endif
+        } else {
+#if SOC_SPI_MEM_SUPPORT_TIMING_TUNING
+            mspi_timing_change_speed_mode_cache_safe(true);
+#endif
+            rtc_clk_cpu_freq_set_config_fast(&new_config);
+        }
         if (!switch_down) {
             on_freq_update(old_ticks_per_us, new_ticks_per_us);
         }