diff --git a/components/esp_hw_support/port/esp32c6/rtc_clk.c b/components/esp_hw_support/port/esp32c6/rtc_clk.c
index 7e73d999e4..54ae585173 100644
--- a/components/esp_hw_support/port/esp32c6/rtc_clk.c
+++ b/components/esp_hw_support/port/esp32c6/rtc_clk.c
@@ -1,5 +1,5 @@
 /*
- * SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD
+ * SPDX-FileCopyrightText: 2022-2023 Espressif Systems (Shanghai) CO LTD
  *
  * SPDX-License-Identifier: Apache-2.0
  */
@@ -19,12 +19,15 @@
 #include "hal/regi2c_ctrl_ll.h"
 #include "soc/io_mux_reg.h"
 #include "soc/lp_aon_reg.h"
+#include "hal/usb_serial_jtag_ll.h"
 
 static const char *TAG = "rtc_clk";
 
 // Current PLL frequency, in 480MHz. Zero if PLL is not enabled.
 static int s_cur_pll_freq;
 
+static bool rtc_clk_set_bbpll_always_on(void);
+
 void rtc_clk_32k_enable(bool enable)
 {
     if (enable) {
@@ -221,7 +224,7 @@ void rtc_clk_cpu_freq_set_config(const rtc_cpu_freq_config_t *config)
     soc_cpu_clk_src_t old_cpu_clk_src = clk_ll_cpu_get_src();
     if (config->source == SOC_CPU_CLK_SRC_XTAL) {
         rtc_clk_cpu_freq_to_xtal(config->freq_mhz, config->div);
-        if (old_cpu_clk_src == SOC_CPU_CLK_SRC_PLL) {
+        if ((old_cpu_clk_src == SOC_CPU_CLK_SRC_PLL) && !rtc_clk_set_bbpll_always_on()) {
             rtc_clk_bbpll_disable();
         }
     } else if (config->source == SOC_CPU_CLK_SRC_PLL) {
@@ -232,7 +235,7 @@ void rtc_clk_cpu_freq_set_config(const rtc_cpu_freq_config_t *config)
         rtc_clk_cpu_freq_to_pll_mhz(config->freq_mhz);
     } else if (config->source == SOC_CPU_CLK_SRC_RC_FAST) {
         rtc_clk_cpu_freq_to_8m();
-        if (old_cpu_clk_src == SOC_CPU_CLK_SRC_PLL) {
+        if ((old_cpu_clk_src == SOC_CPU_CLK_SRC_PLL) && !rtc_clk_set_bbpll_always_on()) {
             rtc_clk_bbpll_disable();
         }
     }
@@ -294,7 +297,10 @@ void rtc_clk_cpu_freq_set_xtal(void)
     int freq_mhz = (int)rtc_clk_xtal_freq_get();
 
     rtc_clk_cpu_freq_to_xtal(freq_mhz, 1);
-    rtc_clk_bbpll_disable();
+    // We don't turn off the bbpll if some consumers only depends on bbpll
+    if (!rtc_clk_set_bbpll_always_on()) {
+        rtc_clk_bbpll_disable();
+    }
 }
 
 rtc_xtal_freq_t rtc_clk_xtal_freq_get(void)
@@ -362,6 +368,21 @@ bool rtc_dig_8m_enabled(void)
     return clk_ll_rc_fast_digi_is_enabled();
 }
 
+static bool rtc_clk_set_bbpll_always_on(void)
+{
+    /* We just keep the rtc bbpll clock on just under the case that
+    user selects the `RTC_CLOCK_BBPLL_POWER_ON_WITH_USB` as well as
+    the USB_SERIAL_JTAG is connected with PC.
+    */
+    bool is_bbpll_on = false;
+#if CONFIG_RTC_CLOCK_BBPLL_POWER_ON_WITH_USB
+    if (usb_serial_jtag_ll_txfifo_writable() == 1) {
+        is_bbpll_on = true;
+    }
+#endif
+    return is_bbpll_on;
+}
+
 /* Name used in libphy.a:phy_chip_v7.o
  * TODO: update the library to use rtc_clk_xtal_freq_get
  */