diff --git a/components/esp_phy/Kconfig b/components/esp_phy/Kconfig
index 59d7fe783d..7a2b7a811f 100644
--- a/components/esp_phy/Kconfig
+++ b/components/esp_phy/Kconfig
@@ -154,4 +154,10 @@ menu "PHY"
             high interference, enable this option will sacrifice Wi-Fi OFDM receive performance.
             But to guarantee 11b receive performance serves as a bottom line in this case.
 
+    config ESP_PHY_PLL_TRACK_DEBUG
+        bool "Enable pll track logging"
+        default n
+        help
+            If enabled, there will be some logs while pll tracking
+
 endmenu  # PHY
diff --git a/components/esp_phy/include/esp_private/phy.h b/components/esp_phy/include/esp_private/phy.h
index 8968e00246..bee81286a4 100644
--- a/components/esp_phy/include/esp_private/phy.h
+++ b/components/esp_phy/include/esp_private/phy.h
@@ -1,5 +1,5 @@
 /*
- * SPDX-FileCopyrightText: 2015-2023 Espressif Systems (Shanghai) CO LTD
+ * SPDX-FileCopyrightText: 2015-2024 Espressif Systems (Shanghai) CO LTD
  *
  * SPDX-License-Identifier: Apache-2.0
  */
@@ -174,6 +174,12 @@ esp_phy_modem_t phy_get_modem_flag(void);
  *
  */
 _lock_t phy_get_lock(void);
+
+/**
+ * @brief Call this funnction to track pll immediately.
+ *
+ */
+void phy_track_pll(void);
 #ifdef __cplusplus
 }
 #endif
diff --git a/components/esp_phy/src/phy_common.c b/components/esp_phy/src/phy_common.c
index ad4a3b388e..759f3b5de8 100644
--- a/components/esp_phy/src/phy_common.c
+++ b/components/esp_phy/src/phy_common.c
@@ -1,9 +1,10 @@
 /*
- * SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD
+ * SPDX-FileCopyrightText: 2023-2024 Espressif Systems (Shanghai) CO LTD
  *
  * SPDX-License-Identifier: Apache-2.0
  */
 
+#include "esp_log.h"
 #include "esp_timer.h"
 #include "esp_phy_init.h"
 #include "esp_private/phy.h"
@@ -20,6 +21,7 @@ static volatile int64_t s_wifi_prev_timestamp;
 static volatile int64_t s_bt_154_prev_timestamp;
 #endif
 #define PHY_TRACK_PLL_PERIOD_IN_US 1000000
+static void phy_track_pll_internal(void);
 
 #if CONFIG_IEEE802154_ENABLED || CONFIG_BT_ENABLED || CONFIG_ESP_WIFI_ENABLED
 bool phy_enabled_modem_contains(esp_phy_modem_t modem)
@@ -28,7 +30,28 @@ bool phy_enabled_modem_contains(esp_phy_modem_t modem)
 }
 #endif
 
-static void phy_track_pll(void)
+void phy_track_pll(void)
+{
+    // Light sleep scenario: enabling and disabling PHY frequently, the timer will not get triggered.
+    // Using a variable to record the previously tracked time when PLL was last called.
+    // If the duration is larger than PHY_TRACK_PLL_PERIOD_IN_US, then track PLL.
+    bool need_track_pll = false;
+#if CONFIG_ESP_WIFI_ENABLED
+    if (phy_enabled_modem_contains(PHY_MODEM_WIFI)) {
+        need_track_pll = need_track_pll || ((esp_timer_get_time() - s_wifi_prev_timestamp) > PHY_TRACK_PLL_PERIOD_IN_US);
+    }
+#endif
+#if CONFIG_IEEE802154_ENABLED || CONFIG_BT_ENABLED
+    if (phy_enabled_modem_contains(PHY_MODEM_BT | PHY_MODEM_IEEE802154)) {
+        need_track_pll = need_track_pll || ((esp_timer_get_time() - s_bt_154_prev_timestamp) > PHY_TRACK_PLL_PERIOD_IN_US);
+    }
+#endif
+    if (need_track_pll) {
+        phy_track_pll_internal();
+    }
+}
+
+static void phy_track_pll_internal(void)
 {
     bool wifi_track_pll = false;
     bool ble_154_track_pll = false;
@@ -46,6 +69,14 @@ static void phy_track_pll(void)
     }
 #endif
     if (wifi_track_pll || ble_154_track_pll) {
+#if CONFIG_ESP_PHY_PLL_TRACK_DEBUG
+#if CONFIG_IEEE802154_ENABLED || CONFIG_BT_ENABLED
+        ESP_LOGI("PLL_TRACK", "BT or IEEE802154 tracks PLL: %s", ble_154_track_pll ? "True" : "False");
+#endif
+#if CONFIG_ESP_WIFI_ENABLED
+        ESP_LOGI("PLL_TRACK", "Wi-Fi tracks PLL: %s", wifi_track_pll ? "True" : "False");
+#endif
+#endif
         phy_param_track_tot(wifi_track_pll, ble_154_track_pll);
     }
 }
@@ -54,26 +85,12 @@ static void phy_track_pll_timer_callback(void* arg)
 {
     _lock_t phy_lock = phy_get_lock();
     _lock_acquire(&phy_lock);
-    phy_track_pll();
+    phy_track_pll_internal();
     _lock_release(&phy_lock);
 }
 
 void phy_track_pll_init(void)
 {
-    // Light sleep scenario: enabling and disabling PHY frequently, the timer will not get triggered.
-    // Using a variable to record the previously tracked time when PLL was last called.
-    // If the duration is larger than PHY_TRACK_PLL_PERIOD_IN_US, then track PLL.
-    bool need_track_pll = false;
-#if CONFIG_ESP_WIFI_ENABLED
-    need_track_pll = need_track_pll || ((esp_timer_get_time() - s_wifi_prev_timestamp) > PHY_TRACK_PLL_PERIOD_IN_US);
-#endif
-#if CONFIG_IEEE802154_ENABLED || CONFIG_BT_ENABLED
-    need_track_pll = need_track_pll || ((esp_timer_get_time() - s_bt_154_prev_timestamp) > PHY_TRACK_PLL_PERIOD_IN_US);
-#endif
-    if (need_track_pll) {
-        phy_track_pll();
-    }
-
     const esp_timer_create_args_t phy_track_pll_timer_args = {
             .callback = &phy_track_pll_timer_callback,
             .name = "phy-track-pll-timer"
diff --git a/components/esp_phy/src/phy_init.c b/components/esp_phy/src/phy_init.c
index b2588161ec..2b029d6c30 100644
--- a/components/esp_phy/src/phy_init.c
+++ b/components/esp_phy/src/phy_init.c
@@ -278,6 +278,10 @@ void esp_phy_enable(esp_phy_modem_t modem)
 #endif
     }
     phy_set_modem_flag(modem);
+#if !CONFIG_IDF_TARGET_ESP32
+    // Immediately track pll when phy enabled.
+    phy_track_pll();
+#endif
 
     _lock_release(&s_phy_access_lock);
 }
diff --git a/components/esp_phy/src/phy_init_esp32hxx.c b/components/esp_phy/src/phy_init_esp32hxx.c
index 9639ffccd3..d0d3e2a20b 100644
--- a/components/esp_phy/src/phy_init_esp32hxx.c
+++ b/components/esp_phy/src/phy_init_esp32hxx.c
@@ -1,5 +1,5 @@
 /*
- * SPDX-FileCopyrightText: 2015-2023 Espressif Systems (Shanghai) CO LTD
+ * SPDX-FileCopyrightText: 2015-2024 Espressif Systems (Shanghai) CO LTD
  *
  * SPDX-License-Identifier: Apache-2.0
  */
@@ -21,7 +21,6 @@ extern void phy_version_print(void);
 static _lock_t s_phy_access_lock;
 
 /* Reference count of enabling PHY */
-static uint8_t s_phy_access_ref = 0;
 static bool s_phy_is_enabled = false;
 
 uint32_t IRAM_ATTR phy_enter_critical(void)
@@ -49,8 +48,7 @@ void IRAM_ATTR phy_exit_critical(uint32_t level)
 void esp_phy_enable(esp_phy_modem_t modem)
 {
     _lock_acquire(&s_phy_access_lock);
-    phy_set_modem_flag(modem);
-    if (s_phy_access_ref == 0) {
+    if (phy_get_modem_flag() == 0) {
 #if SOC_MODEM_CLOCK_IS_INDEPENDENT
         modem_clock_module_enable(PERIPH_PHY_MODULE);
 #endif
@@ -63,9 +61,9 @@ void esp_phy_enable(esp_phy_modem_t modem)
         }
         phy_track_pll_init();
     }
-
-    s_phy_access_ref++;
-
+    phy_set_modem_flag(modem);
+    // Immediately track pll when phy enabled.
+    phy_track_pll();
     _lock_release(&s_phy_access_lock);
 }
 
@@ -73,11 +71,9 @@ void esp_phy_disable(esp_phy_modem_t modem)
 {
     _lock_acquire(&s_phy_access_lock);
 
-    if (s_phy_access_ref) {
-        s_phy_access_ref--;
-    }
+    phy_clr_modem_flag(modem);
+    if (phy_get_modem_flag() == 0) {
 
-    if (s_phy_access_ref == 0) {
         phy_track_pll_deinit();
         phy_close_rf();
         phy_xpd_tsens();
@@ -85,8 +81,6 @@ void esp_phy_disable(esp_phy_modem_t modem)
         modem_clock_module_disable(PERIPH_PHY_MODULE);
 #endif
     }
-
-    phy_clr_modem_flag(modem);
     _lock_release(&s_phy_access_lock);
 }