diff --git a/components/bt/controller/esp32c3/bt.c b/components/bt/controller/esp32c3/bt.c
index 8615c4935e..04c35768e4 100644
--- a/components/bt/controller/esp32c3/bt.c
+++ b/components/bt/controller/esp32c3/bt.c
@@ -955,6 +955,11 @@ esp_err_t esp_bt_controller_init(esp_bt_controller_config_t *cfg)
 
     sdk_config_extend_set_pll_track(false);
 
+#if CONFIG_MAC_BB_PD
+    esp_mac_bb_pd_mem_init();
+#endif
+    esp_phy_pd_mem_init();
+
     btdm_controller_mem_init();
 
 #if CONFIG_MAC_BB_PD
@@ -1161,6 +1166,11 @@ error:
     esp_unregister_mac_bb_pu_callback(btdm_mac_bb_power_up_cb);
 #endif
 
+
+#if CONFIG_MAC_BB_PD
+    esp_mac_bb_pd_mem_deinit();
+#endif
+    esp_phy_pd_mem_deinit();
     if (osi_funcs_p != NULL) {
         free(osi_funcs_p);
         osi_funcs_p = NULL;
diff --git a/components/esp_wifi/include/esp_phy_init.h b/components/esp_wifi/include/esp_phy_init.h
index f85d8b38b1..a22ac25a00 100644
--- a/components/esp_wifi/include/esp_phy_init.h
+++ b/components/esp_wifi/include/esp_phy_init.h
@@ -178,12 +178,27 @@ void esp_phy_disable(void);
  */
 void esp_phy_load_cal_and_init(void);
 
+/**
+ * @brief Initialize backup memory for Phy power up/down
+ */
+void esp_phy_pd_mem_init(void);
+
+/**
+ * @brief Deinitialize backup memory for Phy power up/down
+ */
+void esp_phy_pd_mem_deinit(void);
+
 #if CONFIG_MAC_BB_PD
 /**
  * @brief Initialize backup memory for MAC and Baseband power up/down
  */
 void esp_mac_bb_pd_mem_init(void);
 
+/**
+ * @brief Deinitialize backup memory for MAC and Baseband power up/down
+ */
+void esp_mac_bb_pd_mem_deinit(void);
+
 /**
  * @brief Power up MAC and Baseband
  */
diff --git a/components/esp_wifi/src/phy_init.c b/components/esp_wifi/src/phy_init.c
index bb5c0625f5..82b11f11fe 100644
--- a/components/esp_wifi/src/phy_init.c
+++ b/components/esp_wifi/src/phy_init.c
@@ -79,9 +79,12 @@ static DRAM_ATTR portMUX_TYPE s_phy_int_mux = portMUX_INITIALIZER_UNLOCKED;
 
 /* Memory to store PHY digital registers */
 static uint32_t* s_phy_digital_regs_mem = NULL;
+static uint8_t s_phy_backup_mem_ref = 0;
 
 #if CONFIG_MAC_BB_PD
 uint32_t* s_mac_bb_pd_mem = NULL;
+/* Reference count of MAC BB backup memory */
+static uint8_t s_macbb_backup_mem_ref = 0;
 #endif
 
 #if CONFIG_ESP32_SUPPORT_MULTIPLE_PHY_INIT_DATA_BIN
@@ -210,10 +213,6 @@ IRAM_ATTR void esp_phy_common_clock_disable(void)
 
 static inline void phy_digital_regs_store(void)
 {
-    if (s_phy_digital_regs_mem == NULL) {
-        s_phy_digital_regs_mem = (uint32_t *)malloc(SOC_PHY_DIG_REGS_MEM_SIZE);
-    }
-
     if (s_phy_digital_regs_mem != NULL) {
         phy_dig_reg_backup(true, s_phy_digital_regs_mem);
     }
@@ -287,14 +286,54 @@ void esp_phy_disable(void)
     _lock_release(&s_phy_access_lock);
 }
 
+void esp_phy_pd_mem_init(void)
+{
+    _lock_acquire(&s_phy_access_lock);
+
+    s_phy_backup_mem_ref++;
+    if (s_phy_digital_regs_mem == NULL) {
+        s_phy_digital_regs_mem = (uint32_t *)heap_caps_malloc(SOC_PHY_DIG_REGS_MEM_SIZE, MALLOC_CAP_DMA|MALLOC_CAP_INTERNAL);
+    }
+
+    _lock_release(&s_phy_access_lock);
+
+}
+
+void esp_phy_pd_mem_deinit(void)
+{
+    _lock_acquire(&s_phy_access_lock);
+
+    s_phy_backup_mem_ref--;
+    if (s_phy_backup_mem_ref == 0) {
+        s_is_phy_calibrated = false;
+        free(s_phy_digital_regs_mem);
+        s_phy_digital_regs_mem = NULL;
+    }
+
+    _lock_release(&s_phy_access_lock);
+}
+
 #if CONFIG_MAC_BB_PD
 void esp_mac_bb_pd_mem_init(void)
 {
     _lock_acquire(&s_phy_access_lock);
 
+    s_macbb_backup_mem_ref++;
     if (s_mac_bb_pd_mem == NULL) {
         s_mac_bb_pd_mem = (uint32_t *)heap_caps_malloc(SOC_MAC_BB_PD_MEM_SIZE, MALLOC_CAP_DMA|MALLOC_CAP_INTERNAL);
     }
+    _lock_release(&s_phy_access_lock);
+}
+
+void esp_mac_bb_pd_mem_deinit(void)
+{
+    _lock_acquire(&s_phy_access_lock);
+
+    s_macbb_backup_mem_ref--;
+    if (s_macbb_backup_mem_ref == 0) {
+        free(s_mac_bb_pd_mem);
+        s_mac_bb_pd_mem = NULL;
+    }
 
     _lock_release(&s_phy_access_lock);
 }
diff --git a/components/esp_wifi/src/wifi_init.c b/components/esp_wifi/src/wifi_init.c
index 0e4a6aed2d..0c3caba29b 100644
--- a/components/esp_wifi/src/wifi_init.c
+++ b/components/esp_wifi/src/wifi_init.c
@@ -136,6 +136,11 @@ esp_err_t esp_wifi_deinit(void)
     esp_unregister_mac_bb_pd_callback(pm_mac_sleep);
     esp_unregister_mac_bb_pu_callback(pm_mac_wakeup);
 #endif
+
+#if CONFIG_MAC_BB_PD
+    esp_mac_bb_pd_mem_deinit();
+#endif
+    esp_phy_pd_mem_deinit();
     return err;
 }
 
@@ -250,6 +255,7 @@ esp_err_t esp_wifi_init(const wifi_init_config_t *config)
         esp_mac_bb_pd_mem_init();
         esp_wifi_internal_set_mac_sleep(true);
 #endif
+        esp_phy_pd_mem_init();
 #if CONFIG_IDF_TARGET_ESP32
         s_wifi_mac_time_update_cb = esp_wifi_internal_update_mac_time;
 #endif