spi_flash: Fix issue that cannot get accurate flash size when encounter large size memory,

Closes https://github.com/espressif/esp-idf/pull/9566

components/spi_flash/esp_flash_api.c

@@ -53,6 +53,17 @@ static const char TAG[] = "spi_flash";
     } while(0)
 #endif // CONFIG_SPI_FLASH_DANGEROUS_WRITE_ALLOWED
 
+/* Convenience macro for beginning of all API functions.
+ * Check the return value of `rom_spiflash_api_funcs->chip_check` is correct,
+ * and the chip supports the operation in question.
+ */
+#define VERIFY_CHIP_OP(op) do {                                  \
+        if (err != ESP_OK) return err; \
+        if (chip->chip_drv->op == NULL) {                        \
+            return ESP_ERR_FLASH_UNSUPPORTED_CHIP;              \
+        }                                                   \
+    } while (0)
+
 #define IO_STR_LEN  10
 
 static const char io_mode_str[][IO_STR_LEN] = {
@@ -211,7 +222,7 @@ esp_err_t IRAM_ATTR esp_flash_init(esp_flash_t *chip)
 
     // Detect flash size
     uint32_t size;
-    err = esp_flash_get_size(chip, &size);
+    err = esp_flash_get_physical_size(chip, &size);
     if (err != ESP_OK) {
         ESP_LOGE(TAG, "failed to get chip size");
         return err;
@@ -291,7 +302,7 @@ esp_err_t IRAM_ATTR esp_flash_init_main(esp_flash_t *chip)
 
     // Detect flash size
     uint32_t size;
-    err = esp_flash_get_size(chip, &size);
+    err = esp_flash_get_physical_size(chip, &size);
     if (err != ESP_OK) {
         ESP_LOGE(TAG, "failed to get chip size");
         return err;
@@ -449,32 +460,15 @@ static esp_err_t IRAM_ATTR detect_spi_flash_chip(esp_flash_t *chip)
     return ESP_OK;
 }
 
-#ifndef CONFIG_SPI_FLASH_ROM_IMPL
+esp_err_t IRAM_ATTR esp_flash_get_physical_size(esp_flash_t *chip, uint32_t *flash_size)
-
-/* Convenience macro for beginning of all API functions.
- * Check the return value of `rom_spiflash_api_funcs->chip_check` is correct,
- * and the chip supports the operation in question.
- */
-#define VERIFY_CHIP_OP(OP) do {                                  \
-        if (err != ESP_OK) return err; \
-        if (chip->chip_drv->OP == NULL) {                        \
-            return ESP_ERR_FLASH_UNSUPPORTED_CHIP;              \
-        }                                                   \
-    } while (0)
-
-/* Return true if regions 'a' and 'b' overlap at all, based on their start offsets and lengths. */
-inline static bool regions_overlap(uint32_t a_start, uint32_t a_len,uint32_t b_start, uint32_t b_len);
-
-esp_err_t IRAM_ATTR esp_flash_get_size(esp_flash_t *chip, uint32_t *out_size)
 {
     esp_err_t err = rom_spiflash_api_funcs->chip_check(&chip);
-    VERIFY_CHIP_OP(detect_size);
+    if (err != ESP_OK) {
-    if (out_size == NULL) {
+        return err;
-        return ESP_ERR_INVALID_ARG;
     }
-    if (chip->size != 0) {
+    VERIFY_CHIP_OP(detect_size);
-        *out_size = chip->size;
+    if (flash_size == NULL) {
-        return ESP_OK;
+        return ESP_ERR_INVALID_ARG;
     }
 
     err = rom_spiflash_api_funcs->start(chip);
@@ -484,12 +478,38 @@ esp_err_t IRAM_ATTR esp_flash_get_size(esp_flash_t *chip, uint32_t *out_size)
     uint32_t detect_size;
     err = chip->chip_drv->detect_size(chip, &detect_size);
     if (err == ESP_OK) {
-        chip->size = detect_size;
+        if (chip->size == 0) {
-        *out_size = chip->size;
+            // chip->size will not be changed if detected, it will always be equal to configured flash size.
+            chip->size = detect_size;
+        }
+        *flash_size = detect_size;
     }
     return rom_spiflash_api_funcs->end(chip, err);
 }
 
+#ifndef CONFIG_SPI_FLASH_ROM_IMPL
+
+/* Return true if regions 'a' and 'b' overlap at all, based on their start offsets and lengths. */
+inline static bool regions_overlap(uint32_t a_start, uint32_t a_len,uint32_t b_start, uint32_t b_len);
+
+esp_err_t IRAM_ATTR esp_flash_get_size(esp_flash_t *chip, uint32_t *out_size)
+{
+    esp_err_t err = rom_spiflash_api_funcs->chip_check(&chip);
+    if (err != ESP_OK) {
+        return err;
+    }
+    if (out_size == NULL) {
+        return ESP_ERR_INVALID_ARG;
+    }
+    if (chip->size != 0) {
+        *out_size = chip->size;
+        return ESP_OK;
+    }
+    //Return flash chip physical size, when this API is called before flash initialisation,
+    //After initialization will return available size.
+    return esp_flash_get_physical_size(chip, out_size);
+}
+
 esp_err_t IRAM_ATTR esp_flash_erase_chip(esp_flash_t *chip)
 {
     esp_err_t err = rom_spiflash_api_funcs->chip_check(&chip);

components/spi_flash/esp_flash_spi_init.c

@@ -409,6 +409,7 @@ esp_err_t esp_flash_init_default_chip(void)
     if (default_chip.size > legacy_chip->chip_size) {
         ESP_EARLY_LOGW(TAG, "Detected size(%dk) larger than the size in the binary image header(%dk). Using the size in the binary image header.", default_chip.size/1024, legacy_chip->chip_size/1024);
     }
+    // Set chip->size equal to ROM flash size(also equal to menuconfig flash size), which means the available size that can be used
     default_chip.size = legacy_chip->chip_size;
 
     esp_flash_default_chip = &default_chip;

components/spi_flash/include/esp_flash.h

@@ -97,7 +97,7 @@ struct esp_flash_t {
     void *os_func_data;                         ///< Pointer to argument for os-specific hooks. Left NULL and will be initialized with ``os_func``.
 
     esp_flash_io_mode_t read_mode; ///< Configured SPI flash read mode. Set before ``esp_flash_init`` is called.
-    uint32_t size;                   ///< Size of SPI flash in bytes. If 0, size will be detected during initialisation.
+    uint32_t size;                   ///< Size of SPI flash in bytes. If 0, size will be detected during initialisation. Note: Only stands for the available size (`CONFIG_ESPTOOLPY_FLASHSIZE`), If you want to get the flash physical size, please call `esp_flash_get_physical_size`.
     uint32_t chip_id;               ///< Detected chip id.
     uint32_t busy             :1;   ///< This flag is used to verify chip's status.
     uint32_t hpm_dummy_ena    :1;   ///< This flag is used to verify whether flash works under HPM status.
@@ -145,14 +145,28 @@ esp_err_t esp_flash_read_id(esp_flash_t *chip, uint32_t *out_id);
 /** @brief Detect flash size based on flash ID.
  *
  * @param chip Pointer to identify flash chip. Must have been successfully initialised via esp_flash_init()
- * @param[out] out_size Detected size in bytes.
+ * @param[out] out_size Detected size in bytes, standing for the available size (`CONFIG_ESPTOOLPY_FLASHSIZE`).
+ *
+ * @note 1. Most flash chips use a common format for flash ID, where the lower 4 bits specify the size as a power of 2. If
+ * the manufacturer doesn't follow this convention, the size may be incorrectly detected.
+ *       2. The out_size returned only stands for the size selected in menuconfig.
+ *  If you want to get the real size of the chip, please call `esp_flash_get_physical_size` instead.
+ *
+ * @return ESP_OK on success, or a flash error code if operation failed.
+ */
+esp_err_t esp_flash_get_size(esp_flash_t *chip, uint32_t *out_size);
+
+/** @brief Detect flash size based on flash ID.
+ *
+ * @param chip Pointer to identify flash chip. Must have been successfully initialised via esp_flash_init()
+ * @param[out] flash_size Detected size in bytes.
  *
  * @note Most flash chips use a common format for flash ID, where the lower 4 bits specify the size as a power of 2. If
  * the manufacturer doesn't follow this convention, the size may be incorrectly detected.
  *
  * @return ESP_OK on success, or a flash error code if operation failed.
  */
-esp_err_t esp_flash_get_size(esp_flash_t *chip, uint32_t *out_size);
+esp_err_t esp_flash_get_physical_size(esp_flash_t *chip, uint32_t *flash_size);
 
 /** @brief Read flash unique ID via the common "RDUID" SPI flash command.
  *

components/spi_flash/spi_flash_chip_gd.c

@@ -1,16 +1,8 @@
-// Copyright 2015-2019 Espressif Systems (Shanghai) PTE LTD
+/*
-//
+ * SPDX-FileCopyrightText: 2015-2022 Espressif Systems (Shanghai) CO LTD
-// Licensed under the Apache License, Version 2.0 (the "License");
+ *
-// you may not use this file except in compliance with the License.
+ * SPDX-License-Identifier: Apache-2.0
-// You may obtain a copy of the License at
+ */
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
 
 #include <stdlib.h>
 #include <string.h>
@@ -47,6 +39,21 @@ spi_flash_caps_t spi_flash_chip_gd_get_caps(esp_flash_t *chip)
     return caps_flags;
 }
 
+esp_err_t spi_flash_chip_gd_detect_size(esp_flash_t *chip, uint32_t *size)
+{
+    uint32_t id = chip->chip_id;
+    *size = 0;
+
+    /* Can't detect size unless the high byte of the product ID matches the same convention, which is usually 0x40 or
+     * 0xC0 or similar. */
+    if (((id & 0xFFFF) == 0x0000) || ((id & 0xFFFF) == 0xFFFF)) {
+        return ESP_ERR_FLASH_UNSUPPORTED_CHIP;
+    }
+
+    *size = 1 << (id & 0xFF);
+    return ESP_OK;
+}
+
 #ifndef CONFIG_SPI_FLASH_ROM_IMPL
 
 #define FLASH_ID_MASK       0xFF00
@@ -114,7 +121,7 @@ const spi_flash_chip_t esp_flash_chip_gd = {
     .timeout = &spi_flash_chip_generic_timeout,
     .probe = spi_flash_chip_gd_probe,
     .reset = spi_flash_chip_generic_reset,
-    .detect_size = spi_flash_chip_generic_detect_size,
+    .detect_size = spi_flash_chip_gd_detect_size,
     .erase_chip = spi_flash_chip_generic_erase_chip,
     .erase_sector = spi_flash_chip_gd_erase_sector,
     .erase_block = spi_flash_chip_gd_erase_block,

components/spi_flash/spi_flash_chip_generic.c

@@ -66,6 +66,9 @@ DRAM_ATTR static spi_flash_encryption_t esp_flash_encryption_default __attribute
 #define HOST_DELAY_INTERVAL_US                      1
 #define CHIP_WAIT_IDLE_INTERVAL_US                  20
 
+#define SPI_FLASH_LINEAR_DENSITY_LAST_VALUE        (0x19)
+#define SPI_FLASH_HEX_A_F_RANGE                    (6)
+
 const DRAM_ATTR flash_chip_op_timeout_t spi_flash_chip_generic_timeout = {
     .idle_timeout = SPI_FLASH_DEFAULT_IDLE_TIMEOUT_MS * 1000,
     .chip_erase_timeout = SPI_FLASH_GENERIC_CHIP_ERASE_TIMEOUT_MS * 1000,
@@ -82,6 +85,30 @@ const DRAM_ATTR flash_chip_op_timeout_t spi_flash_chip_generic_timeout = {
 
 static const char TAG[] = "chip_generic";
 
+esp_err_t spi_flash_chip_generic_detect_size(esp_flash_t *chip, uint32_t *size)
+{
+    uint32_t id = chip->chip_id;
+    *size = 0;
+
+    /* Can't detect size unless the high byte of the product ID matches the same convention, which is usually 0x40 or
+     * 0xC0 or similar. */
+    if (((id & 0xFFFF) == 0x0000) || ((id & 0xFFFF) == 0xFFFF)) {
+        return ESP_ERR_FLASH_UNSUPPORTED_CHIP;
+    }
+
+    /* Get flash capacity from flash chip id depends on different vendors. According to majority of flash datasheets,
+       Flash 256Mb to 512Mb directly from 0x19 to 0x20, instead of from 0x19 to 0x1a. So here we leave the common behavior.
+       However, some other flash vendors also have their own rule, we will add them in chip specific files.
+     */
+    uint32_t mem_density = (id & 0xFF);
+    if (mem_density > SPI_FLASH_LINEAR_DENSITY_LAST_VALUE ) {
+        mem_density -= SPI_FLASH_HEX_A_F_RANGE;
+    }
+
+    *size = 1 << mem_density;
+    return ESP_OK;
+}
+
 #ifndef CONFIG_SPI_FLASH_ROM_IMPL
 
 esp_err_t spi_flash_chip_generic_probe(esp_flash_t *chip, uint32_t flash_id)
@@ -115,22 +142,6 @@ esp_err_t spi_flash_chip_generic_reset(esp_flash_t *chip)
     return err;
 }
 
-esp_err_t spi_flash_chip_generic_detect_size(esp_flash_t *chip, uint32_t *size)
-{
-    uint32_t id = chip->chip_id;
-    *size = 0;
-
-    /* Can't detect size unless the high byte of the product ID matches the same convention, which is usually 0x40 or
-     * 0xC0 or similar. */
-    if (((id & 0xFFFF) == 0x0000) || ((id & 0xFFFF) == 0xFFFF)) {
-        return ESP_ERR_FLASH_UNSUPPORTED_CHIP;
-    }
-
-    *size = 1 << (id & 0xFF);
-    return ESP_OK;
-}
-
-
 esp_err_t spi_flash_chip_generic_erase_chip(esp_flash_t *chip)
 {
     esp_err_t err;

components/spi_flash/spi_flash_chip_mxic_opi.c

@@ -1,16 +1,8 @@
-// Copyright 2015-2020 Espressif Systems (Shanghai) PTE LTD
+/*
-//
+ * SPDX-FileCopyrightText: 2015-2022 Espressif Systems (Shanghai) CO LTD
-// Licensed under the Apache License, Version 2.0 (the "License");
+ *
-// you may not use this file except in compliance with the License.
+ * SPDX-License-Identifier: Apache-2.0
-// You may obtain a copy of the License at
+ */
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
 
 #include <stdlib.h>
 #include "spi_flash_chip_generic.h"
@@ -46,6 +38,21 @@ esp_err_t spi_flash_chip_mxic_opi_probe(esp_flash_t *chip, uint32_t flash_id)
     return ESP_OK;
 }
 
+esp_err_t spi_flash_chip_mxic_opi_detect_size(esp_flash_t *chip, uint32_t *size)
+{
+    uint32_t id = chip->chip_id;
+    *size = 0;
+
+    /* Can't detect size unless the high byte of the product ID matches the same convention, which is usually 0x40 or
+     * 0xC0 or similar. */
+    if (((id & 0xFFFF) == 0x0000) || ((id & 0xFFFF) == 0xFFFF)) {
+        return ESP_ERR_FLASH_UNSUPPORTED_CHIP;
+    }
+
+    *size = 1 << ((id & 0xFF) - 0x20);
+    return ESP_OK;
+}
+
 spi_flash_caps_t spi_flash_chip_mxic_opi_get_caps(esp_flash_t *chip)
 {
     spi_flash_caps_t caps_flags = 0;
@@ -384,7 +391,7 @@ const spi_flash_chip_t esp_flash_chip_mxic_opi = {
     .timeout = &spi_flash_chip_generic_timeout,
     .probe = spi_flash_chip_mxic_opi_probe,
     .reset = spi_flash_chip_generic_reset,
-    .detect_size = spi_flash_chip_generic_detect_size,
+    .detect_size = spi_flash_chip_mxic_opi_detect_size,
     .erase_chip = spi_flash_chip_mxic_opi_erase_chip,
     .erase_sector = spi_flash_chip_mxic_opi_erase_sector,
     .erase_block = spi_flash_chip_mxic_opi_erase_block,

components/spi_flash/test_apps/esp_flash/main/test_esp_flash_drv.c

@@ -249,10 +249,11 @@ static void flash_test_func(flash_test_func_t func, int test_num)
 static void test_metadata(const esp_partition_t* part)
 {
     esp_flash_t* chip = part->flash_chip;
-    uint32_t id, size;
+    uint32_t id, size, actual_size;
     TEST_ESP_OK(esp_flash_read_id(chip, &id));
     TEST_ESP_OK(esp_flash_get_size(chip, &size));
-    printf("Flash ID %08lx detected size %" PRIu32 "bytes\n", id, size);
+    TEST_ESP_OK(esp_flash_get_physical_size(chip, &actual_size));
+    printf("Flash ID %08lx, size %" PRIu32 "bytes can be mapped, actual flash size is %" PRIu32 "bytes\n", id, size, actual_size);
 }
 
 TEST_CASE_FLASH("SPI flash metadata functions", test_metadata);

tools/ci/check_copyright_ignore.txt

@@ -1228,10 +1228,8 @@ components/spi_flash/sim/flash_mock_util.c
 components/spi_flash/sim/sdkconfig/sdkconfig.h
 components/spi_flash/sim/stubs/bsd/strlcpy.c
 components/spi_flash/spi_flash_chip_boya.c
-components/spi_flash/spi_flash_chip_gd.c
 components/spi_flash/spi_flash_chip_issi.c
 components/spi_flash/spi_flash_chip_mxic.c
-components/spi_flash/spi_flash_chip_mxic_opi.c
 components/spi_flash/spi_flash_chip_winbond.c
 components/spi_flash/test/test_esp_flash.c
 components/spi_flash/test/test_flash_encryption.c