esp-idf/components/spi_flash/cache_utils.c
Ivan Grokhotkov f9f2937694 spi_flash: raise priority of the task performing spi_flash operation
The fix is for the situation when cache disabling mechanism causes
a deadlock with user tasks. Situation is as follows:

1. spi_flash operation is started from low-priority task on CPU0
2. It uses IPC to wake up high-priority IPC1 task on CPU1, preventing
   all other tasks on CPU1 from running. This is needed to safely
   disable the cache.
3. While the task which started spi_flash operation is waiting for IPC1
   task to acknowledge that CPU1 is not using cache anymore, it is
   preempted by a higher priority application task ("app0").
4. Task app0 busy-waits for some operation on CPU1 to complete. But
   since application tasks are blocked out by IPC1 task, this never
   happens. Since app0 is busy-waiting, the task doing spi flash
   operation never runs.

The more or less logical soltion to the problem would be to also do
cache disabling on CPU0 and the SPI flash operation itself from IPC0
task. However IPC0 task stack would need to be increased to allow doing
SPI flash operation (and IPC1 stack as well). This would waste some
memory. An alternative approach adopted in this fix is to call FreeRTOS
functions to temporary increase the priority of SPI flash operation task
to the same level as the IPC task.

Fixes https://github.com/espressif/arduino-esp32/issues/740
Fixes https://github.com/espressif/esp-idf/issues/1157
2018-07-01 20:44:42 +08:00

302 lines
11 KiB
C

// Copyright 2015-2016 Espressif Systems (Shanghai) PTE LTD
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <stdlib.h>
#include <assert.h>
#include <string.h>
#include <stdio.h>
#include <freertos/FreeRTOS.h>
#include <freertos/task.h>
#include <freertos/semphr.h>
#include <rom/spi_flash.h>
#include <rom/cache.h>
#include <soc/soc.h>
#include <soc/dport_reg.h>
#include "sdkconfig.h"
#include "esp_ipc.h"
#include "esp_attr.h"
#include "esp_intr_alloc.h"
#include "esp_spi_flash.h"
#include "esp_log.h"
static void IRAM_ATTR spi_flash_disable_cache(uint32_t cpuid, uint32_t* saved_state);
static void IRAM_ATTR spi_flash_restore_cache(uint32_t cpuid, uint32_t saved_state);
static uint32_t s_flash_op_cache_state[2];
#ifndef CONFIG_FREERTOS_UNICORE
static SemaphoreHandle_t s_flash_op_mutex;
static volatile bool s_flash_op_can_start = false;
static volatile bool s_flash_op_complete = false;
#ifndef NDEBUG
static volatile int s_flash_op_cpu = -1;
#endif
void spi_flash_init_lock()
{
s_flash_op_mutex = xSemaphoreCreateRecursiveMutex();
assert(s_flash_op_mutex != NULL);
}
void spi_flash_op_lock()
{
xSemaphoreTakeRecursive(s_flash_op_mutex, portMAX_DELAY);
}
void spi_flash_op_unlock()
{
xSemaphoreGiveRecursive(s_flash_op_mutex);
}
/*
If you're going to modify this, keep in mind that while the flash caches of the pro and app
cpu are separate, the psram cache is *not*. If one of the CPUs returns from a flash routine
with its cache enabled but the other CPUs cache is not enabled yet, you will have problems
when accessing psram from the former CPU.
*/
void IRAM_ATTR spi_flash_op_block_func(void* arg)
{
// Disable scheduler on this CPU
vTaskSuspendAll();
// Restore interrupts that aren't located in IRAM
esp_intr_noniram_disable();
uint32_t cpuid = (uint32_t) arg;
// s_flash_op_complete flag is cleared on *this* CPU, otherwise the other
// CPU may reset the flag back to false before IPC task has a chance to check it
// (if it is preempted by an ISR taking non-trivial amount of time)
s_flash_op_complete = false;
s_flash_op_can_start = true;
while (!s_flash_op_complete) {
// busy loop here and wait for the other CPU to finish flash operation
}
// Flash operation is complete, re-enable cache
spi_flash_restore_cache(cpuid, s_flash_op_cache_state[cpuid]);
// Restore interrupts that aren't located in IRAM
esp_intr_noniram_enable();
// Re-enable scheduler
xTaskResumeAll();
}
void IRAM_ATTR spi_flash_disable_interrupts_caches_and_other_cpu()
{
spi_flash_op_lock();
const uint32_t cpuid = xPortGetCoreID();
const uint32_t other_cpuid = (cpuid == 0) ? 1 : 0;
#ifndef NDEBUG
// For sanity check later: record the CPU which has started doing flash operation
assert(s_flash_op_cpu == -1);
s_flash_op_cpu = cpuid;
#endif
if (xTaskGetSchedulerState() == taskSCHEDULER_NOT_STARTED) {
// Scheduler hasn't been started yet, it means that spi_flash API is being
// called from the 2nd stage bootloader or from user_start_cpu0, i.e. from
// PRO CPU. APP CPU is either in reset or spinning inside user_start_cpu1,
// which is in IRAM. So it is safe to disable cache for the other_cpuid here.
assert(other_cpuid == 1);
spi_flash_disable_cache(other_cpuid, &s_flash_op_cache_state[other_cpuid]);
} else {
// Temporarily raise current task priority to prevent a deadlock while
// waiting for IPC task to start on the other CPU
int old_prio = uxTaskPriorityGet(NULL);
vTaskPrioritySet(NULL, configMAX_PRIORITIES - 1);
// Signal to the spi_flash_op_block_task on the other CPU that we need it to
// disable cache there and block other tasks from executing.
s_flash_op_can_start = false;
esp_err_t ret = esp_ipc_call(other_cpuid, &spi_flash_op_block_func, (void*) other_cpuid);
assert(ret == ESP_OK);
while (!s_flash_op_can_start) {
// Busy loop and wait for spi_flash_op_block_func to disable cache
// on the other CPU
}
// Disable scheduler on the current CPU
vTaskSuspendAll();
// Can now set the priority back to the normal one
vTaskPrioritySet(NULL, old_prio);
// This is guaranteed to run on CPU <cpuid> because the other CPU is now
// occupied by highest priority task
assert(xPortGetCoreID() == cpuid);
}
// Kill interrupts that aren't located in IRAM
esp_intr_noniram_disable();
// This CPU executes this routine, with non-IRAM interrupts and the scheduler
// disabled. The other CPU is spinning in the spi_flash_op_block_func task, also
// with non-iram interrupts and the scheduler disabled. None of these CPUs will
// touch external RAM or flash this way, so we can safely disable caches.
spi_flash_disable_cache(cpuid, &s_flash_op_cache_state[cpuid]);
spi_flash_disable_cache(other_cpuid, &s_flash_op_cache_state[other_cpuid]);
}
void IRAM_ATTR spi_flash_enable_interrupts_caches_and_other_cpu()
{
const uint32_t cpuid = xPortGetCoreID();
const uint32_t other_cpuid = (cpuid == 0) ? 1 : 0;
#ifndef NDEBUG
// Sanity check: flash operation ends on the same CPU as it has started
assert(cpuid == s_flash_op_cpu);
// More sanity check: if scheduler isn't started, only CPU0 can call this.
assert(!(xTaskGetSchedulerState() == taskSCHEDULER_NOT_STARTED && cpuid != 0));
s_flash_op_cpu = -1;
#endif
// Re-enable cache on both CPUs. After this, cache (flash and external RAM) should work again.
spi_flash_restore_cache(cpuid, s_flash_op_cache_state[cpuid]);
spi_flash_restore_cache(other_cpuid, s_flash_op_cache_state[other_cpuid]);
if (xTaskGetSchedulerState() != taskSCHEDULER_NOT_STARTED) {
// Signal to spi_flash_op_block_task that flash operation is complete
s_flash_op_complete = true;
}
// Re-enable non-iram interrupts
esp_intr_noniram_enable();
// Resume tasks on the current CPU, if the scheduler has started.
// NOTE: enabling non-IRAM interrupts has to happen before this,
// because once the scheduler has started, due to preemption the
// current task can end up being moved to the other CPU.
// But esp_intr_noniram_enable has to be called on the same CPU which
// called esp_intr_noniram_disable
if (xTaskGetSchedulerState() != taskSCHEDULER_NOT_STARTED) {
xTaskResumeAll();
}
// Release API lock
spi_flash_op_unlock();
}
void IRAM_ATTR spi_flash_disable_interrupts_caches_and_other_cpu_no_os()
{
const uint32_t cpuid = xPortGetCoreID();
const uint32_t other_cpuid = (cpuid == 0) ? 1 : 0;
// do not care about other CPU, it was halted upon entering panic handler
spi_flash_disable_cache(other_cpuid, &s_flash_op_cache_state[other_cpuid]);
// Kill interrupts that aren't located in IRAM
esp_intr_noniram_disable();
// Disable cache on this CPU as well
spi_flash_disable_cache(cpuid, &s_flash_op_cache_state[cpuid]);
}
void IRAM_ATTR spi_flash_enable_interrupts_caches_no_os()
{
const uint32_t cpuid = xPortGetCoreID();
// Re-enable cache on this CPU
spi_flash_restore_cache(cpuid, s_flash_op_cache_state[cpuid]);
// Re-enable non-iram interrupts
esp_intr_noniram_enable();
}
#else // CONFIG_FREERTOS_UNICORE
void spi_flash_init_lock()
{
}
void spi_flash_op_lock()
{
vTaskSuspendAll();
}
void spi_flash_op_unlock()
{
xTaskResumeAll();
}
void IRAM_ATTR spi_flash_disable_interrupts_caches_and_other_cpu()
{
spi_flash_op_lock();
esp_intr_noniram_disable();
spi_flash_disable_cache(0, &s_flash_op_cache_state[0]);
}
void IRAM_ATTR spi_flash_enable_interrupts_caches_and_other_cpu()
{
spi_flash_restore_cache(0, s_flash_op_cache_state[0]);
esp_intr_noniram_enable();
spi_flash_op_unlock();
}
void IRAM_ATTR spi_flash_disable_interrupts_caches_and_other_cpu_no_os()
{
// Kill interrupts that aren't located in IRAM
esp_intr_noniram_disable();
// Disable cache on this CPU as well
spi_flash_disable_cache(0, &s_flash_op_cache_state[0]);
}
void IRAM_ATTR spi_flash_enable_interrupts_caches_no_os()
{
// Re-enable cache on this CPU
spi_flash_restore_cache(0, s_flash_op_cache_state[0]);
// Re-enable non-iram interrupts
esp_intr_noniram_enable();
}
#endif // CONFIG_FREERTOS_UNICORE
/**
* The following two functions are replacements for Cache_Read_Disable and Cache_Read_Enable
* function in ROM. They are used to work around a bug where Cache_Read_Disable requires a call to
* Cache_Flush before Cache_Read_Enable, even if cached data was not modified.
*/
static const uint32_t cache_mask = DPORT_APP_CACHE_MASK_OPSDRAM | DPORT_APP_CACHE_MASK_DROM0 |
DPORT_APP_CACHE_MASK_DRAM1 | DPORT_APP_CACHE_MASK_IROM0 |
DPORT_APP_CACHE_MASK_IRAM1 | DPORT_APP_CACHE_MASK_IRAM0;
static void IRAM_ATTR spi_flash_disable_cache(uint32_t cpuid, uint32_t* saved_state)
{
uint32_t ret = 0;
if (cpuid == 0) {
ret |= DPORT_GET_PERI_REG_BITS2(DPORT_PRO_CACHE_CTRL1_REG, cache_mask, 0);
while (DPORT_GET_PERI_REG_BITS2(DPORT_PRO_DCACHE_DBUG0_REG, DPORT_PRO_CACHE_STATE, DPORT_PRO_CACHE_STATE_S) != 1) {
;
}
DPORT_SET_PERI_REG_BITS(DPORT_PRO_CACHE_CTRL_REG, 1, 0, DPORT_PRO_CACHE_ENABLE_S);
} else {
ret |= DPORT_GET_PERI_REG_BITS2(DPORT_APP_CACHE_CTRL1_REG, cache_mask, 0);
while (DPORT_GET_PERI_REG_BITS2(DPORT_APP_DCACHE_DBUG0_REG, DPORT_APP_CACHE_STATE, DPORT_APP_CACHE_STATE_S) != 1) {
;
}
DPORT_SET_PERI_REG_BITS(DPORT_APP_CACHE_CTRL_REG, 1, 0, DPORT_APP_CACHE_ENABLE_S);
}
*saved_state = ret;
}
static void IRAM_ATTR spi_flash_restore_cache(uint32_t cpuid, uint32_t saved_state)
{
if (cpuid == 0) {
DPORT_SET_PERI_REG_BITS(DPORT_PRO_CACHE_CTRL_REG, 1, 1, DPORT_PRO_CACHE_ENABLE_S);
DPORT_SET_PERI_REG_BITS(DPORT_PRO_CACHE_CTRL1_REG, cache_mask, saved_state, 0);
} else {
DPORT_SET_PERI_REG_BITS(DPORT_APP_CACHE_CTRL_REG, 1, 1, DPORT_APP_CACHE_ENABLE_S);
DPORT_SET_PERI_REG_BITS(DPORT_APP_CACHE_CTRL1_REG, cache_mask, saved_state, 0);
}
}
IRAM_ATTR bool spi_flash_cache_enabled()
{
bool result = (DPORT_REG_GET_BIT(DPORT_PRO_CACHE_CTRL_REG, DPORT_PRO_CACHE_ENABLE) != 0);
#if portNUM_PROCESSORS == 2
result = result && (DPORT_REG_GET_BIT(DPORT_APP_CACHE_CTRL_REG, DPORT_APP_CACHE_ENABLE) != 0);
#endif
return result;
}