From 251fb331d2e633afa06dd87e64105b9ef82e95d6 Mon Sep 17 00:00:00 2001 From: laokaiyao Date: Sat, 1 Jun 2024 01:50:25 +0800 Subject: [PATCH] feat(isp_ccm): support isp color correction matrix --- components/esp_driver_isp/CMakeLists.txt | 1 + .../esp_driver_isp/include/driver/isp.h | 1 + .../esp_driver_isp/include/driver/isp_af.h | 2 +- .../esp_driver_isp/include/driver/isp_awb.h | 2 +- .../esp_driver_isp/include/driver/isp_ccm.h | 70 +++++++++++++++++++ components/esp_driver_isp/src/isp_ccm.c | 55 +++++++++++++++ .../test_apps/isp/main/test_isp_driver.c | 35 ++++++++++ components/hal/esp32p4/include/hal/isp_ll.h | 35 ++++++++++ components/hal/include/hal/isp_hal.h | 15 ++++ components/hal/include/hal/isp_types.h | 9 +++ components/hal/isp_hal.c | 25 +++++++ .../esp32p4/include/soc/Kconfig.soc_caps.in | 8 +++ components/soc/esp32p4/include/soc/soc_caps.h | 6 +- docs/doxygen/Doxyfile_esp32p4 | 2 + docs/en/api-reference/peripherals/isp.rst | 33 +++++++++ 15 files changed, 295 insertions(+), 4 deletions(-) create mode 100644 components/esp_driver_isp/include/driver/isp_ccm.h create mode 100644 components/esp_driver_isp/src/isp_ccm.c diff --git a/components/esp_driver_isp/CMakeLists.txt b/components/esp_driver_isp/CMakeLists.txt index 7ce185791b..b3b04938f6 100644 --- a/components/esp_driver_isp/CMakeLists.txt +++ b/components/esp_driver_isp/CMakeLists.txt @@ -11,6 +11,7 @@ set(requires) if(CONFIG_SOC_ISP_SUPPORTED) list(APPEND srcs "src/isp_core.c" "src/isp_af.c" + "src/isp_ccm.c" "src/isp_awb.c") endif() diff --git a/components/esp_driver_isp/include/driver/isp.h b/components/esp_driver_isp/include/driver/isp.h index 38088eb938..7a36bb172c 100644 --- a/components/esp_driver_isp/include/driver/isp.h +++ b/components/esp_driver_isp/include/driver/isp.h @@ -15,3 +15,4 @@ #include "driver/isp_af.h" #include "driver/isp_awb.h" #include "driver/isp_bf.h" +#include "driver/isp_ccm.h" diff --git a/components/esp_driver_isp/include/driver/isp_af.h b/components/esp_driver_isp/include/driver/isp_af.h index 8a69698d24..67bfa58b52 100644 --- a/components/esp_driver_isp/include/driver/isp_af.h +++ b/components/esp_driver_isp/include/driver/isp_af.h @@ -178,7 +178,7 @@ typedef struct { /** * @brief Prototype of ISP AF Env detector event callback * - * @param[in] af_ctrlr ISP AF controller handle + * @param[in] af_ctrlr ISP AF controller handle * @param[in] edata ISP AF Env detector event data * @param[in] user_data User registered context, registered when in `esp_isp_af_env_detector_register_event_callbacks()` * diff --git a/components/esp_driver_isp/include/driver/isp_awb.h b/components/esp_driver_isp/include/driver/isp_awb.h index 64dab9e787..592b43c700 100644 --- a/components/esp_driver_isp/include/driver/isp_awb.h +++ b/components/esp_driver_isp/include/driver/isp_awb.h @@ -164,7 +164,7 @@ typedef struct { /** * @brief Prototype of ISP AWB event callback * - * @param[in] handle ISP AWB controller handle + * @param[in] awb_ctlr ISP AWB controller handle * @param[in] edata ISP AWB event data * @param[in] user_data User registered context, registered when in `esp_isp_awb_env_detector_register_event_callbacks()` * diff --git a/components/esp_driver_isp/include/driver/isp_ccm.h b/components/esp_driver_isp/include/driver/isp_ccm.h new file mode 100644 index 0000000000..451c69bcba --- /dev/null +++ b/components/esp_driver_isp/include/driver/isp_ccm.h @@ -0,0 +1,70 @@ +/* + * SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include +#include +#include "esp_err.h" +#include "driver/isp_types.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * @brief Color Correction Matrix configurations + * + */ +typedef struct { + float matrix[ISP_CCM_DIMENSION][ISP_CCM_DIMENSION]; /*!< The color correction matrix in float, range (-4.0, 4.0) */ + bool saturation; /*!< Whether to use saturation when the float data in the matrix is out of the range, + * For example, if one of the matrix data is 5.0, + * When saturation is true, and final value will be limited to 4.0, and won't rise error + * When saturation is false, `esp_isp_ccm_configure` will rise ESP_ERR_INVALID_ARG error + */ +} esp_isp_ccm_config_t; + +/** + * @brief ISP Color Correction Matrix (CCM) configuration + * + * @note This function is allowed to be called before or after `esp_isp_ccm_enable`, + * but it only takes effect until `esp_isp_ccm_enable` is called + * + * @param[in] proc Processor handle + * @param[in] ccm_cfg CCM configurations, set NULL to de-configure the ISP CCM + * + * @return + * - ESP_OK On success + * - ESP_ERR_INVALID_ARG If the combination of arguments is invalid + */ +esp_err_t esp_isp_ccm_configure(isp_proc_handle_t proc, const esp_isp_ccm_config_t *ccm_cfg); + +/** + * @brief Enable ISP CCM function + * + * @param[in] proc Processor handle + * + * @return + * - ESP_OK On success + * - ESP_ERR_INVALID_ARG If the combination of arguments is invalid. + */ +esp_err_t esp_isp_ccm_enable(isp_proc_handle_t proc); + +/** + * @brief Disable ISP CCM function + * + * @param[in] proc Processor handle + * + * @return + * - ESP_OK On success + * - ESP_ERR_INVALID_ARG If the combination of arguments is invalid. + */ +esp_err_t esp_isp_ccm_disable(isp_proc_handle_t proc); + +#ifdef __cplusplus +} +#endif diff --git a/components/esp_driver_isp/src/isp_ccm.c b/components/esp_driver_isp/src/isp_ccm.c new file mode 100644 index 0000000000..5fb4598488 --- /dev/null +++ b/components/esp_driver_isp/src/isp_ccm.c @@ -0,0 +1,55 @@ +/* + * SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include "esp_log.h" +#include "esp_check.h" +#include "freertos/FreeRTOS.h" +#include "driver/isp_core.h" +#include "driver/isp_ccm.h" +#include "esp_private/isp_private.h" + +static const char *TAG = "ISP_CCM"; + +/*--------------------------------------------------------------- + CCM +---------------------------------------------------------------*/ +esp_err_t esp_isp_ccm_configure(isp_proc_handle_t proc, const esp_isp_ccm_config_t *ccm_cfg) +{ + ESP_RETURN_ON_FALSE(proc && ccm_cfg, ESP_ERR_INVALID_ARG, TAG, "invalid argument: null pointer"); + + bool ret = true; + portENTER_CRITICAL(&proc->spinlock); + ret = isp_hal_ccm_set_matrix(&proc->hal, ccm_cfg->saturation, ccm_cfg->matrix); + portEXIT_CRITICAL(&proc->spinlock); + ESP_RETURN_ON_FALSE(ret, ESP_ERR_INVALID_ARG, TAG, "invalid argument: ccm matrix contain NaN or out of range"); + + return ESP_OK; +} + +esp_err_t esp_isp_ccm_enable(isp_proc_handle_t proc) +{ + ESP_RETURN_ON_FALSE(proc, ESP_ERR_INVALID_ARG, TAG, "invalid argument: null pointer"); + + portENTER_CRITICAL(&proc->spinlock); + isp_ll_ccm_clk_enable(proc->hal.hw, true); + isp_ll_ccm_enable(proc->hal.hw, true); + portEXIT_CRITICAL(&proc->spinlock); + + return ESP_OK; +} + +esp_err_t esp_isp_ccm_disable(isp_proc_handle_t proc) +{ + ESP_RETURN_ON_FALSE(proc, ESP_ERR_INVALID_ARG, TAG, "invalid argument: null pointer"); + + portENTER_CRITICAL(&proc->spinlock); + isp_ll_ccm_enable(proc->hal.hw, false); + isp_ll_ccm_clk_enable(proc->hal.hw, false); + portEXIT_CRITICAL(&proc->spinlock); + + return ESP_OK; +} diff --git a/components/esp_driver_isp/test_apps/isp/main/test_isp_driver.c b/components/esp_driver_isp/test_apps/isp/main/test_isp_driver.c index 3ed3467242..d2f660ea5d 100644 --- a/components/esp_driver_isp/test_apps/isp/main/test_isp_driver.c +++ b/components/esp_driver_isp/test_apps/isp/main/test_isp_driver.c @@ -118,3 +118,38 @@ TEST_CASE("ISP AWB driver basic function", "[isp]") TEST_ESP_OK(esp_isp_disable(isp_proc)); TEST_ESP_OK(esp_isp_del_processor(isp_proc)); } + +TEST_CASE("ISP CCM basic function", "[isp]") +{ + esp_isp_processor_cfg_t isp_config = { + .clk_hz = 80 * 1000 * 1000, + .input_data_source = ISP_INPUT_DATA_SOURCE_CSI, + .input_data_color_type = ISP_COLOR_RAW8, + .output_data_color_type = ISP_COLOR_RGB565, + }; + isp_proc_handle_t isp_proc = NULL; + TEST_ESP_OK(esp_isp_new_processor(&isp_config, &isp_proc)); + TEST_ESP_OK(esp_isp_enable(isp_proc)); + + esp_isp_ccm_config_t ccm_cfg = { + .matrix = { + {5.0, 0.0, 0.0}, + {0.0, 1.0, 0.0}, + {0.0, 0.0, 1.0} + }, + .saturation = false, + }; + // Out of range case + TEST_ESP_ERR(ESP_ERR_INVALID_ARG, esp_isp_ccm_configure(isp_proc, &ccm_cfg)); + // saturation case + ccm_cfg.saturation = true; + TEST_ESP_OK(esp_isp_ccm_configure(isp_proc, &ccm_cfg)); + TEST_ESP_OK(esp_isp_ccm_enable(isp_proc)); + // Allow to be called after enabled + ccm_cfg.matrix[0][0] = -1.1; + TEST_ESP_OK(esp_isp_ccm_configure(isp_proc, &ccm_cfg)); + TEST_ESP_OK(esp_isp_ccm_disable(isp_proc)); + + TEST_ESP_OK(esp_isp_disable(isp_proc)); + TEST_ESP_OK(esp_isp_del_processor(isp_proc)); +} diff --git a/components/hal/esp32p4/include/hal/isp_ll.h b/components/hal/esp32p4/include/hal/isp_ll.h index fbac2e908c..8792659fcb 100644 --- a/components/hal/esp32p4/include/hal/isp_ll.h +++ b/components/hal/esp32p4/include/hal/isp_ll.h @@ -100,6 +100,22 @@ typedef union { uint32_t val; } isp_ll_awb_rgb_ratio_t; +/*--------------------------------------------------------------- + CCM +---------------------------------------------------------------*/ +#define ISP_LL_CCM_MATRIX_INT_BITS (2) +#define ISP_LL_CCM_MATRIX_FRAC_BITS (10) +#define ISP_LL_CCM_MATRIX_TOT_BITS (ISP_LL_CCM_MATRIX_INT_BITS + ISP_LL_CCM_MATRIX_FRAC_BITS + 1) // including one sign bit + +typedef union { + struct { + uint32_t fraction: ISP_LL_AWB_RGB_RATIO_FRAC_BITS; + uint32_t integer: ISP_LL_AWB_RGB_RATIO_INT_BITS; + uint32_t sign: 1; + }; + uint32_t val; +} isp_ll_ccm_gain_t; + /** * @brief Env monitor mode */ @@ -790,6 +806,25 @@ static inline void isp_ll_ccm_enable(isp_dev_t *hw, bool enable) hw->cntl.ccm_en = enable; } +/** + * @brief Set the Color Correction Matrix + * + * @param[in] hw Hardware instance address + * @param[in] fixed_point_matrix Color Correction Matrix in fixed-point format + */ +static inline void isp_ll_ccm_set_matrix(isp_dev_t *hw, isp_ll_ccm_gain_t fixed_point_matrix[ISP_CCM_DIMENSION][ISP_CCM_DIMENSION]) +{ + hw->ccm_coef0.ccm_rr = fixed_point_matrix[0][0].val; + hw->ccm_coef0.ccm_rg = fixed_point_matrix[0][1].val; + hw->ccm_coef1.ccm_rb = fixed_point_matrix[0][2].val; + hw->ccm_coef1.ccm_gr = fixed_point_matrix[1][0].val; + hw->ccm_coef3.ccm_gg = fixed_point_matrix[1][1].val; + hw->ccm_coef3.ccm_gb = fixed_point_matrix[1][2].val; + hw->ccm_coef4.ccm_br = fixed_point_matrix[2][0].val; + hw->ccm_coef4.ccm_bg = fixed_point_matrix[2][1].val; + hw->ccm_coef5.ccm_bb = fixed_point_matrix[2][2].val; +} + /*--------------------------------------------------------------- Color ---------------------------------------------------------------*/ diff --git a/components/hal/include/hal/isp_hal.h b/components/hal/include/hal/isp_hal.h index fd291e7ec5..d8566a9ec4 100644 --- a/components/hal/include/hal/isp_hal.h +++ b/components/hal/include/hal/isp_hal.h @@ -88,6 +88,21 @@ uint32_t isp_hal_check_clear_intr_event(const isp_hal_context_t *hal, uint32_t m */ void isp_hal_bf_config(isp_hal_context_t *hal, isp_hal_bf_cfg_t *config); +/*--------------------------------------------------------------- + Color Correction Matrix +---------------------------------------------------------------*/ +/** + * @brief Set Color Correction Matrix + * + * @param[in] hal Context of the HAL layer + * @param[in] saturation Whether to enable saturation when float data overflow + * @param[in] flt_matrix 3x3 RGB correction matrix + * @return + * - true Set success + * - false Invalid argument + */ +bool isp_hal_ccm_set_matrix(const isp_hal_context_t *hal, bool saturation, const float flt_matrix[ISP_CCM_DIMENSION][ISP_CCM_DIMENSION]); + /*--------------------------------------------------------------- AWB ---------------------------------------------------------------*/ diff --git a/components/hal/include/hal/isp_types.h b/components/hal/include/hal/isp_types.h index a664d8001a..d36b8a09fc 100644 --- a/components/hal/include/hal/isp_types.h +++ b/components/hal/include/hal/isp_types.h @@ -100,6 +100,15 @@ typedef enum { ISP_BF_EDGE_PADDING_MODE_CUSTOM_DATA, ///< Fill BF edge padding data with custom pixel data } isp_bf_edge_padding_mode_t; +/*--------------------------------------------------------------- + CCM +---------------------------------------------------------------*/ +#if SOC_ISP_CCM_SUPPORTED +#define ISP_CCM_DIMENSION SOC_ISP_CCM_DIMENSION ///< ISP Color Correction Matrix dimension +#else +#define ISP_CCM_DIMENSION 0 ///< Not support CCM +#endif + /*--------------------------------------------------------------- AWB ---------------------------------------------------------------*/ diff --git a/components/hal/isp_hal.c b/components/hal/isp_hal.c index 7d33af5c5d..37e69fd772 100644 --- a/components/hal/isp_hal.c +++ b/components/hal/isp_hal.c @@ -70,6 +70,31 @@ uint32_t isp_hal_check_clear_intr_event(const isp_hal_context_t *hal, uint32_t m return triggered_events; } + +/*--------------------------------------------------------------- + Color Correction Matrix +---------------------------------------------------------------*/ +bool isp_hal_ccm_set_matrix(const isp_hal_context_t *hal, bool saturation, const float flt_matrix[ISP_CCM_DIMENSION][ISP_CCM_DIMENSION]) +{ + isp_ll_ccm_gain_t fp_matrix[ISP_CCM_DIMENSION][ISP_CCM_DIMENSION] = {}; + hal_utils_fixed_point_t fp_cfg = { + .int_bit = ISP_LL_CCM_MATRIX_INT_BITS, + .frac_bit = ISP_LL_CCM_MATRIX_FRAC_BITS, + .saturation = saturation, + }; + int err_level = saturation ? -1 : 0; + /* Transfer the float type to fixed point */ + for (int i = 0; i < ISP_CCM_DIMENSION; i++) { + for (int j = 0; j < ISP_CCM_DIMENSION; j++) { + if (hal_utils_float_to_fixed_point_32b(flt_matrix[i][j], &fp_cfg, &fp_matrix[i][j].val) < err_level) { + return false; + } + } + } + isp_ll_ccm_set_matrix(hal->hw, fp_matrix); + return true; +} + /*--------------------------------------------------------------- AWB ---------------------------------------------------------------*/ diff --git a/components/soc/esp32p4/include/soc/Kconfig.soc_caps.in b/components/soc/esp32p4/include/soc/Kconfig.soc_caps.in index 75ef779745..675d627a55 100644 --- a/components/soc/esp32p4/include/soc/Kconfig.soc_caps.in +++ b/components/soc/esp32p4/include/soc/Kconfig.soc_caps.in @@ -783,6 +783,10 @@ config SOC_ISP_BF_SUPPORTED bool default y +config SOC_ISP_CCM_SUPPORTED + bool + default y + config SOC_ISP_DVP_SUPPORTED bool default y @@ -815,6 +819,10 @@ config SOC_ISP_BF_TEMPLATE_Y_NUMS int default 3 +config SOC_ISP_CCM_DIMENSION + int + default 3 + config SOC_ISP_DVP_DATA_WIDTH_MAX int default 16 diff --git a/components/soc/esp32p4/include/soc/soc_caps.h b/components/soc/esp32p4/include/soc/soc_caps.h index d28c597697..a6ccd7c8b0 100644 --- a/components/soc/esp32p4/include/soc/soc_caps.h +++ b/components/soc/esp32p4/include/soc/soc_caps.h @@ -317,6 +317,7 @@ /*-------------------------- ISP CAPS ----------------------------------------*/ #define SOC_ISP_BF_SUPPORTED 1 +#define SOC_ISP_CCM_SUPPORTED 1 #define SOC_ISP_DVP_SUPPORTED 1 #define SOC_ISP_NUMS 1U @@ -326,6 +327,7 @@ #define SOC_ISP_SHARE_CSI_BRG 1 #define SOC_ISP_BF_TEMPLATE_X_NUMS 3 #define SOC_ISP_BF_TEMPLATE_Y_NUMS 3 +#define SOC_ISP_CCM_DIMENSION 3 #define SOC_ISP_DVP_DATA_WIDTH_MAX 16 /*-------------------------- LEDC CAPS ---------------------------------------*/ @@ -546,8 +548,8 @@ /*-------------------------- TOUCH SENSOR CAPS -------------------------------*/ #define SOC_TOUCH_SENSOR_VERSION (3) // Hardware version of touch sensor #define SOC_TOUCH_SENSOR_NUM (14) // Touch available channel number. Actually there are 15 Touch channels, but channel 14 is not pinned out, limit to 14 channels -#define SOC_TOUCH_PROXIMITY_CHANNEL_NUM (3) // Sopport touch proximity channel number. -#define SOC_TOUCH_PROXIMITY_MEAS_DONE_SUPPORTED (1) // Sopport touch proximity channel measure done interrupt type. +#define SOC_TOUCH_PROXIMITY_CHANNEL_NUM (3) // Support touch proximity channel number. +#define SOC_TOUCH_PROXIMITY_MEAS_DONE_SUPPORTED (1) // Support touch proximity channel measure done interrupt type. #define SOC_TOUCH_SAMPLER_NUM (3) // The sampler number in total, each sampler can be used to sample on one frequency /*-------------------------- TWAI CAPS ---------------------------------------*/ diff --git a/docs/doxygen/Doxyfile_esp32p4 b/docs/doxygen/Doxyfile_esp32p4 index 32f0c5817b..0003223864 100644 --- a/docs/doxygen/Doxyfile_esp32p4 +++ b/docs/doxygen/Doxyfile_esp32p4 @@ -22,6 +22,8 @@ INPUT += \ $(PROJECT_PATH)/components/esp_driver_isp/include/driver/isp.h \ $(PROJECT_PATH)/components/esp_driver_isp/include/driver/isp_types.h \ $(PROJECT_PATH)/components/esp_driver_isp/include/driver/isp_af.h \ + $(PROJECT_PATH)/components/esp_driver_isp/include/driver/isp_awb.h \ + $(PROJECT_PATH)/components/esp_driver_isp/include/driver/isp_ccm.h \ $(PROJECT_PATH)/components/esp_driver_jpeg/include/driver/jpeg_decode.h \ $(PROJECT_PATH)/components/esp_driver_jpeg/include/driver/jpeg_encode.h \ $(PROJECT_PATH)/components/esp_driver_ppa/include/driver/ppa.h \ diff --git a/docs/en/api-reference/peripherals/isp.rst b/docs/en/api-reference/peripherals/isp.rst index 15220418a2..ed4e725085 100644 --- a/docs/en/api-reference/peripherals/isp.rst +++ b/docs/en/api-reference/peripherals/isp.rst @@ -15,8 +15,10 @@ Terminology - RAW: Unprocessed data directly output from an image sensor, typically divided into R, Gr, Gb, and B four channels classified into RAW8, RAW10, RAW12, etc., based on bit width - RGB: Colored image format composed of red, green, and blue colors classified into RGB888, RGB565, etc., based on the bit width of each color - YUV: Colored image format composed of luminance and chrominance classified into YUV444, YUV422, YUV420, etc., based on the data arrangement + - BF: Bayer Domain Filter - AF: Auto-focus - AWB: Auto-white balance + - CCM: Color correction matrix ISP Pipeline ------------ @@ -59,6 +61,7 @@ The ISP driver offers following services: - `Enable and disable ISP processor <#isp-enable-disable>`__ - covers how to enable and disable an ISP processor. - `Get AF statistics in one shot or continuous way <#isp-af-statistics>`__ - covers how to get AF statistics one-shot or continuously. - `Get AWB statistics in one shot or continuous way <#isp-awb-statistics>`__ - covers how to get AWB white patches statistics one-shot or continuously. +- `Configure CCM <#isp-ccm-config>`__ - covers how to config the Color Correction Matrix. - `Register callback <#isp-callback>`__ - covers how to hook user specific code to ISP driver event callback function. - `Thread Safety <#isp-thread-safety>`__ - lists which APIs are guaranteed to be thread safe by the driver. - `Kconfig Options <#isp-kconfig-options>`__ - lists the supported Kconfig options that can bring different effects to the driver. @@ -304,6 +307,36 @@ Note that if you want to use the continuous statistics, you need to register the /* Delete the awb controller and free the resources */ ESP_ERROR_CHECK(esp_isp_del_awb_controller(awb_ctlr)); +.. _isp-ccm-config: + +Configure CCM +^^^^^^^^^^^^^ + +Color Correction Matrix can scale the color ratio of RGB888 pixels. It can be used for adjusting the image color via some algorithms, for example, used for white balance by inputting the AWB computed result, or used as a Filter with some filter algorithms. + +To adjust the color correction matrix, you can refer to the following code: + +.. code-block:: c + + // ... + // Configure CCM + esp_isp_ccm_config_t ccm_cfg = { + .matrix = { + 1.0, 0.0, 0.0, + 0.0, 1.0, 0.0, + 0.0, 0.0, 1.0 + }, + .saturation = false, + }; + ESP_ERROR_CHECK(esp_isp_ccm_configure(isp_proc, &ccm_cfg)); + // The configured CCM will be applied to the image once the CCM module is enabled + ESP_ERROR_CHECK(esp_isp_ccm_enable(isp_proc)); + // CCM can also be configured after it is enabled + ccm_cfg.matrix[0][0] = 2.0; + ESP_ERROR_CHECK(esp_isp_ccm_configure(isp_proc, &ccm_cfg)); + // Disable CCM if no longer needed + ESP_ERROR_CHECK(esp_isp_ccm_disable(isp_proc)); + .. _isp-callback: Register Event Callbacks