From 1bd0c0f6f6431bc0ac7cf57551545b89fae9a9f0 Mon Sep 17 00:00:00 2001 From: Roman Leonov Date: Wed, 2 Aug 2023 19:15:39 +0200 Subject: [PATCH] feat(usb/host): add Kconfig for negotiation protocol configuration in UVC example --- examples/peripherals/usb/host/uvc/README.md | 238 ++++++++++-------- .../usb/host/uvc/main/Kconfig.projbuild | 101 +++++++- examples/peripherals/usb/host/uvc/main/main.c | 119 ++++++--- 3 files changed, 317 insertions(+), 141 deletions(-) diff --git a/examples/peripherals/usb/host/uvc/README.md b/examples/peripherals/usb/host/uvc/README.md index 3fd8e0cfaa..70e32a43b0 100644 --- a/examples/peripherals/usb/host/uvc/README.md +++ b/examples/peripherals/usb/host/uvc/README.md @@ -13,9 +13,9 @@ This example demonstrates how to: The example enumerates connected camera, negotiates selected resolution together with `FPS` and starts capturing video. `frame_callback` function is then invoked after receiving each frame. User can process received frame according to his needs. -Optionally, captured video can be visualized on computer with help of `player.py` script located in this example. +Optionally, captured video can be visualized on computer with help of `player.py` script located in this example. After setting `Enable streaming` menuconfig option, example will create TCP server upon start, and wait until `player.py` connects to server. -Once connection is established, example streams each received frame to computer for visualization. +Once connection is established, example streams each received frame to computer for visualization. **Notice** that `libuvc` selects highest possible `dwMaxPayloadTransferSize` by default, so example has to manually overwrite this value to 512 bytes (maximum transfer size supported by ESP32-S2/S3). @@ -32,7 +32,7 @@ This example requires any ESP32-S2 or ESP32-S3 with external PSRAM and exposed U Following configuration is needed for streaming video: -Open the project configuration menu (`idf.py menuconfig`). +Open the project configuration menu (`idf.py menuconfig`). In the `Example Connection Configuration` menu: @@ -45,6 +45,10 @@ In the `Example Configuration` menu: * Set the Example configuration * `Enable streaming` +* Select one of UVC Protocol Mode + * `Auto` + * `Custom` + Optional: If you need, change the other options according to your requirements. Additionally, `player.py` python script makes use of `opencv-python` and `numpy` packages, @@ -52,11 +56,31 @@ not included in `idf-env` environment. Run following commands to install: * `pip install opencv-python` * `pip install numpy` +#### UVC Protocol Mode: Auto + +When protocol mode set to Auto, the example tries to make three attempts to negotiatiate +the protocol with following parameters: + +1 Attempt: 640x480, 15 FPS, MJPEG +2 Attempt: 320x240, 30 FPS, MJPEG +3 Attempt: 320x240, first available FPS, MJPEG + +If all three attempts result in an error, the example displays the error message and +suggests to try another USB UVC Device. + +#### UVC Protocol Mode: Custom + +When protocol mode set to Custom, the example tries to negotiate protocol with +configured parameters: Attempts, Width, Heighs, FPS, Frame Coding format. +After all attemts result in an error, the example displays the error message and +suggests to try another USB UVC Device. + + ### Build and Flash Build the project and flash it to the board, then run the monitor tool to view the serial output: -Run `idf.py set-target esp32s2` to set target chip. +Run `idf.py set-target esp32s2` to set target chip. Run `idf.py -p PORT flash monitor` to build, flash and monitor the project. @@ -66,7 +90,7 @@ See the Getting Started Guide for all the steps to configure and use the ESP-IDF ## Known limitations -Having only Full Speed USB peripheral and hardware limited MPS (maximum packet size) to 512 bytes, ESP32-S2/S3 is capable of reading about 0.5 MB of data per second. When connected to Full Speed USB host, cameras normally provide resolution no larger than 640x480 pixels. +Having only Full Speed USB peripheral and hardware limited MPS (maximum packet size) to 512 bytes, ESP32-S2/S3 is capable of reading about 0.5 MB of data per second. When connected to Full Speed USB host, cameras normally provide resolution no larger than 640x480 pixels. Following two supported formats are the most common (both encoded in MJPEG): * 320x240 30 FPS * 640x480 15 FPS @@ -79,121 +103,115 @@ Following two supported formats are the most common (both encoded in MJPEG): ``` ... -Waiting for device -Device found -DEVICE CONFIGURATION (0c45:6340/ S) --- +I (1186) example: Waiting for USB UVC device connection ... +I (1606) example: Device found +DEVICE CONFIGURATION (0c45:6340/ S) --- Status: idle VideoControl: - bcdUVC: 0x0100 + bcdUVC: 0x0100 VideoStreaming(1): - bEndpointAddress: 129 - Formats: - MJPEGFormat(1) - bits per pixel: 0 - GUID: 4d4a5047000000000000000000000000 (MJPG) - default frame: 1 - aspect ratio: 0x0 - interlace flags: 00 - copy protect: 00 - FrameDescriptor(1) - capabilities: 00 - size: 640x480 - bit rate: 24576000-147456000 - max frame size: 614400 - default interval: 1/30 - interval[0]: 1/30 - interval[1]: 1/25 - interval[2]: 1/20 - interval[3]: 1/15 - interval[4]: 1/10 - interval[5]: 1/5 - FrameDescriptor(2) - capabilities: 00 - size: 352x288 - bit rate: 8110080-48660480 - max frame size: 202752 - default interval: 1/30 - interval[0]: 1/30 - interval[1]: 1/25 - interval[2]: 1/20 - interval[3]: 1/15 - interval[4]: 1/10 - interval[5]: 1/5 - FrameDescriptor(3) - capabilities: 00 - size: 320x240 - bit rate: 6144000-36864000 - max frame size: 153600 - default interval: 1/30 - interval[0]: 1/30 - interval[1]: 1/25 - interval[2]: 1/20 - interval[3]: 1/15 - interval[4]: 1/10 - interval[5]: 1/5 - FrameDescriptor(4) - capabilities: 00 - size: 176x144 - bit rate: 2027520-12165120 - max frame size: 50688 - default interval: 1/30 - interval[0]: 1/30 - interval[1]: 1/25 - interval[2]: 1/20 - interval[3]: 1/15 - interval[4]: 1/10 - interval[5]: 1/5 - FrameDescriptor(5) - capabilities: 00 - size: 160x120 - bit rate: 1536000-9216000 - max frame size: 38400 - default interval: 1/30 - interval[0]: 1/30 - interval[1]: 1/25 - interval[2]: 1/20 - interval[3]: 1/15 - interval[4]: 1/10 - interval[5]: 1/5 - StillFrameDescriptor - bEndPointAddress: 00 - wWidth(1) = 640 - wHeight(1) = 480 - wWidth(2) = 352 - wHeight(2) = 288 - wWidth(3) = 320 - wHeight(3) = 240 - wWidth(4) = 176 - wHeight(4) = 144 - wWidth(5) = 160 - wHeight(5) = 120 + bEndpointAddress: 129 + Formats: + MJPEGFormat(1) + bits per pixel: 0 + GUID: 4d4a5047000000000000000000000000 (MJPG) + default frame: 1 + aspect ratio: 0x0 + interlace flags: 00 + copy protect: 00 + FrameDescriptor(1) + capabilities: 00 + size: 640x480 + bit rate: 24576000-147456000 + max frame size: 614400 + default interval: 1/30 + interval[0]: 1/30 + interval[1]: 1/25 + interval[2]: 1/20 + interval[3]: 1/15 + interval[4]: 1/10 + interval[5]: 1/5 + FrameDescriptor(2) + capabilities: 00 + size: 352x288 + bit rate: 8110080-48660480 + max frame size: 202752 + default interval: 1/30 + interval[0]: 1/30 + interval[1]: 1/25 + interval[2]: 1/20 + interval[3]: 1/15 + interval[4]: 1/10 + interval[5]: 1/5 + FrameDescriptor(3) + capabilities: 00 + size: 320x240 + bit rate: 6144000-36864000 + max frame size: 153600 + default interval: 1/30 + interval[0]: 1/30 + interval[1]: 1/25 + interval[2]: 1/20 + interval[3]: 1/15 + interval[4]: 1/10 + interval[5]: 1/5 + FrameDescriptor(4) + capabilities: 00 + size: 176x144 + bit rate: 2027520-12165120 + max frame size: 50688 + default interval: 1/30 + interval[0]: 1/30 + interval[1]: 1/25 + interval[2]: 1/20 + interval[3]: 1/15 + interval[4]: 1/10 + interval[5]: 1/5 + FrameDescriptor(5) + capabilities: 00 + size: 160x120 + bit rate: 1536000-9216000 + max frame size: 38400 + default interval: 1/30 + interval[0]: 1/30 + interval[1]: 1/25 + interval[2]: 1/20 + interval[3]: 1/15 + interval[4]: 1/10 + interval[5]: 1/5 + StillFrameDescriptor + bEndPointAddress: 00 + wWidth(1) = 640 + wHeight(1) = 480 + wWidth(2) = 352 + wHeight(2) = 288 + wWidth(3) = 320 + wHeight(3) = 240 + wWidth(4) = 176 + wHeight(4) = 144 + wWidth(5) = 160 + wHeight(5) = 120 END DEVICE CONFIGURATION +I (1796) example: Negotiate streaming profile 640x480, fps 15 ... +I (1816) example: Negotiation complete. bmHint: 0001 bFormatIndex: 1 -bFrameIndex: 3 -dwFrameInterval: 333333 +bFrameIndex: 1 +dwFrameInterval: 666666 wKeyFrameRate: 0 wPFrameRate: 0 wCompQuality: 0 wCompWindowSize: 0 -wDelay: 20905 -dwMaxVideoFrameSize: 153600 +wDelay: 23469 +dwMaxVideoFrameSize: 614400 dwMaxPayloadTransferSize: 512 bInterfaceNumber: 1 -Streaming... -I (4801) example: fps: 35, bytes per second: 170480 -I (5821) example: fps: 34, bytes per second: 172448 -I (6841) example: fps: 34, bytes per second: 172448 -I (7871) example: fps: 34, bytes per second: 172448 -I (8891) example: fps: 34, bytes per second: 172448 -I (9921) example: fps: 35, bytes per second: 177520 -I (10941) example: fps: 34, bytes per second: 172448 -I (11961) example: fps: 34, bytes per second: 172448 -I (12991) example: fps: 34, bytes per second: 172448 -I (14011) example: fps: 34, bytes per second: 172448 -I (15041) example: fps: 34, bytes per second: 172448 -I (16061) example: fps: 34, bytes per second: 172448 -I (17081) example: fps: 34, bytes per second: 172448 -Done streaming. -UVC exited +I (1836) example: Streaming... +I (4016) example: fps: 10, bytes per second: 69920 + +... + +I (44916) example: fps: 9, bytes per second: 62928 +E (45626) USBH: Device 1 gone +I (45636) example: Done streaming. ``` diff --git a/examples/peripherals/usb/host/uvc/main/Kconfig.projbuild b/examples/peripherals/usb/host/uvc/main/Kconfig.projbuild index 2dac27bf66..33c2002c07 100644 --- a/examples/peripherals/usb/host/uvc/main/Kconfig.projbuild +++ b/examples/peripherals/usb/host/uvc/main/Kconfig.projbuild @@ -3,5 +3,104 @@ menu "Example Configuration" bool "Enable streaming" default n help - Enables streaming of captured video + Enables streaming of captured video. + + choice EXAMPLE_UVC_PROTOCOL_MODE + prompt "UVC Protocol mode" + default EXAMPLE_UVC_PROTOCOL_MODE_AUTO + + config EXAMPLE_UVC_PROTOCOL_MODE_AUTO + bool "Auto" + help + When protocol mode set to Auto, the example tries to make three attempts to negotiatiate + the protocol with following parameters: + 1 Attempt: 640x480, 15 FPS, MJPEG + 2 Attempt: 320x240, 30 FPS, MJPEG + 3 Attempt: 320x240, first available FPS, MJPEG + If all three attempts result in an error, the example displays the error message and + suggests to try another USB UVC Device. + + config EXAMPLE_UVC_PROTOCOL_MODE_CUSTOM + bool "Custom" + help + When protocol mode set to Custom, the example tries to negotiate protocol with + configured parameters: Attempts, Width, Heighs, FPS, Frame Coding format. + After all attemts result in an error, the example displays the error message and + suggests to try another USB UVC Device. + + endchoice + + menu "UVC Protocol parameters" + depends on EXAMPLE_UVC_PROTOCOL_MODE_CUSTOM + + config EXAMPLE_NEGOTIATION_ATTEMPTS + int "Attempts" + default 3 + help + Number of attempts to negotiate custom protocol parameters. + + config EXAMPLE_WIDTH_PARAM + int "Width resolution in pixels" + default 320 + help + Configure the negotiation width parameter during UVC device stream getting. + config EXAMPLE_HEIGHT_PARAM + int "Heigth resolution in pixels" + default 240 + help + Configure the negotiation height parameter during UVC device stream getting. + config EXAMPLE_FPS_PARAM + int "FPS" + default 30 + help + Configure the negotiation FPS parameter during UVC device stream getting. + Can be any value, available from the Frame Descriptor of the UVC device. + When 0 - negotiation accept the first rate available. + + choice EXAMPLE_NEGOTIATION_FORMAT + prompt "Frame coding format of the stream, transport-independent" + default UVC_FRAME_FORMAT_MJPEG + help + Configure the negotiation frame coding of stream. + + config UVC_FRAME_FORMAT_UNKNOWN + bool "Unknown" + config UVC_FRAME_FORMAT_UNCOMPRESSED + bool "Uncompressed" + config UVC_FRAME_FORMAT_COMPRESSED + bool "Compressed" + config UVC_FRAME_FORMAT_YUYV + bool "YUYV" + config UVC_FRAME_FORMAT_UYVY + bool "UYVY" + config UVC_FRAME_FORMAT_RGB + bool "RGB" + config UVC_FRAME_FORMAT_BGR + bool "BGR" + config UVC_FRAME_FORMAT_MJPEG + bool "MJPEG" + config UVC_FRAME_FORMAT_GRAY8 + bool "GRAY8" + config UVC_FRAME_FORMAT_GRAY16 + bool "GRAY16" + config UVC_FRAME_FORMAT_NV12 + bool "NV12" + + endchoice + + config EXAMPLE_FORMAT_PARAM + int + default 0 if UVC_FRAME_FORMAT_UNKNOWN + default 1 if UVC_FRAME_FORMAT_UNCOMPRESSED + default 2 if UVC_FRAME_FORMAT_COMPRESSED + default 3 if UVC_FRAME_FORMAT_YUYV + default 4 if UVC_FRAME_FORMAT_UYVY + default 5 if UVC_FRAME_FORMAT_RGB + default 6 if UVC_FRAME_FORMAT_BGR + default 7 if UVC_FRAME_FORMAT_MJPEG + default 9 if UVC_FRAME_FORMAT_GRAY8 + default 10 if UVC_FRAME_FORMAT_GRAY16 + default 17 if UVC_FRAME_FORMAT_NV12 + endmenu + endmenu diff --git a/examples/peripherals/usb/host/uvc/main/main.c b/examples/peripherals/usb/host/uvc/main/main.c index 9f87bdf412..9ed0761291 100644 --- a/examples/peripherals/usb/host/uvc/main/main.c +++ b/examples/peripherals/usb/host/uvc/main/main.c @@ -24,10 +24,29 @@ static const char *TAG = "example"; #define USB_DISCONNECT_PIN GPIO_NUM_0 -#define FPS 30 -#define WIDTH 320 -#define HEIGHT 240 -#define FORMAT UVC_COLOR_FORMAT_MJPEG // UVC_COLOR_FORMAT_YUYV +#if (CONFIG_EXAMPLE_UVC_PROTOCOL_MODE_AUTO) +#define EXAMPLE_UVC_PROTOCOL_AUTO_COUNT 3 +typedef struct { + enum uvc_frame_format format; + int width; + int height; + int fps; + const char* name; +} uvc_stream_profile_t; + +uvc_stream_profile_t uvc_stream_profiles[EXAMPLE_UVC_PROTOCOL_AUTO_COUNT] = { + {UVC_FRAME_FORMAT_MJPEG, 640, 480, 15, "640x480, fps 15"}, + {UVC_FRAME_FORMAT_MJPEG, 320, 240, 30, "320x240, fps 30"}, + {UVC_FRAME_FORMAT_MJPEG, 320, 240, 0, "320x240, any fps"} +}; +#endif // CONFIG_EXAMPLE_UVC_PROTOCOL_MODE_AUTO + +#if (CONFIG_EXAMPLE_UVC_PROTOCOL_MODE_CUSTOM) +#define FPS CONFIG_EXAMPLE_FPS_PARAM +#define WIDTH CONFIG_EXAMPLE_WIDTH_PARAM +#define HEIGHT CONFIG_EXAMPLE_HEIGHT_PARAM +#define FORMAT CONFIG_EXAMPLE_FORMAT_PARAM +#endif // CONFIG_EXAMPLE_UVC_PROTOCOL_MODE_CUSTOM // Attached camera can be filtered out based on (non-zero value of) PID, VID, SERIAL_NUMBER #define PID 0 @@ -147,13 +166,57 @@ static EventBits_t wait_for_event(EventBits_t event) return xEventGroupWaitBits(app_flags, event, pdTRUE, pdFALSE, portMAX_DELAY) & event; } +static uvc_error_t uvc_negotiate_stream_profile(uvc_device_handle_t *devh, + uvc_stream_ctrl_t *ctrl) +{ + uvc_error_t res; +#if (CONFIG_EXAMPLE_UVC_PROTOCOL_MODE_AUTO) + for (int idx = 0; idx < EXAMPLE_UVC_PROTOCOL_AUTO_COUNT; idx++) { + ESP_LOGI(TAG, "Negotiate streaming profile %s ...", uvc_stream_profiles[idx].name); + res = uvc_get_stream_ctrl_format_size(devh, + ctrl, + uvc_stream_profiles[idx].format, + uvc_stream_profiles[idx].width, + uvc_stream_profiles[idx].height, + uvc_stream_profiles[idx].fps); + + if (UVC_SUCCESS == res) { + break; // stream profile negotiated + } + sleep(1); + ESP_LOGE(TAG, "Negotiation failed with error %d.", res); + } +#endif // CONFIG_EXAMPLE_UVC_PROTOCOL_MODE_AUTO + +#if (CONFIG_EXAMPLE_UVC_PROTOCOL_MODE_CUSTOM) + int attempt = CONFIG_EXAMPLE_NEGOTIATION_ATTEMPTS; + while (attempt--) { + ESP_LOGI(TAG, "Negotiate streaming profile %dx%d, %d fps ...", WIDTH, HEIGHT, FPS); + res = uvc_get_stream_ctrl_format_size(devh, ctrl, FORMAT, WIDTH, HEIGHT, FPS); + if (UVC_SUCCESS == res) { + break; + } + sleep(1); + + ESP_LOGE(TAG, "Negotiation failed. Try again (%d) ...", attempt); + } +#endif // CONFIG_EXAMPLE_UVC_PROTOCOL_MODE_CUSTOM + + if (UVC_SUCCESS == res) { + ESP_LOGI(TAG, "Negotiation complete."); + } else { + ESP_LOGE(TAG, "Try another UVC USB device of change negotiation parameters."); + } + + return res; +} + int app_main(int argc, char **argv) { uvc_context_t *ctx; uvc_device_t *dev; uvc_device_handle_t *devh; uvc_stream_ctrl_t ctrl; - uvc_error_t res; app_flags = xEventGroupCreate(); assert(app_flags); @@ -183,12 +246,13 @@ int app_main(int argc, char **argv) do { - printf("Waiting for device\n"); + ESP_LOGI(TAG, "Waiting for USB UVC device connection ..."); wait_for_event(UVC_DEVICE_CONNECTED); UVC_CHECK(uvc_find_device(ctx, &dev, PID, VID, SERIAL_NUMBER)); - puts("Device found"); + ESP_LOGI(TAG, "Device found"); + // UVC Device open UVC_CHECK(uvc_open(dev, &devh)); // Uncomment to print configuration descriptor @@ -198,37 +262,32 @@ int app_main(int argc, char **argv) // Print known device information uvc_print_diag(devh, stderr); - // Negotiate stream profile - res = uvc_get_stream_ctrl_format_size(devh, &ctrl, FORMAT, WIDTH, HEIGHT, FPS); - while (res != UVC_SUCCESS) { - printf("Negotiating streaming format failed, trying again...\n"); - res = uvc_get_stream_ctrl_format_size(devh, &ctrl, FORMAT, WIDTH, HEIGHT, FPS); - sleep(1); + if (UVC_SUCCESS == uvc_negotiate_stream_profile(devh, &ctrl)) { + // dwMaxPayloadTransferSize has to be overwritten to MPS (maximum packet size) + // supported by ESP32-S2(S3), as libuvc selects the highest possible MPS by default. + ctrl.dwMaxPayloadTransferSize = 512; + + uvc_print_stream_ctrl(&ctrl, stderr); + + UVC_CHECK(uvc_start_streaming(devh, &ctrl, frame_callback, NULL, 0)); + ESP_LOGI(TAG, "Streaming..."); + + wait_for_event(UVC_DEVICE_DISCONNECTED); + + uvc_stop_streaming(devh); + ESP_LOGI(TAG, "Done streaming."); + } else { + wait_for_event(UVC_DEVICE_DISCONNECTED); } - - // dwMaxPayloadTransferSize has to be overwritten to MPS (maximum packet size) - // supported by ESP32-S2(S3), as libuvc selects the highest possible MPS by default. - ctrl.dwMaxPayloadTransferSize = 512; - - uvc_print_stream_ctrl(&ctrl, stderr); - - UVC_CHECK(uvc_start_streaming(devh, &ctrl, frame_callback, NULL, 0)); - puts("Streaming..."); - - wait_for_event(UVC_DEVICE_DISCONNECTED); - - uvc_stop_streaming(devh); - puts("Done streaming."); - + // UVC Device close uvc_close(devh); - } while (gpio_get_level(USB_DISCONNECT_PIN) != 0); tcp_server_close_when_done(); uvc_exit(ctx); - puts("UVC exited"); + ESP_LOGI(TAG, "UVC exited"); uninitialize_usb_host_lib();