From dfafa98253554e16f6de0fd9e952d23c1d82d388 Mon Sep 17 00:00:00 2001 From: Sergei Silnov Date: Mon, 18 Nov 2019 17:34:33 +0100 Subject: [PATCH] Use component manager if available as python package --- .gitignore | 6 ++++- .gitmodules | 1 + requirements.txt | 1 + tools/ci/test_build_system_cmake.sh | 15 +++++++++-- tools/cmake/component.cmake | 27 ++++++++++++++++++-- tools/cmake/project.cmake | 39 +++++++++++++++++++++++++++++ tools/idf.py | 11 ++++++++ tools/idf_py_actions/errors.py | 2 +- 8 files changed, 96 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index 56013d428c..029e060e1b 100644 --- a/.gitignore +++ b/.gitignore @@ -71,8 +71,12 @@ test_multi_heap_host .idea/ cmake-build-*/ -# Results for the checking of the Python coding style +# Results for the checking of the Python coding style and static analysis +.mypy_cache flake8_output.txt # ESP-IDF default build directory name build + +# lock files for examples and components +dependencies.lock \ No newline at end of file diff --git a/.gitmodules b/.gitmodules index 53315b72d0..b87c42cfdb 100644 --- a/.gitmodules +++ b/.gitmodules @@ -78,3 +78,4 @@ [submodule "components/esp_wifi/lib"] path = components/esp_wifi/lib url = ../../espressif/esp32-wifi-lib.git + diff --git a/requirements.txt b/requirements.txt index d80a7e1163..630349873c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,6 +11,7 @@ future>=0.15.2 cryptography>=2.1.4 pyparsing>=2.0.3,<2.4.0 pyelftools>=0.22 + # windows-curses are required in Windows command line but cannot be installed in MSYS2. A requirement like # "windows-curses; sys_platform == 'win32'" would want to install the package on both of them. There is no environment # marker for detecting MSYS2. So instead, a dummy custom package is used with "windows-curses" dependency for Windows diff --git a/tools/ci/test_build_system_cmake.sh b/tools/ci/test_build_system_cmake.sh index f809ee8962..fd825e9a0e 100755 --- a/tools/ci/test_build_system_cmake.sh +++ b/tools/ci/test_build_system_cmake.sh @@ -556,8 +556,8 @@ endmenu\n" >> ${IDF_PATH}/Kconfig idf.py -DSDKCONFIG_DEFAULTS="sdkconfig.defaults1;sdkconfig.defaults2" reconfigure > /dev/null grep "CONFIG_PARTITION_TABLE_OFFSET=0x10000" sdkconfig || failure "The define from sdkconfig.defaults1 should be in sdkconfig" grep "CONFIG_PARTITION_TABLE_TWO_OTA=y" sdkconfig || failure "The define from sdkconfig.defaults2 should be in sdkconfig" - rm sdkconfig.defaults1 - rm sdkconfig.defaults2 + rm sdkconfig.defaults1 sdkconfig.defaults2 sdkconfig + git checkout sdkconfig.defaults print_status "Supports git worktree" clean_build_dir @@ -577,6 +577,17 @@ endmenu\n" >> ${IDF_PATH}/Kconfig git checkout CMakeLists.txt rm -f log.txt + print_status "Compiles with dependencies delivered by component manager" + clean_build_dir + printf "\n#include \"test_component.h\"\n" >> main/main.c + printf "dependencies:\n test_component:\n path: test_component\n git: ${COMPONENT_MANAGER_TEST_REPO}\n" >> idf_project.yml + ! idf.py build || failure "Build should fail if dependencies are not installed" + pip install ${COMPONENT_MANAGER_REPO} + idf.py reconfigure build || failure "Build succeeds once requirements are installed" + pip uninstall -y idf_component_manager + rm idf_project.yml + git checkout main/main.c + print_status "All tests completed" if [ -n "${FAILURES}" ]; then echo "Some failures were detected:" diff --git a/tools/cmake/component.cmake b/tools/cmake/component.cmake index 5591ad44f4..b9b035bc50 100644 --- a/tools/cmake/component.cmake +++ b/tools/cmake/component.cmake @@ -215,13 +215,32 @@ function(__component_get_requirements) -D "COMPONENT_REQUIRES_FILE=${component_requires_file}" -P "${idf_path}/tools/cmake/scripts/component_get_requirements.cmake" RESULT_VARIABLE result - ERROR_VARIABLE error - ) + ERROR_VARIABLE error) if(NOT result EQUAL 0) message(FATAL_ERROR "${error}") endif() + idf_build_get_property(idf_component_manager IDF_COMPONENT_MANAGER) + if(idf_component_manager AND idf_component_manager EQUAL "1") + # Call for component manager once again to inject dependencies + idf_build_get_property(python PYTHON) + execute_process(COMMAND ${python} + "-m" + "idf_component_manager.prepare_components" + "--project_dir=${project_dir}" + "inject_requrements" + "--idf_path=${idf_path}" + "--build_dir=${build_dir}" + "--component_requires_file=${component_requires_file}" + RESULT_VARIABLE result + ERROR_VARIABLE error) + + if(NOT result EQUAL 0) + message(FATAL_ERROR "${error}") + endif() + endif() + include(${component_requires_file}) file(REMOVE ${build_properties_file}) @@ -409,6 +428,10 @@ function(idf_component_register) __component_check_target() __component_add_sources(sources) + # Add component manifest and lock files to list of dependencies + set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS "${COMPONENT_DIR}/idf_component.yml") + set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS "${COMPONENT_DIR}/dependencies.lock") + # Create the final target for the component. This target is the target that is # visible outside the build system. __component_get_target(component_target ${COMPONENT_ALIAS}) diff --git a/tools/cmake/project.cmake b/tools/cmake/project.cmake index d5a7254aa4..78d7ae30e7 100644 --- a/tools/cmake/project.cmake +++ b/tools/cmake/project.cmake @@ -169,6 +169,8 @@ function(__project_init components_var test_components_var) endif() endfunction() + idf_build_set_property(IDF_COMPONENT_MANAGER "$ENV{IDF_COMPONENT_MANAGER}") + # Add component directories to the build, given the component filters, exclusions # extra directories, etc. passed from the root CMakeLists.txt. if(COMPONENT_DIRS) @@ -179,6 +181,43 @@ function(__project_init components_var test_components_var) __project_component_dir(${component_dir}) endforeach() else() + # Add project manifest and lock file to the list of dependencies + set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS "${CMAKE_CURRENT_LIST_DIR}/idf_project.yml") + set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS "${CMAKE_CURRENT_LIST_DIR}/dependencies.lock") + + idf_build_get_property(idf_component_manager IDF_COMPONENT_MANAGER) + if(idf_component_manager) + if(idf_component_manager EQUAL "0") + message(VERBOSE "IDF Component manager was explicitly disabled by setting IDF_COMPONENT_MANAGER=0") + elseif(idf_component_manager EQUAL "1") + set(managed_components_list_file ${CMAKE_BINARY_DIR}/managed_components_list.temp.cmake) + + # Call for package manager to prepare remote dependencies + execute_process(COMMAND ${PYTHON} + "-m" + "idf_component_manager.prepare_components" + "--project_dir=${CMAKE_CURRENT_LIST_DIR}" + "prepare_dependencies" + "--managed_components_list_file=${managed_components_list_file}" + RESULT_VARIABLE result + ERROR_VARIABLE error) + + if(NOT result EQUAL 0) + message(FATAL_ERROR "${error}") + endif() + + # Include managed components + include(${managed_components_list_file}) + file(REMOVE ${managed_components_list_file}) + else() + message(WARNING "IDF_COMPONENT_MANAGER environment variable is set to unknown value " + "\"${idf_component_manager}\". If you want to use component manager set it to 1.") + endif() + elseif(EXISTS "${CMAKE_CURRENT_LIST_DIR}/idf_project.yml") + message(WARNING "\"idf_project.yml\" file is found in project directory, " + "but component manager is not enabled. Please set IDF_COMPONENT_MANAGER environment variable.") + endif() + # Look for components in the usual places: CMAKE_CURRENT_LIST_DIR/main, # CMAKE_CURRENT_LIST_DIR/components, and the extra component dirs if(EXISTS "${CMAKE_CURRENT_LIST_DIR}/main") diff --git a/tools/idf.py b/tools/idf.py index 6937eadcdf..39716210d0 100755 --- a/tools/idf.py +++ b/tools/idf.py @@ -649,6 +649,17 @@ def init_cli(verbose_output=None): if name.endswith('_ext'): extensions[name] = import_module(name) + # Load component manager if available and not explicitly disabled + if os.getenv('IDF_COMPONENT_MANAGER', None) != '0': + try: + from idf_component_manager import idf_extensions + + extensions['component_manager_ext'] = idf_extensions + os.environ['IDF_COMPONENT_MANAGER'] = '1' + + except ImportError: + pass + for name, extension in extensions.items(): try: all_actions = merge_action_lists(all_actions, extension.action_extensions(all_actions, project_dir)) diff --git a/tools/idf_py_actions/errors.py b/tools/idf_py_actions/errors.py index d3d5c9e941..20a5922958 100644 --- a/tools/idf_py_actions/errors.py +++ b/tools/idf_py_actions/errors.py @@ -1,6 +1,6 @@ class FatalError(RuntimeError): """ - Wrapper class for runtime errors that aren't caused by bugs in idf.py or the build proces.s + Wrapper class for runtime errors that aren't caused by bugs in idf.py or the build process. """ pass