Tools: Make easier the detection of the list of Python features

This commit is contained in:
Roland Dobai 2022-02-01 19:11:46 +01:00
parent a470ae224d
commit fddc73759e
12 changed files with 146 additions and 7 deletions

View File

@ -201,8 +201,6 @@
/tools/unit-test-app/ @esp-idf-codeowners/system @esp-idf-codeowners/tools
requirements.*.txt @esp-idf-codeowners/tools
# sort-order-reset
/components/**/test_apps/**/*.py @esp-idf-codeowners/ci @esp-idf-codeowners/tools

View File

@ -47,6 +47,7 @@
- "tools/cmake/**/*"
- "tools/kconfig_new/**/*"
- "tools/tools.json"
- "tools/requirements.json"
- "tools/ci/test_build_system*.sh"
.patterns-custom_test: &patterns-custom_test
@ -127,6 +128,10 @@
- "tools/test_idf_tools/**/*"
- "tools/install_util.py"
- "tools/requirements/*"
- "tools/requirements.json"
- "tools/requirements_schema.json"
- "tools/mkdfu.py"
- "tools/test_mkdfu/**/*"

View File

@ -111,6 +111,14 @@ repos:
language: python
files: \.(py|c|h|cpp|hpp|ld)$
require_serial: true
- id: check-requirement-files
name: Check requirement files
entry: tools/ci/check_requirement_files.py
additional_dependencies:
- 'jsonschema'
language: python
files: 'tools/requirements.+|tools/requirements/.+'
pass_filenames: false
- id: check-tools-files-patterns
name: Check tools dir files patterns
entry: tools/ci/check_tools_files_patterns.py

View File

@ -104,9 +104,9 @@ Any mirror server can be used provided the URL matches the ``github.com`` downlo
* ``check``: For each tool, checks whether the tool is available in the system path and in ``IDF_TOOLS_PATH``.
* ``install-python-env``: Create a Python virtual environment in the ``${IDF_TOOLS_PATH}/python_env`` directory and install there the required Python packages. An optional ``--features`` argument allows one to specify a comma-separated list of features. For each feature a requirements file must exist. For example, feature ``XY`` is a valid feature if ``${IDF_PATH}/requirements.XY.txt`` is an existing file with a list of Python packages to be installed. There is one mandatory ``core`` feature ensuring core functionality of ESP-IDF (build, flash, monitor, debug in console). There can be an arbitrary number of optional features. The selected list of features is stored in ``idf-env.json``. The requirement files contain a list of the desired Python packages to be installed and ``espidf.constraints.*.txt`` downloaded from https://dl.espressif.com and stored in ``${IDF_TOOLS_PATH}`` the package version requirements for a given ESP-IDF version.
* ``install-python-env``: Create a Python virtual environment in the ``${IDF_TOOLS_PATH}/python_env`` directory and install there the required Python packages. An optional ``--features`` argument allows one to specify a comma-separated list of features. For each feature a requirements file must exist. For example, feature ``XY`` is a valid feature if ``${IDF_PATH}/tools/requirements/requirements.XY.txt`` is an existing file with a list of Python packages to be installed. There is one mandatory ``core`` feature ensuring core functionality of ESP-IDF (build, flash, monitor, debug in console). There can be an arbitrary number of optional features. The selected list of features is stored in ``idf-env.json``. The requirement files contain a list of the desired Python packages to be installed and ``espidf.constraints.*.txt`` downloaded from https://dl.espressif.com and stored in ``${IDF_TOOLS_PATH}`` the package version requirements for a given ESP-IDF version.
* ``check-python-dependencies``: Checks if all required Python packages are installed. Packages from ``${IDF_PATH}/requirements.*.txt`` files selected by the feature list of ``idf-env.json`` are checked with the package versions specified in the ``espidf.constraints.*.txt`` file. The constraint file will be downloaded from https://dl.espressif.com if this step hasn't been done already in the last day.
* ``check-python-dependencies``: Checks if all required Python packages are installed. Packages from ``${IDF_PATH}/tools/requirements/requirements.*.txt`` files selected by the feature list of ``idf-env.json`` are checked with the package versions specified in the ``espidf.constraints.*.txt`` file. The constraint file will be downloaded from https://dl.espressif.com if this step hasn't been done already in the last day.
.. _idf-tools-install:

View File

@ -0,0 +1,58 @@
#!/usr/bin/env python
# SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD
# SPDX-License-Identifier: Apache-2.0
import json
import os
import re
from typing import Any
import jsonschema
IDF_PATH = os.environ['IDF_PATH']
JSON_PATH = os.path.join(IDF_PATH, 'tools', 'requirements.json')
SCHEMA_PATH = os.path.join(IDF_PATH, 'tools', 'requirements_schema.json')
REQ_DIR = os.path.join(IDF_PATH, 'tools', 'requirements')
RE_FEATURE = re.compile(r'requirements\.(\w+)\.txt')
def action_validate(req_obj: Any) -> None: # "Any" because we are checking this in this script
'''
Check that the parsed JSON object is valid according to the JSON schema provided
'''
with open(SCHEMA_PATH, 'r') as schema_file:
schema_json = json.load(schema_file)
jsonschema.validate(req_obj, schema_json)
def action_check_directory(req_obj: Any) -> None: # "Any" because we are checking this in this script
'''
Check that all directory items are listed in the JSON file
'''
features = set(d['name'] for d in req_obj['features'])
features_found = set()
for file_name in os.listdir(REQ_DIR):
m = re.match(RE_FEATURE, file_name)
if m:
if m.group(1) not in features:
raise RuntimeError(f'Cannot find a feature for {file_name} in {JSON_PATH}')
features_found.add(m.group(1))
else:
raise RuntimeError(f'{file_name} in {REQ_DIR} doesn\'t match the expected name')
features_not_found = features - features_found
if len(features_not_found) > 0:
raise RuntimeError(f'There are no requirements file in {REQ_DIR} for {features_not_found}')
def main() -> None:
with open(JSON_PATH, 'r') as f:
req_obj = json.load(f)
action_validate(req_obj)
action_check_directory(req_obj)
if __name__ == '__main__':
main()

View File

@ -60,6 +60,7 @@ tools/ci/check_executables.py
tools/ci/check_idf_version.sh
tools/ci/check_kconfigs.py
tools/ci/check_readme_links.py
tools/ci/check_requirement_files.py
tools/ci/check_rules_yml.py
tools/ci/check_soc_struct_headers.py
tools/ci/check_tools_files_patterns.py

View File

@ -1091,7 +1091,7 @@ def add_and_save_targets(targets_str): # type: (str) -> list[str]
def feature_to_requirements_path(feature): # type: (str) -> str
return os.path.join(global_idf_path or '', 'requirements.{}.txt'.format(feature))
return os.path.join(global_idf_path or '', 'tools', 'requirements', 'requirements.{}.txt'.format(feature))
def add_and_save_features(features_str): # type: (str) -> list[str]

17
tools/requirements.json Normal file
View File

@ -0,0 +1,17 @@
{
"version": 1,
"features": [
{
"name": "core",
"description": "Core packages necessary for ESP-IDF",
"optional": false,
"requirement_path": "tools/requirements/requirements.core.txt"
},
{
"name": "gdbgui",
"description": "Packages for supporting debugging from web browser",
"optional": true,
"requirement_path": "tools/requirements/requirements.gdbgui.txt"
}
]
}

View File

@ -0,0 +1,52 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"$id": "https://github.com/espressif/esp-idf/blob/master/tools/requirements_schema.json",
"type": "object",
"properties": {
"version": {
"type": "integer",
"description": "Metadata file version"
},
"features": {
"type": "array",
"description": "List of features",
"items": {
"$ref": "#/definitions/featInfo"
}
}
},
"required": [
"version",
"features"
],
"definitions": {
"featInfo": {
"type": "object",
"description": "Information about one feature",
"properties": {
"name" : {
"description": "Feature name",
"type": "string"
},
"description" : {
"description": "A short description of the feature",
"type": "string"
},
"optional": {
"description": "The feature is optional if the user can choose to not install it",
"type": "boolean"
},
"requirement_path": {
"description": "Path to the requirements file with Python packages",
"type": "string",
"pattern": "^tools/requirements/requirements\\..+\\.txt$"
}
},
"required": [
"name",
"optional",
"requirement_path"
]
}
}
}

View File

@ -18,8 +18,8 @@ IDF_PATH = os.environ.get('IDF_PATH', '../..')
TOOLS_DIR = os.environ.get('IDF_TOOLS_PATH') or os.path.expanduser(idf_tools.IDF_TOOLS_PATH_DEFAULT)
PYTHON_DIR = os.path.join(TOOLS_DIR, 'python_env')
REQ_SATISFIED = 'Python requirements are satisfied'
REQ_CORE = '- {}/requirements.core.txt'.format(IDF_PATH)
REQ_GDBGUI = '- {}/requirements.gdbgui.txt'.format(IDF_PATH)
REQ_CORE = '- {}'.format(os.path.join(IDF_PATH, 'tools', 'requirements', 'requirements.core.txt'))
REQ_GDBGUI = '- {}'.format(os.path.join(IDF_PATH, 'tools', 'requirements', 'requirements.gdbgui.txt'))
CONSTR = 'Constraint file: {}/espidf.constraints'.format(TOOLS_DIR)