Add mypy check to pre-commit-config

This commit is contained in:
Tomas Sebestik 2021-02-01 09:18:16 +01:00 committed by bot
parent daf429c67f
commit a22597a2d6
10 changed files with 371 additions and 15 deletions

26
.mypy.ini Normal file
View File

@ -0,0 +1,26 @@
[mypy]
# Specifies the Python version used to parse and check the target program
python_version = 3.9
# Disallows defining functions without type annotations or with incomplete type annotations
# True => enforce type annotation in all function definitions
disallow_untyped_defs = True
# Shows a warning when returning a value with type Any from a function declared with a non- Any return type
warn_return_any = True
# Shows errors for missing return statements on some execution paths
warn_no_return = True
# Suppress error messages about imports that cannot be resolved
# True => ignore all import errors
ignore_missing_imports = True
# Disallows defining functions with incomplete type annotations
disallow_incomplete_defs = False
# Directs what to do with imports when the imported module is found as a .py file and not part of the files,
# modules and packages provided on the command line.
# SKIP -> mypy checks only single file, not included imports
follow_imports = skip

View File

@ -91,3 +91,9 @@ repos:
pass_filenames: false pass_filenames: false
additional_dependencies: additional_dependencies:
- PyYAML == 5.3.1 - PyYAML == 5.3.1
- id: mypy-check
name: Check type annotations in python files
entry: tools/ci/check_type_comments.py
additional_dependencies: ['mypy==0.800', 'mypy-extensions==0.4.3']
language: python
types: [python]

View File

@ -16,6 +16,12 @@ from corefile.gdb import EspGDB
from corefile.loader import ESPCoreDumpFileLoader, ESPCoreDumpFlashLoader from corefile.loader import ESPCoreDumpFileLoader, ESPCoreDumpFlashLoader
from pygdbmi.gdbcontroller import DEFAULT_GDB_TIMEOUT_SEC from pygdbmi.gdbcontroller import DEFAULT_GDB_TIMEOUT_SEC
try:
from typing import Optional, Tuple
except ImportError:
# Only used for type annotations
pass
IDF_PATH = os.getenv('IDF_PATH') IDF_PATH = os.getenv('IDF_PATH')
if not IDF_PATH: if not IDF_PATH:
sys.stderr.write('IDF_PATH is not found! Set proper IDF_PATH in environment.\n') sys.stderr.write('IDF_PATH is not found! Set proper IDF_PATH in environment.\n')
@ -34,7 +40,7 @@ else:
CLOSE_FDS = True CLOSE_FDS = True
def load_aux_elf(elf_path): # type: (str) -> (ElfFile, str) def load_aux_elf(elf_path): # type: (str) -> Tuple[ElfFile, str]
""" """
Loads auxiliary ELF file and composes GDB command to read its symbols. Loads auxiliary ELF file and composes GDB command to read its symbols.
""" """
@ -48,7 +54,7 @@ def load_aux_elf(elf_path): # type: (str) -> (ElfFile, str)
return elf, sym_cmd return elf, sym_cmd
def core_prepare(): def core_prepare(): # type: () -> Tuple[Optional[str], ESPCoreDumpFlashLoader]
loader = None loader = None
core_filename = None core_filename = None
if not args.core: if not args.core:
@ -72,7 +78,7 @@ def core_prepare():
return core_filename, loader return core_filename, loader
def dbg_corefile(): def dbg_corefile(): # type: () -> None
""" """
Command to load core dump from file or flash and run GDB debug session with it Command to load core dump from file or flash and run GDB debug session with it
""" """
@ -91,7 +97,7 @@ def dbg_corefile():
print('Done!') print('Done!')
def info_corefile(): def info_corefile(): # type: () -> None
""" """
Command to load core dump from file or flash and print it's data in user friendly form Command to load core dump from file or flash and print it's data in user friendly form
""" """

View File

@ -1,4 +1,5 @@
#!/usr/bin/env python #!/usr/bin/env python
# coding=utf-8 # coding=utf-8
# #
# ESP-IDF helper script to build multiple applications. Consumes the input of find_apps.py. # ESP-IDF helper script to build multiple applications. Consumes the input of find_apps.py.
@ -12,7 +13,7 @@ from find_build_apps import BUILD_SYSTEMS, BuildError, BuildItem, setup_logging
from find_build_apps.common import SIZE_JSON_FN, rmdir from find_build_apps.common import SIZE_JSON_FN, rmdir
def main(): def main(): # type: () -> None
parser = argparse.ArgumentParser(description='ESP-IDF app builder') parser = argparse.ArgumentParser(description='ESP-IDF app builder')
parser.add_argument( parser.add_argument(
'-v', '-v',

94
tools/ci/check_type_comments.py Executable file
View File

@ -0,0 +1,94 @@
#!/usr/bin/env python
#
# Copyright 2021 Espressif Systems (Shanghai) CO LTD
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import argparse
import re
import subprocess
from sys import exit
try:
from typing import List
except ImportError:
# Only used for type annotations
pass
IGNORE_LIST_MYPY = 'tools/ci/mypy_ignore_list.txt'
COMMENT_RE = re.compile(r'#\s+type:\s+\(', re.MULTILINE) # REGEX: "# type: ("
def type_comments_in_file(file_name): # type: (str) -> bool
""" Check if type annotations (type comments) are present in the file """
with open(file_name, 'r') as f:
return bool(COMMENT_RE.search(f.read()))
def types_valid(file_name): # type: (str) -> bool
""" Run Mypy check on the given file, return TRUE if Mypy check passes """
mypy_exit_code = subprocess.call(('mypy %s' % file_name), shell=True)
return not bool(mypy_exit_code)
def check_files(files): # type: (List[str]) -> List[str]
"""
Check files for type annotatins:
- new python file -> run Mypy check
- existed file updated by type annotations -> run Mypy check, remove from ignore list
- existed file updated, but no type annotations added -> skip Mypy check on file
"""
type_issues = []
with open(IGNORE_LIST_MYPY, 'r') as f:
ignore_list = [item.strip() for item in f.readlines()]
updated_list = ignore_list.copy()
for file_name in files:
if file_name in ignore_list:
if type_comments_in_file(file_name):
if types_valid(file_name):
updated_list.remove(file_name)
print('\33[93m\n File %s removed automatically from ignore list - run commit again! \n\33[0m' % file_name)
else:
type_issues.append(file_name)
else:
if not types_valid(file_name):
type_issues.append(file_name)
ignore_list.sort()
updated_list.sort()
if updated_list != ignore_list:
with open(IGNORE_LIST_MYPY, 'w') as f:
for item in updated_list:
f.write('%s\n' % item)
return type_issues
def main(): # type: () -> None
parser = argparse.ArgumentParser()
parser.add_argument('filenames', nargs='*', help='Filenames to check.')
args = parser.parse_args()
type_issues = check_files(args.filenames)
if type_issues:
print('mypy check failed for:')
for file_name in type_issues:
print('\t', file_name)
exit(1)
if __name__ == '__main__':
main()

View File

@ -24,8 +24,14 @@ import subprocess
from gitlab_api import Gitlab from gitlab_api import Gitlab
try:
from typing import Any, Union
except ImportError:
# Only used for type annotations
pass
def _get_mr_obj(source_branch):
def _get_mr_obj(source_branch): # type: (str) -> Union[Gitlab, None]
if not source_branch: if not source_branch:
return None return None
gl = Gitlab(os.getenv('CI_PROJECT_ID', 'espressif/esp-idf')) gl = Gitlab(os.getenv('CI_PROJECT_ID', 'espressif/esp-idf'))
@ -46,7 +52,7 @@ def get_mr_iid(source_branch): # type: (str) -> str
return str(mr.iid) return str(mr.iid)
def get_mr_changed_files(source_branch): def get_mr_changed_files(source_branch): # type: (str) -> Any
mr = _get_mr_obj(source_branch) mr = _get_mr_obj(source_branch)
if not mr: if not mr:
return '' return ''
@ -55,7 +61,7 @@ def get_mr_changed_files(source_branch):
'origin/{}...origin/{}'.format(mr.target_branch, source_branch)]).decode('utf8') 'origin/{}...origin/{}'.format(mr.target_branch, source_branch)]).decode('utf8')
def get_mr_commits(source_branch): def get_mr_commits(source_branch): # type: (str) -> str
mr = _get_mr_obj(source_branch) mr = _get_mr_obj(source_branch)
if not mr: if not mr:
return '' return ''

View File

@ -48,6 +48,7 @@ tools/ci/check_readme_links.py
tools/ci/check_rom_apis.sh tools/ci/check_rom_apis.sh
tools/ci/check_rules_yml.py tools/ci/check_rules_yml.py
tools/ci/check_tools_files_patterns.py tools/ci/check_tools_files_patterns.py
tools/ci/check_type_comments.py
tools/ci/check_ut_cmake_make.sh tools/ci/check_ut_cmake_make.sh
tools/ci/checkout_project_ref.py tools/ci/checkout_project_ref.py
tools/ci/deploy_docs.py tools/ci/deploy_docs.py

View File

@ -0,0 +1,216 @@
components/app_update/otatool.py
components/efuse/efuse_table_gen.py
components/efuse/test_efuse_host/efuse_tests.py
components/esp32s2/test/gen_digital_signature_tests.py
components/esp_local_ctrl/python/esp_local_ctrl_pb2.py
components/espcoredump/corefile/elf.py
components/espcoredump/corefile/gdb.py
components/espcoredump/corefile/loader.py
components/espcoredump/corefile/xtensa.py
components/espcoredump/test/test_espcoredump.py
components/lwip/weekend_test/net_suite_test.py
components/mbedtls/esp_crt_bundle/gen_crt_bundle.py
components/mbedtls/esp_crt_bundle/test_gen_crt_bundle/test_gen_crt_bundle.py
components/mbedtls/test/test_aes.c
components/mbedtls/test/test_aes_gcm.c
components/mqtt/weekend_test/mqtt_publish_test.py
components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py
components/partition_table/gen_empty_partition.py
components/partition_table/gen_esp32part.py
components/partition_table/parttool.py
components/partition_table/test_gen_esp32part_host/gen_esp32part_tests.py
components/protocomm/python/constants_pb2.py
components/protocomm/python/sec0_pb2.py
components/protocomm/python/sec1_pb2.py
components/protocomm/python/session_pb2.py
components/spiffs/spiffsgen.py
components/ulp/esp32ulp_mapgen.py
components/wifi_provisioning/python/wifi_config_pb2.py
components/wifi_provisioning/python/wifi_constants_pb2.py
components/wifi_provisioning/python/wifi_scan_pb2.py
components/xtensa/trax/traceparse.py
docs/build_docs.py
docs/conf_common.py
docs/en/conf.py
docs/extensions/html_redirects.py
docs/extensions/list_filter.py
docs/extensions/toctree_filter.py
docs/generate_chart.py
docs/get_github_rev.py
docs/idf_extensions/esp_err_definitions.py
docs/idf_extensions/exclude_docs.py
docs/idf_extensions/format_idf_target.py
docs/idf_extensions/gen_defines.py
docs/idf_extensions/gen_idf_tools_links.py
docs/idf_extensions/gen_toolchain_links.py
docs/idf_extensions/gen_version_specific_includes.py
docs/idf_extensions/include_build_file.py
docs/idf_extensions/kconfig_reference.py
docs/idf_extensions/latex_builder.py
docs/idf_extensions/link_roles.py
docs/idf_extensions/run_doxygen.py
docs/idf_extensions/util.py
docs/sanitize_version.py
docs/test/en/conf.py
docs/test/test_docs.py
docs/test/test_sphinx_idf_extensions.py
docs/zh_CN/conf.py
examples/bluetooth/nimble/blecent/blecent_test.py
examples/bluetooth/nimble/blehr/blehr_test.py
examples/bluetooth/nimble/bleprph/bleprph_test.py
examples/get-started/blink/example_test.py
examples/get-started/hello_world/example_test.py
examples/peripherals/gpio/generic_gpio/example_test.py
examples/peripherals/sdio/sdio_test.py
examples/protocols/esp_local_ctrl/scripts/esp_local_ctrl.py
examples/protocols/esp_local_ctrl/scripts/proto.py
examples/protocols/http_server/advanced_tests/http_server_advanced_test.py
examples/protocols/http_server/advanced_tests/scripts/test.py
examples/protocols/http_server/persistent_sockets/http_server_persistence_test.py
examples/protocols/http_server/simple/http_server_simple_test.py
examples/protocols/http_server/ws_echo_server/ws_server_example_test.py
examples/protocols/mdns/mdns_example_test.py
examples/protocols/modbus/serial/example_test.py
examples/protocols/modbus/tcp/example_test.py
examples/protocols/mqtt/ssl/mqtt_ssl_example_test.py
examples/protocols/mqtt/ssl_ds/configure_ds.py
examples/protocols/pppos_client/example_test.py
examples/protocols/sockets/tcp_client/example_test.py
examples/protocols/sockets/udp_client/example_test.py
examples/protocols/websocket/example_test.py
examples/provisioning/legacy/ble_prov/ble_prov_test.py
examples/provisioning/legacy/custom_config/components/custom_provisioning/python/custom_config_pb2.py
examples/provisioning/legacy/softap_prov/softap_prov_test.py
examples/provisioning/wifi_prov_mgr/wifi_prov_mgr_test.py
examples/storage/parttool/parttool_example.py
examples/system/ota/advanced_https_ota/example_test.py
examples/system/ota/native_ota_example/example_test.py
examples/system/ota/otatool/get_running_partition.py
examples/system/ota/otatool/otatool_example.py
examples/system/ota/simple_ota_example/example_test.py
tools/ble/lib_ble_client.py
tools/ble/lib_gap.py
tools/ble/lib_gatt.py
tools/check_python_dependencies.py
tools/check_term.py
tools/ci/apply_bot_filter.py
tools/ci/check_artifacts_expire_time.py
tools/ci/check_build_warnings.py
tools/ci/check_callgraph.py
tools/ci/check_codeowners.py
tools/ci/check_deprecated_kconfigs.py
tools/ci/check_executables.py
tools/ci/check_kconfigs.py
tools/ci/check_public_headers.py
tools/ci/check_readme_links.py
tools/ci/checkout_project_ref.py
tools/ci/ci_fetch_submodule.py
tools/ci/deploy_docs.py
tools/ci/envsubst.py
tools/ci/idf_ci_utils.py
tools/ci/normalize_clangtidy_path.py
tools/ci/python_packages/gitlab_api.py
tools/ci/python_packages/idf_http_server_test/adder.py
tools/ci/python_packages/idf_http_server_test/client.py
tools/ci/python_packages/idf_http_server_test/test.py
tools/ci/python_packages/idf_iperf_test_util/LineChart.py
tools/ci/python_packages/idf_iperf_test_util/PowerControl.py
tools/ci/python_packages/tiny_test_fw/App.py
tools/ci/python_packages/tiny_test_fw/DUT.py
tools/ci/python_packages/tiny_test_fw/Env.py
tools/ci/python_packages/tiny_test_fw/EnvConfig.py
tools/ci/python_packages/tiny_test_fw/TinyFW.py
tools/ci/python_packages/tiny_test_fw/Utility/CIAssignTest.py
tools/ci/python_packages/tiny_test_fw/Utility/CaseConfig.py
tools/ci/python_packages/tiny_test_fw/Utility/GitlabCIJob.py
tools/ci/python_packages/tiny_test_fw/Utility/SearchCases.py
tools/ci/python_packages/tiny_test_fw/Utility/TestCase.py
tools/ci/python_packages/tiny_test_fw/bin/Runner.py
tools/ci/python_packages/ttfw_idf/CIScanTests.py
tools/ci/python_packages/ttfw_idf/DebugUtils.py
tools/ci/python_packages/ttfw_idf/IDFApp.py
tools/ci/python_packages/ttfw_idf/IDFDUT.py
tools/ci/python_packages/wifi_tools.py
tools/ci/test_autocomplete.py
tools/ci/test_check_kconfigs.py
tools/cmake/convert_to_cmake.py
tools/esp_app_trace/espytrace/apptrace.py
tools/esp_app_trace/espytrace/sysview.py
tools/esp_app_trace/logtrace_proc.py
tools/esp_app_trace/sysviewtrace_proc.py
tools/esp_prov/esp_prov.py
tools/esp_prov/prov/custom_prov.py
tools/esp_prov/prov/wifi_prov.py
tools/esp_prov/prov/wifi_scan.py
tools/esp_prov/security/security.py
tools/esp_prov/security/security0.py
tools/esp_prov/security/security1.py
tools/esp_prov/transport/ble_cli.py
tools/esp_prov/transport/transport.py
tools/esp_prov/transport/transport_ble.py
tools/esp_prov/transport/transport_console.py
tools/esp_prov/transport/transport_http.py
tools/esp_prov/utils/convenience.py
tools/find_apps.py
tools/find_build_apps/cmake.py
tools/find_build_apps/common.py
tools/find_build_apps/make.py
tools/gdb_panic_server.py
tools/gen_esp_err_to_name.py
tools/idf.py
tools/idf_monitor.py
tools/idf_py_actions/core_ext.py
tools/idf_py_actions/create_ext.py
tools/idf_py_actions/debug_ext.py
tools/idf_py_actions/dfu_ext.py
tools/idf_py_actions/errors.py
tools/idf_py_actions/serial_ext.py
tools/idf_py_actions/tools.py
tools/idf_py_actions/uf2_ext.py
tools/idf_size.py
tools/idf_tools.py
tools/kconfig_new/confgen.py
tools/kconfig_new/confserver.py
tools/kconfig_new/esp-windows-curses/setup.py
tools/kconfig_new/gen_kconfig_doc.py
tools/kconfig_new/prepare_kconfig_files.py
tools/kconfig_new/test/confgen/test_confgen.py
tools/kconfig_new/test/confserver/test_confserver.py
tools/kconfig_new/test/gen_kconfig_doc/test_kconfig_out.py
tools/kconfig_new/test/gen_kconfig_doc/test_target_visibility.py
tools/ldgen/fragments.py
tools/ldgen/generation.py
tools/ldgen/ldgen.py
tools/ldgen/ldgen_common.py
tools/ldgen/sdkconfig.py
tools/ldgen/test/test_fragments.py
tools/ldgen/test/test_generation.py
tools/mass_mfg/mfg_gen.py
tools/mkuf2.py
tools/test_apps/protocols/mqtt/publish_connect_test/app_test.py
tools/test_apps/protocols/openssl/app_test.py
tools/test_apps/protocols/pppos/app_test.py
tools/test_apps/system/gdb_loadable_elf/app_test.py
tools/test_apps/system/memprot/app_test.py
tools/test_apps/system/monitor_ide_integration/app_test.py
tools/test_apps/system/panic/app_test.py
tools/test_apps/system/panic/panic_tests.py
tools/test_apps/system/panic/test_panic_util/test_panic_util.py
tools/test_apps/system/startup/app_test.py
tools/test_idf_monitor/run_test_idf_monitor.py
tools/test_idf_py/extra_path/some_ext.py
tools/test_idf_py/idf_ext.py
tools/test_idf_py/test_idf_extensions/test_ext/test_extension.py
tools/test_idf_py/test_idf_py.py
tools/test_idf_size/test_idf_size.py
tools/test_idf_tools/test_idf_tools.py
tools/test_mkdfu/test_mkdfu.py
tools/test_mkuf2/test_mkuf2.py
tools/unit-test-app/idf_ext.py
tools/unit-test-app/tools/CreateSectionTable.py
tools/unit-test-app/tools/UnitTestParser.py
tools/unit-test-app/unit_test.py
tools/windows/eclipse_make.py
tools/windows/tool_setup/system_check/system_check_download.py
tools/windows/tool_setup/system_check/system_check_subprocess.py
tools/windows/tool_setup/system_check/system_check_virtualenv.py

View File

@ -38,7 +38,7 @@ except ImportError:
pass pass
try: try:
from itertools import izip as zip from itertools import izip as zip # type: ignore
except ImportError: except ImportError:
# Python 3 # Python 3
pass pass
@ -125,7 +125,7 @@ def pad_bytes(b, multiple, padding=b'\x00'): # type: (bytes, int, bytes) -> byt
class EspDfuWriter(object): class EspDfuWriter(object):
def __init__(self, dest_file, pid): # type: (typing.BinaryIO) -> None def __init__(self, dest_file, pid): # type: (typing.BinaryIO, int) -> None
self.dest = dest_file self.dest = dest_file
self.pid = pid self.pid = pid
self.entries = [] # type: typing.List[bytes] self.entries = [] # type: typing.List[bytes]
@ -187,7 +187,7 @@ class EspDfuWriter(object):
self.entries.insert(0, entry) self.entries.insert(0, entry)
def action_write(args): def action_write(args): # type: (typing.Mapping[str, typing.Any]) -> None
writer = EspDfuWriter(args['output_file'], args['pid']) writer = EspDfuWriter(args['output_file'], args['pid'])
for addr, f in args['files']: for addr, f in args['files']:
print('Adding {} at {:#x}'.format(f, addr)) print('Adding {} at {:#x}'.format(f, addr))
@ -196,7 +196,7 @@ def action_write(args):
print('"{}" has been written. You may proceed with DFU flashing.'.format(args['output_file'].name)) print('"{}" has been written. You may proceed with DFU flashing.'.format(args['output_file'].name))
def main(): def main(): # type: () -> None
parser = argparse.ArgumentParser() parser = argparse.ArgumentParser()
# Provision to add "info" command # Provision to add "info" command
@ -218,7 +218,7 @@ def main():
args = parser.parse_args() args = parser.parse_args()
def check_file(file_name): def check_file(file_name): # type: (str) -> str
if not os.path.isfile(file_name): if not os.path.isfile(file_name):
raise RuntimeError('{} is not a regular file!'.format(file_name)) raise RuntimeError('{} is not a regular file!'.format(file_name))
return file_name return file_name
@ -230,7 +230,7 @@ def main():
if args.json: if args.json:
json_dir = os.path.dirname(os.path.abspath(args.json)) json_dir = os.path.dirname(os.path.abspath(args.json))
def process_json_file(path): def process_json_file(path): # type: (str) -> str
''' '''
The input path is relative to json_dir. This function makes it relative to the current working The input path is relative to json_dir. This function makes it relative to the current working
directory. directory.

View File

@ -13,7 +13,7 @@ import yaml
try: try:
from yaml import CLoader as Loader from yaml import CLoader as Loader
except ImportError: except ImportError:
from yaml import Loader as Loader from yaml import Loader as Loader # type: ignore
TEST_CASE_PATTERN = { TEST_CASE_PATTERN = {
'initial condition': 'UTINIT1', 'initial condition': 'UTINIT1',