mirror of
https://github.com/espressif/esp-idf.git
synced 2024-10-05 20:47:46 -04:00
b70d65444a
set it under `default` does not work due to gitlab bugs. will use `default` back once the bugs are resolved.
133 lines
5.1 KiB
Python
Executable File
133 lines
5.1 KiB
Python
Executable File
#!/usr/bin/env python
|
|
# SPDX-FileCopyrightText: 2023-2024 Espressif Systems (Shanghai) CO LTD
|
|
# SPDX-License-Identifier: Apache-2.0
|
|
"""
|
|
Check gitlab ci yaml files
|
|
"""
|
|
import argparse
|
|
import os
|
|
import typing as t
|
|
from functools import cached_property
|
|
|
|
from idf_ci_utils import get_submodule_dirs
|
|
from idf_ci_utils import GitlabYmlConfig
|
|
from idf_ci_utils import IDF_PATH
|
|
|
|
|
|
class YmlLinter:
|
|
def __init__(self, yml_config: GitlabYmlConfig) -> None:
|
|
self.yml_config = yml_config
|
|
|
|
self._errors: t.List[str] = []
|
|
|
|
@cached_property
|
|
def lint_functions(self) -> t.List[str]:
|
|
funcs = []
|
|
for func in dir(self):
|
|
if func.startswith('_lint_'):
|
|
funcs.append(func)
|
|
|
|
return funcs
|
|
|
|
def lint(self) -> None:
|
|
exit_code = 0
|
|
|
|
for func in self.lint_functions:
|
|
getattr(self, func)()
|
|
|
|
if self._errors:
|
|
print(f'Errors found while running {func}:')
|
|
exit_code = 1
|
|
print('\t- ' + '\n\t- '.join(self._errors))
|
|
self._errors = [] # reset
|
|
|
|
exit(exit_code)
|
|
|
|
# name it like _1_ to make it run first
|
|
def _lint_1_yml_parser(self) -> None:
|
|
for k, v in self.yml_config.config.items():
|
|
if (
|
|
k not in self.yml_config.global_keys
|
|
and k not in self.yml_config.anchors
|
|
and k not in self.yml_config.templates
|
|
and k not in self.yml_config.jobs
|
|
):
|
|
raise SystemExit(f'Parser incorrect. Key {k} not in global keys, anchors, templates, or jobs')
|
|
|
|
def _lint_default_values_artifacts(self) -> None:
|
|
defaults_artifacts = self.yml_config.default.get('artifacts', {})
|
|
|
|
for job_name, d in self.yml_config.jobs.items():
|
|
for k, v in d.get('artifacts', {}).items():
|
|
if k not in defaults_artifacts:
|
|
continue
|
|
|
|
if v == defaults_artifacts[k]:
|
|
self._errors.append(f'job {job_name} key {k} has same value as default value {v}')
|
|
|
|
def _lint_submodule_patterns(self) -> None:
|
|
submodule_paths = sorted(['.gitmodules'] + get_submodule_dirs())
|
|
submodule_paths_in_patterns = sorted(self.yml_config.config.get('.patterns-submodule', []))
|
|
|
|
if submodule_paths != submodule_paths_in_patterns:
|
|
unused_patterns = set(submodule_paths_in_patterns) - set(submodule_paths)
|
|
if unused_patterns:
|
|
for item in unused_patterns:
|
|
self._errors.append(f'non-exist pattern {item}. Please remove {item} from .patterns-submodule')
|
|
undefined_patterns = set(submodule_paths) - set(submodule_paths_in_patterns)
|
|
if undefined_patterns:
|
|
for item in undefined_patterns:
|
|
self._errors.append(f'undefined pattern {item}. Please add {item} to .patterns-submodule')
|
|
|
|
def _lint_gitlab_yml_templates(self) -> None:
|
|
unused_templates = self.yml_config.templates.keys() - self.yml_config.used_templates
|
|
for item in unused_templates:
|
|
# known unused ones
|
|
if item not in [
|
|
'.before_script:fetch:target_test', # used in dynamic pipeline
|
|
]:
|
|
self._errors.append(f'Unused template: {item}, please remove it')
|
|
|
|
undefined_templates = self.yml_config.used_templates - self.yml_config.templates.keys()
|
|
for item in undefined_templates:
|
|
self._errors.append(f'Undefined template: {item}')
|
|
|
|
def _lint_dependencies_and_needs(self) -> None:
|
|
"""
|
|
Use `dependencies: []` together with `needs: []` could cause missing artifacts issue.
|
|
"""
|
|
for job_name, d in self.yml_config.jobs.items():
|
|
if 'dependencies' in d and 'needs' in d:
|
|
if d['dependencies'] is not None and d['needs']:
|
|
self._errors.append(
|
|
f'job {job_name} has both `dependencies` and `needs` defined. '
|
|
f'Please set `dependencies:` (to null) explicitly to avoid missing artifacts issue'
|
|
)
|
|
|
|
def _lint_artifacts_expire_in_and_when(self) -> None:
|
|
"""
|
|
Set `artifacts: expire_in` and `artifacts: when` together since gitlab has bugs:
|
|
|
|
- https://gitlab.com/gitlab-org/gitlab/-/issues/404563 (expire_in)
|
|
- https://gitlab.com/gitlab-org/gitlab/-/issues/440672 (when)
|
|
"""
|
|
for job_name, d in self.yml_config.jobs.items():
|
|
if 'artifacts' in d:
|
|
if 'expire_in' not in d['artifacts']:
|
|
self._errors.append(f'job {job_name} missing `artifacts: expire_in`. (suggest to set to `1 week`)')
|
|
|
|
if 'when' not in d['artifacts']:
|
|
self._errors.append(f'job {job_name} missing `artifacts: when`. (suggest to set to `always`)')
|
|
|
|
|
|
if __name__ == '__main__':
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument(
|
|
'--root-yml-filepath', help='root yml file path', default=os.path.join(IDF_PATH, '.gitlab-ci.yml')
|
|
)
|
|
args = parser.parse_args()
|
|
|
|
config = GitlabYmlConfig(args.root_yml_filepath)
|
|
linter = YmlLinter(config)
|
|
linter.lint()
|