esp-idf/tools/ci/gitlab_yaml_linter.py

#!/usr/bin/env python
# SPDX-FileCopyrightText: 2023-2024 Espressif Systems (Shanghai) CO LTD
# SPDX-License-Identifier: Apache-2.0
"""
Check gitlab ci yaml files
"""
import argparse
import os
import typing as t
from functools import cached_property

from idf_ci_utils import get_submodule_dirs
from idf_ci_utils import GitlabYmlConfig
from idf_ci_utils import IDF_PATH


class YmlLinter:
    def __init__(self, yml_config: GitlabYmlConfig) -> None:
        self.yml_config = yml_config

        self._errors: t.List[str] = []

    @cached_property
    def lint_functions(self) -> t.List[str]:
        funcs = []
        for func in dir(self):
            if func.startswith('_lint_'):
                funcs.append(func)

        return funcs

    def lint(self) -> None:
        exit_code = 0

        for func in self.lint_functions:
            getattr(self, func)()

            if self._errors:
                print(f'Errors found while running {func}:')
                exit_code = 1
                print('\t- ' + '\n\t- '.join(self._errors))
                self._errors = []  # reset

        exit(exit_code)

    # name it like _1_ to make it run first
    def _lint_1_yml_parser(self) -> None:
        for k, v in self.yml_config.config.items():
            if (
                k not in self.yml_config.global_keys
                and k not in self.yml_config.anchors
                and k not in self.yml_config.templates
                and k not in self.yml_config.jobs
            ):
                raise SystemExit(f'Parser incorrect. Key {k} not in global keys, anchors, templates,  or jobs')

    def _lint_default_values_artifacts(self) -> None:
        defaults_artifacts = self.yml_config.default.get('artifacts', {})

        for job_name, d in self.yml_config.jobs.items():
            for k, v in d.get('artifacts', {}).items():
                if k not in defaults_artifacts:
                    continue

                if v == defaults_artifacts[k]:
                    self._errors.append(f'job {job_name} key {k} has same value as default value {v}')

    def _lint_submodule_patterns(self) -> None:
        submodule_paths = sorted(['.gitmodules'] + get_submodule_dirs())
        submodule_paths_in_patterns = sorted(self.yml_config.config.get('.patterns-submodule', []))

        if submodule_paths != submodule_paths_in_patterns:
            unused_patterns = set(submodule_paths_in_patterns) - set(submodule_paths)
            if unused_patterns:
                for item in unused_patterns:
                    self._errors.append(f'non-exist pattern {item}. Please remove {item} from .patterns-submodule')
            undefined_patterns = set(submodule_paths) - set(submodule_paths_in_patterns)
            if undefined_patterns:
                for item in undefined_patterns:
                    self._errors.append(f'undefined pattern {item}. Please add {item} to .patterns-submodule')

    def _lint_gitlab_yml_templates(self) -> None:
        unused_templates = self.yml_config.templates.keys() - self.yml_config.used_templates
        for item in unused_templates:
            # known unused ones
            if item not in [
                '.before_script:fetch:target_test',  # used in dynamic pipeline
            ]:
                self._errors.append(f'Unused template: {item}, please remove it')

        undefined_templates = self.yml_config.used_templates - self.yml_config.templates.keys()
        for item in undefined_templates:
            self._errors.append(f'Undefined template: {item}')

    def _lint_dependencies_and_needs(self) -> None:
        """
        Use `dependencies: []` together with `needs: []` could cause missing artifacts issue.
        """
        for job_name, d in self.yml_config.jobs.items():
            if 'dependencies' in d and 'needs' in d:
                if d['dependencies'] is not None and d['needs']:
                    self._errors.append(
                        f'job {job_name} has both `dependencies` and `needs` defined. '
                        f'Please set `dependencies:` (to null) explicitly to avoid missing artifacts issue'
                    )

    def _lint_artifacts_expire_in_and_when(self) -> None:
        """
        Set `artifacts: expire_in` and `artifacts: when` together since gitlab has bugs:

        - https://gitlab.com/gitlab-org/gitlab/-/issues/404563 (expire_in)
        - https://gitlab.com/gitlab-org/gitlab/-/issues/440672 (when)
        """
        for job_name, d in self.yml_config.jobs.items():
            if 'artifacts' in d:
                if 'expire_in' not in d['artifacts']:
                    self._errors.append(f'job {job_name} missing `artifacts: expire_in`. (suggest to set to `1 week`)')

                if 'when' not in d['artifacts']:
                    self._errors.append(f'job {job_name} missing `artifacts: when`. (suggest to set to `always`)')


if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument(
        '--root-yml-filepath', help='root yml file path', default=os.path.join(IDF_PATH, '.gitlab-ci.yml')
    )
    args = parser.parse_args()

    config = GitlabYmlConfig(args.root_yml_filepath)
    linter = YmlLinter(config)
    linter.lint()