diff --git a/tools/ci/python_packages/gitlab_api.py b/tools/ci/python_packages/gitlab_api.py index a5d58906d1..8e8df4b231 100644 --- a/tools/ci/python_packages/gitlab_api.py +++ b/tools/ci/python_packages/gitlab_api.py @@ -1,34 +1,78 @@ +import argparse import os import re -import argparse -import tempfile import tarfile +import tempfile +import time import zipfile +from functools import wraps import gitlab +try: + from typing import Any, Callable, Dict, List, Optional + TR = Callable[..., Any] +except ImportError: + pass + + +def retry(func): # type: (TR) -> TR + """ + This wrapper will only catch several exception types associated with + "network issues" and retry the whole function. + """ + @wraps(func) + def wrapper(self, *args, **kwargs): # type: (Gitlab, Any, Any) -> Any + retried = 0 + while True: + try: + res = func(self, *args, **kwargs) + except (IOError, EOFError, gitlab.exceptions.GitlabError) as e: + if isinstance(e, gitlab.exceptions.GitlabError) and e.response_code != 500: + # Only retry on error 500 + raise e + retried += 1 + if retried > self.DOWNLOAD_ERROR_MAX_RETRIES: + raise e # get out of the loop + else: + print('Network failure in {}, retrying ({})'.format(getattr(func, '__name__', '(unknown callable)'), retried)) + time.sleep(2 ** retried) # wait a bit more after each retry + continue + else: + break + return res + return wrapper + class Gitlab(object): JOB_NAME_PATTERN = re.compile(r"(\w+)(\s+(\d+)/(\d+))?") - def __init__(self, project_id=None): + DOWNLOAD_ERROR_MAX_RETRIES = 3 + + def __init__(self, project_id=None): # type: (Optional[int]) -> None config_data_from_env = os.getenv("PYTHON_GITLAB_CONFIG") if config_data_from_env: # prefer to load config from env variable - with tempfile.NamedTemporaryFile("w", delete=False) as temp_file: + with tempfile.NamedTemporaryFile('w', delete=False) as temp_file: temp_file.write(config_data_from_env) - config_files = [temp_file.name] + config_files = [temp_file.name] # type: Optional[List[str]] else: # otherwise try to use config file at local filesystem config_files = None - self.gitlab_inst = gitlab.Gitlab.from_config(config_files=config_files) + self._init_gitlab_inst(project_id, config_files) + + @retry + def _init_gitlab_inst(self, project_id, config_files): # type: (Optional[int], Optional[List[str]]) -> None + gitlab_id = os.getenv('LOCAL_GITLAB_HTTPS_HOST') # if None, will use the default gitlab server + self.gitlab_inst = gitlab.Gitlab.from_config(gitlab_id=gitlab_id, config_files=config_files) self.gitlab_inst.auth() if project_id: self.project = self.gitlab_inst.projects.get(project_id) else: self.project = None - def get_project_id(self, name, namespace=None): + @retry + def get_project_id(self, name, namespace=None): # type: (str, Optional[str]) -> int """ search project ID by name @@ -52,9 +96,10 @@ class Gitlab(object): if not res: raise ValueError("Can't find project") - return res[0] + return int(res[0]) - def download_artifacts(self, job_id, destination): + @retry + def download_artifacts(self, job_id, destination): # type (int, str) -> None """ download full job artifacts and extract to destination. @@ -69,7 +114,8 @@ class Gitlab(object): with zipfile.ZipFile(temp_file.name, "r") as archive_file: archive_file.extractall(destination) - def download_artifact(self, job_id, artifact_path, destination=None): + @retry + def download_artifact(self, job_id, artifact_path, destination=None): # type: (int, str, Optional[str]) -> List[bytes] """ download specific path of job artifacts and extract to destination. @@ -84,7 +130,7 @@ class Gitlab(object): for a_path in artifact_path: try: - data = job.artifact(a_path) + data = job.artifact(a_path) # type: bytes except gitlab.GitlabGetError as e: print("Failed to download '{}' form job {}".format(a_path, job_id)) raise e @@ -101,7 +147,8 @@ class Gitlab(object): return raw_data_list - def find_job_id(self, job_name, pipeline_id=None, job_status="success"): + @retry + def find_job_id(self, job_name, pipeline_id=None, job_status='success'): # type: (str, Optional[str], str) -> List[Dict] """ Get Job ID from job name of specific pipeline @@ -123,7 +170,8 @@ class Gitlab(object): job_id_list.append({"id": job.id, "parallel_num": match.group(3)}) return job_id_list - def download_archive(self, ref, destination, project_id=None): + @retry + def download_archive(self, ref, destination, project_id=None): # type: (str, str, Optional[int]) -> str """ Download archive of certain commit of a repository and extract to destination path @@ -153,7 +201,7 @@ class Gitlab(object): return os.path.join(os.path.realpath(destination), root_name) -if __name__ == '__main__': +def main(): # type: () -> None parser = argparse.ArgumentParser() parser.add_argument("action") parser.add_argument("project_id", type=int) @@ -179,3 +227,7 @@ if __name__ == '__main__': elif args.action == "get_project_id": ret = gitlab_inst.get_project_id(args.project_name) print("project id: {}".format(ret)) + + +if __name__ == '__main__': + main()