Merge branch 'bugfix/retry_download_submodule_v4.1' into 'release/v4.1'

ci: retry download from Gitlab on error 500 (v4.1)

See merge request espressif/esp-idf!15560

tools/ci/python_packages/gitlab_api.py

@@ -1,34 +1,78 @@
+import argparse
 import os
 import re
-import argparse
-import tempfile
 import tarfile
+import tempfile
+import time
 import zipfile
+from functools import wraps
 
 import gitlab
 
+try:
+    from typing import Any, Callable, Dict, List, Optional
+    TR = Callable[..., Any]
+except ImportError:
+    pass
+
+
+def retry(func):  # type: (TR) -> TR
+    """
+    This wrapper will only catch several exception types associated with
+    "network issues" and retry the whole function.
+    """
+    @wraps(func)
+    def wrapper(self, *args, **kwargs):  # type: (Gitlab, Any, Any) -> Any
+        retried = 0
+        while True:
+            try:
+                res = func(self, *args, **kwargs)
+            except (IOError, EOFError, gitlab.exceptions.GitlabError) as e:
+                if isinstance(e, gitlab.exceptions.GitlabError) and e.response_code != 500:
+                    # Only retry on error 500
+                    raise e
+                retried += 1
+                if retried > self.DOWNLOAD_ERROR_MAX_RETRIES:
+                    raise e  # get out of the loop
+                else:
+                    print('Network failure in {}, retrying ({})'.format(getattr(func, '__name__', '(unknown callable)'), retried))
+                    time.sleep(2 ** retried)  # wait a bit more after each retry
+                    continue
+            else:
+                break
+        return res
+    return wrapper
+
 
 class Gitlab(object):
     JOB_NAME_PATTERN = re.compile(r"(\w+)(\s+(\d+)/(\d+))?")
 
-    def __init__(self, project_id=None):
+    DOWNLOAD_ERROR_MAX_RETRIES = 3
+
+    def __init__(self, project_id=None):  # type: (Optional[int]) -> None
         config_data_from_env = os.getenv("PYTHON_GITLAB_CONFIG")
         if config_data_from_env:
             # prefer to load config from env variable
-            with tempfile.NamedTemporaryFile("w", delete=False) as temp_file:
+            with tempfile.NamedTemporaryFile('w', delete=False) as temp_file:
                 temp_file.write(config_data_from_env)
-            config_files = [temp_file.name]
+            config_files = [temp_file.name]  # type: Optional[List[str]]
         else:
             # otherwise try to use config file at local filesystem
             config_files = None
-        self.gitlab_inst = gitlab.Gitlab.from_config(config_files=config_files)
+        self._init_gitlab_inst(project_id, config_files)
+
+    @retry
+    def _init_gitlab_inst(self, project_id, config_files):  # type: (Optional[int], Optional[List[str]]) -> None
+        gitlab_id = os.getenv('LOCAL_GITLAB_HTTPS_HOST')  # if None, will use the default gitlab server
+        self.gitlab_inst = gitlab.Gitlab.from_config(gitlab_id=gitlab_id, config_files=config_files)
         self.gitlab_inst.auth()
         if project_id:
             self.project = self.gitlab_inst.projects.get(project_id)
         else:
             self.project = None
 
-    def get_project_id(self, name, namespace=None):
+    @retry
+    def get_project_id(self, name, namespace=None):  # type: (str, Optional[str]) -> int
         """
         search project ID by name
 
@@ -52,9 +96,10 @@ class Gitlab(object):
 
         if not res:
             raise ValueError("Can't find project")
-        return res[0]
+        return int(res[0])
 
-    def download_artifacts(self, job_id, destination):
+    @retry
+    def download_artifacts(self, job_id, destination):  # type (int, str) -> None
         """
         download full job artifacts and extract to destination.
 
@@ -69,7 +114,8 @@ class Gitlab(object):
         with zipfile.ZipFile(temp_file.name, "r") as archive_file:
             archive_file.extractall(destination)
 
-    def download_artifact(self, job_id, artifact_path, destination=None):
+    @retry
+    def download_artifact(self, job_id, artifact_path, destination=None):  # type: (int, str, Optional[str]) -> List[bytes]
         """
         download specific path of job artifacts and extract to destination.
 
@@ -84,7 +130,7 @@ class Gitlab(object):
 
         for a_path in artifact_path:
             try:
-                data = job.artifact(a_path)
+                data = job.artifact(a_path)  # type: bytes
             except gitlab.GitlabGetError as e:
                 print("Failed to download '{}' form job {}".format(a_path, job_id))
                 raise e
@@ -101,7 +147,8 @@ class Gitlab(object):
 
         return raw_data_list
 
-    def find_job_id(self, job_name, pipeline_id=None, job_status="success"):
+    @retry
+    def find_job_id(self, job_name, pipeline_id=None, job_status='success'):  # type: (str, Optional[str], str) -> List[Dict]
         """
         Get Job ID from job name of specific pipeline
 
@@ -123,7 +170,8 @@ class Gitlab(object):
                     job_id_list.append({"id": job.id, "parallel_num": match.group(3)})
         return job_id_list
 
-    def download_archive(self, ref, destination, project_id=None):
+    @retry
+    def download_archive(self, ref, destination, project_id=None):  # type: (str, str, Optional[int]) -> str
         """
         Download archive of certain commit of a repository and extract to destination path
 
@@ -153,7 +201,7 @@ class Gitlab(object):
         return os.path.join(os.path.realpath(destination), root_name)
 
 
-if __name__ == '__main__':
+def main():  # type: () -> None
     parser = argparse.ArgumentParser()
     parser.add_argument("action")
     parser.add_argument("project_id", type=int)
@@ -179,3 +227,7 @@ if __name__ == '__main__':
     elif args.action == "get_project_id":
         ret = gitlab_inst.get_project_id(args.project_name)
         print("project id: {}".format(ret))
+
+
+if __name__ == '__main__':
+    main()