fix: TMS-39305 add connection error retries

testit-adapter-behave/setup.py

@@ -1,6 +1,6 @@
 from setuptools import find_packages, setup
 
-VERSION = "4.1.14"
+VERSION = "4.2.0"
 
 setup(
     name='testit-adapter-behave',

testit-adapter-nose/setup.py

@@ -1,6 +1,6 @@
 from setuptools import setup, find_packages
 
-VERSION = "4.1.14"
+VERSION = "4.2.0"
 
 setup(
     name='testit-adapter-nose',

testit-adapter-pytest/setup.py

@@ -1,6 +1,6 @@
 from setuptools import find_packages, setup
 
-VERSION = "4.1.14"
+VERSION = "4.2.0"
 
 setup(
     name='testit-adapter-pytest',

testit-adapter-robotframework/setup.py

@@ -1,6 +1,6 @@
 from setuptools import find_packages, setup
 
-VERSION = "4.1.14"
+VERSION = "4.2.0"
 
 setup(
     name='testit-adapter-robotframework',

testit-python-commons/setup.py

@@ -1,6 +1,6 @@
 from setuptools import find_packages, setup
 
-VERSION = "4.1.14"
+VERSION = "4.2.0"
 
 setup(
     name='testit-python-commons',

testit-python-commons/src/testit_python_commons/client/api_client.py

@@ -2,6 +2,7 @@
 import os
 from datetime import datetime
 
+import testit_api_client
 from testit_api_client import ApiClient, Configuration
 from testit_api_client.apis import (
     AttachmentsApi,
@@ -32,7 +33,12 @@
 from testit_python_commons.client.helpers.bulk_autotest_helper import BulkAutotestHelper
 from testit_python_commons.models.test_result import TestResult
 from testit_python_commons.services.logger import adapter_logger
-from testit_python_commons.services.retry import retry
+from testit_python_commons.services.retry import (
+    is_non_retriable_api_exception,
+    is_retriable_connection_error,
+    retry,
+    retry_on_connection_error,
+)
 from typing import List
 
 
@@ -164,6 +170,7 @@ def __get_autotests_by_external_id(self, external_id: str) -> List[AutoTestApiRe
         return self.__autotest_api.api_v2_auto_tests_search_post(api_v2_auto_tests_search_post_request=model)
 
     @adapter_logger
+    @retry
     def write_test(self, test_result: TestResult) -> str:
         model = Converter.project_id_and_external_id_to_auto_tests_search_post_request(
             self.__config.get_project_id(),
@@ -198,6 +205,7 @@ def write_test(self, test_result: TestResult) -> str:
         return self.__load_test_result(test_result)
 
     @adapter_logger
+    @retry
     def write_tests(self, test_results: List[TestResult], fixture_containers: dict) -> None:
         logging.debug("call __write_tests")
         bulk_autotest_helper = BulkAutotestHelper(self.__autotest_api, self.__test_run_api, self.__config)
@@ -335,11 +343,20 @@ def __get_work_item_uuid_by_work_item_id(self, work_item_id: str) -> str or None
             # logging.debug(f'Got workitem {work_item}')
 
             return work_item.id
+        except testit_api_client.exceptions.ApiException as exc:
+            if is_retriable_connection_error(exc):
+                raise
+            if is_non_retriable_api_exception(exc):
+                logging.warning(f'Getting workitem by id {work_item_id} status: {exc}')
+                return
+            raise
         except Exception as exc:
+            if is_retriable_connection_error(exc):
+                raise
             logging.error(f'Getting workitem by id {work_item_id} status: {exc}')
 
     @adapter_logger
-    #@retry # disabled
+    @retry_on_connection_error
     def __get_work_items_linked_to_autotest(self, autotest_global_id: str) -> List[AutoTestWorkItemIdentifierApiResult]:
         return self.__autotest_api.get_work_items_linked_to_auto_test(id=autotest_global_id)
 
@@ -400,6 +417,8 @@ def __update_auto_test(self, test_result: TestResult, autotest: AutoTestApiResul
         try:
             self.__autotest_api.update_auto_test(update_auto_test_request=model)
         except Exception as exc:
+            if is_retriable_connection_error(exc):
+                raise
             logging.error(f'Cannot update autotest "{test_result.get_autotest_name()}" status: {exc}')
 
         logging.debug(f'Autotest "{test_result.get_autotest_name()}" was updated')
@@ -416,18 +435,40 @@ def __update_tests(self, autotests_for_update: List[AutoTestUpdateApiModel]) ->
     @adapter_logger
     @retry
     def __unlink_test_to_work_item(self, autotest_global_id: str, work_item_id: str) -> None:
-        self.__autotest_api.delete_auto_test_link_from_work_item(
-            id=autotest_global_id,
-            work_item_id=work_item_id)
+        try:
+            self.__autotest_api.delete_auto_test_link_from_work_item(
+                id=autotest_global_id,
+                work_item_id=work_item_id)
+        except testit_api_client.exceptions.ApiException as exc:
+            if is_non_retriable_api_exception(exc):
+                logging.warning(
+                    'Cannot unlink autotest %s from work item %s: %s',
+                    autotest_global_id,
+                    work_item_id,
+                    exc,
+                )
+                return
+            raise
 
         logging.debug(f'Autotest was unlinked with workItem "{work_item_id}" by global id "{autotest_global_id}')
 
     @adapter_logger
     @retry
     def __link_test_to_work_item(self, autotest_global_id: str, work_item_id: str) -> None:
-        self.__autotest_api.link_auto_test_to_work_item(
-            autotest_global_id,
-            link_auto_test_to_work_item_request=LinkAutoTestToWorkItemRequest(id=work_item_id))
+        try:
+            self.__autotest_api.link_auto_test_to_work_item(
+                autotest_global_id,
+                link_auto_test_to_work_item_request=LinkAutoTestToWorkItemRequest(id=work_item_id))
+        except testit_api_client.exceptions.ApiException as exc:
+            if is_non_retriable_api_exception(exc):
+                logging.warning(
+                    'Cannot link autotest %s to work item %s: %s',
+                    autotest_global_id,
+                    work_item_id,
+                    exc,
+                )
+                return
+            raise
 
         logging.debug(f'Autotest was linked with workItem "{work_item_id}" by global id "{autotest_global_id}')
 
@@ -472,7 +513,17 @@ def update_test_results(self, fixtures_containers: dict, test_result_ids: dict)
                     id=test_result.get_test_result_id(),
                     api_v2_test_results_id_put_request=model)
             except Exception as exc:
-                logging.error(f'Cannot update test result with id "{test_result.get_test_result_id()}" status: {exc}')
+                if is_retriable_connection_error(exc):
+                    raise
+                logging.error(
+                    f'Cannot update test result with id "{test_result.get_test_result_id()}" status: {exc}')
+
+    @adapter_logger
+    @retry
+    def __upload_attachment(self, path: str) -> AttachmentPutModel:
+        with open(path, "rb") as file:
+            attachment_response = self.__attachments_api.api_v2_attachments_post(file=file)
+        return AttachmentPutModel(attachment_response['id'])
 
     @adapter_logger
     def load_attachments(self, attach_paths: list or tuple) -> List[AttachmentPutModel]:
@@ -481,12 +532,11 @@ def load_attachments(self, attach_paths: list or tuple) -> List[AttachmentPutMod
         for path in attach_paths:
             if os.path.isfile(path):
                 try:
-                    attachment_response = self.__attachments_api.api_v2_attachments_post(file=open(path, "rb"))
-
-                    attachments.append(AttachmentPutModel(attachment_response['id']))
-
+                    attachments.append(self.__upload_attachment(path))
                     logging.debug(f'Attachment "{path}" was uploaded')
                 except Exception as exc:
+                    if is_retriable_connection_error(exc):
+                        raise
                     logging.error(f'Upload attachment "{path}" status: {exc}')
             else:
                 logging.error(f'File "{path}" was not found!')
@@ -496,10 +546,12 @@ def get_configuration_id(self):
         return self.__config.get_configuration_id()
 
     @adapter_logger
+    @retry
     def __get_project(self) -> ProjectModel:
         return self.__projects_api.get_project_by_id(id=self.__config.get_project_id())
 
     @adapter_logger
+    @retry
     def __get_workflow_by_id(self, workflow_id: str) -> WorkflowApiResult:
         return self.__workflows_api.api_v2_workflows_id_get(id=workflow_id)
 

testit-python-commons/src/testit_python_commons/client/helpers/bulk_autotest_helper.py

@@ -18,7 +18,9 @@
     ThreadsForUpdateAndResult
 )
 from testit_python_commons.services.logger import adapter_logger
-from testit_python_commons.services.retry import retry
+import testit_api_client
+
+from testit_python_commons.services.retry import is_non_retriable_api_exception, retry
 from testit_python_commons.utils.html_escape_utils import HtmlEscapeUtils
 from typing import Dict, List
 
@@ -186,19 +188,41 @@ def __get_work_items_linked_to_autotest(self, autotest_global_id: str) -> List[A
     @adapter_logger
     @retry
     def __unlink_test_to_work_item(self, autotest_global_id: str, work_item_id: str):
-        self.__autotests_api.delete_auto_test_link_from_work_item(
-            id=autotest_global_id,
-            work_item_id=work_item_id)
+        try:
+            self.__autotests_api.delete_auto_test_link_from_work_item(
+                id=autotest_global_id,
+                work_item_id=work_item_id)
+        except testit_api_client.exceptions.ApiException as exc:
+            if is_non_retriable_api_exception(exc):
+                logging.warning(
+                    'Cannot unlink autotest %s from work item %s: %s',
+                    autotest_global_id,
+                    work_item_id,
+                    exc,
+                )
+                return
+            raise
 
         logging.debug(f'Autotest was unlinked with workItem "{work_item_id}" by global id "{autotest_global_id}')
 
     # TODO: delete after fix PUT/api/v2/autoTests
     @adapter_logger
     @retry
     def __link_test_to_work_item(self, autotest_global_id: str, work_item_id: str):
-        self.__autotests_api.link_auto_test_to_work_item(
-            autotest_global_id,
-            link_auto_test_to_work_item_request=LinkAutoTestToWorkItemRequest(id=work_item_id))
+        try:
+            self.__autotests_api.link_auto_test_to_work_item(
+                autotest_global_id,
+                link_auto_test_to_work_item_request=LinkAutoTestToWorkItemRequest(id=work_item_id))
+        except testit_api_client.exceptions.ApiException as exc:
+            if is_non_retriable_api_exception(exc):
+                logging.warning(
+                    'Cannot link autotest %s to work item %s: %s',
+                    autotest_global_id,
+                    work_item_id,
+                    exc,
+                )
+                return
+            raise
 
         logging.debug(f'Autotest was linked with workItem "{work_item_id}" by global id "{autotest_global_id}')
 

testit-python-commons/src/testit_python_commons/services/retry.py

@@ -1,29 +1,92 @@
 import logging
 import random
 import time
+from http.client import RemoteDisconnected
 
 import testit_api_client
+import urllib3
+
+CONNECTION_RETRIES = 3
+CONNECTION_RETRY_DELAY_SEC = 1
+API_EXCEPTION_RETRIES = 10
+NON_RETRIABLE_API_STATUS_CODES = (400, 404)
+
+_RETRIABLE_CONNECTION_TYPES = (
+    urllib3.exceptions.ProtocolError,
+    urllib3.exceptions.NewConnectionError,
+    ConnectionError,
+    ConnectionResetError,
+    ConnectionAbortedError,
+    RemoteDisconnected,
+    TimeoutError,
+)
+
+
+def is_non_retriable_api_exception(exc: BaseException) -> bool:
+    return (
+        isinstance(exc, testit_api_client.exceptions.ApiException)
+        and int(exc.status) in NON_RETRIABLE_API_STATUS_CODES
+    )
+
+
+def is_retriable_connection_error(exc: BaseException) -> bool:
+    seen = set()
+    current = exc
+    while current is not None and id(current) not in seen:
+        seen.add(id(current))
+        if isinstance(current, _RETRIABLE_CONNECTION_TYPES):
+            return True
+        current = current.__cause__ or current.__context__
+    return False
+
+
+def _execute_with_connection_retries(func, args, kwargs):
+    connection_attempts = 0
+
+    while True:
+        try:
+            return func(*args, **kwargs)
+        except BaseException as e:
+            if not is_retriable_connection_error(e):
+                raise
+
+            connection_attempts += 1
+            logging.warning(
+                'Connection error in %s (attempt %d/%d): %s',
+                func.__name__,
+                connection_attempts,
+                CONNECTION_RETRIES,
+                e,
+            )
+            if connection_attempts > CONNECTION_RETRIES:
+                raise
+
+            time.sleep(CONNECTION_RETRY_DELAY_SEC)
+
+
+def retry_on_connection_error(func):
+    def retry_wrapper(*args, **kwargs):
+        return _execute_with_connection_retries(func, args, kwargs)
+
+    return retry_wrapper
 
 
 def retry(func):
     def retry_wrapper(*args, **kwargs):
-        attempts = 0
-        retries = 10
+        api_attempts = 0
 
-        while attempts < retries:
+        while True:
             try:
-                return func(*args, **kwargs)
+                return _execute_with_connection_retries(func, args, kwargs)
             except testit_api_client.exceptions.ApiException as e:
-                sleep_time = random.randrange(0, 100)
-                time.sleep(sleep_time/100)
-                attempts += 1
+                if is_non_retriable_api_exception(e):
+                    raise
 
+                api_attempts += 1
                 logging.error(e)
-                if e.status == '404':
-                    attempts = retries
-                    return 
-                if e.status == '400':
-                    attempts = retries
-                    return 
+                if api_attempts >= API_EXCEPTION_RETRIES:
+                    raise
+
+                time.sleep(random.randrange(0, 100) / 100)
 
     return retry_wrapper