diff --git a/testit-adapter-behave/setup.py b/testit-adapter-behave/setup.py index 617458b4..f807fb5b 100644 --- a/testit-adapter-behave/setup.py +++ b/testit-adapter-behave/setup.py @@ -1,6 +1,6 @@ from setuptools import find_packages, setup -VERSION = "4.1.14" +VERSION = "4.2.0" setup( name='testit-adapter-behave', diff --git a/testit-adapter-nose/setup.py b/testit-adapter-nose/setup.py index c2e2b9e2..13c2cffc 100644 --- a/testit-adapter-nose/setup.py +++ b/testit-adapter-nose/setup.py @@ -1,6 +1,6 @@ from setuptools import setup, find_packages -VERSION = "4.1.14" +VERSION = "4.2.0" setup( name='testit-adapter-nose', diff --git a/testit-adapter-pytest/setup.py b/testit-adapter-pytest/setup.py index 2839efd4..a81a3b90 100644 --- a/testit-adapter-pytest/setup.py +++ b/testit-adapter-pytest/setup.py @@ -1,6 +1,6 @@ from setuptools import find_packages, setup -VERSION = "4.1.14" +VERSION = "4.2.0" setup( name='testit-adapter-pytest', diff --git a/testit-adapter-robotframework/setup.py b/testit-adapter-robotframework/setup.py index 802e7497..58cbaea2 100644 --- a/testit-adapter-robotframework/setup.py +++ b/testit-adapter-robotframework/setup.py @@ -1,6 +1,6 @@ from setuptools import find_packages, setup -VERSION = "4.1.14" +VERSION = "4.2.0" setup( name='testit-adapter-robotframework', diff --git a/testit-python-commons/setup.py b/testit-python-commons/setup.py index d5ea982b..f90eab5f 100644 --- a/testit-python-commons/setup.py +++ b/testit-python-commons/setup.py @@ -1,6 +1,6 @@ from setuptools import find_packages, setup -VERSION = "4.1.14" +VERSION = "4.2.0" setup( name='testit-python-commons', diff --git a/testit-python-commons/src/testit_python_commons/client/api_client.py b/testit-python-commons/src/testit_python_commons/client/api_client.py index 0d124e4c..20ddeda0 100644 --- a/testit-python-commons/src/testit_python_commons/client/api_client.py +++ b/testit-python-commons/src/testit_python_commons/client/api_client.py @@ -2,6 +2,7 @@ import os from datetime import datetime +import testit_api_client from testit_api_client import ApiClient, Configuration from testit_api_client.apis import ( AttachmentsApi, @@ -32,7 +33,12 @@ from testit_python_commons.client.helpers.bulk_autotest_helper import BulkAutotestHelper from testit_python_commons.models.test_result import TestResult from testit_python_commons.services.logger import adapter_logger -from testit_python_commons.services.retry import retry +from testit_python_commons.services.retry import ( + is_non_retriable_api_exception, + is_retriable_connection_error, + retry, + retry_on_connection_error, +) from typing import List @@ -164,6 +170,7 @@ def __get_autotests_by_external_id(self, external_id: str) -> List[AutoTestApiRe return self.__autotest_api.api_v2_auto_tests_search_post(api_v2_auto_tests_search_post_request=model) @adapter_logger + @retry def write_test(self, test_result: TestResult) -> str: model = Converter.project_id_and_external_id_to_auto_tests_search_post_request( self.__config.get_project_id(), @@ -198,6 +205,7 @@ def write_test(self, test_result: TestResult) -> str: return self.__load_test_result(test_result) @adapter_logger + @retry def write_tests(self, test_results: List[TestResult], fixture_containers: dict) -> None: logging.debug("call __write_tests") bulk_autotest_helper = BulkAutotestHelper(self.__autotest_api, self.__test_run_api, self.__config) @@ -335,11 +343,20 @@ def __get_work_item_uuid_by_work_item_id(self, work_item_id: str) -> str or None # logging.debug(f'Got workitem {work_item}') return work_item.id + except testit_api_client.exceptions.ApiException as exc: + if is_retriable_connection_error(exc): + raise + if is_non_retriable_api_exception(exc): + logging.warning(f'Getting workitem by id {work_item_id} status: {exc}') + return + raise except Exception as exc: + if is_retriable_connection_error(exc): + raise logging.error(f'Getting workitem by id {work_item_id} status: {exc}') @adapter_logger - #@retry # disabled + @retry_on_connection_error def __get_work_items_linked_to_autotest(self, autotest_global_id: str) -> List[AutoTestWorkItemIdentifierApiResult]: return self.__autotest_api.get_work_items_linked_to_auto_test(id=autotest_global_id) @@ -400,6 +417,8 @@ def __update_auto_test(self, test_result: TestResult, autotest: AutoTestApiResul try: self.__autotest_api.update_auto_test(update_auto_test_request=model) except Exception as exc: + if is_retriable_connection_error(exc): + raise logging.error(f'Cannot update autotest "{test_result.get_autotest_name()}" status: {exc}') logging.debug(f'Autotest "{test_result.get_autotest_name()}" was updated') @@ -416,18 +435,40 @@ def __update_tests(self, autotests_for_update: List[AutoTestUpdateApiModel]) -> @adapter_logger @retry def __unlink_test_to_work_item(self, autotest_global_id: str, work_item_id: str) -> None: - self.__autotest_api.delete_auto_test_link_from_work_item( - id=autotest_global_id, - work_item_id=work_item_id) + try: + self.__autotest_api.delete_auto_test_link_from_work_item( + id=autotest_global_id, + work_item_id=work_item_id) + except testit_api_client.exceptions.ApiException as exc: + if is_non_retriable_api_exception(exc): + logging.warning( + 'Cannot unlink autotest %s from work item %s: %s', + autotest_global_id, + work_item_id, + exc, + ) + return + raise logging.debug(f'Autotest was unlinked with workItem "{work_item_id}" by global id "{autotest_global_id}') @adapter_logger @retry def __link_test_to_work_item(self, autotest_global_id: str, work_item_id: str) -> None: - self.__autotest_api.link_auto_test_to_work_item( - autotest_global_id, - link_auto_test_to_work_item_request=LinkAutoTestToWorkItemRequest(id=work_item_id)) + try: + self.__autotest_api.link_auto_test_to_work_item( + autotest_global_id, + link_auto_test_to_work_item_request=LinkAutoTestToWorkItemRequest(id=work_item_id)) + except testit_api_client.exceptions.ApiException as exc: + if is_non_retriable_api_exception(exc): + logging.warning( + 'Cannot link autotest %s to work item %s: %s', + autotest_global_id, + work_item_id, + exc, + ) + return + raise logging.debug(f'Autotest was linked with workItem "{work_item_id}" by global id "{autotest_global_id}') @@ -472,7 +513,17 @@ def update_test_results(self, fixtures_containers: dict, test_result_ids: dict) id=test_result.get_test_result_id(), api_v2_test_results_id_put_request=model) except Exception as exc: - logging.error(f'Cannot update test result with id "{test_result.get_test_result_id()}" status: {exc}') + if is_retriable_connection_error(exc): + raise + logging.error( + f'Cannot update test result with id "{test_result.get_test_result_id()}" status: {exc}') + + @adapter_logger + @retry + def __upload_attachment(self, path: str) -> AttachmentPutModel: + with open(path, "rb") as file: + attachment_response = self.__attachments_api.api_v2_attachments_post(file=file) + return AttachmentPutModel(attachment_response['id']) @adapter_logger def load_attachments(self, attach_paths: list or tuple) -> List[AttachmentPutModel]: @@ -481,12 +532,11 @@ def load_attachments(self, attach_paths: list or tuple) -> List[AttachmentPutMod for path in attach_paths: if os.path.isfile(path): try: - attachment_response = self.__attachments_api.api_v2_attachments_post(file=open(path, "rb")) - - attachments.append(AttachmentPutModel(attachment_response['id'])) - + attachments.append(self.__upload_attachment(path)) logging.debug(f'Attachment "{path}" was uploaded') except Exception as exc: + if is_retriable_connection_error(exc): + raise logging.error(f'Upload attachment "{path}" status: {exc}') else: logging.error(f'File "{path}" was not found!') @@ -496,10 +546,12 @@ def get_configuration_id(self): return self.__config.get_configuration_id() @adapter_logger + @retry def __get_project(self) -> ProjectModel: return self.__projects_api.get_project_by_id(id=self.__config.get_project_id()) @adapter_logger + @retry def __get_workflow_by_id(self, workflow_id: str) -> WorkflowApiResult: return self.__workflows_api.api_v2_workflows_id_get(id=workflow_id) diff --git a/testit-python-commons/src/testit_python_commons/client/helpers/bulk_autotest_helper.py b/testit-python-commons/src/testit_python_commons/client/helpers/bulk_autotest_helper.py index 83449f37..5de63cd0 100644 --- a/testit-python-commons/src/testit_python_commons/client/helpers/bulk_autotest_helper.py +++ b/testit-python-commons/src/testit_python_commons/client/helpers/bulk_autotest_helper.py @@ -18,7 +18,9 @@ ThreadsForUpdateAndResult ) from testit_python_commons.services.logger import adapter_logger -from testit_python_commons.services.retry import retry +import testit_api_client + +from testit_python_commons.services.retry import is_non_retriable_api_exception, retry from testit_python_commons.utils.html_escape_utils import HtmlEscapeUtils from typing import Dict, List @@ -186,9 +188,20 @@ def __get_work_items_linked_to_autotest(self, autotest_global_id: str) -> List[A @adapter_logger @retry def __unlink_test_to_work_item(self, autotest_global_id: str, work_item_id: str): - self.__autotests_api.delete_auto_test_link_from_work_item( - id=autotest_global_id, - work_item_id=work_item_id) + try: + self.__autotests_api.delete_auto_test_link_from_work_item( + id=autotest_global_id, + work_item_id=work_item_id) + except testit_api_client.exceptions.ApiException as exc: + if is_non_retriable_api_exception(exc): + logging.warning( + 'Cannot unlink autotest %s from work item %s: %s', + autotest_global_id, + work_item_id, + exc, + ) + return + raise logging.debug(f'Autotest was unlinked with workItem "{work_item_id}" by global id "{autotest_global_id}') @@ -196,9 +209,20 @@ def __unlink_test_to_work_item(self, autotest_global_id: str, work_item_id: str) @adapter_logger @retry def __link_test_to_work_item(self, autotest_global_id: str, work_item_id: str): - self.__autotests_api.link_auto_test_to_work_item( - autotest_global_id, - link_auto_test_to_work_item_request=LinkAutoTestToWorkItemRequest(id=work_item_id)) + try: + self.__autotests_api.link_auto_test_to_work_item( + autotest_global_id, + link_auto_test_to_work_item_request=LinkAutoTestToWorkItemRequest(id=work_item_id)) + except testit_api_client.exceptions.ApiException as exc: + if is_non_retriable_api_exception(exc): + logging.warning( + 'Cannot link autotest %s to work item %s: %s', + autotest_global_id, + work_item_id, + exc, + ) + return + raise logging.debug(f'Autotest was linked with workItem "{work_item_id}" by global id "{autotest_global_id}') diff --git a/testit-python-commons/src/testit_python_commons/services/retry.py b/testit-python-commons/src/testit_python_commons/services/retry.py index 93b9bc13..2750a16d 100644 --- a/testit-python-commons/src/testit_python_commons/services/retry.py +++ b/testit-python-commons/src/testit_python_commons/services/retry.py @@ -1,29 +1,92 @@ import logging import random import time +from http.client import RemoteDisconnected import testit_api_client +import urllib3 + +CONNECTION_RETRIES = 3 +CONNECTION_RETRY_DELAY_SEC = 1 +API_EXCEPTION_RETRIES = 10 +NON_RETRIABLE_API_STATUS_CODES = (400, 404) + +_RETRIABLE_CONNECTION_TYPES = ( + urllib3.exceptions.ProtocolError, + urllib3.exceptions.NewConnectionError, + ConnectionError, + ConnectionResetError, + ConnectionAbortedError, + RemoteDisconnected, + TimeoutError, +) + + +def is_non_retriable_api_exception(exc: BaseException) -> bool: + return ( + isinstance(exc, testit_api_client.exceptions.ApiException) + and int(exc.status) in NON_RETRIABLE_API_STATUS_CODES + ) + + +def is_retriable_connection_error(exc: BaseException) -> bool: + seen = set() + current = exc + while current is not None and id(current) not in seen: + seen.add(id(current)) + if isinstance(current, _RETRIABLE_CONNECTION_TYPES): + return True + current = current.__cause__ or current.__context__ + return False + + +def _execute_with_connection_retries(func, args, kwargs): + connection_attempts = 0 + + while True: + try: + return func(*args, **kwargs) + except BaseException as e: + if not is_retriable_connection_error(e): + raise + + connection_attempts += 1 + logging.warning( + 'Connection error in %s (attempt %d/%d): %s', + func.__name__, + connection_attempts, + CONNECTION_RETRIES, + e, + ) + if connection_attempts > CONNECTION_RETRIES: + raise + + time.sleep(CONNECTION_RETRY_DELAY_SEC) + + +def retry_on_connection_error(func): + def retry_wrapper(*args, **kwargs): + return _execute_with_connection_retries(func, args, kwargs) + + return retry_wrapper def retry(func): def retry_wrapper(*args, **kwargs): - attempts = 0 - retries = 10 + api_attempts = 0 - while attempts < retries: + while True: try: - return func(*args, **kwargs) + return _execute_with_connection_retries(func, args, kwargs) except testit_api_client.exceptions.ApiException as e: - sleep_time = random.randrange(0, 100) - time.sleep(sleep_time/100) - attempts += 1 + if is_non_retriable_api_exception(e): + raise + api_attempts += 1 logging.error(e) - if e.status == '404': - attempts = retries - return - if e.status == '400': - attempts = retries - return + if api_attempts >= API_EXCEPTION_RETRIES: + raise + + time.sleep(random.randrange(0, 100) / 100) return retry_wrapper diff --git a/testit-python-commons/tests/services/test_retry.py b/testit-python-commons/tests/services/test_retry.py new file mode 100644 index 00000000..53131750 --- /dev/null +++ b/testit-python-commons/tests/services/test_retry.py @@ -0,0 +1,73 @@ +import urllib3 +from testit_api_client.exceptions import ApiException + +from testit_python_commons.services.retry import ( + CONNECTION_RETRIES, + is_non_retriable_api_exception, + is_retriable_connection_error, + retry, + retry_on_connection_error, +) + + +def test_is_retriable_connection_error_protocol_error(): + exc = urllib3.exceptions.ProtocolError( + 'Connection aborted.', + ConnectionResetError('Remote end closed connection without response'), + ) + assert is_retriable_connection_error(exc) + + +def test_retry_connection_error_retries_and_raises(): + calls = {'count': 0} + + @retry + def flaky(): + calls['count'] += 1 + raise urllib3.exceptions.ProtocolError('Connection aborted.') + + try: + flaky() + except urllib3.exceptions.ProtocolError: + pass + + assert calls['count'] == CONNECTION_RETRIES + 1 + + +def test_is_non_retriable_api_exception_for_not_found_subclass(): + from testit_api_client.exceptions import NotFoundException + + exc = NotFoundException(status=404, reason='Not Found') + assert is_non_retriable_api_exception(exc) + + +def test_retry_does_not_retry_on_404(): + calls = {'count': 0} + + @retry + def flaky(): + calls['count'] += 1 + raise ApiException(status=404, reason='Not Found') + + try: + flaky() + except ApiException: + pass + + assert calls['count'] == 1 + + +def test_retry_on_connection_error_does_not_retry_api_exception(): + calls = {'count': 0} + + @retry_on_connection_error + def flaky(): + calls['count'] += 1 + raise ApiException(status=500, reason='error') + + try: + flaky() + except ApiException: + pass + + assert calls['count'] == 1 diff --git a/update_versions.sh b/update_versions.sh index 9b6be65c..b13d17d1 100644 --- a/update_versions.sh +++ b/update_versions.sh @@ -1,6 +1,6 @@ #!/bin/bash -NEW_VERSION="4.1.14" +NEW_VERSION="4.2.0" TESTIT_API_CLIENT_VERSION="7.5.6" echo "Updating all adapters to version: $NEW_VERSION"