diff --git a/.bumpversion.cfg b/.bumpversion.cfg new file mode 100644 index 0000000..86c2681 --- /dev/null +++ b/.bumpversion.cfg @@ -0,0 +1,17 @@ +[bumpversion] +current_version = 0.1.0 +parse = (?P\d+)\.(?P\d+)\.(?P\d+)(-(?Prc)(?P\d+))? +serialize = + {major}.{minor}.{patch}-{release}{rc_version} + {major}.{minor}.{patch} +files = pypi_boilerplate/__init__.py +message = Release version {new_version} +tag_name = {new_version} +tag = true +commit = true + +[bumpversion:part:release] +optional_value = production +values = + rc + production diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..00062ca --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,2 @@ +include *.txt +recursive-exclude tests * diff --git a/README.rst b/README.rst index 548a7d4..b76fd53 100644 --- a/README.rst +++ b/README.rst @@ -88,9 +88,11 @@ Contributors - Michael Ortali - Adam Griffiths - Raphael Antonmattei +- John Penner .. _xethorn: github.com/xethorn .. _rantonmattei: github.com/rantonmattei +.. _someboredkiddo: github.com/someboredkiddo .. |Build Status| image:: https://travis-ci.org/xethorn/garcon.svg :target: https://travis-ci.org/xethorn/garcon diff --git a/garcon/__init__.py b/garcon/__init__.py index 39fe762..c154567 100644 --- a/garcon/__init__.py +++ b/garcon/__init__.py @@ -1,2 +1,4 @@ -from pkgutil import extend_path -__path__ = extend_path(__path__, __name__) +from pkgutil import extend_path +__path__ = extend_path(__path__, __name__) + +__version__ = '0.1.0' diff --git a/garcon/activity.py b/garcon/activity.py index 10c26da..e10c227 100755 --- a/garcon/activity.py +++ b/garcon/activity.py @@ -38,6 +38,8 @@ """ from threading import Thread +import backoff +import boto.exception as boto_exception import boto.swf.layer2 as swf import itertools import json @@ -245,6 +247,24 @@ class Activity(swf.ActivityWorker, log.GarconLogger): version = '1.0' task_list = None + @backoff.on_exception( + backoff.expo, + boto_exception.SWFResponseError, + max_tries=5, + giveup=utils.non_throttle_error, + on_backoff=utils.throttle_backoff_handler, + jitter=backoff.full_jitter) + def poll_for_activity(self): + """Runs Activity Poll. + + If a SWF throttling exception is raised during a poll, the poll will + be retried up to 5 times using exponential backoff algorithm. + + Upgrading to boto3 would make this retry logic redundant. + """ + + return self.poll() + def run(self): """Activity Runner. @@ -254,7 +274,7 @@ def run(self): """ try: - activity_task = self.poll() + activity_task = self.poll_for_activity() except Exception as error: # Catch exceptions raised during poll() to avoid an Activity thread # dying & worker daemon unable to process the affected Activity. diff --git a/garcon/utils.py b/garcon/utils.py index 372119a..3cba14e 100644 --- a/garcon/utils.py +++ b/garcon/utils.py @@ -29,3 +29,38 @@ def create_dictionary_key(dictionary): for (key, val) in sorted_dict]) return hashlib.sha1(key_parts.encode('utf-8')).hexdigest() + +def non_throttle_error(swf_response_error): + """Activity Runner. + + Determine whether SWF Exception was a throttle or a different error. + + Args: + error: boto.exception.SWFResponseError instance. + Return: + bool: True if SWFResponseError was a throttle, False otherwise. + """ + + return swf_response_error.error_code != 'ThrottlingException' + +def throttle_backoff_handler(details): + """Callback to be used when a throttle backoff is invoked. + + For more details see: https://github.com/litl/backoff/#event-handlers + + Args: + dictionary (dict): Details of the backoff invocation. Valid keys + include: + target: reference to the function or method being invoked. + args: positional arguments to func. + kwargs: keyword arguments to func. + tries: number of invocation tries so far. + wait: seconds to wait (on_backoff handler only). + value: value triggering backoff (on_predicate decorator only). + """ + + activity = details['args'][0] + activity.logger.info( + 'Throttle Exception occurred on try {}. ' + 'Sleeping for {} seconds'.format( + details['tries'], details['wait'])) diff --git a/requirements.txt b/requirements.txt index b0096e6..8dab7c7 100755 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,6 @@ +backoff==1.4.3 boto==2.35.1 +bumpversion==0.5.3 pytest-cov==1.8.1 pytest==2.6.4 -python-coveralls==2.4.3 \ No newline at end of file +python-coveralls==2.4.3 diff --git a/setup.py b/setup.py index 2dc21c9..02494e8 100755 --- a/setup.py +++ b/setup.py @@ -3,6 +3,9 @@ from codecs import open from os import path +from garcon import __version__ + + here = path.abspath(path.dirname(__file__)) # Get the long description from the README file with open(path.join(here, 'README.rst'), encoding='utf-8') as f: @@ -10,18 +13,18 @@ setup( - name='Garcon', - version='0.3.4', - url='https://github.com/xethorn/garcon/', - author='Michael Ortali', - author_email='hello@xethorn.net', + name='owsgarcon', + version=__version__, + url='https://github.com/theorchard/garcon/', + author='The Orchard', + author_email='webdev@theorchard.com', description=( - 'Lightweight library for AWS SWF.'), + 'Orchard Fork of Lightweight library for AWS SWF.'), long_description=long_description, license='MIT', packages=find_packages(), include_package_data=True, - install_requires=['boto'], + install_requires=['boto', 'backoff'], zip_safe=False, classifiers=[ 'Programming Language :: Python :: 2.7', diff --git a/tests/test_activity.py b/tests/test_activity.py index 7213801..255a696 100755 --- a/tests/test_activity.py +++ b/tests/test_activity.py @@ -14,6 +14,8 @@ from garcon import utils from tests.fixtures import decider +import boto.exception as boto_exception + def activity_run( monkeypatch, poll=None, complete=None, fail=None, execute=None): @@ -62,6 +64,52 @@ def poll(): return dict(activityId='something') +def test_poll_for_activity(monkeypatch, poll=poll): + """Test that poll_for_activity successfully polls. + """ + + current_activity = activity_run(monkeypatch, poll) + current_activity.poll.return_value = 'foo' + + activity_results = current_activity.poll_for_activity() + assert current_activity.poll.called + assert activity_results == 'foo' + + +def test_poll_for_activity_throttle_retry(monkeypatch, poll=poll): + """Test that SWF throttles are retried during polling. + """ + + current_activity = activity_run(monkeypatch, poll) + + response_status = 400 + response_reason = 'Bad Request' + reponse_body = ( + '{"__type": "com.amazon.coral.availability#ThrottlingException",' + '"message": "Rate exceeded"}') + json_body = json.loads(reponse_body) + exception = boto_exception.SWFResponseError( + response_status, response_reason, body=json_body) + current_activity.poll.side_effect = exception + + with pytest.raises(boto_exception.SWFResponseError): + current_activity.poll_for_activity() + assert current_activity.poll.call_count == 5 + + +def test_poll_for_activity_error(monkeypatch, poll=poll): + """Test that non-throttle errors during poll are thrown. + """ + + current_activity = activity_run(monkeypatch, poll) + + exception = Exception() + current_activity.poll.side_effect = exception + + with pytest.raises(Exception): + current_activity.poll_for_activity() + + def test_run_activity(monkeypatch, poll): """Run an activity. """ diff --git a/tests/test_utils.py b/tests/test_utils.py index 69b702f..44564c7 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,5 +1,11 @@ -import pytest +try: + from unittest.mock import MagicMock +except: + from mock import MagicMock +import boto.exception as boto_exception import datetime +import pytest +import json from garcon import utils @@ -21,7 +27,18 @@ def test_create_dictionary_key_with_empty_dict(): def test_create_dictionary_key(): - """Try craeting a unique key from a dict. + """Try creating a unique key from a dict. + """ + + values = [ + dict(foo=10), + dict(foo2=datetime.datetime.now())] + + for value in values: + assert len(utils.create_dictionary_key(value)) == 40 + +def test_create_dictionary_key(): + """Try creating a unique key from a dict. """ values = [ @@ -30,3 +47,42 @@ def test_create_dictionary_key(): for value in values: assert len(utils.create_dictionary_key(value)) == 40 + + +def test_non_throttle_error(): + """Assert SWF error is evaluated as non-throttle error properly. + """ + + response_status = 400 + response_reason = 'Bad Request' + reponse_body = ( + '{"__type": "com.amazon.coral.availability#ThrottlingException",' + '"message": "Rate exceeded"}') + json_body = json.loads(reponse_body) + exception = boto_exception.SWFResponseError( + response_status, response_reason, body=json_body) + result = utils.non_throttle_error(exception) + assert not utils.non_throttle_error(exception) + + reponse_body = ( + '{"__type": "com.amazon.coral.availability#OtheException",' + '"message": "Rate exceeded"}') + json_body = json.loads(reponse_body) + exception = boto_exception.SWFResponseError( + response_status, response_reason, body=json_body) + assert utils.non_throttle_error(exception) + +def test_throttle_backoff_handler(): + """Assert backoff is logged correctly. + """ + + mock_activity = MagicMock() + details = dict( + args=(mock_activity,), + tries=5, + wait=10) + utils.throttle_backoff_handler(details) + mock_activity.logger.info.assert_called_with( + 'Throttle Exception occurred on try {}. ' + 'Sleeping for {} seconds'.format( + details['tries'], details['wait']))