diff --git a/nodes/src/nodes/tool_gmail/gmail_client.py b/nodes/src/nodes/tool_gmail/gmail_client.py index c2254c4a1..9f0c132b1 100644 --- a/nodes/src/nodes/tool_gmail/gmail_client.py +++ b/nodes/src/nodes/tool_gmail/gmail_client.py @@ -90,6 +90,40 @@ def resolve_refresh_url(token_url: object) -> str | None: # HTTP status codes worth retrying (rate-limit + transient server errors). _RETRY_STATUSES = {429, 500, 502, 503, 504} +# Structured error reasons that make a 403 a retryable rate-limit/quota error +# rather than a hard auth/scope failure. +_RATE_LIMIT_403_REASONS = frozenset({'userRateLimitExceeded', 'rateLimitExceeded', 'quotaExceeded'}) + + +def _is_rate_limit_403(exc: object) -> bool: + """True when a 403 is a retryable rate-limit/quota error, not an auth/scope 403. + + Gmail reports the specific cause in the structured error body + (``error.status`` or ``error.errors[].reason``). Only rate-limit/quota + reasons are transient and worth retrying; scope/permission 403s must fail + fast. Any parse failure conservatively returns False (treat as non-retryable). + """ + content = getattr(exc, 'content', None) + if not content: + return False + try: + if isinstance(content, (bytes, bytearray)): + content = content.decode('utf-8', 'replace') + body = json.loads(content) + except (ValueError, TypeError): + return False + + error = body.get('error') if isinstance(body, dict) else None + if not isinstance(error, dict): + return False + if error.get('status') in _RATE_LIMIT_403_REASONS: + return True + for item in error.get('errors') or []: + if isinstance(item, dict) and item.get('reason') in _RATE_LIMIT_403_REASONS: + return True + return False + + # Google's full-access Gmail scope — a superset of all granular Gmail scopes. _GMAIL_FULL_SCOPE = 'https://mail.google.com/' @@ -242,7 +276,11 @@ def execute(request: Any) -> dict: return request.execute() or {} except Exception as exc: # googleapiclient.errors.HttpError and transport errors status = getattr(getattr(exc, 'resp', None), 'status', None) - if status and int(status) in _RETRY_STATUSES and attempt < 3: + istatus = int(status) if status else None + # Retry transient statuses, plus 403s that the structured error body + # identifies as rate-limit/quota (not scope/permission) failures. + retryable = istatus in _RETRY_STATUSES or (istatus == 403 and _is_rate_limit_403(exc)) + if retryable and attempt < 3: _time.sleep(min(base_delay * (2**attempt), 60.0)) continue detail = getattr(exc, 'reason', None) or str(exc) diff --git a/nodes/test/tool_gmail/test_gmail.py b/nodes/test/tool_gmail/test_gmail.py index 814add131..043820ed7 100644 --- a/nodes/test/tool_gmail/test_gmail.py +++ b/nodes/test/tool_gmail/test_gmail.py @@ -241,6 +241,65 @@ def test_error_path_wraps_http_error(): assert 'Gmail request failed' in str(exc.value) +# --------------------------------------------------------------------------- +# execute() retry: rate-limit 403s are retryable, scope 403s are not (#1560) +# --------------------------------------------------------------------------- + + +class _FakeHttpError(Exception): + """Stand-in for googleapiclient.errors.HttpError with a status + JSON body.""" + + def __init__(self, status, content=b''): + super().__init__(f'HTTP {status}') + self.resp = types.SimpleNamespace(status=status) + self.content = content + self.reason = 'Forbidden' + + +def _rate_limit_body(reason): + return json.dumps({'error': {'errors': [{'reason': reason}], 'status': 'PERMISSION_DENIED'}}).encode() + + +@pytest.mark.parametrize('reason', ['userRateLimitExceeded', 'rateLimitExceeded', 'quotaExceeded']) +def test_is_rate_limit_403_detects_reasons(reason): + assert gmail_client._is_rate_limit_403(_FakeHttpError(403, _rate_limit_body(reason))) + + +def test_is_rate_limit_403_ignores_scope_and_garbage(): + assert not gmail_client._is_rate_limit_403(_FakeHttpError(403, _rate_limit_body('insufficientPermissions'))) + assert not gmail_client._is_rate_limit_403(_FakeHttpError(403, b'not json')) + assert not gmail_client._is_rate_limit_403(_FakeHttpError(403, b'')) + + +def test_execute_retries_rate_limit_403_then_succeeds(monkeypatch): + monkeypatch.setattr(gmail_client._time, 'sleep', lambda *a, **kw: None) + calls = {'n': 0} + + class _R: + def execute(self): + calls['n'] += 1 + if calls['n'] < 3: + raise _FakeHttpError(403, _rate_limit_body('rateLimitExceeded')) + return {'ok': True} + + assert gmail_client.execute(_R()) == {'ok': True} + assert calls['n'] == 3 + + +def test_execute_does_not_retry_scope_403(monkeypatch): + monkeypatch.setattr(gmail_client._time, 'sleep', lambda *a, **kw: None) + calls = {'n': 0} + + class _R: + def execute(self): + calls['n'] += 1 + raise _FakeHttpError(403, _rate_limit_body('insufficientPermissions')) + + with pytest.raises(ValueError, match='403'): + gmail_client.execute(_R()) + assert calls['n'] == 1 # failed fast, no retry + + # --------------------------------------------------------------------------- # Write gating # ---------------------------------------------------------------------------