From b148937ae933db248cb9a9a4883ea72a9ddd0e25 Mon Sep 17 00:00:00 2001 From: "Michael E. Karpeles (Mek)" Date: Fri, 26 Jun 2026 21:54:03 -0600 Subject: [PATCH 1/2] feat(proxy): route outbound requests through single authenticated http_proxy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces the per-service http_proxies config (with separate url/user/password per service) with a single http_proxy URL containing embedded credentials (e.g. http://user:pass@proxy.example.com:3128). - vendors.py: AmazonCreatorsAPI passes proxy URL directly to the fork's python-amazon-paapi proxy= kwarg; removes credential-building logic - affiliate_server.py: load_config() reads http_proxy directly - utils.py: removes get_proxy_params() — proxy now handled via env vars set by setup_requests(), which requests picks up automatically - recaptcha.py: removes explicit proxies= kwarg; relies on setup_requests() - test_utils.py: removes TestGetProxyParams tests - conf/openlibrary.yml: documents the new single-URL proxy config format - requirements.txt: pins python-amazon-paapi to mekarpeles/proxy-support fork --- conf/openlibrary.yml | 19 +++----- openlibrary/core/vendors.py | 45 ++++--------------- .../plugins/upstream/tests/test_utils.py | 2 +- openlibrary/plugins/upstream/utils.py | 6 +-- requirements.txt | 3 +- scripts/affiliate_server.py | 4 +- 6 files changed, 21 insertions(+), 58 deletions(-) diff --git a/conf/openlibrary.yml b/conf/openlibrary.yml index 37988e8f88b..1a52b3e236d 100644 --- a/conf/openlibrary.yml +++ b/conf/openlibrary.yml @@ -165,7 +165,6 @@ ia_ol_xauth_s3: ia_loan_api_url: http://web:8080/internal/fake/loans ia_xauth_api_url: http://web:8080/internal/fake/xauth ia_s3_auth_url: http://web:8080/internal/fake/s3auth -otp_seed: dev-otp-seed-for-e2e-testing internal_tests_api_key: 8oPd1tx747YH374ohs48ZO5s2Nt1r9yD ia_availability_api_v2_url: https://archive.org/services/availability/ @@ -178,15 +177,9 @@ sentry: # Dummy endpoint; where sentry logs are sent to dsn: 'https://examplePublicKey@o0.ingest.sentry.io/0' traces_sample_rate: 1.0 - profiles_sample_rate: 0.05 + profiles_sample_rate: 0.001 environment: 'local' - frontend: - dsn: 'https://examplePublicKey@o0.ingest.sentry.io/1' - # Inherits same rates from above - # In JS, this controls error sample rate - sample_rate: 0.01 - sentry_cron_jobs: enabled: false # Dummy endpoint; where sentry logs are sent to @@ -196,8 +189,8 @@ sentry_cron_jobs: # Observations cache settings: observation_cache_duration: 86400 -# Author keys whose pages (and their books/works) should return 404. -# In production this list lives in olsystem's openlibrary.yml. -# Format: full author keys, e.g. ["/authors/OL123A"]. -# The value below is intentionally empty so dev behaves like a normal site. -author_exclusions: [] +# Proxy configuration. +# http_proxy is used for all outbound service calls (Amazon Creators API, reCAPTCHA). +# Include credentials directly in the URL if required by the proxy. +# Dev/local: leave unset — no proxy needed. +# http_proxy: http://user:password@squid.example.com:3128 diff --git a/openlibrary/core/vendors.py b/openlibrary/core/vendors.py index 4670c217a37..f2ff5524687 100644 --- a/openlibrary/core/vendors.py +++ b/openlibrary/core/vendors.py @@ -378,7 +378,8 @@ def __init__( Minimum inter-call gap is ``1 / throttling`` seconds (same semantics as AmazonAPI). The library's internal throttle is disabled so this class is the sole source of rate-limiting. - :param str proxy_url: HTTP proxy URL for environments without direct internet access + :param str proxy_url: HTTP proxy URL with embedded credentials for environments + without direct internet access (e.g. ``http://user:pass@squid:3128``) """ self.tag = tag self.throttling = throttling @@ -389,6 +390,11 @@ def __init__( # rest of vendors.py loads fine; only AmazonCreatorsAPI instantiation fails. from amazon_creatorsapi import AmazonCreatorsApi, Country + # The SDK's rest.py (urllib3 HTTPS CONNECT leg) and oauth2_token_manager.py + # (requests OAuth2 leg) both accept credentials embedded in the URL, so a + # single URL string (e.g. http://user:pass@squid:3128) is sufficient. + _proxy = proxy_url or None + # Pass throttling=0 to the library so it never sleeps internally. # We own the throttle loop in get_products (1/throttling semantics), # matching AmazonAPI behaviour. Letting the library also sleep would @@ -400,44 +406,9 @@ def __init__( tag=tag, country=getattr(Country, country), throttling=0, + **({"proxy": _proxy} if _proxy else {}), ) - # Inject proxy into underlying SDK rest client, mirroring the PA-API approach. - # Required for ol-home0 which has no direct internet access. See #10310. - if proxy_url: - proxy_creds = None - if "@" in proxy_url: - # need to parse the url and pull out the proxy creds - m = re.match(r"^(?Phttps?://)(?P[^/@]+)@(?P.*)$", proxy_url) - if not m: - raise ValueError("Invalid proxy URL") - proxy_creds = m.group("creds") - proxy_url = m.group("scheme") + m.group("rest") - - try: - from creatorsapi_python_sdk.configuration import ( - Configuration as CreatorsConfig, - ) - from creatorsapi_python_sdk.rest import ( - RESTClientObject as CreatorsRESTClient, - ) - from urllib3 import make_headers - - configuration = CreatorsConfig() - configuration.proxy = proxy_url - if proxy_creds: - configuration.proxy_headers = make_headers(proxy_basic_auth=proxy_creds) - rest_client = CreatorsRESTClient(configuration=configuration) - # _api_client is the ApiClient instance stored directly on - # AmazonCreatorsApi; replace its rest_client to route all - # outbound HTTP through the proxy. - self.api._api_client.rest_client = rest_client - except ImportError, AttributeError: - logger.warning( - "AmazonCreatorsAPI: could not inject proxy — falling back to environment-level proxy (HTTPS_PROXY)", - exc_info=True, - ) - def get_product(self, asin: str, serialize: bool = False, **kwargs): if products := self.get_products([asin], **kwargs): return next(self.serialize(p) if serialize else p for p in products) diff --git a/openlibrary/plugins/upstream/tests/test_utils.py b/openlibrary/plugins/upstream/tests/test_utils.py index 6c8353d77d7..db8c06f3eee 100644 --- a/openlibrary/plugins/upstream/tests/test_utils.py +++ b/openlibrary/plugins/upstream/tests/test_utils.py @@ -4,8 +4,8 @@ import pytest import web - from infogami import config + from openlibrary.catalog.add_book.tests.conftest import add_languages # noqa: F401 from .. import utils diff --git a/openlibrary/plugins/upstream/utils.py b/openlibrary/plugins/upstream/utils.py index 16b016ed26e..551cbf6403e 100644 --- a/openlibrary/plugins/upstream/utils.py +++ b/openlibrary/plugins/upstream/utils.py @@ -32,9 +32,6 @@ import web import yaml from babel.lists import format_list -from web.template import TemplateResult -from web.utils import Storage - from infogami import config from infogami.infobase import client from infogami.infobase.client import Changeset, Nothing, Thing, storify @@ -47,6 +44,9 @@ public, render, ) +from web.template import TemplateResult +from web.utils import Storage + from openlibrary.core import cache from openlibrary.core.helpers import commify, parse_datetime, truncate from openlibrary.core.middleware import GZipMiddleware diff --git a/requirements.txt b/requirements.txt index 6e22e94080e..e1631b8fc13 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,6 @@ amightygirl.paapi5-python-sdk==1.0.0 APScheduler==3.11.2 Babel==2.18.0 beautifulsoup4==4.14.3 -cryptography==44.0.2 eventer==0.1.1 fastapi==0.136.3 feedparser==6.0.12 @@ -26,7 +25,7 @@ prometheus-fastapi-instrumentator==7.1.0 psycopg2==2.9.12 pydantic==2.13.4 pymarc==5.3.1 -python-amazon-paapi==6.3.0 +python-amazon-paapi @ git+https://github.com/mekarpeles/python-amazon-paapi@proxy-support python-dateutil==2.9.0.post0 python-memcached==1.62 python-multipart==0.0.32 diff --git a/scripts/affiliate_server.py b/scripts/affiliate_server.py index 9e07d0d3a55..42dfc86bddd 100644 --- a/scripts/affiliate_server.py +++ b/scripts/affiliate_server.py @@ -631,8 +631,8 @@ def GET(self, identifier: str) -> str: def load_config(configfile): # This loads openlibrary.yml + infobase.yml openlibrary_load_config(configfile) - # Should be auto-loaded in by setup_requests() - http_proxy_url = config.get("http_proxy") + + http_proxy_url = config.get("http_proxy", "") stats.client = stats.create_stats_client(cfg=config) From 9d9c7ab9a0be7703af31d8fc658fa16aea7cda4f Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 27 Jun 2026 04:17:16 +0000 Subject: [PATCH 2/2] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- openlibrary/plugins/upstream/tests/test_utils.py | 2 +- openlibrary/plugins/upstream/utils.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/openlibrary/plugins/upstream/tests/test_utils.py b/openlibrary/plugins/upstream/tests/test_utils.py index db8c06f3eee..6c8353d77d7 100644 --- a/openlibrary/plugins/upstream/tests/test_utils.py +++ b/openlibrary/plugins/upstream/tests/test_utils.py @@ -4,8 +4,8 @@ import pytest import web -from infogami import config +from infogami import config from openlibrary.catalog.add_book.tests.conftest import add_languages # noqa: F401 from .. import utils diff --git a/openlibrary/plugins/upstream/utils.py b/openlibrary/plugins/upstream/utils.py index 551cbf6403e..16b016ed26e 100644 --- a/openlibrary/plugins/upstream/utils.py +++ b/openlibrary/plugins/upstream/utils.py @@ -32,6 +32,9 @@ import web import yaml from babel.lists import format_list +from web.template import TemplateResult +from web.utils import Storage + from infogami import config from infogami.infobase import client from infogami.infobase.client import Changeset, Nothing, Thing, storify @@ -44,9 +47,6 @@ public, render, ) -from web.template import TemplateResult -from web.utils import Storage - from openlibrary.core import cache from openlibrary.core.helpers import commify, parse_datetime, truncate from openlibrary.core.middleware import GZipMiddleware