feat(proxy): route outbound requests through single authenticated http_proxy

conf/openlibrary.yml

@@ -165,7 +165,6 @@ ia_ol_xauth_s3:
 ia_loan_api_url: http://web:8080/internal/fake/loans
 ia_xauth_api_url: http://web:8080/internal/fake/xauth
 ia_s3_auth_url: http://web:8080/internal/fake/s3auth
-otp_seed: dev-otp-seed-for-e2e-testing
 
 internal_tests_api_key: 8oPd1tx747YH374ohs48ZO5s2Nt1r9yD
 ia_availability_api_v2_url: https://archive.org/services/availability/
@@ -178,15 +177,9 @@ sentry:
     # Dummy endpoint; where sentry logs are sent to
     dsn: 'https://examplePublicKey@o0.ingest.sentry.io/0'
     traces_sample_rate: 1.0
-    profiles_sample_rate: 0.05
+    profiles_sample_rate: 0.001
     environment: 'local'
 
-    frontend:
-        dsn: 'https://examplePublicKey@o0.ingest.sentry.io/1'
-        # Inherits same rates from above
-        # In JS, this controls error sample rate
-        sample_rate: 0.01
-
 sentry_cron_jobs:
     enabled: false
     # Dummy endpoint; where sentry logs are sent to
@@ -196,8 +189,8 @@ sentry_cron_jobs:
 # Observations cache settings:
 observation_cache_duration: 86400
 
-# Author keys whose pages (and their books/works) should return 404.
-# In production this list lives in olsystem's openlibrary.yml.
-# Format: full author keys, e.g. ["/authors/OL123A"].
-# The value below is intentionally empty so dev behaves like a normal site.
-author_exclusions: []
+# Proxy configuration.
+# http_proxy is used for all outbound service calls (Amazon Creators API, reCAPTCHA).
+# Include credentials directly in the URL if required by the proxy.
+# Dev/local: leave unset — no proxy needed.
+# http_proxy: http://user:password@squid.example.com:3128

openlibrary/core/vendors.py

@@ -378,7 +378,8 @@ def __init__(
             Minimum inter-call gap is ``1 / throttling`` seconds (same semantics as
             AmazonAPI).  The library's internal throttle is disabled so this class
             is the sole source of rate-limiting.
-        :param str proxy_url: HTTP proxy URL for environments without direct internet access
+        :param str proxy_url: HTTP proxy URL with embedded credentials for environments
+            without direct internet access (e.g. ``http://user:pass@squid:3128``)
         """
         self.tag = tag
         self.throttling = throttling
@@ -389,6 +390,11 @@ def __init__(
         # rest of vendors.py loads fine; only AmazonCreatorsAPI instantiation fails.
         from amazon_creatorsapi import AmazonCreatorsApi, Country
 
+        # The SDK's rest.py (urllib3 HTTPS CONNECT leg) and oauth2_token_manager.py
+        # (requests OAuth2 leg) both accept credentials embedded in the URL, so a
+        # single URL string (e.g. http://user:pass@squid:3128) is sufficient.
+        _proxy = proxy_url or None
+
         # Pass throttling=0 to the library so it never sleeps internally.
         # We own the throttle loop in get_products (1/throttling semantics),
         # matching AmazonAPI behaviour. Letting the library also sleep would
@@ -400,44 +406,9 @@ def __init__(
             tag=tag,
             country=getattr(Country, country),
             throttling=0,
+            **({"proxy": _proxy} if _proxy else {}),
         )
 
-        # Inject proxy into underlying SDK rest client, mirroring the PA-API approach.
-        # Required for ol-home0 which has no direct internet access. See #10310.
-        if proxy_url:
-            proxy_creds = None
-            if "@" in proxy_url:
-                # need to parse the url and pull out the proxy creds
-                m = re.match(r"^(?P<scheme>https?://)(?P<creds>[^/@]+)@(?P<rest>.*)$", proxy_url)
-                if not m:
-                    raise ValueError("Invalid proxy URL")
-                proxy_creds = m.group("creds")
-                proxy_url = m.group("scheme") + m.group("rest")
-
-            try:
-                from creatorsapi_python_sdk.configuration import (
-                    Configuration as CreatorsConfig,
-                )
-                from creatorsapi_python_sdk.rest import (
-                    RESTClientObject as CreatorsRESTClient,
-                )
-                from urllib3 import make_headers
-
-                configuration = CreatorsConfig()
-                configuration.proxy = proxy_url
-                if proxy_creds:
-                    configuration.proxy_headers = make_headers(proxy_basic_auth=proxy_creds)
-                rest_client = CreatorsRESTClient(configuration=configuration)
-                # _api_client is the ApiClient instance stored directly on
-                # AmazonCreatorsApi; replace its rest_client to route all
-                # outbound HTTP through the proxy.
-                self.api._api_client.rest_client = rest_client
-            except ImportError, AttributeError:
-                logger.warning(
-                    "AmazonCreatorsAPI: could not inject proxy — falling back to environment-level proxy (HTTPS_PROXY)",
-                    exc_info=True,
-                )
-
     def get_product(self, asin: str, serialize: bool = False, **kwargs):
         if products := self.get_products([asin], **kwargs):
             return next(self.serialize(p) if serialize else p for p in products)

requirements.txt

@@ -3,7 +3,6 @@ amightygirl.paapi5-python-sdk==1.0.0
 APScheduler==3.11.2
 Babel==2.18.0
 beautifulsoup4==4.14.3
-cryptography==44.0.2
 eventer==0.1.1
 fastapi==0.136.3
 feedparser==6.0.12
@@ -26,7 +25,7 @@ prometheus-fastapi-instrumentator==7.1.0
 psycopg2==2.9.12
 pydantic==2.13.4
 pymarc==5.3.1
-python-amazon-paapi==6.3.0
+python-amazon-paapi @ git+https://github.com/mekarpeles/python-amazon-paapi@proxy-support
 python-dateutil==2.9.0.post0
 python-memcached==1.62
 python-multipart==0.0.32

scripts/affiliate_server.py

@@ -631,8 +631,8 @@ def GET(self, identifier: str) -> str:
 def load_config(configfile):
     # This loads openlibrary.yml + infobase.yml
     openlibrary_load_config(configfile)
-    # Should be auto-loaded in by setup_requests()
-    http_proxy_url = config.get("http_proxy")
+
+    http_proxy_url = config.get("http_proxy", "")
 
     stats.client = stats.create_stats_client(cfg=config)