diff --git a/conf/openlibrary.yml b/conf/openlibrary.yml index 37988e8f88b..1a52b3e236d 100644 --- a/conf/openlibrary.yml +++ b/conf/openlibrary.yml @@ -165,7 +165,6 @@ ia_ol_xauth_s3: ia_loan_api_url: http://web:8080/internal/fake/loans ia_xauth_api_url: http://web:8080/internal/fake/xauth ia_s3_auth_url: http://web:8080/internal/fake/s3auth -otp_seed: dev-otp-seed-for-e2e-testing internal_tests_api_key: 8oPd1tx747YH374ohs48ZO5s2Nt1r9yD ia_availability_api_v2_url: https://archive.org/services/availability/ @@ -178,15 +177,9 @@ sentry: # Dummy endpoint; where sentry logs are sent to dsn: 'https://examplePublicKey@o0.ingest.sentry.io/0' traces_sample_rate: 1.0 - profiles_sample_rate: 0.05 + profiles_sample_rate: 0.001 environment: 'local' - frontend: - dsn: 'https://examplePublicKey@o0.ingest.sentry.io/1' - # Inherits same rates from above - # In JS, this controls error sample rate - sample_rate: 0.01 - sentry_cron_jobs: enabled: false # Dummy endpoint; where sentry logs are sent to @@ -196,8 +189,8 @@ sentry_cron_jobs: # Observations cache settings: observation_cache_duration: 86400 -# Author keys whose pages (and their books/works) should return 404. -# In production this list lives in olsystem's openlibrary.yml. -# Format: full author keys, e.g. ["/authors/OL123A"]. -# The value below is intentionally empty so dev behaves like a normal site. -author_exclusions: [] +# Proxy configuration. +# http_proxy is used for all outbound service calls (Amazon Creators API, reCAPTCHA). +# Include credentials directly in the URL if required by the proxy. +# Dev/local: leave unset — no proxy needed. +# http_proxy: http://user:password@squid.example.com:3128 diff --git a/openlibrary/core/vendors.py b/openlibrary/core/vendors.py index 4670c217a37..f2ff5524687 100644 --- a/openlibrary/core/vendors.py +++ b/openlibrary/core/vendors.py @@ -378,7 +378,8 @@ def __init__( Minimum inter-call gap is ``1 / throttling`` seconds (same semantics as AmazonAPI). The library's internal throttle is disabled so this class is the sole source of rate-limiting. - :param str proxy_url: HTTP proxy URL for environments without direct internet access + :param str proxy_url: HTTP proxy URL with embedded credentials for environments + without direct internet access (e.g. ``http://user:pass@squid:3128``) """ self.tag = tag self.throttling = throttling @@ -389,6 +390,11 @@ def __init__( # rest of vendors.py loads fine; only AmazonCreatorsAPI instantiation fails. from amazon_creatorsapi import AmazonCreatorsApi, Country + # The SDK's rest.py (urllib3 HTTPS CONNECT leg) and oauth2_token_manager.py + # (requests OAuth2 leg) both accept credentials embedded in the URL, so a + # single URL string (e.g. http://user:pass@squid:3128) is sufficient. + _proxy = proxy_url or None + # Pass throttling=0 to the library so it never sleeps internally. # We own the throttle loop in get_products (1/throttling semantics), # matching AmazonAPI behaviour. Letting the library also sleep would @@ -400,44 +406,9 @@ def __init__( tag=tag, country=getattr(Country, country), throttling=0, + **({"proxy": _proxy} if _proxy else {}), ) - # Inject proxy into underlying SDK rest client, mirroring the PA-API approach. - # Required for ol-home0 which has no direct internet access. See #10310. - if proxy_url: - proxy_creds = None - if "@" in proxy_url: - # need to parse the url and pull out the proxy creds - m = re.match(r"^(?Phttps?://)(?P[^/@]+)@(?P.*)$", proxy_url) - if not m: - raise ValueError("Invalid proxy URL") - proxy_creds = m.group("creds") - proxy_url = m.group("scheme") + m.group("rest") - - try: - from creatorsapi_python_sdk.configuration import ( - Configuration as CreatorsConfig, - ) - from creatorsapi_python_sdk.rest import ( - RESTClientObject as CreatorsRESTClient, - ) - from urllib3 import make_headers - - configuration = CreatorsConfig() - configuration.proxy = proxy_url - if proxy_creds: - configuration.proxy_headers = make_headers(proxy_basic_auth=proxy_creds) - rest_client = CreatorsRESTClient(configuration=configuration) - # _api_client is the ApiClient instance stored directly on - # AmazonCreatorsApi; replace its rest_client to route all - # outbound HTTP through the proxy. - self.api._api_client.rest_client = rest_client - except ImportError, AttributeError: - logger.warning( - "AmazonCreatorsAPI: could not inject proxy — falling back to environment-level proxy (HTTPS_PROXY)", - exc_info=True, - ) - def get_product(self, asin: str, serialize: bool = False, **kwargs): if products := self.get_products([asin], **kwargs): return next(self.serialize(p) if serialize else p for p in products) diff --git a/requirements.txt b/requirements.txt index 6e22e94080e..e1631b8fc13 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,6 @@ amightygirl.paapi5-python-sdk==1.0.0 APScheduler==3.11.2 Babel==2.18.0 beautifulsoup4==4.14.3 -cryptography==44.0.2 eventer==0.1.1 fastapi==0.136.3 feedparser==6.0.12 @@ -26,7 +25,7 @@ prometheus-fastapi-instrumentator==7.1.0 psycopg2==2.9.12 pydantic==2.13.4 pymarc==5.3.1 -python-amazon-paapi==6.3.0 +python-amazon-paapi @ git+https://github.com/mekarpeles/python-amazon-paapi@proxy-support python-dateutil==2.9.0.post0 python-memcached==1.62 python-multipart==0.0.32 diff --git a/scripts/affiliate_server.py b/scripts/affiliate_server.py index 9e07d0d3a55..42dfc86bddd 100644 --- a/scripts/affiliate_server.py +++ b/scripts/affiliate_server.py @@ -631,8 +631,8 @@ def GET(self, identifier: str) -> str: def load_config(configfile): # This loads openlibrary.yml + infobase.yml openlibrary_load_config(configfile) - # Should be auto-loaded in by setup_requests() - http_proxy_url = config.get("http_proxy") + + http_proxy_url = config.get("http_proxy", "") stats.client = stats.create_stats_client(cfg=config)