From 0c407e3614fa849985bca1d0e6be27e20ef3251f Mon Sep 17 00:00:00 2001 From: "Michael E. Karpeles" Date: Mon, 11 May 2026 22:52:16 -0600 Subject: [PATCH 1/7] feat(proxy): add per-service proxy auth via get_proxy_params() The global setup_requests() approach (HTTP_PROXY env vars, no auth) is insufficient for services that require per-service proxy credentials. Adds get_proxy_params(service_tag) to utils.py: reads from a new http_proxies config section (url/user/password per service) and returns a requests-compatible proxies dict with credentials embedded in the URL, or None to fall through to the global env var default. Updates recaptcha to pass proxies=get_proxy_params("recaptcha") so that service-specific proxy auth is used when configured. Updates affiliate_server load_config to read amazon proxy settings from http_proxies.amazon, falling back to the legacy http_proxy / http_proxy_creds flat keys for backward compatibility. Documents the new config section in conf/openlibrary.yml. Closes #12715 [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- conf/openlibrary.yml | 15 +++++++++++ openlibrary/plugins/recaptcha/recaptcha.py | 4 ++- openlibrary/plugins/upstream/utils.py | 31 ++++++++++++++++++++++ scripts/affiliate_server.py | 11 ++++++-- 4 files changed, 58 insertions(+), 3 deletions(-) diff --git a/conf/openlibrary.yml b/conf/openlibrary.yml index e75fd373088..68d6c5db1ee 100644 --- a/conf/openlibrary.yml +++ b/conf/openlibrary.yml @@ -190,3 +190,18 @@ sentry_cron_jobs: # Observations cache settings: observation_cache_duration: 86400 + +# Proxy configuration. +# http_proxy sets the global default (no auth) via HTTP_PROXY/HTTPS_PROXY env vars. +# http_proxies overrides per service with credentials; each entry has url/user/password. +# Dev/local: leave both unset — no proxy needed. +# http_proxy: http://squid.example.com:3128 +# http_proxies: +# recaptcha: +# url: http://squid.example.com:3128 +# user: '' +# password: '' +# amazon: +# url: http://squid.example.com:3128 +# user: '' +# password: '' diff --git a/openlibrary/plugins/recaptcha/recaptcha.py b/openlibrary/plugins/recaptcha/recaptcha.py index 08d09a04a6f..ff47b70414f 100644 --- a/openlibrary/plugins/recaptcha/recaptcha.py +++ b/openlibrary/plugins/recaptcha/recaptcha.py @@ -7,6 +7,8 @@ from infogami import config +from openlibrary.plugins.upstream.utils import get_proxy_params + logger = logging.getLogger("openlibrary") INVALIDATING_ERRORS = [ @@ -44,7 +46,7 @@ def accept_error(error_codes: list[str]) -> bool: } try: - r = requests.get(url, params=params, timeout=3) + r = requests.get(url, params=params, timeout=3, proxies=get_proxy_params("recaptcha")) except requests.exceptions.RequestException: logger.exception("Recaptcha call failed: letting user through") return True diff --git a/openlibrary/plugins/upstream/utils.py b/openlibrary/plugins/upstream/utils.py index 82eb0a96f91..eec4505ec7b 100644 --- a/openlibrary/plugins/upstream/utils.py +++ b/openlibrary/plugins/upstream/utils.py @@ -1623,6 +1623,37 @@ def setup_requests(config=config) -> None: logger.info("Requests set up") +def get_proxy_params(service_tag: str) -> dict[str, str] | None: + """Return a requests-compatible proxies dict for a service requiring proxy auth. + + Reads from the ``http_proxies`` config section. Each entry may have: + url: proxy base URL + user: proxy username + password: proxy password + + Returns None when no service-specific config exists so that callers can + pass the result directly as ``proxies=`` to requests — None means requests + will fall back to the global HTTP_PROXY/HTTPS_PROXY env vars set by + setup_requests(). + """ + service = config.get("http_proxies", {}).get(service_tag) + if not service: + return None + + proxy_url = service.get("url", "") + user = service.get("user", "") + password = service.get("password", "") + + if user and password and proxy_url: + parsed = urlparse(proxy_url) + netloc = f"{user}:{password}@{parsed.hostname}" + if parsed.port: + netloc += f":{parsed.port}" + proxy_url = urlunparse(parsed._replace(netloc=netloc)) + + return {"http": proxy_url, "https": proxy_url} if proxy_url else None + + def setup() -> None: """Do required initialization""" # monkey-patch get_markdown to use OL Flavored Markdown diff --git a/scripts/affiliate_server.py b/scripts/affiliate_server.py index 00c648bcd03..00ac9a2f2e7 100644 --- a/scripts/affiliate_server.py +++ b/scripts/affiliate_server.py @@ -638,8 +638,15 @@ def GET(self, identifier: str) -> str: def load_config(configfile): # This loads openlibrary.yml + infobase.yml openlibrary_load_config(configfile) - http_proxy_url = config.get("http_proxy") - http_proxy_creds = config.get("http_proxy_creds") + + # Prefer per-service proxy config under http_proxies.amazon; fall back to the + # legacy flat keys http_proxy / http_proxy_creds for backward compatibility. + amazon_proxy_cfg = config.get("http_proxies", {}).get("amazon", {}) + http_proxy_url = amazon_proxy_cfg.get("url") or config.get("http_proxy") + if amazon_proxy_cfg.get("user"): + http_proxy_creds = f"{amazon_proxy_cfg['user']}:{amazon_proxy_cfg.get('password', '')}" + else: + http_proxy_creds = config.get("http_proxy_creds", "") stats.client = stats.create_stats_client(cfg=config) From a194263b2721b8c58c2c4f3b43c81bb097baaa26 Mon Sep 17 00:00:00 2001 From: "Michael E. Karpeles" Date: Mon, 11 May 2026 23:02:40 -0600 Subject: [PATCH 2/7] fix(proxy): url-encode credentials in get_proxy_params; add tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit URL-encode user and password before embedding in the proxy netloc so special characters (@ : % etc.) don't break the URL. Also relax the auth guard to inject credentials whenever user is set, even when password is empty. Adds TestGetProxyParams unit tests covering: no config → None, unknown service → None, url-only, url+auth, and special-char encoding. [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../plugins/upstream/tests/test_utils.py | 48 +++++++++++++++++++ openlibrary/plugins/upstream/utils.py | 4 +- 2 files changed, 50 insertions(+), 2 deletions(-) diff --git a/openlibrary/plugins/upstream/tests/test_utils.py b/openlibrary/plugins/upstream/tests/test_utils.py index b9acbfee836..7a36a159346 100644 --- a/openlibrary/plugins/upstream/tests/test_utils.py +++ b/openlibrary/plugins/upstream/tests/test_utils.py @@ -383,3 +383,51 @@ def test_get_language_name(add_languages): # noqa: F811 assert utils.get_language_name("/languages/ger", "en") == "German" # Falls back to name when translation missing for requested language assert utils.get_language_name("/languages/ger", "fr") == "Deutsch" + + +class TestGetProxyParams: + def test_no_http_proxies_config(self): + with patch("openlibrary.plugins.upstream.utils.config") as mock_config: + mock_config.get.return_value = {} + assert utils.get_proxy_params("recaptcha") is None + + def test_unknown_service_tag(self): + with patch("openlibrary.plugins.upstream.utils.config") as mock_config: + mock_config.get.return_value = {"amazon": {"url": "http://proxy:3128"}} + assert utils.get_proxy_params("recaptcha") is None + + def test_url_only_no_auth(self): + with patch("openlibrary.plugins.upstream.utils.config") as mock_config: + mock_config.get.return_value = {"recaptcha": {"url": "http://proxy:3128"}} + result = utils.get_proxy_params("recaptcha") + assert result == {"http": "http://proxy:3128", "https": "http://proxy:3128"} + + def test_url_with_auth(self): + with patch("openlibrary.plugins.upstream.utils.config") as mock_config: + mock_config.get.return_value = { + "recaptcha": { + "url": "http://proxy:3128", + "user": "myuser", + "password": "mypass", + } + } + result = utils.get_proxy_params("recaptcha") + assert result == { + "http": "http://myuser:mypass@proxy:3128", + "https": "http://myuser:mypass@proxy:3128", + } + + def test_special_chars_in_credentials_are_encoded(self): + with patch("openlibrary.plugins.upstream.utils.config") as mock_config: + mock_config.get.return_value = { + "recaptcha": { + "url": "http://proxy:3128", + "user": "u@ser", + "password": "p@ss:word", + } + } + result = utils.get_proxy_params("recaptcha") + assert result == { + "http": "http://u%40ser:p%40ss%3Aword@proxy:3128", + "https": "http://u%40ser:p%40ss%3Aword@proxy:3128", + } diff --git a/openlibrary/plugins/upstream/utils.py b/openlibrary/plugins/upstream/utils.py index eec4505ec7b..0531540c45b 100644 --- a/openlibrary/plugins/upstream/utils.py +++ b/openlibrary/plugins/upstream/utils.py @@ -1644,9 +1644,9 @@ def get_proxy_params(service_tag: str) -> dict[str, str] | None: user = service.get("user", "") password = service.get("password", "") - if user and password and proxy_url: + if user and proxy_url: parsed = urlparse(proxy_url) - netloc = f"{user}:{password}@{parsed.hostname}" + netloc = f"{quote(user, safe='')}:{quote(password, safe='')}@{parsed.hostname}" if parsed.port: netloc += f":{parsed.port}" proxy_url = urlunparse(parsed._replace(netloc=netloc)) From ed7d782d2ed5d1b9c486f5ef124bf0e23fcdc709 Mon Sep 17 00:00:00 2001 From: "Michael E. Karpeles" Date: Tue, 12 May 2026 10:05:00 -0600 Subject: [PATCH 3/7] fix(proxy): route Amazon OAuth token refresh through authenticated proxy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit OAuth2TokenManager.refresh_token() calls bare requests.post(), which reads HTTP_PROXY from the environment. After the per-service proxy config lands (http_proxy = bare squid URL, http_proxies.amazon = creds), that env var no longer carries auth → Amazon token endpoint returns 403. Inject a _ProxyAwareTokenManager subclass into the SDK's ApiClient when proxy_creds are present. The subclass overrides refresh_token() to use a requests.Session with the authenticated proxy URL embedded directly, bypassing env-var lookup entirely. The existing rest_client injection (urllib3 / RESTClientObject) already handles all other API calls; this commit covers the one OAuth path that goes through requests directly. --- openlibrary/core/vendors.py | 87 +++++++++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) diff --git a/openlibrary/core/vendors.py b/openlibrary/core/vendors.py index a038af74625..44d1a6a5aef 100644 --- a/openlibrary/core/vendors.py +++ b/openlibrary/core/vendors.py @@ -423,6 +423,93 @@ def __init__( # AmazonCreatorsApi; replace its rest_client to route all # outbound HTTP through the proxy. self.api._api_client.rest_client = rest_client + + # Also inject a proxy-aware OAuth2 token manager. The SDK's + # OAuth2TokenManager.refresh_token() calls bare requests.post() + # which reads HTTP_PROXY from the environment. After this PR + # lands, HTTP_PROXY will be a bare (unauthenticated) squid URL; + # Amazon's token endpoint requires authenticated proxy access, so + # the bare URL would produce a 403. We override refresh_token() + # here to use a requests.Session with the authenticated proxy URL + # embedded directly, bypassing the env-var lookup entirely. + if proxy_creds: + from urllib.parse import ( + quote as _urlquote, + ) + from urllib.parse import ( + urlparse as _urlparse, + ) + from urllib.parse import ( + urlunparse as _urlunparse, + ) + + from creatorsapi_python_sdk.auth.oauth2_config import ( + OAuth2Config as _OAuth2Config, + ) + from creatorsapi_python_sdk.auth.oauth2_token_manager import ( + OAuth2TokenManager as _OAuth2TokenManager, + ) + + _user, _, _password = proxy_creds.partition(":") + _parsed = _urlparse(proxy_url) + _netloc = f"{_urlquote(_user, safe='')}:{_urlquote(_password, safe='')}@{_parsed.hostname}" + if _parsed.port: + _netloc += f":{_parsed.port}" + _auth_proxy_url = _urlunparse(_parsed._replace(netloc=_netloc)) + _proxies = {"http": _auth_proxy_url, "https": _auth_proxy_url} + + class _ProxyAwareTokenManager(_OAuth2TokenManager): + """Routes OAuth2 token refresh through authenticated proxy.""" + + def refresh_token(self): + import requests as _req + + session = _req.Session() + session.proxies = _proxies + try: + if self.config.is_lwa(): + resp = session.post( + self.config.get_cognito_endpoint(), + json={ + "grant_type": self.config.get_grant_type(), + "client_id": self.config.get_credential_id(), + "client_secret": self.config.get_credential_secret(), + "scope": self.config.get_scope(), + }, + headers={"Content-Type": "application/json"}, + ) + else: + resp = session.post( + self.config.get_cognito_endpoint(), + data={ + "grant_type": self.config.get_grant_type(), + "client_id": self.config.get_credential_id(), + "client_secret": self.config.get_credential_secret(), + "scope": self.config.get_scope(), + }, + headers={"Content-Type": "application/x-www-form-urlencoded"}, + ) + if resp.status_code != 200: + raise Exception(f"OAuth2 token request failed with status {resp.status_code}: {resp.text}") + data = resp.json() + if "access_token" not in data: + raise Exception("No access token received from OAuth2 endpoint") + self.access_token = data["access_token"] + self.expires_at = time.time() + data.get("expires_in", 3600) - 30 + return self.access_token + except Exception: + self.clear_token() + raise + + api_client = self.api._api_client + _oauth_config = _OAuth2Config( + api_client.credential_id, + api_client.credential_secret, + api_client.version, + api_client.auth_endpoint, + ) + api_client._token_manager = _ProxyAwareTokenManager(_oauth_config) + except (ImportError, AttributeError): logger.warning( "AmazonCreatorsAPI: could not inject proxy — falling back to environment-level proxy (HTTPS_PROXY)", From 70282ac83e27113c0ec3f3740d702c7fd98773af Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 20 May 2026 23:10:22 +0000 Subject: [PATCH 4/7] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- openlibrary/plugins/recaptcha/recaptcha.py | 1 - 1 file changed, 1 deletion(-) diff --git a/openlibrary/plugins/recaptcha/recaptcha.py b/openlibrary/plugins/recaptcha/recaptcha.py index ff47b70414f..803556d9915 100644 --- a/openlibrary/plugins/recaptcha/recaptcha.py +++ b/openlibrary/plugins/recaptcha/recaptcha.py @@ -6,7 +6,6 @@ import web from infogami import config - from openlibrary.plugins.upstream.utils import get_proxy_params logger = logging.getLogger("openlibrary") From 1a4cd7c0f3f6525b2d5676e6daf59f9e01c748f5 Mon Sep 17 00:00:00 2001 From: Drini Cami Date: Thu, 11 Jun 2026 18:51:17 +0000 Subject: [PATCH 5/7] Code review feedback --- openlibrary/core/vendors.py | 107 ++++++++++++++---------------------- 1 file changed, 41 insertions(+), 66 deletions(-) diff --git a/openlibrary/core/vendors.py b/openlibrary/core/vendors.py index 44d1a6a5aef..efb1c96df28 100644 --- a/openlibrary/core/vendors.py +++ b/openlibrary/core/vendors.py @@ -429,86 +429,42 @@ def __init__( # which reads HTTP_PROXY from the environment. After this PR # lands, HTTP_PROXY will be a bare (unauthenticated) squid URL; # Amazon's token endpoint requires authenticated proxy access, so - # the bare URL would produce a 403. We override refresh_token() - # here to use a requests.Session with the authenticated proxy URL - # embedded directly, bypassing the env-var lookup entirely. + # the bare URL would produce a 403. We inject a custom token + # manager subclass that patches requests.post() during refresh + # to use a session with embedded proxy credentials. if proxy_creds: - from urllib.parse import ( - quote as _urlquote, - ) - from urllib.parse import ( - urlparse as _urlparse, - ) - from urllib.parse import ( - urlunparse as _urlunparse, - ) + from creatorsapi_python_sdk.auth.oauth2_config import OAuth2Config + from creatorsapi_python_sdk.auth.oauth2_token_manager import OAuth2TokenManager - from creatorsapi_python_sdk.auth.oauth2_config import ( - OAuth2Config as _OAuth2Config, + session = requests.Session() + session.proxies = _build_authenticated_proxy_config( + proxy_url, proxy_creds ) - from creatorsapi_python_sdk.auth.oauth2_token_manager import ( - OAuth2TokenManager as _OAuth2TokenManager, - ) - - _user, _, _password = proxy_creds.partition(":") - _parsed = _urlparse(proxy_url) - _netloc = f"{_urlquote(_user, safe='')}:{_urlquote(_password, safe='')}@{_parsed.hostname}" - if _parsed.port: - _netloc += f":{_parsed.port}" - _auth_proxy_url = _urlunparse(_parsed._replace(netloc=_netloc)) - _proxies = {"http": _auth_proxy_url, "https": _auth_proxy_url} - class _ProxyAwareTokenManager(_OAuth2TokenManager): + class ProxyAwareTokenManager(OAuth2TokenManager): """Routes OAuth2 token refresh through authenticated proxy.""" def refresh_token(self): - import requests as _req - - session = _req.Session() - session.proxies = _proxies - try: - if self.config.is_lwa(): - resp = session.post( - self.config.get_cognito_endpoint(), - json={ - "grant_type": self.config.get_grant_type(), - "client_id": self.config.get_credential_id(), - "client_secret": self.config.get_credential_secret(), - "scope": self.config.get_scope(), - }, - headers={"Content-Type": "application/json"}, - ) - else: - resp = session.post( - self.config.get_cognito_endpoint(), - data={ - "grant_type": self.config.get_grant_type(), - "client_id": self.config.get_credential_id(), - "client_secret": self.config.get_credential_secret(), - "scope": self.config.get_scope(), - }, - headers={"Content-Type": "application/x-www-form-urlencoded"}, - ) - if resp.status_code != 200: - raise Exception(f"OAuth2 token request failed with status {resp.status_code}: {resp.text}") - data = resp.json() - if "access_token" not in data: - raise Exception("No access token received from OAuth2 endpoint") - self.access_token = data["access_token"] - self.expires_at = time.time() + data.get("expires_in", 3600) - 30 - return self.access_token - except Exception: - self.clear_token() - raise + from unittest.mock import patch + from creatorsapi_python_sdk.auth import ( + oauth2_token_manager, + ) + + with patch.object( + oauth2_token_manager.requests, + "post", + side_effect=session.post, + ): + return super().refresh_token() api_client = self.api._api_client - _oauth_config = _OAuth2Config( + oauth_config = OAuth2Config( api_client.credential_id, api_client.credential_secret, api_client.version, api_client.auth_endpoint, ) - api_client._token_manager = _ProxyAwareTokenManager(_oauth_config) + api_client._token_manager = ProxyAwareTokenManager(oauth_config) except (ImportError, AttributeError): logger.warning( @@ -994,3 +950,22 @@ def betterworldbooks_fmt( "price_amt": price, "qlt": qlt, } + + +def _build_authenticated_proxy_config(proxy_url: str, proxy_creds: str) -> dict[str, str]: + """ + Parses proxy URL and credentials, returning a proxies dict with embedded auth. + + :param str proxy_url: HTTP proxy URL (e.g., 'http://proxy.example.com:3128') + :param str proxy_creds: Proxy credentials in 'user:password' format + :return: Dict of {"http": auth_proxy_url, "https": auth_proxy_url} + """ + from urllib.parse import quote, urlparse, urlunparse + + user, _, password = proxy_creds.partition(":") + parsed = urlparse(proxy_url) + netloc = f"{quote(user, safe='')}:{quote(password, safe='')}@{parsed.hostname}" + if parsed.port: + netloc += f":{parsed.port}" + auth_proxy_url = urlunparse(parsed._replace(netloc=netloc)) + return {"http": auth_proxy_url, "https": auth_proxy_url} From d8b7b04e8559f529062c96e55385272d5ddbdafe Mon Sep 17 00:00:00 2001 From: Drini Cami Date: Thu, 11 Jun 2026 19:22:42 +0000 Subject: [PATCH 6/7] switch to using fork of python api package for proxy support --- openlibrary/core/vendors.py | 82 ++++--------------------------------- requirements.txt | 2 +- 2 files changed, 10 insertions(+), 74 deletions(-) diff --git a/openlibrary/core/vendors.py b/openlibrary/core/vendors.py index efb1c96df28..5c86c323b9f 100644 --- a/openlibrary/core/vendors.py +++ b/openlibrary/core/vendors.py @@ -401,77 +401,13 @@ def __init__( tag=tag, country=getattr(Country, country), throttling=0, + proxy=( + _build_authenticated_proxy_url(proxy_url, proxy_creds) + if proxy_url and proxy_creds + else (proxy_url or None) + ), ) - # Inject proxy into underlying SDK rest client, mirroring the PA-API approach. - # Required for ol-home0 which has no direct internet access. See #10310. - if proxy_url: - try: - from creatorsapi_python_sdk.configuration import ( - Configuration as CreatorsConfig, - ) - from creatorsapi_python_sdk.rest import ( - RESTClientObject as CreatorsRESTClient, - ) - from urllib3 import make_headers - - configuration = CreatorsConfig() - configuration.proxy = proxy_url - configuration.proxy_headers = make_headers(proxy_basic_auth=proxy_creds) - rest_client = CreatorsRESTClient(configuration=configuration) - # _api_client is the ApiClient instance stored directly on - # AmazonCreatorsApi; replace its rest_client to route all - # outbound HTTP through the proxy. - self.api._api_client.rest_client = rest_client - - # Also inject a proxy-aware OAuth2 token manager. The SDK's - # OAuth2TokenManager.refresh_token() calls bare requests.post() - # which reads HTTP_PROXY from the environment. After this PR - # lands, HTTP_PROXY will be a bare (unauthenticated) squid URL; - # Amazon's token endpoint requires authenticated proxy access, so - # the bare URL would produce a 403. We inject a custom token - # manager subclass that patches requests.post() during refresh - # to use a session with embedded proxy credentials. - if proxy_creds: - from creatorsapi_python_sdk.auth.oauth2_config import OAuth2Config - from creatorsapi_python_sdk.auth.oauth2_token_manager import OAuth2TokenManager - - session = requests.Session() - session.proxies = _build_authenticated_proxy_config( - proxy_url, proxy_creds - ) - - class ProxyAwareTokenManager(OAuth2TokenManager): - """Routes OAuth2 token refresh through authenticated proxy.""" - - def refresh_token(self): - from unittest.mock import patch - from creatorsapi_python_sdk.auth import ( - oauth2_token_manager, - ) - - with patch.object( - oauth2_token_manager.requests, - "post", - side_effect=session.post, - ): - return super().refresh_token() - - api_client = self.api._api_client - oauth_config = OAuth2Config( - api_client.credential_id, - api_client.credential_secret, - api_client.version, - api_client.auth_endpoint, - ) - api_client._token_manager = ProxyAwareTokenManager(oauth_config) - - except (ImportError, AttributeError): - logger.warning( - "AmazonCreatorsAPI: could not inject proxy — falling back to environment-level proxy (HTTPS_PROXY)", - exc_info=True, - ) - def get_product(self, asin: str, serialize: bool = False, **kwargs): if products := self.get_products([asin], **kwargs): return next(self.serialize(p) if serialize else p for p in products) @@ -952,13 +888,13 @@ def betterworldbooks_fmt( } -def _build_authenticated_proxy_config(proxy_url: str, proxy_creds: str) -> dict[str, str]: +def _build_authenticated_proxy_url(proxy_url: str, proxy_creds: str) -> str: """ - Parses proxy URL and credentials, returning a proxies dict with embedded auth. + Parses proxy URL and credentials, returning a proxy URL with embedded auth. :param str proxy_url: HTTP proxy URL (e.g., 'http://proxy.example.com:3128') :param str proxy_creds: Proxy credentials in 'user:password' format - :return: Dict of {"http": auth_proxy_url, "https": auth_proxy_url} + :return: Proxy URL including credentials """ from urllib.parse import quote, urlparse, urlunparse @@ -968,4 +904,4 @@ def _build_authenticated_proxy_config(proxy_url: str, proxy_creds: str) -> dict[ if parsed.port: netloc += f":{parsed.port}" auth_proxy_url = urlunparse(parsed._replace(netloc=netloc)) - return {"http": auth_proxy_url, "https": auth_proxy_url} + return auth_proxy_url diff --git a/requirements.txt b/requirements.txt index 7e6f63dde76..1749c3e8729 100644 --- a/requirements.txt +++ b/requirements.txt @@ -24,7 +24,7 @@ prometheus-fastapi-instrumentator==7.1.0 psycopg2==2.9.12 pydantic==2.13.4 pymarc==5.3.1 -python-amazon-paapi==6.2.0 +git+https://github.com/mekarpeles/python-amazon-paapi.git@proxy-support python-dateutil==2.9.0.post0 python-memcached==1.62 python-multipart==0.0.28 From 1608180b0a2549ba8c06a476a82892961cdb0b7a Mon Sep 17 00:00:00 2001 From: Drini Cami Date: Thu, 11 Jun 2026 19:36:37 +0000 Subject: [PATCH 7/7] Experiment with affiliate server/squid proxy for near prod --- compose.near-prod.yaml | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/compose.near-prod.yaml b/compose.near-prod.yaml index a8e2f53087a..63077f51c75 100644 --- a/compose.near-prod.yaml +++ b/compose.near-prod.yaml @@ -11,6 +11,8 @@ ## You probably want to run: ## COMPOSE_FILE="compose.yaml:compose.override.yaml:compose.near-prod.yaml" docker compose up -d ## +## COMPOSE_FILE="compose.yaml:compose.override.yaml:compose.near-prod.yaml" docker compose up -d affiliate-server squid +## services: solr: @@ -45,5 +47,32 @@ services: # Override with own copy of solr data - solr-replica-data:/var/solr + affiliate-server: + image: "${OLIMAGE:-openlibrary/olbase:latest}" + environment: + - AFFILIATE_CONFIG=./conf/openlibrary.yml + command: docker/ol-affiliate-server-start.sh + ports: + - 31337:31337 + networks: + - webnet + volumes: + - ${OL_MOUNT_DIR:-.}:/openlibrary + logging: + options: + max-size: "512m" + max-file: "4" + + squid: + image: ubuntu/squid:5.2-22.04_beta + ports: + - 3128:3128 + networks: + - webnet + logging: + options: + max-size: "512m" + max-file: "4" + volumes: solr-replica-data: