diff --git a/ddgs/api_server/__init__.py b/ddgs/api_server/__init__.py index 751dc60..3c5d90a 100644 --- a/ddgs/api_server/__init__.py +++ b/ddgs/api_server/__init__.py @@ -4,9 +4,15 @@ and the distributed network cache service. """ -from ddgs.api_server.api import app as fastapi_app +__all__: list[str] = [] -__all__ = ["fastapi_app"] +try: + from ddgs.api_server.api import app as fastapi_app + + __all__ += ["fastapi_app"] +except ImportError: + # API dependencies not installed + pass try: from ddgs.api_server.dht_service import get_dht_service diff --git a/ddgs/engines/duckduckgo.py b/ddgs/engines/duckduckgo.py index 6396dba..9e03c1d 100644 --- a/ddgs/engines/duckduckgo.py +++ b/ddgs/engines/duckduckgo.py @@ -1,11 +1,18 @@ """Duckduckgo search engine implementation.""" from collections.abc import Mapping -from typing import Any, ClassVar +from typing import Any, ClassVar, TypeVar + +from fake_useragent import UserAgent from ddgs.base import BaseSearchEngine +from ddgs.http_client2 import HttpClient2 from ddgs.results import TextResult +ua = UserAgent() + +T = TypeVar("T") + class Duckduckgo(BaseSearchEngine[TextResult]): """Duckduckgo search engine.""" @@ -20,6 +27,13 @@ class Duckduckgo(BaseSearchEngine[TextResult]): items_xpath = "//div[contains(@class, 'body')]" elements_xpath: ClassVar[Mapping[str, str]] = {"title": ".//h2//text()", "href": "./a/@href", "body": "./a//text()"} + headers: ClassVar[dict[str, str]] = {"User-Agent": ua.random} + + def __init__(self, proxy: str | None = None, timeout: int | None = None, *, verify: bool = True) -> None: + """Temporary, delete when HttpClient is fixed.""" + self.http_client = HttpClient2(headers=self.headers, proxy=proxy, timeout=timeout, verify=verify) # type: ignore[assignment] + self.results: list[T] = [] # type: ignore[valid-type] + def build_payload( self, query: str, diff --git a/ddgs/http_client2.py b/ddgs/http_client2.py new file mode 100644 index 0000000..cb03be2 --- /dev/null +++ b/ddgs/http_client2.py @@ -0,0 +1,151 @@ +"""Temporary HTTP client for 'backend=duckduckgo'. Delete when HttpClient is fixed.""" + +import logging +import ssl +from random import SystemRandom +from types import TracebackType +from typing import TYPE_CHECKING, Any + +import h2 +import httpcore +import httpx + +from .exceptions import DDGSException, TimeoutException + +if TYPE_CHECKING: + from collections.abc import Callable + + +logger = logging.getLogger(__name__) +random = SystemRandom() + + +class Response: + """HTTP response.""" + + __slots__ = ("content", "status_code", "text") + + def __init__(self, status_code: int, content: bytes, text: str) -> None: + self.status_code = status_code + self.content = content + self.text = text + + +class HttpClient2: + """Temporary HTTP client.""" + + def __init__( + self, + headers: dict[str, str] | None = None, + proxy: str | None = None, + timeout: int | None = 10, + *, + verify: bool | str = True, + ) -> None: + """Initialize the HttpClient object. + + Args: + headers (dict, optional): headers for the HTTP client. + proxy (str, optional): proxy for the HTTP client, supports http/https/socks5 protocols. + example: "http://user:pass@example.com:3128". Defaults to None. + timeout (int, optional): Timeout value for the HTTP client. Defaults to 10. + verify: (bool | str): True to verify, False to skip or str path to a PEM file. Defaults to True. + + """ + self.client = httpx.Client( + headers=headers, + proxy=proxy, + timeout=timeout, + verify=_get_random_ssl_context(verify=verify) if verify else False, + follow_redirects=False, + http2=True, + ) + + def request(self, *args: Any, **kwargs: Any) -> Response: # noqa: ANN401 + """Make a request to the HTTP client.""" + with Patch(): + try: + resp = self.client.request(*args, **kwargs) + return Response(status_code=resp.status_code, content=resp.content, text=resp.text) + except Exception as ex: + if "timed out" in f"{ex}": + msg = f"Request timed out: {ex!r}" + raise TimeoutException(msg) from ex + msg = f"{type(ex).__name__}: {ex!r}" + raise DDGSException(msg) from ex + + def get(self, *args: Any, **kwargs: Any) -> Response: # noqa: ANN401 + """Make a GET request to the HTTP client.""" + return self.request(*args, method="GET", **kwargs) + + def post(self, *args: Any, **kwargs: Any) -> Response: # noqa: ANN401 + """Make a POST request to the HTTP client.""" + return self.request(*args, method="POST", **kwargs) + + +# SSL +DEFAULT_CIPHERS = [ # https://developers.cloudflare.com/ssl/reference/cipher-suites/recommendations/ + "TLS_AES_128_GCM_SHA256", "TLS_AES_256_GCM_SHA384", "TLS_CHACHA20_POLY1305_SHA256", + # Modern: + "ECDHE-ECDSA-AES128-GCM-SHA256", "ECDHE-ECDSA-CHACHA20-POLY1305", "ECDHE-RSA-AES128-GCM-SHA256", + "ECDHE-RSA-CHACHA20-POLY1305", "ECDHE-ECDSA-AES256-GCM-SHA384", "ECDHE-RSA-AES256-GCM-SHA384", + # Compatible: + "ECDHE-ECDSA-AES128-GCM-SHA256", "ECDHE-ECDSA-CHACHA20-POLY1305", "ECDHE-RSA-AES128-GCM-SHA256", + "ECDHE-RSA-CHACHA20-POLY1305", "ECDHE-ECDSA-AES256-GCM-SHA384", "ECDHE-RSA-AES256-GCM-SHA384", + "ECDHE-ECDSA-AES128-SHA256", "ECDHE-RSA-AES128-SHA256", "ECDHE-ECDSA-AES256-SHA384", "ECDHE-RSA-AES256-SHA384", + # Legacy: + "ECDHE-ECDSA-AES128-SHA", "ECDHE-RSA-AES128-SHA", "AES128-GCM-SHA256", "AES128-SHA256", "AES128-SHA", + "ECDHE-RSA-AES256-SHA", "AES256-GCM-SHA384", "AES256-SHA256", "AES256-SHA", "DES-CBC3-SHA", +] # fmt: skip + + +def _get_random_ssl_context(*, verify: bool | str) -> ssl.SSLContext: + ssl_context = ssl.create_default_context(cafile=verify if isinstance(verify, str) else None) + shuffled_ciphers = random.sample(DEFAULT_CIPHERS[9:], len(DEFAULT_CIPHERS) - 9) + ssl_context.set_ciphers(":".join(DEFAULT_CIPHERS[:9] + shuffled_ciphers)) + commands: list[None | Callable[[ssl.SSLContext], None]] = [ + None, + lambda context: setattr(context, "maximum_version", ssl.TLSVersion.TLSv1_2), + lambda context: setattr(context, "minimum_version", ssl.TLSVersion.TLSv1_3), + lambda context: setattr(context, "options", context.options | ssl.OP_NO_TICKET), + ] + random_command = random.choice(commands) + if random_command: + random_command(ssl_context) + return ssl_context + + +class Patch: + """Patch the HTTP2Connection._send_connection_init method.""" + + def __enter__(self) -> None: + """Enter the context manager.""" + + def _send_connection_init(self: httpcore._sync.http2.HTTP2Connection, request: httpcore.Request) -> None: + self._h2_state.local_settings = h2.settings.Settings( + client=True, + initial_values={ + h2.settings.SettingCodes.INITIAL_WINDOW_SIZE: random.randint(100, 200), + h2.settings.SettingCodes.HEADER_TABLE_SIZE: random.randint(4000, 5000), + h2.settings.SettingCodes.MAX_FRAME_SIZE: random.randint(16384, 65535), + h2.settings.SettingCodes.MAX_CONCURRENT_STREAMS: random.randint(100, 200), + h2.settings.SettingCodes.MAX_HEADER_LIST_SIZE: random.randint(65500, 66500), + h2.settings.SettingCodes.ENABLE_CONNECT_PROTOCOL: random.randint(0, 1), + h2.settings.SettingCodes.ENABLE_PUSH: random.randint(0, 1), + }, + ) + self._h2_state.initiate_connection() + self._h2_state.increment_flow_control_window(2**24) + self._write_outgoing_data(request) + + self.original_send_connection_init = httpcore._sync.http2.HTTP2Connection._send_connection_init + httpcore._sync.http2.HTTP2Connection._send_connection_init = _send_connection_init # type: ignore[method-assign] + + def __exit__( + self, + exc_type: type[BaseException] | None = None, + exc_val: BaseException | None = None, + exc_tb: TracebackType | None = None, + ) -> None: + """Exit the context manager.""" + httpcore._sync.http2.HTTP2Connection._send_connection_init = self.original_send_connection_init # type: ignore[method-assign] diff --git a/pyproject.toml b/pyproject.toml index 5b46d0d..3bbcc33 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,6 +30,8 @@ dependencies = [ "click>=8.1.8", "primp>=1.2.3", "lxml>=4.9.4", + "httpx[http2,socks,brotli]>=0.28.1", # temporarily + "fake-useragent>=2.2.0", ] dynamic = ["version"] @@ -62,7 +64,15 @@ dev = [ "types-Pygments", "types-pexpect", "types-PyYAML", - "types-ujson" + "types-ujson", + + # for mypy (httpx) + "types-PySocks", + "types-colorama", + "types-decorator", + "types-jsonschema", + "types-psutil", + "types-pyasn1" ] mcp = [ "mcp>=1.26.0",