diff --git a/ddgs/base.py b/ddgs/base.py index c8a1027..2c46531 100644 --- a/ddgs/base.py +++ b/ddgs/base.py @@ -110,14 +110,18 @@ def search( **kwargs: str, ) -> list[T] | None: """Search the engine.""" - payload = self.build_payload( - query=query, region=region, safesearch=safesearch, timelimit=timelimit, page=page, **kwargs - ) - if self.search_method == "GET": - html_text = self.request(self.search_method, self.search_url, params=payload) - else: - html_text = self.request(self.search_method, self.search_url, data=payload) - if not html_text: - return None - results = self.extract_results(html_text) + max_results = int(kwargs.get("max_results", 10)) + results = list() + while len(results) <= max_results: + payload = self.build_payload( + query=query, region=region, safesearch=safesearch, timelimit=timelimit, page=page, **kwargs + ) + if self.search_method == "GET": + html_text = self.request(self.search_method, self.search_url, params=payload) + else: + html_text = self.request(self.search_method, self.search_url, data=payload) + if not html_text: + return None + page += 1 + results += self.extract_results(html_text) return self.post_extract_results(results) diff --git a/ddgs/ddgs.py b/ddgs/ddgs.py index 004c7da..e5fb1fa 100644 --- a/ddgs/ddgs.py +++ b/ddgs/ddgs.py @@ -141,9 +141,7 @@ def _search_sync( # noqa: C901 region: str = "us-en", safesearch: str = "moderate", timelimit: str | None = None, - max_results: int | None = 10, page: int = 1, - backend: str = "auto", **kwargs: str, ) -> list[dict[str, Any]]: """Perform a search across engines in the given category. @@ -169,12 +167,14 @@ def _search_sync( # noqa: C901 msg = "query is mandatory." raise DDGSException(msg) + backend = kwargs.get("backend", "auto") engines = self._get_engines(category, backend) len_unique_providers = len({engine.provider for engine in engines}) seen_providers: set[str] = set() # Perform search results_aggregator: ResultsAggregator[set[str]] = ResultsAggregator({"href", "image", "url", "embed_url"}) + max_results = int(kwargs.get("max_results", 0)) max_workers = min(len_unique_providers, ceil(max_results / 10) + 1) if max_results else len_unique_providers if DDGS.threads: max_workers = min(max_workers, DDGS.threads)