Skip to content

fix: fallback to English for invalid Wikipedia language codes #474

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
lennney wants to merge 2 commits into
base: main
Choose a base branch
from
Open

fix: fallback to English for invalid Wikipedia language codes #474

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
bc77179
feat: add --json flag for stdout JSON output
lennney Jun 9, 2026
e6bee49
fix: fallback to English for invalid Wikipedia language codes
lennney Jun 10, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 32 additions & 5 deletions ddgs/cli.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -217,6 +217,7 @@ def version() -> str:
@click.option("-pr", "--proxy", help="the proxy to send requests, example: socks5h://127.0.0.1:9150")
@click.option("-v", "--verify", default=True, help="verify SSL when making the request")
@click.option("-nc", "--no-color", is_flag=True, default=False, help="disable color output")
@click.option("--json", "as_json", is_flag=True, default=False, help="output results as JSON to stdout")
def text(
query: str,
keywords: str | None, # deprecated
Expand All @@ -234,6 +235,7 @@ def text(
download: bool,
verify: bool,
no_color: bool,
as_json: bool,
) -> None:
"""CLI function to perform a DDGS text metasearch."""
data = DDGS(proxy=_expand_proxy_tb_alias(proxy), verify=verify).text(
Expand All @@ -259,7 +261,9 @@ def text(
verify=verify,
pathname=download_directory,
)
if not output and not download:
if as_json:
click.echo(json.dumps(data, ensure_ascii=False, indent=2))
elif not output and not download:
_print_data(data, no_color=no_color)
Comment on lines +264 to 267


Expand Down Expand Up @@ -316,6 +320,7 @@ def text(
@click.option("-pr", "--proxy", help="the proxy to send requests, example: socks5h://127.0.0.1:9150")
@click.option("-v", "--verify", default=True, help="verify SSL when making the request")
@click.option("-nc", "--no-color", is_flag=True, default=False, help="disable color output")
@click.option("--json", "as_json", is_flag=True, default=False, help="output results as JSON to stdout")
def images(
query: str,
keywords: str | None, # deprecated
Expand All @@ -338,6 +343,7 @@ def images(
download: bool,
verify: bool,
no_color: bool,
as_json: bool,
) -> None:
"""CLI function to perform a DDGS images metasearch."""
data = DDGS(proxy=_expand_proxy_tb_alias(proxy), verify=verify).images(
Expand Down Expand Up @@ -368,7 +374,9 @@ def images(
verify=verify,
pathname=download_directory,
)
if not output and not download:
if as_json:
click.echo(json.dumps(data, ensure_ascii=False, indent=2))
elif not output and not download:
_print_data(data, no_color=no_color)
Comment on lines +377 to 380


Expand All @@ -395,6 +403,7 @@ def images(
@click.option("-pr", "--proxy", help="the proxy to send requests, example: socks5h://127.0.0.1:9150")
@click.option("-v", "--verify", default=True, help="verify SSL when making the request")
@click.option("-nc", "--no-color", is_flag=True, default=False, help="disable color output")
@click.option("--json", "as_json", is_flag=True, default=False, help="output results as JSON to stdout")
def videos(
query: str,
keywords: str | None, # deprecated
Expand All @@ -412,6 +421,7 @@ def videos(
*,
verify: bool,
no_color: bool,
as_json: bool,
) -> None:
"""CLI function to perform a DDGS videos metasearch."""
data = DDGS(proxy=_expand_proxy_tb_alias(proxy), verify=verify).videos(
Expand All @@ -430,7 +440,9 @@ def videos(
query = _sanitize_query(keywords or query)
if output:
_save_data(query, data, function_name="videos", filename=output)
else:
if as_json:
click.echo(json.dumps(data, ensure_ascii=False, indent=2))
elif not output:
_print_data(data, no_color=no_color)


Expand All @@ -454,6 +466,7 @@ def videos(
@click.option("-pr", "--proxy", help="the proxy to send requests, example: socks5h://127.0.0.1:9150")
@click.option("-v", "--verify", default=True, help="verify SSL when making the request")
@click.option("-nc", "--no-color", is_flag=True, default=False, help="disable color output")
@click.option("--json", "as_json", is_flag=True, default=False, help="output results as JSON to stdout")
def news(
query: str,
keywords: str | None, # deprecated
Expand All @@ -468,6 +481,7 @@ def news(
*,
verify: bool,
no_color: bool,
as_json: bool,
) -> None:
"""CLI function to perform a DDGS news metasearch."""
data = DDGS(proxy=_expand_proxy_tb_alias(proxy), verify=verify).news(
Expand All @@ -483,7 +497,9 @@ def news(
query = _sanitize_query(keywords or query)
if output:
_save_data(query, data, function_name="news", filename=output)
else:
if as_json:
click.echo(json.dumps(data, ensure_ascii=False, indent=2))
elif not output:
_print_data(data, no_color=no_color)


Expand All @@ -504,6 +520,7 @@ def news(
@click.option("-pr", "--proxy", help="the proxy to send requests, example: socks5h://127.0.0.1:9150")
@click.option("-v", "--verify", default=True, help="verify SSL when making the request")
@click.option("-nc", "--no-color", is_flag=True, default=False, help="disable color output")
@click.option("--json", "as_json", is_flag=True, default=False, help="output results as JSON to stdout")
def books(
query: str,
keywords: str | None, # deprecated
Expand All @@ -515,6 +532,7 @@ def books(
*,
verify: bool,
no_color: bool,
as_json: bool,
) -> None:
"""CLI function to perform a DDGS books metasearch."""
data = DDGS(proxy=_expand_proxy_tb_alias(proxy), verify=verify).books(
Expand All @@ -526,7 +544,9 @@ def books(
)
if output:
_save_data(query, data, function_name="books", filename=output)
else:
if as_json:
click.echo(json.dumps(data, ensure_ascii=False, indent=2))
elif not output:
_print_data(data, no_color=no_color)


Expand All @@ -543,19 +563,26 @@ def books(
@click.option("-o", "--output", help="json or filename.json (save the results to a file)")
@click.option("-pr", "--proxy", help="the proxy to send requests, example: socks5h://127.0.0.1:9150")
@click.option("-v", "--verify", default=True, help="verify SSL when making the request")
@click.option("-nc", "--no-color", is_flag=True, default=False, help="disable color output")
@click.option("--json", "as_json", is_flag=True, default=False, help="output results as JSON to stdout")
def extract(
url: str,
fmt: str,
output: str | None,
proxy: str | None,
*,
verify: bool,
no_color: bool,
as_json: bool,
Comment on lines +566 to +576
) -> None:
"""CLI function to extract content from a URL."""
data = DDGS(proxy=_expand_proxy_tb_alias(proxy), verify=verify).extract(url=url, fmt=fmt)
if output:
str_data: dict[str, str] = {k: v.decode() if isinstance(v, bytes) else v for k, v in data.items()}
_save_data(_sanitize_query(url), [str_data], "extract", filename=output)
elif as_json:
str_data = {k: v.decode() if isinstance(v, bytes) else v for k, v in data.items()}
click.echo(json.dumps([str_data], ensure_ascii=False, indent=2))
else:
click.echo(f"URL: {url}\n")
content = data["content"]
Expand Down
7 changes: 7 additions & 0 deletions ddgs/engines/wikipedia.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,10 @@

logger = logging.getLogger(__name__)

# DuckDuckGo uses some region codes (e.g. "wt" for "worldwide") that are not
# valid Wikipedia language codes. Fall back to English when we encounter one.
_INVALID_LANG_CODES: frozenset[str] = frozenset({"wt"})


class Wikipedia(BaseSearchEngine[TextResult]):
"""Wikipedia text search engine."""
Expand All @@ -33,6 +37,9 @@ def build_payload(
) -> dict[str, Any]:
"""Build a payload for the search request."""
_country, lang = region.lower().split("-")
if lang in _INVALID_LANG_CODES:
logger.debug("Region %s maps to invalid Wikipedia lang %r, falling back to 'en'", region, lang)
lang = "en"
encoded_query = quote(query)
self.search_url = (
f"https://{lang}.wikipedia.org/w/api.php?action=opensearch&profile=fuzzy&limit=1&search={encoded_query}"
Expand Down