From 2f9ecb1a4bab50fa17386bfb89712a1cb680965b Mon Sep 17 00:00:00 2001 From: Slobodan Stanojevikj Date: Sun, 7 Jun 2026 09:09:45 +0200 Subject: [PATCH 1/2] -carried over the `domain` to Concept -added global UI filter for concept domains -added transient (calculated) field display name which will "humanize" the concept name -search is also fixed (normalized) --- web/.env.example | 3 ++ web/concepts/models.py | 47 ++++++++++++++++++--- web/concepts/templates/detail.html | 4 +- web/concepts/templates/results.html | 2 +- web/concepts/utils.py | 20 ++++++++- web/concepts/views.py | 63 +++++++++++++++++++---------- web/web/settings.py | 7 +++- 7 files changed, 113 insertions(+), 33 deletions(-) diff --git a/web/.env.example b/web/.env.example index 97d162a..4fc8d5d 100644 --- a/web/.env.example +++ b/web/.env.example @@ -6,6 +6,9 @@ WIKIPEDIA_CONTACT_EMAIL=my@email.com IMPORT_MATH_ENTITIES=True IMPORT_PHYSICS_ENTITIES=False +# UI domain filter — comma-separated list (math, phys). Default: math. +UI_DOMAINS=math + # DB USE_POSTGRES=True POSTGRES_DB=mathsdb diff --git a/web/concepts/models.py b/web/concepts/models.py index fd1a0d4..e3a8500 100644 --- a/web/concepts/models.py +++ b/web/concepts/models.py @@ -1,14 +1,26 @@ import logging -from concepts.utils import UnionFind, normalize_concept_name +from concepts.utils import ( + UnionFind, + humanize_concept_name, + normalize_concept_name, +) from django.db import models from django.db.utils import IntegrityError +class Domain(models.TextChoices): + MATHEMATICS = "math", "Mathematics" + PHYSICS = "phys", "Physics" + + class Concept(models.Model): name = models.CharField(max_length=200, null=True) normal_name = models.CharField(max_length=200, null=True) description = models.TextField(null=True) + domain = models.CharField( + max_length=4, choices=Domain.choices, default=Domain.MATHEMATICS + ) class Meta: ordering = ["name", "description"] @@ -16,6 +28,16 @@ class Meta: models.UniqueConstraint(fields=["normal_name"], name="unique_normal_name") ] + @property + def display_name(self): + """Human-readable label for the UI, derived on the fly from ``name``. + + This is a transient field (no database column): it is computed at + access time and is not stored or queryable. ``name`` remains the raw + source label and ``normal_name`` the canonical slug used for routing. + """ + return humanize_concept_name(self.name) + class LinkQuerySet(models.QuerySet): def to_tuples(self): @@ -47,8 +69,15 @@ def take_first(lst): name = take_first([item.name for item in concept_items]) normal_name = normalize_concept_name(name) description = take_first([item.description for item in concept_items]) + domain = ( + take_first([item.domain for item in concept_items]) + or Domain.MATHEMATICS + ) new_concept = Concept( - name=name, normal_name=normal_name, description=description + name=name, + normal_name=normal_name, + description=description, + domain=domain, ) try: new_concept.save() @@ -64,9 +93,7 @@ def take_first(lst): class Item(models.Model): - class Domain(models.TextChoices): - MATHEMATICS = "math", "Mathematics" - PHYSICS = "phys", "Physics" + Domain = Domain class Source(models.TextChoices): WIKIDATA = "Wd", "Wikidata" @@ -117,7 +144,14 @@ class Meta: unique_together = ["source", "identifier"] def to_dict(self): - return {"name": self.name, "source": self.get_source_display(), "url": self.url} + normal_name = self.concept.normal_name if self.concept_id else None + return { + "name": self.name, + "normal_name": normal_name or self.name, + "display_name": humanize_concept_name(self.name), + "source": self.get_source_display(), + "url": self.url, + } def get_linked_items(self): linked_destinations = Link.objects.filter(source=self.id).map( @@ -136,6 +170,7 @@ def to_concept(self): name=self.name, normal_name=normalize_concept_name(self.name), description=self.description, + domain=self.domain, ) def __str__(self): diff --git a/web/concepts/templates/detail.html b/web/concepts/templates/detail.html index 1fdcb32..f520771 100644 --- a/web/concepts/templates/detail.html +++ b/web/concepts/templates/detail.html @@ -1,13 +1,13 @@ {% extends 'base.html' %} {% block content %} -

{{ concept.name }}

+

{{ concept.display_name }}

{% if concept.description != None %}

{{ concept.description }}

{% else %}

(no description is available)

{% endif %} {% for item in concept.items %} - {{ item.source }}: {{ item.name }}
+ {{ item.source }}: {{ item.display_name }}
{% endfor %} {% endblock %} diff --git a/web/concepts/templates/results.html b/web/concepts/templates/results.html index 33589cb..6e185dd 100644 --- a/web/concepts/templates/results.html +++ b/web/concepts/templates/results.html @@ -8,7 +8,7 @@

Search results

diff --git a/web/concepts/utils.py b/web/concepts/utils.py index 5ca03d6..0a84d47 100644 --- a/web/concepts/utils.py +++ b/web/concepts/utils.py @@ -1,4 +1,5 @@ from typing import Any, Dict, List, Tuple +from urllib.parse import unquote from unidecode import unidecode @@ -62,5 +63,22 @@ def elements_to_sorted_items(elements: List[int]): def normalize_concept_name(name: str) -> str: - nyoo = unidecode(name).replace("_", "-").replace(" ", "-").lower() + decoded = unquote(name) if name else "" + nyoo = unidecode(decoded).replace("_", "-").replace(" ", "-").lower() return "".join(filter(lambda x: x.isalnum() or x == "-", nyoo)) + + +def humanize_concept_name(name: str) -> str: + """Turn a raw source name into a human-readable UI label. + + Unlike :func:`normalize_concept_name` (which produces a lowercase, ASCII, + URL-safe slug for lookups and routing), this keeps the original casing and + diacritics and only undoes the artifacts of URL-derived names: it + percent-decodes the string and turns underscores back into spaces. + + Examples: + ``Sasakian_manifold`` -> ``Sasakian manifold`` + ``M%C3%B6bius_strip`` -> ``Möbius strip`` + ``Möbius strip`` -> ``Möbius strip`` (unchanged) + """ + return unquote(name).replace("_", " ") if name else "" diff --git a/web/concepts/views.py b/web/concepts/views.py index dd3ab3f..f0f5575 100644 --- a/web/concepts/views.py +++ b/web/concepts/views.py @@ -1,16 +1,29 @@ from concepts.models import Concept, Item +from concepts.utils import normalize_concept_name from django.shortcuts import get_object_or_404, redirect, render +from web.settings import UI_DOMAINS + + +def _concepts_in_scope(): + return Concept.objects.filter(domain__in=UI_DOMAINS) + + +def _items_in_scope(): + return Item.objects.filter(domain__in=UI_DOMAINS) + def concept(request, name): try: - concept = Concept.objects.get(name=name) + concept = _concepts_in_scope().get(normal_name=name) context = { "concept": { - "name": concept.name, + "normal_name": concept.normal_name, + "display_name": concept.display_name, "description": concept.description, "items": [ - item.to_dict() for item in Item.objects.filter(concept=concept.id) + item.to_dict() + for item in _items_in_scope().filter(concept=concept.id) ], } } @@ -20,41 +33,47 @@ def concept(request, name): def home(request): - autocomplete_names = [c.name for c in Concept.objects.all() if c.name is not None] + autocomplete_names = [ + c.display_name for c in _concepts_in_scope() if c.normal_name is not None + ] + items = _items_in_scope() context = { "concepts": autocomplete_names, "number_of_links": { - "wikidata": Item.objects.filter(source=Item.Source.WIKIDATA).count(), - "wikipedia_en": Item.objects.filter( - source=Item.Source.WIKIPEDIA_EN - ).count(), - "nlab": Item.objects.filter(source=Item.Source.NLAB).count(), - "mathworld": Item.objects.filter(source=Item.Source.MATHWORLD).count(), - "proof_wiki": Item.objects.filter(source=Item.Source.PROOF_WIKI).count(), - "encyclopedia_of_mathematics": Item.objects.filter( + "wikidata": items.filter(source=Item.Source.WIKIDATA).count(), + "wikipedia_en": items.filter(source=Item.Source.WIKIPEDIA_EN).count(), + "nlab": items.filter(source=Item.Source.NLAB).count(), + "mathworld": items.filter(source=Item.Source.MATHWORLD).count(), + "proof_wiki": items.filter(source=Item.Source.PROOF_WIKI).count(), + "encyclopedia_of_mathematics": items.filter( source=Item.Source.ENCYCLOPEDIA_OF_MATHEMATICS ).count(), - "agda_unimath": Item.objects.filter( - source=Item.Source.AGDA_UNIMATH - ).count(), - "lmfdb": Item.objects.filter(source=Item.Source.LMFDB).count(), + "agda_unimath": items.filter(source=Item.Source.AGDA_UNIMATH).count(), + "lmfdb": items.filter(source=Item.Source.LMFDB).count(), }, } return render(request, "index.html", context) def search(request): - search_value = request.GET.get("q") - return redirect("/concept/" + search_value) + search_value = request.GET.get("q") or "" + return redirect("/concept/" + normalize_concept_name(search_value)) def redirect_item_to_concept(request, source, identifier): # should this be a permanent redirect? - item = get_object_or_404(Item, source=source, identifier=identifier) - return redirect("/concept/" + item.concept.name) + item = get_object_or_404(_items_in_scope(), source=source, identifier=identifier) + return redirect("/concept/" + item.concept.normal_name) def results(request, query): - concepts = Concept.objects.filter(name__contains=query) - context = {"query": query, "results": [concept.name for concept in concepts]} + normalized = normalize_concept_name(query) + concepts = _concepts_in_scope().filter(normal_name__contains=normalized) + context = { + "query": query, + "results": [ + {"normal_name": concept.normal_name, "display_name": concept.display_name} + for concept in concepts + ], + } return render(request, "results.html", context) diff --git a/web/web/settings.py b/web/web/settings.py index 349bdad..e2120ef 100644 --- a/web/web/settings.py +++ b/web/web/settings.py @@ -13,7 +13,7 @@ from os import path from pathlib import Path -from decouple import config +from decouple import Csv, config # Build paths inside the project like this: BASE_DIR / 'subdir'. BASE_DIR = Path(__file__).resolve().parent.parent @@ -187,3 +187,8 @@ # Feature flags IMPORT_MATH_ENTITIES = config("IMPORT_MATH_ENTITIES", default=True, cast=bool) IMPORT_PHYSICS_ENTITIES = config("IMPORT_PHYSICS_ENTITIES", default=False, cast=bool) + +# Which Item/Concept domains the UI exposes. Comma-separated list of domain +# codes (see concepts.models.Domain), e.g. "math" or "math,phys". A single +# entry locks the entire UI to that domain. +UI_DOMAINS = config("UI_DOMAINS", default="math", cast=Csv()) From 044bb1e0abfc1c0ced9cb1a050434da04328064d Mon Sep 17 00:00:00 2001 From: Slobodan Stanojevikj Date: Sun, 7 Jun 2026 12:48:07 +0200 Subject: [PATCH 2/2] -added handling for umlauts and search with multiple possible variants -added "Other search results for... " if a search is happening but navigated to the exact concept -added missing migration script --- .../migrations/0022_concept_domain.py | 33 +++++++++++++++++++ web/concepts/templates/detail.html | 3 ++ web/concepts/utils.py | 20 +++++++++++ web/concepts/views.py | 17 +++++++--- 4 files changed, 69 insertions(+), 4 deletions(-) create mode 100644 web/concepts/migrations/0022_concept_domain.py diff --git a/web/concepts/migrations/0022_concept_domain.py b/web/concepts/migrations/0022_concept_domain.py new file mode 100644 index 0000000..62c620d --- /dev/null +++ b/web/concepts/migrations/0022_concept_domain.py @@ -0,0 +1,33 @@ +from django.db import migrations, models + + +def backfill_concept_domain(apps, schema_editor): + Concept = apps.get_model("concepts", "Concept") + Item = apps.get_model("concepts", "Item") + for concept in Concept.objects.all(): + first_item = ( + Item.objects.filter(concept=concept).exclude(domain="").first() + ) + if first_item and first_item.domain: + concept.domain = first_item.domain + concept.save(update_fields=["domain"]) + + +class Migration(migrations.Migration): + + dependencies = [ + ("concepts", "0021_alter_item_identifier_alter_item_name_alter_item_url"), + ] + + operations = [ + migrations.AddField( + model_name="concept", + name="domain", + field=models.CharField( + choices=[("math", "Mathematics"), ("phys", "Physics")], + default="math", + max_length=4, + ), + ), + migrations.RunPython(backfill_concept_domain, lambda *_args: None), + ] diff --git a/web/concepts/templates/detail.html b/web/concepts/templates/detail.html index f520771..363f0ba 100644 --- a/web/concepts/templates/detail.html +++ b/web/concepts/templates/detail.html @@ -10,4 +10,7 @@

{{ concept.display_name }}

{% for item in concept.items %} {{ item.source }}: {{ item.display_name }}
{% endfor %} + {% if search_query %} +

Other search results for “{{ concept.display_name }}”

+ {% endif %} {% endblock %} diff --git a/web/concepts/utils.py b/web/concepts/utils.py index 0a84d47..a0d9d3d 100644 --- a/web/concepts/utils.py +++ b/web/concepts/utils.py @@ -68,6 +68,26 @@ def normalize_concept_name(name: str) -> str: return "".join(filter(lambda x: x.isalnum() or x == "-", nyoo)) +# German-umlaut digraphs and the single-letter form unidecode already produces +# for the diacritic itself (ö -> o, ä -> a, ü -> u). +_UMLAUT_DIGRAPHS = (("oe", "o"), ("ae", "a"), ("ue", "u")) + + +def search_variants(name: str) -> set: + """Expand a search query into the slugs worth matching against. + + Returns the normalized slug plus a German-digraph-collapsed variant, so a + typed ``moebius`` also matches a stored ``mobius``. This is query-only: it + does not touch ``normalize_concept_name`` or any stored data. If new fuzzing + cases come up, add them here rather than re-normalizing the database. + """ + base = normalize_concept_name(name) + collapsed = base + for digraph, single in _UMLAUT_DIGRAPHS: + collapsed = collapsed.replace(digraph, single) + return {base, collapsed} + + def humanize_concept_name(name: str) -> str: """Turn a raw source name into a human-readable UI label. diff --git a/web/concepts/views.py b/web/concepts/views.py index f0f5575..6dba863 100644 --- a/web/concepts/views.py +++ b/web/concepts/views.py @@ -1,5 +1,6 @@ from concepts.models import Concept, Item -from concepts.utils import normalize_concept_name +from concepts.utils import normalize_concept_name, search_variants +from django.db.models import Q from django.shortcuts import get_object_or_404, redirect, render from web.settings import UI_DOMAINS @@ -27,6 +28,11 @@ def concept(request, name): ], } } + # Only when reached via search (carries ?q=) do we offer the broader + # results list, so direct/linked visits stay clean. + query = request.GET.get("q") + if query: + context["search_query"] = query return render(request, "detail.html", context) except Concept.DoesNotExist: return redirect("/results/" + name) @@ -57,7 +63,8 @@ def home(request): def search(request): search_value = request.GET.get("q") or "" - return redirect("/concept/" + normalize_concept_name(search_value)) + slug = normalize_concept_name(search_value) + return redirect(f"/concept/{slug}?q={slug}") def redirect_item_to_concept(request, source, identifier): @@ -67,8 +74,10 @@ def redirect_item_to_concept(request, source, identifier): def results(request, query): - normalized = normalize_concept_name(query) - concepts = _concepts_in_scope().filter(normal_name__contains=normalized) + name_filter = Q() + for variant in search_variants(query): + name_filter |= Q(normal_name__contains=variant) + concepts = _concepts_in_scope().filter(name_filter) context = { "query": query, "results": [