Normal name improvements

web/.env.example

@@ -6,6 +6,9 @@ WIKIPEDIA_CONTACT_EMAIL=my@email.com
 IMPORT_MATH_ENTITIES=True
 IMPORT_PHYSICS_ENTITIES=False
 
+# UI domain filter — comma-separated list (math, phys). Default: math.
+UI_DOMAINS=math
+
 # DB
 USE_POSTGRES=True
 POSTGRES_DB=mathsdb

web/concepts/migrations/0022_concept_domain.py

@@ -0,0 +1,33 @@
+from django.db import migrations, models
+
+
+def backfill_concept_domain(apps, schema_editor):
+    Concept = apps.get_model("concepts", "Concept")
+    Item = apps.get_model("concepts", "Item")
+    for concept in Concept.objects.all():
+        first_item = (
+            Item.objects.filter(concept=concept).exclude(domain="").first()
+        )
+        if first_item and first_item.domain:
+            concept.domain = first_item.domain
+            concept.save(update_fields=["domain"])
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("concepts", "0021_alter_item_identifier_alter_item_name_alter_item_url"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="concept",
+            name="domain",
+            field=models.CharField(
+                choices=[("math", "Mathematics"), ("phys", "Physics")],
+                default="math",
+                max_length=4,
+            ),
+        ),
+        migrations.RunPython(backfill_concept_domain, lambda *_args: None),
+    ]

web/concepts/models.py

@@ -1,21 +1,43 @@
 import logging
 
-from concepts.utils import UnionFind, normalize_concept_name
+from concepts.utils import (
+    UnionFind,
+    humanize_concept_name,
+    normalize_concept_name,
+)
 from django.db import models
 from django.db.utils import IntegrityError
 
 
+class Domain(models.TextChoices):
+    MATHEMATICS = "math", "Mathematics"
+    PHYSICS = "phys", "Physics"
+
+
 class Concept(models.Model):
     name = models.CharField(max_length=200, null=True)
     normal_name = models.CharField(max_length=200, null=True)
     description = models.TextField(null=True)
+    domain = models.CharField(
+        max_length=4, choices=Domain.choices, default=Domain.MATHEMATICS
+    )
 
     class Meta:
         ordering = ["name", "description"]
         constraints = [
             models.UniqueConstraint(fields=["normal_name"], name="unique_normal_name")
         ]
 
+    @property
+    def display_name(self):
+        """Human-readable label for the UI, derived on the fly from ``name``.
+
+        This is a transient field (no database column): it is computed at
+        access time and is not stored or queryable. ``name`` remains the raw
+        source label and ``normal_name`` the canonical slug used for routing.
+        """
+        return humanize_concept_name(self.name)
+
 
 class LinkQuerySet(models.QuerySet):
     def to_tuples(self):
@@ -47,8 +69,15 @@ def take_first(lst):
             name = take_first([item.name for item in concept_items])
             normal_name = normalize_concept_name(name)
             description = take_first([item.description for item in concept_items])
+            domain = (
+                take_first([item.domain for item in concept_items])
+                or Domain.MATHEMATICS
+            )
             new_concept = Concept(
-                name=name, normal_name=normal_name, description=description
+                name=name,
+                normal_name=normal_name,
+                description=description,
+                domain=domain,
             )
             try:
                 new_concept.save()
@@ -64,9 +93,7 @@ def take_first(lst):
 
 
 class Item(models.Model):
-    class Domain(models.TextChoices):
-        MATHEMATICS = "math", "Mathematics"
-        PHYSICS = "phys", "Physics"
+    Domain = Domain
 
     class Source(models.TextChoices):
         WIKIDATA = "Wd", "Wikidata"
@@ -117,7 +144,14 @@ class Meta:
         unique_together = ["source", "identifier"]
 
     def to_dict(self):
-        return {"name": self.name, "source": self.get_source_display(), "url": self.url}
+        normal_name = self.concept.normal_name if self.concept_id else None
+        return {
+            "name": self.name,
+            "normal_name": normal_name or self.name,
+            "display_name": humanize_concept_name(self.name),
+            "source": self.get_source_display(),
+            "url": self.url,
+        }
 
     def get_linked_items(self):
         linked_destinations = Link.objects.filter(source=self.id).map(
@@ -136,6 +170,7 @@ def to_concept(self):
             name=self.name,
             normal_name=normalize_concept_name(self.name),
             description=self.description,
+            domain=self.domain,
         )
 
     def __str__(self):

web/concepts/templates/detail.html

@@ -1,13 +1,16 @@
 {% extends 'base.html' %}
 
 {% block content %}
-  <h2>{{ concept.name }}</h2>
+  <h2>{{ concept.display_name }}</h2>
   {% if concept.description != None %}
     <p>{{ concept.description }}</p>
   {% else %}
   <p>(no description is available)</p>
   {% endif %}
   {% for item in concept.items %}
-    <a href="{{ item.url }}">{{ item.source }}: {{ item.name }}</a> <br>
+    <a href="{{ item.url }}">{{ item.source }}: {{ item.display_name }}</a> <br>
   {% endfor %}
+  {% if search_query %}
+    <p><a href="/results/{{ search_query }}">Other search results for “{{ concept.display_name }}”</a></p>
+  {% endif %}
 {% endblock %}

web/concepts/templates/results.html

@@ -8,7 +8,7 @@ <h2>Search results</h2>
 
 <ul id="results">
   {% for concept in results %}
-    <li><a href="/concept/{{ concept }}">{{ concept }}</a></li>
+    <li><a href="/concept/{{ concept.normal_name }}">{{ concept.display_name }}</a></li>
   {% endfor %}
 </ul>
 

web/concepts/utils.py

@@ -1,4 +1,5 @@
 from typing import Any, Dict, List, Tuple
+from urllib.parse import unquote
 
 from unidecode import unidecode
 
@@ -62,5 +63,42 @@ def elements_to_sorted_items(elements: List[int]):
 
 
 def normalize_concept_name(name: str) -> str:
-    nyoo = unidecode(name).replace("_", "-").replace(" ", "-").lower()
+    decoded = unquote(name) if name else ""
+    nyoo = unidecode(decoded).replace("_", "-").replace(" ", "-").lower()
     return "".join(filter(lambda x: x.isalnum() or x == "-", nyoo))
+
+
+# German-umlaut digraphs and the single-letter form unidecode already produces
+# for the diacritic itself (ö -> o, ä -> a, ü -> u).
+_UMLAUT_DIGRAPHS = (("oe", "o"), ("ae", "a"), ("ue", "u"))
+
+
+def search_variants(name: str) -> set:
+    """Expand a search query into the slugs worth matching against.
+
+    Returns the normalized slug plus a German-digraph-collapsed variant, so a
+    typed ``moebius`` also matches a stored ``mobius``. This is query-only: it
+    does not touch ``normalize_concept_name`` or any stored data. If new fuzzing
+    cases come up, add them here rather than re-normalizing the database.
+    """
+    base = normalize_concept_name(name)
+    collapsed = base
+    for digraph, single in _UMLAUT_DIGRAPHS:
+        collapsed = collapsed.replace(digraph, single)
+    return {base, collapsed}
+
+
+def humanize_concept_name(name: str) -> str:
+    """Turn a raw source name into a human-readable UI label.
+
+    Unlike :func:`normalize_concept_name` (which produces a lowercase, ASCII,
+    URL-safe slug for lookups and routing), this keeps the original casing and
+    diacritics and only undoes the artifacts of URL-derived names: it
+    percent-decodes the string and turns underscores back into spaces.
+
+    Examples:
+        ``Sasakian_manifold``   -> ``Sasakian manifold``
+        ``M%C3%B6bius_strip``   -> ``Möbius strip``
+        ``Möbius strip``        -> ``Möbius strip`` (unchanged)
+    """
+    return unquote(name).replace("_", " ") if name else ""

web/concepts/views.py

@@ -1,60 +1,88 @@
 from concepts.models import Concept, Item
+from concepts.utils import normalize_concept_name, search_variants
+from django.db.models import Q
 from django.shortcuts import get_object_or_404, redirect, render
 
+from web.settings import UI_DOMAINS
+
+
+def _concepts_in_scope():
+    return Concept.objects.filter(domain__in=UI_DOMAINS)
+
+
+def _items_in_scope():
+    return Item.objects.filter(domain__in=UI_DOMAINS)
+
 
 def concept(request, name):
     try:
-        concept = Concept.objects.get(name=name)
+        concept = _concepts_in_scope().get(normal_name=name)
         context = {
             "concept": {
-                "name": concept.name,
+                "normal_name": concept.normal_name,
+                "display_name": concept.display_name,
                 "description": concept.description,
                 "items": [
-                    item.to_dict() for item in Item.objects.filter(concept=concept.id)
+                    item.to_dict()
+                    for item in _items_in_scope().filter(concept=concept.id)
                 ],
             }
         }
+        # Only when reached via search (carries ?q=) do we offer the broader
+        # results list, so direct/linked visits stay clean.
+        query = request.GET.get("q")
+        if query:
+            context["search_query"] = query
         return render(request, "detail.html", context)
     except Concept.DoesNotExist:
         return redirect("/results/" + name)
 
 
 def home(request):
-    autocomplete_names = [c.name for c in Concept.objects.all() if c.name is not None]
+    autocomplete_names = [
+        c.display_name for c in _concepts_in_scope() if c.normal_name is not None
+    ]
+    items = _items_in_scope()
     context = {
         "concepts": autocomplete_names,
         "number_of_links": {
-            "wikidata": Item.objects.filter(source=Item.Source.WIKIDATA).count(),
-            "wikipedia_en": Item.objects.filter(
-                source=Item.Source.WIKIPEDIA_EN
-            ).count(),
-            "nlab": Item.objects.filter(source=Item.Source.NLAB).count(),
-            "mathworld": Item.objects.filter(source=Item.Source.MATHWORLD).count(),
-            "proof_wiki": Item.objects.filter(source=Item.Source.PROOF_WIKI).count(),
-            "encyclopedia_of_mathematics": Item.objects.filter(
+            "wikidata": items.filter(source=Item.Source.WIKIDATA).count(),
+            "wikipedia_en": items.filter(source=Item.Source.WIKIPEDIA_EN).count(),
+            "nlab": items.filter(source=Item.Source.NLAB).count(),
+            "mathworld": items.filter(source=Item.Source.MATHWORLD).count(),
+            "proof_wiki": items.filter(source=Item.Source.PROOF_WIKI).count(),
+            "encyclopedia_of_mathematics": items.filter(
                 source=Item.Source.ENCYCLOPEDIA_OF_MATHEMATICS
             ).count(),
-            "agda_unimath": Item.objects.filter(
-                source=Item.Source.AGDA_UNIMATH
-            ).count(),
-            "lmfdb": Item.objects.filter(source=Item.Source.LMFDB).count(),
+            "agda_unimath": items.filter(source=Item.Source.AGDA_UNIMATH).count(),
+            "lmfdb": items.filter(source=Item.Source.LMFDB).count(),
         },
     }
     return render(request, "index.html", context)
 
 
 def search(request):
-    search_value = request.GET.get("q")
-    return redirect("/concept/" + search_value)
+    search_value = request.GET.get("q") or ""
+    slug = normalize_concept_name(search_value)
+    return redirect(f"/concept/{slug}?q={slug}")
 
 
 def redirect_item_to_concept(request, source, identifier):
     # should this be a permanent redirect?
-    item = get_object_or_404(Item, source=source, identifier=identifier)
-    return redirect("/concept/" + item.concept.name)
+    item = get_object_or_404(_items_in_scope(), source=source, identifier=identifier)
+    return redirect("/concept/" + item.concept.normal_name)
 
 
 def results(request, query):
-    concepts = Concept.objects.filter(name__contains=query)
-    context = {"query": query, "results": [concept.name for concept in concepts]}
+    name_filter = Q()
+    for variant in search_variants(query):
+        name_filter |= Q(normal_name__contains=variant)
+    concepts = _concepts_in_scope().filter(name_filter)
+    context = {
+        "query": query,
+        "results": [
+            {"normal_name": concept.normal_name, "display_name": concept.display_name}
+            for concept in concepts
+        ],
+    }
     return render(request, "results.html", context)

web/web/settings.py

@@ -13,7 +13,7 @@
 from os import path
 from pathlib import Path
 
-from decouple import config
+from decouple import Csv, config
 
 # Build paths inside the project like this: BASE_DIR / 'subdir'.
 BASE_DIR = Path(__file__).resolve().parent.parent
@@ -187,3 +187,8 @@
 # Feature flags
 IMPORT_MATH_ENTITIES = config("IMPORT_MATH_ENTITIES", default=True, cast=bool)
 IMPORT_PHYSICS_ENTITIES = config("IMPORT_PHYSICS_ENTITIES", default=False, cast=bool)
+
+# Which Item/Concept domains the UI exposes. Comma-separated list of domain
+# codes (see concepts.models.Domain), e.g. "math" or "math,phys". A single
+# entry locks the entire UI to that domain.
+UI_DOMAINS = config("UI_DOMAINS", default="math", cast=Csv())