From 7a931c78cc95adbb8bcf6164bf6d9a42881f7db7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Katja=20Ber=C4=8Di=C4=8D?= Date: Fri, 24 Apr 2026 16:19:52 +0200 Subject: [PATCH 1/6] Some Wikipedia entries need more than 200 chars, increase to 300; add the source LMFDB --- web/concepts/models.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/web/concepts/models.py b/web/concepts/models.py index cd5bb71..fd1a0d4 100644 --- a/web/concepts/models.py +++ b/web/concepts/models.py @@ -76,6 +76,7 @@ class Source(models.TextChoices): ENCYCLOPEDIA_OF_MATHEMATICS = "EoM", "Encyclopedia of Mathematics" WIKIPEDIA_EN = "WpEN", "Wikipedia (English)" AGDA_UNIMATH = "AUm", "Agda Unimath" + LMFDB = "LMF", "The L-functions and modular forms database" @staticmethod def key(): @@ -87,6 +88,7 @@ def key(): Item.Source.PROOF_WIKI, Item.Source.ENCYCLOPEDIA_OF_MATHEMATICS, Item.Source.AGDA_UNIMATH, + Item.Source.LMFDB, ] return lambda item: SOURCES.index(item.source) @@ -94,9 +96,9 @@ def key(): max_length=4, choices=Domain.choices, default=Domain.MATHEMATICS ) source = models.CharField(max_length=4, choices=Source.choices) - identifier = models.CharField(max_length=200) - url = models.URLField(max_length=200) - name = models.CharField(max_length=200, null=True) + identifier = models.CharField(max_length=300) + url = models.URLField(max_length=300) + name = models.CharField(max_length=300, null=True) description = models.TextField(null=True) keywords = models.TextField(null=True, blank=True) article_text = models.TextField(null=True, blank=True) From 2588ea6f6f96bd9f4ffea35469366cb90bbadf64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Katja=20Ber=C4=8Di=C4=8D?= Date: Fri, 24 Apr 2026 16:26:50 +0200 Subject: [PATCH 2/6] Add a slurper for the LMFDB --- .../migrations/0020_alter_item_source.py | 18 +++++++ ...entifier_alter_item_name_alter_item_url.py | 28 +++++++++++ web/concepts/templates/index.html | 1 + web/concepts/views.py | 1 + .../management/commands/clear_lmfdb.py | 7 +++ .../management/commands/import_lmfdb.py | 7 +++ web/slurper/source_lmfdb.py | 47 +++++++++++++++++++ 7 files changed, 109 insertions(+) create mode 100644 web/concepts/migrations/0020_alter_item_source.py create mode 100644 web/concepts/migrations/0021_alter_item_identifier_alter_item_name_alter_item_url.py create mode 100644 web/slurper/management/commands/clear_lmfdb.py create mode 100644 web/slurper/management/commands/import_lmfdb.py create mode 100644 web/slurper/source_lmfdb.py diff --git a/web/concepts/migrations/0020_alter_item_source.py b/web/concepts/migrations/0020_alter_item_source.py new file mode 100644 index 0000000..e68e742 --- /dev/null +++ b/web/concepts/migrations/0020_alter_item_source.py @@ -0,0 +1,18 @@ +# Generated by Django 4.2.28 on 2026-04-22 07:46 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('concepts', '0019_remove_concept_unique_lower_name_concept_normal_name_and_more'), + ] + + operations = [ + migrations.AlterField( + model_name='item', + name='source', + field=models.CharField(choices=[('Wd', 'Wikidata'), ('nL', 'nLab'), ('MW', 'MathWorld'), ('PW', 'ProofWiki'), ('EoM', 'Encyclopedia of Mathematics'), ('WpEN', 'Wikipedia (English)'), ('AUm', 'Agda Unimath'), ('LMF', 'The L-functions and modular forms database')], max_length=4), + ), + ] diff --git a/web/concepts/migrations/0021_alter_item_identifier_alter_item_name_alter_item_url.py b/web/concepts/migrations/0021_alter_item_identifier_alter_item_name_alter_item_url.py new file mode 100644 index 0000000..ab5353d --- /dev/null +++ b/web/concepts/migrations/0021_alter_item_identifier_alter_item_name_alter_item_url.py @@ -0,0 +1,28 @@ +# Generated by Django 4.2.28 on 2026-04-24 10:08 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('concepts', '0020_alter_item_source'), + ] + + operations = [ + migrations.AlterField( + model_name='item', + name='identifier', + field=models.CharField(max_length=300), + ), + migrations.AlterField( + model_name='item', + name='name', + field=models.CharField(max_length=300, null=True), + ), + migrations.AlterField( + model_name='item', + name='url', + field=models.URLField(max_length=300), + ), + ] diff --git a/web/concepts/templates/index.html b/web/concepts/templates/index.html index 9f7ca54..554f332 100644 --- a/web/concepts/templates/index.html +++ b/web/concepts/templates/index.html @@ -28,6 +28,7 @@

About

  • ProofWiki ({{ number_of_links.proof_wiki }} entries),
  • Encyclopedia of mathematics ({{ number_of_links.encyclopedia_of_mathematics }} entries),
  • Agda Unimath ({{ number_of_links.agda_unimath }} entries)
    • +
  • LMFDB ({{ number_of_links.lmfdb }} entries)

    • The entries are organized into a concept network that connects the same concept appearing in different sources.

    The team: Katja Berčič and Slobodan Stanojevikj.

    diff --git a/web/concepts/views.py b/web/concepts/views.py index 81b0ec7..dd3ab3f 100644 --- a/web/concepts/views.py +++ b/web/concepts/views.py @@ -37,6 +37,7 @@ def home(request): "agda_unimath": Item.objects.filter( source=Item.Source.AGDA_UNIMATH ).count(), + "lmfdb": Item.objects.filter(source=Item.Source.LMFDB).count(), }, } return render(request, "index.html", context) diff --git a/web/slurper/management/commands/clear_lmfdb.py b/web/slurper/management/commands/clear_lmfdb.py new file mode 100644 index 0000000..2ef2c94 --- /dev/null +++ b/web/slurper/management/commands/clear_lmfdb.py @@ -0,0 +1,7 @@ +from concepts.models import Item +from django.core.management.base import BaseCommand + + +class Command(BaseCommand): + def handle(self, *args, **options): + Item.objects.filter(source=Item.Source.LMFDB).delete() diff --git a/web/slurper/management/commands/import_lmfdb.py b/web/slurper/management/commands/import_lmfdb.py new file mode 100644 index 0000000..e22c50b --- /dev/null +++ b/web/slurper/management/commands/import_lmfdb.py @@ -0,0 +1,7 @@ +from django.core.management.base import BaseCommand +from slurper import source_lmfdb + + +class Command(BaseCommand): + def handle(self, *args, **options): + source_lmfdb.LMFDB_SLURPER.save_items() diff --git a/web/slurper/source_lmfdb.py b/web/slurper/source_lmfdb.py new file mode 100644 index 0000000..24bccea --- /dev/null +++ b/web/slurper/source_lmfdb.py @@ -0,0 +1,47 @@ +import logging + +from concepts.models import Item +from django.db.utils import IntegrityError +from psycopg2.sql import SQL + + +class LmfdbSlurper: + KNOWL_URL_PREFIX = "https://www.lmfdb.org/knowledge/show/" + + def __init__(self): + self.source = Item.Source.LMFDB + + def fetch_rows(self): + from lmf import db + + cur = db._execute(SQL("SELECT id, title, content FROM kwl_knowls")) + columns = [desc[0] for desc in cur.description] + for row in cur: + yield dict(zip(columns, row)) + + def row_to_item(self, row) -> Item: + return Item( + source=self.source, + identifier=row["id"], + url=self.KNOWL_URL_PREFIX + row["id"], + name=row["title"], + description=row["content"], + ) + + def save_items(self): + total_saved = 0 + for row in self.fetch_rows(): + item = self.row_to_item(row) + try: + item.save() + total_saved += 1 + except IntegrityError: + logging.info( + f"Item {item.source} {item.identifier} is already in the database." + ) + logging.info( + f"[{self.source.label}] save_items finished: {total_saved} items saved." + ) + + +LMFDB_SLURPER = LmfdbSlurper() From cf237ef156bc9d354f760d96f82934166aabe17c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Katja=20Ber=C4=8Di=C4=8D?= Date: Fri, 24 Apr 2026 17:42:58 +0200 Subject: [PATCH 3/6] Add slurper runs to ensure we don't run certain slurpers too often --- .../management/commands/import_lmfdb.py | 11 ++++++-- web/slurper/migrations/0001_initial.py | 22 ++++++++++++++++ web/slurper/migrations/__init__.py | 0 web/slurper/models.py | 26 +++++++++++++++++++ web/slurper/source_lmfdb.py | 12 ++++++++- 5 files changed, 68 insertions(+), 3 deletions(-) create mode 100644 web/slurper/migrations/0001_initial.py create mode 100644 web/slurper/migrations/__init__.py create mode 100644 web/slurper/models.py diff --git a/web/slurper/management/commands/import_lmfdb.py b/web/slurper/management/commands/import_lmfdb.py index e22c50b..d8c964b 100644 --- a/web/slurper/management/commands/import_lmfdb.py +++ b/web/slurper/management/commands/import_lmfdb.py @@ -3,5 +3,12 @@ class Command(BaseCommand): - def handle(self, *args, **options): - source_lmfdb.LMFDB_SLURPER.save_items() + def add_arguments(self, parser): + parser.add_argument( + "--force", + action="store_true", + help="Bypass the 7-day throttle and run anyway.", + ) + + def handle(self, *args, force=False, **options): + source_lmfdb.LMFDB_SLURPER.save_items(force=force) diff --git a/web/slurper/migrations/0001_initial.py b/web/slurper/migrations/0001_initial.py new file mode 100644 index 0000000..b4d716c --- /dev/null +++ b/web/slurper/migrations/0001_initial.py @@ -0,0 +1,22 @@ +# Generated by Django 4.2.28 on 2026-04-24 15:38 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='SlurperRun', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('source', models.CharField(max_length=8, unique=True)), + ('last_succeeded_at', models.DateTimeField()), + ], + ), + ] diff --git a/web/slurper/migrations/__init__.py b/web/slurper/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/web/slurper/models.py b/web/slurper/models.py new file mode 100644 index 0000000..60dc770 --- /dev/null +++ b/web/slurper/models.py @@ -0,0 +1,26 @@ +from datetime import timedelta + +from django.db import models +from django.utils import timezone + + +class SlurperRun(models.Model): + """Tracks the last successful run of a named slurper for throttling.""" + + source = models.CharField(max_length=8, unique=True) + last_succeeded_at = models.DateTimeField() + + @classmethod + def should_run(cls, source: str, min_interval: timedelta) -> bool: + try: + last = cls.objects.get(source=source).last_succeeded_at + except cls.DoesNotExist: + return True + return timezone.now() - last >= min_interval + + @classmethod + def mark_ran(cls, source: str) -> None: + cls.objects.update_or_create( + source=source, + defaults={"last_succeeded_at": timezone.now()}, + ) diff --git a/web/slurper/source_lmfdb.py b/web/slurper/source_lmfdb.py index 24bccea..d613281 100644 --- a/web/slurper/source_lmfdb.py +++ b/web/slurper/source_lmfdb.py @@ -1,12 +1,15 @@ import logging +from datetime import timedelta from concepts.models import Item from django.db.utils import IntegrityError from psycopg2.sql import SQL +from slurper.models import SlurperRun class LmfdbSlurper: KNOWL_URL_PREFIX = "https://www.lmfdb.org/knowledge/show/" + MIN_INTERVAL = timedelta(days=7) def __init__(self): self.source = Item.Source.LMFDB @@ -28,7 +31,13 @@ def row_to_item(self, row) -> Item: description=row["content"], ) - def save_items(self): + def save_items(self, force: bool = False): + if not force and not SlurperRun.should_run(self.source, self.MIN_INTERVAL): + logging.info( + f"[{self.source.label}] skipped: ran less than " + f"{self.MIN_INTERVAL.days} days ago (use --force to override)." + ) + return total_saved = 0 for row in self.fetch_rows(): item = self.row_to_item(row) @@ -39,6 +48,7 @@ def save_items(self): logging.info( f"Item {item.source} {item.identifier} is already in the database." ) + SlurperRun.mark_ran(self.source) logging.info( f"[{self.source.label}] save_items finished: {total_saved} items saved." ) From 82082c4c86cef1b9402bd65f7a8e7cdafbbbb766 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Katja=20Ber=C4=8Di=C4=8D?= Date: Fri, 24 Apr 2026 17:48:59 +0200 Subject: [PATCH 4/6] Add guard for clearing LMFDB, messages in rebuild_db --- .../management/commands/clear_lmfdb.py | 37 ++++++++++++++++++- web/slurper/models.py | 2 +- web/slurper/source_lmfdb.py | 2 +- web/web/management/commands/rebuild_db.py | 12 ++++++ 4 files changed, 49 insertions(+), 4 deletions(-) diff --git a/web/slurper/management/commands/clear_lmfdb.py b/web/slurper/management/commands/clear_lmfdb.py index 2ef2c94..8087af8 100644 --- a/web/slurper/management/commands/clear_lmfdb.py +++ b/web/slurper/management/commands/clear_lmfdb.py @@ -1,7 +1,40 @@ +import logging +import sys +from datetime import timedelta + from concepts.models import Item from django.core.management.base import BaseCommand +from slurper.models import SlurperRun + +MIN_INTERVAL = timedelta(days=7) class Command(BaseCommand): - def handle(self, *args, **options): - Item.objects.filter(source=Item.Source.LMFDB).delete() + help = "Delete all LMFDB items. Guarded by a 7-day throttle; use --force to override." + + def add_arguments(self, parser): + parser.add_argument( + "--force", + action="store_true", + help="Clear even if the LMFDB slurper ran within the last 7 days.", + ) + + def handle(self, *args, force=False, **options): + source = Item.Source.LMFDB + if not force and not SlurperRun.can_run(source, MIN_INTERVAL): + if sys.stdin.isatty(): + answer = input( + f"LMFDB slurper ran within the last {MIN_INTERVAL.days} days. " + f"Clear anyway? [y/N] " + ).strip().lower() + if answer not in ("y", "yes"): + logging.info(f"[{source.label}] clear cancelled.") + return + else: + logging.info( + f"[{source.label}] clear skipped: ran less than " + f"{MIN_INTERVAL.days} days ago (use --force to override)." + ) + return + deleted, _ = Item.objects.filter(source=source).delete() + logging.info(f"[{source.label}] cleared {deleted} items.") diff --git a/web/slurper/models.py b/web/slurper/models.py index 60dc770..319ef24 100644 --- a/web/slurper/models.py +++ b/web/slurper/models.py @@ -11,7 +11,7 @@ class SlurperRun(models.Model): last_succeeded_at = models.DateTimeField() @classmethod - def should_run(cls, source: str, min_interval: timedelta) -> bool: + def can_run(cls, source: str, min_interval: timedelta) -> bool: try: last = cls.objects.get(source=source).last_succeeded_at except cls.DoesNotExist: diff --git a/web/slurper/source_lmfdb.py b/web/slurper/source_lmfdb.py index d613281..59dc2e6 100644 --- a/web/slurper/source_lmfdb.py +++ b/web/slurper/source_lmfdb.py @@ -32,7 +32,7 @@ def row_to_item(self, row) -> Item: ) def save_items(self, force: bool = False): - if not force and not SlurperRun.should_run(self.source, self.MIN_INTERVAL): + if not force and not SlurperRun.can_run(self.source, self.MIN_INTERVAL): logging.info( f"[{self.source.label}] skipped: ran less than " f"{self.MIN_INTERVAL.days} days ago (use --force to override)." diff --git a/web/web/management/commands/rebuild_db.py b/web/web/management/commands/rebuild_db.py index e0f347f..5677a68 100644 --- a/web/web/management/commands/rebuild_db.py +++ b/web/web/management/commands/rebuild_db.py @@ -6,6 +6,12 @@ class Command(BaseCommand): def handle(self, *args, **options): print("clearing data: agda-unimath") call_command("clear_agda_unimath") + print( + "clearing data: LMFDB " + "(skipped if the LMFDB slurper ran in the last 7 days; " + "use `clear_lmfdb --force` to override)" + ) + call_command("clear_lmfdb") print("clearing data: Wikidata") call_command("clear_wikidata") print("clearing data: concepts") @@ -15,6 +21,12 @@ def handle(self, *args, **options): call_command("import_wikidata") print("importing data: agda-unimath") call_command("import_agda_unimath") + print( + "importing data: LMFDB " + "(skipped if the LMFDB slurper ran in the last 7 days; " + "use `import_lmfdb --force` to override)" + ) + call_command("import_lmfdb") print("linking: items with the same name") call_command("link_same") print("computing concepts") From 4ae98737ebb15ed510a0fceb4fb03721c6d86c2d Mon Sep 17 00:00:00 2001 From: Slobodan Stanojevikj Date: Wed, 6 May 2026 21:18:10 +0200 Subject: [PATCH 5/6] -minor reformatting fix --- web/slurper/management/commands/clear_lmfdb.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/web/slurper/management/commands/clear_lmfdb.py b/web/slurper/management/commands/clear_lmfdb.py index 8087af8..0c0f599 100644 --- a/web/slurper/management/commands/clear_lmfdb.py +++ b/web/slurper/management/commands/clear_lmfdb.py @@ -10,7 +10,9 @@ class Command(BaseCommand): - help = "Delete all LMFDB items. Guarded by a 7-day throttle; use --force to override." + help = ( + "Delete all LMFDB items. Guarded by a 7-day throttle; use --force to override." + ) def add_arguments(self, parser): parser.add_argument( @@ -23,10 +25,14 @@ def handle(self, *args, force=False, **options): source = Item.Source.LMFDB if not force and not SlurperRun.can_run(source, MIN_INTERVAL): if sys.stdin.isatty(): - answer = input( - f"LMFDB slurper ran within the last {MIN_INTERVAL.days} days. " - f"Clear anyway? [y/N] " - ).strip().lower() + answer = ( + input( + f"LMFDB slurper ran within the last {MIN_INTERVAL.days} days. " + f"Clear anyway? [y/N] " + ) + .strip() + .lower() + ) if answer not in ("y", "yes"): logging.info(f"[{source.label}] clear cancelled.") return From c680015db77af4afe5826b8a03a09b288cd42ad8 Mon Sep 17 00:00:00 2001 From: Slobodan Stanojevikj Date: Wed, 6 May 2026 21:33:14 +0200 Subject: [PATCH 6/6] -added Makefile command -added requirement for lmfdb-dblite --- Makefile | 5 ++++- web/requirements.txt | 1 + 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 643a8c9..ffee521 100644 --- a/Makefile +++ b/Makefile @@ -23,7 +23,7 @@ install-scispacy: start: python ./web/manage.py runserver -populate-db: +import-wikidata: python ./web/manage.py import_wikidata clear-db: @@ -46,3 +46,6 @@ create-migrations: migrate: python ./web/manage.py migrate + +import-lmfdb: + python ./web/manage.py import_lmfdb diff --git a/web/requirements.txt b/web/requirements.txt index ebbf308..be3fa51 100644 --- a/web/requirements.txt +++ b/web/requirements.txt @@ -5,6 +5,7 @@ spacy~=3.7.0 --prefer-binary scispacy~=0.6.2 python-decouple~=3.8 unidecode~=1.4.0 +lmfdb-lite[pgbinary] @ git+https://github.com/roed314/lmfdb-lite.git # LLM dependencies (optional, install based on which LLM you want to use) # For paid APIs: