The entries are organized into a concept network that connects the same concept appearing in different sources.
The team: Katja Berčič and Slobodan Stanojevikj.
diff --git a/web/concepts/views.py b/web/concepts/views.py index 81b0ec7..dd3ab3f 100644 --- a/web/concepts/views.py +++ b/web/concepts/views.py @@ -37,6 +37,7 @@ def home(request): "agda_unimath": Item.objects.filter( source=Item.Source.AGDA_UNIMATH ).count(), + "lmfdb": Item.objects.filter(source=Item.Source.LMFDB).count(), }, } return render(request, "index.html", context) diff --git a/web/requirements.txt b/web/requirements.txt index ebbf308..be3fa51 100644 --- a/web/requirements.txt +++ b/web/requirements.txt @@ -5,6 +5,7 @@ spacy~=3.7.0 --prefer-binary scispacy~=0.6.2 python-decouple~=3.8 unidecode~=1.4.0 +lmfdb-lite[pgbinary] @ git+https://github.com/roed314/lmfdb-lite.git # LLM dependencies (optional, install based on which LLM you want to use) # For paid APIs: diff --git a/web/slurper/management/commands/clear_lmfdb.py b/web/slurper/management/commands/clear_lmfdb.py new file mode 100644 index 0000000..0c0f599 --- /dev/null +++ b/web/slurper/management/commands/clear_lmfdb.py @@ -0,0 +1,46 @@ +import logging +import sys +from datetime import timedelta + +from concepts.models import Item +from django.core.management.base import BaseCommand +from slurper.models import SlurperRun + +MIN_INTERVAL = timedelta(days=7) + + +class Command(BaseCommand): + help = ( + "Delete all LMFDB items. Guarded by a 7-day throttle; use --force to override." + ) + + def add_arguments(self, parser): + parser.add_argument( + "--force", + action="store_true", + help="Clear even if the LMFDB slurper ran within the last 7 days.", + ) + + def handle(self, *args, force=False, **options): + source = Item.Source.LMFDB + if not force and not SlurperRun.can_run(source, MIN_INTERVAL): + if sys.stdin.isatty(): + answer = ( + input( + f"LMFDB slurper ran within the last {MIN_INTERVAL.days} days. " + f"Clear anyway? [y/N] " + ) + .strip() + .lower() + ) + if answer not in ("y", "yes"): + logging.info(f"[{source.label}] clear cancelled.") + return + else: + logging.info( + f"[{source.label}] clear skipped: ran less than " + f"{MIN_INTERVAL.days} days ago (use --force to override)." + ) + return + deleted, _ = Item.objects.filter(source=source).delete() + logging.info(f"[{source.label}] cleared {deleted} items.") diff --git a/web/slurper/management/commands/import_lmfdb.py b/web/slurper/management/commands/import_lmfdb.py new file mode 100644 index 0000000..d8c964b --- /dev/null +++ b/web/slurper/management/commands/import_lmfdb.py @@ -0,0 +1,14 @@ +from django.core.management.base import BaseCommand +from slurper import source_lmfdb + + +class Command(BaseCommand): + def add_arguments(self, parser): + parser.add_argument( + "--force", + action="store_true", + help="Bypass the 7-day throttle and run anyway.", + ) + + def handle(self, *args, force=False, **options): + source_lmfdb.LMFDB_SLURPER.save_items(force=force) diff --git a/web/slurper/migrations/0001_initial.py b/web/slurper/migrations/0001_initial.py new file mode 100644 index 0000000..b4d716c --- /dev/null +++ b/web/slurper/migrations/0001_initial.py @@ -0,0 +1,22 @@ +# Generated by Django 4.2.28 on 2026-04-24 15:38 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='SlurperRun', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('source', models.CharField(max_length=8, unique=True)), + ('last_succeeded_at', models.DateTimeField()), + ], + ), + ] diff --git a/web/slurper/migrations/__init__.py b/web/slurper/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/web/slurper/models.py b/web/slurper/models.py new file mode 100644 index 0000000..319ef24 --- /dev/null +++ b/web/slurper/models.py @@ -0,0 +1,26 @@ +from datetime import timedelta + +from django.db import models +from django.utils import timezone + + +class SlurperRun(models.Model): + """Tracks the last successful run of a named slurper for throttling.""" + + source = models.CharField(max_length=8, unique=True) + last_succeeded_at = models.DateTimeField() + + @classmethod + def can_run(cls, source: str, min_interval: timedelta) -> bool: + try: + last = cls.objects.get(source=source).last_succeeded_at + except cls.DoesNotExist: + return True + return timezone.now() - last >= min_interval + + @classmethod + def mark_ran(cls, source: str) -> None: + cls.objects.update_or_create( + source=source, + defaults={"last_succeeded_at": timezone.now()}, + ) diff --git a/web/slurper/source_lmfdb.py b/web/slurper/source_lmfdb.py new file mode 100644 index 0000000..59dc2e6 --- /dev/null +++ b/web/slurper/source_lmfdb.py @@ -0,0 +1,57 @@ +import logging +from datetime import timedelta + +from concepts.models import Item +from django.db.utils import IntegrityError +from psycopg2.sql import SQL +from slurper.models import SlurperRun + + +class LmfdbSlurper: + KNOWL_URL_PREFIX = "https://www.lmfdb.org/knowledge/show/" + MIN_INTERVAL = timedelta(days=7) + + def __init__(self): + self.source = Item.Source.LMFDB + + def fetch_rows(self): + from lmf import db + + cur = db._execute(SQL("SELECT id, title, content FROM kwl_knowls")) + columns = [desc[0] for desc in cur.description] + for row in cur: + yield dict(zip(columns, row)) + + def row_to_item(self, row) -> Item: + return Item( + source=self.source, + identifier=row["id"], + url=self.KNOWL_URL_PREFIX + row["id"], + name=row["title"], + description=row["content"], + ) + + def save_items(self, force: bool = False): + if not force and not SlurperRun.can_run(self.source, self.MIN_INTERVAL): + logging.info( + f"[{self.source.label}] skipped: ran less than " + f"{self.MIN_INTERVAL.days} days ago (use --force to override)." + ) + return + total_saved = 0 + for row in self.fetch_rows(): + item = self.row_to_item(row) + try: + item.save() + total_saved += 1 + except IntegrityError: + logging.info( + f"Item {item.source} {item.identifier} is already in the database." + ) + SlurperRun.mark_ran(self.source) + logging.info( + f"[{self.source.label}] save_items finished: {total_saved} items saved." + ) + + +LMFDB_SLURPER = LmfdbSlurper() diff --git a/web/web/management/commands/rebuild_db.py b/web/web/management/commands/rebuild_db.py index e0f347f..5677a68 100644 --- a/web/web/management/commands/rebuild_db.py +++ b/web/web/management/commands/rebuild_db.py @@ -6,6 +6,12 @@ class Command(BaseCommand): def handle(self, *args, **options): print("clearing data: agda-unimath") call_command("clear_agda_unimath") + print( + "clearing data: LMFDB " + "(skipped if the LMFDB slurper ran in the last 7 days; " + "use `clear_lmfdb --force` to override)" + ) + call_command("clear_lmfdb") print("clearing data: Wikidata") call_command("clear_wikidata") print("clearing data: concepts") @@ -15,6 +21,12 @@ def handle(self, *args, **options): call_command("import_wikidata") print("importing data: agda-unimath") call_command("import_agda_unimath") + print( + "importing data: LMFDB " + "(skipped if the LMFDB slurper ran in the last 7 days; " + "use `import_lmfdb --force` to override)" + ) + call_command("import_lmfdb") print("linking: items with the same name") call_command("link_same") print("computing concepts")