Delete some solr updater dead code + small local solr-updater perf improvements

Makefile

@@ -69,13 +69,10 @@ distclean:
 reindex-solr:
     # Keep link in sync with ol-solr-updater-start and Jenkinsfile
 	curl -C - -L "https://archive.org/download/2023_openlibrary_osp_counts/osp_totals.db" -o $(OSP_DUMP_LOCATION)
-	psql --host db openlibrary -t -c 'select key from thing' | sed 's/ *//' | grep '^/books/' | PYTHONPATH=$(PWD) xargs python openlibrary/solr/update.py --ol-url http://web:8080/ --osp-dump $(OSP_DUMP_LOCATION) --ol-config conf/openlibrary.yml --data-provider=legacy --solr-next
-	psql --host db openlibrary -t -c 'select key from thing' | sed 's/ *//' | grep '^/authors/' | PYTHONPATH=$(PWD) xargs python openlibrary/solr/update.py --ol-url http://web:8080/ --osp-dump $(OSP_DUMP_LOCATION) --ol-config conf/openlibrary.yml --data-provider=legacy --solr-next
-	psql --host db openlibrary -t -c 'select key from thing' | sed 's/ *//' | grep -E '/(lists|series)/' | PYTHONPATH=$(PWD) xargs python openlibrary/solr/update.py --ol-url http://web:8080/ --osp-dump $(OSP_DUMP_LOCATION) --ol-config conf/openlibrary.yml --data-provider=legacy --solr-next
-	PYTHONPATH=$(PWD) python ./scripts/solr_builder/solr_builder/index_subjects.py subject
-	PYTHONPATH=$(PWD) python ./scripts/solr_builder/solr_builder/index_subjects.py person
-	PYTHONPATH=$(PWD) python ./scripts/solr_builder/solr_builder/index_subjects.py place
-	PYTHONPATH=$(PWD) python ./scripts/solr_builder/solr_builder/index_subjects.py time
+	psql --host db openlibrary -t -c 'select key from thing' | sed 's/ *//' | grep '^/books/' | xargs python openlibrary/solr/update.py --ol-url http://web:8080/ --osp-dump $(OSP_DUMP_LOCATION) --ol-config conf/openlibrary.yml --solr-next
+	psql --host db openlibrary -t -c 'select key from thing' | sed 's/ *//' | grep '^/authors/' | xargs python openlibrary/solr/update.py --ol-url http://web:8080/ --osp-dump $(OSP_DUMP_LOCATION) --ol-config conf/openlibrary.yml --solr-next
+	psql --host db openlibrary -t -c 'select key from thing' | sed 's/ *//' | grep -E '/(lists|series)/' | xargs python openlibrary/solr/update.py --ol-url http://web:8080/ --osp-dump $(OSP_DUMP_LOCATION) --ol-config conf/openlibrary.yml --solr-next
+	parallel -j4 python ./scripts/solr_builder/solr_builder/index_subjects.py ::: subject person place time
 
 lint:
 	# See the pyproject.toml file for ruff's settings

compose.override.yaml

@@ -59,8 +59,8 @@ services:
       test: ["CMD", "curl", "-f", "http://localhost:8983/solr/openlibrary/admin/ping"]
       interval: 30s
       start_interval: 1s
-      start_period: 20s
-      timeout: 1s
+      start_period: 60s
+      timeout: 5s
       retries: 24
     volumes:
       - ./docker/ol-local-solr-start.sh:/docker/ol-solr-start.sh:ro
@@ -75,6 +75,7 @@ services:
       dockerfile: docker/Dockerfile.oldev
     environment:
       - OL_SOLR_NEXT=true
+      - LOCAL_DEV=true
     depends_on:
       solr:
         condition: service_healthy

openlibrary/core/env.py

@@ -7,6 +7,10 @@ class OLEnv:
     def OL_EXPOSE_SOLR_INTERNALS_PARAMS(self) -> bool:
         return os.environ.get("OL_EXPOSE_SOLR_INTERNALS_PARAMS") == "true"
 
+    @cached_property
+    def LOCAL_DEV(self) -> bool:
+        return os.environ.get("LOCAL_DEV") == "true"
+
 
 _ol_env = OLEnv()
 

openlibrary/core/lending.py

@@ -19,6 +19,7 @@
 from infogami.utils.view import public
 from openlibrary.accounts.model import OpenLibraryAccount
 from openlibrary.core import cache, stats
+from openlibrary.core.env import get_ol_env
 from openlibrary.plugins.upstream.utils import urlencode
 from openlibrary.utils import dateutil, uniq
 from openlibrary.utils.async_utils import async_bridge
@@ -113,7 +114,6 @@ def compose_ia_url(
     query=None,
     sorts=None,
     advanced: bool = True,
-    rate_limit_exempt: bool = True,
     safe_mode: bool = False,
 ) -> str | None:
     """This needs to be exposed by a generalized API endpoint within
@@ -175,7 +175,8 @@ def compose_ia_url(
         ("page", page),
         ("output", "json"),
     ]
-    if rate_limit_exempt:
+    if not get_ol_env().LOCAL_DEV:
+        # This flag is only available on prod
         params.append(("service", "metadata__unlimited"))
     if not sorts or not isinstance(sorts, list):
         sorts = [""]

openlibrary/core/ratings.py

@@ -81,18 +81,15 @@ def get_rating_stats(cls, work_id) -> dict:
     @classmethod
     def get_work_ratings_summary(cls, work_id: int) -> WorkRatingsSummary | None:
         oldb = db.get_db()
-        # NOTE: Using some old postgres syntax here :/ for modern postgres syntax,
-        # see the query in solr_builder.py
         query = """
             SELECT
-                sum( CASE WHEN rating = 1 THEN 1 ELSE 0 END ) as ratings_count_1,
-                sum( CASE WHEN rating = 2 THEN 1 ELSE 0 END ) as ratings_count_2,
-                sum( CASE WHEN rating = 3 THEN 1 ELSE 0 END ) as ratings_count_3,
-                sum( CASE WHEN rating = 4 THEN 1 ELSE 0 END ) as ratings_count_4,
-                sum( CASE WHEN rating = 5 THEN 1 ELSE 0 END ) as ratings_count_5
+                count(*) FILTER (WHERE rating = 1) AS ratings_count_1,
+                count(*) FILTER (WHERE rating = 2) AS ratings_count_2,
+                count(*) FILTER (WHERE rating = 3) AS ratings_count_3,
+                count(*) FILTER (WHERE rating = 4) AS ratings_count_4,
+                count(*) FILTER (WHERE rating = 5) AS ratings_count_5
             FROM ratings
             WHERE work_id = $work_id
-            GROUP BY work_id
         """
         result = oldb.query(query, vars={"work_id": work_id})
         if not result: