diff --git a/.jules/bolt.md b/.jules/bolt.md index 533e206..80c9409 100644 --- a/.jules/bolt.md +++ b/.jules/bolt.md @@ -13,3 +13,7 @@ ## 2026-06-11 - [ML Prediction Path Optimization] **Learning:** For low-latency ML services, the overhead of creating NumPy arrays and importing NumPy in the hot path can be significant (~10% of execution time for simple models). Furthermore, instance-level caching using `lru_cache` inside a `cached_property` provides massive speedups for repeated requests without the memory risks of global caches or unhashable `self` issues. **Action:** Use `tuple` conversion and per-instance `lru_cache` for numerical feature caching. Pass lists directly to scikit-learn's `predict` to avoid unnecessary NumPy allocations in the hot path. + +## 2026-06-12 - [LLM Pipeline Caching & Truncation] +**Learning:** Using `@lru_cache` on instance methods leads to memory leaks and hashability issues. Combining `@cached_property` for the heavy pipeline object with an internal cached function for results ensures both fast loading and efficient inference without reloading the model. Enabling `truncation=True` is critical for robustness against long inputs. +**Action:** Use the per-instance caching pattern (cached property returning an inner decorated function) for all model inference services. Always enable truncation for LLM pipelines unless full context is strictly required. diff --git a/llm_service.py b/llm_service.py index 831fda6..4192380 100644 --- a/llm_service.py +++ b/llm_service.py @@ -1,17 +1,43 @@ -from functools import cached_property, lru_cache +import functools +import logging + +# Configure logging +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) class LLMService: - @cached_property + @functools.cached_property def classifier(self): - # Lazy load transformers and the pipeline to improve startup time - from transformers import pipeline - return pipeline("sentiment-analysis", model="distilbert-base-uncased-finetuned-sst-2-english") + """Lazy load the sentiment analysis pipeline with truncation enabled.""" + try: + # Local import to speed up initial service instantiation + from transformers import pipeline + logger.info("Loading sentiment-analysis pipeline...") + # DistilBERT is used for efficient inference. + # truncation=True ensures inputs > 512 tokens are handled without error. + return pipeline( + "sentiment-analysis", + model="distilbert-base-uncased-finetuned-sst-2-english", + truncation=True + ) + except Exception as e: + logger.error(f"Failed to load LLM pipeline: {e}") + raise RuntimeError(f"Could not initialize LLM classifier: {e}") + + @functools.cached_property + def _cached_analyze_sentiment(self): + """Internal cached function to provide per-instance result caching.""" + @functools.lru_cache(maxsize=128) + def _analyze(text: str): + # Accessing self.classifier triggers the lazy loading (if not already loaded) + # and returns the pipeline object which is then called. + result = self.classifier(text) + return result[0] + return _analyze - @lru_cache(maxsize=128) def analyze_sentiment(self, text: str): - # Cache results to speed up repeated requests with the same text - result = self.classifier(text) - return result[0] + """Analyze text sentiment with per-instance caching and automatic truncation.""" + return self._cached_analyze_sentiment(text) if __name__ == "__main__": service = LLMService()