diff --git a/.jules/bolt.md b/.jules/bolt.md index 7e410c4..de404ad 100644 --- a/.jules/bolt.md +++ b/.jules/bolt.md @@ -9,3 +9,7 @@ ## 2026-06-10 - [Lazy Loading Scikit-learn & NumPy] **Learning:** Top-level imports of `numpy` and `joblib` plus loading a serialized model file (`.joblib`) during FastAPI startup adds significant overhead (e.g., ~3 seconds). Refactoring this into a service with `cached_property` and local imports deferred the cost until the first request. **Action:** Move all heavy ML model loading and their dependencies into lazy-loaded properties to ensure near-instant application startup. + +## 2026-06-11 - [ML Prediction Caching & NumPy Overhead] +**Learning:** Standard `lru_cache` on instance methods can lead to memory leaks and issues with unhashable `self`. Using a `cached_property` that returns a decorated inner function allows for per-instance caching while keeping the cache key simple and avoiding unhashable `self` issues. Additionally, passing a list of tuples instead of a `numpy` array for single-sample predictions in Scikit-learn models reduces overhead by eliminating `numpy` imports and array allocations in the hot path. +**Action:** Implement per-instance caching for ML predictions using `cached_property` and closures, and favor plain Python structures over `numpy` for single-sample inference in high-frequency paths. diff --git a/ml_service.py b/ml_service.py index bee0bde..aa64ef1 100644 --- a/ml_service.py +++ b/ml_service.py @@ -3,6 +3,7 @@ logger = logging.getLogger(__name__) + class MLService: @cached_property def model(self): @@ -16,11 +17,24 @@ def model(self): return None def predict(self, features_list): - """Perform prediction using the lazy-loaded model.""" + """Perform prediction using the lazy-loaded model with caching.""" if self.model is None: raise RuntimeError("ML model could not be loaded") - import numpy as np - features = np.array(features_list).reshape(1, -1) - prediction = self.model.predict(features) - return int(prediction[0]) + # Convert list to tuple for caching (tuples are hashable) + features_tuple = tuple(features_list) + prediction = self._predict_with_cache(features_tuple) + return int(prediction) + + @cached_property + def _predict_with_cache(self): + """Lazy-loaded LRU cache for predictions.""" + from functools import lru_cache + + @lru_cache(maxsize=128) + def _get_prediction(features_tuple): + # The model is accessed from the outer scope (the MLService instance) + # This avoids including the unhashable model object in the cache key. + return self.model.predict([features_tuple])[0] + + return _get_prediction