diff --git a/.jules/bolt.md b/.jules/bolt.md index 7e410c4..103a693 100644 --- a/.jules/bolt.md +++ b/.jules/bolt.md @@ -9,3 +9,7 @@ ## 2026-06-10 - [Lazy Loading Scikit-learn & NumPy] **Learning:** Top-level imports of `numpy` and `joblib` plus loading a serialized model file (`.joblib`) during FastAPI startup adds significant overhead (e.g., ~3 seconds). Refactoring this into a service with `cached_property` and local imports deferred the cost until the first request. **Action:** Move all heavy ML model loading and their dependencies into lazy-loaded properties to ensure near-instant application startup. + +## 2026-06-11 - [ML Inference Caching & NumPy Overhead] +**Learning:** Using a per-instance `lru_cache` (via a `cached_property` returning a decorated inner function) for ML inference avoids "unhashable self" errors and prevents cross-instance memory leaks. Additionally, for single-sample predictions in Scikit-learn, passing a list of tuples directly is faster than the overhead of NumPy array allocation and reshaping. +**Action:** Implement per-instance caching for model inference and evaluate if passing raw Python types to ML models is more efficient than NumPy for small/single-sample inputs. diff --git a/ml_service.py b/ml_service.py index bee0bde..7d0fccf 100644 --- a/ml_service.py +++ b/ml_service.py @@ -1,4 +1,4 @@ -from functools import cached_property +from functools import cached_property, lru_cache import logging logger = logging.getLogger(__name__) @@ -15,12 +15,22 @@ def model(self): logger.error(f"Error loading ML model: {e}") return None + @cached_property + def _cached_predict(self): + """Internal cached prediction function to avoid unhashable 'self'.""" + @lru_cache(maxsize=128) + def _predict(features_tuple): + # Pass a list containing the features tuple directly to the model. + # This avoids the overhead of numpy array creation and reshaping. + prediction = self.model.predict([features_tuple]) + return int(prediction[0]) + return _predict + def predict(self, features_list): - """Perform prediction using the lazy-loaded model.""" + """Perform prediction using the lazy-loaded model with caching.""" if self.model is None: raise RuntimeError("ML model could not be loaded") - import numpy as np - features = np.array(features_list).reshape(1, -1) - prediction = self.model.predict(features) - return int(prediction[0]) + # Convert list to tuple to make it hashable for LRU cache + features_tuple = tuple(features_list) + return self._cached_predict(features_tuple)