diff --git a/.jules/bolt.md b/.jules/bolt.md index 7e410c4..a5309cd 100644 --- a/.jules/bolt.md +++ b/.jules/bolt.md @@ -9,3 +9,7 @@ ## 2026-06-10 - [Lazy Loading Scikit-learn & NumPy] **Learning:** Top-level imports of `numpy` and `joblib` plus loading a serialized model file (`.joblib`) during FastAPI startup adds significant overhead (e.g., ~3 seconds). Refactoring this into a service with `cached_property` and local imports deferred the cost until the first request. **Action:** Move all heavy ML model loading and their dependencies into lazy-loaded properties to ensure near-instant application startup. + +## 2026-06-11 - [ML Service Hot-Path Optimization] +**Learning:** Applying `lru_cache` to an instance method can lead to memory leaks as the cache holds a strong reference to `self`. Using a module-level function for caching instead is safer. Additionally, for single-sample predictions, passing a list containing a tuple (e.g., `model.predict([features_tuple])`) to Scikit-learn is faster than creating and reshaping a Numpy array, especially when combined with removing local `numpy` imports. +**Action:** Use module-level functions for caching to avoid instance leaks, and minimize object creation/conversion (like Numpy array initialization) in high-frequency prediction loops. diff --git a/ml_service.py b/ml_service.py index bee0bde..6ced81e 100644 --- a/ml_service.py +++ b/ml_service.py @@ -1,8 +1,20 @@ -from functools import cached_property +from functools import cached_property, lru_cache import logging logger = logging.getLogger(__name__) +@lru_cache(maxsize=128) +def _get_cached_prediction(model, features_tuple): + """ + Internal cached prediction function. + Taking model as an argument and being top-level avoids the lru_cache + memory leak associated with instance methods. + """ + # Scikit-learn can accept a list containing a tuple. + # This avoids converting the tuple back to a list in the hot path. + prediction = model.predict([features_tuple]) + return int(prediction[0]) + class MLService: @cached_property def model(self): @@ -16,11 +28,10 @@ def model(self): return None def predict(self, features_list): - """Perform prediction using the lazy-loaded model.""" + """Perform prediction with caching and low overhead.""" if self.model is None: raise RuntimeError("ML model could not be loaded") - import numpy as np - features = np.array(features_list).reshape(1, -1) - prediction = self.model.predict(features) - return int(prediction[0]) + # Convert list to tuple for hashability in lru_cache + features_tuple = tuple(features_list) + return _get_cached_prediction(self.model, features_tuple)