diff --git a/.jules/bolt.md b/.jules/bolt.md index 699e7a3..3fb069f 100644 --- a/.jules/bolt.md +++ b/.jules/bolt.md @@ -13,3 +13,7 @@ ## 2024-03-16 - Batch JSON File I/O Operations **Learning:** When using JSON file-backed repositories, iterating over items sequentially and calling `save_item` or `record_attempt` inside a loop leads to N+1 file read/write operations. This creates a significant performance bottleneck, especially when importing progress snapshots with numerous items and attempts. **Action:** Prefer batch processing methods (e.g., `save_items`, `record_attempts`) so file-backed adapters can load storage once, update it in memory, and write it back in a single pass. + +## 2025-03-01 - Avoid full deserialization during batch updates +**Learning:** Fully deserializing and re-serializing all JSON records during a batch update (like `save_items`) causes unnecessary O(N) CPU and memory overhead, slowing down updates when the dataset is large. +**Action:** Update the raw JSON dictionary entries in-place by matching string-casted IDs, preventing full repository deserialization and speeding up partial batch updates. diff --git a/src/python_learning_orchestrated/adapters/json_file_practice_repository.py b/src/python_learning_orchestrated/adapters/json_file_practice_repository.py index 15da61b..c53f4f9 100644 --- a/src/python_learning_orchestrated/adapters/json_file_practice_repository.py +++ b/src/python_learning_orchestrated/adapters/json_file_practice_repository.py @@ -46,15 +46,25 @@ def save_items(self, items: list[LearningItem]) -> None: storage = self._load_storage() raw_items = storage.get("items", []) - existing_items = [] - if isinstance(raw_items, list): - existing_items = [ - _item_from_dict(entry) for entry in raw_items if isinstance(entry, dict) - ] - by_id = {existing.id: existing for existing in existing_items} - for item in items: - by_id[item.id] = item - storage["items"] = [_item_to_dict(entry) for entry in by_id.values()] + if not isinstance(raw_items, list): + raw_items = [] + + # Performance optimization: + # Instead of fully deserializing all items (O(N) CPU overhead), + # we convert the new items to dicts and update the raw JSON list in-place. + # This speeds up partial batch updates significantly when the repository is large. + items_to_save = {str(item.id): _item_to_dict(item) for item in items} + + for i, entry in enumerate(raw_items): + if isinstance(entry, dict): + entry_id = str(entry.get("id")) + if entry_id in items_to_save: + raw_items[i] = items_to_save.pop(entry_id) + + for new_item in items_to_save.values(): + raw_items.append(new_item) + + storage["items"] = raw_items self._save_storage(storage) def list_attempts(self) -> list[Attempt]: