diff --git a/README.md b/README.md index 84953ba..bbb61d6 100644 --- a/README.md +++ b/README.md @@ -175,10 +175,34 @@ net_profiler = NetworkProfiler(network_metrics={'bytes_sent': True, 'bytes_recv' with net_profiler.profile_block("api_request"): requests.get('https://api.github.com') -# Manually print stats (since logging is disabled) +# Manually print stats to show they were collected stats = net_profiler.get_stats() -print("Network stats:", stats) +print("Network stats (manually printed):", stats) ``` + +**1.4 Exporting Statistics** + +SmartProfiler allows you to export profiling data to JSON or CSV files, which can be useful for further analysis with tools like pandas or for generating custom reports. + +```python +from smartprofiler import CPUProfiler + +cpu_profiler = CPUProfiler() + +@cpu_profiler.profile_function +def sample_task(): + # Simulate some work + sum(i for i in range(100000)) + +sample_task() + +# Export stats to a JSON file +cpu_profiler.export_stats('cpu_stats.json', format='json') + +# Export stats to a CSV file +cpu_profiler.export_stats('cpu_stats.csv', format='csv') +``` + See `examples/examples_general_usage.py` for more usage examples, including profiling function calls, memory usage, and multithreaded scenarios. ### 2. Visualization Examples diff --git a/examples/examples_general_usage.py b/examples/examples_general_usage.py index cd26dfc..ce95c1d 100644 --- a/examples/examples_general_usage.py +++ b/examples/examples_general_usage.py @@ -2,11 +2,13 @@ import requests import logging import argparse +import tempfile +import os from smartprofiler import CPUProfiler, DiskProfiler, FunctionProfiler, MemoryProfiler, NetworkProfiler def setup_logger(logger_type: str = None) -> logging.Logger: """ - Set up and return a logger based on the specified type. + ... (rest of the code remains the same) Args: logger_type: Type of logger to use ('loguru', 'structlog', or None for default) @@ -54,9 +56,12 @@ def main(): logger = setup_logger(args.logger) print(f"\nUsing {args.logger if args.logger else 'default'} logger\n") + # Get a temporary directory for the current OS + temp_dir = tempfile.gettempdir() + # Initialize profilers with custom logging settings cpu_profiler = CPUProfiler(time_func='execution_time', logger=logger, log_level=logging.DEBUG) - disk_profiler = DiskProfiler(disk_path='/tmp', disk_metrics={'write_bytes': True, 'disk_usage': False}, logger=logger, log_level=logging.DEBUG) + disk_profiler = DiskProfiler(disk_path=temp_dir, disk_metrics={'write_bytes': True, 'disk_usage': False}, logger=logger, log_level=logging.DEBUG) func_profiler = FunctionProfiler(logger=logger, log_level=logging.DEBUG) mem_profiler = MemoryProfiler(logger=logger, log_level=logging.DEBUG) net_profiler = NetworkProfiler(network_metrics={'bytes_sent': True, 'bytes_recv': True}, logger=logger, log_level=logging.DEBUG) @@ -76,7 +81,7 @@ def compute_fibonacci(n): # Example 2: Profile a disk-intensive block print("Profiling a disk-intensive block (writing to a file):") with disk_profiler.profile_block("disk_write_block"): - with open('/tmp/test_file.txt', 'w') as f: + with open(os.path.join(temp_dir, 'test_file.txt'), 'w') as f: f.write("Sample data " * 1000) disk_profiler.summarize_stats() print() @@ -107,6 +112,35 @@ def inner_func(): # Manually print stats to show they were collected stats = net_profiler_silent.get_stats() print("Network stats (manually printed):", stats) + print() + + # Example 6: Exporting statistics to JSON and CSV + print("\nExporting all profiler stats to JSON and CSV files:") + + # Export CPU stats + cpu_profiler.export_stats('cpu_stats.json', format='json') + cpu_profiler.export_stats('cpu_stats.csv', format='csv') + print("- Exported CPU stats to 'cpu_stats.json' and 'cpu_stats.csv'") + + # Export Disk stats + disk_profiler.export_stats('disk_stats.json', format='json') + disk_profiler.export_stats('disk_stats.csv', format='csv') + print("- Exported Disk stats to 'disk_stats.json' and 'disk_stats.csv'") + + # Export Function stats + func_profiler.export_stats('func_stats.json', format='json') + func_profiler.export_stats('func_stats.csv', format='csv') + print("- Exported Function stats to 'func_stats.json' and 'func_stats.csv'") + + # Export Memory stats + mem_profiler.export_stats('mem_stats.json', format='json') + mem_profiler.export_stats('mem_stats.csv', format='csv') + print("- Exported Memory stats to 'mem_stats.json' and 'mem_stats.csv'") + + # Export Network stats + net_profiler_silent.export_stats('network_stats.json', format='json') + net_profiler_silent.export_stats('network_stats.csv', format='csv') + print("- Exported Network stats to 'network_stats.json' and 'network_stats.csv'") if __name__ == "__main__": main() diff --git a/requirements.txt b/requirements.txt index 222fbea..80cc60a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ psutil~=6.1.0 +filelock~=3.18.0 smartprofiler~=0.3.2 setuptools~=75.6.0 requests~=2.32.3 diff --git a/setup.py b/setup.py index 91dcd63..b87ae48 100644 --- a/setup.py +++ b/setup.py @@ -15,6 +15,7 @@ 'matplotlib>=3.5.0', 'numpy>=1.21.0', 'requests>=2.28.0', + 'filelock', 'psutil>=5.9.0', # Added for NetworkProfiler and DiskProfiler ], description='A comprehensive Python library for profiling CPU, disk, memory, network I/O, and function calls with integrated visualization.', diff --git a/smartprofiler/base_profiler.py b/smartprofiler/base_profiler.py index f6dceaa..8389d7c 100644 --- a/smartprofiler/base_profiler.py +++ b/smartprofiler/base_profiler.py @@ -1,7 +1,11 @@ import logging import threading from abc import ABC, abstractmethod -from typing import Optional, Callable, Dict, List, Any +import os +import json +import csv +from typing import Any, Callable, Dict, List, Optional +from filelock import FileLock, Timeout from functools import wraps from .logger_adapter import LoggerAdapter @@ -76,3 +80,64 @@ def summarize_stats(self): self.logger.log(self.log_level, f"Summary of {len(self.stats)} profiling events:") for stat in self.stats: self.logger.log(self.log_level, f"{stat['label']}: {stat['metrics']}") + + def export_stats(self, file_path: str, format: str = 'json'): + """ + Export profiling statistics to a file in the specified format. + + Args: + file_path (str): The path to the output file. + format (str): The format for exporting. Can be 'json' or 'csv'. + Defaults to 'json'. + """ + if format not in ['json', 'csv']: + self.logger.error(f"Unsupported format: '{format}'. Please use 'json' or 'csv'.") + raise ValueError(f"Unsupported format: '{format}'. Please use 'json' or 'csv'.") + + lock_path = f"{file_path}.lock" + try: + with FileLock(lock_path, timeout=10): + if format == 'json': + self._export_to_json(file_path) + elif format == 'csv': + self._export_to_csv(file_path) + except Timeout: + self.logger.error(f"Could not acquire lock on {file_path} after 10 seconds. Another process may be holding it.") + except Exception as e: + self.logger.error(f"An unexpected error occurred during export: {e}") + finally: + if os.path.exists(lock_path): + os.remove(lock_path) + + def _export_to_json(self, file_path: str): + """Private helper method to export stats to a JSON file.""" + try: + with open(file_path, 'w', encoding='utf-8') as f: + json.dump(self.stats, f, indent=4) + except (IOError, PermissionError) as e: + self.logger.error(f"Error writing to JSON file {file_path}: {e}") + + def _export_to_csv(self, file_path: str): + """Private helper method to export stats to a CSV file.""" + if not self.stats: + return + + flattened_data = [] + for item in self.stats: + flat_record = {'label': item.get('label')} + flat_record.update(item.get('metrics', {})) + flattened_data.append(flat_record) + + header_fields = set(['label']) + for record in flattened_data: + header_fields.update(record.keys()) + + sorted_header = sorted(list(header_fields), key=lambda x: (x != 'label', x)) + + try: + with open(file_path, 'w', newline='', encoding='utf-8') as f: + writer = csv.DictWriter(f, fieldnames=sorted_header) + writer.writeheader() + writer.writerows(flattened_data) + except (IOError, PermissionError) as e: + self.logger.error(f"Error writing to CSV file {file_path}: {e}") diff --git a/tests/test_profilers.py b/tests/test_profilers.py index bf12874..654425d 100644 --- a/tests/test_profilers.py +++ b/tests/test_profilers.py @@ -2,6 +2,9 @@ import time import logging import sys +import os +import json +import csv import tracemalloc from unittest.mock import patch, MagicMock from smartprofiler import CPUProfiler, DiskProfiler, FunctionProfiler, MemoryProfiler, NetworkProfiler @@ -321,6 +324,46 @@ def test_cpu_profiler_disable_logging(self): self.assertEqual(len(stats), 1) self.assertIn('execution_time', stats[0]['metrics']) + def test_export_stats_json(self): + with self.cpu_profiler.profile_block("test_json_export"): + time.sleep(0.01) + + json_path = 'test_stats.json' + self.cpu_profiler.export_stats(json_path, format='json') + + self.assertTrue(os.path.exists(json_path)) + + with open(json_path, 'r') as f: + data = json.load(f) + + self.assertEqual(len(data), 1) + self.assertEqual(data[0]['label'], 'test_json_export') + self.assertIn('execution_time', data[0]['metrics']) + + os.remove(json_path) + + def test_export_stats_csv(self): + with self.cpu_profiler.profile_block("test_csv_export_1"): + time.sleep(0.01) + with self.cpu_profiler.profile_block("test_csv_export_2"): + time.sleep(0.01) + + csv_path = 'test_stats.csv' + self.cpu_profiler.export_stats(csv_path, format='csv') + + self.assertTrue(os.path.exists(csv_path)) + + with open(csv_path, 'r') as f: + reader = csv.reader(f) + header = next(reader) + self.assertEqual(header, ['label', 'execution_time']) + rows = list(reader) + self.assertEqual(len(rows), 2) + self.assertEqual(rows[0][0], 'test_csv_export_1') + self.assertEqual(rows[1][0], 'test_csv_export_2') + + os.remove(csv_path) + if __name__ == '__main__': unittest.main()