diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml index 383e65c..e44f0a7 100644 --- a/.github/workflows/pylint.yml +++ b/.github/workflows/pylint.yml @@ -18,6 +18,7 @@ jobs: run: | python -m pip install --upgrade pip pip install pylint + pip install -r requirements.txt - name: Analysing the code with pylint run: | pylint $(git ls-files '*.py') diff --git a/.vscode/launch.json b/.vscode/launch.json index 4254917..53a103f 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -1,10 +1,9 @@ { "version": "0.2.0", - "configurations": [ - + "configurations": [ { "name": "main", - "type": "python", + "type": "debugpy", "request": "launch", "program": "main.py", "console": "integratedTerminal", diff --git a/README.md b/README.md index 2c0105e..1c3cbf7 100644 --- a/README.md +++ b/README.md @@ -1,34 +1,22 @@ -# PythonProfiler - -# PythonProfiler - -Light-weight, decorator-based **time & memory profiler** that streams events to a central TCP server. -Add `@time_profile` or `@memory_profile` to any Python function and get a unified, time-ordered trace across multiple scripts, processes, or hosts without installing heavyweight agents or sharing files. -![image](https://github.com/user-attachments/assets/0e3f7a74-de75-4c52-9826-9d0c21f9de36) +# Python Profiler Viewer +A **streaming log viewer and profiler visualizer** built for large-scale Python profiling data. --- -## ✨ Features +## πŸš€ Features -| | What it does | Why it matters | -|-----------------------------------|--------------|----------------| -| **Two-line integration** | `from logic.decorators import ProfilingDecorators as P` β†’ decorate functions. | Leaves your business-logic untouched; `functools.wraps` keeps signatures & docstrings. | -| **Wall-clock latency** | Captures `datetime.now()` on entry/exit and ships a pair of timestamps. | Perfect for I/O-bound or distributed workflows where β€œwhat waited for what?” matters. :contentReference[oaicite:0]{index=0} | -| **Resident-set memory** | Reads `ru_maxrss` via the `resource` module before & after the call. | Spot leaks and unexpected growth at function granularity on Unix systems. :contentReference[oaicite:1]{index=1} | -| **Global ordering** | Every event carries a Version-1 UUID. | Collision-free merging when dozens of clients fire events in parallel. :contentReference[oaicite:2]{index=2} | -| **Zero external deps** | Uses only the Python standard library (`socket`, `uuid`, `resource`, `datetime`). | Works anywhere Python β‰₯ 3.8 runs. | +- πŸ”„ **Streaming JSONL file processing** β€” processes log files while loading +- 🌳 **Virtualized expandable Call Tree** β€” smooth scroll & expand, even for millions of calls +- πŸ“Š **Timeline view (Plotly powered)** β€” visual execution timeline +- πŸ“ˆ **Summary table** β€” aggregated total time per function +- ⚑ **Fully browser-based** β€” no backend server required --- -## Quick start - -```bash -git clone https://github.com/Saher-Amasha/PythonProfiler.git +## πŸ“„ Log Format -cd PythonProfiler -python3 -m venv venv -source venv/bin/activate -pip install -r requirements.txt # only std-lib, but black & pytest for dev helpers +Profiler expects newline-delimited JSON arrays (`.jsonl`), where each line represents one event: -``` +```json +["17:54:12.457", "start", "function_name", 123, 122, "sync"] \ No newline at end of file diff --git a/app.py b/app.py deleted file mode 100644 index cee096d..0000000 --- a/app.py +++ /dev/null @@ -1,23 +0,0 @@ - -import threading -from example_usage.example1 import Example -from profiler_server import ProfilerServer -from view.main_view import MainView - - - -class app: - server_instance = ProfilerServer() - MainWindow = MainView() - - @staticmethod - def run_test_mode(): - threading.Thread(target=app.server_instance.run).start() - threading.Thread(target=Example.test).start() - app.MainWindow.run() - - @staticmethod - def run(): - threading.Thread(target=app.server_instance.run).start() - app.MainWindow.run() - \ No newline at end of file diff --git a/controller.py b/controller.py deleted file mode 100644 index bc20479..0000000 --- a/controller.py +++ /dev/null @@ -1,15 +0,0 @@ -from datetime import datetime - - -class Model: - TIME_STAMPS = dict() - MEMORY_STAMPS = dict() - START_TIME= datetime.now() - @staticmethod - def add_time_stamp(id,val): - Model.TIME_STAMPS[id]= val - # MainWindow.update() - @staticmethod - def add_memmory_stamp(id,val): - Model.MEMORY_STAMPS[id]= val - # MainWindow.update() \ No newline at end of file diff --git a/example_usage/example1.py b/example_usage/example1.py deleted file mode 100644 index d98dd79..0000000 --- a/example_usage/example1.py +++ /dev/null @@ -1,53 +0,0 @@ -import random -import resource -from time import sleep - -from logic.profiling_meta import ProfilingMeta - - - -from resource import * -import time - -# a non CPU-bound task -time.sleep(3) - -class Example(metaclass=ProfilingMeta): - - @staticmethod - def ex1_func(): - a = [1] * (10 ** 6) - b = [1] * (10 ** 6) - print("1") - sleep(2) - Example.ex2_func() - sleep(3) - print("2") - @staticmethod - def ex2_func(): - print("1") - sleep(2) - print("2") - - @staticmethod - def ex3_func(): - a = [1] * (10 ** 7) - b = [1] * (10 ** 7) - print("1") - sleep(random.randint(0,5)) - print("2") - - @staticmethod - def test(): - i = 0 - while i < 10 : - sleep(random.randint(0,5)) - match random.randint(0,2): - case 0: - Example.ex1_func() - case 1: - Example.ex2_func() - case 2: - Example.ex3_func() - - i+=1 \ No newline at end of file diff --git a/index.html b/index.html new file mode 100644 index 0000000..7780672 --- /dev/null +++ b/index.html @@ -0,0 +1,394 @@ + + + + + + Python Profiler Viewer + + + + + + + + + + + + + + + + + +

Python Profiler Viewer

+ +
+ + +
+ +
+
+
Call Tree
+
+
+ + +
+ + + + + + + diff --git a/inject_profiler.py b/inject_profiler.py new file mode 100644 index 0000000..802f2a8 --- /dev/null +++ b/inject_profiler.py @@ -0,0 +1,218 @@ +""" +Code responsible for injecting the decorator into all .py files +""" + +import os +import sys +import ast +import shutil +import astor + +# Constant strings for marker and import line + +INJECTION_MARKER: str = "# PROFILER_INJECTED" + +# Excluded directories and files from injection process +EXCLUDED_DIRS = { + "venv", + "__pycache__", + "site-packages", + "env", + ".venv", + ".git", + "profiler_internal_files", +} +EXCLUDED_FILES = {"__init__.py", "setup.py", "profiler.py"} + + +class DecoratorInjector(ast.NodeTransformer): + """ + AST Transformer that injects the @profile decorator + into every function and async function. + """ + + def visit_FunctionDef(self, node: ast.FunctionDef) -> ast.FunctionDef: # pylint: disable=invalid-name + """ + Visits all functions in the module + """ + if not any( + isinstance(d, ast.Name) and d.id == "profile" for d in node.decorator_list + ): + node.decorator_list.insert(0, ast.Name(id="profile", ctx=ast.Load())) + return self.generic_visit(node) + + def visit_AsyncFunctionDef( # pylint: disable=invalid-name + self, node: ast.AsyncFunctionDef + ) -> ast.AsyncFunctionDef: + """ + Visits all async functions in the module + """ + if not any( + isinstance(d, ast.Name) and d.id == "profile" for d in node.decorator_list + ): + node.decorator_list.insert(0, ast.Name(id="profile", ctx=ast.Load())) + return self.generic_visit(node) + + +def should_process_file(filepath: str) -> bool: + """ + Determine if the file should be processed: + - Python file (.py) + - Not in excluded files + - Not already injected (check marker) + """ + filename = os.path.basename(filepath) + if filename in EXCLUDED_FILES or not filename.endswith(".py"): + return False + with open(filepath, "r", encoding="utf-8") as f: + first_line = f.readline() + if first_line.strip() == INJECTION_MARKER: + return False + return True + + +def copy_profiler_module(base_dir: str) -> None: + """ + Copy profiler.py into the target project directory if not already exists. + """ + dest_profiler_path = os.path.join(base_dir, "profiler.py") + if os.path.exists(dest_profiler_path): + print("profiler.py already exists in target project, skipping copy.") + return + + # Determine path of this script's directory + current_dir = os.path.dirname(os.path.abspath(__file__)) + src_profiler_path = os.path.join(current_dir, "profiler.py") + + if not os.path.exists(src_profiler_path): + print("ERROR: Cannot find profiler.py next to injector!") + sys.exit(1) + + shutil.copy2(src_profiler_path, dest_profiler_path) + print(f"Copied profiler.py into {base_dir}") + +def copy_file(base_dir: str,file_name) -> None: + """ + Copy profiler.py into the target project directory if not already exists. + """ + dest_profiler_path = os.path.join(base_dir, file_name) + if os.path.exists(dest_profiler_path): + print("file_name already exists in target project, skipping copy.") + return + + # Determine path of this script's directory + current_dir = os.path.dirname(os.path.abspath(__file__)) + src_profiler_path = os.path.join(current_dir, file_name) + + if not os.path.exists(src_profiler_path): + print(f"ERROR: Cannot find {file_name} next to injector!") + sys.exit(1) + + shutil.copy2(src_profiler_path, dest_profiler_path) + print(f"Copied profiler.py into {base_dir}") + + +def backup_file(filepath: str, base_dir: str, backup_dir: str) -> None: + """ + Backup the file to the backup directory before modification. + """ + rel_path = os.path.relpath(filepath, base_dir) + backup_path = os.path.join(backup_dir, rel_path) + os.makedirs(os.path.dirname(backup_path), exist_ok=True) + shutil.copy2(filepath, backup_path) + + +def process_file(filepath: str, profiler_import: str) -> None: + """ + Parse file, inject decorators, backup, and overwrite file. + """ + + with open(filepath, "r", encoding="utf-8") as f: + source = f.read() + + try: + tree = ast.parse(source) + except SyntaxError: + print(f"Skipping invalid file (syntax error): {filepath}") + return + + injector = DecoratorInjector() + new_tree = injector.visit(tree) + new_source = astor.to_source(new_tree) + + # Inject marker and import on top + new_source = INJECTION_MARKER + "\n" + profiler_import + "\n" + new_source + + with open(filepath, "w", encoding="utf-8") as f: + f.write(new_source) + + print(f"Injected: {filepath}") + + +def restore_backups(base_dir: str, backup_dir: str, internal_dir: str) -> None: + """ + Restore all backed-up files from backup directory to original state. + """ + if not os.path.exists(backup_dir): + print("No backup found.") + return + for root, _, files in os.walk(backup_dir): + for file in files: + src_path = os.path.join(root, file) + rel_path = os.path.relpath(src_path, backup_dir) + dest_path = os.path.join(base_dir, rel_path) + shutil.copy2(src_path, dest_path) + print(f"Restored: {dest_path}") + shutil.rmtree(internal_dir) + + +def create_backup_all(internal_files: str, base_dir: str): + """creates a backup for all files pre modification""" + # Handle backup + backup_dir = os.path.join(internal_files, "backup") + os.makedirs(backup_dir, exist_ok=True) + + original_files = [] + for root, dirs, files in os.walk(base_dir): + dirs[:] = [d for d in dirs if d not in EXCLUDED_DIRS] + for file in files: + full_path = os.path.join(root, file) + if should_process_file(full_path) and os.path.abspath( + full_path + ) != os.path.abspath(__file__): + original_files.append((full_path, base_dir, backup_dir)) + + for filepath, c_base_dir, backup_dir in original_files: + backup_file(filepath, c_base_dir, backup_dir) + + +def inject_all(base_dir_path: str) -> None: + """ + Walk through entire project directory and inject profiling + into all eligible Python files. + """ + # create internal dir + internal_files_dir_name = "profiler_internal_files" + internal_files_path = os.path.join(base_dir_path, internal_files_dir_name) + os.makedirs(internal_files_path, exist_ok=True) + + # Add profiler code + copy_profiler_module(internal_files_path) + + # Add ui code + copy_file(base_dir_path,"profiler.js") + copy_file(base_dir_path,"index.html") + # create backup + create_backup_all(internal_files_path, base_dir_path) + + # inject all files with decorator + profiler_import: str = f"from {internal_files_dir_name}.profiler import profile" + + for root, dirs, files in os.walk(base_dir_path): + dirs[:] = [d for d in dirs if d not in EXCLUDED_DIRS] + for file in files: + full_path = os.path.join(root, file) + if should_process_file(full_path) and os.path.abspath( + full_path + ) != os.path.abspath(__file__): + process_file(full_path, profiler_import) diff --git a/logic/metta_adder.py b/logic/metta_adder.py deleted file mode 100644 index e2a5af9..0000000 --- a/logic/metta_adder.py +++ /dev/null @@ -1,68 +0,0 @@ -import os -import re - -class MetaAdder: - def add_metaclass_to_files(self,root_folder): - # Loop through all files and subfiles in the current folder - for foldername, subfolders, filenames in os.walk(root_folder): - for filename in filenames: - # Check if the file has a .py extension - if filename.endswith('.py'): - file_path = os.path.join(foldername, filename) - - # Read the content of the file - with open(file_path, 'r') as file: - content = file.read() - - # Use regular expression to find and add metaclass to class definitions - content_with_metaclass = re.sub(r'class\s+(\w+)\(([^)]*)\)\s*:', r'class \1(\2, metaclass=MyMeta):', content) - - # Write the modified content back to the file - with open(file_path, 'w') as file: - file.write(content_with_metaclass) - - print("Metaclass added to all class definitions.") - - def remove_metaclass_from_files(self,root_folder): - # Loop through all files and subfiles in the current folder - for foldername, subfolders, filenames in os.walk(root_folder): - for filename in filenames: - # Check if the file has a .py extension - if filename.endswith('.py'): - file_path = os.path.join(foldername, filename) - - # Read the content of the file - with open(file_path, 'r') as file: - content = file.read() - - # Use regular expression to remove metaclass from class definitions - content_without_metaclass = re.sub(r'class\s+(\w+)\([^)]*,\s*metaclass=MyMeta\)\s*:', r'class \1:', content) - - # Write the modified content back to the file - with open(file_path, 'w') as file: - file.write(content_without_metaclass) - - print("Metaclass removed from all class definitions.") - -# # Ask the user whether to add or remove the metaclass -# action = input("Do you want to add or remove the metaclass? (Type 'add' or 'remove'): ").lower() - -# # Specify the root folder (current directory in this case) -# root_folder = os.getcwd() - -# if action == 'add': -# add_metaclass_to_files(root_folder) -# input("Metaclass added. Press Enter to continue.") -# elif action == 'remove': -# remove_metaclass_from_files(root_folder) -# input("Metaclass removed. Press Enter to continue.") -# else: -# print("Invalid action. Please type 'add' or 'remove'.") - - - - # Finished - # def[ ]+class[ ]+(.*)(:) , def class \1 (metaclass=MyMeta): - # def( )+class([ ]+.*?[ ]*)[ ]*([\)]*:) , def class \1 \2,metaclass=MyMeta): - - \ No newline at end of file diff --git a/logic/profiling_decorators.py b/logic/profiling_decorators.py deleted file mode 100644 index fefc23c..0000000 --- a/logic/profiling_decorators.py +++ /dev/null @@ -1,35 +0,0 @@ - -import functools -import os -import resource -import socket -from datetime import datetime -import uuid - -class ProfilingDecorators: - HOST = os.environ.get('HOST', "127.0.0.1") - PORT = int(os.environ.get('PORT', "65433") ) - - def time_profile(func): - @functools.wraps(func) - def wrapper(*args, **kwargs): - start = datetime.now() - val = func(*args, **kwargs) - end = datetime.now() - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: - s.connect((ProfilingDecorators.HOST, ProfilingDecorators.PORT)) - s.sendall(bytes('time;'+str(uuid.uuid1()) +';'+str(func.__name__) +';'+ str(start) + ';'+str(end), 'utf-8')) - return val - return wrapper - def memory_profile(func): - @functools.wraps(func) - def wrapper(*args, **kwargs): - start = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss - val = func(*args, **kwargs) - end = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: - s.connect((ProfilingDecorators.HOST, ProfilingDecorators.PORT)) - s.sendall(bytes('memory;'+str(uuid.uuid1()) +';'+str(func.__name__) +';'+ str(start) + ';'+str(end), 'utf-8')) - return val - return wrapper - \ No newline at end of file diff --git a/logic/profiling_meta.py b/logic/profiling_meta.py deleted file mode 100644 index be5aeb8..0000000 --- a/logic/profiling_meta.py +++ /dev/null @@ -1,8 +0,0 @@ -from logic.profiling_decorators import ProfilingDecorators - -class ProfilingMeta(type): - def __new__(cls, name, bases, class_dict): - for key, value in class_dict.items(): - if callable(value): - class_dict[key] = ProfilingDecorators.time_profile(ProfilingDecorators.memory_profile(value)) - return super().__new__(cls, name, bases, class_dict) diff --git a/logic/stamps/base_stamp.py b/logic/stamps/base_stamp.py deleted file mode 100644 index 72a5f47..0000000 --- a/logic/stamps/base_stamp.py +++ /dev/null @@ -1,10 +0,0 @@ -from dataclasses import dataclass - - -@dataclass -class BaseStamp(): - """ - object that holds a time stap and additional info about the function - """ - id:str - name:str \ No newline at end of file diff --git a/logic/stamps/memory_stamp.py b/logic/stamps/memory_stamp.py deleted file mode 100644 index 9719dda..0000000 --- a/logic/stamps/memory_stamp.py +++ /dev/null @@ -1,29 +0,0 @@ -from dataclasses import dataclass - -from logic.stamps.base_stamp import BaseStamp - - -@dataclass -class MemoryStamp(BaseStamp): - """ - object that holds a time stap and additional info about the function - """ - start_memory:float - end_memory:float - - @staticmethod - def init_from_bytes(split_desc_string): - """ - init date time from a string - """ - if len(split_desc_string) > 0: - - id = split_desc_string[0] - name = split_desc_string[1] - start_memory = float(split_desc_string[2]) - end_memory = float(split_desc_string[3]) - - return MemoryStamp(id=id,name=name,start_memory=start_memory,end_memory=end_memory) - - def __lt__(self, other): - return self.end_memory - self.start_memory < other.end_memory - other.start_memory \ No newline at end of file diff --git a/logic/stamps/time_stamp.py b/logic/stamps/time_stamp.py deleted file mode 100644 index ac10027..0000000 --- a/logic/stamps/time_stamp.py +++ /dev/null @@ -1,30 +0,0 @@ -from dataclasses import dataclass -from datetime import datetime - -from logic.stamps.base_stamp import BaseStamp - - -@dataclass -class TimeStamp(BaseStamp): - """ - object that holds a time stap and additional info about the function - """ - start:datetime - end:datetime - format:str ="%Y-%m-%d %H:%M:%S.%f" - - @staticmethod - def init_from_bytes(split_desc_string): - """ - init date time from a string - """ - if len(split_desc_string) > 0: - - id = split_desc_string[0] - name = split_desc_string[1] - start = datetime.strptime(split_desc_string[2], TimeStamp.format) - end = datetime.strptime(split_desc_string[3], TimeStamp.format) - - return TimeStamp(id=id,name=name,start=start,end=end) - def __lt__(self, other): - return self.start < other.start \ No newline at end of file diff --git a/main.py b/main.py index 092d557..a922b10 100644 --- a/main.py +++ b/main.py @@ -1,10 +1,25 @@ -from app import app +""" +Main function of the profiler +""" +import os +import sys +from inject_profiler import inject_all, restore_backups -def main(): - app.run_test_mode() +if __name__ == "__main__": + if len(sys.argv) < 2: + print("Usage:") + print(" Inject: python inject_profiler.py ") + print(" Restore: python inject_profiler.py restore") + sys.exit(1) + provided_base_dir: str = os.path.abspath(sys.argv[1]) -if __name__ == "__main__": - main() + generated_internal_dir: str = os.path.join(provided_base_dir, "profiler_internal_files") + generated_backup_dir: str = os.path.join(generated_internal_dir, "backup") + + if len(sys.argv) == 3 and sys.argv[2] == "restore": + restore_backups(provided_base_dir, generated_backup_dir, generated_internal_dir) + else: + inject_all(provided_base_dir) diff --git a/profiler.js b/profiler.js new file mode 100644 index 0000000..d2f155c --- /dev/null +++ b/profiler.js @@ -0,0 +1,252 @@ +// GLOBAL STATE +let calls = {}; +let idMap = {}; +let treeRoot = null; +let processedBatches = 0; +const renderEvery = 5; +const MAX_NODES = 5000; +const MAX_TIMELINE = 5000; + +// Streaming batch processing +function processBatch(batch) { + for (const entry of batch) { + const id = entry.call_id; + if (!calls[id]) { + calls[id] = { + call_id: id, + parent_call_id: entry.parent_call_id, + function: entry.function, + type: entry.type, + start: null, + end: null + }; + } + if (entry.event === "start") { + calls[id].start = parseTime(entry.time); + } else if (entry.event === "end") { + calls[id].end = parseTime(entry.time); + } + } + processedBatches++; +} + +function maybeRender() { + if (processedBatches % renderEvery === 0) { + renderPartial(); + } +} + +function renderPartial() { + const partialAgg = computeAggregates(Object.values(calls)); + renderSummary(partialAgg); +} + +function finalizeProcessing() { + idMap = buildCallMap(calls); + treeRoot = buildTree(idMap); + renderTree(treeRoot); + renderTimeline(Object.values(calls)); + renderSummary(computeAggregates(Object.values(calls))); + renderFlamegraph(buildFlamegraphTree(calls)); + if (treeRoot) showFunctionDetails(treeRoot); +} + +function parseTime(timeStr) { + const [h, m, s] = timeStr.split(':'); + const [sec, ms = "0"] = s.split('.'); + const date = new Date(); + date.setHours(parseInt(h), parseInt(m), parseInt(sec), parseInt(ms)); + return date; +} + +function buildCallMap(calls) { + const map = {}; + for (const id in calls) { + map[id] = { ...calls[id], children: [], expanded: false }; + } + for (const id in map) { + const node = map[id]; + if (node.parent_call_id != null && map[node.parent_call_id]) { + map[node.parent_call_id].children.push(node); + } + } + return map; +} +function flattenVisibleTree(node, result = [], depth = 0) { + result.push({ node, depth }); + if (node.expanded) { + for (const child of node.children) { + flattenVisibleTree(child, result, depth + 1); + } + } + return result; +} + +function buildTree(idMap) { + for (const id in idMap) { + if (idMap[id].parent_call_id == null || !idMap[idMap[id].parent_call_id]) { + return idMap[id]; + } + } + return null; +} + +function renderTree(treeData) { + if (!treeData) return; + + const container = document.getElementById("tree"); + container.innerHTML = ""; + + const table = document.createElement("table"); + table.className = "profiler-table"; + container.appendChild(table); + + const visibleNodes = flattenVisibleTree(treeData); + + for (const { node, depth } of visibleNodes) { + const row = table.insertRow(); + const cell = row.insertCell(); + + cell.style.paddingLeft = (depth * 20) + "px"; + + // Expand/collapse toggle + if (node.children.length > 0) { + const toggle = document.createElement("span"); + toggle.textContent = node.expanded ? "β–Ό " : "β–Ά "; + toggle.style.cursor = "pointer"; + toggle.onclick = () => { + node.expanded = !node.expanded; + renderTree(treeData); // re-render after toggle + }; + cell.appendChild(toggle); + } else { + cell.textContent = "β€’ "; + } + + const label = document.createElement("span"); + label.textContent = `${node.function} (${node.call_id})`; + label.style.cursor = "pointer"; + label.onclick = () => showFunctionDetails(node); + cell.appendChild(label); + } +} +// === SAFE TIMELINE RENDERING === +function renderTimeline(callArray) { + const limited = callArray.slice(0, MAX_TIMELINE); + const bars = limited.map(call => { + let end = call.end || new Date(); + return { + x: [call.start, end], + y: [call.function + ` [${call.call_id}]`, call.function + ` [${call.call_id}]`], + mode: 'lines', + type: 'scatter', + name: `${call.function} [${call.call_id}]`, + call_id: call.call_id + }; + }); + + Plotly.newPlot('timeline', bars, { + title: 'Global Timeline', + xaxis: { type: 'date' }, + paper_bgcolor: '#1e1e1e', + plot_bgcolor: '#1e1e1e', + font: { color: '#dcdcdc' }, + height: document.getElementById("timeline").clientHeight + }); +} + +function showFunctionDetails(data) { + let html = ""; + html += ""; + html += ``; + html += ``; + if (data.parent_call_id && idMap[data.parent_call_id]) { + html += ``; + } + html += ``; + html += ``; + if (data.start && data.end) + html += ``; + else + html += ``; + html += "
Function Details
Function:${data.function}
Call ID:${data.call_id}
Parent:${idMap[data.parent_call_id].function} (${data.parent_call_id})
Start:${data.start || "missing"}
End:${data.end || "missing"}
Duration:${data.end - data.start} ms
Duration:incomplete
"; + document.getElementById("functionInfo").innerHTML = html; +} + +function computeAggregates(callArray) { + const agg = {}; + for (const call of callArray) { + if (!agg[call.function]) { + agg[call.function] = { count: 0, total: 0 }; + } + agg[call.function].count += 1; + if (call.start && call.end) { + agg[call.function].total += (call.end - call.start); + } + } + return agg; +} + +function renderSummary(aggregates) { + let html = ""; + html += ""; + const sorted = Object.entries(aggregates).sort((a, b) => b[1].total - a[1].total); + for (const [func, stat] of sorted) { + html += ``; + } + html += "
FunctionCallsTotal Time (ms)
${func}${stat.count}${stat.total}
"; + document.getElementById("summaryTable").innerHTML = html; +} + +function buildFlamegraphTree(calls) { + const root = { name: "__ROOT__", value: 0, children: [] }; + + for (const call of Object.values(calls)) { + let current = call; + const path = []; + while (current) { + path.unshift(current.function); + current = current.parent_call_id ? calls[current.parent_call_id] : null; + } + + let node = root; + for (const func of path) { + let child = node.children.find(c => c.name === func); + if (!child) { + child = { name: func, value: 0, children: [] }; + node.children.push(child); + } + node = child; + } + if (call.start && call.end) + node.value += call.end - call.start; + } + return root; +} + +function renderFlamegraph(data) { + document.getElementById("flamegraphView").innerHTML = ""; + const width = document.getElementById("flamegraphView").clientWidth; + const height = 400; + + const partition = d3.partition().size([width, height]); + const root = d3.hierarchy(data).sum(d => d.value); + partition(root); + + const color = d3.scaleOrdinal(d3.schemeTableau10); + + const svg = d3.select("#flamegraphView").append("svg") + .attr("width", width) + .attr("height", height); + + svg.selectAll("rect") + .data(root.descendants()) + .enter().append("rect") + .attr("x", d => d.x0) + .attr("y", d => d.y0) + .attr("width", d => d.x1 - d.x0) + .attr("height", d => d.y1 - d.y0) + .attr("fill", d => color(d.data.name)) + .append("title") + .text(d => `\${d.data.name}: \${Math.round(d.value)} ms`); +} diff --git a/profiler.py b/profiler.py new file mode 100644 index 0000000..286c38f --- /dev/null +++ b/profiler.py @@ -0,0 +1,123 @@ +'''Main profiler code that will be injected''' +import functools as functoolsProfilerProtected +import inspect as inspectProfilerProtected +import json as jsonProfilerProtected +import threading as threadingProfilerProtected +import datetime as datetimeProfilerProtected +import os as osProfilerProtected +import itertools +from typing import Callable, Any, Dict, Optional + + +# Base directory to store the log file (default = current dir or env var override) +BASE_DIR: str = osProfilerProtected.getenv("PROFILER_BASE_DIR", osProfilerProtected.getcwd()) +LOG_FILE: str = osProfilerProtected.path.join(BASE_DIR, "profiler_log.jsonl") +LOCK: threadingProfilerProtected.Lock = threadingProfilerProtected.Lock() + +# Global unique call ID generator +CALL_ID_GENERATOR = itertools.count(1) + +# Thread-local call stack per thread +CALL_STACK = threadingProfilerProtected.local() + +def enter_function(function_name: str) -> Dict[str, Optional[int]]: + """ + Push current function onto thread-local call stack and return parent info. + """ + if not hasattr(CALL_STACK, 'stack'): + CALL_STACK.stack = [] + + call_id = next(CALL_ID_GENERATOR) + parent_call_id = CALL_STACK.stack[-1]['call_id'] if CALL_STACK.stack else None + + CALL_STACK.stack.append({ + 'function': function_name, + 'call_id': call_id + }) + + return {'call_id': call_id, 'parent_call_id': parent_call_id} + +def exit_function() -> None: + """ + Pop current function from call stack. + """ + if hasattr(CALL_STACK, 'stack') and CALL_STACK.stack: + CALL_STACK.stack.pop() + +def log_record(record: Dict[str, Any]) -> None: + """ + Thread-safe log writer. + """ + with LOCK: + with open(LOG_FILE, "a",encoding='utf-8') as f: + f.write(jsonProfilerProtected.dumps(record) + "\n") + +def profile(func: Callable[..., Any]) -> Callable[..., Any]: + """ + Decorator that wraps both synchronous and asynchronous functions + to log start and end events with call instance tracking. + """ + if inspectProfilerProtected.iscoroutinefunction(func): + @functoolsProfilerProtected.wraps(func) + async def async_wrapper(*args: Any, **kwargs: Any) -> Any: + context = enter_function(func.__qualname__) + call_id = context['call_id'] + parent_call_id = context['parent_call_id'] + now = datetimeProfilerProtected.datetime.now() + start_time = now.strftime('%H:%M:%S') + f".{now.microsecond // 1000:09d}" + log_record([ + start_time, + "start", + func.__qualname__, + call_id, + parent_call_id, + "async" + ]) + + try: + return await func(*args, **kwargs) + finally: + now = datetimeProfilerProtected.datetime.now() + end_time = now.strftime('%H:%M:%S') + f".{now.microsecond // 1000:09d}" + log_record([ + end_time, + "end", + func.__qualname__, + call_id, + parent_call_id, + "async" + ]) + exit_function() + + return async_wrapper + + @functoolsProfilerProtected.wraps(func) + def sync_wrapper(*args: Any, **kwargs: Any) -> Any: + context = enter_function(func.__qualname__) + call_id = context['call_id'] + parent_call_id = context['parent_call_id'] + now = datetimeProfilerProtected.datetime.now() + start_time = now.strftime('%H:%M:%S') + f".{now.microsecond // 1000:09d}" + log_record([ + start_time, + "start", + func.__qualname__, + call_id, + parent_call_id, + "sync" + ]) + try: + return func(*args, **kwargs) + finally: + now = datetimeProfilerProtected.datetime.now() + end_time = now.strftime('%H:%M:%S') + f".{now.microsecond // 1000:09d}" + log_record([ + end_time, + "end", + func.__qualname__, + call_id, + parent_call_id, + "sync" + ]) + exit_function() + return sync_wrapper diff --git a/profiler_server.py b/profiler_server.py deleted file mode 100644 index 596072d..0000000 --- a/profiler_server.py +++ /dev/null @@ -1,39 +0,0 @@ -import os -import socket -from controller import Model -from logic.stamps.memory_stamp import MemoryStamp - -from logic.stamps.time_stamp import TimeStamp - - - -class ProfilerServer: - HOST = os.environ.get('HOST', "127.0.0.1") - PORT = int(os.environ.get('PORT', "65433") ) - @staticmethod - def run(): - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: - s.bind((ProfilerServer.HOST, ProfilerServer.PORT)) - s.listen() - while True: - conn, addr = s.accept() - with conn: - print(f"Connected by {addr}") - while True: - data = conn.recv(1024) - ProfilerServer.init_stamp_from_bytes(data) - if not data: - break - print(data) - - @staticmethod - def init_stamp_from_bytes(data): - split_desc_string = data.decode('utf-8').split(';') - if len(split_desc_string) > 0 : - match split_desc_string[0]: - case 'time': - Model.add_time_stamp(split_desc_string[1],TimeStamp.init_from_bytes(split_desc_string[1:])) - case 'memory': - # TODO FIX - Model.add_memmory_stamp(split_desc_string[1],MemoryStamp.init_from_bytes(split_desc_string[1:])) - diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..22a99d3 --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +astor==0.8.1 \ No newline at end of file diff --git a/requirments.txt b/requirments.txt deleted file mode 100644 index 35c0477..0000000 --- a/requirments.txt +++ /dev/null @@ -1 +0,0 @@ -Pillow==10.1.0 \ No newline at end of file diff --git a/resources/profiling_logo.png b/resources/profiling_logo.png deleted file mode 100644 index ad71820..0000000 Binary files a/resources/profiling_logo.png and /dev/null differ diff --git a/setup/build_deps.sh b/setup/build_deps.sh deleted file mode 100644 index e494760..0000000 --- a/setup/build_deps.sh +++ /dev/null @@ -1 +0,0 @@ -sudo apt-get install python3-tk \ No newline at end of file diff --git a/setup_env.sh b/setup_env.sh new file mode 100755 index 0000000..e3ac6a7 --- /dev/null +++ b/setup_env.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +set -e + +VENV_DIR="venv" +REQUIREMENTS_FILE="requirements.txt" + +# Create venv if doesn't exist +if [ ! -d "$VENV_DIR" ]; then + echo "Creating virtual environment in ./$VENV_DIR ..." + python3 -m venv "$VENV_DIR" +else + echo "Virtual environment already exists in ./$VENV_DIR" +fi + +# Activate venv +if [[ "$OSTYPE" == "msys" || "$OSTYPE" == "win32" || "$OSTYPE" == "cygwin" ]]; then + # Windows (Git Bash or native Bash) + source "$VENV_DIR/Scripts/activate" +else + # Unix/Linux/Mac + source "$VENV_DIR/bin/activate" +fi + +# Install requirements +if [ -f "$REQUIREMENTS_FILE" ]; then + echo "Installing requirements from $REQUIREMENTS_FILE ..." + pip install -r "$REQUIREMENTS_FILE" +else + echo "ERROR: $REQUIREMENTS_FILE not found." + exit 1 +fi + +echo "Environment setup complete." diff --git a/view/base_view.py b/view/base_view.py deleted file mode 100644 index 07b23ba..0000000 --- a/view/base_view.py +++ /dev/null @@ -1,16 +0,0 @@ -from dataclasses import dataclass - - -@dataclass -class Baseview: - width:int = 500 - height:int = 500 - color:str = '#FFFFFF' - canvas_scaler:int=40 - rectange_height:int = 60 - rectange_width:int = 20 - rectange_spacing:int = 5 - def draw_rect(self,x1,x2,y1,y2): - self.canvas.create_rectangle((x1,x2,y1,y2),fill='red') - def destroy(self): - self.frame.destroy() diff --git a/view/main_view.py b/view/main_view.py deleted file mode 100644 index 5451511..0000000 --- a/view/main_view.py +++ /dev/null @@ -1,76 +0,0 @@ -import tkinter -from PIL import ImageTk -from tkinter import * -from view.memory_view import MemoryView - -from view.time_view import TimeView - -class MainView(): - TITLE = "Python profiler" - CurrentWindow = None - def __init__(self) -> None: - self.root = tkinter.Tk() - - - # Set title - self.root.title=MainView.TITLE - - - # Set logo - logo = ImageTk.PhotoImage(file="resources/profiling_logo.png") - self.root.iconphoto(True,logo) - self.root.iconbitmap(default=None) - - - self.buttotns=[] - self.frames= [] - - # set buttons to move between views - self.time_button=tkinter.Button( self.root,text="time",command=self.time_window) - self.time_button.pack() - self.buttotns.append(self.time_button) - - - self.memoryButton=tkinter.Button( self.root,text="memory",command=self.memory_view) - self.memoryButton.pack() - self.buttotns.append(self.memoryButton) - - # self.networkButton=tkinter.Button( self.root,text="network",command=self.network_window) - # self.networkButton.grid(row=0,column=2,columnspan=5,sticky='w') - @staticmethod - def update(): - MainView.CurrentWindow.update() - def memory_view(self): - for button in self.buttotns: - button["state"] = "active" - for frame in self.frames: - frame.destroy() - - self.memory_view = MemoryView(self.root) - MainView.CurrentWindow = self.time_window - self.frames.append(self.memory_view) - self.memoryButton["state"] = "disabled" - self.memory_view.draw() - - def time_window(self): - for button in self.buttotns: - button["state"] = "active" - for frame in self.frames: - frame.destroy() - self.time_window = TimeView(self.root) - MainView.CurrentWindow = self.time_window - self.frames.append(self.time_window) - self.time_button["state"] = "disabled" - self.time_window.draw() - - - - # def network_window(self): - # for button in self.buttotns: - # button["state"] = "active" - # label = tkinter.Label(self.root,text="network profiling") - # label.grid(row=1,column=1) - - def run(self): - # Event loop - self.root.mainloop() \ No newline at end of file diff --git a/view/memory_view.py b/view/memory_view.py deleted file mode 100644 index 5ce7621..0000000 --- a/view/memory_view.py +++ /dev/null @@ -1,55 +0,0 @@ -from tkinter import * -from controller import Model - -from view.base_view import Baseview - - -class MemoryView(Baseview): - - def __init__(self,root) -> None: - self.root=root - self.frame=Frame(self.root,width=self.width,height=self.height) - label = Label(self.frame,text="Memory profiling") - copy = Model.MEMORY_STAMPS.copy() - - mlist = list(copy.values()) - if len(copy) > 0: - self.canvas=Canvas(self.frame,bg=self.color,width=self.width,height=self.height,scrollregion=(0,0,max(self.rectange_width*len(copy),self.width),max(max(mlist).end_memory - max(mlist).start_memory,self.height*self.canvas_scaler))) - else : - self.canvas=Canvas(self.frame,bg=self.color,width=self.width,height=self.height,scrollregion=(0,0,self.width*self.canvas_scaler,self.height*self.canvas_scaler)) - def draw(self): - self.frame.pack(expand=True, fill=BOTH) - - hbar=Scrollbar(self.frame,orient=HORIZONTAL) - hbar.pack(side=BOTTOM,fill=X) - hbar.config(command=self.canvas.xview) - vbar=Scrollbar(self.frame,orient=VERTICAL) - vbar.pack(side=RIGHT,fill=Y) - vbar.config(command=self.canvas.yview) - self.canvas.config(width=self.width,height=self.height) - self.canvas.config(xscrollcommand=hbar.set, yscrollcommand=vbar.set) - self.canvas.pack(side=LEFT,expand=True,fill=BOTH) - self.update() - - def update(self): - copy = Model.MEMORY_STAMPS.copy() - if len(copy) <=0: - return - mlist = list(copy.values()) - mlist_sorted= sorted(mlist) - max_mlist=max(mlist) - for index,i in enumerate(mlist_sorted): - memory_stamp = i - memory_leaked = memory_stamp.end_memory - memory_stamp.start_memory - if memory_leaked == 0 : - continue - self.scaler = 1/10 - x = index * (self.rectange_width + self.rectange_spacing) - y = (max_mlist.end_memory - max_mlist.start_memory) *self.scaler - width = x + self.rectange_width + 50 - height = memory_leaked *self.scaler# it is in kb - self.draw_rect( x ,y ,width ,height) - - self.canvas.create_text((x + width )/2 ,(y +height)/2 -self.rectange_height/3 , text=memory_stamp.name , fill="white") - self.canvas.create_text((x + width )/2 ,(y +height)/2 , text='memory_leaked: '+str(memory_leaked), fill="white") - self.canvas.create_text((x + width )/2 ,(y +height)/2 +self.rectange_height/3, text=memory_stamp.id, fill="white") \ No newline at end of file diff --git a/view/time_view.py b/view/time_view.py deleted file mode 100644 index fb864a4..0000000 --- a/view/time_view.py +++ /dev/null @@ -1,50 +0,0 @@ -from tkinter import * -from controller import Model -from view.base_view import Baseview - -class TimeView(Baseview): - def __init__(self,root) -> None: - self.root=root - self.frame=Frame(self.root,width=self.width,height=self.height) - label = Label(self.frame,text="time profiling") - label.pack() - self.canvas=Canvas(self.frame,bg=self.color,width=self.width,height=self.height,scrollregion=(0,0,self.width*self.canvas_scaler,self.height*self.canvas_scaler)) - def draw(self): - self.frame.pack(expand=True, fill=BOTH) - - hbar=Scrollbar(self.frame,orient=HORIZONTAL) - hbar.pack(side=BOTTOM,fill=X) - hbar.config(command=self.canvas.xview) - vbar=Scrollbar(self.frame,orient=VERTICAL) - vbar.pack(side=RIGHT,fill=Y) - vbar.config(command=self.canvas.yview) - self.canvas.config(width=self.width,height=self.height) - self.canvas.config(xscrollcommand=hbar.set, yscrollcommand=vbar.set) - self.canvas.pack(side=LEFT,expand=True,fill=BOTH) - self.update() - - def update(self): - copy = Model.TIME_STAMPS.copy() - if len(copy) <=0: - return - - mlist = list(copy.values()) - mlist_sorted= sorted(mlist) - # start =mlist_sorted[0].start - - for index,i in enumerate(mlist_sorted): - time_stamp = i - beggening = time_stamp.start - Model.START_TIME - time = time_stamp.end - time_stamp.start - self.scaler = 50 - # x, y, x+width, y+height, fill='red' - x = beggening.seconds *self.scaler - y = index * (self.rectange_height + self.rectange_spacing) - - width = x + (time.seconds ) *self.scaler - height = y + self.rectange_height - self.draw_rect( x ,y ,width ,height) - - self.canvas.create_text((x + width )/2 ,(y +height)/2 -self.rectange_height/3 , text=time_stamp.name , fill="white") - self.canvas.create_text((x + width )/2 ,(y +height)/2 , text=str(time.seconds)+' seconds', fill="white") - self.canvas.create_text((x + width )/2 ,(y +height)/2 +self.rectange_height/3, text=time_stamp.id, fill="white")