Theelx main

.github/workflows/tests.yml

@@ -17,7 +17,7 @@ jobs:
     - name: Checkout source
       uses: actions/checkout@v3
     - name: Set up Python 3.8
-      uses: actions/setup-python@v4.2.0
+      uses: actions/setup-python@v4.3.0
       with:
         python-version: 3.8
     - name: Install dependencies
@@ -40,15 +40,14 @@ jobs:
     - name: Checkout source
       uses: actions/checkout@v3
     - name: Set up Python 3.8
-      uses: actions/setup-python@v4.2.0
+      uses: actions/setup-python@v4.3.0
       with:
         python-version: 3.8
     - name: Upgrade pip
       run: |-
         python -m pip install --upgrade pip
         python -m pip install -r requirements/tests.txt
         python -m pip install -r requirements/runtime.txt
-        true
     - name: Build sdist
       shell: bash
       run: |-
@@ -99,9 +98,9 @@ jobs:
     strategy:
       matrix:
         os:
-        - windows-latest
         - ubuntu-latest
         - macOS-latest
+        - windows-latest
         cibw_skip:
         - '*-win32'
         arch:
@@ -127,7 +126,7 @@ jobs:
       with:
         platforms: all
     - name: Build binary wheels
-      uses: pypa/cibuildwheel@v2.9.0
+      uses: pypa/cibuildwheel@v2.10.0
       with:
         output-dir: wheelhouse
         config-file: pyproject.toml
@@ -138,7 +137,7 @@ jobs:
       shell: bash
       run: ls -la wheelhouse
     - name: Set up Python 3.8 to combine coverage Linux
-      uses: actions/setup-python@v4.2.0
+      uses: actions/setup-python@v4.3.0
       if: runner.os == 'Linux'
       with:
         python-version: 3.8
@@ -204,7 +203,8 @@ jobs:
         $GPG_EXECUTABLE --list-keys  || echo "first invocation of gpg creates directories and returns 1"
         $GPG_EXECUTABLE --list-keys
         VERSION=$(python -c "import setup; print(setup.VERSION)")
-        pip install urllib3 requests[security] twine -U
+        pip install twine
+        pip install urllib3 requests[security] twine
         GPG_KEYID=$(cat dev/public_gpg_key)
         echo "GPG_KEYID = '$GPG_KEYID'"
         DO_GPG=True GPG_KEYID=$GPG_KEYID TWINE_REPOSITORY_URL=${TWINE_REPOSITORY_URL} TWINE_PASSWORD=$TWINE_PASSWORD TWINE_USERNAME=$TWINE_USERNAME GPG_EXECUTABLE=$GPG_EXECUTABLE DO_UPLOAD=True DO_TAG=False ./publish.sh
@@ -246,7 +246,8 @@ jobs:
         $GPG_EXECUTABLE --list-keys  || echo "first invocation of gpg creates directories and returns 1"
         $GPG_EXECUTABLE --list-keys
         VERSION=$(python -c "import setup; print(setup.VERSION)")
-        pip install urllib3 requests[security] twine -U
+        pip install twine
+        pip install urllib3 requests[security] twine
         GPG_KEYID=$(cat dev/public_gpg_key)
         echo "GPG_KEYID = '$GPG_KEYID'"
         DO_GPG=True GPG_KEYID=$GPG_KEYID TWINE_REPOSITORY_URL=${TWINE_REPOSITORY_URL} TWINE_PASSWORD=$TWINE_PASSWORD TWINE_USERNAME=$TWINE_USERNAME GPG_EXECUTABLE=$GPG_EXECUTABLE DO_UPLOAD=True DO_TAG=False ./publish.sh

.gitignore

@@ -9,6 +9,7 @@
 *.so
 *.o
 *.a
+*.cpp
 
 build/
 dist/
@@ -18,6 +19,9 @@ line_profiler.egg-info/
 MANIFEST
 pypi-site-docs.zip
 index.html
+CMake*
+!line_profiler/CMakeLists.txt
+Makefile
 
 .coverage
 tests/coverage.xml

CHANGELOG.rst

@@ -1,6 +1,14 @@
 Changes
 =======
 
+4.0.0
+~~~~~
+* ENH: Python 3.11 is now supported.
+* ENH: Profiling overhead is now drastically smaller, thanks to reimplementing almost all of the tracing callback in C++. You can expect to see reductions of between 0.3 and 1 microseconds per line hit, resulting in a speedup of up to 4x for codebases with many lines of Python that only do a little work per line.
+* ENH: Added the ``-i <# of seconds>`` option to the ``kernprof`` script. This uses the threading module to output profiling data to the output file every n seconds, and is useful for long-running tasks that shouldn't be stopped in the middle of processing.
+* CHANGE: Cython's native cythonize function is now used to compile the project, instead of scikit-build's convoluted process.
+* CHANGE: Due to optimizations done while reimplementing the callback in C++, the profiler's code_map and last_time attributes now are indexed by a hash of the code block's bytecode and its line number. Any code that directly reads (and processes) or edits the code_map and/or last_time attributes will likely break.
+
 3.5.2
 ~~~~~
 * FIX: filepath test in is_ipython_kernel_cell for Windows #161

README.rst

@@ -4,13 +4,13 @@ line_profiler and kernprof
 |Pypi| |Downloads| |CircleCI| |GithubActions| |Codecov|
 
 
-NOTICE: This is the official `line_profiler` repository. The most recent
+NOTICE: This is **not** the official `line_profiler` repository. The most recent
 version of `line-profiler <https://pypi.org/project/line_profiler/>`_ on pypi
-points to this repo. The the original 
-`line_profiler <https://github.com/rkern/line_profiler/>`_ package by  
-`@rkern <https://github.com/rkern/>`_ is currently unmaintained. This fork
-seeks to simply maintain the original code so it continues to work in new
-versions of Python.
+points to the repo this was forked from. This fork makes a few ABI changes for
+the sake of massively decreased overhead. You can install this fork with
+``pip install git+https://github.com/Theelx/line_profiler.git``
+Eventually this fork may be uploaded to PyPi, or upstreamed to pyutils, but not
+for now.
 
 ----
 
@@ -31,19 +31,15 @@ Installation
 
 Releases of `line_profiler` can be installed using pip::
 
-    $ pip install line_profiler
+    $ pip install git+https://github.com/Theelx/line_profiler.git
 
 Installation while ensuring a compatible IPython version can also be installed using pip::
 
-    $ pip install line_profiler[ipython]
-
-Source releases and any binaries can be downloaded from the PyPI link.
-
-    http://pypi.python.org/pypi/line_profiler
+    $ pip install git+https://github.com/Theelx/line_profiler.git[ipython]
 
 To check out the development sources, you can use Git_::
 
-    $ git clone https://github.com/pyutils/line_profiler.git
+    $ git clone https://github.com/Theelx/line_profiler.git
 
 You may also download source tarballs of any snapshot from that URL.
 

clean.sh

@@ -14,6 +14,10 @@ rm -rf wheelhouse
 rm -rf pip-wheel-metadata
 rm -rf htmlcov
 rm -rf tests/htmlcov
+rm -rf CMakeCache.txt
+rm -rf CMakeTmp
+rm -rf CMakeFiles
+rm -rf tests/htmlcov
 
 
 if [ -f "distutils.errors" ]; then

comp.py

@@ -0,0 +1,29 @@
+import multiprocessing
+import os
+import sys
+from setuptools import Extension, setup
+
+from Cython.Build import cythonize
+
+def run_cythonize(force=False):
+    return cythonize(
+            Extension(
+                name="line_profiler._line_profiler",
+                sources=[f"line_profiler/_line_profiler.pyx", "line_profiler/timers.c", "line_profiler/unset_trace.c"],
+                language="c++",
+                define_macros=[("CYTHON_TRACE", (1 if os.getenv("DEV") == "true" else 0))],
+            ),
+            compiler_directives={"language_level": 3, "infer_types": True, "linetrace": (True if os.getenv("DEV") == "true" else False)},
+            include_path=["line_profiler/python25.pxd"],
+            force=force,
+            nthreads=multiprocessing.cpu_count(),
+        )
+
+def run_setup(force=False):
+    setup(
+        ext_modules=run_cythonize()
+    )
+
+
+if __name__ == "__main__":
+    run_setup(force=True)

kernprof.py

@@ -5,11 +5,15 @@
 import functools
 import os
 import sys
+import threading
+import asyncio
+import concurrent.futures
+import time
 from argparse import ArgumentError, ArgumentParser
 
 # NOTE: This version needs to be manually maintained with the line_profiler
 # __version__ for now.
-__version__ = '3.5.2'
+__version__ = '4.0.0'
 
 # Guard the import of cProfile such that 3.x people
 # without lsprof can still use this script.
@@ -125,6 +129,38 @@ def __exit__(self, exc_type, exc_val, exc_tb):
         self.disable_by_count()
 
 
+class RepeatedTimer(object):
+    """
+    Background timer for outputting file every n seconds.
+
+    Adapted from
+    https://stackoverflow.com/questions/474528/what-is-the-best-way-to-repeatedly-execute-a-function-every-x-seconds/40965385#40965385
+    """
+    def __init__(self, interval, dump_func, outfile):
+        self._timer = None
+        self.interval = interval
+        self.dump_func = dump_func
+        self.outfile = outfile
+        self.is_running = False
+        self.next_call = time.time()
+        self.start()
+
+    def _run(self):
+        self.is_running = False
+        self.start()
+        self.dump_func(self.outfile)
+
+    def start(self):
+        if not self.is_running:
+            self.next_call += self.interval
+            self._timer = threading.Timer(self.next_call - time.time(), self._run)
+            self._timer.start()
+            self.is_running = True
+
+    def stop(self):
+        self._timer.cancel()
+        self.is_running = False
+
 def find_script(script_name):
     """ Find the script.
 
@@ -171,7 +207,10 @@ def positive_float(value):
                         help='Output unit (in seconds) in which the timing info is '
                         'displayed (default: 1e-6)')
     parser.add_argument('-z', '--skip-zero', action='store_true',
-                        help='Hide functions which have not been called')
+                        help="Hide functions which have not been called")
+    parser.add_argument('-i', '--output-interval', type=int, default=0, const=0, nargs='?',
+        help="Enables outputting of cumulative profiling results to file every n seconds. Uses the threading module."
+            "Minimum value is 1 (second). Defaults to disabled.")
 
     parser.add_argument('script', help='The python script file to run')
     parser.add_argument('args', nargs='...', help='Optional script arguments')
@@ -211,7 +250,12 @@ def positive_float(value):
     # kernprof.py's.
     sys.path.insert(0, os.path.dirname(script_file))
 
+
+    if options.output_interval:
+        rt = RepeatedTimer(max(options.output_interval, 1), prof.dump_stats, options.outfile)
     original_stdout = sys.stdout
+    if options.output_interval:
+        rt = RepeatedTimer(max(options.output_interval, 1), prof.dump_stats, options.outfile)
     try:
         try:
             execfile_ = execfile
@@ -223,6 +267,8 @@ def positive_float(value):
         except (KeyboardInterrupt, SystemExit):
             pass
     finally:
+        if options.output_interval:
+            rt.stop()
         prof.dump_stats(options.outfile)
         print('Wrote profile results to %s' % options.outfile)
         if options.view: