Adding typing, changing the data model, probably a v3.0 rewrite

.github/workflows/deploy-docs.yml

@@ -18,33 +18,33 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - name: checkout
-      uses: actions/checkout@v3
-      with:
-        fetch-depth: 0
-
-    - name: Setup Mamba
-      uses: mamba-org/provision-with-micromamba@v15
-      with:
-        environment-file: false
-
-    - name: Build environment
-      run: |
-        micromamba create --name TEST python=3 pip --file requirements-dev.txt --channel conda-forge
-        micromamba activate TEST
-        python -m pip install -e . --force-reinstall
-
-    - name: Build documentation
-      run: |
-        set -e
-        micromamba activate TEST
-        pushd doc
-        make clean html linkcheck
-        popd
-
-    - name: Deploy
-      if: github.event_name == 'release' || github.event_name == 'push'
-      uses: peaceiris/actions-gh-pages@v3
-      with:
-        github_token: ${{ secrets.GITHUB_TOKEN }}
-        publish_dir: doc/_build/html
+      - name: checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Setup Mamba
+        uses: mamba-org/provision-with-micromamba@v15
+        with:
+          environment-file: false
+
+      - name: Build environment
+        run: |
+          micromamba create --name TEST python=3 pip --file requirements-dev.txt  --channel conda-forge
+          micromamba activate TEST
+          python -m pip install -e . --force-reinstall
+
+      - name: Build documentation
+        run: |
+          set -e
+          micromamba activate TEST
+          pushd doc
+          make clean html linkcheck
+          popd
+
+      - name: Deploy
+        if: github.event_name == 'release' || github.event_name == 'push'
+        uses: peaceiris/actions-gh-pages@v3
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          publish_dir: doc/_build/html

.github/workflows/tests.yml

@@ -14,7 +14,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        python-version: ["3.6", "3.7", "3.8", "3.9", "3.10", "3.11"]
+        python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"]
         os: [windows-latest, ubuntu-latest, macos-latest]
       fail-fast: false
 
@@ -29,8 +29,8 @@ jobs:
       - name: Setup Micromamba
         uses: mamba-org/provision-with-micromamba@v15
         with:
-         environment-file: false
- 
+          environment-file: false
+
       - name: Python ${{ matrix.python-version }}
         run: |
           micromamba create --name TEST python=${{ matrix.python-version }} pip --file requirements-dev.txt --channel conda-forge
@@ -47,4 +47,3 @@ jobs:
           micromamba activate TEST
           coverage report -m
           codecov
-

depfinder/cli.py

@@ -29,22 +29,21 @@
 
 from __future__ import absolute_import, division, print_function
 from argparse import ArgumentParser
-from collections import defaultdict
 import logging
 import os
 from pprint import pprint
 import itertools
 import pdb
 import sys
+from typing import Dict, Iterable, Set
 
 import yaml
 
 from . import main
 from .inspection import parse_file
-from .main import (simple_import_search, notebook_path_to_dependencies,
-                   sanitize_deps)
+from .main import simple_import_search, notebook_path_to_dependencies, sanitize_deps
 
-logger = logging.getLogger('depfinder')
+logger = logging.getLogger("depfinder")
 
 
 class InvalidSelection(RuntimeError):
@@ -68,81 +67,92 @@ def _init_parser():
         nargs="?",
     )
     p.add_argument(
-        '-y',
-        '--yaml',
-        action='store_true',
+        "-y",
+        "--yaml",
+        action="store_true",
         default=False,
-        help=("Output in syntactically valid yaml when true. Defaults to "
-              "%(default)s"))
+        help=(
+            "Output in syntactically valid yaml when true. Defaults to " "%(default)s"
+        ),
+    )
     p.add_argument(
-        '-V',
-        '--version',
-        action='store_true',
+        "-V",
+        "--version",
+        action="store_true",
         default=False,
-        help="Print out the version of depfinder and exit"
+        help="Print out the version of depfinder and exit",
     )
     p.add_argument(
-        '--no-remap',
-        action='store_true',
+        "--no-remap",
+        action="store_true",
         default=False,
-        help=("Do not remap the names of the imported libraries to their "
-              "proper conda name")
+        help=(
+            "Do not remap the names of the imported libraries to their "
+            "proper conda name"
+        ),
     )
     p.add_argument(
-        '-v',
-        '--verbose',
-        action='store_true',
+        "-v",
+        "--verbose",
+        action="store_true",
         default=False,
-        help="Enable debug level logging info from depfinder"
+        help="Enable debug level logging info from depfinder",
     )
     p.add_argument(
-        '-q',
-        '--quiet',
-        action='store_true',
+        "-q",
+        "--quiet",
+        action="store_true",
         default=False,
-        help="Turn off all logging from depfinder"
+        help="Turn off all logging from depfinder",
     )
     p.add_argument(
-        '-k', '--key',
+        "-k",
+        "--key",
         action="append",
         default=[],
-        help=("Select some or all of the output keys. Valid options are "
-              "'required', 'optional', 'builtin', 'relative', 'all'. Defaults "
-              "to 'all'")
+        help=(
+            "Select some or all of the output keys. Valid options are "
+            "'required', 'optional', 'builtin', 'relative', 'all'. Defaults "
+            "to 'all'"
+        ),
     )
     p.add_argument(
-        '--conda',
+        "--conda",
         action="store_true",
         default=False,
-        help=("Format output so it can be passed as an argument to conda "
-              "install or conda create")
+        help=(
+            "Format output so it can be passed as an argument to conda "
+            "install or conda create"
+        ),
     )
     p.add_argument(
-        '--pdb',
+        "--pdb",
         action="store_true",
         help="Enable PDB debugging on exception",
         default=False,
     )
     p.add_argument(
-        '--ignore',
-        default='',
-        help="Comma separated list of file patterns not to inspect"
+        "--ignore",
+        default="",
+        help="Comma separated list of file patterns not to inspect",
     )
     p.add_argument(
-        '--strict',
+        "--strict",
         default=False,
         action="store_true",
-        help=("Immediately raise an Exception if any files fail to parse. Defaults to off.")
+        help=(
+            "Immediately raise an Exception if any files fail to parse. Defaults to off."
+        ),
     )
     p.add_argument(
-        '--custom-namespaces',
-        default='',
+        "--custom-namespaces",
+        default="",
         help=(
             "A comma-separated list of custom namespace packages. "
             "Listing namespaces here will enable depfinder to properly report "
             "dependencies when a namespace is in use (e.g., depfinder will report "
             "both foo.bar and foo.baz instead of foo when run with --custom-namespaces=foo)."
-        )
+        ),
     )
     return p
 
@@ -151,14 +161,17 @@ def cli():
     p = _init_parser()
     args = p.parse_args()
     if args.verbose and args.quiet:
-        msg = ("You have enabled both verbose mode (--verbose or -v) and "
-               "quiet mode (-q or --quiet).  Please pick one. Exiting...")
+        msg = (
+            "You have enabled both verbose mode (--verbose or -v) and "
+            "quiet mode (-q or --quiet).  Please pick one. Exiting..."
+        )
         raise InvalidSelection(msg)
 
     if args.pdb:
         # set the pdb_hook as the except hook for all exceptions
-        def pdb_hook(exctype, value, traceback):
-            pdb.post_mortem(traceback)
+        def pdb_hook(exctype, value, traceback):  # type: ignore
+            pdb.post_mortem(traceback)  # type: ignore
+
         sys.excepthook = pdb_hook
 
     cs = args.custom_namespaces.split(",")
@@ -173,7 +186,7 @@ def pdb_hook(exctype, value, traceback):
         loglevel = logging.DEBUG
     stream_handler = logging.StreamHandler()
     stream_handler.setLevel(loglevel)
-    f = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+    f = "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
     formatter = logging.Formatter(f)
     stream_handler.setFormatter(formatter)
     logger.setLevel(loglevel)
@@ -183,6 +196,7 @@ def pdb_hook(exctype, value, traceback):
         # handle the case where the user just cares about the version. Print
         # version and exit
         from . import __version__
+
         print(__version__)
         return 0
 
@@ -192,11 +206,9 @@ def pdb_hook(exctype, value, traceback):
         logger.warning("positional argument `file_or_directory` not provided.")
         raise RuntimeError("positional argument `file_or_directory` is required")
     keys = args.key
-    if keys == []:
-        keys = None
-    logger.debug('keys: %s', keys)
+    logger.debug("keys: %s", keys)
 
-    def dump_deps(deps, keys):
+    def dump_deps(deps: Dict[str, Set[str]], keys: Iterable[str]):
         """
         Helper function to print the dependencies to the console.
 
@@ -205,34 +217,45 @@ def dump_deps(deps, keys):
         deps : dict
             Dictionary of dependencies that were found
         """
-        if keys is None:
+        if not keys:
             keys = list(deps.keys())
-        deps = {k: list(v) for k, v in deps.items() if k in keys}
+        sorted_deps = {k: sorted(v) for k, v in deps.items() if k in keys}
         if args.yaml:
-            print(yaml.dump(deps, default_flow_style=False))
+            print(yaml.dump(sorted_deps, default_flow_style=False))
         elif args.conda:
-            list_of_deps = [item for sublist in itertools.chain(deps.values())
-                            for item in sublist]
-            print(' '.join(list_of_deps))
+            list_of_deps = [
+                item
+                for sublist in itertools.chain(
+                    deps.get("required", set()), deps["questionable"]
+                )
+                for item in sublist
+            ]
+            print(" ".join(list_of_deps))
         else:
             pprint(deps)
 
     if os.path.isdir(file_or_dir):
-        logger.debug("Treating {} as a directory and recursively searching "
-                     "it for python files".format(file_or_dir))
+        logger.debug(
+            "Treating {} as a directory and recursively searching "
+            "it for python files".format(file_or_dir)
+        )
         # directories are a little easier from the purpose of the API call.
         # print the dependencies to the console and then exit
-        ignore = args.ignore.split(',')
+        ignore = args.ignore.split(",")
         deps = simple_import_search(
-            file_or_dir, remap=not args.no_remap,
-            ignore=ignore, custom_namespaces=cs,
+            file_or_dir,
+            remap=not args.no_remap,
+            ignore=ignore,
+            custom_namespaces=cs,
         )
         dump_deps(deps, keys)
         return 0
     elif os.path.isfile(file_or_dir):
-        if file_or_dir.endswith('ipynb'):
-            logger.debug("Treating {} as a jupyter notebook and searching "
-                         "all of its code cells".format(file_or_dir))
+        if file_or_dir.endswith("ipynb"):
+            logger.debug(
+                "Treating {} as a jupyter notebook and searching "
+                "all of its code cells".format(file_or_dir)
+            )
             deps = notebook_path_to_dependencies(
                 file_or_dir,
                 remap=not args.no_remap,
@@ -242,24 +265,22 @@ def dump_deps(deps, keys):
             # print the dependencies to the console and then exit
             dump_deps(sanitized, keys)
             return 0
-        elif file_or_dir.endswith('.py'):
-            logger.debug("Treating {} as a single python file"
-                         "".format(file_or_dir))
-            mod, path, import_finder = parse_file(file_or_dir, custom_namespaces=cs)
-            mods = defaultdict(set)
-            for k, v in import_finder.describe().items():
-                mods[k].update(v)
-            deps = {k: sorted(list(v)) for k, v in mods.items() if v}
+        elif file_or_dir.endswith(".py"):
+            logger.debug("Treating {} as a single python file" "".format(file_or_dir))
+            _, _, import_finder = parse_file(file_or_dir, custom_namespaces=cs)
+            mods = import_finder.describe()
 
-            sanitized = sanitize_deps(deps)
+            sanitized = sanitize_deps(mods)
             # print the dependencies to the console and then exit
             dump_deps(sanitized, keys)
             return 0
         else:
             # Any file with a suffix that is not ".ipynb" or ".py" will not
             # be parsed correctly
-            msg = ("depfinder is only configured to work with jupyter "
-                   "notebooks and python source code files. It is anticipated "
-                   "that the file {} will not work with depfinder"
-                   "".format(file_or_dir))
+            msg = (
+                "depfinder is only configured to work with jupyter "
+                "notebooks and python source code files. It is anticipated "
+                "that the file {} will not work with depfinder"
+                "".format(file_or_dir)
+            )
             raise RuntimeError(msg)