diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml new file mode 100644 index 0000000..ec4d9b3 --- /dev/null +++ b/.github/workflows/pre-commit.yml @@ -0,0 +1,14 @@ +name: pre-commit + +on: + pull_request: + push: + branches: [main] + +jobs: + pre-commit: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v3 + - uses: pre-commit/action@v3.0.0 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 9344aae..23c07d3 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -30,7 +30,7 @@ jobs: uses: mamba-org/provision-with-micromamba@v15 with: environment-file: false - + - name: Python ${{ matrix.python-version }} run: | micromamba create --name TEST python=${{ matrix.python-version }} pip --file requirements-dev.txt --channel conda-forge @@ -47,4 +47,3 @@ jobs: micromamba activate TEST coverage report -m codecov - diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..3854349 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,11 @@ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v2.3.0 + hooks: + - id: check-yaml + - id: end-of-file-fixer + - id: trailing-whitespace + - repo: https://github.com/psf/black + rev: 22.10.0 + hooks: + - id: black diff --git a/CHANGELOG.rst b/CHANGELOG.rst index d3a255c..7a85f69 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -117,5 +117,3 @@ v2.4.0 * Eric Dill * Christopher J. Wright - - diff --git a/depfinder/__init__.py b/depfinder/__init__.py index e796c9f..5cb3566 100644 --- a/depfinder/__init__.py +++ b/depfinder/__init__.py @@ -35,7 +35,8 @@ from .inspection import parse_file -from .main import (iterate_over_library, notebook_path_to_dependencies) +from .main import iterate_over_library, notebook_path_to_dependencies import logging -logger = logging.getLogger('depfinder') + +logger = logging.getLogger("depfinder") diff --git a/depfinder/cli.py b/depfinder/cli.py index 9d80fec..94df740 100644 --- a/depfinder/cli.py +++ b/depfinder/cli.py @@ -41,10 +41,9 @@ from . import main from .inspection import parse_file -from .main import (simple_import_search, notebook_path_to_dependencies, - sanitize_deps) +from .main import simple_import_search, notebook_path_to_dependencies, sanitize_deps -logger = logging.getLogger('depfinder') +logger = logging.getLogger("depfinder") class InvalidSelection(RuntimeError): @@ -68,81 +67,92 @@ def _init_parser(): nargs="?", ) p.add_argument( - '-y', - '--yaml', - action='store_true', + "-y", + "--yaml", + action="store_true", default=False, - help=("Output in syntactically valid yaml when true. Defaults to " - "%(default)s")) + help=( + "Output in syntactically valid yaml when true. Defaults to " "%(default)s" + ), + ) p.add_argument( - '-V', - '--version', - action='store_true', + "-V", + "--version", + action="store_true", default=False, - help="Print out the version of depfinder and exit" + help="Print out the version of depfinder and exit", ) p.add_argument( - '--no-remap', - action='store_true', + "--no-remap", + action="store_true", default=False, - help=("Do not remap the names of the imported libraries to their " - "proper conda name") + help=( + "Do not remap the names of the imported libraries to their " + "proper conda name" + ), ) p.add_argument( - '-v', - '--verbose', - action='store_true', + "-v", + "--verbose", + action="store_true", default=False, - help="Enable debug level logging info from depfinder" + help="Enable debug level logging info from depfinder", ) p.add_argument( - '-q', - '--quiet', - action='store_true', + "-q", + "--quiet", + action="store_true", default=False, - help="Turn off all logging from depfinder" + help="Turn off all logging from depfinder", ) p.add_argument( - '-k', '--key', + "-k", + "--key", action="append", default=[], - help=("Select some or all of the output keys. Valid options are " - "'required', 'optional', 'builtin', 'relative', 'all'. Defaults " - "to 'all'") + help=( + "Select some or all of the output keys. Valid options are " + "'required', 'optional', 'builtin', 'relative', 'all'. Defaults " + "to 'all'" + ), ) p.add_argument( - '--conda', + "--conda", action="store_true", default=False, - help=("Format output so it can be passed as an argument to conda " - "install or conda create") + help=( + "Format output so it can be passed as an argument to conda " + "install or conda create" + ), ) p.add_argument( - '--pdb', + "--pdb", action="store_true", help="Enable PDB debugging on exception", default=False, ) p.add_argument( - '--ignore', - default='', - help="Comma separated list of file patterns not to inspect" + "--ignore", + default="", + help="Comma separated list of file patterns not to inspect", ) p.add_argument( - '--strict', + "--strict", default=False, action="store_true", - help=("Immediately raise an Exception if any files fail to parse. Defaults to off.") + help=( + "Immediately raise an Exception if any files fail to parse. Defaults to off." + ), ) p.add_argument( - '--custom-namespaces', - default='', + "--custom-namespaces", + default="", help=( "A comma-separated list of custom namespace packages. " "Listing namespaces here will enable depfinder to properly report " "dependencies when a namespace is in use (e.g., depfinder will report " "both foo.bar and foo.baz instead of foo when run with --custom-namespaces=foo)." - ) + ), ) return p @@ -151,14 +161,17 @@ def cli(): p = _init_parser() args = p.parse_args() if args.verbose and args.quiet: - msg = ("You have enabled both verbose mode (--verbose or -v) and " - "quiet mode (-q or --quiet). Please pick one. Exiting...") + msg = ( + "You have enabled both verbose mode (--verbose or -v) and " + "quiet mode (-q or --quiet). Please pick one. Exiting..." + ) raise InvalidSelection(msg) if args.pdb: # set the pdb_hook as the except hook for all exceptions def pdb_hook(exctype, value, traceback): pdb.post_mortem(traceback) + sys.excepthook = pdb_hook cs = args.custom_namespaces.split(",") @@ -173,7 +186,7 @@ def pdb_hook(exctype, value, traceback): loglevel = logging.DEBUG stream_handler = logging.StreamHandler() stream_handler.setLevel(loglevel) - f = '%(asctime)s - %(name)s - %(levelname)s - %(message)s' + f = "%(asctime)s - %(name)s - %(levelname)s - %(message)s" formatter = logging.Formatter(f) stream_handler.setFormatter(formatter) logger.setLevel(loglevel) @@ -183,6 +196,7 @@ def pdb_hook(exctype, value, traceback): # handle the case where the user just cares about the version. Print # version and exit from . import __version__ + print(__version__) return 0 @@ -194,7 +208,7 @@ def pdb_hook(exctype, value, traceback): keys = args.key if keys == []: keys = None - logger.debug('keys: %s', keys) + logger.debug("keys: %s", keys) def dump_deps(deps, keys): """ @@ -211,28 +225,35 @@ def dump_deps(deps, keys): if args.yaml: print(yaml.dump(deps, default_flow_style=False)) elif args.conda: - list_of_deps = [item for sublist in itertools.chain(deps.values()) - for item in sublist] - print(' '.join(list_of_deps)) + list_of_deps = [ + item for sublist in itertools.chain(deps.values()) for item in sublist + ] + print(" ".join(list_of_deps)) else: pprint(deps) if os.path.isdir(file_or_dir): - logger.debug("Treating {} as a directory and recursively searching " - "it for python files".format(file_or_dir)) + logger.debug( + "Treating {} as a directory and recursively searching " + "it for python files".format(file_or_dir) + ) # directories are a little easier from the purpose of the API call. # print the dependencies to the console and then exit - ignore = args.ignore.split(',') + ignore = args.ignore.split(",") deps = simple_import_search( - file_or_dir, remap=not args.no_remap, - ignore=ignore, custom_namespaces=cs, + file_or_dir, + remap=not args.no_remap, + ignore=ignore, + custom_namespaces=cs, ) dump_deps(deps, keys) return 0 elif os.path.isfile(file_or_dir): - if file_or_dir.endswith('ipynb'): - logger.debug("Treating {} as a jupyter notebook and searching " - "all of its code cells".format(file_or_dir)) + if file_or_dir.endswith("ipynb"): + logger.debug( + "Treating {} as a jupyter notebook and searching " + "all of its code cells".format(file_or_dir) + ) deps = notebook_path_to_dependencies( file_or_dir, remap=not args.no_remap, @@ -242,9 +263,8 @@ def dump_deps(deps, keys): # print the dependencies to the console and then exit dump_deps(sanitized, keys) return 0 - elif file_or_dir.endswith('.py'): - logger.debug("Treating {} as a single python file" - "".format(file_or_dir)) + elif file_or_dir.endswith(".py"): + logger.debug("Treating {} as a single python file" "".format(file_or_dir)) mod, path, import_finder = parse_file(file_or_dir, custom_namespaces=cs) mods = defaultdict(set) for k, v in import_finder.describe().items(): @@ -258,8 +278,10 @@ def dump_deps(deps, keys): else: # Any file with a suffix that is not ".ipynb" or ".py" will not # be parsed correctly - msg = ("depfinder is only configured to work with jupyter " - "notebooks and python source code files. It is anticipated " - "that the file {} will not work with depfinder" - "".format(file_or_dir)) + msg = ( + "depfinder is only configured to work with jupyter " + "notebooks and python source code files. It is anticipated " + "that the file {} will not work with depfinder" + "".format(file_or_dir) + ) raise RuntimeError(msg) diff --git a/depfinder/inspection.py b/depfinder/inspection.py index 8d3f713..76b9e76 100644 --- a/depfinder/inspection.py +++ b/depfinder/inspection.py @@ -44,7 +44,7 @@ SKETCHY_TYPES_TABLE, ) -logger = logging.getLogger('depfinder') +logger = logging.getLogger("depfinder") PACKAGE_NAME = None @@ -55,7 +55,11 @@ def get_top_level_import_name(name, custom_namespaces=None): num_dot = name.count(".") custom_namespaces = custom_namespaces or [] - if name in namespace_packages or name in custom_namespaces or name in builtin_modules: + if ( + name in namespace_packages + or name in custom_namespaces + or name in builtin_modules + ): return name elif any( ((num_dot - nsp.count(".")) == 1) and name.startswith(nsp + ".") @@ -65,12 +69,11 @@ def get_top_level_import_name(name, custom_namespaces=None): # foo.bar return name else: - if '.' not in name: + if "." not in name: return name else: return get_top_level_import_name( - name.rsplit('.', 1)[0], - custom_namespaces=custom_namespaces + name.rsplit(".", 1)[0], custom_namespaces=custom_namespaces ) @@ -92,7 +95,7 @@ class ImportFinder(ast.NodeVisitor): """ - def __init__(self, filename='', custom_namespaces=None): + def __init__(self, filename="", custom_namespaces=None): self.filename = filename self.required_modules = set() self.sketchy_modules = set() @@ -139,10 +142,14 @@ def visit_Import(self, node: ast.Import): self.imports.append(node) self._add_to_total_imports(node) - mods = set([ - get_top_level_import_name(name.name, custom_namespaces=self.custom_namespaces) - for name in node.names - ]) + mods = set( + [ + get_top_level_import_name( + name.name, custom_namespaces=self.custom_namespaces + ) + for name in node.names + ] + ) for mod in mods: self._add_import_node(mod) @@ -179,25 +186,31 @@ def visit_ImportFrom(self, node: ast.ImportFrom): def _add_to_total_imports(self, node: Union[ast.Import, ast.ImportFrom]): import_metadata = {} try: - import_metadata.update({'exact_line': ast.unparse(node)}) + import_metadata.update({"exact_line": ast.unparse(node)}) except AttributeError: pass import_metadata.update({v: False for v in SKETCHY_TYPES_TABLE.values()}) - import_metadata.update({SKETCHY_TYPES_TABLE[node.__class__]: True for node in self.sketchy_nodes}) + import_metadata.update( + {SKETCHY_TYPES_TABLE[node.__class__]: True for node in self.sketchy_nodes} + ) names = set() if isinstance(node, ast.Import): _names = set(name.name for name in node.names) - import_metadata['import'] = _names + import_metadata["import"] = _names names.update(_names) elif isinstance(node, ast.ImportFrom): - import_metadata['import_from'] = {node.module} + import_metadata["import_from"] = {node.module} names.add(node.module) else: - raise NotImplementedError(f"Expected ast.Import or ast.ImportFrom this is {type(node)}") + raise NotImplementedError( + f"Expected ast.Import or ast.ImportFrom this is {type(node)}" + ) for name in names: - self.total_imports[name].update({(self.filename, node.lineno): import_metadata}) + self.total_imports[name].update( + {(self.filename, node.lineno): import_metadata} + ) def _add_import_node(self, node_name): # see if the module is a builtin @@ -229,19 +242,19 @@ def describe(self): 'builtin' : The modules that are part of the standard library """ desc = { - 'required': self.required_modules, - 'relative': self.relative_modules, - 'questionable': self.sketchy_modules, - 'builtin': self.builtin_modules + "required": self.required_modules, + "relative": self.relative_modules, + "questionable": self.sketchy_modules, + "builtin": self.builtin_modules, } desc = {k: v for k, v in desc.items() if v} return desc def __repr__(self): - return 'ImportCatcher: %s' % repr(self.describe()) + return "ImportCatcher: %s" % repr(self.describe()) -def get_imported_libs(code, filename='', custom_namespaces=None): +def get_imported_libs(code, filename="", custom_namespaces=None): """Given a code snippet, return a list of the imported libraries Parameters @@ -269,8 +282,7 @@ def get_imported_libs(code, filename='', custom_namespaces=None): 'required': {'stdlib_list'}} """ # skip ipython notebook lines - code = '\n'.join([line for line in code.split('\n') - if not line.startswith('%')]) + code = "\n".join([line for line in code.split("\n") if not line.startswith("%")]) tree = ast.parse(code) import_finder = ImportFinder(filename=filename, custom_namespaces=custom_namespaces) import_finder.visit(tree) @@ -292,19 +304,20 @@ def parse_file(python_file, custom_namespaces=None): """ global PACKAGE_NAME if PACKAGE_NAME is None: - PACKAGE_NAME = os.path.basename(python_file).split('.')[0] - logger.debug("Setting PACKAGE_NAME global variable to {}" - "".format(PACKAGE_NAME)) + PACKAGE_NAME = os.path.basename(python_file).split(".")[0] + logger.debug( + "Setting PACKAGE_NAME global variable to {}" "".format(PACKAGE_NAME) + ) # Try except block added for adal package which has a BOM at the beginning, # requiring a different encoding to load properly try: - with open(python_file, 'r') as f: + with open(python_file, "r") as f: code = f.read() catcher = get_imported_libs( code, filename=python_file, custom_namespaces=custom_namespaces ) except SyntaxError: - with open(python_file, 'r', encoding='utf-8-sig') as f: + with open(python_file, "r", encoding="utf-8-sig") as f: code = f.read() catcher = get_imported_libs( code, filename=python_file, custom_namespaces=custom_namespaces @@ -334,18 +347,21 @@ def iterate_over_library(path_to_source_code, custom_namespaces=None): global PACKAGE_NAME global STRICT_CHECKING if PACKAGE_NAME is None: - PACKAGE_NAME = os.path.basename(path_to_source_code).split('.')[0] - logger.debug("Setting PACKAGE_NAME global variable to {}" - "".format(PACKAGE_NAME)) + PACKAGE_NAME = os.path.basename(path_to_source_code).split(".")[0] + logger.debug( + "Setting PACKAGE_NAME global variable to {}" "".format(PACKAGE_NAME) + ) skipped_files = [] all_files = [] for parent, folders, files in os.walk(path_to_source_code): for f in files: - if f.endswith('.py'): + if f.endswith(".py"): full_file_path = os.path.join(parent, f) all_files.append(full_file_path) try: - yield parse_file(full_file_path, custom_namespaces=custom_namespaces) + yield parse_file( + full_file_path, custom_namespaces=custom_namespaces + ) except Exception: logger.exception("Could not parse file: {}".format(full_file_path)) skipped_files.append(full_file_path) @@ -355,4 +371,6 @@ def iterate_over_library(path_to_source_code, custom_namespaces=None): logger.warn("%s: %s" % (str(idx), f)) if skipped_files and STRICT_CHECKING: - raise RuntimeError("Some files failed to parse. See logs for full stack traces.") + raise RuntimeError( + "Some files failed to parse. See logs for full stack traces." + ) diff --git a/depfinder/main.py b/depfinder/main.py index 99976d0..3267a8a 100644 --- a/depfinder/main.py +++ b/depfinder/main.py @@ -39,12 +39,14 @@ from .inspection import iterate_over_library, get_imported_libs from .utils import pkg_data -logger = logging.getLogger('depfinder') +logger = logging.getLogger("depfinder") STRICT_CHECKING = False -def simple_import_search(path_to_source_code, remap=True, ignore=None, custom_namespaces=None): +def simple_import_search( + path_to_source_code, remap=True, ignore=None, custom_namespaces=None +): """Return all imported modules in all .py files in `path_to_source_code` Parameters @@ -85,7 +87,9 @@ def simple_import_search(path_to_source_code, remap=True, ignore=None, custom_na 'test_with_code']} """ all_deps = defaultdict(set) - catchers = iterate_over_library(path_to_source_code, custom_namespaces=custom_namespaces) + catchers = iterate_over_library( + path_to_source_code, custom_namespaces=custom_namespaces + ) for mod, path, catcher in catchers: # if ignore provided skip things which match the ignore pattern if ignore and any(fnmatch(path, i) for i in ignore): @@ -129,13 +133,15 @@ def notebook_path_to_dependencies(path_to_notebook, remap=True, custom_namespace """ try: from IPython.core.inputsplitter import IPythonInputSplitter + transform = IPythonInputSplitter(line_input_checker=False).transform_cell except: transform = lambda code: code - nb = json.load(io.open(path_to_notebook, encoding='utf8')) - codeblocks = [''.join(cell['source']) for cell in nb['cells'] - if cell['cell_type'] == 'code'] + nb = json.load(io.open(path_to_notebook, encoding="utf8")) + codeblocks = [ + "".join(cell["source"]) for cell in nb["cells"] if cell["cell_type"] == "code" + ] all_deps = defaultdict(set) for codeblock in codeblocks: @@ -143,7 +149,9 @@ def notebook_path_to_dependencies(path_to_notebook, remap=True, custom_namespace # TODO this may fail on py2/py3 syntax when running in the other runtime. # May want to consider updating some error handling around that case. # Will wait until that use case surfaces before modifying - deps_dict = get_imported_libs(codeblock, custom_namespaces=custom_namespaces).describe() + deps_dict = get_imported_libs( + codeblock, custom_namespaces=custom_namespaces + ).describe() for k, v in deps_dict.items(): all_deps[k].update(v) @@ -170,8 +178,11 @@ def sanitize_deps(deps_dict): If remap is False: `deps_dict` """ from .inspection import PACKAGE_NAME + new_deps_dict = {} - list_of_possible_fakes = set([v for val in pkg_data['_FAKE_PACKAGES'].values() for v in val]) + list_of_possible_fakes = set( + [v for val in pkg_data["_FAKE_PACKAGES"].values() for v in val] + ) for k, packages_list in deps_dict.items(): pkgs = copy.copy(packages_list) @@ -179,18 +190,22 @@ def sanitize_deps(deps_dict): for pkg in pkgs: # drop fake packages if pkg in list_of_possible_fakes: - logger.debug("Ignoring {} from the list of imports. It is " - "installed as part of another package. Set the " - "`--no-remap` cli flag if you want to disable " - "this".format(pkg)) + logger.debug( + "Ignoring {} from the list of imports. It is " + "installed as part of another package. Set the " + "`--no-remap` cli flag if you want to disable " + "this".format(pkg) + ) continue if pkg == PACKAGE_NAME: - logger.debug("Ignoring {} from the list of imports. It is " - "the name of the package that we are trying to " - "find the dependencies for. Set the `--no-remap` " - "cli flag if you want to disable this.".format(pkg)) + logger.debug( + "Ignoring {} from the list of imports. It is " + "the name of the package that we are trying to " + "find the dependencies for. Set the `--no-remap` " + "cli flag if you want to disable this.".format(pkg) + ) continue - pkg_to_add = pkg_data['_PACKAGE_MAPPING'].get(pkg, pkg) + pkg_to_add = pkg_data["_PACKAGE_MAPPING"].get(pkg, pkg) if pkg != pkg_to_add: logger.debug("Renaming {} to {}".format(pkg, pkg_to_add)) new_deps_dict[k].add(pkg_to_add) @@ -198,7 +213,9 @@ def sanitize_deps(deps_dict): return new_deps_dict -def simple_import_search_conda_forge_import_map(path_to_source_code, builtins=None, ignore=None, custom_namespaces=None): +def simple_import_search_conda_forge_import_map( + path_to_source_code, builtins=None, ignore=None, custom_namespaces=None +): """Return all conda-forge packages used in all .py files in `path_to_source_code` Parameters @@ -242,20 +259,25 @@ def simple_import_search_conda_forge_import_map(path_to_source_code, builtins=No if ignore is None: ignore = [] total_imports_list = [] - for _, _, c in iterate_over_library(path_to_source_code, custom_namespaces=custom_namespaces): + for _, _, c in iterate_over_library( + path_to_source_code, custom_namespaces=custom_namespaces + ): total_imports_list.append(c.total_imports) total_imports = defaultdict(dict) for total_import in total_imports_list: for name, md in total_import.items(): total_imports[name].update(md) from .reports import report_conda_forge_names_from_import_map + imports, _, _ = report_conda_forge_names_from_import_map( total_imports, builtin_modules=builtins, ignore=ignore ) return {k: sorted(list(v)) for k, v in imports.items()} -def simple_import_to_pkg_map(path_to_source_code, builtins=None, ignore=None, custom_namespaces=None): +def simple_import_to_pkg_map( + path_to_source_code, builtins=None, ignore=None, custom_namespaces=None +): """Provide the map beteen all the imports and their possible packages Parameters @@ -281,13 +303,16 @@ def simple_import_to_pkg_map(path_to_source_code, builtins=None, ignore=None, cu if ignore is None: ignore = [] total_imports_list = [] - for _, _, c in iterate_over_library(path_to_source_code, custom_namespaces=custom_namespaces): + for _, _, c in iterate_over_library( + path_to_source_code, custom_namespaces=custom_namespaces + ): total_imports_list.append(c.total_imports) total_imports = defaultdict(dict) for total_import in total_imports_list: for name, md in total_import.items(): total_imports[name].update(md) from .reports import report_conda_forge_names_from_import_map + _, _, import_to_artifact = report_conda_forge_names_from_import_map( total_imports, builtin_modules=builtins, ignore=ignore ) diff --git a/depfinder/reports.py b/depfinder/reports.py index bc748f2..b5bc0b4 100644 --- a/depfinder/reports.py +++ b/depfinder/reports.py @@ -41,34 +41,43 @@ from .stdliblist import builtin_modules as _builtin_modules from .utils import SKETCHY_TYPES_TABLE -logger = logging.getLogger('depfinder') +logger = logging.getLogger("depfinder") @lru_cache() def _import_map_num_letters(): req = requests.get( - 'https://raw.githubusercontent.com/regro/libcfgraph/master' - '/import_maps_meta.json') + "https://raw.githubusercontent.com/regro/libcfgraph/master" + "/import_maps_meta.json" + ) req.raise_for_status() - return int(req.json()['num_letters']) + return int(req.json()["num_letters"]) @lru_cache() def _import_map_cache(import_first_letters): req = requests.get( - f'https://raw.githubusercontent.com/regro/libcfgraph' - f'/master/import_maps/{import_first_letters.lower()}.json') + f"https://raw.githubusercontent.com/regro/libcfgraph" + f"/master/import_maps/{import_first_letters.lower()}.json" + ) if not req.ok: - print('Request to {req_url} failed'.format(req_url=req.url)) + print("Request to {req_url} failed".format(req_url=req.url)) return {} - return {k: set(v['elements']) for k, v in req.json().items()} + return {k: set(v["elements"]) for k, v in req.json().items()} -FILE_LISTING = requests.get('https://raw.githubusercontent.com/regro/libcfgraph/master/.file_listing.json').json() +FILE_LISTING = requests.get( + "https://raw.githubusercontent.com/regro/libcfgraph/master/.file_listing.json" +).json() # TODO: upstream this to libcfgraph so we just request it, so we reduce bandwidth requirements -ARTIFACT_TO_PKG = {v.split('/')[-1].rsplit('.', 1)[0]: v.split('/')[1] for v in FILE_LISTING if 'artifacts' in v} +ARTIFACT_TO_PKG = { + v.split("/")[-1].rsplit(".", 1)[0]: v.split("/")[1] + for v in FILE_LISTING + if "artifacts" in v +} hubs_auths = requests.get( - 'https://raw.githubusercontent.com/regro/cf-graph-countyfair/master/ranked_hubs_authorities.json').json() + "https://raw.githubusercontent.com/regro/cf-graph-countyfair/master/ranked_hubs_authorities.json" +).json() def extract_pkg_from_import(name): @@ -88,13 +97,13 @@ def extract_pkg_from_import(name): original_name = name while True: try: - fllt = name[:min(len(name), num_letters)] + fllt = name[: min(len(name), num_letters)] import_map = _import_map_cache(fllt) supplying_artifacts = import_map[name] except KeyError: - if '.' not in name: + if "." not in name: return original_name, {}, {} - name = name.rsplit('.', 1)[0] + name = name.rsplit(".", 1)[0] pass else: break @@ -104,7 +113,11 @@ def extract_pkg_from_import(name): supplying_pkgs = {ARTIFACT_TO_PKG[k] for k in supplying_artifacts} import_to_pkg = {name: supplying_pkgs} - return next(iter(k for k in hubs_auths if k in supplying_pkgs), original_name), import_to_artifact, import_to_pkg + return ( + next(iter(k for k in hubs_auths if k in supplying_pkgs), original_name), + import_to_artifact, + import_to_pkg, + ) def recursively_search_for_name(name, module_names): @@ -112,18 +125,26 @@ def recursively_search_for_name(name, module_names): if name in module_names: return name else: - if '.' in name: - name = name.rsplit('.', 1)[0] + if "." in name: + name = name.rsplit(".", 1)[0] else: return False -def report_conda_forge_names_from_import_map(total_imports, builtin_modules=None, ignore=None): +def report_conda_forge_names_from_import_map( + total_imports, builtin_modules=None, ignore=None +): if ignore is None: ignore = [] if builtin_modules is None: builtin_modules = _builtin_modules - report_keys = ['required', 'questionable', 'builtin', 'questionable no match', 'required no match'] + report_keys = [ + "required", + "questionable", + "builtin", + "questionable no match", + "required no match", + ] report = {k: set() for k in report_keys} import_to_pkg = {k: {} for k in report_keys} import_to_artifact = {k: {} for k in report_keys} @@ -131,10 +152,15 @@ def report_conda_forge_names_from_import_map(total_imports, builtin_modules=None with ThreadPoolExecutor() as pool: for name, md in total_imports.items(): - if all([any(fnmatch(filename, ignore_element) for ignore_element in ignore) for filename, _ in md]): + if all( + [ + any(fnmatch(filename, ignore_element) for ignore_element in ignore) + for filename, _ in md + ] + ): continue elif recursively_search_for_name(name, builtin_modules): - report['builtin'].add(name) + report["builtin"].add(name) continue future = pool.submit(extract_pkg_from_import, name) futures[future] = md @@ -151,15 +177,15 @@ def report_conda_forge_names_from_import_map(total_imports, builtin_modules=None if any(import_metadata.get(v, False) for v in SKETCHY_TYPES_TABLE.values()): # if we couldn't find any artifacts to represent this then it doesn't exist in our maps if not _import_to_artifact: - report_key = 'questionable no match' + report_key = "questionable no match" else: - report_key = 'questionable' + report_key = "questionable" else: # if we couldn't find any artifacts to represent this then it doesn't exist in our maps if not _import_to_artifact: - report_key = 'required no match' + report_key = "required no match" else: - report_key = 'required' + report_key = "required" report[report_key].add(most_likely_pkg) import_to_pkg[report_key].update(_import_to_pkg) diff --git a/depfinder/stdliblist.py b/depfinder/stdliblist.py index bfd4b88..bff5caa 100644 --- a/depfinder/stdliblist.py +++ b/depfinder/stdliblist.py @@ -1,17 +1,20 @@ import sys import logging -logger = logging.getLogger('depfinder') +logger = logging.getLogger("depfinder") MAJOR, MINOR = sys.version_info.major, sys.version_info.minor if MAJOR == 3 and MINOR >= 10: - builtin_modules = list(set(list(sys.stdlib_module_names) + list(sys.builtin_module_names))) + builtin_modules = list( + set(list(sys.stdlib_module_names) + list(sys.builtin_module_names)) + ) else: try: from stdlib_list import stdlib_list - pyver = '%s.%s' % (MAJOR, MINOR) + + pyver = "%s.%s" % (MAJOR, MINOR) builtin_modules = stdlib_list(pyver) del pyver except ImportError: - logger.exception('stdlib-list required for python <= 3.9') - raise \ No newline at end of file + logger.exception("stdlib-list required for python <= 3.9") + raise diff --git a/depfinder/utils.py b/depfinder/utils.py index e582c3c..f43f87a 100644 --- a/depfinder/utils.py +++ b/depfinder/utils.py @@ -15,18 +15,18 @@ try: # python 3 AST_TRY = [ast.Try] - SKETCHY_TYPES_TABLE[ast.Try] = 'try' + SKETCHY_TYPES_TABLE[ast.Try] = "try" except AttributeError: # python 2.7 AST_TRY = [ast.TryExcept, ast.TryFinally] - SKETCHY_TYPES_TABLE[ast.TryExcept] = 'try' - SKETCHY_TYPES_TABLE[ast.TryFinally] = 'try' + SKETCHY_TYPES_TABLE[ast.TryExcept] = "try" + SKETCHY_TYPES_TABLE[ast.TryFinally] = "try" try: # python 3.10+ AST_MATCH = [ast.match_case] - SKETCHY_TYPES_TABLE[ast.match_case] = 'match' + SKETCHY_TYPES_TABLE[ast.match_case] = "match" except AttributeError: # match/case does not exist before 3.10 AST_MATCH = [] @@ -38,20 +38,24 @@ # 3. part of an if/elif/else # 4. inside a loop # 5. (for Python 3.10+) inside a match/case -AST_QUESTIONABLE = tuple(AST_TRY + AST_MATCH + [ - ast.FunctionDef, - ast.AsyncFunctionDef, - ast.If, - ast.While, - ast.For, - ast.AsyncFor, -]) -SKETCHY_TYPES_TABLE[ast.FunctionDef] = 'function' -SKETCHY_TYPES_TABLE[ast.AsyncFunctionDef] = 'async-function' -SKETCHY_TYPES_TABLE[ast.If] = 'if' -SKETCHY_TYPES_TABLE[ast.While] = 'while' -SKETCHY_TYPES_TABLE[ast.For] = 'for' -SKETCHY_TYPES_TABLE[ast.AsyncFor] = 'async-for' +AST_QUESTIONABLE = tuple( + AST_TRY + + AST_MATCH + + [ + ast.FunctionDef, + ast.AsyncFunctionDef, + ast.If, + ast.While, + ast.For, + ast.AsyncFor, + ] +) +SKETCHY_TYPES_TABLE[ast.FunctionDef] = "function" +SKETCHY_TYPES_TABLE[ast.AsyncFunctionDef] = "async-function" +SKETCHY_TYPES_TABLE[ast.If] = "if" +SKETCHY_TYPES_TABLE[ast.While] = "while" +SKETCHY_TYPES_TABLE[ast.For] = "for" +SKETCHY_TYPES_TABLE[ast.AsyncFor] = "async-for" del AST_TRY del AST_MATCH @@ -64,17 +68,21 @@ yaml_loader = yaml.SafeLoader pkg_data = yaml.load( - pkgutil.get_data(__name__, 'pkg_data/pkg_data.yml').decode(), + pkgutil.get_data(__name__, "pkg_data/pkg_data.yml").decode(), Loader=yaml_loader, ) -req = requests.get('https://raw.githubusercontent.com/regro/cf-graph-countyfair/master/mappings/pypi/name_mapping.yaml') +req = requests.get( + "https://raw.githubusercontent.com/regro/cf-graph-countyfair/master/mappings/pypi/name_mapping.yaml" +) if req.status_code == 200: mapping_list = yaml.load(req.text, Loader=yaml_loader) else: mapping_list = yaml.load( - pkgutil.get_data(__name__, 'pkg_data/name_mapping.yml').decode(), + pkgutil.get_data(__name__, "pkg_data/name_mapping.yml").decode(), Loader=yaml_loader, ) -namespace_packages = {pkg['import_name'] for pkg in mapping_list if '.' in pkg['import_name']} +namespace_packages = { + pkg["import_name"] for pkg in mapping_list if "." in pkg["import_name"] +} diff --git a/doc/conf.py b/doc/conf.py index 8e53e55..30d7535 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -17,52 +17,53 @@ import os import shlex import sphinx_rtd_theme + # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. -#sys.path.insert(0, os.path.abspath('.')) +# sys.path.insert(0, os.path.abspath('.')) # -- General configuration ------------------------------------------------ # If your documentation needs a minimal Sphinx version, state it here. -#needs_sphinx = '1.0' +# needs_sphinx = '1.0' # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ - 'myst_parser', - 'sphinx.ext.autosummary', - 'sphinx.ext.autodoc', - 'sphinx.ext.viewcode', - 'sphinx.ext.todo', - 'numpydoc', - 'sphinx.ext.extlinks', - 'sphinx.ext.intersphinx', - 'sphinx.ext.napoleon', + "myst_parser", + "sphinx.ext.autosummary", + "sphinx.ext.autodoc", + "sphinx.ext.viewcode", + "sphinx.ext.todo", + "numpydoc", + "sphinx.ext.extlinks", + "sphinx.ext.intersphinx", + "sphinx.ext.napoleon", ] # map to the standard library -intersphinx_mapping = {'python': ('https://docs.python.org/3.4', None)} +intersphinx_mapping = {"python": ("https://docs.python.org/3.4", None)} numpydoc_show_class_members = False # Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] +templates_path = ["_templates"] # The suffix(es) of source filenames. # You can specify multiple suffix as a list of string: source_suffix = [".rst", ".md"] # The encoding of source files. -#source_encoding = 'utf-8-sig' +# source_encoding = 'utf-8-sig' # The master toctree document. -master_doc = 'index' +master_doc = "index" # General information about the project. -project = 'depfinder' -copyright = '2015, Eric Dill' -author = 'Eric Dill' +project = "depfinder" +copyright = "2015, Eric Dill" +author = "Eric Dill" # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -81,37 +82,37 @@ # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: -#today = '' +# today = '' # Else, today_fmt is used as the format for a strftime call. -#today_fmt = '%B %d, %Y' +# today_fmt = '%B %d, %Y' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. -exclude_patterns = ['_build'] +exclude_patterns = ["_build"] # The reST default role (used for this markup: `text`) to use for all # documents. -#default_role = None +# default_role = None # If true, '()' will be appended to :func: etc. cross-reference text. -#add_function_parentheses = True +# add_function_parentheses = True # If true, the current module name will be prepended to all description # unit titles (such as .. function::). -#add_module_names = True +# add_module_names = True # If true, sectionauthor and moduleauthor directives will be shown in the # output. They are ignored by default. -#show_authors = False +# show_authors = False # The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' +pygments_style = "sphinx" # A list of ignored prefixes for module index sorting. -#modindex_common_prefix = [] +# modindex_common_prefix = [] # If true, keep warnings as "system message" paragraphs in the built documents. -#keep_warnings = False +# keep_warnings = False # If true, `todo` and `todoList` produce output, else they produce nothing. todo_include_todos = False @@ -121,31 +122,31 @@ # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. -html_theme = 'sphinx_rtd_theme' +html_theme = "sphinx_rtd_theme" # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. -#html_theme_options = {} +# html_theme_options = {} # Add any paths that contain custom themes here, relative to this directory. html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] # The name for this set of Sphinx documents. If None, it defaults to # " v documentation". -#html_title = None +# html_title = None # A shorter title for the navigation bar. Default is the same as html_title. -#html_short_title = None +# html_short_title = None # The name of an image file (relative to this directory) to place at the top # of the sidebar. -#html_logo = None +# html_logo = None # The name of an image file (within the static path) to use as favicon of the # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 # pixels large. -#html_favicon = None +# html_favicon = None # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, @@ -155,122 +156,115 @@ # Add any extra paths that contain custom files (such as robots.txt or # .htaccess) here, relative to this directory. These files are copied # directly to the root of the documentation. -#html_extra_path = [] +# html_extra_path = [] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. -#html_last_updated_fmt = '%b %d, %Y' +# html_last_updated_fmt = '%b %d, %Y' # If true, SmartyPants will be used to convert quotes and dashes to # typographically correct entities. -#html_use_smartypants = True +# html_use_smartypants = True # Custom sidebar templates, maps document names to template names. -#html_sidebars = {} +# html_sidebars = {} # Additional templates that should be rendered to pages, maps page names to # template names. -#html_additional_pages = {} +# html_additional_pages = {} # If false, no module index is generated. -#html_domain_indices = True +# html_domain_indices = True # If false, no index is generated. -#html_use_index = True +# html_use_index = True # If true, the index is split into individual pages for each letter. -#html_split_index = False +# html_split_index = False # If true, links to the reST sources are added to the pages. -#html_show_sourcelink = True +# html_show_sourcelink = True # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. -#html_show_sphinx = True +# html_show_sphinx = True # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. -#html_show_copyright = True +# html_show_copyright = True # If true, an OpenSearch description file will be output, and all pages will # contain a tag referring to it. The value of this option must be the # base URL from which the finished HTML is served. -#html_use_opensearch = '' +# html_use_opensearch = '' # This is the file name suffix for HTML files (e.g. ".xhtml"). -#html_file_suffix = None +# html_file_suffix = None # Language to be used for generating the HTML full-text search index. # Sphinx supports the following languages: # 'da', 'de', 'en', 'es', 'fi', 'fr', 'h', 'it', 'ja' # 'nl', 'no', 'pt', 'ro', 'r', 'sv', 'tr' -#html_search_language = 'en' +# html_search_language = 'en' # A dictionary with options for the search language support, empty by default. # Now only 'ja' uses this config value -#html_search_options = {'type': 'default'} +# html_search_options = {'type': 'default'} # The name of a javascript file (relative to the configuration directory) that # implements a search results scorer. If empty, the default will be used. -#html_search_scorer = 'scorer.js' +# html_search_scorer = 'scorer.js' # Output file base name for HTML help builder. -htmlhelp_basename = 'depfinderdoc' +htmlhelp_basename = "depfinderdoc" # -- Options for LaTeX output --------------------------------------------- latex_elements = { -# The paper size ('letterpaper' or 'a4paper'). -#'papersize': 'letterpaper', - -# The font size ('10pt', '11pt' or '12pt'). -#'pointsize': '10pt', - -# Additional stuff for the LaTeX preamble. -#'preamble': '', - -# Latex figure (float) alignment -#'figure_align': 'htbp', + # The paper size ('letterpaper' or 'a4paper'). + #'papersize': 'letterpaper', + # The font size ('10pt', '11pt' or '12pt'). + #'pointsize': '10pt', + # Additional stuff for the LaTeX preamble. + #'preamble': '', + # Latex figure (float) alignment + #'figure_align': 'htbp', } # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ - (master_doc, 'depfinder.tex', 'depfinder Documentation', - 'Eric Dill', 'manual'), + (master_doc, "depfinder.tex", "depfinder Documentation", "Eric Dill", "manual"), ] # The name of an image file (relative to this directory) to place at the top of # the title page. -#latex_logo = None +# latex_logo = None # For "manual" documents, if this is true, then toplevel headings are parts, # not chapters. -#latex_use_parts = False +# latex_use_parts = False # If true, show page references after internal links. -#latex_show_pagerefs = False +# latex_show_pagerefs = False # If true, show URL addresses after external links. -#latex_show_urls = False +# latex_show_urls = False # Documents to append as an appendix to all manuals. -#latex_appendices = [] +# latex_appendices = [] # If false, no module index is generated. -#latex_domain_indices = True +# latex_domain_indices = True # -- Options for manual page output --------------------------------------- # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). -man_pages = [ - (master_doc, 'depfinder', 'depfinder Documentation', - [author], 1) -] +man_pages = [(master_doc, "depfinder", "depfinder Documentation", [author], 1)] # If true, show URL addresses after external links. -#man_show_urls = False +# man_show_urls = False # -- Options for Texinfo output ------------------------------------------- @@ -279,19 +273,25 @@ # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - (master_doc, 'depfinder', 'depfinder Documentation', - author, 'depfinder', 'One line description of project.', - 'Miscellaneous'), + ( + master_doc, + "depfinder", + "depfinder Documentation", + author, + "depfinder", + "One line description of project.", + "Miscellaneous", + ), ] # Documents to append as an appendix to all manuals. -#texinfo_appendices = [] +# texinfo_appendices = [] # If false, no module index is generated. -#texinfo_domain_indices = True +# texinfo_domain_indices = True # How to display URL addresses: 'footnote', 'no', or 'inline'. -#texinfo_show_urls = 'footnote' +# texinfo_show_urls = 'footnote' # If true, do not generate a @detailmenu in the "Top" node's menu. -#texinfo_no_detailmenu = False +# texinfo_no_detailmenu = False diff --git a/doc/index.rst b/doc/index.rst index a3ebd4a..fd1804b 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -9,4 +9,4 @@ Indices and tables * :ref:`genindex` * :ref:`modindex` -* :ref:`search` \ No newline at end of file +* :ref:`search` diff --git a/requirements.txt b/requirements.txt index 30e1887..77ff476 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,3 @@ pyyaml stdlib-list; python_version < "3.10" -requests \ No newline at end of file +requests diff --git a/run_tests.py b/run_tests.py index e47c159..fb83999 100755 --- a/run_tests.py +++ b/run_tests.py @@ -2,11 +2,11 @@ import sys import pytest -if __name__ == '__main__': +if __name__ == "__main__": # show output results from every test function - args = ['-v', 'test.py'] + args = ["-v", "test.py"] # show the message output for skipped and expected failure tests - args.append('-rxs') + args.append("-rxs") args.extend(sys.argv[1:]) # call pytest and exit with the return code from pytest so that # travis will fail correctly if tests fail diff --git a/test.py b/test.py index 1024942..05e86f2 100644 --- a/test.py +++ b/test.py @@ -1,5 +1,4 @@ -from __future__ import (unicode_literals, print_function, division, - absolute_import) +from __future__ import unicode_literals, print_function, division, absolute_import import contextlib import itertools @@ -17,9 +16,16 @@ import depfinder from depfinder import cli, main, inspection, parse_file -from depfinder.main import simple_import_search_conda_forge_import_map, simple_import_to_pkg_map -from depfinder.reports import report_conda_forge_names_from_import_map, extract_pkg_from_import, \ - recursively_search_for_name, _builtin_modules +from depfinder.main import ( + simple_import_search_conda_forge_import_map, + simple_import_to_pkg_map, +) +from depfinder.reports import ( + report_conda_forge_names_from_import_map, + extract_pkg_from_import, + recursively_search_for_name, + _builtin_modules, +) random.seed(12345) @@ -31,9 +37,9 @@ complex_imports = [ - {'targets': - {'questionable': ['atom', 'chemist', 'molecule', 'physicist']}, - 'code': """ + { + "targets": {"questionable": ["atom", "chemist", "molecule", "physicist"]}, + "code": """ try: import molecule except ImportError: @@ -41,10 +47,11 @@ else: import chemist finally: - import physicist""" - }, - {'targets': {'required': ['foo'], 'builtin': ['os']}, - 'code': """ + import physicist""", + }, + { + "targets": {"required": ["foo"], "builtin": ["os"]}, + "code": """ import foo try: import os @@ -132,60 +139,56 @@ async def async_function(): elif that: import harpo else: - import chico""" - }, + import chico""", + }, ] simple_imports = [ - {'targets': {'required': ['foo']}, - 'code': 'import foo'}, - {'targets': {'required': ['bar', 'foo']}, - 'code': 'import foo, bar'}, - {'targets': {'required': ['numpy']}, - 'code': 'import numpy'}, - {'targets': {'required': ['matplotlib']}, - 'code': 'from matplotlib import pyplot'}, + {"targets": {"required": ["foo"]}, "code": "import foo"}, + {"targets": {"required": ["bar", "foo"]}, "code": "import foo, bar"}, + {"targets": {"required": ["numpy"]}, "code": "import numpy"}, + {"targets": {"required": ["matplotlib"]}, "code": "from matplotlib import pyplot"}, # Hit the fake packages code block in main.sanitize_deps() - {'targets': {'required': ['numpy']}, - 'code': 'from numpy import warnings as npwarn'}, + { + "targets": {"required": ["numpy"]}, + "code": "from numpy import warnings as npwarn", + }, ] relative_imports = [ - {'targets': {}, - 'code': 'from . import bar'}, - {'targets': {'relative': ['bar']}, - 'code': 'from .bar import baz'}, - {'targets': {'relative': ['bar']}, - 'code': 'from ..bar import baz'}, + {"targets": {}, "code": "from . import bar"}, + {"targets": {"relative": ["bar"]}, "code": "from .bar import baz"}, + {"targets": {"relative": ["bar"]}, "code": "from ..bar import baz"}, ] -@pytest.fixture(scope='module') + +@pytest.fixture(scope="module") def using_stdlib_list(): try: import stdlib_list + return True except ImportError: return False + def test_nested_namespace_builtins(using_stdlib_list): if using_stdlib_list: - expected = {'builtin': ['concurrent.futures']} + expected = {"builtin": ["concurrent.futures"]} else: - expected = {'builtin': ['concurrent']} - code = 'import concurrent.futures' + expected = {"builtin": ["concurrent"]} + code = "import concurrent.futures" - - - test_object = Initter({'targets': expected, 'code': code}) + test_object = Initter({"targets": expected, "code": code}) imports = main.get_imported_libs(test_object.code) assert imports.describe() == test_object.targets class Initter(object): def __init__(self, artifact): - targets = artifact.get('targets', {}) + targets = artifact.get("targets", {}) self.targets = {k: set(v) for k, v in targets.items()} - self.code = artifact['code'] + self.code = artifact["code"] def test_imports(): @@ -206,22 +209,23 @@ def test_for_smoke(): """Do not validate the output of the functions, just make sure that calling them does not make depfinder blow up """ - deps = list(main.iterate_over_library('.')) + deps = list(main.iterate_over_library(".")) assert deps is not None assert str(deps) is not None assert repr(deps) is not None # hit the simple api - assert main.simple_import_search('.') is not None + assert main.simple_import_search(".") is not None ### NOTEBOOK TESTING CODE ### + @contextlib.contextmanager def write_notebook(cells): nb = v4.new_notebook() - nb['cells'] = [v4.new_code_cell(code_cell) for code_cell in cells] - fname = tempfile.NamedTemporaryFile(suffix='.ipynb').name - with open(fname, 'w') as f: + nb["cells"] = [v4.new_code_cell(code_cell) for code_cell in cells] + fname = tempfile.NamedTemporaryFile(suffix=".ipynb").name + with open(fname, "w") as f: f.write(v4.writes(nb)) try: @@ -234,18 +238,18 @@ def test_notebook_remapping(): code = "import mpl_toolkits" with write_notebook([code]) as fname: deps = main.notebook_path_to_dependencies(fname, remap=False) - assert {'required': ['mpl_toolkits']} == deps + assert {"required": ["mpl_toolkits"]} == deps assert {} == main.notebook_path_to_dependencies(fname) -@pytest.mark.parametrize("import_list_dict", [complex_imports, - simple_imports, - relative_imports]) +@pytest.mark.parametrize( + "import_list_dict", [complex_imports, simple_imports, relative_imports] +) def tester(import_list_dict, capsys): # http://nbviewer.ipython.org/gist/fperez/9716279 for import_dict in import_list_dict: - cell_code = [import_dict['code']] - target = import_dict['targets'] + cell_code = [import_dict["code"]] + target = import_dict["targets"] with write_notebook(cell_code) as fname: # parse the notebook! assert set(target) == set(main.notebook_path_to_dependencies(fname)) @@ -261,8 +265,8 @@ def test_multiple_code_cells(capsys): # http://nbviewer.ipython.org/gist/fperez/9716279 code_for_cells = [] for import_dict in import_list_dict: - code_for_cells.append(import_dict['code']) - target = import_dict['targets'] + code_for_cells.append(import_dict["code"]) + target = import_dict["targets"] for k, v in target.items(): targets[k].update(set(v)) @@ -279,6 +283,7 @@ def test_multiple_code_cells(capsys): ### CLI TESTING CODE ### + def _process_args(path_to_check, extra_flags): """ Parameters @@ -302,10 +307,10 @@ def _process_args(path_to_check, extra_flags): def _subprocess_cli(path_to_check=None, extra_flags=None): path_to_check, extra_flags = _process_args(path_to_check, extra_flags) p = subprocess.Popen( - ['depfinder', path_to_check] + extra_flags, + ["depfinder", path_to_check] + extra_flags, env=dict(os.environ), stderr=subprocess.PIPE, - stdout=subprocess.PIPE + stdout=subprocess.PIPE, ) stdout, stderr = p.communicate() @@ -318,7 +323,7 @@ def _run_cli(path_to_check=None, extra_flags=None): Helper function to run depfinder in its cli mode """ path_to_check, extra_flags = _process_args(path_to_check, extra_flags) - sys.argv = ['depfinder', path_to_check] + extra_flags + sys.argv = ["depfinder", path_to_check] + extra_flags cli.cli() return None @@ -331,13 +336,12 @@ def known_flags(): flags = [flag for flag in flags if flag] # now flatten the nested list flags = [flag for flag_twins in flags for flag in flag_twins] - flags.remove('-k') - flags.remove('--key') - flags.remove('--pdb') - flags.remove('--ignore') - flags.remove('--custom-namespaces') - flags.extend(['-k all', '-k required', '-k optional', '-k builtin', - '-k relative']) + flags.remove("-k") + flags.remove("--key") + flags.remove("--pdb") + flags.remove("--ignore") + flags.remove("--custom-namespaces") + flags.extend(["-k all", "-k required", "-k optional", "-k builtin", "-k relative"]) return flags @@ -347,10 +351,10 @@ def flags(): @pytest.mark.parametrize( - 'flags', + "flags", itertools.chain( [random.sample(known_flags(), i) for i in range(1, len(known_flags()))] - ) + ), ) def test_cli_with_random_flags(flags): """ @@ -366,8 +370,8 @@ def test_cli_with_random_flags(flags): # The only thing that I know of that will exit with a nonzero status # is if you try to combine quiet mode and verbose mode. This handles # that case - quiet = {'-q', '--quiet'} - verbose = {'-v', '--verbose'} + quiet = {"-q", "--quiet"} + verbose = {"-v", "--verbose"} flags = set(flags) assert flags & quiet != set() and flags & verbose != set() return @@ -376,9 +380,11 @@ def test_cli_with_random_flags(flags): @pytest.mark.parametrize( - 'path, req', - ((dirname(depfinder.__file__), None), - (join(dirname(depfinder.__file__), 'main.py'), set())) + "path, req", + ( + (dirname(depfinder.__file__), None), + (join(dirname(depfinder.__file__), "main.py"), set()), + ), ) def test_cli(path, req, capsys): """ @@ -387,26 +393,34 @@ def test_cli(path, req, capsys): """ main.PACKAGE_NAME = None old_argv = sys.argv - sys.argv = ['depfinder'] + sys.argv = ["depfinder"] _run_cli(path_to_check=path) sys.argv = old_argv # read stdout and stderr with pytest's built-in capturing mechanism stdout, stderr = capsys.readouterr() - print('stdout\n{}'.format(stdout)) - print('stderr\n{}'.format(stderr)) + print("stdout\n{}".format(stdout)) + print("stderr\n{}".format(stderr)) if req is None: - dependencies_file = join(dirname(dirname(depfinder.__file__)), - 'requirements.txt') - dependencies = set([dep for dep in open(dependencies_file, 'r').read().split('\n') if not dep.startswith("stdlib")]) + dependencies_file = join( + dirname(dirname(depfinder.__file__)), "requirements.txt" + ) + dependencies = set( + [ + dep + for dep in open(dependencies_file, "r").read().split("\n") + if dep and not dep.startswith("stdlib") + ] + ) else: dependencies = req - assert dependencies == set(eval(stdout).get('required', set())) + assert dependencies == set(eval(stdout).get("required", set())) def test_known_fail_cli(tmpdir): - tmpfile = os.path.join(str(tmpdir), 'bad_file.txt') + tmpfile = os.path.join(str(tmpdir), "bad_file.txt") import this - with open(tmpfile, 'w') as f: + + with open(tmpfile, "w") as f: f.write("".join([this.d.get(this.c, this.c) for this.c in this.s])) with pytest.raises(RuntimeError): @@ -415,17 +429,15 @@ def test_known_fail_cli(tmpdir): def test_known_fail_cli2(): with pytest.raises(cli.InvalidSelection): - _run_cli(extra_flags=['-q', '-v']) + _run_cli(extra_flags=["-q", "-v"]) @pytest.mark.parametrize( - 'path', - (dirname(depfinder.__file__), - join(dirname(depfinder.__file__), 'main.py')) + "path", (dirname(depfinder.__file__), join(dirname(depfinder.__file__), "main.py")) ) def test_individual_args(path, flags): for flag in flags: - if flag in ['-h', '--help']: + if flag in ["-h", "--help"]: # skip the help messages since they cause the system to exit continue _run_cli(path_to_check=path, extra_flags=[flag]) @@ -435,55 +447,68 @@ def test_individual_args(path, flags): def test_fake_packages(): fake_import = "import mpl_toolkits" imports = main.get_imported_libs(fake_import) - assert imports.describe() == {'required': {'mpl_toolkits'}} + assert imports.describe() == {"required": {"mpl_toolkits"}} assert main.sanitize_deps(imports.describe()) == {} def test_get_top_level_import(): - name = 'this.that.something' + name = "this.that.something" top_level_name = inspection.get_top_level_import_name(name) - assert top_level_name == 'this' + assert top_level_name == "this" - name = 'google.cloud.storage.something' + name = "google.cloud.storage.something" top_level_name = inspection.get_top_level_import_name(name) - assert top_level_name == 'google.cloud.storage' + assert top_level_name == "google.cloud.storage" def test_report_conda_forge_names_from_import_map(): - m, f, c = parse_file(join(dirname(depfinder.__file__), 'utils.py')) - report, import_to_artifact, import_to_pkg = report_conda_forge_names_from_import_map(c.total_imports) - assert report['required'] == {'pyyaml', 'requests'} + m, f, c = parse_file(join(dirname(depfinder.__file__), "utils.py")) + ( + report, + import_to_artifact, + import_to_pkg, + ) = report_conda_forge_names_from_import_map(c.total_imports) + assert report["required"] == {"pyyaml", "requests"} def test_report_conda_forge_names_from_import_map_ignore(): - m, f, c = parse_file(join(dirname(depfinder.__file__), 'inspection.py')) - report, import_to_artifact, import_to_pkg = report_conda_forge_names_from_import_map(c.total_imports, - ignore=['*insp*']) - assert report['required'] == set() + m, f, c = parse_file(join(dirname(depfinder.__file__), "inspection.py")) + ( + report, + import_to_artifact, + import_to_pkg, + ) = report_conda_forge_names_from_import_map(c.total_imports, ignore=["*insp*"]) + assert report["required"] == set() def test_simple_import_search_conda_forge_import_map(): path_to_source = dirname(depfinder.__file__) expected_result = sorted(list({"pyyaml", "requests"})) report = simple_import_search_conda_forge_import_map(path_to_source) - assert report['required'] == expected_result + assert report["required"] == expected_result -@pytest.mark.parametrize('import_name, expected_result', [ - ('six.moves', 'six'), - ('win32com.shell', 'pywin32'), - ('win32com', 'pywin32'), - # this comes from cython but doesn't seem to be a real pkg - ('refnanny.hi', 'refnanny.hi') -]) +@pytest.mark.parametrize( + "import_name, expected_result", + [ + ("six.moves", "six"), + ("win32com.shell", "pywin32"), + ("win32com", "pywin32"), + # this comes from cython but doesn't seem to be a real pkg + ("refnanny.hi", "refnanny.hi"), + ], +) def test_extract_pkg_from_import_for_complex_imports(import_name, expected_result): result, _, _ = extract_pkg_from_import(import_name) assert result == expected_result -@pytest.mark.parametrize('import_name, expected_result', [ - ('six.moves', False), -]) +@pytest.mark.parametrize( + "import_name, expected_result", + [ + ("six.moves", False), + ], +) def test_search_for_name(import_name, expected_result): builtin_name_maybe = recursively_search_for_name(import_name, _builtin_modules) assert builtin_name_maybe == expected_result @@ -492,16 +517,25 @@ def test_search_for_name(import_name, expected_result): def test_simple_import_to_pkg_map(): path_to_source = dirname(depfinder.__file__) import_to_artifact = simple_import_to_pkg_map(path_to_source) - expected_result = {'builtin': {}, - 'questionable': {'stdlib_list': {'stdlib-list'}, 'IPython.core.inputsplitter': {'ipython', 'autovizwidget'}}, - 'questionable no match': {}, - 'required': {'requests': {'apache-libcloud', - 'arm_pyart', - 'autovizwidget', - 'dbxfs', - 'google-api-core', - 'google-cloud-bigquery-storage-core', - 'requests'}, - 'yaml': {'google-cloud-bigquery-storage-core', 'pyyaml'}}, - 'required no match': {}} - assert import_to_artifact == expected_result \ No newline at end of file + expected_result = { + "builtin": {}, + "questionable": { + "stdlib_list": {"stdlib-list"}, + "IPython.core.inputsplitter": {"ipython", "autovizwidget"}, + }, + "questionable no match": {}, + "required": { + "requests": { + "apache-libcloud", + "arm_pyart", + "autovizwidget", + "dbxfs", + "google-api-core", + "google-cloud-bigquery-storage-core", + "requests", + }, + "yaml": {"google-cloud-bigquery-storage-core", "pyyaml"}, + }, + "required no match": {}, + } + assert import_to_artifact == expected_result