From 4f12e1dd97562cf3008f44f79611c5a59e67fc03 Mon Sep 17 00:00:00 2001 From: Javier Santacruz Date: Tue, 28 May 2024 07:58:33 +0200 Subject: [PATCH 1/4] Adds tests github workflow pipeline --- .github/workflows/tests.yaml | 22 ++++++++++++++++++++++ .travis.yml | 12 ------------ README.md | 2 +- 3 files changed, 23 insertions(+), 13 deletions(-) create mode 100644 .github/workflows/tests.yaml delete mode 100644 .travis.yml diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml new file mode 100644 index 0000000..796244e --- /dev/null +++ b/.github/workflows/tests.yaml @@ -0,0 +1,22 @@ +--- +name: Python tests +on: [push] + +jobs: + build: + runs-on: ubuntu-latest + strategy: + matrix: + python: ["3.8", "3.9", "3.10", "3.11", "3.12"] + format: ["basic", "yaml", "hcl"] + + steps: + - uses: actions/checkout@v4 + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python }} + - name: Install tox and any other packages + run: pip install tox + - name: Run tox + run: tox -e py-${{ matrix.format }} diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 6384c1d..0000000 --- a/.travis.yml +++ /dev/null @@ -1,12 +0,0 @@ -language: python -sudo: false -python: - - "2.7" - - "3.4" - - "3.5" - - "3.6" -# - "3.7" Not yet in Travis -install: - - pip install tox-travis -script: - - tox diff --git a/README.md b/README.md index 37af046..2aa0fe4 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ confight [![PyPI](https://img.shields.io/pypi/v/confight.svg)](https://pypi.org/project/confight/) ![PyPI - Python Version](https://img.shields.io/pypi/pyversions/confight.svg) -[![Build Status](https://travis-ci.org/Avature/confight.svg?branch=master)](https://travis-ci.org/Avature/confight) +![Build Status](https://github.com/Avature/confight/actions/workflows/tests.yaml/badge.svg) One simple way of parsing configs From c1ae1b8e14d1fd7fbf0ab66266be34d386552fec Mon Sep 17 00:00:00 2001 From: Javier Santacruz Date: Tue, 28 May 2024 08:44:50 +0200 Subject: [PATCH 2/4] Adds pre-commit config --- .pre-commit-config.yaml | 68 +++++++++++++++++++++++++++++++++++++++++ README.md | 4 +++ pyproject.toml | 2 ++ 3 files changed, 74 insertions(+) create mode 100644 .pre-commit-config.yaml create mode 100644 pyproject.toml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..6a57d1f --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,68 @@ +--- +default_stages: [pre-commit] +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.5.0 + hooks: + # Attempts to load all json files to verify syntax. + - id: check-json + + # Prettify JSON files + - id: pretty-format-json + args: [--autofix, --indent=4, --no-sort-keys] + + # Check for files that contain merge conflict strings. + - id: check-merge-conflict + + # Attempts to load all TOML files to verify syntax. + - id: check-toml + + # Makes sure files end in a newline and only a newline. + - id: end-of-file-fixer + stages: [pre-commit, manual] + + # Replaces or checks mixed line ending. + - id: mixed-line-ending + args: [--fix=no] + + # Trims trailing whitespace. + - id: trailing-whitespace + args: [--markdown-linebreak-ext=md] + stages: [pre-commit, manual] + + # Sorts entries in requirements.txt + - id: requirements-txt-fixer + + # Sort imports + - repo: https://github.com/timothycrosley/isort + rev: 5.12.0 + hooks: + - id: isort + args: [--profile=black] + + # Fix common misspellings in source code + - repo: https://github.com/codespell-project/codespell + rev: v2.2.6 + hooks: + - id: codespell + entry: codespell --write-changes + + # Fast Python linter + - repo: https://github.com/charliermarsh/ruff-pre-commit + rev: v0.1.6 + hooks: + - id: ruff + args: [--target-version, "py312"] + + # Shell script linter + - repo: https://github.com/shellcheck-py/shellcheck-py + rev: v0.9.0.6 + hooks: + - id: shellcheck + + # YAML linter + - repo: https://github.com/adrienverge/yamllint + rev: v1.33.0 + hooks: + - id: yamllint + args: [--config-data, "{extends: relaxed, rules: {line-length: {max: 100}}}"] diff --git a/README.md b/README.md index 2aa0fe4..0b3c71f 100644 --- a/README.md +++ b/README.md @@ -351,6 +351,10 @@ Install the application and run tests in development: pip install -e . python -m pytest +Make sure to install pre-commit before checking in any changes: + + pre-commit install + Changelog ========= diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..57a5583 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,2 @@ +[tool.black] +line-length = 99 From 66fc7ed6f8df9a020f655760c9dbca477c3c733f Mon Sep 17 00:00:00 2001 From: Javier Santacruz Date: Tue, 28 May 2024 08:45:53 +0200 Subject: [PATCH 3/4] Black format all code --- confight.py | 168 +++++++-------- dev-requirements.txt | 1 + setup.py | 37 ++-- test_confight.py | 476 ++++++++++++++++++++++++------------------- write_changelog.py | 30 +-- 5 files changed, 378 insertions(+), 334 deletions(-) diff --git a/confight.py b/confight.py index 11f0f70..763a0b8 100644 --- a/confight.py +++ b/confight.py @@ -1,25 +1,27 @@ from __future__ import print_function -import io -import os -import sys +import argparse import glob +import io +import itertools import json import logging -import argparse -import itertools +import os +import sys from collections import OrderedDict + try: from ConfigParser import ConfigParser + # Monkey patch python 2.7 version to avoid deprecation warnings - setattr(ConfigParser, 'read_file', getattr(ConfigParser, 'readfp')) + setattr(ConfigParser, "read_file", getattr(ConfigParser, "readfp")) except ImportError: from configparser import ConfigParser, ExtendedInterpolation import toml -__version__ = '2.0.0-2' -logger = logging.getLogger('confight') +__version__ = "2.0.0-2" +logger = logging.getLogger("confight") def load_user_app(name, extension="toml", user_prefix=None, **kwargs): @@ -35,10 +37,10 @@ def load_user_app(name, extension="toml", user_prefix=None, **kwargs): :returns: Single dict with all the loaded config """ if user_prefix is None: - user_prefix = os.path.join('~/.config', name) - filename = 'config.{ext}'.format(ext=extension) - kwargs.setdefault('user_file_path', os.path.join(user_prefix, filename)) - kwargs.setdefault('user_dir_path', os.path.join(user_prefix, 'conf.d')) + user_prefix = os.path.join("~/.config", name) + filename = "config.{ext}".format(ext=extension) + kwargs.setdefault("user_file_path", os.path.join(user_prefix, filename)) + kwargs.setdefault("user_dir_path", os.path.join(user_prefix, "conf.d")) return load_app(name, extension, **kwargs) @@ -53,15 +55,22 @@ def load_app(name, extension="toml", prefix=None, **kwargs): :returns: Single dict with all the loaded config """ if prefix is None: - prefix = os.path.join('/etc', name) - filename = 'config.{ext}'.format(ext=extension) - kwargs.setdefault('file_path', os.path.join(prefix, filename)) - kwargs.setdefault('dir_path', os.path.join(prefix, 'conf.d')) + prefix = os.path.join("/etc", name) + filename = "config.{ext}".format(ext=extension) + kwargs.setdefault("file_path", os.path.join(prefix, filename)) + kwargs.setdefault("dir_path", os.path.join(prefix, "conf.d")) return load_app_paths(extension=extension, **kwargs) -def load_app_paths(file_path=None, dir_path=None, user_file_path=None, - user_dir_path=None, default=None, paths=None, **kwargs): +def load_app_paths( + file_path=None, + dir_path=None, + user_file_path=None, + user_dir_path=None, + default=None, + paths=None, + **kwargs +): """Parse and merge user and app config files User config will have precedence @@ -70,19 +79,18 @@ def load_app_paths(file_path=None, dir_path=None, user_file_path=None, :param dir_path: Path to the extension config directory :param user_file_path: Path to the user base config file :param user_dir_path: Path to the user base config file - :param default: Path to be preppended as the default config file embedded + :param default: Path to be prepended as the default config file embedded in the app :param paths: Extra paths to add to the parsing after the defaults :param force_extension: only read files with given extension. :returns: Single dict with all the loaded config """ files = [default, file_path, dir_path, user_file_path, user_dir_path] - files += (paths or []) + files += paths or [] return load_paths([path for path in files if path], **kwargs) -def load_paths(paths, finder=None, extension=None, - force_extension=False, **kwargs): +def load_paths(paths, finder=None, extension=None, force_extension=False, **kwargs): """Parse and merge config in path and directories :param finder: Finder function(dir_path) returning ordered list of paths @@ -92,7 +100,7 @@ def load_paths(paths, finder=None, extension=None, finder = find if finder is None else finder files = itertools.chain.from_iterable(finder(path) for path in paths) if extension and force_extension: - files = (path for path in files if path.endswith('.' + extension)) + files = (path for path in files if path.endswith("." + extension)) return load(files, **kwargs) @@ -118,11 +126,11 @@ def parse(path, format=None): :returns: dict with the parsed contents """ format = format_from_path(path) if format is None else format - logger.info('Parsing %r config file from %r', format, path) + logger.info("Parsing %r config file from %r", format, path) if format not in FORMATS: - raise ValueError('Unknown format {} for file {}'.format(format, path)) + raise ValueError("Unknown format {} for file {}".format(format, path)) loader = FORMAT_LOADERS[format] - with io.open(path, 'r', encoding='utf8') as stream: + with io.open(path, "r", encoding="utf8") as stream: return loader(stream) @@ -136,12 +144,10 @@ def merge(configs): :param configs: List of parsed config dicts in order :returns: dict with the merged resulting config """ - logger.debug('Merging config data %r', configs) + logger.debug("Merging config data %r", configs) result = OrderedDict() # No OrderedSets available - keys = OrderedDict( - (key, None) for config in configs for key in config - ) + keys = OrderedDict((key, None) for config in configs for key in config) for key in keys: values = [config[key] for config in configs if key in config] merges = [v for v in values if isinstance(v, dict)] @@ -164,7 +170,7 @@ def find(path): return [] if os.path.isfile(path): return [path] - return sorted(glob.glob(os.path.join(path, '*'))) + return sorted(glob.glob(os.path.join(path, "*"))) def check_access(path): @@ -172,43 +178,40 @@ def check_access(path): if not path: return False elif not os.path.exists(path): - logger.debug('Could not find %r', path) + logger.debug("Could not find %r", path) return False elif not os.access(path, os.R_OK): - logger.error('Could not read %r', path) + logger.error("Could not read %r", path) return False elif os.path.isdir(path) and not os.access(path, os.X_OK): - logger.error('Could not list directory %r', path) + logger.error("Could not list directory %r", path) return False elif os.path.isfile(path) and os.access(path, os.X_OK): - logger.warning('Config file %r has exec permissions', path) + logger.warning("Config file %r has exec permissions", path) return True def load_ini(stream): - if 'ExtendedInterpolation' in globals(): + if "ExtendedInterpolation" in globals(): parser = ConfigParser(interpolation=ExtendedInterpolation()) else: parser = ConfigParser() parser.read_file(stream) - return { - section: OrderedDict(parser.items(section)) - for section in parser.sections() - } + return {section: OrderedDict(parser.items(section)) for section in parser.sections()} -FORMATS = ('toml', 'ini', 'json') +FORMATS = ("toml", "ini", "json") FORMAT_EXTENSIONS = { - 'js': 'json', - 'json': 'json', - 'toml': 'toml', - 'ini': 'ini', - 'cfg': 'ini', + "js": "json", + "json": "json", + "toml": "toml", + "ini": "ini", + "cfg": "ini", } FORMAT_LOADERS = { - 'json': lambda *args: json.load(*args, object_pairs_hook=OrderedDict), - 'toml': lambda *args: toml.load(*args, _dict=OrderedDict), - 'ini': load_ini + "json": lambda *args: json.load(*args, object_pairs_hook=OrderedDict), + "toml": lambda *args: toml.load(*args, _dict=OrderedDict), + "ini": load_ini, } @@ -218,18 +221,14 @@ def load_ini(stream): except ImportError: pass else: + def load_yaml(stream): yaml = YAML(typ="rt") return yaml.load(stream) - FORMATS = FORMATS + ('yaml',) - FORMAT_EXTENSIONS.update({ - 'yml': 'yaml', - 'yaml': 'yaml' - }) - FORMAT_LOADERS.update({ - 'yaml': load_yaml - }) + FORMATS = FORMATS + ("yaml",) + FORMAT_EXTENSIONS.update({"yml": "yaml", "yaml": "yaml"}) + FORMAT_LOADERS.update({"yaml": load_yaml}) # Optional dependency HCL try: @@ -237,32 +236,28 @@ def load_yaml(stream): except ImportError: pass else: + def load_hcl(stream): return hcl.load(stream) - FORMATS = FORMATS + ('hcl',) - FORMAT_EXTENSIONS.update({ - 'hcl': 'hcl' - }) - FORMAT_LOADERS.update({ - 'hcl': load_hcl - }) + FORMATS = FORMATS + ("hcl",) + FORMAT_EXTENSIONS.update({"hcl": "hcl"}) + FORMAT_LOADERS.update({"hcl": load_hcl}) def format_from_path(path): - """Get file format from a given path based on exension""" + """Get file format from a given path based on extension""" ext = os.path.splitext(path)[1][1:] # extension without dot format = FORMAT_EXTENSIONS.get(ext) if not format: - raise ValueError( - 'Unknown format extension {!r} for {!r}'.format(ext, path) - ) + raise ValueError("Unknown format extension {!r} for {!r}".format(ext, path)) return format def get_version(): import pkg_resources - return 'confight ' + pkg_resources.get_distribution('confight').version + + return "confight " + pkg_resources.get_distribution("confight").version def cli_configure_logging(args): @@ -272,40 +267,33 @@ def cli_configure_logging(args): def cli_show(args): """Load config and show it""" - config = load_user_app( - args.name, prefix=args.prefix, user_prefix=args.user_prefix - ) - print(toml.dumps(config), end='') + config = load_user_app(args.name, prefix=args.prefix, user_prefix=args.user_prefix) + print(toml.dumps(config), end="") def cli(): - LOG_LEVELS = ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'] - parser = argparse.ArgumentParser( - description='One simple way of parsing configs' - ) - parser.add_argument('--version', action='version', version=get_version()) + LOG_LEVELS = ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"] + parser = argparse.ArgumentParser(description="One simple way of parsing configs") + parser.add_argument("--version", action="version", version=get_version()) parser.add_argument( - '-v', '--verbose', choices=LOG_LEVELS, default='ERROR', - help='Logging level default: ERROR' - ) - subparsers = parser.add_subparsers(title='subcommands', dest='command') - show_parser = subparsers.add_parser('show') - show_parser.add_argument('name', help='Name of the application') - show_parser.add_argument('--prefix', help='Base for default paths') - show_parser.add_argument( - '--user-prefix', help='Base for default user paths' + "-v", "--verbose", choices=LOG_LEVELS, default="ERROR", help="Logging level default: ERROR" ) + subparsers = parser.add_subparsers(title="subcommands", dest="command") + show_parser = subparsers.add_parser("show") + show_parser.add_argument("name", help="Name of the application") + show_parser.add_argument("--prefix", help="Base for default paths") + show_parser.add_argument("--user-prefix", help="Base for default user paths") args = parser.parse_args() cli_configure_logging(args) # Use callbacks, parser.set_defaults(func=) does not work in Python3.3 callbacks = { - 'show': cli_show, + "show": cli_show, None: lambda args: parser.print_help(file=sys.stderr), } try: callbacks[args.command](args) except Exception as error: - log = logger.exception if args.verbose == 'DEBUG' else logger.error - log('Error: %s', error) + log = logger.exception if args.verbose == "DEBUG" else logger.error + log("Error: %s", error) sys.exit(1) diff --git a/dev-requirements.txt b/dev-requirements.txt index 0f56f76..b60033d 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,2 +1,3 @@ +black pyHamcrest pytest diff --git a/setup.py b/setup.py index 46e5efc..262f69c 100644 --- a/setup.py +++ b/setup.py @@ -1,36 +1,37 @@ import io import re + from setuptools import setup def get_version_from_debian_changelog(): try: - with io.open('debian/changelog', encoding='utf8') as stream: - return re.search(r'\((.+)\)', next(stream)).group(1) + with io.open("debian/changelog", encoding="utf8") as stream: + return re.search(r"\((.+)\)", next(stream)).group(1) except Exception: - return '0.0.1' + return "0.0.1" setup( - name='confight', + name="confight", version=get_version_from_debian_changelog(), - description='Common config loading for Python and the command line', - license='MIT', - author='Avature', - author_email='platform@avature.net', - url='https://github.com/avature/confight', - keywords='config configuration droplets toml json ini yaml', - long_description=io.open('README.md', encoding='utf8').read(), - long_description_content_type='text/markdown', - py_modules=['confight'], - install_requires=io.open('requirements.txt').read().splitlines(), + description="Common config loading for Python and the command line", + license="MIT", + author="Avature", + author_email="platform@avature.net", + url="https://github.com/avature/confight", + keywords="config configuration droplets toml json ini yaml", + long_description=io.open("README.md", encoding="utf8").read(), + long_description_content_type="text/markdown", + py_modules=["confight"], + install_requires=io.open("requirements.txt").read().splitlines(), extras_require={ - 'yaml': ["ruamel.yaml>=0.18.0"], - 'hcl': ["pyhcl"], + "yaml": ["ruamel.yaml>=0.18.0"], + "hcl": ["pyhcl"], }, entry_points={ - 'console_scripts': [ - 'confight = confight:cli', + "console_scripts": [ + "confight = confight:cli", ] }, classifiers=[ diff --git a/test_confight.py b/test_confight.py index bc142b9..fb89e0a 100644 --- a/test_confight.py +++ b/test_confight.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- import os + try: import subprocess32 as subprocess except ImportError: @@ -10,11 +11,28 @@ from unittest import mock import pytest -from hamcrest import (assert_that, has_entry, has_key, has_entries, is_, empty, - only_contains, contains_exactly, contains_string) - -from confight import (parse, merge, find, load, load_paths, load_app, - load_user_app, FORMATS) +from hamcrest import ( + assert_that, + contains_exactly, + contains_string, + empty, + has_entries, + has_entry, + has_key, + is_, + only_contains, +) + +from confight import ( + FORMATS, + find, + load, + load_app, + load_paths, + load_user_app, + merge, + parse, +) @pytest.fixture @@ -22,20 +40,15 @@ def examples(tmpdir): return Repository(tmpdir) -FILES = [ - 'basic_file.toml', 'basic_file.ini', 'basic_file.json', 'basic_file.cfg', - 'basic_file.js' -] +FILES = ["basic_file.toml", "basic_file.ini", "basic_file.json", "basic_file.cfg", "basic_file.js"] if "yaml" in FORMATS: FILES.extend(["basic_file.yaml", "basic_file.yml"]) if "hcl" in FORMATS: FILES.extend(["basic_file.hcl"]) -INVALID_FILES = [ - 'invalid.toml', 'invalid.ini', 'invalid.json', 'invalid.cfg', 'invalid.js' -] -INVALID_EXTENSIONS = ['bad_ext.ext', 'bad_ext.j'] -SORTED_FILES = ['00_base.toml', '01_first.json', 'AA_second.ini'] +INVALID_FILES = ["invalid.toml", "invalid.ini", "invalid.json", "invalid.cfg", "invalid.js"] +INVALID_EXTENSIONS = ["bad_ext.ext", "bad_ext.j"] +SORTED_FILES = ["00_base.toml", "01_first.json", "AA_second.ini"] class TestParse(object): @@ -43,29 +56,30 @@ class TestParse(object): def test_it_should_detect_format_from_extension(self, name, examples): config = parse(examples.get(name)) - assert_that(config, has_entry('section', has_key('string'))) + assert_that(config, has_entry("section", has_key("string"))) @pytest.mark.parametrize("name", FILES) def test_it_loads_strings_as_unicode(self, name, examples): config = parse(examples.get(name)) - assert_that(config, has_entry( - 'section', has_entry('unicode', is_(u'💩')) - )) + assert_that(config, has_entry("section", has_entry("unicode", is_("💩")))) - @pytest.mark.parametrize("name, format", [ - ('basic_file_toml', 'toml'), - ('basic_file_ini', 'ini'), - ('basic_file_json', 'json'), - ]) + @pytest.mark.parametrize( + "name, format", + [ + ("basic_file_toml", "toml"), + ("basic_file_ini", "ini"), + ("basic_file_json", "json"), + ], + ) def test_it_should_load_for_given_format(self, name, format, examples): config = parse(examples.get(name), format) - assert_that(config, has_entry('section', has_key('string'))) + assert_that(config, has_entry("section", has_key("string"))) def test_it_should_fail_with_missing_files(self): with pytest.raises(Exception): - parse('/path/to/nowhere.json') + parse("/path/to/nowhere.json") @pytest.mark.parametrize("name", INVALID_FILES) def test_it_should_fail_with_invalid_files(self, name, examples): @@ -81,58 +95,60 @@ def test_it_should_fail_with_invalid_extensions(self, name, examples): class TestMerge(object): def test_it_should_give_priority_to_last_value(self): configs = [ - {'key': 1}, - {'key': 2}, - {'key': 3}, + {"key": 1}, + {"key": 2}, + {"key": 3}, ] result = merge(configs) - assert_that(result, has_entry('key', 3)) + assert_that(result, has_entry("key", 3)) def test_it_should_add_all_values(self): configs = [ - {'section1': {'key1': 1}}, - {'section2': {'key2': 2}}, - {'section3': {'key3': 3}}, + {"section1": {"key1": 1}}, + {"section2": {"key2": 2}}, + {"section3": {"key3": 3}}, ] result = merge(configs) - assert_that(result, has_entries({ - 'section1': has_entry('key1', 1), - 'section2': has_entry('key2', 2), - 'section3': has_entry('key3', 3), - })) + assert_that( + result, + has_entries( + { + "section1": has_entry("key1", 1), + "section2": has_entry("key2", 2), + "section3": has_entry("key3", 3), + } + ), + ) def test_it_should_merge_dicts_recursively(self): configs = [ - {'section': {'lv1': {'lv2': {'lv3': 1}}}}, - {'section': {'lv1': {'lv2': {'lv3': 2}}}}, - {'section': {'lv1': {'lv2': {'lv3': 3}}}}, + {"section": {"lv1": {"lv2": {"lv3": 1}}}}, + {"section": {"lv1": {"lv2": {"lv3": 2}}}}, + {"section": {"lv1": {"lv2": {"lv3": 3}}}}, ] result = merge(configs) - assert_that(result, has_entry( - 'section', has_entry( - 'lv1', has_entry( - 'lv2', has_entry( - 'lv3', 3))) - )) + assert_that( + result, has_entry("section", has_entry("lv1", has_entry("lv2", has_entry("lv3", 3)))) + ) def test_it_should_ignore_scalar_values_given_as_configs(self): configs = [ - {'section': {'key': 1}}, - {'section': None}, - {'section': {'key': 2}}, - {'section': []}, - {'section': {'key': 3}}, + {"section": {"key": 1}}, + {"section": None}, + {"section": {"key": 2}}, + {"section": []}, + {"section": {"key": 3}}, ] result = merge(configs) - assert_that(result, has_entry('section', has_entry('key', 3))) + assert_that(result, has_entry("section", has_entry("key", 3))) class TestFind(object): @@ -150,13 +166,10 @@ def test_it_should_load_full_paths(self, examples): found = find(str(examples.tmpdir)) - assert_that( - all(os.path.isabs(path) for path in found), - is_(True) - ) + assert_that(all(os.path.isabs(path) for path in found), is_(True)) def test_it_should_normalize_relative_paths(self): - path = os.path.join('.', os.path.basename(__file__)) + path = os.path.join(".", os.path.basename(__file__)) found = find(path) @@ -171,15 +184,15 @@ def test_it_should_load_existing_files(self, examples): def test_it_should_expand_user_variables(self, tmpdir, examples): path = examples.get(FILES[0]) - home = os.path.expanduser('~') - relpath = os.path.join('~', os.path.relpath(path, start=home)) + home = os.path.expanduser("~") + relpath = os.path.join("~", os.path.relpath(path, start=home)) found = find(relpath) assert_that(found, contains_exactly(path)) def test_it_should_return_nothing_for_missing_directories(self): - assert_that(find('/path/to/nowhere'), is_(empty())) + assert_that(find("/path/to/nowhere"), is_(empty())) def test_it_should_ignore_invalid_files(self): found = find(None) @@ -202,7 +215,7 @@ def test_it_should_ignore_unexplorable_dirs(self, tmpdir): assert_that(found, is_(empty())) - @mock.patch('confight.logger') + @mock.patch("confight.logger") def test_it_should_warn_about_executable_config_files(self, logger, examples): executable_file = examples.load(FILES[0]) os.chmod(executable_file, 0o777) @@ -219,27 +232,24 @@ def test_it_should_load_and_merge_lists_of_paths(self, examples): config = load(paths) - assert_that(config, has_entry('section', has_entry('key', 'second'))) + assert_that(config, has_entry("section", has_entry("key", "second"))) def test_it_should_load_paths_for_given_format(self, examples): - paths = examples.get_many(['00_base.toml', 'basic_file_toml']) + paths = examples.get_many(["00_base.toml", "basic_file_toml"]) - config = load(paths, format='toml') + config = load(paths, format="toml") - assert_that(config, has_entry('section', has_entry('key', 'basic'))) + assert_that(config, has_entry("section", has_entry("key", "basic"))) def test_it_should_use_given_parser(self): - paths = ['/path/to/1', '/path/to/2'] + paths = ["/path/to/1", "/path/to/2"] def myparse(path, format=None): - return {'path': path, 'format': format} + return {"path": path, "format": format} - config = load(paths, format='toml', parser=myparse) + config = load(paths, format="toml", parser=myparse) - assert_that(config, has_entries({ - 'path': paths[-1], - 'format': 'toml' - })) + assert_that(config, has_entries({"path": paths[-1], "format": "toml"})) def test_it_should_use_given_merger(self, examples): paths = examples.get_many(SORTED_FILES) @@ -249,7 +259,7 @@ def mymerge(configs): config = load(paths, merger=mymerge) - assert_that(config, only_contains(has_key('section'))) + assert_that(config, only_contains(has_key("section"))) class TestLoadPaths(object): @@ -259,16 +269,14 @@ def test_it_should_load_from_file_and_directory(self, examples): config = load_paths([paths[0], str(examples.tmpdir)]) - assert_that(config, has_entry('section', has_entry('key', 'second'))) + assert_that(config, has_entry("section", has_entry("key", "second"))) def test_it_should_filter_extensions(self, examples): examples.clear() examples.get_many(FILES) expected_contents = parse(examples.get(FILES[0])) # toml file - config = load_paths( - [str(examples.tmpdir)], extension='toml', force_extension=True - ) + config = load_paths([str(examples.tmpdir)], extension="toml", force_extension=True) # Only reads the toml file assert_that(config, is_(expected_contents)) @@ -280,8 +288,15 @@ def test_merges_must_retain_order(self, examples): config = load(paths) good_data = [ - 'string', 'integer', 'float', 'boolean', 'list', 'key', 'unicode', - 'subsection', 'null' + "string", + "integer", + "float", + "boolean", + "list", + "key", + "unicode", + "subsection", + "null", ] assert_that(config["section"].keys(), contains_exactly(*good_data)) @@ -297,81 +312,107 @@ def call_config_loader(self, loader, *args, **kwargs): Result is a dictionary with all loaded d[key, int] with every loaded path the order in which they were loaded. """ - def myparser(path, format=None, _data={'n': 0}): - _data['n'] += 1 - return {path: _data['n']} + + def myparser(path, format=None, _data={"n": 0}): + _data["n"] += 1 + return {path: _data["n"]} def myfinder(path): return [path] - kwargs.setdefault('parser', myparser) - kwargs.setdefault('finder', myfinder) + kwargs.setdefault("parser", myparser) + kwargs.setdefault("finder", myfinder) return loader(*args, **kwargs) class TestLoadApp(LoadAppBehaviour): def test_it_should_load_from_default_path(self): - config = self.load_app('myapp') + config = self.load_app("myapp") - assert_that(self.loaded_paths(config), contains_exactly( - '/etc/myapp/config.toml', '/etc/myapp/conf.d', - )) + assert_that( + self.loaded_paths(config), + contains_exactly( + "/etc/myapp/config.toml", + "/etc/myapp/conf.d", + ), + ) def test_it_should_be_able_to_ignore_file(self): - config = self.load_app('myapp', file_path=None) + config = self.load_app("myapp", file_path=None) - assert_that(self.loaded_paths(config), contains_exactly( - '/etc/myapp/conf.d', - )) + assert_that( + self.loaded_paths(config), + contains_exactly( + "/etc/myapp/conf.d", + ), + ) def test_it_should_be_able_to_ignore_directory(self): - config = self.load_app('myapp', dir_path=None) + config = self.load_app("myapp", dir_path=None) - assert_that(self.loaded_paths(config), contains_exactly( - '/etc/myapp/config.toml', - )) + assert_that( + self.loaded_paths(config), + contains_exactly( + "/etc/myapp/config.toml", + ), + ) def test_it_should_load_extra_paths(self): - config = self.load_app('myapp', paths=['/extra/path']) + config = self.load_app("myapp", paths=["/extra/path"]) - assert_that(self.loaded_paths(config), contains_exactly( - '/etc/myapp/config.toml', '/etc/myapp/conf.d', '/extra/path' - )) + assert_that( + self.loaded_paths(config), + contains_exactly("/etc/myapp/config.toml", "/etc/myapp/conf.d", "/extra/path"), + ) def test_it_should_add_default_as_priority_location(self): - config = self.load_app('myapp', default='/path/to/default') + config = self.load_app("myapp", default="/path/to/default") - assert_that(self.loaded_paths(config), contains_exactly( - '/path/to/default', - '/etc/myapp/config.toml', - '/etc/myapp/conf.d', - )) + assert_that( + self.loaded_paths(config), + contains_exactly( + "/path/to/default", + "/etc/myapp/config.toml", + "/etc/myapp/conf.d", + ), + ) def test_it_should_allow_using_known_extensions(self): - config = self.load_app('myapp', extension='json') + config = self.load_app("myapp", extension="json") - assert_that(self.loaded_paths(config), contains_exactly( - '/etc/myapp/config.json', '/etc/myapp/conf.d', - )) + assert_that( + self.loaded_paths(config), + contains_exactly( + "/etc/myapp/config.json", + "/etc/myapp/conf.d", + ), + ) def test_it_should_reject_custom_extensions(self): with pytest.raises(ValueError): - self.load_app('myapp', extension='jsn', parser=parse) + self.load_app("myapp", extension="jsn", parser=parse) def test_it_should_allow_using_custom_extensions_with_format(self): - config = self.load_app('myapp', extension='jsn', format='json') + config = self.load_app("myapp", extension="jsn", format="json") - assert_that(self.loaded_paths(config), contains_exactly( - '/etc/myapp/config.jsn', '/etc/myapp/conf.d', - )) + assert_that( + self.loaded_paths(config), + contains_exactly( + "/etc/myapp/config.jsn", + "/etc/myapp/conf.d", + ), + ) def test_it_should_use_prefix_for_default_locations(self): - config = self.load_app('myapp', prefix='/my/path') - - assert_that(self.loaded_paths(config), contains_exactly( - '/my/path/config.toml', '/my/path/conf.d', - )) + config = self.load_app("myapp", prefix="/my/path") + assert_that( + self.loaded_paths(config), + contains_exactly( + "/my/path/config.toml", + "/my/path/conf.d", + ), + ) def load_app(self, *args, **kwargs): return self.call_config_loader(load_app, *args, **kwargs) @@ -379,69 +420,87 @@ def load_app(self, *args, **kwargs): class TestLoadUserApp(LoadAppBehaviour): def test_it_should_load_from_default_user_path(self): - config = self.load_app('myapp') + config = self.load_app("myapp") - assert_that(self.loaded_paths(config), contains_exactly( - '/etc/myapp/config.toml', - '/etc/myapp/conf.d', - '~/.config/myapp/config.toml', - '~/.config/myapp/conf.d' - )) + assert_that( + self.loaded_paths(config), + contains_exactly( + "/etc/myapp/config.toml", + "/etc/myapp/conf.d", + "~/.config/myapp/config.toml", + "~/.config/myapp/conf.d", + ), + ) def test_it_should_load_extra_paths(self): - config = self.load_app('myapp', paths=['/extra/path']) + config = self.load_app("myapp", paths=["/extra/path"]) - assert_that(self.loaded_paths(config), contains_exactly( - '/etc/myapp/config.toml', - '/etc/myapp/conf.d', - '~/.config/myapp/config.toml', - '~/.config/myapp/conf.d', - '/extra/path' - )) + assert_that( + self.loaded_paths(config), + contains_exactly( + "/etc/myapp/config.toml", + "/etc/myapp/conf.d", + "~/.config/myapp/config.toml", + "~/.config/myapp/conf.d", + "/extra/path", + ), + ) def test_it_should_allow_using_known_extensions(self): - config = self.load_app('myapp', extension='json') + config = self.load_app("myapp", extension="json") - assert_that(self.loaded_paths(config), contains_exactly( - '/etc/myapp/config.json', - '/etc/myapp/conf.d', - '~/.config/myapp/config.json', - '~/.config/myapp/conf.d', - )) + assert_that( + self.loaded_paths(config), + contains_exactly( + "/etc/myapp/config.json", + "/etc/myapp/conf.d", + "~/.config/myapp/config.json", + "~/.config/myapp/conf.d", + ), + ) def test_it_should_reject_custom_extensions(self): with pytest.raises(ValueError): - self.load_app('myapp', extension='jsn', parser=parse) + self.load_app("myapp", extension="jsn", parser=parse) def test_it_should_allow_using_custom_extensions_with_format(self): - config = self.load_app('myapp', extension='jsn', format='json') + config = self.load_app("myapp", extension="jsn", format="json") - assert_that(self.loaded_paths(config), contains_exactly( - '/etc/myapp/config.jsn', - '/etc/myapp/conf.d', - '~/.config/myapp/config.jsn', - '~/.config/myapp/conf.d', - )) + assert_that( + self.loaded_paths(config), + contains_exactly( + "/etc/myapp/config.jsn", + "/etc/myapp/conf.d", + "~/.config/myapp/config.jsn", + "~/.config/myapp/conf.d", + ), + ) def test_it_should_use_prefix_for_default_locations(self): - config = self.load_app('myapp', prefix='/my/path') + config = self.load_app("myapp", prefix="/my/path") - assert_that(self.loaded_paths(config), contains_exactly( - '/my/path/config.toml', - '/my/path/conf.d', - '~/.config/myapp/config.toml', - '~/.config/myapp/conf.d', - )) + assert_that( + self.loaded_paths(config), + contains_exactly( + "/my/path/config.toml", + "/my/path/conf.d", + "~/.config/myapp/config.toml", + "~/.config/myapp/conf.d", + ), + ) def test_it_should_use_prefix_for_default_user_locations(self): - config = self.load_app('myapp', user_prefix='/my/path') + config = self.load_app("myapp", user_prefix="/my/path") - assert_that(self.loaded_paths(config), contains_exactly( - '/etc/myapp/config.toml', - '/etc/myapp/conf.d', - '/my/path/config.toml', - '/my/path/conf.d', - )) + assert_that( + self.loaded_paths(config), + contains_exactly( + "/etc/myapp/config.toml", + "/etc/myapp/conf.d", + "/my/path/config.toml", + "/my/path/conf.d", + ), + ) def load_app(self, *args, **kwargs): return self.call_config_loader(load_user_app, *args, **kwargs) @@ -451,45 +510,44 @@ class TestCli(object): def test_it_should_print_help(self): out = subprocess.run([self.bin], stderr=subprocess.PIPE) - assert_that(out.stderr.decode('utf8'), contains_string('usage:')) + assert_that(out.stderr.decode("utf8"), contains_string("usage:")) def test_it_should_show_message_on_exit(self, examples): examples.clear() - examples.create('config.toml', b'[broken') + examples.create("config.toml", b"[broken") - out = self.run(['show', 'name', '--prefix', str(examples.tmpdir)]) + out = self.run(["show", "name", "--prefix", str(examples.tmpdir)]) - assert_that(out.stderr.decode('utf8'), contains_string('Error:')) + assert_that(out.stderr.decode("utf8"), contains_string("Error:")) assert_that(out.returncode, is_(1)) def test_it_should_show_config(self, examples): examples.clear() - examples.get('config.toml') - contents = examples.get_contents('config.toml') + examples.get("config.toml") + contents = examples.get_contents("config.toml") - out = self.run(['show', 'name', '--prefix', str(examples.tmpdir)]) + out = self.run(["show", "name", "--prefix", str(examples.tmpdir)]) - assert_that(out.stdout.decode('utf8'), is_(contents)) + assert_that(out.stdout.decode("utf8"), is_(contents)) assert_that(out.returncode, is_(0)) def run(self, args): return subprocess.run( - [self.bin] + list(args), - stdout=subprocess.PIPE, - stderr=subprocess.PIPE + [self.bin] + list(args), stdout=subprocess.PIPE, stderr=subprocess.PIPE ) @property def bin(self): - name = 'confight' - prefix = os.getenv('VIRTUAL_ENV', '') + name = "confight" + prefix = os.getenv("VIRTUAL_ENV", "") if prefix: - name = os.path.join(prefix, 'bin', name) + name = os.path.join(prefix, "bin", name) return name # Evil monkeypatching for python 3.3 and python 3.4 -if getattr(subprocess, 'run', None) is None: +if getattr(subprocess, "run", None) is None: + class CompletedProcess: def __init__(self, **kwargs): self.__dict__.update(kwargs) @@ -505,10 +563,7 @@ def maimed_run(*args, **kwargs): retcode = process.poll() return CompletedProcess( - args=process.args, - returncode=retcode, - stdout=stdout, - stderr=stderr + args=process.args, returncode=retcode, stdout=stdout, stderr=stderr ) subprocess.run = maimed_run @@ -534,19 +589,19 @@ def clear(self): def create(self, name, contents): fileobj = self.tmpdir.join(name) - fileobj.write(contents, 'wb') + fileobj.write(contents, "wb") return str(fileobj) def load(self, name): - return self.create(name, self._contents[name].encode('utf8')) + return self.create(name, self._contents[name].encode("utf8")) _files = {} _contents = { - 'config.toml': u"""\ + "config.toml": """\ [section] string = "toml" """, - 'basic_file.toml': u""" + "basic_file.toml": """ # Basic toml file [section] string = "toml" @@ -560,13 +615,13 @@ def load(self, name): [section.subsection] key = "value" """, - 'basic_file.ini': u""" + "basic_file.ini": """ # Basic ini file [section] string = string unicode = 💩 """, - 'basic_file.json': u""" + "basic_file.json": """ { "section": { "string": "json", @@ -579,7 +634,7 @@ def load(self, name): } } """, - 'basic_file.yaml': u""" + "basic_file.yaml": """ section: string: "json" integer: 3 @@ -591,7 +646,7 @@ def load(self, name): - fourth unicode: "💩" """, - 'basic_file.hcl': u""" + "basic_file.hcl": """ section { string = "hcl" integer = 3 @@ -602,43 +657,42 @@ def load(self, name): unicode = "💩" } """, - 'invalid.toml': """ + "invalid.toml": """ [section] key = null """, - 'invalid.ini': """ + "invalid.ini": """ = """, - 'invalid.json': """ + "invalid.json": """ {"invalid"} """, - '00_base.toml': """ + "00_base.toml": """ [section] key = "zero" """, - '01_first.json': """ + "01_first.json": """ { "section": { "key": "first" } } """, - 'AA_second.ini': """ + "AA_second.ini": """ [section] key = second """, } - _contents['basic_file_toml'] = _contents['basic_file.toml'] - _contents['basic_file_ini'] = _contents['basic_file.ini'] - _contents['basic_file_json'] = _contents['basic_file.json'] - _contents['basic_file.js'] = _contents['basic_file.json'] - _contents['basic_file.cfg'] = _contents['basic_file.ini'] - _contents['invalid.js'] = _contents['invalid.json'] - _contents['invalid.cfg'] = _contents['invalid.ini'] - _contents['bad_ext.ext'] = _contents['basic_file.toml'] - _contents['bad_ext.j'] = _contents['basic_file.json'] - _contents['bad_ext.u'] = _contents['basic_file.yaml'] - _contents['basic_file_yaml'] = _contents['basic_file.yaml'] - _contents['basic_file.yml'] = _contents['basic_file.yaml'] - _contents['basic_file_hcl'] = _contents['basic_file.hcl'] - + _contents["basic_file_toml"] = _contents["basic_file.toml"] + _contents["basic_file_ini"] = _contents["basic_file.ini"] + _contents["basic_file_json"] = _contents["basic_file.json"] + _contents["basic_file.js"] = _contents["basic_file.json"] + _contents["basic_file.cfg"] = _contents["basic_file.ini"] + _contents["invalid.js"] = _contents["invalid.json"] + _contents["invalid.cfg"] = _contents["invalid.ini"] + _contents["bad_ext.ext"] = _contents["basic_file.toml"] + _contents["bad_ext.j"] = _contents["basic_file.json"] + _contents["bad_ext.u"] = _contents["basic_file.yaml"] + _contents["basic_file_yaml"] = _contents["basic_file.yaml"] + _contents["basic_file.yml"] = _contents["basic_file.yaml"] + _contents["basic_file_hcl"] = _contents["basic_file.hcl"] diff --git a/write_changelog.py b/write_changelog.py index 9d19a3b..469cbbe 100644 --- a/write_changelog.py +++ b/write_changelog.py @@ -1,23 +1,23 @@ """Write README changelog section from debian Changelog section""" from __future__ import print_function -import io -import re import datetime import email.utils +import io +import re def write_changelog(): - with io.open('README.md', 'r+', encoding='utf8') as stream: + with io.open("README.md", "r+", encoding="utf8") as stream: remove_old_changelog(stream) stream.writelines(get_changes()) def get_changes(): - with io.open('debian/changelog', encoding='utf8') as stream: - yield u"\n" + with io.open("debian/changelog", encoding="utf8") as stream: + yield "\n" for change in parse_changelog(stream): - yield u"* {version} ({date})\n{changes}".format(**change) + yield "* {version} ({date})\n{changes}".format(**change) def parse_changelog(stream): @@ -31,14 +31,14 @@ def parse_changelog(stream): context.update(footer) yield context else: - context['changes'] = context.get('changes', '') + line + context["changes"] = context.get("changes", "") + line -_changelog_re = re.compile('^Changelog *$') +_changelog_re = re.compile("^Changelog *$") def remove_old_changelog(stream): - for line in iter(stream.readline, ''): + for line in iter(stream.readline, ""): if _changelog_re.match(line): # truncate at next line stream.readline() @@ -46,10 +46,10 @@ def remove_old_changelog(stream): stream.seek(0, 1) return - raise Exception(u'Could not find Changelog section') + raise Exception("Could not find Changelog section") -_header_re = re.compile('^(?P\w+) \((?P.+)\) \w+; \w+=\w+') +_header_re = re.compile("^(?P\w+) \((?P.+)\) \w+; \w+=\w+") def _detect_header(line): @@ -58,18 +58,18 @@ def _detect_header(line): return match.groupdict() -_footer_re = re.compile('^ -- [^<]+ \<[^>]+\> (?P.*)') +_footer_re = re.compile("^ -- [^<]+ \<[^>]+\> (?P.*)") def _detect_footer(line): match = _footer_re.match(line) if match: - return dict(date=parse_changelog_date(match.group('date'))) + return dict(date=parse_changelog_date(match.group("date"))) def parse_changelog_date(text): - return datetime.datetime(*email.utils.parsedate(text)[:6]).strftime('%Y-%m-%d') + return datetime.datetime(*email.utils.parsedate(text)[:6]).strftime("%Y-%m-%d") -if __name__ == '__main__': +if __name__ == "__main__": write_changelog() From abf446e64f2ae300139a6ccb634a91fef453a6b2 Mon Sep 17 00:00:00 2001 From: Javier Santacruz Date: Tue, 28 May 2024 08:50:52 +0200 Subject: [PATCH 4/4] Allows to register custom formats fixes #26 - Changes loader API to take paths rather than open streams - Makes loader API public through the `register_format` function so library users can define their own custom loaders. --- README.md | 28 +++++++++++-- confight.py | 84 +++++++++++++++++++++++++++----------- test_confight.py | 102 ++++++++++++++++++++++++++++++++++++++++------- tox.ini | 2 +- 4 files changed, 175 insertions(+), 41 deletions(-) diff --git a/README.md b/README.md index 0b3c71f..d840593 100644 --- a/README.md +++ b/README.md @@ -136,6 +136,29 @@ The list of _optional_ file formats: In order to install confight with _optional_ formats see [installation](#installation) with [optional features][]. +## Custom Formats + +Other formats can be registered by name providing a function with the loader signature +`loader(path)`. + +```python +from confight import register_loader, register_extension + +def load_assign(path): + """Parse files with KEY=VALUE lines """ + with open(path, 'r') as stream: + return dict(line.split('=', 1) for line in stream] + +register_loader('equal', load_assign) +``` + +Extensions can also be associated to previously registered formats by adding them to the extension +register as alias so it automatically detects with `.eq` extension: + +```python +register_extension('eq', format='equal') +``` + ## Parsing Given a path to an existing configuration file, it will be loaded in memory @@ -151,7 +174,7 @@ confight.parse('/path/to/config', format='toml') When no format is given, it tries to guess by looking at file extensions: ``` -confight.parse('/path/to/config.json') # will gess json format +confight.parse('/path/to/config.json') # will guess json format ``` You can see the list of all available extensions at `confight.FORMAT_EXTENSIONS`. @@ -375,7 +398,7 @@ Changelog * 1.4.0 (2023-12-12) [ Federico Fapitalle ] - * [3e618f3b] feat: adds support for HCL languaje + * [3e618f3b] feat: adds support for HCL language [ Frank Lenormand ] * [a9b3b9a2] fix(confight): Stick to older `ruamel.yaml` API @@ -469,4 +492,3 @@ Changelog * 0.0.1 (2018-03-27) * Initial release. - diff --git a/confight.py b/confight.py index 763a0b8..5d7edf9 100644 --- a/confight.py +++ b/confight.py @@ -127,11 +127,10 @@ def parse(path, format=None): """ format = format_from_path(path) if format is None else format logger.info("Parsing %r config file from %r", format, path) - if format not in FORMATS: + if format not in FORMAT_LOADERS: raise ValueError("Unknown format {} for file {}".format(format, path)) loader = FORMAT_LOADERS[format] - with io.open(path, "r", encoding="utf8") as stream: - return loader(stream) + return loader(path) def merge(configs): @@ -191,16 +190,35 @@ def check_access(path): return True -def load_ini(stream): +def open_text(path): + return io.open(path, mode="r", encoding="utf8") + + +def open_bin(path): + return io.open(path, mode="rb", encoding="utf8") + + +def load_json(path): + with open_text(path) as stream: + return json.load(stream, object_pairs_hook=OrderedDict) + + +def load_toml(path): + with open_text(path) as stream: + return toml.load(stream, _dict=OrderedDict) + + +def load_ini(path): if "ExtendedInterpolation" in globals(): parser = ConfigParser(interpolation=ExtendedInterpolation()) else: parser = ConfigParser() - parser.read_file(stream) + with open_text(path) as stream: + parser.read_file(stream) return {section: OrderedDict(parser.items(section)) for section in parser.sections()} -FORMATS = ("toml", "ini", "json") +FORMAT_LOADERS = {"json": load_json, "toml": load_toml, "ini": load_ini} FORMAT_EXTENSIONS = { "js": "json", "json": "json", @@ -208,11 +226,26 @@ def load_ini(stream): "ini": "ini", "cfg": "ini", } -FORMAT_LOADERS = { - "json": lambda *args: json.load(*args, object_pairs_hook=OrderedDict), - "toml": lambda *args: toml.load(*args, _dict=OrderedDict), - "ini": load_ini, -} + + +def register_format(name, function, override=False, _loaders=FORMAT_LOADERS): + if not override and name in _loaders: + raise ValueError("Format already registered: {}".format(name)) + _loaders[name] = function + + +def register_extension( + alias, + format, + override=False, + _loaders=FORMAT_LOADERS, + _extensions=FORMAT_EXTENSIONS, +): + if format not in _loaders: + raise ValueError("Format does not exists: {}".format(format)) + if not override and alias in _extensions: + raise ValueError("Format extension already registered: {}".format(alias)) + _extensions[alias] = format # Optional dependency yaml @@ -222,13 +255,14 @@ def load_ini(stream): pass else: - def load_yaml(stream): - yaml = YAML(typ="rt") - return yaml.load(stream) + def load_yaml(path): + with open_text(path) as stream: + yaml = YAML(typ="rt") + return yaml.load(stream) - FORMATS = FORMATS + ("yaml",) - FORMAT_EXTENSIONS.update({"yml": "yaml", "yaml": "yaml"}) - FORMAT_LOADERS.update({"yaml": load_yaml}) + register_format("yaml", load_yaml) + register_extension("yaml", format="yaml") + register_extension("yml", format="yaml") # Optional dependency HCL try: @@ -237,12 +271,12 @@ def load_yaml(stream): pass else: - def load_hcl(stream): - return hcl.load(stream) + def load_hcl(path): + with open_text(path) as stream: + return hcl.load(stream) - FORMATS = FORMATS + ("hcl",) - FORMAT_EXTENSIONS.update({"hcl": "hcl"}) - FORMAT_LOADERS.update({"hcl": load_hcl}) + register_format("hcl", load_hcl) + register_extension("hcl", format="hcl") def format_from_path(path): @@ -276,7 +310,11 @@ def cli(): parser = argparse.ArgumentParser(description="One simple way of parsing configs") parser.add_argument("--version", action="version", version=get_version()) parser.add_argument( - "-v", "--verbose", choices=LOG_LEVELS, default="ERROR", help="Logging level default: ERROR" + "-v", + "--verbose", + choices=LOG_LEVELS, + default="ERROR", + help="Logging level default: ERROR", ) subparsers = parser.add_subparsers(title="subcommands", dest="command") show_parser = subparsers.add_parser("show") diff --git a/test_confight.py b/test_confight.py index fb89e0a..fd7d54f 100644 --- a/test_confight.py +++ b/test_confight.py @@ -24,7 +24,7 @@ ) from confight import ( - FORMATS, + FORMAT_LOADERS, find, load, load_app, @@ -32,6 +32,8 @@ load_user_app, merge, parse, + register_extension, + register_format, ) @@ -40,18 +42,30 @@ def examples(tmpdir): return Repository(tmpdir) -FILES = ["basic_file.toml", "basic_file.ini", "basic_file.json", "basic_file.cfg", "basic_file.js"] -if "yaml" in FORMATS: +FILES = [ + "basic_file.toml", + "basic_file.ini", + "basic_file.json", + "basic_file.cfg", + "basic_file.js", +] +if "yaml" in FORMAT_LOADERS: FILES.extend(["basic_file.yaml", "basic_file.yml"]) -if "hcl" in FORMATS: +if "hcl" in FORMAT_LOADERS: FILES.extend(["basic_file.hcl"]) -INVALID_FILES = ["invalid.toml", "invalid.ini", "invalid.json", "invalid.cfg", "invalid.js"] +INVALID_FILES = [ + "invalid.toml", + "invalid.ini", + "invalid.json", + "invalid.cfg", + "invalid.js", +] INVALID_EXTENSIONS = ["bad_ext.ext", "bad_ext.j"] SORTED_FILES = ["00_base.toml", "01_first.json", "AA_second.ini"] -class TestParse(object): +class TestParse: @pytest.mark.parametrize("name", FILES) def test_it_should_detect_format_from_extension(self, name, examples): config = parse(examples.get(name)) @@ -92,7 +106,7 @@ def test_it_should_fail_with_invalid_extensions(self, name, examples): parse(examples.get(name)) -class TestMerge(object): +class TestMerge: def test_it_should_give_priority_to_last_value(self): configs = [ {"key": 1}, @@ -134,7 +148,8 @@ def test_it_should_merge_dicts_recursively(self): result = merge(configs) assert_that( - result, has_entry("section", has_entry("lv1", has_entry("lv2", has_entry("lv3", 3)))) + result, + has_entry("section", has_entry("lv1", has_entry("lv2", has_entry("lv3", 3)))), ) def test_it_should_ignore_scalar_values_given_as_configs(self): @@ -151,7 +166,7 @@ def test_it_should_ignore_scalar_values_given_as_configs(self): assert_that(result, has_entry("section", has_entry("key", 3))) -class TestFind(object): +class TestFind: def test_it_should_load_files_in_order(self, examples): examples.clear() expected_files = sorted(examples.get_many(SORTED_FILES)) @@ -226,7 +241,7 @@ def test_it_should_warn_about_executable_config_files(self, logger, examples): logger.warning.assert_called() -class TestLoad(object): +class TestLoad: def test_it_should_load_and_merge_lists_of_paths(self, examples): paths = sorted(examples.get_many(SORTED_FILES)) @@ -262,7 +277,7 @@ def mymerge(configs): assert_that(config, only_contains(has_key("section"))) -class TestLoadPaths(object): +class TestLoadPaths: def test_it_should_load_from_file_and_directory(self, examples): examples.clear() paths = sorted(examples.get_many(SORTED_FILES)) @@ -302,7 +317,7 @@ def test_merges_must_retain_order(self, examples): assert_that(config["section"].keys(), contains_exactly(*good_data)) -class LoadAppBehaviour(object): +class LoadAppBehaviour: def loaded_paths(self, config): return sorted(config, key=lambda k: config[k]) @@ -506,7 +521,7 @@ def load_app(self, *args, **kwargs): return self.call_config_loader(load_user_app, *args, **kwargs) -class TestCli(object): +class TestCli: def test_it_should_print_help(self): out = subprocess.run([self.bin], stderr=subprocess.PIPE) @@ -569,7 +584,7 @@ def maimed_run(*args, **kwargs): subprocess.run = maimed_run -class Repository(object): +class Repository: def __init__(self, tmpdir): self.tmpdir = tmpdir @@ -696,3 +711,62 @@ def load(self, name): _contents["basic_file_yaml"] = _contents["basic_file.yaml"] _contents["basic_file.yml"] = _contents["basic_file.yaml"] _contents["basic_file_hcl"] = _contents["basic_file.hcl"] + + +def function(path): + return {} + + +def function2(path): + return {} + + +class TestRegister: + def test_it_should_add_a_new_format(self): + register = {} + + register_format("test", function, _loaders=register) + + assert_that(register, has_entry("test", function)) + + def test_it_should_fail_with_existing_formats(self): + register = {"test": function} + + with pytest.raises(ValueError): + register_format("test", function, _loaders=register) + + def test_it_should_override_existing_formats(self): + register = {"test": function} + + register_format("test", function2, override=True, _loaders=register) + + assert_that(register, has_entry("test", function2)) + + def test_it_should_register_new_extensions(self): + loaders = {"test": function} + extensions = {} + + register_extension("t", format="test", _loaders=loaders, _extensions=extensions) + + assert_that(extensions, has_entry("t", "test")) + + def test_it_should_fail_to_register_missing_formats(self): + with pytest.raises(ValueError): + register_extension("t", format="test", _loaders={}, _extensions={}) + + def test_it_should_fail_with_existing_aliases(self): + loaders = {"test": lambda path: {}} + extensions = {"t": "test"} + + with pytest.raises(ValueError): + register_extension("t", format="test", _loaders=loaders, _extensions=extensions) + + def test_it_should_override_existing_aliases(self): + loaders = {"test1": lambda path: {}, "test2": lambda path: {}} + extensions = {"t": "test"} + + register_extension( + "t", format="test2", override=True, _loaders=loaders, _extensions=extensions + ) + + assert_that(extensions, has_entry("t", "test2")) diff --git a/tox.ini b/tox.ini index a4e1117..5b64d77 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py{38,39,310,311}-{basic,yaml,hcl} +envlist = py{38,39,310,311,312}-{basic,yaml,hcl} skip_missing_interpreters=True [testenv]