From cae0f8f0fb3f6e4e90ec8552035064d9a7c4fb9c Mon Sep 17 00:00:00 2001 From: Trevor Gattis Date: Wed, 18 Oct 2017 16:22:27 -0700 Subject: [PATCH 1/5] Add json and csv formatters --- aq/__init__.py | 15 +++-- aq/formatters.py | 62 ++++++++++++++++-- setup.py | 162 +++++++++++++++++++++++------------------------ 3 files changed, 148 insertions(+), 91 deletions(-) diff --git a/aq/__init__.py b/aq/__init__.py index 83ad78f..5d75eff 100644 --- a/aq/__init__.py +++ b/aq/__init__.py @@ -2,7 +2,7 @@ Usage: aq [--profile=] [--region=] [--table-cache-ttl=] [-v] [--debug] - aq [--profile=] [--region=] [--table-cache-ttl=] [-v] [--debug] + aq [--profile=] [--region=] [--table-cache-ttl=] [-v] [--debug] [--format=] Sample queries: aq "select tags->'Name' from ec2_instances" @@ -13,6 +13,8 @@ --region= The region to use. Overrides config/env settings --table-cache-ttl= number of seconds to cache the tables before we update them from AWS again [default: 300] + --format= Choose a table, json, or csv formatter [default: table] + -v, --verbose enable verbose logging --debug enable debug mode """ @@ -25,12 +27,12 @@ from aq.engines import BotoSqliteEngine from aq.errors import QueryError -from aq.formatters import TableFormatter +import aq.formatters from aq.logger import initialize_logger from aq.parsers import SelectParser from aq.prompt import AqPrompt -__version__ = '0.1.1' +__version__ = '0.1.2' QueryResult = namedtuple('QueryResult', ('parsed_query', 'query_metadata', 'columns', 'rows')) @@ -44,7 +46,12 @@ def get_parser(options): def get_formatter(options): - return TableFormatter(options) + if (options.get("--format") == "json"): + return formatters.JsonFormatter(options) + elif (options.get("--format") == "csv"): + return formatters.CsvFormatter(options) + else: + return formatters.TableFormatter(options) def get_prompt(parser, engine, options): diff --git a/aq/formatters.py b/aq/formatters.py index e131021..6ebf04f 100644 --- a/aq/formatters.py +++ b/aq/formatters.py @@ -1,10 +1,60 @@ from tabulate import tabulate - +import io +import json +import csv class TableFormatter(object): - def __init__(self, options=None): - self.options = options if options else {} + def __init__(self, options=None): + self.options = options if options else {} + + @staticmethod + def format(columns, rows): + return tabulate(rows, headers=columns, tablefmt='psql', missingval='NULL') + + +class JsonFormatter(object): + def __init__(self, options=None): + self.options = options if options else {} + + @staticmethod + def format(columns, rows): + rowsData = [] + l = len(columns) + + for row in rows: + r = {} + + for i in range(0, l): + r[columns[i]] = row[i] + + rowsData.append(r) + + + return json.dumps(rowsData) + + +class CsvFormatter(object): + def __init__(self, options=None): + self.options = options if options else {} + + @staticmethod + def format(columns, rows): + l = len(columns) + + # Note: + rowsData = io.BytesIO() + writer = csv.DictWriter(rowsData, fieldnames=columns) + writer.writeheader() + + for row in rows: + r = {} + + for i in range(0, l): + r[columns[i]] = row[i] + + writer.writerow(r) - @staticmethod - def format(columns, rows): - return tabulate(rows, headers=columns, tablefmt='psql', missingval='NULL') + # Force string. + # Python2.7 getvalue() returns string + # Python 3.x returns bytes + return str(rowsData.getvalue()) diff --git a/setup.py b/setup.py index 03c88e8..62b292f 100644 --- a/setup.py +++ b/setup.py @@ -8,87 +8,87 @@ # Get the long description from the README file with open(path.join(here, 'README.rst'), encoding='utf-8') as f: - long_description = f.read() + long_description = f.read() setup( - name='aq', - - # Versions should comply with PEP440. For a discussion on single-sourcing - # the version across setup.py and the project code, see - # https://packaging.python.org/en/latest/single_source_version.html - version='0.1.1', - - description='Query AWS resources with SQL', - long_description=long_description, - - # The project's main homepage. - url='https://github.com/lebinh/aq', - - # Author details - author='Binh Le', - author_email='lebinh.it@gmail.com', - - # Choose your license - license='MIT', - - # See https://pypi.python.org/pypi?%3Aaction=list_classifiers - classifiers=[ - # How mature is this project? Common values are - # 3 - Alpha - # 4 - Beta - # 5 - Production/Stable - 'Development Status :: 3 - Alpha', - - # Indicate who your project is intended for - 'Intended Audience :: Developers', - - # Pick your license as you wish (should match "license" above) - 'License :: OSI Approved :: MIT License', - - # Specify the Python versions you support here. In particular, ensure - # that you indicate whether you support Python 2, Python 3 or both. - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', - 'Programming Language :: Python :: 3.4', - 'Programming Language :: Python :: 3.5', - ], - - # What does your project relate to? - keywords='aws sql query', - - # You can just specify the packages manually here if your project is - # simple. Or you can use find_packages(). - packages=find_packages(exclude=['contrib', 'docs', 'tests']), - - # List run-time dependencies here. These will be installed by pip when - # your project is installed. For an analysis of "install_requires" vs pip's - # requirements files see: - # https://packaging.python.org/en/latest/requirements.html - install_requires=[ - 'boto3', - 'docopt', - 'pyparsing', - 'tabulate', - 'prompt_toolkit', - 'pygments', - 'six', - ], - - # List additional groups of dependencies here (e.g. development - # dependencies). You can install these using the following syntax, - # for example: - # $ pip install -e .[dev,test] - extras_require={ - 'dev': [], - 'test': [], - }, - - entry_points={ - 'console_scripts': [ - 'aq = aq:main' - ] - }, + name='aq', + + # Versions should comply with PEP440. For a discussion on single-sourcing + # the version across setup.py and the project code, see + # https://packaging.python.org/en/latest/single_source_version.html + version='0.1.2', + + description='Query AWS resources with SQL', + long_description=long_description, + + # The project's main homepage. + url='https://github.com/lebinh/aq', + + # Author details + author='Binh Le', + author_email='lebinh.it@gmail.com', + + # Choose your license + license='MIT', + + # See https://pypi.python.org/pypi?%3Aaction=list_classifiers + classifiers=[ + # How mature is this project? Common values are + # 3 - Alpha + # 4 - Beta + # 5 - Production/Stable + 'Development Status :: 3 - Alpha', + + # Indicate who your project is intended for + 'Intended Audience :: Developers', + + # Pick your license as you wish (should match "license" above) + 'License :: OSI Approved :: MIT License', + + # Specify the Python versions you support here. In particular, ensure + # that you indicate whether you support Python 2, Python 3 or both. + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 2.6', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.3', + 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', + ], + + # What does your project relate to? + keywords='aws sql query', + + # You can just specify the packages manually here if your project is + # simple. Or you can use find_packages(). + packages=find_packages(exclude=['contrib', 'docs', 'tests']), + + # List run-time dependencies here. These will be installed by pip when + # your project is installed. For an analysis of "install_requires" vs pip's + # requirements files see: + # https://packaging.python.org/en/latest/requirements.html + install_requires=[ + 'boto3', + 'docopt', + 'pyparsing', + 'tabulate', + 'prompt_toolkit', + 'pygments', + 'six', + ], + + # List additional groups of dependencies here (e.g. development + # dependencies). You can install these using the following syntax, + # for example: + # $ pip install -e .[dev,test] + extras_require={ + 'dev': [], + 'test': [], + }, + + entry_points={ + 'console_scripts': [ + 'aq = aq:main' + ] + }, ) From 9ec4296251ce329929a9d1d144a2c99129de761b Mon Sep 17 00:00:00 2001 From: Trevor Gattis Date: Wed, 18 Oct 2017 16:27:21 -0700 Subject: [PATCH 2/5] Update usage information --- README.rst | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 08cac2f..c606ed5 100644 --- a/README.rst +++ b/README.rst @@ -20,14 +20,21 @@ Usage Usage: aq [--profile=] [--region=] [--table-cache-ttl=] [-v] [--debug] - aq [--profile=] [--region=] [--table-cache-ttl=] [-v] [--debug] + aq [--profile=] [--region=] [--table-cache-ttl=] [-v] [--debug] [--format=] + + Sample queries: + aq "select tags->'Name' from ec2_instances" + aq "select count(*) from us_west_1.ec2_instances" Options: --profile= Use a specific profile from your credential file --region= The region to use. Overrides config/env settings --table-cache-ttl= number of seconds to cache the tables before we update them from AWS again [default: 300] + --format= Choose a table, json, or csv formatter [default: table] + -v, --verbose enable verbose logging + --debug enable debug mode Running ``aq`` without specifying any query will start a REPL to run your queries interactively. From e25b5acba455344878debc864c0cc7f17bdc8b5c Mon Sep 17 00:00:00 2001 From: Trevor Gattis Date: Thu, 19 Oct 2017 09:18:18 -0700 Subject: [PATCH 3/5] Add in yaml support --- README.rst | 2 +- aq/__init__.py | 9 ++++++--- aq/formatters.py | 48 ++++++++++++++++++++++++++++-------------------- setup.py | 1 + 4 files changed, 36 insertions(+), 24 deletions(-) diff --git a/README.rst b/README.rst index c606ed5..4b3db41 100644 --- a/README.rst +++ b/README.rst @@ -31,7 +31,7 @@ Usage --region= The region to use. Overrides config/env settings --table-cache-ttl= number of seconds to cache the tables before we update them from AWS again [default: 300] - --format= Choose a table, json, or csv formatter [default: table] + --format= Choose a table, json, yaml, or csv formatter [default: table] -v, --verbose enable verbose logging --debug enable debug mode diff --git a/aq/__init__.py b/aq/__init__.py index 5d75eff..6ddd83e 100644 --- a/aq/__init__.py +++ b/aq/__init__.py @@ -13,7 +13,7 @@ --region= The region to use. Overrides config/env settings --table-cache-ttl= number of seconds to cache the tables before we update them from AWS again [default: 300] - --format= Choose a table, json, or csv formatter [default: table] + --format= Choose a table, json, yaml, or csv formatter [default: table] -v, --verbose enable verbose logging --debug enable debug mode @@ -46,10 +46,13 @@ def get_parser(options): def get_formatter(options): - if (options.get("--format") == "json"): + format = options.get("--format") + if (format == "json"): return formatters.JsonFormatter(options) - elif (options.get("--format") == "csv"): + elif (format == "csv"): return formatters.CsvFormatter(options) + elif (format == "yaml"): + return formatters.YamlFormatter(options) else: return formatters.TableFormatter(options) diff --git a/aq/formatters.py b/aq/formatters.py index 6ebf04f..aca66e1 100644 --- a/aq/formatters.py +++ b/aq/formatters.py @@ -2,6 +2,7 @@ import io import json import csv +import yaml class TableFormatter(object): def __init__(self, options=None): @@ -12,25 +13,39 @@ def format(columns, rows): return tabulate(rows, headers=columns, tablefmt='psql', missingval='NULL') +# Returns an array of hashes with the columns as the keys +def mixinColumns(columns, rows): + rowsArr = [] + cLen = len(columns) + + for row in rows: + r = {} + + for i in range(0, cLen): + r[columns[i]] = row[i] + + rowsArr.append(r) + + return rowsArr + class JsonFormatter(object): def __init__(self, options=None): self.options = options if options else {} @staticmethod def format(columns, rows): - rowsData = [] - l = len(columns) - - for row in rows: - r = {} - - for i in range(0, l): - r[columns[i]] = row[i] + rowsArr = mixinColumns(columns, rows) + return json.dumps(rowsArr) - rowsData.append(r) +class YamlFormatter(object): + def __init__(self, options=None): + self.options = options if options else {} - return json.dumps(rowsData) + @staticmethod + def format(columns, rows): + rowsArr = mixinColumns(columns, rows) + return yaml.safe_dump(rowsArr, default_flow_style=False) class CsvFormatter(object): @@ -39,20 +54,13 @@ def __init__(self, options=None): @staticmethod def format(columns, rows): - l = len(columns) - - # Note: rowsData = io.BytesIO() writer = csv.DictWriter(rowsData, fieldnames=columns) writer.writeheader() - for row in rows: - r = {} - - for i in range(0, l): - r[columns[i]] = row[i] - - writer.writerow(r) + rowsArr = mixinColumns(columns, rows) + for row in rowsArr: + writer.writerow(row) # Force string. # Python2.7 getvalue() returns string diff --git a/setup.py b/setup.py index 62b292f..60b893a 100644 --- a/setup.py +++ b/setup.py @@ -75,6 +75,7 @@ 'prompt_toolkit', 'pygments', 'six', + 'pyyaml' ], # List additional groups of dependencies here (e.g. development From 1567e3f3400f40a24586acb3a6fdb33ac2ab069c Mon Sep 17 00:00:00 2001 From: Trevor Gattis Date: Thu, 19 Oct 2017 09:23:27 -0700 Subject: [PATCH 4/5] Update Tabs -> spaces --- setup.py | 164 +++++++++++++++++++++++++++---------------------------- 1 file changed, 82 insertions(+), 82 deletions(-) diff --git a/setup.py b/setup.py index 60b893a..96e4f73 100644 --- a/setup.py +++ b/setup.py @@ -8,88 +8,88 @@ # Get the long description from the README file with open(path.join(here, 'README.rst'), encoding='utf-8') as f: - long_description = f.read() + long_description = f.read() setup( - name='aq', - - # Versions should comply with PEP440. For a discussion on single-sourcing - # the version across setup.py and the project code, see - # https://packaging.python.org/en/latest/single_source_version.html - version='0.1.2', - - description='Query AWS resources with SQL', - long_description=long_description, - - # The project's main homepage. - url='https://github.com/lebinh/aq', - - # Author details - author='Binh Le', - author_email='lebinh.it@gmail.com', - - # Choose your license - license='MIT', - - # See https://pypi.python.org/pypi?%3Aaction=list_classifiers - classifiers=[ - # How mature is this project? Common values are - # 3 - Alpha - # 4 - Beta - # 5 - Production/Stable - 'Development Status :: 3 - Alpha', - - # Indicate who your project is intended for - 'Intended Audience :: Developers', - - # Pick your license as you wish (should match "license" above) - 'License :: OSI Approved :: MIT License', - - # Specify the Python versions you support here. In particular, ensure - # that you indicate whether you support Python 2, Python 3 or both. - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', - 'Programming Language :: Python :: 3.4', - 'Programming Language :: Python :: 3.5', - ], - - # What does your project relate to? - keywords='aws sql query', - - # You can just specify the packages manually here if your project is - # simple. Or you can use find_packages(). - packages=find_packages(exclude=['contrib', 'docs', 'tests']), - - # List run-time dependencies here. These will be installed by pip when - # your project is installed. For an analysis of "install_requires" vs pip's - # requirements files see: - # https://packaging.python.org/en/latest/requirements.html - install_requires=[ - 'boto3', - 'docopt', - 'pyparsing', - 'tabulate', - 'prompt_toolkit', - 'pygments', - 'six', - 'pyyaml' - ], - - # List additional groups of dependencies here (e.g. development - # dependencies). You can install these using the following syntax, - # for example: - # $ pip install -e .[dev,test] - extras_require={ - 'dev': [], - 'test': [], - }, - - entry_points={ - 'console_scripts': [ - 'aq = aq:main' - ] - }, + name='aq', + + # Versions should comply with PEP440. For a discussion on single-sourcing + # the version across setup.py and the project code, see + # https://packaging.python.org/en/latest/single_source_version.html + version='0.1.2', + + description='Query AWS resources with SQL', + long_description=long_description, + + # The project's main homepage. + url='https://github.com/lebinh/aq', + + # Author details + author='Binh Le', + author_email='lebinh.it@gmail.com', + + # Choose your license + license='MIT', + + # See https://pypi.python.org/pypi?%3Aaction=list_classifiers + classifiers=[ + # How mature is this project? Common values are + # 3 - Alpha + # 4 - Beta + # 5 - Production/Stable + 'Development Status :: 3 - Alpha', + + # Indicate who your project is intended for + 'Intended Audience :: Developers', + + # Pick your license as you wish (should match "license" above) + 'License :: OSI Approved :: MIT License', + + # Specify the Python versions you support here. In particular, ensure + # that you indicate whether you support Python 2, Python 3 or both. + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 2.6', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.3', + 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', + ], + + # What does your project relate to? + keywords='aws sql query', + + # You can just specify the packages manually here if your project is + # simple. Or you can use find_packages(). + packages=find_packages(exclude=['contrib', 'docs', 'tests']), + + # List run-time dependencies here. These will be installed by pip when + # your project is installed. For an analysis of "install_requires" vs pip's + # requirements files see: + # https://packaging.python.org/en/latest/requirements.html + install_requires=[ + 'boto3', + 'docopt', + 'pyparsing', + 'tabulate', + 'prompt_toolkit', + 'pygments', + 'six', + 'pyyaml' + ], + + # List additional groups of dependencies here (e.g. development + # dependencies). You can install these using the following syntax, + # for example: + # $ pip install -e .[dev,test] + extras_require={ + 'dev': [], + 'test': [], + }, + + entry_points={ + 'console_scripts': [ + 'aq = aq:main' + ] + }, ) From bb07a159dce479051dc8fc3fd0d0311c4b8782be Mon Sep 17 00:00:00 2001 From: Trevor Gattis Date: Thu, 19 Oct 2017 09:25:00 -0700 Subject: [PATCH 5/5] Update Tabs -> spaces --- aq/formatters.py | 84 ++++++++++++++++++++++++------------------------ 1 file changed, 42 insertions(+), 42 deletions(-) diff --git a/aq/formatters.py b/aq/formatters.py index aca66e1..7f3681d 100644 --- a/aq/formatters.py +++ b/aq/formatters.py @@ -5,64 +5,64 @@ import yaml class TableFormatter(object): - def __init__(self, options=None): - self.options = options if options else {} + def __init__(self, options=None): + self.options = options if options else {} - @staticmethod - def format(columns, rows): - return tabulate(rows, headers=columns, tablefmt='psql', missingval='NULL') + @staticmethod + def format(columns, rows): + return tabulate(rows, headers=columns, tablefmt='psql', missingval='NULL') # Returns an array of hashes with the columns as the keys def mixinColumns(columns, rows): - rowsArr = [] - cLen = len(columns) + rowsArr = [] + cLen = len(columns) - for row in rows: - r = {} + for row in rows: + r = {} - for i in range(0, cLen): - r[columns[i]] = row[i] + for i in range(0, cLen): + r[columns[i]] = row[i] - rowsArr.append(r) + rowsArr.append(r) - return rowsArr + return rowsArr class JsonFormatter(object): - def __init__(self, options=None): - self.options = options if options else {} + def __init__(self, options=None): + self.options = options if options else {} - @staticmethod - def format(columns, rows): - rowsArr = mixinColumns(columns, rows) - return json.dumps(rowsArr) + @staticmethod + def format(columns, rows): + rowsArr = mixinColumns(columns, rows) + return json.dumps(rowsArr) class YamlFormatter(object): - def __init__(self, options=None): - self.options = options if options else {} + def __init__(self, options=None): + self.options = options if options else {} - @staticmethod - def format(columns, rows): - rowsArr = mixinColumns(columns, rows) - return yaml.safe_dump(rowsArr, default_flow_style=False) + @staticmethod + def format(columns, rows): + rowsArr = mixinColumns(columns, rows) + return yaml.safe_dump(rowsArr, default_flow_style=False ) class CsvFormatter(object): - def __init__(self, options=None): - self.options = options if options else {} - - @staticmethod - def format(columns, rows): - rowsData = io.BytesIO() - writer = csv.DictWriter(rowsData, fieldnames=columns) - writer.writeheader() - - rowsArr = mixinColumns(columns, rows) - for row in rowsArr: - writer.writerow(row) - - # Force string. - # Python2.7 getvalue() returns string - # Python 3.x returns bytes - return str(rowsData.getvalue()) + def __init__(self, options=None): + self.options = options if options else {} + + @staticmethod + def format(columns, rows): + rowsData = io.BytesIO() + writer = csv.DictWriter(rowsData, fieldnames=columns) + writer.writeheader() + + rowsArr = mixinColumns(columns, rows) + for row in rowsArr: + writer.writerow(row) + + # Force string. + # Python2.7 getvalue() returns string + # Python 3.x returns bytes + return str(rowsData.getvalue())