From 85fa788add6f137da18f4922b15da555f96fb6c1 Mon Sep 17 00:00:00 2001 From: pkimrapdev Date: Fri, 4 Feb 2022 16:36:51 -0500 Subject: [PATCH] Initial commit of the rapdev host tag puller. Contains .env.template file. --- host_tag_puller/.env.template | 3 ++ host_tag_puller/.gitignore | 2 + host_tag_puller/README.md | 39 +++++++++++++++ host_tag_puller/host_tag.py | 90 +++++++++++++++++++++++++++++++++++ 4 files changed, 134 insertions(+) create mode 100644 host_tag_puller/.env.template create mode 100644 host_tag_puller/.gitignore create mode 100644 host_tag_puller/README.md create mode 100644 host_tag_puller/host_tag.py diff --git a/host_tag_puller/.env.template b/host_tag_puller/.env.template new file mode 100644 index 0000000..4fa04ad --- /dev/null +++ b/host_tag_puller/.env.template @@ -0,0 +1,3 @@ +DD_API_KEY="" +DD_APP_KEY="" +DD_CUSTOMER="" \ No newline at end of file diff --git a/host_tag_puller/.gitignore b/host_tag_puller/.gitignore new file mode 100644 index 0000000..43b6d21 --- /dev/null +++ b/host_tag_puller/.gitignore @@ -0,0 +1,2 @@ +.env +out diff --git a/host_tag_puller/README.md b/host_tag_puller/README.md new file mode 100644 index 0000000..6c6689c --- /dev/null +++ b/host_tag_puller/README.md @@ -0,0 +1,39 @@ +# RapDev Tag + Key Puller + +## Introduction + +The goal of the tag puller is to assist teams with auditing all of the Datadog tags across their hosts in their Datadog instance. +It works by searching through all hosts in the provided Datadog account and generating a report with a list of all tags in the datadog account. + +## Pre-reqs + +- [Python3.x Environment](https://realpython.com/installing-python/) + +You will need to install the following Python librarys: + +- [Datadog Python library](https://datadogpy.readthedocs.io/en/latest/) +- [python-dotenv Library](https://pypi.org/project/python-dotenv/) + + +## Configuration + +### Environment File + +Begin by providing a `.env` in the same directory you are running the `host_tag.py` from. The `.env` file should contain the following information: + + DD_API_KEY="" + DD_APP_KEY="" + DD_CUSTOMER="" + +Provided in the repository is a skeleton `.env.template` that can be renamed to `.env` with the appropriate keys and configurations. +## Running the Script + +Using your `python3.X` command (make sure you use your installed version), you can run the script in one simple command from within the directory: + + # Only run one of these based on your python version: + + python3 host_tag.py + python3.7 host_tag.py + python3.8 host_tag.py + python3.9 host_tag.py + \ No newline at end of file diff --git a/host_tag_puller/host_tag.py b/host_tag_puller/host_tag.py new file mode 100644 index 0000000..16a79e3 --- /dev/null +++ b/host_tag_puller/host_tag.py @@ -0,0 +1,90 @@ +from datadog import initialize, api +from dotenv import load_dotenv +import os + +def get_tags(dd_api_key, dd_app_key,dd_customer,outpath): + """Gets all tags in a DD account then outputs keys and tags (key:value pairs) to a flatfile + + :param string dd_api_key: Datadog api key used to authenticate to Host endpoint + :param string dd_app_key: Datadog app key used to authenticate to Host endpoint + :param boolean eu_customer: True if customer is in EU, else false + :return: + """ + options = { + "api_key": dd_api_key, + "app_key": dd_app_key + } + + outdir = outpath + customer = dd_customer + + # Initialize Datadog module for API usage + initialize(**options) + + # Grabbing the hosts list + total_hosts = api.Hosts.totals().get('total_active') + + # Find all tags in each host and put them in a single list + total_tag_list=[] + n = 0 + while n < total_hosts: + response = api.Hosts.search(start=n, count=1000) + for host in response.get("host_list",""): + for tags in host.get("tags_by_source","").values(): + total_tag_list.append(tags) + n+=1000 + + # Join all lists to single list + total_tag_list = sum(total_tag_list,[]) + + # Track each key and value to write to the report + lines_seen = set() + final_value_list = open("{}/{}_vals".format(outdir, customer), "w") + for item in total_tag_list: + if "host" in item: + pass + else: + if item not in lines_seen: + final_value_list.write(item + "\n") + lines_seen.add(item) + + # Unzip key value pairs to pull just the keys + file_key_list = open("{}/{}_keys".format(outdir, customer), "w") + keys_seen = set() + for item in lines_seen: + if ":" in item: + key = item.split(":", 1)[0] + else: + key = item + if key not in keys_seen: + file_key_list.write(key + "\n") + keys_seen.add(key) + + final_value_list.close() + file_key_list.close() + +def main(): + # Get environment variables from .env + if "DD_API_KEY" in os.environ and "DD_APP_KEY" in os.environ: + dd_api_key = os.environ.get('DD_API_KEY') + dd_app_key = os.environ.get('DD_APP_KEY') + dd_customer = os.environ.get('DD_CUSTOMER') + else: + raise Exception("Datadog API and APP keys are required. Please provide both via environment variables.") + + # Create output directory + outpath = os.path.join(os.getcwd(),"out") + if not os.path.exists(outpath): + os.mkdir(outpath) + + # Pull tags and generate report + get_tags(dd_api_key,dd_app_key, dd_customer,outpath) + +if __name__ == '__main__': + # loads environment variables + load_dotenv() + # Get run mode, if not set defaults to test + global RUN_MODE + RUN_MODE = os.environ.get("RUN_MODE", "test") + + main() \ No newline at end of file