From a1be59a6bf6d750e04cf8d274c1faa172d23c2c8 Mon Sep 17 00:00:00 2001
From: RodneyCodess <160431998+RodneyCodess@users.noreply.github.com>
Date: Thu, 30 May 2024 10:33:50 -0400
Subject: [PATCH 01/72] Add citation for the DAF paper.

---
 README.md | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 0c9263a9..9aca07db 100644
--- a/README.md
+++ b/README.md
@@ -55,7 +55,13 @@ For support, please see [this page](https://open.xdmod.org/support.html). If you
 ## License
 `xdmod-data` is released under the GNU Lesser General Public License ("LGPL") Version 3.0. See the [LICENSE](LICENSE) file for details.
 
-## Reference
+## References
+
+When referencing the Data Analytics Framework for XDMoD, please cite the following publication:
+
+> Weeden, A., White, J.P., DeLeon, R.L., Rathsam, R., Simakov, N.A., Saeli, C., and Furlani, T.R. The Data Analytics Framework for XDMoD. _SN COMPUT. SCI._ 5, 462 (2024). https://doi.org/10.1007/s42979-024-02789-2
+
 When referencing XDMoD, please cite the following publication:
 
-> Jeffrey T. Palmer, Steven M. Gallo, Thomas R. Furlani, Matthew D. Jones, Robert L. DeLeon, Joseph P. White, Nikolay Simakov, Abani K. Patra, Jeanette Sperhac, Thomas Yearke, Ryan Rathsam, Martins Innus, Cynthia D. Cornelius, James C. Browne, William L. Barth, Richard T. Evans, "Open XDMoD: A Tool for the Comprehensive Management of High-Performance Computing Resources", *Computing in Science & Engineering*, Vol 17, Issue 4, 2015, pp. 52-62. DOI:10.1109/MCSE.2015.68
+> Jeffrey T. Palmer, Steven M. Gallo, Thomas R. Furlani, Matthew D. Jones, Robert L. DeLeon, Joseph P. White, Nikolay Simakov, Abani K. Patra, Jeanette Sperhac, Thomas Yearke, Ryan Rathsam, Martins Innus, Cynthia D. Cornelius, James C. Browne, William L. Barth, Richard T. Evans, "Open XDMoD: A Tool for the Comprehensive Management of High-Performance Computing Resources", *Computing in Science & Engineering*, Vol 17, Issue 4, 2015, pp. 52-62. DOI:[10.1109/MCSE.2015.68](https://doi.org/10.1109/MCSE.2015.68)
+

From 82edbdc6a98d7fcdf558c5baf85c0ddb340b921d Mon Sep 17 00:00:00 2001
From: RodneyCodess <160431998+RodneyCodess@users.noreply.github.com>
Date: Wed, 5 Jun 2024 14:12:26 -0400
Subject: [PATCH 02/72] Fix bug with trailing slashes in `xdmod_host`. (#24)

Co-authored-by: Aaron Weeden <31246768+aaronweeden@users.noreply.github.com>
---
 CHANGELOG.md                                  |  1 +
 .../test_datawarehouse_integration.py         | 43 ++++++++++++-------
 tests/unit/test_datawarehouse_unit.py         |  6 ++-
 xdmod_data/_http_requester.py                 |  2 +
 4 files changed, 35 insertions(+), 17 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index aada8df2..9c0a77b4 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,6 +5,7 @@
 - Remove limit on number of results returned from `get_filter_values()` ([\#21](https://github.com/ubccr/xdmod-data/pull/21)).
 - Add a "Feedback / Feature Requests" section to the README ([\#22](https://github.com/ubccr/xdmod-notebooks/pull/22)).
 - Improve performance of validation of filters and raw fields ([\#18](https://github.com/ubccr/xdmod-data/pull/18)).
+- Fix bug with trailing slashes in `xdmod_host` ([\#24](https://github.com/ubccr/xdmod-data/pull/24)).
 
 ## v1.0.0 (2023-07-21)
 - Initial release.
diff --git a/tests/integration/test_datawarehouse_integration.py b/tests/integration/test_datawarehouse_integration.py
index 9e296fec..1688b5c0 100644
--- a/tests/integration/test_datawarehouse_integration.py
+++ b/tests/integration/test_datawarehouse_integration.py
@@ -35,7 +35,7 @@
 VALID_DATE = '2020-01-01'
 VALID_DIMENSION = 'Resource'
 VALID_VALUES = {
-  'duration': 'Previous month',
+  'duration': 'Yesterday',
   'realm': 'Jobs',
   'metric': 'CPU Hours: Total',
   'dimension': VALID_DIMENSION,
@@ -44,7 +44,7 @@
   'aggregation_unit': 'Auto',
   'parameter': 'duration',
   'fields': ['Nodes'],
-  'show_progress': True,
+  'show_progress': False,
 }
 KEY_ERROR_TEST_VALUES_AND_MATCHES = {
     'duration': (INVALID_STR, 'Invalid value for `duration`'),
@@ -92,8 +92,16 @@
                 method + ':end_date',
             ]
             date_malformed_test_params += [
-                (method, 'start_date', {'duration': (INVALID_STR, VALID_DATE)}),
-                (method, 'end_date', {'duration': (VALID_DATE, INVALID_STR)}),
+                (
+                    method,
+                    'start_date',
+                    {'duration': (INVALID_STR, VALID_DATE)},
+                ),
+                (
+                    method,
+                    'end_date',
+                    {'duration': (VALID_DATE, INVALID_STR)},
+                ),
             ]
             value_error_test_methods += [method]
     if 'filters' in METHOD_PARAMS[method]:
@@ -107,8 +115,11 @@
 
 
 @pytest.fixture(scope='module')
-def dw_methods():
-    with DataWarehouse(VALID_XDMOD_URL) as dw:
+def dw_methods(request):
+    xdmod_host = VALID_XDMOD_URL
+    if hasattr(request, 'param'):
+        xdmod_host = request.param
+    with DataWarehouse(xdmod_host) as dw:
         yield __get_dw_methods(dw)
 
 
@@ -152,16 +163,7 @@ def test_KeyError(dw_methods, method, params, match):
 
 @pytest.mark.parametrize(
     'method',
-    [
-        'get_data',
-        'get_raw_data',
-        'describe_realms',
-        'describe_metrics',
-        'describe_dimensions',
-        'get_filter_values',
-        'describe_raw_realms',
-        'describe_raw_fields',
-    ],
+    list(METHOD_PARAMS.keys()),
 )
 def test_RuntimeError_outside_context(
     dw_methods_outside_runtime_context,
@@ -439,3 +441,12 @@ def test_case_insensitive(dw_methods, method, param, value1, value2):
     data1 = __run_method(dw_methods, method, {param: value1})
     data2 = __run_method(dw_methods, method, {param: value2})
     assert data1.equals(data2)
+
+
+@pytest.mark.parametrize(
+    'dw_methods,method',
+    [(VALID_XDMOD_URL + '/', method) for method in list(METHOD_PARAMS.keys())],
+    indirect=['dw_methods'],
+)
+def test_trailing_slashes(dw_methods, method):
+    __run_method(dw_methods, method)
diff --git a/tests/unit/test_datawarehouse_unit.py b/tests/unit/test_datawarehouse_unit.py
index 7af2d989..95b0bf95 100644
--- a/tests/unit/test_datawarehouse_unit.py
+++ b/tests/unit/test_datawarehouse_unit.py
@@ -10,7 +10,11 @@
 
 @pytest.fixture(scope='module', autouse=True)
 def set_environ():
-    token = os.environ['XDMOD_API_TOKEN'] if 'XDMOD_API_TOKEN' in os.environ else ''
+    token = (
+        os.environ['XDMOD_API_TOKEN']
+        if 'XDMOD_API_TOKEN' in os.environ
+        else ''
+    )
     os.environ['XDMOD_API_TOKEN'] = INVALID_STR
     yield
     os.environ['XDMOD_API_TOKEN'] = token
diff --git a/xdmod_data/_http_requester.py b/xdmod_data/_http_requester.py
index d90e3af7..cc089003 100644
--- a/xdmod_data/_http_requester.py
+++ b/xdmod_data/_http_requester.py
@@ -1,5 +1,6 @@
 import json
 import os
+import re
 import requests
 from urllib.parse import urlencode
 import xdmod_data._validator as _validator
@@ -10,6 +11,7 @@ class _HttpRequester:
     def __init__(self, xdmod_host):
         self.__in_runtime_context = False
         _validator._assert_str('xdmod_host', xdmod_host)
+        xdmod_host = re.sub('/+$', '', xdmod_host)
         self.__xdmod_host = xdmod_host
         try:
             self.__api_token = os.environ['XDMOD_API_TOKEN']

From ac99530c90fcaa833473e364fb76e394c6f662e5 Mon Sep 17 00:00:00 2001
From: Aaron Weeden <31246768+aaronweeden@users.noreply.github.com>
Date: Wed, 5 Jun 2024 14:30:24 -0400
Subject: [PATCH 03/72] Use streaming for raw data requests. (#19)

---
 CHANGELOG.md                  |  1 +
 xdmod_data/_http_requester.py | 80 +++++++++++++++++++++++++++--------
 2 files changed, 63 insertions(+), 18 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 9c0a77b4..52c4c3bc 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -6,6 +6,7 @@
 - Add a "Feedback / Feature Requests" section to the README ([\#22](https://github.com/ubccr/xdmod-notebooks/pull/22)).
 - Improve performance of validation of filters and raw fields ([\#18](https://github.com/ubccr/xdmod-data/pull/18)).
 - Fix bug with trailing slashes in `xdmod_host` ([\#24](https://github.com/ubccr/xdmod-data/pull/24)).
+- Use streaming for raw data requests ([\#19](https://github.com/ubccr/xdmod-data/pull/19)).
 
 ## v1.0.0 (2023-07-21)
 - Initial release.
diff --git a/xdmod_data/_http_requester.py b/xdmod_data/_http_requester.py
index cc089003..ad5bc939 100644
--- a/xdmod_data/_http_requester.py
+++ b/xdmod_data/_http_requester.py
@@ -44,24 +44,55 @@ def _request_data(self, params):
 
     def _request_raw_data(self, params):
         url_params = self.__get_raw_data_url_params(params)
+        # Once XDMoD 10.5 is no longer supported, there will be no need to call
+        # __get_raw_data_limit(), and the if/else statement below will not be
+        # necessary — only the body of the 'if' branch will be needed.
         limit = self.__get_raw_data_limit()
         data = []
-        num_rows = limit
-        offset = 0
-        while num_rows == limit:
-            response = self._request_json(
-                path='/rest/v1/warehouse/raw-data?' + url_params
-                + '&offset=' + str(offset)
+        if limit == 'NA':
+            response_iter_lines = self.__request(
+                path='/rest/v1/warehouse/raw-data?' + url_params,
+                post_fields=None,
+                stream=True,
             )
-            partial_data = response['data']
-            data += partial_data
+            i = 0
+            for line in response_iter_lines:
+                line_text = line.decode('utf-8').replace('\x1e', '')
+                line_json = json.loads(line_text)
+                if i == 0:
+                    response = {'fields': line_json}
+                else:
+                    data.append(line_json)
+                    if params['show_progress']:
+                        progress_msg = (
+                            'Got ' + str(i) + ' row' + ('' if i == 1 else 's')
+                            + '...'
+                        )
+                        print(progress_msg, end='\r')
+                i += 1
             if params['show_progress']:
-                progress_msg = 'Got ' + str(len(data)) + ' rows...'
-                print(progress_msg, end='\r')
-            num_rows = len(partial_data)
-            offset += limit
-        if params['show_progress']:
-            print(progress_msg + 'DONE')
+                print(progress_msg + 'DONE')
+        else:
+            num_rows = limit
+            offset = 0
+            while num_rows == limit:
+                response = self._request_json(
+                    path='/rest/v1/warehouse/raw-data?' + url_params
+                    + '&offset=' + str(offset)
+                )
+                partial_data = response['data']
+                data += partial_data
+                if params['show_progress']:
+                    progress_msg = (
+                        'Got ' + str(len(data)) + ' row'
+                        + ('' if len(data) == 1 else 's')
+                        + '...'
+                    )
+                    print(progress_msg, end='\r')
+                num_rows = len(partial_data)
+                offset += limit
+            if params['show_progress']:
+                print(progress_msg + 'DONE')
         return (data, response['fields'])
 
     def _request_filter_values(self, realm_id, dimension_id):
@@ -98,7 +129,7 @@ def __assert_connection_to_xdmod_host(self):
                 + '\': ' + str(e)
             ) from None
 
-    def __request(self, path='', post_fields=None):
+    def __request(self, path='', post_fields=None, stream=False):
         _validator._assert_runtime_context(self.__in_runtime_context)
         url = self.__xdmod_host + path
         if post_fields:
@@ -126,7 +157,10 @@ def __request(self, path='', post_fields=None):
             raise RuntimeError(
                 'Error ' + str(response.status_code) + msg
             ) from None
-        return response.text
+        if stream:
+            return response.iter_lines()
+        else:
+            return response.text
 
     def __get_data_post_fields(self, params):
         post_fields = {
@@ -161,8 +195,18 @@ def __get_raw_data_url_params(self, params):
                 )
         return urlencode(results)
 
+    # Once XDMoD 10.5 is no longer supported, there will be no need for this
+    # method.
     def __get_raw_data_limit(self):
         if self.__raw_data_limit is None:
-            response = self._request_json('/rest/v1/warehouse/raw-data/limit')
-            self.__raw_data_limit = int(response['data'])
+            try:
+                response = self._request_json(
+                    '/rest/v1/warehouse/raw-data/limit'
+                )
+                self.__raw_data_limit = int(response['data'])
+            except RuntimeError as e:
+                if '404' in str(e):
+                    self.__raw_data_limit = 'NA'
+                else:
+                    raise
         return self.__raw_data_limit

From ca445dd1042291d89ae7f02e1be79ac078a9ae7d Mon Sep 17 00:00:00 2001
From: RodneyCodess <160431998+RodneyCodess@users.noreply.github.com>
Date: Fri, 7 Jun 2024 10:35:49 -0400
Subject: [PATCH 04/72] Specify minimum version numbers for dependencies. (#25)

---
 .github/workflows/python-package.yml  |  8 +++++++-
 CHANGELOG.md                          |  1 +
 docs/developing.md                    | 11 +++++++++--
 setup.cfg                             | 10 +++++-----
 tests/unit/test_datawarehouse_unit.py | 12 ++++++++++--
 5 files changed, 32 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml
index 3f9ee2bc..5718eea3 100644
--- a/.github/workflows/python-package.yml
+++ b/.github/workflows/python-package.yml
@@ -16,7 +16,13 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.10"]
+        python-version: [
+          "3.12",
+          "3.11",
+          "3.10",
+          "3.9",
+          "3.8",
+        ]
 
     steps:
     - uses: actions/checkout@v3
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 52c4c3bc..238d4ac2 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -7,6 +7,7 @@
 - Improve performance of validation of filters and raw fields ([\#18](https://github.com/ubccr/xdmod-data/pull/18)).
 - Fix bug with trailing slashes in `xdmod_host` ([\#24](https://github.com/ubccr/xdmod-data/pull/24)).
 - Use streaming for raw data requests ([\#19](https://github.com/ubccr/xdmod-data/pull/19)).
+- Specify minimum version numbers for dependencies ([\#25](https://github.com/ubccr/xdmod-data/pull/25)). 
 
 ## v1.0.0 (2023-07-21)
 - Initial release.
diff --git a/docs/developing.md b/docs/developing.md
index 23045149..27488560 100644
--- a/docs/developing.md
+++ b/docs/developing.md
@@ -20,11 +20,18 @@
     XDMOD_API_TOKEN=<token>
     ```
 1. Change directories to your local development copy of `xdmod-data`.
-1. Run the following command:
+1. Run the following command and make sure all the tests pass:
     ```
     (env) $ pytest -vvs -o log_cli=true tests/ 
     ```
-
+1. Downgrade to the minimum version of the dependencies. Replace the version numbers below with the values from `setup.cfg`.
+    ```
+    (env) $ python3 -m pip install --force-reinstall numpy==1.23.0 pandas==1.5.0 plotly==5.8.0 requests==2.19.0
+    ```
+1. Run the following command again and make sure all the tests pass (Deprecation warnings in `urllib3` are Ok).
+    ```
+    (env) $ pytest -vvs -o log_cli=true tests/
+    ```
 # Developing a new version
 1. Make sure the version number is updated in `xdmod_data/__version__.py`.
 1. Create a Pull Request for the new version.
diff --git a/setup.cfg b/setup.cfg
index 0333c62a..9fa7fab0 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -12,9 +12,9 @@ project_urls =
 
 [options]
 packages = find:
-python_requires = >=3
+python_requires = >=3.8
 install_requires =
-    numpy
-    pandas
-    plotly
-    requests
+    numpy >= 1.23.0
+    pandas >= 1.5.0
+    plotly >= 5.8.0
+    requests >= 2.19.0
diff --git a/tests/unit/test_datawarehouse_unit.py b/tests/unit/test_datawarehouse_unit.py
index 95b0bf95..d73b04df 100644
--- a/tests/unit/test_datawarehouse_unit.py
+++ b/tests/unit/test_datawarehouse_unit.py
@@ -38,8 +38,16 @@ def test___init___KeyError():
 
 def test___enter___RuntimeError_xdmod_host_malformed():
     with pytest.raises(
-        requests.exceptions.InvalidURL,
-        match=r'Invalid URL \'.*\': No host supplied'
+        (
+            requests.exceptions.InvalidURL,
+            requests.exceptions.MissingSchema
+        ),
+        match=(
+            r'(Invalid URL \'.*\': No host supplied|'
+            + r'Invalid URL \'https:\?Bearer=' + INVALID_STR + '\': '
+            + r'No schema supplied. Perhaps you meant http://https:\?Bearer='
+            + INVALID_STR + r'\?)'
+        )
     ):
         with DataWarehouse('https://'):
             pass

From eb590f97097b918af9828ccaa60eaff57d6c4ea1 Mon Sep 17 00:00:00 2001
From: Aaron Weeden <aaronwee@buffalo.edu>
Date: Mon, 27 Nov 2023 15:54:06 -0500
Subject: [PATCH 05/72] Document compatibility with different versions of Open
 XDMoD.

---
 README.md | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 9aca07db..da1d4659 100644
--- a/README.md
+++ b/README.md
@@ -1,5 +1,5 @@
 # xdmod-data
-As part of the [XDMoD](https://open.xdmod.org) Data Analytics Framework, this Python package provides API access to the data warehouse of an instance of XDMoD version ≥10.5.
+As part of the Data Analytics Framework for [XDMoD](https://open.xdmod.org), this Python package provides API access to the data warehouse of instances of Open XDMoD.
 
 The package can be installed from PyPI via `pip install xdmod-data`.
 
@@ -7,6 +7,14 @@ It has dependencies on [NumPy](https://pypi.org/project/numpy/), [Pandas](https:
 
 Example usage is documented through Jupyter notebooks in the [xdmod-notebooks](https://github.com/ubccr/xdmod-notebooks) repository.
 
+## Compatibility with Open XDMoD
+Specific versions of this `xdmod-data` API are compatible with specific versions of Open XDMoD as indicated in the table below.
+
+| `xdmod-data` version | Open XDMoD versions |
+| -------------------- | ------------------- |
+| 1.0.1                | 11.0.x, 10.5.x      |
+| 1.0.0                | 10.5.x              |
+
 ## API Token Access
 Use of the Data Analytics Framework requires an API token. To obtain an API token, follow the steps below to obtain an API token from the XDMoD portal.
 

From 4c475abaea5dfa39dd3f05dce46f37dce6da2ce4 Mon Sep 17 00:00:00 2001
From: Rodney Garnett <garnettr35@gmail.com>
Date: Fri, 7 Jun 2024 14:26:23 -0400
Subject: [PATCH 06/72] prepare to release version 1.0.1

---
 CHANGELOG.md              |  2 +-
 docs/developing.md        | 56 ++++++++++++++++++++++++++-------------
 setup.cfg                 |  2 +-
 xdmod_data/__version__.py |  2 +-
 4 files changed, 40 insertions(+), 22 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 238d4ac2..a953b917 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -9,5 +9,5 @@
 - Use streaming for raw data requests ([\#19](https://github.com/ubccr/xdmod-data/pull/19)).
 - Specify minimum version numbers for dependencies ([\#25](https://github.com/ubccr/xdmod-data/pull/25)). 
 
-## v1.0.0 (2023-07-21)
+## v1.0.1 (2024-06-XX)
 - Initial release.
diff --git a/docs/developing.md b/docs/developing.md
index 27488560..66627fa4 100644
--- a/docs/developing.md
+++ b/docs/developing.md
@@ -1,17 +1,17 @@
 # Testing the code
 1. Start up a virtual environment, e.g.:
+    ```                                            
+    python3 -m venv ~/xdmod-data-test-env                                             
+    source ~/xdmod-data-test-env/bin/activate
     ```
-    $ env_dir=~/xdmod-data-test-env                                            
-    $ python3 -m venv ${env_dir}                                                
-    $ source ${env_dir}/bin/activate
-    ```
+    Your command prompt should now start with `(xdmod-data-test-env)`.
 1. Install your local development copy of `xdmod-data` in editable mode:
     ```
-    (env) $ python3 -m pip install --force-reinstall -e /path/to/your/xdmod-data
+    python3 -m pip install --force-reinstall -e /path/to/your/xdmod-data
     ```
 1. Install `python-dotenv` and `pytest`:
     ```
-    (env) $ python3 -m pip install --upgrade python-dotenv pytest
+    python3 -m pip install --upgrade python-dotenv pytest
     ```
 1. Create an empty file in your home directory at `~/.xdmod-data-token` with permissions set to 600.
 1. With an [https://xdmod.access-ci.org](ACCESS XDMoD) account with "User" as the Top Role, create an API token if you do not already have one (sign in and click My Profile -> API Token).
@@ -22,39 +22,57 @@
 1. Change directories to your local development copy of `xdmod-data`.
 1. Run the following command and make sure all the tests pass:
     ```
-    (env) $ pytest -vvs -o log_cli=true tests/ 
+    pytest -vvs -o log_cli=true tests/ 
     ```
 1. Downgrade to the minimum version of the dependencies. Replace the version numbers below with the values from `setup.cfg`.
     ```
-    (env) $ python3 -m pip install --force-reinstall numpy==1.23.0 pandas==1.5.0 plotly==5.8.0 requests==2.19.0
+    python3 -m pip install --force-reinstall numpy==1.23.0 pandas==1.5.0 plotly==5.8.0 requests==2.19.0
     ```
 1. Run the following command again and make sure all the tests pass (Deprecation warnings in `urllib3` are Ok).
     ```
-    (env) $ pytest -vvs -o log_cli=true tests/
+    pytest -vvs -o log_cli=true tests/
     ```
-# Developing a new version
+    
+# Preparing a new version for release
 1. Make sure the version number is updated in `xdmod_data/__version__.py`.
+1. Update the Open XDMoD compatibility matrix in the `README.md`
+1. Update CHANGELOG.md to change "Main development branch" to v1.0.1 (2024-06-XX)
 1. Create a Pull Request for the new version.
+1.Do not merge the Pull Request until following the instructions below for distributing the new version.
 
 # Distributing the new version to PyPI
-After the Pull Request is approved, follow these steps in the `xdmod-data` root directory:
+After the Pull Request is approved (but not merged), follow these steps in the `xdmod-data` root directory:
 1. Start up a virtual environment, e.g.:
     ```
-    $ env_dir=~/xdmod-data-build-env
-    $ python3 -m venv ${env_dir}
-    $ source ${env_dir}/bin/activate
+    python3 -m venv ~/xdmod-data-build-env
+    source ~/xdmod-data-build-env/bin/activate
     ```
+    Your command prompt should now start with `(xdmod-data-build-env)`.
 1. Make sure the required packages are installed:
     ```
-    (env) $ python3 -m pip install --upgrade pip build setuptools twine
+    python3 -m pip install --upgrade pip build setuptools twine
     ```
 1. Build the built distribution:
     ```
-    (env) $ python3 -m build --wheel
+    python3 -m build --wheel
+    ```
+1. Validate `setup.cfg`, e.g., for version 1.0.0-beta1:
+    ```
+    version=1.0.0b1
+    twine check dist/xdmod_data-${version}-py3-none-any.whl
+    ```
+    Make sure you recieve `PASSED`.
+1. Upload the built distribution to TestPyPI:
+    ```
+    twine upload --repository testpypi dist/xdmod_data-${version}-py3-none-any.whl
     ```
-1. Upload the built distribution to PyPI, e.g., for version 1.0.0-beta1:
+    Enter your TestPyPI API token.
+1.Go to https://testpypi.org/project/xdmod-data and confirm that everything looks right.
+1. Upload the built distribution to PyPI:
     ```
-    (env) $ version=1.0.0b1
-    (env) $ twine upload dist/xdmod_data-${version}-py3-none-any.whl
+    twine upload dist/xdmod_data-${version}-py3-none-any.whl
     ```
     Enter your PyPI username and password.
+1. Go to https://pypi.org/project/xdmod-data and confirm the new version is the latest release.
+1. In the Pull Request, update the date of the release in `CHANGELOG.md`.
+1. Merge the Pull Request.
diff --git a/setup.cfg b/setup.cfg
index 9fa7fab0..0b224c44 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -4,7 +4,7 @@ version = attr: xdmod_data.__version__
 author = Aaron Weeden, Joseph P. White
 maintainer = Aaron Weeden, Joseph P. White
 description = Python package for XDMoD data access
-long_description = file: README.md
+long_description = See [https://github.com/ubccr/xdmod-data](the README) for instructions on use.
 long_description_content_type = text/markdown
 license_files = LICENSE
 project_urls =
diff --git a/xdmod_data/__version__.py b/xdmod_data/__version__.py
index 10f5e82c..09869ff8 100644
--- a/xdmod_data/__version__.py
+++ b/xdmod_data/__version__.py
@@ -1,2 +1,2 @@
 __title__ = 'xdmod-data'
-__version__ = '1.0.1-beta.1'
+__version__ = '1.0.1'

From c1465dae499cb6baaeeadbb4806886872ce7101d Mon Sep 17 00:00:00 2001
From: Rodney Garnett <garnettr35@gmail.com>
Date: Fri, 7 Jun 2024 15:55:42 -0400
Subject: [PATCH 07/72] Update changelog and instructions and fix link
 formatting

---
 CHANGELOG.md       | 1 +
 docs/developing.md | 2 +-
 setup.cfg          | 4 ++--
 3 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index a953b917..fbd1dd54 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -11,3 +11,4 @@
 
 ## v1.0.1 (2024-06-XX)
 - Initial release.
+
diff --git a/docs/developing.md b/docs/developing.md
index 66627fa4..6fb6f9d3 100644
--- a/docs/developing.md
+++ b/docs/developing.md
@@ -67,7 +67,7 @@ After the Pull Request is approved (but not merged), follow these steps in the `
     twine upload --repository testpypi dist/xdmod_data-${version}-py3-none-any.whl
     ```
     Enter your TestPyPI API token.
-1.Go to https://testpypi.org/project/xdmod-data and confirm that everything looks right.
+1. Go to https://testpypi.org/project/xdmod-data and confirm that everything looks right.
 1. Upload the built distribution to PyPI:
     ```
     twine upload dist/xdmod_data-${version}-py3-none-any.whl
diff --git a/setup.cfg b/setup.cfg
index 0b224c44..405e0514 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -4,8 +4,8 @@ version = attr: xdmod_data.__version__
 author = Aaron Weeden, Joseph P. White
 maintainer = Aaron Weeden, Joseph P. White
 description = Python package for XDMoD data access
-long_description = See [https://github.com/ubccr/xdmod-data](the README) for instructions on use.
-long_description_content_type = text/markdown
+long_description = See `the README <https://github.com/ubccr/xdmod-data?tab=readme-ov-file#xdmod-data>`_ for instructions on use.
+long_description_content_type = text/x-rst
 license_files = LICENSE
 project_urls =
     Source Code = https://github.com/ubccr/xdmod-data

From f36c4baa45a0874a2abb82ed59d55f16f870d00d Mon Sep 17 00:00:00 2001
From: Rodney Garnett <garnettr35@gmail.com>
Date: Fri, 7 Jun 2024 16:03:27 -0400
Subject: [PATCH 08/72] Updat PR template

---
 .github/PULL_REQUEST_TEMPLATE.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
index 9c7cbddd..c8f84986 100644
--- a/.github/PULL_REQUEST_TEMPLATE.md
+++ b/.github/PULL_REQUEST_TEMPLATE.md
@@ -21,6 +21,7 @@
 - [ ] Bug fix (non-breaking change which fixes an issue)
 - [ ] New feature (non-breaking change which adds functionality)
 - [ ] Breaking change (fix or feature that would cause existing functionality to change)
+- [ ] Release preparation
 
 ## Checklist:
 <!--- Go over all the following points and make sure they have all been completed -->

From 9500be273a19b20f913a4e142b9745afc6485f65 Mon Sep 17 00:00:00 2001
From: Rodney Garnett <garnettr35@gmail.com>
Date: Fri, 7 Jun 2024 16:42:05 -0400
Subject: [PATCH 09/72] Quick changelog fix

---
 CHANGELOG.md | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index fbd1dd54..0913f81d 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,6 +1,6 @@
 # xdmod-data Changelog
 
-## Main development branch
+## v1.0.1 (2024-06-XX)
 - Update tests and testing instructions ([\#14](https://github.com/ubccr/xdmod-data/pull/14)).
 - Remove limit on number of results returned from `get_filter_values()` ([\#21](https://github.com/ubccr/xdmod-data/pull/21)).
 - Add a "Feedback / Feature Requests" section to the README ([\#22](https://github.com/ubccr/xdmod-notebooks/pull/22)).
@@ -9,6 +9,5 @@
 - Use streaming for raw data requests ([\#19](https://github.com/ubccr/xdmod-data/pull/19)).
 - Specify minimum version numbers for dependencies ([\#25](https://github.com/ubccr/xdmod-data/pull/25)). 
 
-## v1.0.1 (2024-06-XX)
+## v1.0.0 (2023-07-21)
 - Initial release.
-

From 7b29b49293861a636db5b46b075b996aca345512 Mon Sep 17 00:00:00 2001
From: Rodney Garnett <garnettr35@gmail.com>
Date: Fri, 7 Jun 2024 17:02:55 -0400
Subject: [PATCH 10/72] Style fixes

---
 docs/developing.md | 13 ++++++-------
 setup.cfg          |  4 ++--
 2 files changed, 8 insertions(+), 9 deletions(-)

diff --git a/docs/developing.md b/docs/developing.md
index 6fb6f9d3..93c6b5e1 100644
--- a/docs/developing.md
+++ b/docs/developing.md
@@ -14,7 +14,7 @@
     python3 -m pip install --upgrade python-dotenv pytest
     ```
 1. Create an empty file in your home directory at `~/.xdmod-data-token` with permissions set to 600.
-1. With an [https://xdmod.access-ci.org](ACCESS XDMoD) account with "User" as the Top Role, create an API token if you do not already have one (sign in and click My Profile -> API Token).
+1. With an [ACCESS XDMoD](https://xdmod.access-ci.org) account with "User" as the Top Role, create an API token if you do not already have one (sign in and click My Profile -> API Token).
 1. Add the following line to the file `~/.xdmod-data-token`, replacing `<token>` with your token.
     ```
     XDMOD_API_TOKEN=<token>
@@ -35,13 +35,12 @@
     
 # Preparing a new version for release
 1. Make sure the version number is updated in `xdmod_data/__version__.py`.
-1. Update the Open XDMoD compatibility matrix in the `README.md`
-1. Update CHANGELOG.md to change "Main development branch" to v1.0.1 (2024-06-XX)
-1. Create a Pull Request for the new version.
-1.Do not merge the Pull Request until following the instructions below for distributing the new version.
+1. Update the Open XDMoD compatibility matrix in the `README.md`.
+1. Update `CHANGELOG.md` to change "Main development branch" to, e.g., `v1.0.1 (2024-06-XX)`.
+1. Create a Pull Request for the new version. Do not merge the Pull Request until following the instructions below for distributing the new version.
 
 # Distributing the new version to PyPI
-After the Pull Request is approved (but not merged), follow these steps in the `xdmod-data` root directory:
+After the Pull Request is approved (but not merged yet), follow these steps in the `xdmod-data` root directory:
 1. Start up a virtual environment, e.g.:
     ```
     python3 -m venv ~/xdmod-data-build-env
@@ -61,7 +60,7 @@ After the Pull Request is approved (but not merged), follow these steps in the `
     version=1.0.0b1
     twine check dist/xdmod_data-${version}-py3-none-any.whl
     ```
-    Make sure you recieve `PASSED`.
+    Make sure you receive `PASSED`.
 1. Upload the built distribution to TestPyPI:
     ```
     twine upload --repository testpypi dist/xdmod_data-${version}-py3-none-any.whl
diff --git a/setup.cfg b/setup.cfg
index 405e0514..9839013b 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -4,8 +4,8 @@ version = attr: xdmod_data.__version__
 author = Aaron Weeden, Joseph P. White
 maintainer = Aaron Weeden, Joseph P. White
 description = Python package for XDMoD data access
-long_description = See `the README <https://github.com/ubccr/xdmod-data?tab=readme-ov-file#xdmod-data>`_ for instructions on use.
-long_description_content_type = text/x-rst
+long_description = See [the README](https://github.com/ubccr/xdmod-data?tab=readme-ov-file#xdmod-data) for instructions on use.
+long_description_content_type = text/markdown
 license_files = LICENSE
 project_urls =
     Source Code = https://github.com/ubccr/xdmod-data

From c0bdeec9bdc1d7cb07c6cb6832f6871d2f5e8308 Mon Sep 17 00:00:00 2001
From: Rodney Garnett <garnettr35@gmail.com>
Date: Mon, 10 Jun 2024 14:54:16 -0400
Subject: [PATCH 11/72] update changelog

---
 CHANGELOG.md       | 6 ++++++
 docs/developing.md | 7 +++++++
 2 files changed, 13 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 0913f81d..b36c3ff1 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,8 +1,14 @@
 # xdmod-data Changelog
 
 ## v1.0.1 (2024-06-XX)
+- Add borders to README images ([\#12](https://github.com/ubccr/xdmod-data/pull/12)).
+- `Create PULL_REQUEST_TEMPLATE.md'([\#13](https://github.com/ubccr/xdmod-data/pull/13))
+- Add types of changes to pull request template ([#\15](https://github.com/ubccr/xdmod-data/pull/15))
+- Add Changelog ([\#17](https://github.com/ubccr/xdmod-data/pull/17)).
 - Update tests and testing instructions ([\#14](https://github.com/ubccr/xdmod-data/pull/14)).
 - Remove limit on number of results returned from `get_filter_values()` ([\#21](https://github.com/ubccr/xdmod-data/pull/21)).
+
+- Add citation for the DAF paper.([\#23](https://github.com/ubccr/xdmod-data/pull/12))
 - Add a "Feedback / Feature Requests" section to the README ([\#22](https://github.com/ubccr/xdmod-notebooks/pull/22)).
 - Improve performance of validation of filters and raw fields ([\#18](https://github.com/ubccr/xdmod-data/pull/18)).
 - Fix bug with trailing slashes in `xdmod_host` ([\#24](https://github.com/ubccr/xdmod-data/pull/24)).
diff --git a/docs/developing.md b/docs/developing.md
index 93c6b5e1..1e640afd 100644
--- a/docs/developing.md
+++ b/docs/developing.md
@@ -75,3 +75,10 @@ After the Pull Request is approved (but not merged yet), follow these steps in t
 1. Go to https://pypi.org/project/xdmod-data and confirm the new version is the latest release.
 1. In the Pull Request, update the date of the release in `CHANGELOG.md`.
 1. Merge the Pull Request.
+1. 
+1. Apply Git tag to the pull request:
+    ```
+    git tag <tagname>
+    ```
+    Replace `<tagname>` with the actual name of the tag you want
+ 
\ No newline at end of file

From 8e63da68cfcff6e87bc4ae7d8a2e8fbbfb914596 Mon Sep 17 00:00:00 2001
From: Rodney Garnett <garnettr35@gmail.com>
Date: Mon, 10 Jun 2024 15:00:53 -0400
Subject: [PATCH 12/72] Update changelog 2

---
 CHANGELOG.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index b36c3ff1..0671b73b 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,13 +2,13 @@
 
 ## v1.0.1 (2024-06-XX)
 - Add borders to README images ([\#12](https://github.com/ubccr/xdmod-data/pull/12)).
-- `Create PULL_REQUEST_TEMPLATE.md'([\#13](https://github.com/ubccr/xdmod-data/pull/13))
-- Add types of changes to pull request template ([#\15](https://github.com/ubccr/xdmod-data/pull/15))
+- `Create PULL_REQUEST_TEMPLATE.md'([\#13](https://github.com/ubccr/xdmod-data/pull/13)).
+- Add types of changes to pull request template ([#\15](https://github.com/ubccr/xdmod-data/pull/15)).
 - Add Changelog ([\#17](https://github.com/ubccr/xdmod-data/pull/17)).
 - Update tests and testing instructions ([\#14](https://github.com/ubccr/xdmod-data/pull/14)).
 - Remove limit on number of results returned from `get_filter_values()` ([\#21](https://github.com/ubccr/xdmod-data/pull/21)).
 
-- Add citation for the DAF paper.([\#23](https://github.com/ubccr/xdmod-data/pull/12))
+- Add citation for the DAF paper.([\#23](https://github.com/ubccr/xdmod-data/pull/12)).
 - Add a "Feedback / Feature Requests" section to the README ([\#22](https://github.com/ubccr/xdmod-notebooks/pull/22)).
 - Improve performance of validation of filters and raw fields ([\#18](https://github.com/ubccr/xdmod-data/pull/18)).
 - Fix bug with trailing slashes in `xdmod_host` ([\#24](https://github.com/ubccr/xdmod-data/pull/24)).

From 9ebf528f3b09e3b7880b5892d9bccc4c4e32b20c Mon Sep 17 00:00:00 2001
From: Rodney Garnett <garnettr35@gmail.com>
Date: Mon, 10 Jun 2024 15:36:27 -0400
Subject: [PATCH 13/72] Update changelog 3

---
 CHANGELOG.md | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 0671b73b..40cf97ff 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,15 +1,16 @@
 # xdmod-data Changelog
 
 ## v1.0.1 (2024-06-XX)
+This release has bug fixes, performance improvements, and updates for compatibility, tests, and documentation.
 - Add borders to README images ([\#12](https://github.com/ubccr/xdmod-data/pull/12)).
-- `Create PULL_REQUEST_TEMPLATE.md'([\#13](https://github.com/ubccr/xdmod-data/pull/13)).
-- Add types of changes to pull request template ([#\15](https://github.com/ubccr/xdmod-data/pull/15)).
+- Create `PULL_REQUEST_TEMPLATE.md` ([\#13](https://github.com/ubccr/xdmod-data/pull/13)).
+- Add types of changes to pull request template ([\#15](https://github.com/ubccr/xdmod-data/pull/15)).
 - Add Changelog ([\#17](https://github.com/ubccr/xdmod-data/pull/17)).
 - Update tests and testing instructions ([\#14](https://github.com/ubccr/xdmod-data/pull/14)).
 - Remove limit on number of results returned from `get_filter_values()` ([\#21](https://github.com/ubccr/xdmod-data/pull/21)).
 
-- Add citation for the DAF paper.([\#23](https://github.com/ubccr/xdmod-data/pull/12)).
-- Add a "Feedback / Feature Requests" section to the README ([\#22](https://github.com/ubccr/xdmod-notebooks/pull/22)).
+- Add citation for the DAF paper ([\#23](https://github.com/ubccr/xdmod-data/pull/23)).
+- Add a "Feedback / Feature Requests" section to the README ([\#22](https://github.com/ubccr/xdmod-data/pull/22)).
 - Improve performance of validation of filters and raw fields ([\#18](https://github.com/ubccr/xdmod-data/pull/18)).
 - Fix bug with trailing slashes in `xdmod_host` ([\#24](https://github.com/ubccr/xdmod-data/pull/24)).
 - Use streaming for raw data requests ([\#19](https://github.com/ubccr/xdmod-data/pull/19)).

From 38e6fa7af3584627cc79bbb712b96e58424c7e1f Mon Sep 17 00:00:00 2001
From: Rodney Garnett <garnettr35@gmail.com>
Date: Mon, 10 Jun 2024 16:41:33 -0400
Subject: [PATCH 14/72] Update developer instructions

---
 docs/developing.md | 108 +++++++++++++++++++++++++--------------------
 1 file changed, 60 insertions(+), 48 deletions(-)

diff --git a/docs/developing.md b/docs/developing.md
index 1e640afd..ef980de4 100644
--- a/docs/developing.md
+++ b/docs/developing.md
@@ -1,4 +1,6 @@
-# Testing the code
+# Instructions for developers
+
+## Testing the code
 1. Start up a virtual environment, e.g.:
     ```                                            
     python3 -m venv ~/xdmod-data-test-env                                             
@@ -33,52 +35,62 @@
     pytest -vvs -o log_cli=true tests/
     ```
     
-# Preparing a new version for release
-1. Make sure the version number is updated in `xdmod_data/__version__.py`.
-1. Update the Open XDMoD compatibility matrix in the `README.md`.
-1. Update `CHANGELOG.md` to change "Main development branch" to, e.g., `v1.0.1 (2024-06-XX)`.
-1. Create a Pull Request for the new version. Do not merge the Pull Request until following the instructions below for distributing the new version.
+## Releasing a new version
+1. Make a new branch of `xdmod-data` and:
 
-# Distributing the new version to PyPI
-After the Pull Request is approved (but not merged yet), follow these steps in the `xdmod-data` root directory:
-1. Start up a virtual environment, e.g.:
-    ```
-    python3 -m venv ~/xdmod-data-build-env
-    source ~/xdmod-data-build-env/bin/activate
-    ```
-    Your command prompt should now start with `(xdmod-data-build-env)`.
-1. Make sure the required packages are installed:
-    ```
-    python3 -m pip install --upgrade pip build setuptools twine
-    ```
-1. Build the built distribution:
-    ```
-    python3 -m build --wheel
-    ```
-1. Validate `setup.cfg`, e.g., for version 1.0.0-beta1:
-    ```
-    version=1.0.0b1
-    twine check dist/xdmod_data-${version}-py3-none-any.whl
-    ```
-    Make sure you receive `PASSED`.
-1. Upload the built distribution to TestPyPI:
-    ```
-    twine upload --repository testpypi dist/xdmod_data-${version}-py3-none-any.whl
-    ```
-    Enter your TestPyPI API token.
-1. Go to https://testpypi.org/project/xdmod-data and confirm that everything looks right.
-1. Upload the built distribution to PyPI:
-    ```
-    twine upload dist/xdmod_data-${version}-py3-none-any.whl
-    ```
-    Enter your PyPI username and password.
-1. Go to https://pypi.org/project/xdmod-data and confirm the new version is the latest release.
-1. In the Pull Request, update the date of the release in `CHANGELOG.md`.
+    1. Make sure the version number is updated in `xdmod_data/__version__.py`.
+    1. Update the Open XDMoD compatibility matrix in the `README.md`.
+    1. Update `CHANGELOG.md` to change "Main development branch" to, e.g., `v1.0.1 (2024-06-XX)`.
+    1. Create a Pull Request for the new version.
+
+1. After the Pull Request is approved (but not merged yet), follow these steps in a cloned copy of the branch:
+    1. Start up a virtual environment, e.g.:
+        ```
+        python3 -m venv ~/xdmod-data-build-env
+        source ~/xdmod-data-build-env/bin/activate
+        ```
+        Your command prompt should now start with `(xdmod-data-build-env)`.
+    1. Make sure the required packages are installed:
+        ```
+        python3 -m pip install --upgrade pip build setuptools twine
+        ```
+    1. Build the built distribution:
+        ```
+        python3 -m build --wheel
+        ```
+    1. Validate `setup.cfg`, e.g., for version 1.0.0-beta1:
+        ```
+        version=1.0.0b1
+        twine check dist/xdmod_data-${version}-py3-none-any.whl
+        ```
+        Make sure you receive `PASSED`.
+    1. Upload the built distribution to TestPyPI:
+        ```
+        twine upload --repository testpypi dist/xdmod_data-${version}-py3-none-any.whl
+        ```
+        Enter your TestPyPI API token.
+    1. Go to https://testpypi.org/project/xdmod-data and confirm that everything looks right.
+    1. Upload the built distribution to PyPI:
+        ```
+        twine upload dist/xdmod_data-${version}-py3-none-any.whl
+        ```
+        Enter your PyPI username and password.
+    1. Go to https://pypi.org/project/xdmod-data and confirm the new version is the latest release.
+1. Update the date of the release in `CHANGELOG.md` and commit it to the Pull Request.
 1. Merge the Pull Request.
-1. 
-1. Apply Git tag to the pull request:
-    ```
-    git tag <tagname>
-    ```
-    Replace `<tagname>` with the actual name of the tag you want
- 
\ No newline at end of file
+
+1. Go to [create a new release on GitHub](https://github.com/ubccr/xdmod-data/releases/new) and:
+    1. Click `Choose a tag`.
+    1. Type in a tag similar to `v1.0.0` and choose `Create new tag`.
+    1. Make the release title the same as the tag name.
+    1. Where it says `Describe this release` paste in the contents of `CHANGELOG.md`.
+    1. Where it says `Attach binaries` attach the built distribution that was uploaded to PyPI.
+    1. Click `Publish release`.
+
+## After release
+
+1. Make a new branch of `xdmod-data` and:
+
+    1. Make sure the version number is updated in `xdmod_data/__version__.py` to a beta release of the next version, e.g., `1.0.1-beta.1`.
+    1. Update `CHANGELOG.md` to add a section at the top called `Main development branch`.
+    1. Create a Pull Request for the new version.
\ No newline at end of file

From df1f00ebe007c2e2949d66045b5a726c61160ae4 Mon Sep 17 00:00:00 2001
From: Rodney Garnett <garnettr35@gmail.com>
Date: Fri, 14 Jun 2024 15:43:54 -0400
Subject: [PATCH 15/72] Updated code to receive 100% code coverage

---
 docs/developing.md                            | 10 ++-
 .../test_datawarehouse_integration.py         |  1 +
 .../regression/data/jobs-2022-2023-years.csv  |  3 +
 tests/regression/data/jobs-2023-quarters.csv  |  5 ++
 .../test_datawarehouse_regression.py          | 75 ++++++++++++++++++-
 tests/unit/test_datawarehouse_unit.py         |  6 +-
 xdmod_data/_descriptors.py                    |  2 +-
 xdmod_data/_http_requester.py                 | 28 +++++--
 xdmod_data/_response_processor.py             |  9 +--
 xdmod_data/_validator.py                      |  8 +-
 10 files changed, 121 insertions(+), 26 deletions(-)
 create mode 100644 tests/regression/data/jobs-2022-2023-years.csv
 create mode 100644 tests/regression/data/jobs-2023-quarters.csv

diff --git a/docs/developing.md b/docs/developing.md
index ef980de4..8aaa9bbc 100644
--- a/docs/developing.md
+++ b/docs/developing.md
@@ -11,9 +11,9 @@
     ```
     python3 -m pip install --force-reinstall -e /path/to/your/xdmod-data
     ```
-1. Install `python-dotenv` and `pytest`:
+1. Install `python-dotenv` and `pytest` and `coverage`:
     ```
-    python3 -m pip install --upgrade python-dotenv pytest
+    python3 -m pip install --upgrade python-dotenv pytest coverage
     ```
 1. Create an empty file in your home directory at `~/.xdmod-data-token` with permissions set to 600.
 1. With an [ACCESS XDMoD](https://xdmod.access-ci.org) account with "User" as the Top Role, create an API token if you do not already have one (sign in and click My Profile -> API Token).
@@ -24,7 +24,11 @@
 1. Change directories to your local development copy of `xdmod-data`.
 1. Run the following command and make sure all the tests pass:
     ```
-    pytest -vvs -o log_cli=true tests/ 
+    coverage run -m pytest -vvs -o log_cli=true tests/
+    ```
+1. Run the following command and make sure code is 100% covered by tests
+    ```
+    coverage report -m
     ```
 1. Downgrade to the minimum version of the dependencies. Replace the version numbers below with the values from `setup.cfg`.
     ```
diff --git a/tests/integration/test_datawarehouse_integration.py b/tests/integration/test_datawarehouse_integration.py
index 1688b5c0..bca063e7 100644
--- a/tests/integration/test_datawarehouse_integration.py
+++ b/tests/integration/test_datawarehouse_integration.py
@@ -447,6 +447,7 @@ def test_case_insensitive(dw_methods, method, param, value1, value2):
     'dw_methods,method',
     [(VALID_XDMOD_URL + '/', method) for method in list(METHOD_PARAMS.keys())],
     indirect=['dw_methods'],
+    ids=[method for method in list(METHOD_PARAMS.keys())],
 )
 def test_trailing_slashes(dw_methods, method):
     __run_method(dw_methods, method)
diff --git a/tests/regression/data/jobs-2022-2023-years.csv b/tests/regression/data/jobs-2022-2023-years.csv
new file mode 100644
index 00000000..0958a21a
--- /dev/null
+++ b/tests/regression/data/jobs-2022-2023-years.csv
@@ -0,0 +1,3 @@
+Time,CPU Hours: Total
+2022-01-01,2537086425.0939
+2023-01-01,2913432363.5808
diff --git a/tests/regression/data/jobs-2023-quarters.csv b/tests/regression/data/jobs-2023-quarters.csv
new file mode 100644
index 00000000..86e584cb
--- /dev/null
+++ b/tests/regression/data/jobs-2023-quarters.csv
@@ -0,0 +1,5 @@
+Time,CPU Hours: Total
+2023-01-01,874332559.6222
+2023-04-01,801997902.9242
+2023-07-01,617633340.4103
+2023-10-01,619468560.6242
diff --git a/tests/regression/test_datawarehouse_regression.py b/tests/regression/test_datawarehouse_regression.py
index 437f5821..66514368 100644
--- a/tests/regression/test_datawarehouse_regression.py
+++ b/tests/regression/test_datawarehouse_regression.py
@@ -1,3 +1,4 @@
+from datetime import date
 from dotenv import load_dotenv
 import numpy
 from os.path import dirname, expanduser
@@ -25,6 +26,7 @@ def __assert_dfs_equal(
     actual,
     dtype='object',
     index_col='id',
+    columns_name=None
 ):
     expected = pandas.read_csv(
         DATA_DIR + '/' + data_file,
@@ -34,10 +36,13 @@ def __assert_dfs_equal(
         na_values=[''],
     ).fillna(numpy.nan)
     expected.columns = expected.columns.astype('string')
+    expected.columns.name = columns_name
+    if (index_col == 'Time'):
+        expected.index = pandas.to_datetime(expected.index)
     assert expected.equals(actual)
 
 
-def test_get_raw_data(valid_dw):
+def test_get_raw_data(valid_dw, capsys):
     data = valid_dw.get_raw_data(
         duration=('2023-05-01', '2023-05-02'),
         realm='SUPREMM',
@@ -57,6 +62,7 @@ def test_get_raw_data(valid_dw):
                 'Bridges 2 RM',
             ],
         },
+        show_progress=True
     ).iloc[::1000]
     data.index = data.index.astype('string')
     __assert_dfs_equal(
@@ -65,6 +71,7 @@ def test_get_raw_data(valid_dw):
         dtype='string',
         index_col=0,
     )
+    assert 'Got 42637 rows...DONE' in capsys.readouterr().out
 
 
 def __assert_descriptor_dfs_equal(data_file, actual):
@@ -109,3 +116,69 @@ def test_get_data_filter_user(valid_dw):
         dataset_type='aggregate',
         filters={'User': '10332'},
     )
+
+
+def test_get_quarter(valid_dw):
+    data = valid_dw.get_data(
+        duration=('2023-01-01', '2023-12-31'),
+        realm='Jobs',
+        metric='CPU Hours: Total',
+        aggregation_unit='Quarter'
+        )
+    __assert_dfs_equal(
+        "jobs-2023-quarters.csv",
+        data,
+        index_col='Time',
+        columns_name='Metric',
+        dtype={'CPU Hours: Total': 'Float64'}
+        )
+
+
+def test_get_years(valid_dw):
+    data = valid_dw.get_data(
+        duration=('2022-01-01', '2023-12-31'),
+        realm='Jobs',
+        metric='CPU Hours: Total',
+        aggregation_unit='Year'
+        )
+    __assert_dfs_equal(
+        "jobs-2022-2023-years.csv",
+        data,
+        index_col='Time',
+        columns_name='Metric',
+        dtype={'CPU Hours: Total': 'Float64'}
+        )
+
+
+def test_get_aggregation_units(valid_dw):
+    expected_agg_units = ('Auto', 'Day', 'Month', 'Quarter', 'Year')
+    actual_agg_units = valid_dw.get_aggregation_units()
+    assert expected_agg_units == actual_agg_units
+
+
+def test_get_durations(valid_dw):
+    expected_durations = [
+        'Yesterday',
+        '7 day',
+        '30 day',
+        '90 day',
+        'Month to date',
+        'Previous month',
+        'Quarter to date',
+        'Previous quarter',
+        'Year to date',
+        'Previous year',
+        '1 year',
+        '2 year',
+        '3 year',
+        '5 year',
+        '10 year',
+    ]
+    today_date = date.today()
+    current_year = today_date.year
+    for count in range(0, 7):
+        year = current_year - count
+        year = str(year)
+        expected_durations.append(year)
+    actual_durations = list(valid_dw.get_durations())
+    assert expected_durations == actual_durations
diff --git a/tests/unit/test_datawarehouse_unit.py b/tests/unit/test_datawarehouse_unit.py
index d73b04df..2f3d55e7 100644
--- a/tests/unit/test_datawarehouse_unit.py
+++ b/tests/unit/test_datawarehouse_unit.py
@@ -50,14 +50,14 @@ def test___enter___RuntimeError_xdmod_host_malformed():
         )
     ):
         with DataWarehouse('https://'):
-            pass
+            pass  # pragma: no cover
 
 
 def test___enter___RuntimeError_xdmod_host_unresolved():
     invalid_host = 'https://' + INVALID_STR + '.xdmod.org'
     with pytest.raises(Exception):
         with DataWarehouse(invalid_host):
-            pass
+            pass  # pragma: no cover
 
 
 def test___enter___RuntimeError_xdmod_host_unsupported_protocol():
@@ -67,7 +67,7 @@ def test___enter___RuntimeError_xdmod_host_unsupported_protocol():
         match='No connection adapters were found for \'' + invalid_host
     ):
         with DataWarehouse(invalid_host):
-            pass
+            pass  # pragma: no cover
 
 
 def test___enter___RuntimeError_401():
diff --git a/xdmod_data/_descriptors.py b/xdmod_data/_descriptors.py
index 6387ba68..d5a24ff6 100644
--- a/xdmod_data/_descriptors.py
+++ b/xdmod_data/_descriptors.py
@@ -19,7 +19,7 @@ def __request_aggregate(self):
             '/controllers/metric_explorer.php',
             {'operation': 'get_dw_descripter'},
         )
-        if response['totalCount'] != 1:
+        if response['totalCount'] != 1:  # pragma: no cover
             raise RuntimeError(
                 'Descriptor received with unexpected structure.'
             )
diff --git a/xdmod_data/_http_requester.py b/xdmod_data/_http_requester.py
index ad5bc939..3e6cee68 100644
--- a/xdmod_data/_http_requester.py
+++ b/xdmod_data/_http_requester.py
@@ -49,7 +49,11 @@ def _request_raw_data(self, params):
         # necessary — only the body of the 'if' branch will be needed.
         limit = self.__get_raw_data_limit()
         data = []
-        if limit == 'NA':
+        # Once ACCESS XDMoD is on version 11.0 then the `pragma: no cover`
+        # below will need to be moved to the `else` statement so that
+        # Coverage.py will know that branch is not
+        # expected to be covered by tests.
+        if limit == 'NA':  # pragma: no cover
             response_iter_lines = self.__request(
                 path='/rest/v1/warehouse/raw-data?' + url_params,
                 post_fields=None,
@@ -123,7 +127,7 @@ def _request_json(self, path, post_fields=None):
     def __assert_connection_to_xdmod_host(self):
         try:
             self.__request()
-        except RuntimeError as e:
+        except RuntimeError as e:  # pragma: no cover
             raise RuntimeError(
                 'Could not connect to xdmod_host \'' + self.__xdmod_host
                 + '\': ' + str(e)
@@ -148,7 +152,7 @@ def __request(self, path='', post_fields=None, stream=False):
             try:
                 response_json = json.loads(response.text)
                 msg = ': ' + response_json['message']
-            except json.JSONDecodeError:
+            except json.JSONDecodeError:  # pragma: no cover
                 pass
             if response.status_code == 401:
                 msg = (
@@ -157,7 +161,11 @@ def __request(self, path='', post_fields=None, stream=False):
             raise RuntimeError(
                 'Error ' + str(response.status_code) + msg
             ) from None
-        if stream:
+        # Once ACCESS XDMoD is on version 11.0 then
+        # the `pragma: no cover` below
+        # will need to be removed so that Coverage.py
+        # will know that branch is expected to be covered by tests.
+        if stream:  # pragma: no cover
             return response.iter_lines()
         else:
             return response.text
@@ -195,8 +203,8 @@ def __get_raw_data_url_params(self, params):
                 )
         return urlencode(results)
 
-    # Once XDMoD 10.5 is no longer supported, there will be no need for this
-    # method.
+    #  Once XDMoD 10.5 is no longer supported,
+    #  there will be no need for this method
     def __get_raw_data_limit(self):
         if self.__raw_data_limit is None:
             try:
@@ -204,9 +212,13 @@ def __get_raw_data_limit(self):
                     '/rest/v1/warehouse/raw-data/limit'
                 )
                 self.__raw_data_limit = int(response['data'])
-            except RuntimeError as e:
+                # Once ACCESS XDMoD is on version 11.0 then
+                # the `pragma: no cover` below
+                # will need to be removed so that Coverage.py
+                # will know that branch is expected to be covered by tests.
+            except RuntimeError as e:  # pragma: no cover
                 if '404' in str(e):
                     self.__raw_data_limit = 'NA'
-                else:
+                else:  # pragma: no cover
                     raise
         return self.__raw_data_limit
diff --git a/xdmod_data/_response_processor.py b/xdmod_data/_response_processor.py
index ba06f925..661806c0 100644
--- a/xdmod_data/_response_processor.py
+++ b/xdmod_data/_response_processor.py
@@ -55,10 +55,7 @@ def __parse_timeseries_dimension_values(labels):
     dimension_values = []
     for label in labels:
         match = label_re.match(label)
-        if match:
-            dimension_values.append(html.unescape(match.group(1)))
-        else:
-            dimension_values.append(html.unescape(label))
+        dimension_values.append(html.unescape(match.group(1)))
     return dimension_values
 
 
@@ -75,7 +72,7 @@ def __parse_timeseries_date_string(date_string):
     # Match YYYY Q#
     elif re.match(r'^[0-9]{4} Q[0-9]$', date_string):
         (date_string, format_) = __parse_quarter_date_string(date_string)
-    else:
+    else:  # pragma: no cover
         raise Exception(
             'Unsupported date specification ' + date_string + '.'
         )
@@ -134,7 +131,7 @@ def __parse_quarter_date_string(date_string):
         month = '07'
     elif quarter == 'Q4':
         month = '10'
-    else:
+    else:  # pragma: no cover
         raise Exception(
             'Unsupported date quarter specification '
             + date_string + '.'
diff --git a/xdmod_data/_validator.py b/xdmod_data/_validator.py
index d80f399a..6a4e6da6 100644
--- a/xdmod_data/_validator.py
+++ b/xdmod_data/_validator.py
@@ -255,8 +255,8 @@ def __get_dates_from_duration(duration):
     last_quarter = today + timedelta(days=-90)
     this_month_start = date(today.year, today.month, 1)
     if today.month == 1:
-        last_full_month_start_year = today.year - 1
-        last_full_month_start_month = 12
+        last_full_month_start_year = today.year - 1  # pragma: no cover
+        last_full_month_start_month = 12  # pragma: no cover
     else:
         last_full_month_start_year = today.year
         last_full_month_start_month = today.month - 1
@@ -272,7 +272,7 @@ def __get_dates_from_duration(duration):
         1,
     )
     if today.month < 4:
-        last_quarter_start_year = today.year - 1
+        last_quarter_start_year = today.year - 1  # pragma: no cover
     else:
         last_quarter_start_year = today.year
     last_quarter_start = date(
@@ -340,7 +340,7 @@ def __date_add_years(old_date, year_delta):
         try:
             new_date = date(new_date_year, old_date.month, new_date_day)
             keep_going = False
-        except ValueError:
+        except ValueError:  # pragma: no cover
             new_date_day -= 1
             days_above += 1
     return new_date + timedelta(days=days_above)

From aacd5d67e044b02aed39bf0a8818cb94022ad6f2 Mon Sep 17 00:00:00 2001
From: Rodney Garnett <garnettr35@gmail.com>
Date: Fri, 14 Jun 2024 15:59:44 -0400
Subject: [PATCH 16/72] Revert "Updated code to receive 100% code coverage"

This reverts commit df1f00ebe007c2e2949d66045b5a726c61160ae4.
---
 docs/developing.md                            | 10 +--
 .../test_datawarehouse_integration.py         |  1 -
 .../regression/data/jobs-2022-2023-years.csv  |  3 -
 tests/regression/data/jobs-2023-quarters.csv  |  5 --
 .../test_datawarehouse_regression.py          | 75 +------------------
 tests/unit/test_datawarehouse_unit.py         |  6 +-
 xdmod_data/_descriptors.py                    |  2 +-
 xdmod_data/_http_requester.py                 | 28 ++-----
 xdmod_data/_response_processor.py             |  9 ++-
 xdmod_data/_validator.py                      |  8 +-
 10 files changed, 26 insertions(+), 121 deletions(-)
 delete mode 100644 tests/regression/data/jobs-2022-2023-years.csv
 delete mode 100644 tests/regression/data/jobs-2023-quarters.csv

diff --git a/docs/developing.md b/docs/developing.md
index 8aaa9bbc..ef980de4 100644
--- a/docs/developing.md
+++ b/docs/developing.md
@@ -11,9 +11,9 @@
     ```
     python3 -m pip install --force-reinstall -e /path/to/your/xdmod-data
     ```
-1. Install `python-dotenv` and `pytest` and `coverage`:
+1. Install `python-dotenv` and `pytest`:
     ```
-    python3 -m pip install --upgrade python-dotenv pytest coverage
+    python3 -m pip install --upgrade python-dotenv pytest
     ```
 1. Create an empty file in your home directory at `~/.xdmod-data-token` with permissions set to 600.
 1. With an [ACCESS XDMoD](https://xdmod.access-ci.org) account with "User" as the Top Role, create an API token if you do not already have one (sign in and click My Profile -> API Token).
@@ -24,11 +24,7 @@
 1. Change directories to your local development copy of `xdmod-data`.
 1. Run the following command and make sure all the tests pass:
     ```
-    coverage run -m pytest -vvs -o log_cli=true tests/
-    ```
-1. Run the following command and make sure code is 100% covered by tests
-    ```
-    coverage report -m
+    pytest -vvs -o log_cli=true tests/ 
     ```
 1. Downgrade to the minimum version of the dependencies. Replace the version numbers below with the values from `setup.cfg`.
     ```
diff --git a/tests/integration/test_datawarehouse_integration.py b/tests/integration/test_datawarehouse_integration.py
index bca063e7..1688b5c0 100644
--- a/tests/integration/test_datawarehouse_integration.py
+++ b/tests/integration/test_datawarehouse_integration.py
@@ -447,7 +447,6 @@ def test_case_insensitive(dw_methods, method, param, value1, value2):
     'dw_methods,method',
     [(VALID_XDMOD_URL + '/', method) for method in list(METHOD_PARAMS.keys())],
     indirect=['dw_methods'],
-    ids=[method for method in list(METHOD_PARAMS.keys())],
 )
 def test_trailing_slashes(dw_methods, method):
     __run_method(dw_methods, method)
diff --git a/tests/regression/data/jobs-2022-2023-years.csv b/tests/regression/data/jobs-2022-2023-years.csv
deleted file mode 100644
index 0958a21a..00000000
--- a/tests/regression/data/jobs-2022-2023-years.csv
+++ /dev/null
@@ -1,3 +0,0 @@
-Time,CPU Hours: Total
-2022-01-01,2537086425.0939
-2023-01-01,2913432363.5808
diff --git a/tests/regression/data/jobs-2023-quarters.csv b/tests/regression/data/jobs-2023-quarters.csv
deleted file mode 100644
index 86e584cb..00000000
--- a/tests/regression/data/jobs-2023-quarters.csv
+++ /dev/null
@@ -1,5 +0,0 @@
-Time,CPU Hours: Total
-2023-01-01,874332559.6222
-2023-04-01,801997902.9242
-2023-07-01,617633340.4103
-2023-10-01,619468560.6242
diff --git a/tests/regression/test_datawarehouse_regression.py b/tests/regression/test_datawarehouse_regression.py
index 66514368..437f5821 100644
--- a/tests/regression/test_datawarehouse_regression.py
+++ b/tests/regression/test_datawarehouse_regression.py
@@ -1,4 +1,3 @@
-from datetime import date
 from dotenv import load_dotenv
 import numpy
 from os.path import dirname, expanduser
@@ -26,7 +25,6 @@ def __assert_dfs_equal(
     actual,
     dtype='object',
     index_col='id',
-    columns_name=None
 ):
     expected = pandas.read_csv(
         DATA_DIR + '/' + data_file,
@@ -36,13 +34,10 @@ def __assert_dfs_equal(
         na_values=[''],
     ).fillna(numpy.nan)
     expected.columns = expected.columns.astype('string')
-    expected.columns.name = columns_name
-    if (index_col == 'Time'):
-        expected.index = pandas.to_datetime(expected.index)
     assert expected.equals(actual)
 
 
-def test_get_raw_data(valid_dw, capsys):
+def test_get_raw_data(valid_dw):
     data = valid_dw.get_raw_data(
         duration=('2023-05-01', '2023-05-02'),
         realm='SUPREMM',
@@ -62,7 +57,6 @@ def test_get_raw_data(valid_dw, capsys):
                 'Bridges 2 RM',
             ],
         },
-        show_progress=True
     ).iloc[::1000]
     data.index = data.index.astype('string')
     __assert_dfs_equal(
@@ -71,7 +65,6 @@ def test_get_raw_data(valid_dw, capsys):
         dtype='string',
         index_col=0,
     )
-    assert 'Got 42637 rows...DONE' in capsys.readouterr().out
 
 
 def __assert_descriptor_dfs_equal(data_file, actual):
@@ -116,69 +109,3 @@ def test_get_data_filter_user(valid_dw):
         dataset_type='aggregate',
         filters={'User': '10332'},
     )
-
-
-def test_get_quarter(valid_dw):
-    data = valid_dw.get_data(
-        duration=('2023-01-01', '2023-12-31'),
-        realm='Jobs',
-        metric='CPU Hours: Total',
-        aggregation_unit='Quarter'
-        )
-    __assert_dfs_equal(
-        "jobs-2023-quarters.csv",
-        data,
-        index_col='Time',
-        columns_name='Metric',
-        dtype={'CPU Hours: Total': 'Float64'}
-        )
-
-
-def test_get_years(valid_dw):
-    data = valid_dw.get_data(
-        duration=('2022-01-01', '2023-12-31'),
-        realm='Jobs',
-        metric='CPU Hours: Total',
-        aggregation_unit='Year'
-        )
-    __assert_dfs_equal(
-        "jobs-2022-2023-years.csv",
-        data,
-        index_col='Time',
-        columns_name='Metric',
-        dtype={'CPU Hours: Total': 'Float64'}
-        )
-
-
-def test_get_aggregation_units(valid_dw):
-    expected_agg_units = ('Auto', 'Day', 'Month', 'Quarter', 'Year')
-    actual_agg_units = valid_dw.get_aggregation_units()
-    assert expected_agg_units == actual_agg_units
-
-
-def test_get_durations(valid_dw):
-    expected_durations = [
-        'Yesterday',
-        '7 day',
-        '30 day',
-        '90 day',
-        'Month to date',
-        'Previous month',
-        'Quarter to date',
-        'Previous quarter',
-        'Year to date',
-        'Previous year',
-        '1 year',
-        '2 year',
-        '3 year',
-        '5 year',
-        '10 year',
-    ]
-    today_date = date.today()
-    current_year = today_date.year
-    for count in range(0, 7):
-        year = current_year - count
-        year = str(year)
-        expected_durations.append(year)
-    actual_durations = list(valid_dw.get_durations())
-    assert expected_durations == actual_durations
diff --git a/tests/unit/test_datawarehouse_unit.py b/tests/unit/test_datawarehouse_unit.py
index 2f3d55e7..d73b04df 100644
--- a/tests/unit/test_datawarehouse_unit.py
+++ b/tests/unit/test_datawarehouse_unit.py
@@ -50,14 +50,14 @@ def test___enter___RuntimeError_xdmod_host_malformed():
         )
     ):
         with DataWarehouse('https://'):
-            pass  # pragma: no cover
+            pass
 
 
 def test___enter___RuntimeError_xdmod_host_unresolved():
     invalid_host = 'https://' + INVALID_STR + '.xdmod.org'
     with pytest.raises(Exception):
         with DataWarehouse(invalid_host):
-            pass  # pragma: no cover
+            pass
 
 
 def test___enter___RuntimeError_xdmod_host_unsupported_protocol():
@@ -67,7 +67,7 @@ def test___enter___RuntimeError_xdmod_host_unsupported_protocol():
         match='No connection adapters were found for \'' + invalid_host
     ):
         with DataWarehouse(invalid_host):
-            pass  # pragma: no cover
+            pass
 
 
 def test___enter___RuntimeError_401():
diff --git a/xdmod_data/_descriptors.py b/xdmod_data/_descriptors.py
index d5a24ff6..6387ba68 100644
--- a/xdmod_data/_descriptors.py
+++ b/xdmod_data/_descriptors.py
@@ -19,7 +19,7 @@ def __request_aggregate(self):
             '/controllers/metric_explorer.php',
             {'operation': 'get_dw_descripter'},
         )
-        if response['totalCount'] != 1:  # pragma: no cover
+        if response['totalCount'] != 1:
             raise RuntimeError(
                 'Descriptor received with unexpected structure.'
             )
diff --git a/xdmod_data/_http_requester.py b/xdmod_data/_http_requester.py
index 3e6cee68..ad5bc939 100644
--- a/xdmod_data/_http_requester.py
+++ b/xdmod_data/_http_requester.py
@@ -49,11 +49,7 @@ def _request_raw_data(self, params):
         # necessary — only the body of the 'if' branch will be needed.
         limit = self.__get_raw_data_limit()
         data = []
-        # Once ACCESS XDMoD is on version 11.0 then the `pragma: no cover`
-        # below will need to be moved to the `else` statement so that
-        # Coverage.py will know that branch is not
-        # expected to be covered by tests.
-        if limit == 'NA':  # pragma: no cover
+        if limit == 'NA':
             response_iter_lines = self.__request(
                 path='/rest/v1/warehouse/raw-data?' + url_params,
                 post_fields=None,
@@ -127,7 +123,7 @@ def _request_json(self, path, post_fields=None):
     def __assert_connection_to_xdmod_host(self):
         try:
             self.__request()
-        except RuntimeError as e:  # pragma: no cover
+        except RuntimeError as e:
             raise RuntimeError(
                 'Could not connect to xdmod_host \'' + self.__xdmod_host
                 + '\': ' + str(e)
@@ -152,7 +148,7 @@ def __request(self, path='', post_fields=None, stream=False):
             try:
                 response_json = json.loads(response.text)
                 msg = ': ' + response_json['message']
-            except json.JSONDecodeError:  # pragma: no cover
+            except json.JSONDecodeError:
                 pass
             if response.status_code == 401:
                 msg = (
@@ -161,11 +157,7 @@ def __request(self, path='', post_fields=None, stream=False):
             raise RuntimeError(
                 'Error ' + str(response.status_code) + msg
             ) from None
-        # Once ACCESS XDMoD is on version 11.0 then
-        # the `pragma: no cover` below
-        # will need to be removed so that Coverage.py
-        # will know that branch is expected to be covered by tests.
-        if stream:  # pragma: no cover
+        if stream:
             return response.iter_lines()
         else:
             return response.text
@@ -203,8 +195,8 @@ def __get_raw_data_url_params(self, params):
                 )
         return urlencode(results)
 
-    #  Once XDMoD 10.5 is no longer supported,
-    #  there will be no need for this method
+    # Once XDMoD 10.5 is no longer supported, there will be no need for this
+    # method.
     def __get_raw_data_limit(self):
         if self.__raw_data_limit is None:
             try:
@@ -212,13 +204,9 @@ def __get_raw_data_limit(self):
                     '/rest/v1/warehouse/raw-data/limit'
                 )
                 self.__raw_data_limit = int(response['data'])
-                # Once ACCESS XDMoD is on version 11.0 then
-                # the `pragma: no cover` below
-                # will need to be removed so that Coverage.py
-                # will know that branch is expected to be covered by tests.
-            except RuntimeError as e:  # pragma: no cover
+            except RuntimeError as e:
                 if '404' in str(e):
                     self.__raw_data_limit = 'NA'
-                else:  # pragma: no cover
+                else:
                     raise
         return self.__raw_data_limit
diff --git a/xdmod_data/_response_processor.py b/xdmod_data/_response_processor.py
index 661806c0..ba06f925 100644
--- a/xdmod_data/_response_processor.py
+++ b/xdmod_data/_response_processor.py
@@ -55,7 +55,10 @@ def __parse_timeseries_dimension_values(labels):
     dimension_values = []
     for label in labels:
         match = label_re.match(label)
-        dimension_values.append(html.unescape(match.group(1)))
+        if match:
+            dimension_values.append(html.unescape(match.group(1)))
+        else:
+            dimension_values.append(html.unescape(label))
     return dimension_values
 
 
@@ -72,7 +75,7 @@ def __parse_timeseries_date_string(date_string):
     # Match YYYY Q#
     elif re.match(r'^[0-9]{4} Q[0-9]$', date_string):
         (date_string, format_) = __parse_quarter_date_string(date_string)
-    else:  # pragma: no cover
+    else:
         raise Exception(
             'Unsupported date specification ' + date_string + '.'
         )
@@ -131,7 +134,7 @@ def __parse_quarter_date_string(date_string):
         month = '07'
     elif quarter == 'Q4':
         month = '10'
-    else:  # pragma: no cover
+    else:
         raise Exception(
             'Unsupported date quarter specification '
             + date_string + '.'
diff --git a/xdmod_data/_validator.py b/xdmod_data/_validator.py
index 6a4e6da6..d80f399a 100644
--- a/xdmod_data/_validator.py
+++ b/xdmod_data/_validator.py
@@ -255,8 +255,8 @@ def __get_dates_from_duration(duration):
     last_quarter = today + timedelta(days=-90)
     this_month_start = date(today.year, today.month, 1)
     if today.month == 1:
-        last_full_month_start_year = today.year - 1  # pragma: no cover
-        last_full_month_start_month = 12  # pragma: no cover
+        last_full_month_start_year = today.year - 1
+        last_full_month_start_month = 12
     else:
         last_full_month_start_year = today.year
         last_full_month_start_month = today.month - 1
@@ -272,7 +272,7 @@ def __get_dates_from_duration(duration):
         1,
     )
     if today.month < 4:
-        last_quarter_start_year = today.year - 1  # pragma: no cover
+        last_quarter_start_year = today.year - 1
     else:
         last_quarter_start_year = today.year
     last_quarter_start = date(
@@ -340,7 +340,7 @@ def __date_add_years(old_date, year_delta):
         try:
             new_date = date(new_date_year, old_date.month, new_date_day)
             keep_going = False
-        except ValueError:  # pragma: no cover
+        except ValueError:
             new_date_day -= 1
             days_above += 1
     return new_date + timedelta(days=days_above)

From 018b3ce5960e89900d1ba424be41539e7fa11534 Mon Sep 17 00:00:00 2001
From: RodneyCodess <160431998+RodneyCodess@users.noreply.github.com>
Date: Mon, 24 Jun 2024 09:58:08 -0400
Subject: [PATCH 17/72] Update CHANGELOG.md

Co-authored-by: Aaron Weeden <31246768+aaronweeden@users.noreply.github.com>
---
 CHANGELOG.md | 1 -
 1 file changed, 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 40cf97ff..33cb26ad 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -8,7 +8,6 @@ This release has bug fixes, performance improvements, and updates for compatibil
 - Add Changelog ([\#17](https://github.com/ubccr/xdmod-data/pull/17)).
 - Update tests and testing instructions ([\#14](https://github.com/ubccr/xdmod-data/pull/14)).
 - Remove limit on number of results returned from `get_filter_values()` ([\#21](https://github.com/ubccr/xdmod-data/pull/21)).
-
 - Add citation for the DAF paper ([\#23](https://github.com/ubccr/xdmod-data/pull/23)).
 - Add a "Feedback / Feature Requests" section to the README ([\#22](https://github.com/ubccr/xdmod-data/pull/22)).
 - Improve performance of validation of filters and raw fields ([\#18](https://github.com/ubccr/xdmod-data/pull/18)).

From 8334e9274dd00223e65638a7d4ca7d9d31287430 Mon Sep 17 00:00:00 2001
From: Rodney Garnett <garnettr35@gmail.com>
Date: Mon, 24 Jun 2024 10:12:15 -0400
Subject: [PATCH 18/72] Revert "Revert "Updated code to receive 100% code
 coverage""

This reverts commit aacd5d67e044b02aed39bf0a8818cb94022ad6f2.
---
 docs/developing.md                            | 10 ++-
 .../test_datawarehouse_integration.py         |  1 +
 .../regression/data/jobs-2022-2023-years.csv  |  3 +
 tests/regression/data/jobs-2023-quarters.csv  |  5 ++
 .../test_datawarehouse_regression.py          | 75 ++++++++++++++++++-
 tests/unit/test_datawarehouse_unit.py         |  6 +-
 xdmod_data/_descriptors.py                    |  2 +-
 xdmod_data/_http_requester.py                 | 28 +++++--
 xdmod_data/_response_processor.py             |  9 +--
 xdmod_data/_validator.py                      |  8 +-
 10 files changed, 121 insertions(+), 26 deletions(-)
 create mode 100644 tests/regression/data/jobs-2022-2023-years.csv
 create mode 100644 tests/regression/data/jobs-2023-quarters.csv

diff --git a/docs/developing.md b/docs/developing.md
index ef980de4..8aaa9bbc 100644
--- a/docs/developing.md
+++ b/docs/developing.md
@@ -11,9 +11,9 @@
     ```
     python3 -m pip install --force-reinstall -e /path/to/your/xdmod-data
     ```
-1. Install `python-dotenv` and `pytest`:
+1. Install `python-dotenv` and `pytest` and `coverage`:
     ```
-    python3 -m pip install --upgrade python-dotenv pytest
+    python3 -m pip install --upgrade python-dotenv pytest coverage
     ```
 1. Create an empty file in your home directory at `~/.xdmod-data-token` with permissions set to 600.
 1. With an [ACCESS XDMoD](https://xdmod.access-ci.org) account with "User" as the Top Role, create an API token if you do not already have one (sign in and click My Profile -> API Token).
@@ -24,7 +24,11 @@
 1. Change directories to your local development copy of `xdmod-data`.
 1. Run the following command and make sure all the tests pass:
     ```
-    pytest -vvs -o log_cli=true tests/ 
+    coverage run -m pytest -vvs -o log_cli=true tests/
+    ```
+1. Run the following command and make sure code is 100% covered by tests
+    ```
+    coverage report -m
     ```
 1. Downgrade to the minimum version of the dependencies. Replace the version numbers below with the values from `setup.cfg`.
     ```
diff --git a/tests/integration/test_datawarehouse_integration.py b/tests/integration/test_datawarehouse_integration.py
index 1688b5c0..bca063e7 100644
--- a/tests/integration/test_datawarehouse_integration.py
+++ b/tests/integration/test_datawarehouse_integration.py
@@ -447,6 +447,7 @@ def test_case_insensitive(dw_methods, method, param, value1, value2):
     'dw_methods,method',
     [(VALID_XDMOD_URL + '/', method) for method in list(METHOD_PARAMS.keys())],
     indirect=['dw_methods'],
+    ids=[method for method in list(METHOD_PARAMS.keys())],
 )
 def test_trailing_slashes(dw_methods, method):
     __run_method(dw_methods, method)
diff --git a/tests/regression/data/jobs-2022-2023-years.csv b/tests/regression/data/jobs-2022-2023-years.csv
new file mode 100644
index 00000000..0958a21a
--- /dev/null
+++ b/tests/regression/data/jobs-2022-2023-years.csv
@@ -0,0 +1,3 @@
+Time,CPU Hours: Total
+2022-01-01,2537086425.0939
+2023-01-01,2913432363.5808
diff --git a/tests/regression/data/jobs-2023-quarters.csv b/tests/regression/data/jobs-2023-quarters.csv
new file mode 100644
index 00000000..86e584cb
--- /dev/null
+++ b/tests/regression/data/jobs-2023-quarters.csv
@@ -0,0 +1,5 @@
+Time,CPU Hours: Total
+2023-01-01,874332559.6222
+2023-04-01,801997902.9242
+2023-07-01,617633340.4103
+2023-10-01,619468560.6242
diff --git a/tests/regression/test_datawarehouse_regression.py b/tests/regression/test_datawarehouse_regression.py
index 437f5821..66514368 100644
--- a/tests/regression/test_datawarehouse_regression.py
+++ b/tests/regression/test_datawarehouse_regression.py
@@ -1,3 +1,4 @@
+from datetime import date
 from dotenv import load_dotenv
 import numpy
 from os.path import dirname, expanduser
@@ -25,6 +26,7 @@ def __assert_dfs_equal(
     actual,
     dtype='object',
     index_col='id',
+    columns_name=None
 ):
     expected = pandas.read_csv(
         DATA_DIR + '/' + data_file,
@@ -34,10 +36,13 @@ def __assert_dfs_equal(
         na_values=[''],
     ).fillna(numpy.nan)
     expected.columns = expected.columns.astype('string')
+    expected.columns.name = columns_name
+    if (index_col == 'Time'):
+        expected.index = pandas.to_datetime(expected.index)
     assert expected.equals(actual)
 
 
-def test_get_raw_data(valid_dw):
+def test_get_raw_data(valid_dw, capsys):
     data = valid_dw.get_raw_data(
         duration=('2023-05-01', '2023-05-02'),
         realm='SUPREMM',
@@ -57,6 +62,7 @@ def test_get_raw_data(valid_dw):
                 'Bridges 2 RM',
             ],
         },
+        show_progress=True
     ).iloc[::1000]
     data.index = data.index.astype('string')
     __assert_dfs_equal(
@@ -65,6 +71,7 @@ def test_get_raw_data(valid_dw):
         dtype='string',
         index_col=0,
     )
+    assert 'Got 42637 rows...DONE' in capsys.readouterr().out
 
 
 def __assert_descriptor_dfs_equal(data_file, actual):
@@ -109,3 +116,69 @@ def test_get_data_filter_user(valid_dw):
         dataset_type='aggregate',
         filters={'User': '10332'},
     )
+
+
+def test_get_quarter(valid_dw):
+    data = valid_dw.get_data(
+        duration=('2023-01-01', '2023-12-31'),
+        realm='Jobs',
+        metric='CPU Hours: Total',
+        aggregation_unit='Quarter'
+        )
+    __assert_dfs_equal(
+        "jobs-2023-quarters.csv",
+        data,
+        index_col='Time',
+        columns_name='Metric',
+        dtype={'CPU Hours: Total': 'Float64'}
+        )
+
+
+def test_get_years(valid_dw):
+    data = valid_dw.get_data(
+        duration=('2022-01-01', '2023-12-31'),
+        realm='Jobs',
+        metric='CPU Hours: Total',
+        aggregation_unit='Year'
+        )
+    __assert_dfs_equal(
+        "jobs-2022-2023-years.csv",
+        data,
+        index_col='Time',
+        columns_name='Metric',
+        dtype={'CPU Hours: Total': 'Float64'}
+        )
+
+
+def test_get_aggregation_units(valid_dw):
+    expected_agg_units = ('Auto', 'Day', 'Month', 'Quarter', 'Year')
+    actual_agg_units = valid_dw.get_aggregation_units()
+    assert expected_agg_units == actual_agg_units
+
+
+def test_get_durations(valid_dw):
+    expected_durations = [
+        'Yesterday',
+        '7 day',
+        '30 day',
+        '90 day',
+        'Month to date',
+        'Previous month',
+        'Quarter to date',
+        'Previous quarter',
+        'Year to date',
+        'Previous year',
+        '1 year',
+        '2 year',
+        '3 year',
+        '5 year',
+        '10 year',
+    ]
+    today_date = date.today()
+    current_year = today_date.year
+    for count in range(0, 7):
+        year = current_year - count
+        year = str(year)
+        expected_durations.append(year)
+    actual_durations = list(valid_dw.get_durations())
+    assert expected_durations == actual_durations
diff --git a/tests/unit/test_datawarehouse_unit.py b/tests/unit/test_datawarehouse_unit.py
index d73b04df..2f3d55e7 100644
--- a/tests/unit/test_datawarehouse_unit.py
+++ b/tests/unit/test_datawarehouse_unit.py
@@ -50,14 +50,14 @@ def test___enter___RuntimeError_xdmod_host_malformed():
         )
     ):
         with DataWarehouse('https://'):
-            pass
+            pass  # pragma: no cover
 
 
 def test___enter___RuntimeError_xdmod_host_unresolved():
     invalid_host = 'https://' + INVALID_STR + '.xdmod.org'
     with pytest.raises(Exception):
         with DataWarehouse(invalid_host):
-            pass
+            pass  # pragma: no cover
 
 
 def test___enter___RuntimeError_xdmod_host_unsupported_protocol():
@@ -67,7 +67,7 @@ def test___enter___RuntimeError_xdmod_host_unsupported_protocol():
         match='No connection adapters were found for \'' + invalid_host
     ):
         with DataWarehouse(invalid_host):
-            pass
+            pass  # pragma: no cover
 
 
 def test___enter___RuntimeError_401():
diff --git a/xdmod_data/_descriptors.py b/xdmod_data/_descriptors.py
index 6387ba68..d5a24ff6 100644
--- a/xdmod_data/_descriptors.py
+++ b/xdmod_data/_descriptors.py
@@ -19,7 +19,7 @@ def __request_aggregate(self):
             '/controllers/metric_explorer.php',
             {'operation': 'get_dw_descripter'},
         )
-        if response['totalCount'] != 1:
+        if response['totalCount'] != 1:  # pragma: no cover
             raise RuntimeError(
                 'Descriptor received with unexpected structure.'
             )
diff --git a/xdmod_data/_http_requester.py b/xdmod_data/_http_requester.py
index ad5bc939..3e6cee68 100644
--- a/xdmod_data/_http_requester.py
+++ b/xdmod_data/_http_requester.py
@@ -49,7 +49,11 @@ def _request_raw_data(self, params):
         # necessary — only the body of the 'if' branch will be needed.
         limit = self.__get_raw_data_limit()
         data = []
-        if limit == 'NA':
+        # Once ACCESS XDMoD is on version 11.0 then the `pragma: no cover`
+        # below will need to be moved to the `else` statement so that
+        # Coverage.py will know that branch is not
+        # expected to be covered by tests.
+        if limit == 'NA':  # pragma: no cover
             response_iter_lines = self.__request(
                 path='/rest/v1/warehouse/raw-data?' + url_params,
                 post_fields=None,
@@ -123,7 +127,7 @@ def _request_json(self, path, post_fields=None):
     def __assert_connection_to_xdmod_host(self):
         try:
             self.__request()
-        except RuntimeError as e:
+        except RuntimeError as e:  # pragma: no cover
             raise RuntimeError(
                 'Could not connect to xdmod_host \'' + self.__xdmod_host
                 + '\': ' + str(e)
@@ -148,7 +152,7 @@ def __request(self, path='', post_fields=None, stream=False):
             try:
                 response_json = json.loads(response.text)
                 msg = ': ' + response_json['message']
-            except json.JSONDecodeError:
+            except json.JSONDecodeError:  # pragma: no cover
                 pass
             if response.status_code == 401:
                 msg = (
@@ -157,7 +161,11 @@ def __request(self, path='', post_fields=None, stream=False):
             raise RuntimeError(
                 'Error ' + str(response.status_code) + msg
             ) from None
-        if stream:
+        # Once ACCESS XDMoD is on version 11.0 then
+        # the `pragma: no cover` below
+        # will need to be removed so that Coverage.py
+        # will know that branch is expected to be covered by tests.
+        if stream:  # pragma: no cover
             return response.iter_lines()
         else:
             return response.text
@@ -195,8 +203,8 @@ def __get_raw_data_url_params(self, params):
                 )
         return urlencode(results)
 
-    # Once XDMoD 10.5 is no longer supported, there will be no need for this
-    # method.
+    #  Once XDMoD 10.5 is no longer supported,
+    #  there will be no need for this method
     def __get_raw_data_limit(self):
         if self.__raw_data_limit is None:
             try:
@@ -204,9 +212,13 @@ def __get_raw_data_limit(self):
                     '/rest/v1/warehouse/raw-data/limit'
                 )
                 self.__raw_data_limit = int(response['data'])
-            except RuntimeError as e:
+                # Once ACCESS XDMoD is on version 11.0 then
+                # the `pragma: no cover` below
+                # will need to be removed so that Coverage.py
+                # will know that branch is expected to be covered by tests.
+            except RuntimeError as e:  # pragma: no cover
                 if '404' in str(e):
                     self.__raw_data_limit = 'NA'
-                else:
+                else:  # pragma: no cover
                     raise
         return self.__raw_data_limit
diff --git a/xdmod_data/_response_processor.py b/xdmod_data/_response_processor.py
index ba06f925..661806c0 100644
--- a/xdmod_data/_response_processor.py
+++ b/xdmod_data/_response_processor.py
@@ -55,10 +55,7 @@ def __parse_timeseries_dimension_values(labels):
     dimension_values = []
     for label in labels:
         match = label_re.match(label)
-        if match:
-            dimension_values.append(html.unescape(match.group(1)))
-        else:
-            dimension_values.append(html.unescape(label))
+        dimension_values.append(html.unescape(match.group(1)))
     return dimension_values
 
 
@@ -75,7 +72,7 @@ def __parse_timeseries_date_string(date_string):
     # Match YYYY Q#
     elif re.match(r'^[0-9]{4} Q[0-9]$', date_string):
         (date_string, format_) = __parse_quarter_date_string(date_string)
-    else:
+    else:  # pragma: no cover
         raise Exception(
             'Unsupported date specification ' + date_string + '.'
         )
@@ -134,7 +131,7 @@ def __parse_quarter_date_string(date_string):
         month = '07'
     elif quarter == 'Q4':
         month = '10'
-    else:
+    else:  # pragma: no cover
         raise Exception(
             'Unsupported date quarter specification '
             + date_string + '.'
diff --git a/xdmod_data/_validator.py b/xdmod_data/_validator.py
index d80f399a..6a4e6da6 100644
--- a/xdmod_data/_validator.py
+++ b/xdmod_data/_validator.py
@@ -255,8 +255,8 @@ def __get_dates_from_duration(duration):
     last_quarter = today + timedelta(days=-90)
     this_month_start = date(today.year, today.month, 1)
     if today.month == 1:
-        last_full_month_start_year = today.year - 1
-        last_full_month_start_month = 12
+        last_full_month_start_year = today.year - 1  # pragma: no cover
+        last_full_month_start_month = 12  # pragma: no cover
     else:
         last_full_month_start_year = today.year
         last_full_month_start_month = today.month - 1
@@ -272,7 +272,7 @@ def __get_dates_from_duration(duration):
         1,
     )
     if today.month < 4:
-        last_quarter_start_year = today.year - 1
+        last_quarter_start_year = today.year - 1  # pragma: no cover
     else:
         last_quarter_start_year = today.year
     last_quarter_start = date(
@@ -340,7 +340,7 @@ def __date_add_years(old_date, year_delta):
         try:
             new_date = date(new_date_year, old_date.month, new_date_day)
             keep_going = False
-        except ValueError:
+        except ValueError:  # pragma: no cover
             new_date_day -= 1
             days_above += 1
     return new_date + timedelta(days=days_above)

From ff61aa04ef18fb2e30524b6fb7a70fb6fa9b1149 Mon Sep 17 00:00:00 2001
From: RodneyCodess <160431998+RodneyCodess@users.noreply.github.com>
Date: Mon, 24 Jun 2024 10:16:33 -0400
Subject: [PATCH 19/72] Apply suggestions from code review

Co-authored-by: Aaron Weeden <31246768+aaronweeden@users.noreply.github.com>
---
 docs/developing.md                                |  4 ++--
 tests/regression/test_datawarehouse_regression.py | 14 +++++++-------
 xdmod_data/_http_requester.py                     |  4 ++--
 3 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/docs/developing.md b/docs/developing.md
index 8aaa9bbc..5793d71b 100644
--- a/docs/developing.md
+++ b/docs/developing.md
@@ -11,7 +11,7 @@
     ```
     python3 -m pip install --force-reinstall -e /path/to/your/xdmod-data
     ```
-1. Install `python-dotenv` and `pytest` and `coverage`:
+1. Install `python-dotenv`, `pytest`, and `coverage`:
     ```
     python3 -m pip install --upgrade python-dotenv pytest coverage
     ```
@@ -26,7 +26,7 @@
     ```
     coverage run -m pytest -vvs -o log_cli=true tests/
     ```
-1. Run the following command and make sure code is 100% covered by tests
+1. Run the following command and make sure the code is 100% covered by tests:
     ```
     coverage report -m
     ```
diff --git a/tests/regression/test_datawarehouse_regression.py b/tests/regression/test_datawarehouse_regression.py
index 66514368..c601e1ef 100644
--- a/tests/regression/test_datawarehouse_regression.py
+++ b/tests/regression/test_datawarehouse_regression.py
@@ -26,7 +26,7 @@ def __assert_dfs_equal(
     actual,
     dtype='object',
     index_col='id',
-    columns_name=None
+    columns_name=None,
 ):
     expected = pandas.read_csv(
         DATA_DIR + '/' + data_file,
@@ -37,7 +37,7 @@ def __assert_dfs_equal(
     ).fillna(numpy.nan)
     expected.columns = expected.columns.astype('string')
     expected.columns.name = columns_name
-    if (index_col == 'Time'):
+    if index_col == 'Time':
         expected.index = pandas.to_datetime(expected.index)
     assert expected.equals(actual)
 
@@ -62,7 +62,7 @@ def test_get_raw_data(valid_dw, capsys):
                 'Bridges 2 RM',
             ],
         },
-        show_progress=True
+        show_progress=True,
     ).iloc[::1000]
     data.index = data.index.astype('string')
     __assert_dfs_equal(
@@ -123,15 +123,15 @@ def test_get_quarter(valid_dw):
         duration=('2023-01-01', '2023-12-31'),
         realm='Jobs',
         metric='CPU Hours: Total',
-        aggregation_unit='Quarter'
-        )
+        aggregation_unit='Quarter',
+    )
     __assert_dfs_equal(
         "jobs-2023-quarters.csv",
         data,
         index_col='Time',
         columns_name='Metric',
-        dtype={'CPU Hours: Total': 'Float64'}
-        )
+        dtype={'CPU Hours: Total': 'Float64'},
+    )
 
 
 def test_get_years(valid_dw):
diff --git a/xdmod_data/_http_requester.py b/xdmod_data/_http_requester.py
index 3e6cee68..93dc8112 100644
--- a/xdmod_data/_http_requester.py
+++ b/xdmod_data/_http_requester.py
@@ -203,8 +203,8 @@ def __get_raw_data_url_params(self, params):
                 )
         return urlencode(results)
 
-    #  Once XDMoD 10.5 is no longer supported,
-    #  there will be no need for this method
+    # Once XDMoD 10.5 is no longer supported,
+    # there will be no need for this method.
     def __get_raw_data_limit(self):
         if self.__raw_data_limit is None:
             try:

From 9a68d3afb9804ee22e05e4564170446b0c633528 Mon Sep 17 00:00:00 2001
From: Rodney Garnett <garnettr35@gmail.com>
Date: Mon, 24 Jun 2024 11:06:24 -0400
Subject: [PATCH 20/72] combined two tests into one

---
 .../test_datawarehouse_regression.py          | 32 +++++++------------
 xdmod_data/_validator.py                      | 12 +++----
 2 files changed, 18 insertions(+), 26 deletions(-)

diff --git a/tests/regression/test_datawarehouse_regression.py b/tests/regression/test_datawarehouse_regression.py
index c601e1ef..4f15f17c 100644
--- a/tests/regression/test_datawarehouse_regression.py
+++ b/tests/regression/test_datawarehouse_regression.py
@@ -118,15 +118,23 @@ def test_get_data_filter_user(valid_dw):
     )
 
 
-def test_get_quarter(valid_dw):
+@pytest.mark.parametrize(
+    "duration,aggregation_unit,data_file",
+    [
+        (('2023-01-01', '2023-12-31'), 'Quarter', "jobs-2023-quarters.csv"),
+        (('2022-01-01', '2023-12-31'), 'Year', "jobs-2022-2023-years.csv"),
+    ],
+    ids=('quarter', 'year'),
+)
+def test_get_data(valid_dw, duration, aggregation_unit, data_file):
     data = valid_dw.get_data(
-        duration=('2023-01-01', '2023-12-31'),
+        duration=duration,
         realm='Jobs',
         metric='CPU Hours: Total',
-        aggregation_unit='Quarter',
+        aggregation_unit=aggregation_unit,
     )
     __assert_dfs_equal(
-        "jobs-2023-quarters.csv",
+        data_file,
         data,
         index_col='Time',
         columns_name='Metric',
@@ -134,22 +142,6 @@ def test_get_quarter(valid_dw):
     )
 
 
-def test_get_years(valid_dw):
-    data = valid_dw.get_data(
-        duration=('2022-01-01', '2023-12-31'),
-        realm='Jobs',
-        metric='CPU Hours: Total',
-        aggregation_unit='Year'
-        )
-    __assert_dfs_equal(
-        "jobs-2022-2023-years.csv",
-        data,
-        index_col='Time',
-        columns_name='Metric',
-        dtype={'CPU Hours: Total': 'Float64'}
-        )
-
-
 def test_get_aggregation_units(valid_dw):
     expected_agg_units = ('Auto', 'Day', 'Month', 'Quarter', 'Year')
     actual_agg_units = valid_dw.get_aggregation_units()
diff --git a/xdmod_data/_validator.py b/xdmod_data/_validator.py
index 6a4e6da6..6df0e321 100644
--- a/xdmod_data/_validator.py
+++ b/xdmod_data/_validator.py
@@ -254,9 +254,9 @@ def __get_dates_from_duration(duration):
     last_month = today + timedelta(days=-30)
     last_quarter = today + timedelta(days=-90)
     this_month_start = date(today.year, today.month, 1)
-    if today.month == 1:
-        last_full_month_start_year = today.year - 1  # pragma: no cover
-        last_full_month_start_month = 12  # pragma: no cover
+    if today.month == 1:  # pragma: no cover
+        last_full_month_start_year = today.year - 1
+        last_full_month_start_month = 12
     else:
         last_full_month_start_year = today.year
         last_full_month_start_month = today.month - 1
@@ -271,9 +271,9 @@ def __get_dates_from_duration(duration):
         ((today.month - 1) // 3) * 3 + 1,
         1,
     )
-    if today.month < 4:
-        last_quarter_start_year = today.year - 1  # pragma: no cover
-    else:
+    if today.month < 4:  # pragma: no cover
+        last_quarter_start_year = today.year - 1
+    else:  # pragma: no cover
         last_quarter_start_year = today.year
     last_quarter_start = date(
         last_quarter_start_year,

From 50bc530a4e050b7b1fac48547a56e41482f63327 Mon Sep 17 00:00:00 2001
From: Rodney Garnett <garnettr35@gmail.com>
Date: Mon, 24 Jun 2024 11:22:12 -0400
Subject: [PATCH 21/72] updated CHANGELOG.md

---
 CHANGELOG.md | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 33cb26ad..b2d69432 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,8 @@
 # xdmod-data Changelog
 
+## Main development branch
+- Implement 100% test coverage ([\#27](https://github.com/ubccr/xdmod-data/pull/27)).
+
 ## v1.0.1 (2024-06-XX)
 This release has bug fixes, performance improvements, and updates for compatibility, tests, and documentation.
 - Add borders to README images ([\#12](https://github.com/ubccr/xdmod-data/pull/12)).

From 65b1e6c1a56d860dc0e22ebd54429daf017aa5f0 Mon Sep 17 00:00:00 2001
From: RodneyCodess <160431998+RodneyCodess@users.noreply.github.com>
Date: Mon, 24 Jun 2024 11:27:33 -0400
Subject: [PATCH 22/72] Update
 tests/regression/test_datawarehouse_regression.py

Co-authored-by: Aaron Weeden <31246768+aaronweeden@users.noreply.github.com>
---
 tests/regression/test_datawarehouse_regression.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/regression/test_datawarehouse_regression.py b/tests/regression/test_datawarehouse_regression.py
index 4f15f17c..11ccbbe4 100644
--- a/tests/regression/test_datawarehouse_regression.py
+++ b/tests/regression/test_datawarehouse_regression.py
@@ -119,7 +119,7 @@ def test_get_data_filter_user(valid_dw):
 
 
 @pytest.mark.parametrize(
-    "duration,aggregation_unit,data_file",
+    'duration,aggregation_unit,data_file',
     [
         (('2023-01-01', '2023-12-31'), 'Quarter', "jobs-2023-quarters.csv"),
         (('2022-01-01', '2023-12-31'), 'Year', "jobs-2022-2023-years.csv"),

From d419b74bd404547e537e2f4efe76dfbd26ef672d Mon Sep 17 00:00:00 2001
From: RodneyCodess <160431998+RodneyCodess@users.noreply.github.com>
Date: Mon, 24 Jun 2024 11:27:50 -0400
Subject: [PATCH 23/72] Update
 tests/regression/test_datawarehouse_regression.py

Co-authored-by: Aaron Weeden <31246768+aaronweeden@users.noreply.github.com>
---
 tests/regression/test_datawarehouse_regression.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/regression/test_datawarehouse_regression.py b/tests/regression/test_datawarehouse_regression.py
index 11ccbbe4..34ff1600 100644
--- a/tests/regression/test_datawarehouse_regression.py
+++ b/tests/regression/test_datawarehouse_regression.py
@@ -121,8 +121,8 @@ def test_get_data_filter_user(valid_dw):
 @pytest.mark.parametrize(
     'duration,aggregation_unit,data_file',
     [
-        (('2023-01-01', '2023-12-31'), 'Quarter', "jobs-2023-quarters.csv"),
-        (('2022-01-01', '2023-12-31'), 'Year', "jobs-2022-2023-years.csv"),
+        (('2023-01-01', '2023-12-31'), 'Quarter', 'jobs-2023-quarters.csv'),
+        (('2022-01-01', '2023-12-31'), 'Year', 'jobs-2022-2023-years.csv'),
     ],
     ids=('quarter', 'year'),
 )

From 6eb6a0338d55ba935f85eb5feacbc16ea4949d97 Mon Sep 17 00:00:00 2001
From: Rodney Garnett <garnettr35@gmail.com>
Date: Mon, 24 Jun 2024 11:30:30 -0400
Subject: [PATCH 24/72] changed version

---
 xdmod_data/__version__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/xdmod_data/__version__.py b/xdmod_data/__version__.py
index 09869ff8..475e210b 100644
--- a/xdmod_data/__version__.py
+++ b/xdmod_data/__version__.py
@@ -1,2 +1,2 @@
 __title__ = 'xdmod-data'
-__version__ = '1.0.1'
+__version__ = '1.1.0-beta.1'

From a3ee3e1ea472d412ac051f223ade03e47b4258e6 Mon Sep 17 00:00:00 2001
From: Rodney Garnett <garnettr35@gmail.com>
Date: Mon, 24 Jun 2024 12:04:43 -0400
Subject: [PATCH 25/72] Flake8

---
 .github/workflows/python-package.yml |  4 +---
 docs/developing.md                   | 16 +++++++++++++++-
 2 files changed, 16 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml
index 5718eea3..3a722f94 100644
--- a/.github/workflows/python-package.yml
+++ b/.github/workflows/python-package.yml
@@ -39,9 +39,7 @@ jobs:
     - name: Lint with flake8
       run: |
         # stop the build if there are linter errors (ignore complexity metrics)
-        flake8 . --count --max-line-length=127 --max-complexity=1000 --show-source --statistics --exclude __init__.py
-        # output complexity metrics
-        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics --exclude __init__.py
+        flake8 . --max-complexity=10 --show-source --exclude __init__.py
     - name: Test with pytest
       run: |
         pytest tests/unit
diff --git a/docs/developing.md b/docs/developing.md
index 5793d71b..3b6b2b06 100644
--- a/docs/developing.md
+++ b/docs/developing.md
@@ -38,7 +38,21 @@
     ```
     pytest -vvs -o log_cli=true tests/
     ```
-    
+## Linting the code
+1. Start up a virtual environment, e.g.:
+    ```                                            
+    python3 -m venv ~/xdmod-data-test-env                                             
+    source ~/xdmod-data-test-env/bin/activate
+    ```
+    Your command prompt should now start with `(xdmod-data-test-env)`.
+1. Install Flake8
+    ```
+    python3 -m pip install flake8
+    ```
+1. Run Flake8
+    ```
+    flake8 --max-complexity=10 --show-source --exclude __init__.py /path/to/your/xdmod-data
+    ```
 ## Releasing a new version
 1. Make a new branch of `xdmod-data` and:
 

From 2a52a69c6b2eca7487d69420ae50a285d2fa662b Mon Sep 17 00:00:00 2001
From: Rodney Garnett <garnettr35@gmail.com>
Date: Mon, 24 Jun 2024 14:35:34 -0400
Subject: [PATCH 26/72] Flake8 additional packages

---
 docs/developing.md                            |   4 +-
 .../test_datawarehouse_integration.py         |   9 +-
 tests/unit/test_datawarehouse_unit.py         |  10 +-
 xdmod_data/_descriptors.py                    |   4 +-
 xdmod_data/_http_requester.py                 |  16 +--
 xdmod_data/_response_processor.py             |   4 +-
 xdmod_data/_validator.py                      |  18 +--
 xdmod_data/themes.py                          | 132 +++++++++---------
 8 files changed, 99 insertions(+), 98 deletions(-)

diff --git a/docs/developing.md b/docs/developing.md
index 3b6b2b06..816746f7 100644
--- a/docs/developing.md
+++ b/docs/developing.md
@@ -45,9 +45,9 @@
     source ~/xdmod-data-test-env/bin/activate
     ```
     Your command prompt should now start with `(xdmod-data-test-env)`.
-1. Install Flake8
+1. Install Flake8 and additional packages
     ```
-    python3 -m pip install flake8
+    python3 -m pip install flake8 flake8-quotes flake8-commas
     ```
 1. Run Flake8
     ```
diff --git a/tests/integration/test_datawarehouse_integration.py b/tests/integration/test_datawarehouse_integration.py
index bca063e7..65dc0eb5 100644
--- a/tests/integration/test_datawarehouse_integration.py
+++ b/tests/integration/test_datawarehouse_integration.py
@@ -28,7 +28,7 @@
     'describe_realms': (),
     'describe_metrics': ('realm',),
     'describe_dimensions': ('realm',),
-    'get_filter_values': ('realm', 'dimension',),
+    'get_filter_values': ('realm', 'dimension'),
     'describe_raw_realms': (),
     'describe_raw_fields': ('realm',),
 }
@@ -53,7 +53,8 @@
     'dimension': (INVALID_STR, r'Dimension .* not found'),
     'filter_key': ({INVALID_STR: INVALID_STR}, r'Dimension .* not found'),
     'filter_value': (
-        {VALID_DIMENSION: INVALID_STR}, r'Filter value .* not found'
+        {VALID_DIMENSION: INVALID_STR},
+        r'Filter value .* not found',
     ),
     'dataset_type': (INVALID_STR, 'Invalid value for `dataset_type`'),
     'aggregation_unit': (INVALID_STR, 'Invalid value for `aggregation_unit`'),
@@ -281,7 +282,7 @@ def __test_DataFrame_return_value(
         (
             {
                 'dimension': 'Resource',
-                'filters': {'Service Provider': 'StonyBrook'}
+                'filters': {'Service Provider': 'StonyBrook'},
             },
             'Resource',
             0,
@@ -352,7 +353,7 @@ def test_get_data_timeseries_return_value(
         (
             {
                 'dimension': 'Resource',
-                'filters': {'Service Provider': 'StonyBrook'}
+                'filters': {'Service Provider': 'StonyBrook'},
             },
             'Resource',
             0,
diff --git a/tests/unit/test_datawarehouse_unit.py b/tests/unit/test_datawarehouse_unit.py
index 2f3d55e7..1cf04f43 100644
--- a/tests/unit/test_datawarehouse_unit.py
+++ b/tests/unit/test_datawarehouse_unit.py
@@ -40,14 +40,14 @@ def test___enter___RuntimeError_xdmod_host_malformed():
     with pytest.raises(
         (
             requests.exceptions.InvalidURL,
-            requests.exceptions.MissingSchema
+            requests.exceptions.MissingSchema,
         ),
         match=(
             r'(Invalid URL \'.*\': No host supplied|'
-            + r'Invalid URL \'https:\?Bearer=' + INVALID_STR + '\': '
+            + r'Invalid URL \'https:\?Bearer=' + INVALID_STR + "': "
             + r'No schema supplied. Perhaps you meant http://https:\?Bearer='
             + INVALID_STR + r'\?)'
-        )
+        ),
     ):
         with DataWarehouse('https://'):
             pass  # pragma: no cover
@@ -64,7 +64,7 @@ def test___enter___RuntimeError_xdmod_host_unsupported_protocol():
     invalid_host = INVALID_STR + '://' + INVALID_STR
     with pytest.raises(
         requests.exceptions.InvalidSchema,
-        match='No connection adapters were found for \'' + invalid_host
+        match="No connection adapters were found for '" + invalid_host,
     ):
         with DataWarehouse(invalid_host):
             pass  # pragma: no cover
@@ -74,7 +74,7 @@ def test___enter___RuntimeError_401():
     with pytest.raises(
         RuntimeError,
         match='Error 401: Make sure XDMOD_API_TOKEN is set'
-        + ' to a valid API token.'
+        + ' to a valid API token.',
     ):
         with DataWarehouse(VALID_XDMOD_URL) as dw:
             dw.describe_realms()
diff --git a/xdmod_data/_descriptors.py b/xdmod_data/_descriptors.py
index d5a24ff6..5826737e 100644
--- a/xdmod_data/_descriptors.py
+++ b/xdmod_data/_descriptors.py
@@ -21,13 +21,13 @@ def __request_aggregate(self):
         )
         if response['totalCount'] != 1:  # pragma: no cover
             raise RuntimeError(
-                'Descriptor received with unexpected structure.'
+                'Descriptor received with unexpected structure.',
             )
         return self.__deserialize_aggregate(response['data'][0]['realms'])
 
     def __request_raw(self):
         response = self.__http_requester._request_json(
-            '/rest/v1/warehouse/export/realms'
+            '/rest/v1/warehouse/export/realms',
         )
         return self.__deserialize_raw(response['data'])
 
diff --git a/xdmod_data/_http_requester.py b/xdmod_data/_http_requester.py
index 93dc8112..26206668 100644
--- a/xdmod_data/_http_requester.py
+++ b/xdmod_data/_http_requester.py
@@ -17,7 +17,7 @@ def __init__(self, xdmod_host):
             self.__api_token = os.environ['XDMOD_API_TOKEN']
         except KeyError:
             raise KeyError(
-                '`XDMOD_API_TOKEN` environment variable has not been set.'
+                '`XDMOD_API_TOKEN` environment variable has not been set.',
             ) from None
         self.__headers = {
             'Authorization': 'Bearer ' + self.__api_token,
@@ -82,7 +82,7 @@ def _request_raw_data(self, params):
             while num_rows == limit:
                 response = self._request_json(
                     path='/rest/v1/warehouse/raw-data?' + url_params
-                    + '&offset=' + str(offset)
+                    + '&offset=' + str(offset),
                 )
                 partial_data = response['data']
                 data += partial_data
@@ -129,8 +129,8 @@ def __assert_connection_to_xdmod_host(self):
             self.__request()
         except RuntimeError as e:  # pragma: no cover
             raise RuntimeError(
-                'Could not connect to xdmod_host \'' + self.__xdmod_host
-                + '\': ' + str(e)
+                "Could not connect to xdmod_host '" + self.__xdmod_host
+                + "': " + str(e),
             ) from None
 
     def __request(self, path='', post_fields=None, stream=False):
@@ -159,7 +159,7 @@ def __request(self, path='', post_fields=None, stream=False):
                     ': Make sure XDMOD_API_TOKEN is set to a valid API token.'
                 )
             raise RuntimeError(
-                'Error ' + str(response.status_code) + msg
+                'Error ' + str(response.status_code) + msg,
             ) from None
         # Once ACCESS XDMoD is on version 11.0 then
         # the `pragma: no cover` below
@@ -184,7 +184,7 @@ def __get_data_post_fields(self, params):
         }
         for dimension in params['filters']:
             post_fields[dimension + '_filter'] = ','.join(
-                params['filters'][dimension]
+                params['filters'][dimension],
             )
         return post_fields
 
@@ -199,7 +199,7 @@ def __get_raw_data_url_params(self, params):
         if (params['filters']):
             for dimension in params['filters']:
                 results['filters[' + dimension + ']'] = ','.join(
-                    params['filters'][dimension]
+                    params['filters'][dimension],
                 )
         return urlencode(results)
 
@@ -209,7 +209,7 @@ def __get_raw_data_limit(self):
         if self.__raw_data_limit is None:
             try:
                 response = self._request_json(
-                    '/rest/v1/warehouse/raw-data/limit'
+                    '/rest/v1/warehouse/raw-data/limit',
                 )
                 self.__raw_data_limit = int(response['data'])
                 # Once ACCESS XDMoD is on version 11.0 then
diff --git a/xdmod_data/_response_processor.py b/xdmod_data/_response_processor.py
index 661806c0..9a72d9fc 100644
--- a/xdmod_data/_response_processor.py
+++ b/xdmod_data/_response_processor.py
@@ -74,7 +74,7 @@ def __parse_timeseries_date_string(date_string):
         (date_string, format_) = __parse_quarter_date_string(date_string)
     else:  # pragma: no cover
         raise Exception(
-            'Unsupported date specification ' + date_string + '.'
+            'Unsupported date specification ' + date_string + '.',
         )
     return datetime.strptime(date_string, format_)
 
@@ -134,7 +134,7 @@ def __parse_quarter_date_string(date_string):
     else:  # pragma: no cover
         raise Exception(
             'Unsupported date quarter specification '
-            + date_string + '.'
+            + date_string + '.',
         )
     date_string = year + '-' + month + '-01'
     format_ = '%Y-%m-%d'
diff --git a/xdmod_data/_validator.py b/xdmod_data/_validator.py
index 6df0e321..93fb27e6 100644
--- a/xdmod_data/_validator.py
+++ b/xdmod_data/_validator.py
@@ -10,7 +10,7 @@ def _assert_runtime_context(in_runtime_context):
         raise RuntimeError(
             'Method is being called outside of the runtime context.'
             + ' Make sure this method is only called within the body'
-            + ' of a `with` statement.'
+            + ' of a `with` statement.',
         )
 
 
@@ -94,7 +94,7 @@ def _get_durations():
     this_year = date.today().year
     six_years_ago = this_year - 6
     last_seven_years = tuple(
-        map(str, reversed(range(six_years_ago, this_year + 1)))
+        map(str, reversed(range(six_years_ago, this_year + 1))),
     )
     return (
         (
@@ -156,7 +156,7 @@ def __validate_duration(duration):
         except (TypeError, ValueError) as error:
             raise type(error)(
                 '`duration` must be a string or an object'
-                + ' with 2 items.'
+                + ' with 2 items.',
             ) from None
     return (start_date, end_date)
 
@@ -194,7 +194,7 @@ def __validate_filters(data_warehouse, descriptors, realm, filters):
     except TypeError:
         raise TypeError(
             '`filters` must be a mapping whose keys are strings and whose'
-            + ' values are strings or sequences of strings.'
+            + ' values are strings or sequences of strings.',
         ) from None
 
 
@@ -210,8 +210,8 @@ def __find_str_in_sequence(value, sequence, label):
         if transformed_valid_value == transformed_value:
             return valid_value
     raise KeyError(
-        'Invalid value for `' + label + '`: \'' + value + '\''
-        + '. Valid values are: \'' + '\', \''.join(sequence) + '\'.'
+        'Invalid value for `' + label + "`: '" + value + "'"
+        + ". Valid values are: '" + "', '".join(sequence) + "'.",
     ) from None
 
 
@@ -225,7 +225,7 @@ def __validate_raw_fields(data_warehouse, realm, fields):
         return results
     except TypeError:
         raise TypeError(
-            '`fields` must be a sequence of strings.'
+            '`fields` must be a sequence of strings.',
         ) from None
 
 
@@ -235,7 +235,7 @@ def __find_id_in_descriptor(descriptor, name, value):
         if id_ == value or descriptor[id_]['label'] == value:
             return id_
     raise KeyError(
-        name.capitalize() + ' \'' + value + '\' not found.'
+        name.capitalize() + " '" + value + "' not found.",
     )
 
 
@@ -321,7 +321,7 @@ def __find_value_in_df(label, df, value):
     elif value in df['label'].values:
         return df.index[df['label'] == value].tolist()[0]
     else:
-        raise KeyError(label + ' \'' + value + '\' not found.')
+        raise KeyError(label + " '" + value + "' not found.")
 
 
 def __lowercase_and_remove_spaces(value):
diff --git a/xdmod_data/themes.py b/xdmod_data/themes.py
index 14d4bf29..ebf41167 100644
--- a/xdmod_data/themes.py
+++ b/xdmod_data/themes.py
@@ -1,76 +1,76 @@
 import plotly.graph_objects as go
 import plotly.io as pio
 
-pio.templates["timeseries"] = go.layout.Template(
+pio.templates['timeseries'] = go.layout.Template(
     layout={
         'hoverlabel': {
-            'bgcolor': 'white'
+            'bgcolor': 'white',
         },
         'plot_bgcolor': 'white',
         'colorway': [
-            "#1199FF",
-            "#DB4230",
-            "#4E665D",
-            "#F4A221",
-            "#66FF00",
-            "#33ABAB",
-            "#A88D95",
-            "#789ABC",
-            "#FF99CC",
-            "#00CCFF",
-            "#FFBC71",
-            "#A57E81",
-            "#8D4DFF",
-            "#FF6666",
-            "#CC99FF",
-            "#2F7ED8",
-            "#0D233A",
-            "#8BBC21",
-            "#910000",
-            "#1AADCE",
-            "#492970",
-            "#F28F43",
-            "#77A1E5",
-            "#3366FF",
-            "#FF6600",
-            "#808000",
-            "#CC99FF",
-            "#008080",
-            "#CC6600",
-            "#9999FF",
-            "#99FF99",
-            "#969696",
-            "#FF00FF",
-            "#FFCC00",
-            "#666699",
-            "#00FFFF",
-            "#00CCFF",
-            "#993366",
-            "#3AAAAA",
-            "#C0C0C0",
-            "#FF99CC",
-            "#FFCC99",
-            "#CCFFCC",
-            "#CCFFFF",
-            "#99CCFF",
-            "#339966",
-            "#FF9966",
-            "#69BBED",
-            "#33FF33",
-            "#6666FF",
-            "#FF66FF",
-            "#99ABAB",
-            "#AB8722",
-            "#AB6565",
-            "#990099",
-            "#999900",
-            "#CC3300",
-            "#669999",
-            "#993333",
-            "#339966",
-            "#C42525",
-            "#A6C96A",
-            "#111111",
+            '#1199FF',
+            '#DB4230',
+            '#4E665D',
+            '#F4A221',
+            '#66FF00',
+            '#33ABAB',
+            '#A88D95',
+            '#789ABC',
+            '#FF99CC',
+            '#00CCFF',
+            '#FFBC71',
+            '#A57E81',
+            '#8D4DFF',
+            '#FF6666',
+            '#CC99FF',
+            '#2F7ED8',
+            '#0D233A',
+            '#8BBC21',
+            '#910000',
+            '#1AADCE',
+            '#492970',
+            '#F28F43',
+            '#77A1E5',
+            '#3366FF',
+            '#FF6600',
+            '#808000',
+            '#CC99FF',
+            '#008080',
+            '#CC6600',
+            '#9999FF',
+            '#99FF99',
+            '#969696',
+            '#FF00FF',
+            '#FFCC00',
+            '#666699',
+            '#00FFFF',
+            '#00CCFF',
+            '#993366',
+            '#3AAAAA',
+            '#C0C0C0',
+            '#FF99CC',
+            '#FFCC99',
+            '#CCFFCC',
+            '#CCFFFF',
+            '#99CCFF',
+            '#339966',
+            '#FF9966',
+            '#69BBED',
+            '#33FF33',
+            '#6666FF',
+            '#FF66FF',
+            '#99ABAB',
+            '#AB8722',
+            '#AB6565',
+            '#990099',
+            '#999900',
+            '#CC3300',
+            '#669999',
+            '#993333',
+            '#339966',
+            '#C42525',
+            '#A6C96A',
+            '#111111',
         ],
         'xaxis': {
             'titlefont': {
@@ -88,7 +88,7 @@
             'zerolinecolor': '#c0cfe1',
             'showline': True,
             'zerolinewidth': 4,
-            'tickformat': "%Y-%m-%d",
+            'tickformat': '%Y-%m-%d',
         },
         'yaxis': {
             'titlefont': {

From bc64e3f9fae39af67a1c2c306fb5e98981c9c661 Mon Sep 17 00:00:00 2001
From: Rodney Garnett <garnettr35@gmail.com>
Date: Mon, 24 Jun 2024 14:41:25 -0400
Subject: [PATCH 27/72] github updated flake8 check

---
 .github/workflows/python-package.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml
index 3a722f94..396aa290 100644
--- a/.github/workflows/python-package.yml
+++ b/.github/workflows/python-package.yml
@@ -33,7 +33,7 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        python -m pip install flake8 pytest pandas
+        python -m pip install flake8 flake8-commas flake8-quotes pytest pandas
         if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
         python -m pip install -e .
     - name: Lint with flake8

From ce66badaec84b2ce59c8fad607759dc17359a63c Mon Sep 17 00:00:00 2001
From: Rodney Garnett <garnettr35@gmail.com>
Date: Mon, 24 Jun 2024 15:14:16 -0400
Subject: [PATCH 28/72] updated CHANGELOG.md

---
 CHANGELOG.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index b2d69432..59f2daf5 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,7 @@
 
 ## Main development branch
 - Implement 100% test coverage ([\#27](https://github.com/ubccr/xdmod-data/pull/27)).
+- Update Flake8 rules ([\#28](https://github.com/ubccr/xdmod-data/pull/28)).
 
 ## v1.0.1 (2024-06-XX)
 This release has bug fixes, performance improvements, and updates for compatibility, tests, and documentation.

From 34bebb09579f1ba3e342b779f48f2b69741bebb8 Mon Sep 17 00:00:00 2001
From: Rodney Garnett <garnettr35@gmail.com>
Date: Tue, 25 Jun 2024 16:08:11 -0400
Subject: [PATCH 29/72] github action test

---
 .github/workflows/python-package.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml
index 396aa290..9698d6d8 100644
--- a/.github/workflows/python-package.yml
+++ b/.github/workflows/python-package.yml
@@ -43,3 +43,5 @@ jobs:
     - name: Test with pytest
       run: |
         pytest tests/unit
+    - name: Hello world!!!
+      run: echo "name"

From 256030294445bbce2febae21d01640cae8c172f3 Mon Sep 17 00:00:00 2001
From: Rodney Garnett <garnettr35@gmail.com>
Date: Tue, 25 Jun 2024 16:27:35 -0400
Subject: [PATCH 30/72] added compose file

---
 .github/workflows/docker-compose.yaml | 14 ++++++++++++++
 .github/workflows/python-package.yml  |  4 ++++
 2 files changed, 18 insertions(+)
 create mode 100644 .github/workflows/docker-compose.yaml

diff --git a/.github/workflows/docker-compose.yaml b/.github/workflows/docker-compose.yaml
new file mode 100644
index 00000000..b0de57c5
--- /dev/null
+++ b/.github/workflows/docker-compose.yaml
@@ -0,0 +1,14 @@
+services:
+    xdmod10.5:
+        container_name: xdmod10.5
+        image: tools-ext-01.ccr.xdmod.org/xdmod-10.5.0-x86_64:rockylinux8.5-0.3
+        tty: true
+        ports: 
+          - "8081:443"
+           
+    xdmod11.0:
+        container_name: xdmod11.0
+        image: tools-ext-01.ccr.xdmod.org/xdmod-10.5.0-x86_64:rockylinux8.5-0.3
+        tty: true
+        ports:
+          - "8082:443"
diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml
index 9698d6d8..b74d8a1c 100644
--- a/.github/workflows/python-package.yml
+++ b/.github/workflows/python-package.yml
@@ -40,6 +40,10 @@ jobs:
       run: |
         # stop the build if there are linter errors (ignore complexity metrics)
         flake8 . --max-complexity=10 --show-source --exclude __init__.py
+    - name: Spin up containers for testing XDMoD webservers
+      uses: hoverkraft-tech/compose-action@v2.0.1
+      with: 
+        compose-file: .github/workflows/docker-compose.yaml
     - name: Test with pytest
       run: |
         pytest tests/unit

From e19ae8c562b2276ab5543c806304c636b840840c Mon Sep 17 00:00:00 2001
From: Rodney Garnett <garnettr35@gmail.com>
Date: Thu, 27 Jun 2024 11:13:46 -0400
Subject: [PATCH 31/72] Github actions check 1

---
 .github/workflows/docker-compose.yaml |  3 +--
 .github/workflows/python-package.yml  | 21 +++++++++++++++++----
 2 files changed, 18 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/docker-compose.yaml b/.github/workflows/docker-compose.yaml
index b0de57c5..fc2469c3 100644
--- a/.github/workflows/docker-compose.yaml
+++ b/.github/workflows/docker-compose.yaml
@@ -4,8 +4,7 @@ services:
         image: tools-ext-01.ccr.xdmod.org/xdmod-10.5.0-x86_64:rockylinux8.5-0.3
         tty: true
         ports: 
-          - "8081:443"
-           
+          - "8081:443"      
     xdmod11.0:
         container_name: xdmod11.0
         image: tools-ext-01.ccr.xdmod.org/xdmod-10.5.0-x86_64:rockylinux8.5-0.3
diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml
index b74d8a1c..ca968d33 100644
--- a/.github/workflows/python-package.yml
+++ b/.github/workflows/python-package.yml
@@ -44,8 +44,21 @@ jobs:
       uses: hoverkraft-tech/compose-action@v2.0.1
       with: 
         compose-file: .github/workflows/docker-compose.yaml
-    - name: Test with pytest
+    - name: Install yq 
       run: |
-        pytest tests/unit
-    - name: Hello world!!!
-      run: echo "name"
+        sudo add-apt-repository ppa:rmescandon/yq
+        sudo apt update
+        sudo apt install yq -y
+    - name: Test against each supported XDMoD portal version
+      run: |
+        declare -a portal_versions=$(yq '.services | keys[]' .github/workflows/docker-compose.yaml)
+        for version in portal_versions; do
+          docker exec $version /root/bin/services start
+          port_version=$(yq ".services.$version.ports")
+          port=${port_version:0:4}
+          token=$(curl -X POST -c xdmod.cookie -d "username=normaluser&password=normaluser" https://localhost:$port/rest/auth/login -k | jq -r '.results.token')
+          curl -b xdmod.cookie https://localhost:port/rest/users/current/api/token?token=$token -k -X POST | jq -r '.data.token' > ~/.xdmod-data-token
+          coverage erase
+          coverage run --branch --append -m pytest -vvs -o log_cli=true tests/
+        done
+        coverage report -m --fail-under=100
\ No newline at end of file

From f5929295ec7f674bd8530be40f54d72852c51d87 Mon Sep 17 00:00:00 2001
From: Rodney Garnett <garnettr35@gmail.com>
Date: Thu, 27 Jun 2024 11:28:55 -0400
Subject: [PATCH 32/72] Github actions check 2

---
 .github/workflows/python-package.yml | 14 ++++++--------
 1 file changed, 6 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml
index ca968d33..30e45951 100644
--- a/.github/workflows/python-package.yml
+++ b/.github/workflows/python-package.yml
@@ -46,19 +46,17 @@ jobs:
         compose-file: .github/workflows/docker-compose.yaml
     - name: Install yq 
       run: |
-        sudo add-apt-repository ppa:rmescandon/yq
-        sudo apt update
-        sudo apt install yq -y
+        wget https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 -O /usr/bin/yq && chmod +x /usr/bin/yq
     - name: Test against each supported XDMoD portal version
       run: |
+        coverage erase
         declare -a portal_versions=$(yq '.services | keys[]' .github/workflows/docker-compose.yaml)
         for version in portal_versions; do
           docker exec $version /root/bin/services start
-          port_version=$(yq ".services.$version.ports")
-          port=${port_version:0:4}
-          token=$(curl -X POST -c xdmod.cookie -d "username=normaluser&password=normaluser" https://localhost:$port/rest/auth/login -k | jq -r '.results.token')
-          curl -b xdmod.cookie https://localhost:port/rest/users/current/api/token?token=$token -k -X POST | jq -r '.data.token' > ~/.xdmod-data-token
-          coverage erase
+          ports=$(yq ".services.$version.ports")
+          port=$(echo $ports | cut -d ':' -f 1)
+          token=$(curl -k -X POST -c xdmod.cookie -d "username=normaluser&password=normaluser" https://localhost:$port/rest/auth/login | jq -r '.results.token')
+          curl -k -X POST -b xdmod.cookie https://localhost:port/rest/users/current/api/token?token=$token | jq -r '.data.token' > ~/.xdmod-data-token
           coverage run --branch --append -m pytest -vvs -o log_cli=true tests/
         done
         coverage report -m --fail-under=100
\ No newline at end of file

From db7f6b2ac83e41b80ba8768c1049af221e9f864d Mon Sep 17 00:00:00 2001
From: Rodney Garnett <garnettr35@gmail.com>
Date: Thu, 27 Jun 2024 11:39:08 -0400
Subject: [PATCH 33/72] Github actions check 3

---
 .github/workflows/python-package.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml
index 30e45951..dd726546 100644
--- a/.github/workflows/python-package.yml
+++ b/.github/workflows/python-package.yml
@@ -46,7 +46,7 @@ jobs:
         compose-file: .github/workflows/docker-compose.yaml
     - name: Install yq 
       run: |
-        wget https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 -O /usr/bin/yq && chmod +x /usr/bin/yq
+        sudo wget https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 -O /usr/bin/yq && chmod +x /usr/bin/yq
     - name: Test against each supported XDMoD portal version
       run: |
         coverage erase

From 6947f34faa5c593e17fc9210650f80a6e9d4a792 Mon Sep 17 00:00:00 2001
From: Rodney Garnett <garnettr35@gmail.com>
Date: Thu, 27 Jun 2024 11:49:26 -0400
Subject: [PATCH 34/72] Github actions check 4

---
 .github/workflows/python-package.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml
index dd726546..b00e0c64 100644
--- a/.github/workflows/python-package.yml
+++ b/.github/workflows/python-package.yml
@@ -46,7 +46,7 @@ jobs:
         compose-file: .github/workflows/docker-compose.yaml
     - name: Install yq 
       run: |
-        sudo wget https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 -O /usr/bin/yq && chmod +x /usr/bin/yq
+        sudo chmod wget https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 -O /usr/bin/yq && chmod +x /usr/bin/yq
     - name: Test against each supported XDMoD portal version
       run: |
         coverage erase

From 3f0fede0ba1677f817f041c5578828d2205c93e0 Mon Sep 17 00:00:00 2001
From: Rodney Garnett <garnettr35@gmail.com>
Date: Thu, 27 Jun 2024 12:08:01 -0400
Subject: [PATCH 35/72] Github actions check 5

---
 .github/workflows/python-package.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml
index b00e0c64..46f59ce5 100644
--- a/.github/workflows/python-package.yml
+++ b/.github/workflows/python-package.yml
@@ -46,7 +46,7 @@ jobs:
         compose-file: .github/workflows/docker-compose.yaml
     - name: Install yq 
       run: |
-        sudo chmod wget https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 -O /usr/bin/yq && chmod +x /usr/bin/yq
+        sudo chmod wget https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 -O /usr/bin/yq && sudo chmod +x /usr/bin/yq
     - name: Test against each supported XDMoD portal version
       run: |
         coverage erase

From 33e3708a80f2b39af7532943cfcc4364f54ad67c Mon Sep 17 00:00:00 2001
From: Rodney Garnett <garnettr35@gmail.com>
Date: Thu, 27 Jun 2024 12:12:44 -0400
Subject: [PATCH 36/72] Github actions check 6

---
 .github/workflows/python-package.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml
index 46f59ce5..2bc379ef 100644
--- a/.github/workflows/python-package.yml
+++ b/.github/workflows/python-package.yml
@@ -46,7 +46,7 @@ jobs:
         compose-file: .github/workflows/docker-compose.yaml
     - name: Install yq 
       run: |
-        sudo chmod wget https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 -O /usr/bin/yq && sudo chmod +x /usr/bin/yq
+        sudo wget https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 -O /usr/bin/yq && sudo chmod +x /usr/bin/yq
     - name: Test against each supported XDMoD portal version
       run: |
         coverage erase

From f0ef7eef0a58c73afd1e61951826670e0cad3827 Mon Sep 17 00:00:00 2001
From: Rodney Garnett <garnettr35@gmail.com>
Date: Thu, 27 Jun 2024 13:22:26 -0400
Subject: [PATCH 37/72] Github actions check 7

---
 .github/workflows/python-package.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml
index 2bc379ef..7e8d790c 100644
--- a/.github/workflows/python-package.yml
+++ b/.github/workflows/python-package.yml
@@ -33,7 +33,7 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        python -m pip install flake8 flake8-commas flake8-quotes pytest pandas
+        python -m pip install flake8 flake8-commas flake8-quotes pytest pandas coverage
         if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
         python -m pip install -e .
     - name: Lint with flake8

From 21b9e6eebba5f3329d3ea4a9d18ed7eee910071f Mon Sep 17 00:00:00 2001
From: Rodney Garnett <garnettr35@gmail.com>
Date: Thu, 27 Jun 2024 13:37:04 -0400
Subject: [PATCH 38/72] Github actions check 9

---
 .github/workflows/python-package.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml
index 7e8d790c..4ac8425f 100644
--- a/.github/workflows/python-package.yml
+++ b/.github/workflows/python-package.yml
@@ -51,6 +51,7 @@ jobs:
       run: |
         coverage erase
         declare -a portal_versions=$(yq '.services | keys[]' .github/workflows/docker-compose.yaml)
+        echo $portal_versions
         for version in portal_versions; do
           docker exec $version /root/bin/services start
           ports=$(yq ".services.$version.ports")

From 368be3217d02cd5ddd6f7773e51721d8e4750ce2 Mon Sep 17 00:00:00 2001
From: Rodney Garnett <garnettr35@gmail.com>
Date: Thu, 27 Jun 2024 13:41:08 -0400
Subject: [PATCH 39/72] Github actions check 10

---
 .github/workflows/python-package.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml
index 4ac8425f..0c0d9035 100644
--- a/.github/workflows/python-package.yml
+++ b/.github/workflows/python-package.yml
@@ -51,8 +51,7 @@ jobs:
       run: |
         coverage erase
         declare -a portal_versions=$(yq '.services | keys[]' .github/workflows/docker-compose.yaml)
-        echo $portal_versions
-        for version in portal_versions; do
+        for version in $portal_versions; do
           docker exec $version /root/bin/services start
           ports=$(yq ".services.$version.ports")
           port=$(echo $ports | cut -d ':' -f 1)

From 737a6536af53d930b385b57ba57a0a6908111ac7 Mon Sep 17 00:00:00 2001
From: Rodney Garnett <garnettr35@gmail.com>
Date: Thu, 27 Jun 2024 14:06:30 -0400
Subject: [PATCH 40/72] Github actions check 11

---
 .github/workflows/python-package.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml
index 0c0d9035..8f7fbc46 100644
--- a/.github/workflows/python-package.yml
+++ b/.github/workflows/python-package.yml
@@ -56,7 +56,7 @@ jobs:
           ports=$(yq ".services.$version.ports")
           port=$(echo $ports | cut -d ':' -f 1)
           token=$(curl -k -X POST -c xdmod.cookie -d "username=normaluser&password=normaluser" https://localhost:$port/rest/auth/login | jq -r '.results.token')
-          curl -k -X POST -b xdmod.cookie https://localhost:port/rest/users/current/api/token?token=$token | jq -r '.data.token' > ~/.xdmod-data-token
+          curl -k -X POST -b xdmod.cookie https://localhost:$port/rest/users/current/api/token?token=$token | jq -r '.data.token' > ~/.xdmod-data-token
           coverage run --branch --append -m pytest -vvs -o log_cli=true tests/
         done
         coverage report -m --fail-under=100
\ No newline at end of file

From 92b86ef12e69d5761a95bf5a257b1433ed51a90a Mon Sep 17 00:00:00 2001
From: Rodney Garnett <garnettr35@gmail.com>
Date: Thu, 27 Jun 2024 14:29:32 -0400
Subject: [PATCH 41/72] Github actions check 12

---
 .github/workflows/python-package.yml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml
index 8f7fbc46..f59a87f0 100644
--- a/.github/workflows/python-package.yml
+++ b/.github/workflows/python-package.yml
@@ -55,8 +55,13 @@ jobs:
           docker exec $version /root/bin/services start
           ports=$(yq ".services.$version.ports")
           port=$(echo $ports | cut -d ':' -f 1)
+          echo "PORTS: $ports"
+          echo "PORT: $port"
           token=$(curl -k -X POST -c xdmod.cookie -d "username=normaluser&password=normaluser" https://localhost:$port/rest/auth/login | jq -r '.results.token')
+          echo "TOKEN: $token"
           curl -k -X POST -b xdmod.cookie https://localhost:$port/rest/users/current/api/token?token=$token | jq -r '.data.token' > ~/.xdmod-data-token
+          echo "FILE: "
+          cat ~/.xdmod-data-token
           coverage run --branch --append -m pytest -vvs -o log_cli=true tests/
         done
         coverage report -m --fail-under=100
\ No newline at end of file

From db9a1d3b41bd6cb8bd9391819b62ff9e54869d2c Mon Sep 17 00:00:00 2001
From: Rodney Garnett <garnettr35@gmail.com>
Date: Thu, 27 Jun 2024 14:34:20 -0400
Subject: [PATCH 42/72] Github actions check 13

---
 .github/workflows/python-package.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml
index f59a87f0..bf38cefa 100644
--- a/.github/workflows/python-package.yml
+++ b/.github/workflows/python-package.yml
@@ -53,7 +53,7 @@ jobs:
         declare -a portal_versions=$(yq '.services | keys[]' .github/workflows/docker-compose.yaml)
         for version in $portal_versions; do
           docker exec $version /root/bin/services start
-          ports=$(yq ".services.$version.ports")
+          ports=$(yq ".services.$version.ports" .github/workflows/docker-compose.yaml)
           port=$(echo $ports | cut -d ':' -f 1)
           echo "PORTS: $ports"
           echo "PORT: $port"

From 7f678436b0f26bbad3cbefdeac636e1dd41851b3 Mon Sep 17 00:00:00 2001
From: Rodney Garnett <garnettr35@gmail.com>
Date: Thu, 27 Jun 2024 14:38:49 -0400
Subject: [PATCH 43/72] Github actions check 14

---
 .github/workflows/python-package.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml
index bf38cefa..3d3ee843 100644
--- a/.github/workflows/python-package.yml
+++ b/.github/workflows/python-package.yml
@@ -53,6 +53,7 @@ jobs:
         declare -a portal_versions=$(yq '.services | keys[]' .github/workflows/docker-compose.yaml)
         for version in $portal_versions; do
           docker exec $version /root/bin/services start
+          echo "versions $portal versions"
           ports=$(yq ".services.$version.ports" .github/workflows/docker-compose.yaml)
           port=$(echo $ports | cut -d ':' -f 1)
           echo "PORTS: $ports"

From 84f208a7accc9dce5c90134b82ac79211e59407d Mon Sep 17 00:00:00 2001
From: Rodney Garnett <garnettr35@gmail.com>
Date: Thu, 27 Jun 2024 14:44:44 -0400
Subject: [PATCH 44/72] Github actions check 15

---
 .github/workflows/python-package.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml
index 3d3ee843..e6a6a017 100644
--- a/.github/workflows/python-package.yml
+++ b/.github/workflows/python-package.yml
@@ -53,8 +53,8 @@ jobs:
         declare -a portal_versions=$(yq '.services | keys[]' .github/workflows/docker-compose.yaml)
         for version in $portal_versions; do
           docker exec $version /root/bin/services start
-          echo "versions $portal versions"
-          ports=$(yq ".services.$version.ports" .github/workflows/docker-compose.yaml)
+          echo "versions $portal_versions"
+          ports=$(yq ".services.$version.ports[0]" .github/workflows/docker-compose.yaml)
           port=$(echo $ports | cut -d ':' -f 1)
           echo "PORTS: $ports"
           echo "PORT: $port"

From 37317940281372e263c48afe2fa7520debb7c626 Mon Sep 17 00:00:00 2001
From: Rodney Garnett <garnettr35@gmail.com>
Date: Thu, 27 Jun 2024 14:53:50 -0400
Subject: [PATCH 45/72] Github actions check 16

---
 .github/workflows/python-package.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml
index e6a6a017..176d5f62 100644
--- a/.github/workflows/python-package.yml
+++ b/.github/workflows/python-package.yml
@@ -54,7 +54,7 @@ jobs:
         for version in $portal_versions; do
           docker exec $version /root/bin/services start
           echo "versions $portal_versions"
-          ports=$(yq ".services.$version.ports[0]" .github/workflows/docker-compose.yaml)
+          ports=$(yq ".services.\"$version\".ports[0]" .github/workflows/docker-compose.yaml)
           port=$(echo $ports | cut -d ':' -f 1)
           echo "PORTS: $ports"
           echo "PORT: $port"

From 17ec0b0010de9432a8db7f30d9de1dfc6d8cfed5 Mon Sep 17 00:00:00 2001
From: Rodney Garnett <garnettr35@gmail.com>
Date: Thu, 27 Jun 2024 14:58:31 -0400
Subject: [PATCH 46/72] Github actions check 17

---
 .github/workflows/python-package.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml
index 176d5f62..ee6c9eb7 100644
--- a/.github/workflows/python-package.yml
+++ b/.github/workflows/python-package.yml
@@ -33,7 +33,7 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        python -m pip install flake8 flake8-commas flake8-quotes pytest pandas coverage
+        python -m pip install flake8 flake8-commas flake8-quotes pandas python-dotenv pytest coverage
         if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
         python -m pip install -e .
     - name: Lint with flake8

From 8a3b37c1b1d965eba31f733ec3256aad5947c9d3 Mon Sep 17 00:00:00 2001
From: Rodney Garnett <garnettr35@gmail.com>
Date: Thu, 27 Jun 2024 15:13:04 -0400
Subject: [PATCH 47/72] Github actions check 18

---
 .github/workflows/python-package.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml
index ee6c9eb7..62f882c3 100644
--- a/.github/workflows/python-package.yml
+++ b/.github/workflows/python-package.yml
@@ -60,7 +60,8 @@ jobs:
           echo "PORT: $port"
           token=$(curl -k -X POST -c xdmod.cookie -d "username=normaluser&password=normaluser" https://localhost:$port/rest/auth/login | jq -r '.results.token')
           echo "TOKEN: $token"
-          curl -k -X POST -b xdmod.cookie https://localhost:$port/rest/users/current/api/token?token=$token | jq -r '.data.token' > ~/.xdmod-data-token
+          echo -n "XDMOD_API_TOKEN=" > ~/.xdmod-data-token 
+          curl -k -X POST -b xdmod.cookie https://localhost:$port/rest/users/current/api/token?token=$token | jq -r '.data.token' >> ~/.xdmod-data-token
           echo "FILE: "
           cat ~/.xdmod-data-token
           coverage run --branch --append -m pytest -vvs -o log_cli=true tests/

From 1280af559c574c26b95c749ee09b94cbc8794b0c Mon Sep 17 00:00:00 2001
From: Rodney Garnett <garnettr35@gmail.com>
Date: Thu, 27 Jun 2024 15:40:51 -0400
Subject: [PATCH 48/72] Github actions check 20

---
 .github/workflows/python-package.yml                | 5 +++--
 tests/integration/test_datawarehouse_integration.py | 3 ++-
 tests/regression/test_datawarehouse_regression.py   | 3 ++-
 3 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml
index 62f882c3..71b94163 100644
--- a/.github/workflows/python-package.yml
+++ b/.github/workflows/python-package.yml
@@ -58,10 +58,11 @@ jobs:
           port=$(echo $ports | cut -d ':' -f 1)
           echo "PORTS: $ports"
           echo "PORT: $port"
-          token=$(curl -k -X POST -c xdmod.cookie -d "username=normaluser&password=normaluser" https://localhost:$port/rest/auth/login | jq -r '.results.token')
+          export XDMOD_HOST="https://localhost:$port"
+          token=$(curl -k -X POST -c xdmod.cookie -d "username=normaluser&password=normaluser" $XDMOD_HOST/rest/auth/login | jq -r '.results.token')
           echo "TOKEN: $token"
           echo -n "XDMOD_API_TOKEN=" > ~/.xdmod-data-token 
-          curl -k -X POST -b xdmod.cookie https://localhost:$port/rest/users/current/api/token?token=$token | jq -r '.data.token' >> ~/.xdmod-data-token
+          curl -k -X POST -b xdmod.cookie $XDMOD_HOST/rest/users/current/api/token?token=$token | jq -r '.data.token' >> ~/.xdmod-data-token
           echo "FILE: "
           cat ~/.xdmod-data-token
           coverage run --branch --append -m pytest -vvs -o log_cli=true tests/
diff --git a/tests/integration/test_datawarehouse_integration.py b/tests/integration/test_datawarehouse_integration.py
index 65dc0eb5..1e1e111b 100644
--- a/tests/integration/test_datawarehouse_integration.py
+++ b/tests/integration/test_datawarehouse_integration.py
@@ -1,11 +1,12 @@
 from dotenv import load_dotenv
+import os 
 from os.path import expanduser
 import pandas
 from pathlib import Path
 import pytest
 from xdmod_data.warehouse import DataWarehouse
 
-VALID_XDMOD_URL = 'https://xdmod.access-ci.org'
+VALID_XDMOD_URL = os.environ['XDMOD_HOST']
 TOKEN_PATH = '~/.xdmod-data-token'
 INVALID_STR = 'asdlkfjsdlkfisdjkfjd'
 METHOD_PARAMS = {
diff --git a/tests/regression/test_datawarehouse_regression.py b/tests/regression/test_datawarehouse_regression.py
index 34ff1600..0128b7a8 100644
--- a/tests/regression/test_datawarehouse_regression.py
+++ b/tests/regression/test_datawarehouse_regression.py
@@ -1,13 +1,14 @@
 from datetime import date
 from dotenv import load_dotenv
 import numpy
+import os
 from os.path import dirname, expanduser
 import pandas
 from pathlib import Path
 import pytest
 from xdmod_data.warehouse import DataWarehouse
 
-XDMOD_URL = 'https://xdmod.access-ci.org'
+XDMOD_URL = os.environ['XDMOD_HOST']
 TOKEN_PATH = '~/.xdmod-data-token'
 DATA_DIR = dirname(__file__) + '/data'
 

From 91f7227100ea4c2a5a753f2c448e66b810a46112 Mon Sep 17 00:00:00 2001
From: Rodney Garnett <garnettr35@gmail.com>
Date: Thu, 27 Jun 2024 15:42:58 -0400
Subject: [PATCH 49/72] Github actions check 21

---
 tests/integration/test_datawarehouse_integration.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/integration/test_datawarehouse_integration.py b/tests/integration/test_datawarehouse_integration.py
index 1e1e111b..3215c284 100644
--- a/tests/integration/test_datawarehouse_integration.py
+++ b/tests/integration/test_datawarehouse_integration.py
@@ -1,5 +1,5 @@
 from dotenv import load_dotenv
-import os 
+import os
 from os.path import expanduser
 import pandas
 from pathlib import Path

From e451b712302bc75c790d309b56a37af26f0f90b3 Mon Sep 17 00:00:00 2001
From: Rodney Garnett <garnettr35@gmail.com>
Date: Thu, 27 Jun 2024 16:30:21 -0400
Subject: [PATCH 50/72] Github actions check 22

---
 .github/workflows/python-package.yml | 17 ++++++++++-------
 1 file changed, 10 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml
index 71b94163..86fb0bb7 100644
--- a/.github/workflows/python-package.yml
+++ b/.github/workflows/python-package.yml
@@ -52,18 +52,21 @@ jobs:
         coverage erase
         declare -a portal_versions=$(yq '.services | keys[]' .github/workflows/docker-compose.yaml)
         for version in $portal_versions; do
+          # Generate OpenSSL Key
+          docker exec xdmod10.5 openssl genrsa -rand /proc/cpuinfo:/proc/filesystems:/proc/interrupts:/proc/ioports:/proc/uptime 2048 > /etc/pki/tls/private/localhost.key
+          # Generate Certificate 
+          docker exec xdmod10.5 /usr/bin/openssl req -new -key /etc/pki/tls/private/localhost.key -x509 -sha256 -days 365 -set_serial $RANDOM -extensions v3_req -out /etc/pki/tls/certs/localhost.crt -subj "/C=XX/L=Default City/O=Default Company Ltd"
+          # Copy Certificate from container
+          docker cp $version:/etc/pki/tls/certs/localhost.crt $version.crt 
+          # Prepare to use certificate for curl and python requests
+          export CURL_CA_BUNDLE="$version.crt"
+          # Start up web server in container
           docker exec $version /root/bin/services start
-          echo "versions $portal_versions"
           ports=$(yq ".services.\"$version\".ports[0]" .github/workflows/docker-compose.yaml)
           port=$(echo $ports | cut -d ':' -f 1)
-          echo "PORTS: $ports"
-          echo "PORT: $port"
           export XDMOD_HOST="https://localhost:$port"
-          token=$(curl -k -X POST -c xdmod.cookie -d "username=normaluser&password=normaluser" $XDMOD_HOST/rest/auth/login | jq -r '.results.token')
-          echo "TOKEN: $token"
-          echo -n "XDMOD_API_TOKEN=" > ~/.xdmod-data-token 
+          token=$(curl -k -X POST -c xdmod.cookie -d "username=normaluser&password=normaluser" $XDMOD_HOST/rest/auth/login | jq -r '.results.token') 
           curl -k -X POST -b xdmod.cookie $XDMOD_HOST/rest/users/current/api/token?token=$token | jq -r '.data.token' >> ~/.xdmod-data-token
-          echo "FILE: "
           cat ~/.xdmod-data-token
           coverage run --branch --append -m pytest -vvs -o log_cli=true tests/
         done

From 256cbcc97a798c69a6bab2a0d8212da3c868158c Mon Sep 17 00:00:00 2001
From: Rodney Garnett <garnettr35@gmail.com>
Date: Thu, 27 Jun 2024 16:40:34 -0400
Subject: [PATCH 51/72] Github actions check 23

---
 .github/workflows/python-package.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml
index 86fb0bb7..b8396923 100644
--- a/.github/workflows/python-package.yml
+++ b/.github/workflows/python-package.yml
@@ -53,7 +53,7 @@ jobs:
         declare -a portal_versions=$(yq '.services | keys[]' .github/workflows/docker-compose.yaml)
         for version in $portal_versions; do
           # Generate OpenSSL Key
-          docker exec xdmod10.5 openssl genrsa -rand /proc/cpuinfo:/proc/filesystems:/proc/interrupts:/proc/ioports:/proc/uptime 2048 > /etc/pki/tls/private/localhost.key
+          docker exec xdmod10.5 sh -c 'openssl genrsa -rand /proc/cpuinfo:/proc/filesystems:/proc/interrupts:/proc/ioports:/proc/uptime 2048 > /etc/pki/tls/private/localhost.key'
           # Generate Certificate 
           docker exec xdmod10.5 /usr/bin/openssl req -new -key /etc/pki/tls/private/localhost.key -x509 -sha256 -days 365 -set_serial $RANDOM -extensions v3_req -out /etc/pki/tls/certs/localhost.crt -subj "/C=XX/L=Default City/O=Default Company Ltd"
           # Copy Certificate from container

From 13a8afabc500ca4730c372ac02e16ba78d22a661 Mon Sep 17 00:00:00 2001
From: Rodney Garnett <garnettr35@gmail.com>
Date: Thu, 27 Jun 2024 16:57:42 -0400
Subject: [PATCH 52/72] Github actions check 24

---
 .github/workflows/python-package.yml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml
index b8396923..691e850c 100644
--- a/.github/workflows/python-package.yml
+++ b/.github/workflows/python-package.yml
@@ -65,8 +65,9 @@ jobs:
           ports=$(yq ".services.\"$version\".ports[0]" .github/workflows/docker-compose.yaml)
           port=$(echo $ports | cut -d ':' -f 1)
           export XDMOD_HOST="https://localhost:$port"
-          token=$(curl -k -X POST -c xdmod.cookie -d "username=normaluser&password=normaluser" $XDMOD_HOST/rest/auth/login | jq -r '.results.token') 
-          curl -k -X POST -b xdmod.cookie $XDMOD_HOST/rest/users/current/api/token?token=$token | jq -r '.data.token' >> ~/.xdmod-data-token
+          token=$(curl -X POST -c xdmod.cookie -d "username=normaluser&password=normaluser" $XDMOD_HOST/rest/auth/login | jq -r '.results.token')
+          echo -n 'XDMOD_API_TOKEN=' > ~/.xdmod-data-token
+          curl -X POST -b xdmod.cookie $XDMOD_HOST/rest/users/current/api/token?token=$token | jq -r '.data.token' >> ~/.xdmod-data-token
           cat ~/.xdmod-data-token
           coverage run --branch --append -m pytest -vvs -o log_cli=true tests/
         done

From e987bcc6c8ee00f272036e3cd0d7256f8586be1f Mon Sep 17 00:00:00 2001
From: Rodney Garnett <garnettr35@gmail.com>
Date: Fri, 28 Jun 2024 11:46:27 -0400
Subject: [PATCH 53/72] Github actions check 24

---
 .github/workflows/python-package.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml
index 691e850c..a9cb6b2e 100644
--- a/.github/workflows/python-package.yml
+++ b/.github/workflows/python-package.yml
@@ -53,9 +53,9 @@ jobs:
         declare -a portal_versions=$(yq '.services | keys[]' .github/workflows/docker-compose.yaml)
         for version in $portal_versions; do
           # Generate OpenSSL Key
-          docker exec xdmod10.5 sh -c 'openssl genrsa -rand /proc/cpuinfo:/proc/filesystems:/proc/interrupts:/proc/ioports:/proc/uptime 2048 > /etc/pki/tls/private/localhost.key'
+          docker exec $version sh -c 'openssl genrsa -rand /proc/cpuinfo:/proc/filesystems:/proc/interrupts:/proc/ioports:/proc/uptime 2048 > /etc/pki/tls/private/localhost.key'
           # Generate Certificate 
-          docker exec xdmod10.5 /usr/bin/openssl req -new -key /etc/pki/tls/private/localhost.key -x509 -sha256 -days 365 -set_serial $RANDOM -extensions v3_req -out /etc/pki/tls/certs/localhost.crt -subj "/C=XX/L=Default City/O=Default Company Ltd"
+          docker exec $version /usr/bin/openssl req -new -key /etc/pki/tls/private/localhost.key -x509 -sha256 -days 365 -set_serial $RANDOM -extensions v3_req -out /etc/pki/tls/certs/localhost.crt -subj "/C=XX/L=Default City/O=Default Company Ltd/CN=localhost"
           # Copy Certificate from container
           docker cp $version:/etc/pki/tls/certs/localhost.crt $version.crt 
           # Prepare to use certificate for curl and python requests

From 12823798a71ce3a8b433e87477b546fdb9bf972a Mon Sep 17 00:00:00 2001
From: Rodney Garnett <garnettr35@gmail.com>
Date: Fri, 28 Jun 2024 12:31:49 -0400
Subject: [PATCH 54/72] Github actions check 25

---
 .github/workflows/python-package.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml
index a9cb6b2e..8bbf2e22 100644
--- a/.github/workflows/python-package.yml
+++ b/.github/workflows/python-package.yml
@@ -55,7 +55,7 @@ jobs:
           # Generate OpenSSL Key
           docker exec $version sh -c 'openssl genrsa -rand /proc/cpuinfo:/proc/filesystems:/proc/interrupts:/proc/ioports:/proc/uptime 2048 > /etc/pki/tls/private/localhost.key'
           # Generate Certificate 
-          docker exec $version /usr/bin/openssl req -new -key /etc/pki/tls/private/localhost.key -x509 -sha256 -days 365 -set_serial $RANDOM -extensions v3_req -out /etc/pki/tls/certs/localhost.crt -subj "/C=XX/L=Default City/O=Default Company Ltd/CN=localhost"
+          docker exec $version /usr/bin/openssl req -new -key /etc/pki/tls/private/localhost.key -x509 -sha256 -days 365 -set_serial $RANDOM -extensions v3_req -out /etc/pki/tls/certs/localhost.crt -subj "/C=XX/L=Default City/O=Default Company Ltd/CN=localhost" -addext 'subjectAltName=DNS:localhost'
           # Copy Certificate from container
           docker cp $version:/etc/pki/tls/certs/localhost.crt $version.crt 
           # Prepare to use certificate for curl and python requests

From fa01e2b2e9a3f1d6c0d27cf35e75da140f0e2643 Mon Sep 17 00:00:00 2001
From: Rodney Garnett <garnettr35@gmail.com>
Date: Tue, 2 Jul 2024 16:39:26 -0400
Subject: [PATCH 55/72] Github actions check 26

---
 .github/workflows/python-package.yml          |  12 +-
 .../test_datawarehouse_integration.py         |   2 +-
 .../data/jobs-2016-2017-quarters.csv          |   3 +
 .../regression/data/jobs-2016-2017-years.csv  |   3 +
 .../regression/data/jobs-2022-2023-years.csv  |   3 -
 tests/regression/data/jobs-2023-quarters.csv  |   5 -
 tests/regression/data/jobs-dimensions.csv     |  21 +---
 tests/regression/data/jobs-metrics.csv        | 117 ++++--------------
 .../data/jobs-pi-group-filter-values.csv      |  41 ++++++
 ...e-learning-notebook-example-every-1000.csv |  44 -------
 tests/regression/data/raw-data-every-1000.csv |  35 ++++++
 tests/regression/data/realms.csv              |   6 +-
 .../test_datawarehouse_regression.py          |  67 +++++-----
 tests/unit/test_datawarehouse_unit.py         |   2 +-
 14 files changed, 161 insertions(+), 200 deletions(-)
 create mode 100644 tests/regression/data/jobs-2016-2017-quarters.csv
 create mode 100644 tests/regression/data/jobs-2016-2017-years.csv
 delete mode 100644 tests/regression/data/jobs-2022-2023-years.csv
 delete mode 100644 tests/regression/data/jobs-2023-quarters.csv
 create mode 100644 tests/regression/data/jobs-pi-group-filter-values.csv
 delete mode 100644 tests/regression/data/machine-learning-notebook-example-every-1000.csv
 create mode 100644 tests/regression/data/raw-data-every-1000.csv

diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml
index 8bbf2e22..a5833b78 100644
--- a/.github/workflows/python-package.yml
+++ b/.github/workflows/python-package.yml
@@ -49,6 +49,12 @@ jobs:
         sudo wget https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 -O /usr/bin/yq && sudo chmod +x /usr/bin/yq
     - name: Test against each supported XDMoD portal version
       run: |
+        # Create a file with 10,000 users so that we can test filters that have more than 10,000 values
+        for i in {1..10000}; do
+          unix=$((1704085200+$i))
+          echo "$i|2269212|frearson|black|banana-cream|caste|caste|45580|user$i|379330|$(date +'%Y-%m-%dT%H:%M:%S' -d @$unix)|$(date +'%Y-%m-%dT%H:%M:%S' -d @$(($unix+1)))|$(date +'%Y-%m-%dT%H:%M:%S' -d @$(($unix+2)))|$(date +'%Y-%m-%dT%H:%M:%S' -d @$(($unix+3)))|2-23:24:40|0:0|COMPLETED|1|16|16|62.50Gn|cpu=16,mem=62.50G,node=1|billing=16,cpu=16,mem=62.50G,node=1|3-00:00:00|cpn-m25-02-01|310f6011b278ebb15df2d199e0119de78d261d8c478a203abc97dc3dda7702ae" >> 10000users.log
+        done
+        #run tests
         coverage erase
         declare -a portal_versions=$(yq '.services | keys[]' .github/workflows/docker-compose.yaml)
         for version in $portal_versions; do
@@ -68,7 +74,11 @@ jobs:
           token=$(curl -X POST -c xdmod.cookie -d "username=normaluser&password=normaluser" $XDMOD_HOST/rest/auth/login | jq -r '.results.token')
           echo -n 'XDMOD_API_TOKEN=' > ~/.xdmod-data-token
           curl -X POST -b xdmod.cookie $XDMOD_HOST/rest/users/current/api/token?token=$token | jq -r '.data.token' >> ~/.xdmod-data-token
-          cat ~/.xdmod-data-token
+          docker cp 10000users.log $version:10000users.log
+          docker exec $version xdmod-shredder -r frearson -f slurm -i 10000users.log
+          date=$(date +%Y-%m-%d)
+          docker exec $version xdmod-ingestor --ingest
+          docker exec $version xdmod-ingestor --aggregate=job --last-modified-start-date $date
           coverage run --branch --append -m pytest -vvs -o log_cli=true tests/
         done
         coverage report -m --fail-under=100
\ No newline at end of file
diff --git a/tests/integration/test_datawarehouse_integration.py b/tests/integration/test_datawarehouse_integration.py
index 3215c284..8cc76939 100644
--- a/tests/integration/test_datawarehouse_integration.py
+++ b/tests/integration/test_datawarehouse_integration.py
@@ -40,7 +40,7 @@
   'realm': 'Jobs',
   'metric': 'CPU Hours: Total',
   'dimension': VALID_DIMENSION,
-  'filters': {VALID_DIMENSION: 'Expanse'},
+  'filters': {VALID_DIMENSION: 'frearson'},
   'dataset_type': 'timeseries',
   'aggregation_unit': 'Auto',
   'parameter': 'duration',
diff --git a/tests/regression/data/jobs-2016-2017-quarters.csv b/tests/regression/data/jobs-2016-2017-quarters.csv
new file mode 100644
index 00000000..7d5b24e2
--- /dev/null
+++ b/tests/regression/data/jobs-2016-2017-quarters.csv
@@ -0,0 +1,3 @@
+Time,CPU Hours: Total
+2016-10-01,770775.9989
+2017-01-01,69779.8381
diff --git a/tests/regression/data/jobs-2016-2017-years.csv b/tests/regression/data/jobs-2016-2017-years.csv
new file mode 100644
index 00000000..7e9cf848
--- /dev/null
+++ b/tests/regression/data/jobs-2016-2017-years.csv
@@ -0,0 +1,3 @@
+Time,CPU Hours: Total
+2016-01-01,770775.9989
+2017-01-01,69779.8381
diff --git a/tests/regression/data/jobs-2022-2023-years.csv b/tests/regression/data/jobs-2022-2023-years.csv
deleted file mode 100644
index 0958a21a..00000000
--- a/tests/regression/data/jobs-2022-2023-years.csv
+++ /dev/null
@@ -1,3 +0,0 @@
-Time,CPU Hours: Total
-2022-01-01,2537086425.0939
-2023-01-01,2913432363.5808
diff --git a/tests/regression/data/jobs-2023-quarters.csv b/tests/regression/data/jobs-2023-quarters.csv
deleted file mode 100644
index 86e584cb..00000000
--- a/tests/regression/data/jobs-2023-quarters.csv
+++ /dev/null
@@ -1,5 +0,0 @@
-Time,CPU Hours: Total
-2023-01-01,874332559.6222
-2023-04-01,801997902.9242
-2023-07-01,617633340.4103
-2023-10-01,619468560.6242
diff --git a/tests/regression/data/jobs-dimensions.csv b/tests/regression/data/jobs-dimensions.csv
index 798d3541..10887b04 100644
--- a/tests/regression/data/jobs-dimensions.csv
+++ b/tests/regression/data/jobs-dimensions.csv
@@ -1,26 +1,17 @@
 id,label,description
-none,None,Summarizes jobs reported to the ACCESS allocations service (excludes non-ACCESS usage of the resource).
-allocation,Allocation,A funded project that is allowed to run jobs on resources.
-fieldofscience,Field of Science,The field of science indicated on the allocation request pertaining to the running jobs.
-gateway,Gateway,A science gateway is a portal set up to aid submiting jobs to resources.
-grant_type,Grant Type,A categorization of the projects/allocations.
+none,None,Summarizes Jobs data reported to the Screwdriver database.
+nsfdirectorate,Decanal Unit,Decanal Unit
+parentscience,Department,Department
+gpucount,GPU Count,A categorization of jobs into discrete groups based on the number of GPUs used by each job.
 jobsize,Job Size,A categorization of jobs into discrete groups based on the number of cores used by each job.
 jobwaittime,Job Wait Time,A categorization of jobs into discrete groups based on the total linear time each job waited.
 jobwalltime,Job Wall Time,A categorization of jobs into discrete groups based on the total linear time each job took to execute.
-nsfdirectorate,NSF Directorate,The NSF directorate of the field of science indiciated on the allocation request pertaining to the running jobs.
 nodecount,Node Count,A categorization of jobs into discrete groups based on node count.
 pi,PI,The principal investigator of a project.
-pi_institution,PI Institution,Organizations that have PIs with allocations.
-pi_institution_country,PI Institution Country,The country of the institution of the PI of the project associated with compute jobs.
-pi_institution_state,PI Institution State,The location of the institution of the PI of the project associated with the compute jobs.
-parentscience,Parent Science,The parent of the field of science indiciated on the allocation request pertaining to the running jobs.
+fieldofscience,PI Group,PI Group
+qos,QOS,The quality of service of the job
 queue,Queue,Queue pertains to the low level job queues on each resource.
 resource,Resource,A resource is a remote computer that can run jobs.
 resource_type,Resource Type,A categorization of resources into by their general capabilities.
-provider,Service Provider,A service provider is an institution that hosts resources.
 username,System Username,The specific system username of the users who ran jobs.
 person,User,"A person who is on a PIs allocation, hence able run jobs on resources."
-institution,User Institution,Organizations that have users with allocations.
-institution_country,User Institution Country,The name of the country of the institution of the person who ran the compute job.
-institution_state,User Institution State,The location of the institution of the person who ran the compute job.
-nsfstatus,User NSF Status,Categorization of the users who ran jobs.
diff --git a/tests/regression/data/jobs-metrics.csv b/tests/regression/data/jobs-metrics.csv
index f5a9f14e..2f4c6d7e 100644
--- a/tests/regression/data/jobs-metrics.csv
+++ b/tests/regression/data/jobs-metrics.csv
@@ -1,94 +1,27 @@
 id,label,description
-avg_ace,ACCESS Credit Equivalents Charged: Per Job (SU),"The average amount of ACCESS Credit Equivalents charged per compute job.<br/>
-
-The ACCESS Credit Equivalent is a measure of how much compute time was used on each resource.
-One ACCESS Credit Equivalent is defined as one CPU Hour on SDSC Expanse (an AMD EPYC 7742 based compute resource).
-The ACCESS Credit Equivalent allows comparison between usage of node-allocated, core-allocated and GPU-allocated 
-resources. It also allows a comparison between resources with different compute power per core.
-The <a href=""https://allocations.access-ci.org/exchange_calculator"" target=""_blank"" rel=""noopener noreferrer"">ACCESS allocations exchange calculator</a>
-lists conversion rates between an ACCESS Credit Equivalent and a service unit on a resource."
-total_ace,ACCESS Credit Equivalents Charged: Total (SU),"The total amount of ACCESS Credit Equivalents charged.<br/>
-
-The ACCESS Credit Equivalent is a measure of how much compute time was used on each resource.
-One ACCESS Credit Equivalent is defined as one CPU Hour on SDSC Expanse (an AMD EPYC 7742 based compute resource).
-The ACCESS Credit Equivalent allows comparison between usage of node-allocated, core-allocated and GPU-allocated 
-resources. It also allows a comparison between resources with different compute power per core.
-The <a href=""https://allocations.access-ci.org/exchange_calculator"" target=""_blank"" rel=""noopener noreferrer"">ACCESS allocations exchange calculator</a>
-lists conversion rates between an ACCESS Credit Equivalent and a service unit on a resource."
-utilization,ACCESS Utilization (%),"The percentage of the ACCESS obligation of a resource that has been utilized by ACCESS jobs.<br/><i> ACCESS Utilization:</i> The ratio of the total CPU hours consumed by ACCESS jobs over a given time period divided by the total CPU hours that the system is contractually required to provide to ACCESS during that period. It does not include non-ACCESS jobs.<br/>It is worth noting that this value is a rough estimate in certain cases where the resource providers don't provide accurate records of their system specifications, over time."
-rate_of_usage,Allocation Usage Rate (XD SU/Hour),The rate of ACCESS allocation usage in XD SUs per hour.
-rate_of_usage_ace,Allocation Usage Rate ACEs (SU/Hour),The rate of ACCESS allocation usage in ACCESS Credit Equivalents per hour.
-avg_cpu_hours,CPU Hours: Per Job,"The average CPU hours (number of CPU cores x wall time hours) per ACCESS job.<br/>For each job, the CPU usage is aggregated. For example, if a job used 1000 CPUs for one minute, it would be aggregated as 1000 CPU minutes or 16.67 CPU hours."
-total_cpu_hours,CPU Hours: Total,"The total CPU hours (number of CPU cores x wall time hours) used by ACCESS jobs.<br/>For each job, the CPU usage is aggregated. For example, if a job used 1000 CPUs for one minute, it would be aggregated as 1000 CPU minutes or 16.67 CPU hours."
-max_processors,Job Size: Max (Core Count),The maximum size ACCESS job in number of cores.<br/><i>Job Size: </i>The total number of processor cores used by a (parallel) job.
-min_processors,Job Size: Min (Core Count),The minimum size ACCESS job in number of cores.<br/><i>Job Size: </i>The total number of processor cores used by a (parallel) job.
-normalized_avg_processors,Job Size: Normalized (% of Total Cores),The percentage average size ACCESS job over total machine cores.<br><i>Normalized Job Size: </i>The percentage total number of processor cores used by a (parallel) job over the total number of cores on the machine.
-avg_processors,Job Size: Per Job (Core Count),The average job size per ACCESS job.<br><i>Job Size: </i>The number of processor cores used by a (parallel) job.
-avg_job_size_weighted_by_ace,Job Size: Weighted By ACEs (Core Count),The average job size weighted by charge in ACCESS Credit Equivalents (ACEs). Defined as <br><i>Average Job Size Weighted By ACEs: </i> sum(i = 0 to n){job i core count*job i charge in ACEs}/sum(i =  0 to n){job i charge in ACEs}
-avg_job_size_weighted_by_cpu_hours,Job Size: Weighted By CPU Hours (Core Count),The average ACCESS job size weighted by CPU Hours. Defined as <br><i>Average Job Size Weighted By CPU Hours: </i> sum(i = 0 to n){ job i core count * job i cpu hours}/sum(i =  0 to n){job i cpu hours}
-avg_job_size_weighted_by_xd_su,Job Size: Weighted By XD SUs (Core Count),The average ACCESS job size weighted by charge in XD SUs. Defined as <br><i>Average Job Size Weighted By XD SUs: </i> sum(i = 0 to n){job i core count*job i charge in xd sus}/sum(i =  0 to n){job i charge in xd sus}
-avg_nu,NUs Charged: Per Job,"The average amount of NUs charged per ACCESS job.<br/>
-<i>NU - Normalized Units: </i>Roaming allocations are awarded in XSEDE Service Units (SUs). 1 XSEDE SU is defined as one CPU-hour on a Phase-1 DTF cluster. 
-For usage on a resource that is charged to a Roaming allocation, a normalization factor is applied. The normalization factor is based on
-the method historically used to calculate 'Normalized Units' (Cray X-MP-equivalent SUs), which derives from a resource's performance on the HPL
-benchmark.<br/>
-
-Specifically, 1 Phase-1 DTF SU = 21.576 NUs, and the XD SU conversion factor for a resource is calculated by taking its NU conversion factor
-and dividing it by 21.576. The standard formula for calculating a resource's NU conversion factor is: (Rmax * 1000 / 191) / P
-where Rmax is the resource's Rmax result on the HPL benchmark in Gflops and P is the number of processors used in the benchmark.
-In the absence of an HPL benchmark run, a conversion factor can be agreed upon, based on that of an architecturally similar platform
-and scaled according to processor performance differences.<br/>
-
-Conversion to Roaming SUs is handled by the XSEDE central accounting system, and RPs are only required to report usage in local SUs for all allocations.<br/>
-
-Defining an SU charge for specialized compute resources (such as visualization hardware) or non-compute resources (such as storage) is possible, but there is no XSEDE-wide policy for doing so."
-total_nu,NUs Charged: Total,"The total amount of NUs charged by ACCESS jobs.<br/>
-<i>NU - Normalized Units: </i>Roaming allocations are awarded in XSEDE Service Units (SUs). 1 XSEDE SU is defined as one CPU-hour on a Phase-1 DTF cluster.
-For usage on a resource that is charged to a Roaming allocation, a normalization factor is applied. The normalization factor is based on 
-the method historically used to calculate 'Normalized Units' (Cray X-MP-equivalent SUs), which derives from a resource's performance on the HPL 
-benchmark.<br/>
-
-Specifically, 1 Phase-1 DTF SU = 21.576 NUs, and the XD SU conversion factor for a resource is calculated by taking its NU conversion factor 
-and dividing it by 21.576. The standard formula for calculating a resource's NU conversion factor is: (Rmax * 1000 / 191) / P 
-where Rmax is the resource's Rmax result on the HPL benchmark in Gflops and P is the number of processors used in the benchmark. 
-In the absence of an HPL benchmark run, a conversion factor can be agreed upon, based on that of an architecturally similar platform 
-and scaled according to processor performance differences.<br/>
-
-Conversion to Roaming SUs is handled by the XSEDE central accounting system, and RPs are only required to report usage in local SUs for all allocations.<br/>
-
-Defining an SU charge for specialized compute resources (such as visualization hardware) or non-compute resources (such as storage) is possible, but there is no XSEDE-wide policy for doing so."
-avg_node_hours,Node Hours: Per Job,The average node hours (number of nodes x wall time hours) per ACCESS job.
-total_node_hours,Node Hours: Total,The total node hours (number of nodes x wall time hours) used by ACCESS jobs.
-active_allocation_count,Number of Allocations: Active,The total number of funded projects that used ACCESS resources.
-active_institution_count,Number of Institutions: Active,The total number of institutions that used ACCESS resources.
-job_count,Number of Jobs Ended,The total number of ACCESS jobs that ended within the selected duration.
-running_job_count,Number of Jobs Running,The total number of ACCESS jobs that are running.
-started_job_count,Number of Jobs Started,The total number of ACCESS jobs that started executing within the selected duration.
-submitted_job_count,Number of Jobs Submitted,The total number of ACCESS jobs that submitted/queued within the selected duration.<i>
-gateway_job_count,Number of Jobs via Gateway,"The total number of ACCESS  jobs submitted through gateways (e.g., via a community user account) that ended within the selected duration.<br/><i>Job: </i>A scheduled process for a computer resource in a batch processing environment."
-active_pi_count,Number of PIs: Active,The total number of PIs that used ACCESS resources.
-active_resource_count,Number of Resources: Active,The total number of active ACCESS resources.
-active_person_count,Number of Users: Active,The total number of users that used ACCESS resources.
-expansion_factor,User Expansion Factor,"Gauging ACCESS job-turnaround time, it measures the ratio of wait time and the total time from submission to end of execution.<br/><i>User Expansion Factor = ((wait duration + wall duration) / wall duration). </i>"
-avg_waitduration_hours,Wait Hours: Per Job,"The average time, in hours, a ACCESS job waits before execution on the designated resource.<br/><i>Wait Time: </i>Wait time is defined as the linear time between submission of a job by a user until it begins to execute."
-total_waitduration_hours,Wait Hours: Total,"The total time, in hours, ACCESS jobs waited before execution on their designated resource.<br/><i>Wait Time: </i>Wait time is defined as the linear time between submission of a job by a user until it begins to execute."
+avg_cpu_hours,CPU Hours: Per Job,"The average CPU hours (number of CPU cores x wall time hours) per Screwdriver job.<br/>For each job, the CPU usage is aggregated. For example, if a job used 1000 CPUs for one minute, it would be aggregated as 1000 CPU minutes or 16.67 CPU hours."
+total_cpu_hours,CPU Hours: Total,"The total CPU hours (number of CPU cores x wall time hours) used by Screwdriver jobs.<br/>For each job, the CPU usage is aggregated. For example, if a job used 1000 CPUs for one minute, it would be aggregated as 1000 CPU minutes or 16.67 CPU hours."
+avg_gpus,GPU Count: Per Job,The average job size per Screwdriver job.<br><i>Job Size: </i>The number of GPUs used by a (parallel) job.
+avg_gpu_hours,GPU Hours: Per Job,"The average GPU hours (number of GPU cores x wall time hours) per Screwdriver job.<br/>For each job, the GPU usage is aggregated. For example, if a job used 1000 GPUs for one minute, it would be aggregated as 1000 GPU minutes or 16.67 GPU hours."
+total_gpu_hours,GPU Hours: Total,"The total GPU hours (number of GPUs x wall time hours) used by Screwdriver jobs.<br/>For each job, the GPU usage is aggregated. For example, if a job used 1000 GPUs for one minute, it would be aggregated as 1000 GPU minutes or 16.67 GPU hours."
+max_processors,Job Size: Max (Core Count),The maximum size Screwdriver job in number of cores.<br/><i>Job Size: </i>The total number of processor cores used by a (parallel) job.
+min_processors,Job Size: Min (Core Count),The minimum size Screwdriver job in number of cores.<br/><i>Job Size: </i>The total number of processor cores used by a (parallel) job.
+normalized_avg_processors,Job Size: Normalized (% of Total Cores),The percentage average size Screwdriver job over total machine cores.<br><i>Normalized Job Size: </i>The percentage total number of processor cores used by a (parallel) job over the total number of cores on the machine.
+avg_processors,Job Size: Per Job (Core Count),The average job size per Screwdriver job.<br><i>Job Size: </i>The number of processor cores used by a (parallel) job.
+avg_job_size_weighted_by_cpu_hours,Job Size: Weighted By CPU Hours (Core Count),The average Screwdriver job size weighted by CPU Hours. Defined as <br><i>Average Job Size Weighted By CPU Hours: </i> sum(i = 0 to n){ job i core count * job i cpu hours}/sum(i =  0 to n){job i cpu hours}
+avg_job_size_weighted_by_gpu_hours,Job Size: Weighted By GPU Hours (GPU Count),The average Screwdriver job size weighted by GPU Hours. Defined as <br><i>Average Job Size Weighted By GPU Hours: </i> sum(i = 0 to n){ job i core count * job i gpu hours}/sum(i =  0 to n){job i gpu hours}
+avg_node_hours,Node Hours: Per Job,The average node hours (number of nodes x wall time hours) per Screwdriver job.
+total_node_hours,Node Hours: Total,The total node hours (number of nodes x wall time hours) used by Screwdriver jobs.
+job_count,Number of Jobs Ended,The total number of Screwdriver jobs that ended within the selected duration.
+running_job_count,Number of Jobs Running,The total number of Screwdriver jobs that are running.
+started_job_count,Number of Jobs Started,The total number of Screwdriver jobs that started executing within the selected duration.
+submitted_job_count,Number of Jobs Submitted,The total number of Screwdriver jobs that submitted/queued within the selected duration.<i>
+active_pi_count,Number of PIs: Active,The total number of PIs that used Screwdriver resources.
+active_resource_count,Number of Resources: Active,The total number of active Screwdriver resources.
+active_person_count,Number of Users: Active,The total number of users that used Screwdriver resources.
+utilization,Screwdriver Utilization (%),The percentage of core time that a resource has been running jobs.<br/><i> Screwdriver Utilization:</i> The ratio of the total CPU hours consumed by jobs over a given time period divided by the maximum CPU hours that the system could deliver (based on the number of cores present on the resources). This value does not take into account downtimes or outages. It is just calculated based on the number of cores in the resource specifications.
+expansion_factor,User Expansion Factor,The average job size per Screwdriver job.<br><i>Job Size: </i>The number of processor cores used by a (parallel) job.
+avg_waitduration_hours,Wait Hours: Per Job,"The average time, in hours, a Screwdriver job waits before execution on the designated resource.<br/><i>Wait Time: </i>Wait time is defined as the linear time between submission of a job by a user until it begins to execute."
+total_waitduration_hours,Wait Hours: Total,"The total time, in hours, Screwdriver jobs waited before execution on their designated resource.<br/><i>Wait Time: </i>Wait time is defined as the linear time between submission of a job by a user until it begins to execute."
 avg_wallduration_hours,Wall Hours: Per Job,"The average time, in hours, a job takes to execute.<br/>In timeseries view mode, the statistic shows the average wall time per job per time period. In aggregate view mode the statistic only includes the job wall hours between the defined time range. The wall hours outside the time range are not included in the calculation.<br /> <i>Wall Time:</i> Wall time is defined as the linear time between start and end time of execution for a particular job."
-total_wallduration_hours,Wall Hours: Total,"The total time, in hours, ACCESS jobs took to execute.<br/><i>Wall Time:</i> Wall time is defined as the linear time between start and end time of execution for a particular job."
-avg_su,XD SUs Charged: Per Job,"The average amount of XD SUs charged per ACCESS job.<br/>
-<i>XD SU: </i>1 XSEDE SU is defined as one CPU-hour on a Phase-1 DTF cluster.<br/>
-<i>SU - Service Units: </i>Computational resources on the XSEDE are allocated and charged in service units (SUs). SUs are defined locally on each system, with conversion factors among systems based on HPL benchmark results.<br/>
-
-Current TeraGrid supercomputers have complex multi-core and memory hierarchies. Each resource has a specific configuration that determines the number (N) of cores that can be dedicated to a job without slowing the code (and other user and system codes). Each resource defines for its system the minimum number of SUs charged for a job running in the default batch queue, calculated as wallclock runtime multiplied by N. Minimum charges may apply.<br/>
-
-Note: The actual charge will depend on the specific requirements of the job (e.g., the mapping of the cores across the machine, or the priority you wish to obtain).<br/>
-
-Note 2: The SUs show here have been normalized against the XSEDE Roaming service. Therefore they are comparable across resources."
-total_su,XD SUs Charged: Total,"The total amount of XD SUs charged by ACCESS jobs.<br/>
-<i>XD SU: </i>1 XSEDE SU is defined as one CPU-hour on a Phase-1 DTF cluster.<br/>
-<i>SU - Service Units: </i>Computational resources on the XSEDE are allocated and charged in service units (SUs). SUs are defined locally on each system, with conversion factors among systems based on HPL benchmark results.<br/>
-
-Current TeraGrid supercomputers have complex multi-core and memory hierarchies. Each resource has a specific configuration that determines the number (N) of cores that can be dedicated to a job without slowing the code (and other user and system codes). Each resource defines for its system the minimum number of SUs charged for a job running in the default batch queue, calculated as wallclock runtime multiplied by N. Minimum charges may apply.<br/>
-
-Note: The actual charge will depend on the specific requirements of the job (e.g., the mapping of the cores across the machine, or the priority you wish to obtain).<br/>
-
-Note 2: The SUs show here have been normalized against the XSEDE Roaming service. Therefore they are comparable across resources."
+total_wallduration_hours,Wall Hours: Total,"The total time, in hours, Screwdriver jobs took to execute.<br/><i>Wall Time:</i> Wall time is defined as the linear time between start and end time of execution for a particular job."
diff --git a/tests/regression/data/jobs-pi-group-filter-values.csv b/tests/regression/data/jobs-pi-group-filter-values.csv
new file mode 100644
index 00000000..a31287cc
--- /dev/null
+++ b/tests/regression/data/jobs-pi-group-filter-values.csv
@@ -0,0 +1,41 @@
+id,label
+53,Algebra and Number Theory
+10,Arts
+44,Biochemistry and Molecular Structure and Function
+45,Biophysics
+47,Cell Biology
+127,Computer and Computation Theory
+61,"Decision, Risk, and Management Science"
+99,Design and Computer-Integrated Engineering
+139,"Design, Tools, and Test"
+55,Economics
+97,Emerging Technologies Initiation
+137,Experimental Systems
+112,Extragalactic Astronomy and Cosmology
+88,"Fluid, Particulate, and Hydraulic Systems"
+115,Galactic Astronomy
+68,Geology and Paleontology
+72,Geophysics
+65,Global Atmospheric Research
+60,Law and Social Sciences
+91,Mechanics and Materials
+126,"Metals, Ceramics, and Electronic Materials"
+98,Operations Research and Production Systems
+122,Organic and Macromolecular Chemistry
+123,Physical Chemistry
+81,Polar Aeronomy and Astrophysics
+84,Polar Meteorology
+59,Polar Ocean and Climate Systems
+93,"Quantum Electronics, Waves, and Beams"
+71,Seismology
+57,Sociology
+124,Solid State Chemistry and Polymers
+94,Solid-State and Microstructures
+69,Statistics and Probability
+114,Stellar Astronomy and Astrophysics
+90,Structures and Building Systems
+49,Systematic and Population Biology
+138,Systems Prototyping and Fabrication
+70,Tectonics
+119,Theoretical Physics
+74,Volcanology and Mantle Geochemistry
diff --git a/tests/regression/data/machine-learning-notebook-example-every-1000.csv b/tests/regression/data/machine-learning-notebook-example-every-1000.csv
deleted file mode 100644
index 7f44b29e..00000000
--- a/tests/regression/data/machine-learning-notebook-example-every-1000.csv
+++ /dev/null
@@ -1,44 +0,0 @@
-,Nodes,Requested Wall Time,Wait Time,Wall Time,CPU User,"Mount point ""home"" data written","Mount point ""scratch"" data written",Total memory used
-0,1,172800,11,506,,,,
-1000,1,86400,1,66,,,,
-2000,1,86400,18,752,,,,
-3000,1,86400,8,5434,,,,
-4000,1,86400,6,1572,,,,
-5000,1,172800,7,2592,,,,
-6000,1,14400,7,2800,,,,
-7000,1,3600,2894,1357,,,,
-8000,1,21600,116,7277,,,,
-9000,1,21600,2173,6764,,,,
-10000,1,21600,3574,7095,,,,
-11000,1,9000,4,3564,88.01518903173182,992.4354304606816,267087841.6178405,811231171.4285715
-12000,1,21600,158,5565,,,,
-13000,1,21600,59,6965,,,,
-14000,1,21600,9,7760,,,,
-15000,1,3600,22122,1335,,,,
-16000,1,28800,130,9421,12.262731018331898,19749.432156075072,0,787292327.46875
-17000,1,28800,6,1990,,,,
-18000,1,172800,13,73,,,,
-19000,1,172800,7,129,,,,
-20000,1,25200,4,25211,82.16279473845965,0,5113844.667942916,240912572.72941175
-21000,1,21600,18,6099,,,,
-22000,1,21600,27,7131,,,,
-23000,1,1800,61,1079,35.02319701051263,5818814744.200479,0,91742777.25
-24000,1,3600,5,2306,0.11814596015380158,,,33983854837.760006
-25000,1,960,1,59,2.025062333453586,0,0,118141168
-26000,1,172800,1,20494,87.54061396105656,548.1123048956289,0,224020798.15942028
-27000,4,7200,2,7214,99.2396948622311,441.0514348202534,34392345950.27519,1104895888.4
-28000,1,21600,13,55,1.2148444482641405,,,
-29000,1,21600,171,40,,,,
-30000,1,960,0,42,1.5504320217730077,0,0,112133180
-31000,1,1800,11,183,25.94758412119134,,,129784697856
-32000,1,21600,372,114,1.5571541609296509,,,92681043968
-33000,2,1800,134,139,55.875186246345784,533.6963000565588,754284385.8435647,137041136
-34000,1,7200,74,9,0.94096807333301,4681.666820975073,0,145688575
-35000,1,172800,22,83953,98.95217460976379,,,91787361316.73543
-36000,1,6000,8,152,0.4601673251104144,,,85277047466.66667
-37000,1,900,124,137,96.18834348033303,,,35703571797.333336
-38000,1,21600,12,56,24.892228849477622,,,
-39000,1,21600,12,134,26.487756894710913,,,113801609216
-40000,1,21600,12,229,45.74138522053433,,,42761861802.66667
-41000,1,21600,20,307,0.9428384414161763,,,27184930360.88889
-42000,1,21600,130,386,1.68608777466353,,,49510031732.36363
diff --git a/tests/regression/data/raw-data-every-1000.csv b/tests/regression/data/raw-data-every-1000.csv
new file mode 100644
index 00000000..5c9e0675
--- /dev/null
+++ b/tests/regression/data/raw-data-every-1000.csv
@@ -0,0 +1,35 @@
+,Local Job Id,Quality of Service,Start Time (Timestamp),GPUs,Department
+0,2286873,banana-cream,1483160526,0,Social and Economic Science
+1000,2293183,banana-cream,1483184848,0,Social and Economic Science
+2000,2302227,banana-cream,1483211690,0,Social and Economic Science
+3000,2289494,banana-cream,1483172442,0,Social and Economic Science
+4000,2299179,banana-cream,1483202557,0,Social and Economic Science
+5000,2284778,banana-cream,1483141002,0,Social and Economic Science
+6000,2296200,banana-cream,1483193678,0,Social and Economic Science
+7000,2287312,banana-cream,1483163089,0,Social and Economic Science
+8000,2300378,banana-cream,1483206150,0,Social and Economic Science
+9000,2280848,banana-cream,1483105401,0,Social and Economic Science
+10000,2281978,banana-cream,1483115753,0,Social and Economic Science
+11000,2281388,banana-cream,1483110374,0,Social and Economic Science
+12000,2280890,banana-cream,1483105946,0,Social and Economic Science
+13000,1008307,banana-cream,1483219024,0,Social and Economic Science
+14000,998351,banana-cream,1483159887,0,Social and Economic Science
+15000,1006328,banana-cream,1483212749,0,Social and Economic Science
+16000,1009986,banana-cream,1483223948,0,Social and Economic Science
+17000,994932,banana-cream,1483148678,0,Social and Economic Science
+18000,998665,banana-cream,1483160915,0,Social and Economic Science
+19000,1001452,banana-cream,1483173813,0,Social and Economic Science
+20000,1003324,boysenberry,1483204008,0,Social and Economic Science
+21000,1002994,banana-cream,1483181740,0,Social and Economic Science
+22000,971318,banana-cream,1483067878,0,Social and Economic Science
+23000,983519,banana-cream,1483109661,0,Social and Economic Science
+24000,992780,banana-cream,1483139789,0,Social and Economic Science
+25000,976041,banana-cream,1483083428,0,Social and Economic Science
+26000,983897,banana-cream,1483111546,0,Social and Economic Science
+27000,973004,banana-cream,1483072678,0,Social and Economic Science
+28000,978764,banana-cream,1483091343,0,Social and Economic Science
+29000,983635,banana-cream,1483110039,0,Social and Economic Science
+30000,988096,banana-cream,1483123810,0,Social and Economic Science
+31000,991564,banana-cream,1483135262,0,Social and Economic Science
+32000,973969,banana-cream,1483076149,0,Social and Economic Science
+33000,987124,banana-cream,1483120872,0,Social and Economic Science
diff --git a/tests/regression/data/realms.csv b/tests/regression/data/realms.csv
index f0fb63e9..5e555121 100644
--- a/tests/regression/data/realms.csv
+++ b/tests/regression/data/realms.csv
@@ -1,8 +1,4 @@
 id,label
-Accounts,Accounts
-Allocations,Allocations
 Cloud,Cloud
-Gateways,Gateways
 Jobs,Jobs
-Requests,Requests
-SUPREMM,SUPREMM
+Storage,Storage
diff --git a/tests/regression/test_datawarehouse_regression.py b/tests/regression/test_datawarehouse_regression.py
index 0128b7a8..98f5b1e3 100644
--- a/tests/regression/test_datawarehouse_regression.py
+++ b/tests/regression/test_datawarehouse_regression.py
@@ -29,50 +29,50 @@ def __assert_dfs_equal(
     index_col='id',
     columns_name=None,
 ):
-    expected = pandas.read_csv(
-        DATA_DIR + '/' + data_file,
-        dtype=dtype,
-        index_col=index_col,
-        keep_default_na=False,
-        na_values=[''],
-    ).fillna(numpy.nan)
-    expected.columns = expected.columns.astype('string')
-    expected.columns.name = columns_name
-    if index_col == 'Time':
-        expected.index = pandas.to_datetime(expected.index)
-    assert expected.equals(actual)
+    if "GENERATE_DATA_FILES" in os.environ:
+        actual.to_csv(DATA_DIR + '/' + data_file)
+    else:
+        expected = pandas.read_csv(
+            DATA_DIR + '/' + data_file,
+            dtype=dtype,
+            index_col=index_col,
+            keep_default_na=False,
+            na_values=[''],
+        ).fillna(numpy.nan)
+        expected.columns = expected.columns.astype('string')
+        expected.columns.name = columns_name
+        if index_col == 'Time':
+            expected.index = pandas.to_datetime(expected.index)
+        assert expected.equals(actual)
 
 
 def test_get_raw_data(valid_dw, capsys):
     data = valid_dw.get_raw_data(
-        duration=('2023-05-01', '2023-05-02'),
-        realm='SUPREMM',
+        duration=('2016-01-01', '2016-12-31'),
+        realm='Jobs',
         fields=(
-            'CPU User',
-            'Nodes',
-            'Wall Time',
-            'Wait Time',
-            'Requested Wall Time',
-            'Total memory used',
-            'Mount point "home" data written',
-            'Mount point "scratch" data written',
+            'Local Job Id',
+            'Quality of Service',
+            'GPUs',
+            'Start Time',
+            'Department',
         ),
         filters={
             'Resource': [
-                'STAMPEDE2 TACC',
-                'Bridges 2 RM',
+                'mortorq',
+                'frearson',
             ],
         },
         show_progress=True,
     ).iloc[::1000]
     data.index = data.index.astype('string')
     __assert_dfs_equal(
-        'machine-learning-notebook-example-every-1000.csv',
+        'raw-data-every-1000.csv',
         data,
         dtype='string',
         index_col=0,
     )
-    assert 'Got 42637 rows...DONE' in capsys.readouterr().out
+    assert 'Got 33345 rows...DONE' in capsys.readouterr().out
 
 
 def __assert_descriptor_dfs_equal(data_file, actual):
@@ -80,6 +80,7 @@ def __assert_descriptor_dfs_equal(data_file, actual):
 
 
 def test_describe_realms(valid_dw):
+
     __assert_descriptor_dfs_equal(
         'realms.csv',
         valid_dw.describe_realms(),
@@ -102,8 +103,8 @@ def test_describe_dimensions(valid_dw):
 
 def test_get_filter_values(valid_dw):
     __assert_descriptor_dfs_equal(
-        'jobs-fieldofscience-filter-values.csv',
-        valid_dw.get_filter_values('Jobs', 'Field of Science'),
+        'jobs-pi-group-filter-values.csv',
+        valid_dw.get_filter_values('Jobs', 'PI Group'),
     )
 
 
@@ -120,16 +121,16 @@ def test_get_data_filter_user(valid_dw):
 
 
 @pytest.mark.parametrize(
-    'duration,aggregation_unit,data_file',
+    'aggregation_unit,data_file',
     [
-        (('2023-01-01', '2023-12-31'), 'Quarter', 'jobs-2023-quarters.csv'),
-        (('2022-01-01', '2023-12-31'), 'Year', 'jobs-2022-2023-years.csv'),
+        ('Quarter', 'jobs-2016-2017-quarters.csv'),
+        ('Year', 'jobs-2016-2017-years.csv'),
     ],
     ids=('quarter', 'year'),
 )
-def test_get_data(valid_dw, duration, aggregation_unit, data_file):
+def test_get_data(valid_dw, aggregation_unit, data_file):
     data = valid_dw.get_data(
-        duration=duration,
+        duration= ('2016-01-01', '2017-12-31'),
         realm='Jobs',
         metric='CPU Hours: Total',
         aggregation_unit=aggregation_unit,
diff --git a/tests/unit/test_datawarehouse_unit.py b/tests/unit/test_datawarehouse_unit.py
index 1cf04f43..e7a829cc 100644
--- a/tests/unit/test_datawarehouse_unit.py
+++ b/tests/unit/test_datawarehouse_unit.py
@@ -4,7 +4,7 @@
 from xdmod_data.warehouse import DataWarehouse
 
 
-VALID_XDMOD_URL = 'https://xdmod.access-ci.org'
+VALID_XDMOD_URL = os.environ['XDMOD_HOST']
 INVALID_STR = 'asdlkfjsdlkfisdjkfjd'
 
 

From cff8fe22972afe165e0bb05b984422d9a78d9d63 Mon Sep 17 00:00:00 2001
From: Rodney Garnett <garnettr35@gmail.com>
Date: Wed, 3 Jul 2024 11:17:08 -0400
Subject: [PATCH 56/72] Github actions check 26

---
 .../test_datawarehouse_integration.py          | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/tests/integration/test_datawarehouse_integration.py b/tests/integration/test_datawarehouse_integration.py
index 8cc76939..18c809fc 100644
--- a/tests/integration/test_datawarehouse_integration.py
+++ b/tests/integration/test_datawarehouse_integration.py
@@ -33,14 +33,14 @@
     'describe_raw_realms': (),
     'describe_raw_fields': ('realm',),
 }
-VALID_DATE = '2020-01-01'
+VALID_DATE = '2016-12-25'
 VALID_DIMENSION = 'Resource'
 VALID_VALUES = {
   'duration': 'Yesterday',
   'realm': 'Jobs',
   'metric': 'CPU Hours: Total',
   'dimension': VALID_DIMENSION,
-  'filters': {VALID_DIMENSION: 'frearson'},
+  'filters': {VALID_DIMENSION: 'phillips'},
   'dataset_type': 'timeseries',
   'aggregation_unit': 'Auto',
   'parameter': 'duration',
@@ -268,22 +268,22 @@ def __test_DataFrame_return_value(
         (
             {},
             'Metric',
-            31,
+            0,
         ),
         (
-            {'filters': {'Service Provider': 'StonyBrook'}},
+            {'filters': {'Resource': 'robertson'}},
             'Metric',
             0,
         ),
         (
             {'dimension': 'Resource'},
             'Resource',
-            31,
+            0,
         ),
         (
             {
                 'dimension': 'Resource',
-                'filters': {'Service Provider': 'StonyBrook'},
+                'filters': {'Resource': 'robertson'},
             },
             'Resource',
             0,
@@ -342,19 +342,19 @@ def test_get_data_timeseries_return_value(
             1,
         ),
         (
-            {'filters': {'Service Provider': 'StonyBrook'}},
+            {'filters': {'Resource': 'robertson'}},
             None,
             1,
         ),
         (
             {'dimension': 'Resource'},
             'Resource',
-            8,
+            0,
         ),
         (
             {
                 'dimension': 'Resource',
-                'filters': {'Service Provider': 'StonyBrook'},
+                'filters': {'Resource': 'robertson'},
             },
             'Resource',
             0,

From ff9f38a01c3c73eea46ef363d5e202a1caa047de Mon Sep 17 00:00:00 2001
From: Rodney Garnett <garnettr35@gmail.com>
Date: Wed, 3 Jul 2024 11:21:11 -0400
Subject: [PATCH 57/72] GIthub actions check 27

---
 tests/regression/test_datawarehouse_regression.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/regression/test_datawarehouse_regression.py b/tests/regression/test_datawarehouse_regression.py
index 98f5b1e3..de3fe9e7 100644
--- a/tests/regression/test_datawarehouse_regression.py
+++ b/tests/regression/test_datawarehouse_regression.py
@@ -29,7 +29,7 @@ def __assert_dfs_equal(
     index_col='id',
     columns_name=None,
 ):
-    if "GENERATE_DATA_FILES" in os.environ:
+    if 'GENERATE_DATA_FILES' in os.environ:
         actual.to_csv(DATA_DIR + '/' + data_file)
     else:
         expected = pandas.read_csv(
@@ -130,7 +130,7 @@ def test_get_data_filter_user(valid_dw):
 )
 def test_get_data(valid_dw, aggregation_unit, data_file):
     data = valid_dw.get_data(
-        duration= ('2016-01-01', '2017-12-31'),
+        duration=('2016-01-01', '2017-12-31'),
         realm='Jobs',
         metric='CPU Hours: Total',
         aggregation_unit=aggregation_unit,

From 22adc79dcd8b7ddb602bd5a2bf2f890c35320b9f Mon Sep 17 00:00:00 2001
From: Rodney Garnett <garnettr35@gmail.com>
Date: Wed, 3 Jul 2024 14:54:05 -0400
Subject: [PATCH 58/72] Github actions check 28

---
 .github/workflows/python-package.yml               |  2 +-
 .../integration/test_datawarehouse_integration.py  | 14 +++++++-------
 tests/regression/data/jobs-2016-2017-month.csv     |  3 +++
 tests/regression/test_datawarehouse_regression.py  |  5 +++--
 4 files changed, 14 insertions(+), 10 deletions(-)
 create mode 100644 tests/regression/data/jobs-2016-2017-month.csv

diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml
index a5833b78..199a8c27 100644
--- a/.github/workflows/python-package.yml
+++ b/.github/workflows/python-package.yml
@@ -51,7 +51,7 @@ jobs:
       run: |
         # Create a file with 10,000 users so that we can test filters that have more than 10,000 values
         for i in {1..10000}; do
-          unix=$((1704085200+$i))
+          unix=$((149361090+$i))
           echo "$i|2269212|frearson|black|banana-cream|caste|caste|45580|user$i|379330|$(date +'%Y-%m-%dT%H:%M:%S' -d @$unix)|$(date +'%Y-%m-%dT%H:%M:%S' -d @$(($unix+1)))|$(date +'%Y-%m-%dT%H:%M:%S' -d @$(($unix+2)))|$(date +'%Y-%m-%dT%H:%M:%S' -d @$(($unix+3)))|2-23:24:40|0:0|COMPLETED|1|16|16|62.50Gn|cpu=16,mem=62.50G,node=1|billing=16,cpu=16,mem=62.50G,node=1|3-00:00:00|cpn-m25-02-01|310f6011b278ebb15df2d199e0119de78d261d8c478a203abc97dc3dda7702ae" >> 10000users.log
         done
         #run tests
diff --git a/tests/integration/test_datawarehouse_integration.py b/tests/integration/test_datawarehouse_integration.py
index 18c809fc..ecf2101c 100644
--- a/tests/integration/test_datawarehouse_integration.py
+++ b/tests/integration/test_datawarehouse_integration.py
@@ -252,7 +252,7 @@ def __test_DataFrame_return_value(
 
 
 get_data_return_value_test_params = {
-    'duration': ('2020-01-01', '2020-01-31'),
+    'duration': ('2016-12-22', '2017-01-31'),
     'realm': 'Jobs',
     'metric': 'Number of Users: Active',
     'dimension': 'None',
@@ -268,17 +268,17 @@ def __test_DataFrame_return_value(
         (
             {},
             'Metric',
-            0,
+            11,
         ),
         (
             {'filters': {'Resource': 'robertson'}},
             'Metric',
-            0,
+            6,
         ),
         (
             {'dimension': 'Resource'},
             'Resource',
-            0,
+            11,
         ),
         (
             {
@@ -286,7 +286,7 @@ def __test_DataFrame_return_value(
                 'filters': {'Resource': 'robertson'},
             },
             'Resource',
-            0,
+            6,
         ),
     ],
     ids=(
@@ -349,7 +349,7 @@ def test_get_data_timeseries_return_value(
         (
             {'dimension': 'Resource'},
             'Resource',
-            0,
+            5,
         ),
         (
             {
@@ -357,7 +357,7 @@ def test_get_data_timeseries_return_value(
                 'filters': {'Resource': 'robertson'},
             },
             'Resource',
-            0,
+            1,
         ),
     ],
     ids=(
diff --git a/tests/regression/data/jobs-2016-2017-month.csv b/tests/regression/data/jobs-2016-2017-month.csv
new file mode 100644
index 00000000..22a503c0
--- /dev/null
+++ b/tests/regression/data/jobs-2016-2017-month.csv
@@ -0,0 +1,3 @@
+Time,CPU Hours: Total
+2016-12-01,770775.9989
+2017-01-01,69779.8381
diff --git a/tests/regression/test_datawarehouse_regression.py b/tests/regression/test_datawarehouse_regression.py
index de3fe9e7..bdbbbf1a 100644
--- a/tests/regression/test_datawarehouse_regression.py
+++ b/tests/regression/test_datawarehouse_regression.py
@@ -30,7 +30,7 @@ def __assert_dfs_equal(
     columns_name=None,
 ):
     if 'GENERATE_DATA_FILES' in os.environ:
-        actual.to_csv(DATA_DIR + '/' + data_file)
+        actual.to_csv(DATA_DIR + '/' + data_file)  # pragma: no cover
     else:
         expected = pandas.read_csv(
             DATA_DIR + '/' + data_file,
@@ -123,10 +123,11 @@ def test_get_data_filter_user(valid_dw):
 @pytest.mark.parametrize(
     'aggregation_unit,data_file',
     [
+        ('Month', 'jobs-2016-2017-month.csv'),
         ('Quarter', 'jobs-2016-2017-quarters.csv'),
         ('Year', 'jobs-2016-2017-years.csv'),
     ],
-    ids=('quarter', 'year'),
+    ids=('month', 'quarter', 'year'),
 )
 def test_get_data(valid_dw, aggregation_unit, data_file):
     data = valid_dw.get_data(

From f4ab22c68f0cf831696bc453d891136af907213d Mon Sep 17 00:00:00 2001
From: Rodney Garnett <garnettr35@gmail.com>
Date: Wed, 3 Jul 2024 17:04:35 -0400
Subject: [PATCH 59/72] Github actions check 29

---
 .github/workflows/python-package.yml              | 2 +-
 tests/regression/data/jobs-2016-2017-month.csv    | 1 +
 tests/regression/data/jobs-2016-2017-quarters.csv | 1 +
 tests/regression/data/jobs-2016-2017-years.csv    | 2 +-
 tests/regression/test_datawarehouse_regression.py | 2 +-
 tests/unit/test_datawarehouse_unit.py             | 4 ++++
 6 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml
index 199a8c27..c38ffe9c 100644
--- a/.github/workflows/python-package.yml
+++ b/.github/workflows/python-package.yml
@@ -51,7 +51,7 @@ jobs:
       run: |
         # Create a file with 10,000 users so that we can test filters that have more than 10,000 values
         for i in {1..10000}; do
-          unix=$((149361090+$i))
+          unix=$((1493596500+$i))
           echo "$i|2269212|frearson|black|banana-cream|caste|caste|45580|user$i|379330|$(date +'%Y-%m-%dT%H:%M:%S' -d @$unix)|$(date +'%Y-%m-%dT%H:%M:%S' -d @$(($unix+1)))|$(date +'%Y-%m-%dT%H:%M:%S' -d @$(($unix+2)))|$(date +'%Y-%m-%dT%H:%M:%S' -d @$(($unix+3)))|2-23:24:40|0:0|COMPLETED|1|16|16|62.50Gn|cpu=16,mem=62.50G,node=1|billing=16,cpu=16,mem=62.50G,node=1|3-00:00:00|cpn-m25-02-01|310f6011b278ebb15df2d199e0119de78d261d8c478a203abc97dc3dda7702ae" >> 10000users.log
         done
         #run tests
diff --git a/tests/regression/data/jobs-2016-2017-month.csv b/tests/regression/data/jobs-2016-2017-month.csv
index 22a503c0..b4959c8f 100644
--- a/tests/regression/data/jobs-2016-2017-month.csv
+++ b/tests/regression/data/jobs-2016-2017-month.csv
@@ -1,3 +1,4 @@
 Time,CPU Hours: Total
 2016-12-01,770775.9989
 2017-01-01,69779.8381
+2017-04-01,44.4444
diff --git a/tests/regression/data/jobs-2016-2017-quarters.csv b/tests/regression/data/jobs-2016-2017-quarters.csv
index 7d5b24e2..cc2316af 100644
--- a/tests/regression/data/jobs-2016-2017-quarters.csv
+++ b/tests/regression/data/jobs-2016-2017-quarters.csv
@@ -1,3 +1,4 @@
 Time,CPU Hours: Total
 2016-10-01,770775.9989
 2017-01-01,69779.8381
+2017-04-01,44.4444
diff --git a/tests/regression/data/jobs-2016-2017-years.csv b/tests/regression/data/jobs-2016-2017-years.csv
index 7e9cf848..d6faef19 100644
--- a/tests/regression/data/jobs-2016-2017-years.csv
+++ b/tests/regression/data/jobs-2016-2017-years.csv
@@ -1,3 +1,3 @@
 Time,CPU Hours: Total
 2016-01-01,770775.9989
-2017-01-01,69779.8381
+2017-01-01,69824.2825
diff --git a/tests/regression/test_datawarehouse_regression.py b/tests/regression/test_datawarehouse_regression.py
index bdbbbf1a..51fed6e3 100644
--- a/tests/regression/test_datawarehouse_regression.py
+++ b/tests/regression/test_datawarehouse_regression.py
@@ -112,7 +112,7 @@ def test_get_data_filter_user(valid_dw):
     # Make sure the filter validation works for a user whose list position is
     # greater than 10000 — this will raise an exception if it doesn't work.
     valid_dw.get_data(
-        duration=('2023-01-01', '2023-01-01'),
+        duration=('2016-01-01', '2017-12-31'),
         realm='Jobs',
         metric='CPU Hours: Total',
         dataset_type='aggregate',
diff --git a/tests/unit/test_datawarehouse_unit.py b/tests/unit/test_datawarehouse_unit.py
index e7a829cc..b3b04810 100644
--- a/tests/unit/test_datawarehouse_unit.py
+++ b/tests/unit/test_datawarehouse_unit.py
@@ -78,3 +78,7 @@ def test___enter___RuntimeError_401():
     ):
         with DataWarehouse(VALID_XDMOD_URL) as dw:
             dw.describe_realms()
+
+def test_exit_without_enter():
+    dw = DataWarehouse(VALID_XDMOD_URL)
+    dw.__exit__(None, None, None)

From aa8175c8e6566a210842f2b6694075f436e4274e Mon Sep 17 00:00:00 2001
From: Rodney Garnett <garnettr35@gmail.com>
Date: Tue, 9 Jul 2024 15:43:08 -0400
Subject: [PATCH 60/72] Github actions check 30

---
 .coveragerc                                   |  2 +
 .github/workflows/python-package.yml          |  4 +-
 .../regression/data/jobs-2016-2017-month.csv  |  3 +-
 .../data/jobs-2016-2017-quarters.csv          |  1 +
 .../regression/data/jobs-2016-2017-years.csv  |  2 +-
 .../data/raw-data-every-1000-pt2.csv          | 56 +++++++++++++++++
 .../data/raw-data-every-1000-pt3.csv          | 35 +++++++++++
 .../test_datawarehouse_regression.py          | 62 +++++++++++++------
 xdmod_data/_response_processor.py             |  2 +-
 9 files changed, 143 insertions(+), 24 deletions(-)
 create mode 100644 .coveragerc
 create mode 100644 tests/regression/data/raw-data-every-1000-pt2.csv
 create mode 100644 tests/regression/data/raw-data-every-1000-pt3.csv

diff --git a/.coveragerc b/.coveragerc
new file mode 100644
index 00000000..d07bcb56
--- /dev/null
+++ b/.coveragerc
@@ -0,0 +1,2 @@
+[run]
+dynamic_context = test_function
\ No newline at end of file
diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml
index c38ffe9c..fe73d097 100644
--- a/.github/workflows/python-package.yml
+++ b/.github/workflows/python-package.yml
@@ -51,9 +51,11 @@ jobs:
       run: |
         # Create a file with 10,000 users so that we can test filters that have more than 10,000 values
         for i in {1..10000}; do
-          unix=$((1493596500+$i))
+          unix=$((1498866840+$i))
           echo "$i|2269212|frearson|black|banana-cream|caste|caste|45580|user$i|379330|$(date +'%Y-%m-%dT%H:%M:%S' -d @$unix)|$(date +'%Y-%m-%dT%H:%M:%S' -d @$(($unix+1)))|$(date +'%Y-%m-%dT%H:%M:%S' -d @$(($unix+2)))|$(date +'%Y-%m-%dT%H:%M:%S' -d @$(($unix+3)))|2-23:24:40|0:0|COMPLETED|1|16|16|62.50Gn|cpu=16,mem=62.50G,node=1|billing=16,cpu=16,mem=62.50G,node=1|3-00:00:00|cpn-m25-02-01|310f6011b278ebb15df2d199e0119de78d261d8c478a203abc97dc3dda7702ae" >> 10000users.log
         done
+        unix=1498886840
+        echo "01|2269212|mortorq|black|banana-cream|caste|caste|45580|user01|379330|$(date +'%Y-%m-%dT%H:%M:%S' -d @$unix)|$(date +'%Y-%m-%dT%H:%M:%S' -d @$(($unix+1)))|$(date +'%Y-%m-%dT%H:%M:%S' -d @$(($unix+2)))|$(date +'%Y-%m-%dT%H:%M:%S' -d @$(($unix+3)))|2-23:24:40|0:0|COMPLETED|1|16|16|62.50Gn|cpu=16,mem=62.50G,node=1|billing=16,cpu=16,mem=62.50G,node=1|3-00:00:00|cpn-m25-02-01|310f6011b278ebb15df2d199e0119de78d261d8c478a203abc97dc3dda7702ae" >> 10000users.log
         #run tests
         coverage erase
         declare -a portal_versions=$(yq '.services | keys[]' .github/workflows/docker-compose.yaml)
diff --git a/tests/regression/data/jobs-2016-2017-month.csv b/tests/regression/data/jobs-2016-2017-month.csv
index b4959c8f..048783c1 100644
--- a/tests/regression/data/jobs-2016-2017-month.csv
+++ b/tests/regression/data/jobs-2016-2017-month.csv
@@ -1,4 +1,5 @@
 Time,CPU Hours: Total
 2016-12-01,770775.9989
 2017-01-01,69779.8381
-2017-04-01,44.4444
+2017-06-01,44.4444
+2017-07-01,0.0044
diff --git a/tests/regression/data/jobs-2016-2017-quarters.csv b/tests/regression/data/jobs-2016-2017-quarters.csv
index cc2316af..df7ce99c 100644
--- a/tests/regression/data/jobs-2016-2017-quarters.csv
+++ b/tests/regression/data/jobs-2016-2017-quarters.csv
@@ -2,3 +2,4 @@ Time,CPU Hours: Total
 2016-10-01,770775.9989
 2017-01-01,69779.8381
 2017-04-01,44.4444
+2017-07-01,0.0044
diff --git a/tests/regression/data/jobs-2016-2017-years.csv b/tests/regression/data/jobs-2016-2017-years.csv
index d6faef19..326a2c29 100644
--- a/tests/regression/data/jobs-2016-2017-years.csv
+++ b/tests/regression/data/jobs-2016-2017-years.csv
@@ -1,3 +1,3 @@
 Time,CPU Hours: Total
 2016-01-01,770775.9989
-2017-01-01,69824.2825
+2017-01-01,69824.2869
diff --git a/tests/regression/data/raw-data-every-1000-pt2.csv b/tests/regression/data/raw-data-every-1000-pt2.csv
new file mode 100644
index 00000000..f830643e
--- /dev/null
+++ b/tests/regression/data/raw-data-every-1000-pt2.csv
@@ -0,0 +1,56 @@
+,Local Job Id,Resource,Timezone,System Username (Deidentified),User,Organization,Quality of Service,Submit Time (Timestamp),Start Time (Timestamp),End Time (Timestamp),Eligible Time (Timestamp),Nodes,Cores,GPUs,Memory Used,Wall Time,Wait Time,Core Time,GPU Time,Exit Code,Exit State,Requested Cores,Requested memory,Requested Wall Time,Queue,Decanal Unit,Department,PI Group
+0,977669,Posidriv,UTC,2e73831287e6f71196828b70000ec53828d6148b,"Dove, Turtle",Screwdriver,apple,1482356614,1482390210,1483069389,1482356614,1,12,0,-1,679179,33596,8150148,0,0:0,COMPLETED,12,48000Mn,10800000,croutons,Mathematical and Physical Sciences,Chemistry,Physical Chemistry
+1000,6106087,Robertson,UTC,ea56fb19a839cb996b0e014f70d35a03606aa19f,Moorhen,Screwdriver,curry,1482980998,1482981066,1483080831,1482980998,1,1,0,-1,99765,68,99765,0,0:0,COMPLETED,1,2147486448Mn,259200,white,Computer and Information Science and Engineering,Microelectronic Information Processing Systems,Systems Prototyping and Fabrication
+2000,975068,Mortorq,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483080229,1483080229,1483080249,1483080229,1,8,0,-1,20,0,160,0,1:0,FAILED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+3000,974176,Mortorq,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483076858,1483076859,1483076886,1483076858,1,8,0,-1,27,1,216,0,1:0,FAILED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+4000,985572,Mortorq,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483116354,1483116924,1483116945,1483116354,1,8,0,-1,21,570,168,0,1:0,FAILED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+5000,990168,Mortorq,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483124951,1483130983,1483131006,1483124951,1,8,0,-1,23,6032,184,0,0:0,COMPLETED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+6000,969126,Mortorq,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483059498,1483059498,1483059563,1483059498,1,8,0,-1,65,0,520,0,0:0,COMPLETED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+7000,978262,Mortorq,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483089659,1483089659,1483089703,1483089659,1,8,0,-1,44,0,352,0,1:0,FAILED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+8000,171798,Phillips,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483083351,1483083351,1483083386,1483083351,1,8,0,-1,35,0,280,0,1:0,FAILED,8,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+9000,968562,Mortorq,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483057298,1483057299,1483057345,1483057298,1,8,0,-1,46,1,368,0,0:0,COMPLETED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+10000,974174,Mortorq,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483076846,1483076847,1483076913,1483076846,1,8,0,-1,66,1,528,0,1:0,FAILED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+11000,979960,Mortorq,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483095351,1483095353,1483095404,1483095351,1,8,0,-1,51,2,408,0,1:0,FAILED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+12000,984736,Mortorq,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483114441,1483114442,1483114544,1483114441,1,8,0,-1,102,1,816,0,1:0,FAILED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+13000,2277762,Frearson,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483076479,1483076480,1483078267,1483076479,1,12,0,-1,1787,1,21444,0,0:0,COMPLETED,12,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+14000,6112330,Robertson,UTC,c73e24b5c1a6de69db0bd9bf67fb461cf4da104f,Lapwing,Screwdriver,curry,1483136546,1483136628,1483138226,1483136546,1,1,0,-1,1598,82,1598,0,0:0,CANCELLED by 0,1,8Gn,36000,white,Engineering,Design and Manufacturing Systems,Operations Research and Production Systems
+15000,1006230,Posidriv,UTC,cdd144ba0c808c63ec0bbdeb890cd90b74587c9b,"Bunting, Reed",Screwdriver,banana-cream,1483083136,1483083193,1483083332,1483083136,1,8,0,-1,139,57,1112,0,0:0,COMPLETED,8,2147486448Mn,259200,black,Mathematical and Physical Sciences,Chemistry,Physical Chemistry
+16000,1008815,Posidriv,UTC,cdd144ba0c808c63ec0bbdeb890cd90b74587c9b,"Bunting, Reed",Screwdriver,banana-cream,1483118123,1483118171,1483118309,1483118123,1,8,0,-1,138,48,1104,0,0:0,COMPLETED,8,2147486448Mn,259200,black,Mathematical and Physical Sciences,Chemistry,Physical Chemistry
+17000,6111037[445],Robertson,UTC,420f3254a8b357efce12369d50ff33dc437f4f0b,"Grey, Great",Screwdriver,curry,1483125912,1483129321,1483129510,1483125913,1,1,0,-1,189,3409,189,0,0:0,COMPLETED,1,2147487648Mn,900,white,"Social, Behavioral, and Economic Sciences",Social and Economic Science,"Decision, Risk, and Management Science"
+18000,6112283[353],Robertson,UTC,420f3254a8b357efce12369d50ff33dc437f4f0b,"Grey, Great",Screwdriver,curry,1483135648,1483139250,1483139291,1483135648,1,1,0,-1,41,3602,41,0,0:0,COMPLETED,1,48000Mn,10800,white,"Social, Behavioral, and Economic Sciences",Social and Economic Science,"Decision, Risk, and Management Science"
+19000,989648,Mortorq,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483123196,1483129202,1483129233,1483123196,1,8,0,-1,31,6006,248,0,1:0,FAILED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+20000,993076,Mortorq,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483134403,1483140976,1483141013,1483134403,1,8,0,-1,37,6573,296,0,0:0,COMPLETED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+21000,6082994,Robertson,UTC,d2e5a44089d3217a3c4538f936d800471b145a4c,"Thrush, Swainson's",Screwdriver,curry,1482264225,1483074239,1483076033,1482264225,1,4,0,-1,1794,810014,7176,0,0:0,COMPLETED,4,125Gn,7200,white,Mathematical and Physical Sciences,Astronomical Sciences,Galactic Astronomy
+22000,2279431,Frearson,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483092109,1483092109,1483094048,1483092109,1,12,0,-1,1939,0,23268,0,0:0,COMPLETED,12,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+23000,2277824,Frearson,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483077039,1483077040,1483079075,1483077039,1,12,0,-1,2035,1,24420,0,0:0,COMPLETED,12,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+24000,1003828,Posidriv,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483050771,1483057342,1483059250,1483050771,1,12,0,-1,1908,6571,22896,0,0:0,COMPLETED,12,2147486448Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+25000,6083488,Robertson,UTC,d2e5a44089d3217a3c4538f936d800471b145a4c,"Thrush, Swainson's",Screwdriver,curry,1482264356,1483114035,1483115957,1482264356,1,4,0,-1,1922,849679,7688,0,0:0,COMPLETED,4,125Gn,7200,white,Mathematical and Physical Sciences,Astronomical Sciences,Galactic Astronomy
+26000,984588,Mortorq,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483114013,1483114015,1483118919,1483114013,1,8,0,-1,4904,2,39232,0,1:0,FAILED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+27000,6106282,Robertson,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,curry,1482988989,1483091687,1483096295,1482988989,1,8,0,-1,4608,102698,36864,0,0:0,COMPLETED,8,2147486448Mn,108000,white,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+28000,6110529,Robertson,UTC,a3a4db35c34bc1ae71d8ba76352b66796cddb74e,"Grey, Lesser",Screwdriver,curry,1483118933,1483140830,1483172555,1483118933,1,1,0,-1,31725,21897,31725,0,0:0,COMPLETED,1,2147486448Mn,259200,white,Geosciences,Earth Sciences,Tectonics
+29000,1005802,Mortorq,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483210021,1483210931,1483210958,1483210021,1,8,0,-1,27,910,216,0,0:0,COMPLETED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+30000,186979,Phillips,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483207102,1483209435,1483209451,1483207102,1,8,0,-1,16,2333,128,0,0:0,COMPLETED,8,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+31000,998331,Mortorq,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483154514,1483159821,1483159839,1483154514,1,8,0,-1,18,5307,144,0,0:0,COMPLETED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+32000,184674,Phillips,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483161882,1483195550,1483195576,1483161882,1,8,0,-1,26,33668,208,0,0:0,COMPLETED,8,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+33000,2293381,Frearson,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483185388,1483185388,1483187114,1483185388,1,12,0,-1,1726,0,20712,0,0:0,COMPLETED,12,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+34000,2302343,Frearson,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483212133,1483212133,1483213855,1483212133,1,12,0,-1,1722,0,20664,0,0:0,COMPLETED,12,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+35000,2286627,Frearson,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483158678,1483158679,1483159873,1483158678,1,12,0,-1,1194,1,14328,0,0:0,COMPLETED,12,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+36000,2296175,Frearson,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483193616,1483193617,1483195331,1483193616,1,12,0,-1,1714,1,20568,0,0:0,COMPLETED,12,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+37000,2304947,Frearson,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483220143,1483220144,1483221600,1483220143,1,12,0,-1,1456,1,17472,0,0:0,COMPLETED,12,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+38000,1004771,Mortorq,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483207312,1483207904,1483207951,1483207312,1,8,0,-1,47,592,376,0,0:0,COMPLETED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+39000,1008199,Mortorq,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483217878,1483218733,1483218884,1483217878,1,8,0,-1,151,855,1208,0,0:0,COMPLETED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+40000,1010919,Posidriv,UTC,cdd144ba0c808c63ec0bbdeb890cd90b74587c9b,"Bunting, Reed",Screwdriver,banana-cream,1483143792,1483143885,1483144233,1483143792,1,8,0,-1,348,93,2784,0,0:0,COMPLETED,8,2147486448Mn,259200,black,Mathematical and Physical Sciences,Chemistry,Physical Chemistry
+41000,1014861,Posidriv,UTC,cdd144ba0c808c63ec0bbdeb890cd90b74587c9b,"Bunting, Reed",Screwdriver,banana-cream,1483180668,1483180806,1483180917,1483180668,1,8,0,-1,111,138,888,0,0:0,COMPLETED,8,2147486448Mn,259200,black,Mathematical and Physical Sciences,Chemistry,Physical Chemistry
+42000,6113756[245],Robertson,UTC,420f3254a8b357efce12369d50ff33dc437f4f0b,"Grey, Great",Screwdriver,curry,1483199295,1483201396,1483201474,1483199297,1,1,0,-1,78,2101,78,0,0:0,COMPLETED,1,2147531648Mn,1800,white,"Social, Behavioral, and Economic Sciences",Social and Economic Science,"Decision, Risk, and Management Science"
+43000,994128,Mortorq,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483138376,1483145047,1483145213,1483138376,1,8,0,-1,166,6671,1328,0,0:0,COMPLETED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+44000,997525,Mortorq,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483151518,1483157350,1483157384,1483151518,1,8,0,-1,34,5832,272,0,0:0,COMPLETED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+45000,1000594,Mortorq,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483161348,1483168897,1483168986,1483161348,1,8,0,-1,89,7549,712,0,0:0,COMPLETED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+46000,1010026,Posidriv,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483132282,1483143164,1483144904,1483132282,1,12,0,-1,1740,10882,20880,0,0:0,COMPLETED,12,2147486448Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+47000,184170,Phillips,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483158181,1483190013,1483190125,1483158181,1,8,0,-1,112,31832,896,0,0:0,COMPLETED,8,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+48000,2288942,Frearson,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483170250,1483170250,1483172607,1483170250,1,12,0,-1,2357,0,28284,0,0:0,COMPLETED,12,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+49000,2302966,Frearson,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483214056,1483214056,1483216359,1483214056,1,12,0,-1,2303,0,27636,0,0:0,COMPLETED,12,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+50000,2291733,Frearson,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483180588,1483180589,1483182543,1483180588,1,12,0,-1,1954,1,23448,0,0:0,COMPLETED,12,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+51000,2306026,Frearson,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483223552,1483223553,1483225424,1483223552,1,12,0,-1,1871,1,22452,0,0:0,COMPLETED,12,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+52000,1014640,Posidriv,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483179697,1483191989,1483193818,1483179697,1,12,0,-1,1829,12292,21948,0,0:0,COMPLETED,12,2147486448Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+53000,1012058,Posidriv,UTC,2e73831287e6f71196828b70000ec53828d6148b,"Dove, Turtle",Screwdriver,apple,1483157875,1483157966,1483167364,1483157875,1,8,0,-1,9398,91,75184,0,0:0,COMPLETED,8,23000Mn,10800000,croutons,Mathematical and Physical Sciences,Chemistry,Physical Chemistry
+54000,6108290,Robertson,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,curry,1483054131,1483207468,1483213641,1483054131,1,8,0,-1,6173,153337,49384,0,0:0,COMPLETED,8,2147486448Mn,108000,white,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
diff --git a/tests/regression/data/raw-data-every-1000-pt3.csv b/tests/regression/data/raw-data-every-1000-pt3.csv
new file mode 100644
index 00000000..5c9e0675
--- /dev/null
+++ b/tests/regression/data/raw-data-every-1000-pt3.csv
@@ -0,0 +1,35 @@
+,Local Job Id,Quality of Service,Start Time (Timestamp),GPUs,Department
+0,2286873,banana-cream,1483160526,0,Social and Economic Science
+1000,2293183,banana-cream,1483184848,0,Social and Economic Science
+2000,2302227,banana-cream,1483211690,0,Social and Economic Science
+3000,2289494,banana-cream,1483172442,0,Social and Economic Science
+4000,2299179,banana-cream,1483202557,0,Social and Economic Science
+5000,2284778,banana-cream,1483141002,0,Social and Economic Science
+6000,2296200,banana-cream,1483193678,0,Social and Economic Science
+7000,2287312,banana-cream,1483163089,0,Social and Economic Science
+8000,2300378,banana-cream,1483206150,0,Social and Economic Science
+9000,2280848,banana-cream,1483105401,0,Social and Economic Science
+10000,2281978,banana-cream,1483115753,0,Social and Economic Science
+11000,2281388,banana-cream,1483110374,0,Social and Economic Science
+12000,2280890,banana-cream,1483105946,0,Social and Economic Science
+13000,1008307,banana-cream,1483219024,0,Social and Economic Science
+14000,998351,banana-cream,1483159887,0,Social and Economic Science
+15000,1006328,banana-cream,1483212749,0,Social and Economic Science
+16000,1009986,banana-cream,1483223948,0,Social and Economic Science
+17000,994932,banana-cream,1483148678,0,Social and Economic Science
+18000,998665,banana-cream,1483160915,0,Social and Economic Science
+19000,1001452,banana-cream,1483173813,0,Social and Economic Science
+20000,1003324,boysenberry,1483204008,0,Social and Economic Science
+21000,1002994,banana-cream,1483181740,0,Social and Economic Science
+22000,971318,banana-cream,1483067878,0,Social and Economic Science
+23000,983519,banana-cream,1483109661,0,Social and Economic Science
+24000,992780,banana-cream,1483139789,0,Social and Economic Science
+25000,976041,banana-cream,1483083428,0,Social and Economic Science
+26000,983897,banana-cream,1483111546,0,Social and Economic Science
+27000,973004,banana-cream,1483072678,0,Social and Economic Science
+28000,978764,banana-cream,1483091343,0,Social and Economic Science
+29000,983635,banana-cream,1483110039,0,Social and Economic Science
+30000,988096,banana-cream,1483123810,0,Social and Economic Science
+31000,991564,banana-cream,1483135262,0,Social and Economic Science
+32000,973969,banana-cream,1483076149,0,Social and Economic Science
+33000,987124,banana-cream,1483120872,0,Social and Economic Science
diff --git a/tests/regression/test_datawarehouse_regression.py b/tests/regression/test_datawarehouse_regression.py
index 51fed6e3..591e6607 100644
--- a/tests/regression/test_datawarehouse_regression.py
+++ b/tests/regression/test_datawarehouse_regression.py
@@ -46,33 +46,55 @@ def __assert_dfs_equal(
         assert expected.equals(actual)
 
 
-def test_get_raw_data(valid_dw, capsys):
-    data = valid_dw.get_raw_data(
-        duration=('2016-01-01', '2016-12-31'),
-        realm='Jobs',
-        fields=(
-            'Local Job Id',
-            'Quality of Service',
-            'GPUs',
-            'Start Time',
-            'Department',
+@pytest.mark.parametrize(
+    'additional_params, number, csv_title',
+    [
+        (
+            {},
+            '54747',
+            'raw-data-every-1000-pt2.csv',
+        ),
+        (
+            {
+                'fields':
+                (
+                'Local Job Id',
+                'Quality of Service',
+                'GPUs',
+                'Start Time',
+                'Department',
+                ),
+                'filters':
+                {
+                'Resource':
+                [
+                    'mortorq',
+                    'frearson',
+                ],
+                },
+            },
+            '33345',
+            'raw-data-every-1000-pt3.csv',
         ),
-        filters={
-            'Resource': [
-                'mortorq',
-                'frearson',
-            ],
-        },
-        show_progress=True,
-    ).iloc[::1000]
+    ]
+)
+
+def test_get_raw_data(valid_dw, capsys,additional_params,number,csv_title):
+    defult_params = {
+        'duration':('2016-01-01', '2016-12-31'),
+        'realm':'Jobs',
+        'show_progress':True,
+    }
+    params={**defult_params,**additional_params}
+    data = valid_dw.get_raw_data(**params).iloc[::1000]
     data.index = data.index.astype('string')
     __assert_dfs_equal(
-        'raw-data-every-1000.csv',
+        csv_title,
         data,
         dtype='string',
         index_col=0,
     )
-    assert 'Got 33345 rows...DONE' in capsys.readouterr().out
+    assert 'Got '+ number +' rows...DONE' in capsys.readouterr().out
 
 
 def __assert_descriptor_dfs_equal(data_file, actual):
diff --git a/xdmod_data/_response_processor.py b/xdmod_data/_response_processor.py
index 9a72d9fc..134b0329 100644
--- a/xdmod_data/_response_processor.py
+++ b/xdmod_data/_response_processor.py
@@ -18,7 +18,7 @@ def _process_get_data_response(dw, params, response):
     csv_data = csv.reader(response.splitlines())
     if params['dataset_type'] == 'timeseries':
         return __parse_timeseries_csv_data(dw, params, csv_data)
-    elif params['dataset_type'] == 'aggregate':
+    else: 
         return __parse_aggregate_csv_data(params, csv_data)
 
 

From 165a93b31922e00afdd5555b1c23144d351cb157 Mon Sep 17 00:00:00 2001
From: Rodney Garnett <garnettr35@gmail.com>
Date: Tue, 9 Jul 2024 15:53:32 -0400
Subject: [PATCH 61/72] Github actions check 31

---
 .../test_datawarehouse_regression.py          | 35 +++++++++----------
 tests/unit/test_datawarehouse_unit.py         |  1 +
 xdmod_data/_response_processor.py             |  2 +-
 3 files changed, 19 insertions(+), 19 deletions(-)

diff --git a/tests/regression/test_datawarehouse_regression.py b/tests/regression/test_datawarehouse_regression.py
index 591e6607..8446fcec 100644
--- a/tests/regression/test_datawarehouse_regression.py
+++ b/tests/regression/test_datawarehouse_regression.py
@@ -58,34 +58,33 @@ def __assert_dfs_equal(
             {
                 'fields':
                 (
-                'Local Job Id',
-                'Quality of Service',
-                'GPUs',
-                'Start Time',
-                'Department',
+                    'Local Job Id',
+                    'Quality of Service',
+                    'GPUs',
+                    'Start Time',
+                    'Department',
                 ),
                 'filters':
                 {
-                'Resource':
-                [
-                    'mortorq',
-                    'frearson',
-                ],
+                    'Resource':
+                    [
+                        'mortorq',
+                        'frearson',
+                    ],
                 },
             },
             '33345',
             'raw-data-every-1000-pt3.csv',
         ),
-    ]
+    ],
 )
-
-def test_get_raw_data(valid_dw, capsys,additional_params,number,csv_title):
+def test_get_raw_data(valid_dw, capsys, additional_params, number, csv_title):
     defult_params = {
-        'duration':('2016-01-01', '2016-12-31'),
-        'realm':'Jobs',
-        'show_progress':True,
+        'duration': ('2016-01-01', '2016-12-31'),
+        'realm': 'Jobs',
+        'show_progress': True,
     }
-    params={**defult_params,**additional_params}
+    params = {**defult_params, **additional_params}
     data = valid_dw.get_raw_data(**params).iloc[::1000]
     data.index = data.index.astype('string')
     __assert_dfs_equal(
@@ -94,7 +93,7 @@ def test_get_raw_data(valid_dw, capsys,additional_params,number,csv_title):
         dtype='string',
         index_col=0,
     )
-    assert 'Got '+ number +' rows...DONE' in capsys.readouterr().out
+    assert 'Got ' + number + ' rows...DONE' in capsys.readouterr().out
 
 
 def __assert_descriptor_dfs_equal(data_file, actual):
diff --git a/tests/unit/test_datawarehouse_unit.py b/tests/unit/test_datawarehouse_unit.py
index b3b04810..4749e13e 100644
--- a/tests/unit/test_datawarehouse_unit.py
+++ b/tests/unit/test_datawarehouse_unit.py
@@ -79,6 +79,7 @@ def test___enter___RuntimeError_401():
         with DataWarehouse(VALID_XDMOD_URL) as dw:
             dw.describe_realms()
 
+
 def test_exit_without_enter():
     dw = DataWarehouse(VALID_XDMOD_URL)
     dw.__exit__(None, None, None)
diff --git a/xdmod_data/_response_processor.py b/xdmod_data/_response_processor.py
index 134b0329..e015634f 100644
--- a/xdmod_data/_response_processor.py
+++ b/xdmod_data/_response_processor.py
@@ -18,7 +18,7 @@ def _process_get_data_response(dw, params, response):
     csv_data = csv.reader(response.splitlines())
     if params['dataset_type'] == 'timeseries':
         return __parse_timeseries_csv_data(dw, params, csv_data)
-    else: 
+    else:
         return __parse_aggregate_csv_data(params, csv_data)
 
 

From fc52b2d77a5aa814b1b0e3ce995fef97715c6bcf Mon Sep 17 00:00:00 2001
From: Rodney Garnett <garnettr35@gmail.com>
Date: Tue, 9 Jul 2024 16:44:47 -0400
Subject: [PATCH 62/72] Github actions check 31

---
 .github/workflows/python-package.yml              | 4 +---
 tests/regression/data/jobs-2016-2017-month.csv    | 4 ++--
 tests/regression/data/jobs-2016-2017-quarters.csv | 4 ++--
 tests/regression/data/jobs-2016-2017-years.csv    | 2 +-
 4 files changed, 6 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml
index fe73d097..347cf9ed 100644
--- a/.github/workflows/python-package.yml
+++ b/.github/workflows/python-package.yml
@@ -51,11 +51,9 @@ jobs:
       run: |
         # Create a file with 10,000 users so that we can test filters that have more than 10,000 values
         for i in {1..10000}; do
-          unix=$((1498866840+$i))
+          unix=$((1498866840+$i*2))
           echo "$i|2269212|frearson|black|banana-cream|caste|caste|45580|user$i|379330|$(date +'%Y-%m-%dT%H:%M:%S' -d @$unix)|$(date +'%Y-%m-%dT%H:%M:%S' -d @$(($unix+1)))|$(date +'%Y-%m-%dT%H:%M:%S' -d @$(($unix+2)))|$(date +'%Y-%m-%dT%H:%M:%S' -d @$(($unix+3)))|2-23:24:40|0:0|COMPLETED|1|16|16|62.50Gn|cpu=16,mem=62.50G,node=1|billing=16,cpu=16,mem=62.50G,node=1|3-00:00:00|cpn-m25-02-01|310f6011b278ebb15df2d199e0119de78d261d8c478a203abc97dc3dda7702ae" >> 10000users.log
         done
-        unix=1498886840
-        echo "01|2269212|mortorq|black|banana-cream|caste|caste|45580|user01|379330|$(date +'%Y-%m-%dT%H:%M:%S' -d @$unix)|$(date +'%Y-%m-%dT%H:%M:%S' -d @$(($unix+1)))|$(date +'%Y-%m-%dT%H:%M:%S' -d @$(($unix+2)))|$(date +'%Y-%m-%dT%H:%M:%S' -d @$(($unix+3)))|2-23:24:40|0:0|COMPLETED|1|16|16|62.50Gn|cpu=16,mem=62.50G,node=1|billing=16,cpu=16,mem=62.50G,node=1|3-00:00:00|cpn-m25-02-01|310f6011b278ebb15df2d199e0119de78d261d8c478a203abc97dc3dda7702ae" >> 10000users.log
         #run tests
         coverage erase
         declare -a portal_versions=$(yq '.services | keys[]' .github/workflows/docker-compose.yaml)
diff --git a/tests/regression/data/jobs-2016-2017-month.csv b/tests/regression/data/jobs-2016-2017-month.csv
index 048783c1..103b32a0 100644
--- a/tests/regression/data/jobs-2016-2017-month.csv
+++ b/tests/regression/data/jobs-2016-2017-month.csv
@@ -1,5 +1,5 @@
 Time,CPU Hours: Total
 2016-12-01,770775.9989
 2017-01-01,69779.8381
-2017-06-01,44.4444
-2017-07-01,0.0044
+2017-06-01,32.7911
+2017-07-01,11.6533
diff --git a/tests/regression/data/jobs-2016-2017-quarters.csv b/tests/regression/data/jobs-2016-2017-quarters.csv
index df7ce99c..dc2ace64 100644
--- a/tests/regression/data/jobs-2016-2017-quarters.csv
+++ b/tests/regression/data/jobs-2016-2017-quarters.csv
@@ -1,5 +1,5 @@
 Time,CPU Hours: Total
 2016-10-01,770775.9989
 2017-01-01,69779.8381
-2017-04-01,44.4444
-2017-07-01,0.0044
+2017-04-01,32.7911
+2017-07-01,11.6533
diff --git a/tests/regression/data/jobs-2016-2017-years.csv b/tests/regression/data/jobs-2016-2017-years.csv
index 326a2c29..d6faef19 100644
--- a/tests/regression/data/jobs-2016-2017-years.csv
+++ b/tests/regression/data/jobs-2016-2017-years.csv
@@ -1,3 +1,3 @@
 Time,CPU Hours: Total
 2016-01-01,770775.9989
-2017-01-01,69824.2869
+2017-01-01,69824.2825

From f93a148b2e6bc88c8f75f8ca55308992f6613797 Mon Sep 17 00:00:00 2001
From: Rodney Garnett <garnettr35@gmail.com>
Date: Wed, 10 Jul 2024 14:07:34 -0400
Subject: [PATCH 63/72] Github actions check 32

---
 .github/workflows/python-package.yml              | 3 ++-
 tests/regression/test_datawarehouse_regression.py | 2 ++
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml
index 347cf9ed..82484bdd 100644
--- a/.github/workflows/python-package.yml
+++ b/.github/workflows/python-package.yml
@@ -52,9 +52,10 @@ jobs:
         # Create a file with 10,000 users so that we can test filters that have more than 10,000 values
         for i in {1..10000}; do
           unix=$((1498866840+$i*2))
+          echo $unix $(date +'%Y-%m-%dT%H:%M:%S' -d @$unix)
           echo "$i|2269212|frearson|black|banana-cream|caste|caste|45580|user$i|379330|$(date +'%Y-%m-%dT%H:%M:%S' -d @$unix)|$(date +'%Y-%m-%dT%H:%M:%S' -d @$(($unix+1)))|$(date +'%Y-%m-%dT%H:%M:%S' -d @$(($unix+2)))|$(date +'%Y-%m-%dT%H:%M:%S' -d @$(($unix+3)))|2-23:24:40|0:0|COMPLETED|1|16|16|62.50Gn|cpu=16,mem=62.50G,node=1|billing=16,cpu=16,mem=62.50G,node=1|3-00:00:00|cpn-m25-02-01|310f6011b278ebb15df2d199e0119de78d261d8c478a203abc97dc3dda7702ae" >> 10000users.log
         done
-        #run tests
+        #run test
         coverage erase
         declare -a portal_versions=$(yq '.services | keys[]' .github/workflows/docker-compose.yaml)
         for version in $portal_versions; do
diff --git a/tests/regression/test_datawarehouse_regression.py b/tests/regression/test_datawarehouse_regression.py
index 8446fcec..dce591f6 100644
--- a/tests/regression/test_datawarehouse_regression.py
+++ b/tests/regression/test_datawarehouse_regression.py
@@ -41,6 +41,8 @@ def __assert_dfs_equal(
         ).fillna(numpy.nan)
         expected.columns = expected.columns.astype('string')
         expected.columns.name = columns_name
+        print('expected: ', expected)
+        print('actual: ', actual)
         if index_col == 'Time':
             expected.index = pandas.to_datetime(expected.index)
         assert expected.equals(actual)

From 7d40148eb88c17f0e5c4f36464ca6e55eee6faed Mon Sep 17 00:00:00 2001
From: Rodney Garnett <garnettr35@gmail.com>
Date: Wed, 10 Jul 2024 14:30:12 -0400
Subject: [PATCH 64/72] Github actions check 33O

---
 .github/workflows/python-package.yml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml
index 82484bdd..3c9e2969 100644
--- a/.github/workflows/python-package.yml
+++ b/.github/workflows/python-package.yml
@@ -51,9 +51,10 @@ jobs:
       run: |
         # Create a file with 10,000 users so that we can test filters that have more than 10,000 values
         for i in {1..10000}; do
+          # sets the unix at the end of quarter 2
           unix=$((1498866840+$i*2))
-          echo $unix $(date +'%Y-%m-%dT%H:%M:%S' -d @$unix)
-          echo "$i|2269212|frearson|black|banana-cream|caste|caste|45580|user$i|379330|$(date +'%Y-%m-%dT%H:%M:%S' -d @$unix)|$(date +'%Y-%m-%dT%H:%M:%S' -d @$(($unix+1)))|$(date +'%Y-%m-%dT%H:%M:%S' -d @$(($unix+2)))|$(date +'%Y-%m-%dT%H:%M:%S' -d @$(($unix+3)))|2-23:24:40|0:0|COMPLETED|1|16|16|62.50Gn|cpu=16,mem=62.50G,node=1|billing=16,cpu=16,mem=62.50G,node=1|3-00:00:00|cpn-m25-02-01|310f6011b278ebb15df2d199e0119de78d261d8c478a203abc97dc3dda7702ae" >> 10000users.log
+          # each user is added in the file later to be injested
+          echo "$i|2269212|frearson|black|banana-cream|caste|caste|45580|user$i|379330|$(date --utc +'%Y-%m-%dT%H:%M:%S' -d @$unix)|$(date --utc +'%Y-%m-%dT%H:%M:%S' -d @$(($unix+1)))|$(date --utc +'%Y-%m-%dT%H:%M:%S' -d @$(($unix+2)))|$(date --utc +'%Y-%m-%dT%H:%M:%S' -d @$(($unix+3)))|2-23:24:40|0:0|COMPLETED|1|16|16|62.50Gn|cpu=16,mem=62.50G,node=1|billing=16,cpu=16,mem=62.50G,node=1|3-00:00:00|cpn-m25-02-01|310f6011b278ebb15df2d199e0119de78d261d8c478a203abc97dc3dda7702ae" >> 10000users.log
         done
         #run test
         coverage erase

From 9706abe4cae41558347fd94b7989c63db2833207 Mon Sep 17 00:00:00 2001
From: Rodney Garnett <garnettr35@gmail.com>
Date: Wed, 10 Jul 2024 15:26:12 -0400
Subject: [PATCH 65/72] Github actions check 34

---
 .github/workflows/python-package.yml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml
index 3c9e2969..a1397371 100644
--- a/.github/workflows/python-package.yml
+++ b/.github/workflows/python-package.yml
@@ -50,11 +50,16 @@ jobs:
     - name: Test against each supported XDMoD portal version
       run: |
         # Create a file with 10,000 users so that we can test filters that have more than 10,000 values
+        date
+
         for i in {1..10000}; do
           # sets the unix at the end of quarter 2
           unix=$((1498866840+$i*2))
           # each user is added in the file later to be injested
           echo "$i|2269212|frearson|black|banana-cream|caste|caste|45580|user$i|379330|$(date --utc +'%Y-%m-%dT%H:%M:%S' -d @$unix)|$(date --utc +'%Y-%m-%dT%H:%M:%S' -d @$(($unix+1)))|$(date --utc +'%Y-%m-%dT%H:%M:%S' -d @$(($unix+2)))|$(date --utc +'%Y-%m-%dT%H:%M:%S' -d @$(($unix+3)))|2-23:24:40|0:0|COMPLETED|1|16|16|62.50Gn|cpu=16,mem=62.50G,node=1|billing=16,cpu=16,mem=62.50G,node=1|3-00:00:00|cpn-m25-02-01|310f6011b278ebb15df2d199e0119de78d261d8c478a203abc97dc3dda7702ae" >> 10000users.log
+          if [[ $unix -eq 1 ]]; then
+           echo $(date --utc +'%Y-%m-%dT%H:%M:%S' -d @$unix)
+          fi
         done
         #run test
         coverage erase
@@ -70,6 +75,7 @@ jobs:
           export CURL_CA_BUNDLE="$version.crt"
           # Start up web server in container
           docker exec $version /root/bin/services start
+          docker exec $version date
           ports=$(yq ".services.\"$version\".ports[0]" .github/workflows/docker-compose.yaml)
           port=$(echo $ports | cut -d ':' -f 1)
           export XDMOD_HOST="https://localhost:$port"

From e85145194c927e6af979f468f0930e3054d7e7eb Mon Sep 17 00:00:00 2001
From: Rodney Garnett <garnettr35@gmail.com>
Date: Wed, 10 Jul 2024 15:52:15 -0400
Subject: [PATCH 66/72] Github actions please work

---
 .github/workflows/python-package.yml              | 6 +-----
 tests/regression/data/jobs-2016-2017-month.csv    | 4 ++--
 tests/regression/data/jobs-2016-2017-quarters.csv | 4 ++--
 3 files changed, 5 insertions(+), 9 deletions(-)

diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml
index a1397371..2a878f9b 100644
--- a/.github/workflows/python-package.yml
+++ b/.github/workflows/python-package.yml
@@ -57,9 +57,6 @@ jobs:
           unix=$((1498866840+$i*2))
           # each user is added in the file later to be injested
           echo "$i|2269212|frearson|black|banana-cream|caste|caste|45580|user$i|379330|$(date --utc +'%Y-%m-%dT%H:%M:%S' -d @$unix)|$(date --utc +'%Y-%m-%dT%H:%M:%S' -d @$(($unix+1)))|$(date --utc +'%Y-%m-%dT%H:%M:%S' -d @$(($unix+2)))|$(date --utc +'%Y-%m-%dT%H:%M:%S' -d @$(($unix+3)))|2-23:24:40|0:0|COMPLETED|1|16|16|62.50Gn|cpu=16,mem=62.50G,node=1|billing=16,cpu=16,mem=62.50G,node=1|3-00:00:00|cpn-m25-02-01|310f6011b278ebb15df2d199e0119de78d261d8c478a203abc97dc3dda7702ae" >> 10000users.log
-          if [[ $unix -eq 1 ]]; then
-           echo $(date --utc +'%Y-%m-%dT%H:%M:%S' -d @$unix)
-          fi
         done
         #run test
         coverage erase
@@ -75,7 +72,6 @@ jobs:
           export CURL_CA_BUNDLE="$version.crt"
           # Start up web server in container
           docker exec $version /root/bin/services start
-          docker exec $version date
           ports=$(yq ".services.\"$version\".ports[0]" .github/workflows/docker-compose.yaml)
           port=$(echo $ports | cut -d ':' -f 1)
           export XDMOD_HOST="https://localhost:$port"
@@ -84,7 +80,7 @@ jobs:
           curl -X POST -b xdmod.cookie $XDMOD_HOST/rest/users/current/api/token?token=$token | jq -r '.data.token' >> ~/.xdmod-data-token
           docker cp 10000users.log $version:10000users.log
           docker exec $version xdmod-shredder -r frearson -f slurm -i 10000users.log
-          date=$(date +%Y-%m-%d)
+          date=$(date --utc +%Y-%m-%d)
           docker exec $version xdmod-ingestor --ingest
           docker exec $version xdmod-ingestor --aggregate=job --last-modified-start-date $date
           coverage run --branch --append -m pytest -vvs -o log_cli=true tests/
diff --git a/tests/regression/data/jobs-2016-2017-month.csv b/tests/regression/data/jobs-2016-2017-month.csv
index 103b32a0..ff9c7baa 100644
--- a/tests/regression/data/jobs-2016-2017-month.csv
+++ b/tests/regression/data/jobs-2016-2017-month.csv
@@ -1,5 +1,5 @@
 Time,CPU Hours: Total
 2016-12-01,770775.9989
 2017-01-01,69779.8381
-2017-06-01,32.7911
-2017-07-01,11.6533
+2017-06-01,0.7911
+2017-07-01,43.6533
diff --git a/tests/regression/data/jobs-2016-2017-quarters.csv b/tests/regression/data/jobs-2016-2017-quarters.csv
index dc2ace64..c3ae1193 100644
--- a/tests/regression/data/jobs-2016-2017-quarters.csv
+++ b/tests/regression/data/jobs-2016-2017-quarters.csv
@@ -1,5 +1,5 @@
 Time,CPU Hours: Total
 2016-10-01,770775.9989
 2017-01-01,69779.8381
-2017-04-01,32.7911
-2017-07-01,11.6533
+2017-04-01,0.7911
+2017-07-01,43.6533

From 49ecc9df20d002cedaaa8e8662d385304e32ec7a Mon Sep 17 00:00:00 2001
From: Rodney Garnett <garnettr35@gmail.com>
Date: Mon, 15 Jul 2024 11:52:12 -0400
Subject: [PATCH 67/72] Github actions 11.0 container

---
 .github/workflows/docker-compose.yaml |  4 ++--
 .github/workflows/python-package.yml  | 19 +++++++++++++++----
 tests/unit/test_datawarehouse_unit.py |  6 +++---
 xdmod_data/_descriptors.py            |  2 +-
 xdmod_data/_http_requester.py         | 24 ++++++------------------
 xdmod_data/_response_processor.py     |  4 ++--
 xdmod_data/_validator.py              |  8 ++++----
 7 files changed, 33 insertions(+), 34 deletions(-)

diff --git a/.github/workflows/docker-compose.yaml b/.github/workflows/docker-compose.yaml
index fc2469c3..ab3608ab 100644
--- a/.github/workflows/docker-compose.yaml
+++ b/.github/workflows/docker-compose.yaml
@@ -5,8 +5,8 @@ services:
         tty: true
         ports: 
           - "8081:443"      
-    xdmod11.0:
-        container_name: xdmod11.0
+    xdmod-main-dev:
+        container_name: xdmod-main-dev
         image: tools-ext-01.ccr.xdmod.org/xdmod-10.5.0-x86_64:rockylinux8.5-0.3
         tty: true
         ports:
diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml
index 2a878f9b..05c30253 100644
--- a/.github/workflows/python-package.yml
+++ b/.github/workflows/python-package.yml
@@ -50,8 +50,6 @@ jobs:
     - name: Test against each supported XDMoD portal version
       run: |
         # Create a file with 10,000 users so that we can test filters that have more than 10,000 values
-        date
-
         for i in {1..10000}; do
           # sets the unix at the end of quarter 2
           unix=$((1498866840+$i*2))
@@ -68,10 +66,23 @@ jobs:
           docker exec $version /usr/bin/openssl req -new -key /etc/pki/tls/private/localhost.key -x509 -sha256 -days 365 -set_serial $RANDOM -extensions v3_req -out /etc/pki/tls/certs/localhost.crt -subj "/C=XX/L=Default City/O=Default Company Ltd/CN=localhost" -addext 'subjectAltName=DNS:localhost'
           # Copy Certificate from container
           docker cp $version:/etc/pki/tls/certs/localhost.crt $version.crt 
+          if [ "$version" = "xdmod-main-dev" ]; then
+            # Install the latest development version of xdmod, 
+            # these steps come from xdmod CircleCI configuration file
+            docker exec xdmod-main-dev bash -c "dnf module -y reset php && dnf module -y enable php:7.4"
+            docker exec xdmod-main-dev bash -c "dnf install -y php libzip-devel php-pear php-devel"
+            docker exec xdmod-main-dev bash -c "yes '' | pecl install mongodb || true"
+            docker exec xdmod-main-dev bash -c "git clone --depth=1 https://github.com/ubccr/xdmod.git /root/xdmod"
+            docker exec -w /root/xdmod xdmod-main-dev bash -c "composer install"
+            docker exec -w /root/xdmod xdmod-main-dev bash -c "~/bin/buildrpm xdmod"
+            docker exec -w /root/xdmod xdmod-main-dev bash -c "XDMOD_TEST_MODE=upgrade ./tests/ci/bootstrap.sh"
+            docker exec -w /root/xdmod xdmod-main-dev bash -c "./tests/ci/validate.sh"
+          else
+            # Start up web server in container
+            docker exec $version /root/bin/services start
+          fi
           # Prepare to use certificate for curl and python requests
           export CURL_CA_BUNDLE="$version.crt"
-          # Start up web server in container
-          docker exec $version /root/bin/services start
           ports=$(yq ".services.\"$version\".ports[0]" .github/workflows/docker-compose.yaml)
           port=$(echo $ports | cut -d ':' -f 1)
           export XDMOD_HOST="https://localhost:$port"
diff --git a/tests/unit/test_datawarehouse_unit.py b/tests/unit/test_datawarehouse_unit.py
index 4749e13e..a3565af2 100644
--- a/tests/unit/test_datawarehouse_unit.py
+++ b/tests/unit/test_datawarehouse_unit.py
@@ -50,14 +50,14 @@ def test___enter___RuntimeError_xdmod_host_malformed():
         ),
     ):
         with DataWarehouse('https://'):
-            pass  # pragma: no cover
+            pass
 
 
 def test___enter___RuntimeError_xdmod_host_unresolved():
     invalid_host = 'https://' + INVALID_STR + '.xdmod.org'
     with pytest.raises(Exception):
         with DataWarehouse(invalid_host):
-            pass  # pragma: no cover
+            pass
 
 
 def test___enter___RuntimeError_xdmod_host_unsupported_protocol():
@@ -67,7 +67,7 @@ def test___enter___RuntimeError_xdmod_host_unsupported_protocol():
         match="No connection adapters were found for '" + invalid_host,
     ):
         with DataWarehouse(invalid_host):
-            pass  # pragma: no cover
+            pass
 
 
 def test___enter___RuntimeError_401():
diff --git a/xdmod_data/_descriptors.py b/xdmod_data/_descriptors.py
index 5826737e..19e3a151 100644
--- a/xdmod_data/_descriptors.py
+++ b/xdmod_data/_descriptors.py
@@ -19,7 +19,7 @@ def __request_aggregate(self):
             '/controllers/metric_explorer.php',
             {'operation': 'get_dw_descripter'},
         )
-        if response['totalCount'] != 1:  # pragma: no cover
+        if response['totalCount'] != 1:
             raise RuntimeError(
                 'Descriptor received with unexpected structure.',
             )
diff --git a/xdmod_data/_http_requester.py b/xdmod_data/_http_requester.py
index 26206668..33345fd4 100644
--- a/xdmod_data/_http_requester.py
+++ b/xdmod_data/_http_requester.py
@@ -49,11 +49,7 @@ def _request_raw_data(self, params):
         # necessary — only the body of the 'if' branch will be needed.
         limit = self.__get_raw_data_limit()
         data = []
-        # Once ACCESS XDMoD is on version 11.0 then the `pragma: no cover`
-        # below will need to be moved to the `else` statement so that
-        # Coverage.py will know that branch is not
-        # expected to be covered by tests.
-        if limit == 'NA':  # pragma: no cover
+        if limit == 'NA':
             response_iter_lines = self.__request(
                 path='/rest/v1/warehouse/raw-data?' + url_params,
                 post_fields=None,
@@ -127,7 +123,7 @@ def _request_json(self, path, post_fields=None):
     def __assert_connection_to_xdmod_host(self):
         try:
             self.__request()
-        except RuntimeError as e:  # pragma: no cover
+        except RuntimeError as e:
             raise RuntimeError(
                 "Could not connect to xdmod_host '" + self.__xdmod_host
                 + "': " + str(e),
@@ -152,7 +148,7 @@ def __request(self, path='', post_fields=None, stream=False):
             try:
                 response_json = json.loads(response.text)
                 msg = ': ' + response_json['message']
-            except json.JSONDecodeError:  # pragma: no cover
+            except json.JSONDecodeError:
                 pass
             if response.status_code == 401:
                 msg = (
@@ -161,11 +157,7 @@ def __request(self, path='', post_fields=None, stream=False):
             raise RuntimeError(
                 'Error ' + str(response.status_code) + msg,
             ) from None
-        # Once ACCESS XDMoD is on version 11.0 then
-        # the `pragma: no cover` below
-        # will need to be removed so that Coverage.py
-        # will know that branch is expected to be covered by tests.
-        if stream:  # pragma: no cover
+        if stream:
             return response.iter_lines()
         else:
             return response.text
@@ -212,13 +204,9 @@ def __get_raw_data_limit(self):
                     '/rest/v1/warehouse/raw-data/limit',
                 )
                 self.__raw_data_limit = int(response['data'])
-                # Once ACCESS XDMoD is on version 11.0 then
-                # the `pragma: no cover` below
-                # will need to be removed so that Coverage.py
-                # will know that branch is expected to be covered by tests.
-            except RuntimeError as e:  # pragma: no cover
+            except RuntimeError as e:
                 if '404' in str(e):
                     self.__raw_data_limit = 'NA'
-                else:  # pragma: no cover
+                else:
                     raise
         return self.__raw_data_limit
diff --git a/xdmod_data/_response_processor.py b/xdmod_data/_response_processor.py
index e015634f..b2f7958b 100644
--- a/xdmod_data/_response_processor.py
+++ b/xdmod_data/_response_processor.py
@@ -72,7 +72,7 @@ def __parse_timeseries_date_string(date_string):
     # Match YYYY Q#
     elif re.match(r'^[0-9]{4} Q[0-9]$', date_string):
         (date_string, format_) = __parse_quarter_date_string(date_string)
-    else:  # pragma: no cover
+    else:
         raise Exception(
             'Unsupported date specification ' + date_string + '.',
         )
@@ -131,7 +131,7 @@ def __parse_quarter_date_string(date_string):
         month = '07'
     elif quarter == 'Q4':
         month = '10'
-    else:  # pragma: no cover
+    else:
         raise Exception(
             'Unsupported date quarter specification '
             + date_string + '.',
diff --git a/xdmod_data/_validator.py b/xdmod_data/_validator.py
index 93fb27e6..4ffc3092 100644
--- a/xdmod_data/_validator.py
+++ b/xdmod_data/_validator.py
@@ -254,7 +254,7 @@ def __get_dates_from_duration(duration):
     last_month = today + timedelta(days=-30)
     last_quarter = today + timedelta(days=-90)
     this_month_start = date(today.year, today.month, 1)
-    if today.month == 1:  # pragma: no cover
+    if today.month == 1:
         last_full_month_start_year = today.year - 1
         last_full_month_start_month = 12
     else:
@@ -271,9 +271,9 @@ def __get_dates_from_duration(duration):
         ((today.month - 1) // 3) * 3 + 1,
         1,
     )
-    if today.month < 4:  # pragma: no cover
+    if today.month < 4:
         last_quarter_start_year = today.year - 1
-    else:  # pragma: no cover
+    else:
         last_quarter_start_year = today.year
     last_quarter_start = date(
         last_quarter_start_year,
@@ -340,7 +340,7 @@ def __date_add_years(old_date, year_delta):
         try:
             new_date = date(new_date_year, old_date.month, new_date_day)
             keep_going = False
-        except ValueError:  # pragma: no cover
+        except ValueError:
             new_date_day -= 1
             days_above += 1
     return new_date + timedelta(days=days_above)

From 2642f226325dbc15aa64b57becebf3a2bd395d22 Mon Sep 17 00:00:00 2001
From: Rodney Garnett <garnettr35@gmail.com>
Date: Mon, 15 Jul 2024 12:07:06 -0400
Subject: [PATCH 68/72] Github actions check 37

---
 .github/workflows/python-package.yml              | 1 +
 tests/regression/test_datawarehouse_regression.py | 2 --
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml
index 05c30253..50c9d72d 100644
--- a/.github/workflows/python-package.yml
+++ b/.github/workflows/python-package.yml
@@ -60,6 +60,7 @@ jobs:
         coverage erase
         declare -a portal_versions=$(yq '.services | keys[]' .github/workflows/docker-compose.yaml)
         for version in $portal_versions; do
+          echo $version
           # Generate OpenSSL Key
           docker exec $version sh -c 'openssl genrsa -rand /proc/cpuinfo:/proc/filesystems:/proc/interrupts:/proc/ioports:/proc/uptime 2048 > /etc/pki/tls/private/localhost.key'
           # Generate Certificate 
diff --git a/tests/regression/test_datawarehouse_regression.py b/tests/regression/test_datawarehouse_regression.py
index dce591f6..8446fcec 100644
--- a/tests/regression/test_datawarehouse_regression.py
+++ b/tests/regression/test_datawarehouse_regression.py
@@ -41,8 +41,6 @@ def __assert_dfs_equal(
         ).fillna(numpy.nan)
         expected.columns = expected.columns.astype('string')
         expected.columns.name = columns_name
-        print('expected: ', expected)
-        print('actual: ', actual)
         if index_col == 'Time':
             expected.index = pandas.to_datetime(expected.index)
         assert expected.equals(actual)

From 972d99af4d11b2eb8424b369378cedfcf4ddae6d Mon Sep 17 00:00:00 2001
From: Rodney Garnett <garnettr35@gmail.com>
Date: Mon, 15 Jul 2024 16:27:05 -0400
Subject: [PATCH 69/72] Git actions check final

---
 .coveragerc                                   |  2 -
 .github/workflows/python-package.yml          |  3 +-
 .../test_datawarehouse_integration.py         |  8 +-
 .../jobs-fieldofscience-filter-values.csv     | 82 -------------------
 tests/regression/data/raw-data-every-1000.csv | 35 --------
 .../jobs-2016-2017-month.csv                  |  0
 .../jobs-2016-2017-quarters.csv               |  0
 .../jobs-2016-2017-years.csv                  |  0
 .../{ => xdmod-main-dev}/jobs-dimensions.csv  |  0
 .../data/xdmod-main-dev/jobs-metrics.csv      | 27 ++++++
 .../jobs-pi-group-filter-values.csv           |  0
 ...w-data-every-1000-no-fields-no-filters.csv | 56 +++++++++++++
 ...ata-every-1000-with-fields-and-filters.csv | 35 ++++++++
 .../regression/data/xdmod-main-dev/realms.csv |  5 ++
 .../data/xdmod10.5/jobs-2016-2017-month.csv   |  5 ++
 .../xdmod10.5/jobs-2016-2017-quarters.csv     |  5 ++
 .../data/xdmod10.5/jobs-2016-2017-years.csv   |  3 +
 .../data/xdmod10.5/jobs-dimensions.csv        | 17 ++++
 .../data/{ => xdmod10.5}/jobs-metrics.csv     |  0
 .../xdmod10.5/jobs-pi-group-filter-values.csv | 41 ++++++++++
 ...-data-every-1000-no-fields-no-filters.csv} |  0
 ...ta-every-1000-with-fields-and-filters.csv} |  0
 .../data/{ => xdmod10.5}/realms.csv           |  0
 .../test_datawarehouse_regression.py          | 19 +++--
 tests/unit/test_datawarehouse_unit.py         |  8 +-
 25 files changed, 216 insertions(+), 135 deletions(-)
 delete mode 100644 .coveragerc
 delete mode 100644 tests/regression/data/jobs-fieldofscience-filter-values.csv
 delete mode 100644 tests/regression/data/raw-data-every-1000.csv
 rename tests/regression/data/{ => xdmod-main-dev}/jobs-2016-2017-month.csv (100%)
 rename tests/regression/data/{ => xdmod-main-dev}/jobs-2016-2017-quarters.csv (100%)
 rename tests/regression/data/{ => xdmod-main-dev}/jobs-2016-2017-years.csv (100%)
 rename tests/regression/data/{ => xdmod-main-dev}/jobs-dimensions.csv (100%)
 create mode 100644 tests/regression/data/xdmod-main-dev/jobs-metrics.csv
 rename tests/regression/data/{ => xdmod-main-dev}/jobs-pi-group-filter-values.csv (100%)
 create mode 100644 tests/regression/data/xdmod-main-dev/raw-data-every-1000-no-fields-no-filters.csv
 create mode 100644 tests/regression/data/xdmod-main-dev/raw-data-every-1000-with-fields-and-filters.csv
 create mode 100644 tests/regression/data/xdmod-main-dev/realms.csv
 create mode 100644 tests/regression/data/xdmod10.5/jobs-2016-2017-month.csv
 create mode 100644 tests/regression/data/xdmod10.5/jobs-2016-2017-quarters.csv
 create mode 100644 tests/regression/data/xdmod10.5/jobs-2016-2017-years.csv
 create mode 100644 tests/regression/data/xdmod10.5/jobs-dimensions.csv
 rename tests/regression/data/{ => xdmod10.5}/jobs-metrics.csv (100%)
 create mode 100644 tests/regression/data/xdmod10.5/jobs-pi-group-filter-values.csv
 rename tests/regression/data/{raw-data-every-1000-pt2.csv => xdmod10.5/raw-data-every-1000-no-fields-no-filters.csv} (100%)
 rename tests/regression/data/{raw-data-every-1000-pt3.csv => xdmod10.5/raw-data-every-1000-with-fields-and-filters.csv} (100%)
 rename tests/regression/data/{ => xdmod10.5}/realms.csv (100%)

diff --git a/.coveragerc b/.coveragerc
deleted file mode 100644
index d07bcb56..00000000
--- a/.coveragerc
+++ /dev/null
@@ -1,2 +0,0 @@
-[run]
-dynamic_context = test_function
\ No newline at end of file
diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml
index 50c9d72d..2d5da2c4 100644
--- a/.github/workflows/python-package.yml
+++ b/.github/workflows/python-package.yml
@@ -60,7 +60,7 @@ jobs:
         coverage erase
         declare -a portal_versions=$(yq '.services | keys[]' .github/workflows/docker-compose.yaml)
         for version in $portal_versions; do
-          echo $version
+          echo "starting up $version container"
           # Generate OpenSSL Key
           docker exec $version sh -c 'openssl genrsa -rand /proc/cpuinfo:/proc/filesystems:/proc/interrupts:/proc/ioports:/proc/uptime 2048 > /etc/pki/tls/private/localhost.key'
           # Generate Certificate 
@@ -95,6 +95,7 @@ jobs:
           date=$(date --utc +%Y-%m-%d)
           docker exec $version xdmod-ingestor --ingest
           docker exec $version xdmod-ingestor --aggregate=job --last-modified-start-date $date
+          export XDMOD_VERSION="$version"
           coverage run --branch --append -m pytest -vvs -o log_cli=true tests/
         done
         coverage report -m --fail-under=100
\ No newline at end of file
diff --git a/tests/integration/test_datawarehouse_integration.py b/tests/integration/test_datawarehouse_integration.py
index ecf2101c..5c4b7794 100644
--- a/tests/integration/test_datawarehouse_integration.py
+++ b/tests/integration/test_datawarehouse_integration.py
@@ -6,7 +6,7 @@
 import pytest
 from xdmod_data.warehouse import DataWarehouse
 
-VALID_XDMOD_URL = os.environ['XDMOD_HOST']
+VALID_XDMOD_HOST = os.environ['XDMOD_HOST']
 TOKEN_PATH = '~/.xdmod-data-token'
 INVALID_STR = 'asdlkfjsdlkfisdjkfjd'
 METHOD_PARAMS = {
@@ -118,7 +118,7 @@
 
 @pytest.fixture(scope='module')
 def dw_methods(request):
-    xdmod_host = VALID_XDMOD_URL
+    xdmod_host = VALID_XDMOD_HOST
     if hasattr(request, 'param'):
         xdmod_host = request.param
     with DataWarehouse(xdmod_host) as dw:
@@ -127,7 +127,7 @@ def dw_methods(request):
 
 @pytest.fixture(scope='module')
 def dw_methods_outside_runtime_context():
-    dw = DataWarehouse(VALID_XDMOD_URL)
+    dw = DataWarehouse(VALID_XDMOD_HOST)
     return __get_dw_methods(dw)
 
 
@@ -447,7 +447,7 @@ def test_case_insensitive(dw_methods, method, param, value1, value2):
 
 @pytest.mark.parametrize(
     'dw_methods,method',
-    [(VALID_XDMOD_URL + '/', method) for method in list(METHOD_PARAMS.keys())],
+    [(VALID_XDMOD_HOST + '/', method) for method in list(METHOD_PARAMS.keys())],
     indirect=['dw_methods'],
     ids=[method for method in list(METHOD_PARAMS.keys())],
 )
diff --git a/tests/regression/data/jobs-fieldofscience-filter-values.csv b/tests/regression/data/jobs-fieldofscience-filter-values.csv
deleted file mode 100644
index 86ccb36e..00000000
--- a/tests/regression/data/jobs-fieldofscience-filter-values.csv
+++ /dev/null
@@ -1,82 +0,0 @@
-id,label
-189,"Agriculture, Forestry, and Fisheries"
-192,Agriculural Biotechnology
-236,Analytical and Materials Chemistry
-219,Applied Computer Science
-215,Applied Mathematics
-221,Artificial Intelligence and Intelligent Systems
-206,Arts
-225,Astronomy and Planetary Sciences
-238,Atmospheric Sciences
-228,"Atomic, Molecular, and Optical Physics"
-184,Basic Medicine
-254,Behavioral Sciences Biology
-247,Biochemistry and Molecular Biology
-246,Biophysics
-248,Cell Biology
-176,Chemical Engineering
-173,Civil Engineering
-239,Climate and Global Dynamics
-185,Clinical Medicine
-218,Computer Science
-226,Condensed Matter Physics
-250,Developmental Biology
-252,Ecology
-195,Economics and Business
-196,Educational Sciences
-174,"Electrical, Electronic, and Information Engineering"
-253,Environmental Biology
-180,Environmental Biotechnology
-179,Environmental Engineering
-231,Fluid and Plasma Physics
-249,Genetics
-242,Geology and Solid Earth Sciences
-241,Geophysics and Geochemistry
-227,Gravitational Physics
-186,Health Sciences
-203,History and Archaeology
-244,Hydrology and Water Resources
-181,Industrial Biotechnology
-223,"Informatics, Analytics and Information Science"
-211,Infrastructure and Instrumentation
-235,Inorganic and Nuclear Chemistry
-198,Law
-240,Magnetospheric and Upper Atmospheric Physics
-177,Materials Engineering
-175,Mechanical Engineering
-201,Media and communications
-187,Medical Biotechnology
-178,Medical Engineering
-182,Nanotechnology
-229,Nuclear Physics
-243,Oceanography and Ocean Sciences
-234,Organic Chemistry
-213,Other
-193,Other Agricultural Sciences
-255,Other Biological Sciences
-237,Other Chemical Sciences
-224,Other Computer and Information Sciences
-245,Other Earth and Environmental Sciences
-183,Other Engineering and Technologies
-207,Other humanities
-217,Other Mathematics
-188,Other Medical Sciences
-172,Other Natural Sciences
-232,Other Physical Sciences
-202,Other social sciences
-230,Particle and High-Energy Physics
-210,Performance Evaluation and Benchmarking
-233,Physical Chemistry
-199,Political Science
-194,Psychology
-214,Pure Mathematics
-212,Science and Engineering Education
-200,Social and Economic Geography
-197,Sociology
-220,"Software Engineering, Systems, and Development"
-209,Staff Activities (ACCESS or SP)
-216,Statistics and Probability
-251,Systematics and Population Biology
-208,Training
-191,Veterinary Science
-222,Visualization and Human-Computer Systems
diff --git a/tests/regression/data/raw-data-every-1000.csv b/tests/regression/data/raw-data-every-1000.csv
deleted file mode 100644
index 5c9e0675..00000000
--- a/tests/regression/data/raw-data-every-1000.csv
+++ /dev/null
@@ -1,35 +0,0 @@
-,Local Job Id,Quality of Service,Start Time (Timestamp),GPUs,Department
-0,2286873,banana-cream,1483160526,0,Social and Economic Science
-1000,2293183,banana-cream,1483184848,0,Social and Economic Science
-2000,2302227,banana-cream,1483211690,0,Social and Economic Science
-3000,2289494,banana-cream,1483172442,0,Social and Economic Science
-4000,2299179,banana-cream,1483202557,0,Social and Economic Science
-5000,2284778,banana-cream,1483141002,0,Social and Economic Science
-6000,2296200,banana-cream,1483193678,0,Social and Economic Science
-7000,2287312,banana-cream,1483163089,0,Social and Economic Science
-8000,2300378,banana-cream,1483206150,0,Social and Economic Science
-9000,2280848,banana-cream,1483105401,0,Social and Economic Science
-10000,2281978,banana-cream,1483115753,0,Social and Economic Science
-11000,2281388,banana-cream,1483110374,0,Social and Economic Science
-12000,2280890,banana-cream,1483105946,0,Social and Economic Science
-13000,1008307,banana-cream,1483219024,0,Social and Economic Science
-14000,998351,banana-cream,1483159887,0,Social and Economic Science
-15000,1006328,banana-cream,1483212749,0,Social and Economic Science
-16000,1009986,banana-cream,1483223948,0,Social and Economic Science
-17000,994932,banana-cream,1483148678,0,Social and Economic Science
-18000,998665,banana-cream,1483160915,0,Social and Economic Science
-19000,1001452,banana-cream,1483173813,0,Social and Economic Science
-20000,1003324,boysenberry,1483204008,0,Social and Economic Science
-21000,1002994,banana-cream,1483181740,0,Social and Economic Science
-22000,971318,banana-cream,1483067878,0,Social and Economic Science
-23000,983519,banana-cream,1483109661,0,Social and Economic Science
-24000,992780,banana-cream,1483139789,0,Social and Economic Science
-25000,976041,banana-cream,1483083428,0,Social and Economic Science
-26000,983897,banana-cream,1483111546,0,Social and Economic Science
-27000,973004,banana-cream,1483072678,0,Social and Economic Science
-28000,978764,banana-cream,1483091343,0,Social and Economic Science
-29000,983635,banana-cream,1483110039,0,Social and Economic Science
-30000,988096,banana-cream,1483123810,0,Social and Economic Science
-31000,991564,banana-cream,1483135262,0,Social and Economic Science
-32000,973969,banana-cream,1483076149,0,Social and Economic Science
-33000,987124,banana-cream,1483120872,0,Social and Economic Science
diff --git a/tests/regression/data/jobs-2016-2017-month.csv b/tests/regression/data/xdmod-main-dev/jobs-2016-2017-month.csv
similarity index 100%
rename from tests/regression/data/jobs-2016-2017-month.csv
rename to tests/regression/data/xdmod-main-dev/jobs-2016-2017-month.csv
diff --git a/tests/regression/data/jobs-2016-2017-quarters.csv b/tests/regression/data/xdmod-main-dev/jobs-2016-2017-quarters.csv
similarity index 100%
rename from tests/regression/data/jobs-2016-2017-quarters.csv
rename to tests/regression/data/xdmod-main-dev/jobs-2016-2017-quarters.csv
diff --git a/tests/regression/data/jobs-2016-2017-years.csv b/tests/regression/data/xdmod-main-dev/jobs-2016-2017-years.csv
similarity index 100%
rename from tests/regression/data/jobs-2016-2017-years.csv
rename to tests/regression/data/xdmod-main-dev/jobs-2016-2017-years.csv
diff --git a/tests/regression/data/jobs-dimensions.csv b/tests/regression/data/xdmod-main-dev/jobs-dimensions.csv
similarity index 100%
rename from tests/regression/data/jobs-dimensions.csv
rename to tests/regression/data/xdmod-main-dev/jobs-dimensions.csv
diff --git a/tests/regression/data/xdmod-main-dev/jobs-metrics.csv b/tests/regression/data/xdmod-main-dev/jobs-metrics.csv
new file mode 100644
index 00000000..90c0bfe5
--- /dev/null
+++ b/tests/regression/data/xdmod-main-dev/jobs-metrics.csv
@@ -0,0 +1,27 @@
+id,label,description
+avg_cpu_hours,CPU Hours: Per Job,"The average CPU hours (number of CPU cores x wall time hours) per Screwdriver job.<br/>For each job, the CPU usage is aggregated. For example, if a job used 1000 CPUs for one minute, it would be aggregated as 1000 CPU minutes or 16.67 CPU hours."
+total_cpu_hours,CPU Hours: Total,"The total CPU hours (number of CPU cores x wall time hours) used by Screwdriver jobs.<br/>For each job, the CPU usage is aggregated. For example, if a job used 1000 CPUs for one minute, it would be aggregated as 1000 CPU minutes or 16.67 CPU hours."
+avg_gpus,GPU Count: Per Job,The average job size per Screwdriver job.<br><i>Job Size: </i>The number of GPUs used by a (parallel) job.
+avg_gpu_hours,GPU Hours: Per Job,"The average GPU hours (number of GPU cores x wall time hours) per Screwdriver job.<br/>For each job, the GPU usage is aggregated. For example, if a job used 1000 GPUs for one minute, it would be aggregated as 1000 GPU minutes or 16.67 GPU hours."
+total_gpu_hours,GPU Hours: Total,"The total GPU hours (number of GPUs x wall time hours) used by Screwdriver jobs.<br/>For each job, the GPU usage is aggregated. For example, if a job used 1000 GPUs for one minute, it would be aggregated as 1000 GPU minutes or 16.67 GPU hours."
+max_processors,Job Size: Max (Core Count),The maximum size Screwdriver job in number of cores.<br/><i>Job Size: </i>The total number of processor cores used by a (parallel) job.
+min_processors,Job Size: Min (Core Count),The minimum size Screwdriver job in number of cores.<br/><i>Job Size: </i>The total number of processor cores used by a (parallel) job.
+normalized_avg_processors,Job Size: Normalized (% of Total Cores),"The percentage average size Screwdriver job divided by the total number of cores in the resource where the job ran. The job normalization calculation assumes that the resource size is constant. This statistic should not be used with a time range where the resource size changes, because the statistic will be incorrect.<br><i>Normalized Job Size: </i>The ratio of the total number of processor cores used by a (parallel) job over the total number of cores on the resource."
+avg_processors,Job Size: Per Job (Core Count),The average job size per Screwdriver job.<br><i>Job Size: </i>The number of processor cores used by a (parallel) job.
+avg_job_size_weighted_by_cpu_hours,Job Size: Weighted By CPU Hours (Core Count),The average Screwdriver job size weighted by CPU Hours. Defined as <br><i>Average Job Size Weighted By CPU Hours: </i> sum(i = 0 to n){ job i core count * job i cpu hours}/sum(i =  0 to n){job i cpu hours}
+avg_job_size_weighted_by_gpu_hours,Job Size: Weighted By GPU Hours (GPU Count),The average Screwdriver job size weighted by GPU Hours. Defined as <br><i>Average Job Size Weighted By GPU Hours: </i> sum(i = 0 to n){ job i core count * job i gpu hours}/sum(i =  0 to n){job i gpu hours}
+avg_node_hours,Node Hours: Per Job,The average node hours (number of nodes x wall time hours) per Screwdriver job.
+total_node_hours,Node Hours: Total,The total node hours (number of nodes x wall time hours) used by Screwdriver jobs.
+job_count,Number of Jobs Ended,The total number of Screwdriver jobs that ended within the selected duration.
+running_job_count,Number of Jobs Running,The total number of Screwdriver jobs that are running.
+started_job_count,Number of Jobs Started,The total number of Screwdriver jobs that started executing within the selected duration.
+submitted_job_count,Number of Jobs Submitted,The total number of Screwdriver jobs that submitted/queued within the selected duration.<i>
+active_pi_count,Number of PIs: Active,The total number of PIs that used Screwdriver resources.
+active_resource_count,Number of Resources: Active,The total number of active Screwdriver resources.
+active_person_count,Number of Users: Active,The total number of users that used Screwdriver resources.
+utilization,Screwdriver Utilization (%),The percentage of core time that a resource has been running jobs.<br/><i> Screwdriver Utilization:</i> The ratio of the total CPU hours consumed by jobs over a given time period divided by the maximum CPU hours that the system could deliver (based on the number of cores present on the resource and the percent of the resource allocated through Screwdriver). This value does not take into account downtimes or outages. It is just calculated based on the number of cores in the resource specifications.
+expansion_factor,User Expansion Factor,The average job size per Screwdriver job.<br><i>Job Size: </i>The number of processor cores used by a (parallel) job.
+avg_waitduration_hours,Wait Hours: Per Job,"The average time, in hours, a Screwdriver job waits before execution on the designated resource.<br/><i>Wait Time: </i>Wait time is defined as the linear time between submission of a job by a user until it begins to execute."
+total_waitduration_hours,Wait Hours: Total,"The total time, in hours, Screwdriver jobs waited before execution on their designated resource.<br/><i>Wait Time: </i>Wait time is defined as the linear time between submission of a job by a user until it begins to execute."
+avg_wallduration_hours,Wall Hours: Per Job,"The average time, in hours, a job takes to execute.<br/>In timeseries view mode, the statistic shows the average wall time per job per time period. In aggregate view mode the statistic only includes the job wall hours between the defined time range. The wall hours outside the time range are not included in the calculation.<br /> <i>Wall Time:</i> Wall time is defined as the linear time between start and end time of execution for a particular job."
+total_wallduration_hours,Wall Hours: Total,"The total time, in hours, Screwdriver jobs took to execute.<br/><i>Wall Time:</i> Wall time is defined as the linear time between start and end time of execution for a particular job."
diff --git a/tests/regression/data/jobs-pi-group-filter-values.csv b/tests/regression/data/xdmod-main-dev/jobs-pi-group-filter-values.csv
similarity index 100%
rename from tests/regression/data/jobs-pi-group-filter-values.csv
rename to tests/regression/data/xdmod-main-dev/jobs-pi-group-filter-values.csv
diff --git a/tests/regression/data/xdmod-main-dev/raw-data-every-1000-no-fields-no-filters.csv b/tests/regression/data/xdmod-main-dev/raw-data-every-1000-no-fields-no-filters.csv
new file mode 100644
index 00000000..758d84d3
--- /dev/null
+++ b/tests/regression/data/xdmod-main-dev/raw-data-every-1000-no-fields-no-filters.csv
@@ -0,0 +1,56 @@
+,Local Job Id,Resource,Timezone,System Username (Deidentified),User,Organization,Quality of Service,Submit Time (Timestamp),Start Time (Timestamp),End Time (Timestamp),Eligible Time (Timestamp),Nodes,Cores,GPUs,Memory Used,Wall Time,Wait Time,Core Time,GPU Time,Exit Code,Exit State,Requested Cores,Requested memory,Requested Wall Time,Queue,Decanal Unit,Department,PI Group
+0,6110386,Robertson,UTC,a19290db301c97aa78a57524bc42c8c78bf455de,"Harrier, Hen",Screwdriver,curry,1483118438,1483118438,1483118443,1483118438,1,1,0,-1,5,0,5,0,0:0,COMPLETED,1,48000Mn,172800,white,Computer and Information Science and Engineering,Computer and Computation Research,Computer and Computation Theory
+1000,970348,Mortorq,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483064151,1483064152,1483064172,1483064151,1,8,0,-1,20,1,160,0,1:0,FAILED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+2000,981745,Mortorq,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483102502,1483102504,1483102531,1483102502,1,8,0,-1,27,2,216,0,1:0,FAILED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+3000,6112283[436],Robertson,UTC,420f3254a8b357efce12369d50ff33dc437f4f0b,"Grey, Great",Screwdriver,curry,1483135648,1483139913,1483139937,1483135648,1,1,0,-1,24,4265,24,0,0:0,COMPLETED,1,48000Mn,10800,white,"Social, Behavioral, and Economic Sciences",Social and Economic Science,"Decision, Risk, and Management Science"
+4000,178502,Phillips,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483116701,1483130204,1483130230,1483116701,1,8,0,-1,26,13503,208,0,1:0,FAILED,8,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+5000,974487,Mortorq,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483077974,1483077974,1483078015,1483077974,1,8,0,-1,41,0,328,0,1:0,FAILED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+6000,982727,Mortorq,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483106505,1483106505,1483106556,1483106505,1,8,0,-1,51,0,408,0,1:0,FAILED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+7000,2281895,Frearson,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483115132,1483115132,1483116616,1483115132,1,12,0,-1,1484,0,17808,0,0:0,COMPLETED,12,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+8000,972196,Mortorq,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483070299,1483070301,1483070337,1483070299,1,8,0,-1,36,2,288,0,1:0,FAILED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+9000,978066,Mortorq,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483088977,1483088978,1483089027,1483088977,1,8,0,-1,49,1,392,0,1:0,FAILED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+10000,983035,Mortorq,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483107862,1483107863,1483108007,1483107862,1,8,0,-1,144,1,1152,0,1:0,FAILED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+11000,1009935,Posidriv,UTC,cdd144ba0c808c63ec0bbdeb890cd90b74587c9b,"Bunting, Reed",Screwdriver,banana-cream,1483130897,1483130908,1483131130,1483130897,1,8,0,-1,222,11,1776,0,0:0,COMPLETED,8,2147486448Mn,259200,black,Mathematical and Physical Sciences,Chemistry,Physical Chemistry
+12000,2284391,Frearson,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483137557,1483137558,1483139202,1483137557,1,12,0,-1,1644,1,19728,0,0:0,COMPLETED,12,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+13000,1005349,Posidriv,UTC,cdd144ba0c808c63ec0bbdeb890cd90b74587c9b,"Bunting, Reed",Screwdriver,banana-cream,1483071422,1483071534,1483071778,1483071422,1,8,0,-1,244,112,1952,0,0:0,COMPLETED,8,2147486448Mn,259200,black,Mathematical and Physical Sciences,Chemistry,Physical Chemistry
+14000,1007824,Posidriv,UTC,cdd144ba0c808c63ec0bbdeb890cd90b74587c9b,"Bunting, Reed",Screwdriver,banana-cream,1483105333,1483105429,1483105948,1483105333,1,8,0,-1,519,96,4152,0,0:0,COMPLETED,8,2147486448Mn,259200,black,Mathematical and Physical Sciences,Chemistry,Physical Chemistry
+15000,1010629,Posidriv,UTC,cdd144ba0c808c63ec0bbdeb890cd90b74587c9b,"Bunting, Reed",Screwdriver,banana-cream,1483140307,1483140400,1483140550,1483140307,1,8,0,-1,150,93,1200,0,0:0,COMPLETED,8,2147486448Mn,259200,black,Mathematical and Physical Sciences,Chemistry,Physical Chemistry
+16000,6111037[473],Robertson,UTC,420f3254a8b357efce12369d50ff33dc437f4f0b,"Grey, Great",Screwdriver,curry,1483125912,1483129603,1483129747,1483125913,1,1,0,-1,144,3691,144,0,0:0,COMPLETED,1,2147487648Mn,900,white,"Social, Behavioral, and Economic Sciences",Social and Economic Science,"Decision, Risk, and Management Science"
+17000,988528,Mortorq,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483119249,1483125280,1483125321,1483119249,1,8,0,-1,41,6031,328,0,1:0,FAILED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+18000,991993,Mortorq,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483131196,1483136908,1483136994,1483131196,1,8,0,-1,86,5712,688,0,0:0,COMPLETED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+19000,1008740,Posidriv,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483117402,1483124540,1483126240,1483117402,1,12,0,-1,1700,7138,20400,0,0:0,COMPLETED,12,2147486448Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+20000,2276090,Frearson,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483060299,1483060299,1483062513,1483060299,1,12,0,-1,2214,0,26568,0,0:0,COMPLETED,12,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+21000,967122,Mortorq,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483053110,1483053111,1483056506,1483053110,1,8,0,-1,3395,1,27160,0,0:0,COMPLETED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+22000,2283293,Frearson,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483126817,1483126818,1483128722,1483126817,1,12,0,-1,1904,1,22848,0,0:0,COMPLETED,12,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+23000,1008638,Posidriv,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483116706,1483122377,1483124429,1483116706,1,12,0,-1,2052,5671,24624,0,0:0,COMPLETED,12,2147486448Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+24000,972040,Mortorq,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483069980,1483069981,1483074019,1483069980,1,8,0,-1,4038,1,32304,0,1:0,FAILED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+25000,991884,Mortorq,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483130901,1483136404,1483141877,1483130901,1,8,0,-1,5473,5503,43784,0,0:0,COMPLETED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+26000,6105349,Robertson,UTC,ea56fb19a839cb996b0e014f70d35a03606aa19f,Moorhen,Screwdriver,curry,1482948669,1482948735,1483066377,1482948669,1,1,0,-1,117642,66,117642,0,0:0,COMPLETED,1,2147486448Mn,259200,white,Computer and Information Science and Engineering,Microelectronic Information Processing Systems,Systems Prototyping and Fabrication
+27000,1005309,Mortorq,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483208573,1483209389,1483209414,1483208573,1,8,0,-1,25,816,200,0,0:0,COMPLETED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+28000,6114243[425],Robertson,UTC,420f3254a8b357efce12369d50ff33dc437f4f0b,"Grey, Great",Screwdriver,curry,1483202456,1483205646,1483205659,1483202456,1,1,0,-1,13,3190,13,0,0:0,COMPLETED,1,2147531648Mn,1800,white,"Social, Behavioral, and Economic Sciences",Social and Economic Science,"Decision, Risk, and Management Science"
+29000,998101,Mortorq,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483154003,1483159238,1483159264,1483154003,1,8,0,-1,26,5235,208,0,0:0,COMPLETED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+30000,184276,Phillips,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483158935,1483191164,1483191182,1483158935,1,8,0,-1,18,32229,144,0,0:0,COMPLETED,8,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+31000,2292895,Frearson,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483183969,1483183969,1483185436,1483183969,1,12,0,-1,1467,0,17604,0,0:0,COMPLETED,12,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+32000,2302043,Frearson,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483211208,1483211208,1483212911,1483211208,1,12,0,-1,1703,0,20436,0,0:0,COMPLETED,12,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+33000,2286102,Frearson,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483154003,1483154004,1483155247,1483154003,1,12,0,-1,1243,1,14916,0,0:0,COMPLETED,12,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+34000,2295330,Frearson,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483191135,1483191136,1483192490,1483191135,1,12,0,-1,1354,1,16248,0,0:0,COMPLETED,12,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+35000,2304491,Frearson,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483218813,1483218814,1483219999,1483218813,1,12,0,-1,1185,1,14220,0,0:0,COMPLETED,12,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+36000,1004397,Mortorq,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483206039,1483207040,1483207079,1483206039,1,8,0,-1,39,1001,312,0,0:0,COMPLETED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+37000,1007907,Mortorq,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483216840,1483217894,1483217930,1483216840,1,8,0,-1,36,1054,288,0,0:0,COMPLETED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+38000,189492,Phillips,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483219052,1483220591,1483220646,1483219052,1,8,0,-1,55,1539,440,0,0:0,COMPLETED,8,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+39000,1014185,Posidriv,UTC,cdd144ba0c808c63ec0bbdeb890cd90b74587c9b,"Bunting, Reed",Screwdriver,banana-cream,1483177983,1483178402,1483178494,1483177983,1,8,0,-1,92,419,736,0,0:0,COMPLETED,8,2147486448Mn,259200,black,Mathematical and Physical Sciences,Chemistry,Physical Chemistry
+40000,6112860,Robertson,UTC,7cdc338fbe4024d633ba19deca41e22c78704105,Dunlin,Screwdriver,curry,1483142479,1483143257,1483143695,1483142479,1,12,0,-1,438,778,5256,0,0:0,COMPLETED,12,2147486448Mn,540,white,Biological Sciences,Environmental Biology,Systematic and Population Biology
+41000,993914,Mortorq,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483137599,1483144206,1483144241,1483137599,1,8,0,-1,35,6607,280,0,0:0,COMPLETED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+42000,997151,Mortorq,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483150168,1483156370,1483156401,1483150168,1,8,0,-1,31,6202,248,0,0:0,COMPLETED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+43000,1000391,Mortorq,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483160667,1483167946,1483167986,1483160667,1,8,0,-1,40,7279,320,0,0:0,COMPLETED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+44000,1003122,Mortorq,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483172712,1483182324,1483182384,1483172712,1,8,0,-1,60,9612,480,0,0:0,COMPLETED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+45000,183442,Phillips,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483151833,1483184199,1483184231,1483151833,1,8,0,-1,32,32366,256,0,0:0,COMPLETED,8,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+46000,2288079,Frearson,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483166466,1483166466,1483168335,1483166466,1,12,0,-1,1869,0,22428,0,0:0,COMPLETED,12,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+47000,2300772,Frearson,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483207320,1483207320,1483209262,1483207320,1,12,0,-1,1942,0,23304,0,0:0,COMPLETED,12,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+48000,2290078,Frearson,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483174731,1483174734,1483177171,1483174731,1,12,0,-1,2437,3,29244,0,0:0,COMPLETED,12,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+49000,2304088,Frearson,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483217556,1483217557,1483219647,1483217556,1,12,0,-1,2090,1,25080,0,0:0,COMPLETED,12,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+50000,1011153,Posidriv,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483146809,1483157125,1483159016,1483146809,1,12,0,-1,1891,10316,22692,0,0:0,COMPLETED,12,2147486448Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+51000,1006854,Mortorq,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483213460,1483214511,1483219230,1483213460,1,8,0,-1,4719,1051,37752,0,0:0,COMPLETED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+52000,6106621,Robertson,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,curry,1483007306,1483150677,1483157807,1483007306,1,8,0,-1,7130,143371,57040,0,0:0,COMPLETED,8,2147486448Mn,108000,white,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology
+53000,6110124,Robertson,UTC,a3a4db35c34bc1ae71d8ba76352b66796cddb74e,"Grey, Lesser",Screwdriver,curry,1483117470,1483124196,1483154281,1483117470,1,1,0,-1,30085,6726,30085,0,0:0,COMPLETED,1,2147486448Mn,259200,white,Geosciences,Earth Sciences,Tectonics
+54000,6107522,Robertson,UTC,ea56fb19a839cb996b0e014f70d35a03606aa19f,Moorhen,Screwdriver,curry,1483033412,1483033544,1483158212,1483033412,1,1,0,-1,124668,132,124668,0,0:0,COMPLETED,1,2147486448Mn,259200,white,Computer and Information Science and Engineering,Microelectronic Information Processing Systems,Systems Prototyping and Fabrication
diff --git a/tests/regression/data/xdmod-main-dev/raw-data-every-1000-with-fields-and-filters.csv b/tests/regression/data/xdmod-main-dev/raw-data-every-1000-with-fields-and-filters.csv
new file mode 100644
index 00000000..0685e971
--- /dev/null
+++ b/tests/regression/data/xdmod-main-dev/raw-data-every-1000-with-fields-and-filters.csv
@@ -0,0 +1,35 @@
+,Local Job Id,Quality of Service,Start Time (Timestamp),GPUs,Department
+0,968193,banana-cream,1483056132,0,Social and Economic Science
+1000,971442,banana-cream,1483068165,0,Social and Economic Science
+2000,984175,banana-cream,1483112456,0,Social and Economic Science
+3000,993014,banana-cream,1483140821,0,Social and Economic Science
+4000,976375,banana-cream,1483084157,0,Social and Economic Science
+5000,983957,banana-cream,1483111812,0,Social and Economic Science
+6000,2284603,banana-cream,1483139473,0,Social and Economic Science
+7000,973762,banana-cream,1483075149,0,Social and Economic Science
+8000,979555,banana-cream,1483093851,0,Social and Economic Science
+9000,984311,banana-cream,1483112900,0,Social and Economic Science
+10000,2283845,banana-cream,1483132302,0,Social and Economic Science
+11000,988946,banana-cream,1483126657,0,Social and Economic Science
+12000,992368,banana-cream,1483138292,0,Social and Economic Science
+13000,2279980,banana-cream,1483097026,0,Social and Economic Science
+14000,2278883,banana-cream,1483086983,0,Social and Economic Science
+15000,979689,banana-cream,1483094414,0,Social and Economic Science
+16000,988604,banana-cream,1483125536,0,Social and Economic Science
+17000,1008884,banana-cream,1483220332,0,Social and Economic Science
+18000,999879,banana-cream,1483165819,0,Social and Economic Science
+19000,2292962,banana-cream,1483184182,0,Social and Economic Science
+20000,2302093,banana-cream,1483211322,0,Social and Economic Science
+21000,2288974,banana-cream,1483170354,0,Social and Economic Science
+22000,2298407,banana-cream,1483200279,0,Social and Economic Science
+23000,2306930,banana-cream,1483226400,0,Social and Economic Science
+24000,1006973,banana-cream,1483214989,0,Social and Economic Science
+25000,1010648,banana-cream,1483226156,0,Social and Economic Science
+26000,995617,banana-cream,1483151188,0,Social and Economic Science
+27000,999309,banana-cream,1483163285,0,Social and Economic Science
+28000,1002056,banana-cream,1483177400,0,Social and Economic Science
+29000,2289840,banana-cream,1483173994,0,Social and Economic Science
+30000,2303857,banana-cream,1483216696,0,Social and Economic Science
+31000,2293260,banana-cream,1483185052,0,Social and Economic Science
+32000,1005259,banana-cream,1483209237,0,Social and Economic Science
+33000,998935,banana-cream,1483161891,0,Social and Economic Science
diff --git a/tests/regression/data/xdmod-main-dev/realms.csv b/tests/regression/data/xdmod-main-dev/realms.csv
new file mode 100644
index 00000000..e6cc8eb1
--- /dev/null
+++ b/tests/regression/data/xdmod-main-dev/realms.csv
@@ -0,0 +1,5 @@
+id,label
+Cloud,Cloud
+Jobs,Jobs
+ResourceSpecifications,Resource Specifications
+Storage,Storage
diff --git a/tests/regression/data/xdmod10.5/jobs-2016-2017-month.csv b/tests/regression/data/xdmod10.5/jobs-2016-2017-month.csv
new file mode 100644
index 00000000..ff9c7baa
--- /dev/null
+++ b/tests/regression/data/xdmod10.5/jobs-2016-2017-month.csv
@@ -0,0 +1,5 @@
+Time,CPU Hours: Total
+2016-12-01,770775.9989
+2017-01-01,69779.8381
+2017-06-01,0.7911
+2017-07-01,43.6533
diff --git a/tests/regression/data/xdmod10.5/jobs-2016-2017-quarters.csv b/tests/regression/data/xdmod10.5/jobs-2016-2017-quarters.csv
new file mode 100644
index 00000000..c3ae1193
--- /dev/null
+++ b/tests/regression/data/xdmod10.5/jobs-2016-2017-quarters.csv
@@ -0,0 +1,5 @@
+Time,CPU Hours: Total
+2016-10-01,770775.9989
+2017-01-01,69779.8381
+2017-04-01,0.7911
+2017-07-01,43.6533
diff --git a/tests/regression/data/xdmod10.5/jobs-2016-2017-years.csv b/tests/regression/data/xdmod10.5/jobs-2016-2017-years.csv
new file mode 100644
index 00000000..d6faef19
--- /dev/null
+++ b/tests/regression/data/xdmod10.5/jobs-2016-2017-years.csv
@@ -0,0 +1,3 @@
+Time,CPU Hours: Total
+2016-01-01,770775.9989
+2017-01-01,69824.2825
diff --git a/tests/regression/data/xdmod10.5/jobs-dimensions.csv b/tests/regression/data/xdmod10.5/jobs-dimensions.csv
new file mode 100644
index 00000000..10887b04
--- /dev/null
+++ b/tests/regression/data/xdmod10.5/jobs-dimensions.csv
@@ -0,0 +1,17 @@
+id,label,description
+none,None,Summarizes Jobs data reported to the Screwdriver database.
+nsfdirectorate,Decanal Unit,Decanal Unit
+parentscience,Department,Department
+gpucount,GPU Count,A categorization of jobs into discrete groups based on the number of GPUs used by each job.
+jobsize,Job Size,A categorization of jobs into discrete groups based on the number of cores used by each job.
+jobwaittime,Job Wait Time,A categorization of jobs into discrete groups based on the total linear time each job waited.
+jobwalltime,Job Wall Time,A categorization of jobs into discrete groups based on the total linear time each job took to execute.
+nodecount,Node Count,A categorization of jobs into discrete groups based on node count.
+pi,PI,The principal investigator of a project.
+fieldofscience,PI Group,PI Group
+qos,QOS,The quality of service of the job
+queue,Queue,Queue pertains to the low level job queues on each resource.
+resource,Resource,A resource is a remote computer that can run jobs.
+resource_type,Resource Type,A categorization of resources into by their general capabilities.
+username,System Username,The specific system username of the users who ran jobs.
+person,User,"A person who is on a PIs allocation, hence able run jobs on resources."
diff --git a/tests/regression/data/jobs-metrics.csv b/tests/regression/data/xdmod10.5/jobs-metrics.csv
similarity index 100%
rename from tests/regression/data/jobs-metrics.csv
rename to tests/regression/data/xdmod10.5/jobs-metrics.csv
diff --git a/tests/regression/data/xdmod10.5/jobs-pi-group-filter-values.csv b/tests/regression/data/xdmod10.5/jobs-pi-group-filter-values.csv
new file mode 100644
index 00000000..a31287cc
--- /dev/null
+++ b/tests/regression/data/xdmod10.5/jobs-pi-group-filter-values.csv
@@ -0,0 +1,41 @@
+id,label
+53,Algebra and Number Theory
+10,Arts
+44,Biochemistry and Molecular Structure and Function
+45,Biophysics
+47,Cell Biology
+127,Computer and Computation Theory
+61,"Decision, Risk, and Management Science"
+99,Design and Computer-Integrated Engineering
+139,"Design, Tools, and Test"
+55,Economics
+97,Emerging Technologies Initiation
+137,Experimental Systems
+112,Extragalactic Astronomy and Cosmology
+88,"Fluid, Particulate, and Hydraulic Systems"
+115,Galactic Astronomy
+68,Geology and Paleontology
+72,Geophysics
+65,Global Atmospheric Research
+60,Law and Social Sciences
+91,Mechanics and Materials
+126,"Metals, Ceramics, and Electronic Materials"
+98,Operations Research and Production Systems
+122,Organic and Macromolecular Chemistry
+123,Physical Chemistry
+81,Polar Aeronomy and Astrophysics
+84,Polar Meteorology
+59,Polar Ocean and Climate Systems
+93,"Quantum Electronics, Waves, and Beams"
+71,Seismology
+57,Sociology
+124,Solid State Chemistry and Polymers
+94,Solid-State and Microstructures
+69,Statistics and Probability
+114,Stellar Astronomy and Astrophysics
+90,Structures and Building Systems
+49,Systematic and Population Biology
+138,Systems Prototyping and Fabrication
+70,Tectonics
+119,Theoretical Physics
+74,Volcanology and Mantle Geochemistry
diff --git a/tests/regression/data/raw-data-every-1000-pt2.csv b/tests/regression/data/xdmod10.5/raw-data-every-1000-no-fields-no-filters.csv
similarity index 100%
rename from tests/regression/data/raw-data-every-1000-pt2.csv
rename to tests/regression/data/xdmod10.5/raw-data-every-1000-no-fields-no-filters.csv
diff --git a/tests/regression/data/raw-data-every-1000-pt3.csv b/tests/regression/data/xdmod10.5/raw-data-every-1000-with-fields-and-filters.csv
similarity index 100%
rename from tests/regression/data/raw-data-every-1000-pt3.csv
rename to tests/regression/data/xdmod10.5/raw-data-every-1000-with-fields-and-filters.csv
diff --git a/tests/regression/data/realms.csv b/tests/regression/data/xdmod10.5/realms.csv
similarity index 100%
rename from tests/regression/data/realms.csv
rename to tests/regression/data/xdmod10.5/realms.csv
diff --git a/tests/regression/test_datawarehouse_regression.py b/tests/regression/test_datawarehouse_regression.py
index 8446fcec..546ac928 100644
--- a/tests/regression/test_datawarehouse_regression.py
+++ b/tests/regression/test_datawarehouse_regression.py
@@ -8,9 +8,10 @@
 import pytest
 from xdmod_data.warehouse import DataWarehouse
 
-XDMOD_URL = os.environ['XDMOD_HOST']
+XDMOD_HOST = os.environ['XDMOD_HOST']
+XDMOD_VERSION = os.environ['XDMOD_VERSION']
 TOKEN_PATH = '~/.xdmod-data-token'
-DATA_DIR = dirname(__file__) + '/data'
+DATA_DIR = dirname(__file__) + '/data/' + XDMOD_VERSION
 
 
 load_dotenv(Path(expanduser(TOKEN_PATH)), override=True)
@@ -18,7 +19,7 @@
 
 @pytest.fixture(scope='module')
 def valid_dw():
-    with DataWarehouse(XDMOD_URL) as dw:
+    with DataWarehouse(XDMOD_HOST) as dw:
         yield dw
 
 
@@ -29,8 +30,12 @@ def __assert_dfs_equal(
     index_col='id',
     columns_name=None,
 ):
-    if 'GENERATE_DATA_FILES' in os.environ:
-        actual.to_csv(DATA_DIR + '/' + data_file)  # pragma: no cover
+    if 'GENERATE_DATA_FILES' in os.environ:  # pragma: no cover
+        try:
+            os.mkdir(DATA_DIR)
+        except FileExistsError:
+            pass 
+        actual.to_csv(DATA_DIR + '/' + data_file)
     else:
         expected = pandas.read_csv(
             DATA_DIR + '/' + data_file,
@@ -52,7 +57,7 @@ def __assert_dfs_equal(
         (
             {},
             '54747',
-            'raw-data-every-1000-pt2.csv',
+            'raw-data-every-1000-no-fields-no-filters.csv',
         ),
         (
             {
@@ -74,7 +79,7 @@ def __assert_dfs_equal(
                 },
             },
             '33345',
-            'raw-data-every-1000-pt3.csv',
+            'raw-data-every-1000-with-fields-and-filters.csv',
         ),
     ],
 )
diff --git a/tests/unit/test_datawarehouse_unit.py b/tests/unit/test_datawarehouse_unit.py
index a3565af2..a40d9162 100644
--- a/tests/unit/test_datawarehouse_unit.py
+++ b/tests/unit/test_datawarehouse_unit.py
@@ -4,7 +4,7 @@
 from xdmod_data.warehouse import DataWarehouse
 
 
-VALID_XDMOD_URL = os.environ['XDMOD_HOST']
+VALID_XDMOD_HOST = os.environ['XDMOD_HOST']
 INVALID_STR = 'asdlkfjsdlkfisdjkfjd'
 
 
@@ -32,7 +32,7 @@ def test___init___KeyError():
         KeyError,
         match='`XDMOD_API_TOKEN` environment variable has not been set.',
     ):
-        DataWarehouse(VALID_XDMOD_URL)
+        DataWarehouse(VALID_XDMOD_HOST)
     os.environ['XDMOD_API_TOKEN'] = token
 
 
@@ -76,10 +76,10 @@ def test___enter___RuntimeError_401():
         match='Error 401: Make sure XDMOD_API_TOKEN is set'
         + ' to a valid API token.',
     ):
-        with DataWarehouse(VALID_XDMOD_URL) as dw:
+        with DataWarehouse(VALID_XDMOD_HOST) as dw:
             dw.describe_realms()
 
 
 def test_exit_without_enter():
-    dw = DataWarehouse(VALID_XDMOD_URL)
+    dw = DataWarehouse(VALID_XDMOD_HOST)
     dw.__exit__(None, None, None)

From 92dcdb1c21a08d2b87edcd1ce13e34059e062f47 Mon Sep 17 00:00:00 2001
From: Rodney Garnett <garnettr35@gmail.com>
Date: Mon, 15 Jul 2024 16:41:22 -0400
Subject: [PATCH 70/72] Git actions check final_ linter fix

---
 tests/integration/test_datawarehouse_integration.py | 7 ++++++-
 tests/regression/test_datawarehouse_regression.py   | 2 +-
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/tests/integration/test_datawarehouse_integration.py b/tests/integration/test_datawarehouse_integration.py
index 5c4b7794..6f7ee5df 100644
--- a/tests/integration/test_datawarehouse_integration.py
+++ b/tests/integration/test_datawarehouse_integration.py
@@ -447,7 +447,12 @@ def test_case_insensitive(dw_methods, method, param, value1, value2):
 
 @pytest.mark.parametrize(
     'dw_methods,method',
-    [(VALID_XDMOD_HOST + '/', method) for method in list(METHOD_PARAMS.keys())],
+    [
+        (
+            VALID_XDMOD_HOST + '/',
+            method,
+        ) for method in list(METHOD_PARAMS.keys())
+    ],
     indirect=['dw_methods'],
     ids=[method for method in list(METHOD_PARAMS.keys())],
 )
diff --git a/tests/regression/test_datawarehouse_regression.py b/tests/regression/test_datawarehouse_regression.py
index 546ac928..dde44fc3 100644
--- a/tests/regression/test_datawarehouse_regression.py
+++ b/tests/regression/test_datawarehouse_regression.py
@@ -34,7 +34,7 @@ def __assert_dfs_equal(
         try:
             os.mkdir(DATA_DIR)
         except FileExistsError:
-            pass 
+            pass
         actual.to_csv(DATA_DIR + '/' + data_file)
     else:
         expected = pandas.read_csv(

From c6a470ef7584987929d598129571a72d84a8699d Mon Sep 17 00:00:00 2001
From: Rodney Garnett <garnettr35@gmail.com>
Date: Tue, 16 Jul 2024 14:58:46 -0400
Subject: [PATCH 71/72] Github actions check 40

---
 tests/unit/test_datawarehouse_unit.py |  6 +++---
 xdmod_data/_descriptors.py            |  2 +-
 xdmod_data/_http_requester.py         |  6 +++---
 xdmod_data/_response_processor.py     |  4 ++--
 xdmod_data/_validator.py              | 10 +++++-----
 5 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/tests/unit/test_datawarehouse_unit.py b/tests/unit/test_datawarehouse_unit.py
index a40d9162..ba177341 100644
--- a/tests/unit/test_datawarehouse_unit.py
+++ b/tests/unit/test_datawarehouse_unit.py
@@ -49,14 +49,14 @@ def test___enter___RuntimeError_xdmod_host_malformed():
             + INVALID_STR + r'\?)'
         ),
     ):
-        with DataWarehouse('https://'):
+        with DataWarehouse('https://'):  # pragma: no cover
             pass
 
 
 def test___enter___RuntimeError_xdmod_host_unresolved():
     invalid_host = 'https://' + INVALID_STR + '.xdmod.org'
     with pytest.raises(Exception):
-        with DataWarehouse(invalid_host):
+        with DataWarehouse(invalid_host):  # pragma: no cover
             pass
 
 
@@ -66,7 +66,7 @@ def test___enter___RuntimeError_xdmod_host_unsupported_protocol():
         requests.exceptions.InvalidSchema,
         match="No connection adapters were found for '" + invalid_host,
     ):
-        with DataWarehouse(invalid_host):
+        with DataWarehouse(invalid_host): # pragma no cover
             pass
 
 
diff --git a/xdmod_data/_descriptors.py b/xdmod_data/_descriptors.py
index 19e3a151..5826737e 100644
--- a/xdmod_data/_descriptors.py
+++ b/xdmod_data/_descriptors.py
@@ -19,7 +19,7 @@ def __request_aggregate(self):
             '/controllers/metric_explorer.php',
             {'operation': 'get_dw_descripter'},
         )
-        if response['totalCount'] != 1:
+        if response['totalCount'] != 1:  # pragma: no cover
             raise RuntimeError(
                 'Descriptor received with unexpected structure.',
             )
diff --git a/xdmod_data/_http_requester.py b/xdmod_data/_http_requester.py
index 33345fd4..69916bbe 100644
--- a/xdmod_data/_http_requester.py
+++ b/xdmod_data/_http_requester.py
@@ -123,7 +123,7 @@ def _request_json(self, path, post_fields=None):
     def __assert_connection_to_xdmod_host(self):
         try:
             self.__request()
-        except RuntimeError as e:
+        except RuntimeError as e:  # pragma: no cover
             raise RuntimeError(
                 "Could not connect to xdmod_host '" + self.__xdmod_host
                 + "': " + str(e),
@@ -148,7 +148,7 @@ def __request(self, path='', post_fields=None, stream=False):
             try:
                 response_json = json.loads(response.text)
                 msg = ': ' + response_json['message']
-            except json.JSONDecodeError:
+            except json.JSONDecodeError:  # pragma: no cover
                 pass
             if response.status_code == 401:
                 msg = (
@@ -207,6 +207,6 @@ def __get_raw_data_limit(self):
             except RuntimeError as e:
                 if '404' in str(e):
                     self.__raw_data_limit = 'NA'
-                else:
+                else:  # pragma: no cover
                     raise
         return self.__raw_data_limit
diff --git a/xdmod_data/_response_processor.py b/xdmod_data/_response_processor.py
index b2f7958b..656c6e3d 100644
--- a/xdmod_data/_response_processor.py
+++ b/xdmod_data/_response_processor.py
@@ -72,7 +72,7 @@ def __parse_timeseries_date_string(date_string):
     # Match YYYY Q#
     elif re.match(r'^[0-9]{4} Q[0-9]$', date_string):
         (date_string, format_) = __parse_quarter_date_string(date_string)
-    else:
+    else:  #pragma: no cover
         raise Exception(
             'Unsupported date specification ' + date_string + '.',
         )
@@ -131,7 +131,7 @@ def __parse_quarter_date_string(date_string):
         month = '07'
     elif quarter == 'Q4':
         month = '10'
-    else:
+    else:  # pragma: no cover
         raise Exception(
             'Unsupported date quarter specification '
             + date_string + '.',
diff --git a/xdmod_data/_validator.py b/xdmod_data/_validator.py
index 4ffc3092..646aa7f1 100644
--- a/xdmod_data/_validator.py
+++ b/xdmod_data/_validator.py
@@ -254,10 +254,10 @@ def __get_dates_from_duration(duration):
     last_month = today + timedelta(days=-30)
     last_quarter = today + timedelta(days=-90)
     this_month_start = date(today.year, today.month, 1)
-    if today.month == 1:
+    if today.month == 1:  # pragma: no cover
         last_full_month_start_year = today.year - 1
         last_full_month_start_month = 12
-    else:
+    else:  # pragma: no cover
         last_full_month_start_year = today.year
         last_full_month_start_month = today.month - 1
     last_full_month_start = date(
@@ -271,9 +271,9 @@ def __get_dates_from_duration(duration):
         ((today.month - 1) // 3) * 3 + 1,
         1,
     )
-    if today.month < 4:
+    if today.month < 4:  # pragma: no cover
         last_quarter_start_year = today.year - 1
-    else:
+    else:  # pragma: no cover
         last_quarter_start_year = today.year
     last_quarter_start = date(
         last_quarter_start_year,
@@ -340,7 +340,7 @@ def __date_add_years(old_date, year_delta):
         try:
             new_date = date(new_date_year, old_date.month, new_date_day)
             keep_going = False
-        except ValueError:
+        except ValueError:  # pragma: no cover
             new_date_day -= 1
             days_above += 1
     return new_date + timedelta(days=days_above)

From 477eb7b7293ea84651498b3791600a0630c9b2db Mon Sep 17 00:00:00 2001
From: Rodney Garnett <garnettr35@gmail.com>
Date: Tue, 16 Jul 2024 15:01:02 -0400
Subject: [PATCH 72/72] Github actions check 40 fix

---
 tests/unit/test_datawarehouse_unit.py | 2 +-
 xdmod_data/_response_processor.py     | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/unit/test_datawarehouse_unit.py b/tests/unit/test_datawarehouse_unit.py
index ba177341..9627e0f1 100644
--- a/tests/unit/test_datawarehouse_unit.py
+++ b/tests/unit/test_datawarehouse_unit.py
@@ -66,7 +66,7 @@ def test___enter___RuntimeError_xdmod_host_unsupported_protocol():
         requests.exceptions.InvalidSchema,
         match="No connection adapters were found for '" + invalid_host,
     ):
-        with DataWarehouse(invalid_host): # pragma no cover
+        with DataWarehouse(invalid_host):  # pragma no cover
             pass
 
 
diff --git a/xdmod_data/_response_processor.py b/xdmod_data/_response_processor.py
index 656c6e3d..e015634f 100644
--- a/xdmod_data/_response_processor.py
+++ b/xdmod_data/_response_processor.py
@@ -72,7 +72,7 @@ def __parse_timeseries_date_string(date_string):
     # Match YYYY Q#
     elif re.match(r'^[0-9]{4} Q[0-9]$', date_string):
         (date_string, format_) = __parse_quarter_date_string(date_string)
-    else:  #pragma: no cover
+    else:  # pragma: no cover
         raise Exception(
             'Unsupported date specification ' + date_string + '.',
         )